Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

⚡ feat: Reliable Interrupt & Steer Escalation and Recovery (#14558)

Danny Avilae7f183851516
diff --git a/.env.example b/.env.example
index 19570809a2..b9bd77b768 100644
--- a/.env.example
+++ b/.env.example
@@ -985,6 +985,13 @@ HELP_AND_FAQ_URL=https://librechat.ai
# Set to false to use in-memory storage for streams while keeping Redis for other caches
# USE_REDIS_STREAMS=true
+# Generation stream wire/state protocol. Redis-backed deployments default to the
+# rolling-upgrade-safe v1 protocol when this is unset; in-memory deployments use v2.
+# After EVERY replica is running a v2-capable LibreChat build and all v1-owned active
+# generations have drained, set this to 2 in a second rollout. Do not roll back to a
+# pre-v2 build while v2 generations remain active in Redis.
+# GENERATION_PROTOCOL_VERSION=2
+
# Single Redis instance
# REDIS_URI=redis://127.0.0.1:6379
diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js
index f85d53b1da..efa4c88fb7 100644
--- a/api/app/clients/BaseClient.js
+++ b/api/app/clients/BaseClient.js
@@ -849,6 +849,19 @@ class BaseClient {
responseMessage.contextMeta = this.contextMeta;
}
+ /** Resumable generation controllers must win the generation's terminal
+ * CAS before this outcome-defining `unfinished:false` write can begin.
+ * The hook is deliberately narrow: ordinary clients omit it, and `false`
+ * means another terminal owner (for example Stop) already won, so this
+ * stale completion must return without writing the response row. */
+ if (typeof opts.beforeResponsePersistence === 'function') {
+ const ownsTerminalPersistence = await opts.beforeResponsePersistence(responseMessage);
+ if (ownsTerminalPersistence === false) {
+ responseMessage.databasePromise = Promise.resolve({ persistenceSkipped: true });
+ return responseMessage;
+ }
+ }
+
responseMessage.databasePromise = this.saveMessageToDatabase(
responseMessage,
saveOptions,
diff --git a/api/app/clients/specs/BaseClient.test.js b/api/app/clients/specs/BaseClient.test.js
index acf3c0c69d..1cf3b45431 100644
--- a/api/app/clients/specs/BaseClient.test.js
+++ b/api/app/clients/specs/BaseClient.test.js
@@ -754,6 +754,64 @@ describe('BaseClient', () => {
);
});
+ test('does not start the completed response write when terminal ownership is denied', async () => {
+ const hookStarted = deferred();
+ const terminalDecision = deferred();
+ const beforeResponsePersistence = jest.fn(() => {
+ hookStarted.resolve();
+ return terminalDecision.promise;
+ });
+ const saveSpy = jest.spyOn(TestClient, 'saveMessageToDatabase');
+
+ const responsePromise = TestClient.sendMessage('Race Stop against completion.', {
+ user: {},
+ beforeResponsePersistence,
+ });
+ await hookStarted.promise;
+
+ expect(beforeResponsePersistence).toHaveBeenCalledTimes(1);
+ expect(
+ saveSpy.mock.calls.filter(([message]) => message?.isCreatedByUser === false),
+ ).toHaveLength(0);
+
+ terminalDecision.resolve(false);
+ const response = await responsePromise;
+
+ expect(beforeResponsePersistence).toHaveBeenCalledWith(response);
+ expect(
+ saveSpy.mock.calls.filter(([message]) => message?.isCreatedByUser === false),
+ ).toHaveLength(0);
+ expect(TestClient.savedMessageIds.has(response.messageId)).toBe(false);
+ await expect(response.databasePromise).resolves.toEqual({ persistenceSkipped: true });
+ });
+
+ test('starts the completed response write only after terminal ownership is granted', async () => {
+ const hookStarted = deferred();
+ const terminalDecision = deferred();
+ const beforeResponsePersistence = jest.fn(() => {
+ hookStarted.resolve();
+ return terminalDecision.promise;
+ });
+ const saveSpy = jest.spyOn(TestClient, 'saveMessageToDatabase');
+
+ const responsePromise = TestClient.sendMessage('Complete after winning ownership.', {
+ user: {},
+ beforeResponsePersistence,
+ });
+ await hookStarted.promise;
+ expect(
+ saveSpy.mock.calls.filter(([message]) => message?.isCreatedByUser === false),
+ ).toHaveLength(0);
+
+ terminalDecision.resolve(true);
+ const response = await responsePromise;
+
+ expect(
+ saveSpy.mock.calls.filter(([message]) => message?.isCreatedByUser === false),
+ ).toHaveLength(1);
+ await expect(response.databasePromise).resolves.toEqual(expect.any(Object));
+ });
+
test('persists the generation-time Langfuse sampling decision for agent responses', async () => {
const previousSampleRate = process.env.LANGFUSE_SAMPLE_RATE;
process.env.LANGFUSE_SAMPLE_RATE = '0';
diff --git a/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js b/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js
index 4613c50cbb..3a331a791a 100644
--- a/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js
+++ b/api/server/controllers/agents/__tests__/askUserQuestion.e2e.spec.js
@@ -55,6 +55,7 @@ const {
buildPendingAction,
getAgentCheckpointer,
deleteAgentCheckpoint,
+ LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
__resetCheckpointerForTests,
} = require('@librechat/api');
const ResumeAgentController = require('~/server/controllers/agents/resume');
@@ -159,9 +160,14 @@ async function buildAskRunEventMode({ saver, responses, toolCalls, runId }) {
return run;
}
-const runConfig = (conversationId) => ({
+const runConfig = (conversationId, checkpointNamespace = '') => ({
runName: 'AgentRun',
- configurable: { thread_id: conversationId, user_id: USER_ID },
+ configurable: {
+ thread_id: conversationId,
+ checkpoint_ns: '',
+ [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
+ user_id: USER_ID,
+ },
streamMode: 'values',
version: 'v2',
});
@@ -221,6 +227,11 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
test('a tool-body interrupt pauses durably and the REAL /resume controller delivers the answer as the tool result', async () => {
const conversationId = `ask-e2e-resume-${Date.now()}`;
const responseMessageId = 'resp-ask-1';
+ const job = await GenerationJobManager.createJob(conversationId, USER_ID, conversationId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const checkpointNamespace = job.metadata.checkpointNamespace;
+ expect(checkpointNamespace).toBe(String(job.createdAt));
// --- Turn 1: the model calls the ask tool → interrupt() from inside the tool body. ---
const run = await buildAskRun({
@@ -244,7 +255,7 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
});
await run.processStream(
{ messages: [new HumanMessage('deploy the app')] },
- runConfig(conversationId),
+ runConfig(conversationId, checkpointNamespace),
);
const interrupt = run.getInterrupt();
@@ -262,7 +273,6 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
expect(paused.checkpoints).toBeGreaterThan(0); // the interrupt checkpoint is durable
// --- Pause bookkeeping (mirrors AgentClient.handleRunInterrupt). ---
- await GenerationJobManager.createJob(conversationId, USER_ID, conversationId);
await GenerationJobManager.updateMetadata(conversationId, {
endpoint: 'agents',
agent_id: 'agent-ask-e2e',
@@ -291,7 +301,7 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
runId: responseMessageId,
});
await resumed.resume(resumeValue, {
- ...runConfig(conversationId),
+ ...runConfig(conversationId, checkpointNamespace),
signal: (abortController ?? new AbortController()).signal,
});
const reInterrupt = resumed.getInterrupt?.();
@@ -345,6 +355,11 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
test('EVENT-DRIVEN mode (production shape): the graphTools ask tool pauses and resumes over the REAL /resume controller', async () => {
const conversationId = `ask-e2e-event-${Date.now()}`;
const responseMessageId = 'resp-ask-event-1';
+ const job = await GenerationJobManager.createJob(conversationId, USER_ID, conversationId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const checkpointNamespace = job.metadata.checkpointNamespace;
+ expect(checkpointNamespace).toBe(String(job.createdAt));
const run = await buildAskRunEventMode({
saver,
@@ -361,7 +376,7 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
});
await run.processStream(
{ messages: [new HumanMessage('run the migration')] },
- runConfig(conversationId),
+ runConfig(conversationId, checkpointNamespace),
);
const interrupt = run.getInterrupt();
@@ -370,7 +385,6 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
expect(bodyRuns).toBe(1);
expect((await checkpointCounts(conversationId)).checkpoints).toBeGreaterThan(0);
- await GenerationJobManager.createJob(conversationId, USER_ID, conversationId);
await GenerationJobManager.updateMetadata(conversationId, {
endpoint: 'agents',
agent_id: 'agent-ask-e2e',
@@ -398,7 +412,7 @@ describe('ask_user_question lifecycle (full wiring, approval policy disabled)',
runId: responseMessageId,
});
await resumed.resume(resumeValue, {
- ...runConfig(conversationId),
+ ...runConfig(conversationId, checkpointNamespace),
signal: (abortController ?? new AbortController()).signal,
});
this.contentParts.push({ type: 'text', text: 'Migration underway.' });
diff --git a/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js b/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js
index ada484c43e..8015224398 100644
--- a/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js
+++ b/api/server/controllers/agents/__tests__/hitlCheckpoint.e2e.spec.js
@@ -55,6 +55,7 @@ const {
deleteAgentCheckpoint,
buildHITLRunWiring,
resolveToolApprovalPolicy,
+ LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
__resetCheckpointerForTests,
} = require('@librechat/api');
const ResumeAgentController = require('~/server/controllers/agents/resume');
@@ -105,9 +106,14 @@ async function buildHitlRun({ saver, conversationId, responses, toolCalls, runId
return run;
}
-const runConfig = (conversationId) => ({
+const runConfig = (conversationId, checkpointNamespace = '') => ({
runName: 'AgentRun',
- configurable: { thread_id: conversationId, user_id: USER_ID },
+ configurable: {
+ thread_id: conversationId,
+ checkpoint_ns: '',
+ [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: checkpointNamespace,
+ user_id: USER_ID,
+ },
streamMode: 'values',
version: 'v2',
});
@@ -203,6 +209,13 @@ describe('HITL checkpoint lifecycle (full wiring)', () => {
test('pause → approve over the REAL /resume controller → tool runs once → checkpoint pruned', async () => {
const conversationId = `e2e-resume-${Date.now()}`;
const responseMessageId = 'resp-pause-1';
+ // Production creates the v2 generation before compiling/running the graph,
+ // then scopes every checkpoint operation to that immutable generation.
+ const job = await GenerationJobManager.createJob(conversationId, USER_ID, conversationId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const checkpointNamespace = job.metadata.checkpointNamespace;
+ expect(checkpointNamespace).toBe(String(job.createdAt));
// --- Turn 1: the model calls the gated tool → PreToolUse 'ask' → interrupt. ---
const run = await buildHitlRun({
@@ -214,7 +227,7 @@ describe('HITL checkpoint lifecycle (full wiring)', () => {
});
await run.processStream(
{ messages: [new HumanMessage('run the guarded tool')] },
- runConfig(conversationId),
+ runConfig(conversationId, checkpointNamespace),
);
const interrupt = run.getInterrupt();
@@ -224,7 +237,6 @@ describe('HITL checkpoint lifecycle (full wiring)', () => {
expect(paused.checkpoints).toBeGreaterThan(0); // the interrupt checkpoint is durable
// --- Pause bookkeeping (mirrors AgentClient.handleRunInterrupt). ---
- const job = await GenerationJobManager.createJob(conversationId, USER_ID, conversationId);
await GenerationJobManager.updateMetadata(conversationId, {
endpoint: 'agents',
agent_id: 'agent-e2e',
@@ -254,7 +266,7 @@ describe('HITL checkpoint lifecycle (full wiring)', () => {
runId: responseMessageId,
});
await resumed.resume(resumeValue, {
- ...runConfig(conversationId),
+ ...runConfig(conversationId, checkpointNamespace),
signal: (abortController ?? new AbortController()).signal,
});
const reInterrupt = resumed.getInterrupt?.();
diff --git a/api/server/controllers/agents/__tests__/protocol.spec.js b/api/server/controllers/agents/__tests__/protocol.spec.js
new file mode 100644
index 0000000000..5b535d956c
--- /dev/null
+++ b/api/server/controllers/agents/__tests__/protocol.spec.js
@@ -0,0 +1,88 @@
+const {
+ GENERATION_PROTOCOL_V1,
+ GENERATION_PROTOCOL_V2,
+ getRequestedGenerationProtocol,
+ getServerGenerationProtocol,
+ negotiateNewGenerationProtocol,
+ negotiateExistingGenerationProtocol,
+} = require('../protocol');
+
+describe('generation protocol negotiation', () => {
+ const configuredProtocol = process.env.GENERATION_PROTOCOL_VERSION;
+
+ afterEach(() => {
+ if (configuredProtocol == null) {
+ delete process.env.GENERATION_PROTOCOL_VERSION;
+ } else {
+ process.env.GENERATION_PROTOCOL_VERSION = configuredProtocol;
+ }
+ });
+
+ test('missing, invalid, or conflicting advertisements fail closed to v1', () => {
+ expect(getRequestedGenerationProtocol({})).toBe(GENERATION_PROTOCOL_V1);
+ expect(
+ getRequestedGenerationProtocol({
+ body: { generationProtocolVersion: 2 },
+ headers: { 'x-librechat-generation-protocol': 'bogus' },
+ }),
+ ).toBe(GENERATION_PROTOCOL_V1);
+ expect(
+ getRequestedGenerationProtocol({
+ body: { generationProtocolVersion: 2 },
+ headers: { 'x-librechat-generation-protocol': '1' },
+ }),
+ ).toBe(GENERATION_PROTOCOL_V1);
+ });
+
+ test('accepts a consistent v2 advertisement across body, query, and header', () => {
+ expect(
+ getRequestedGenerationProtocol({
+ body: { generationProtocolVersion: 2 },
+ query: { generationProtocolVersion: '2' },
+ headers: { 'x-librechat-generation-protocol': '2' },
+ }),
+ ).toBe(GENERATION_PROTOCOL_V2);
+ });
+
+ test('defaults Redis to the bridge protocol and in-memory storage to v2', () => {
+ delete process.env.GENERATION_PROTOCOL_VERSION;
+ expect(getServerGenerationProtocol({ isRedis: true })).toBe(GENERATION_PROTOCOL_V1);
+ expect(getServerGenerationProtocol({ isRedis: false })).toBe(GENERATION_PROTOCOL_V2);
+ });
+
+ test('honors an explicit fleet-wide protocol gate', () => {
+ process.env.GENERATION_PROTOCOL_VERSION = '2';
+ expect(getServerGenerationProtocol({ isRedis: true })).toBe(GENERATION_PROTOCOL_V2);
+ process.env.GENERATION_PROTOCOL_VERSION = '1';
+ expect(getServerGenerationProtocol({ isRedis: false })).toBe(GENERATION_PROTOCOL_V1);
+ });
+
+ test('negotiates new jobs against the server ceiling', () => {
+ delete process.env.GENERATION_PROTOCOL_VERSION;
+ const req = {
+ body: { generationProtocolVersion: 2 },
+ headers: { 'x-librechat-generation-protocol': '2' },
+ };
+ expect(negotiateNewGenerationProtocol(req, { isRedis: true })).toBe(GENERATION_PROTOCOL_V1);
+ expect(negotiateNewGenerationProtocol(req, { isRedis: false })).toBe(GENERATION_PROTOCOL_V2);
+ });
+
+ test('never upgrades a live v1 job after the global gate flips', () => {
+ process.env.GENERATION_PROTOCOL_VERSION = '2';
+ const req = {
+ query: { generationProtocolVersion: '2' },
+ headers: { 'x-librechat-generation-protocol': '2' },
+ };
+ expect(
+ negotiateExistingGenerationProtocol(req, {
+ metadata: { generationProtocolVersion: 1 },
+ }),
+ ).toBe(GENERATION_PROTOCOL_V1);
+ expect(
+ negotiateExistingGenerationProtocol(req, {
+ metadata: { generationProtocolVersion: 2 },
+ }),
+ ).toBe(GENERATION_PROTOCOL_V2);
+ expect(negotiateExistingGenerationProtocol(req, { metadata: {} })).toBe(GENERATION_PROTOCOL_V1);
+ });
+});
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index be73779101..66aec8b9d1 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -9,19 +9,44 @@ const mockLogger = {
const mockGenerationJobManager = {
createJob: jest.fn(),
+ getJob: jest.fn(),
emitError: jest.fn(),
+ emitChunk: jest.fn(),
+ emitDone: jest.fn(),
+ claimTerminalJob: jest.fn(),
+ publishTerminalClaim: jest.fn(),
+ finishTerminalJob: jest.fn(),
completeJob: jest.fn(),
+ failPausePersistence: jest.fn(),
getResumeState: jest.fn(),
updateMetadata: jest.fn(),
claimGeneration: jest.fn(),
+ resumeClaimedGeneration: jest.fn(),
+ takeoverGeneration: jest.fn(),
releaseGeneration: jest.fn(),
hasJob: jest.fn(),
+ approvals: {
+ ownsPausePersistence: jest.fn(),
+ finishPausePersistence: jest.fn(),
+ },
steering: {
closeAndDrain: jest.fn(),
park: jest.fn(),
+ consumeRecovered: jest.fn(),
},
};
+const DEFAULT_OWNED_CLAIM = Object.freeze({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ claimedAt: 100,
+ claimToken: 'claim-token',
+});
+
+function wonGenerationClaim(overrides = {}) {
+ return { claimed: true, existing: { ...DEFAULT_OWNED_CLAIM, ...overrides } };
+}
+
const mockCheckAndIncrementPendingRequest = jest.fn();
const mockDecrementPendingRequest = jest.fn();
const mockGetViolationInfo = jest.fn(() => ({
@@ -36,6 +61,7 @@ const mockFilterPersistableAbortContent = jest.fn((content) =>
const mockGetConvo = jest.fn();
const mockGetMessages = jest.fn();
const mockSaveMessage = jest.fn();
+const mockDeleteAgentCheckpoint = jest.fn();
const mockStartupTelemetry = {
mark: jest.fn(),
setStreamId: jest.fn(),
@@ -103,9 +129,20 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
sendEvent: jest.fn(),
+ toPendingSteer: jest.fn((item) => item),
/** Recorded onto the job so the steer route can honour the OWNING replica's
* seal capability rather than its own probe. */
isSteerPreemptSupported: jest.fn(() => true),
+ buildRecoveredSteerPayload: jest.fn((text, files) => {
+ if (typeof text !== 'string' || (files != null && !Array.isArray(files))) {
+ return null;
+ }
+ const fileIds = [...new Set((files ?? []).map((file) => file?.file_id))];
+ if (fileIds.some((id) => typeof id !== 'string' || id.length === 0)) {
+ return null;
+ }
+ return { text, fileIds: fileIds.sort() };
+ }),
getViolationInfo: (...args) => mockGetViolationInfo(...args),
buildMessageFiles: jest.fn(() => []),
resolveTitleTiming: jest.fn(() => 'immediate'),
@@ -148,6 +185,7 @@ jest.mock('@librechat/api', () => ({
const messages = await getMessages(filter, '_id');
return messages.length === 0;
},
+ deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
}));
jest.mock('~/server/cleanup', () => ({
@@ -190,6 +228,7 @@ function createResumableResponse() {
return res;
});
res.status = jest.fn(() => res);
+ res.set = jest.fn(() => res);
return res;
}
@@ -207,20 +246,178 @@ describe('ResumableAgentController resume metadata', () => {
mockGetMessages.mockResolvedValue([]);
mockGenerationJobManager.createJob.mockResolvedValue({
createdAt: 1000,
+ metadata: { checkpointNamespace: '1000' },
readyPromise: Promise.resolve(),
abortController: new AbortController(),
emitter: { on: jest.fn() },
});
mockGenerationJobManager.getResumeState.mockResolvedValue(null);
+ mockGenerationJobManager.getJob.mockResolvedValue(undefined);
mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
+ mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);
+ mockGenerationJobManager.emitDone.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
+ mockGenerationJobManager.claimTerminalJob.mockResolvedValue({
+ streamId: 'conversation-123',
+ createdAt: 1000,
+ status: 'complete',
+ persistencePending: true,
+ drainedSteers: [],
+ });
+ mockGenerationJobManager.publishTerminalClaim.mockImplementation(
+ async (_claim, finalEvent) => ({
+ finalEvent: finalEvent ?? {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'error',
+ },
+ persistenceFailed: finalEvent == null,
+ }),
+ );
+ mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
+ mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
+ mockGenerationJobManager.resumeClaimedGeneration.mockResolvedValue(null);
+ mockGenerationJobManager.takeoverGeneration.mockResolvedValue({ claimed: false });
mockGenerationJobManager.releaseGeneration.mockResolvedValue(undefined);
mockGenerationJobManager.hasJob.mockResolvedValue(true);
+ mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);
+ mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.steering.closeAndDrain.mockResolvedValue([]);
mockGenerationJobManager.steering.park.mockResolvedValue(undefined);
+ mockGenerationJobManager.steering.consumeRecovered.mockResolvedValue(true);
mockSaveMessage.mockResolvedValue({});
+ mockDeleteAgentCheckpoint.mockResolvedValue(undefined);
+ });
+
+ it.each([
+ ['non-string', { arbitrary: true }],
+ ['empty', ''],
+ ['oversized', 'a'.repeat(129)],
+ ['unsafe characters', 'request id with spaces'],
+ ])(
+ 'rejects a %s clientRequestId before creating durable state',
+ async (_label, clientRequestId) => {
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Invalid request identity',
+ messageId: 'user-message',
+ clientRequestId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res) };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'INVALID_CLIENT_REQUEST_ID' }),
+ );
+ expect(mockGenerationJobManager.claimGeneration).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([-1, 1.5, Number.MAX_SAFE_INTEGER + 1, '1000'])(
+ 'rejects invalid expected predecessor epoch %p before creating durable state',
+ async (expectedPredecessorCreatedAt) => {
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Conditional queued follow-up',
+ messageId: 'user-message',
+ clientRequestId: 'conditional-request',
+ expectedPredecessorCreatedAt,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'INVALID_GENERATION_PREDECESSOR' }),
+ );
+ expect(mockGenerationJobManager.claimGeneration).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ },
+ );
+
+ it.each([
+ ['empty recovery id', { clientRequestId: 'steer-recovery:' }],
+ ['regenerate', { isRegenerate: true }],
+ ['continued response', { isContinued: true }],
+ ['content edit', { editedContent: { index: 0, type: 'text', text: 'edited' } }],
+ ['response reuse', { responseMessageId: 'existing-response' }],
+ ['mismatched user-row override', { overrideUserMessageId: 'existing-user__1' }],
+ ['malformed files', { files: [{}] }],
+ ])('rejects a recovered steer submitted as an incompatible %s shape', async (_label, shape) => {
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Recovered words',
+ messageId: 'recovered-user-message',
+ clientRequestId: 'steer-recovery:server-steer-1',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ ...shape,
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res) };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'INVALID_RECOVERY_REQUEST' }),
+ );
+ expect(mockGenerationJobManager.claimGeneration).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.steering.consumeRecovered).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ['missing attempt id', { clientRequestId: undefined, recoverySteerId: 'server-steer-1' }],
+ ['invalid source id', { clientRequestId: 'attempt-1', recoverySteerId: 'bad source id' }],
+ [
+ 'mismatched legacy and explicit ids',
+ { clientRequestId: 'steer-recovery:legacy-source', recoverySteerId: 'explicit-source' },
+ ],
+ ])('rejects an explicit recovery with %s', async (_label, recoveryFields) => {
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Recovered words',
+ messageId: 'recovered-user-message',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ ...recoveryFields,
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res) };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(400);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'INVALID_RECOVERY_REQUEST' }),
+ );
+ expect(mockGenerationJobManager.claimGeneration).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
});
it('rejects an underscore-suffixed parent that is not persisted', async () => {
@@ -347,6 +544,7 @@ describe('ResumableAgentController resume metadata', () => {
startupTelemetry: mockStartupTelemetry,
initialMetadata: {
conversationId,
+ generationProtocolVersion: 1,
endpoint: 'agents',
iconURL: 'https://example.com/spec-icon.png',
model: 'gpt-3.5-turbo',
@@ -367,6 +565,9 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockGenerationJobManager.createJob.mock.invocationCallOrder[0]).toBeLessThan(
initializeClient.mock.invocationCallOrder[0],
);
+ expect(initializeClient).toHaveBeenCalledWith(
+ expect.objectContaining({ checkpointNamespace: '1000', jobCreatedAt: 1000 }),
+ );
expect(mockGenerationJobManager.updateMetadata).not.toHaveBeenCalled();
const startupMilestones = mockStartupTelemetry.mark.mock.calls.map(([milestone]) => milestone);
expect(startupMilestones.slice(0, 2)).toEqual(['request_admitted', 'job_created']);
@@ -377,6 +578,48 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
});
+ it('persists and exactly echoes protocol v2 on a newly created generation', async () => {
+ mockGenerationJobManager.createJob.mockResolvedValue({
+ createdAt: 1000,
+ metadata: { checkpointNamespace: '1000', generationProtocolVersion: 2 },
+ readyPromise: Promise.resolve(),
+ abortController: new AbortController(),
+ emitter: { on: jest.fn() },
+ });
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Negotiate the rollout protocol.',
+ messageId: 'user-message',
+ conversationId: 'conversation-123',
+ generationProtocolVersion: 2,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+ const initializeClient = jest.fn().mockRejectedValue(new Error('stop after negotiation'));
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'user-123',
+ 'conversation-123',
+ expect.objectContaining({
+ initialMetadata: expect.objectContaining({ generationProtocolVersion: 2 }),
+ }),
+ );
+ expect(res.set).toHaveBeenCalledWith('x-librechat-generation-protocol', '2');
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ generationCreatedAt: 1000,
+ status: 'started',
+ generationProtocolVersion: 2,
+ });
+ });
+
it('prefetches conversation state before admission and joins it with job metadata', async () => {
let resolveConversation;
let signalMetadataStarted;
@@ -425,7 +668,9 @@ describe('ResumableAgentController resume metadata', () => {
expect(res.json).toHaveBeenCalledWith({
streamId: conversationId,
conversationId,
+ generationCreatedAt: 1000,
status: 'started',
+ generationProtocolVersion: 1,
});
expect(initializeClient).not.toHaveBeenCalled();
@@ -468,7 +713,9 @@ describe('ResumableAgentController resume metadata', () => {
expect(res.json).toHaveBeenCalledWith({
streamId: conversationId,
conversationId,
+ generationCreatedAt: 1000,
status: 'started',
+ generationProtocolVersion: 1,
});
expect(disconnect).toHaveBeenCalledTimes(1);
expect(disconnect.mock.invocationCallOrder[0]).toBeLessThan(
@@ -649,6 +896,8 @@ describe('ResumableAgentController resume metadata', () => {
};
const textPart = { type: 'text', text: 'Partial response...' };
+ mockSaveMessage.mockResolvedValueOnce(undefined).mockResolvedValueOnce({});
+ await allSubscribersLeftHandler([oauthPart, textPart]);
await allSubscribersLeftHandler([oauthPart, textPart]);
expect(mockFilterPersistableAbortContent).toHaveBeenCalledWith([oauthPart, textPart]);
@@ -663,6 +912,13 @@ describe('ResumableAgentController resume metadata', () => {
}),
expect.any(Object),
);
+ expect(mockSaveMessage).toHaveBeenCalledTimes(2);
+ expect(mockLogger.error).toHaveBeenCalledWith(
+ '[ResumableAgentController] Error saving partial response:',
+ expect.objectContaining({
+ message: 'Partial response could not be persisted after disconnect',
+ }),
+ );
});
it('uses model spec and agent fallbacks when saving partial responses on disconnect', async () => {
@@ -750,9 +1006,19 @@ describe('ResumableAgentController resume metadata', () => {
it('dedups a retried start-generation request to the original stream', async () => {
mockGenerationJobManager.claimGeneration.mockResolvedValue({
claimed: false,
- existing: { streamId: 'orig-stream', conversationId: 'orig-convo' },
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: 100,
+ claimToken: 'existing-token',
+ startedAt: 1000,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 1000,
+ status: 'running',
+ metadata: { userId: 'user-123', idempotencyClientRequestId: 'req-abc' },
});
- mockGenerationJobManager.hasJob.mockResolvedValue(true);
const initializeClient = jest.fn();
const req = {
user: { id: 'user-123' },
@@ -760,7 +1026,7 @@ describe('ResumableAgentController resume metadata', () => {
text: 'Retried after a lost response.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ conversationId: 'orig-stream',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
@@ -771,8 +1037,10 @@ describe('ResumableAgentController resume metadata', () => {
expect(res.json).toHaveBeenCalledWith({
streamId: 'orig-stream',
- conversationId: 'orig-convo',
+ conversationId: 'orig-stream',
+ generationCreatedAt: 1000,
status: 'resumed',
+ generationProtocolVersion: 1,
});
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
@@ -780,25 +1048,29 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('deduplicated');
});
- it('resumes when the job is missing but the claim is old (original completed and was cleaned up)', async () => {
- // An old claim with no job means the original already ran and was cleaned up; the deduped
- // response must attach (client 404 handler refetches) rather than loop on readiness.
+ it('attaches a new-chat retry to a tokenless legacy random stream after owner validation', async () => {
mockGenerationJobManager.claimGeneration.mockResolvedValue({
claimed: false,
+ source: 'legacy',
existing: {
- streamId: 'orig-stream',
- conversationId: 'orig-convo',
- claimedAt: Date.now() - 60000,
+ streamId: 'legacy-random-stream',
+ conversationId: 'legacy-random-stream',
+ claimedAt: Date.now() - 100,
},
});
- mockGenerationJobManager.hasJob.mockResolvedValue(false);
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 1000,
+ status: 'running',
+ metadata: { userId: 'user-123' },
+ });
const req = {
user: { id: 'user-123' },
body: {
- text: 'Retry after a fast, already-cleaned-up generation.',
+ text: 'Retry an old-server new chat.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ conversationId: 'new',
+ generationProtocolVersion: 2,
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
@@ -808,31 +1080,35 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), jest.fn(), null);
expect(res.json).toHaveBeenCalledWith({
- streamId: 'orig-stream',
- conversationId: 'orig-convo',
+ streamId: 'legacy-random-stream',
+ conversationId: 'legacy-random-stream',
+ generationCreatedAt: 1000,
status: 'resumed',
+ generationProtocolVersion: 1,
});
- expect(res.status).not.toHaveBeenCalledWith(503);
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.takeoverGeneration).not.toHaveBeenCalled();
});
- it('returns 503 SERVER_NOT_READY when a fresh claim still has no job (winner is between claim and createJob)', async () => {
+ it('never takes over a tokenless legacy claim after its job has disappeared', async () => {
mockGenerationJobManager.claimGeneration.mockResolvedValue({
claimed: false,
+ source: 'legacy',
existing: {
- streamId: 'orig-stream',
- conversationId: 'orig-convo',
- claimedAt: Date.now(),
+ streamId: 'legacy-random-stream',
+ conversationId: 'legacy-random-stream',
+ claimedAt: Date.now() - 60_000,
},
});
- mockGenerationJobManager.hasJob.mockResolvedValue(false);
+ mockGenerationJobManager.getJob.mockResolvedValue(undefined);
const req = {
user: { id: 'user-123' },
body: {
- text: 'Concurrent duplicate before the winner wrote its job.',
+ text: 'Retry after an old generation was cleaned.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ conversationId: 'new',
+ generationProtocolVersion: 2,
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
@@ -841,23 +1117,30 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), jest.fn(), null);
- expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
- expect(res.status).toHaveBeenCalledWith(503);
- expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'legacy-random-stream',
+ conversationId: 'legacy-random-stream',
+ status: 'resumed',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.takeoverGeneration).not.toHaveBeenCalled();
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
});
- it('never starts a second generation when the job lookup fails for a confirmed duplicate', async () => {
- // A store hiccup while checking an existing claim must not fail open into createJob.
+ it('rejects a tokenless legacy claim that miscorrelates an existing conversation', async () => {
mockGenerationJobManager.claimGeneration.mockResolvedValue({
claimed: false,
- existing: { streamId: 'orig-stream', conversationId: 'orig-convo', claimedAt: Date.now() },
+ source: 'legacy',
+ existing: {
+ streamId: 'different-conversation',
+ conversationId: 'different-conversation',
+ claimedAt: Date.now() - 100,
+ },
});
- mockGenerationJobManager.hasJob.mockRejectedValue(new Error('redis down'));
const req = {
user: { id: 'user-123' },
body: {
- text: 'Duplicate during a Redis hiccup.',
+ text: 'Do not cross-wire this retry.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
conversationId: 'conversation-123',
@@ -870,163 +1153,1439 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), jest.fn(), null);
expect(res.status).toHaveBeenCalledWith(503);
- expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
});
- it('does not finalize an unscoped generation when job creation rejects before returning', async () => {
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
- mockGenerationJobManager.createJob.mockRejectedValue(new Error('create failed before return'));
+ it.each([
+ ['a different stream', { streamId: 'foreign-stream' }],
+ ['a different conversation', { conversationId: 'foreign-conversation' }],
+ ['a missing claim token', { claimToken: undefined }],
+ ['an invalid claim timestamp', { claimedAt: 'not-a-timestamp' }],
+ ['an invalid started timestamp', { startedAt: -1 }],
+ ])('fails closed when the idempotency claim contains %s', async (_name, override) => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: 100,
+ claimToken: 'existing-token',
+ ...override,
+ },
+ });
const req = {
user: { id: 'user-123' },
body: {
- text: 'Fail before receiving a job epoch.',
+ text: 'Retry against corrupt claim state.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ conversationId: 'orig-stream',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
- const res = createResumableResponse();
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
await AgentController(req, res, jest.fn(), jest.fn(), null);
- expect(res.status).toHaveBeenCalledWith(500);
- expect(res.json).toHaveBeenCalledWith({ error: 'create failed before return' });
- expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
- expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ expect(res.status).toHaveBeenCalledWith(503);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.takeoverGeneration).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
});
- it('finalizes the failed job before releasing the idempotency claim', async () => {
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
- const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
+ it.each([
+ ['another user', { userId: 'someone-else' }, { id: 'user-123' }],
+ ['a missing owner', {}, { id: 'user-123' }],
+ [
+ 'another tenant',
+ { userId: 'user-123', tenantId: 'tenant-b' },
+ { id: 'user-123', tenantId: 'tenant-a' },
+ ],
+ ])(
+ 'fails closed when a valid idempotency claim resolves to %s',
+ async (_name, jobMetadata, requestUser) => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: 100,
+ claimToken: 'existing-token',
+ startedAt: 1000,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 1000,
+ status: 'running',
+ metadata: { ...jobMetadata, idempotencyClientRequestId: 'req-abc' },
+ });
+ const req = {
+ user: requestUser,
+ body: {
+ text: 'Do not attach this retry to a foreign run.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'orig-stream',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(503);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ },
+ );
+
+ it('keeps a terminal duplicate without a durable payload on the readiness path', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: 100,
+ claimToken: 'existing-token',
+ startedAt: 1000,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 1000,
+ status: 'aborted',
+ metadata: {
+ userId: 'user-123',
+ idempotencyClientRequestId: 'req-abc',
+ terminalPersistencePending: true,
+ },
+ });
const req = {
user: { id: 'user-123' },
body: {
- text: 'Start fails after the initial JSON.',
+ text: 'Retry while abort persistence is still in flight.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ conversationId: 'orig-stream',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
- const res = createResumableResponse();
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
- await AgentController(req, res, jest.fn(), initializeClient, null);
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
- 'conversation-123',
- expect.any(String),
- 1000,
- );
- expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
- // completeJob must finalize the failed job BEFORE the claim is released, or a racing
- // retry could win the key, createJob the same streamId, and be aborted by this completeJob.
- expect(mockGenerationJobManager.completeJob.mock.invocationCallOrder[0]).toBeLessThan(
- mockGenerationJobManager.releaseGeneration.mock.invocationCallOrder[0],
- );
+ expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
+ expect(res.status).toHaveBeenCalledWith(503);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
});
- it('still releases the claim and pending slot when completeJob fails during init-error cleanup', async () => {
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
- mockGenerationJobManager.completeJob.mockRejectedValue(new Error('store hiccup'));
- const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
+ it('allows a terminal duplicate with a stored FINAL to replay through SSE', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: 100,
+ claimToken: 'existing-token',
+ startedAt: 1000,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 1000,
+ status: 'complete',
+ finalEvent: JSON.stringify({ final: true }),
+ metadata: { userId: 'user-123', idempotencyClientRequestId: 'req-abc' },
+ });
const req = {
user: { id: 'user-123' },
body: {
- text: 'Start fails while the store is degraded.',
+ text: 'Retry after the terminal payload was persisted.',
messageId: 'user-msg',
clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ conversationId: 'orig-stream',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
- const res = createResumableResponse();
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
- await AgentController(req, res, jest.fn(), initializeClient, null);
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
- // A completeJob rejection must not wedge the retry behind the claim or leak the slot.
- expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
- expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ generationCreatedAt: 1000,
+ status: 'resumed',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
});
- it('still finalizes and releases when streaming the initialization error fails', async () => {
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
- mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
- const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
- const req = {
+ it('derives the same new-conversation stream for a lost-response retry', async () => {
+ mockGenerationJobManager.claimGeneration.mockImplementation(
+ async (_userId, _requestId, streamId, conversationId) =>
+ wonGenerationClaim({ streamId, conversationId }),
+ );
+ const initializeClient = jest.fn().mockRejectedValue(new Error('stop after deterministic id'));
+ const makeReq = () => ({
user: { id: 'user-123' },
body: {
- text: 'Start fails while Redis publish is degraded.',
+ text: 'Create a stable new chat.',
messageId: 'user-msg',
- clientRequestId: 'req-abc',
- conversationId: 'conversation-123',
+ clientRequestId: '4ea9fc40-f28f-4f89-a575-aa5854d10c19',
+ conversationId: 'new',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
- };
- const res = createResumableResponse();
+ });
- await AgentController(req, res, jest.fn(), initializeClient, null);
+ await AgentController(makeReq(), createResumableResponse(), jest.fn(), initializeClient, null);
+ await AgentController(makeReq(), createResumableResponse(), jest.fn(), initializeClient, null);
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
- 'conversation-123',
- 'init boom after res.json',
- 1000,
+ const firstStreamId = mockGenerationJobManager.claimGeneration.mock.calls[0][2];
+ const retryStreamId = mockGenerationJobManager.claimGeneration.mock.calls[1][2];
+ expect(firstStreamId).toMatch(
+ /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
);
- expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
- expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
- expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
+ expect(retryStreamId).toBe(firstStreamId);
});
- it('finalizes and disposes a client aborted during initialization before releasing the slot', async () => {
- const abortController = new AbortController();
- let resolveCompletion;
- let signalCompletionStarted;
- const completionStarted = new Promise((resolve) => {
- signalCompletionStarted = resolve;
- });
- mockGenerationJobManager.createJob.mockResolvedValue({
- createdAt: 1000,
- readyPromise: Promise.resolve(),
- abortController,
- emitter: { on: jest.fn() },
- });
- mockGenerationJobManager.completeJob.mockImplementation(() => {
- signalCompletionStarted();
- return new Promise((resolve) => {
- resolveCompletion = resolve;
- });
+ it('resumes a matching live generation when its fixed claim lease was reacquired', async () => {
+ const reacquired = wonGenerationClaim({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ claimToken: 'reacquired-token',
});
- const client = { options: {} };
- const initializeClient = jest.fn(async ({ signal }) => {
- expect(signal).toBe(abortController.signal);
- abortController.abort();
- return { client };
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(reacquired);
+ mockGenerationJobManager.resumeClaimedGeneration.mockResolvedValue({
+ ...reacquired.existing,
+ startedAt: 42,
});
- const conversationId = 'conversation-123';
const req = {
user: { id: 'user-123' },
body: {
- text: 'Stop during initialization.',
+ text: 'Retry after sleeping through a long approval.',
messageId: 'user-msg',
- conversationId,
+ clientRequestId: 'req-abc',
+ conversationId: 'conversation-123',
endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
},
config: {},
};
- const res = createResumableResponse();
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
- const controllerPromise = AgentController(req, res, jest.fn(), initializeClient, null);
- await completionStarted;
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
- conversationId,
+ expect(mockGenerationJobManager.resumeClaimedGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'conversation-123',
+ reacquired.existing,
+ );
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ generationCreatedAt: 42,
+ status: 'resumed',
+ generationProtocolVersion: 1,
+ });
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ });
+
+ it('caps a reacquired v2 lease to the immutable v1 live-job protocol', async () => {
+ const reacquired = wonGenerationClaim({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ claimToken: 'reacquired-v2-token',
+ generationProtocolVersion: 2,
+ });
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(reacquired);
+ mockGenerationJobManager.resumeClaimedGeneration.mockResolvedValue({
+ ...reacquired.existing,
+ startedAt: 42,
+ generationProtocolVersion: 1,
+ });
+ const req = {
+ user: { id: 'user-123' },
+ headers: { 'x-librechat-generation-protocol': '2' },
+ body: {
+ text: 'Retry a v1 job from an upgraded client.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'conversation-123',
+ generationProtocolVersion: 2,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ generationCreatedAt: 42,
+ status: 'resumed',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ });
+
+ it('returns a settled response when the original job completed and was cleaned up', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: Date.now() - 60000,
+ claimToken: 'existing-token',
+ startedAt: Date.now() - 59000,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue(undefined);
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Retry after a fast, already-cleaned-up generation.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'orig-stream',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ status: 'resumed',
+ generationProtocolVersion: 1,
+ });
+ expect(res.status).not.toHaveBeenCalledWith(503);
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ });
+
+ it('uses the v2 settled control only when the durable claim confirms protocol v2', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: Date.now() - 60_000,
+ claimToken: 'existing-token',
+ startedAt: Date.now() - 59_000,
+ generationProtocolVersion: 2,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue(undefined);
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Retry a completed v2 generation.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'orig-stream',
+ generationProtocolVersion: 2,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.json).toHaveBeenCalledWith({
+ conversationId: 'orig-stream',
+ status: 'settled',
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('does not attach a stale retry to a newer generation on the same conversation stream', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ claimedAt: Date.now() - 60_000,
+ claimToken: 'existing-token',
+ startedAt: 1000,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 2000,
+ status: 'running',
+ metadata: { userId: 'user-123', idempotencyClientRequestId: 'newer-request' },
+ });
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Retry the earlier turn after its response was lost.',
+ messageId: 'old-user-msg',
+ clientRequestId: 'older-request',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({
+ code: 'RUN_REPLACED',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ });
+
+ it('uses the v2 replacement handoff only when both claim and live job confirm v2', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ claimedAt: Date.now() - 60_000,
+ claimToken: 'existing-token',
+ startedAt: 1000,
+ generationProtocolVersion: 2,
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 2000,
+ status: 'running',
+ metadata: {
+ userId: 'user-123',
+ idempotencyClientRequestId: 'newer-request',
+ generationProtocolVersion: 2,
+ },
+ });
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Retry the earlier v2 turn.',
+ messageId: 'old-user-msg',
+ clientRequestId: 'older-request',
+ conversationId: 'conversation-123',
+ generationProtocolVersion: 2,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(200);
+ expect(res.json).toHaveBeenCalledWith({
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ generationCreatedAt: 2000,
+ status: 'replaced',
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('takes over an abandoned old pre-create claim and fences the prior owner', async () => {
+ const abandonedClaim = {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: Date.now() - 60000,
+ claimToken: 'abandoned-token',
+ };
+ const takeover = wonGenerationClaim({
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimToken: 'takeover-token',
+ });
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: abandonedClaim,
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue(undefined);
+ mockGenerationJobManager.takeoverGeneration.mockResolvedValue(takeover);
+ const initializeClient = jest.fn().mockRejectedValue(new Error('stop after takeover'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Recover the abandoned request.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'orig-stream',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.takeoverGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'orig-stream',
+ abandonedClaim,
+ );
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'orig-stream',
+ 'user-123',
+ 'orig-stream',
+ expect.objectContaining({
+ idempotencyClientRequestId: 'req-abc',
+ idempotencyClaimToken: 'takeover-token',
+ }),
+ );
+ });
+
+ it('returns 503 SERVER_NOT_READY when a fresh claim still has no job (winner is between claim and createJob)', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: Date.now(),
+ claimToken: 'existing-token',
+ },
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue(undefined);
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Concurrent duplicate before the winner wrote its job.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'orig-stream',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.set).toHaveBeenCalledWith('Retry-After', '1');
+ expect(res.status).toHaveBeenCalledWith(503);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ });
+
+ it('never starts a second generation when the job lookup fails for a confirmed duplicate', async () => {
+ // A store hiccup while checking an existing claim must not fail open into createJob.
+ mockGenerationJobManager.claimGeneration.mockResolvedValue({
+ claimed: false,
+ existing: {
+ streamId: 'orig-stream',
+ conversationId: 'orig-stream',
+ claimedAt: Date.now(),
+ claimToken: 'existing-token',
+ },
+ });
+ mockGenerationJobManager.getJob.mockRejectedValue(new Error('redis down'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Duplicate during a Redis hiccup.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'orig-stream',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = { json: jest.fn(), status: jest.fn(() => res), set: jest.fn() };
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(503);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
+ });
+
+ it('does not finalize an unscoped generation when job creation rejects before returning', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
+ mockGenerationJobManager.createJob.mockRejectedValue(new Error('create failed before return'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Fail before receiving a job epoch.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(500);
+ expect(res.json).toHaveBeenCalledWith({
+ error: 'create failed before return',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'conversation-123',
+ DEFAULT_OWNED_CLAIM,
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ });
+
+ it('returns a recovery conflict when the atomic store rejects changed source content', async () => {
+ const mismatch = new Error('recovery mismatch');
+ mismatch.code = 'RECOVERY_PAYLOAD_MISMATCH';
+ mockGenerationJobManager.createJob.mockRejectedValue(mismatch);
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Changed words',
+ messageId: 'recovered-user-msg',
+ clientRequestId: 'steer-recovery:server-steer-1',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'user-123',
+ 'conversation-123',
+ expect.objectContaining({
+ recoveredSteerId: 'server-steer-1',
+ recoveredSteerPayload: { text: 'Changed words', fileIds: [] },
+ }),
+ );
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith(
+ expect.objectContaining({ code: 'RECOVERY_PAYLOAD_MISMATCH' }),
+ );
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.steering.consumeRecovered).not.toHaveBeenCalled();
+ });
+
+ it('restores a conditional queued send when a newer generation wins the create CAS', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(
+ wonGenerationClaim({ generationProtocolVersion: 2 }),
+ );
+ const mismatch = new Error('predecessor changed');
+ mismatch.code = 'GENERATION_PREDECESSOR_MISMATCH';
+ mismatch.currentJob = {
+ createdAt: 2000,
+ status: 'running',
+ conversationId: 'conversation-123',
+ verified: true,
+ };
+ mockGenerationJobManager.createJob.mockRejectedValue(mismatch);
+ const req = {
+ user: { id: 'user-123' },
+ headers: { 'x-librechat-generation-protocol': '2' },
+ body: {
+ text: 'Queued follow-up C',
+ messageId: 'queued-message-c',
+ clientRequestId: 'queued-attempt-c',
+ expectedPredecessorCreatedAt: 1000,
+ conversationId: 'conversation-123',
+ generationProtocolVersion: 2,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'user-123',
+ 'conversation-123',
+ expect.objectContaining({ expectedPredecessorCreatedAt: 1000 }),
+ );
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({
+ status: 'predecessor_mismatch',
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ error: 'A newer generation became current before this request could start.',
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ generationCreatedAt: 2000,
+ predecessorVerified: true,
+ active: true,
+ generationProtocolVersion: 2,
+ });
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'queued-attempt-c',
+ 'conversation-123',
+ expect.objectContaining(DEFAULT_OWNED_CLAIM),
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ });
+
+ it('returns a finite fail-closed mismatch when predecessor evidence expired', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(
+ wonGenerationClaim({ generationProtocolVersion: 2 }),
+ );
+ const mismatch = new Error('predecessor evidence expired');
+ mismatch.code = 'GENERATION_PREDECESSOR_MISMATCH';
+ mismatch.currentJob = {
+ createdAt: 1000,
+ active: false,
+ verified: false,
+ };
+ mockGenerationJobManager.createJob.mockRejectedValue(mismatch);
+ const req = {
+ user: { id: 'user-123' },
+ headers: { 'x-librechat-generation-protocol': '2' },
+ body: {
+ text: 'Queued follow-up C',
+ messageId: 'queued-message-c',
+ clientRequestId: 'queued-attempt-c',
+ expectedPredecessorCreatedAt: 1000,
+ conversationId: 'conversation-123',
+ generationProtocolVersion: 2,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), jest.fn(), null);
+
+ expect(res.status).toHaveBeenCalledWith(409);
+ expect(res.json).toHaveBeenCalledWith({
+ status: 'predecessor_mismatch',
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ error: 'The prior generation could not be verified. Please retry.',
+ streamId: 'conversation-123',
+ conversationId: 'conversation-123',
+ generationCreatedAt: 1000,
+ predecessorVerified: false,
+ active: false,
+ generationProtocolVersion: 2,
+ });
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'queued-attempt-c',
+ 'conversation-123',
+ expect.objectContaining(DEFAULT_OWNED_CLAIM),
+ );
+ });
+
+ it('does not consume a recovered steer when client initialization fails before persistence', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(
+ wonGenerationClaim({ claimToken: 'recovery-claim-token' }),
+ );
+ const initializeClient = jest.fn().mockRejectedValue(new Error('provider init failed'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Recovered words',
+ messageId: 'recovered-user-msg',
+ clientRequestId: 'recovery-attempt-1',
+ recoverySteerId: 'server-steer-1',
+ overrideUserMessageId: 'server-steer-1',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.createJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'user-123',
+ 'conversation-123',
+ expect.objectContaining({ recoveredSteerId: 'server-steer-1' }),
+ );
+ expect(mockGenerationJobManager.claimGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'recovery-attempt-1',
+ 'conversation-123',
+ 'conversation-123',
+ 1,
+ );
+ expect(req.body.overrideUserMessageId).toBe('server-steer-1__0');
+ expect(mockGenerationJobManager.steering.consumeRecovered).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'provider init failed',
+ 1000,
+ );
+ });
+
+ it('fails closed when a recovered-turn client derives skipSaveUserMessage', async () => {
+ const client = {
+ options: {},
+ skipSaveUserMessage: true,
+ sendMessage: jest.fn(),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Recovered words',
+ messageId: 'recovered-user-msg',
+ clientRequestId: 'steer-recovery:server-steer-1',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+
+ expect(client.sendMessage).not.toHaveBeenCalled();
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.steering.consumeRecovered).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'Recovered steer cannot skip user message persistence',
+ 1000,
+ );
+ });
+
+ it('does not consume a recovered steer when the explicit normal-flow user save returns no row', async () => {
+ const userMessage = {
+ messageId: 'recovered-user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Recovered words',
+ };
+ mockSaveMessage.mockImplementation(async (_reqCtx, message) => {
+ if (message?.messageId === userMessage.messageId) {
+ return undefined;
+ }
+ return {};
+ });
+ let signalFinished;
+ const finished = new Promise((resolve) => {
+ signalFinished = resolve;
+ });
+ mockGenerationJobManager.finishTerminalJob.mockImplementation(async (...args) => {
+ signalFinished(args);
+ });
+ const client = {
+ options: {},
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ clientRequestId: 'steer-recovery:server-steer-1',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ const finishArgs = await finished;
+
+ expect(mockSaveMessage).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: 'user-123' }),
+ userMessage,
+ expect.objectContaining({ context: expect.stringContaining('resumable user message') }),
+ );
+ expect(mockGenerationJobManager.steering.consumeRecovered).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: 'conversation-123', status: 'complete' }),
+ null,
+ );
+ expect(finishArgs).toEqual([
+ expect.objectContaining({ streamId: 'conversation-123', status: 'complete' }),
+ ]);
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ });
+
+ it('does not consume a recovered steer when the explicit HITL user save fails', async () => {
+ const userMessage = {
+ messageId: 'recovered-user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Recovered words before approval',
+ };
+ const userSaveError = new Error('paused user row unavailable');
+ mockSaveMessage.mockRejectedValue(userSaveError);
+ let signalPauseFailed;
+ const pauseFailed = new Promise((resolve) => {
+ signalPauseFailed = resolve;
+ });
+ mockGenerationJobManager.failPausePersistence.mockImplementation(async (...args) => {
+ signalPauseFailed(args);
+ return true;
+ });
+ const client = {
+ options: {},
+ pendingApproval: { actionId: 'action-paused-save-fails' },
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ clientRequestId: 'steer-recovery:server-steer-1',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ const failArgs = await pauseFailed;
+ await nextTick();
+
+ expect(mockSaveMessage).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: 'user-123' }),
+ userMessage,
+ expect.objectContaining({ context: expect.stringContaining('before HITL pause') }),
+ );
+ expect(mockGenerationJobManager.steering.consumeRecovered).not.toHaveBeenCalled();
+ expect(failArgs).toEqual([
+ 'conversation-123',
+ 'action-paused-save-fails',
+ userSaveError.message,
+ 1000,
+ ]);
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ 'conversation-123',
+ undefined,
+ undefined,
+ { checkpointNamespace: '1000' },
+ );
+ });
+
+ it('fails the pause when a normal user-row retry resolves without a durable row', async () => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Pause only after my row is durable.',
+ };
+ mockSaveMessage.mockResolvedValueOnce(undefined);
+ let signalPauseFailed;
+ const pauseFailed = new Promise((resolve) => {
+ signalPauseFailed = resolve;
+ });
+ mockGenerationJobManager.failPausePersistence.mockImplementation(async (...args) => {
+ signalPauseFailed(args);
+ return true;
+ });
+ const client = {
+ options: {},
+ pendingApproval: { actionId: 'action-normal-user-falsy' },
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ const failArgs = await pauseFailed;
+ await nextTick();
+
+ expect(mockSaveMessage).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: 'user-123' }),
+ userMessage,
+ expect.objectContaining({ context: expect.stringContaining('before HITL pause') }),
+ );
+ expect(failArgs).toEqual([
+ 'conversation-123',
+ 'action-normal-user-falsy',
+ 'User message could not be persisted before HITL pause',
+ 1000,
+ ]);
+ expect(mockSaveMessage).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ [
+ 'message row',
+ { message: undefined, conversation: { conversationId: 'conversation-123' } },
+ 'User message could not be persisted before HITL pause',
+ ],
+ [
+ 'conversation row',
+ { message: { messageId: 'user-msg' }, conversation: undefined },
+ 'Conversation could not be persisted before HITL pause',
+ ],
+ ])(
+ 'fails the pause when the BaseClient retry returns no %s',
+ async (_label, retryResult, expectedError) => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Repair the whole user turn before approval.',
+ };
+ let signalPauseFailed;
+ const pauseFailed = new Promise((resolve) => {
+ signalPauseFailed = resolve;
+ });
+ mockGenerationJobManager.failPausePersistence.mockImplementation(async (...args) => {
+ signalPauseFailed(args);
+ return true;
+ });
+ const client = {
+ options: {},
+ pendingApproval: { actionId: 'action-base-client-retry-falsy' },
+ skipSaveUserMessage: false,
+ skipSaveConvo: false,
+ getSaveOptions: jest.fn(() => ({ endpoint: 'agents' })),
+ saveMessageToDatabase: jest.fn().mockResolvedValue(retryResult),
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ const failArgs = await pauseFailed;
+ await nextTick();
+
+ expect(client.saveMessageToDatabase).toHaveBeenCalledWith(
+ userMessage,
+ { endpoint: 'agents' },
+ 'user-123',
+ );
+ expect(failArgs).toEqual([
+ 'conversation-123',
+ 'action-base-client-retry-falsy',
+ expectedError,
+ 1000,
+ ]);
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ },
+ );
+
+ it('does not let generic cleanup touch a newer generation after paused persistence loses ownership', async () => {
+ let signalPauseFailed;
+ const pauseFailed = new Promise((resolve) => {
+ signalPauseFailed = resolve;
+ });
+ mockSaveMessage.mockResolvedValue(undefined);
+ mockGenerationJobManager.failPausePersistence.mockImplementation(async (...args) => {
+ signalPauseFailed(args);
+ return false;
+ });
+ const client = {
+ options: {},
+ jobCreatedAt: 1000,
+ pendingApproval: { actionId: 'action-replaced-before-pause-save' },
+ skipSaveUserMessage: true,
+ sendMessage: jest.fn(async () => ({
+ messageId: 'response-msg',
+ content: [{ type: 'text', text: 'Waiting for approval.' }],
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123' },
+ }),
+ })),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Pause before the tool runs.',
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ const failArgs = await pauseFailed;
+ await nextTick();
+
+ expect(failArgs).toEqual([
+ 'conversation-123',
+ 'action-replaced-before-pause-save',
+ 'Paused response could not be persisted as unfinished',
+ 1000,
+ ]);
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ });
+
+ it('persists a paused response as unfinished before releasing its Stop/resume barrier', async () => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Pause before the tool runs.',
+ };
+ const response = {
+ messageId: 'response-msg',
+ parentMessageId: userMessage.messageId,
+ conversationId: 'conversation-123',
+ content: [{ type: 'text', text: 'Waiting for approval.' }],
+ };
+ let signalPauseSaveStarted;
+ const pauseSaveStarted = new Promise((resolve) => {
+ signalPauseSaveStarted = resolve;
+ });
+ let releasePauseSave;
+ const pauseSaveGate = new Promise((resolve) => {
+ releasePauseSave = resolve;
+ });
+ mockSaveMessage.mockImplementation(async () => {
+ signalPauseSaveStarted();
+ await pauseSaveGate;
+ return {};
+ });
+ let observedHookResult;
+ const completedResponseWrite = jest.fn();
+ const client = {
+ options: {},
+ jobCreatedAt: 1000,
+ pendingApproval: { actionId: 'action-pause-barrier' },
+ skipSaveUserMessage: false,
+ skipSaveConvo: false,
+ getSaveOptions: jest.fn(() => ({ endpoint: 'agents' })),
+ saveMessageToDatabase: jest.fn().mockResolvedValue({
+ message: userMessage,
+ conversation: { conversationId: 'conversation-123' },
+ }),
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, response.messageId);
+ observedHookResult = await options.beforeResponsePersistence(response);
+ response.databasePromise = observedHookResult
+ ? completedResponseWrite()
+ : Promise.resolve({
+ persistenceSkipped: true,
+ conversation: { conversationId: 'conversation-123' },
+ });
+ return response;
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ await pauseSaveStarted;
+
+ expect(observedHookResult).toBe(false);
+ expect(completedResponseWrite).not.toHaveBeenCalled();
+ expect(client.saveMessageToDatabase).toHaveBeenCalledWith(
+ userMessage,
+ { endpoint: 'agents' },
+ 'user-123',
+ );
+ expect(mockSaveMessage).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: 'user-123' }),
+ expect.objectContaining({
+ messageId: response.messageId,
+ content: response.content,
+ unfinished: true,
+ }),
+ expect.objectContaining({ context: expect.stringContaining('persist unfinished') }),
+ );
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+
+ releasePauseSave();
+ await nextTick();
+ await nextTick();
+
+ expect(mockGenerationJobManager.approvals.ownsPausePersistence).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'action-pause-barrier',
+ 1000,
+ );
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'action-pause-barrier',
+ 1000,
+ );
+ expect(mockGenerationJobManager.failPausePersistence).not.toHaveBeenCalled();
+ expect(client.saveMessageToDatabase.mock.invocationCallOrder[0]).toBeLessThan(
+ mockSaveMessage.mock.invocationCallOrder[0],
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.approvals.finishPausePersistence.mock.invocationCallOrder[0],
+ );
+ expect(mockGenerationJobManager.claimTerminalJob).not.toHaveBeenCalled();
+ });
+
+ it('finalizes the failed job before releasing the idempotency claim', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
+ const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Start fails after the initial JSON.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ expect.any(String),
+ 1000,
+ );
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'conversation-123',
+ DEFAULT_OWNED_CLAIM,
+ );
+ // completeJob must finalize the failed job BEFORE the claim is released, or a racing
+ // retry could win the key, createJob the same streamId, and be aborted by this completeJob.
+ expect(mockGenerationJobManager.completeJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.releaseGeneration.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('still releases the claim and pending slot when completeJob fails during init-error cleanup', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
+ mockGenerationJobManager.completeJob.mockRejectedValue(new Error('store hiccup'));
+ const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Start fails while the store is degraded.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ // A completeJob rejection must not wedge the retry behind the claim or leak the slot.
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'conversation-123',
+ DEFAULT_OWNED_CLAIM,
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ });
+
+ it('still finalizes and releases when streaming the initialization error fails', async () => {
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
+ mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
+ const initializeClient = jest.fn().mockRejectedValue(new Error('init boom after res.json'));
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Start fails while Redis publish is degraded.',
+ messageId: 'user-msg',
+ clientRequestId: 'req-abc',
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ await AgentController(req, res, jest.fn(), initializeClient, null);
+
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'init boom after res.json',
+ 1000,
+ );
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'conversation-123',
+ DEFAULT_OWNED_CLAIM,
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith('user-123');
+ expect(mockStartupTelemetry.end).toHaveBeenCalledWith('error', expect.any(Error));
+ });
+
+ it('finalizes and disposes a client aborted during initialization before releasing the slot', async () => {
+ const abortController = new AbortController();
+ let resolveCompletion;
+ let signalCompletionStarted;
+ const completionStarted = new Promise((resolve) => {
+ signalCompletionStarted = resolve;
+ });
+ mockGenerationJobManager.createJob.mockResolvedValue({
+ createdAt: 1000,
+ readyPromise: Promise.resolve(),
+ abortController,
+ emitter: { on: jest.fn() },
+ });
+ mockGenerationJobManager.completeJob.mockImplementation(() => {
+ signalCompletionStarted();
+ return new Promise((resolve) => {
+ resolveCompletion = resolve;
+ });
+ });
+ const client = { options: {} };
+ const initializeClient = jest.fn(async ({ signal }) => {
+ expect(signal).toBe(abortController.signal);
+ abortController.abort();
+ return { client };
+ });
+ const conversationId = 'conversation-123';
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: 'Stop during initialization.',
+ messageId: 'user-msg',
+ conversationId,
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+ const res = createResumableResponse();
+
+ const controllerPromise = AgentController(req, res, jest.fn(), initializeClient, null);
+ await completionStarted;
+
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ conversationId,
'Request aborted during initialization',
1000,
);
@@ -1049,7 +2608,6 @@ describe('ResumableAgentController resume metadata', () => {
const completionStarted = new Promise((resolve) => {
signalCompletionStarted = resolve;
});
- mockGenerationJobManager.emitError.mockRejectedValue(new Error('publish failed'));
mockGenerationJobManager.completeJob.mockImplementation(() => {
signalCompletionStarted();
return new Promise((_, reject) => {
@@ -1076,11 +2634,9 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), initializeClient, null);
await completionStarted;
- expect(mockGenerationJobManager.emitError).toHaveBeenCalledWith(
- 'conversation-123',
- generationError.message,
- 1000,
- );
+ // completeJob owns both the terminal CAS and error publication; the
+ // controller must not publish before terminal ownership is established.
+ expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
expect(mockDisposeClient).not.toHaveBeenCalled();
@@ -1099,8 +2655,344 @@ describe('ResumableAgentController resume metadata', () => {
expect(mockDisposeClient).toHaveBeenCalledWith(client);
});
+ it('claims terminal ownership before FINAL and always finishes the winning claim', async () => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Finish normally.',
+ };
+ const terminalClaim = {
+ streamId: 'conversation-123',
+ createdAt: 1000,
+ status: 'complete',
+ persistencePending: true,
+ drainedSteers: [],
+ };
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ createdAt: 1000,
+ status: 'running',
+ });
+ mockGenerationJobManager.claimTerminalJob.mockResolvedValue(terminalClaim);
+ let signalFinished;
+ const finished = new Promise((resolve) => {
+ signalFinished = resolve;
+ });
+ mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());
+ const client = {
+ options: {},
+ savedMessageIds: new Set(),
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ parentMessageId: userMessage.messageId,
+ conversationId: 'conversation-123',
+ content: [{ type: 'text', text: 'Done.' }],
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123', title: 'Existing' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ await finished;
+ await nextTick();
+
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'complete',
+ undefined,
+ 1000,
+ { persistencePending: true },
+ );
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(
+ terminalClaim,
+ expect.objectContaining({ final: true }),
+ );
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(terminalClaim);
+ expect(mockGenerationJobManager.claimTerminalJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
+ );
+ expect(mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.finishTerminalJob.mock.invocationCallOrder[0],
+ );
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.steering.closeAndDrain).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ [
+ 'completed',
+ undefined,
+ 'api/server/controllers/agents/request.js - resumable response end',
+ 'Response message could not be persisted before terminal publication',
+ ],
+ [
+ 'unfinished preempt',
+ {
+ getPreemptStats: () => ({ emptyBoundaries: 1 }),
+ getHaltReason: () => 'preempt_incomplete',
+ },
+ 'api/server/controllers/agents/request.js - terminal response unfinished',
+ 'Terminal response could not be persisted as unfinished',
+ ],
+ ])(
+ 'publishes reconciliation when a BaseClient-marked %s response save returned no row',
+ async (_label, run, expectedContext, expectedError) => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Verify terminal persistence.',
+ };
+ const terminalClaim = {
+ streamId: 'conversation-123',
+ createdAt: 1000,
+ status: 'complete',
+ persistencePending: true,
+ drainedSteers: [],
+ };
+ mockGenerationJobManager.claimTerminalJob.mockResolvedValue(terminalClaim);
+ mockSaveMessage.mockImplementation(async (_reqCtx, message) =>
+ message?.messageId === userMessage.messageId ? {} : undefined,
+ );
+ let signalFinished;
+ const finished = new Promise((resolve) => {
+ signalFinished = resolve;
+ });
+ mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());
+ const client = {
+ options: {},
+ savedMessageIds: new Set(['response-msg']),
+ skipSaveUserMessage: false,
+ ...(run && { run }),
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ parentMessageId: userMessage.messageId,
+ conversationId: 'conversation-123',
+ content: [{ type: 'text', text: 'Done.' }],
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123', title: 'Existing' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ await finished;
+ await nextTick();
+
+ expect(mockSaveMessage).toHaveBeenCalledWith(
+ expect.anything(),
+ expect.objectContaining({
+ messageId: 'response-msg',
+ unfinished: _label === 'unfinished preempt',
+ }),
+ { context: expectedContext },
+ );
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(
+ terminalClaim,
+ null,
+ );
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(terminalClaim);
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockLogger.error).toHaveBeenCalledWith(
+ expect.stringContaining('Terminal persistence failed'),
+ expect.objectContaining({ message: expectedError }),
+ );
+ },
+ );
+
+ it('suppresses the completed response write when Stop already won terminal ownership', async () => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Race Stop against completion.',
+ };
+ let signalClaimStarted;
+ const claimStarted = new Promise((resolve) => {
+ signalClaimStarted = resolve;
+ });
+ let resolveTerminalClaim;
+ const terminalClaimGate = new Promise((resolve) => {
+ resolveTerminalClaim = resolve;
+ });
+ mockGenerationJobManager.claimTerminalJob.mockImplementation(async () => {
+ signalClaimStarted();
+ return terminalClaimGate;
+ });
+ const completedResponseWrite = jest.fn(() =>
+ Promise.resolve({ conversation: { conversationId: 'conversation-123' } }),
+ );
+ let observedHookResult;
+ const client = {
+ options: {},
+ savedMessageIds: new Set(),
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ const response = {
+ messageId: 'response-msg',
+ parentMessageId: userMessage.messageId,
+ conversationId: 'conversation-123',
+ content: [{ type: 'text', text: 'Stale completion.' }],
+ };
+ observedHookResult = await options.beforeResponsePersistence(response);
+ response.databasePromise = observedHookResult
+ ? completedResponseWrite()
+ : Promise.resolve({ persistenceSkipped: true });
+ return response;
+ }),
+ };
+ let signalSettled;
+ const settled = new Promise((resolve) => {
+ signalSettled = resolve;
+ });
+ mockDecrementPendingRequest.mockImplementationOnce(async () => signalSettled());
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ await claimStarted;
+ expect(completedResponseWrite).not.toHaveBeenCalled();
+
+ // Model the Stop endpoint winning the durable CAS while completion is
+ // blocked at the exact pre-write ownership hook.
+ resolveTerminalClaim(null);
+ await settled;
+ await nextTick();
+
+ expect(observedHookResult).toBe(false);
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
+ 'conversation-123',
+ 'complete',
+ undefined,
+ 1000,
+ { persistencePending: true },
+ );
+ expect(completedResponseWrite).not.toHaveBeenCalled();
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.finishTerminalJob).not.toHaveBeenCalled();
+ });
+
+ it('finishes a claimed request job when FINAL publication throws', async () => {
+ const userMessage = {
+ messageId: 'user-msg',
+ parentMessageId: 'parent-msg',
+ conversationId: 'conversation-123',
+ text: 'Finish despite transport failure.',
+ };
+ mockGenerationJobManager.getJob.mockResolvedValue({ createdAt: 1000, status: 'running' });
+ mockGenerationJobManager.publishTerminalClaim.mockRejectedValue(
+ new Error('done publish failed'),
+ );
+ let signalFinished;
+ const finished = new Promise((resolve) => {
+ signalFinished = resolve;
+ });
+ mockGenerationJobManager.finishTerminalJob.mockImplementation(async () => signalFinished());
+ const client = {
+ options: {},
+ savedMessageIds: new Set(),
+ skipSaveUserMessage: false,
+ sendMessage: jest.fn(async (_text, options) => {
+ options.onStart(userMessage, 'response-msg');
+ return {
+ messageId: 'response-msg',
+ databasePromise: Promise.resolve({
+ conversation: { conversationId: 'conversation-123', title: 'Existing' },
+ }),
+ };
+ }),
+ };
+ const req = {
+ user: { id: 'user-123' },
+ body: {
+ text: userMessage.text,
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: 'conversation-123',
+ endpointOption: { endpoint: 'agents', modelOptions: { model: 'gpt-4.1' } },
+ },
+ config: {},
+ };
+
+ await AgentController(
+ req,
+ createResumableResponse(),
+ jest.fn(),
+ jest.fn().mockResolvedValue({ client }),
+ null,
+ );
+ await finished;
+ await nextTick();
+
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.finishTerminalJob.mock.invocationCallOrder[0]).toBeGreaterThan(
+ mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
+ );
+ });
+
it('proceeds to create the job when it wins the idempotency claim', async () => {
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
const initializeClient = jest.fn().mockRejectedValue(new Error('stop before tool loading'));
const req = {
user: { id: 'user-123' },
@@ -1140,7 +3032,7 @@ describe('ResumableAgentController resume metadata', () => {
});
it('releases the idempotency claim on a 429 only when it won the claim', async () => {
- mockGenerationJobManager.claimGeneration.mockResolvedValue({ claimed: true });
+ mockGenerationJobManager.claimGeneration.mockResolvedValue(wonGenerationClaim());
mockCheckAndIncrementPendingRequest.mockResolvedValue({
allowed: false,
pendingRequests: 3,
@@ -1162,11 +3054,16 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), jest.fn(), null);
expect(res.status).toHaveBeenCalledWith(429);
- expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith('user-123', 'req-abc');
+ expect(mockGenerationJobManager.releaseGeneration).toHaveBeenCalledWith(
+ 'user-123',
+ 'req-abc',
+ 'conversation-123',
+ DEFAULT_OWNED_CLAIM,
+ );
expect(mockStartupTelemetry.end).toHaveBeenCalledWith('rejected');
});
- it('does not release a claim it never won when a fail-open duplicate hits the limiter', async () => {
+ it('fails closed before the limiter when generation ownership is ambiguous', async () => {
mockGenerationJobManager.claimGeneration.mockRejectedValue(new Error('redis down'));
mockCheckAndIncrementPendingRequest.mockResolvedValue({
allowed: false,
@@ -1188,7 +3085,10 @@ describe('ResumableAgentController resume metadata', () => {
await AgentController(req, res, jest.fn(), jest.fn(), null);
- expect(res.status).toHaveBeenCalledWith(429);
+ expect(res.status).toHaveBeenCalledWith(503);
+ expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'SERVER_NOT_READY' }));
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.createJob).not.toHaveBeenCalled();
expect(mockGenerationJobManager.releaseGeneration).not.toHaveBeenCalled();
});
});
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js
index 56e0a7c8c9..1fa910dbc9 100644
--- a/api/server/controllers/agents/__tests__/resume.spec.js
+++ b/api/server/controllers/agents/__tests__/resume.spec.js
@@ -29,6 +29,7 @@ const TENANT_ID = 'tenant-1';
const AGENT_ID = 'agent-abc';
const CONVO_ID = 'convo-123';
const ACTION_ID = 'action-xyz';
+const NEXT_ACTION_ID = 'action-next';
const RESPONSE_MSG_ID = 'resp-1';
const USER_MSG_ID = 'umsg-1';
const THREAD_PARENT_ID = 'thread-parent-1';
@@ -50,16 +51,22 @@ const mockGenerationJobManager = {
getJobStore: jest.fn(() => mockJobStore),
getResumeState: jest.fn(),
setContentParts: jest.fn(),
- /** Resume moves ownership: the new owner records its own seal capability
- * and rebuilds armed interrupts from the durable queue. */
- updateMetadata: jest.fn().mockResolvedValue(undefined),
+ /** Resume moves ownership and rebuilds armed interrupts from the durable queue. */
rearmQueuedPreempts: jest.fn().mockResolvedValue(0),
emitChunk: jest.fn(),
emitDone: jest.fn(),
emitError: jest.fn(),
+ claimTerminalJob: jest.fn(),
+ publishTerminalClaim: jest.fn(),
+ finishTerminalJob: jest.fn(),
completeJob: jest.fn(),
+ failPausePersistence: jest.fn(),
expireApproval: jest.fn(),
- approvals: { resolve: jest.fn() },
+ approvals: {
+ resolve: jest.fn(),
+ ownsPausePersistence: jest.fn(),
+ finishPausePersistence: jest.fn(),
+ },
};
const mockDeleteAgentCheckpoint = jest.fn();
@@ -86,6 +93,7 @@ jest.mock('@librechat/api', () => ({
deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
checkAndIncrementPendingRequest: (...args) => mockCheckAndIncrementPendingRequest(...args),
+ isSteerPreemptSupported: jest.fn(() => true),
}));
jest.mock('~/models', () => ({
@@ -191,7 +199,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
checkpointIds: ['checkpoint-old'],
});
mockCleanupMCPRequestContextForReq.mockResolvedValue(undefined);
- mockSaveMessage.mockResolvedValue(undefined);
+ mockSaveMessage.mockResolvedValue({});
mockGetConvo.mockResolvedValue(null);
mockGetMessages.mockResolvedValue([]);
mockJobStore.getJob.mockResolvedValue({
@@ -204,8 +212,30 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.emitDone.mockResolvedValue(undefined);
mockGenerationJobManager.emitError.mockResolvedValue(undefined);
mockGenerationJobManager.emitChunk.mockResolvedValue(undefined);
- mockGenerationJobManager.completeJob.mockResolvedValue(undefined);
+ mockGenerationJobManager.claimTerminalJob.mockResolvedValue({
+ streamId: CONVO_ID,
+ createdAt: 1000,
+ status: 'complete',
+ persistencePending: true,
+ drainedSteers: [],
+ });
+ mockGenerationJobManager.publishTerminalClaim.mockImplementation(
+ async (_claim, finalEvent) => ({
+ finalEvent: finalEvent ?? {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'error',
+ },
+ persistenceFailed: finalEvent == null,
+ }),
+ );
+ mockGenerationJobManager.finishTerminalJob.mockResolvedValue(undefined);
+ mockGenerationJobManager.completeJob.mockResolvedValue(true);
+ mockGenerationJobManager.failPausePersistence.mockResolvedValue(true);
mockGenerationJobManager.approvals.resolve.mockResolvedValue(true);
+ mockGenerationJobManager.approvals.ownsPausePersistence.mockResolvedValue(true);
+ mockGenerationJobManager.approvals.finishPausePersistence.mockResolvedValue(true);
// `decrementPendingRequest` runs in the controller's `finally` on every
// post-ACK path, so resolving on it signals the async continuation is done.
@@ -217,13 +247,15 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
});
mockAddTitle = jest.fn().mockResolvedValue(undefined);
- mockInitializeClient = jest.fn(async ({ req }) => {
+ mockInitializeClient = jest.fn(async ({ req, checkpointNamespace }) => {
// Capture the request state the controller seeds BEFORE reconstruction.
capturedInit = {
parentMessageId: req.body.parentMessageId,
files: req.body.files,
+ isTemporary: req.body.isTemporary,
conversationCreatedAt: req.conversationCreatedAt,
timezone: req.body.timezone,
+ checkpointNamespace,
};
return { client: makeClient(), userMCPAuthMap: { server1: { token: 't' } } };
});
@@ -266,26 +298,19 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
});
/**
- * `approvals.resolve` has already consumed the action and flipped the job
- * to running by the time steering bookkeeping runs. ioredis queues rather
- * than rejects during an outage, so `.catch` never fires and an unbounded
- * await would strand the client: its retry gets a 409 for a spent action,
- * and neither the continuation nor the failed-resume cleanup runs.
+ * `approvals.resolve` atomically records capability and flips the job to
+ * running. Durable-arm reconstruction follows; ioredis queues rather than
+ * rejects during an outage, so an unbounded await there would strand the
+ * client after the action is already spent.
*/
it('answers the resume even when steering bookkeeping never settles', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
let releaseRearm;
- let releaseMetadata;
mockGenerationJobManager.rearmQueuedPreempts.mockReturnValue(
new Promise((resolve) => {
releaseRearm = () => resolve(0);
}),
);
- mockGenerationJobManager.updateMetadata.mockReturnValue(
- new Promise((resolve) => {
- releaseMetadata = () => resolve(undefined);
- }),
- );
try {
const res = await post(approveBody());
@@ -293,9 +318,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(res.body.status).toBe('resuming');
} finally {
releaseRearm?.();
- releaseMetadata?.();
mockGenerationJobManager.rearmQueuedPreempts.mockResolvedValue(0);
- mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
}
}, 15000);
@@ -360,6 +383,18 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
+ it('fails closed when the stored job owner is missing', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(
+ makeToolApprovalJob({ metadata: { userId: undefined } }),
+ );
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(403);
+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
+ expect(mockInitializeClient).not.toHaveBeenCalled();
+ });
+
it('403 on a tenant mismatch', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(
makeToolApprovalJob({ metadata: { tenantId: 'other-tenant' } }),
@@ -397,6 +432,17 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
});
+ it('409s a resume targeting a replaced generation before claiming the action', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+
+ const res = await post(approveBody({ generationCreatedAt: 999 }));
+
+ expect(res.status).toBe(409);
+ expect(res.body).toEqual({ code: 'RUN_REPLACED', generationProtocolVersion: 1 });
+ expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
+ expect(mockInitializeClient).not.toHaveBeenCalled();
+ });
+
it('409 when the job is not in requires_action (already terminal; no expire)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob({ status: 'running' }));
const res = await post(approveBody());
@@ -415,7 +461,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(res.body.error).toMatch(/no live pending action/i);
// The stale action is expired NOW (expire CAS + terminal SSE) so an attached SSE
// client gets a terminal event instead of hanging until the periodic sweeper runs.
- expect(mockGenerationJobManager.expireApproval).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
+ expect(mockGenerationJobManager.expireApproval).toHaveBeenCalledWith(
+ CONVO_ID,
+ ACTION_ID,
+ 1000,
+ );
});
it('400 when actionId is missing', async () => {
@@ -527,7 +577,12 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(200);
- expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
+ CONVO_ID,
+ ACTION_ID,
+ { preemptCapable: true },
+ 1000,
+ );
await settled;
await flush();
});
@@ -591,7 +646,12 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.getJob.mockResolvedValue(job);
const res = await post(approveBody());
expect(res.status).toBe(200);
- expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
+ CONVO_ID,
+ ACTION_ID,
+ { preemptCapable: true },
+ 1000,
+ );
await settled;
await flush();
});
@@ -625,7 +685,33 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
const res = await post(approveBody());
expect(res.status).toBe(409);
expect(res.body.error).toMatch(/already resolved or has expired/i);
- expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
+ CONVO_ID,
+ ACTION_ID,
+ { preemptCapable: true },
+ 1000,
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
+ expect(mockInitializeClient).not.toHaveBeenCalled();
+ });
+
+ it('409s RUN_REPLACED when the generation changes before the resume CAS', async () => {
+ const original = makeToolApprovalJob();
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce(original)
+ .mockResolvedValueOnce(makeToolApprovalJob({ createdAt: 2000 }));
+ mockGenerationJobManager.approvals.resolve.mockResolvedValue(false);
+
+ const res = await post(approveBody({ generationCreatedAt: 1000 }));
+
+ expect(res.status).toBe(409);
+ expect(res.body).toEqual({ code: 'RUN_REPLACED', generationProtocolVersion: 1 });
+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
+ CONVO_ID,
+ ACTION_ID,
+ { preemptCapable: true },
+ 1000,
+ );
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockInitializeClient).not.toHaveBeenCalled();
});
@@ -652,11 +738,17 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
streamId: CONVO_ID,
conversationId: CONVO_ID,
status: 'resuming',
+ generationProtocolVersion: 1,
});
expect(mockCaptureAgentCheckpointGeneration).toHaveBeenCalledWith(CONVO_ID, {
type: 'mongo',
});
- expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(CONVO_ID, ACTION_ID);
+ expect(mockGenerationJobManager.approvals.resolve).toHaveBeenCalledWith(
+ CONVO_ID,
+ ACTION_ID,
+ { preemptCapable: true },
+ 1000,
+ );
expect(mockCaptureAgentCheckpointGeneration.mock.invocationCallOrder[0]).toBeLessThan(
mockGenerationJobManager.approvals.resolve.mock.invocationCallOrder[0],
);
@@ -664,6 +756,25 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await flush();
});
+ it('preserves and exactly echoes a paused generation protocol v2 marker', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(
+ makeToolApprovalJob({ metadata: { generationProtocolVersion: 2 } }),
+ );
+
+ const res = await post(approveBody({ generationProtocolVersion: 2 }));
+
+ expect(res.status).toBe(200);
+ expect(res.headers['x-librechat-generation-protocol']).toBe('2');
+ expect(res.body).toEqual({
+ streamId: CONVO_ID,
+ conversationId: CONVO_ID,
+ status: 'resuming',
+ generationProtocolVersion: 2,
+ });
+ await settled;
+ await flush();
+ });
+
it('seeds the thread parent before reconstruction and maps the decision to the SDK', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
await post(approveBody());
@@ -684,6 +795,24 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
);
});
+ it('reuses the persisted generation checkpoint namespace and keeps legacy fallback explicit', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(
+ makeToolApprovalJob({ metadata: { checkpointNamespace: 'generation-1000' } }),
+ );
+ await post(approveBody());
+ await settled;
+ await flush();
+
+ expect(capturedInit.checkpointNamespace).toBe('generation-1000');
+ expect(mockCaptureAgentCheckpointGeneration).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ CONVO_ID,
+ { type: 'mongo' },
+ undefined,
+ { checkpointNamespace: 'generation-1000' },
+ );
+ });
+
it('passes persisted run steps into the rebuilt run for tool-result correlation', async () => {
const runSteps = [
{
@@ -724,6 +853,24 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(capturedInit.files).toEqual([{ file_id: 'f1' }]);
});
+ it.each([
+ { stored: true, supplied: false },
+ { stored: false, supplied: true },
+ ])(
+ 'restores authoritative isTemporary=$stored before reconstruction',
+ async ({ stored, supplied }) => {
+ mockGenerationJobManager.getJob.mockResolvedValue(
+ makeToolApprovalJob({ metadata: { isTemporary: stored } }),
+ );
+
+ const res = await post(approveBody({ isTemporary: supplied }));
+ expect(res.status).toBe(200);
+ await settled;
+
+ expect(capturedInit.isTemporary).toBe(stored);
+ },
+ );
+
it('ignores client-supplied resume files, sourcing from the paused job (security)', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
// The paused turn's authoritative files (DB row); a crafted client tries to swap them.
@@ -780,7 +927,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await settled;
await flush();
- const [, finalEvent] = mockGenerationJobManager.emitDone.mock.calls[0];
+ const [, finalEvent] = mockGenerationJobManager.publishTerminalClaim.mock.calls[0];
expect(finalEvent.requestMessage).toMatchObject({
messageId: USER_MSG_ID,
isCreatedByUser: true,
@@ -788,7 +935,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
});
});
- it('persists the finished response, emits done, completes the job, and prunes the checkpoint', async () => {
+ it('persists the response, claims terminal ownership, emits done, finishes, and prunes', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
await post(approveBody());
await settled;
@@ -815,8 +962,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
// Assert the finalEvent STRUCTURE, not just the hardcoded `final: true` literal —
// a `final: true`-only check would still pass if the entire content / title /
// requestMessage build in finalizeResumedTurn were deleted.
- const [doneStreamId, finalEvent] = mockGenerationJobManager.emitDone.mock.calls[0];
- expect(doneStreamId).toBe(CONVO_ID);
+ const [terminalClaim, finalEvent] =
+ mockGenerationJobManager.publishTerminalClaim.mock.calls[0];
+ expect(terminalClaim).toMatchObject({ streamId: CONVO_ID, persistencePending: true });
expect(finalEvent).toMatchObject({
final: true,
conversation: { conversationId: CONVO_ID },
@@ -829,7 +977,24 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
});
expect(typeof finalEvent.title).toBe('string');
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, undefined, 1000);
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
+ CONVO_ID,
+ 'complete',
+ undefined,
+ 1000,
+ { persistencePending: true },
+ );
+ const claim = await mockGenerationJobManager.claimTerminalJob.mock.results[0].value;
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(claim);
+ expect(mockGenerationJobManager.claimTerminalJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockSaveMessage.mock.invocationCallOrder[0],
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
+ );
+ expect(
+ mockGenerationJobManager.publishTerminalClaim.mock.invocationCallOrder[0],
+ ).toBeLessThan(mockGenerationJobManager.finishTerminalJob.mock.invocationCallOrder[0]);
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
{ type: 'mongo' },
@@ -839,6 +1004,27 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
});
+ it('publishes reconciliation instead of a normal FINAL when the resumed response save returns no row', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockGenerationJobManager.completeJob.mockResolvedValue(false);
+ mockSaveMessage.mockResolvedValue(undefined);
+
+ await post(approveBody());
+ await settled;
+ await flush();
+
+ const claim = await mockGenerationJobManager.claimTerminalJob.mock.results[0].value;
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledTimes(1);
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(claim, null);
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(claim);
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ CONVO_ID,
+ 'Resumed response could not be persisted before terminal publication',
+ 1000,
+ );
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledTimes(1);
+ });
+
it('degrades a failed checkpoint snapshot to scoped no-op cleanup', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockCaptureAgentCheckpointGeneration.mockRejectedValue(new Error('mongo down'));
@@ -854,7 +1040,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
);
});
- it('skips finalization (no save/emitDone/complete) when the job was replaced mid-resume', async () => {
+ it('skips finalization when the job was replaced mid-resume', async () => {
// The paused job has createdAt 1000; a concurrent request reused this conversationId,
// so the live job now has a different createdAt — finalizing would clobber the newer
// turn's job. The finally still runs (slot release), so `settled` resolves.
@@ -869,8 +1055,69 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await flush();
expect(mockSaveMessage).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.claimTerminalJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.finishTerminalJob).not.toHaveBeenCalled();
+ });
+
+ it('does not write a completed response when a pause or abort wins the terminal claim', async () => {
+ const job = makeToolApprovalJob();
+ mockGenerationJobManager.getJob.mockResolvedValue(job);
+ let signalResumeBoundary;
+ const resumeBoundary = new Promise((resolve) => {
+ signalResumeBoundary = resolve;
+ });
+ let releaseResume;
+ const resumeGate = new Promise((resolve) => {
+ releaseResume = resolve;
+ });
+ const client = makeClient({
+ resumeCompletion: jest.fn(async () => {
+ signalResumeBoundary();
+ await resumeGate;
+ }),
+ });
+ mockInitializeClient.mockResolvedValue({ client, userMCPAuthMap: {} });
+
+ await post(approveBody());
+ await resumeBoundary;
+ // Cross-replica Stop wins durably while its abort pub/sub is delayed, so
+ // the local signal remains live and only the terminal CAS can fence this
+ // resumed completion's Mongo write.
+ mockGenerationJobManager.claimTerminalJob.mockResolvedValue(null);
+ releaseResume();
+ await settled;
+ await flush();
+
+ expect(job.abortController.signal.aborted).toBe(false);
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
+ CONVO_ID,
+ 'complete',
+ undefined,
+ 1000,
+ { persistencePending: true },
+ );
+ expect(mockSaveMessage).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.finishTerminalJob).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ });
+
+ it('finishes a winning terminal claim even when FINAL publication throws', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockGenerationJobManager.publishTerminalClaim.mockRejectedValue(new Error('transport down'));
+
+ await post(approveBody());
+ await settled;
+ await flush();
+
+ const claim = await mockGenerationJobManager.claimTerminalJob.mock.results[0].value;
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledWith(claim);
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(
+ CONVO_ID,
+ 'transport down',
+ 1000,
+ );
});
it('does not release the slot in the finally when the client already released it on pause', async () => {
@@ -1014,7 +1261,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await settled;
await flush();
- const [, finalEvent] = mockGenerationJobManager.emitDone.mock.calls[0];
+ const [, finalEvent] = mockGenerationJobManager.publishTerminalClaim.mock.calls[0];
expect(finalEvent.requestMessage).toMatchObject({
manualSkills: ['skill-a'],
alwaysAppliedSkills: ['skill-b'],
@@ -1057,7 +1304,13 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(client.resumeCompletion).toHaveBeenCalledWith(
expect.objectContaining({ resumeValue: { answer: 'call it report.pdf' } }),
);
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, undefined, 1000);
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
+ CONVO_ID,
+ 'complete',
+ undefined,
+ 1000,
+ { persistencePending: true },
+ );
});
it('generates a title for a first-turn pause before completing the stream', async () => {
@@ -1071,8 +1324,17 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await flush();
expect(mockAddTitle).toHaveBeenCalledTimes(1);
- // Title is emitted (and the job completed) — order matters but both must happen.
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, undefined, 1000);
+ // Terminal ownership precedes title persistence and final stream publication.
+ expect(mockGenerationJobManager.claimTerminalJob).toHaveBeenCalledWith(
+ CONVO_ID,
+ 'complete',
+ undefined,
+ 1000,
+ { persistencePending: true },
+ );
+ expect(mockGenerationJobManager.claimTerminalJob.mock.invocationCallOrder[0]).toBeLessThan(
+ mockAddTitle.mock.invocationCallOrder[0],
+ );
});
it('still finalizes the turn when first-turn title generation throws', async () => {
@@ -1089,12 +1351,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockLogger.error).toHaveBeenCalled();
expect(mockSaveMessage).toHaveBeenCalledTimes(1);
- expect(mockGenerationJobManager.emitDone).toHaveBeenCalledWith(
- CONVO_ID,
+ expect(mockGenerationJobManager.publishTerminalClaim).toHaveBeenCalledWith(
+ expect.objectContaining({ streamId: CONVO_ID }),
expect.any(Object),
- 1000,
);
- expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, undefined, 1000);
+ expect(mockGenerationJobManager.finishTerminalJob).toHaveBeenCalledTimes(1);
});
});
@@ -1102,7 +1363,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
it('re-pause: does not finalize when the run pauses again', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
- client: makeClient({ pendingApproval: true }),
+ client: makeClient({ pendingApproval: { actionId: NEXT_ACTION_ID } }),
userMCPAuthMap: {},
});
@@ -1117,9 +1378,21 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect.objectContaining({ unfinished: true }),
expect.anything(),
);
- expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.claimTerminalJob).not.toHaveBeenCalled();
expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.approvals.ownsPausePersistence).toHaveBeenCalledWith(
+ CONVO_ID,
+ NEXT_ACTION_ID,
+ 1000,
+ );
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).toHaveBeenCalledWith(
+ CONVO_ID,
+ NEXT_ACTION_ID,
+ 1000,
+ );
+ expect(mockGenerationJobManager.failPausePersistence).not.toHaveBeenCalled();
// The slot is still released and the client disposed.
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
expect(mockDisposeClient).toHaveBeenCalledTimes(1);
@@ -1129,7 +1402,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
- pendingApproval: true,
+ pendingApproval: { actionId: NEXT_ACTION_ID },
contentParts: [{ type: 'text', text: 'streamed this segment' }],
artifactPromises: [],
}),
@@ -1151,7 +1424,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
context: 'api/server/controllers/agents/resume.js - re-pause progress persist',
}),
);
- expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
});
it('re-pause: persists artifacts produced before pausing again (unfinished)', async () => {
@@ -1159,7 +1432,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
- pendingApproval: true,
+ pendingApproval: { actionId: NEXT_ACTION_ID },
artifactPromises: [Promise.resolve(artifact)],
}),
userMCPAuthMap: {},
@@ -1172,7 +1445,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
// No finalize, but the segment's artifact is persisted unfinished so the next
// resume's finalize can merge it (otherwise the fresh client drops it).
- expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockSaveMessage).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ attachments: [artifact], unfinished: true }),
@@ -1182,6 +1455,145 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
);
});
+ it('re-pause: holds the exact Stop/resume barrier until progress persistence settles', async () => {
+ let signalSaveStarted;
+ const saveStarted = new Promise((resolve) => {
+ signalSaveStarted = resolve;
+ });
+ let releaseSave;
+ const saveGate = new Promise((resolve) => {
+ releaseSave = resolve;
+ });
+ mockSaveMessage.mockImplementation(async () => {
+ signalSaveStarted();
+ await saveGate;
+ return {};
+ });
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ pendingApproval: { actionId: NEXT_ACTION_ID },
+ contentParts: [{ type: 'text', text: 'must be durable before another action' }],
+ }),
+ userMCPAuthMap: {},
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await saveStarted;
+
+ expect(mockGenerationJobManager.approvals.ownsPausePersistence).toHaveBeenCalledWith(
+ CONVO_ID,
+ NEXT_ACTION_ID,
+ 1000,
+ );
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockDecrementPendingRequest).not.toHaveBeenCalled();
+
+ releaseSave();
+ await settled;
+ await flush();
+
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).toHaveBeenCalledWith(
+ CONVO_ID,
+ NEXT_ACTION_ID,
+ 1000,
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.approvals.finishPausePersistence.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('re-pause: terminalizes instead of exposing the next action when persistence fails', async () => {
+ mockSaveMessage.mockRejectedValue(new Error('re-pause save failed'));
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ pendingApproval: { actionId: NEXT_ACTION_ID },
+ contentParts: [{ type: 'text', text: 'progress that must not be lost' }],
+ }),
+ userMCPAuthMap: {},
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockGenerationJobManager.failPausePersistence).toHaveBeenCalledWith(
+ CONVO_ID,
+ NEXT_ACTION_ID,
+ 're-pause save failed',
+ 1000,
+ );
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ CONVO_ID,
+ { type: 'mongo' },
+ { threadId: CONVO_ID, checkpointIds: ['checkpoint-old'] },
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
+ mockGenerationJobManager.failPausePersistence.mock.invocationCallOrder[0],
+ );
+ });
+
+ it('re-pause: contains checkpoint deletion failure after exact persistence terminalization', async () => {
+ const checkpointError = new Error('checkpoint delete unavailable');
+ mockSaveMessage.mockRejectedValue(new Error('re-pause save failed'));
+ mockDeleteAgentCheckpoint.mockRejectedValue(checkpointError);
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ pendingApproval: { actionId: NEXT_ACTION_ID },
+ contentParts: [{ type: 'text', text: 'progress that must not be lost' }],
+ }),
+ userMCPAuthMap: {},
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockGenerationJobManager.failPausePersistence).toHaveBeenCalledTimes(1);
+ expect(mockLogger.error).toHaveBeenCalledWith(
+ '[ResumeAgentController] Failed to prune checkpoint after re-pause persistence failure',
+ checkpointError,
+ );
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
+ expect(mockDisposeClient).toHaveBeenCalledTimes(1);
+ });
+
+ it('re-pause: leaves a replacement checkpoint untouched when exact persistence failure loses ownership', async () => {
+ mockSaveMessage.mockResolvedValue(undefined);
+ mockGenerationJobManager.failPausePersistence.mockResolvedValue(false);
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ pendingApproval: { actionId: NEXT_ACTION_ID },
+ contentParts: [{ type: 'text', text: 'stale segment progress' }],
+ }),
+ userMCPAuthMap: {},
+ });
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockGenerationJobManager.failPausePersistence).toHaveBeenCalledWith(
+ CONVO_ID,
+ NEXT_ACTION_ID,
+ 'Re-pause response progress could not be persisted',
+ 1000,
+ );
+ expect(mockGenerationJobManager.approvals.finishPausePersistence).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ });
+
it('abort-during-resume: lets the abort route finalize, does not double-save', async () => {
const job = makeToolApprovalJob();
mockGenerationJobManager.getJob.mockResolvedValue(job);
@@ -1196,12 +1608,12 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await flush();
expect(mockSaveMessage).not.toHaveBeenCalled();
- expect(mockGenerationJobManager.emitDone).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.publishTerminalClaim).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
});
- it('resume failure: emits an error, finalizes the job, and prunes the checkpoint', async () => {
+ it('resume failure delegates single-winner error publication and prunes the checkpoint', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
@@ -1215,7 +1627,7 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
await settled;
await flush();
- expect(mockGenerationJobManager.emitError).toHaveBeenCalledWith(CONVO_ID, 'boom', 1000);
+ expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, 'boom', 1000);
expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
CONVO_ID,
@@ -1226,7 +1638,9 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockSaveMessage).not.toHaveBeenCalled();
});
- it('forces a terminal job state when completeJob also fails during a resume error', async () => {
+ it('contains checkpoint deletion failure after winning ordinary resume-error finalization', async () => {
+ const checkpointError = new Error('checkpoint delete unavailable');
+ mockDeleteAgentCheckpoint.mockRejectedValue(checkpointError);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
mockInitializeClient.mockResolvedValue({
client: makeClient({
@@ -1234,19 +1648,61 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
}),
userMCPAuthMap: {},
});
- // The error path's completeJob also fails → last-resort updateJob must force a
- // terminal state so the job isn't orphaned in `running`.
+
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ await settled;
+ await flush();
+
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, 'boom', 1000);
+ expect(mockLogger.error).toHaveBeenCalledWith(
+ '[ResumeAgentController] Failed to prune checkpoint after failed resume finalization',
+ checkpointError,
+ );
+ expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
+ expect(mockDisposeClient).toHaveBeenCalledTimes(1);
+ });
+
+ it('keeps a re-pause checkpoint when failed-resume terminal ownership is lost', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(
+ makeToolApprovalJob({ metadata: { checkpointNamespace: 'generation-1000' } }),
+ );
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
+ }),
+ userMCPAuthMap: {},
+ });
+ // A fresh interrupt moved the same epoch back to requires_action before
+ // the error path's terminal CAS. Its newly written checkpoint is live.
+ mockGenerationJobManager.completeJob.mockResolvedValue(false);
+
+ await post(approveBody());
+ await settled;
+ await flush();
+
+ expect(mockGenerationJobManager.completeJob).toHaveBeenCalledWith(CONVO_ID, 'boom', 1000);
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ });
+
+ it('does not overwrite the job when guarded error finalization itself fails', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ mockInitializeClient.mockResolvedValue({
+ client: makeClient({
+ resumeCompletion: jest.fn().mockRejectedValue(new Error('boom')),
+ }),
+ userMCPAuthMap: {},
+ });
+ // An unconditional fallback update could overwrite a pause or replacement
+ // that won while completeJob was failing.
mockGenerationJobManager.completeJob.mockRejectedValue(new Error('complete failed'));
await post(approveBody());
await settled;
await flush();
- expect(mockJobStore.updateJob).toHaveBeenCalledWith(
- CONVO_ID,
- expect.objectContaining({ status: 'error', error: 'Resume failed' }),
- 1000,
- );
+ expect(mockJobStore.updateJob).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
expect(mockDecrementPendingRequest).toHaveBeenCalledWith(USER_ID);
});
});
diff --git a/api/server/controllers/agents/__tests__/steer.spec.js b/api/server/controllers/agents/__tests__/steer.spec.js
index a0108b3cbc..1ff636e1d1 100644
--- a/api/server/controllers/agents/__tests__/steer.spec.js
+++ b/api/server/controllers/agents/__tests__/steer.spec.js
@@ -2,6 +2,8 @@ const express = require('express');
const request = require('supertest');
const mockHandleSteerRequest = jest.fn();
+const mockHandleSteerCancel = jest.fn();
+const mockHandleSteerArm = jest.fn();
const mockCheckAccess = jest.fn();
const mockCheckPermission = jest.fn();
const mockHasCapability = jest.fn();
@@ -15,7 +17,10 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
+ GenerationJobManager: { isRedis: false },
handleSteerRequest: (...args) => mockHandleSteerRequest(...args),
+ handleSteerCancel: (...args) => mockHandleSteerCancel(...args),
+ handleSteerArm: (...args) => mockHandleSteerArm(...args),
checkAccess: (...args) => mockCheckAccess(...args),
}));
@@ -36,6 +41,9 @@ jest.mock('~/models', () => ({
const { Permissions, PermissionTypes, PermissionBits } = require('librechat-data-provider');
const SteerController = require('~/server/controllers/agents/steer');
+const { SteerCancelController, SteerArmController } = SteerController;
+
+const GENERATION_PROTOCOL_HEADER = 'x-librechat-generation-protocol';
/**
* The guard ladder itself (validation, file sanitization, ownership, enqueue
@@ -52,6 +60,8 @@ function buildApp(user = { id: 'user-1', tenantId: 'tenant-1' }) {
next();
});
app.post('/chat/steer', SteerController);
+ app.post('/chat/steer/cancel', SteerCancelController);
+ app.post('/chat/steer/arm', SteerArmController);
return app;
}
@@ -60,10 +70,16 @@ describe('SteerController (wrapper)', () => {
jest.clearAllMocks();
});
- it('serializes the handler result verbatim', async () => {
+ it('defaults an unmarked request to v1 and serializes the marker in body and header', async () => {
mockHandleSteerRequest.mockResolvedValue({
status: 202,
- body: { status: 'queued', steerId: 's1', position: 1, conversationId: 'c1' },
+ body: {
+ status: 'queued',
+ steerId: 's1',
+ position: 1,
+ conversationId: 'c1',
+ generationProtocolVersion: 1,
+ },
});
const res = await request(buildApp())
@@ -76,11 +92,14 @@ describe('SteerController (wrapper)', () => {
steerId: 's1',
position: 1,
conversationId: 'c1',
+ generationProtocolVersion: 1,
});
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('1');
expect(mockHandleSteerRequest).toHaveBeenCalledWith(
{ id: 'user-1', tenantId: 'tenant-1' },
{ conversationId: 'c1', text: 'hello', files: [{ file_id: 'f1' }] },
{
+ generationProtocolVersion: 1,
getFiles: expect.any(Function),
updateFilesUsage: expect.any(Function),
checkAgentAccess: expect.any(Function),
@@ -89,12 +108,17 @@ describe('SteerController (wrapper)', () => {
});
it('passes rejection statuses through untouched', async () => {
- mockHandleSteerRequest.mockResolvedValue({ status: 409, body: { code: 'RUN_PAUSED' } });
+ mockHandleSteerRequest.mockResolvedValue({
+ status: 409,
+ body: { code: 'RUN_PAUSED', generationProtocolVersion: 1 },
+ });
const res = await request(buildApp()).post('/chat/steer').send({ conversationId: 'c1' });
expect(res.status).toBe(409);
expect(res.body.code).toBe('RUN_PAUSED');
+ expect(res.body.generationProtocolVersion).toBe(1);
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('1');
});
it('500s with STEER_FAILED when the handler throws', async () => {
@@ -105,9 +129,113 @@ describe('SteerController (wrapper)', () => {
.send({ conversationId: 'c1', text: 'x' });
expect(res.status).toBe(500);
- expect(res.body.code).toBe('STEER_FAILED');
+ expect(res.body).toEqual({ code: 'STEER_FAILED', generationProtocolVersion: 1 });
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('1');
expect(mockLogger.error).toHaveBeenCalled();
});
+
+ it('passes an exact body+header v2 marker through the server rollout gate', async () => {
+ mockHandleSteerRequest.mockResolvedValue({
+ status: 202,
+ body: { status: 'queued', generationProtocolVersion: 2 },
+ });
+
+ const res = await request(buildApp())
+ .post('/chat/steer')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'c1', text: 'hello', generationProtocolVersion: 2 });
+
+ expect(mockHandleSteerRequest.mock.calls[0][2]).toEqual(
+ expect.objectContaining({ generationProtocolVersion: 2 }),
+ );
+ expect(res.body.generationProtocolVersion).toBe(2);
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('2');
+ });
+
+ it.each([
+ ['conflicting', '1'],
+ ['malformed', 'not-a-version'],
+ ])('downgrades %s body/header markers to v1', async (_label, header) => {
+ mockHandleSteerRequest.mockResolvedValue({
+ status: 202,
+ body: { status: 'queued', generationProtocolVersion: 1 },
+ });
+
+ const res = await request(buildApp())
+ .post('/chat/steer')
+ .set('X-LibreChat-Generation-Protocol', header)
+ .send({ conversationId: 'c1', text: 'hello', generationProtocolVersion: 2 });
+
+ expect(mockHandleSteerRequest.mock.calls[0][2]).toEqual(
+ expect.objectContaining({ generationProtocolVersion: 1 }),
+ );
+ expect(res.body.generationProtocolVersion).toBe(1);
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('1');
+ });
+
+ it('honors a v1 server rollout gate even when the request advertises exact v2', async () => {
+ const previous = process.env.GENERATION_PROTOCOL_VERSION;
+ process.env.GENERATION_PROTOCOL_VERSION = '1';
+ mockHandleSteerRequest.mockResolvedValue({
+ status: 202,
+ body: { status: 'queued', generationProtocolVersion: 1 },
+ });
+ try {
+ await request(buildApp())
+ .post('/chat/steer')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'c1', text: 'hello', generationProtocolVersion: 2 });
+
+ expect(mockHandleSteerRequest.mock.calls[0][2]).toEqual(
+ expect.objectContaining({ generationProtocolVersion: 1 }),
+ );
+ } finally {
+ if (previous == null) {
+ delete process.env.GENERATION_PROTOCOL_VERSION;
+ } else {
+ process.env.GENERATION_PROTOCOL_VERSION = previous;
+ }
+ }
+ });
+
+ it('uses the package job cap, not the host maximum, for the final response marker', async () => {
+ mockHandleSteerRequest.mockResolvedValue({
+ status: 202,
+ body: { status: 'queued', generationProtocolVersion: 1 },
+ });
+
+ const res = await request(buildApp())
+ .post('/chat/steer')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: 'c1', text: 'hello', generationProtocolVersion: 2 });
+
+ expect(mockHandleSteerRequest.mock.calls[0][2].generationProtocolVersion).toBe(2);
+ expect(res.body.generationProtocolVersion).toBe(1);
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('1');
+ });
+
+ it.each([
+ ['/chat/steer/cancel', mockHandleSteerCancel, { conversationId: 'c1', steerId: 's1' }],
+ ['/chat/steer/arm', mockHandleSteerArm, { conversationId: 'c1', steerId: 's1' }],
+ ])('negotiates and echoes protocol markers for %s', async (path, handler, body) => {
+ handler.mockResolvedValue({
+ status: 200,
+ body: { ok: true, generationProtocolVersion: 2 },
+ });
+
+ const res = await request(buildApp())
+ .post(path)
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ ...body, generationProtocolVersion: 2 });
+
+ expect(handler).toHaveBeenCalledWith(
+ { id: 'user-1', tenantId: 'tenant-1' },
+ { ...body, generationProtocolVersion: 2 },
+ { generationProtocolVersion: 2 },
+ );
+ expect(res.body.generationProtocolVersion).toBe(2);
+ expect(res.headers[GENERATION_PROTOCOL_HEADER]).toBe('2');
+ });
});
describe('createAgentAccessCheck (chat-route parity via job identity)', () => {
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index a01a74b839..da758b384c 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -41,7 +41,9 @@ const {
pickResumeContext,
getApprovalTtlMs,
isHITLEnabled,
+ captureAgentCheckpointGeneration,
deleteAgentCheckpoint,
+ LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
agentRequestsAskUserQuestion,
attachAskUserQuestionArgs,
hydrateResumeRunSteps,
@@ -138,6 +140,9 @@ class AgentClient extends BaseClient {
this.isChatCompletion = true;
/** @type {number | undefined} */
this.jobCreatedAt = options.jobCreatedAt;
+ /** Generation-scoped LangGraph checkpoint namespace. Legacy paused jobs
+ * intentionally use the historical empty namespace. @type {string} */
+ this.checkpointNamespace = options.checkpointNamespace ?? '';
/** @type {AgentRun} */
this.run;
@@ -300,6 +305,7 @@ class AgentClient extends BaseClient {
type: ContentTypes.STEER,
[ContentTypes.STEER]: item.text,
steerId: item.steerId,
+ ...(item.clientSteerId && { clientSteerId: item.clientSteerId }),
createdAt: item.createdAt,
...(item.files?.length && { files: item.files }),
};
@@ -308,20 +314,36 @@ class AgentClient extends BaseClient {
// durable: the chunk-log XADD is this event's recovery record — it must
// commit before the publish or a cross-replica reconnect that missed the
// pub/sub delivery reconstructs content without the steer part.
- await GenerationJobManager.emitChunk(
- streamId,
- {
- event: SteerEvents.ON_STEER_APPLIED,
- data: {
- steerId: item.steerId,
- index,
- part,
- responseMessageId: this.responseMessageId,
- conversationId: this.conversationId,
+ try {
+ await GenerationJobManager.emitChunk(
+ streamId,
+ {
+ event: SteerEvents.ON_STEER_APPLIED,
+ data: {
+ steerId: item.steerId,
+ ...(item.clientSteerId && { clientSteerId: item.clientSteerId }),
+ index,
+ part,
+ responseMessageId: this.responseMessageId,
+ conversationId: this.conversationId,
+ },
},
- },
- { durable: true, expectedCreatedAt: this.jobCreatedAt },
- );
+ {
+ durable: true,
+ expectedCreatedAt: this.jobCreatedAt,
+ deliveredSteer: item,
+ },
+ );
+ } catch (error) {
+ /** The part and its receipt commit as one durable unit. Roll the local
+ * projection back when that commit fails so the drain can restore the
+ * claimed item instead of injecting an instruction absent from replay. */
+ if (this.contentParts[index] === part) {
+ this.contentParts.splice(index, 1);
+ this.steerOffsetState.offset -= 1;
+ }
+ throw error;
+ }
}
/**
@@ -799,8 +821,11 @@ class AgentClient extends BaseClient {
* emit, whose ordering against shifted SDK indices is load-bearing.
* The chain settles on failure (warned retry), so a lost write can
* never wedge run startup. */
- this.activityLabelsMarkedPromise = GenerationJobManager.markActivityLabels(streamId).catch(() =>
- GenerationJobManager.markActivityLabels(streamId).catch(() => {
+ this.activityLabelsMarkedPromise = GenerationJobManager.markActivityLabels(
+ streamId,
+ this.jobCreatedAt,
+ ).catch(() =>
+ GenerationJobManager.markActivityLabels(streamId, this.jobCreatedAt).catch(() => {
logger.warn(
`[AgentClient] Could not flag activity labels for ${streamId}; a label resolving during a resume gap may not be reconciled.`,
);
@@ -2139,6 +2164,7 @@ class AgentClient extends BaseClient {
const paused = await GenerationJobManager.approvals.pause(streamId, pendingAction, {
expectedCreatedAt: this.jobCreatedAt,
...(discoveredTools.length > 0 ? { discoveredTools } : {}),
+ persistencePending: true,
});
if (!paused) {
logger.debug(
@@ -2214,6 +2240,11 @@ class AgentClient extends BaseClient {
runName: 'AgentRun',
configurable: {
thread_id: this.conversationId,
+ // LangGraph owns `checkpoint_ns` and resets it to '' at every root
+ // invocation. The saver maps this private immutable generation key
+ // into its physical namespace while tools keep the conversation id.
+ checkpoint_ns: '',
+ [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: this.checkpointNamespace,
last_agent_index: this.agentConfigs?.size ?? 0,
user_id: this.user ?? this.options.req.user?.id,
hide_sequential_outputs: this.options.agent.hide_sequential_outputs,
@@ -2430,27 +2461,61 @@ class AgentClient extends BaseClient {
}
const streamId = this.options.req?._resumableStreamId;
- // HITL: clear any checkpoint orphaned by a prior paused turn in this
- // conversation (one that expired or was aborted while paused) so this fresh
- // turn starts clean instead of rehydrating a stale interrupt — thread_id is
- // the stable conversationId. No-op when HITL is off or nothing is orphaned.
- // Deliberately UNCONDITIONAL per HITL turn: any cheaper gate (job metadata,
- // a Redis flag) can go stale across replicas/restarts and skip the prune
- // exactly when an orphan exists, while these are two indexed, usually-empty
- // deleteMany ops — correctness over a micro-optimization.
+ // HITL: establish an empty checkpoint barrier for THIS immutable generation
+ // before exposing its graph. A retried/recovered initialization may have left
+ // partial state in the same saver scope; a predecessor uses a different scope,
+ // so even a late remote write cannot be rehydrated or deleted here. No-op when
+ // HITL is off or the generation has no remnants. Deliberately unconditional
+ // per HITL turn: any cheaper Redis flag can go stale across replicas/restarts,
+ // while these are two indexed, usually-empty deleteMany operations.
// The gate mirrors createRun's checkpointer condition: the approval policy
// OR an ask_user_question-capable agent (which attaches a checkpointer
- // WITHOUT the approval policy) — an ask pause abandoned via job replacement
- // or Stop would otherwise rehydrate here and silently duplicate context.
+ // WITHOUT the approval policy).
//
// Start the prune alongside graph construction. The all-settled barrier
// below still guarantees it completes before the graph is exposed or run.
const shouldPruneCheckpoint =
streamId &&
(isHITLEnabled(agentsEConfig?.toolApproval) || agents.some(agentRequestsAskUserQuestion));
- const checkpointPrunePromise = shouldPruneCheckpoint
- ? deleteAgentCheckpoint(this.conversationId, agentsEConfig?.checkpointer)
- : Promise.resolve();
+ let checkpointPrunePromise = Promise.resolve();
+ if (shouldPruneCheckpoint && this.checkpointNamespace !== '') {
+ checkpointPrunePromise = deleteAgentCheckpoint(
+ this.conversationId,
+ agentsEConfig?.checkpointer,
+ undefined,
+ {
+ throwOnError: true,
+ checkpointNamespace: this.checkpointNamespace,
+ },
+ );
+ } else if (shouldPruneCheckpoint) {
+ checkpointPrunePromise = captureAgentCheckpointGeneration(
+ this.conversationId,
+ agentsEConfig?.checkpointer,
+ { throwOnError: true },
+ ).then(async (checkpointGeneration) => {
+ /** Legacy jobs share LangGraph's root/nested namespaces. Capture
+ * their immutable ids first, then prove this client still owns
+ * the exact job epoch before deleting that set. If a replacement
+ * arrived before/during capture the check fails; if it arrives
+ * after the check, its newly-written checkpoint ids are outside
+ * the snapshot and therefore cannot be deleted. */
+ const liveJob = await GenerationJobManager.getJobStore().getJob(streamId);
+ if (
+ !liveJob ||
+ liveJob.createdAt !== this.jobCreatedAt ||
+ liveJob.status !== 'running'
+ ) {
+ throw new Error('Generation replaced before legacy checkpoint cleanup');
+ }
+ await deleteAgentCheckpoint(
+ this.conversationId,
+ agentsEConfig?.checkpointer,
+ checkpointGeneration,
+ { throwOnError: true },
+ );
+ });
+ }
const createRunPromise = createRun({
agents,
@@ -2685,9 +2750,9 @@ class AgentClient extends BaseClient {
// HITL: a non-paused turn deliberately prunes nothing here. The lazy checkpointer
// (LazyMongoSaver) never persists a clean-exit checkpoint, so there is
- // nothing this turn left to delete. A checkpoint orphaned by a PRIOR abandoned pause
- // is cleared by the pre-run prune (before processStream) on the next turn, with the
- // Mongo TTL as the backstop. Dropping this post-completion prune also removes its
+ // nothing this turn left to delete. Terminal HITL owners eagerly delete their exact
+ // saver scope, with the Mongo TTL as the backstop for a crashed owner. Dropping a
+ // generic post-completion prune also removes its
// job-replacement race: an older run's late finally can no longer delete a newer
// paused run's checkpoint, because there is no longer a clean-path prune to race.
@@ -2702,11 +2767,11 @@ class AgentClient extends BaseClient {
*
* The original run lives in a detached background task that exits when the run
* pauses, so resume REBUILDS the run on a fresh graph bound to the same
- * `thread_id` (= conversationId) and the durable checkpointer. LangGraph rehydrates
- * the paused graph state from the checkpoint; `run.resume(value)` re-enters the
- * interrupted node with the user's decision. State comes from the checkpoint, so
- * no message history is rebuilt here — `createRun` only needs the agent(s) to
- * reconstruct the graph structure.
+ * `thread_id` (= conversationId), immutable saver scope, and durable checkpointer.
+ * LangGraph rehydrates the paused graph state from that scoped checkpoint;
+ * `run.resume(value)` re-enters the interrupted node with the user's decision.
+ * State comes from the checkpoint, so no message history is rebuilt here —
+ * `createRun` only needs the agent(s) to reconstruct the graph structure.
*
* `seedContent` is the content streamed before the pause (the assistant message +
* its tool call). In Redis mode the job store's append log already spans the pause,
@@ -2753,6 +2818,8 @@ class AgentClient extends BaseClient {
runName: 'AgentRun',
configurable: {
thread_id: this.conversationId,
+ checkpoint_ns: '',
+ [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: this.checkpointNamespace,
last_agent_index: this.agentConfigs?.size ?? 0,
user_id: this.user ?? this.options.req.user?.id,
hide_sequential_outputs: this.options.agent.hide_sequential_outputs,
diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js
index 72b663c5cf..8c0f643507 100644
--- a/api/server/controllers/agents/client.test.js
+++ b/api/server/controllers/agents/client.test.js
@@ -1,4 +1,5 @@
const mockCreateRun = jest.fn();
+const mockCaptureAgentCheckpointGeneration = jest.fn();
const mockDeleteAgentCheckpoint = jest.fn();
const mockIsHITLEnabled = jest.fn().mockReturnValue(false);
const mockBuildAgentScopedContext = jest.fn((...args) =>
@@ -42,6 +43,7 @@ jest.mock('@librechat/api', () => ({
countFormattedMessageTokens: jest.fn(() => 42),
countTokens: jest.fn((text) => Math.ceil(String(text ?? '').length / 4)),
createTokenCounter: jest.fn(() => jest.fn(() => 0)),
+ captureAgentCheckpointGeneration: (...args) => mockCaptureAgentCheckpointGeneration(...args),
deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
decrementPendingRequest: jest.fn(async () => {}),
initializeAgent: jest.fn(),
@@ -163,6 +165,10 @@ describe('AgentClient - applyHideSequentialOutputsFilter', () => {
});
describe('AgentClient - startup telemetry', () => {
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
it('overlaps run creation with checkpoint pruning and joins both before stream processing', async () => {
let releaseCheckpoint;
let checkpointStarted;
@@ -213,6 +219,7 @@ describe('AgentClient - startup telemetry', () => {
collectedUsage: [],
artifactPromises: [],
startupTelemetry,
+ checkpointNamespace: '1000',
});
client.conversationId = 'conversation-123';
client.responseMessageId = 'response-123';
@@ -223,7 +230,12 @@ describe('AgentClient - startup telemetry', () => {
await checkpointStartedPromise;
expect(mockCreateRun).toHaveBeenCalledTimes(1);
- expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith('conversation-123', undefined);
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ 'conversation-123',
+ undefined,
+ undefined,
+ { throwOnError: true, checkpointNamespace: '1000' },
+ );
expect(startupTelemetry.mark.mock.calls.map(([milestone]) => milestone)).toEqual([
'run_input_prepared',
]);
@@ -247,6 +259,158 @@ describe('AgentClient - startup telemetry', () => {
'stream_processing_started',
]);
expect(processStream).toHaveBeenCalledTimes(1);
+ expect(processStream.mock.calls[0][1]).toEqual(
+ expect.objectContaining({
+ configurable: expect.objectContaining({
+ thread_id: 'conversation-123',
+ checkpoint_ns: '',
+ __librechat_checkpoint_ns: '1000',
+ }),
+ }),
+ );
+ });
+
+ it('does not expose or process a fresh graph when strict checkpoint pruning fails', async () => {
+ jest.clearAllMocks();
+ const checkpointGeneration = {
+ threadId: 'conversation-123',
+ checkpointIds: ['legacy-root', 'legacy-child'],
+ };
+ const processStream = jest.fn().mockResolvedValue();
+ const run = {
+ Graph: { id: 'must-not-be-exposed' },
+ processStream,
+ getCalibrationRatio: jest.fn(() => 0),
+ };
+ mockCreateRun.mockResolvedValue(run);
+ mockIsHITLEnabled.mockReturnValue(true);
+ mockCaptureAgentCheckpointGeneration.mockResolvedValue(checkpointGeneration);
+ mockDeleteAgentCheckpoint.mockRejectedValue(new Error('checkpoint prune failed'));
+ jest.spyOn(GenerationJobManager, 'getJobStore').mockReturnValue({
+ getJob: jest.fn().mockResolvedValue({ createdAt: 1000, status: 'running' }),
+ });
+
+ const client = new AgentClient({
+ req: {
+ user: { id: 'user-123' },
+ body: {},
+ config: { endpoints: { [EModelEndpoint.agents]: { toolApproval: {} } } },
+ _resumableStreamId: 'conversation-123',
+ },
+ res: {},
+ agent: {
+ id: 'agent-123',
+ endpoint: EModelEndpoint.openAI,
+ provider: EModelEndpoint.openAI,
+ model_parameters: { model: 'gpt-4' },
+ hide_sequential_outputs: false,
+ },
+ endpointTokenConfig: {},
+ eventHandlers: {},
+ contentParts: [],
+ collectedUsage: [],
+ artifactPromises: [],
+ startupTelemetry: {
+ mark: jest.fn(),
+ setStreamId: jest.fn(),
+ recordGenerationEvent: jest.fn(),
+ end: jest.fn(),
+ },
+ });
+ client.conversationId = 'conversation-123';
+ client.jobCreatedAt = 1000;
+ client.responseMessageId = 'response-123';
+ client.parentMessageId = 'parent-123';
+ client.recordCollectedUsage = jest.fn().mockResolvedValue();
+
+ await client.chatCompletion({ payload: [] });
+
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ 'conversation-123',
+ undefined,
+ checkpointGeneration,
+ { throwOnError: true },
+ );
+ expect(processStream).not.toHaveBeenCalled();
+ expect(client.run).not.toBe(run);
+ expect(client.contentParts).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ [ContentTypes.ERROR]: expect.stringContaining('checkpoint prune failed'),
+ }),
+ ]),
+ );
+ });
+
+ it('does not let a stale v1 fresh prune delete a paused v2 replacement generation', async () => {
+ jest.clearAllMocks();
+ const checkpointGeneration = {
+ threadId: 'conversation-123',
+ checkpointIds: ['legacy-root', 'legacy-child'],
+ };
+ const processStream = jest.fn().mockResolvedValue();
+ const run = {
+ Graph: { id: 'stale-v1-graph' },
+ processStream,
+ getCalibrationRatio: jest.fn(() => 0),
+ };
+ mockCreateRun.mockResolvedValue(run);
+ mockIsHITLEnabled.mockReturnValue(true);
+ mockCaptureAgentCheckpointGeneration.mockResolvedValue(checkpointGeneration);
+ const getJob = jest.fn().mockResolvedValue({
+ createdAt: 2000,
+ status: 'requires_action',
+ checkpointNamespace: '2000',
+ });
+ jest.spyOn(GenerationJobManager, 'getJobStore').mockReturnValue({ getJob });
+
+ const client = new AgentClient({
+ req: {
+ user: { id: 'user-123' },
+ body: {},
+ config: { endpoints: { [EModelEndpoint.agents]: { toolApproval: {} } } },
+ _resumableStreamId: 'conversation-123',
+ },
+ res: {},
+ agent: {
+ id: 'agent-123',
+ endpoint: EModelEndpoint.openAI,
+ provider: EModelEndpoint.openAI,
+ model_parameters: { model: 'gpt-4' },
+ hide_sequential_outputs: false,
+ },
+ endpointTokenConfig: {},
+ eventHandlers: {},
+ contentParts: [],
+ collectedUsage: [],
+ artifactPromises: [],
+ });
+ client.conversationId = 'conversation-123';
+ client.jobCreatedAt = 1000;
+ client.responseMessageId = 'response-123';
+ client.parentMessageId = 'parent-123';
+ client.recordCollectedUsage = jest.fn().mockResolvedValue();
+
+ await client.chatCompletion({ payload: [] });
+
+ expect(mockCaptureAgentCheckpointGeneration).toHaveBeenCalledWith(
+ 'conversation-123',
+ undefined,
+ { throwOnError: true },
+ );
+ expect(getJob).toHaveBeenCalledTimes(1);
+ expect(mockDeleteAgentCheckpoint).not.toHaveBeenCalled();
+ expect(processStream).not.toHaveBeenCalled();
+ expect(client.run).not.toBe(run);
+ expect(client.contentParts).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ [ContentTypes.ERROR]: expect.stringContaining(
+ 'Generation replaced before legacy checkpoint cleanup',
+ ),
+ }),
+ ]),
+ );
});
});
diff --git a/api/server/controllers/agents/protocol.js b/api/server/controllers/agents/protocol.js
new file mode 100644
index 0000000000..cc8e043865
--- /dev/null
+++ b/api/server/controllers/agents/protocol.js
@@ -0,0 +1,77 @@
+const GENERATION_PROTOCOL_HEADER = 'x-librechat-generation-protocol';
+const GENERATION_PROTOCOL_V1 = 1;
+const GENERATION_PROTOCOL_V2 = 2;
+
+function parseProtocolVersion(value) {
+ if (value === GENERATION_PROTOCOL_V1 || value === '1') {
+ return GENERATION_PROTOCOL_V1;
+ }
+ if (value === GENERATION_PROTOCOL_V2 || value === '2') {
+ return GENERATION_PROTOCOL_V2;
+ }
+ return undefined;
+}
+
+/**
+ * Reads every transport carrying the negotiation marker. If multiple markers
+ * disagree, the lower protocol wins; a proxy or token-refresh path dropping
+ * one marker must never upgrade a legacy request accidentally.
+ */
+function getRequestedGenerationProtocol(req) {
+ const advertised = [
+ req?.body?.generationProtocolVersion,
+ req?.query?.generationProtocolVersion,
+ req?.headers?.[GENERATION_PROTOCOL_HEADER],
+ ]
+ .filter((value) => value != null)
+ .map(parseProtocolVersion);
+
+ if (advertised.length === 0 || advertised.some((value) => value == null)) {
+ return GENERATION_PROTOCOL_V1;
+ }
+ return advertised.every((value) => value === GENERATION_PROTOCOL_V2)
+ ? GENERATION_PROTOCOL_V2
+ : GENERATION_PROTOCOL_V1;
+}
+
+/**
+ * Redis state is shared by every replica, including an older binary during a
+ * rolling deployment. Default it to the bridge-safe v1 protocol until an
+ * operator completes the documented homogeneous-fleet cutover. In-memory
+ * state cannot be touched by another process, so it can use v2 immediately.
+ */
+function getServerGenerationProtocol(manager) {
+ const configured = parseProtocolVersion(process.env.GENERATION_PROTOCOL_VERSION);
+ if (configured != null) {
+ return configured;
+ }
+ return manager?.isRedis === true ? GENERATION_PROTOCOL_V1 : GENERATION_PROTOCOL_V2;
+}
+
+function getJobGenerationProtocol(job) {
+ return parseProtocolVersion(job?.metadata?.generationProtocolVersion) ?? GENERATION_PROTOCOL_V1;
+}
+
+/** Selects an immutable protocol for a newly created generation. */
+function negotiateNewGenerationProtocol(req, manager) {
+ return Math.min(getRequestedGenerationProtocol(req), getServerGenerationProtocol(manager));
+}
+
+/**
+ * Existing jobs keep the protocol they were created with. This is what makes
+ * v1 jobs safe to finish after the fleet-wide v2 gate is enabled.
+ */
+function negotiateExistingGenerationProtocol(req, job) {
+ return Math.min(getRequestedGenerationProtocol(req), getJobGenerationProtocol(job));
+}
+
+module.exports = {
+ GENERATION_PROTOCOL_HEADER,
+ GENERATION_PROTOCOL_V1,
+ GENERATION_PROTOCOL_V2,
+ getRequestedGenerationProtocol,
+ getServerGenerationProtocol,
+ getJobGenerationProtocol,
+ negotiateNewGenerationProtocol,
+ negotiateExistingGenerationProtocol,
+};
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index 109581f38e..16b47ea3e5 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -1,5 +1,11 @@
const { logger } = require('@librechat/data-schemas');
-const { Constants, ViolationTypes, isEphemeralAgentId } = require('librechat-data-provider');
+const { v5: uuidv5 } = require('uuid');
+const {
+ Constants,
+ EModelEndpoint,
+ ViolationTypes,
+ isEphemeralAgentId,
+} = require('librechat-data-provider');
const {
sendEvent,
toPendingSteer,
@@ -17,6 +23,8 @@ const {
getAgentStartupTelemetry,
acceptAgentStartupTelemetry,
isSteerPreemptSupported,
+ buildRecoveredSteerPayload,
+ deleteAgentCheckpoint,
} = require('@librechat/api');
const { disposeClient, clientRegistry, requestDataMap } = require('~/server/cleanup');
const {
@@ -26,6 +34,21 @@ const {
const { handleAbortError } = require('~/server/middleware');
const { logViolation } = require('~/cache');
const { saveMessage, getMessages, getConvo } = require('~/models');
+const {
+ GENERATION_PROTOCOL_HEADER,
+ GENERATION_PROTOCOL_V2,
+ negotiateNewGenerationProtocol,
+ negotiateExistingGenerationProtocol,
+} = require('./protocol');
+
+function sendGenerationJson(res, status, body, generationProtocolVersion) {
+ if (typeof res.set === 'function') {
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ } else if (typeof res.setHeader === 'function') {
+ res.setHeader(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ }
+ return res.status(status).json({ ...body, generationProtocolVersion });
+}
function createCloseHandler(abortController) {
return function (manual) {
@@ -168,6 +191,63 @@ const JOB_RECORD_WAIT_DELAY_MS = 60;
// than hand back a stream that would 404). Past it, a missing job means the original
// already completed and was cleaned up (attach and let the client refetch).
const IDEMPOTENCY_STARTUP_GRACE_MS = 5000;
+const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
+/** New-chat retries do not carry a conversation id, so derive the stream id
+ * from their stable per-submission id. This keeps both the dedupe key and the
+ * Redis hash slot identical across a lost-response retry. */
+const NEW_CONVERSATION_IDEMPOTENCY_NAMESPACE = 'd7f2518c-94b8-4fe8-97ad-2d4bdb2c9f43';
+
+function isValidGenerationClaim(value, streamId, conversationId, requireStarted = false) {
+ return (
+ value != null &&
+ typeof value === 'object' &&
+ value.streamId === streamId &&
+ value.conversationId === conversationId &&
+ Number.isSafeInteger(value.claimedAt) &&
+ value.claimedAt >= 0 &&
+ typeof value.claimToken === 'string' &&
+ value.claimToken.length > 0 &&
+ value.claimToken.length <= 128 &&
+ (value.generationProtocolVersion == null ||
+ value.generationProtocolVersion === 1 ||
+ value.generationProtocolVersion === GENERATION_PROTOCOL_V2) &&
+ (value.startedAt == null || (Number.isSafeInteger(value.startedAt) && value.startedAt >= 0)) &&
+ (!requireStarted || value.startedAt != null)
+ );
+}
+
+/** Pre-bridge servers wrote the legacy global key without a claim token and,
+ * for a new conversation, chose a random stream before claiming it. Accept
+ * only that tightly bounded legacy shape: existing conversations must still
+ * match the requested stream exactly; new-chat claims may point to the old
+ * random stream only when streamId === conversationId. Ownership is verified
+ * against the live job before attachment. */
+function isValidLegacyGenerationClaim(value, streamId, isNewConvo) {
+ return (
+ value != null &&
+ typeof value === 'object' &&
+ typeof value.streamId === 'string' &&
+ value.streamId.length > 0 &&
+ value.streamId.length <= 512 &&
+ value.conversationId === value.streamId &&
+ (isNewConvo || value.streamId === streamId) &&
+ Number.isSafeInteger(value.claimedAt) &&
+ value.claimedAt >= 0 &&
+ value.claimToken == null &&
+ value.startedAt == null &&
+ (value.generationProtocolVersion == null || value.generationProtocolVersion === 1)
+ );
+}
+
+/** Store corruption must not turn a user-scoped idempotency claim into a
+ * pointer to another user's/tenant's live stream. Missing tenant metadata is
+ * kept as the explicit legacy case, but missing ownership never authorizes. */
+function liveJobBelongsToRequester(job, user) {
+ return (
+ job?.metadata?.userId === user.id &&
+ (job.metadata?.tenantId == null || job.metadata.tenantId === user.tenantId)
+ );
+}
/**
* Poll briefly for a job record to appear. A deduped retry that loses the idempotency
@@ -176,19 +256,53 @@ const IDEMPOTENCY_STARTUP_GRACE_MS = 5000;
*/
async function waitForJobRecord(streamId) {
for (let attempt = 0; attempt < JOB_RECORD_WAIT_ATTEMPTS; attempt++) {
- if (await GenerationJobManager.hasJob(streamId)) {
- return true;
+ const job = await GenerationJobManager.getJob(streamId);
+ if (job) {
+ return job;
}
await new Promise((resolve) => setTimeout(resolve, JOB_RECORD_WAIT_DELAY_MS));
}
- return GenerationJobManager.hasJob(streamId);
+ return GenerationJobManager.getJob(streamId);
}
-function rejectPreliminaryParentMessageId(res) {
- return res.status(409).json({
- error:
- 'Cannot submit a follow-up while the selected parent response is still being saved. Please wait and try again.',
- });
+/** The claimed generation already reached durable/terminal history, but its
+ * conversation stream id now belongs to no job or to a newer submission. A
+ * success shape with that streamId would attach the stale submission to the
+ * replacement, so tell the client to refetch without opening SSE. */
+function sendSettledGeneration(
+ res,
+ streamId,
+ conversationId,
+ startupTelemetry,
+ generationProtocolVersion,
+) {
+ startupTelemetry?.end('deduplicated');
+ if (generationProtocolVersion < GENERATION_PROTOCOL_V2) {
+ return sendGenerationJson(
+ res,
+ 200,
+ { streamId, conversationId, status: 'resumed' },
+ generationProtocolVersion,
+ );
+ }
+ return sendGenerationJson(
+ res,
+ 200,
+ { conversationId, status: 'settled' },
+ generationProtocolVersion,
+ );
+}
+
+function rejectPreliminaryParentMessageId(res, generationProtocolVersion) {
+ return sendGenerationJson(
+ res,
+ 409,
+ {
+ error:
+ 'Cannot submit a follow-up while the selected parent response is still being saved. Please wait and try again.',
+ },
+ generationProtocolVersion,
+ );
}
/**
@@ -197,6 +311,7 @@ function rejectPreliminaryParentMessageId(res) {
*/
const ResumableAgentController = async (req, res, next, initializeClient, addTitle) => {
const startupTelemetry = getAgentStartupTelemetry(req);
+ let generationProtocolVersion = negotiateNewGenerationProtocol(req, GenerationJobManager);
const {
text,
isRegenerate,
@@ -210,8 +325,117 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
} = req.body;
const userId = req.user.id;
+ const rawClientRequestId = req.body?.clientRequestId;
+ if (
+ rawClientRequestId != null &&
+ (typeof rawClientRequestId !== 'string' || !CLIENT_REQUEST_ID_PATTERN.test(rawClientRequestId))
+ ) {
+ startupTelemetry?.end('rejected');
+ return sendGenerationJson(
+ res,
+ 400,
+ {
+ code: 'INVALID_CLIENT_REQUEST_ID',
+ error: 'clientRequestId must be a 1-128 character identifier.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ const clientRequestId = rawClientRequestId;
+ const rawExpectedPredecessorCreatedAt = req.body?.expectedPredecessorCreatedAt;
+ if (
+ rawExpectedPredecessorCreatedAt != null &&
+ (!Number.isSafeInteger(rawExpectedPredecessorCreatedAt) || rawExpectedPredecessorCreatedAt < 0)
+ ) {
+ startupTelemetry?.end('rejected');
+ return sendGenerationJson(
+ res,
+ 400,
+ {
+ code: 'INVALID_GENERATION_PREDECESSOR',
+ error: 'expectedPredecessorCreatedAt must be a non-negative safe integer.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ const expectedPredecessorCreatedAt = rawExpectedPredecessorCreatedAt;
+ const legacyRecoveredSteerId =
+ clientRequestId?.startsWith('steer-recovery:') === true
+ ? clientRequestId.slice('steer-recovery:'.length)
+ : undefined;
+ const explicitRecoveredSteerId = req.body?.recoverySteerId;
+ const invalidExplicitRecoveryId =
+ explicitRecoveredSteerId != null &&
+ (typeof explicitRecoveredSteerId !== 'string' ||
+ !CLIENT_REQUEST_ID_PATTERN.test(explicitRecoveredSteerId));
+ const mismatchedRecoveryIds =
+ explicitRecoveredSteerId != null &&
+ legacyRecoveredSteerId != null &&
+ explicitRecoveredSteerId !== legacyRecoveredSteerId;
+ if (invalidExplicitRecoveryId || mismatchedRecoveryIds) {
+ startupTelemetry?.end('rejected');
+ return sendGenerationJson(
+ res,
+ 400,
+ {
+ code: 'INVALID_RECOVERY_REQUEST',
+ error: 'recoverySteerId must identify exactly one parked steer source.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ const recoveredSteerId = explicitRecoveredSteerId ?? legacyRecoveredSteerId;
+ const isRecoveredSteerRequest = recoveredSteerId != null;
+ const recoveryUserMessageId = req.body?.overrideUserMessageId;
+ const recoveredSteerPayload = isRecoveredSteerRequest
+ ? buildRecoveredSteerPayload(text, req.body?.files)
+ : undefined;
+ /** A recovered steer is handed off as a new ordinary user turn. Edit,
+ * regenerate, continue, and arbitrary override-id shapes can reuse an
+ * existing user row (or deliberately skip its save); consuming the parked
+ * source from one of those shapes would therefore erase the only durable
+ * copy of the recovered words without proving that a new user row contains
+ * them. The source steer id itself is the one permitted user-row override:
+ * retries intentionally upsert that stable recovery row while each
+ * generation attempt uses a fresh clientRequestId. */
+ if (
+ isRecoveredSteerRequest &&
+ (!clientRequestId ||
+ !recoveredSteerId ||
+ !recoveredSteerPayload ||
+ !!isRegenerate ||
+ !!isContinued ||
+ editedContent != null ||
+ overrideParentMessageId != null ||
+ editedResponseMessageId != null ||
+ (recoveryUserMessageId != null && recoveryUserMessageId !== recoveredSteerId) ||
+ !!req.body?.overrideConvoId)
+ ) {
+ startupTelemetry?.end('rejected');
+ return sendGenerationJson(
+ res,
+ 400,
+ {
+ code: 'INVALID_RECOVERY_REQUEST',
+ error: 'A recovered steer must be submitted as a new user turn.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ if (isRecoveredSteerRequest && recoveryUserMessageId === recoveredSteerId) {
+ /** BaseClient treats a bare override id as an already-persisted row and
+ * skips its save. Recovery instead needs an idempotent upsert: preserve
+ * the source-derived row id while explicitly selecting save index zero. */
+ req.body.overrideUserMessageId = `${recoveredSteerId}${Constants.COMMON_DIVIDER}0`;
+ }
const isNewConvo = !reqConversationId || reqConversationId === 'new';
- const conversationId = isNewConvo ? crypto.randomUUID() : reqConversationId;
+ let conversationId = reqConversationId;
+ if (isNewConvo) {
+ conversationId =
+ typeof clientRequestId === 'string' && clientRequestId.length > 0
+ ? uuidv5(`${userId}:${clientRequestId}`, NEW_CONVERSATION_IDEMPOTENCY_NAMESPACE)
+ : crypto.randomUUID();
+ }
const conversationAnchorPromise = resolveConversationCreatedAt({
userId,
conversationId,
@@ -227,7 +451,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
})
) {
startupTelemetry?.end('rejected');
- return rejectPreliminaryParentMessageId(res);
+ return rejectPreliminaryParentMessageId(res, generationProtocolVersion);
}
/** When to generate the conversation title. `immediate` (default) fires title
@@ -245,9 +469,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// identical payload, which would otherwise start a second fully-billed generation.
// Claim the submission's clientRequestId before creating the job so a retry attaches
// to the original stream instead of spawning a duplicate. Runs before the concurrency
- // check so a deduped retry is never counted against the limiter. Fail-open on errors.
- const clientRequestId = req.body?.clientRequestId;
- let ownsIdempotencyClaim = false;
+ // check so a deduped retry is never counted against the limiter. Once a
+ // stable id is present, an ambiguous store outcome must fail closed.
+ let ownedIdempotencyClaim = null;
if (clientRequestId) {
let claim = null;
try {
@@ -256,29 +480,117 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
clientRequestId,
streamId,
conversationId,
+ generationProtocolVersion,
);
} catch (err) {
- // The claim itself could not be determined (store unavailable): fail open and proceed
- // as a fresh request rather than blocking the send. This is the ONLY fail-open path —
- // once a duplicate is confirmed below, an error must never fall through to a second
- // billed generation.
logger.error(
- '[ResumableAgentController] Idempotency claim failed; proceeding without dedup',
+ '[ResumableAgentController] Idempotency claim outcome is unknown; asking the client to retry',
err,
);
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership could not be confirmed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
}
- if (claim?.claimed) {
- ownsIdempotencyClaim = true;
+ if (claim?.existing != null) {
+ generationProtocolVersion = Math.min(
+ generationProtocolVersion,
+ claim.existing.generationProtocolVersion === GENERATION_PROTOCOL_V2
+ ? GENERATION_PROTOCOL_V2
+ : 1,
+ );
+ }
+
+ const isLegacyTokenlessClaim =
+ claim?.source === 'legacy' && claim?.existing != null && claim.existing.claimToken == null;
+ const validClaim = isLegacyTokenlessClaim
+ ? isValidLegacyGenerationClaim(claim.existing, streamId, isNewConvo)
+ : isValidGenerationClaim(claim?.existing, streamId, conversationId);
+ if (claim?.existing != null && !validClaim) {
+ logger.error('[ResumableAgentController] Invalid or miscorrelated idempotency claim');
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership could not be confirmed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+
+ if (claim?.claimed && claim.existing?.claimToken) {
+ ownedIdempotencyClaim = claim.existing;
+ try {
+ const existingLiveGeneration = await GenerationJobManager.resumeClaimedGeneration(
+ userId,
+ clientRequestId,
+ streamId,
+ ownedIdempotencyClaim,
+ );
+ if (
+ existingLiveGeneration &&
+ isValidGenerationClaim(existingLiveGeneration, streamId, conversationId, true)
+ ) {
+ // A fresh lease may have been negotiated under a different rollout
+ // cap than the still-live job it was atomically rebound to. The
+ // job's immutable protocol wins; echoing the fresh request's marker
+ // would make the client use v2-only recovery against a v1 run (or
+ // unnecessarily downgrade a v2 run).
+ generationProtocolVersion = Math.min(
+ generationProtocolVersion,
+ existingLiveGeneration.generationProtocolVersion === GENERATION_PROTOCOL_V2
+ ? GENERATION_PROTOCOL_V2
+ : 1,
+ );
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 200,
+ {
+ streamId: existingLiveGeneration.streamId,
+ conversationId: existingLiveGeneration.conversationId,
+ generationCreatedAt: existingLiveGeneration.startedAt,
+ status: 'resumed',
+ },
+ generationProtocolVersion,
+ );
+ } else if (existingLiveGeneration) {
+ throw new Error('Live generation idempotency adoption returned invalid ownership');
+ }
+ } catch (err) {
+ logger.error('[ResumableAgentController] Live generation idempotency adoption failed', err);
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership changed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
} else if (claim?.existing) {
// A duplicate is confirmed. Attach to the original stream — and never fall through to
// a second generation, even if the job lookup hiccups.
const existingStreamId = claim.existing.streamId;
- let jobExists = false;
+ let liveJob;
try {
// Wait briefly for the winner to write the job record (it does so a few ms after
// claiming) so a still-live stream isn't handed back before its job exists.
- jobExists = await waitForJobRecord(existingStreamId);
+ liveJob = await waitForJobRecord(existingStreamId);
} catch (err) {
// Store hiccup while checking the job: ask the client to retry rather than starting
// a second generation for a request we know is a duplicate.
@@ -288,51 +600,256 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
);
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
- return res.status(503).json({
- code: 'SERVER_NOT_READY',
- error: 'Generation is still starting. Please retry shortly.',
- });
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation is still starting. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
}
const claimAgeMs = Date.now() - (claim.existing.claimedAt ?? 0);
- if (!jobExists && claimAgeMs < IDEMPOTENCY_STARTUP_GRACE_MS) {
+ if (!liveJob && claim.existing.startedAt != null) {
+ // createJob marked this claim in the same transaction that installed
+ // the job. A now-missing record therefore represents an already-owned
+ // generation (usually fast completion + cleanup), never an abandoned
+ // pre-create lease that may be taken over and billed again. There is no
+ // attachable stream; the settled response refetches persisted history.
+ return sendSettledGeneration(
+ res,
+ existingStreamId,
+ claim.existing.conversationId,
+ startupTelemetry,
+ generationProtocolVersion,
+ );
+ }
+ if (!liveJob && isLegacyTokenlessClaim && claimAgeMs >= IDEMPOTENCY_STARTUP_GRACE_MS) {
+ /** A legacy owner cannot be fenced (its value has no token), so it is
+ * never safe to take over. Return its original stream on the legacy
+ * attach/refetch path: this covers fast completion without starting a
+ * second billed generation, while an abandoned pre-create claim ages
+ * out under the old server's bounded TTL. */
+ return sendSettledGeneration(
+ res,
+ existingStreamId,
+ claim.existing.conversationId,
+ startupTelemetry,
+ generationProtocolVersion,
+ );
+ }
+ if (!liveJob && claimAgeMs < IDEMPOTENCY_STARTUP_GRACE_MS) {
// The winner claimed but has not written the job yet (still between claim and
// createJob). Handing back the stream now would 404 and tear down the client while
// the winner goes on to generate and bill with no UI attached — ask the client to
// retry via the readiness path instead.
res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
- return res.status(503).json({
- code: 'SERVER_NOT_READY',
- error: 'Generation is still starting. Please retry shortly.',
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation is still starting. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ if (liveJob) {
+ generationProtocolVersion = negotiateExistingGenerationProtocol(req, liveJob);
+ if (!liveJobBelongsToRequester(liveJob, req.user)) {
+ logger.error(
+ '[ResumableAgentController] Existing idempotency claim resolved to a foreign generation',
+ );
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership could not be confirmed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ const liveClientRequestId = liveJob.metadata?.idempotencyClientRequestId;
+ const startedAt = claim.existing.startedAt;
+ if (liveJob.metadata?.terminalPersistencePending === true) {
+ /** The terminal owner has claimed the outcome but has not yet
+ * finished the required persistence hook. Do not let a duplicate
+ * start refetch history until that single-winner publication is
+ * finalized (or stale-pending recovery publishes failure). */
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation is finalizing. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ const terminalWithoutPayload =
+ ['complete', 'error', 'aborted'].includes(liveJob.status) &&
+ !liveJob.finalEvent &&
+ !liveJob.error;
+ if (terminalWithoutPayload) {
+ /** A terminal CAS can precede its required DB save and durable FINAL
+ * by a narrow window. Returning an attachable/settled success here
+ * lets the retry refetch before persistence is complete. Keep the
+ * duplicate on the readiness path until the owner publishes its
+ * terminal payload (or cleanup makes the job disappear). */
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation is finalizing. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ const replacedGeneration =
+ (startedAt != null && liveJob.createdAt !== startedAt) ||
+ (liveClientRequestId != null && liveClientRequestId !== clientRequestId);
+ if (replacedGeneration) {
+ // streamId === conversationId, so a later turn reuses the same route.
+ // Never pair this stale POST's optimistic submission with that newer
+ // job's SSE snapshot. If the replacement is still active, distinguish
+ // it from an ordinary settled retry so the client hands off to the
+ // authoritative B submission instead of going idle and starting C.
+ if (liveJob.status === 'running' || liveJob.status === 'requires_action') {
+ startupTelemetry?.end('deduplicated');
+ if (generationProtocolVersion < GENERATION_PROTOCOL_V2) {
+ return sendGenerationJson(
+ res,
+ 409,
+ { code: 'RUN_REPLACED' },
+ generationProtocolVersion,
+ );
+ }
+ return sendGenerationJson(
+ res,
+ 200,
+ {
+ streamId: existingStreamId,
+ conversationId: claim.existing.conversationId,
+ generationCreatedAt: liveJob.createdAt,
+ status: 'replaced',
+ },
+ generationProtocolVersion,
+ );
+ }
+ return sendSettledGeneration(
+ res,
+ existingStreamId,
+ claim.existing.conversationId,
+ startupTelemetry,
+ generationProtocolVersion,
+ );
+ }
+ if (liveClientRequestId == null && !isLegacyTokenlessClaim) {
+ // A syntactically valid claim plus an uncorrelated live job is
+ // outcome-ambiguous (legacy/corrupt/partially written state). Attaching
+ // risks cross-wiring two submissions; starting risks double billing.
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership could not be confirmed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ logger.debug('[ResumableAgentController] Deduped retried start-generation request', {
+ userId,
+ clientRequestId,
+ streamId: existingStreamId,
});
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 200,
+ {
+ streamId: existingStreamId,
+ conversationId: claim.existing.conversationId,
+ generationCreatedAt: liveJob.createdAt,
+ status: 'resumed',
+ },
+ generationProtocolVersion,
+ );
}
- // Job exists (live), or the grace elapsed with none (the original already completed
- // and was cleaned up, or the winner died): attach. A then-missing job recovers via
- // the client's subscribe 404 handler (refetch persisted messages) rather than an
- // indefinite readiness loop.
- logger.debug('[ResumableAgentController] Deduped retried start-generation request', {
+
+ // The creator held the claim beyond the startup grace but never made a
+ // job. Atomically take over its lease; createJob verifies this token in
+ // the same Redis transaction as job creation, so the abandoned winner
+ // can no longer wake up and start a second generation.
+ const takeover = await GenerationJobManager.takeoverGeneration(
userId,
clientRequestId,
- streamId: existingStreamId,
+ existingStreamId,
+ claim.existing,
+ ).catch((err) => {
+ logger.error('[ResumableAgentController] Stale idempotency takeover failed', err);
+ return null;
});
+ if (
+ !takeover?.claimed ||
+ !isValidGenerationClaim(takeover.existing, streamId, conversationId)
+ ) {
+ res.set('Retry-After', '1');
+ startupTelemetry?.end('deduplicated');
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership changed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
+ }
+ ownedIdempotencyClaim = takeover.existing;
+ } else {
+ // A malformed/unreadable existing claim is outcome-ambiguous. Starting
+ // anyway would turn a store parsing failure into duplicate generation.
+ res.set('Retry-After', '1');
startupTelemetry?.end('deduplicated');
- return res.json({
- streamId: existingStreamId,
- conversationId: claim.existing.conversationId,
- status: 'resumed',
- });
+ return sendGenerationJson(
+ res,
+ 503,
+ {
+ code: 'SERVER_NOT_READY',
+ error: 'Generation ownership could not be confirmed. Please retry shortly.',
+ },
+ generationProtocolVersion,
+ );
}
}
const { allowed, pendingRequests, limit } = await checkAndIncrementPendingRequest(userId);
if (!allowed) {
- if (ownsIdempotencyClaim) {
- await GenerationJobManager.releaseGeneration(userId, clientRequestId).catch(() => {});
+ if (ownedIdempotencyClaim) {
+ await GenerationJobManager.releaseGeneration(
+ userId,
+ clientRequestId,
+ streamId,
+ ownedIdempotencyClaim,
+ ).catch(() => {});
}
const violationInfo = getViolationInfo(pendingRequests, limit);
await logViolation(req, res, ViolationTypes.CONCURRENT, violationInfo, violationInfo.score);
startupTelemetry?.end('rejected');
- return res.status(429).json(violationInfo);
+ return sendGenerationJson(res, 429, violationInfo, generationProtocolVersion);
}
startupTelemetry?.mark('request_admitted');
@@ -353,8 +870,16 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
const preliminaryResponseMessageId = getPreliminaryResponseMessageId(req.body);
const job = await GenerationJobManager.createJob(streamId, userId, conversationId, {
startupTelemetry,
+ ...(recoveredSteerId && { recoveredSteerId }),
+ ...(recoveredSteerPayload && { recoveredSteerPayload }),
+ ...(expectedPredecessorCreatedAt != null && { expectedPredecessorCreatedAt }),
+ ...(ownedIdempotencyClaim?.claimToken && {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: ownedIdempotencyClaim.claimToken,
+ }),
initialMetadata: {
conversationId,
+ generationProtocolVersion,
endpoint: endpointOption.endpoint,
iconURL: endpointIconURL,
model: responseModel,
@@ -375,13 +900,38 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
startupTelemetry?.mark('job_created');
acceptAgentStartupTelemetry(req, streamId);
startupTelemetry?.mark('metadata_persisted');
+ generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
jobCreatedAt = job.createdAt; // Capture creation time to detect job replacement
req._resumableStreamId = streamId;
getMCPRequestContext(req, undefined, { cleanupOnResponse: false });
+ let recoveredSteerCommitted = false;
+ const commitRecoveredSteer = async () => {
+ if (!recoveredSteerId || recoveredSteerCommitted) {
+ return;
+ }
+ if (client?.skipSaveUserMessage) {
+ throw new Error('Recovered steer cannot skip user message persistence');
+ }
+ const committed = await GenerationJobManager.steering.consumeRecovered(
+ streamId,
+ recoveredSteerId,
+ { userId, tenantId: req.user?.tenantId },
+ jobCreatedAt,
+ );
+ if (!committed) {
+ throw new Error('Recovered steer could not be committed after message persistence');
+ }
+ recoveredSteerCommitted = true;
+ };
// Send JSON response IMMEDIATELY so client can connect to SSE stream
// This is critical: tool loading (MCP OAuth) may emit events that the client needs to receive
- res.json({ streamId, conversationId, status: 'started' });
+ sendGenerationJson(
+ res,
+ 200,
+ { streamId, conversationId, generationCreatedAt: jobCreatedAt, status: 'started' },
+ generationProtocolVersion,
+ );
await attachConversationCreatedAt(req, conversationId, conversationAnchorPromise).then(() =>
startupTelemetry?.mark('conversation_resolved'),
@@ -414,7 +964,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
return;
}
- const resumeState = await GenerationJobManager.getResumeState(streamId);
+ const resumeState = await GenerationJobManager.getResumeState(streamId, jobCreatedAt);
if (!resumeState?.userMessage) {
logger.debug('[ResumableAgentController] No user message to save partial response for');
return;
@@ -443,7 +993,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
partialMessage.agent_id = req.body.agent_id;
}
- await saveMessage(
+ const savedPartialMessage = await saveMessage(
{
userId: req?.user?.id,
isTemporary: req?.body?.isTemporary,
@@ -452,6 +1002,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
partialMessage,
{ context: 'api/server/controllers/agents/request.js - partial response on disconnect' },
);
+ if (!savedPartialMessage) {
+ throw new Error('Partial response could not be persisted after disconnect');
+ }
logger.debug(
`[ResumableAgentController] Saved partial response for ${streamId}, content parts: ${persistableContent.length}`,
@@ -471,10 +1024,21 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// Use the job's abort controller signal - allows abort via GenerationJobManager.abortJob()
signal: job.abortController.signal,
jobCreatedAt,
+ checkpointNamespace: job.metadata?.checkpointNamespace,
});
startupTelemetry?.mark('client_initialized');
client = result.client;
+ /** Request-shape validation rejects every known edit/regenerate path, but
+ * the client owns the final persistence decision. Fail closed if a future
+ * or provider-specific path still derives skip-save for a recovered turn;
+ * consuming its parked source would otherwise erase the only durable copy
+ * of the user's words. Re-checked inside commitRecoveredSteer in case a
+ * client mutates the flag while sending. */
+ if (recoveredSteerId && client?.skipSaveUserMessage) {
+ throw new Error('Recovered steer cannot skip user message persistence');
+ }
+
if (job.abortController.signal.aborted) {
await GenerationJobManager.completeJob(
streamId,
@@ -536,6 +1100,56 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
let immediateTitlePromise = null;
let backgroundClientCleanupScheduled = false;
+ let terminalClaim = null;
+ let terminalClaimFinished = false;
+ let terminalPersistenceChecked = false;
+ let terminalWasAborted = false;
+ let preemptIncomplete = false;
+ /** A pause-row write failure is terminalized through the exact action/epoch
+ * barrier. Once that path starts, neither generic background error handler
+ * may call completeJob: the pause may already have been replaced by a newer
+ * action or generation by the time the persistence failure is observed. */
+ let pausePersistenceFailed = false;
+ const finishOwnedTerminalClaim = async () => {
+ if (!terminalClaim || terminalClaimFinished) {
+ return;
+ }
+ try {
+ await GenerationJobManager.finishTerminalJob(terminalClaim);
+ } finally {
+ terminalClaimFinished = true;
+ }
+ };
+ /** Runs inside BaseClient immediately before it can start the completed
+ * response write. A lost claim returns false, and BaseClient skips that
+ * stale `unfinished:false` write entirely. The fallback invocation below
+ * supports test/custom clients that do not derive from BaseClient. */
+ const claimBeforeResponsePersistence = async () => {
+ if (terminalPersistenceChecked) {
+ return terminalClaim != null;
+ }
+ terminalPersistenceChecked = true;
+ if (client?.pendingApproval) {
+ // AgentClient installed a durable pause-persistence barrier in the
+ // running→requires_action CAS. BaseClient must not start its ordinary
+ // `unfinished:false` response write; the HITL branch persists the
+ // partial row as unfinished before releasing that barrier.
+ return false;
+ }
+ terminalWasAborted = job.abortController.signal.aborted;
+ const preemptStats = client?.run?.getPreemptStats?.();
+ preemptIncomplete =
+ (preemptStats?.emptyBoundaries ?? 0) > 0 ||
+ client?.run?.getHaltReason?.() === 'preempt_incomplete';
+ terminalClaim = await GenerationJobManager.claimTerminalJob(
+ streamId,
+ terminalWasAborted ? 'aborted' : 'complete',
+ undefined,
+ jobCreatedAt,
+ { persistencePending: true },
+ );
+ return terminalClaim != null;
+ };
const disposeBackgroundClient = () => {
if (backgroundClientCleanupScheduled) {
return;
@@ -693,6 +1307,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
abortController: job.abortController,
overrideParentMessageId,
isEdited: !!editedContent,
+ beforeResponsePersistence: claimBeforeResponsePersistence,
userMCPAuthMap: result.userMCPAuthMap,
responseMessageId: editedResponseMessageId,
progressOptions: {
@@ -726,9 +1341,9 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// HITL: the turn paused for human review (see AgentClient.handleRunInterrupt).
// The job is already `requires_action` with the pending action persisted and
- // emitted to the client; the resume route owns finishing this turn. Settle the
- // in-flight user-message / conversation save, then tear down WITHOUT saving a
- // partial response, emitting a terminal event, or completing the job.
+ // emitted to the client; the resume route owns finishing this turn. Settle and
+ // verify the required unfinished history, then tear down without publishing a
+ // terminal event or completing a successfully persisted paused job.
if (client?.pendingApproval) {
if (response?.databasePromise) {
try {
@@ -741,62 +1356,142 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
}
delete response.databasePromise;
}
- // BaseClient saved the response as completed (unfinished:false), but the turn
- // is paused awaiting a decision. Re-mark it unfinished so an expired / never-
- // resumed approval doesn't leave a "finished" response in history; the resume
- // path overwrites it with the full completed message on success.
- if (response?.messageId) {
- // Guard against a fast /resume: the user can approve the instant the
- // pending-action SSE lands, and resume.js can then claim + finalize — saving
- // the COMPLETED response — while we're still awaiting `response.databasePromise`
- // above. Marking the row unfinished now would clobber that completed content
- // with this stale pre-pause response. Only mark unfinished while the job is
- // STILL paused on THIS generation's action: a claim transitions it out of
- // `requires_action`, and a replacement bumps `createdAt`. Fail open on a read
- // error so a genuinely never-resumed approval isn't left looking "finished".
- let stillPaused = true;
+ const pauseActionId = client.pendingApproval.actionId;
+ const pauseCreatedAt = client.jobCreatedAt ?? jobCreatedAt;
+ const ownsPausePersistence = await GenerationJobManager.approvals.ownsPausePersistence(
+ streamId,
+ pauseActionId,
+ pauseCreatedAt,
+ );
+ if (ownsPausePersistence) {
try {
- const liveJob = await GenerationJobManager.getJob(streamId);
- stillPaused =
- !!liveJob &&
- liveJob.status === 'requires_action' &&
- (client?.jobCreatedAt == null || liveJob.createdAt === client.jobCreatedAt);
- } catch (readErr) {
- logger.warn(
- '[ResumableAgentController] Pause unfinished-save liveness check failed; proceeding',
- readErr?.message ?? readErr,
- );
- }
- if (!stillPaused) {
- logger.debug(
- `[ResumableAgentController] Skipping pause unfinished-save — ${streamId} already resumed/replaced`,
+ /** BaseClient awaits its first user/conversation write before the
+ * pause hook, but deliberately swallows a failed/falsy user save
+ * and may still record the id locally. Re-save idempotently for
+ * every ordinary user turn before exposing the approval. */
+ if (!client?.skipSaveUserMessage) {
+ if (!userMessage) {
+ throw new Error('User message was unavailable before HITL pause');
+ }
+ if (
+ typeof client.saveMessageToDatabase === 'function' &&
+ typeof client.getSaveOptions === 'function'
+ ) {
+ /** Retry through BaseClient so a failure before its original
+ * saveConvo is repaired along with the message row. Direct
+ * saveMessage alone cannot recreate that conversation. */
+ const savedUserTurn = await client.saveMessageToDatabase(
+ userMessage,
+ client.getSaveOptions(),
+ userId,
+ );
+ if (!savedUserTurn?.message) {
+ throw new Error('User message could not be persisted before HITL pause');
+ }
+ if (!client.skipSaveConvo && !savedUserTurn.conversation) {
+ throw new Error('Conversation could not be persisted before HITL pause');
+ }
+ } else {
+ // Custom clients used by integrations/tests may not inherit BaseClient.
+ const savedUserMessage = await saveMessage(
+ {
+ userId,
+ isTemporary: req?.body?.isTemporary,
+ interfaceConfig: req?.config?.interfaceConfig,
+ },
+ userMessage,
+ {
+ context:
+ 'api/server/controllers/agents/request.js - user message before HITL pause',
+ },
+ );
+ if (!savedUserMessage) {
+ throw new Error('User message could not be persisted before HITL pause');
+ }
+ }
+ }
+ if (!response?.messageId) {
+ throw new Error('Response message was unavailable before HITL pause');
+ }
+ const savedResponseMessage = await saveMessage(
+ {
+ userId,
+ isTemporary: req?.body?.isTemporary,
+ interfaceConfig: req?.config?.interfaceConfig,
+ },
+ {
+ ...response,
+ endpoint: endpointOption.endpoint,
+ unfinished: true,
+ user: userId,
+ },
+ {
+ context:
+ 'api/server/controllers/agents/request.js - HITL pause (persist unfinished)',
+ },
);
- } else {
+ if (!savedResponseMessage) {
+ throw new Error('Paused response could not be persisted as unfinished');
+ }
+ await commitRecoveredSteer();
+ } catch (pausePersistenceError) {
+ pausePersistenceFailed = true;
+ let failed;
try {
- await saveMessage(
- {
- userId,
- isTemporary: req?.body?.isTemporary,
- interfaceConfig: req?.config?.interfaceConfig,
- },
- {
- ...response,
- endpoint: endpointOption.endpoint,
- unfinished: true,
- user: userId,
- },
- {
- context:
- 'api/server/controllers/agents/request.js - HITL pause (mark unfinished)',
- },
+ failed = await GenerationJobManager.failPausePersistence(
+ streamId,
+ pauseActionId,
+ pausePersistenceError?.message ?? 'Pause persistence failed',
+ pauseCreatedAt,
);
- } catch (saveErr) {
+ } catch (failError) {
logger.error(
- '[ResumableAgentController] Failed to mark paused response unfinished',
- saveErr,
+ `[ResumableAgentController] Failed to terminalize pause persistence error for ${streamId}`,
+ failError,
+ );
+ }
+ if (failed === true) {
+ /** Namespaced checkpoints belong exclusively to this epoch,
+ * so the exact pause-failure CAS winner can safely remove the
+ * now-unresumable graph state. Legacy shared namespaces are
+ * left to their guarded/TTL cleanup path. */
+ const checkpointNamespace = job.metadata?.checkpointNamespace;
+ if (typeof checkpointNamespace === 'string' && checkpointNamespace !== '') {
+ try {
+ await deleteAgentCheckpoint(
+ conversationId,
+ req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
+ undefined,
+ { checkpointNamespace },
+ );
+ } catch (checkpointError) {
+ logger.error(
+ `[ResumableAgentController] Failed to prune checkpoint after pause persistence error for ${streamId}`,
+ checkpointError,
+ );
+ }
+ }
+ } else if (failed === false) {
+ logger.warn(
+ `[ResumableAgentController] Skipping stale pause persistence failure — ${streamId} no longer owns its barrier`,
);
}
+ throw pausePersistenceError;
}
+ const released = await GenerationJobManager.approvals.finishPausePersistence(
+ streamId,
+ pauseActionId,
+ pauseCreatedAt,
+ );
+ if (!released) {
+ logger.warn(
+ `[ResumableAgentController] Pause persistence barrier changed before release: ${streamId}`,
+ );
+ }
+ } else {
+ logger.debug(
+ `[ResumableAgentController] Skipping stale pause persistence — ${streamId} no longer owns its barrier`,
+ );
}
titleAbortController.abort();
acceptsTitleEvents = false;
@@ -818,7 +1513,11 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
return;
}
- const messageId = response.messageId;
+ // BaseClient invokes this before starting its response write. Custom
+ // clients/tests may return a database promise directly, so keep the
+ // controller-side fallback before awaiting that promise.
+ await claimBeforeResponsePersistence();
+
const endpoint = endpointOption.endpoint;
response.endpoint = endpoint;
@@ -830,6 +1529,22 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
conversation.title =
conversation && !conversation.title ? null : conversation?.title || 'New Chat';
+ if (!terminalClaim) {
+ /** Stop/replacement won before the response persistence hook. The
+ * BaseClient contract skipped its completed response write; cancel
+ * title work and leave terminal publication/persistence to the
+ * actual winner. */
+ titleAbortController.abort();
+ titleDiscardController.abort();
+ job.abortController.signal.removeEventListener('abort', abortTitleOnJobAbort);
+ acceptsTitleEvents = false;
+ resolveConvoReady();
+ await finishResumableRequest(req, userId);
+ disposeBackgroundClient();
+ startupTelemetry?.end(job.abortController.signal.aborted ? 'aborted' : 'replaced');
+ return;
+ }
+
if (req.body.files && Array.isArray(client.options.attachments)) {
const files = buildMessageFiles(req.body.files, client.options.attachments);
if (files.length > 0) {
@@ -838,24 +1553,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
delete userMessage.image_urls;
}
- // Check abort state BEFORE calling completeJob (which triggers abort signal for cleanup)
- const wasAbortedBeforeComplete = job.abortController.signal.aborted;
- /**
- * A preempt boundary that had nothing to inject (cancelled/stale
- * request) ends the turn with a genuinely truncated answer — the SDK
- * reports it via preempt stats and the halt reason. Persist it with
- * the same honest `unfinished` contract an abort gets, never as a
- * silent completion.
- */
- const preemptStats = client?.run?.getPreemptStats?.();
- const preemptIncomplete =
- (preemptStats?.emptyBoundaries ?? 0) > 0 ||
- client?.run?.getHaltReason?.() === 'preempt_incomplete';
const shouldGenerateTitle =
- addTitle &&
- parentMessageId === Constants.NO_PARENT &&
- isNewConvo &&
- !wasAbortedBeforeComplete;
+ addTitle && parentMessageId === Constants.NO_PARENT && isNewConvo && !terminalWasAborted;
// Save user message BEFORE sending final event to avoid race condition
// where client refetch happens before database is updated
@@ -865,79 +1564,54 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
interfaceConfig: req?.config?.interfaceConfig,
};
- if (!client.skipSaveUserMessage && userMessage) {
- await saveMessage(reqCtx, userMessage, {
+ if (!client.skipSaveUserMessage) {
+ if (!userMessage) {
+ throw new Error('User message was unavailable before terminal persistence');
+ }
+ const savedUserMessage = await saveMessage(reqCtx, userMessage, {
context: 'api/server/controllers/agents/request.js - resumable user message',
});
+ if (!savedUserMessage) {
+ throw new Error('User message could not be persisted before terminal publication');
+ }
}
+ // Only consume the parked recovery source after the explicit user-row
+ // write above succeeds. `response.databasePromise` alone is insufficient:
+ // BaseClient intentionally swallows a failed first user-message save.
+ await commitRecoveredSteer();
// CRITICAL: Save response message BEFORE emitting final event.
// This prevents race conditions where the client sends a follow-up message
// before the response is saved to the database, causing orphaned parentMessageIds.
- if (client.savedMessageIds && !client.savedMessageIds.has(messageId)) {
- await saveMessage(
- reqCtx,
- {
- ...response,
- user: userId,
- unfinished: wasAbortedBeforeComplete || preemptIncomplete,
- },
- { context: 'api/server/controllers/agents/request.js - resumable response end' },
- );
- } else if (preemptIncomplete) {
- /**
- * A completed send already saved this row as `unfinished: false`
- * from `BaseClient.sendMessage`, and registered it in
- * `savedMessageIds` — so the branch above is skipped and the flag
- * would never reach the database. An empty preempt boundary IS a
- * completed send (the SDK halts and returns content), so re-mark
- * it explicitly, the same way the HITL pause re-marks above.
- */
- await saveMessage(
- reqCtx,
- { ...response, user: userId, unfinished: true },
- { context: 'api/server/controllers/agents/request.js - preempt incomplete' },
+ /** BaseClient can add the id to savedMessageIds even when its model-layer
+ * save resolved falsy. Re-save the terminal row idempotently and require
+ * the returned durable row before publishing the normal FINAL. */
+ const responseIsUnfinished = terminalWasAborted || preemptIncomplete;
+ const savedResponseMessage = await saveMessage(
+ reqCtx,
+ {
+ ...response,
+ user: userId,
+ unfinished: responseIsUnfinished,
+ },
+ {
+ context: responseIsUnfinished
+ ? 'api/server/controllers/agents/request.js - terminal response unfinished'
+ : 'api/server/controllers/agents/request.js - resumable response end',
+ },
+ );
+ if (!savedResponseMessage) {
+ throw new Error(
+ responseIsUnfinished
+ ? 'Terminal response could not be persisted as unfinished'
+ : 'Response message could not be persisted before terminal publication',
);
}
- // Check if our job was replaced by a new request before emitting
- // This prevents stale requests from emitting events to newer jobs
- const currentJob = await GenerationJobManager.getJob(streamId);
- const jobWasReplaced = !currentJob || currentJob.createdAt !== jobCreatedAt;
-
- if (jobWasReplaced) {
- logger.debug(`[ResumableAgentController] Skipping FINAL emit - job was replaced`, {
- streamId,
- originalCreatedAt: jobCreatedAt,
- currentCreatedAt: currentJob?.createdAt,
- });
- // Discard the stale title from this replaced stream: cancel it and
- // unblock its persistence wait without letting it save (the newer job
- // owns the conversation now).
- titleAbortController.abort();
- titleDiscardController.abort();
- job.abortController.signal.removeEventListener('abort', abortTitleOnJobAbort);
- acceptsTitleEvents = false;
- resolveConvoReady();
- // Still decrement pending request since we incremented at start
- await finishResumableRequest(req, userId);
- startupTelemetry?.end('replaced');
- if (immediateTitlePromise) {
- immediateTitlePromise.finally(() => {
- if (client) {
- disposeClient(client);
- }
- });
- } else if (client) {
- disposeClient(client);
- }
- return;
- }
-
// If the user stopped this turn, cancel the title BEFORE unblocking its
// persistence wait — otherwise resolving `convoReady` lets the title task
// resume and save before the later abort runs.
- if (wasAbortedBeforeComplete) {
+ if (terminalWasAborted) {
titleAbortController.abort();
} else {
job.abortController.signal.removeEventListener('abort', abortTitleOnJobAbort);
@@ -952,88 +1626,70 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
await titleEventPromise;
}
- // Steers that never reached an injection boundary (queued after the last
- // tool batch, or the run had none). The close-and-drain atomically stops
- // new enqueues first — a steer POST racing this finalization gets 404
- // (client sends it as a normal message) instead of a 202 whose payload
- // completeJob would then silently clear. Reported on the final event so
- // the client converts them to queued follow-up messages.
- let pendingSteers;
+ let terminalPublicationStarted = false;
try {
- const leftoverSteers = await GenerationJobManager.steering.closeAndDrain(
- streamId,
- jobCreatedAt,
- );
- if (leftoverSteers.length > 0) {
- pendingSteers = leftoverSteers.map(toPendingSteer);
- // Parked BEFORE the final event: a client with no live subscriber
- // recovers these via /chat/status (claim-on-read) within the
- // recovery TTL — the SSE copy alone is transient.
- await GenerationJobManager.steering.park(
- streamId,
- pendingSteers,
- {
- userId,
- tenantId: req.user?.tenantId,
- },
- jobCreatedAt,
- );
- }
- } catch (err) {
- logger.warn(`[ResumableAgentController] Failed to drain leftover steers`, err);
- }
-
- if (!wasAbortedBeforeComplete) {
+ const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);
const finalEvent = {
final: true,
conversation,
title: conversation.title,
requestMessage: sanitizeMessageForTransmit(userMessage),
- responseMessage: { ...response, ...(preemptIncomplete && { unfinished: true }) },
- ...(pendingSteers && { pendingSteers }),
- };
-
- logger.debug(`[ResumableAgentController] Emitting FINAL event`, {
- streamId,
- wasAbortedBeforeComplete,
- userMessageId: userMessage?.messageId,
- responseMessageId: response?.messageId,
- conversationId: conversation?.conversationId,
- });
-
- await GenerationJobManager.emitDone(streamId, finalEvent, jobCreatedAt);
- startupTelemetry?.end('completed_without_delta');
- void GenerationJobManager.completeJob(streamId, undefined, jobCreatedAt).catch((err) => {
- logger.warn('[ResumableAgentController] Failed to finalize completed job', err);
- });
- await finishResumableRequest(req, userId);
- } else {
- const finalEvent = {
- final: true,
- conversation,
- title: conversation.title,
- requestMessage: sanitizeMessageForTransmit(userMessage),
- responseMessage: { ...response, unfinished: true },
- ...(pendingSteers && { pendingSteers }),
+ responseMessage: {
+ ...response,
+ ...((terminalWasAborted || preemptIncomplete) && { unfinished: true }),
+ },
+ ...(pendingSteers.length > 0 && { pendingSteers }),
};
- logger.debug(`[ResumableAgentController] Emitting ABORTED FINAL event`, {
- streamId,
- wasAbortedBeforeComplete,
- userMessageId: userMessage?.messageId,
- responseMessageId: response?.messageId,
- conversationId: conversation?.conversationId,
- });
-
- await GenerationJobManager.emitDone(streamId, finalEvent, jobCreatedAt);
- startupTelemetry?.end('aborted');
- void GenerationJobManager.completeJob(streamId, 'Request aborted', jobCreatedAt).catch(
- (err) => {
- logger.warn('[ResumableAgentController] Failed to finalize aborted job', err);
+ logger.debug(
+ terminalWasAborted
+ ? `[ResumableAgentController] Emitting ABORTED FINAL event`
+ : `[ResumableAgentController] Emitting FINAL event`,
+ {
+ streamId,
+ wasAbortedBeforeComplete: terminalWasAborted,
+ userMessageId: userMessage?.messageId,
+ responseMessageId: response?.messageId,
+ conversationId: conversation?.conversationId,
},
);
- await finishResumableRequest(req, userId);
+
+ terminalPublicationStarted = true;
+ const publication = await GenerationJobManager.publishTerminalClaim(
+ terminalClaim,
+ finalEvent,
+ );
+ let terminalOutcome = 'completed_without_delta';
+ if (publication.persistenceFailed) {
+ terminalOutcome = 'error';
+ } else if (terminalWasAborted) {
+ terminalOutcome = 'aborted';
+ }
+ startupTelemetry?.end(terminalOutcome);
+ } catch (terminalError) {
+ /** A failure while constructing the payload happened after this
+ * controller's terminal CAS but before the manager could durably
+ * settle it. Publish conservative reconciliation immediately. Once
+ * publication starts, the manager either stores the payload or owns
+ * its bounded recovery marker, so retrying with a different payload
+ * here would only risk duplicate delivery. */
+ if (!terminalPublicationStarted) {
+ try {
+ await GenerationJobManager.publishTerminalClaim(terminalClaim, null);
+ } catch (reconcileError) {
+ logger.warn(
+ '[ResumableAgentController] Failed to publish terminal persistence reconciliation',
+ reconcileError,
+ );
+ }
+ }
+ throw terminalError;
+ } finally {
+ // Pair every successful claim even when final-event construction or
+ // transport publication throws. Cleanup is epoch/runtime guarded.
+ await finishOwnedTerminalClaim();
}
+ await finishResumableRequest(req, userId);
if (titleTiming === 'immediate') {
// Title was fired in parallel above (if eligible); a stopped turn already
@@ -1078,63 +1734,59 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
acceptsTitleEvents = false;
resolveConvoReady();
- // Check if this was an abort (not a real error)
- const wasAborted = job.abortController.signal.aborted || error.message?.includes('abort');
-
- if (wasAborted) {
+ // Once this controller owns terminal persistence, no competing error
+ // transition can win. Settle its pending marker with conservative
+ // reconciliation on any required-write/final-construction failure,
+ // then release exactly that claim.
+ if (terminalClaim && !terminalClaimFinished) {
+ try {
+ await GenerationJobManager.publishTerminalClaim(terminalClaim, null);
+ } catch (publishError) {
+ logger.warn(
+ '[ResumableAgentController] Failed to publish terminal persistence reconciliation',
+ publishError,
+ );
+ } finally {
+ await finishOwnedTerminalClaim().catch((finishError) => {
+ logger.warn(
+ '[ResumableAgentController] Failed to finish terminal persistence claim',
+ finishError,
+ );
+ });
+ }
+ logger.error(
+ `[ResumableAgentController] Terminal persistence failed for ${streamId}:`,
+ error,
+ );
+ startupTelemetry?.end('error', error);
+ } else if (pausePersistenceFailed) {
+ // failPausePersistence owns the only legal requires_action -> error
+ // transition for this exact action/epoch. Never fall through to
+ // completeJob, which could race a newer action or replacement job.
+ logger.error(
+ `[ResumableAgentController] Pause persistence failed for ${streamId}:`,
+ error,
+ );
+ startupTelemetry?.end('error', error);
+ } else if (job.abortController.signal.aborted || error.message?.includes('abort')) {
logger.debug(`[ResumableAgentController] Generation aborted for ${streamId}`);
startupTelemetry?.end('aborted');
// abortJob already handled emitDone and completeJob
} else {
logger.error(`[ResumableAgentController] Generation error for ${streamId}:`, error);
- // Close the steer queue BEFORE the error event reaches clients: a
- // steer POST racing this failure gets 404 (client queues or sends it)
- // instead of a 202 whose payload would vanish with the job. Text
- // recovery is client-side — acknowledged chips convert to queued.
+ const generationError = error.message || 'Generation failed';
try {
- const erroredLeftovers = await GenerationJobManager.steering.closeAndDrain(
- streamId,
- jobCreatedAt,
- );
- if (erroredLeftovers.length > 0) {
- // The error event is a bare string — park the acknowledged
- // steers so a reloaded/disconnected client can still recover
- // them via /chat/status instead of losing them with the queue.
- await GenerationJobManager.steering.park(
- streamId,
- erroredLeftovers.map(toPendingSteer),
- { userId, tenantId: req.user?.tenantId },
- jobCreatedAt,
- );
- }
- } catch (drainErr) {
- logger.warn(
- `[ResumableAgentController] Failed to close steer queue on error`,
- drainErr,
- );
- }
- try {
- await GenerationJobManager.emitError(
- streamId,
- error.message || 'Generation failed',
- jobCreatedAt,
- );
- } catch (notificationError) {
+ // completeJob first wins running -> error and atomically parks
+ // steers, then publishes. A competing abort/pause emits nothing.
+ await GenerationJobManager.completeJob(streamId, generationError, jobCreatedAt);
+ } catch (completeErr) {
logger.warn(
- '[ResumableAgentController] Failed to notify client of generation error',
- notificationError,
+ '[ResumableAgentController] completeJob failed during generation-error cleanup',
+ completeErr,
);
} finally {
startupTelemetry?.end('error', error);
}
- await GenerationJobManager.completeJob(streamId, error.message, jobCreatedAt).catch(
- (completeErr) => {
- logger.warn(
- '[ResumableAgentController] completeJob failed during generation-error cleanup',
- completeErr,
- );
- },
- );
}
try {
@@ -1154,14 +1806,16 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
`[ResumableAgentController] Unhandled error in background generation: ${err.message}`,
);
startupTelemetry?.end('error', err);
- await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
- (completeErr) => {
- logger.warn(
- '[ResumableAgentController] completeJob failed during background-error cleanup',
- completeErr,
- );
- },
- );
+ if (!pausePersistenceFailed) {
+ await GenerationJobManager.completeJob(streamId, err.message, jobCreatedAt).catch(
+ (completeErr) => {
+ logger.warn(
+ '[ResumableAgentController] completeJob failed during background-error cleanup',
+ completeErr,
+ );
+ },
+ );
+ }
try {
await finishResumableRequest(req, userId);
} finally {
@@ -1172,22 +1826,63 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
logger.error('[ResumableAgentController] Initialization error:', error);
try {
if (!res.headersSent) {
- res.status(500).json({ error: error.message || 'Failed to start generation' });
- } else if (jobCreatedAt != null) {
- // JSON already sent, emit error to stream so client can receive it
- await GenerationJobManager.emitError(
- streamId,
- error.message || 'Failed to start generation',
- jobCreatedAt,
- );
+ if (error?.code === 'GENERATION_PREDECESSOR_MISMATCH') {
+ const currentJob = error.currentJob;
+ const currentStatus = currentJob?.status;
+ const predecessorVerified =
+ currentJob != null &&
+ Number.isSafeInteger(currentJob.createdAt) &&
+ currentJob.createdAt >= 0 &&
+ currentJob.verified !== false;
+ sendGenerationJson(
+ res,
+ 409,
+ {
+ status: 'predecessor_mismatch',
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ error: predecessorVerified
+ ? 'A newer generation became current before this request could start.'
+ : 'The prior generation could not be verified. Please retry.',
+ streamId,
+ conversationId: currentJob?.conversationId ?? conversationId,
+ generationCreatedAt: currentJob?.createdAt,
+ predecessorVerified,
+ active:
+ typeof currentJob?.active === 'boolean'
+ ? currentJob.active
+ : currentStatus === 'running' || currentStatus === 'requires_action',
+ },
+ generationProtocolVersion,
+ );
+ } else if (error?.code === 'RECOVERY_PAYLOAD_MISMATCH') {
+ sendGenerationJson(
+ res,
+ 409,
+ {
+ code: 'RECOVERY_PAYLOAD_MISMATCH',
+ error: 'The queued message changed before it could be recovered. Please retry.',
+ },
+ generationProtocolVersion,
+ );
+ } else {
+ sendGenerationJson(
+ res,
+ 500,
+ { error: error.message || 'Failed to start generation' },
+ generationProtocolVersion,
+ );
+ }
}
} catch (notificationError) {
logger.warn(
- '[ResumableAgentController] Failed to notify client of initialization error',
+ '[ResumableAgentController] Failed to send initialization error response',
notificationError,
);
} finally {
- startupTelemetry?.end('error', error);
+ startupTelemetry?.end(
+ error?.code === 'GENERATION_PREDECESSOR_MISMATCH' ? 'deduplicated' : 'error',
+ error,
+ );
}
// Finalize THIS failed job before releasing the idempotency claim. Releasing first would
// let the client's retry win the same key and createJob() the same streamId while we are
@@ -1197,7 +1892,8 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// and the concurrency slot leaks — so swallow its error. (A failed completeJob did not
// finalize anything, so releasing afterward can't let it abort a later replacement.)
if (jobCreatedAt != null) {
- await GenerationJobManager.completeJob(streamId, error.message, jobCreatedAt).catch(
+ const initializationError = error.message || 'Failed to start generation';
+ await GenerationJobManager.completeJob(streamId, initializationError, jobCreatedAt).catch(
(completeErr) => {
logger.warn(
'[ResumableAgentController] completeJob failed during init-error cleanup',
@@ -1206,8 +1902,13 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
},
);
}
- if (ownsIdempotencyClaim) {
- await GenerationJobManager.releaseGeneration(userId, clientRequestId).catch(() => {});
+ if (ownedIdempotencyClaim) {
+ await GenerationJobManager.releaseGeneration(
+ userId,
+ clientRequestId,
+ streamId,
+ ownedIdempotencyClaim,
+ ).catch(() => {});
}
await finishResumableRequest(req, userId);
if (client) {
@@ -1373,6 +2074,7 @@ const _LegacyAgentController = async (req, res, next, initializeClient, addTitle
const job = await GenerationJobManager.createJob(streamId, userId, conversationId);
jobCreatedAt = job.createdAt;
client.jobCreatedAt = jobCreatedAt;
+ client.checkpointNamespace = job.metadata?.checkpointNamespace ?? '';
// Store endpoint metadata for abort handling
GenerationJobManager.updateMetadata(
diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js
index 0d3e557acf..2bd1b50b5d 100644
--- a/api/server/controllers/agents/resume.js
+++ b/api/server/controllers/agents/resume.js
@@ -26,6 +26,20 @@ const {
cleanupMCPRequestContextForReq,
} = require('~/server/services/MCPRequestContext');
const { saveMessage, getConvo, getMessages } = require('~/models');
+const {
+ GENERATION_PROTOCOL_HEADER,
+ negotiateNewGenerationProtocol,
+ negotiateExistingGenerationProtocol,
+} = require('./protocol');
+
+function sendGenerationJson(res, status, body, generationProtocolVersion) {
+ if (typeof res.set === 'function') {
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ } else if (typeof res.setHeader === 'function') {
+ res.setHeader(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ }
+ return res.status(status).json({ ...body, generationProtocolVersion });
+}
/**
* Upper bound on an `ask_user_question` answer (characters). Generous for any real
@@ -41,6 +55,39 @@ const MAX_ASK_ANSWER_LENGTH = 16_000;
*/
const STEER_RESUME_SETUP_TIMEOUT_MS = 1000;
+/**
+ * New jobs are physically isolated by an immutable saver namespace, so a
+ * terminal owner deletes the whole namespace and catches writes that landed
+ * after an earlier read. Pre-isolation jobs share the root namespace and must
+ * retain captured-id cleanup to avoid pruning a replacement.
+ */
+function deleteResumedGenerationCheckpoint({
+ conversationId,
+ checkpointerCfg,
+ job,
+ checkpointGeneration,
+}) {
+ const checkpointNamespace =
+ typeof job?.metadata?.checkpointNamespace === 'string' ? job.metadata.checkpointNamespace : '';
+ if (checkpointNamespace !== '') {
+ return deleteAgentCheckpoint(conversationId, checkpointerCfg, undefined, {
+ checkpointNamespace,
+ });
+ }
+ return deleteAgentCheckpoint(conversationId, checkpointerCfg, checkpointGeneration);
+}
+
+/** Error-path checkpoint cleanup runs after the HTTP ACK. A storage failure
+ * must be observable, but must not escape the controller catch and bypass the
+ * remaining request-context/concurrency/client cleanup in `finally`. */
+async function deleteFailedResumeCheckpoint(args, context) {
+ try {
+ await deleteResumedGenerationCheckpoint(args);
+ } catch (error) {
+ logger.error(`[ResumeAgentController] Failed to prune checkpoint after ${context}`, error);
+ }
+}
+
/** De-duplicate a merged attachment list by a stable artifact identity. */
function mergeAttachments(existing, incoming) {
const seen = new Set();
@@ -92,12 +139,13 @@ async function resolveAccumulatedAttachments({ client, conversationId, responseM
}
/** Resolve the segment's content for an unfinished save (mirrors finalize's source). */
-async function resolveSegmentContent(client, streamId) {
+async function resolveSegmentContent(client, streamId, expectedCreatedAt) {
const liveContent = Array.isArray(client?.contentParts) ? client.contentParts : [];
const rawContent =
liveContent.length > 0
? liveContent
- : ((await GenerationJobManager.getResumeState(streamId))?.aggregatedContent ?? []);
+ : ((await GenerationJobManager.getResumeState(streamId, expectedCreatedAt))
+ ?.aggregatedContent ?? []);
return filterMalformedContentParts(rawContent);
}
@@ -116,7 +164,7 @@ async function persistRePauseProgress({ req, client, job, streamId, conversation
if (!responseMessageId) {
return;
}
- const content = await resolveSegmentContent(client, streamId);
+ const content = await resolveSegmentContent(client, streamId, job.createdAt);
const attachments = await resolveAccumulatedAttachments({
client,
conversationId,
@@ -125,25 +173,24 @@ async function persistRePauseProgress({ req, client, job, streamId, conversation
if (content.length === 0 && attachments.length === 0) {
return;
}
- try {
- await saveMessage(
- {
- userId,
- isTemporary: meta.isTemporary ?? req.body?.isTemporary,
- interfaceConfig: req?.config?.interfaceConfig,
- },
- {
- messageId: responseMessageId,
- conversationId,
- ...(content.length > 0 && { content }),
- ...(attachments.length > 0 && { attachments }),
- unfinished: true,
- user: userId,
- },
- { context: 'api/server/controllers/agents/resume.js - re-pause progress persist' },
- );
- } catch (err) {
- logger.error('[ResumeAgentController] Failed to persist re-pause progress', err);
+ const savedResponseMessage = await saveMessage(
+ {
+ userId,
+ isTemporary: meta.isTemporary ?? req.body?.isTemporary,
+ interfaceConfig: req?.config?.interfaceConfig,
+ },
+ {
+ messageId: responseMessageId,
+ conversationId,
+ ...(content.length > 0 && { content }),
+ ...(attachments.length > 0 && { attachments }),
+ unfinished: true,
+ user: userId,
+ },
+ { context: 'api/server/controllers/agents/resume.js - re-pause progress persist' },
+ );
+ if (!savedResponseMessage) {
+ throw new Error('Re-pause response progress could not be persisted');
}
}
@@ -248,7 +295,8 @@ async function finalizeResumedTurn({
const rawContent =
liveContent.length > 0
? liveContent
- : ((await GenerationJobManager.getResumeState(streamId))?.aggregatedContent ?? []);
+ : ((await GenerationJobManager.getResumeState(streamId, job.createdAt))?.aggregatedContent ??
+ []);
// Parity with the normal agents path (AgentClient strips these before saving):
// drop empty/malformed tool_call parts so a resumed turn can't persist an invalid
// part that breaks reload/rendering.
@@ -319,124 +367,123 @@ async function finalizeResumedTurn({
responseMessage.contextMeta = client.contextMeta;
}
- await saveMessage(
- { userId, isTemporary, interfaceConfig: req?.config?.interfaceConfig },
- responseMessage,
- { context: 'api/server/controllers/agents/resume.js - resumed response end' },
+ // Win terminal ownership BEFORE the outcome-defining response write. Stop
+ // and completion both write the same Mongo row; a later liveness read cannot
+ // fence that external write, while this CAS gives exactly one side authority.
+ // The durable pending marker keeps status/subscribers on the readiness path
+ // until the winner has persisted and published its FINAL.
+ const terminalClaim = await GenerationJobManager.claimTerminalJob(
+ streamId,
+ 'complete',
+ undefined,
+ job.createdAt,
+ { persistencePending: true },
);
-
- const convo = await getConvo(userId, conversationId);
- const conversation = { ...(convo ?? {}), conversationId };
-
- // First-turn pause: the title was deferred when the turn paused. Generate it BEFORE
- // completing the stream so the `title` event still reaches the live client (emitChunk
- // no-ops once completeJob tears down the runtime) and the final event carries the real
- // title instead of "New Chat". Best-effort — a failure must not fail the resumed turn.
- if (
- addTitle &&
- isFirstTurn &&
- !isTemporary &&
- userMessage?.text &&
- (!convo || !convo.title || convo.title === 'New Chat')
- ) {
- try {
- await addTitle(req, {
- text: userMessage.text,
- conversationId,
- client,
- onTitleGenerated: ({ conversationId: titleConvoId, title }) => {
- conversation.title = title;
- return GenerationJobManager.emitChunk(
- streamId,
- {
- event: 'title',
- data: { conversationId: titleConvoId, title },
- },
- { expectedCreatedAt: job.createdAt },
- );
- },
- });
- } catch (err) {
- logger.error('[ResumeAgentController] Title generation failed after resume', err);
- }
- }
- conversation.title = conversation.title || 'New Chat';
-
- // Re-check ownership immediately before the terminal writes. The start-of-function
- // guard can go stale across the awaits above: saveMessage and (first-turn) title
- // generation can take long enough for a new request to replace this job on the same
- // conversationId (streamId == conversationId). Without this second read, emitDone /
- // completeJob / prune below would emit `done` to and tear down the REPLACEMENT job —
- // the same hazard the catch-path guard prevents on the failure path.
- const liveJobBeforeFinalize = await GenerationJobManager.getJobStore().getJob(streamId);
- if (!liveJobBeforeFinalize || liveJobBeforeFinalize.createdAt !== job.createdAt) {
+ if (!terminalClaim) {
logger.warn(
- `[ResumeAgentController] Skipping resumed terminal writes — job ${streamId} was replaced mid-finalize`,
+ `[ResumeAgentController] Skipping resumed FINAL — another terminal/pause transition won for ${streamId}`,
);
return;
}
-
- // Steers that never reached an injection boundary during the resumed
- // segment — mirror the normal request path's terminal drain: the atomic
- // close (createdAt-guarded) rejects a steer POST racing this finalization,
- // and the leftovers ride the final event as queued follow-ups instead of
- // being 202-ACKed and then silently cleared by completeJob.
- let pendingSteers;
+ let terminalPublicationStarted = false;
try {
- const leftoverSteers = await GenerationJobManager.steering.closeAndDrain(
- streamId,
- job.createdAt,
+ const savedResponseMessage = await saveMessage(
+ { userId, isTemporary, interfaceConfig: req?.config?.interfaceConfig },
+ responseMessage,
+ { context: 'api/server/controllers/agents/resume.js - resumed response end' },
);
- if (leftoverSteers.length > 0) {
- pendingSteers = leftoverSteers.map(toPendingSteer);
- // Same no-subscriber recovery as the normal final path (claim-on-read
- // via /chat/status within the recovery TTL). NOTE: `job` is the manager
- // facade — owner fields live under `metadata` (a bare `job.userId` is
- // undefined and would make the parked payload unclaimable).
- await GenerationJobManager.steering.park(
- streamId,
- pendingSteers,
- {
- userId: job.metadata?.userId,
- tenantId: job.metadata?.tenantId,
- },
- job.createdAt,
- );
+ if (!savedResponseMessage) {
+ throw new Error('Resumed response could not be persisted before terminal publication');
}
- } catch (drainErr) {
- logger.warn('[ResumeAgentController] Failed to drain leftover steers', drainErr);
- }
- const finalEvent = {
- final: true,
- conversation,
- title: conversation.title,
- requestMessage: userMessage
- ? sanitizeMessageForTransmit({
- ...userMessage,
+ const convo = await getConvo(userId, conversationId);
+ const conversation = { ...(convo ?? {}), conversationId };
+
+ // First-turn pause: the title was deferred when the turn paused. Generate it BEFORE
+ // completing the stream so the `title` event still reaches the live client (emitChunk
+ // no-ops once completeJob tears down the runtime) and the final event carries the real
+ // title instead of "New Chat". Best-effort — a failure must not fail the resumed turn.
+ if (
+ addTitle &&
+ isFirstTurn &&
+ !isTemporary &&
+ userMessage?.text &&
+ (!convo || !convo.title || convo.title === 'New Chat')
+ ) {
+ try {
+ await addTitle(req, {
+ text: userMessage.text,
conversationId,
- isCreatedByUser: true,
- // job.metadata.userMessage is persisted without files; carry the restored
- // uploads (seeded onto req.body.files before reconstruction) so the final SSE
- // doesn't blank the user bubble's attachments — matching the normal path.
- ...(Array.isArray(req.body?.files) && req.body.files.length > 0
- ? { files: req.body.files }
- : {}),
- })
- : null,
- responseMessage: { ...responseMessage },
- ...(pendingSteers && { pendingSteers }),
- };
+ client,
+ onTitleGenerated: ({ conversationId: titleConvoId, title }) => {
+ conversation.title = title;
+ return GenerationJobManager.emitChunk(
+ streamId,
+ {
+ event: 'title',
+ data: { conversationId: titleConvoId, title },
+ },
+ { expectedCreatedAt: job.createdAt },
+ );
+ },
+ });
+ } catch (err) {
+ logger.error('[ResumeAgentController] Title generation failed after resume', err);
+ }
+ }
+ conversation.title = conversation.title || 'New Chat';
- await GenerationJobManager.emitDone(streamId, finalEvent, job.createdAt);
- // Awaited (not fire-and-forget) so the job's terminal write lands before the
- // checkpoint prune, and so a failure here doesn't race the controller's error path.
- try {
- await GenerationJobManager.completeJob(streamId, undefined, job.createdAt);
- } catch (completeErr) {
- logger.error('[ResumeAgentController] Failed to complete resumed turn', completeErr);
+ const pendingSteers = terminalClaim.drainedSteers.map(toPendingSteer);
+ const finalEvent = {
+ final: true,
+ conversation,
+ title: conversation.title,
+ requestMessage: userMessage
+ ? sanitizeMessageForTransmit({
+ ...userMessage,
+ conversationId,
+ isCreatedByUser: true,
+ // job.metadata.userMessage is persisted without files; carry the restored
+ // uploads (seeded onto req.body.files before reconstruction) so the final SSE
+ // doesn't blank the user bubble's attachments — matching the normal path.
+ ...(Array.isArray(req.body?.files) && req.body.files.length > 0
+ ? { files: req.body.files }
+ : {}),
+ })
+ : null,
+ responseMessage: { ...responseMessage },
+ ...(pendingSteers.length > 0 && { pendingSteers }),
+ };
+
+ terminalPublicationStarted = true;
+ await GenerationJobManager.publishTerminalClaim(terminalClaim, finalEvent);
+ } catch (error) {
+ if (!terminalPublicationStarted) {
+ try {
+ await GenerationJobManager.publishTerminalClaim(terminalClaim, null);
+ } catch (publishError) {
+ logger.error(
+ '[ResumeAgentController] Failed to publish terminal persistence reconciliation',
+ publishError,
+ );
+ }
+ }
+ throw error;
+ } finally {
+ try {
+ // Cleanup must run even if persistence/publication fails. The claim
+ // carries the exact generation/runtime identity, so this cannot tear
+ // down a later run.
+ await GenerationJobManager.finishTerminalJob(terminalClaim);
+ } finally {
+ await deleteResumedGenerationCheckpoint({
+ conversationId,
+ checkpointerCfg,
+ job,
+ checkpointGeneration,
+ });
+ }
}
- await deleteAgentCheckpoint(conversationId, checkpointerCfg, checkpointGeneration);
}
/**
@@ -461,22 +508,50 @@ async function finalizeResumedTurn({
*/
const ResumeAgentController = async (req, res, next, initializeClient, addTitle) => {
const userId = req.user.id;
- const { conversationId, actionId } = req.body;
+ let generationProtocolVersion = negotiateNewGenerationProtocol(req, GenerationJobManager);
+ const { conversationId, actionId, generationCreatedAt } = req.body;
const streamId = conversationId;
if (!streamId || streamId === 'new') {
- return res.status(400).json({ error: 'conversationId is required to resume' });
+ return sendGenerationJson(
+ res,
+ 400,
+ { error: 'conversationId is required to resume' },
+ generationProtocolVersion,
+ );
+ }
+ if (
+ generationCreatedAt != null &&
+ (!Number.isSafeInteger(generationCreatedAt) || generationCreatedAt < 0)
+ ) {
+ return sendGenerationJson(
+ res,
+ 400,
+ { code: 'INVALID_GENERATION_IDENTITY' },
+ generationProtocolVersion,
+ );
}
const job = await GenerationJobManager.getJob(streamId);
if (!job) {
- return res.status(404).json({ error: 'No paused generation for this conversation' });
+ return sendGenerationJson(
+ res,
+ 404,
+ { error: 'No paused generation for this conversation' },
+ generationProtocolVersion,
+ );
}
- if (job.metadata?.userId && job.metadata.userId !== userId) {
- return res.status(403).json({ error: 'Unauthorized' });
+ // Every persisted generation is owner-scoped. A missing/corrupt owner is
+ // not a legacy wildcard: fail closed before reading or resolving its action.
+ if (job.metadata?.userId !== userId) {
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, generationProtocolVersion);
}
if (hasTenantMismatch(job, req.user)) {
- return res.status(403).json({ error: 'Unauthorized' });
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, generationProtocolVersion);
+ }
+ generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
+ if (generationCreatedAt != null && job.createdAt !== generationCreatedAt) {
+ return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion);
}
// The resume must rebuild the SAME agent/endpoint that paused. Require an EXACT
@@ -486,7 +561,12 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// sends the right one.
const originalAgentId = job.metadata?.agent_id;
if (originalAgentId && req.body.agent_id !== originalAgentId) {
- return res.status(403).json({ error: 'Cannot resume with a different agent' });
+ return sendGenerationJson(
+ res,
+ 403,
+ { error: 'Cannot resume with a different agent' },
+ generationProtocolVersion,
+ );
}
// Require an EXACT endpoint match (like agent_id): a request that OMITS endpoint must
// not fall through — the shared chat middleware treats a missing/non-agents endpoint
@@ -494,12 +574,22 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// different graph. A correct client always echoes the paused endpoint.
const originalEndpoint = job.metadata?.endpoint;
if (originalEndpoint && req.body.endpoint !== originalEndpoint) {
- return res.status(403).json({ error: 'Cannot resume on a different endpoint' });
+ return sendGenerationJson(
+ res,
+ 403,
+ { error: 'Cannot resume on a different endpoint' },
+ generationProtocolVersion,
+ );
}
const pendingAction = job.metadata?.pendingAction;
if (job.status !== 'requires_action') {
- return res.status(409).json({ error: 'No live pending action to resume' });
+ return sendGenerationJson(
+ res,
+ 409,
+ { error: 'No live pending action to resume' },
+ generationProtocolVersion,
+ );
}
if (isPendingActionStale({ pendingAction })) {
// The action expired between the pending-action SSE and this submit. Drive the expiry
@@ -508,22 +598,37 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// client never gets a terminal event, so the stream appears to hang even though the
// UI already reported the action as expired.
try {
- await GenerationJobManager.expireApproval(streamId, pendingAction?.actionId);
+ await GenerationJobManager.expireApproval(streamId, pendingAction?.actionId, job.createdAt);
} catch (err) {
logger.warn(
'[ResumeAgentController] Failed to expire stale action on submit',
err?.message ?? err,
);
}
- return res.status(409).json({ error: 'No live pending action to resume' });
+ return sendGenerationJson(
+ res,
+ 409,
+ { error: 'No live pending action to resume' },
+ generationProtocolVersion,
+ );
}
// Require the actionId the UI sends: without it, a stale/malformed client could
// resolve whatever action is currently pending (e.g. answer a different question).
if (!actionId) {
- return res.status(400).json({ error: 'actionId is required to resume' });
+ return sendGenerationJson(
+ res,
+ 400,
+ { error: 'actionId is required to resume' },
+ generationProtocolVersion,
+ );
}
if (pendingAction.actionId !== actionId) {
- return res.status(409).json({ error: 'This decision targets a stale action' });
+ return sendGenerationJson(
+ res,
+ 409,
+ { error: 'This decision targets a stale action' },
+ generationProtocolVersion,
+ );
}
// Pin the graph identity: the resume must rebuild the SAME agent/graph + tool set the
@@ -534,42 +639,61 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// change won't), and recomputed from the resume body's graph-determining fields.
const pinnedFingerprint = pendingAction.requestFingerprint;
if (pinnedFingerprint && pinnedFingerprint !== computeAgentRequestFingerprint(req.body ?? {})) {
- return res.status(403).json({ error: 'Cannot resume with a different agent configuration' });
+ return sendGenerationJson(
+ res,
+ 403,
+ { error: 'Cannot resume with a different agent configuration' },
+ generationProtocolVersion,
+ );
}
const mapped = resolveResumeValue(pendingAction, req.body);
if (mapped.error) {
- return res.status(mapped.status).json({
- error: mapped.error,
- ...(mapped.undecided && { undecided: mapped.undecided }),
- ...(mapped.disallowed && { disallowed: mapped.disallowed }),
- ...(mapped.incomplete && { incomplete: mapped.incomplete }),
- });
+ return sendGenerationJson(
+ res,
+ mapped.status,
+ {
+ error: mapped.error,
+ ...(mapped.undecided && { undecided: mapped.undecided }),
+ ...(mapped.disallowed && { disallowed: mapped.disallowed }),
+ ...(mapped.incomplete && { incomplete: mapped.incomplete }),
+ },
+ generationProtocolVersion,
+ );
}
- // Snapshot the exact durable checkpoint ids before the atomic resume claim. The
- // claim is the linearization point: a replacement that already owns this stream
- // makes it fail, while one that starts afterward writes fresh ids outside the
- // snapshot. Terminal cleanup can therefore delete this generation without a
- // check-then-delete race against a later pause on the same conversation.
+ // A legacy job has no saver-level generation namespace, so snapshot its exact
+ // durable ids before the atomic resume claim. New jobs can skip this indexed
+ // read: terminal cleanup deletes their whole immutable namespace, including
+ // writes that land while the continuation is running.
//
// Start the indexed read alongside the independent concurrency check so the
// generation guard adds minimal time to the resume ACK path.
const checkpointerCfg = req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer;
- const checkpointGenerationPromise = captureAgentCheckpointGeneration(
- conversationId,
- checkpointerCfg,
- ).catch((err) => {
- logger.warn('[ResumeAgentController] Failed to capture checkpoint generation', err);
- return { threadId: conversationId, checkpointIds: [] };
- });
+ const checkpointNamespace =
+ typeof job.metadata?.checkpointNamespace === 'string' ? job.metadata.checkpointNamespace : '';
+ const checkpointGenerationPromise =
+ checkpointNamespace !== ''
+ ? Promise.resolve(undefined)
+ : captureAgentCheckpointGeneration(conversationId, checkpointerCfg).catch((err) => {
+ logger.warn('[ResumeAgentController] Failed to capture checkpoint generation', err);
+ return {
+ threadId: conversationId,
+ checkpointIds: [],
+ };
+ });
// Count the resume against the concurrency limit. The original turn released its slot
// when it paused, so resuming must re-acquire one — otherwise pausing several turns
// and resuming them at once would bypass LIMIT_CONCURRENT_MESSAGES.
const { allowed } = await checkAndIncrementPendingRequest(userId);
if (!allowed) {
- return res.status(429).json({ error: 'Too many concurrent requests' });
+ return sendGenerationJson(
+ res,
+ 429,
+ { error: 'Too many concurrent requests' },
+ generationProtocolVersion,
+ );
}
// Atomically claim the resume. The single winner drives the run; a racing second
@@ -584,40 +708,36 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
let checkpointGeneration;
try {
checkpointGeneration = await checkpointGenerationPromise;
- claimed = await GenerationJobManager.approvals.resolve(streamId, pendingAction.actionId);
+ /** The CAS that reopens steering must also publish THIS owner's seal
+ * capability. A separate write after status=`running` leaves a window in
+ * which steer/arm requests read the previous replica's capability. */
+ claimed = await GenerationJobManager.approvals.resolve(
+ streamId,
+ pendingAction.actionId,
+ {
+ preemptCapable: isSteerPreemptSupported(),
+ },
+ job.createdAt,
+ );
} catch (err) {
await decrementPendingRequest(userId);
logger.error('[ResumeAgentController] Failed to claim resume', err);
- return res.status(500).json({ error: 'Failed to resume' });
+ return sendGenerationJson(res, 500, { error: 'Failed to resume' }, generationProtocolVersion);
}
if (!claimed) {
await decrementPendingRequest(userId);
- return res.status(409).json({ error: 'This action was already resolved or has expired' });
+ const currentJob = await GenerationJobManager.getJob(streamId).catch(() => null);
+ if (currentJob != null && currentJob.createdAt !== job.createdAt) {
+ return sendGenerationJson(res, 409, { code: 'RUN_REPLACED' }, generationProtocolVersion);
+ }
+ return sendGenerationJson(
+ res,
+ 409,
+ { error: 'This action was already resolved or has expired' },
+ generationProtocolVersion,
+ );
}
- /**
- * Ownership moves on resume, so the job's recorded seal capability must
- * describe THIS replica — otherwise a job created on a capable replica that
- * resumes on an older one during a rolling deploy keeps acknowledging
- * steers as interrupting. Written IMMEDIATELY after the claim, before
- * client reconstruction: `approvals.resolve` has already flipped the job
- * back to `running`, so the steer route is accepting requests from here on
- * and every one of them reads this flag.
- */
- const capabilityRefresh = GenerationJobManager.updateMetadata(
- streamId,
- { preemptCapable: isSteerPreemptSupported() },
- job.createdAt,
- ).catch((error) => {
- /**
- * Logged, not fatal: this is a chip-label accuracy write, and failing the
- * user's resume over it would be disproportionate. The write only fails
- * when the job store is erroring, in which case the steer route's own
- * `getJob` is degraded too — it cannot read a stale flag it cannot read.
- */
- logger.error('[ResumeAgentController] Failed to refresh preempt capability', error);
- });
-
/**
* An interrupt steer enqueued just before the pause survives durably with
* its `preempt` flag, but the ARM lived only in the previous owner's
@@ -640,13 +760,14 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
* because the action is spent, and neither the continuation nor the
* failed-resume cleanup ever runs.
*
- * Both writes are steering bookkeeping — a chip label and an arm that the
- * next tool boundary would honour anyway — so they finish in the background
- * rather than holding a resume the user is waiting on.
+ * Re-arming is steering bookkeeping that the next tool boundary would
+ * honour anyway, so it finishes in the background rather than holding a
+ * resume the user is waiting on. Capability is not in this best-effort path:
+ * it was committed atomically by the resume claim above.
*/
let steeringSetupTimer;
await Promise.race([
- Promise.all([capabilityRefresh, preemptRearm]),
+ preemptRearm,
new Promise((resolve) => {
steeringSetupTimer = setTimeout(() => {
logger.warn(
@@ -669,7 +790,12 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
getMCPRequestContext(req, undefined, { cleanupOnResponse: false });
// ACK immediately; the continuation streams over the client's existing SSE.
- res.json({ streamId, conversationId, status: 'resuming' });
+ sendGenerationJson(
+ res,
+ 200,
+ { streamId, conversationId, status: 'resuming' },
+ generationProtocolVersion,
+ );
// Seed the original thread parent BEFORE initializeClient: initializeAgent scopes
// thread files / code artifacts off `req.body.parentMessageId`, and the resume body
@@ -678,6 +804,12 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// the user message id.
req.body.parentMessageId = job.metadata.userMessage?.parentMessageId ?? Constants.NO_PARENT;
+ // Rebuild the same persistence/retention mode as the paused turn. The resume body is
+ // not authoritative here: image/code tools inspect `req.body.isTemporary` during
+ // initializeClient, and a missing or crafted value must not make a temporary chat's
+ // artifacts durable (or make a durable chat's artifacts ephemeral).
+ req.body.isTemporary = job.metadata.isTemporary === true;
+
// Restore the paused user message's OWN uploaded files. initializeAgent rebuilds
// code/file sessions by walking the conversation from `parentMessageId`, but
// execute-code files are excluded from that lookup, so files uploaded on the paused
@@ -740,6 +872,11 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
}
let client = null;
+ /** Re-pause progress failures use the action/epoch-scoped terminal CAS. The
+ * generic resume catch must not subsequently call completeJob, because the
+ * failed pause may have lost ownership to a newer action or generation. */
+ let pausePersistenceFailed = false;
+ let pausePersistenceFailureFinalized = false;
try {
const result = await initializeClient({
req,
@@ -747,6 +884,7 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
endpointOption: req.body.endpointOption,
signal: job.abortController.signal,
jobCreatedAt: job.createdAt,
+ checkpointNamespace,
});
client = result.client;
@@ -756,12 +894,13 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// the paused job's createdAt — used by the re-pause CAS pre-check + checkpoint prune to
// avoid acting on a job a newer request has since replaced.
client.jobCreatedAt = job.createdAt;
+ client.checkpointNamespace = checkpointNamespace;
client.responseMessageId = job.metadata.responseMessageId;
client.parentMessageId = job.metadata.userMessage?.messageId ?? Constants.NO_PARENT;
// Read the pre-pause content BEFORE swapping the store's content reference: the
// in-memory store's setContentParts REPLACES the stored array, so reading the
// resume state afterward would see the new (empty) client array and lose the seed.
- const resumeState = await GenerationJobManager.getResumeState(streamId);
+ const resumeState = await GenerationJobManager.getResumeState(streamId, job.createdAt);
let seedContent = resumeState?.aggregatedContent ?? [];
// Stamp the answered question onto the paused ask_user_question tool-call part
// (args = the pendingAction's authoritative question, output = the user's answer):
@@ -796,10 +935,59 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
// action is already persisted + emitted; leave the job `requires_action`.
if (client.pendingApproval) {
logger.debug(`[ResumeAgentController] Re-paused for approval: ${streamId}`);
- // Persist this segment's content + artifacts before the fresh client (next
- // resume) drops them, so an expiring re-pause doesn't lose them; finalize later
- // overwrites content and merges attachments onto the saved message.
- await persistRePauseProgress({ req, client, job, streamId, conversationId });
+ const pauseActionId = client.pendingApproval.actionId;
+ const pauseCreatedAt = client.jobCreatedAt ?? job.createdAt;
+ const ownsPausePersistence = await GenerationJobManager.approvals.ownsPausePersistence(
+ streamId,
+ pauseActionId,
+ pauseCreatedAt,
+ );
+ if (ownsPausePersistence) {
+ try {
+ // Persist this segment's content + artifacts before the fresh client (next
+ // resume) drops them, so an expiring re-pause doesn't lose them; finalize later
+ // overwrites content and merges attachments onto the saved message. A failed
+ // required write must reject into the error-finalization path rather than expose
+ // the next action while its preceding segment is absent from durable history.
+ await persistRePauseProgress({ req, client, job, streamId, conversationId });
+ } catch (pausePersistenceError) {
+ pausePersistenceFailed = true;
+ try {
+ pausePersistenceFailureFinalized =
+ (await GenerationJobManager.failPausePersistence(
+ streamId,
+ pauseActionId,
+ pausePersistenceError?.message ?? 'Re-pause persistence failed',
+ pauseCreatedAt,
+ )) === true;
+ if (!pausePersistenceFailureFinalized) {
+ logger.warn(
+ `[ResumeAgentController] Skipping stale re-pause persistence failure — ${streamId} no longer owns its barrier`,
+ );
+ }
+ } catch (failError) {
+ logger.error(
+ `[ResumeAgentController] Failed to terminalize re-pause persistence error for ${streamId}`,
+ failError,
+ );
+ }
+ throw pausePersistenceError;
+ }
+ const released = await GenerationJobManager.approvals.finishPausePersistence(
+ streamId,
+ pauseActionId,
+ pauseCreatedAt,
+ );
+ if (!released) {
+ logger.warn(
+ `[ResumeAgentController] Re-pause persistence barrier changed before release: ${streamId}`,
+ );
+ }
+ } else {
+ logger.debug(
+ `[ResumeAgentController] Skipping stale re-pause persistence — ${streamId} no longer owns its barrier`,
+ );
+ }
return;
}
@@ -823,6 +1011,23 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
});
} catch (err) {
logger.error('[ResumeAgentController] Resume failed', err);
+ if (pausePersistenceFailed) {
+ // failPausePersistence already performed the exact requires_action ->
+ // error transition. Only its CAS winner owns this generation's checkpoint
+ // cleanup; a stale/mismatched failure must leave the live scope intact.
+ if (pausePersistenceFailureFinalized) {
+ await deleteFailedResumeCheckpoint(
+ {
+ conversationId,
+ checkpointerCfg,
+ job,
+ checkpointGeneration,
+ },
+ 're-pause persistence failure',
+ );
+ }
+ return;
+ }
// Job-replacement guard (mirrors finalizeResumedTurn's success-path guard): if a
// newer request reused this conversationId while the resume was failing, do NOT emit
// the error to / complete / prune the NEWER turn's job. The finally still releases
@@ -839,67 +1044,31 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
`[ResumeAgentController] Skipping failed-resume finalization — job ${streamId} was replaced`,
);
} else {
- // A steer 202-accepted during the failed resume segment would otherwise
- // be silently cleared by completeJob's backstop — mirror the normal
- // request error path: close the queue BEFORE the error event (racing
- // steer POSTs get 404) and park the leftovers for /chat/status recovery.
+ // completeJob atomically claims running -> error and parks steers before
+ // publishing. If abort or a re-pause won, it returns false; only the
+ // terminal-CAS winner may delete this generation's checkpoint scope.
+ let errorFinalized = false;
try {
- const leftoverSteers = await GenerationJobManager.steering.closeAndDrain(
- streamId,
- job.createdAt,
- );
- if (leftoverSteers.length > 0) {
- // Facade shape: owner fields are under `metadata` (see finalize).
- await GenerationJobManager.steering.park(
+ errorFinalized =
+ (await GenerationJobManager.completeJob(
streamId,
- leftoverSteers.map(toPendingSteer),
- {
- userId: job.metadata?.userId,
- tenantId: job.metadata?.tenantId,
- },
+ err?.message ?? 'Resume failed',
job.createdAt,
- );
- }
- } catch (drainErr) {
- logger.warn('[ResumeAgentController] Failed to drain steers on resume failure', drainErr);
- }
- try {
- await GenerationJobManager.emitError(
- streamId,
- err?.message ?? 'Resume failed',
- job.createdAt,
- );
- } catch (emitErr) {
- logger.error('[ResumeAgentController] Failed to emit resume error', emitErr);
- }
- try {
- await GenerationJobManager.completeJob(
- streamId,
- err?.message ?? 'Resume failed',
- job.createdAt,
- );
+ )) === true;
} catch (completeErr) {
logger.error('[ResumeAgentController] Failed to finalize failed resume', completeErr);
- // Last resort: force a terminal state so the job isn't orphaned in `running`.
- await GenerationJobManager.getJobStore()
- .updateJob(
- streamId,
- {
- status: 'error',
- completedAt: Date.now(),
- error: 'Resume failed',
- },
- job.createdAt,
- )
- .catch((updErr) =>
- logger.error('[ResumeAgentController] Fallback job finalize failed', updErr),
- );
}
- await deleteAgentCheckpoint(
- conversationId,
- req.config?.endpoints?.[EModelEndpoint.agents]?.checkpointer,
- checkpointGeneration,
- );
+ if (errorFinalized) {
+ await deleteFailedResumeCheckpoint(
+ {
+ conversationId,
+ checkpointerCfg,
+ job,
+ checkpointGeneration,
+ },
+ 'failed resume finalization',
+ );
+ }
}
} finally {
// Tear down the MCP request-context store seeded before the ACK (parity with
diff --git a/api/server/controllers/agents/steer.js b/api/server/controllers/agents/steer.js
index baa86de010..4ee163c3a7 100644
--- a/api/server/controllers/agents/steer.js
+++ b/api/server/controllers/agents/steer.js
@@ -1,4 +1,10 @@
-const { checkAccess, handleSteerRequest, handleSteerCancel } = require('@librechat/api');
+const {
+ checkAccess,
+ GenerationJobManager,
+ handleSteerRequest,
+ handleSteerCancel,
+ handleSteerArm,
+} = require('@librechat/api');
const { logger, ResourceCapabilityMap } = require('@librechat/data-schemas');
const {
Permissions,
@@ -10,8 +16,30 @@ const {
} = require('librechat-data-provider');
const { checkPermission } = require('~/server/services/PermissionService');
const { hasCapability } = require('~/server/middleware/roles/capabilities');
+const {
+ GENERATION_PROTOCOL_HEADER,
+ getRequestedGenerationProtocol,
+ getServerGenerationProtocol,
+} = require('~/server/controllers/agents/protocol');
const db = require('~/models');
+/** Upper bound before the package reads the immutable live-job marker. */
+const getHostGenerationProtocol = (req) =>
+ Math.min(getRequestedGenerationProtocol(req), getServerGenerationProtocol(GenerationJobManager));
+
+/** The package returns its job-capped effective marker in every body. Keep the
+ * header and JSON inseparable at this final serialization boundary. */
+const sendProtocolResult = (res, status, body) => {
+ const generationProtocolVersion = body?.generationProtocolVersion === 2 ? 2 : 1;
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ return res.status(status).json({ ...body, generationProtocolVersion });
+};
+
+const sendProtocolFailure = (res, status, code) => {
+ res.set(GENERATION_PROTOCOL_HEADER, '1');
+ return res.status(status).json({ code, generationProtocolVersion: 1 });
+};
+
/**
* Steer-time agent authorization, mirroring the chat route's middlewares
* (`checkAgentAccess` + `canAccessAgentFromBody`) against the ORIGINATING
@@ -77,15 +105,17 @@ const createAgentAccessCheck =
*/
const SteerController = async (req, res) => {
try {
+ const generationProtocolVersion = getHostGenerationProtocol(req);
const { status, body } = await handleSteerRequest(req.user ?? {}, req.body ?? {}, {
+ generationProtocolVersion,
getFiles: db.getFiles,
updateFilesUsage: db.updateFilesUsage,
checkAgentAccess: createAgentAccessCheck(req),
});
- return res.status(status).json(body);
+ return sendProtocolResult(res, status, body);
} catch (error) {
logger.error('[SteerController] Failed to queue steer', error);
- return res.status(500).json({ code: 'STEER_FAILED' });
+ return sendProtocolFailure(res, 500, 'STEER_FAILED');
}
};
@@ -99,13 +129,39 @@ const SteerController = async (req, res) => {
*/
const SteerCancelController = async (req, res) => {
try {
- const { status, body } = await handleSteerCancel(req.user ?? {}, req.body ?? {});
- return res.status(status).json(body);
+ const generationProtocolVersion = getHostGenerationProtocol(req);
+ const { status, body } = await handleSteerCancel(req.user ?? {}, req.body ?? {}, {
+ generationProtocolVersion,
+ });
+ return sendProtocolResult(res, status, body);
} catch (error) {
logger.error('[SteerCancelController] Failed to cancel steer', error);
- return res.status(500).json({ code: 'STEER_CANCEL_FAILED' });
+ return sendProtocolFailure(res, 500, 'STEER_CANCEL_FAILED');
+ }
+};
+
+/**
+ * POST /api/agents/chat/steer/arm
+ *
+ * Escalates a still-queued steer to an interrupt in place (the durable item
+ * keeps its FIFO position). `armed: false` is not an error — the steer
+ * already injected, was cancelled, or the deployment cannot seal mid-stream.
+ * No agent-access check: arming injects nothing model-bound, so ownership
+ * checks suffice, exactly like cancel.
+ */
+const SteerArmController = async (req, res) => {
+ try {
+ const generationProtocolVersion = getHostGenerationProtocol(req);
+ const { status, body } = await handleSteerArm(req.user ?? {}, req.body ?? {}, {
+ generationProtocolVersion,
+ });
+ return sendProtocolResult(res, status, body);
+ } catch (error) {
+ logger.error('[SteerArmController] Failed to arm steer', error);
+ return sendProtocolFailure(res, 500, 'STEER_ARM_FAILED');
}
};
module.exports = SteerController;
module.exports.SteerCancelController = SteerCancelController;
+module.exports.SteerArmController = SteerArmController;
diff --git a/api/server/routes/agents/__tests__/abort.spec.js b/api/server/routes/agents/__tests__/abort.spec.js
index 538e01b243..d9916504af 100644
--- a/api/server/routes/agents/__tests__/abort.spec.js
+++ b/api/server/routes/agents/__tests__/abort.spec.js
@@ -63,6 +63,7 @@ jest.mock('~/server/routes/agents/v1', () => ({
// Import after mocks
const agentRoutes = require('~/server/routes/agents/index');
+const { isUnpersistedPreliminaryParent } = require('@librechat/api');
describe('Agent Abort Endpoint', () => {
let app;
@@ -79,6 +80,7 @@ describe('Agent Abort Endpoint', () => {
mockGenerationJobManager.abortJob.mockReset();
mockGenerationJobManager.getActiveJobIdsForUser.mockReset();
mockSaveMessage.mockReset();
+ mockSaveMessage.mockImplementation(async (_context, message) => message);
});
describe('POST /chat/abort', () => {
@@ -95,7 +97,7 @@ describe('Agent Abort Endpoint', () => {
.send({ conversationId: jobStreamId });
expect(response.status).toBe(403);
- expect(response.body).toEqual({ error: 'Unauthorized' });
+ expect(response.body).toEqual({ error: 'Unauthorized', generationProtocolVersion: 1 });
expect(mockLogger.warn).toHaveBeenCalledWith(
expect.stringContaining('Unauthorized abort attempt'),
);
@@ -121,14 +123,19 @@ describe('Agent Abort Endpoint', () => {
.send({ conversationId: jobStreamId });
expect(response.status).toBe(200);
- expect(response.body).toEqual({ success: true, aborted: jobStreamId });
+ expect(response.body).toEqual({
+ success: true,
+ aborted: jobStreamId,
+ generationProtocolVersion: 1,
+ });
+ expect(response.headers['x-librechat-generation-protocol']).toBe('1');
expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(
jobStreamId,
expect.objectContaining({ transformAbortContent: expect.any(Function) }),
);
});
- it('should allow abort when job has no userId metadata (backwards compatibility)', async () => {
+ it('should fail closed when job has no userId metadata', async () => {
const jobStreamId = 'test-stream-123';
mockGenerationJobManager.getJob.mockResolvedValue({
@@ -146,8 +153,9 @@ describe('Agent Abort Endpoint', () => {
.post('/api/agents/chat/abort')
.send({ conversationId: jobStreamId });
- expect(response.status).toBe(200);
- expect(response.body).toEqual({ success: true, aborted: jobStreamId });
+ expect(response.status).toBe(403);
+ expect(response.body).toEqual({ error: 'Unauthorized', generationProtocolVersion: 1 });
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
});
});
@@ -156,7 +164,7 @@ describe('Agent Abort Endpoint', () => {
const jobStreamId = 'test-stream-123';
mockGenerationJobManager.getJob.mockResolvedValue({
- metadata: { userId: 'test-user-123' },
+ metadata: { userId: 'test-user-123', generationProtocolVersion: 2 },
});
mockGenerationJobManager.abortJob.mockResolvedValue({
@@ -172,7 +180,8 @@ describe('Agent Abort Endpoint', () => {
const response = await request(app)
.post('/api/agents/chat/abort')
- .send({ conversationId: jobStreamId });
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: jobStreamId, generationProtocolVersion: 2 });
expect(response.status).toBe(200);
expect(mockSaveMessage).not.toHaveBeenCalled();
@@ -240,6 +249,91 @@ describe('Agent Abort Endpoint', () => {
expect(response.status).toBe(200);
expect(mockSaveMessage).not.toHaveBeenCalled();
});
+
+ it('persists a created empty turn before FINAL so interrupt-and-send clears the parent fence', async () => {
+ const jobStreamId = 'test-stream-123';
+ const userMessageId = 'user-msg-123';
+ const preliminaryResponseId = `${userMessageId}_`;
+ const persistedMessages = new Map();
+
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'test-user-123', generationProtocolVersion: 2 },
+ });
+
+ const abortResult = {
+ success: true,
+ jobData: {
+ createdEventEmitted: true,
+ userMessage: {
+ messageId: userMessageId,
+ parentMessageId: 'older-response',
+ conversationId: jobStreamId,
+ text: 'Stop before the first model token.',
+ },
+ responseMessageId: preliminaryResponseId,
+ conversationId: jobStreamId,
+ endpoint: 'agents',
+ sender: 'TestAgent',
+ model: 'agent-1',
+ },
+ content: [],
+ text: '',
+ };
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ await options.beforePublish(abortResult);
+ return abortResult;
+ });
+ mockSaveMessage.mockImplementation(async (_context, message) => {
+ persistedMessages.set(message.messageId, message);
+ return message;
+ });
+
+ const response = await request(app)
+ .post('/api/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: jobStreamId, generationProtocolVersion: 2 });
+
+ expect(response.status).toBe(200);
+ expect(mockSaveMessage).toHaveBeenNthCalledWith(
+ 1,
+ expect.anything(),
+ expect.objectContaining({
+ messageId: userMessageId,
+ conversationId: jobStreamId,
+ isCreatedByUser: true,
+ }),
+ expect.objectContaining({ context: expect.stringContaining('user prerequisite') }),
+ );
+ expect(mockSaveMessage).toHaveBeenNthCalledWith(
+ 2,
+ expect.anything(),
+ expect.objectContaining({
+ messageId: preliminaryResponseId,
+ parentMessageId: userMessageId,
+ conversationId: jobStreamId,
+ content: [],
+ text: '',
+ unfinished: true,
+ isCreatedByUser: false,
+ }),
+ expect.objectContaining({ context: expect.stringContaining('abort endpoint') }),
+ );
+
+ /** This is the exact server-side fence hit by the queued submission
+ * after the abort FINAL. It must observe the row written above rather
+ * than reject the drain solely because the stable id ends in `_`. */
+ await expect(
+ isUnpersistedPreliminaryParent({
+ userId: 'test-user-123',
+ conversationId: jobStreamId,
+ parentMessageId: preliminaryResponseId,
+ getMessages: async (filter) =>
+ persistedMessages.has(filter.messageId)
+ ? [{ _id: `persisted:${filter.messageId}` }]
+ : [],
+ }),
+ ).resolves.toBe(false);
+ });
});
describe('Partial Response Saving', () => {
@@ -252,7 +346,7 @@ describe('Agent Abort Endpoint', () => {
metadata: { userId: 'test-user-123' },
});
- mockGenerationJobManager.abortJob.mockResolvedValue({
+ const abortResult = {
success: true,
jobData: {
userMessage: { messageId: userMessageId },
@@ -265,10 +359,12 @@ describe('Agent Abort Endpoint', () => {
},
content: [{ type: 'text', text: 'Partial response...' }],
text: 'Partial response...',
+ };
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ await options.beforePublish(abortResult);
+ return abortResult;
});
- mockSaveMessage.mockResolvedValue();
-
const response = await request(app)
.post('/api/agents/chat/abort')
.send({ conversationId: jobStreamId });
@@ -305,7 +401,7 @@ describe('Agent Abort Endpoint', () => {
});
// The job was a temporary chat; the stop button posts only conversationId.
- mockGenerationJobManager.abortJob.mockResolvedValue({
+ const abortResult = {
success: true,
jobData: {
userMessage: { messageId: 'user-msg-123' },
@@ -315,10 +411,12 @@ describe('Agent Abort Endpoint', () => {
},
content: [{ type: 'text', text: 'Partial...' }],
text: 'Partial...',
+ };
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ await options.beforePublish(abortResult);
+ return abortResult;
});
- mockSaveMessage.mockResolvedValue();
-
const response = await request(app)
.post('/api/agents/chat/abort')
.send({ conversationId: jobStreamId }); // no isTemporary in body
@@ -352,7 +450,7 @@ describe('Agent Abort Endpoint', () => {
{ type: 'tool_call', tool_call: { id: 'tc1', name: 'ask_user_question', args: '' } },
];
const content = capturedTransform ? capturedTransform(rawContent) : rawContent;
- return {
+ const result = {
success: true,
jobData: {
userMessage: { messageId: 'user-msg-123' },
@@ -362,10 +460,10 @@ describe('Agent Abort Endpoint', () => {
content,
text: '',
};
+ await options?.beforePublish?.(result);
+ return result;
});
- mockSaveMessage.mockResolvedValue();
-
const response = await request(app)
.post('/api/agents/chat/abort')
.send({ conversationId: jobStreamId });
@@ -374,7 +472,9 @@ describe('Agent Abort Endpoint', () => {
expect(capturedTransform).toEqual(expect.any(Function));
// The saved (and, in prod, emitted) content carries the stamped args.
// saveMessage(reqLike, responseMessage, opts) — the message is arg #2.
- const savedMessage = mockSaveMessage.mock.calls[0][1];
+ const savedMessage = mockSaveMessage.mock.calls
+ .map(([, message]) => message)
+ .find((message) => message.isCreatedByUser === false);
const askPart = savedMessage.content.find(
(p) => p?.tool_call?.name === 'ask_user_question',
);
@@ -385,10 +485,10 @@ describe('Agent Abort Endpoint', () => {
const jobStreamId = 'test-stream-123';
mockGenerationJobManager.getJob.mockResolvedValue({
- metadata: { userId: 'test-user-123' },
+ metadata: { userId: 'test-user-123', generationProtocolVersion: 2 },
});
- mockGenerationJobManager.abortJob.mockResolvedValue({
+ const abortResult = {
success: true,
jobData: {
userMessage: { messageId: 'user-msg-123' },
@@ -397,25 +497,37 @@ describe('Agent Abort Endpoint', () => {
},
content: [{ type: 'text', text: 'Partial response...' }],
text: 'Partial response...',
+ };
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ try {
+ await options.beforePublish(abortResult);
+ } catch {
+ return { ...abortResult, persistenceFailed: true };
+ }
+ return abortResult;
});
mockSaveMessage.mockRejectedValue(new Error('Database error'));
const response = await request(app)
.post('/api/agents/chat/abort')
- .send({ conversationId: jobStreamId });
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({ conversationId: jobStreamId, generationProtocolVersion: 2 });
// Should still return success even if save fails
expect(response.status).toBe(200);
- expect(response.body).toEqual({ success: true, aborted: jobStreamId });
- expect(mockLogger.error).toHaveBeenCalledWith(
- expect.stringContaining('Failed to save partial response'),
- );
+ expect(response.body).toEqual({
+ success: true,
+ aborted: jobStreamId,
+ persistenceFailed: true,
+ generationProtocolVersion: 2,
+ });
+ expect(response.headers['x-librechat-generation-protocol']).toBe('2');
});
});
describe('Job Not Found', () => {
- it('should skip paused fallback jobs and abort the running job', async () => {
+ it('should reject an unfenced new-placeholder abort when paused and running jobs exist', async () => {
mockGenerationJobManager.getJob
.mockResolvedValueOnce({
status: 'requires_action',
@@ -440,31 +552,41 @@ describe('Agent Abort Endpoint', () => {
.post('/api/agents/chat/abort')
.send({ conversationId: 'new' });
- expect(response.status).toBe(200);
- expect(response.body).toEqual({ success: true, aborted: 'running-stream' });
- expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(
- 'running-stream',
- expect.objectContaining({ transformAbortContent: expect.any(Function) }),
- );
+ expect(response.status).toBe(409);
+ expect(response.body).toEqual({
+ code: 'AMBIGUOUS_ACTIVE_RUN',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
});
- it('should not abort paused fallback jobs', async () => {
+ it('should abort an unambiguous paused fallback job', async () => {
mockGenerationJobManager.getJob.mockResolvedValueOnce({
status: 'requires_action',
metadata: { userId: 'test-user-123' },
});
mockGenerationJobManager.getActiveJobIdsForUser.mockResolvedValue(['paused-stream']);
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ jobData: null,
+ content: [],
+ text: '',
+ });
const response = await request(app)
.post('/api/agents/chat/abort')
.send({ conversationId: 'new' });
- expect(response.status).toBe(404);
+ expect(response.status).toBe(200);
expect(response.body).toEqual({
- error: 'Job not found',
- streamId: null,
+ success: true,
+ aborted: 'paused-stream',
+ generationProtocolVersion: 1,
});
- expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(
+ 'paused-stream',
+ expect.objectContaining({ transformAbortContent: expect.any(Function) }),
+ );
});
it('should return 404 when job is not found', async () => {
@@ -479,6 +601,7 @@ describe('Agent Abort Endpoint', () => {
expect(response.body).toEqual({
error: 'Job not found',
streamId: 'non-existent-job',
+ generationProtocolVersion: 1,
});
});
});
diff --git a/api/server/routes/agents/__tests__/streamTenant.spec.js b/api/server/routes/agents/__tests__/streamTenant.spec.js
index ee55f5e9c8..194caa12fd 100644
--- a/api/server/routes/agents/__tests__/streamTenant.spec.js
+++ b/api/server/routes/agents/__tests__/streamTenant.spec.js
@@ -9,7 +9,14 @@ const mockGenerationJobManager = {
markSyncSent: jest.fn(),
abortJob: jest.fn(),
getActiveJobIdsForUser: jest.fn().mockResolvedValue([]),
+ steering: {
+ claim: jest.fn().mockResolvedValue([]),
+ claimDetailed: jest.fn().mockResolvedValue({ generationProtocolVersion: 1, steers: [] }),
+ },
};
+const mockCaptureAgentCheckpointGeneration = jest.fn();
+const mockDeleteAgentCheckpoint = jest.fn();
+const mockSaveMessage = jest.fn();
jest.mock('@librechat/data-schemas', () => ({
...jest.requireActual('@librechat/data-schemas'),
@@ -25,10 +32,12 @@ jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
isEnabled: jest.fn().mockReturnValue(false),
GenerationJobManager: mockGenerationJobManager,
+ captureAgentCheckpointGeneration: (...args) => mockCaptureAgentCheckpointGeneration(...args),
+ deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
}));
jest.mock('~/models', () => ({
- saveMessage: jest.fn(),
+ saveMessage: (...args) => mockSaveMessage(...args),
}));
let mockUserId = 'user-123';
@@ -60,6 +69,7 @@ const agentsRouter = require('../index');
const app = express();
app.use(express.json());
app.use('/agents', agentsRouter);
+app.use((error, _req, res, _next) => res.status(500).json({ error: error.message }));
function mockSubscribeSuccess() {
mockGenerationJobManager.subscribe.mockImplementation((_streamId, _writeEvent, onDone) => {
@@ -73,6 +83,18 @@ describe('SSE stream tenant isolation', () => {
jest.clearAllMocks();
mockUserId = 'user-123';
mockTenantId = undefined;
+ mockCaptureAgentCheckpointGeneration.mockResolvedValue({
+ threadId: 'stream-123',
+ checkpointIds: ['checkpoint-a'],
+ });
+ mockDeleteAgentCheckpoint.mockResolvedValue(undefined);
+ mockSaveMessage.mockResolvedValue({ persisted: true });
+ mockGenerationJobManager.getActiveJobIdsForUser.mockResolvedValue([]);
+ mockGenerationJobManager.steering.claim.mockResolvedValue([]);
+ mockGenerationJobManager.steering.claimDetailed.mockResolvedValue({
+ generationProtocolVersion: 1,
+ steers: [],
+ });
});
describe('GET /chat/stream/:streamId', () => {
@@ -97,6 +119,206 @@ describe('SSE stream tenant isolation', () => {
expect(res.status).toBe(404);
});
+ it('returns GENERATION_REPLACED before SSE headers when the requested epoch is stale', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 2000,
+ });
+
+ const res = await request(app).get('/agents/chat/stream/stream-123?generationCreatedAt=1000');
+
+ expect(res.status).toBe(409);
+ expect(res.body).toEqual(expect.objectContaining({ code: 'GENERATION_REPLACED' }));
+ expect(res.headers['content-type']).not.toContain('text/event-stream');
+ expect(mockGenerationJobManager.subscribe).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.subscribeWithResume).not.toHaveBeenCalled();
+ });
+
+ it('passes the expected epoch into the manager subscription fence', async () => {
+ mockSubscribeSuccess();
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 1000,
+ });
+
+ const res = await request(app).get('/agents/chat/stream/stream-123?generationCreatedAt=1000');
+
+ expect(res.status).toBe(200);
+ expect(mockGenerationJobManager.subscribe).toHaveBeenCalledWith(
+ 'stream-123',
+ expect.any(Function),
+ expect.any(Function),
+ expect.any(Function),
+ expect.objectContaining({ expectedCreatedAt: 1000 }),
+ );
+ });
+
+ it('pins an unfenced legacy subscription to the generation that passed authorization', async () => {
+ mockSubscribeSuccess();
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123', tenantId: 'tenant-a' },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockTenantId = 'tenant-a';
+
+ const res = await request(app).get('/agents/chat/stream/stream-123');
+
+ expect(res.status).toBe(200);
+ expect(mockGenerationJobManager.subscribe).toHaveBeenCalledWith(
+ 'stream-123',
+ expect.any(Function),
+ expect.any(Function),
+ expect.any(Function),
+ expect.objectContaining({ expectedCreatedAt: 1000 }),
+ );
+ });
+
+ it('never attaches a cross-owner replacement after authorizing the predecessor', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', tenantId: 'tenant-a' },
+ status: 'running',
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'other-user', tenantId: 'tenant-b' },
+ status: 'running',
+ createdAt: 2000,
+ conversationId: 'stream-123',
+ });
+ mockTenantId = 'tenant-a';
+ mockGenerationJobManager.subscribe.mockResolvedValue(null);
+
+ const res = await request(app).get('/agents/chat/stream/stream-123');
+
+ expect(res.status).toBe(200);
+ expect(mockGenerationJobManager.subscribe).toHaveBeenCalledWith(
+ 'stream-123',
+ expect.any(Function),
+ expect.any(Function),
+ expect.any(Function),
+ expect.objectContaining({ expectedCreatedAt: 1000 }),
+ );
+ expect(res.text).toContain('event: error');
+ expect(res.text).not.toContain('generation_replaced');
+ });
+
+ it('fails closed when the job owner is missing', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { tenantId: 'tenant-a' },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockTenantId = 'tenant-a';
+
+ const res = await request(app).get('/agents/chat/stream/stream-123');
+
+ expect(res.status).toBe(403);
+ expect(mockGenerationJobManager.subscribe).not.toHaveBeenCalled();
+ });
+
+ it('reconciles a replacement that wins after headers but before manager attachment', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 2 },
+ status: 'running',
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 2000,
+ conversationId: 'stream-123',
+ });
+ mockGenerationJobManager.subscribe.mockResolvedValue(null);
+
+ const res = await request(app)
+ .get('/agents/chat/stream/stream-123?generationCreatedAt=1000&generationProtocolVersion=2')
+ .set('X-LibreChat-Generation-Protocol', '2');
+ const payload = JSON.parse(res.text.trim().split('\ndata: ')[1]);
+
+ expect(res.status).toBe(200);
+ expect(payload).toEqual({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: 1000,
+ conversation: { conversationId: 'stream-123' },
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('reconciles a generation deleted in the snapshot-to-attach window', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 2 },
+ status: 'running',
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ })
+ .mockResolvedValueOnce(null);
+ mockGenerationJobManager.subscribe.mockResolvedValue(null);
+
+ const res = await request(app)
+ .get('/agents/chat/stream/stream-123?generationCreatedAt=1000&generationProtocolVersion=2')
+ .set('X-LibreChat-Generation-Protocol', '2');
+ const payload = JSON.parse(res.text.trim().split('\ndata: ')[1]);
+
+ expect(res.status).toBe(200);
+ expect(payload).toEqual({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ generationCreatedAt: 1000,
+ conversation: { conversationId: 'stream-123' },
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('translates reconciliation into an error for a v1 subscriber', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 1 },
+ status: 'running',
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ })
+ .mockResolvedValueOnce(null);
+ mockGenerationJobManager.subscribe.mockResolvedValue(null);
+
+ const res = await request(app).get('/agents/chat/stream/stream-123?generationCreatedAt=1000');
+
+ expect(res.status).toBe(200);
+ expect(res.headers['x-librechat-generation-protocol']).toBe('1');
+ expect(res.text).toContain('event: error');
+ expect(res.text).toContain('Generation state changed');
+ expect(res.text).not.toContain('"reconcile":true');
+ });
+
+ it('does not misclassify a failed reconciliation read as a missing terminal job', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ })
+ .mockRejectedValueOnce(new Error('store unavailable'));
+ mockGenerationJobManager.subscribe.mockResolvedValue(null);
+
+ const res = await request(app).get('/agents/chat/stream/stream-123?generationCreatedAt=1000');
+
+ expect(res.status).toBe(200);
+ expect(res.text).toContain('event: error');
+ expect(res.text).toContain('Failed to subscribe to stream');
+ expect(res.text).not.toContain('terminal_payload_missing');
+ });
+
it('proceeds past tenant guard when tenant matches', async () => {
mockUserId = 'user-123';
mockTenantId = 'tenant-a';
@@ -105,6 +327,7 @@ describe('SSE stream tenant isolation', () => {
mockGenerationJobManager.getJob.mockResolvedValue({
metadata: { userId: 'user-123', tenantId: 'tenant-a' },
status: 'running',
+ createdAt: 1000,
});
const res = await request(app).get('/agents/chat/stream/stream-123');
@@ -120,6 +343,7 @@ describe('SSE stream tenant isolation', () => {
mockGenerationJobManager.getJob.mockResolvedValue({
metadata: { userId: 'user-123' },
status: 'running',
+ createdAt: 1000,
});
const res = await request(app).get('/agents/chat/stream/stream-123');
@@ -131,6 +355,7 @@ describe('SSE stream tenant isolation', () => {
mockGenerationJobManager.getJob.mockResolvedValue({
metadata: { userId: 'user-123' },
status: 'running',
+ createdAt: 1000,
});
const activate = jest.fn();
mockGenerationJobManager.subscribeWithResume.mockImplementation(
@@ -161,16 +386,17 @@ describe('SSE stream tenant isolation', () => {
pendingEvents: [],
},
{ event: 'on_message_delta', data: { text: 'live' } },
- { final: true },
+ { final: true, generationProtocolVersion: 1 },
]);
expect(activate).toHaveBeenCalledTimes(1);
- expect(mockGenerationJobManager.markSyncSent).toHaveBeenCalledWith('stream-123');
+ expect(mockGenerationJobManager.markSyncSent).toHaveBeenCalledWith('stream-123', 1000);
});
it('detaches a paused resume subscription when the response ends before activation', async () => {
mockGenerationJobManager.getJob.mockResolvedValue({
metadata: { userId: 'user-123' },
status: 'running',
+ createdAt: 1000,
});
const subscription = {
unsubscribe: jest.fn(),
@@ -209,6 +435,47 @@ describe('SSE stream tenant isolation', () => {
});
describe('GET /chat/status/:conversationId', () => {
+ it('echoes v2 for a clean jobless status after the server rollout gate', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(null);
+ mockGenerationJobManager.steering.claimDetailed.mockResolvedValue({
+ generationProtocolVersion: 2,
+ steers: [],
+ });
+
+ const res = await request(app)
+ .get('/agents/chat/status/conv-123?generationProtocolVersion=2')
+ .set('X-LibreChat-Generation-Protocol', '2');
+
+ expect(res.status).toBe(200);
+ expect(res.headers['x-librechat-generation-protocol']).toBe('2');
+ expect(res.body).toEqual({ active: false, generationProtocolVersion: 2 });
+ expect(mockGenerationJobManager.steering.claimDetailed).toHaveBeenCalledWith(
+ 'conv-123',
+ { userId: 'user-123', tenantId: undefined },
+ 2,
+ );
+ });
+
+ it('downgrades a jobless v2 request when its parked payload is legacy', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(null);
+ mockGenerationJobManager.steering.claimDetailed.mockResolvedValue({
+ generationProtocolVersion: 1,
+ steers: [{ steerId: 'legacy-steer', text: 'legacy words' }],
+ });
+
+ const res = await request(app)
+ .get('/agents/chat/status/conv-123?generationProtocolVersion=2')
+ .set('X-LibreChat-Generation-Protocol', '2');
+
+ expect(res.status).toBe(200);
+ expect(res.headers['x-librechat-generation-protocol']).toBe('1');
+ expect(res.body).toEqual({
+ active: false,
+ generationProtocolVersion: 1,
+ unrecoveredSteers: [{ steerId: 'legacy-steer', text: 'legacy words' }],
+ });
+ });
+
it('returns 403 when tenant does not match', async () => {
mockUserId = 'user-123';
mockTenantId = 'tenant-b';
@@ -238,9 +505,171 @@ describe('SSE stream tenant isolation', () => {
expect(res.status).toBe(200);
expect(res.body.active).toBe(true);
});
+
+ it('preserves the immutable v2 marker on an active status response', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123', generationProtocolVersion: 2 },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.getResumeState.mockResolvedValue(null);
+
+ const res = await request(app)
+ .get('/agents/chat/status/conv-123?generationProtocolVersion=2')
+ .set('X-LibreChat-Generation-Protocol', '2');
+
+ expect(res.status).toBe(200);
+ expect(res.headers['x-librechat-generation-protocol']).toBe('2');
+ expect(res.body).toEqual(
+ expect.objectContaining({
+ active: true,
+ streamId: 'conv-123',
+ generationProtocolVersion: 2,
+ }),
+ );
+ });
+
+ it('discards resume content when a cross-owner replacement wins after authorization', async () => {
+ mockUserId = 'user-123';
+ mockTenantId = 'tenant-a';
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', tenantId: 'tenant-a' },
+ status: 'running',
+ createdAt: 1000,
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'other-user', tenantId: 'tenant-b' },
+ status: 'running',
+ createdAt: 2000,
+ });
+ mockGenerationJobManager.getResumeState.mockResolvedValue({
+ aggregatedContent: [{ text: 'replacement secret' }],
+ });
+
+ const res = await request(app).get('/agents/chat/status/conv-123');
+
+ expect(res.status).toBe(403);
+ expect(res.text).not.toContain('replacement secret');
+ });
+
+ it('retries a same-owner replacement snapshot and returns only its verified state', async () => {
+ mockUserId = 'user-123';
+ mockTenantId = 'tenant-a';
+ const first = {
+ metadata: { userId: 'user-123', tenantId: 'tenant-a' },
+ status: 'running',
+ createdAt: 1000,
+ };
+ const replacement = {
+ metadata: { userId: 'user-123', tenantId: 'tenant-a' },
+ status: 'running',
+ createdAt: 2000,
+ };
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce(first)
+ .mockResolvedValueOnce(replacement)
+ .mockResolvedValueOnce(replacement);
+ mockGenerationJobManager.getResumeState
+ .mockResolvedValueOnce({ aggregatedContent: [{ text: 'discarded mixed snapshot' }] })
+ .mockResolvedValueOnce({ aggregatedContent: [{ text: 'verified replacement' }] });
+
+ const res = await request(app).get('/agents/chat/status/conv-123');
+
+ expect(res.status).toBe(200);
+ expect(res.body.createdAt).toBe(2000);
+ expect(res.body.aggregatedContent).toEqual([{ text: 'verified replacement' }]);
+ expect(res.text).not.toContain('discarded mixed snapshot');
+ expect(mockGenerationJobManager.getResumeState).toHaveBeenCalledTimes(2);
+ });
+
+ it('fails closed when status snapshots keep changing', async () => {
+ mockUserId = 'user-123';
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 1,
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 2,
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 3,
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 4,
+ });
+ mockGenerationJobManager.getResumeState.mockResolvedValue({
+ aggregatedContent: [{ text: 'never verified' }],
+ });
+
+ const res = await request(app).get('/agents/chat/status/conv-123');
+
+ expect(res.status).toBe(503);
+ expect(res.body).toEqual({ code: 'SERVER_NOT_READY', generationProtocolVersion: 1 });
+ expect(res.text).not.toContain('never verified');
+ });
+
+ it('keeps an abort persistence-pending terminal snapshot on the readiness path', async () => {
+ const pendingAbort = {
+ metadata: {
+ userId: 'user-123',
+ terminalPersistencePending: true,
+ },
+ status: 'aborted',
+ createdAt: 1000,
+ };
+ mockGenerationJobManager.getJob.mockResolvedValue(pendingAbort);
+ mockGenerationJobManager.getResumeState.mockResolvedValue({
+ aggregatedContent: [{ text: 'not authoritative yet' }],
+ });
+
+ const res = await request(app).get('/agents/chat/status/conv-123');
+
+ expect(res.status).toBe(503);
+ expect(res.headers['retry-after']).toBe('1');
+ expect(res.body).toEqual({ code: 'SERVER_NOT_READY', generationProtocolVersion: 1 });
+ expect(res.text).not.toContain('not authoritative yet');
+ expect(mockGenerationJobManager.steering.claim).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.steering.claimDetailed).not.toHaveBeenCalled();
+ });
+
+ it('fails closed when the status job owner is missing', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {},
+ status: 'running',
+ createdAt: 1000,
+ });
+
+ const res = await request(app).get('/agents/chat/status/conv-123');
+
+ expect(res.status).toBe(403);
+ expect(mockGenerationJobManager.getResumeState).not.toHaveBeenCalled();
+ });
});
describe('POST /chat/abort', () => {
+ it.each([
+ [{ streamId: { malicious: true } }],
+ [{ conversationId: 123 }],
+ [{ abortKey: { split: 'not a function' } }],
+ [{ streamId: '' }],
+ ])('rejects malformed abort target fields without touching the store', async (body) => {
+ const res = await request(app).post('/agents/chat/abort').send(body);
+
+ expect(res.status).toBe(400);
+ expect(res.body).toEqual({ code: 'INVALID_ABORT_TARGET', generationProtocolVersion: 1 });
+ expect(mockGenerationJobManager.getJob).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ });
+
it('returns 403 when tenant does not match', async () => {
mockUserId = 'user-123';
mockTenantId = 'tenant-b';
@@ -254,5 +683,542 @@ describe('SSE stream tenant isolation', () => {
expect(res.status).toBe(403);
expect(res.body.error).toBe('Unauthorized');
});
+
+ it('fails closed when the stored job owner is missing', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {},
+ status: 'running',
+ createdAt: 1000,
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(403);
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ });
+
+ it('does not fall back from an unknown concrete id to an unrelated active run', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(null);
+ mockGenerationJobManager.getActiveJobIdsForUser.mockResolvedValue(['other-active']);
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stale-or-typo-id',
+ });
+
+ expect(res.status).toBe(404);
+ expect(mockGenerationJobManager.getActiveJobIdsForUser).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ });
+
+ it('uses the generation epoch to resolve the exact new-placeholder run', async () => {
+ mockGenerationJobManager.getActiveJobIdsForUser.mockResolvedValue(['run-a', 'run-b']);
+ mockGenerationJobManager.getJob.mockImplementation(async (streamId) => ({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: streamId === 'run-a' ? 1000 : 2000,
+ }));
+ mockGenerationJobManager.abortJob.mockResolvedValue({ success: true });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ conversationId: 'new',
+ generationCreatedAt: 2000,
+ });
+
+ expect(res.status).toBe(200);
+ expect(res.body.aborted).toBe('run-b');
+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(
+ 'run-b',
+ expect.objectContaining({ expectedCreatedAt: 2000 }),
+ );
+ });
+
+ it('refuses an ambiguous unfenced new-placeholder abort', async () => {
+ mockGenerationJobManager.getActiveJobIdsForUser.mockResolvedValue(['run-a', 'run-b']);
+ mockGenerationJobManager.getJob.mockImplementation(async (streamId) => ({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: streamId === 'run-a' ? 1000 : 2000,
+ }));
+
+ const res = await request(app).post('/agents/chat/abort').send({ conversationId: 'new' });
+
+ expect(res.status).toBe(409);
+ expect(res.body).toEqual({
+ code: 'AMBIGUOUS_ACTIVE_RUN',
+ generationProtocolVersion: 1,
+ });
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ });
+
+ it('409s a stale generation identity without aborting the replacement', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 2000,
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(409);
+ expect(res.body).toEqual({ code: 'RUN_REPLACED', generationProtocolVersion: 1 });
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ });
+
+ it('threads the authenticated epoch into abortJob and reports a manager-side replacement', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: false,
+ failureReason: 'generation_replaced',
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(409);
+ expect(res.body).toEqual({ code: 'RUN_REPLACED', generationProtocolVersion: 1 });
+ expect(mockGenerationJobManager.abortJob).toHaveBeenCalledWith(
+ 'stream-123',
+ expect.objectContaining({ expectedCreatedAt: 1000 }),
+ );
+ });
+
+ it('does not report success when the same generation remains active after an abort race', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: false,
+ failureReason: 'job_still_active',
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(409);
+ expect(res.headers['retry-after']).toBe('1');
+ expect(res.body).toEqual({ code: 'RUN_STILL_ACTIVE', generationProtocolVersion: 1 });
+ });
+
+ it('reports an already-settled generation when natural completion wins the abort CAS', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 2 },
+ status: 'running',
+ createdAt: 1000,
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 2 },
+ status: 'complete',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: false,
+ jobData: { status: 'running', createdAt: 1000 },
+ });
+
+ const res = await request(app)
+ .post('/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+
+ expect(res.status).toBe(200);
+ expect(res.body).toEqual({
+ success: false,
+ settled: true,
+ code: 'RUN_ALREADY_SETTLED',
+ streamId: 'stream-123',
+ terminalStatus: 'complete',
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('keeps the legacy abort ACK shape when natural completion wins for protocol v1', async () => {
+ mockGenerationJobManager.getJob
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 1 },
+ status: 'running',
+ createdAt: 1000,
+ })
+ .mockResolvedValueOnce({
+ metadata: { userId: 'user-123', generationProtocolVersion: 1 },
+ status: 'complete',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: false,
+ jobData: { status: 'running', createdAt: 1000 },
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(200);
+ expect(res.body).toEqual({
+ success: true,
+ aborted: 'stream-123',
+ generationProtocolVersion: 1,
+ });
+ });
+
+ it('returns 404 when the job disappears before the manager can observe it', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123' },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: false,
+ jobData: null,
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(404);
+ expect(res.body).toEqual({
+ success: false,
+ error: 'Job not found',
+ streamId: 'stream-123',
+ generationProtocolVersion: 1,
+ });
+ });
+
+ it('leaves the run untouched when the required checkpoint snapshot cannot be captured', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {
+ userId: 'user-123',
+ pendingAction: { payload: { type: 'tool_approval' } },
+ },
+ status: 'requires_action',
+ createdAt: 1000,
+ });
+ mockCaptureAgentCheckpointGeneration.mockRejectedValue(
+ new Error('checkpoint snapshot unavailable'),
+ );
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(500);
+ expect(res.body).toEqual({
+ code: 'ABORT_FAILED',
+ error: 'Failed to abort generation',
+ generationProtocolVersion: 1,
+ });
+ expect(mockCaptureAgentCheckpointGeneration).toHaveBeenCalledWith('stream-123', undefined, {
+ throwOnError: true,
+ });
+ expect(mockGenerationJobManager.abortJob).not.toHaveBeenCalled();
+ });
+
+ it('persists the aborted parent and prunes the captured checkpoint before normal FINAL', async () => {
+ const publishFinal = jest.fn();
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {
+ userId: 'user-123',
+ pendingAction: { payload: { type: 'tool_approval' } },
+ },
+ status: 'requires_action',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ const abortResult = {
+ success: true,
+ jobData: {
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ responseMessageId: 'response-1',
+ userMessage: { messageId: 'user-1' },
+ sender: 'Agent',
+ endpoint: 'agents',
+ },
+ content: [{ type: 'text', text: 'partial answer' }],
+ text: 'partial answer',
+ pendingSteers: [{ steerId: 'steer-1', text: 'next' }],
+ };
+ await options.beforePublish(abortResult);
+ publishFinal();
+ return abortResult;
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(200);
+ expect(mockCaptureAgentCheckpointGeneration).toHaveBeenCalledWith('stream-123', undefined, {
+ throwOnError: true,
+ });
+ expect(mockSaveMessage).toHaveBeenCalledTimes(2);
+ expect(mockSaveMessage).toHaveBeenNthCalledWith(
+ 1,
+ expect.any(Object),
+ expect.objectContaining({ messageId: 'user-1', isCreatedByUser: true }),
+ expect.any(Object),
+ );
+ expect(mockSaveMessage).toHaveBeenNthCalledWith(
+ 2,
+ expect.any(Object),
+ expect.objectContaining({ messageId: 'response-1', isCreatedByUser: false }),
+ expect.any(Object),
+ );
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ 'stream-123',
+ undefined,
+ { threadId: 'stream-123', checkpointIds: ['checkpoint-a'] },
+ { throwOnError: true },
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
+ publishFinal.mock.invocationCallOrder[0],
+ );
+ expect(mockDeleteAgentCheckpoint.mock.invocationCallOrder[0]).toBeLessThan(
+ publishFinal.mock.invocationCallOrder[0],
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[1]).toBeLessThan(
+ mockDeleteAgentCheckpoint.mock.invocationCallOrder[0],
+ );
+ expect(mockSaveMessage.mock.invocationCallOrder[0]).toBeLessThan(
+ mockSaveMessage.mock.invocationCallOrder[1],
+ );
+ expect(res.body.pendingSteers).toEqual([{ steerId: 'steer-1', text: 'next' }]);
+ });
+
+ it('prunes the whole immutable generation namespace without a racy id snapshot', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {
+ userId: 'user-123',
+ checkpointNamespace: 'generation-1000',
+ pendingAction: { payload: { type: 'tool_approval' } },
+ },
+ status: 'requires_action',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ const abortResult = {
+ success: true,
+ jobData: { createdAt: 1000, conversationId: 'stream-123' },
+ content: [],
+ text: '',
+ pendingSteers: [],
+ };
+ await options.beforePublish(abortResult);
+ return abortResult;
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(200);
+ expect(mockCaptureAgentCheckpointGeneration).not.toHaveBeenCalled();
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith('stream-123', undefined, undefined, {
+ throwOnError: true,
+ checkpointNamespace: 'generation-1000',
+ });
+ });
+
+ it('suppresses interrupt-drain payloads when required abort persistence fails', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123', generationProtocolVersion: 2 },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ persistenceFailed: true,
+ pendingSteers: [{ steerId: 'steer-1', text: 'do not auto-send' }],
+ });
+
+ const res = await request(app)
+ .post('/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+
+ expect(res.status).toBe(200);
+ expect(res.body).toEqual({
+ success: true,
+ aborted: 'stream-123',
+ persistenceFailed: true,
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('fails closed instead of acknowledging abort persistence failure to a v1 client', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: { userId: 'user-123', generationProtocolVersion: 1 },
+ status: 'running',
+ createdAt: 1000,
+ });
+ mockGenerationJobManager.abortJob.mockResolvedValue({
+ success: true,
+ persistenceFailed: true,
+ pendingSteers: [{ steerId: 'steer-1', text: 'do not expose' }],
+ });
+
+ const res = await request(app).post('/agents/chat/abort').send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+
+ expect(res.status).toBe(409);
+ expect(res.headers['retry-after']).toBe('1');
+ expect(res.body).toEqual({
+ code: 'ABORT_PERSISTENCE_FAILED',
+ generationProtocolVersion: 1,
+ });
+ expect(res.text).not.toContain('steer-1');
+ });
+
+ it('still prunes the captured checkpoint when the aborted-message save fails', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {
+ userId: 'user-123',
+ generationProtocolVersion: 2,
+ pendingAction: { payload: { type: 'tool_approval' } },
+ },
+ status: 'requires_action',
+ createdAt: 1000,
+ });
+ mockSaveMessage
+ .mockResolvedValueOnce({ messageId: 'user-1' })
+ .mockRejectedValueOnce(new Error('message store unavailable'));
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ const abortResult = {
+ success: true,
+ jobData: {
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ responseMessageId: 'response-1',
+ userMessage: { messageId: 'user-1' },
+ },
+ content: [{ type: 'text', text: 'partial answer' }],
+ text: 'partial answer',
+ pendingSteers: [{ steerId: 'steer-1', text: 'do not drain' }],
+ };
+ try {
+ await options.beforePublish(abortResult);
+ return abortResult;
+ } catch (_error) {
+ return { ...abortResult, persistenceFailed: true };
+ }
+ });
+
+ const res = await request(app)
+ .post('/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+
+ expect(res.status).toBe(200);
+ expect(mockSaveMessage).toHaveBeenCalledTimes(2);
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ 'stream-123',
+ undefined,
+ { threadId: 'stream-123', checkpointIds: ['checkpoint-a'] },
+ { throwOnError: true },
+ );
+ expect(res.body).toEqual({
+ success: true,
+ aborted: 'stream-123',
+ persistenceFailed: true,
+ generationProtocolVersion: 2,
+ });
+ });
+
+ it('keeps checkpoint-prune failure on the conservative reconciliation path', async () => {
+ const publishNormalFinal = jest.fn();
+ const publishReconcile = jest.fn();
+ mockGenerationJobManager.getJob.mockResolvedValue({
+ metadata: {
+ userId: 'user-123',
+ generationProtocolVersion: 2,
+ pendingAction: { payload: { type: 'tool_approval' } },
+ },
+ status: 'requires_action',
+ createdAt: 1000,
+ });
+ mockDeleteAgentCheckpoint.mockRejectedValue(new Error('checkpoint store unavailable'));
+ mockGenerationJobManager.abortJob.mockImplementation(async (_streamId, options) => {
+ const abortResult = {
+ success: true,
+ jobData: {
+ createdAt: 1000,
+ conversationId: 'stream-123',
+ userMessage: { messageId: 'user-1' },
+ },
+ content: [],
+ text: '',
+ pendingSteers: [{ steerId: 'steer-1', text: 'do not drain' }],
+ };
+ try {
+ await options.beforePublish(abortResult);
+ publishNormalFinal();
+ return abortResult;
+ } catch (_error) {
+ publishReconcile();
+ return { ...abortResult, persistenceFailed: true };
+ }
+ });
+
+ const res = await request(app)
+ .post('/agents/chat/abort')
+ .set('X-LibreChat-Generation-Protocol', '2')
+ .send({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+
+ expect(res.status).toBe(200);
+ expect(publishNormalFinal).not.toHaveBeenCalled();
+ expect(publishReconcile).toHaveBeenCalledTimes(1);
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ 'stream-123',
+ undefined,
+ { threadId: 'stream-123', checkpointIds: ['checkpoint-a'] },
+ { throwOnError: true },
+ );
+ expect(res.body).toEqual({
+ success: true,
+ aborted: 'stream-123',
+ persistenceFailed: true,
+ generationProtocolVersion: 2,
+ });
+ });
});
});
diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.js
index 217c94203b..39d69a2353 100644
--- a/api/server/routes/agents/index.js
+++ b/api/server/routes/agents/index.js
@@ -2,11 +2,13 @@ const express = require('express');
const {
isEnabled,
GenerationJobManager,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
hasPersistableAbortContent,
buildAbortedResponseMetadata,
isPendingActionStale,
toClientPendingAction,
isHITLEnabled,
+ captureAgentCheckpointGeneration,
deleteAgentCheckpoint,
attachAskUserQuestionArgs,
createMessageFilterPii,
@@ -23,6 +25,13 @@ const {
messageUserLimiter,
} = require('~/server/middleware');
const SteerController = require('~/server/controllers/agents/steer');
+const {
+ GENERATION_PROTOCOL_HEADER,
+ GENERATION_PROTOCOL_V2,
+ getRequestedGenerationProtocol,
+ getServerGenerationProtocol,
+ negotiateExistingGenerationProtocol,
+} = require('~/server/controllers/agents/protocol');
const { saveMessage } = require('~/models');
const responses = require('./responses');
const openai = require('./openai');
@@ -36,6 +45,52 @@ function hasTenantMismatch(job, user) {
return job.metadata?.tenantId != null && job.metadata.tenantId !== user.tenantId;
}
+/** Protocol selected before a job has been authorized/read. This is used for
+ * validation, not-found, and authorization envelopes; it never leaks an
+ * existing job's marker to an unauthorized caller. */
+function negotiateRequestGenerationProtocol(req) {
+ return Math.min(
+ getRequestedGenerationProtocol(req),
+ getServerGenerationProtocol(GenerationJobManager),
+ );
+}
+
+/** Every generation-control JSON envelope carries the exact numeric protocol
+ * that governs it. The response header is useful to fetch/Axios callers, while
+ * the body survives auth-refresh adapters and is the client's fail-closed
+ * source of truth. */
+function sendGenerationJson(res, status, body, generationProtocolVersion) {
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ return res.status(status).json({ ...body, generationProtocolVersion });
+}
+
+async function sendJoblessStatus(req, res, conversationId) {
+ // The default completeJob path deletes the job record immediately, so the
+ // jobless branch IS the common reload-after-terminal case — parked steers
+ // live under their own bounded-TTL key and authorize from their stored owner.
+ const requestedProtocolVersion = getRequestedGenerationProtocol(req);
+ const claimed = await GenerationJobManager.steering.claimDetailed(
+ conversationId,
+ {
+ userId: req.user.id,
+ tenantId: req.user.tenantId,
+ },
+ requestedProtocolVersion,
+ );
+ const generationProtocolVersion = Math.min(
+ requestedProtocolVersion,
+ claimed.steers.length > 0
+ ? claimed.generationProtocolVersion
+ : getServerGenerationProtocol(GenerationJobManager),
+ );
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ return res.json({
+ active: false,
+ generationProtocolVersion,
+ ...(claimed.steers.length > 0 && { unrecoveredSteers: claimed.steers }),
+ });
+}
+
const router = express.Router();
/**
@@ -71,6 +126,24 @@ router.use(uaParser);
router.get('/chat/stream/:streamId', async (req, res) => {
const { streamId } = req.params;
const isResume = req.query.resume === 'true';
+ const requestProtocolVersion = negotiateRequestGenerationProtocol(req);
+ const rawGenerationCreatedAt = req.query.generationCreatedAt;
+ let expectedGenerationCreatedAt;
+ if (rawGenerationCreatedAt != null) {
+ if (
+ typeof rawGenerationCreatedAt !== 'string' ||
+ !/^\d+$/.test(rawGenerationCreatedAt) ||
+ !Number.isSafeInteger(Number(rawGenerationCreatedAt))
+ ) {
+ return sendGenerationJson(
+ res,
+ 400,
+ { error: 'Invalid generation identity' },
+ requestProtocolVersion,
+ );
+ }
+ expectedGenerationCreatedAt = Number(rawGenerationCreatedAt);
+ }
let result;
const attachmentAbortController = new AbortController();
req.on('close', () => {
@@ -84,20 +157,56 @@ router.get('/chat/stream/:streamId', async (req, res) => {
return;
}
if (!job) {
- return res.status(404).json({
- error: 'Stream not found',
- message: 'The generation job does not exist or has expired.',
- });
+ return sendGenerationJson(
+ res,
+ 404,
+ {
+ error: 'Stream not found',
+ message: 'The generation job does not exist or has expired.',
+ },
+ requestProtocolVersion,
+ );
}
- if (job.metadata?.userId && job.metadata.userId !== req.user.id) {
- return res.status(403).json({ error: 'Unauthorized' });
+ // Every job has an owner at creation time. Treat a missing/corrupt owner as
+ // unauthorized instead of turning malformed store state into a public
+ // stream for anyone who knows the conversation id.
+ if (job.metadata?.userId !== req.user.id) {
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
}
if (hasTenantMismatch(job, req.user)) {
- return res.status(403).json({ error: 'Unauthorized' });
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
+
+ const generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
+
+ if (expectedGenerationCreatedAt != null && job.createdAt !== expectedGenerationCreatedAt) {
+ // streamId is conversation-scoped and may now belong to a newer turn. A
+ // stale start/reconnect gets a dedicated handoff signal instead of either
+ // following the ordinary terminal path or receiving replacement content.
+ return sendGenerationJson(
+ res,
+ 409,
+ {
+ code: 'GENERATION_REPLACED',
+ error: 'Generation replaced',
+ message: 'The requested generation has completed or was replaced.',
+ },
+ generationProtocolVersion,
+ );
}
+ /** Pin even legacy (unfenced-query) subscribers to the exact job snapshot
+ * that passed the owner + tenant checks above. `streamId` is conversation-
+ * scoped, so a replacement can otherwise land between this authorization
+ * read and the manager attachment and expose the replacement generation
+ * without ever authorizing its owner. */
+ const authorizedGenerationCreatedAt = job.createdAt;
+ if (!Number.isSafeInteger(authorizedGenerationCreatedAt) || authorizedGenerationCreatedAt < 0) {
+ logger.warn(`[AgentStream] Refusing stream with invalid generation identity: ${streamId}`);
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
const streamTelemetry = createSseStreamTelemetry({ req, res, streamId, isResume });
res.setHeader('Content-Encoding', 'identity');
@@ -105,12 +214,16 @@ router.get('/chat/stream/:streamId', async (req, res) => {
res.setHeader('Cache-Control', 'no-cache, no-transform');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Accel-Buffering', 'no');
+ res.setHeader(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
res.flushHeaders();
streamTelemetry.recordHeadersFlushed();
logger.debug(`[AgentStream] Client subscribed to ${streamId}, resume: ${isResume}`);
const writeEvent = (event, options = {}) => {
+ if (generationProtocolVersion < GENERATION_PROTOCOL_V2 && event?.event === 'on_steer_updated') {
+ return true;
+ }
if (!res.writableEnded) {
const eventName = options.eventName ?? 'message';
const payload = `event: ${eventName}\ndata: ${JSON.stringify(event)}\n\n`;
@@ -127,14 +240,42 @@ router.get('/chat/stream/:streamId', async (req, res) => {
const onDone = (event) => {
streamTelemetry.recordFinalEventEmitted();
- writeEvent(event, { final: true });
+ if (event?.reconcile === true && generationProtocolVersion < GENERATION_PROTOCOL_V2) {
+ /** Legacy clients treat an ordinary `final: true` as the completion of
+ * their optimistic submission. A reconciliation frame has no response
+ * payload and may describe a replacement, so expose it only as a
+ * transport error; the v1 reconnect/status path will refetch safely. */
+ writeEvent(
+ {
+ error: 'Generation state changed; reconnect to load the saved response.',
+ generationProtocolVersion,
+ },
+ { eventName: 'error', final: true },
+ );
+ res.end();
+ return;
+ }
+ writeEvent(
+ event != null && typeof event === 'object'
+ ? { ...event, generationProtocolVersion }
+ : { final: true, generationProtocolVersion },
+ { final: true },
+ );
res.end();
};
const onError = (error) => {
if (!res.writableEnded) {
streamTelemetry.recordErrorEventEmitted();
- writeEvent({ error }, { eventName: 'error' });
+ if (error === TERMINAL_PUBLICATION_RECONNECT_ERROR) {
+ /** A durable terminal payload exists, but cross-replica DONE publish
+ * failed. Tear down the HTTP stream without an application error frame:
+ * sse.js treats the transport close as reconnectable, and the retained
+ * terminal job then replays its authoritative final payload. */
+ res.destroy();
+ return;
+ }
+ writeEvent({ error, generationProtocolVersion }, { eventName: 'error' });
res.end();
}
};
@@ -143,12 +284,13 @@ router.get('/chat/stream/:streamId', async (req, res) => {
const { subscription, resumeState, pendingEvents } =
await GenerationJobManager.subscribeWithResume(streamId, writeEvent, onDone, onError, {
signal: attachmentAbortController.signal,
+ expectedCreatedAt: authorizedGenerationCreatedAt,
});
if (subscription && !attachmentAbortController.signal.aborted && !res.writableEnded) {
if (resumeState) {
writeEvent({ sync: true, resumeState, pendingEvents });
- GenerationJobManager.markSyncSent(streamId);
+ GenerationJobManager.markSyncSent(streamId, authorizedGenerationCreatedAt);
logger.debug(
`[AgentStream] Sent sync event for ${streamId} with ${resumeState.runSteps.length} run steps, ${pendingEvents.length} pending events`,
);
@@ -169,6 +311,7 @@ router.get('/chat/stream/:streamId', async (req, res) => {
} else {
result = await GenerationJobManager.subscribe(streamId, writeEvent, onDone, onError, {
signal: attachmentAbortController.signal,
+ expectedCreatedAt: authorizedGenerationCreatedAt,
});
}
@@ -178,6 +321,56 @@ router.get('/chat/stream/:streamId', async (req, res) => {
}
if (!result) {
streamTelemetry.recordSubscribeFailed();
+ {
+ let currentJob;
+ let currentJobReadSucceeded = false;
+ try {
+ currentJob = await GenerationJobManager.getJob(streamId);
+ currentJobReadSucceeded = true;
+ } catch (error) {
+ logger.warn(`[AgentStream] Failed to reconcile fenced subscription for ${streamId}`, error);
+ }
+
+ if (attachmentAbortController.signal.aborted || res.writableEnded) {
+ return;
+ }
+
+ const currentJobAuthorized =
+ currentJobReadSucceeded &&
+ (!currentJob ||
+ (currentJob.metadata?.userId === req.user.id &&
+ !hasTenantMismatch(currentJob, req.user)));
+ const generationReplaced =
+ currentJobAuthorized &&
+ currentJob != null &&
+ currentJob.createdAt !== authorizedGenerationCreatedAt;
+ const expectedGenerationTerminal =
+ currentJobAuthorized &&
+ currentJob?.createdAt === authorizedGenerationCreatedAt &&
+ ['complete', 'error', 'aborted'].includes(currentJob.status);
+
+ /** The route already flushed SSE headers before the manager's final
+ * generation fence ran. A generic error here would misreport the common
+ * snapshot-to-attach race where the requested run terminalized or was
+ * replaced. Send the same control-only reconciliation frame used by the
+ * manager so the client refetches authoritative state instead. */
+ if (
+ currentJobReadSucceeded &&
+ (generationReplaced || !currentJob || expectedGenerationTerminal)
+ ) {
+ onDone({
+ final: true,
+ reconcile: true,
+ reconcileReason: generationReplaced ? 'generation_replaced' : 'terminal_payload_missing',
+ ...(expectedGenerationTerminal && { terminalStatus: currentJob.status }),
+ generationCreatedAt: authorizedGenerationCreatedAt,
+ conversation: {
+ conversationId: currentJob?.conversationId ?? job.conversationId ?? streamId,
+ },
+ });
+ return;
+ }
+ }
onError('Failed to subscribe to stream');
return;
}
@@ -205,36 +398,72 @@ router.get('/chat/active', async (req, res) => {
*/
router.get('/chat/status/:conversationId', async (req, res) => {
const { conversationId } = req.params;
+ const requestProtocolVersion = negotiateRequestGenerationProtocol(req);
// streamId === conversationId, so we can use getJob directly
- const job = await GenerationJobManager.getJob(conversationId);
+ let job = await GenerationJobManager.getJob(conversationId);
if (!job) {
- // The default completeJob path deletes the job record immediately, so the
- // jobless branch IS the common reload-after-terminal case — parked steers
- // live under their own bounded-TTL key and the claim authorizes against
- // the payload's stored owner (no job record is left to check).
- const claimed = await GenerationJobManager.steering.claim(conversationId, {
- userId: req.user.id,
- tenantId: req.user.tenantId,
- });
- return res.json({
- active: false,
- ...(claimed.length > 0 && { unrecoveredSteers: claimed }),
- });
+ return sendJoblessStatus(req, res, conversationId);
}
- if (job.metadata.userId !== req.user.id) {
- return res.status(403).json({ error: 'Unauthorized' });
+ let resumeState;
+ let snapshotVerified = false;
+ /** `getResumeState` begins with its own streamId lookup. A replacement can
+ * land after this route authorizes A but before that lookup and make it read
+ * B's content. Verify the epoch after each read and discard mismatched
+ * snapshots; every replacement snapshot is re-authorized before use. */
+ for (let attempt = 0; attempt < 3; attempt++) {
+ if (job.metadata?.userId !== req.user.id || hasTenantMismatch(job, req.user)) {
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
+ if (!Number.isSafeInteger(job.createdAt) || job.createdAt < 0) {
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
+
+ const authorizedCreatedAt = job.createdAt;
+ resumeState = await GenerationJobManager.getResumeState(conversationId, authorizedCreatedAt);
+ const verifiedJob = await GenerationJobManager.getJob(conversationId);
+ if (!verifiedJob) {
+ return sendJoblessStatus(req, res, conversationId);
+ }
+ if (verifiedJob.createdAt === authorizedCreatedAt) {
+ if (
+ verifiedJob.metadata?.userId !== req.user.id ||
+ hasTenantMismatch(verifiedJob, req.user)
+ ) {
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
+ job = verifiedJob;
+ snapshotVerified = true;
+ break;
+ }
+ job = verifiedJob;
}
- if (hasTenantMismatch(job, req.user)) {
- return res.status(403).json({ error: 'Unauthorized' });
+ if (!snapshotVerified) {
+ res.set('Retry-After', '1');
+ return sendGenerationJson(res, 503, { code: 'SERVER_NOT_READY' }, requestProtocolVersion);
}
- // Get resume state which contains aggregatedContent
- // Avoid calling both getStreamInfo and getResumeState (both fetch content)
- const resumeState = await GenerationJobManager.getResumeState(conversationId);
+ /** Abort has won terminal ownership, but its required message/checkpoint
+ * persistence has not finished yet. Reporting this snapshot as inactive
+ * would let a reloading client clear its live state and refetch history
+ * before the terminal owner has made that history authoritative. `getJob`
+ * recovers a stale pending marker; while the verified marker remains live,
+ * keep every status consumer on the same readiness path as duplicate starts. */
+ if (job.metadata?.terminalPersistencePending === true) {
+ res.set('Retry-After', '1');
+ return sendGenerationJson(
+ res,
+ 503,
+ { code: 'SERVER_NOT_READY' },
+ negotiateExistingGenerationProtocol(req, job),
+ );
+ }
+ let generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+
// A job paused for human review is still active (consistent with /chat/active),
// so the client resumes/subscribes rather than treating it as finished — but
// only while it has a live, resolvable prompt: a missing/malformed or
@@ -244,21 +473,31 @@ router.get('/chat/status/:conversationId', async (req, res) => {
const isActive = job.status === 'running' || pendingLive;
/** Acknowledged steers the terminal drains parked because no subscriber was
- * live to receive the final/abort event — claim-on-read (cleared once
- * returned) so the reloading client restores them as queued follow-ups. */
+ * live to receive the final/abort event. Reads are replayable; a recovery
+ * turn leases its exact source and removes it only after durable persistence. */
let unrecoveredSteers;
- if (!isActive) {
- const claimed = await GenerationJobManager.steering.claim(conversationId, {
- userId: req.user.id,
- tenantId: req.user.tenantId,
- });
- if (claimed.length > 0) {
- unrecoveredSteers = claimed;
+ if (!isActive || job.metadata.steersClosed === true) {
+ const claimed = await GenerationJobManager.steering.claimDetailed(
+ conversationId,
+ {
+ userId: req.user.id,
+ tenantId: req.user.tenantId,
+ },
+ getRequestedGenerationProtocol(req),
+ );
+ if (claimed.steers.length > 0) {
+ generationProtocolVersion = Math.min(
+ generationProtocolVersion,
+ claimed.generationProtocolVersion,
+ );
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ unrecoveredSteers = claimed.steers;
}
}
res.json({
active: isActive,
+ generationProtocolVersion,
...(unrecoveredSteers && { unrecoveredSteers }),
streamId: conversationId,
status: job.status,
@@ -282,157 +521,354 @@ router.get('/chat/status/:conversationId', async (req, res) => {
* @access Private
* @description Mounted before chatRouter to bypass buildEndpointOption middleware
*/
-router.post('/chat/abort', configMiddleware, async (req, res) => {
+router.post('/chat/abort', configMiddleware, async (req, res, next) => {
logger.debug(`[AgentStream] ========== ABORT ENDPOINT HIT ==========`);
logger.debug(`[AgentStream] Method: ${req.method}, Path: ${req.path}`);
logger.debug(`[AgentStream] Body:`, req.body);
- const { streamId, conversationId, abortKey } = req.body;
- const userId = req.user?.id;
-
- // streamId === conversationId, so try any of the provided IDs
- // Skip "new" as it's a placeholder for new conversations, not an actual ID
- let jobStreamId =
- streamId ||
- (conversationId !== 'new' ? conversationId : null) ||
- abortKey?.split(':')[0] ||
- null;
- let job = jobStreamId ? await GenerationJobManager.getJob(jobStreamId) : null;
-
- // Fallback: if job not found and we have a userId, look up active jobs for user
- // This handles the case where frontend sends "new" but job was created with a UUID
- if (!job && userId) {
- logger.debug(`[AgentStream] Job not found by ID, checking active jobs for user: ${userId}`);
- const activeJobIds = await GenerationJobManager.getActiveJobIdsForUser(
- userId,
- req.user.tenantId,
- );
- for (const activeJobId of activeJobIds) {
- const activeJob = await GenerationJobManager.getJob(activeJobId);
- if (activeJob?.status !== 'running') {
- continue;
+ const requestProtocolVersion = negotiateRequestGenerationProtocol(req);
+ let responseProtocolVersion = requestProtocolVersion;
+ try {
+ if (req.body == null || typeof req.body !== 'object' || Array.isArray(req.body)) {
+ return sendGenerationJson(res, 400, { code: 'INVALID_ABORT_TARGET' }, requestProtocolVersion);
+ }
+ const { streamId, conversationId, abortKey, generationCreatedAt } = req.body;
+ for (const value of [streamId, conversationId, abortKey]) {
+ if (
+ value != null &&
+ (typeof value !== 'string' || value.length === 0 || value.length > 512)
+ ) {
+ return sendGenerationJson(
+ res,
+ 400,
+ { code: 'INVALID_ABORT_TARGET' },
+ requestProtocolVersion,
+ );
}
- jobStreamId = activeJobId;
- job = activeJob;
- logger.debug(`[AgentStream] Found active job for user: ${jobStreamId}`);
- break;
}
- }
-
- logger.debug(`[AgentStream] Computed jobStreamId: ${jobStreamId}`);
-
- if (job && jobStreamId) {
- if (job.metadata?.userId && job.metadata.userId !== userId) {
- logger.warn(`[AgentStream] Unauthorized abort attempt for ${jobStreamId} by user ${userId}`);
- return res.status(403).json({ error: 'Unauthorized' });
+ const userId = req.user?.id;
+ if (
+ generationCreatedAt != null &&
+ (!Number.isSafeInteger(generationCreatedAt) || generationCreatedAt < 0)
+ ) {
+ return sendGenerationJson(
+ res,
+ 400,
+ { code: 'INVALID_GENERATION_IDENTITY' },
+ requestProtocolVersion,
+ );
}
- if (hasTenantMismatch(job, req.user)) {
- return res.status(403).json({ error: 'Unauthorized' });
+ // streamId === conversationId, so try any of the provided IDs
+ // Skip "new" as it's a placeholder for new conversations, not an actual ID.
+ const streamCandidate = streamId && streamId !== 'new' ? streamId : null;
+ const conversationCandidate =
+ conversationId && conversationId !== 'new' ? conversationId : null;
+ const abortCandidate = abortKey?.split(':')[0];
+ const abortKeyCandidate = abortCandidate && abortCandidate !== 'new' ? abortCandidate : null;
+ let jobStreamId = streamCandidate || conversationCandidate || abortKeyCandidate || null;
+ let job = jobStreamId ? await GenerationJobManager.getJob(jobStreamId) : null;
+
+ /** Fallback only for the explicit new-conversation placeholder. An unknown
+ * concrete id (including a typo/stale tab) must never abort an unrelated
+ * active job. If several new-chat starts are active, the epoch selects the
+ * exact one; an unfenced legacy request is safe only when unambiguous. */
+ const canResolveNewPlaceholder =
+ !jobStreamId && (streamId === 'new' || conversationId === 'new') && userId;
+ if (!job && canResolveNewPlaceholder) {
+ logger.debug(`[AgentStream] Job not found by ID, checking active jobs for user: ${userId}`);
+ const activeJobIds = await GenerationJobManager.getActiveJobIdsForUser(
+ userId,
+ req.user.tenantId,
+ );
+ const candidates = [];
+ for (const activeJobId of activeJobIds) {
+ const activeJob = await GenerationJobManager.getJob(activeJobId);
+ if (
+ !activeJob ||
+ (activeJob.status !== 'running' && activeJob.status !== 'requires_action') ||
+ activeJob.metadata?.userId !== userId ||
+ hasTenantMismatch(activeJob, req.user) ||
+ (generationCreatedAt != null && activeJob.createdAt !== generationCreatedAt)
+ ) {
+ continue;
+ }
+ candidates.push({ streamId: activeJobId, job: activeJob });
+ }
+ if (candidates.length > 1) {
+ return sendGenerationJson(
+ res,
+ 409,
+ { code: 'AMBIGUOUS_ACTIVE_RUN' },
+ requestProtocolVersion,
+ );
+ }
+ if (candidates.length === 1) {
+ jobStreamId = candidates[0].streamId;
+ job = candidates[0].job;
+ logger.debug(`[AgentStream] Found active job for user: ${jobStreamId}`);
+ }
}
- logger.debug(`[AgentStream] Job found, aborting: ${jobStreamId}`);
- // Re-attach a paused ask_user_question's args to the abort content BEFORE
- // abortJob emits the final SSE. Redis reconstructs abort content from the
- // chunk log, which never saw the pause-time stamp applied to the in-process
- // contentParts — stamping inside abortJob (not after) means the LIVE client
- // gets the question too, not just the saved message on reload.
- const abortedAskPayload = job.metadata?.pendingAction?.payload;
- const abortResult = await GenerationJobManager.abortJob(jobStreamId, {
- transformAbortContent: (content) =>
- abortedAskPayload?.type === 'ask_user_question' && Array.isArray(content)
- ? attachAskUserQuestionArgs(content, abortedAskPayload.question)
- : content,
- });
- logger.debug(`[AgentStream] Job aborted successfully: ${jobStreamId}`, {
- abortResultSuccess: abortResult.success,
- abortResultUserMessageId: abortResult.jobData?.userMessage?.messageId,
- abortResultResponseMessageId: abortResult.jobData?.responseMessageId,
- });
+ logger.debug(`[AgentStream] Computed jobStreamId: ${jobStreamId}`);
- // HITL: prune the durable checkpoint of a run aborted while paused, so a new turn
- // in this conversation can't rehydrate the stale interrupt before the Mongo TTL
- // reclaims it (thread_id is the stable conversationId). Idempotent / no-op when
- // HITL is off or nothing was written. The pendingAction check covers ask-only
- // pauses (ask_user_question attaches a checkpointer WITHOUT the approval policy):
- // a job aborted while paused still carries its pendingAction in metadata, which is
- // exactly the case whose checkpoint would otherwise go stale.
- const agentsCfg = req.config?.endpoints?.agents;
- if (isHITLEnabled(agentsCfg?.toolApproval) || job.metadata?.pendingAction != null) {
- await deleteAgentCheckpoint(jobStreamId, agentsCfg?.checkpointer).catch((err) =>
- logger.error(`[AgentStream] Failed to prune checkpoint on abort: ${jobStreamId}`, err),
- );
- }
+ if (job && jobStreamId) {
+ if (job.metadata?.userId !== userId) {
+ logger.warn(
+ `[AgentStream] Unauthorized abort attempt for ${jobStreamId} by user ${userId}`,
+ );
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
- // CRITICAL: Save partial response BEFORE returning to prevent race condition.
- // If user sends a follow-up immediately after abort, the parentMessageId must exist in DB.
- // Only save if we have a valid responseMessageId (skip early aborts before generation started)
- if (
- abortResult.success &&
- abortResult.jobData?.userMessage?.messageId &&
- abortResult.jobData?.responseMessageId &&
- hasPersistableAbortContent(abortResult.content)
- ) {
- const { jobData, text } = abortResult;
- // `abortResult.content` is already stamped by `transformAbortContent`
- // above (same content the final SSE carried), so the saved message and
- // the live client agree.
- const { content } = abortResult;
- const responseMessage = {
- messageId: jobData.responseMessageId,
- parentMessageId: jobData.userMessage.messageId,
- conversationId: jobData.conversationId,
- content: content || [],
- text: text || '',
- sender: jobData.sender || 'AI',
- endpoint: jobData.endpoint,
- iconURL: jobData.iconURL,
- model: jobData.model,
- unfinished: true,
- error: false,
- isCreatedByUser: false,
- user: userId,
- };
-
- /** Persist the usage/cost rollup + context breakdown for the stopped
- * response (from the job's tracked tokenUsage/contextUsage) so its
- * branch/total cost and granular rows survive a reload — parity with the
- * normal completion path. */
- const abortMetadata = buildAbortedResponseMetadata(jobData);
- if (abortMetadata) {
- responseMessage.metadata = abortMetadata;
+ if (hasTenantMismatch(job, req.user)) {
+ return sendGenerationJson(res, 403, { error: 'Unauthorized' }, requestProtocolVersion);
+ }
+ const generationProtocolVersion = negotiateExistingGenerationProtocol(req, job);
+ responseProtocolVersion = generationProtocolVersion;
+ res.set(GENERATION_PROTOCOL_HEADER, String(generationProtocolVersion));
+ if (generationCreatedAt != null && job.createdAt !== generationCreatedAt) {
+ return res.status(409).json({ code: 'RUN_REPLACED', generationProtocolVersion });
}
- try {
- await saveMessage(
- {
- userId: req?.user?.id,
- // Source from the job, not the request: the stop button posts only the
- // conversationId, so trusting req.body.isTemporary would persist an aborted
- // temporary-chat partial as a normal (orphaned) message.
- isTemporary: jobData?.isTemporary ?? req?.body?.isTemporary,
- interfaceConfig: req?.config?.interfaceConfig,
- },
- responseMessage,
- { context: 'api/server/routes/agents/index.js - abort endpoint' },
- );
- logger.debug(`[AgentStream] Saved partial response for: ${jobStreamId}`);
- } catch (saveError) {
- logger.error(`[AgentStream] Failed to save partial response: ${saveError.message}`);
+ logger.debug(`[AgentStream] Job found, aborting: ${jobStreamId}`);
+ // Re-attach a paused ask_user_question's args to the abort content BEFORE
+ // abortJob emits the final SSE. Redis reconstructs abort content from the
+ // chunk log, which never saw the pause-time stamp applied to the in-process
+ // contentParts — stamping inside abortJob (not after) means the LIVE client
+ // gets the question too, not just the saved message on reload.
+ const abortedAskPayload = job.metadata?.pendingAction?.payload;
+ const agentsCfg = req.config?.endpoints?.agents;
+ const shouldPruneCheckpoint =
+ isHITLEnabled(agentsCfg?.toolApproval) || job.metadata?.pendingAction != null;
+ const checkpointNamespace =
+ typeof job.metadata?.checkpointNamespace === 'string'
+ ? job.metadata.checkpointNamespace
+ : '';
+ /** New jobs have an immutable saver-level namespace, so the terminal
+ * owner can delete that entire namespace (including a checkpoint written
+ * after this route's initial read) without touching a replacement. Legacy
+ * jobs share the root namespace and still need an id snapshot before CAS. */
+ const checkpointGeneration =
+ shouldPruneCheckpoint && checkpointNamespace === ''
+ ? await captureAgentCheckpointGeneration(jobStreamId, agentsCfg?.checkpointer, {
+ throwOnError: true,
+ })
+ : undefined;
+ const abortResult = await GenerationJobManager.abortJob(jobStreamId, {
+ expectedCreatedAt: job.createdAt,
+ transformAbortContent: (content) =>
+ abortedAskPayload?.type === 'ask_user_question' && Array.isArray(content)
+ ? attachAskUserQuestionArgs(content, abortedAskPayload.question)
+ : content,
+ /** Persist every parent-row prerequisite before publishing the ordinary
+ * abort FINAL. That frame can immediately drain a queued follow-up, whose
+ * parent must already exist and whose graph must not see a stale HITL
+ * checkpoint. Throwing makes the manager publish a conservative
+ * reconciliation frame instead of an unsafe normal FINAL. */
+ beforePublish: async (pendingAbortResult) => {
+ const persistenceErrors = [];
+ const { jobData, text, content } = pendingAbortResult;
+ /** `abortJob` treats a delivered `created` event as a real turn even
+ * when every streamed part is filtered out (for example, an
+ * interrupt before the model's first non-whitespace token). Its
+ * normal FINAL therefore carries an empty unfinished assistant.
+ * Persist that same row before publishing, including when its id is
+ * the underscore-suffixed preliminary id rendered by `created`.
+ * Otherwise interrupt-and-send immediately posts that unsaved id as
+ * its parent and the preliminary-parent fence correctly rejects it. */
+ const shouldPersistAbortedTurn =
+ hasPersistableAbortContent(content) || jobData?.createdEventEmitted === true;
+
+ if (
+ jobData?.userMessage?.messageId &&
+ jobData?.responseMessageId &&
+ shouldPersistAbortedTurn
+ ) {
+ const messageContext = {
+ userId: req?.user?.id,
+ // Source from the job: the stop request does not carry the
+ // original temporary-chat flag.
+ isTemporary: jobData?.isTemporary ?? req?.body?.isTemporary,
+ interfaceConfig: req?.config?.interfaceConfig,
+ };
+ const requestMessage = {
+ ...jobData.userMessage,
+ conversationId: jobData.conversationId,
+ sender: 'User',
+ endpoint: jobData.endpoint,
+ isCreatedByUser: true,
+ user: userId,
+ };
+ const responseMessage = {
+ messageId: jobData.responseMessageId,
+ parentMessageId: jobData.userMessage.messageId,
+ conversationId: jobData.conversationId,
+ content: content || [],
+ text: text || '',
+ sender: jobData.sender || 'AI',
+ endpoint: jobData.endpoint,
+ iconURL: jobData.iconURL,
+ model: jobData.model,
+ unfinished: true,
+ error: false,
+ isCreatedByUser: false,
+ user: userId,
+ };
+
+ const abortMetadata = buildAbortedResponseMetadata(jobData);
+ if (abortMetadata) {
+ responseMessage.metadata = abortMetadata;
+ }
+
+ /** `created` fires before BaseClient starts its asynchronous user
+ * write. A very early interrupt can therefore reach this barrier
+ * with neither row stored. Both writes are idempotent upserts;
+ * await the user prerequisite first, but still attempt the child
+ * write and checkpoint cleanup so every independently useful
+ * operation gets a chance to succeed. */
+ try {
+ const persistedRequest = await saveMessage(messageContext, requestMessage, {
+ context: 'api/server/routes/agents/index.js - abort user prerequisite',
+ });
+ if (!persistedRequest) {
+ throw new Error('Abort user prerequisite was not persisted');
+ }
+ } catch (error) {
+ persistenceErrors.push(error);
+ }
+
+ try {
+ const persistedResponse = await saveMessage(messageContext, responseMessage, {
+ context: 'api/server/routes/agents/index.js - abort endpoint',
+ });
+ if (!persistedResponse) {
+ throw new Error('Abort response was not persisted');
+ }
+ logger.debug(`[AgentStream] Saved partial response for: ${jobStreamId}`);
+ } catch (error) {
+ persistenceErrors.push(error);
+ }
+ }
+
+ /** Attempt checkpoint cleanup even when the message write failed, and
+ * attempt the message write even when cleanup will fail. Both are
+ * independently valuable; any failure still suppresses the normal
+ * FINAL after all required work has been attempted. */
+ if (shouldPruneCheckpoint) {
+ try {
+ await deleteAgentCheckpoint(
+ jobStreamId,
+ agentsCfg?.checkpointer,
+ checkpointGeneration,
+ checkpointNamespace !== ''
+ ? { throwOnError: true, checkpointNamespace }
+ : { throwOnError: true },
+ );
+ } catch (error) {
+ persistenceErrors.push(error);
+ }
+ }
+
+ if (persistenceErrors.length === 1) {
+ throw persistenceErrors[0];
+ }
+ if (persistenceErrors.length > 1) {
+ const error = new Error('Abort message persistence and checkpoint cleanup failed');
+ error.causes = persistenceErrors;
+ throw error;
+ }
+ },
+ });
+ if (abortResult.failureReason === 'generation_replaced') {
+ return res.status(409).json({ code: 'RUN_REPLACED', generationProtocolVersion });
+ }
+ if (abortResult.failureReason === 'job_still_active') {
+ res.set('Retry-After', '1');
+ return res.status(409).json({ code: 'RUN_STILL_ACTIVE', generationProtocolVersion });
+ }
+ if (!abortResult.success) {
+ // The route authorized a live generation, but the manager can lose its
+ // terminal CAS to natural completion/error (or observe deletion before
+ // its own lookup). Never claim that Stop won when no abort FINAL exists.
+ if (!abortResult.jobData) {
+ return res.status(404).json({
+ success: false,
+ error: 'Job not found',
+ streamId: jobStreamId,
+ generationProtocolVersion,
+ });
+ }
+
+ const currentJob = await GenerationJobManager.getJob(jobStreamId);
+ if (currentJob && currentJob.createdAt !== job.createdAt) {
+ return res.status(409).json({ code: 'RUN_REPLACED', generationProtocolVersion });
+ }
+ if (currentJob?.status === 'running' || currentJob?.status === 'requires_action') {
+ res.set('Retry-After', '1');
+ return res.status(409).json({ code: 'RUN_STILL_ACTIVE', generationProtocolVersion });
+ }
+
+ if (generationProtocolVersion < GENERATION_PROTOCOL_V2) {
+ return res.json({
+ success: true,
+ aborted: jobStreamId,
+ generationProtocolVersion,
+ });
+ }
+ return res.json({
+ success: false,
+ settled: true,
+ code: 'RUN_ALREADY_SETTLED',
+ streamId: jobStreamId,
+ generationProtocolVersion,
+ ...(currentJob?.status && { terminalStatus: currentJob.status }),
+ });
}
+ logger.debug(`[AgentStream] Job aborted successfully: ${jobStreamId}`, {
+ abortResultSuccess: abortResult.success,
+ abortResultUserMessageId: abortResult.jobData?.userMessage?.messageId,
+ abortResultResponseMessageId: abortResult.jobData?.responseMessageId,
+ });
+
+ if (abortResult.persistenceFailed && generationProtocolVersion < GENERATION_PROTOCOL_V2) {
+ res.set('Retry-After', '1');
+ return res.status(409).json({
+ code: 'ABORT_PERSISTENCE_FAILED',
+ generationProtocolVersion,
+ });
+ }
+
+ return res.json({
+ success: true,
+ aborted: jobStreamId,
+ generationProtocolVersion,
+ ...(abortResult.persistenceFailed && { persistenceFailed: true }),
+ // Steers that never reached an injection boundary — restored client-side
+ // as queued chips so the user's words aren't dropped with the abort.
+ ...(!abortResult.persistenceFailed &&
+ abortResult.pendingSteers?.length > 0 && { pendingSteers: abortResult.pendingSteers }),
+ });
}
- return res.json({
- success: true,
- aborted: jobStreamId,
- // Steers that never reached an injection boundary — restored client-side
- // as queued chips so the user's words aren't dropped with the abort.
- ...(abortResult.pendingSteers?.length > 0 && { pendingSteers: abortResult.pendingSteers }),
- });
+ logger.warn(`[AgentStream] Job not found for streamId: ${jobStreamId}`);
+ return sendGenerationJson(
+ res,
+ 404,
+ { error: 'Job not found', streamId: jobStreamId },
+ requestProtocolVersion,
+ );
+ } catch (error) {
+ logger.error('[AgentStream] Abort request failed', error);
+ if (res.headersSent) {
+ return next(error);
+ }
+ return sendGenerationJson(
+ res,
+ 500,
+ { code: 'ABORT_FAILED', error: 'Failed to abort generation' },
+ responseProtocolVersion,
+ );
}
-
- logger.warn(`[AgentStream] Job not found for streamId: ${jobStreamId}`);
- return res.status(404).json({ error: 'Job not found', streamId: jobStreamId });
});
/**
@@ -474,6 +910,19 @@ router.post(
SteerController.SteerCancelController,
);
+/**
+ * @route POST /chat/steer/arm
+ * @desc Escalate a still-queued steer to an interrupt in place (no new
+ * model-bound content, so no PII/moderation pass — just the shared limiters)
+ * @access Private
+ */
+router.post(
+ '/chat/steer/arm',
+ configMiddleware,
+ ...steerLimiters,
+ SteerController.SteerArmController,
+);
+
router.use('/', v1);
const chatRouter = express.Router();
diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js
index 13dec77866..e0e5b3024a 100644
--- a/api/server/services/Endpoints/agents/initialize.js
+++ b/api/server/services/Endpoints/agents/initialize.js
@@ -115,8 +115,16 @@ function createToolLoader(signal, streamId = null, definitionsOnly = false, jobC
* @param {AbortSignal} params.signal
* @param {Object} params.endpointOption
* @param {number} [params.jobCreatedAt]
+ * @param {string} [params.checkpointNamespace] Immutable saver-level generation scope
*/
-const initializeClient = async ({ req, res, signal, endpointOption, jobCreatedAt }) => {
+const initializeClient = async ({
+ req,
+ res,
+ signal,
+ endpointOption,
+ jobCreatedAt,
+ checkpointNamespace,
+}) => {
if (!endpointOption) {
throw new Error('Endpoint option not provided');
}
@@ -1057,6 +1065,7 @@ const initializeClient = async ({ req, res, signal, endpointOption, jobCreatedAt
startupTelemetry,
toolInputValidationErrors,
jobCreatedAt,
+ checkpointNamespace,
});
if (streamId) {
diff --git a/client/src/common/types.ts b/client/src/common/types.ts
index 66e145fa61..20499a3206 100644
--- a/client/src/common/types.ts
+++ b/client/src/common/types.ts
@@ -377,6 +377,14 @@ export type TOptions = {
overrideQuotes?: string[];
/** Added conversation for multi-convo feature - sent to server as part of submission payload */
addedConvo?: t.TConversation;
+ /** Reuse a durable submission identity (terminal steer recovery). */
+ overrideClientRequestId?: string;
+ /** Exact parked steer source for a recovery attempt. */
+ overrideRecoverySteerId?: string;
+ /** Exact terminal generation observed before an automatic queued start. */
+ overrideExpectedPredecessorCreatedAt?: number;
+ /** Client-only exact queue position restored if admission is rejected. */
+ overrideQueuedMessageOrigin?: unknown;
};
export type TAskFunction = (props: TAskProps, options?: TOptions) => false | void;
@@ -392,7 +400,7 @@ export type TAskFunction = (props: TAskProps, options?: TOptions) => false | voi
* value at call-time (for callback guards) without being a reactive dependency.
*/
export type TMessageChatContext = {
- ask: (...args: Parameters<TAskFunction>) => void;
+ ask: TAskFunction;
index: number;
regenerate: (message: t.TMessage, options?: { addedConvo?: t.TConversation | null }) => void;
conversation: t.TConversation | null;
diff --git a/client/src/components/Chat/Input/ChatForm.tsx b/client/src/components/Chat/Input/ChatForm.tsx
index 784ec144e2..dd01b8154e 100644
--- a/client/src/components/Chat/Input/ChatForm.tsx
+++ b/client/src/components/Chat/Input/ChatForm.tsx
@@ -199,6 +199,10 @@ const ChatForm = memo(function ChatForm({
overrideFiles,
overrideQuotes: context?.quotes ?? [],
overrideManualSkills: context?.manualSkills ?? [],
+ overrideClientRequestId: context?.clientRequestId,
+ overrideRecoverySteerId: context?.recoverySteerId,
+ overrideExpectedPredecessorCreatedAt: context?.expectedPredecessorCreatedAt,
+ overrideQueuedMessageOrigin: context?.queuedMessageOrigin,
}),
[submitMessage],
);
@@ -436,19 +440,24 @@ const ChatForm = memo(function ChatForm({
/** One button slot while a run is generating: with composer text the send
* button takes over (Enter steers/queues; hover reveals all actions);
* clearing the text restores Stop. */
- const duringRunSlot =
- steering.duringRunActive && (textValue?.trim() ?? '') !== '' ? (
- <DuringRunSendButton
- ref={submitButtonRef}
- control={methods.control}
- steering={steering}
- getText={() => methods.getValues('text')}
- onConsumed={() => methods.reset()}
- disabled={filesLoading}
- />
- ) : (
- <StopButton stop={handleStopGenerating} setShowStopButton={setShowStopButton} />
- );
+ const duringRunSlot = (() => {
+ if (steering.duringRunActive && (textValue?.trim() ?? '') !== '') {
+ return (
+ <DuringRunSendButton
+ ref={submitButtonRef}
+ control={methods.control}
+ steering={steering}
+ getText={() => methods.getValues('text')}
+ onConsumed={() => methods.reset()}
+ disabled={filesLoading}
+ />
+ );
+ }
+ if (showStopButton) {
+ return <StopButton stop={handleStopGenerating} setShowStopButton={setShowStopButton} />;
+ }
+ return null;
+ })();
const baseClasses = useMemo(
() =>
@@ -665,18 +674,22 @@ const ChatForm = memo(function ChatForm({
isSubmitting={isSubmitting}
/>
)}
- {steering.duringRunActive && (textValue?.trim() ?? '') !== '' && (
- <div className={`${isRTL ? 'ml-2' : 'mr-2'}`}>
- <InterruptSteerButton
- steering={steering}
- getText={() => methods.getValues('text')}
- onConsumed={() => methods.reset()}
- disabled={filesLoading}
- />
- </div>
- )}
+ {steering.duringRunActive &&
+ steering.canControlGeneration &&
+ (textValue?.trim() ?? '') !== '' && (
+ <div className={`${isRTL ? 'ml-2' : 'mr-2'}`}>
+ <InterruptSteerButton
+ steering={steering}
+ getText={() => methods.getValues('text')}
+ onConsumed={() => methods.reset()}
+ disabled={filesLoading}
+ />
+ </div>
+ )}
<div className={`${isRTL ? 'ml-2' : 'mr-2'}`}>
- {isSubmitting && showStopButton && !answerMode.active
+ {isSubmitting &&
+ (showStopButton || steering.duringRunActive) &&
+ !answerMode.active
? duringRunSlot
: endpoint && (
<SendButton
diff --git a/client/src/components/Chat/Input/DuringRunSendButton.tsx b/client/src/components/Chat/Input/DuringRunSendButton.tsx
index 213aec32d3..d4ad5a718e 100644
--- a/client/src/components/Chat/Input/DuringRunSendButton.tsx
+++ b/client/src/components/Chat/Input/DuringRunSendButton.tsx
@@ -163,7 +163,7 @@ const DuringRunSendButton = React.memo(
// `!canSteer` like the steer row above: `canSteer` is also false before
// a conversation exists, where `interruptSteer` falls back to interrupt
// & send and this row must stay live for the whole first turn.
- disabled: steering.pausedOnApproval,
+ disabled: steering.pausedOnApproval || !steering.canControlGeneration,
onClick: () => runAction((text) => steering.interruptSteer(text)),
};
const interruptRow: ActionRow = {
@@ -171,6 +171,7 @@ const DuringRunSendButton = React.memo(
label: localize('com_ui_interrupt_send'),
kbd: verdicts.altEnter === 'interrupt' ? altEnter : undefined,
icon: <OctagonPause className="h-4 w-4 text-red-500" aria-hidden="true" />,
+ disabled: !steering.canControlGeneration,
onClick: () => runAction((text) => steering.interruptAndSend(text)),
};
const rows = primary === 'steer' ? [steerRow, queueRow] : [queueRow, steerRow];
diff --git a/client/src/components/Chat/Input/InFlightSteers.tsx b/client/src/components/Chat/Input/InFlightSteers.tsx
index cbecaf0d7b..456c7ea0b4 100644
--- a/client/src/components/Chat/Input/InFlightSteers.tsx
+++ b/client/src/components/Chat/Input/InFlightSteers.tsx
@@ -1,5 +1,5 @@
import { memo, useId, useRef, useMemo, useState, useEffect, useCallback } from 'react';
-import { useSetAtom } from 'jotai';
+import { useSetAtom, useAtomValue } from 'jotai';
import { useToastContext } from '@librechat/client';
import { useRecoilValue, useRecoilCallback } from 'recoil';
import { X, Zap, ZapOff, Clock, Pencil, ChevronUp, ChevronDown } from 'lucide-react';
@@ -7,13 +7,19 @@ import type { TFile, TMessage } from 'librechat-data-provider';
import type { SteeringControls, QueuedMessageContext } from '~/hooks/Chat/useSteering';
import type { PendingSteer } from '~/store/families';
import type { MenuEntry } from './SteerMenu';
+import {
+ RowMenu,
+ EscalateNowButton,
+ useDefaultToggleEntry,
+ useInterruptToggleEntry,
+} from './SteerMenu';
import FilePreviewDialog from '~/components/Chat/Messages/Content/FilePreviewDialog';
+import { supportsGenerationProtocolV2, useArmSteerMutation } from '~/data-provider';
+import { steerOverlayHeightFamily, escalatingSteerFamily } from '~/store/steer';
import MarkdownLite from '~/components/Chat/Messages/Content/MarkdownLite';
import FileContainer from '~/components/Chat/Input/Files/FileContainer';
import { useSteerCancel, useSteerReclaim, useLocalize } from '~/hooks';
import ImagePreview from '~/components/Chat/Input/Files/ImagePreview';
-import { RowMenu, useDefaultToggleEntry } from './SteerMenu';
-import { steerOverlayHeightFamily } from '~/store/steer';
import { carriedSteerContext, cn } from '~/utils';
import store from '~/store';
@@ -42,6 +48,24 @@ const splitFiles = (files?: TMessage['files']) => {
* for its own bottom margin does not trip a pointless toggle. */
const STEER_COLLAPSED_MAX_HEIGHT = 128;
const STEER_OVERFLOW_TOLERANCE = 8;
+/** Axios has no default request timeout. Bound the UI lock while preserving an
+ * honest unknown outcome; the idempotent arm may still complete server-side. */
+const ARM_CONFIRM_TIMEOUT_MS = 10_000;
+
+type ArmFailure = {
+ name?: string;
+ response?: { data?: { code?: string } };
+};
+
+/** Only a failure without an HTTP response leaves the server-side outcome
+ * unknown. An HTTP rejection is a known response and must not replay the arm. */
+const isAmbiguousArmFailure = (error: unknown): boolean => {
+ const failure = error as ArmFailure | null | undefined;
+ return failure?.name !== 'AbortError' && failure?.response == null;
+};
+
+const armFailureCode = (error: unknown): string | undefined =>
+ (error as ArmFailure | null | undefined)?.response?.data?.code;
/**
* One steer on its way into the run, anchored above the composer as a message
@@ -64,11 +88,13 @@ const InFlightSteer = memo(function InFlightSteer({
steer,
steering,
conversationId,
+ interruptPending,
onRestoreToComposer,
}: {
steer: PendingSteer;
steering: SteeringControls;
conversationId: string;
+ interruptPending: boolean;
onRestoreToComposer: RestoreToComposer;
}) {
const localize = useLocalize();
@@ -76,8 +102,17 @@ const InFlightSteer = memo(function InFlightSteer({
const cancelSteer = useSteerCancel(conversationId);
const reclaimSteer = useSteerReclaim(conversationId);
const toggleEntry = useDefaultToggleEntry(steering);
+ const interruptToggle = useInterruptToggleEntry();
const enableUserMsgMarkdown = useRecoilValue<boolean>(store.enableUserMsgMarkdown);
+ const activeGenerationCreatedAt = useRecoilValue(
+ store.activeGenerationCreatedAtByConvoId(conversationId),
+ );
+ const activeGenerationProtocolVersion = useRecoilValue(
+ store.activeGenerationProtocolVersionByConvoId(conversationId),
+ );
const [selectedFile, setSelectedFile] = useState<Partial<TFile> | null>(null);
+ const optionsButtonRef = useRef<HTMLButtonElement>(null);
+ const [escalationAnnouncement, setEscalationAnnouncement] = useState('');
const handlePreviewClose = useCallback((open: boolean) => {
if (!open) {
setSelectedFile(null);
@@ -113,19 +148,154 @@ const InFlightSteer = memo(function InFlightSteer({
return () => observer.disconnect();
}, []);
- /** Whether the words have already been re-homed by a terminal conversion (a
- * run that ended/errored mid-reclaim queues the still-present chip). The
- * queue action is safe either way — the conversion dedupes by id — but a
- * composer restore would leave one copy queued and another in the draft. */
- const hasSettled = useRecoilCallback(
- ({ snapshot }) =>
- (steerId: string) =>
- snapshot
- .getLoadable(store.appliedSteerIdsByConvoId(conversationId))
- .getValue()
- .includes(steerId),
+ /** Relabels the chip in place once the server confirms the durable arm —
+ * same steerId, same position, only the `preempt` flag flips. */
+ const markSteerPreempt = useRecoilCallback(
+ ({ set }) =>
+ (steerId: string, revision: number) =>
+ set(store.pendingSteersByConvoId(conversationId), (prev) =>
+ prev.map((item) =>
+ item.steerId === steerId && revision >= (item.preemptRevision ?? 0)
+ ? { ...item, preempt: true, preemptRevision: revision }
+ : item,
+ ),
+ ),
[conversationId],
);
+ const { mutateAsync: armSteer } = useArmSteerMutation();
+ const setEscalating = useSetAtom(escalatingSteerFamily(conversationId));
+
+ /**
+ * Escalate this waiting steer to an interrupt: one idempotent server op
+ * flips `preempt` on the EXISTING queued item, so its FIFO position, id, and
+ * timestamp survive and there is no reclaim window to race. A transport
+ * failure is retried once because the first request may have committed even
+ * though its response was lost. Every "too late" interleaving (drained,
+ * cancelled, run ended or replaced) is the same honest `armed: false`, and
+ * the chip is only relabelled on a confirmed durable arm. The escalating
+ * flag flips synchronously, before the request: the chip-derived gate cannot
+ * see this arm until the response lands, and the other escalation controls
+ * advertise "one interrupt at a time".
+ */
+ const escalate = useCallback(
+ (event: React.MouseEvent<HTMLButtonElement>) => {
+ const generationCreatedAt =
+ steer.generationCreatedAt ?? activeGenerationCreatedAt ?? undefined;
+ if (generationCreatedAt == null) {
+ return;
+ }
+ const trigger = event.currentTarget;
+ setEscalationAnnouncement('');
+ setEscalating(true);
+ const params = {
+ conversationId,
+ steerId: steer.steerId,
+ ...(generationCreatedAt != null && { generationCreatedAt }),
+ };
+ const requestArm = async () => {
+ let firstResponseWasLost = false;
+ let acceptingRetry = true;
+ let timeout: ReturnType<typeof setTimeout> | undefined;
+ try {
+ const firstAttempt = armSteer(params);
+ const attempts =
+ activeGenerationProtocolVersion === 2
+ ? firstAttempt.catch((error) => {
+ /** If the overall confirmation window already closed, do not let a
+ * very late rejection launch a detached retry behind the user's
+ * back. The first request itself may still have committed. */
+ if (!acceptingRetry) {
+ throw error;
+ }
+ if (!isAmbiguousArmFailure(error)) {
+ throw error;
+ }
+ firstResponseWasLost = true;
+ return armSteer(params);
+ })
+ : firstAttempt;
+ const response = await Promise.race([
+ attempts,
+ new Promise<never>((_resolve, reject) => {
+ timeout = setTimeout(
+ () => reject(new Error('Steer arm confirmation timed out')),
+ ARM_CONFIRM_TIMEOUT_MS,
+ );
+ }),
+ ]);
+ const responseSupportsNegotiatedProtocol =
+ activeGenerationProtocolVersion === 1 || supportsGenerationProtocolV2(response);
+ if (responseSupportsNegotiatedProtocol && response.armed === true) {
+ /** The successful state removes the arm button. Move focus to the
+ * stable options control only if the user has not moved elsewhere
+ * while the request was pending. */
+ if (document.activeElement === trigger) {
+ optionsButtonRef.current?.focus();
+ }
+ setEscalationAnnouncement(localize('com_ui_steer_in_flight_preempt'));
+ markSteerPreempt(steer.steerId, response.preemptRevision ?? 0);
+ return;
+ }
+ if (!responseSupportsNegotiatedProtocol) {
+ showToast({ message: localize('com_ui_steer_arm_unconfirmed'), status: 'warning' });
+ return;
+ }
+ /** Once a response was lost, a later `armed: false` cannot prove the
+ * first request did not commit: the steer may have drained or the job
+ * may have paused between attempts. Keep the chip event-driven and
+ * report the result as unknown instead of claiming a lost race. */
+ if (firstResponseWasLost) {
+ showToast({ message: localize('com_ui_steer_arm_unconfirmed'), status: 'warning' });
+ return;
+ }
+ /* `armed: false` is deliberately ambiguous — injected, cancelled,
+ * re-homed, or run over — so the message only says the escalation
+ * lost, and the chip defers to the events for what happened. */
+ showToast({
+ message: localize(
+ response.code === 'PREEMPT_UNSUPPORTED'
+ ? 'com_ui_steer_preempt_unsupported'
+ : 'com_ui_steer_arm_lost_race',
+ ),
+ status: 'info',
+ });
+ } catch (error) {
+ const ambiguous = isAmbiguousArmFailure(error);
+ if (ambiguous) {
+ showToast({
+ message: localize('com_ui_steer_arm_unconfirmed'),
+ status: 'warning',
+ });
+ return;
+ }
+ showToast({
+ message: localize(
+ armFailureCode(error) === 'PREEMPT_UNSUPPORTED'
+ ? 'com_ui_steer_preempt_unsupported'
+ : 'com_ui_steer_arm_lost_race',
+ ),
+ status: 'info',
+ });
+ } finally {
+ acceptingRetry = false;
+ clearTimeout(timeout);
+ }
+ };
+ void requestArm().finally(() => setEscalating(false));
+ },
+ [
+ armSteer,
+ conversationId,
+ steer.steerId,
+ steer.generationCreatedAt,
+ activeGenerationCreatedAt,
+ activeGenerationProtocolVersion,
+ setEscalating,
+ markSteerPreempt,
+ showToast,
+ localize,
+ ],
+ );
/**
* Takes the steer back off the server queue so its words can be re-homed.
@@ -157,12 +327,6 @@ const InFlightSteer = memo(function InFlightSteer({
if (!reclaimed) {
return;
}
- if (hasSettled(steer.steerId)) {
- /* The run ended while the reclaim was in flight and its terminal
- * conversion already queued these words. */
- showToast({ message: localize('com_ui_steer_run_ended_queued'), status: 'info' });
- return;
- }
const restored = onRestoreToComposer(
steer.text,
steer.files,
@@ -181,18 +345,6 @@ const InFlightSteer = memo(function InFlightSteer({
});
},
},
- {
- key: 'queue',
- label: localize('com_ui_convert_to_queue'),
- icon: <Clock className="h-4 w-4 text-cyan-500" aria-hidden="true" />,
- onClick: () => {
- void reclaim().then((reclaimed) => {
- if (reclaimed) {
- steering.queueReclaimedSteer(steer);
- }
- });
- },
- },
{
/* Non-destructive, but only when it is safe: cancel reliably first (the
* optimistic hook removes the chip and restores it if the server would
@@ -209,6 +361,8 @@ const InFlightSteer = memo(function InFlightSteer({
if (outcome !== 'reclaimed') {
return;
}
+ // useSteerReclaim has tombstoned both ids and removed any terminal
+ // recovery copy, so exactly one client destination is restored here.
const restored = onRestoreToComposer(
steer.text,
steer.files,
@@ -225,8 +379,20 @@ const InFlightSteer = memo(function InFlightSteer({
});
},
},
- toggleEntry,
+ {
+ key: 'queue',
+ label: localize('com_ui_convert_to_queue'),
+ icon: <Clock className="h-4 w-4 text-cyan-500" aria-hidden="true" />,
+ onClick: () => {
+ void reclaim().then((reclaimed) => {
+ if (reclaimed) {
+ steering.queueReclaimedSteer(steer);
+ }
+ });
+ },
+ },
];
+ const preferences: MenuEntry[] = [toggleEntry, interruptToggle];
return (
<div
@@ -334,11 +500,32 @@ const InFlightSteer = memo(function InFlightSteer({
* controls on the queued rows). `sticky` keeps it in view while the
* user scrolls through a tall, expanded steer (the stack scrolls once
* it passes 35vh). */
- <div data-testid="steer-controls" className="sticky top-2 flex shrink-0 items-center">
- <RowMenu label={localize('com_ui_more_options')} entries={entries} />
+ <div
+ data-testid="steer-controls"
+ className="sticky top-2 flex shrink-0 items-center gap-1"
+ >
+ {!preempting && (
+ <EscalateNowButton
+ surface="bubble"
+ messageText={steer.text}
+ disabled={
+ interruptPending || steering.pausedOnApproval || !steering.duringRunActive
+ }
+ onClick={escalate}
+ />
+ )}
+ <RowMenu
+ label={localize('com_ui_more_options')}
+ entries={entries}
+ preferences={preferences}
+ buttonRef={optionsButtonRef}
+ />
</div>
)}
</div>
+ <span role="status" aria-live="polite" aria-atomic="true" className="sr-only">
+ {escalationAnnouncement}
+ </span>
{others.length > 0 && (
<FilePreviewDialog
open={selectedFile !== null}
@@ -372,6 +559,15 @@ const InFlightSteers = memo(function InFlightSteers({
const localize = useLocalize();
const steers = useRecoilValue(store.pendingSteersByConvoId(conversationId));
const inFlight = useMemo(() => steers.filter((steer) => steer.status !== 'failed'), [steers]);
+ /** Mirrors `PendingSteerChips`: while one interrupt is unresolved, every
+ * other escalation control disables rather than arming a second seal. The
+ * escalating flag covers an arm request's round trip, before its chip
+ * relabels for the chip-derived check to see. */
+ const escalating = useAtomValue(escalatingSteerFamily(conversationId));
+ const interruptPending = useMemo(
+ () => escalating || inFlight.some((steer) => steer.preempt === true),
+ [escalating, inFlight],
+ );
const setOverlayHeight = useSetAtom(steerOverlayHeightFamily(conversationId));
/** Steers append newest-last, so an overflowing stack would sit scrolled to
@@ -434,6 +630,7 @@ const InFlightSteers = memo(function InFlightSteers({
steer={steer}
steering={steering}
conversationId={conversationId}
+ interruptPending={interruptPending}
onRestoreToComposer={onRestoreToComposer}
/>
))}
diff --git a/client/src/components/Chat/Input/InterruptSteerButton.tsx b/client/src/components/Chat/Input/InterruptSteerButton.tsx
index 3a55d3f851..e5c593a3a6 100644
--- a/client/src/components/Chat/Input/InterruptSteerButton.tsx
+++ b/client/src/components/Chat/Input/InterruptSteerButton.tsx
@@ -27,7 +27,8 @@ const InterruptSteerButton = React.memo((props: InterruptSteerButtonProps) => {
const { steering } = props;
const label = localize('com_ui_interrupt_steer_button');
/** Pre-empts the server's 409: a paused run cannot accept a steer. */
- const disabled = props.disabled === true || steering.pausedOnApproval;
+ const disabled =
+ props.disabled === true || steering.pausedOnApproval || !steering.canControlGeneration;
const onClick = () => {
const text = props.getText().trim();
diff --git a/client/src/components/Chat/Input/PendingSteerChips.tsx b/client/src/components/Chat/Input/PendingSteerChips.tsx
index 34d98324aa..a6f9bff00b 100644
--- a/client/src/components/Chat/Input/PendingSteerChips.tsx
+++ b/client/src/components/Chat/Input/PendingSteerChips.tsx
@@ -1,12 +1,22 @@
-import { memo, useMemo } from 'react';
+import { memo, useMemo, useRef, useState, useCallback } from 'react';
+import { useAtomValue } from 'jotai';
import { useRecoilValue } from 'recoil';
+import { useToastContext } from '@librechat/client';
import { X, Zap, Send, Clock, Pencil, Trash2, Paperclip, RotateCcw } from 'lucide-react';
import type { TMessage } from 'librechat-data-provider';
import type { SteeringControls, QueuedMessageContext } from '~/hooks/Chat/useSteering';
import type { PendingSteer, QueuedMessage } from '~/store/families';
import type { RestoreToComposer } from './InFlightSteers';
import type { MenuEntry } from './SteerMenu';
-import { RowMenu, useDefaultToggleEntry, ICON_BTN_CLASS, PRIMARY_BTN_CLASS } from './SteerMenu';
+import {
+ RowMenu,
+ ICON_BTN_CLASS,
+ PRIMARY_BTN_CLASS,
+ EscalateNowButton,
+ useDefaultToggleEntry,
+ useInterruptToggleEntry,
+} from './SteerMenu';
+import { escalatingSteerFamily } from '~/store/steer';
import { useLocalize } from '~/hooks';
import { cn } from '~/utils';
import store from '~/store';
@@ -31,12 +41,14 @@ function QueuedRow({
message,
steering,
conversationId,
+ interruptPending,
onEditToComposer,
onRestoreToComposer,
}: {
message: QueuedMessage;
steering: SteeringControls;
conversationId: string;
+ interruptPending: boolean;
onEditToComposer: (
text: string,
files?: TMessage['files'],
@@ -45,26 +57,89 @@ function QueuedRow({
onRestoreToComposer: RestoreToComposer;
}) {
const localize = useLocalize();
+ const { showToast } = useToastContext();
const toggleEntry = useDefaultToggleEntry(steering);
+ const interruptToggle = useInterruptToggleEntry();
const fileCount = message.files?.length ?? 0;
- const canSteerNow = steering.duringRunActive && steering.canSteer;
- const showPrimary = canSteerNow || !steering.duringRunActive;
+ const isRecovered = message.recoverySteerId != null;
+ const actionPendingRef = useRef(false);
+ const [actionPending, setActionPending] = useState(false);
+ /** A recovered item has a replayable parked source. Edit/remove must first
+ * cancel that source by receipt; local-only rows settle synchronously through
+ * the same control. The ref closes the pre-render double-click window. */
+ const afterDiscard = useCallback(
+ (action: () => boolean) => {
+ if (actionPendingRef.current) {
+ return;
+ }
+ actionPendingRef.current = true;
+ setActionPending(true);
+ void (async () => {
+ let discarded = false;
+ try {
+ discarded = await steering.discardQueued(message);
+ } catch {
+ // The steering hook reports request failures and leaves the row in
+ // place. Keep this guard for test/custom control implementations.
+ }
+ if (!discarded) {
+ actionPendingRef.current = false;
+ setActionPending(false);
+ return;
+ }
+ if (!action()) {
+ actionPendingRef.current = false;
+ setActionPending(false);
+ }
+ })();
+ },
+ [message, steering],
+ );
+ // A recovered item is consumed atomically only when it starts a normal
+ // generation. Re-steering it would leave or duplicate the parked source;
+ // Edit/remove are safe because `afterDiscard` tombstones that source first.
+ const canSteerNow = steering.duringRunActive && steering.canSteer && !isRecovered;
+ const showPrimary = canSteerNow || (!steering.duringRunActive && steering.canSendQueuedNow);
+ /** `canSteer` is defined as false while paused on approval, but the
+ * escalation control must stay visible-and-disabled there — hiding it
+ * during the pause is exactly the discoverability gap this button fixes. */
+ const showEscalate =
+ !isRecovered && (steering.pausedOnApproval || (steering.duringRunActive && steering.canSteer));
const entries: MenuEntry[] = [
{
key: 'edit',
label: localize('com_ui_edit_message'),
icon: <Pencil className="h-4 w-4" aria-hidden="true" />,
+ disabled: actionPending,
onClick: () => {
- steering.removeQueued(message.id);
- onEditToComposer(message.text, message.files, {
- quotes: message.quotes,
- manualSkills: message.manualSkills,
+ const context = { quotes: message.quotes, manualSkills: message.manualSkills };
+ if (!isRecovered) {
+ steering.removeQueued(message.id);
+ onEditToComposer(message.text, message.files, {
+ quotes: message.quotes,
+ manualSkills: message.manualSkills,
+ });
+ return;
+ }
+ afterDiscard(() => {
+ const restored = onRestoreToComposer(
+ message.text,
+ message.files,
+ context,
+ conversationId,
+ );
+ if (!restored) {
+ showToast({ message: localize('com_ui_steer_edit_queued'), status: 'info' });
+ return false;
+ }
+ steering.removeQueued(message.id);
+ return true;
});
},
},
- toggleEntry,
];
+ const preferences: MenuEntry[] = [toggleEntry, interruptToggle];
return (
<div role="listitem" className={ROW_CLASS} data-testid="queued-message-row">
@@ -80,6 +155,7 @@ function QueuedRow({
<button
type="button"
className={PRIMARY_BTN_CLASS}
+ disabled={actionPending}
onClick={() => steering.sendQueuedNow(message)}
>
{canSteerNow ? (
@@ -95,26 +171,47 @@ function QueuedRow({
)}
</button>
)}
+ {showEscalate && (
+ <EscalateNowButton
+ surface="queued"
+ messageText={message.text}
+ disabled={steering.pausedOnApproval || interruptPending || actionPending}
+ onClick={() => steering.sendQueuedNow(message, { preempt: true })}
+ />
+ )}
<button
type="button"
aria-label={localize('com_ui_remove_queued')}
+ disabled={actionPending}
onClick={() => {
- /* Same safety net as the in-flight cancel: return the words to the
- * composer when it is free (the gated restore refuses rather than
- * clobber a draft), then remove either way. */
- onRestoreToComposer(
- message.text,
- message.files,
- { quotes: message.quotes, manualSkills: message.manualSkills },
- conversationId,
- );
- steering.removeQueued(message.id);
+ const remove = () => {
+ /* Same safety net as the in-flight cancel: once removal is safely
+ * settled, return the words to the composer when it is free (the
+ * gated restore refuses rather than clobber a draft). */
+ onRestoreToComposer(
+ message.text,
+ message.files,
+ { quotes: message.quotes, manualSkills: message.manualSkills },
+ conversationId,
+ );
+ steering.removeQueued(message.id);
+ return true;
+ };
+ if (!isRecovered) {
+ remove();
+ return;
+ }
+ afterDiscard(remove);
}}
className={ICON_BTN_CLASS}
>
<Trash2 className="h-4 w-4" aria-hidden="true" />
</button>
- <RowMenu label={localize('com_ui_more_options')} entries={entries} />
+ <RowMenu
+ label={localize('com_ui_more_options')}
+ entries={entries}
+ preferences={preferences}
+ />
</div>
);
}
@@ -134,32 +231,39 @@ function FailedSteerRow({
}) {
const localize = useLocalize();
const toggleEntry = useDefaultToggleEntry(steering);
+ const interruptToggle = useInterruptToggleEntry();
+ const canRetry = !steer.deliveryUncertain || steer.generationProtocolVersion === 2;
- const entries: MenuEntry[] = [
- {
- key: 'edit',
- label: localize('com_ui_edit_message'),
- icon: <Pencil className="h-4 w-4" aria-hidden="true" />,
- onClick: () => {
- steering.removeSteer(steer.steerId);
- onEditToComposer(steer.text, steer.files, {
- quotes: steer.quotes,
- manualSkills: steer.manualSkills,
- });
- },
- },
- {
- key: 'queue',
- label: localize('com_ui_convert_to_queue'),
- icon: <Clock className="h-4 w-4 text-cyan-500" aria-hidden="true" />,
- onClick: () =>
- steering.convertSteerToQueue(steer.steerId, steer.text, steer.files, {
- quotes: steer.quotes,
- manualSkills: steer.manualSkills,
- }),
- },
- toggleEntry,
- ];
+ const entries: MenuEntry[] = steer.deliveryUncertain
+ ? []
+ : [
+ {
+ key: 'edit',
+ label: localize('com_ui_edit_message'),
+ icon: <Pencil className="h-4 w-4" aria-hidden="true" />,
+ onClick: () => {
+ steering.removeSteer(steer.steerId);
+ onEditToComposer(steer.text, steer.files, {
+ quotes: steer.quotes,
+ manualSkills: steer.manualSkills,
+ });
+ },
+ },
+ {
+ key: 'queue',
+ label: localize('com_ui_convert_to_queue'),
+ icon: <Clock className="h-4 w-4 text-cyan-500" aria-hidden="true" />,
+ onClick: () =>
+ steering.convertSteerToQueue(
+ steer.steerId,
+ steer.text,
+ steer.files,
+ { quotes: steer.quotes, manualSkills: steer.manualSkills },
+ steer.queuedOrigin,
+ ),
+ },
+ ];
+ const preferences: MenuEntry[] = [toggleEntry, interruptToggle];
return (
<div
@@ -171,32 +275,52 @@ function FailedSteerRow({
<span className="min-w-0 flex-1 truncate" title={steer.text}>
{steer.text}
</span>
- <span className="shrink-0 text-xs text-red-500">{localize('com_ui_steer_failed')}</span>
- <button
- type="button"
- className={PRIMARY_BTN_CLASS}
- onClick={() =>
- steering.retrySteer(
- steer.steerId,
- steer.text,
- steer.files,
- { quotes: steer.quotes, manualSkills: steer.manualSkills },
- { preempt: steer.preempt === true },
- )
- }
- >
- <RotateCcw className="h-4 w-4" aria-hidden="true" />
- {localize('com_ui_steer_retry')}
- </button>
- <button
- type="button"
- aria-label={localize('com_ui_remove_queued')}
- onClick={() => steering.removeSteer(steer.steerId)}
- className={ICON_BTN_CLASS}
- >
- <X className="h-4 w-4" aria-hidden="true" />
- </button>
- <RowMenu label={localize('com_ui_more_options')} entries={entries} />
+ <span className="shrink-0 text-xs text-red-500">
+ {localize(
+ steer.deliveryUncertain ? 'com_ui_steer_delivery_unconfirmed' : 'com_ui_steer_failed',
+ )}
+ </span>
+ {canRetry && (
+ <button
+ type="button"
+ className={PRIMARY_BTN_CLASS}
+ onClick={() =>
+ steering.retrySteer(
+ steer.steerId,
+ steer.text,
+ steer.files,
+ { quotes: steer.quotes, manualSkills: steer.manualSkills },
+ {
+ preempt: steer.preempt === true,
+ createdAt: steer.createdAt,
+ generationProtocolVersion: steer.generationProtocolVersion,
+ ...(steer.generationCreatedAt != null && {
+ generationCreatedAt: steer.generationCreatedAt,
+ }),
+ ...(steer.queuedOrigin && { queuedOrigin: steer.queuedOrigin }),
+ },
+ )
+ }
+ >
+ <RotateCcw className="h-4 w-4" aria-hidden="true" />
+ {localize('com_ui_steer_retry')}
+ </button>
+ )}
+ {!steer.deliveryUncertain && (
+ <button
+ type="button"
+ aria-label={localize('com_ui_remove_queued')}
+ onClick={() => steering.removeSteer(steer.steerId)}
+ className={ICON_BTN_CLASS}
+ >
+ <X className="h-4 w-4" aria-hidden="true" />
+ </button>
+ )}
+ <RowMenu
+ label={localize('com_ui_more_options')}
+ entries={entries}
+ preferences={preferences}
+ />
</div>
);
}
@@ -230,6 +354,15 @@ function PendingSteerChips({
const steers = useRecoilValue(store.pendingSteersByConvoId(conversationId));
const queued = useRecoilValue(store.queuedMessagesByConvoId(steering.queueKey));
const failedSteers = useMemo(() => steers.filter((steer) => steer.status === 'failed'), [steers]);
+ /** Only one interrupt can be in flight: a second preempt while one is
+ * unresolved would arm a second seal, so escalation buttons disable. The
+ * escalating flag covers a bubble arm's round trip, before its chip
+ * relabels for the chip-derived check to see. */
+ const escalating = useAtomValue(escalatingSteerFamily(conversationId));
+ const interruptPending = useMemo(
+ () => escalating || steers.some((steer) => steer.preempt === true && steer.status !== 'failed'),
+ [escalating, steers],
+ );
if (failedSteers.length === 0 && queued.length === 0) {
return null;
@@ -256,6 +389,7 @@ function PendingSteerChips({
message={message}
steering={steering}
conversationId={conversationId}
+ interruptPending={interruptPending}
onEditToComposer={onEditToComposer}
onRestoreToComposer={onRestoreToComposer}
/>
diff --git a/client/src/components/Chat/Input/SteerMenu.tsx b/client/src/components/Chat/Input/SteerMenu.tsx
index a5ba0c6fe9..977568b4f4 100644
--- a/client/src/components/Chat/Input/SteerMenu.tsx
+++ b/client/src/components/Chat/Input/SteerMenu.tsx
@@ -1,8 +1,13 @@
-import { useMemo } from 'react';
+import { Fragment, useEffect, useId, useMemo, useState, useSyncExternalStore } from 'react';
+import { useRecoilState } from 'recoil';
import * as Ariakit from '@ariakit/react';
-import { Zap, Clock, MoreHorizontal } from 'lucide-react';
+import { Zap, ZapOff, Clock, ArrowUp, CircleHelp, MoreHorizontal } from 'lucide-react';
+import type { Ref } from 'react';
import type { SteeringControls } from '~/hooks/Chat/useSteering';
+import { useShortcutAriaKey, useShortcutDisplay } from '~/hooks/useKeyboardShortcuts';
import { useLocalize } from '~/hooks';
+import { cn } from '~/utils';
+import store from '~/store';
/** Shared row/bubble affordances for the during-run surfaces: the in-flight
* steer bubbles (`InFlightSteers`) and the queued/failed rows
@@ -15,41 +20,263 @@ const MENU_CLASS =
'z-50 min-w-[13rem] rounded-xl border border-border-light bg-surface-secondary p-1.5 text-text-primary shadow-lg outline-none';
const MENU_ITEM_CLASS =
'flex w-full cursor-pointer items-center gap-2 rounded-lg px-2 py-1.5 text-sm text-text-primary data-[active-item]:bg-surface-tertiary aria-disabled:cursor-not-allowed aria-disabled:opacity-50';
+const ESCALATION_MESSAGE_LABEL_MAX_LENGTH = 80;
+
+const activeEscalateListeners = new Set<() => void>();
+let hoveredEscalateTarget: string | null = null;
+let focusedEscalateTarget: string | null = null;
+
+function getActiveEscalateTarget() {
+ return focusedEscalateTarget ?? hoveredEscalateTarget;
+}
+
+function subscribeToActiveEscalateTarget(listener: () => void) {
+ activeEscalateListeners.add(listener);
+ return () => activeEscalateListeners.delete(listener);
+}
+
+function updateActiveEscalateTarget(kind: 'hover' | 'focus', targetId: string, active: boolean) {
+ const previous = getActiveEscalateTarget();
+ if (kind === 'hover') {
+ if (active) {
+ hoveredEscalateTarget = targetId;
+ } else if (hoveredEscalateTarget === targetId) {
+ hoveredEscalateTarget = null;
+ }
+ } else {
+ if (active) {
+ focusedEscalateTarget = targetId;
+ } else if (focusedEscalateTarget === targetId) {
+ focusedEscalateTarget = null;
+ }
+ }
+ if (previous !== getActiveEscalateTarget()) {
+ activeEscalateListeners.forEach((listener) => listener());
+ }
+}
+
+function clearActiveEscalateTarget(targetId: string) {
+ const previous = getActiveEscalateTarget();
+ if (hoveredEscalateTarget === targetId) {
+ hoveredEscalateTarget = null;
+ }
+ if (focusedEscalateTarget === targetId) {
+ focusedEscalateTarget = null;
+ }
+ if (previous !== getActiveEscalateTarget()) {
+ activeEscalateListeners.forEach((listener) => listener());
+ }
+}
export type MenuEntry = {
key: string;
label: string;
icon: React.ReactNode;
onClick: () => void;
+ disabled?: boolean;
+ /** Localized description announced on the action and exposed by a help disclosure. */
+ info?: string;
};
-/** Per-row "…" overflow menu (edit / mode toggle / conversions). */
-export function RowMenu({ label, entries }: { label: string; entries: MenuEntry[] }) {
+/**
+ * Per-row "…" overflow menu: message actions first, then a visually separated
+ * "Preferences" section for the sticky mode toggles, so one-off actions and
+ * persistent behavior changes never read as the same kind of choice.
+ */
+export function RowMenu({
+ label,
+ entries,
+ preferences,
+ buttonRef,
+}: {
+ label: string;
+ entries: MenuEntry[];
+ preferences?: MenuEntry[];
+ buttonRef?: Ref<HTMLButtonElement>;
+}) {
+ const localize = useLocalize();
const menu = Ariakit.useMenuStore({ placement: 'top-end' });
+ const descriptionPrefix = useId();
+ const preferencesLabelId = `${descriptionPrefix}-preferences`;
+ const [expandedInfoKey, setExpandedInfoKey] = useState<string | null>(null);
+ const renderEntry = (entry: MenuEntry) => {
+ const descriptionId = entry.info == null ? undefined : `${descriptionPrefix}-${entry.key}`;
+ const infoExpanded = entry.info != null && expandedInfoKey === entry.key;
+ const action = (
+ <Ariakit.MenuItem
+ className={MENU_ITEM_CLASS}
+ disabled={entry.disabled === true}
+ accessibleWhenDisabled
+ aria-describedby={descriptionId}
+ onClick={() => {
+ entry.onClick();
+ setExpandedInfoKey(null);
+ menu.hide();
+ }}
+ >
+ {entry.icon}
+ {entry.label}
+ </Ariakit.MenuItem>
+ );
+
+ if (entry.info == null || descriptionId == null) {
+ return <Fragment key={entry.key}>{action}</Fragment>;
+ }
+
+ return (
+ <div key={entry.key} role="none">
+ <div role="none" className="flex items-center gap-0.5">
+ <div role="none" className="min-w-0 flex-1">
+ {action}
+ </div>
+ <Ariakit.MenuItem
+ aria-label={`${localize('com_ui_more_info')}: ${entry.label}`}
+ aria-expanded={infoExpanded}
+ aria-controls={descriptionId}
+ hideOnClick={false}
+ className="flex size-8 shrink-0 items-center justify-center rounded-lg text-text-secondary hover:bg-surface-tertiary hover:text-text-primary focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-xheavy data-[active-item]:bg-surface-tertiary"
+ onClick={() =>
+ setExpandedInfoKey((current) => (current === entry.key ? null : entry.key))
+ }
+ >
+ <CircleHelp className="h-3.5 w-3.5" aria-hidden="true" />
+ </Ariakit.MenuItem>
+ </div>
+ <div
+ id={descriptionId}
+ className={cn(
+ infoExpanded
+ ? 'mx-2 mb-1 rounded-lg bg-surface-tertiary px-2 py-1.5 text-xs leading-relaxed text-text-secondary'
+ : 'sr-only',
+ )}
+ >
+ {entry.info}
+ </div>
+ </div>
+ );
+ };
return (
<>
- <Ariakit.MenuButton store={menu} aria-label={label} className={ICON_BTN_CLASS}>
+ <Ariakit.MenuButton
+ ref={buttonRef}
+ store={menu}
+ aria-label={label}
+ className={ICON_BTN_CLASS}
+ >
<MoreHorizontal className="h-4 w-4" aria-hidden="true" />
</Ariakit.MenuButton>
<Ariakit.Menu store={menu} portal gutter={6} className={MENU_CLASS}>
- {entries.map((entry) => (
- <Ariakit.MenuItem
- key={entry.key}
- className={MENU_ITEM_CLASS}
- onClick={() => {
- entry.onClick();
- menu.hide();
- }}
- >
- {entry.icon}
- {entry.label}
- </Ariakit.MenuItem>
- ))}
+ {entries.map(renderEntry)}
+ {preferences != null && preferences.length > 0 && (
+ <>
+ <div role="separator" className="mx-2 my-1 border-t border-border-light" />
+ <div role="group" aria-labelledby={preferencesLabelId}>
+ <div
+ id={preferencesLabelId}
+ className="px-2 pb-0.5 pt-1 text-[11px] font-medium uppercase tracking-wide text-text-secondary"
+ >
+ {localize('com_ui_preferences')}
+ </div>
+ {preferences.map(renderEntry)}
+ </div>
+ </>
+ )}
</Ariakit.Menu>
</>
);
}
+/**
+ * The always-visible escalation control on a waiting message: interrupt &
+ * steer it now, at the next safe token boundary. The tooltip teaches this
+ * action's OWN shortcut (registry-aware, so a rebinding shows correctly).
+ * Hover/focus marks this exact button as the active shortcut target; without
+ * an active row the shortcut retains its newest-waiting-message fallback.
+ */
+export function EscalateNowButton({
+ surface,
+ disabled,
+ messageText,
+ onClick,
+}: {
+ surface: 'bubble' | 'queued';
+ disabled: boolean;
+ messageText: string;
+ onClick: (event: React.MouseEvent<HTMLButtonElement>) => void;
+}) {
+ const localize = useLocalize();
+ const chord = useShortcutDisplay('escalateSteer');
+ const ariaKey = useShortcutAriaKey('escalateSteer');
+ const targetId = useId();
+ const activeTarget = useSyncExternalStore(
+ subscribeToActiveEscalateTarget,
+ getActiveEscalateTarget,
+ () => null,
+ );
+ const isActive = !disabled && activeTarget === targetId;
+ const label = localize('com_ui_interrupt_steer_now');
+ const normalizedMessageLabel = messageText.trim().replace(/\s+/g, ' ');
+ const messageCharacters = Array.from(normalizedMessageLabel);
+ const messageLabel =
+ messageCharacters.length > ESCALATION_MESSAGE_LABEL_MAX_LENGTH
+ ? `${messageCharacters
+ .slice(0, ESCALATION_MESSAGE_LABEL_MAX_LENGTH - 1)
+ .join('')
+ .trimEnd()}…`
+ : normalizedMessageLabel;
+ const accessibleLabel = messageLabel.length > 0 ? `${label}: ${messageLabel}` : label;
+
+ useEffect(() => {
+ if (disabled) {
+ clearActiveEscalateTarget(targetId);
+ }
+ return () => clearActiveEscalateTarget(targetId);
+ }, [disabled, targetId]);
+
+ return (
+ <>
+ {/* Thin divider binds the arrow to the message on its left, so it can
+ * never read as part of the bare-glyph menu control beside it — the
+ * send-now belongs to THIS chip, and the pairing repeats cleanly when
+ * several messages stack. */}
+ <span aria-hidden="true" className="h-[18px] w-px shrink-0 bg-border-medium" />
+ <Ariakit.TooltipProvider placement="top" timeout={300}>
+ <Ariakit.TooltipAnchor
+ render={
+ <button
+ type="button"
+ aria-label={accessibleLabel}
+ aria-keyshortcuts={isActive ? ariaKey : undefined}
+ data-escalate-steer={surface}
+ data-escalate-steer-active={isActive ? 'true' : undefined}
+ data-testid={surface === 'queued' ? 'queued-interrupt-now' : 'steer-escalate-now'}
+ disabled={disabled}
+ onPointerEnter={() =>
+ !disabled && updateActiveEscalateTarget('hover', targetId, true)
+ }
+ onPointerLeave={() => updateActiveEscalateTarget('hover', targetId, false)}
+ onFocus={() => !disabled && updateActiveEscalateTarget('focus', targetId, true)}
+ onBlur={() => updateActiveEscalateTarget('focus', targetId, false)}
+ onClick={onClick}
+ className={cn(
+ 'flex size-6 shrink-0 items-center justify-center rounded-full',
+ 'bg-text-primary text-surface-primary transition-opacity hover:opacity-85',
+ 'focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-xheavy',
+ 'disabled:cursor-not-allowed disabled:opacity-35 disabled:hover:opacity-35',
+ )}
+ >
+ <ArrowUp className="h-3.5 w-3.5" strokeWidth={2.4} aria-hidden="true" />
+ </button>
+ }
+ />
+ <Ariakit.Tooltip className="z-50 rounded-lg bg-surface-tertiary px-2 py-1 text-xs text-text-primary shadow-lg">
+ {chord && isActive ? `${label} · ${chord}` : label}
+ </Ariakit.Tooltip>
+ </Ariakit.TooltipProvider>
+ </>
+ );
+}
+
/**
* The overflow item that flips the Enter-during-run default. Shown as the
* OPPOSITE of the current default (the action you would switch to), matching
@@ -71,7 +298,49 @@ export function useDefaultToggleEntry(steering: SteeringControls): MenuEntry {
) : (
<Zap className="h-4 w-4 text-amber-500" aria-hidden="true" />
),
+ info: localize('com_nav_info_during_run_action'),
onClick: () => steering.setDefaultAction(next),
};
}, [steering, localize]);
}
+
+/**
+ * The overflow item that flips whether a default steer interrupts generation
+ * (`steerInterruptsByDefault`). Worded as the mode you would switch to, like
+ * the queue/steer toggle above.
+ */
+export function useInterruptToggleEntry(): MenuEntry {
+ const localize = useLocalize();
+ const [interrupts, setInterrupts] = useRecoilState(store.steerInterruptsByDefault);
+ const [defaultAction, setDefaultAction] = useRecoilState(store.duringRunDefaultAction);
+ const interruptsByDefault = defaultAction === 'steer' && interrupts;
+ return useMemo(
+ () => ({
+ key: 'toggle-interrupt',
+ label: interruptsByDefault
+ ? localize('com_ui_wait_for_tool_steps')
+ : localize('com_ui_always_interrupt'),
+ icon: interruptsByDefault ? (
+ <Zap className="h-4 w-4 text-amber-500" aria-hidden="true" />
+ ) : (
+ <ZapOff className="h-4 w-4 text-amber-500" aria-hidden="true" />
+ ),
+ info: localize(
+ !interruptsByDefault && defaultAction === 'queue'
+ ? 'com_ui_steer_interrupts_enable_info'
+ : 'com_ui_steer_interrupts_default_info',
+ ),
+ onClick: () => {
+ if (interruptsByDefault) {
+ setInterrupts(false);
+ return;
+ }
+ if (defaultAction === 'queue') {
+ setDefaultAction('steer');
+ }
+ setInterrupts(true);
+ },
+ }),
+ [defaultAction, interruptsByDefault, localize, setDefaultAction, setInterrupts],
+ );
+}
diff --git a/client/src/components/Chat/Input/__tests__/DuringRunSendButton.test.tsx b/client/src/components/Chat/Input/__tests__/DuringRunSendButton.test.tsx
index d4f1240e4b..677cf30a59 100644
--- a/client/src/components/Chat/Input/__tests__/DuringRunSendButton.test.tsx
+++ b/client/src/components/Chat/Input/__tests__/DuringRunSendButton.test.tsx
@@ -26,22 +26,30 @@ const TEXT = 'stop, do not run that command';
const mockInterruptSteer = jest.fn(() => true);
const mockSteerFromComposer = jest.fn(() => true);
+const mockQueueFromComposer = jest.fn(() => true);
+const mockInterruptAndSend = jest.fn(() => true);
const mockOnConsumed = jest.fn();
type StubOptions = {
pausedOnApproval?: boolean;
canSteer?: boolean;
+ canControlGeneration?: boolean;
};
-const steeringStub = ({ pausedOnApproval = false, canSteer = true }: StubOptions) =>
+const steeringStub = ({
+ pausedOnApproval = false,
+ canSteer = true,
+ canControlGeneration = true,
+}: StubOptions) =>
({
effectiveAction: canSteer ? 'steer' : 'queue',
canSteer,
+ canControlGeneration,
pausedOnApproval,
interruptSteer: mockInterruptSteer,
steerFromComposer: mockSteerFromComposer,
- queueFromComposer: jest.fn(() => true),
- interruptAndSend: jest.fn(() => true),
+ queueFromComposer: mockQueueFromComposer,
+ interruptAndSend: mockInterruptAndSend,
}) as unknown as SteeringControls;
function Harness({ steering }: { steering: SteeringControls }) {
@@ -126,6 +134,31 @@ describe('DuringRunSendButton — Interrupt & steer availability', () => {
expect(mockSteerFromComposer).not.toHaveBeenCalled();
});
+ test('keeps Queue live but disables every generation mutation before the epoch arrives', () => {
+ openMenu({ canSteer: false, canControlGeneration: false });
+
+ expect(screen.getByText('com_ui_steer').closest('button')).toHaveAttribute(
+ 'aria-disabled',
+ 'true',
+ );
+ expect(screen.getByText('com_ui_interrupt_steer').closest('button')).toHaveAttribute(
+ 'aria-disabled',
+ 'true',
+ );
+ expect(screen.getByText('com_ui_interrupt_send').closest('button')).toHaveAttribute(
+ 'aria-disabled',
+ 'true',
+ );
+
+ const queue = screen.getByText('com_ui_queue').closest('button') as HTMLButtonElement;
+ expect(queue).toHaveAttribute('aria-disabled', 'false');
+ fireEvent.click(queue);
+ expect(mockQueueFromComposer).toHaveBeenCalledWith(TEXT);
+ expect(mockOnConsumed).toHaveBeenCalled();
+ expect(mockInterruptSteer).not.toHaveBeenCalled();
+ expect(mockInterruptAndSend).not.toHaveBeenCalled();
+ });
+
test('both actions are available during a normal run', () => {
openMenu({ pausedOnApproval: false, canSteer: true });
diff --git a/client/src/components/Chat/Input/__tests__/InFlightSteers.test.tsx b/client/src/components/Chat/Input/__tests__/InFlightSteers.test.tsx
index 7ee3948505..30486fd3e7 100644
--- a/client/src/components/Chat/Input/__tests__/InFlightSteers.test.tsx
+++ b/client/src/components/Chat/Input/__tests__/InFlightSteers.test.tsx
@@ -1,10 +1,11 @@
import React from 'react';
-import { RecoilRoot } from 'recoil';
import { getDefaultStore } from 'jotai';
+import { RecoilRoot, useRecoilValue } from 'recoil';
import { render, screen, fireEvent, act } from '@testing-library/react';
+import type { PendingSteer, QueuedMessage } from '~/store/families';
import type { SteeringControls } from '~/hooks/Chat/useSteering';
-import type { PendingSteer } from '~/store/families';
-import { steerOverlayHeightFamily } from '~/store/steer';
+import { steerOverlayHeightFamily, escalatingSteerFamily } from '~/store/steer';
+import useSteerConvert from '~/hooks/Chat/useSteerConvert';
import InFlightSteers from '../InFlightSteers';
import store from '~/store';
@@ -12,8 +13,12 @@ const mockCancelMutateAsync = jest.fn();
const mockShowToast = jest.fn();
const mockQueueReclaimedSteer = jest.fn();
const mockRemoveSteer = jest.fn();
+const mockRetrySteer = jest.fn();
+const mockArmMutateAsync = jest.fn();
const mockSetDefaultAction = jest.fn();
const mockRestoreToComposer = jest.fn();
+let convertSteersForTest: ReturnType<typeof useSteerConvert>;
+let observedQueueForTest: QueuedMessage[];
jest.mock('~/hooks', () => ({
useLocalize: () => (key: string) => key,
@@ -23,10 +28,23 @@ jest.mock('~/hooks', () => ({
jest.mock('@librechat/client', () => ({
useToastContext: () => ({ showToast: mockShowToast }),
+ InfoHoverCard: () => null,
+ ESide: { Top: 'top', Bottom: 'bottom' },
}));
jest.mock('~/data-provider', () => ({
useCancelSteerMutation: () => ({ mutateAsync: mockCancelMutateAsync }),
+ useArmSteerMutation: () => ({ mutateAsync: mockArmMutateAsync }),
+ getGenerationProtocolVersion: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2
+ ? 2
+ : 1,
+ supportsGenerationProtocolV2: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2,
}));
jest.mock('~/components/Chat/Input/Files/FileContainer', () => ({
@@ -64,43 +82,69 @@ jest.mock('~/components/Chat/Messages/Content/MarkdownLite', () => ({
const CONVO_ID = 'convo-in-flight';
-const steeringStub = (defaultAction: 'steer' | 'queue' = 'steer') =>
+const steeringStub = (defaultAction: 'steer' | 'queue' = 'steer', duringRunActive = true) =>
({
defaultAction,
+ duringRunActive,
+ pausedOnApproval: false,
removeSteer: mockRemoveSteer,
+ retrySteer: mockRetrySteer,
setDefaultAction: mockSetDefaultAction,
queueReclaimedSteer: mockQueueReclaimedSteer,
}) as unknown as SteeringControls;
-function renderSteers(
- steers: PendingSteer[],
- options?: {
- enableUserMsgMarkdown?: boolean;
- appliedSteerIds?: string[];
- defaultAction?: 'steer' | 'queue';
- },
-) {
- return render(
+type RenderOptions = {
+ enableUserMsgMarkdown?: boolean;
+ appliedSteerIds?: string[];
+ defaultAction?: 'steer' | 'queue';
+ duringRunActive?: boolean;
+ activeGenerationCreatedAt?: number | null;
+ activeGenerationProtocolVersion?: 1 | 2;
+};
+
+/** Element builder shared by first render and rerenders — `initializeState`
+ * only applies on mount, so a rerender just swaps the steering controls. */
+function steersElement(steers: PendingSteer[], options?: RenderOptions) {
+ const SteerRaceProbe = () => {
+ convertSteersForTest = useSteerConvert();
+ observedQueueForTest = useRecoilValue(store.queuedMessagesByConvoId(CONVO_ID));
+ return null;
+ };
+ return (
<RecoilRoot
initializeState={({ set }) => {
set(store.pendingSteersByConvoId(CONVO_ID), steers);
+ const activeGenerationCreatedAt =
+ options?.activeGenerationCreatedAt === undefined ? 41 : options.activeGenerationCreatedAt;
if (options?.appliedSteerIds != null) {
set(store.appliedSteerIdsByConvoId(CONVO_ID), options.appliedSteerIds);
}
if (options?.enableUserMsgMarkdown != null) {
set(store.enableUserMsgMarkdown, options.enableUserMsgMarkdown);
}
+ if (activeGenerationCreatedAt != null) {
+ set(store.activeGenerationCreatedAtByConvoId(CONVO_ID), activeGenerationCreatedAt);
+ }
+ set(
+ store.activeGenerationProtocolVersionByConvoId(CONVO_ID),
+ options?.activeGenerationProtocolVersion ?? 2,
+ );
}}
>
+ <SteerRaceProbe />
<InFlightSteers
conversationId={CONVO_ID}
- steering={steeringStub(options?.defaultAction)}
+ steering={steeringStub(options?.defaultAction, options?.duringRunActive)}
onRestoreToComposer={mockRestoreToComposer}
/>
- </RecoilRoot>,
+ </RecoilRoot>
);
}
+function renderSteers(steers: PendingSteer[], options?: RenderOptions) {
+ return render(steersElement(steers, options));
+}
+
/** Opens a bubble's "…" menu and clicks one of its items, flushing the reclaim
* round-trip the action awaits before it re-homes the text. */
async function clickMenuItem(label: string) {
@@ -116,6 +160,7 @@ describe('InFlightSteers', () => {
jest.clearAllMocks();
mockCancelMutateAsync.mockResolvedValue({ removed: true });
mockRestoreToComposer.mockReturnValue(true);
+ observedQueueForTest = [];
});
it('renders nothing when no steer is in flight', () => {
@@ -171,11 +216,45 @@ describe('InFlightSteers', () => {
expect(mockCancelMutateAsync).toHaveBeenCalledWith({
conversationId: CONVO_ID,
steerId: 's-ack',
+ generationCreatedAt: 41,
});
await act(async () => {});
expect(screen.queryByText('waiting on boundary')).toBeNull();
});
+ it('fences cancel to the generation recorded on the steer', async () => {
+ renderSteers(
+ [
+ {
+ steerId: 's-epoch',
+ text: 'belongs to the old turn',
+ status: 'pending',
+ createdAt: 1,
+ generationCreatedAt: 41,
+ },
+ ],
+ { activeGenerationCreatedAt: 99 },
+ );
+ await clickMenuItem('com_ui_steer_cancel');
+
+ expect(mockCancelMutateAsync).toHaveBeenCalledWith({
+ conversationId: CONVO_ID,
+ steerId: 's-epoch',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('does not send a cancel before any generation epoch is known', async () => {
+ renderSteers(
+ [{ steerId: 's-no-epoch', text: 'wait for start', status: 'pending', createdAt: 1 }],
+ { activeGenerationCreatedAt: null },
+ );
+ await clickMenuItem('com_ui_steer_cancel');
+
+ expect(mockCancelMutateAsync).not.toHaveBeenCalled();
+ expect(screen.getByText('wait for start')).toBeInTheDocument();
+ });
+
it('hands the words back to the composer once the cancel reclaims them', async () => {
// Cancel is non-destructive: on a `reclaimed` outcome (removed:true) the
// steer never reached the run, so its words return to the composer (the
@@ -216,8 +295,11 @@ describe('InFlightSteers', () => {
await clickMenuItem('com_ui_steer_cancel');
await act(async () => {});
expect(mockRestoreToComposer).not.toHaveBeenCalled();
- // The chip still left optimistically; the events own the outcome.
+ // Receiptless v1 cannot distinguish an injected steer from a terminal
+ // conversion race, so keep the original metadata until an event proves
+ // which path won.
expect(mockCancelMutateAsync).toHaveBeenCalled();
+ expect(screen.getByText('too late')).toBeInTheDocument();
});
it('does not restore when the cancel POST fails', async () => {
@@ -239,19 +321,81 @@ describe('InFlightSteers', () => {
expect(screen.getByText('network flake')).toBeInTheDocument();
});
- it('does not restore a steer that settled while the cancel POST was in flight', async () => {
- // The run's final event converted this steer to a queued follow-up, which
- // stamps its id into the applied set. Restoring it on a failed cancel would
- // strand a stale entry that the NEXT run — a queue drain auto-sends one —
- // renders as an in-flight bubble beside that queued copy.
- mockCancelMutateAsync.mockRejectedValue(new Error('network'));
- renderSteers(
- [{ steerId: 's-settled', text: 'already queued', status: 'pending', createdAt: 1 }],
- { appliedSteerIds: ['s-settled'] },
+ it('keeps the terminal recovery copy when an in-flight cancel POST fails', async () => {
+ let rejectCancel: ((reason?: unknown) => void) | undefined;
+ mockCancelMutateAsync.mockReturnValue(
+ new Promise((_resolve, reject) => {
+ rejectCancel = reject;
+ }),
);
+ const steer = {
+ steerId: 's-settled',
+ clientSteerId: 'local-settled',
+ text: 'already queued',
+ status: 'pending' as const,
+ createdAt: 1,
+ };
+ renderSteers([steer]);
await clickMenuItem('com_ui_steer_cancel');
- await act(async () => {});
+
+ act(() => {
+ convertSteersForTest(CONVO_ID, [steer]);
+ });
+ expect(observedQueueForTest).toEqual([
+ expect.objectContaining({
+ id: steer.steerId,
+ recoverySteerId: steer.steerId,
+ recoveryClientSteerId: steer.clientSteerId,
+ }),
+ ]);
+
+ await act(async () => {
+ rejectCancel?.(new Error('network'));
+ await Promise.resolve();
+ });
expect(screen.queryByText('already queued')).toBeNull();
+ expect(observedQueueForTest).toHaveLength(1);
+ expect(mockRestoreToComposer).not.toHaveBeenCalled();
+ expect(mockQueueReclaimedSteer).not.toHaveBeenCalled();
+ });
+
+ it('keeps the terminal recovery copy when cancel reports an applied race', async () => {
+ let resolveCancel: ((value: { removed: boolean }) => void) | undefined;
+ mockCancelMutateAsync.mockReturnValue(
+ new Promise((resolve) => {
+ resolveCancel = resolve;
+ }),
+ );
+ const steer = {
+ steerId: 's-terminal-race',
+ clientSteerId: 'local-terminal-race',
+ text: 'terminal recovery wins',
+ status: 'pending' as const,
+ createdAt: 1,
+ };
+ renderSteers([steer]);
+ await clickMenuItem('com_ui_steer_cancel');
+
+ act(() => {
+ convertSteersForTest(CONVO_ID, [steer]);
+ });
+ expect(observedQueueForTest).toHaveLength(1);
+
+ await act(async () => {
+ resolveCancel?.({ removed: false });
+ await Promise.resolve();
+ });
+
+ expect(screen.queryByText(steer.text)).toBeNull();
+ expect(observedQueueForTest).toEqual([
+ expect.objectContaining({
+ id: steer.steerId,
+ recoverySteerId: steer.steerId,
+ recoveryClientSteerId: steer.clientSteerId,
+ }),
+ ]);
+ expect(mockRestoreToComposer).not.toHaveBeenCalled();
+ expect(mockQueueReclaimedSteer).not.toHaveBeenCalled();
});
it('reclaims a pending steer before queueing it for after the response', async () => {
@@ -269,6 +413,7 @@ describe('InFlightSteers', () => {
expect(mockCancelMutateAsync).toHaveBeenCalledWith({
conversationId: CONVO_ID,
steerId: 's-ack',
+ generationCreatedAt: 41,
});
// Routed through the shared conversion, which preserves the steer's id and
// createdAt so it drains ahead of a follow-up queued after it.
@@ -337,21 +482,36 @@ describe('InFlightSteers', () => {
);
});
- it('does not restore a steer a terminal conversion already queued', async () => {
- // The chip stays interactive during the reclaim round-trip, so a run that
- // ends or errors meanwhile converts it to a queued follow-up (stamping the
- // applied set). Restoring afterwards would leave one copy queued and
- // another in the composer draft.
- renderSteers([{ steerId: 's-ack', text: 'already queued', status: 'pending', createdAt: 1 }], {
- appliedSteerIds: ['s-ack'],
- });
+ it('removes a competing terminal recovery copy before restoring a confirmed reclaim', async () => {
+ let resolveCancel: ((value: { removed: boolean }) => void) | undefined;
+ mockCancelMutateAsync.mockReturnValue(
+ new Promise((resolve) => {
+ resolveCancel = resolve;
+ }),
+ );
+ const steer = {
+ steerId: 's-ack',
+ clientSteerId: 'local-ack',
+ text: 'already queued',
+ status: 'pending' as const,
+ createdAt: 1,
+ };
+ renderSteers([steer]);
await clickMenuItem('com_ui_edit_message');
- expect(mockRestoreToComposer).not.toHaveBeenCalled();
+ act(() => {
+ convertSteersForTest(CONVO_ID, [steer]);
+ });
+ expect(observedQueueForTest).toHaveLength(1);
+
+ await act(async () => {
+ resolveCancel?.({ removed: true });
+ await Promise.resolve();
+ });
+
+ expect(observedQueueForTest).toEqual([]);
+ expect(mockRestoreToComposer).toHaveBeenCalledWith(steer.text, undefined, {}, CONVO_ID);
expect(mockQueueReclaimedSteer).not.toHaveBeenCalled();
- expect(mockShowToast).toHaveBeenCalledWith(
- expect.objectContaining({ message: 'com_ui_steer_run_ended_queued' }),
- );
});
it('never re-homes a steer the server already applied', async () => {
@@ -576,3 +736,379 @@ describe('InFlightSteers', () => {
}
});
});
+
+/**
+ * The bubble's "Interrupt now" escalation is ONE atomic server op: `preempt`
+ * flips on the existing queued item, so its FIFO position, id, and timestamp
+ * survive and no reclaim window exists to race. `armed: false` is the honest
+ * answer for every "too late" interleaving (drained, cancelled, run ended or
+ * replaced) and for a deployment that cannot seal mid-stream.
+ */
+describe('InFlightSteers — interrupt-now escalation', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockArmMutateAsync.mockReset();
+ mockCancelMutateAsync.mockResolvedValue({ removed: true });
+ mockArmMutateAsync.mockResolvedValue({ armed: true, generationProtocolVersion: 2 });
+ mockRestoreToComposer.mockReturnValue(true);
+ act(() => {
+ getDefaultStore().set(escalatingSteerFamily(CONVO_ID), false);
+ });
+ });
+
+ it("arms the interrupt in place, keeping the steer's id and position", async () => {
+ renderSteers([{ steerId: 's1', text: 'hold on', status: 'pending', createdAt: 1 }]);
+ const armButton = screen.getByTestId('steer-escalate-now');
+ act(() => armButton.focus());
+ await act(async () => {
+ fireEvent.click(armButton);
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledWith({
+ conversationId: CONVO_ID,
+ steerId: 's1',
+ generationCreatedAt: 41,
+ });
+ /** No cancel, no resubmission: the durable item never left the queue. */
+ expect(mockCancelMutateAsync).not.toHaveBeenCalled();
+ expect(mockRetrySteer).not.toHaveBeenCalled();
+ expect(screen.getByText('hold on')).toBeInTheDocument();
+
+ /** The chip relabelled in place: an interrupting steer offers no further
+ * escalation, so its arrow control is gone. */
+ expect(screen.queryByTestId('steer-escalate-now')).toBeNull();
+ expect(document.activeElement).toBe(screen.getByLabelText('com_ui_more_options'));
+ expect(screen.getByRole('status')).toHaveTextContent('com_ui_steer_in_flight_preempt');
+ });
+
+ it('names each escalation control with the message it will interrupt for', () => {
+ renderSteers([
+ { steerId: 's1', text: 'correct the city', status: 'pending', createdAt: 1 },
+ { steerId: 's2', text: 'use metric units', status: 'pending', createdAt: 2 },
+ ]);
+
+ expect(
+ screen.getByRole('button', {
+ name: 'com_ui_interrupt_steer_now: correct the city',
+ }),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByRole('button', {
+ name: 'com_ui_interrupt_steer_now: use metric units',
+ }),
+ ).toBeInTheDocument();
+ });
+
+ it('bounds and normalizes the message suffix in an escalation control name', () => {
+ renderSteers([
+ {
+ steerId: 's-long-label',
+ text: ` one two\n${'x'.repeat(100)} `,
+ status: 'pending',
+ createdAt: 1,
+ },
+ ]);
+
+ expect(screen.getByTestId('steer-escalate-now')).toHaveAccessibleName(
+ `com_ui_interrupt_steer_now: one two ${'x'.repeat(71)}…`,
+ );
+ });
+
+ it('fences escalation to the generation recorded on the steer', async () => {
+ renderSteers(
+ [
+ {
+ steerId: 's-epoch',
+ text: 'old generation steer',
+ status: 'pending',
+ createdAt: 1,
+ generationCreatedAt: 41,
+ },
+ ],
+ { activeGenerationCreatedAt: 99 },
+ );
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledWith({
+ conversationId: CONVO_ID,
+ steerId: 's-epoch',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('does not arm an interrupt before any generation epoch is known', async () => {
+ renderSteers(
+ [{ steerId: 's-no-epoch', text: 'wait for start', status: 'pending', createdAt: 1 }],
+ { activeGenerationCreatedAt: null },
+ );
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).not.toHaveBeenCalled();
+ });
+
+ it('does not offer escalation on a steer that is already interrupting', async () => {
+ renderSteers([
+ { steerId: 's1', text: 'already sealing', status: 'pending', createdAt: 1, preempt: true },
+ ]);
+ expect(screen.queryByTestId('steer-escalate-now')).toBeNull();
+ fireEvent.click(screen.getByLabelText('com_ui_more_options'));
+ expect(await screen.findByText('com_ui_steer_cancel')).toBeInTheDocument();
+ });
+
+ it('keeps the chip an ordinary steer when the deployment cannot seal', async () => {
+ mockArmMutateAsync.mockResolvedValue({
+ armed: false,
+ code: 'PREEMPT_UNSUPPORTED',
+ generationProtocolVersion: 2,
+ });
+ renderSteers([{ steerId: 's1', text: 'no seal here', status: 'pending', createdAt: 1 }]);
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockShowToast).toHaveBeenCalledWith(
+ expect.objectContaining({ message: 'com_ui_steer_preempt_unsupported' }),
+ );
+ /** Still an ordinary steer: escalation stays offered. */
+ expect(screen.getByTestId('steer-escalate-now')).toBeEnabled();
+ });
+
+ it('stays neutral when the arm loses its race, whatever the reason', async () => {
+ /** `armed: false` covers injected, cancelled, re-homed, and run-over
+ * alike, so the toast must not claim one specific outcome. */
+ mockArmMutateAsync.mockResolvedValue({ armed: false, generationProtocolVersion: 2 });
+ renderSteers([{ steerId: 's1', text: 'too late', status: 'pending', createdAt: 1 }]);
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockRetrySteer).not.toHaveBeenCalled();
+ expect(mockShowToast).toHaveBeenCalledWith(
+ expect.objectContaining({ message: 'com_ui_steer_arm_lost_race' }),
+ );
+ });
+
+ it('retries an indeterminate arm and relabels after confirmation', async () => {
+ mockArmMutateAsync
+ .mockRejectedValueOnce(new Error('response lost'))
+ .mockResolvedValueOnce({ armed: true, generationProtocolVersion: 2 });
+ renderSteers([{ steerId: 's1', text: 'still queued', status: 'pending', createdAt: 1 }]);
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(2);
+ expect(mockShowToast).not.toHaveBeenCalled();
+ expect(screen.queryByTestId('steer-escalate-now')).toBeNull();
+ });
+
+ it.each([400, 403, 409])('does not retry a definitive HTTP %s rejection', async (status) => {
+ mockArmMutateAsync.mockRejectedValue({
+ response: { status, data: { code: 'RUN_PAUSED' } },
+ });
+ renderSteers([{ steerId: 's1', text: 'pause conflict', status: 'pending', createdAt: 1 }]);
+
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(1);
+ expect(mockShowToast).toHaveBeenCalledWith({
+ message: 'com_ui_steer_arm_lost_race',
+ status: 'info',
+ });
+ });
+
+ it('does not automatically retry an indeterminate arm on protocol v1', async () => {
+ mockArmMutateAsync.mockRejectedValue(new Error('response lost'));
+ renderSteers([{ steerId: 's1', text: 'legacy arm', status: 'pending', createdAt: 1 }], {
+ activeGenerationProtocolVersion: 1,
+ });
+
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(1);
+ expect(mockShowToast).toHaveBeenCalledWith(
+ expect.objectContaining({ message: 'com_ui_steer_arm_unconfirmed', status: 'warning' }),
+ );
+ });
+
+ it('reports an indeterminate result after two transport failures', async () => {
+ mockArmMutateAsync.mockRejectedValue(new Error('network'));
+ renderSteers([{ steerId: 's1', text: 'status unknown', status: 'pending', createdAt: 1 }]);
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(2);
+ expect(mockShowToast).toHaveBeenCalledWith(
+ expect.objectContaining({ message: 'com_ui_steer_arm_unconfirmed', status: 'warning' }),
+ );
+ expect(screen.getByText('status unknown')).toBeInTheDocument();
+ });
+
+ it('keeps a rejected-then-missing retry indeterminate', async () => {
+ mockArmMutateAsync
+ .mockRejectedValueOnce(new Error('response lost after commit'))
+ .mockResolvedValueOnce({ armed: false, generationProtocolVersion: 2 });
+ renderSteers([{ steerId: 's1', text: 'possibly armed', status: 'pending', createdAt: 1 }]);
+ await act(async () => {
+ fireEvent.click(screen.getByTestId('steer-escalate-now'));
+ });
+
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(2);
+ expect(mockShowToast).toHaveBeenCalledWith(
+ expect.objectContaining({ message: 'com_ui_steer_arm_unconfirmed', status: 'warning' }),
+ );
+ expect(screen.getByText('possibly armed')).toBeInTheDocument();
+ });
+
+ it('locks every escalation control while an arm request is in flight', async () => {
+ /** The chip-derived gate cannot see an arm until its response lands, so
+ * the flag flips synchronously at click — otherwise two bubbles could
+ * both arm on a slow connection, belying the disabled controls. */
+ let resolveArm: (value: {
+ armed: boolean;
+ generationProtocolVersion: number;
+ }) => void = () => {};
+ mockArmMutateAsync.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveArm = resolve;
+ }),
+ );
+ renderSteers([
+ { steerId: 's1', text: 'first', status: 'pending', createdAt: 1 },
+ { steerId: 's2', text: 'second', status: 'pending', createdAt: 2 },
+ ]);
+ const buttons = screen.getAllByTestId('steer-escalate-now');
+ fireEvent.click(buttons[0]);
+
+ expect(buttons[1]).toBeDisabled();
+ await act(async () => {
+ fireEvent.click(buttons[1]);
+ });
+
+ await act(async () => {
+ resolveArm({ armed: true, generationProtocolVersion: 2 });
+ });
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(1);
+ expect(mockArmMutateAsync).toHaveBeenCalledWith({
+ conversationId: CONVO_ID,
+ steerId: 's1',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('warns and unlocks when arm confirmation never settles', async () => {
+ jest.useFakeTimers();
+ try {
+ mockArmMutateAsync.mockImplementation(() => new Promise(() => undefined));
+ renderSteers([
+ { steerId: 's1', text: 'stalled arm', status: 'pending', createdAt: 1 },
+ { steerId: 's2', text: 'still available later', status: 'pending', createdAt: 2 },
+ ]);
+ const buttons = screen.getAllByTestId('steer-escalate-now');
+ fireEvent.click(buttons[0]);
+ expect(buttons[1]).toBeDisabled();
+
+ await act(async () => {
+ jest.advanceTimersByTime(10_000);
+ await Promise.resolve();
+ });
+
+ expect(mockShowToast).toHaveBeenCalledWith(
+ expect.objectContaining({ message: 'com_ui_steer_arm_unconfirmed', status: 'warning' }),
+ );
+ expect(buttons[1]).not.toBeDisabled();
+ expect(mockArmMutateAsync).toHaveBeenCalledTimes(1);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('disables escalation on every bubble while an interrupt is unresolved', async () => {
+ renderSteers([
+ { steerId: 's1', text: 'plain steer', status: 'pending', createdAt: 1 },
+ { steerId: 's2', text: 'sealing now', status: 'pending', createdAt: 2, preempt: true },
+ ]);
+ const button = screen.getByTestId('steer-escalate-now');
+ expect(button).toBeDisabled();
+
+ await act(async () => {
+ fireEvent.click(button);
+ });
+ expect(mockArmMutateAsync).not.toHaveBeenCalled();
+ });
+});
+
+/**
+ * Answer mode (`ask_user_question`) sets `duringRunActive` false while
+ * `pausedOnApproval` stays false (it only detects approval-bearing tool
+ * calls). The entry disables there as a UX gate; the server-side arm is the
+ * correctness backstop for states the client cannot see.
+ */
+describe('InFlightSteers — escalation while the run cannot accept a steer', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockArmMutateAsync.mockResolvedValue({ armed: true, generationProtocolVersion: 2 });
+ });
+
+ it('disables escalation in answer mode', async () => {
+ renderSteers([{ steerId: 's1', text: 'waiting', status: 'pending', createdAt: 1 }], {
+ duringRunActive: false,
+ });
+ const button = screen.getByTestId('steer-escalate-now');
+ expect(button).toBeDisabled();
+
+ await act(async () => {
+ fireEvent.click(button);
+ });
+ expect(mockArmMutateAsync).not.toHaveBeenCalled();
+ });
+});
+
+/**
+ * One-off message actions and sticky behavior changes must never read as the
+ * same kind of choice: actions first (edit, cancel, queue), then a separated
+ * "Preferences" section holding the two mode toggles. Escalation is not a
+ * menu entry at all — it has its own always-visible arrow control.
+ */
+describe('InFlightSteers — menu structure', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockCancelMutateAsync.mockResolvedValue({ removed: true });
+ });
+
+ it('orders actions above a separated Preferences section, with no escalation entry', async () => {
+ renderSteers([{ steerId: 's1', text: 'structured', status: 'pending', createdAt: 1 }]);
+ fireEvent.click(screen.getByLabelText('com_ui_more_options'));
+ await screen.findByText('com_ui_edit_message');
+
+ const items = screen
+ .getAllByRole('menuitem')
+ .filter((item) => !item.getAttribute('aria-label')?.startsWith('com_ui_more_info:'))
+ .map((item) => item.textContent);
+ expect(items).toEqual([
+ 'com_ui_edit_message',
+ 'com_ui_steer_cancel',
+ 'com_ui_convert_to_queue',
+ 'com_ui_turn_on_queueing',
+ 'com_ui_always_interrupt',
+ ]);
+ const preferences = screen.getByRole('group', { name: 'com_ui_preferences' });
+ expect(preferences).toContainElement(
+ screen.getByRole('menuitem', { name: 'com_ui_turn_on_queueing' }),
+ );
+ expect(preferences).toContainElement(
+ screen.getByRole('menuitem', { name: 'com_ui_always_interrupt' }),
+ );
+ expect(screen.queryByRole('menuitem', { name: 'com_ui_interrupt_steer_now' })).toBeNull();
+ });
+});
diff --git a/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx b/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx
index e3ebc72ec2..ff87b991b3 100644
--- a/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx
+++ b/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx
@@ -1,57 +1,177 @@
import React from 'react';
-import { RecoilRoot } from 'recoil';
-import { render, screen, fireEvent } from '@testing-library/react';
+import { getDefaultStore } from 'jotai';
+import userEvent from '@testing-library/user-event';
+import { RecoilRoot, useRecoilValue, useSetRecoilState } from 'recoil';
+import { render, screen, fireEvent, act, waitFor } from '@testing-library/react';
+import type { PendingSteer, QueuedMessage } from '~/store/families';
import type { SteeringControls } from '~/hooks/Chat/useSteering';
-import type { QueuedMessage } from '~/store/families';
+import { escalatingSteerFamily } from '~/store/steer';
import PendingSteerChips from '../PendingSteerChips';
import store from '~/store';
const mockRemoveQueued = jest.fn();
+const mockDiscardQueued = jest.fn(async (_message: QueuedMessage) => true);
+const mockSendQueuedNow = jest.fn();
+const mockRetrySteer = jest.fn();
const mockRestoreToComposer = jest.fn(() => true);
const mockEditToComposer = jest.fn();
+const mockShowToast = jest.fn();
+
+jest.mock('@librechat/client', () => ({
+ useToastContext: () => ({ showToast: mockShowToast }),
+}));
jest.mock('~/hooks', () => ({
useLocalize: () => (key: string) => key,
}));
const CONVO_ID = 'convo-q';
+let updateQueueForTest:
+ | ((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void)
+ | undefined;
+
+function QueueState() {
+ const queue = useRecoilValue(store.queuedMessagesByConvoId(CONVO_ID));
+ const setQueue = useSetRecoilState(store.queuedMessagesByConvoId(CONVO_ID));
+ updateQueueForTest = (updater) => setQueue(updater);
+ return <output data-testid="queue-state">{JSON.stringify(queue)}</output>;
+}
+
+function PreferenceState() {
+ const defaultAction = useRecoilValue(store.duringRunDefaultAction);
+ const interrupts = useRecoilValue(store.steerInterruptsByDefault);
+ return (
+ <>
+ <output data-testid="default-action-state">{defaultAction}</output>
+ <output data-testid="interrupts-state">{String(interrupts)}</output>
+ </>
+ );
+}
-const steeringStub = () =>
+const steeringStub = (overrides: Partial<SteeringControls> = {}) =>
({
queueKey: CONVO_ID,
defaultAction: 'steer',
duringRunActive: false,
+ canSendQueuedNow: true,
canSteer: false,
+ pausedOnApproval: false,
removeQueued: mockRemoveQueued,
- sendQueuedNow: jest.fn(),
+ discardQueued: mockDiscardQueued,
+ sendQueuedNow: mockSendQueuedNow,
+ retrySteer: mockRetrySteer,
setDefaultAction: jest.fn(),
+ ...overrides,
}) as unknown as SteeringControls;
-function renderChips(queued: QueuedMessage[]) {
+function renderChips(
+ queued: QueuedMessage[],
+ options?: {
+ steering?: Partial<SteeringControls>;
+ steers?: PendingSteer[];
+ interrupts?: boolean;
+ },
+) {
return render(
<RecoilRoot
initializeState={({ set }) => {
set(store.queuedMessagesByConvoId(CONVO_ID), queued);
+ set(store.duringRunDefaultAction, options?.steering?.defaultAction ?? 'steer');
+ set(store.steerInterruptsByDefault, options?.interrupts ?? false);
+ if (options?.steers != null) {
+ set(store.pendingSteersByConvoId(CONVO_ID), options.steers);
+ }
}}
>
<PendingSteerChips
conversationId={CONVO_ID}
- steering={steeringStub()}
+ steering={steeringStub(options?.steering)}
onEditToComposer={mockEditToComposer}
onRestoreToComposer={mockRestoreToComposer}
/>
+ <PreferenceState />
+ <QueueState />
</RecoilRoot>,
);
}
+describe('PendingSteerChips — ambiguous delivery retry', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it('hides same-id Retry for a legacy ambiguous delivery', () => {
+ renderChips([], {
+ steers: [
+ {
+ steerId: 'legacy-uncertain',
+ text: 'may already be queued',
+ status: 'failed',
+ deliveryUncertain: true,
+ generationProtocolVersion: 1,
+ createdAt: 1,
+ },
+ ],
+ });
+
+ expect(screen.queryByText('com_ui_steer_retry')).toBeNull();
+ });
+
+ it('offers same-id Retry only after protocol v2 was negotiated', () => {
+ renderChips([], {
+ steers: [
+ {
+ steerId: 'v2-uncertain',
+ text: 'dedupe this retry',
+ status: 'failed',
+ deliveryUncertain: true,
+ generationProtocolVersion: 2,
+ createdAt: 1,
+ },
+ ],
+ });
+
+ fireEvent.click(screen.getByText('com_ui_steer_retry'));
+ expect(mockRetrySteer).toHaveBeenCalledWith(
+ 'v2-uncertain',
+ 'dedupe this retry',
+ undefined,
+ { quotes: undefined, manualSkills: undefined },
+ { preempt: false, createdAt: 1, generationProtocolVersion: 2 },
+ );
+ });
+});
+
+describe('PendingSteerChips — queued primary availability', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it('does not offer Send now while answer mode owns the submission slot', () => {
+ renderChips([{ id: 'answer-queued', text: 'wait until the answer resumes', createdAt: 1 }], {
+ steering: {
+ duringRunActive: false,
+ canSendQueuedNow: false,
+ canSteer: false,
+ pausedOnApproval: true,
+ },
+ });
+
+ expect(screen.queryByText('com_ui_send_now')).toBeNull();
+ expect(screen.getByTestId('queued-interrupt-now')).toBeDisabled();
+ expect(mockSendQueuedNow).not.toHaveBeenCalled();
+ });
+});
+
describe('PendingSteerChips — queued trash', () => {
beforeEach(() => {
jest.clearAllMocks();
+ window.localStorage.clear();
});
- it('returns the words to the composer before removing a queued message', () => {
+ it('returns the words to the composer when removing a queued message', async () => {
// The trash is non-destructive: it hands the text (and its carried context)
- // to the gated restore first, so the words are not gone forever.
+ // to the gated restore once removal succeeds, so the words are not gone forever.
renderChips([
{
id: 'q1',
@@ -63,23 +183,361 @@ describe('PendingSteerChips — queued trash', () => {
]);
fireEvent.click(screen.getByLabelText('com_ui_remove_queued'));
- expect(mockRestoreToComposer).toHaveBeenCalledWith(
- 'later thought',
- undefined,
- { quotes: ['a quote'], manualSkills: ['a-skill'] },
- CONVO_ID,
- );
- expect(mockRemoveQueued).toHaveBeenCalledWith('q1');
+ await waitFor(() => {
+ expect(mockRestoreToComposer).toHaveBeenCalledWith(
+ 'later thought',
+ undefined,
+ { quotes: ['a quote'], manualSkills: ['a-skill'] },
+ CONVO_ID,
+ );
+ expect(mockRemoveQueued).toHaveBeenCalledWith('q1');
+ });
});
- it('still removes the message even when the composer refuses the restore', () => {
+ it('still removes the message even when the composer refuses the restore', async () => {
// Occupied composer / other chat: the gated restore returns false, but the
// trash must reliably remove either way.
mockRestoreToComposer.mockReturnValueOnce(false);
renderChips([{ id: 'q2', text: 'drop me', createdAt: 1 }]);
fireEvent.click(screen.getByLabelText('com_ui_remove_queued'));
- expect(mockRestoreToComposer).toHaveBeenCalled();
- expect(mockRemoveQueued).toHaveBeenCalledWith('q2');
+ await waitFor(() => {
+ expect(mockRestoreToComposer).toHaveBeenCalled();
+ expect(mockRemoveQueued).toHaveBeenCalledWith('q2');
+ });
+ });
+
+ it('discards a recovered source before restoring and removing its queued row', async () => {
+ let confirmDiscard: ((discarded: boolean) => void) | undefined;
+ mockDiscardQueued.mockImplementationOnce(
+ () =>
+ new Promise<boolean>((resolve) => {
+ confirmDiscard = resolve;
+ }),
+ );
+ const recovered = {
+ id: 'q-recovered',
+ text: 'recovered words',
+ createdAt: 1,
+ recoverySteerId: 'server-source',
+ recoveryClientSteerId: 'client-source',
+ };
+ renderChips([recovered]);
+
+ fireEvent.click(screen.getByLabelText('com_ui_remove_queued'));
+ expect(mockDiscardQueued).toHaveBeenCalledWith(recovered);
+ expect(mockRestoreToComposer).not.toHaveBeenCalled();
+
+ await act(async () => {
+ confirmDiscard?.(true);
+ });
+ expect(mockRestoreToComposer).toHaveBeenCalledWith(
+ 'recovered words',
+ undefined,
+ { quotes: undefined, manualSkills: undefined },
+ CONVO_ID,
+ );
+ expect(mockRemoveQueued).toHaveBeenCalledWith('q-recovered');
+ });
+
+ it('offers Edit for a recovered row and leaves it untouched when discard is refused', async () => {
+ const user = userEvent.setup();
+ mockDiscardQueued.mockResolvedValueOnce(false);
+ const recovered = {
+ id: 'q-recovered',
+ text: 'safe to edit only after discard',
+ createdAt: 1,
+ recoverySteerId: 'server-source',
+ recoveryClientSteerId: 'client-source',
+ };
+ renderChips([recovered]);
+
+ await user.click(screen.getByLabelText('com_ui_more_options'));
+ await user.click(await screen.findByRole('menuitem', { name: 'com_ui_edit_message' }));
+
+ await waitFor(() => expect(mockDiscardQueued).toHaveBeenCalledWith(recovered));
+ expect(mockRestoreToComposer).not.toHaveBeenCalled();
+ expect(mockEditToComposer).not.toHaveBeenCalled();
+ expect(mockRemoveQueued).not.toHaveBeenCalled();
+ expect(screen.getByText('safe to edit only after discard')).toBeInTheDocument();
+ });
+
+ it('uses the guarded composer restore only after recovered Edit confirms discard', async () => {
+ const user = userEvent.setup();
+ let confirmDiscard: ((discarded: boolean) => void) | undefined;
+ mockDiscardQueued.mockImplementationOnce(
+ () =>
+ new Promise<boolean>((resolve) => {
+ confirmDiscard = resolve;
+ }),
+ );
+ const recovered = {
+ id: 'q-recovered-edit',
+ text: 'edit recovered words',
+ createdAt: 1,
+ recoverySteerId: 'server-source',
+ recoveryClientSteerId: 'client-source',
+ quotes: ['kept quote'],
+ manualSkills: ['kept skill'],
+ };
+ renderChips([recovered]);
+
+ await user.click(screen.getByLabelText('com_ui_more_options'));
+ await user.click(await screen.findByRole('menuitem', { name: 'com_ui_edit_message' }));
+ expect(mockEditToComposer).not.toHaveBeenCalled();
+
+ await act(async () => {
+ confirmDiscard?.(true);
+ });
+ expect(mockRestoreToComposer).toHaveBeenCalledWith(
+ 'edit recovered words',
+ undefined,
+ { quotes: ['kept quote'], manualSkills: ['kept skill'] },
+ CONVO_ID,
+ );
+ expect(mockEditToComposer).not.toHaveBeenCalled();
+ expect(mockRemoveQueued).toHaveBeenCalledWith('q-recovered-edit');
+ });
+
+ it('keeps a safely discarded Edit queued when live composer state changes during cancel', async () => {
+ const user = userEvent.setup();
+ let confirmDiscard: (() => void) | undefined;
+ const recovered = {
+ id: 'q-recovered-race',
+ text: 'preserve after async race',
+ createdAt: 1,
+ clientRequestId: 'recovery-attempt',
+ recoverySteerId: 'server-source',
+ recoveryClientSteerId: 'client-source',
+ quotes: ['kept quote'],
+ };
+ mockDiscardQueued.mockImplementationOnce(
+ () =>
+ new Promise<boolean>((resolve) => {
+ confirmDiscard = () => {
+ updateQueueForTest?.((current) =>
+ current.map((item) => {
+ if (item.id !== recovered.id) {
+ return item;
+ }
+ const {
+ clientRequestId: _clientRequestId,
+ recoverySteerId: _recoverySteerId,
+ recoveryClientSteerId: _recoveryClientSteerId,
+ ...ordinary
+ } = item;
+ return ordinary;
+ }),
+ );
+ resolve(true);
+ };
+ }),
+ );
+ renderChips([recovered]);
+
+ await user.click(screen.getByLabelText('com_ui_more_options'));
+ await user.click(await screen.findByRole('menuitem', { name: 'com_ui_edit_message' }));
+ expect(mockDiscardQueued).toHaveBeenCalledWith(recovered);
+ expect(mockRestoreToComposer).not.toHaveBeenCalled();
+
+ // The live guard can change while the durable cancel request is pending.
+ mockRestoreToComposer.mockReturnValueOnce(false);
+ await act(async () => {
+ confirmDiscard?.();
+ });
+
+ expect(mockRestoreToComposer).toHaveBeenCalledWith(
+ 'preserve after async race',
+ undefined,
+ { quotes: ['kept quote'], manualSkills: undefined },
+ CONVO_ID,
+ );
+ expect(mockEditToComposer).not.toHaveBeenCalled();
+ expect(mockRemoveQueued).not.toHaveBeenCalled();
+ expect(screen.getByText('preserve after async race')).toBeInTheDocument();
+ expect(JSON.parse(screen.getByTestId('queue-state').textContent ?? 'null')).toEqual([
+ {
+ id: 'q-recovered-race',
+ text: 'preserve after async race',
+ createdAt: 1,
+ quotes: ['kept quote'],
+ },
+ ]);
+ expect(mockShowToast).toHaveBeenCalledWith({
+ message: 'com_ui_steer_edit_queued',
+ status: 'info',
+ });
+ });
+});
+
+/**
+ * The queued row's escalation: interrupt & steer this one message now, at the
+ * next safe token boundary, instead of waiting for the run to end. Offered
+ * only while a live run can accept a steer; disabled while another interrupt
+ * is unresolved or the run is paused on approval, since a second preempt
+ * would race the same seal (or draw the server's 409).
+ */
+describe('PendingSteerChips — queued interrupt-now', () => {
+ const liveRun = { duringRunActive: true, canSteer: true };
+ const queuedMessage: QueuedMessage = { id: 'q1', text: 'urgent one', createdAt: 1 };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ window.localStorage.clear();
+ });
+
+ it('escalates a queued message as an interrupt steer', () => {
+ renderChips([queuedMessage], { steering: liveRun });
+ fireEvent.click(screen.getByTestId('queued-interrupt-now'));
+
+ expect(mockSendQueuedNow).toHaveBeenCalledWith(expect.objectContaining({ id: 'q1' }), {
+ preempt: true,
+ });
});
+
+ it('offers no escalation outside a steerable run', () => {
+ renderChips([queuedMessage]);
+ expect(screen.queryByTestId('queued-interrupt-now')).toBeNull();
+ });
+
+ it('disables escalation while another interrupt is unresolved', () => {
+ renderChips([queuedMessage], {
+ steering: liveRun,
+ steers: [{ steerId: 'p1', text: 'sealing', status: 'pending', createdAt: 1, preempt: true }],
+ });
+ const button = screen.getByTestId('queued-interrupt-now');
+ expect(button).toBeDisabled();
+
+ fireEvent.click(button);
+ expect(mockSendQueuedNow).not.toHaveBeenCalled();
+ });
+
+ it('disables escalation while a bubble arm request is in flight', () => {
+ const jotai = getDefaultStore();
+ act(() => {
+ jotai.set(escalatingSteerFamily(CONVO_ID), true);
+ });
+ try {
+ renderChips([queuedMessage], { steering: liveRun });
+ expect(screen.getByTestId('queued-interrupt-now')).toBeDisabled();
+ } finally {
+ act(() => {
+ jotai.set(escalatingSteerFamily(CONVO_ID), false);
+ });
+ }
+ });
+
+ it('stays visible but disabled while the run is paused on approval', () => {
+ /** The real hook invariant: `canSteer = hasRealConvoId && !pausedOnApproval`,
+ * so a paused run always reads `canSteer: false`. The control must remain
+ * discoverable-but-disabled there, not vanish with the primary. */
+ renderChips([queuedMessage], {
+ steering: { duringRunActive: true, canSteer: false, pausedOnApproval: true },
+ });
+ const button = screen.getByTestId('queued-interrupt-now');
+ expect(button).toBeDisabled();
+
+ fireEvent.click(button);
+ expect(mockSendQueuedNow).not.toHaveBeenCalled();
+ });
+
+ it('stays visible but disabled while ask-user answer mode owns the composer', () => {
+ renderChips([queuedMessage], {
+ steering: { duringRunActive: false, canSteer: false, pausedOnApproval: true },
+ });
+ const button = screen.getByTestId('queued-interrupt-now');
+ expect(button).toBeDisabled();
+
+ fireEvent.click(button);
+ expect(mockSendQueuedNow).not.toHaveBeenCalled();
+ });
+
+ it('makes the hovered row the only advertised shortcut target', () => {
+ renderChips([queuedMessage, { id: 'q2', text: 'urgent two', createdAt: 2 }], {
+ steering: liveRun,
+ });
+ const [first, second] = screen.getAllByTestId('queued-interrupt-now');
+
+ expect(first).not.toHaveAttribute('aria-keyshortcuts');
+ expect(second).not.toHaveAttribute('aria-keyshortcuts');
+
+ fireEvent.pointerEnter(first);
+ expect(first).toHaveAttribute('data-escalate-steer-active', 'true');
+ expect(first).toHaveAttribute('aria-keyshortcuts');
+ expect(second).not.toHaveAttribute('data-escalate-steer-active');
+ expect(second).not.toHaveAttribute('aria-keyshortcuts');
+
+ fireEvent.pointerEnter(second);
+ expect(first).not.toHaveAttribute('data-escalate-steer-active');
+ expect(first).not.toHaveAttribute('aria-keyshortcuts');
+ expect(second).toHaveAttribute('data-escalate-steer-active', 'true');
+ expect(second).toHaveAttribute('aria-keyshortcuts');
+ });
+
+ it('names each queued escalation control with its target message', () => {
+ renderChips([queuedMessage, { id: 'q2', text: 'urgent two', createdAt: 2 }], {
+ steering: liveRun,
+ });
+
+ expect(
+ screen.getByRole('button', { name: 'com_ui_interrupt_steer_now: urgent one' }),
+ ).toBeInTheDocument();
+ expect(
+ screen.getByRole('button', { name: 'com_ui_interrupt_steer_now: urgent two' }),
+ ).toBeInTheDocument();
+ });
+
+ it.each(['pointer', 'keyboard'] as const)(
+ 'keeps preference help visibly open after %s activation',
+ async (activation) => {
+ const user = userEvent.setup();
+ renderChips([queuedMessage], { steering: liveRun });
+ const menuButton = screen.getByLabelText('com_ui_more_options');
+ await user.click(menuButton);
+
+ const preference = await screen.findByRole('menuitem', {
+ name: 'com_ui_turn_on_queueing',
+ });
+ expect(preference).toHaveAccessibleDescription('com_nav_info_during_run_action');
+ const help = screen.getByRole('menuitem', {
+ name: 'com_ui_more_info: com_ui_turn_on_queueing',
+ });
+
+ if (activation === 'pointer') {
+ await user.click(help);
+ } else {
+ act(() => help.focus());
+ expect(document.activeElement).toBe(help);
+ await user.keyboard('{Enter}');
+ }
+
+ const menu = screen.getByRole('menu');
+ const description = screen.getByText('com_nav_info_during_run_action');
+ expect(menuButton).toHaveAttribute('aria-expanded', 'true');
+ expect(menu).toBeVisible();
+ expect(help).toHaveAttribute('aria-expanded', 'true');
+ expect(description).not.toHaveClass('sr-only');
+ expect(description).toBeVisible();
+ },
+ );
+
+ it.each([false, true])(
+ 'makes steering the default when enabling always-interrupt from queue mode (latent=%s)',
+ async (interrupts) => {
+ renderChips([queuedMessage], {
+ steering: { ...liveRun, defaultAction: 'queue' },
+ interrupts,
+ });
+ fireEvent.click(screen.getByLabelText('com_ui_more_options'));
+ const preference = await screen.findByRole('menuitem', { name: 'com_ui_always_interrupt' });
+ expect(preference).toHaveAccessibleDescription('com_ui_steer_interrupts_enable_info');
+ fireEvent.click(preference);
+
+ expect(screen.getByTestId('default-action-state')).toHaveTextContent('steer');
+ expect(screen.getByTestId('interrupts-state')).toHaveTextContent('true');
+
+ fireEvent.click(screen.getByLabelText('com_ui_more_options'));
+ expect(await screen.findByText('com_ui_wait_for_tool_steps')).toBeInTheDocument();
+ },
+ );
});
diff --git a/client/src/components/Chat/Messages/Content/ApprovalContext.tsx b/client/src/components/Chat/Messages/Content/ApprovalContext.tsx
index 8fa66614da..fa3e4382de 100644
--- a/client/src/components/Chat/Messages/Content/ApprovalContext.tsx
+++ b/client/src/components/Chat/Messages/Content/ApprovalContext.tsx
@@ -1,6 +1,6 @@
import { createContext, useCallback, useContext, useMemo, useRef, useState } from 'react';
-import { atom, useRecoilState } from 'recoil';
import { Constants } from 'librechat-data-provider';
+import { atom, useRecoilState, useRecoilValue } from 'recoil';
import type { Agents } from 'librechat-data-provider';
import {
useSubmitToolApprovalMutation,
@@ -10,6 +10,7 @@ import {
import { resolveAskUserQuestionPart } from '~/utils/approval';
import { ChatContext } from '~/Providers/ChatContext';
import { useGetEphemeralAgent } from '~/store/agents';
+import store from '~/store';
/** Per-action submission lifecycle, surfaced to the cards so they can disable
* controls and explain a terminal outcome. */
@@ -284,17 +285,26 @@ export function useResumeSubmit() {
/** React state cannot lock a second click in the same browser task. Keep a
* synchronous action-id guard alongside the rendered submission status. */
const submittingToolActionIdsRef = useRef(new Set<string>());
+ const submittingAskActionIdsRef = useRef(new Set<string>());
/** Ask status lives in Recoil so it works from the composer (outside the
* provider); tool-approval status stays on the context. */
const { setAskStatus } = useAskSubmitStatus();
+ const activeGenerationCreatedAt = useRecoilValue(
+ store.activeGenerationCreatedAtByConvoId(conversation?.conversationId ?? Constants.NEW_CONVO),
+ );
const buildResumeFields = useCallback((): ResumeAgentFields | null => {
const conversationId = conversation?.conversationId;
- if (!conversationId || conversationId === Constants.NEW_CONVO) {
+ if (
+ !conversationId ||
+ conversationId === Constants.NEW_CONVO ||
+ activeGenerationCreatedAt == null
+ ) {
return null;
}
return {
conversationId,
+ generationCreatedAt: activeGenerationCreatedAt,
endpoint: conversation?.endpoint,
endpointType: conversation?.endpointType,
agent_id: conversation?.agent_id,
@@ -305,7 +315,7 @@ export function useResumeSubmit() {
promptPrefix: conversation?.promptPrefix,
ephemeralAgent: getEphemeralAgent(conversationId),
};
- }, [conversation, getEphemeralAgent]);
+ }, [conversation, getEphemeralAgent, activeGenerationCreatedAt]);
const submitToolApproval = useCallback(
(actionId: string) => {
@@ -343,6 +353,10 @@ export function useResumeSubmit() {
if (!fields || answer.length === 0) {
return;
}
+ if (submittingAskActionIdsRef.current.has(actionId)) {
+ return;
+ }
+ submittingAskActionIdsRef.current.add(actionId);
setAskStatus(actionId, 'submitting');
askMutation.mutate(
{ ...fields, actionId, answer },
@@ -379,7 +393,13 @@ export function useResumeSubmit() {
*/
opts?.onSuccess?.();
},
- onError: (error) => setAskStatus(actionId, isExpiredError(error) ? 'expired' : 'error'),
+ onError: (error) => {
+ const expired = isExpiredError(error);
+ if (!expired) {
+ submittingAskActionIdsRef.current.delete(actionId);
+ }
+ setAskStatus(actionId, expired ? 'expired' : 'error');
+ },
},
);
},
diff --git a/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx b/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx
index 2ce2ae93bd..78477b3b72 100644
--- a/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx
+++ b/client/src/components/Chat/Messages/Content/__tests__/ApprovalContext.test.tsx
@@ -1,8 +1,10 @@
import React from 'react';
import { RecoilRoot } from 'recoil';
import { act, renderHook } from '@testing-library/react';
+import type { MutableSnapshot } from 'recoil';
import ApprovalProvider, { useApprovalContext, useResumeSubmit } from '../ApprovalContext';
import { ChatContext } from '~/Providers/ChatContext';
+import store from '~/store';
const mockApprovalMutate = jest.fn();
const mockAskMutate = jest.fn();
@@ -24,15 +26,26 @@ const chatContextValue = {
},
} as unknown as React.ContextType<typeof ChatContext>;
-function wrapper({ children }: { children: React.ReactNode }) {
- return (
- <RecoilRoot>
- <ChatContext.Provider value={chatContextValue}>
- <ApprovalProvider>{children}</ApprovalProvider>
- </ChatContext.Provider>
- </RecoilRoot>
- );
-}
+const createWrapper = (generationCreatedAt: number | null) =>
+ function ResumeWrapper({ children }: { children: React.ReactNode }) {
+ const initializeState = (snapshot: MutableSnapshot) => {
+ if (generationCreatedAt != null) {
+ snapshot.set(
+ store.activeGenerationCreatedAtByConvoId('conversation-1'),
+ generationCreatedAt,
+ );
+ }
+ };
+ return (
+ <RecoilRoot initializeState={initializeState}>
+ <ChatContext.Provider value={chatContextValue}>
+ <ApprovalProvider>{children}</ApprovalProvider>
+ </ChatContext.Provider>
+ </RecoilRoot>
+ );
+ };
+
+const wrapper = createWrapper(1000);
describe('useResumeSubmit', () => {
beforeEach(() => {
@@ -61,6 +74,14 @@ describe('useResumeSubmit', () => {
result.current.resume.submitToolApproval('action-1');
});
expect(mockApprovalMutate).toHaveBeenCalledTimes(1);
+ expect(mockApprovalMutate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ conversationId: 'conversation-1',
+ generationCreatedAt: 1000,
+ actionId: 'action-1',
+ }),
+ expect.any(Object),
+ );
const firstOptions = mockApprovalMutate.mock.calls[0][1] as {
onError: (error: unknown) => void;
@@ -73,4 +94,57 @@ describe('useResumeSubmit', () => {
});
expect(mockApprovalMutate).toHaveBeenCalledTimes(2);
});
+
+ test('keeps approval and ask-answer resume controls inert before the generation epoch exists', () => {
+ const { result } = renderHook(
+ () => ({
+ approval: useApprovalContext(),
+ resume: useResumeSubmit(),
+ }),
+ { wrapper: createWrapper(null) },
+ );
+
+ act(() => {
+ result.current.approval.registerToolCall('action-1', 'call-1');
+ result.current.approval.setDecision('action-1', 'call-1', {
+ tool_call_id: 'call-1',
+ decision: 'approve',
+ });
+ result.current.resume.submitToolApproval('action-1');
+ result.current.resume.submitAskAnswer('ask-1', 'answer');
+ });
+
+ expect(mockApprovalMutate).not.toHaveBeenCalled();
+ expect(mockAskMutate).not.toHaveBeenCalled();
+ });
+
+ test('synchronously deduplicates ask answers and unlocks a retryable error', () => {
+ const { result } = renderHook(() => useResumeSubmit(), { wrapper });
+
+ act(() => {
+ result.current.submitAskAnswer('ask-1', 'answer');
+ result.current.submitAskAnswer('ask-1', 'answer');
+ });
+ expect(mockAskMutate).toHaveBeenCalledTimes(1);
+ expect(mockAskMutate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ conversationId: 'conversation-1',
+ generationCreatedAt: 1000,
+ actionId: 'ask-1',
+ answer: 'answer',
+ }),
+ expect.any(Object),
+ );
+
+ const firstOptions = mockAskMutate.mock.calls[0][1] as {
+ onError: (error: unknown) => void;
+ };
+ act(() => firstOptions.onError(new Error('temporary failure')));
+
+ act(() => {
+ result.current.submitAskAnswer('ask-1', 'answer');
+ result.current.submitAskAnswer('ask-1', 'answer');
+ });
+ expect(mockAskMutate).toHaveBeenCalledTimes(2);
+ });
});
diff --git a/client/src/data-provider/SSE/__tests__/protocol.test.ts b/client/src/data-provider/SSE/__tests__/protocol.test.ts
new file mode 100644
index 0000000000..fc8e5a6664
--- /dev/null
+++ b/client/src/data-provider/SSE/__tests__/protocol.test.ts
@@ -0,0 +1,77 @@
+const mockAuthenticatedFetch = jest.fn();
+
+jest.mock('librechat-data-provider', () => ({
+ request: {
+ authenticatedFetch: (...args: unknown[]) => mockAuthenticatedFetch(...args),
+ },
+}));
+
+import {
+ GENERATION_PROTOCOL_HEADER,
+ getGenerationProtocolVersion,
+ postGenerationRequest,
+ supportsGenerationProtocolV2,
+} from '../protocol';
+
+const response = (body: unknown, status = 200): Response =>
+ ({
+ ok: status >= 200 && status < 300,
+ status,
+ headers: new Headers({ 'Retry-After': '3' }),
+ text: jest.fn().mockResolvedValue(body == null ? '' : JSON.stringify(body)),
+ }) as unknown as Response;
+
+describe('generation protocol negotiation', () => {
+ beforeEach(() => {
+ mockAuthenticatedFetch.mockReset();
+ });
+
+ it.each([
+ [{ generationProtocolVersion: 2 }, 2],
+ [{ generationProtocolVersion: 1 }, 1],
+ [{ generationProtocolVersion: '2' }, 1],
+ [{}, 1],
+ [undefined, 1],
+ ] as const)('fails closed for protocol echo %#', (value, expected) => {
+ expect(getGenerationProtocolVersion(value)).toBe(expected);
+ expect(supportsGenerationProtocolV2(value)).toBe(expected === 2);
+ });
+
+ it('advertises v2 in both the request header and JSON body', async () => {
+ mockAuthenticatedFetch.mockResolvedValueOnce(
+ response({ streamId: 'stream-1', generationProtocolVersion: 2 }),
+ );
+
+ await expect(postGenerationRequest('/api/agents/chat', { text: 'hello' })).resolves.toEqual({
+ streamId: 'stream-1',
+ generationProtocolVersion: 2,
+ });
+
+ expect(mockAuthenticatedFetch).toHaveBeenCalledWith(
+ '/api/agents/chat',
+ expect.objectContaining({
+ method: 'POST',
+ headers: expect.objectContaining({
+ 'Content-Type': 'application/json',
+ [GENERATION_PROTOCOL_HEADER]: '2',
+ }),
+ body: JSON.stringify({ text: 'hello', generationProtocolVersion: 2 }),
+ }),
+ );
+ });
+
+ it('keeps Axios-compatible status, body, and headers for existing error callers', async () => {
+ mockAuthenticatedFetch.mockResolvedValueOnce(
+ response({ code: 'SERVER_NOT_READY', message: 'warming up' }, 503),
+ );
+
+ await expect(postGenerationRequest('/api/agents/chat', {})).rejects.toMatchObject({
+ message: 'warming up',
+ response: {
+ status: 503,
+ data: { code: 'SERVER_NOT_READY', message: 'warming up' },
+ headers: { 'retry-after': '3' },
+ },
+ });
+ });
+});
diff --git a/client/src/data-provider/SSE/__tests__/queries.test.ts b/client/src/data-provider/SSE/__tests__/queries.test.ts
index d1e4526512..dce1e86b57 100644
--- a/client/src/data-provider/SSE/__tests__/queries.test.ts
+++ b/client/src/data-provider/SSE/__tests__/queries.test.ts
@@ -35,7 +35,20 @@ jest.mock('../../Endpoints', () => ({
useGetStartupConfig: jest.fn(() => ({ data: undefined })),
}));
-import { genTitleQueryKey, queueTitleGeneration } from '../queries';
+import { request } from 'librechat-data-provider';
+import { fetchStreamStatus, genTitleQueryKey, queueTitleGeneration } from '../queries';
+
+describe('fetchStreamStatus generation protocol advertisement', () => {
+ it('sends v2 in both the query and header', async () => {
+ (request.get as jest.Mock).mockResolvedValueOnce({ active: false });
+
+ await expect(fetchStreamStatus('conversation-1')).resolves.toEqual({ active: false });
+ expect(request.get).toHaveBeenCalledWith(
+ '/api/agents/chat/status/conversation-1?generationProtocolVersion=2',
+ { headers: { 'X-LibreChat-Generation-Protocol': '2' } },
+ );
+ });
+});
/** Build a minimal Axios-shaped error with a given HTTP status. */
function makeAxiosError(status: number): Error {
diff --git a/client/src/data-provider/SSE/index.ts b/client/src/data-provider/SSE/index.ts
index d0720956a0..5074ce40b0 100644
--- a/client/src/data-provider/SSE/index.ts
+++ b/client/src/data-provider/SSE/index.ts
@@ -1,2 +1,3 @@
export * from './queries';
export * from './mutations';
+export * from './protocol';
diff --git a/client/src/data-provider/SSE/mutations.ts b/client/src/data-provider/SSE/mutations.ts
index 0f6b3d0afd..d6bb83667e 100644
--- a/client/src/data-provider/SSE/mutations.ts
+++ b/client/src/data-provider/SSE/mutations.ts
@@ -1,18 +1,31 @@
import { useMutation } from '@tanstack/react-query';
-import { apiBaseUrl, request, EModelEndpoint } from 'librechat-data-provider';
+import { apiBaseUrl, EModelEndpoint } from 'librechat-data-provider';
import type { Agents, TMessage, TEphemeralAgent, TPendingSteer } from 'librechat-data-provider';
+import { postGenerationRequest } from './protocol';
export interface AbortStreamParams {
/** The stream ID to abort (if known) */
streamId?: string;
/** The conversation ID to abort (backend will look up the job) */
conversationId?: string;
+ /** Exact generation epoch returned by start/status. Prevents a stale tab
+ * from aborting a newer turn that reused the conversation-scoped stream id. */
+ generationCreatedAt?: number;
}
export interface AbortStreamResponse {
+ generationProtocolVersion?: number;
success: boolean;
aborted?: string;
+ /** Stop lost to an ordinary terminal transition; no abort FINAL was emitted. */
+ settled?: boolean;
+ code?: 'RUN_ALREADY_SETTLED' | string;
+ streamId?: string;
+ terminalStatus?: 'complete' | 'error' | 'aborted';
error?: string;
+ /** The worker stopped, but its terminal state could not be made durable.
+ * The client must not infer a safe queue-drain boundary from this ACK. */
+ persistenceFailed?: boolean;
/** Steers that never reached an injection boundary; restored as queued chips. */
pendingSteers?: TPendingSteer[];
}
@@ -26,10 +39,10 @@ export interface AbortStreamResponse {
*/
export const abortStream = async (params: AbortStreamParams): Promise<AbortStreamResponse> => {
console.log('[abortStream] Calling abort endpoint with params:', params);
- const result = (await request.post(
+ const result = await postGenerationRequest<AbortStreamResponse>(
`${apiBaseUrl()}/api/agents/chat/abort`,
params,
- )) as AbortStreamResponse;
+ );
console.log('[abortStream] Abort response:', result);
return result;
};
@@ -52,6 +65,8 @@ export function useAbortStreamMutation() {
*/
export interface ResumeAgentFields {
conversationId: string;
+ /** Exact paused generation being resumed. */
+ generationCreatedAt: number;
endpoint?: EModelEndpoint | string | null;
endpointType?: EModelEndpoint | string | null;
agent_id?: string | null;
@@ -66,6 +81,7 @@ export interface ResumeAgentFields {
/** Successful resume ACK. The continuation streams over the existing SSE. */
export interface ResumeResponse {
+ generationProtocolVersion?: number;
streamId: string;
conversationId: string;
status: 'resuming';
@@ -97,11 +113,11 @@ export const submitToolApproval = async (
params: SubmitToolApprovalParams,
): Promise<ResumeResponse> => {
const { actionId, decisions, ...fields } = params;
- return request.post(`${apiBaseUrl()}/api/agents/chat/resume`, {
+ return postGenerationRequest<ResumeResponse>(`${apiBaseUrl()}/api/agents/chat/resume`, {
...buildResumeBase(fields),
actionId,
decisions,
- }) as Promise<ResumeResponse>;
+ });
};
/**
@@ -126,11 +142,11 @@ export interface SubmitAskAnswerParams extends ResumeAgentFields {
*/
export const submitAskAnswer = async (params: SubmitAskAnswerParams): Promise<ResumeResponse> => {
const { actionId, answer, ...fields } = params;
- return request.post(`${apiBaseUrl()}/api/agents/chat/resume`, {
+ return postGenerationRequest<ResumeResponse>(`${apiBaseUrl()}/api/agents/chat/resume`, {
...buildResumeBase(fields),
actionId,
answer,
- }) as Promise<ResumeResponse>;
+ });
};
/**
@@ -145,6 +161,10 @@ export function useSubmitAskAnswerMutation() {
export interface SteerMessageParams {
conversationId: string;
+ /** Exact live generation being steered. */
+ generationCreatedAt?: number;
+ /** Correlates terminal events that can arrive before this POST's response. */
+ clientSteerId?: string;
text: string;
/** Attachment refs steered with the message (already uploaded). */
files?: TMessage['files'];
@@ -159,12 +179,19 @@ export interface SteerMessageParams {
/** Successful steer ACK: the server queued the message for mid-run injection. */
export interface SteerMessageResponse {
+ generationProtocolVersion?: number;
status: 'queued';
steerId: string;
position: number;
conversationId: string;
/** Whether the seal request was actually armed; see {@link SteerMessageParams.preempt}. */
preempt?: boolean;
+ preemptRevision?: number;
+ /** Receipt replay after this item already left the durable queue. */
+ settled?: boolean;
+ /** Settled specifically by terminal drain; restore as a queued follow-up. */
+ leftover?: boolean;
+ replayed?: boolean;
}
/**
@@ -176,10 +203,10 @@ export interface SteerMessageResponse {
* send, RUN_PAUSED / STEER_UNSUPPORTED → client-side queue).
*/
export const steerMessage = async (params: SteerMessageParams): Promise<SteerMessageResponse> => {
- return request.post(
+ return postGenerationRequest<SteerMessageResponse>(
`${apiBaseUrl()}/api/agents/chat/steer`,
params,
- ) as Promise<SteerMessageResponse>;
+ );
};
/** React Query mutation hook for steering; the injection arrives on the SSE. */
@@ -192,9 +219,13 @@ export function useSteerMessageMutation() {
export interface CancelSteerParams {
conversationId: string;
steerId: string;
+ clientSteerId?: string;
+ /** Exact generation that owns this steer receipt. */
+ generationCreatedAt?: number;
}
export interface CancelSteerResponse {
+ generationProtocolVersion?: number;
removed?: boolean;
}
@@ -206,10 +237,10 @@ export interface CancelSteerResponse {
export const cancelSteerMessage = async (
params: CancelSteerParams,
): Promise<CancelSteerResponse> => {
- return request.post(
+ return postGenerationRequest<CancelSteerResponse>(
`${apiBaseUrl()}/api/agents/chat/steer/cancel`,
params,
- ) as Promise<CancelSteerResponse>;
+ );
};
export function useCancelSteerMutation() {
@@ -217,3 +248,36 @@ export function useCancelSteerMutation() {
mutationFn: cancelSteerMessage,
});
}
+
+export interface ArmSteerParams {
+ conversationId: string;
+ steerId: string;
+ /** Exact generation that owns this steer receipt. */
+ generationCreatedAt?: number;
+}
+
+export interface ArmSteerResponse {
+ generationProtocolVersion?: number;
+ armed?: boolean;
+ code?: string;
+ preemptRevision?: number;
+}
+
+/**
+ * Escalates a still-queued steer to an interrupt IN PLACE: the server flips
+ * `preempt` on the existing item, so its FIFO position, id, and timestamp all
+ * survive. `armed: false` is not an error — the steer already injected, was
+ * cancelled, or the deployment cannot seal mid-stream (`PREEMPT_UNSUPPORTED`).
+ */
+export const armSteerMessage = async (params: ArmSteerParams): Promise<ArmSteerResponse> => {
+ return postGenerationRequest<ArmSteerResponse>(
+ `${apiBaseUrl()}/api/agents/chat/steer/arm`,
+ params,
+ );
+};
+
+export function useArmSteerMutation() {
+ return useMutation({
+ mutationFn: armSteerMessage,
+ });
+}
diff --git a/client/src/data-provider/SSE/protocol.ts b/client/src/data-provider/SSE/protocol.ts
new file mode 100644
index 0000000000..1fdb70cae2
--- /dev/null
+++ b/client/src/data-provider/SSE/protocol.ts
@@ -0,0 +1,122 @@
+import { request } from 'librechat-data-provider';
+
+export const GENERATION_PROTOCOL_VERSION = 2 as const;
+export const GENERATION_PROTOCOL_HEADER = 'X-LibreChat-Generation-Protocol';
+
+export type GenerationProtocolVersion = 1 | typeof GENERATION_PROTOCOL_VERSION;
+
+/**
+ * Protocol negotiation is deliberately fail-closed: only an exact numeric
+ * echo of `2` enables v2 behavior. Old servers, stripped fields, strings, and
+ * future versions all remain on the legacy path until the client understands
+ * their contract.
+ */
+export const getGenerationProtocolVersion = (value: unknown): GenerationProtocolVersion => {
+ if (
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion ===
+ GENERATION_PROTOCOL_VERSION
+ ) {
+ return GENERATION_PROTOCOL_VERSION;
+ }
+ return 1;
+};
+
+export const supportsGenerationProtocolV2 = (value: unknown): boolean =>
+ getGenerationProtocolVersion(value) === GENERATION_PROTOCOL_VERSION;
+
+export const generationProtocolHeaders = (): Record<string, string> => ({
+ [GENERATION_PROTOCOL_HEADER]: String(GENERATION_PROTOCOL_VERSION),
+});
+
+export const withGenerationProtocolQuery = (url: string): string =>
+ `${url}${url.includes('?') ? '&' : '?'}generationProtocolVersion=${GENERATION_PROTOCOL_VERSION}`;
+
+const withGenerationProtocolBody = (body: unknown): unknown => {
+ if (body == null || typeof body !== 'object' || Array.isArray(body)) {
+ return body;
+ }
+ return {
+ ...(body as Record<string, unknown>),
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+};
+
+type GenerationRequestError = Error & {
+ code?: string;
+ response?: {
+ status: number;
+ data: unknown;
+ headers: Record<string, string>;
+ };
+};
+
+const parseResponseBody = (text: string): unknown => {
+ if (!text) {
+ return undefined;
+ }
+ try {
+ return JSON.parse(text);
+ } catch {
+ return text;
+ }
+};
+
+const responseHeaders = (headers: Headers): Record<string, string> => {
+ const result: Record<string, string> = {};
+ headers.forEach((value, key) => {
+ result[key] = value;
+ });
+ return result;
+};
+
+/**
+ * JSON POST used by generation-control routes. `authenticatedFetch` preserves
+ * these custom headers when it transparently refreshes a 401 token; the small
+ * Axios-shaped error wrapper keeps existing mutation/retry callers compatible.
+ */
+export async function postGenerationRequest<T>(
+ url: string,
+ body: unknown,
+ options?: { signal?: AbortSignal },
+): Promise<T> {
+ let response: Response;
+ try {
+ response = await request.authenticatedFetch(url, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ ...generationProtocolHeaders(),
+ },
+ body: JSON.stringify(withGenerationProtocolBody(body)),
+ signal: options?.signal,
+ });
+ } catch (error) {
+ /** Fetch does not expose Axios' `ERR_NETWORK`, which the start retry loop
+ * already uses to distinguish an ambiguous transport failure. */
+ if (error instanceof Error && error.name !== 'AbortError') {
+ (error as GenerationRequestError).code ??= 'ERR_NETWORK';
+ }
+ throw error;
+ }
+
+ const data = parseResponseBody(await response.text());
+ if (!response.ok) {
+ const message =
+ data != null &&
+ typeof data === 'object' &&
+ typeof (data as { message?: unknown }).message === 'string'
+ ? (data as { message: string }).message
+ : `Generation request failed with status ${response.status}`;
+ const error = new Error(message) as GenerationRequestError;
+ error.response = {
+ status: response.status,
+ data,
+ headers: responseHeaders(response.headers),
+ };
+ throw error;
+ }
+
+ return data as T;
+}
diff --git a/client/src/data-provider/SSE/queries.ts b/client/src/data-provider/SSE/queries.ts
index 3c20391547..e701e1f248 100644
--- a/client/src/data-provider/SSE/queries.ts
+++ b/client/src/data-provider/SSE/queries.ts
@@ -2,11 +2,17 @@ import { useEffect, useMemo, useState } from 'react';
import { useQuery, useQueries, useQueryClient } from '@tanstack/react-query';
import { apiBaseUrl, QueryKeys, request, dataService } from 'librechat-data-provider';
import type { Agents, TConversation, TPendingSteer } from 'librechat-data-provider';
+import { generationProtocolHeaders, withGenerationProtocolQuery } from './protocol';
import { isNotFoundError, updateConvoInAllQueries } from '~/utils';
import { useGetStartupConfig } from '../Endpoints';
export interface StreamStatusResponse {
+ /** Exact protocol selected by the job. Missing/1 is always legacy. */
+ generationProtocolVersion?: number;
active: boolean;
+ /** Client-local cache marker used to force useResumeOnLoad to replace a
+ * stale generation attachment with the newer epoch occupying this stream. */
+ generationHandoff?: boolean;
streamId?: string;
status?: 'running' | 'complete' | 'error' | 'aborted' | 'requires_action';
aggregatedContent?: Array<{ type: string; text?: string }>;
@@ -16,7 +22,8 @@ export interface StreamStatusResponse {
* `resumeState.pendingAction`, surfaced top-level for the resume-on-load path. */
pendingAction?: Agents.PendingAction;
/** Acknowledged steers a terminal drain parked because no subscriber was
- * live for the final/abort event — claim-on-read; restore as queued chips. */
+ * live for the final/abort event. Reads are replayable until an exact
+ * recovery turn persists or the user discards the item. */
unrecoveredSteers?: TPendingSteer[];
}
@@ -24,7 +31,8 @@ export const streamStatusQueryKey = (conversationId: string) => ['streamStatus',
export const fetchStreamStatus = async (conversationId: string): Promise<StreamStatusResponse> => {
return request.get<StreamStatusResponse>(
- `${apiBaseUrl()}/api/agents/chat/status/${conversationId}`,
+ withGenerationProtocolQuery(`${apiBaseUrl()}/api/agents/chat/status/${conversationId}`),
+ { headers: generationProtocolHeaders() },
);
};
diff --git a/client/src/hooks/Chat/__tests__/abort.spec.tsx b/client/src/hooks/Chat/__tests__/abort.spec.tsx
index 45d481e493..6a7a501430 100644
--- a/client/src/hooks/Chat/__tests__/abort.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/abort.spec.tsx
@@ -4,7 +4,7 @@ import { act, renderHook, waitFor } from '@testing-library/react';
import { RecoilRoot, useSetRecoilState, useRecoilValue } from 'recoil';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import type { TSubmission, TConversation } from 'librechat-data-provider';
-import type { RunEnd } from '~/store/families';
+import type { DrainAfterAbort, RunEnd } from '~/store/families';
import useChatHelpers from '../useChatHelpers';
import { useAbortCleanup } from '../abort';
import store from '~/store';
@@ -14,6 +14,10 @@ const mockConvertSteersToQueued = jest.fn();
jest.mock('~/data-provider', () => ({
useAbortStreamMutation: () => ({ mutateAsync: mockAbortMutateAsync }),
+ supportsGenerationProtocolV2: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2,
}));
jest.mock('~/hooks/Messages/useLatestMessage', () => ({
@@ -167,11 +171,17 @@ describe('useAbortCleanup', () => {
});
describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
- const observed: { runEnd: RunEnd | null } = { runEnd: null };
+ const observed: { runEnd: RunEnd | null; drainAfterAbort: DrainAfterAbort | false } = {
+ runEnd: null,
+ drainAfterAbort: false,
+ };
- function setupStop(conversationId: string) {
+ function setupStop(conversationId: string, generationCreatedAt: number | null = 41) {
+ let setDrainAfterAbort: ((value: DrainAfterAbort | false) => void) | undefined;
function RunEndProbe() {
observed.runEnd = useRecoilValue(store.runEndByIndex(INDEX));
+ observed.drainAfterAbort = useRecoilValue(store.drainAfterAbortByIndex(INDEX));
+ setDrainAfterAbort = useSetRecoilState(store.drainAfterAbortByIndex(INDEX));
return null;
}
const queryClient = new QueryClient();
@@ -184,7 +194,14 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
endpoint: 'agents',
} as TConversation);
// Arm interrupt & send so the abort response writes the drain signal.
- set(store.drainAfterAbortByIndex(INDEX), true);
+ set(store.drainAfterAbortByIndex(INDEX), {
+ conversationId,
+ generationCreatedAt: 41,
+ });
+ if (generationCreatedAt != null) {
+ set(store.activeGenerationCreatedAtByConvoId(conversationId), generationCreatedAt);
+ set(store.activeGenerationProtocolVersionByConvoId(conversationId), 2);
+ }
}}
>
<RunEndProbe />
@@ -192,11 +209,15 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
</RecoilRoot>
</QueryClientProvider>
);
- return renderHook(() => useChatHelpers(INDEX), { wrapper });
+ return {
+ ...renderHook(() => useChatHelpers(INDEX), { wrapper }),
+ setDrainAfterAbort: (value: DrainAfterAbort | false) => setDrainAfterAbort?.(value),
+ };
}
beforeEach(() => {
observed.runEnd = null;
+ observed.drainAfterAbort = false;
mockAbortMutateAsync.mockReset();
mockConvertSteersToQueued.mockReset();
jest.spyOn(console, 'log').mockImplementation(() => undefined);
@@ -212,6 +233,7 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
success: true,
aborted: 'convo-resolved',
pendingSteers,
+ generationProtocolVersion: 2,
});
const { result } = setupStop(String(Constants.NEW_CONVO));
@@ -221,6 +243,7 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
expect(mockAbortMutateAsync).toHaveBeenCalledWith({
conversationId: String(Constants.NEW_CONVO),
+ generationCreatedAt: 41,
});
// The user is still on /c/new (nothing navigated), so the chips and the
// drain signal key under NEW_CONVO — but the parked server copy lives
@@ -228,11 +251,16 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
expect(mockConvertSteersToQueued).toHaveBeenCalledWith(
String(Constants.NEW_CONVO),
pendingSteers,
- { claimParked: true, claimConversationId: 'convo-resolved' },
+ {
+ claimParked: true,
+ claimConversationId: 'convo-resolved',
+ generationProtocolVersion: 2,
+ },
);
expect(observed.runEnd).toMatchObject({
conversationId: String(Constants.NEW_CONVO),
outcome: 'aborted',
+ generationCreatedAt: 41,
});
});
@@ -242,6 +270,7 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
success: true,
aborted: 'convo-resolved',
pendingSteers,
+ generationProtocolVersion: 2,
});
const { result } = setupStop('convo-held');
@@ -252,15 +281,173 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
expect(mockConvertSteersToQueued).toHaveBeenCalledWith('convo-resolved', pendingSteers, {
claimParked: true,
claimConversationId: 'convo-resolved',
+ generationProtocolVersion: 2,
});
expect(observed.runEnd).toMatchObject({
conversationId: 'convo-resolved',
outcome: 'aborted',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('fences abort to the attached generation epoch', async () => {
+ mockAbortMutateAsync.mockResolvedValue({ success: true, aborted: 'convo-held' });
+
+ const { result } = setupStop('convo-held', 41);
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(mockAbortMutateAsync).toHaveBeenCalledWith({
+ conversationId: 'convo-held',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('does not arm a run-end drain when abort targets a replaced generation', async () => {
+ mockAbortMutateAsync.mockRejectedValue({
+ response: {
+ status: 409,
+ data: { code: 'RUN_REPLACED', generationProtocolVersion: 2 },
+ },
+ });
+
+ const { result } = setupStop('convo-held', 41);
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(observed.runEnd).toBeNull();
+ expect(mockConvertSteersToQueued).not.toHaveBeenCalled();
+ });
+
+ it('keeps Stop inert before the generation epoch is known', async () => {
+ const { result } = setupStop('convo-held', null);
+
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(mockAbortMutateAsync).not.toHaveBeenCalled();
+ expect(mockConvertSteersToQueued).not.toHaveBeenCalled();
+ expect(observed.runEnd).toBeNull();
+ expect(observed.drainAfterAbort).toEqual({
+ conversationId: 'convo-held',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('waits for SSE reconciliation when abort persistence fails', async () => {
+ mockAbortMutateAsync.mockResolvedValue({
+ success: true,
+ aborted: 'convo-held',
+ persistenceFailed: true,
+ generationProtocolVersion: 2,
+ });
+ const { result } = setupStop('convo-held');
+
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(mockAbortMutateAsync).toHaveBeenCalledWith({
+ conversationId: 'convo-held',
+ generationCreatedAt: 41,
+ });
+ expect(mockConvertSteersToQueued).not.toHaveBeenCalled();
+ expect(observed.runEnd).toBeNull();
+ expect(observed.drainAfterAbort).toBe(false);
+ });
+
+ it('does not release the abort-drain queue when natural completion already settled the run', async () => {
+ mockAbortMutateAsync.mockResolvedValue({
+ success: false,
+ settled: true,
+ code: 'RUN_ALREADY_SETTLED',
+ streamId: 'convo-held',
+ terminalStatus: 'complete',
+ generationProtocolVersion: 2,
+ });
+ const { result } = setupStop('convo-held');
+
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(mockConvertSteersToQueued).not.toHaveBeenCalled();
+ expect(observed.runEnd).toBeNull();
+ expect(observed.drainAfterAbort).toBe(false);
+ });
+
+ it('does not let a late abort response clear another conversation epoch', async () => {
+ let resolveAbort: ((value: unknown) => void) | undefined;
+ mockAbortMutateAsync.mockReturnValue(
+ new Promise((resolve) => {
+ resolveAbort = resolve;
+ }),
+ );
+ const { result, setDrainAfterAbort } = setupStop('convo-a');
+
+ let stopPromise: Promise<void> | undefined;
+ act(() => {
+ stopPromise = result.current.stopGenerating();
+ });
+ act(() => {
+ setDrainAfterAbort({ conversationId: 'convo-b', generationCreatedAt: 22 });
+ });
+ await act(async () => {
+ resolveAbort?.({
+ success: false,
+ settled: true,
+ code: 'RUN_ALREADY_SETTLED',
+ generationProtocolVersion: 2,
+ });
+ await stopPromise;
+ });
+
+ expect(observed.drainAfterAbort).toEqual({
+ conversationId: 'convo-b',
+ generationCreatedAt: 22,
+ });
+ });
+
+ it('does not release the armed queue for an unknown abort outcome', async () => {
+ mockAbortMutateAsync.mockRejectedValue({ response: { status: 503 } });
+ const { result } = setupStop('convo-held');
+
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(mockConvertSteersToQueued).not.toHaveBeenCalled();
+ expect(observed.runEnd).toBeNull();
+ expect(observed.drainAfterAbort).toEqual({
+ conversationId: 'convo-held',
+ generationCreatedAt: 41,
+ });
+ });
+
+ it('releases the armed queue only when abort authoritatively returns 404', async () => {
+ mockAbortMutateAsync.mockRejectedValue({ response: { status: 404 } });
+ const { result } = setupStop('convo-held');
+
+ await act(async () => {
+ await result.current.stopGenerating();
+ });
+
+ expect(observed.runEnd).toMatchObject({
+ conversationId: 'convo-held',
+ outcome: 'aborted',
+ generationCreatedAt: 41,
});
});
it('falls back to the client-held id when the response carries no resolved id', async () => {
- mockAbortMutateAsync.mockResolvedValue({ success: true, pendingSteers: [] });
+ mockAbortMutateAsync.mockResolvedValue({
+ success: true,
+ pendingSteers: [],
+ generationProtocolVersion: 2,
+ });
const { result } = setupStop('convo-held');
await act(async () => {
@@ -270,6 +457,7 @@ describe('useChatHelpers stopGenerating (abort steer targeting)', () => {
expect(mockConvertSteersToQueued).toHaveBeenCalledWith('convo-held', [], {
claimParked: true,
claimConversationId: 'convo-held',
+ generationProtocolVersion: 2,
});
expect(observed.runEnd).toMatchObject({ conversationId: 'convo-held', outcome: 'aborted' });
});
diff --git a/client/src/hooks/Chat/__tests__/useChatFunctions.regenerate.spec.tsx b/client/src/hooks/Chat/__tests__/useChatFunctions.regenerate.spec.tsx
index ebf46e138d..d984692c57 100644
--- a/client/src/hooks/Chat/__tests__/useChatFunctions.regenerate.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/useChatFunctions.regenerate.spec.tsx
@@ -95,15 +95,23 @@ const conversation = (conversationId: string) =>
agent_id: 'agent-1',
}) as TConversation;
-function renderAsk(messages: TMessage[] | undefined, conversationId = 'conversation-1') {
+function renderAsk(
+ messages: TMessage[] | undefined,
+ conversationId = 'conversation-1',
+ options: { endpoint?: TConversation['endpoint']; isSubmitting?: boolean } = {},
+) {
const setMessages = jest.fn();
const setSubmission = jest.fn();
const getMessages = jest.fn(() => messages);
+ const immutableConversation = conversation(conversationId);
+ if ('endpoint' in options) {
+ immutableConversation.endpoint = options.endpoint ?? null;
+ }
const hook = renderHook(() =>
useChatFunctions({
- isSubmitting: false,
+ isSubmitting: options.isSubmitting ?? false,
latestMessage: messages?.at(-1) ?? null,
- conversation: conversation(conversationId),
+ conversation: immutableConversation,
getMessages,
setMessages,
setSubmission,
@@ -136,6 +144,55 @@ describe('useChatFunctions ask', () => {
);
});
+ it('synchronously reports a refusal while another submit is in flight', () => {
+ const { result, setMessages, setSubmission } = renderAsk([], 'conversation-1', {
+ isSubmitting: true,
+ });
+
+ let askResult: ReturnType<typeof result.current.ask>;
+ act(() => {
+ askResult = result.current.ask({ text: 'queued follow-up' });
+ });
+
+ expect(askResult!).toBe(false);
+ expect(setMessages).not.toHaveBeenCalled();
+ expect(setSubmission).not.toHaveBeenCalled();
+ expect(mockSetShowStopButton).not.toHaveBeenCalled();
+ });
+
+ it('reports a refusal when no endpoint is available', () => {
+ const { result, setMessages, setSubmission } = renderAsk([], 'conversation-1', {
+ endpoint: null,
+ });
+
+ let askResult: ReturnType<typeof result.current.ask>;
+ act(() => {
+ askResult = result.current.ask({ text: 'queued follow-up' });
+ });
+
+ expect(askResult!).toBe(false);
+ expect(setMessages).not.toHaveBeenCalled();
+ expect(setSubmission).not.toHaveBeenCalled();
+ expect(mockSetShowStopButton).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ['empty text', { text: ' ' }, undefined],
+ ['the search view', { text: 'queued follow-up', conversationId: 'search' }, undefined],
+ ['a continue without a latest message', { text: 'continue' }, { isContinued: true }],
+ ])('reports a refusal for %s', (_label, props, askOptions) => {
+ const { result, setMessages, setSubmission } = renderAsk([]);
+
+ let askResult: ReturnType<typeof result.current.ask>;
+ act(() => {
+ askResult = result.current.ask(props, askOptions);
+ });
+
+ expect(askResult!).toBe(false);
+ expect(setMessages).not.toHaveBeenCalled();
+ expect(setSubmission).not.toHaveBeenCalled();
+ });
+
it('allows an existing conversation whose loaded history is empty', () => {
const { result, setMessages, setSubmission } = renderAsk([]);
diff --git a/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx b/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx
index c33b3f979e..7a3f68d099 100644
--- a/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx
@@ -3,7 +3,7 @@ import { Constants } from 'librechat-data-provider';
import { act, renderHook, waitFor } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { RecoilRoot, useRecoilValue, useSetRecoilState, type MutableSnapshot } from 'recoil';
-import type { RunEnd, QueuedMessage } from '~/store/families';
+import type { DrainAfterAbort, RunEnd, QueuedMessage } from '~/store/families';
import useQueueDrain from '../useQueueDrain';
import store from '~/store';
@@ -26,7 +26,7 @@ function setup(
setIsSubmitting?: (value: boolean) => void;
setQueue?: (value: { id: string; text: string; createdAt: number }[]) => void;
setNewConvoQueue?: (value: { id: string; text: string; createdAt: number }[]) => void;
- setInterruptFlag?: (value: boolean) => void;
+ setInterruptFlag?: (value: DrainAfterAbort | false) => void;
queueRef?: { current: { id: string; text: string; createdAt: number }[] };
} = {};
@@ -59,7 +59,12 @@ function setup(
return { ask, setters };
}
-const emptyOverrides = { overrideFiles: [], overrideQuotes: [], overrideManualSkills: [] };
+const emptyOverrides = expect.objectContaining({
+ overrideFiles: [],
+ overrideQuotes: [],
+ overrideManualSkills: [],
+ overrideQueuedMessageOrigin: expect.any(Object),
+});
const queuedMessage = (id: string, text: string) => ({ id, text, createdAt: Date.now() });
@@ -67,6 +72,7 @@ const runEnd = (overrides: Partial<RunEnd> = {}): RunEnd => ({
conversationId: CONVO_ID,
outcome: 'completed',
endedAt: Date.now(),
+ generationCreatedAt: 41,
...overrides,
});
@@ -84,7 +90,7 @@ describe('useQueueDrain', () => {
});
act(() => {
- setters.setRunEnd!(runEnd());
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 1234 }));
});
await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
@@ -131,7 +137,12 @@ describe('useQueueDrain', () => {
await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
expect(ask).toHaveBeenCalledWith(
{ text: 'with media' },
- { overrideFiles: files, overrideQuotes: [], overrideManualSkills: [] },
+ expect.objectContaining({
+ overrideFiles: files,
+ overrideQuotes: [],
+ overrideManualSkills: [],
+ overrideQueuedMessageOrigin: expect.any(Object),
+ }),
);
});
@@ -319,7 +330,39 @@ describe('useQueueDrain', () => {
await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
expect(ask).toHaveBeenCalledWith(
{ text: 'with context' },
- { overrideFiles: [], overrideQuotes: ['quoted excerpt'], overrideManualSkills: ['skill-1'] },
+ expect.objectContaining({
+ overrideFiles: [],
+ overrideQuotes: ['quoted excerpt'],
+ overrideManualSkills: ['skill-1'],
+ overrideQueuedMessageOrigin: expect.any(Object),
+ }),
+ );
+ });
+
+ it('keeps the recovery user row stable while forwarding its per-attempt identity', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ {
+ ...queuedMessage('source-steer', 'recover once'),
+ clientRequestId: 'attempt-uuid',
+ recoverySteerId: 'source-steer',
+ },
+ ]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 1234 }));
+ });
+
+ await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
+ expect(ask).toHaveBeenCalledWith(
+ { text: 'recover once', overrideUserMessageId: 'source-steer' },
+ expect.objectContaining({
+ overrideClientRequestId: 'attempt-uuid',
+ overrideRecoverySteerId: 'source-steer',
+ overrideExpectedPredecessorCreatedAt: 1234,
+ overrideQueuedMessageOrigin: expect.any(Object),
+ }),
);
});
@@ -342,7 +385,10 @@ describe('useQueueDrain', () => {
it('drains on abort when the interrupt & send flag is armed, then disarms', async () => {
const { ask, setters } = setup(({ set }) => {
set(store.queuedMessagesByConvoId(CONVO_ID), [queuedMessage('q1', 'interrupt text')]);
- set(store.drainAfterAbortByIndex(INDEX), true);
+ set(store.drainAfterAbortByIndex(INDEX), {
+ conversationId: CONVO_ID,
+ generationCreatedAt: 41,
+ });
});
act(() => {
@@ -361,6 +407,116 @@ describe('useQueueDrain', () => {
expect(ask).toHaveBeenCalledTimes(1);
});
+ it('does not let a different generation consume an interrupt arm', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [queuedMessage('q1', 'epoch owner')]);
+ set(store.drainAfterAbortByIndex(INDEX), {
+ conversationId: CONVO_ID,
+ generationCreatedAt: 41,
+ });
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ outcome: 'aborted', generationCreatedAt: 42 }));
+ });
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ expect(ask).not.toHaveBeenCalled();
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ outcome: 'aborted', generationCreatedAt: 41 }));
+ });
+ await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
+ expect(ask).toHaveBeenCalledWith({ text: 'epoch owner' }, emptyOverrides);
+ });
+
+ it('parks a foreign epoch while another conversation submits and preserves the next end', async () => {
+ const CONVO_A = 'convo-a';
+ const CONVO_B = 'convo-b';
+ const ask = jest.fn();
+ let setRunEnd: ((value: RunEnd | null) => void) | undefined;
+ let setIsSubmitting: ((value: boolean) => void) | undefined;
+
+ const queryClient = new QueryClient({
+ defaultOptions: { mutations: { retry: false } },
+ });
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ <QueryClientProvider client={queryClient}>
+ <RecoilRoot
+ initializeState={({ set }) => {
+ set(store.isSubmittingFamily(INDEX), true);
+ set(store.queuedMessagesByConvoId(CONVO_A), [queuedMessage('qa', 'send for A')]);
+ set(store.queuedMessagesByConvoId(CONVO_B), [queuedMessage('qb', 'send for B')]);
+ set(store.drainAfterAbortByIndex(INDEX), {
+ conversationId: CONVO_A,
+ generationCreatedAt: 11,
+ });
+ }}
+ >
+ {children}
+ </RecoilRoot>
+ </QueryClientProvider>
+ );
+ const { result, rerender } = renderHook(
+ ({ activeConversationId }: { activeConversationId: string }) => {
+ setRunEnd = useSetRecoilState(store.runEndByIndex(INDEX));
+ setIsSubmitting = useSetRecoilState(store.isSubmittingFamily(INDEX));
+ useQueueDrain(INDEX, activeConversationId, ask);
+ return {
+ indexEnd: useRecoilValue(store.runEndByIndex(INDEX)),
+ parkedA: useRecoilValue(store.pendingRunEndByConvoId(CONVO_A)),
+ };
+ },
+ { wrapper, initialProps: { activeConversationId: CONVO_B } },
+ );
+
+ /** Both terminal callbacks land before React can park the first one. The
+ * pane carrier must retain B behind A instead of replacing A. */
+ act(() => {
+ setRunEnd?.({
+ conversationId: CONVO_A,
+ outcome: 'aborted',
+ endedAt: 1,
+ generationCreatedAt: 11,
+ });
+ setRunEnd?.({
+ conversationId: CONVO_B,
+ outcome: 'completed',
+ endedAt: 2,
+ generationCreatedAt: 22,
+ });
+ });
+
+ await waitFor(() =>
+ expect(result.current.parkedA).toMatchObject({
+ conversationId: CONVO_A,
+ generationCreatedAt: 11,
+ interruptArmed: true,
+ }),
+ );
+ expect(result.current.indexEnd).toMatchObject({
+ conversationId: CONVO_B,
+ generationCreatedAt: 22,
+ });
+ expect(ask).not.toHaveBeenCalled();
+
+ act(() => setIsSubmitting?.(false));
+ await waitFor(() =>
+ expect(ask).toHaveBeenCalledWith(
+ { text: 'send for B' },
+ expect.objectContaining({ overrideExpectedPredecessorCreatedAt: 22 }),
+ ),
+ );
+
+ rerender({ activeConversationId: CONVO_A });
+ await waitFor(() =>
+ expect(ask).toHaveBeenCalledWith(
+ { text: 'send for A' },
+ expect.objectContaining({ overrideExpectedPredecessorCreatedAt: 11 }),
+ ),
+ );
+ expect(ask).toHaveBeenCalledTimes(2);
+ });
+
it('waits for isSubmitting to flip false before draining', async () => {
const { ask, setters } = setup(({ set }) => {
set(store.queuedMessagesByConvoId(CONVO_ID), [queuedMessage('q1', 'deferred')]);
@@ -396,6 +552,34 @@ describe('useQueueDrain', () => {
);
});
+ it('keeps an interrupt queued after URL resolution ahead of pre-migration follow-ups', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(Constants.NEW_CONVO), [
+ { id: 'ordinary-before-url', text: 'ordinary follow-up', createdAt: 1 },
+ ]);
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ { id: 'interrupt-after-url', text: 'interrupt next', createdAt: 2, priority: true },
+ ]);
+ set(store.drainAfterAbortByIndex(INDEX), {
+ conversationId: CONVO_ID,
+ generationCreatedAt: 41,
+ });
+ });
+
+ act(() => {
+ setters.setRunEnd!(
+ runEnd({
+ outcome: 'aborted',
+ startedAsNewConvo: true,
+ }),
+ );
+ });
+
+ await waitFor(() =>
+ expect(ask).toHaveBeenCalledWith({ text: 'interrupt next' }, emptyOverrides),
+ );
+ });
+
describe('early-aborted first turn (NEW_CONVO-keyed signal)', () => {
const OPTIMISTIC_ID = 'optimistic-stream-id';
@@ -403,7 +587,7 @@ describe('useQueueDrain', () => {
const ask = jest.fn();
const setters: {
setRunEnd?: (value: RunEnd | null) => void;
- setInterruptFlag?: (value: boolean) => void;
+ setInterruptFlag?: (value: DrainAfterAbort | false) => void;
} = {};
const state: {
newConvoQueue?: QueuedMessage[];
@@ -465,12 +649,16 @@ describe('useQueueDrain', () => {
set(store.queuedMessagesByConvoId(Constants.NEW_CONVO), [
queuedMessage('q1', 'interrupted first turn'),
]);
- set(store.drainAfterAbortByIndex(INDEX), true);
+ set(store.drainAfterAbortByIndex(INDEX), {
+ conversationId: String(Constants.NEW_CONVO),
+ generationCreatedAt: 41,
+ });
set(store.runEndByIndex(INDEX), {
conversationId: Constants.NEW_CONVO as string,
outcome: 'aborted',
startedAsNewConvo: false,
endedAt: Date.now(),
+ generationCreatedAt: 41,
});
});
diff --git a/client/src/hooks/Chat/__tests__/useSteerConvert.spec.tsx b/client/src/hooks/Chat/__tests__/useSteerConvert.spec.tsx
index da8edcdd7a..b1eb60a204 100644
--- a/client/src/hooks/Chat/__tests__/useSteerConvert.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/useSteerConvert.spec.tsx
@@ -7,13 +7,26 @@ import store from '~/store';
const mockFetchStreamStatus = jest.fn();
jest.mock('~/data-provider', () => ({
fetchStreamStatus: (...args: unknown[]) => mockFetchStreamStatus(...args),
+ getGenerationProtocolVersion: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2
+ ? 2
+ : 1,
}));
const CONVO_ID = 'convo-steer-convert';
function setup(initialize?: (snapshot: MutableSnapshot) => void) {
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot initializeState={initialize}>{children}</RecoilRoot>
+ <RecoilRoot
+ initializeState={(snapshot) => {
+ snapshot.set(store.activeGenerationProtocolVersionByConvoId(CONVO_ID), 2);
+ initialize?.(snapshot);
+ }}
+ >
+ {children}
+ </RecoilRoot>
);
return renderHook(
() => {
@@ -72,6 +85,115 @@ describe('useSteerConvert', () => {
]);
});
+ it('restores an accepted queued-origin steer with its queue identity, position, and receipt', () => {
+ const original = {
+ id: 'queue-original',
+ text: 'sent from queue',
+ createdAt: 10,
+ priority: true,
+ quotes: ['original quote'],
+ };
+ const after = { id: 'queue-after', text: 'still queued', createdAt: 20 };
+ const { result } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [after]);
+ set(store.pendingSteersByConvoId(CONVO_ID), [
+ {
+ steerId: 'server-replacement-id',
+ text: original.text,
+ status: 'pending' as const,
+ createdAt: 999,
+ queuedOrigin: { item: original, beforeIds: [], afterIds: [after.id] },
+ },
+ ]);
+ });
+
+ act(() => {
+ result.current.convert(CONVO_ID, [
+ { steerId: 'server-replacement-id', text: original.text, createdAt: 999 },
+ ]);
+ });
+
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ ...original,
+ clientRequestId: expect.any(String),
+ recoverySteerId: 'server-replacement-id',
+ }),
+ after,
+ ]);
+ expect(result.current.queue[0]).toMatchObject({
+ id: 'queue-original',
+ createdAt: 10,
+ priority: true,
+ quotes: ['original quote'],
+ });
+ expect(result.current.applied).toContain('server-replacement-id');
+ });
+
+ it('recovers a v1 leftover as an ordinary local follow-up without receipt binding', () => {
+ const { result } = setup();
+ act(() => {
+ result.current.convert(
+ CONVO_ID,
+ [{ steerId: 'legacy-leftover', text: 'recover locally', createdAt: 1 }],
+ { generationProtocolVersion: 1 },
+ );
+ });
+
+ expect(result.current.queue).toEqual([
+ {
+ id: 'legacy-leftover',
+ text: 'recover locally',
+ createdAt: 1,
+ },
+ ]);
+ });
+
+ it('correlates a terminal leftover that arrives before the 202 ACK', () => {
+ const original = {
+ id: 'queue-before-ack',
+ text: 'terminal raced the ACK',
+ createdAt: 10,
+ priority: true,
+ };
+ const after = { id: 'queue-after-race', text: 'after', createdAt: 20 };
+ const { result } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [after]);
+ set(store.pendingSteersByConvoId(CONVO_ID), [
+ {
+ steerId: 'local-correlation-id',
+ text: original.text,
+ status: 'sending' as const,
+ createdAt: 999,
+ queuedOrigin: { item: original, beforeIds: [], afterIds: [after.id] },
+ },
+ ]);
+ });
+
+ act(() => {
+ result.current.convert(CONVO_ID, [
+ {
+ steerId: 'server-id-before-ack',
+ clientSteerId: 'local-correlation-id',
+ text: original.text,
+ createdAt: 999,
+ },
+ ]);
+ });
+
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ ...original,
+ clientRequestId: expect.any(String),
+ recoverySteerId: 'server-id-before-ack',
+ recoveryClientSteerId: 'local-correlation-id',
+ }),
+ after,
+ ]);
+ expect(result.current.chips).toEqual([]);
+ expect(result.current.applied).toContain('server-id-before-ack');
+ });
+
it('falls back to context carried on the steer when its chip is already gone', () => {
// A reclaimed steer stays interactive during its cancel round-trip, so a
// competing X can delete the chip before the conversion runs. The steer
@@ -165,7 +287,37 @@ describe('useSteerConvert', () => {
expect(result.current.applied).toEqual(['srv-drained']);
});
- describe('claimParked (clears the parked server copy of live-delivered steers)', () => {
+ it('re-queues a drained recovery source when inactive status proves it was not consumed', () => {
+ const { result } = setup();
+ const steers = [{ steerId: 'srv-retry', text: 'retry after failed recovery', createdAt: 5 }];
+ act(() => {
+ result.current.convert(CONVO_ID, steers);
+ });
+ const failedAttemptId = result.current.queue[0].clientRequestId;
+ act(() => {
+ result.current.drainQueue();
+ });
+
+ expect(result.current.queue).toEqual([]);
+ act(() => {
+ result.current.convert(CONVO_ID, steers, {
+ generationProtocolVersion: 2,
+ allowPreviouslyConvertedIds: ['srv-retry'],
+ });
+ });
+
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ id: 'srv-retry',
+ clientRequestId: expect.any(String),
+ recoverySteerId: 'srv-retry',
+ }),
+ ]);
+ expect(result.current.queue[0].clientRequestId).not.toBe(failedAttemptId);
+ expect(result.current.applied).toEqual(['srv-retry']);
+ });
+
+ describe('claimParked (reconciles the parked server copy of live-delivered steers)', () => {
beforeEach(() => {
mockFetchStreamStatus.mockReset();
});
@@ -173,6 +325,7 @@ describe('useSteerConvert', () => {
it('fires exactly one status fetch per batch and dedupes the claimed steers', async () => {
mockFetchStreamStatus.mockResolvedValue({
active: false,
+ generationProtocolVersion: 2,
unrecoveredSteers: [
{ steerId: 'live-1', text: 'delivered live', createdAt: 1 },
{ steerId: 'parked-1', text: 'parked only', createdAt: 2 },
@@ -207,9 +360,35 @@ describe('useSteerConvert', () => {
expect(result.current.queue).toEqual([expect.objectContaining({ id: 'srv-3' })]);
});
+ it('downgrades an existing bound recovery when a legacy claim deletes its source', async () => {
+ mockFetchStreamStatus.mockResolvedValue({
+ active: false,
+ generationProtocolVersion: 1,
+ unrecoveredSteers: [{ steerId: 'legacy-claimed', text: 'keep these words', createdAt: 1 }],
+ });
+ const { result } = setup();
+
+ await act(async () => {
+ result.current.convert(
+ CONVO_ID,
+ [{ steerId: 'legacy-claimed', text: 'keep these words', createdAt: 1 }],
+ { claimParked: true, generationProtocolVersion: 2 },
+ );
+ });
+
+ expect(result.current.queue).toEqual([
+ {
+ id: 'legacy-claimed',
+ text: 'keep these words',
+ createdAt: 1,
+ },
+ ]);
+ });
+
it('claims under claimConversationId while chips land under the chip key', async () => {
mockFetchStreamStatus.mockResolvedValue({
active: false,
+ generationProtocolVersion: 2,
unrecoveredSteers: [
{ steerId: 'parked-2', text: 'parked under resolved id', createdAt: 2 },
],
diff --git a/client/src/hooks/Chat/__tests__/useSteering.spec.tsx b/client/src/hooks/Chat/__tests__/useSteering.spec.tsx
index 88073ceaea..e8102ecc6f 100644
--- a/client/src/hooks/Chat/__tests__/useSteering.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/useSteering.spec.tsx
@@ -3,22 +3,30 @@ import { act, renderHook } from '@testing-library/react';
import { RecoilRoot, useRecoilValue, useSetRecoilState, type MutableSnapshot } from 'recoil';
import { Constants, ContentTypes, EModelEndpoint, LocalStorageKeys } from 'librechat-data-provider';
import type { TConversation, TMessage } from 'librechat-data-provider';
+import type { QueuedMessage } from '~/store/families';
import useSteering from '../useSteering';
import store from '~/store';
const CONVO_ID = 'convo-steer-ui';
+const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const mockMutate = jest.fn();
+const mockCancelSteer = jest.fn();
const mockShowToast = jest.fn();
const mockMarkUsage = jest.fn();
let mockMessages: TMessage[] | undefined;
jest.mock('~/data-provider', () => ({
+ useCancelSteerMutation: () => ({ mutateAsync: mockCancelSteer }),
useSteerMessageMutation: () => ({ mutate: mockMutate }),
useGetMessagesByConvoId: (_id: string, config?: { select?: (messages: unknown) => unknown }) => ({
data: config?.select ? config.select(mockMessages) : mockMessages,
}),
useMarkFilesUsageMutation: () => ({ mutate: mockMarkUsage }),
+ supportsGenerationProtocolV2: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2,
}));
jest.mock('@librechat/client', () => ({
@@ -32,11 +40,31 @@ const agentsConversation = {
type HookParams = Partial<Parameters<typeof useSteering>[0]>;
+const withActiveGeneration =
+ (initialize?: (snapshot: MutableSnapshot) => void, conversationId = CONVO_ID) =>
+ (snapshot: MutableSnapshot) => {
+ snapshot.set(store.activeGenerationCreatedAtByConvoId(conversationId), 41);
+ snapshot.set(store.activeGenerationProtocolVersionByConvoId(conversationId), 2);
+ initialize?.(snapshot);
+ };
+
function setup(params: HookParams = {}, initialize?: (snapshot: MutableSnapshot) => void) {
const sendNow = jest.fn();
const stopGenerating = jest.fn();
+ let setGenerationCreatedAt: ((value: number | null) => void) | undefined;
+ const GenerationProbe = () => {
+ setGenerationCreatedAt = useSetRecoilState(
+ store.activeGenerationCreatedAtByConvoId(params.conversationId ?? CONVO_ID),
+ );
+ return null;
+ };
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot initializeState={initialize}>{children}</RecoilRoot>
+ <RecoilRoot
+ initializeState={withActiveGeneration(initialize, params.conversationId ?? CONVO_ID)}
+ >
+ <GenerationProbe />
+ {children}
+ </RecoilRoot>
);
const rendered = renderHook(
() =>
@@ -52,7 +80,12 @@ function setup(params: HookParams = {}, initialize?: (snapshot: MutableSnapshot)
}),
{ wrapper },
);
- return { ...rendered, sendNow, stopGenerating };
+ return {
+ ...rendered,
+ sendNow,
+ stopGenerating,
+ setGenerationCreatedAt: (value: number | null) => setGenerationCreatedAt?.(value),
+ };
}
function useQueue(convoId: string) {
@@ -106,13 +139,57 @@ describe('useSteering', () => {
expect(result.current.canSteer).toBe(false);
});
- it('is inactive for assistants endpoints, secondary composers, and answer mode', () => {
+ it('queues during startup and exposes no live mutation until the generation epoch arrives', () => {
+ const { result, stopGenerating, setGenerationCreatedAt } = setup({}, ({ set }) => {
+ set(store.activeGenerationCreatedAtByConvoId(CONVO_ID), null);
+ });
+
+ expect(result.current.duringRunActive).toBe(true);
+ expect(result.current.canControlGeneration).toBe(false);
+ expect(result.current.canSteer).toBe(false);
+ expect(result.current.effectiveAction).toBe('queue');
+
+ let queued = false;
+ let alternateSteer = true;
+ let interrupted = true;
+ act(() => {
+ queued = result.current.submitDuringRun('hold this until startup finishes');
+ // Cmd/Ctrl+Enter reaches this explicit alternate while Queue is the
+ // effective action; it must not bypass the epoch gate.
+ alternateSteer = result.current.steerFromComposer('unsafe startup steer');
+ interrupted = result.current.interruptAndSend('unsafe startup abort');
+ });
+ expect(queued).toBe(true);
+ expect(alternateSteer).toBe(false);
+ expect(interrupted).toBe(false);
+ expect(mockMutate).not.toHaveBeenCalled();
+ expect(stopGenerating).not.toHaveBeenCalled();
+
+ act(() => setGenerationCreatedAt(42));
+ expect(result.current.canControlGeneration).toBe(true);
+ expect(result.current.canSteer).toBe(true);
+
+ act(() => {
+ result.current.steerFromComposer('now fenced');
+ result.current.interruptAndSend('now stoppable');
+ });
+ expect(mockMutate).toHaveBeenCalledWith(
+ expect.objectContaining({ generationCreatedAt: 42, text: 'now fenced' }),
+ expect.anything(),
+ );
+ expect(stopGenerating).toHaveBeenCalledTimes(1);
+ });
+
+ it('is inactive but exposes the paused state for ask-user answer mode', () => {
expect(
setup({ conversation: { endpoint: EModelEndpoint.assistants } as TConversation }).result
.current.duringRunActive,
).toBe(false);
expect(setup({ index: 1 }).result.current.duringRunActive).toBe(false);
- expect(setup({ answerModeActive: true }).result.current.duringRunActive).toBe(false);
+ const answerMode = setup({ answerModeActive: true }).result.current;
+ expect(answerMode.duringRunActive).toBe(false);
+ expect(answerMode.pausedOnApproval).toBe(true);
+ expect(answerMode.canSteer).toBe(false);
});
});
@@ -139,7 +216,7 @@ describe('useSteering', () => {
const sendNow = jest.fn();
const stopGenerating = jest.fn();
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot initializeState={initialize}>{children}</RecoilRoot>
+ <RecoilRoot initializeState={withActiveGeneration(initialize)}>{children}</RecoilRoot>
);
const rendered = renderHook(
() => ({
@@ -314,7 +391,7 @@ describe('useSteering', () => {
) {
const sendNow = jest.fn();
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot initializeState={initialize}>{children}</RecoilRoot>
+ <RecoilRoot initializeState={withActiveGeneration(initialize)}>{children}</RecoilRoot>
);
const rendered = renderHook(
({ convoId, isSubmitting }: { convoId: string; isSubmitting: boolean }) => ({
@@ -366,6 +443,47 @@ describe('useSteering', () => {
expect(sendNow).not.toHaveBeenCalled();
});
+ it('re-arms the origin completion when a delayed refusal lands off-screen', () => {
+ let rejectSteer: ((error: unknown) => void) | undefined;
+ mockMutate.mockImplementationOnce((_params, { onError }) => {
+ rejectSteer = onError;
+ });
+ const { result, rerender, sendNow } = setupNavigable({
+ convoId: CONVO_ID,
+ isSubmitting: true,
+ });
+
+ act(() => {
+ result.current.steering.submitSteer('refused after navigation');
+ });
+ // The origin run completes and its empty-queue drain consumes the signal
+ // before the POST response arrives.
+ act(() => {
+ result.current.consumeIndexSignal(runEnd('completed'));
+ });
+ act(() => {
+ rerender({ convoId: CONVO_ID, isSubmitting: false });
+ });
+ act(() => {
+ result.current.consumeIndexSignal(null);
+ });
+ act(() => {
+ rerender({ convoId: 'convo-elsewhere', isSubmitting: false });
+ });
+ act(() => {
+ rejectSteer?.({ response: { status: 409, data: { code: 'RUN_PAUSED' } } });
+ });
+
+ expect(result.current.queueHere).toEqual([
+ expect.objectContaining({ text: 'refused after navigation' }),
+ ]);
+ expect(result.current.parkedHere).toMatchObject({
+ conversationId: CONVO_ID,
+ outcome: 'completed',
+ });
+ expect(sendNow).not.toHaveBeenCalled();
+ });
+
it('never parks another chat’s run-end under this conversation', () => {
// The run-end is keyed by conversation, so the new chat's end can never
// be mistaken for this one's. Parking it here would give `drainNext` a
@@ -434,6 +552,41 @@ describe('useSteering', () => {
});
expect(result.current.queue.map((item) => item.id)).toEqual(['s-reclaimed']);
});
+
+ it('re-arms a consumed completion drain when receipt replay recovers a leftover', () => {
+ mockMutate.mockImplementationOnce((_params, { onSuccess }) => {
+ onSuccess({
+ steerId: 's-receipt-leftover',
+ status: 'queued',
+ position: 1,
+ conversationId: CONVO_ID,
+ settled: true,
+ leftover: true,
+ replayed: true,
+ generationProtocolVersion: 2,
+ });
+ });
+ const { result } = setupWithQueue({ isSubmitting: false }, ({ set }) => {
+ set(store.runEndByIndex(0), runEnd('completed'));
+ });
+ act(() => {
+ result.current.consumeIndexSignal(null);
+ });
+ act(() => {
+ result.current.steering.submitSteer('recovered after drain');
+ });
+
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ id: 's-receipt-leftover',
+ text: 'recovered after drain',
+ }),
+ ]);
+ expect(result.current.parkedRunEnd).toMatchObject({
+ conversationId: CONVO_ID,
+ outcome: 'completed',
+ });
+ });
});
describe('submitDuringRun', () => {
@@ -445,7 +598,11 @@ describe('useSteering', () => {
});
expect(consumed).toBe(true);
expect(mockMutate).toHaveBeenCalledWith(
- { conversationId: CONVO_ID, text: 'steer this' },
+ expect.objectContaining({
+ conversationId: CONVO_ID,
+ text: 'steer this',
+ clientSteerId: expect.any(String),
+ }),
expect.anything(),
);
});
@@ -493,7 +650,44 @@ describe('useSteering', () => {
act(() => {
result.current.submitSteer('too late');
});
- expect(sendNow).toHaveBeenCalledWith('too late', [], undefined);
+ expect(sendNow).toHaveBeenCalledWith(
+ 'too late',
+ [],
+ expect.objectContaining({
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: {
+ item: expect.objectContaining({
+ id: expect.stringMatching(/^local-/),
+ text: 'too late',
+ createdAt: expect.any(Number),
+ expectedPredecessorCreatedAt: 41,
+ }),
+ beforeIds: [],
+ afterIds: [],
+ },
+ }),
+ );
+ });
+
+ it('fences a queued-origin fallback and carries its exact restoration slot', () => {
+ mockMutate.mockImplementation((_params, { onError }) => {
+ onError({ response: { data: { code: 'NO_ACTIVE_RUN' } } });
+ });
+ const queuedOrigin = {
+ item: { id: 'queued-original', text: 'late queued turn', createdAt: 10 },
+ beforeIds: ['queued-before'],
+ afterIds: ['queued-after'],
+ };
+ const { result, sendNow } = setup({ isSubmitting: false });
+
+ act(() => {
+ result.current.submitSteer('late queued turn', undefined, undefined, { queuedOrigin });
+ });
+
+ expect(sendNow).toHaveBeenCalledWith('late queued turn', [], {
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: queuedOrigin,
+ });
});
it('queues + toasts when the run is paused (RUN_PAUSED)', () => {
@@ -506,6 +700,20 @@ describe('useSteering', () => {
});
expect(mockShowToast).toHaveBeenCalled();
});
+
+ it('never normal-sends into a newer live run on RUN_REPLACED', () => {
+ mockMutate.mockImplementation((_params, { onError }) => {
+ onError({
+ response: { data: { code: 'RUN_REPLACED', generationProtocolVersion: 2 } },
+ });
+ });
+ const { result, sendNow } = setup({ isSubmitting: false });
+ act(() => {
+ result.current.submitSteer('wait for the newer run');
+ });
+ expect(sendNow).not.toHaveBeenCalled();
+ expect(mockShowToast).toHaveBeenCalled();
+ });
});
describe('interrupt & steer (preempt)', () => {
@@ -656,18 +864,121 @@ describe('useSteering', () => {
);
});
- it('is idempotent enough for a double click (two chips, both armed)', () => {
+ it('serializes rapid steer POSTs in submission order', () => {
+ const acknowledge: Array<(serverId: string) => void> = [];
+ mockMutate.mockImplementation((params, { onSuccess }) => {
+ acknowledge.push((serverId) =>
+ onSuccess({
+ status: 'queued',
+ steerId: serverId,
+ position: 1,
+ conversationId: params.conversationId,
+ }),
+ );
+ });
const { result } = setup();
act(() => {
result.current.interruptSteer('first');
result.current.interruptSteer('second');
});
+
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+ expect(mockMutate).toHaveBeenCalledWith(
+ expect.objectContaining({ text: 'first', preempt: true, generationCreatedAt: 41 }),
+ expect.anything(),
+ );
+
+ act(() => {
+ acknowledge[0]('server-first');
+ });
+
+ expect(mockMutate).toHaveBeenCalledTimes(2);
+ expect(mockMutate).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ text: 'second', preempt: true, generationCreatedAt: 41 }),
+ expect.anything(),
+ );
+
+ act(() => {
+ acknowledge[1]('server-second');
+ result.current.interruptSteer('third');
+ });
+ expect(mockMutate).toHaveBeenCalledTimes(3);
+ expect(mockMutate).toHaveBeenNthCalledWith(
+ 3,
+ expect.objectContaining({ text: 'third', preempt: true, generationCreatedAt: 41 }),
+ expect.anything(),
+ );
+ });
+
+ it('keeps different conversations dispatching in parallel', () => {
+ const otherConversationId = 'convo-steer-ui-other';
+ const acknowledge: Array<(serverId: string) => void> = [];
+ mockMutate.mockImplementation((params, { onSuccess }) => {
+ acknowledge.push((serverId) =>
+ onSuccess({
+ status: 'queued',
+ steerId: serverId,
+ position: 1,
+ conversationId: params.conversationId,
+ }),
+ );
+ });
+ const sendNow = jest.fn();
+ const stopGenerating = jest.fn();
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ <RecoilRoot
+ initializeState={({ set }) => {
+ set(store.activeGenerationCreatedAtByConvoId(CONVO_ID), 41);
+ set(store.activeGenerationProtocolVersionByConvoId(CONVO_ID), 2);
+ set(store.activeGenerationCreatedAtByConvoId(otherConversationId), 42);
+ set(store.activeGenerationProtocolVersionByConvoId(otherConversationId), 2);
+ }}
+ >
+ {children}
+ </RecoilRoot>
+ );
+ const { result, rerender } = renderHook(
+ ({ conversationId }: { conversationId: string }) =>
+ useSteering({
+ index: 0,
+ conversationId,
+ conversation: { ...agentsConversation, conversationId } as TConversation,
+ isSubmitting: true,
+ answerModeActive: false,
+ sendNow,
+ stopGenerating,
+ }),
+ { initialProps: { conversationId: CONVO_ID }, wrapper },
+ );
+
+ act(() => {
+ result.current.submitSteer('first conversation');
+ });
+ rerender({ conversationId: otherConversationId });
+ act(() => {
+ result.current.submitSteer('second conversation');
+ });
+
expect(mockMutate).toHaveBeenCalledTimes(2);
+ expect(mockMutate).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ conversationId: CONVO_ID, text: 'first conversation' }),
+ expect.anything(),
+ );
expect(mockMutate).toHaveBeenNthCalledWith(
2,
- expect.objectContaining({ text: 'second', preempt: true }),
+ expect.objectContaining({
+ conversationId: otherConversationId,
+ text: 'second conversation',
+ }),
expect.anything(),
);
+
+ act(() => {
+ acknowledge[0]('server-first-conversation');
+ acknowledge[1]('server-second-conversation');
+ });
});
});
@@ -679,7 +990,7 @@ describe('useSteering', () => {
const sendNow = jest.fn();
const stopGenerating = jest.fn();
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot initializeState={initialize}>{children}</RecoilRoot>
+ <RecoilRoot initializeState={withActiveGeneration(initialize)}>{children}</RecoilRoot>
);
const rendered = renderHook(
() => ({
@@ -694,7 +1005,14 @@ describe('useSteering', () => {
...params,
}),
queue: useQueue(CONVO_ID),
+ setQueue: useSetRecoilState(store.queuedMessagesByConvoId(CONVO_ID)),
+ setActiveEpoch: useSetRecoilState(store.activeGenerationCreatedAtByConvoId(CONVO_ID)),
+ setActiveProtocol: useSetRecoilState(
+ store.activeGenerationProtocolVersionByConvoId(CONVO_ID),
+ ),
+ setAcceptedIds: useSetRecoilState(store.acceptedSteerClientIdsByConvoId(CONVO_ID)),
chips: useRecoilValue(store.pendingSteersByConvoId(CONVO_ID)),
+ setAppliedIds: useSetRecoilState(store.appliedSteerIdsByConvoId(CONVO_ID)),
drainFlag: useRecoilValue(store.drainAfterAbortByIndex(0)),
}),
{ wrapper },
@@ -702,18 +1020,82 @@ describe('useSteering', () => {
return { ...rendered, sendNow, stopGenerating };
}
- it('drops the ACK chip when on_steer_applied already landed (SSE beat the 202)', () => {
- mockMutate.mockImplementation((_params, { onSuccess }) => {
- onSuccess({ steerId: 'srv-1', status: 'queued', position: 1, conversationId: CONVO_ID });
- });
- const { result } = setupWithState({}, ({ set }) => {
- set(store.appliedSteerIdsByConvoId(CONVO_ID), ['srv-1']);
+ it('drops the ACK chip when FINAL settlement lands before the delayed 202', () => {
+ let acknowledge: (() => void) | undefined;
+ let clientSteerId = '';
+ mockMutate.mockImplementation((params, { onSuccess }) => {
+ clientSteerId = params.clientSteerId;
+ acknowledge = () =>
+ onSuccess({
+ steerId: 'srv-1',
+ status: 'queued',
+ position: 1,
+ conversationId: CONVO_ID,
+ });
});
+ const { result } = setupWithState();
act(() => {
result.current.steering.submitSteer('already injected');
});
+ expect(result.current.chips).toEqual([
+ expect.objectContaining({ steerId: clientSteerId, status: 'sending' }),
+ ]);
+ act(() => {
+ result.current.setAppliedIds([clientSteerId, 'srv-1']);
+ });
+ act(() => {
+ acknowledge?.();
+ });
// No pending chip may survive — its only removal event already passed.
expect(result.current.chips).toEqual([]);
+ expect(result.current.queue).toEqual([]);
+ });
+
+ it.each([
+ [
+ 'a definite origin 400',
+ { response: { status: 400, data: { code: 'INVALID_STEER' } } },
+ false,
+ ],
+ ['a transport failure', new Error('network disconnected'), true],
+ ] as const)('marks delivery uncertainty correctly for %s', (_scenario, error, uncertain) => {
+ mockMutate.mockImplementationOnce((_params, { onError }) => onError(error));
+ const { result } = setupWithState();
+
+ act(() => {
+ result.current.steering.submitSteer('classify the failure');
+ });
+
+ expect(result.current.chips).toEqual([
+ expect.objectContaining({
+ status: 'failed',
+ ...(uncertain && { deliveryUncertain: true }),
+ }),
+ ]);
+ if (!uncertain) {
+ expect(result.current.chips[0].deliveryUncertain).toBeUndefined();
+ }
+ });
+
+ it('marks an ambiguous legacy delivery as non-retryable protocol v1', () => {
+ mockMutate.mockImplementationOnce((_params, { onError }) =>
+ onError(new Error('network disconnected')),
+ );
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.activeGenerationProtocolVersionByConvoId(CONVO_ID), 1);
+ });
+
+ act(() => {
+ result.current.steering.submitSteer('legacy ambiguous delivery');
+ });
+
+ expect(result.current.chips).toEqual([
+ expect.objectContaining({
+ status: 'failed',
+ deliveryUncertain: true,
+ generationProtocolVersion: 1,
+ }),
+ ]);
});
it('converts the ACK chip to pending when the steer is not yet applied', () => {
@@ -729,6 +1111,103 @@ describe('useSteering', () => {
]);
});
+ it('recovers a delayed ACK when a replacement generation is already active', () => {
+ let acknowledge: (() => void) | undefined;
+ let clientSteerId = '';
+ mockMutate.mockImplementation((params, { onSuccess }) => {
+ clientSteerId = params.clientSteerId;
+ acknowledge = () =>
+ onSuccess({
+ steerId: 'srv-replaced',
+ status: 'queued',
+ position: 1,
+ conversationId: CONVO_ID,
+ });
+ });
+ const { result } = setupWithState();
+
+ act(() => {
+ result.current.steering.submitSteer('accepted by the old generation');
+ });
+ // The server's correlation update may arrive before both replacement
+ // handoff and the delayed HTTP ACK. Acceptance is not proof of delivery.
+ act(() => {
+ result.current.setAcceptedIds([clientSteerId]);
+ });
+ act(() => {
+ result.current.setActiveEpoch(42);
+ });
+ act(() => {
+ acknowledge?.();
+ });
+
+ expect(result.current.chips).toEqual([]);
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ id: 'srv-replaced',
+ text: 'accepted by the old generation',
+ recoverySteerId: 'srv-replaced',
+ }),
+ ]);
+ });
+
+ it('recovers a delayed ACK after the source generation epoch is cleared', () => {
+ let acknowledge: (() => void) | undefined;
+ mockMutate.mockImplementation((params, { onSuccess }) => {
+ acknowledge = () =>
+ onSuccess({
+ steerId: 'srv-ended',
+ status: 'queued',
+ position: 1,
+ conversationId: params.conversationId,
+ });
+ });
+ const { result } = setupWithState();
+
+ act(() => {
+ result.current.steering.submitSteer('accepted before the epoch cleared');
+ });
+ act(() => {
+ result.current.setActiveEpoch(null);
+ });
+ act(() => {
+ acknowledge?.();
+ });
+
+ expect(result.current.chips).toEqual([]);
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ id: 'srv-ended',
+ text: 'accepted before the epoch cleared',
+ recoverySteerId: 'srv-ended',
+ }),
+ ]);
+ });
+
+ it('fences steer submission and its retry chip to the attached generation epoch', () => {
+ mockMutate.mockImplementation((_params, { onError }) => {
+ onError({ response: { status: 400, data: { code: 'INVALID_STEER' } } });
+ });
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.activeGenerationCreatedAtByConvoId(CONVO_ID), 41);
+ });
+
+ act(() => {
+ result.current.steering.submitSteer('epoch-bound');
+ });
+
+ expect(mockMutate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ conversationId: CONVO_ID,
+ generationCreatedAt: 41,
+ }),
+ expect.anything(),
+ );
+ expect(result.current.chips).toEqual([
+ expect.objectContaining({ generationCreatedAt: 41, status: 'failed' }),
+ ]);
+ });
+
it('carries the submit time through the ACK so the pending chip is not re-timestamped', () => {
// A draft queued during the 202 round-trip must not drain ahead of a
// steer submitted before it — so the ACK'd chip keeps its SUBMIT time,
@@ -750,6 +1229,54 @@ describe('useSteering', () => {
}
});
+ it('preserves the original submit time through a same-id receipt retry', () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+ try {
+ mockMutate.mockImplementationOnce((_params, { onError }) => {
+ onError(new Error('lost response'));
+ });
+ const { result } = setupWithState();
+ act(() => {
+ result.current.steering.submitSteer('retry without reordering');
+ });
+ const failed = result.current.chips[0];
+
+ now.mockReturnValue(9_000);
+ act(() => {
+ // A replacement deployment/run may now advertise v1. The receipt
+ // still belongs to the v2 source generation captured on the chip.
+ result.current.setActiveProtocol(1);
+ });
+ mockMutate.mockImplementationOnce((_params, { onSuccess }) => {
+ onSuccess({
+ steerId: 'srv-retry-leftover',
+ settled: true,
+ leftover: true,
+ replayed: true,
+ generationProtocolVersion: 2,
+ });
+ });
+ act(() => {
+ result.current.steering.retrySteer(failed.steerId, failed.text, failed.files, undefined, {
+ createdAt: failed.createdAt,
+ generationCreatedAt: failed.generationCreatedAt,
+ generationProtocolVersion: failed.generationProtocolVersion,
+ });
+ });
+
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ id: 'srv-retry-leftover',
+ createdAt: 1_000,
+ clientRequestId: expect.stringMatching(UUID_V4_PATTERN),
+ recoverySteerId: 'srv-retry-leftover',
+ }),
+ ]);
+ } finally {
+ now.mockRestore();
+ }
+ });
+
it('does not duplicate a chip already reseeded under the server id (SSE reconnect)', () => {
mockMutate.mockImplementation((_params, { onSuccess }) => {
onSuccess({ steerId: 'srv-3', status: 'queued', position: 1, conversationId: CONVO_ID });
@@ -767,7 +1294,7 @@ describe('useSteering', () => {
expect(result.current.chips.filter((chip) => chip.steerId === 'srv-3')).toHaveLength(1);
});
- it('restores the queued item (same id, front) when sendNow refuses', () => {
+ it('restores the queued item (same id, original slot) when sendNow refuses', () => {
const refusingSendNow = jest.fn().mockReturnValue(false);
const { result } = setupWithState({ isSubmitting: false, sendNow: refusingSendNow });
act(() => {
@@ -778,8 +1305,19 @@ describe('useSteering', () => {
act(() => {
result.current.steering.sendQueuedNow(first);
});
- expect(refusingSendNow).toHaveBeenCalledWith('refused send', [], {});
- // `ask` refused without sending — the ORIGINAL item returns to the front.
+ expect(refusingSendNow).toHaveBeenCalledWith('refused send', [], {
+ quotes: undefined,
+ manualSkills: undefined,
+ clientRequestId: undefined,
+ recoverySteerId: undefined,
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: {
+ item: first,
+ beforeIds: [],
+ afterIds: [second.id],
+ },
+ });
+ // `ask` refused without sending — the ORIGINAL item returns to its slot.
expect(result.current.queue.map((item) => item.id)).toEqual([first.id, second.id]);
expect(result.current.queue[0]).toEqual(first);
});
@@ -796,7 +1334,16 @@ describe('useSteering', () => {
act(() => {
result.current.steering.submitSteer('refused words', steerFiles);
});
- expect(refusingSendNow).toHaveBeenCalledWith('refused words', steerFiles, undefined);
+ expect(refusingSendNow).toHaveBeenCalledWith(
+ 'refused words',
+ steerFiles,
+ expect.objectContaining({
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: expect.objectContaining({
+ item: expect.objectContaining({ text: 'refused words', files: steerFiles }),
+ }),
+ }),
+ );
// The chip is already gone — a refused send must land in the queue, not drop.
expect(result.current.chips).toEqual([]);
expect(result.current.queue).toEqual([
@@ -816,7 +1363,16 @@ describe('useSteering', () => {
act(() => {
result.current.steering.submitSteer('unsupported words', steerFiles);
});
- expect(refusingSendNow).toHaveBeenCalledWith('unsupported words', steerFiles, undefined);
+ expect(refusingSendNow).toHaveBeenCalledWith(
+ 'unsupported words',
+ steerFiles,
+ expect.objectContaining({
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: expect.objectContaining({
+ item: expect.objectContaining({ text: 'unsupported words', files: steerFiles }),
+ }),
+ }),
+ );
expect(result.current.chips).toEqual([]);
expect(result.current.queue).toEqual([
expect.objectContaining({ text: 'unsupported words', files: steerFiles }),
@@ -833,10 +1389,68 @@ describe('useSteering', () => {
act(() => {
result.current.steering.sendQueuedNow(queued);
});
- expect(acceptingSendNow).toHaveBeenCalledWith('accepted send', [], {});
+ expect(acceptingSendNow).toHaveBeenCalledWith('accepted send', [], {
+ quotes: undefined,
+ manualSkills: undefined,
+ clientRequestId: undefined,
+ recoverySteerId: undefined,
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: {
+ item: queued,
+ beforeIds: [],
+ afterIds: [],
+ },
+ });
expect(result.current.queue).toEqual([]);
});
+ it('leaves a queued item untouched when answer mode owns the submission slot', () => {
+ const { result, sendNow } = setupWithState({ answerModeActive: true });
+ act(() => {
+ result.current.steering.enqueue('wait for the answer');
+ });
+ const queued = result.current.queue[0];
+
+ expect(result.current.steering.canSendQueuedNow).toBe(false);
+ act(() => {
+ result.current.steering.sendQueuedNow(queued);
+ });
+
+ expect(result.current.queue).toEqual([queued]);
+ expect(sendNow).not.toHaveBeenCalled();
+ expect(mockMutate).not.toHaveBeenCalled();
+ });
+
+ it('does not lose a second queued item clicked before submitting state renders', () => {
+ const acceptingSendNow = jest.fn().mockReturnValue(undefined);
+ const { result } = setupWithState({ isSubmitting: false, sendNow: acceptingSendNow });
+ act(() => {
+ result.current.steering.enqueue('first send');
+ result.current.steering.enqueue('second send');
+ });
+ const [first, second] = result.current.queue;
+
+ act(() => {
+ result.current.steering.sendQueuedNow(first);
+ result.current.steering.sendQueuedNow(second);
+ });
+
+ expect(acceptingSendNow).toHaveBeenCalledTimes(1);
+ expect(acceptingSendNow).toHaveBeenCalledWith('first send', [], {
+ quotes: undefined,
+ manualSkills: undefined,
+ clientRequestId: undefined,
+ recoverySteerId: undefined,
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: {
+ item: first,
+ beforeIds: [],
+ afterIds: [second.id],
+ },
+ });
+ expect(result.current.queue).toEqual([second]);
+ });
+
it('interruptAndSend queues at the front, arms the drain flag, and stops the run', () => {
const { result, stopGenerating } = setupWithState();
act(() => {
@@ -849,10 +1463,37 @@ describe('useSteering', () => {
'urgent redirect',
'already queued',
]);
- expect(result.current.drainFlag).toBe(true);
+ expect(result.current.drainFlag).toEqual({
+ conversationId: CONVO_ID,
+ generationCreatedAt: 41,
+ });
expect(stopGenerating).toHaveBeenCalledTimes(1);
});
+ it('keeps multiple interrupt-and-send items FIFO ahead of ordinary follow-ups', () => {
+ const now = jest
+ .spyOn(Date, 'now')
+ .mockReturnValueOnce(1)
+ .mockReturnValueOnce(10)
+ .mockReturnValueOnce(20);
+ try {
+ const { result } = setupWithState();
+ act(() => {
+ result.current.steering.enqueue('ordinary');
+ result.current.steering.interruptAndSend('first interrupt');
+ result.current.steering.interruptAndSend('second interrupt');
+ });
+
+ expect(result.current.queue.map((item) => item.text)).toEqual([
+ 'first interrupt',
+ 'second interrupt',
+ 'ordinary',
+ ]);
+ } finally {
+ now.mockRestore();
+ }
+ });
+
it('enqueue appends and removeQueued deletes by id', () => {
const { result } = setupWithState();
act(() => {
@@ -868,6 +1509,101 @@ describe('useSteering', () => {
expect(mockMutate).not.toHaveBeenCalled();
});
+ it('atomically discards a recovered source and downgrades its row in place', async () => {
+ mockCancelSteer.mockResolvedValueOnce({ removed: true, generationProtocolVersion: 2 });
+ const before: QueuedMessage = { id: 'queued-before', text: 'before', createdAt: 0 };
+ const recovered: QueuedMessage = {
+ id: 'queued-leftover',
+ text: 'edit this next',
+ createdAt: 1,
+ clientRequestId: 'recovery-attempt',
+ recoverySteerId: 'server-leftover',
+ recoveryClientSteerId: 'client-leftover',
+ expectedPredecessorCreatedAt: 41,
+ files: [{ file_id: 'file-1', filepath: '/uploads/file-1.txt', type: 'text/plain' }],
+ quotes: ['keep quote'],
+ manualSkills: ['keep skill'],
+ priority: true,
+ };
+ const after: QueuedMessage = { id: 'queued-after', text: 'after', createdAt: 2 };
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [before, recovered, after]);
+ });
+
+ let discarded = false;
+ await act(async () => {
+ discarded = await result.current.steering.discardQueued(recovered);
+ });
+
+ expect(discarded).toBe(true);
+ expect(mockCancelSteer).toHaveBeenCalledWith({
+ conversationId: CONVO_ID,
+ steerId: 'server-leftover',
+ clientSteerId: 'client-leftover',
+ });
+ expect(result.current.queue).toEqual([
+ before,
+ {
+ id: 'queued-leftover',
+ text: 'edit this next',
+ createdAt: 1,
+ expectedPredecessorCreatedAt: 41,
+ files: [{ file_id: 'file-1', filepath: '/uploads/file-1.txt', type: 'text/plain' }],
+ quotes: ['keep quote'],
+ manualSkills: ['keep skill'],
+ priority: true,
+ },
+ after,
+ ]);
+ });
+
+ it('keeps a recovered queue row when its parked source cannot be discarded', async () => {
+ mockCancelSteer.mockResolvedValueOnce({ removed: false, generationProtocolVersion: 2 });
+ const recovered = {
+ id: 'queued-leftover',
+ text: 'do not duplicate me',
+ createdAt: 1,
+ recoverySteerId: 'server-leftover',
+ recoveryClientSteerId: 'client-leftover',
+ };
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [recovered]);
+ });
+
+ let discarded = true;
+ await act(async () => {
+ discarded = await result.current.steering.discardQueued(recovered);
+ });
+
+ expect(discarded).toBe(false);
+ expect(result.current.queue).toEqual([recovered]);
+ expect(mockShowToast).toHaveBeenCalledWith({
+ message: 'Could not cancel the steering message — it may still reach the agent',
+ status: 'error',
+ });
+ });
+
+ it('fails closed for a recovered row without a durable receipt correlation', async () => {
+ const recovered = {
+ id: 'queued-leftover',
+ text: 'keep this visible',
+ createdAt: 1,
+ recoverySteerId: 'server-leftover',
+ };
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [recovered]);
+ });
+
+ let discarded = true;
+ await act(async () => {
+ discarded = await result.current.steering.discardQueued(recovered);
+ });
+
+ expect(discarded).toBe(false);
+ expect(mockCancelSteer).not.toHaveBeenCalled();
+ expect(result.current.queue).toEqual([recovered]);
+ });
+
it('sendQueuedNow steers whenever steering is available, even under the queue preference', () => {
const { result } = setupWithState({}, ({ set }) => {
set(store.duringRunDefaultAction, 'queue');
@@ -881,10 +1617,198 @@ describe('useSteering', () => {
});
expect(result.current.queue).toEqual([]);
expect(mockMutate).toHaveBeenCalledWith(
- { conversationId: CONVO_ID, text: 'send me now' },
+ expect.objectContaining({
+ conversationId: CONVO_ID,
+ text: 'send me now',
+ clientSteerId: expect.any(String),
+ }),
+ expect.anything(),
+ );
+ });
+
+ it.each([
+ ['an ordinary paused steer', 'RUN_PAUSED', undefined],
+ ['a preempt steer rejected at capacity', 'STEER_QUEUE_FULL', { preempt: true }],
+ ['a preempt steer racing run settlement', 'NO_ACTIVE_RUN', { preempt: true }],
+ ] as const)('restores a non-front queued item in place after %s', (_scenario, code, opts) => {
+ mockMutate.mockImplementationOnce((_params, { onError }) => {
+ onError({ response: { data: { code } } });
+ });
+ const before = { id: 'q-before', text: 'before', createdAt: 10 };
+ const selected = {
+ id: 'q-selected',
+ text: 'restore exactly',
+ createdAt: 20,
+ priority: true,
+ quotes: ['carried quote'],
+ manualSkills: ['carried-skill'],
+ files: [
+ {
+ file_id: 'file-selected',
+ filepath: '/uploads/file-selected.txt',
+ type: 'text/plain',
+ },
+ ],
+ };
+ const after = { id: 'q-after', text: 'after', createdAt: 30 };
+ const originalQueue = [before, selected, after];
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), originalQueue);
+ });
+
+ act(() => {
+ result.current.steering.sendQueuedNow(selected, opts);
+ });
+
+ expect(mockMutate).toHaveBeenCalledWith(
+ expect.objectContaining({
+ conversationId: CONVO_ID,
+ text: selected.text,
+ files: selected.files,
+ clientSteerId: expect.any(String),
+ ...(opts?.preempt === true && { preempt: true }),
+ }),
+ expect.anything(),
+ );
+ expect(result.current.chips).toEqual([]);
+ expect(result.current.queue).toEqual(originalQueue);
+ expect(result.current.queue[1]).toBe(selected);
+ });
+
+ it('restores before a surviving successor when the queue drains during the request', () => {
+ let rejectSteer: ((error: unknown) => void) | undefined;
+ mockMutate.mockImplementationOnce((_params, { onError }) => {
+ rejectSteer = onError;
+ });
+ const before = { id: 'q-drained', text: 'drained meanwhile', createdAt: 10 };
+ const selected = { id: 'q-racing', text: 'restore before successor', createdAt: 20 };
+ const after = { id: 'q-survives', text: 'surviving successor', createdAt: 30 };
+ const addedLater = { id: 'q-later', text: 'queued during request', createdAt: 40 };
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [before, selected, after]);
+ });
+
+ act(() => {
+ result.current.steering.sendQueuedNow(selected);
+ });
+ expect(result.current.queue).toEqual([before, after]);
+
+ act(() => {
+ result.current.setQueue([after, addedLater]);
+ rejectSteer?.({ response: { data: { code: 'NO_ACTIVE_RUN' } } });
+ });
+
+ expect(result.current.queue).toEqual([selected, after, addedLater]);
+ expect(result.current.queue[0]).toBe(selected);
+ });
+
+ it('serializes two queue-origin sends and reconstructs their order when both reject', () => {
+ const rejectSteer: Array<(error: unknown) => void> = [];
+ mockMutate.mockImplementation((_params, { onError }) => {
+ rejectSteer.push(onError);
+ });
+ const first = { id: 'q-concurrent-first', text: 'first instruction', createdAt: 10 };
+ const second = { id: 'q-concurrent-second', text: 'second instruction', createdAt: 20 };
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [first, second]);
+ });
+
+ act(() => {
+ result.current.steering.sendQueuedNow(first);
+ });
+ act(() => {
+ result.current.steering.sendQueuedNow(second);
+ });
+ expect(result.current.queue).toEqual([]);
+ expect(mockMutate).toHaveBeenCalledTimes(1);
+ expect(mockMutate).toHaveBeenCalledWith(
+ expect.objectContaining({ text: first.text }),
+ expect.anything(),
+ );
+
+ act(() => {
+ rejectSteer[0]({ response: { data: { code: 'RUN_PAUSED' } } });
+ });
+ expect(mockMutate).toHaveBeenCalledTimes(2);
+ expect(mockMutate).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ text: second.text }),
+ expect.anything(),
+ );
+
+ act(() => {
+ rejectSteer[1]({ response: { data: { code: 'RUN_PAUSED' } } });
+ });
+
+ expect(result.current.queue).toEqual([first, second]);
+ expect(result.current.queue[0]).toBe(first);
+ expect(result.current.queue[1]).toBe(second);
+
+ act(() => {
+ result.current.steering.submitSteer('third after rejections');
+ });
+ expect(mockMutate).toHaveBeenCalledTimes(3);
+ expect(mockMutate).toHaveBeenNthCalledWith(
+ 3,
+ expect.objectContaining({ text: 'third after rejections' }),
expect.anything(),
);
});
+
+ it('reconstructs FIFO when two serialized queue-origin receipts report leftovers', () => {
+ const settle: Array<(serverId: string) => void> = [];
+ mockMutate.mockImplementation((_params, { onSuccess }) => {
+ settle.push((serverId) =>
+ onSuccess({
+ steerId: serverId,
+ settled: true,
+ leftover: true,
+ replayed: true,
+ generationProtocolVersion: 2,
+ }),
+ );
+ });
+ const first = { id: 'q-receipt-first', text: 'first accepted', createdAt: 10 };
+ const second = { id: 'q-receipt-second', text: 'second accepted', createdAt: 20 };
+ const { result } = setupWithState({}, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [first, second]);
+ });
+
+ act(() => {
+ result.current.steering.sendQueuedNow(first);
+ });
+ act(() => {
+ result.current.steering.sendQueuedNow(second);
+ });
+ expect(result.current.queue).toEqual([]);
+ expect(settle).toHaveLength(1);
+
+ act(() => {
+ settle[0]('srv-receipt-first');
+ });
+ expect(settle).toHaveLength(2);
+ act(() => {
+ settle[1]('srv-receipt-second');
+ });
+
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ ...first,
+ clientRequestId: expect.stringMatching(UUID_V4_PATTERN),
+ recoverySteerId: 'srv-receipt-first',
+ }),
+ expect.objectContaining({
+ ...second,
+ clientRequestId: expect.stringMatching(UUID_V4_PATTERN),
+ recoverySteerId: 'srv-receipt-second',
+ }),
+ ]);
+ expect(result.current.queue[0].clientRequestId).not.toBe(
+ result.current.queue[1].clientRequestId,
+ );
+ expect(result.current.queue[0].clientRequestId).not.toBe('steer-recovery:srv-receipt-first');
+ expect(result.current.queue[1].clientRequestId).not.toBe('steer-recovery:srv-receipt-second');
+ });
});
describe('attachments', () => {
@@ -897,7 +1821,7 @@ describe('useSteering', () => {
};
const queuedFiles = [{ file_id: 'file-1', filepath: '/uploads/file-1.png', type: 'image/png' }];
- function setupWithFiles(params: HookParams = {}) {
+ function setupWithFiles(params: HookParams = {}, generationCreatedAt: number | null = 41) {
const setFiles = jest.fn();
const files = new Map([['file-1', composerFile]]) as unknown as NonNullable<
Parameters<typeof useSteering>[0]['files']
@@ -905,7 +1829,15 @@ describe('useSteering', () => {
const sendNow = jest.fn();
const stopGenerating = jest.fn();
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot>{children}</RecoilRoot>
+ <RecoilRoot
+ initializeState={({ set }) => {
+ if (generationCreatedAt != null) {
+ set(store.activeGenerationCreatedAtByConvoId(CONVO_ID), generationCreatedAt);
+ }
+ }}
+ >
+ {children}
+ </RecoilRoot>
);
const rendered = renderHook(
() => ({
@@ -922,6 +1854,7 @@ describe('useSteering', () => {
...params,
}),
queue: useQueue(CONVO_ID),
+ setQueue: useSetRecoilState(store.queuedMessagesByConvoId(CONVO_ID)),
}),
{ wrapper },
);
@@ -936,11 +1869,12 @@ describe('useSteering', () => {
});
expect(consumed).toBe(true);
expect(mockMutate).toHaveBeenCalledWith(
- {
+ expect.objectContaining({
conversationId: CONVO_ID,
text: 'look at this image',
+ clientSteerId: expect.any(String),
files: [expect.objectContaining({ file_id: 'file-1' })],
- },
+ }),
expect.anything(),
);
expect(result.current.queue).toEqual([]);
@@ -958,6 +1892,19 @@ describe('useSteering', () => {
expect(mockMutate).not.toHaveBeenCalled();
});
+ it('does not consume composer attachments when an explicit pre-epoch steer is refused', () => {
+ const { result, setFiles } = setupWithFiles({}, null);
+ let consumed = true;
+
+ act(() => {
+ consumed = result.current.steering.steerFromComposer('wait for startup');
+ });
+
+ expect(consumed).toBe(false);
+ expect(mockMutate).not.toHaveBeenCalled();
+ expect(setFiles).not.toHaveBeenCalled();
+ });
+
it('steers a queued media item with its own files during a live run', () => {
const { result, sendNow } = setupWithFiles();
act(() => {
@@ -970,7 +1917,12 @@ describe('useSteering', () => {
});
expect(sendNow).not.toHaveBeenCalled();
expect(mockMutate).toHaveBeenCalledWith(
- { conversationId: CONVO_ID, text: 'media message', files: queuedFiles },
+ expect.objectContaining({
+ conversationId: CONVO_ID,
+ text: 'media message',
+ files: queuedFiles,
+ clientSteerId: expect.any(String),
+ }),
expect.anything(),
);
expect(result.current.queue).toEqual([]);
@@ -986,7 +1938,23 @@ describe('useSteering', () => {
files: queuedFiles,
});
});
- expect(sendNow).toHaveBeenCalledWith('media message', queuedFiles, {});
+ expect(sendNow).toHaveBeenCalledWith('media message', queuedFiles, {
+ quotes: undefined,
+ manualSkills: undefined,
+ clientRequestId: undefined,
+ recoverySteerId: undefined,
+ expectedPredecessorCreatedAt: undefined,
+ queuedMessageOrigin: {
+ item: {
+ id: 'q-media',
+ text: 'media message',
+ createdAt: expect.any(Number),
+ files: queuedFiles,
+ },
+ beforeIds: [],
+ afterIds: [],
+ },
+ });
});
it('marks queued files used exactly once when queueing from the composer', () => {
@@ -1024,9 +1992,9 @@ describe('useSteering', () => {
expect(mockMarkUsage).not.toHaveBeenCalled();
});
- it('does not re-mark already-queued files on a send-now re-queue', () => {
+ it('restores an unchanged queued media item without re-marking its files', () => {
// Paused on approval → steering unavailable while the run is live, so
- // sendQueuedNow front-requeues the item — its files were already marked.
+ // sendQueuedNow restores the item unchanged — its files were already marked.
mockMessages = [
{
messageId: 'resp-1',
@@ -1040,17 +2008,20 @@ describe('useSteering', () => {
} as unknown as TMessage,
];
const { result } = setupWithFiles();
+ const queued = {
+ id: 'q-requeue',
+ text: 'requeued media',
+ createdAt: Date.now(),
+ files: queuedFiles,
+ };
act(() => {
- result.current.steering.sendQueuedNow({
- id: 'q-requeue',
- text: 'requeued media',
- createdAt: Date.now(),
- files: queuedFiles,
- });
+ result.current.setQueue([queued]);
});
- expect(result.current.queue).toEqual([
- expect.objectContaining({ text: 'requeued media', files: queuedFiles, priority: true }),
- ]);
+ act(() => {
+ result.current.steering.sendQueuedNow(queued);
+ });
+ expect(result.current.queue).toEqual([queued]);
+ expect(result.current.queue[0]).toBe(queued);
expect(mockMarkUsage).not.toHaveBeenCalled();
});
});
@@ -1063,7 +2034,7 @@ describe('useSteering', () => {
const sendNow = jest.fn();
const stopGenerating = jest.fn();
const wrapper = ({ children }: { children: React.ReactNode }) => (
- <RecoilRoot initializeState={initialize}>{children}</RecoilRoot>
+ <RecoilRoot initializeState={withActiveGeneration(initialize)}>{children}</RecoilRoot>
);
const rendered = renderHook(
() => ({
@@ -1162,6 +2133,20 @@ describe('useSteering', () => {
expect(sendNow).toHaveBeenCalledWith('context send', [], {
quotes: ['carried quote'],
manualSkills: ['carried-skill'],
+ clientRequestId: undefined,
+ recoverySteerId: undefined,
+ expectedPredecessorCreatedAt: undefined,
+ queuedMessageOrigin: {
+ item: {
+ id: 'q-ctx',
+ text: 'context send',
+ createdAt: expect.any(Number),
+ quotes: ['carried quote'],
+ manualSkills: ['carried-skill'],
+ },
+ beforeIds: [],
+ afterIds: [],
+ },
});
});
@@ -1204,6 +2189,19 @@ describe('useSteering', () => {
expect(sendNow).toHaveBeenCalledWith('late context', [], {
quotes: ['carried quote'],
manualSkills: ['carried-skill'],
+ expectedPredecessorCreatedAt: 41,
+ queuedMessageOrigin: {
+ item: expect.objectContaining({
+ id: expect.stringMatching(/^local-/),
+ text: 'late context',
+ createdAt: expect.any(Number),
+ expectedPredecessorCreatedAt: 41,
+ quotes: ['carried quote'],
+ manualSkills: ['carried-skill'],
+ }),
+ beforeIds: [],
+ afterIds: [],
+ },
});
});
@@ -1252,6 +2250,55 @@ describe('useSteering', () => {
]);
});
+ it('restores an exact queued-origin item when its 202 lands after run end', () => {
+ let acknowledge: (() => void) | undefined;
+ mockMutate.mockImplementationOnce((_params, { onSuccess }) => {
+ acknowledge = () =>
+ onSuccess({
+ steerId: 'server-late-id',
+ status: 'queued',
+ position: 1,
+ conversationId: CONVO_ID,
+ });
+ });
+ const params: HookParams = { isSubmitting: true };
+ const before = { id: 'late-before', text: 'before', createdAt: 10 };
+ const selected = {
+ id: 'late-selected',
+ text: 'accepted before run end',
+ createdAt: 20,
+ priority: true,
+ quotes: ['original quote'],
+ };
+ const after = { id: 'late-after', text: 'after', createdAt: 30 };
+ const { result, rerender } = setupWithContext(params, ({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [before, selected, after]);
+ });
+
+ act(() => {
+ result.current.steering.sendQueuedNow(selected);
+ });
+ expect(result.current.queue).toEqual([before, after]);
+
+ params.isSubmitting = false;
+ rerender();
+ act(() => {
+ acknowledge?.();
+ });
+
+ expect(result.current.queue).toEqual([
+ before,
+ expect.objectContaining({
+ ...selected,
+ clientRequestId: expect.stringMatching(UUID_V4_PATTERN),
+ recoverySteerId: 'server-late-id',
+ recoveryClientSteerId: expect.any(String),
+ }),
+ after,
+ ]);
+ expect(result.current.queue[1].clientRequestId).not.toBe('steer-recovery:server-late-id');
+ });
+
it('keeps the carried context on a failed chip through retry', () => {
mockMutate.mockImplementationOnce((_params, { onError }) => {
onError({ response: { data: { code: 'SOME_ERROR' } } });
@@ -1267,6 +2314,7 @@ describe('useSteering', () => {
expect(failed).toEqual(
expect.objectContaining({
status: 'failed',
+ deliveryUncertain: true,
quotes: ['carried quote'],
manualSkills: ['carried-skill'],
}),
@@ -1285,6 +2333,10 @@ describe('useSteering', () => {
manualSkills: failed.manualSkills,
});
});
+ expect(mockMutate).toHaveBeenLastCalledWith(
+ expect.objectContaining({ clientSteerId: failed.steerId }),
+ expect.anything(),
+ );
expect(result.current.chips).toEqual([
expect.objectContaining({
steerId: 'srv-retry',
diff --git a/client/src/hooks/Chat/useChatFunctions.ts b/client/src/hooks/Chat/useChatFunctions.ts
index 885a35d226..8bf70611e3 100644
--- a/client/src/hooks/Chat/useChatFunctions.ts
+++ b/client/src/hooks/Chat/useChatFunctions.ts
@@ -285,13 +285,15 @@ export default function useChatFunctions({
overrideManualSkills,
overrideQuotes,
addedConvo,
+ overrideClientRequestId,
+ overrideRecoverySteerId,
+ overrideExpectedPredecessorCreatedAt,
+ overrideQueuedMessageOrigin,
} = {},
) => {
- setShowStopButton(false);
-
text = text.trim();
if (!!isSubmitting || text === '') {
- return;
+ return false;
}
const conversation = cloneDeep(immutableConversation);
@@ -299,13 +301,13 @@ export default function useChatFunctions({
const endpoint = conversation?.endpoint;
if (endpoint === null) {
console.error('No endpoint available');
- return;
+ return false;
}
conversationId = conversationId ?? conversation?.conversationId ?? null;
if (conversationId == 'search') {
console.error('cannot send any message under search view!');
- return;
+ return false;
}
const cachedMessages = getMessages(conversationId);
@@ -340,7 +342,7 @@ export default function useChatFunctions({
if (isContinued && !latestMessage) {
console.error('cannot continue AI message without latestMessage!');
- return;
+ return false;
}
if (parentMessageId == null && hasPendingAssistantParent(latestMessage)) {
@@ -351,6 +353,8 @@ export default function useChatFunctions({
return false;
}
+ setShowStopButton(false);
+
const ephemeralAgent = getEphemeralAgent(conversationId ?? Constants.NEW_CONVO);
/**
* Manual skill selection resolution:
@@ -415,7 +419,7 @@ export default function useChatFunctions({
/** Stable idempotency key for this submission: fresh per `ask()` (so regenerate differs)
* but reused across the client's start-generation network retries, letting the server
* dedup a retried request instead of starting a second billed generation. */
- const clientRequestId = v4();
+ const clientRequestId = overrideClientRequestId ?? v4();
if (parentMessageId == null) {
parentMessageId = getAppendParentMessageId({ latestMessage, currentMessages });
}
@@ -677,6 +681,9 @@ export default function useChatFunctions({
addedConvo,
manualSkills: manualSkills.length > 0 ? manualSkills : undefined,
clientRequestId,
+ recoverySteerId: overrideRecoverySteerId,
+ expectedPredecessorCreatedAt: overrideExpectedPredecessorCreatedAt,
+ queuedMessageOrigin: overrideQueuedMessageOrigin,
};
if (isRegenerate) {
diff --git a/client/src/hooks/Chat/useChatHelpers.ts b/client/src/hooks/Chat/useChatHelpers.ts
index 2f401fb636..7402d1febc 100644
--- a/client/src/hooks/Chat/useChatHelpers.ts
+++ b/client/src/hooks/Chat/useChatHelpers.ts
@@ -1,13 +1,12 @@
import { useCallback, useMemo, useRef, useState } from 'react';
import { useQueryClient } from '@tanstack/react-query';
-import { QueryKeys, isAssistantsEndpoint } from 'librechat-data-provider';
-import { useRecoilState, useSetRecoilState, useRecoilCallback } from 'recoil';
+import { Constants, QueryKeys, isAssistantsEndpoint } from 'librechat-data-provider';
+import { useRecoilState, useRecoilValue, useSetRecoilState, useRecoilCallback } from 'recoil';
import type { TMessage } from 'librechat-data-provider';
-import type { ActiveJobsResponse } from '~/data-provider';
import { useLatestMessage, useLatestMessageId } from '~/hooks/Messages/useLatestMessage';
+import { supportsGenerationProtocolV2, useAbortStreamMutation } from '~/data-provider';
import useChatFunctions from '~/hooks/Chat/useChatFunctions';
import useSteerConvert from '~/hooks/Chat/useSteerConvert';
-import { useAbortStreamMutation } from '~/data-provider';
import { resolveAbortSteerTarget } from '~/utils';
import useNewConvo from '~/hooks/useNewConvo';
import { getMessageCacheIds } from './cache';
@@ -27,6 +26,24 @@ export default function useChatHelpers(index = 0, paramId?: string) {
const convertSteersToQueued = useSteerConvert();
const { captureSubmission, clearSubmissionsUnlessReplaced } = useAbortCleanup(index);
+ /** Async abort responses can settle after this pane has moved to another
+ * conversation and armed its own interrupt. Clear only the intent owned by
+ * the request that produced the response. */
+ const clearInterruptDrain = useRecoilCallback(
+ ({ snapshot, set }) =>
+ (convoId: string, generationCreatedAt: number) => {
+ const armed = snapshot.getLoadable(store.drainAfterAbortByIndex(index)).getValue();
+ if (
+ armed !== false &&
+ armed.conversationId === convoId &&
+ armed.generationCreatedAt === generationCreatedAt
+ ) {
+ set(store.drainAfterAbortByIndex(index), false);
+ }
+ },
+ [index],
+ );
+
/**
* Interrupt & send fallback: clearing submissions below can tear down the
* SSE before its aborted-final event is processed, and only that event
@@ -38,16 +55,23 @@ export default function useChatHelpers(index = 0, paramId?: string) {
*/
const signalInterruptDrain = useRecoilCallback(
({ snapshot, set }) =>
- (convoId: string) => {
+ (convoId: string, generationCreatedAt: number, armedConversationId = convoId) => {
const armed = snapshot.getLoadable(store.drainAfterAbortByIndex(index)).getValue();
const runEnd = snapshot.getLoadable(store.runEndByIndex(index)).getValue();
- if (!armed || runEnd != null) {
+ const matchesArm =
+ armed !== false &&
+ armed.conversationId === armedConversationId &&
+ armed.generationCreatedAt === generationCreatedAt;
+ const alreadySignaled =
+ runEnd?.conversationId === convoId && runEnd.generationCreatedAt === generationCreatedAt;
+ if (!matchesArm || alreadySignaled) {
return;
}
set(store.runEndByIndex(index), {
conversationId: convoId,
outcome: 'aborted',
endedAt: Date.now(),
+ generationCreatedAt,
});
},
[index],
@@ -57,6 +81,12 @@ export default function useChatHelpers(index = 0, paramId?: string) {
const { useCreateConversationAtom } = store;
const { conversation, setConversation } = useCreateConversationAtom(index);
const { conversationId, endpoint, endpointType } = conversation ?? {};
+ const activeGenerationCreatedAt = useRecoilValue(
+ store.activeGenerationCreatedAtByConvoId(conversationId ?? Constants.NEW_CONVO),
+ );
+ const activeGenerationProtocolVersion = useRecoilValue(
+ store.activeGenerationProtocolVersionByConvoId(conversationId ?? Constants.NEW_CONVO),
+ );
/** Use paramId (from URL) as primary source for query key - this must match what ChatView uses
Falling back to conversationId (Recoil) only if paramId is not available */
@@ -179,18 +209,59 @@ export default function useChatHelpers(index = 0, paramId?: string) {
// For non-assistants endpoints (using resumable streams), call abort endpoint first
if (conversationId && !isAssistants) {
- queryClient.setQueryData<ActiveJobsResponse>([QueryKeys.activeJobs], (old) => ({
- activeJobIds: (old?.activeJobIds ?? []).filter((id) => id !== conversationId),
- }));
-
+ /** Stop is a generation-scoped mutation. `isSubmitting` becomes true
+ * before the start POST returns, so a stale/missing epoch must make this
+ * path inert instead of aborting whichever conversation job is current. */
+ if (activeGenerationCreatedAt == null) {
+ return;
+ }
// The aborted run's final SSE can land (and the interrupt drain can
// start the NEXT submission) while the abort response is in flight —
// the fallback clear below must not tear down that new run.
const submissionAtAbort = captureSubmission();
try {
console.log('[useChatHelpers] Calling abort mutation for:', conversationId);
- const response = await abortStream({ conversationId });
+ const response = await abortStream({
+ conversationId,
+ ...(activeGenerationCreatedAt != null && {
+ generationCreatedAt: activeGenerationCreatedAt,
+ }),
+ });
console.log('[useChatHelpers] Abort mutation succeeded');
+ const canUseV2AbortResponse =
+ activeGenerationProtocolVersion === 2 && supportsGenerationProtocolV2(response);
+ /** Conversation-scoped active-job ids cannot be removed optimistically:
+ * a newer generation may have started while this response was in flight.
+ * Refetch instead so that replacement remains represented. */
+ queryClient.invalidateQueries({ queryKey: [QueryKeys.activeJobs] });
+ if (canUseV2AbortResponse && response?.settled === true) {
+ /** Natural completion/error won the terminal CAS, so this request did
+ * not emit an abort FINAL and must not release the abort-drain queue.
+ * The winning terminal event/status owns reconciliation. */
+ clearInterruptDrain(conversationId, activeGenerationCreatedAt);
+ queryClient.invalidateQueries({
+ queryKey: [QueryKeys.messages, conversationId],
+ refetchType: 'all',
+ });
+ queryClient.invalidateQueries({ queryKey: ['streamStatus', conversationId] });
+ clearSubmissionsUnlessReplaced(submissionAtAbort);
+ return;
+ }
+ if (canUseV2AbortResponse && response?.persistenceFailed === true) {
+ /** The process was interrupted, but neither a durable terminal
+ * payload nor a trustworthy leftover-steer set exists. Keep the SSE
+ * submission attached for its reconciliation frame/status retry and
+ * do not consume the armed interrupt queue against unknown history. */
+ clearInterruptDrain(conversationId, activeGenerationCreatedAt);
+ queryClient.invalidateQueries({
+ queryKey: [QueryKeys.messages, conversationId],
+ refetchType: 'all',
+ });
+ queryClient.invalidateQueries({
+ queryKey: ['streamStatus', conversationId],
+ });
+ return;
+ }
// The response's `aborted` field is the RESOLVED job id — authoritative
// when this turn still holds the `new` placeholder. Chips and the drain
// signal land where the mounted composer's queue machinery looks, while
@@ -203,25 +274,52 @@ export default function useChatHelpers(index = 0, paramId?: string) {
// here as well as on the SSE final event — clearing submissions below
// can close the stream before that event lands, and conversion
// dedupes by steer id so double delivery is a no-op. `claimParked`
- // clears the parked server copy so a reload can't re-mint the chips.
+ // reconciles the replayable parked copy if the final raced this response.
if (Array.isArray(response?.pendingSteers)) {
convertSteersToQueued(chipConvoId, response.pendingSteers, {
claimParked: true,
claimConversationId: claimConvoId,
+ generationProtocolVersion: canUseV2AbortResponse ? 2 : 1,
});
}
- signalInterruptDrain(chipConvoId);
+ signalInterruptDrain(chipConvoId, activeGenerationCreatedAt, conversationId);
// The SSE will receive a `done` event with `aborted: true` and clean up
// We still clear submissions as a fallback
clearSubmissionsUnlessReplaced(submissionAtAbort);
} catch (error) {
console.error('[useChatHelpers] Abort failed:', error);
- // An abort that 404s (run completed first) still needs the interrupt
- // fallback: without a run-end signal the queued interrupt message
- // strands and the armed flag would leak onto a later run.
- signalInterruptDrain(conversationId);
- // Fall back to clearing submissions
- clearSubmissionsUnlessReplaced(submissionAtAbort);
+ const errorData = (
+ error as {
+ response?: { data?: { code?: string; generationProtocolVersion?: unknown } };
+ } | null
+ )?.response?.data;
+ const code = errorData?.code;
+ if (
+ code === 'RUN_REPLACED' &&
+ activeGenerationProtocolVersion === 2 &&
+ supportsGenerationProtocolV2(errorData)
+ ) {
+ // This tab targeted an older generation. Do not emit an abort run-end
+ // or consume its armed queue against the newer live owner.
+ queryClient.invalidateQueries({ queryKey: [QueryKeys.activeJobs] });
+ clearSubmissionsUnlessReplaced(submissionAtAbort);
+ return;
+ }
+ const status = (error as { response?: { status?: number } } | null | undefined)?.response
+ ?.status;
+ if (status === 404) {
+ // A missing job cannot still consume the interrupt follow-up. Release
+ // the armed queue even if its ordinary terminal frame raced cleanup.
+ signalInterruptDrain(conversationId, activeGenerationCreatedAt);
+ clearSubmissionsUnlessReplaced(submissionAtAbort);
+ return;
+ }
+ /** A timeout/5xx is an unknown mutation outcome: the job may still be
+ * live, or the abort may have committed without its response. Keep the
+ * attachment and armed queue parked until SSE/status proves a terminal
+ * boundary; firing it now could replace a still-running generation. */
+ queryClient.invalidateQueries({ queryKey: [QueryKeys.activeJobs] });
+ queryClient.invalidateQueries({ queryKey: ['streamStatus', conversationId] });
}
} else {
// For assistants endpoints, just clear submissions (existing behavior)
@@ -232,6 +330,8 @@ export default function useChatHelpers(index = 0, paramId?: string) {
conversationId,
endpoint,
endpointType,
+ activeGenerationCreatedAt,
+ activeGenerationProtocolVersion,
abortStream,
captureSubmission,
convertSteersToQueued,
@@ -239,6 +339,7 @@ export default function useChatHelpers(index = 0, paramId?: string) {
clearSubmissionsUnlessReplaced,
clearAllSubmissions,
queryClient,
+ clearInterruptDrain,
]);
const handleStopGenerating = useCallback(
diff --git a/client/src/hooks/Chat/useQueueDrain.ts b/client/src/hooks/Chat/useQueueDrain.ts
index e4092304a9..b38cc13997 100644
--- a/client/src/hooks/Chat/useQueueDrain.ts
+++ b/client/src/hooks/Chat/useQueueDrain.ts
@@ -1,7 +1,7 @@
import { useEffect, useMemo } from 'react';
import { Constants } from 'librechat-data-provider';
import { useRecoilValue, useRecoilCallback } from 'recoil';
-import type { QueuedMessage } from '~/store/families';
+import type { DrainAfterAbort, QueuedMessage, QueuedMessageOrigin, RunEnd } from '~/store/families';
import type { TAskFunction } from '~/common';
import { useMarkFilesUsageMutation } from '~/data-provider';
import store from '~/store';
@@ -36,6 +36,23 @@ const batchFileIds = (fileIds: string[]): string[][] => {
return batches;
};
+const compareQueuedMessages = (a: QueuedMessage, b: QueuedMessage): number =>
+ Number(b.priority ?? false) - Number(a.priority ?? false) || a.createdAt - b.createdAt;
+
+/** Interrupt intent belongs to one generation, never to whichever terminal
+ * event happens to occupy the shared pane slot next. The NEW_CONVO alias is
+ * the one exception: after its first successful start, the terminal event
+ * carries the resolved conversation id while migrating the NEW_CONVO queue. */
+const matchesInterruptArm = (armed: DrainAfterAbort | false, end: RunEnd): boolean => {
+ if (armed === false || end.generationCreatedAt == null) {
+ return false;
+ }
+ const sameConversation =
+ armed.conversationId === end.conversationId ||
+ (end.startedAsNewConvo === true && armed.conversationId === String(Constants.NEW_CONVO));
+ return sameConversation && armed.generationCreatedAt === end.generationCreatedAt;
+};
+
/**
* Auto-sends queued follow-up messages when a run finishes.
*
@@ -106,12 +123,48 @@ export default function useQueueDrain(
return () => clearInterval(timer);
}, [queuedFileIds, markFilesUsage]);
+ /** Move a foreign pane signal to its conversation before the submission
+ * gate. A different conversation can already be submitting on this pane;
+ * waiting for it to become idle lets its own terminal signal replace the
+ * first one. Matching interrupt intent travels with the exact terminal
+ * epoch, while unrelated intent remains armed for its owner. */
+ const parkForeignRunEnd = useRecoilCallback(
+ ({ snapshot, set }) =>
+ (): boolean => {
+ const end = snapshot.getLoadable(store.runEndByIndex(index)).getValue();
+ if (
+ end == null ||
+ end.conversationId == null ||
+ end.conversationId === activeConversationId
+ ) {
+ return false;
+ }
+ const armed = snapshot.getLoadable(store.drainAfterAbortByIndex(index)).getValue();
+ const interruptArmed = matchesInterruptArm(armed, end);
+ if (interruptArmed) {
+ set(store.drainAfterAbortByIndex(index), false);
+ }
+ set(store.pendingRunEndByConvoId(end.conversationId), {
+ ...end,
+ ...((end.interruptArmed === true || interruptArmed) && { interruptArmed: true }),
+ });
+ set(store.runEndByIndex(index), null);
+ return true;
+ },
+ [index, activeConversationId],
+ );
+
// Fully synchronous reads (getLoadable): a useRecoilCallback snapshot is
// only guaranteed valid for the callback's synchronous execution, so no
// awaits may interleave with the reads.
const drainNext = useRecoilCallback(
({ snapshot, set }) =>
- (): { next: QueuedMessage; conversationId: string } | null => {
+ (): {
+ next: QueuedMessage;
+ conversationId: string;
+ queuedMessageOrigin: QueuedMessageOrigin;
+ expectedPredecessorCreatedAt?: number;
+ } | null => {
let end = snapshot.getLoadable(store.runEndByIndex(index)).getValue();
let fromParked = false;
if (
@@ -129,15 +182,16 @@ export default function useQueueDrain(
* this pane consume it (or drain the wrong conversation).
*/
const armedNow = snapshot.getLoadable(store.drainAfterAbortByIndex(index)).getValue();
- if (armedNow) {
+ const interruptArmed = matchesInterruptArm(armedNow, end);
+ if (interruptArmed) {
set(store.drainAfterAbortByIndex(index), false);
}
set(store.pendingRunEndByConvoId(end.conversationId), {
...end,
- ...(armedNow && { interruptArmed: true }),
+ ...((end.interruptArmed === true || interruptArmed) && { interruptArmed: true }),
});
set(store.runEndByIndex(index), null);
- end = null;
+ return null;
}
if (end == null && activeConversationId) {
const parked = snapshot
@@ -160,10 +214,11 @@ export default function useQueueDrain(
}
const indexArmed = snapshot.getLoadable(store.drainAfterAbortByIndex(index)).getValue();
- if (indexArmed) {
+ const matchingIndexArm = matchesInterruptArm(indexArmed, end);
+ if (matchingIndexArm) {
set(store.drainAfterAbortByIndex(index), false);
}
- const interruptArmed = indexArmed || end.interruptArmed === true;
+ const interruptArmed = matchingIndexArm || end.interruptArmed === true;
const conversationId = end.conversationId;
if (!conversationId) {
@@ -178,7 +233,12 @@ export default function useQueueDrain(
const ownQueue = snapshot
.getLoadable(store.queuedMessagesByConvoId(conversationId))
.getValue();
- const merged = [...newConvoQueue, ...ownQueue];
+ /** Both queues are ordered independently, but migration crosses the
+ * key boundary: an interrupt queued under the resolved conversation
+ * must still outrank an ordinary follow-up captured under NEW_CONVO. */
+ const merged = shouldMigrate
+ ? [...newConvoQueue, ...ownQueue].sort(compareQueuedMessages)
+ : ownQueue;
const shouldDrain = end.outcome === 'completed' || interruptArmed;
const next = shouldDrain ? (merged[0] ?? null) : null;
@@ -190,7 +250,19 @@ export default function useQueueDrain(
if (remainder.length !== ownQueue.length || shouldMigrate || next != null) {
set(store.queuedMessagesByConvoId(conversationId), remainder);
}
- return next ? { next, conversationId } : null;
+ return next
+ ? {
+ next,
+ conversationId,
+ queuedMessageOrigin: {
+ item: next,
+ beforeIds: [],
+ afterIds: remainder.map((item) => item.id),
+ },
+ expectedPredecessorCreatedAt:
+ end.generationCreatedAt ?? next.expectedPredecessorCreatedAt,
+ }
+ : null;
},
[index, activeConversationId],
);
@@ -206,6 +278,14 @@ export default function useQueueDrain(
);
useEffect(() => {
+ if (
+ runEnd != null &&
+ runEnd.conversationId != null &&
+ runEnd.conversationId !== activeConversationId
+ ) {
+ parkForeignRunEnd();
+ return;
+ }
if ((runEnd == null && parkedRunEnd == null) || isSubmitting) {
return;
}
@@ -213,16 +293,28 @@ export default function useQueueDrain(
if (drained == null) {
return;
}
- const { next, conversationId } = drained;
+ const { next, conversationId, queuedMessageOrigin, expectedPredecessorCreatedAt } = drained;
// The queued item is the FULL submission context: explicit (possibly
// empty) overrides stop `ask` from vacuuming up files, quotes, or skill
// picks the user has staged in the composer for their NEXT message.
const accepted = ask(
- { text: next.text },
+ {
+ text: next.text,
+ // Recovery can fail after the user-row save but before the parked
+ // source commit. Every attempt upserts the same source-derived row
+ // while its generation idempotency key rotates independently.
+ ...(next.recoverySteerId != null && {
+ overrideUserMessageId: next.recoverySteerId,
+ }),
+ },
{
overrideFiles: next.files ?? [],
overrideQuotes: next.quotes ?? [],
overrideManualSkills: next.manualSkills ?? [],
+ overrideClientRequestId: next.clientRequestId,
+ overrideRecoverySteerId: next.recoverySteerId,
+ overrideExpectedPredecessorCreatedAt: expectedPredecessorCreatedAt,
+ overrideQueuedMessageOrigin: queuedMessageOrigin,
},
);
if (accepted === false) {
@@ -244,6 +336,7 @@ export default function useQueueDrain(
parkedRunEnd,
isSubmitting,
activeConversationId,
+ parkForeignRunEnd,
drainNext,
restoreQueued,
markFilesUsage,
diff --git a/client/src/hooks/Chat/useSteerCancel.ts b/client/src/hooks/Chat/useSteerCancel.ts
index 85cb1f879b..1d7bd76fb1 100644
--- a/client/src/hooks/Chat/useSteerCancel.ts
+++ b/client/src/hooks/Chat/useSteerCancel.ts
@@ -1,7 +1,8 @@
import { useCallback } from 'react';
-import { useRecoilCallback } from 'recoil';
+import { useRecoilCallback, useRecoilValue } from 'recoil';
import type { PendingSteer } from '~/store/families';
import { useCancelSteerMutation } from '~/data-provider';
+import { appendAppliedSteerIds } from '~/utils';
import store from '~/store';
/**
@@ -24,29 +25,74 @@ export type SteerCancelOutcome = 'reclaimed' | 'applied' | 'failed';
*/
export function useSteerReclaim(conversationId: string) {
const cancelMutation = useCancelSteerMutation();
+ const activeGenerationCreatedAt = useRecoilValue(
+ store.activeGenerationCreatedAtByConvoId(conversationId),
+ );
+ /** A confirmed reclaim is a terminal settlement too. Tombstone both ids so
+ * a final payload captured before the server discard cannot requeue it, and
+ * remove a recovered queue copy if the opposite ordering already occurred. */
+ const settleReclaimed = useRecoilCallback(
+ ({ set }) =>
+ (steer: PendingSteer) => {
+ const ids = [steer.steerId, ...(steer.clientSteerId ? [steer.clientSteerId] : [])];
+ const settled = new Set(ids);
+ set(store.appliedSteerIdsByConvoId(conversationId), (prev) =>
+ appendAppliedSteerIds(prev, ids),
+ );
+ set(store.pendingSteersByConvoId(conversationId), (prev) =>
+ prev.filter(
+ (item) =>
+ !settled.has(item.steerId) &&
+ (item.clientSteerId == null || !settled.has(item.clientSteerId)),
+ ),
+ );
+ set(store.queuedMessagesByConvoId(conversationId), (prev) =>
+ prev.filter(
+ (item) =>
+ !settled.has(item.id) &&
+ (item.recoverySteerId == null || !settled.has(item.recoverySteerId)) &&
+ (item.recoveryClientSteerId == null || !settled.has(item.recoveryClientSteerId)),
+ ),
+ );
+ },
+ [conversationId],
+ );
return useCallback(
async (steer: PendingSteer): Promise<SteerCancelOutcome> => {
try {
+ const generationCreatedAt =
+ steer.generationCreatedAt ?? activeGenerationCreatedAt ?? undefined;
+ if (generationCreatedAt == null) {
+ return 'failed';
+ }
const { removed } = await cancelMutation.mutateAsync({
conversationId,
steerId: steer.steerId,
+ ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
+ generationCreatedAt,
});
- return removed === true ? 'reclaimed' : 'applied';
+ if (removed === true) {
+ settleReclaimed(steer);
+ return 'reclaimed';
+ }
+ return 'applied';
} catch {
return 'failed';
}
},
- [conversationId, cancelMutation],
+ [conversationId, cancelMutation, settleReclaimed, activeGenerationCreatedAt],
);
}
/**
- * Cancels a steer still waiting on its injection boundary. Optimistic: the
- * entry leaves the chip stack immediately; `removed: false` needs no handling
- * (the steer already injected or the run ended — the events own the outcome).
- * Only a failed POST restores the entry, since the server would still inject
- * the supposedly-cancelled words.
+ * Cancels a steer still waiting on its injection boundary. The chip stays
+ * visible until the request settles so capability/update events can still
+ * patch its current server revision while cancel is in flight. Only a
+ * confirmed reclaim removes it here. `removed:false` is ambiguous in the v1
+ * protocol (already injected OR terminal conversion won), so the event path
+ * must decide whether to remove or recover that entry without discarding its
+ * client-only files/quotes/skills first.
*/
export default function useSteerCancel(conversationId: string) {
const reclaim = useSteerReclaim(conversationId);
@@ -60,35 +106,14 @@ export default function useSteerCancel(conversationId: string) {
},
[conversationId],
);
- const restoreEntry = useRecoilCallback(
- ({ snapshot, set }) =>
- (entry: PendingSteer) => {
- /* A steer that settled while the POST was in flight — applied on the
- * server, or converted to a queued follow-up at run end — must NOT come
- * back. The next run (a queue drain auto-sends one) would render this
- * stale entry as an in-flight bubble beside its own queued copy. */
- const settled = snapshot
- .getLoadable(store.appliedSteerIdsByConvoId(conversationId))
- .getValue();
- if (settled.includes(entry.steerId)) {
- return;
- }
- set(store.pendingSteersByConvoId(conversationId), (prev) =>
- prev.some((item) => item.steerId === entry.steerId) ? prev : [...prev, entry],
- );
- },
- [conversationId],
- );
-
return useCallback(
async (steer: PendingSteer): Promise<SteerCancelOutcome> => {
- removeEntry(steer.steerId);
const outcome = await reclaim(steer);
- if (outcome === 'failed') {
- restoreEntry(steer);
+ if (outcome === 'reclaimed') {
+ removeEntry(steer.steerId);
}
return outcome;
},
- [reclaim, removeEntry, restoreEntry],
+ [reclaim, removeEntry],
);
}
diff --git a/client/src/hooks/Chat/useSteerConvert.ts b/client/src/hooks/Chat/useSteerConvert.ts
index 2e4912f076..59229566d1 100644
--- a/client/src/hooks/Chat/useSteerConvert.ts
+++ b/client/src/hooks/Chat/useSteerConvert.ts
@@ -1,24 +1,34 @@
import { useCallback } from 'react';
+import { v4 } from 'uuid';
import { useRecoilCallback } from 'recoil';
import type { TPendingSteer } from 'librechat-data-provider';
-import type { QueuedMessage } from '~/store/families';
+import type { QueuedMessage, QueuedMessageOrigin } from '~/store/families';
+import type { GenerationProtocolVersion } from '~/data-provider';
import type { SteerCarriedContext } from '~/utils';
-import { appendAppliedSteerIds, carriedSteerContext } from '~/utils';
-import { fetchStreamStatus } from '~/data-provider';
+import { appendAppliedSteerIds, carriedSteerContext, insertQueuedOrigin } from '~/utils';
+import { fetchStreamStatus, getGenerationProtocolVersion } from '~/data-provider';
import store from '~/store';
/** A server-reported steer, or a local one that carries its own client-only
* context (quotes / skill picks) because its chip may already be gone. */
-type ConvertibleSteer = TPendingSteer & SteerCarriedContext;
+type ConvertibleSteer = TPendingSteer &
+ SteerCarriedContext & { queuedOrigin?: QueuedMessageOrigin };
interface SteerConvertOptions {
/** Live-delivered terminal steers also have a parked server copy (the
* terminal drain parks before knowing the final reached a subscriber);
- * set on final/abort/error surfaces to claim-and-clear it. */
+ * set on final/abort/error surfaces to reconcile that durable copy. */
claimParked?: boolean;
/** Server-side id the parked copy is keyed under when it differs from the
* chip landing key (a `new`-held abort claims under the resolved job id). */
claimConversationId?: string;
+ /** Protocol selected for the source generation. V2 may bind the queued
+ * follow-up to a durable receipt; v1 must remain an ordinary local item. */
+ generationProtocolVersion?: GenerationProtocolVersion;
+ /** Exact recovery sources that a post-terminal status read proved were not
+ * consumed by THIS recovery generation. Never set for an arbitrary or
+ * pre-start status response: those can be stale after a successful start. */
+ allowPreviouslyConvertedIds?: readonly string[];
}
/**
@@ -31,26 +41,43 @@ interface SteerConvertOptions {
* insertion both dedupe by steer id.
*
* `claimParked` fires ONE fire-and-forget /chat/status fetch per conversion
- * batch: its claim-on-read consumes the parked copy of steers just delivered
- * live, so a later reload cannot resurrect chips the user already dismissed.
- * Claimed steers re-run the same id-deduped conversion (no double-add).
- * Residual race: the claim no-ops if the job hasn't gone terminal by fetch
- * time — acceptable; the parked-copy TTL still bounds it.
+ * batch. The replayable response reconciles the durable parked copy through
+ * the same id-deduped conversion, so a lost final/status response cannot erase
+ * the only recovery source. Exact recovery or explicit dismissal commits the
+ * server-side removal.
*/
export default function useSteerConvert() {
const convert = useRecoilCallback(
({ snapshot, set }) =>
- (conversationId: string, steers: ConvertibleSteer[]) => {
+ (
+ conversationId: string,
+ steers: ConvertibleSteer[],
+ generationProtocolVersion?: GenerationProtocolVersion,
+ allowPreviouslyConvertedIds: readonly string[] = [],
+ ) => {
if (steers.length === 0) {
return;
}
+ const negotiatedVersion =
+ generationProtocolVersion ??
+ snapshot
+ .getLoadable(store.activeGenerationProtocolVersionByConvoId(conversationId))
+ .getValue();
+ const bindRecoverySource = negotiatedVersion === 2;
// Quotes/skill picks never ride the server steer; restore them from
// the local chip (matched by id) before the chips are dropped below.
const localChips = snapshot
.getLoadable(store.pendingSteersByConvoId(conversationId))
.getValue();
const chipById = new Map(localChips.map((chip) => [chip.steerId, chip]));
- const steerIds = new Set(steers.map((steer) => steer.steerId));
+ const localChipFor = (steer: ConvertibleSteer) =>
+ chipById.get(steer.steerId) ??
+ (steer.clientSteerId ? chipById.get(steer.clientSteerId) : undefined);
+ const steerIds = new Set(
+ steers.flatMap((steer) =>
+ steer.clientSteerId ? [steer.steerId, steer.clientSteerId] : [steer.steerId],
+ ),
+ );
// Steers already settled (applied on the server OR converted here on an
// earlier delivery) must not re-enter the queue. Read BEFORE the append
// below so a first-time conversion still queues, but a redelivery whose
@@ -60,44 +87,94 @@ export default function useSteerConvert() {
const settledSteerIds = new Set(
snapshot.getLoadable(store.appliedSteerIdsByConvoId(conversationId)).getValue(),
);
+ const allowedRedeliveries = new Set(allowPreviouslyConvertedIds);
set(store.appliedSteerIdsByConvoId(conversationId), (prev) =>
- appendAppliedSteerIds(
- prev,
- steers.map((steer) => steer.steerId),
- ),
+ appendAppliedSteerIds(prev, [...steerIds]),
);
set(store.pendingSteersByConvoId(conversationId), (prev) =>
prev.filter((steer) => !steerIds.has(steer.steerId)),
);
set(store.queuedMessagesByConvoId(conversationId), (prev) => {
+ /** A legacy status read is destructive: if a v2 live-final path
+ * already created a receipt-bound item before the claim reached an
+ * old replica, that source no longer exists. Downgrade the existing
+ * item in place to an ordinary local follow-up. */
+ const existing = bindRecoverySource
+ ? prev
+ : prev.map((item) => {
+ const matchesClaimedSource =
+ (item.recoverySteerId != null && steerIds.has(item.recoverySteerId)) ||
+ (item.recoveryClientSteerId != null && steerIds.has(item.recoveryClientSteerId));
+ if (!matchesClaimedSource) {
+ return item;
+ }
+ const {
+ clientRequestId: _clientRequestId,
+ recoverySteerId: _recoverySteerId,
+ recoveryClientSteerId: _recoveryClientSteerId,
+ ...ordinary
+ } = item;
+ return ordinary;
+ });
const fresh = steers
.filter(
(steer) =>
- !settledSteerIds.has(steer.steerId) &&
- !prev.some((queued) => queued.id === steer.steerId),
+ allowedRedeliveries.has(steer.steerId) ||
+ (!settledSteerIds.has(steer.steerId) &&
+ (steer.clientSteerId == null || !settledSteerIds.has(steer.clientSteerId))),
)
- .map((steer) => ({
- id: steer.steerId,
- text: steer.text,
- createdAt: steer.createdAt ?? Date.now(),
- ...(steer.files && steer.files.length > 0 && { files: steer.files }),
- // The chip is the usual source, but a reclaimed steer may have
- // lost its chip to a competing cancel mid-round-trip — it carries
- // the context itself so the picks survive either way.
- ...carriedSteerContext(chipById.get(steer.steerId) ?? steer),
- }));
+ .map((steer) => {
+ const local = localChipFor(steer);
+ const source = local ?? steer;
+ const queuedOrigin = source.queuedOrigin;
+ const recoveryFields = bindRecoverySource
+ ? {
+ // One UUID is stable for this queued attempt and all of
+ // its POST retries. A later failed generation re-converts
+ // the durable source and receives a new key, so the old
+ // started idempotency tombstone cannot make it unsendable.
+ clientRequestId: v4(),
+ recoverySteerId: steer.steerId,
+ ...(steer.clientSteerId && { recoveryClientSteerId: steer.clientSteerId }),
+ }
+ : {};
+ const item =
+ queuedOrigin != null
+ ? { ...queuedOrigin.item, ...recoveryFields }
+ : ({
+ id: steer.steerId,
+ text: steer.text,
+ createdAt: steer.createdAt ?? Date.now(),
+ ...recoveryFields,
+ ...(steer.files && steer.files.length > 0 && { files: steer.files }),
+ // The chip is the usual source, but a reclaimed steer may
+ // have lost its chip to a competing cancel mid-round-trip.
+ ...carriedSteerContext(source),
+ } satisfies QueuedMessage);
+ return {
+ item,
+ queuedOrigin: queuedOrigin != null ? { ...queuedOrigin, item } : undefined,
+ };
+ })
+ .filter(({ item }) => !existing.some((queued) => queued.id === item.id));
if (fresh.length === 0) {
- return prev;
+ return existing;
}
- // Merge chronologically — a steer accepted BEFORE the user queued a
- // later follow-up must drain first — EXCEPT explicit front-inserts
- // ("Interrupt & send"), whose urgency outranks age.
- const merged: QueuedMessage[] = [...prev, ...fresh];
- return merged.sort(
+ /** Ordinary steer leftovers merge chronologically. A steer that
+ * originated in this queue restores its exact object/identity and
+ * captured position instead of being re-minted under the server id. */
+ const ordinary = fresh.filter(({ queuedOrigin }) => queuedOrigin == null);
+ let merged: QueuedMessage[] = [...existing, ...ordinary.map(({ item }) => item)].sort(
(a, b) =>
Number(b.priority ?? false) - Number(a.priority ?? false) ||
a.createdAt - b.createdAt,
);
+ for (const { queuedOrigin } of fresh) {
+ if (queuedOrigin != null) {
+ merged = insertQueuedOrigin(merged, queuedOrigin);
+ }
+ }
+ return merged;
});
},
[],
@@ -105,7 +182,12 @@ export default function useSteerConvert() {
return useCallback(
(conversationId: string, steers: ConvertibleSteer[], options?: SteerConvertOptions) => {
- convert(conversationId, steers);
+ convert(
+ conversationId,
+ steers,
+ options?.generationProtocolVersion,
+ options?.allowPreviouslyConvertedIds,
+ );
if (options?.claimParked !== true || steers.length === 0) {
return;
}
@@ -113,7 +195,7 @@ export default function useSteerConvert() {
.then((status) => {
const unrecovered = status.unrecoveredSteers ?? [];
if (unrecovered.length > 0) {
- convert(conversationId, unrecovered);
+ convert(conversationId, unrecovered, getGenerationProtocolVersion(status));
}
})
.catch(() => undefined);
diff --git a/client/src/hooks/Chat/useSteering.ts b/client/src/hooks/Chat/useSteering.ts
index 0799f0a787..c348a7376f 100644
--- a/client/src/hooks/Chat/useSteering.ts
+++ b/client/src/hooks/Chat/useSteering.ts
@@ -1,17 +1,25 @@
-import { useCallback, useMemo, useRef } from 'react';
+import { useCallback, useEffect, useMemo, useRef } from 'react';
import { v4 } from 'uuid';
import { useToastContext } from '@librechat/client';
import { useRecoilValue, useSetRecoilState, useRecoilCallback } from 'recoil';
import { Constants, ContentTypes, isAssistantsEndpoint } from 'librechat-data-provider';
import type { TMessage, TConversation, TMessageContentParts } from 'librechat-data-provider';
-import type { RunEnd, PendingSteer, QueuedMessage } from '~/store/families';
+import type { RunEnd, PendingSteer, QueuedMessage, QueuedMessageOrigin } from '~/store/families';
+import type { GenerationProtocolVersion } from '~/data-provider';
import type { ExtendedFile, FileSetter } from '~/common';
import {
useGetMessagesByConvoId,
+ useCancelSteerMutation,
useSteerMessageMutation,
useMarkFilesUsageMutation,
+ supportsGenerationProtocolV2,
} from '~/data-provider';
-import { carriedSteerContext, clearAllDrafts } from '~/utils';
+import {
+ appendAppliedSteerIds,
+ carriedSteerContext,
+ clearAllDrafts,
+ insertQueuedOrigin,
+} from '~/utils';
import useSteerConvert from '~/hooks/Chat/useSteerConvert';
import { useSetFilesToDelete } from '~/hooks/Files';
import useLocalize from '~/hooks/useLocalize';
@@ -24,6 +32,10 @@ export type DuringRunAction = 'steer' | 'queue';
export interface QueuedMessageContext {
quotes?: string[];
manualSkills?: string[];
+ clientRequestId?: string;
+ recoverySteerId?: string;
+ expectedPredecessorCreatedAt?: number;
+ queuedMessageOrigin?: QueuedMessageOrigin;
}
/** Server-side cap on a usage touch (mirrors `FILES_USAGE_MAX_IDS`). */
@@ -32,15 +44,55 @@ const QUEUE_USAGE_MAX_FILES = 10;
type SteerErrorCode =
| 'NO_ACTIVE_RUN'
| 'RUN_PAUSED'
+ | 'RUN_REPLACED'
| 'STEER_QUEUE_FULL'
| 'STEER_UNSUPPORTED'
| string;
+type SubmitSteerOptions = {
+ preempt?: boolean;
+ queuedOrigin?: QueuedMessageOrigin;
+ /** Stable across transport retries so the server can dedupe a committed POST. */
+ clientSteerId?: string;
+ /** Preserve the generation identity on retries instead of targeting a newer
+ * turn that may now occupy the same conversation-scoped stream id. */
+ generationCreatedAt?: number;
+ /** Same-id retries retain the original submission timestamp so a receipt
+ * replay cannot reorder this steer behind a later queued follow-up. */
+ createdAt?: number;
+ /** Interpret a retry receipt using the source generation's negotiated
+ * protocol, not whichever replacement generation is currently active. */
+ generationProtocolVersion?: GenerationProtocolVersion;
+};
+
function getSteerErrorCode(error: unknown): SteerErrorCode | undefined {
const response = (error as { response?: { data?: { code?: string } } } | undefined)?.response;
return response?.data?.code;
}
+function isDefiniteSteerRejection(error: unknown): boolean {
+ const status = (error as { response?: { status?: number } } | undefined)?.response?.status;
+ return (
+ status != null &&
+ status >= 400 &&
+ status < 500 &&
+ // These statuses can be generated by an intermediary while the origin
+ // request continues; keep their delivery outcome explicitly uncertain.
+ status !== 408 &&
+ status !== 425
+ );
+}
+
+function isSameRunEpoch(a: RunEnd | null, b: RunEnd): boolean {
+ if (a == null || a.conversationId !== b.conversationId) {
+ return false;
+ }
+ if (a.generationCreatedAt != null || b.generationCreatedAt != null) {
+ return a.generationCreatedAt === b.generationCreatedAt;
+ }
+ return a.endedAt === b.endedAt;
+}
+
/** True when the latest assistant message carries an unresolved tool approval —
* the run is (or is about to be) paused, so a steer POST would 409. */
function hasLiveToolApproval(messages: TMessage[] | undefined): boolean {
@@ -121,6 +173,7 @@ export default function useSteering({
/** `mutate` is a stable callback; the mutation result objects are fresh
* every render and would defeat the memoized return value below. */
const { mutate: steerMessage } = useSteerMessageMutation();
+ const { mutateAsync: cancelSteer } = useCancelSteerMutation();
const { mutate: markFilesUsage } = useMarkFilesUsageMutation();
const defaultAction = useRecoilValue<DuringRunAction>(store.duringRunDefaultAction);
const setDefaultAction = useSetRecoilState(store.duringRunDefaultAction);
@@ -132,8 +185,117 @@ export default function useSteering({
conversationId != null && conversationId !== '' && conversationId !== Constants.NEW_CONVO;
/** v1 gates the during-run UI to the primary composer, like the HITL popover. */
const enabled = steerable && index === 0;
- const duringRunActive = enabled && isSubmitting && !answerModeActive;
const queueKey = hasRealConvoId ? conversationId : Constants.NEW_CONVO;
+ const activeGenerationCreatedAt = useRecoilValue(
+ store.activeGenerationCreatedAtByConvoId(queueKey),
+ );
+ const activeGenerationProtocolVersion = useRecoilValue(
+ store.activeGenerationProtocolVersionByConvoId(queueKey),
+ );
+ /** The start POST installs this epoch before any live mutation may be sent.
+ * `isSubmitting` flips earlier so the user can keep typing; during that
+ * bounded interval submits degrade to the local queue and Stop/steer refuse. */
+ const canControlGeneration = activeGenerationCreatedAt != null;
+ const duringRunActive = enabled && isSubmitting && !answerModeActive;
+ /** Queue rows can be sent concurrently. Keep their captured origins while
+ * absent so a later capture still sees the complete logical queue. Origins
+ * are isolated per conversation because this hook survives navigation. */
+ const queuedOriginsRef = useRef<Map<string, Map<string, QueuedMessageOrigin>>>(new Map());
+ /** First-party FIFO admission: validation inside the steer route can await
+ * agent ACL and attachment reads, so overlapping POSTs could otherwise reach
+ * the durable queue out of submission order. Keep one request in flight per
+ * conversation; later optimistic chips retain the epoch/id/time captured at
+ * submission and dispatch only after the prior request settles. */
+ const steerDispatchQueuesRef = useRef<
+ Map<
+ string,
+ {
+ active: boolean;
+ pending: Array<(settle: () => void) => void>;
+ }
+ >
+ >(new Map());
+ const scheduleSteerDispatch = useCallback(
+ (convoId: string, dispatch: (settle: () => void) => void) => {
+ const existing = steerDispatchQueuesRef.current.get(convoId);
+ const state = existing ?? { active: false, pending: [] };
+ if (existing == null) {
+ steerDispatchQueuesRef.current.set(convoId, state);
+ }
+ state.pending.push(dispatch);
+
+ const startNext = () => {
+ if (state.active) {
+ return;
+ }
+ const next = state.pending[0];
+ if (next == null) {
+ if (steerDispatchQueuesRef.current.get(convoId) === state) {
+ steerDispatchQueuesRef.current.delete(convoId);
+ }
+ return;
+ }
+ state.active = true;
+ let settled = false;
+ const settle = () => {
+ if (settled) {
+ return;
+ }
+ settled = true;
+ const index = state.pending.indexOf(next);
+ if (index >= 0) {
+ state.pending.splice(index, 1);
+ }
+ state.active = false;
+ startNext();
+ };
+ try {
+ next(settle);
+ } catch (error) {
+ settle();
+ throw error;
+ }
+ };
+
+ startNext();
+ },
+ [],
+ );
+ /** `ask()` does not flip `isSubmitting` until the submission effect renders.
+ * Two queued-row clicks in the same browser task would otherwise both remove
+ * their items and the second submission could overwrite the first. */
+ const idleSendInFlightRef = useRef(false);
+ const pendingSteers = useRecoilValue(store.pendingSteersByConvoId(queueKey));
+
+ useEffect(() => {
+ if (!isSubmitting) {
+ idleSendInFlightRef.current = false;
+ }
+ }, [isSubmitting]);
+
+ /** Keep logical queue origins through the 202 ACK. They are needed to
+ * reconstruct FIFO if several accepted queue items later return as terminal
+ * leftovers. Once no live/failed chip references an origin, application or
+ * conversion has settled it and the reconstruction entry can be released. */
+ useEffect(() => {
+ const origins = queuedOriginsRef.current.get(queueKey);
+ if (origins == null) {
+ return;
+ }
+ const referenced = new Set(
+ pendingSteers.flatMap((steer) =>
+ steer.queuedOrigin != null ? [steer.queuedOrigin.item.id] : [],
+ ),
+ );
+ for (const originId of origins.keys()) {
+ if (!referenced.has(originId)) {
+ origins.delete(originId);
+ }
+ }
+ if (origins.size === 0) {
+ queuedOriginsRef.current.delete(queueKey);
+ }
+ }, [pendingSteers, queueKey]);
/** Boolean `select` so streaming deltas don't notify this subscription:
* structural sharing only re-renders the composer when the flag flips. */
@@ -144,12 +306,22 @@ export default function useSteering({
select: hasLiveToolApproval,
},
);
- const pausedOnApproval = duringRunActive ? (liveToolApproval ?? false) : false;
+ /** Both approval cards and `ask_user_question` suspend the current
+ * generation while keeping its submission slot occupied. Answer mode hides
+ * the ordinary during-run composer, but waiting-message controls still need
+ * to know a live generation exists so they remain discoverable and disabled
+ * instead of looking as though the action disappeared. */
+ const pausedOnApproval =
+ enabled && isSubmitting ? answerModeActive || (liveToolApproval ?? false) : false;
/** Whether a steer can reach the live run right now — independent of the
* user's default action, so the per-send menu can always override to
* steer when it's genuinely available. */
- const canSteer = hasRealConvoId && !pausedOnApproval;
+ const canSteer = hasRealConvoId && canControlGeneration && !pausedOnApproval;
+ /** Whether a queued row has a real immediate-send path. Answer mode keeps
+ * `isSubmitting` true while hiding during-run steering, so presenting Send
+ * now there would be an enabled no-op. */
+ const canSendQueuedNow = !isSubmitting || (duringRunActive && canSteer);
/** Steering needs a live server-side job; degrade to queue otherwise. */
const effectiveAction: DuringRunAction = canSteer ? defaultAction : 'queue';
@@ -157,6 +329,8 @@ export default function useSteering({
* stale by the time the steer response arrives. */
const isSubmittingRef = useRef(isSubmitting);
isSubmittingRef.current = isSubmitting;
+ const visibleConversationRef = useRef(conversationId);
+ visibleConversationRef.current = conversationId;
/**
* How each conversation's last run ended, kept because `useQueueDrain`
@@ -220,34 +394,103 @@ export default function useSteering({
*/
const acknowledgeSteer = useRecoilCallback(
({ snapshot, set }) =>
- (convoId: string, localId: string, steer: PendingSteer) => {
+ (convoId: string, localId: string, steer: PendingSteer): boolean => {
const applied = snapshot.getLoadable(store.appliedSteerIdsByConvoId(convoId)).getValue();
- const alreadyApplied = applied.includes(steer.steerId);
- const runOver = !isSubmittingRef.current;
+ const alreadyApplied = applied.includes(steer.steerId) || applied.includes(localId);
+ const alreadyAccepted = snapshot
+ .getLoadable(store.acceptedSteerClientIdsByConvoId(convoId))
+ .getValue()
+ .includes(localId);
+ const activeEpoch = snapshot
+ .getLoadable(store.activeGenerationCreatedAtByConvoId(convoId))
+ .getValue();
+ /** A replacement can already be submitting when an older generation's
+ * delayed 202 arrives. `isSubmitting` alone would re-mint the old chip
+ * after replacement reconciliation, with no old apply event left to
+ * settle it. Every POST reaching this callback captured a non-null
+ * source epoch before dispatch, so either a mismatch or a later clear
+ * proves that source generation is no longer active. */
+ const runReplaced =
+ steer.generationCreatedAt != null &&
+ (activeEpoch == null || steer.generationCreatedAt !== activeEpoch);
+ // This hook instance survives navigation. The visible pane's idle flag
+ // says nothing about an off-screen origin run, so only infer ordinary
+ // terminal state while the callback still belongs to the visible chat.
+ const runOver =
+ runReplaced || (visibleConversationRef.current === convoId && !isSubmittingRef.current);
+ set(store.acceptedSteerClientIdsByConvoId(convoId), (prev) =>
+ appendAppliedSteerIds(prev, [localId]),
+ );
set(store.pendingSteersByConvoId(convoId), (prev) => {
+ const local = prev.find((item) => item.steerId === localId);
+ let acknowledged = steer;
+ if ((local?.preemptRevision ?? 0) > (steer.preemptRevision ?? 0)) {
+ const { preempt: _stalePreempt, ...withoutStaleLabel } = steer;
+ acknowledged = {
+ ...withoutStaleLabel,
+ ...(local?.preempt === true && { preempt: true }),
+ preemptRevision: local?.preemptRevision,
+ };
+ }
const next = prev.filter((item) => item.steerId !== localId);
// Upsert: an SSE reconnect may have reseeded the chip under the
// server id already — appending again would duplicate it.
const alreadySeeded = next.some((item) => item.steerId === steer.steerId);
- return alreadyApplied || runOver || alreadySeeded ? next : [...next, steer];
+ return alreadyApplied || alreadyAccepted || runOver || alreadySeeded
+ ? next
+ : [...next, acknowledged];
});
+ /** `alreadyAccepted` can come from the correlation update that precedes
+ * the delayed HTTP response; it proves server ownership, not delivery.
+ * Once the source epoch is over/replaced we still must recover it. */
if (alreadyApplied || !runOver) {
- return;
+ return false;
}
- set(store.queuedMessagesByConvoId(convoId), (prev) =>
- prev.some((queued) => queued.id === steer.steerId)
- ? prev
- : [
- ...prev,
- {
- id: steer.steerId,
- text: steer.text,
- createdAt: steer.createdAt,
- ...(steer.files && steer.files.length > 0 && { files: steer.files }),
- ...carriedSteerContext(steer),
- },
- ],
+ // The server accepted but this visible run is already terminal. Its
+ // terminal close may have parked the item, so the caller must route it
+ // through the recovery converter (deterministic next-turn id), not an
+ // ordinary editable queue row.
+ return true;
+ },
+ [],
+ );
+
+ const settleReceiptReplay = useRecoilCallback(
+ ({ set }) =>
+ (convoId: string, localId: string, serverId: string) => {
+ set(store.appliedSteerIdsByConvoId(convoId), (prev) =>
+ appendAppliedSteerIds(prev, [localId, serverId]),
+ );
+ set(store.acceptedSteerClientIdsByConvoId(convoId), (prev) =>
+ appendAppliedSteerIds(prev, [localId]),
);
+ set(store.pendingSteersByConvoId(convoId), (prev) =>
+ prev.filter((steer) => steer.steerId !== localId && steer.steerId !== serverId),
+ );
+ },
+ [],
+ );
+
+ /** Terminal/applied SSE can settle the optimistic client id before either
+ * HTTP callback. Those callbacks must become no-ops or a lost response can
+ * resurrect already-delivered words as a failed chip or queued fallback. */
+ const isSteerAcceptedOrSettled = useRecoilCallback(
+ ({ snapshot }) =>
+ (convoId: string, clientSteerId: string) => {
+ const applied = snapshot.getLoadable(store.appliedSteerIdsByConvoId(convoId)).getValue();
+ if (applied.includes(clientSteerId)) {
+ return true;
+ }
+ const accepted = snapshot
+ .getLoadable(store.acceptedSteerClientIdsByConvoId(convoId))
+ .getValue();
+ if (accepted.includes(clientSteerId)) {
+ return true;
+ }
+ return snapshot
+ .getLoadable(store.pendingSteersByConvoId(convoId))
+ .getValue()
+ .some((steer) => steer.status === 'pending' && steer.clientSteerId === clientSteerId);
},
[],
);
@@ -290,6 +533,10 @@ export default function useSteering({
/** Set when the files were ALREADY queued/steered: their TTL was
* held when they first entered the queue (or at the steer 202). */
skipUsageMark?: boolean;
+ id?: string;
+ createdAt?: number;
+ clientRequestId?: string;
+ expectedPredecessorCreatedAt?: number;
},
) => {
const trimmed = text.trim();
@@ -297,9 +544,14 @@ export default function useSteering({
return;
}
const item = {
- id: v4(),
+ id: options?.id ?? v4(),
text: trimmed,
- createdAt: Date.now(),
+ createdAt: options?.createdAt ?? Date.now(),
+ ...(options?.clientRequestId && { clientRequestId: options.clientRequestId }),
+ ...((options?.expectedPredecessorCreatedAt ?? activeGenerationCreatedAt) != null && {
+ expectedPredecessorCreatedAt:
+ options?.expectedPredecessorCreatedAt ?? activeGenerationCreatedAt ?? undefined,
+ }),
...(options?.files && options.files.length > 0 && { files: options.files }),
...(options?.quotes && options.quotes.length > 0 && { quotes: options.quotes }),
...(options?.manualSkills &&
@@ -307,13 +559,17 @@ export default function useSteering({
...(options?.front && { priority: true }),
};
set(store.queuedMessagesByConvoId(queueKey), (prev) =>
- options?.front ? [item, ...prev] : [...prev, item],
+ [...prev, item].sort(
+ (a, b) =>
+ Number(b.priority ?? false) - Number(a.priority ?? false) ||
+ a.createdAt - b.createdAt,
+ ),
);
if (options?.skipUsageMark !== true) {
markQueuedFilesUsage(options?.files);
}
},
- [queueKey, markQueuedFilesUsage],
+ [queueKey, markQueuedFilesUsage, activeGenerationCreatedAt],
);
/** Consumes completed composer attachments into message file refs (clearing
@@ -387,29 +643,139 @@ export default function useSteering({
[queueKey],
);
- /** Capture-then-remove, so a refused send can restore the ORIGINAL item. */
+ /** Once a parked source is discarded it must never be retried as a recovery
+ * attempt. Downgrade the row in place so a guarded Edit that finds a newer
+ * draft can leave the same words, context, identity, and queue position as
+ * an ordinary local follow-up. */
+ const downgradeQueuedRecovery = useRecoilCallback(
+ ({ snapshot, set }) =>
+ (id: string): boolean => {
+ const queue = snapshot.getLoadable(store.queuedMessagesByConvoId(queueKey)).getValue();
+ let found = false;
+ const next = queue.map((item) => {
+ if (item.id !== id) {
+ return item;
+ }
+ found = true;
+ const {
+ clientRequestId: _clientRequestId,
+ recoverySteerId: _recoverySteerId,
+ recoveryClientSteerId: _recoveryClientSteerId,
+ ...ordinary
+ } = item;
+ return ordinary;
+ });
+ if (found) {
+ set(store.queuedMessagesByConvoId(queueKey), next);
+ }
+ return found;
+ },
+ [queueKey],
+ );
+
+ /** Settle a queued row's terminal recovery source before an Edit/Remove.
+ * Ordinary rows have no server copy. A v2 leftover first uses its durable
+ * receipt to atomically discard the parked copy, then becomes an ordinary
+ * local row; the caller decides whether the live composer can consume it.
+ * The cancel deliberately omits the active epoch: the receipt belongs to the
+ * terminal source generation, not whichever run now occupies the chat. */
+ const discardQueued = useCallback(
+ async (item: QueuedMessage): Promise<boolean> => {
+ if (item.recoverySteerId == null) {
+ return true;
+ }
+ if (item.recoveryClientSteerId == null || !hasRealConvoId) {
+ showToast({ message: localize('com_ui_steer_cancel_failed'), status: 'error' });
+ return false;
+ }
+ try {
+ const { removed } = await cancelSteer({
+ conversationId,
+ steerId: item.recoverySteerId,
+ clientSteerId: item.recoveryClientSteerId,
+ });
+ if (removed !== true) {
+ showToast({ message: localize('com_ui_steer_cancel_failed'), status: 'error' });
+ return false;
+ }
+ return downgradeQueuedRecovery(item.id);
+ } catch {
+ showToast({ message: localize('com_ui_steer_cancel_failed'), status: 'error' });
+ return false;
+ }
+ },
+ [cancelSteer, conversationId, downgradeQueuedRecovery, hasRealConvoId, localize, showToast],
+ );
+
+ /** Capture-then-remove, including the item's neighbours, so any refused send
+ * or rejected steer can restore the ORIGINAL item in place even if the run
+ * drains an adjacent entry while the request is in flight. */
const takeQueued = useRecoilCallback(
({ snapshot, set }) =>
- (id: string): QueuedMessage | undefined => {
+ (id: string): QueuedMessageOrigin | undefined => {
const queue = snapshot.getLoadable(store.queuedMessagesByConvoId(queueKey)).getValue();
- const taken = queue.find((item) => item.id === id);
+ const index = queue.findIndex((item) => item.id === id);
+ if (index < 0) {
+ return undefined;
+ }
+ const item = queue[index];
+ let queuedOrigins = queuedOriginsRef.current.get(queueKey);
+ if (queuedOrigins == null) {
+ queuedOrigins = new Map();
+ queuedOriginsRef.current.set(queueKey, queuedOrigins);
+ }
+ let logicalQueue = queue;
+ for (const pendingOrigin of queuedOrigins.values()) {
+ logicalQueue = insertQueuedOrigin(logicalQueue, pendingOrigin);
+ }
+ const logicalIndex = logicalQueue.findIndex((queued) => queued.id === id);
+ const origin = {
+ item,
+ beforeIds: logicalQueue.slice(0, logicalIndex).map((queued) => queued.id),
+ afterIds: logicalQueue.slice(logicalIndex + 1).map((queued) => queued.id),
+ };
+ queuedOrigins.set(id, origin);
set(store.queuedMessagesByConvoId(queueKey), (prev) =>
prev.filter((item) => item.id !== id),
);
- return taken;
+ return origin;
},
[queueKey],
);
- /** Front restore mirroring useQueueDrain: same id, never duplicated. */
+ const releaseQueuedOrigin = useCallback(
+ (origin: QueuedMessageOrigin) => {
+ const queuedOrigins = queuedOriginsRef.current.get(queueKey);
+ queuedOrigins?.delete(origin.item.id);
+ if (queuedOrigins?.size === 0) {
+ queuedOriginsRef.current.delete(queueKey);
+ }
+ },
+ [queueKey],
+ );
+
+ const registerQueuedOrigin = useCallback(
+ (origin: QueuedMessageOrigin) => {
+ let queuedOrigins = queuedOriginsRef.current.get(queueKey);
+ if (queuedOrigins == null) {
+ queuedOrigins = new Map();
+ queuedOriginsRef.current.set(queueKey, queuedOrigins);
+ }
+ queuedOrigins.set(origin.item.id, origin);
+ },
+ [queueKey],
+ );
+
+ /** Restore the exact item between its nearest surviving original neighbours,
+ * never duplicated. If every neighbour disappeared, newly front-prioritized
+ * entries stay ahead while ordinary entries created later stay behind. */
const restoreQueued = useRecoilCallback(
({ set }) =>
- (item: QueuedMessage) => {
- set(store.queuedMessagesByConvoId(queueKey), (prev) =>
- prev.some((queued) => queued.id === item.id) ? prev : [item, ...prev],
- );
+ (origin: QueuedMessageOrigin) => {
+ releaseQueuedOrigin(origin);
+ set(store.queuedMessagesByConvoId(queueKey), (prev) => insertQueuedOrigin(prev, origin));
},
- [queueKey],
+ [queueKey, releaseQueuedOrigin],
);
/**
@@ -428,7 +794,7 @@ export default function useSteering({
(convoId: string, end: RunEnd) => {
const indexArmed = snapshot.getLoadable(store.runEndByIndex(index)).getValue();
const parkedArmed = snapshot.getLoadable(store.pendingRunEndByConvoId(convoId)).getValue();
- if (indexArmed?.conversationId === convoId || parkedArmed != null) {
+ if (isSameRunEpoch(indexArmed, end) || isSameRunEpoch(parkedArmed, end)) {
return;
}
set(store.pendingRunEndByConvoId(convoId), end);
@@ -436,12 +802,53 @@ export default function useSteering({
[index],
);
+ /** Restore a server-owned steer as the same queued follow-up and wake a
+ * drain whose one-shot completion signal may already have been consumed
+ * while the recovery request was in flight. Shared by reclaim and durable
+ * receipt replay so neither round-trip can strand accepted words. */
+ const queueRecoveredSteer = useCallback(
+ (steer: PendingSteer) => {
+ convertSteersToQueued(
+ conversationId,
+ [
+ {
+ steerId: steer.steerId,
+ ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
+ text: steer.text,
+ createdAt: steer.createdAt,
+ ...(steer.files && steer.files.length > 0 && { files: steer.files }),
+ ...carriedSteerContext(steer),
+ ...(steer.queuedOrigin && { queuedOrigin: steer.queuedOrigin }),
+ },
+ ],
+ {
+ /** A durable receipt belongs to the generation that produced it. The
+ * visible conversation may already advertise a replacement/legacy
+ * generation by the time this late ACK lands, but that must not strip
+ * the receipt-bound recovery identity from the queued item. */
+ generationProtocolVersion: steer.generationProtocolVersion,
+ },
+ );
+ const lastRunEnd = runEndsRef.current.get(conversationId);
+ if (lastRunEnd?.outcome === 'completed') {
+ rearmDrain(conversationId, lastRunEnd);
+ }
+ },
+ [conversationId, convertSteersToQueued, rearmDrain],
+ );
+
const armDrainAfterAbort = useRecoilCallback(
({ set }) =>
() => {
- set(store.drainAfterAbortByIndex(index), true);
+ if (activeGenerationCreatedAt == null) {
+ return;
+ }
+ set(store.drainAfterAbortByIndex(index), {
+ conversationId: queueKey,
+ generationCreatedAt: activeGenerationCreatedAt,
+ });
},
- [index],
+ [index, queueKey, activeGenerationCreatedAt],
);
/** POSTs a steer (text + files only; the server never carries quotes or
@@ -454,23 +861,73 @@ export default function useSteering({
text: string,
steerFiles?: TMessage['files'],
context?: QueuedMessageContext,
- opts?: { preempt?: boolean },
+ opts?: SubmitSteerOptions,
): boolean => {
const trimmed = text.trim();
if (trimmed.length === 0 || !hasRealConvoId) {
return false;
}
const preempt = opts?.preempt === true;
+ const targetGenerationCreatedAt =
+ opts?.generationCreatedAt ?? activeGenerationCreatedAt ?? undefined;
+ if (targetGenerationCreatedAt == null) {
+ return false;
+ }
const files = steerFiles && steerFiles.length > 0 ? steerFiles : undefined;
+ const targetGenerationProtocolVersion =
+ opts?.generationProtocolVersion ?? activeGenerationProtocolVersion;
/** Rides every chip state so a terminal conversion (late ACK, run-end
* leftover report) can restore the queued item's full context. */
const carried = carriedSteerContext(context);
- const localId = `local-${v4()}`;
+ const localId = opts?.clientSteerId ?? `local-${v4()}`;
/** The true submission time, carried through the ACK and failure chips so
* a later conversion sorts this steer by when it was SENT, not when its
* 202 landed — otherwise a draft queued during the round-trip would drain
* ahead of a steer submitted before it. */
- const createdAt = Date.now();
+ const createdAt = opts?.createdAt ?? Date.now();
+ /** If the steer loses its race with run finalization, its normal-send
+ * fallback must still be tied to the generation it targeted. Otherwise
+ * a newer run started by another tab can be replaced by this stale
+ * fallback. Every fallback also carries a queue origin: an existing row
+ * keeps its exact logical slot, while a composer-origin steer gets a
+ * stable synthetic row so a predecessor 409 cannot discard its text or
+ * files when the newer run wins. */
+ const fallbackOrigin: QueuedMessageOrigin = opts?.queuedOrigin ?? {
+ item: {
+ id: localId,
+ text: trimmed,
+ createdAt,
+ expectedPredecessorCreatedAt: targetGenerationCreatedAt,
+ ...(files && { files }),
+ ...carried,
+ ...(context?.clientRequestId && { clientRequestId: context.clientRequestId }),
+ ...(context?.recoverySteerId && { recoverySteerId: context.recoverySteerId }),
+ },
+ beforeIds: [],
+ afterIds: [],
+ };
+ const fallbackContext: QueuedMessageContext = {
+ ...context,
+ expectedPredecessorCreatedAt: targetGenerationCreatedAt,
+ queuedMessageOrigin: fallbackOrigin,
+ };
+ /** A queued-origin attempt must be reversible as the same queue item,
+ * not merely reproducible from its text. This preserves its identity,
+ * timestamp, priority, future fields, and original array position. */
+ const restoreRejectedSteer = () => {
+ if (opts?.queuedOrigin != null) {
+ restoreQueued(opts.queuedOrigin);
+ } else {
+ enqueue(trimmed, { id: localId, createdAt, files, ...context });
+ }
+ const lastRunEnd = runEndsRef.current.get(conversationId);
+ if (lastRunEnd?.outcome === 'completed') {
+ rearmDrain(conversationId, lastRunEnd);
+ }
+ };
+ if (opts?.queuedOrigin != null) {
+ registerQueuedOrigin(opts.queuedOrigin);
+ }
upsertSteerChip(conversationId, {
steerId: localId,
text: trimmed,
@@ -478,75 +935,187 @@ export default function useSteering({
createdAt,
...(files && { files }),
...(preempt && { preempt: true }),
+ ...(targetGenerationCreatedAt != null && {
+ generationCreatedAt: targetGenerationCreatedAt,
+ }),
+ generationProtocolVersion: targetGenerationProtocolVersion,
+ ...(opts?.queuedOrigin && { queuedOrigin: opts.queuedOrigin }),
...carried,
});
- steerMessage(
- { conversationId, text: trimmed, ...(files && { files }), ...(preempt && { preempt }) },
- {
- onSuccess: (response) => {
- /** The server's echo is authoritative: a deployment whose SDK
- * cannot seal mid-stream still queues the steer and answers
- * `preempt: false`, which relabels the chip to the ordinary
- * wording instead of surfacing an error. */
- acknowledgeSteer(conversationId, localId, {
- steerId: response.steerId,
- text: trimmed,
- status: 'pending',
- createdAt,
- ...(files && { files }),
- ...(response.preempt === true && { preempt: true }),
- ...carried,
- });
+ scheduleSteerDispatch(conversationId, (settleDispatch) => {
+ steerMessage(
+ {
+ conversationId,
+ clientSteerId: localId,
+ text: trimmed,
+ ...(files && { files }),
+ ...(preempt && { preempt }),
+ ...(targetGenerationCreatedAt != null && {
+ generationCreatedAt: targetGenerationCreatedAt,
+ }),
},
- onError: (error) => {
- const code = getSteerErrorCode(error);
- if (code === 'NO_ACTIVE_RUN') {
- // The run finished before the steer landed. While the final SSE
- // is still settling, `ask()`'s in-flight guard would drop a
- // direct send — queue it so the run-end drain fires it instead.
- // The explicit empty array stops `ask` from vacuuming composer
- // files staged for a DIFFERENT draft. A refused send (`false`)
- // falls back to the queue too — the chip is already gone, so
- // dropping the text here would lose it silently.
- replaceSteerChip(conversationId, localId, null);
- if (isSubmittingRef.current || sendNow(trimmed, files ?? [], context) === false) {
- enqueue(trimmed, { files, ...context });
+ {
+ onSuccess: (response) => {
+ try {
+ const canUseV2Receipt =
+ targetGenerationProtocolVersion === 2 && supportsGenerationProtocolV2(response);
+ if (canUseV2Receipt && response.settled === true) {
+ if (response.leftover === true) {
+ queueRecoveredSteer({
+ steerId: response.steerId,
+ clientSteerId: localId,
+ text: trimmed,
+ status: 'pending',
+ createdAt,
+ ...(files && { files }),
+ ...(targetGenerationCreatedAt != null && {
+ generationCreatedAt: targetGenerationCreatedAt,
+ }),
+ generationProtocolVersion: targetGenerationProtocolVersion,
+ ...(opts?.queuedOrigin && { queuedOrigin: opts.queuedOrigin }),
+ ...carried,
+ });
+ } else {
+ settleReceiptReplay(conversationId, localId, response.steerId);
+ }
+ return;
+ }
+ /** The server's echo is authoritative: a deployment whose SDK
+ * cannot seal mid-stream still queues the steer and answers
+ * `preempt: false`, which relabels the chip to the ordinary
+ * wording instead of surfacing an error. */
+ const acknowledged = {
+ steerId: response.steerId,
+ clientSteerId: localId,
+ text: trimmed,
+ status: 'pending',
+ createdAt,
+ ...(files && { files }),
+ ...(response.preempt === true && { preempt: true }),
+ ...(response.preemptRevision != null && {
+ preemptRevision: response.preemptRevision,
+ }),
+ ...(targetGenerationCreatedAt != null && {
+ generationCreatedAt: targetGenerationCreatedAt,
+ }),
+ generationProtocolVersion: targetGenerationProtocolVersion,
+ ...(opts?.queuedOrigin && { queuedOrigin: opts.queuedOrigin }),
+ ...carried,
+ } satisfies PendingSteer;
+ if (acknowledgeSteer(conversationId, localId, acknowledged)) {
+ queueRecoveredSteer(acknowledged);
+ }
+ } finally {
+ settleDispatch();
}
- return;
- }
- if (
- code === 'RUN_PAUSED' ||
- code === 'STEER_UNSUPPORTED' ||
- code === 'STEER_QUEUE_FULL'
- ) {
- replaceSteerChip(conversationId, localId, null);
- // The rejection can land AFTER the final SSE consumed the
- // run-end signal (common on an unsupported SDK near run end):
- // queueing then has nothing left to drain it, so mirror the
- // NO_ACTIVE_RUN fallback and send once submission settled —
- // queueing the refusal instead of dropping the text.
- if (!isSubmittingRef.current) {
- if (sendNow(trimmed, files ?? [], context) === false) {
- enqueue(trimmed, { files, ...context });
+ },
+ onError: (error) => {
+ try {
+ if (isSteerAcceptedOrSettled(conversationId, localId)) {
+ replaceSteerChip(conversationId, localId, null);
+ return;
+ }
+ const code = getSteerErrorCode(error);
+ const originStillVisible = visibleConversationRef.current === conversationId;
+ if (code === 'NO_ACTIVE_RUN') {
+ // The run finished before the steer landed. While the final SSE
+ // is still settling, `ask()`'s in-flight guard would drop a
+ // direct send — queue it so the run-end drain fires it instead.
+ // The explicit empty array stops `ask` from vacuuming composer
+ // files staged for a DIFFERENT draft. A refused send (`false`)
+ // falls back to the queue too — the chip is already gone, so
+ // dropping the text here would lose it silently.
+ replaceSteerChip(conversationId, localId, null);
+ if (
+ !originStillVisible ||
+ isSubmittingRef.current ||
+ sendNow(trimmed, files ?? [], fallbackContext) === false
+ ) {
+ restoreRejectedSteer();
+ }
+ return;
+ }
+ if (
+ code === 'RUN_PAUSED' ||
+ code === 'STEER_UNSUPPORTED' ||
+ code === 'STEER_QUEUE_FULL'
+ ) {
+ replaceSteerChip(conversationId, localId, null);
+ // The rejection can land AFTER the final SSE consumed the
+ // run-end signal (common on an unsupported SDK near run end):
+ // queueing then has nothing left to drain it, so mirror the
+ // NO_ACTIVE_RUN fallback and send once submission settled —
+ // queueing the refusal instead of dropping the text.
+ if (originStillVisible && !isSubmittingRef.current) {
+ if (sendNow(trimmed, files ?? [], fallbackContext) === false) {
+ restoreRejectedSteer();
+ }
+ return;
+ }
+ restoreRejectedSteer();
+ showToast({ message: localize('com_ui_steer_paused_queued'), status: 'info' });
+ return;
+ }
+ const errorData = (
+ error as { response?: { data?: { generationProtocolVersion?: unknown } } }
+ )?.response?.data;
+ if (
+ code === 'RUN_REPLACED' &&
+ targetGenerationProtocolVersion === 2 &&
+ supportsGenerationProtocolV2(errorData)
+ ) {
+ /** A different live generation now owns this conversation. A
+ * normal-send fallback would replace that newer run (especially
+ * from a stale tab), so retain the instruction for its run-end
+ * drain regardless of this tab's local submission state. */
+ replaceSteerChip(conversationId, localId, null);
+ restoreRejectedSteer();
+ showToast({ message: localize('com_ui_steer_paused_queued'), status: 'info' });
+ return;
+ }
+ if (isDefiniteSteerRejection(error)) {
+ // A concrete origin 4xx proves the steer was rejected. Keep the
+ // ordinary failed controls (edit/queue/remove/retry) instead of
+ // the locked "delivery unknown" surface reserved for transport
+ // and 5xx ambiguity.
+ upsertSteerChip(conversationId, {
+ steerId: localId,
+ text: trimmed,
+ status: 'failed',
+ createdAt,
+ ...(files && { files }),
+ ...(preempt && { preempt: true }),
+ ...(targetGenerationCreatedAt != null && {
+ generationCreatedAt: targetGenerationCreatedAt,
+ }),
+ generationProtocolVersion: targetGenerationProtocolVersion,
+ ...(opts?.queuedOrigin && { queuedOrigin: opts.queuedOrigin }),
+ ...carried,
+ });
+ return;
}
- return;
+ upsertSteerChip(conversationId, {
+ steerId: localId,
+ text: trimmed,
+ status: 'failed',
+ deliveryUncertain: true,
+ createdAt,
+ ...(files && { files }),
+ ...(preempt && { preempt: true }),
+ ...(targetGenerationCreatedAt != null && {
+ generationCreatedAt: targetGenerationCreatedAt,
+ }),
+ generationProtocolVersion: targetGenerationProtocolVersion,
+ ...(opts?.queuedOrigin && { queuedOrigin: opts.queuedOrigin }),
+ ...carried,
+ });
+ } finally {
+ settleDispatch();
}
- enqueue(trimmed, { files, ...context });
- showToast({ message: localize('com_ui_steer_paused_queued'), status: 'info' });
- return;
- }
- upsertSteerChip(conversationId, {
- steerId: localId,
- text: trimmed,
- status: 'failed',
- createdAt,
- ...(files && { files }),
- ...(preempt && { preempt: true }),
- ...carried,
- });
+ },
},
- },
- );
+ );
+ });
return true;
},
[
@@ -555,11 +1124,20 @@ export default function useSteering({
upsertSteerChip,
replaceSteerChip,
acknowledgeSteer,
+ settleReceiptReplay,
+ queueRecoveredSteer,
steerMessage,
sendNow,
enqueue,
+ restoreQueued,
+ rearmDrain,
+ registerQueuedOrigin,
+ isSteerAcceptedOrSettled,
showToast,
localize,
+ activeGenerationCreatedAt,
+ activeGenerationProtocolVersion,
+ scheduleSteerDispatch,
],
);
@@ -569,7 +1147,7 @@ export default function useSteering({
const steerFromComposer = useCallback(
(text: string, preempt = false): boolean => {
const trimmed = text.trim();
- if (trimmed.length === 0 || filesLoading || !hasRealConvoId) {
+ if (trimmed.length === 0 || filesLoading || !canSteer) {
return false;
}
const consumed = submitSteer(trimmed, takeComposerFiles(), undefined, { preempt });
@@ -578,7 +1156,7 @@ export default function useSteering({
}
return consumed;
},
- [filesLoading, hasRealConvoId, takeComposerFiles, takeComposerDraft, submitSteer],
+ [filesLoading, canSteer, takeComposerFiles, takeComposerDraft, submitSteer],
);
/** Composer-originated queue: carries the composer's attachments, quote
@@ -603,14 +1181,13 @@ export default function useSteering({
text: string,
steerFiles?: TMessage['files'],
context?: QueuedMessageContext,
- opts?: { preempt?: boolean },
+ opts?: SubmitSteerOptions,
) => {
- replaceSteerChip(conversationId, steerId, null);
/** A failed interrupt-steer must retry AS an interrupt — resubmitting it
* as an ordinary steer would silently let generation run on. */
- submitSteer(text, steerFiles, context, opts);
+ submitSteer(text, steerFiles, context, { ...opts, clientSteerId: steerId });
},
- [conversationId, replaceSteerChip, submitSteer],
+ [submitSteer],
);
const removeSteer = useCallback(
@@ -642,31 +1219,26 @@ export default function useSteering({
*/
const queueReclaimedSteer = useCallback(
(steer: PendingSteer) => {
- convertSteersToQueued(conversationId, [
- {
- steerId: steer.steerId,
- text: steer.text,
+ // A confirmed cancel has no parked server source. Restore one ordinary
+ // queue item explicitly (the reclaim tombstone intentionally makes the
+ // shared terminal converter ignore late payloads for these ids).
+ if (steer.queuedOrigin != null) {
+ restoreQueued(steer.queuedOrigin);
+ } else {
+ enqueue(steer.text, {
+ id: steer.steerId,
createdAt: steer.createdAt,
- ...(steer.files && steer.files.length > 0 && { files: steer.files }),
- /** Carried on the steer itself: the conversion normally recovers this
- * from the chip, which a competing cancel can remove mid-reclaim. */
+ files: steer.files,
...carriedSteerContext(steer),
- },
- ]);
- /**
- * Read by conversation, so it describes THIS steer's run even once the
- * hook has moved to another chat — the user navigating away does not make
- * their words any less owed a send. No entry means that run is still going
- * (or has started another): its own end is ahead of this item and drains
- * it, so there is nothing to re-arm.
- */
+ skipUsageMark: true,
+ });
+ }
const lastRunEnd = runEndsRef.current.get(conversationId);
- if (lastRunEnd == null || lastRunEnd.outcome !== 'completed') {
- return;
+ if (lastRunEnd?.outcome === 'completed') {
+ rearmDrain(conversationId, lastRunEnd);
}
- rearmDrain(conversationId, lastRunEnd);
},
- [conversationId, convertSteersToQueued, rearmDrain],
+ [conversationId, enqueue, rearmDrain, restoreQueued],
);
/** Convert a failed/unsent steer chip into a queued follow-up. */
@@ -676,50 +1248,88 @@ export default function useSteering({
text: string,
steerFiles?: TMessage['files'],
context?: QueuedMessageContext,
+ queuedOrigin?: QueuedMessageOrigin,
) => {
replaceSteerChip(conversationId, steerId, null);
+ if (queuedOrigin != null) {
+ restoreQueued(queuedOrigin);
+ return;
+ }
enqueue(text, { files: steerFiles, ...context });
},
- [conversationId, replaceSteerChip, enqueue],
+ [conversationId, replaceSteerChip, restoreQueued, enqueue],
);
/** Chip action: send a queued message into the live run instead. Keys on
* steer availability, not the default action — a queue-preferring user
* clicking send-now explicitly asked to inject into the live run. The
* item's attachments ride the steer; its quotes/skills travel as the
- * restore context so a degraded steer requeues/sends with them intact. */
+ * restore context so a degraded steer requeues/sends with them intact.
+ * `preempt` escalates the steer to interrupt at the next safe token
+ * boundary; it only means something on the live-run path. */
const sendQueuedNow = useCallback(
- (item: QueuedMessage) => {
- const taken = takeQueued(item.id) ?? item;
+ (item: QueuedMessage, opts?: { preempt?: boolean }) => {
+ /** In answer mode (and any other submission-owned non-steerable state)
+ * there is no immediate path. Refuse before touching queue state so a
+ * stale/direct caller cannot perform the old remove-and-restore no-op. */
+ if (isSubmitting && (!duringRunActive || !canSteer || item.recoverySteerId != null)) {
+ return;
+ }
+ /** UI callers always find the item; a stale/direct caller has no original
+ * neighbours, so restoration falls back to the queue's priority split. */
+ const origin = takeQueued(item.id) ?? { item, beforeIds: [], afterIds: [] };
+ const taken = origin.item;
if (duringRunActive && canSteer) {
- submitSteer(taken.text, taken.files, {
- quotes: taken.quotes,
- manualSkills: taken.manualSkills,
- });
+ const consumed = submitSteer(
+ taken.text,
+ taken.files,
+ { quotes: taken.quotes, manualSkills: taken.manualSkills },
+ {
+ ...(opts?.preempt === true && { preempt: true }),
+ queuedOrigin: origin,
+ },
+ );
+ if (!consumed) {
+ restoreQueued(origin);
+ }
return;
}
if (!isSubmitting) {
+ if (idleSendInFlightRef.current) {
+ restoreQueued(origin);
+ return;
+ }
+ idleSendInFlightRef.current = true;
// Explicit (possibly empty) overrides: the queued item is the full
// submission context, never the composer's staged files/quotes/picks.
- const accepted = sendNow(taken.text, taken.files ?? [], {
- quotes: taken.quotes,
- manualSkills: taken.manualSkills,
- });
+ let accepted: false | void;
+ try {
+ accepted = sendNow(taken.text, taken.files ?? [], {
+ quotes: taken.quotes,
+ manualSkills: taken.manualSkills,
+ clientRequestId: taken.clientRequestId,
+ recoverySteerId: taken.recoverySteerId,
+ expectedPredecessorCreatedAt: taken.expectedPredecessorCreatedAt,
+ queuedMessageOrigin: origin,
+ });
+ } catch (error) {
+ idleSendInFlightRef.current = false;
+ restoreQueued(origin);
+ throw error;
+ }
if (accepted === false) {
+ idleSendInFlightRef.current = false;
// `ask` refused without sending — restore the chip so the user's
// text is never silently dropped (mirrors useQueueDrain).
- restoreQueued(taken);
+ restoreQueued(origin);
+ } else {
+ releaseQueuedOrigin(origin);
}
return;
}
- // Files were already TTL-held when the item first entered the queue.
- enqueue(taken.text, {
- front: true,
- files: taken.files,
- quotes: taken.quotes,
- manualSkills: taken.manualSkills,
- skipUsageMark: true,
- });
+ /** The run still owns the send slot but cannot accept a steer (approval
+ * pause/answer transition): put the untouched item back where it was. */
+ restoreQueued(origin);
},
[
takeQueued,
@@ -729,7 +1339,7 @@ export default function useSteering({
isSubmitting,
sendNow,
restoreQueued,
- enqueue,
+ releaseQueuedOrigin,
],
);
@@ -737,7 +1347,7 @@ export default function useSteering({
const interruptAndSend = useCallback(
(text: string): boolean => {
const trimmed = text.trim();
- if (trimmed.length === 0 || filesLoading) {
+ if (trimmed.length === 0 || filesLoading || !canControlGeneration) {
return false;
}
enqueue(trimmed, { front: true, files: takeComposerFiles(), ...takeComposerContext() });
@@ -748,6 +1358,7 @@ export default function useSteering({
},
[
filesLoading,
+ canControlGeneration,
enqueue,
takeComposerFiles,
takeComposerContext,
@@ -779,7 +1390,7 @@ export default function useSteering({
const interruptSteer = useCallback(
(text: string): boolean => {
const trimmed = text.trim();
- if (trimmed.length === 0 || filesLoading || pausedOnApproval) {
+ if (trimmed.length === 0 || filesLoading || pausedOnApproval || !canControlGeneration) {
return false;
}
if (!hasRealConvoId) {
@@ -794,6 +1405,7 @@ export default function useSteering({
[
filesLoading,
pausedOnApproval,
+ canControlGeneration,
hasRealConvoId,
interruptAndSend,
takeComposerFiles,
@@ -832,7 +1444,9 @@ export default function useSteering({
enabled,
queueKey,
canSteer,
+ canControlGeneration,
duringRunActive,
+ canSendQueuedNow,
effectiveAction,
defaultAction,
pausedOnApproval,
@@ -847,6 +1461,7 @@ export default function useSteering({
queueReclaimedSteer,
enqueue,
removeQueued,
+ discardQueued,
sendQueuedNow,
interruptAndSend,
interruptSteer,
@@ -855,7 +1470,9 @@ export default function useSteering({
enabled,
queueKey,
canSteer,
+ canControlGeneration,
duringRunActive,
+ canSendQueuedNow,
effectiveAction,
defaultAction,
pausedOnApproval,
@@ -870,6 +1487,7 @@ export default function useSteering({
queueReclaimedSteer,
enqueue,
removeQueued,
+ discardQueued,
sendQueuedNow,
interruptAndSend,
interruptSteer,
diff --git a/client/src/hooks/Input/useAskAnswerMode.spec.ts b/client/src/hooks/Input/useAskAnswerMode.spec.ts
index 00b67a0f38..148fb45565 100644
--- a/client/src/hooks/Input/useAskAnswerMode.spec.ts
+++ b/client/src/hooks/Input/useAskAnswerMode.spec.ts
@@ -1,14 +1,34 @@
+const mockSubmitAskAnswer = jest.fn();
+const mockResetComposer = jest.fn();
+const mockGetComposerText = jest.fn(() => 'answer from A');
+const mockSetSelected = jest.fn();
+const mockSetChecked = jest.fn();
+let mockSaveDrafts = false;
+
jest.mock('~/data-provider', () => ({ useGetMessagesByConvoId: jest.fn() }));
jest.mock('~/components/Chat/Messages/Content/ApprovalContext', () => ({
useAskSubmitStatus: () => ({ getAskStatus: () => 'idle' }),
- useResumeSubmit: () => ({ submitAskAnswer: jest.fn() }),
+ useResumeSubmit: () => ({ submitAskAnswer: mockSubmitAskAnswer }),
+}));
+jest.mock('~/Providers', () => ({
+ useOptionalChatFormContext: () => ({
+ reset: mockResetComposer,
+ getValues: mockGetComposerText,
+ }),
}));
-jest.mock('~/Providers', () => ({ useOptionalChatFormContext: () => undefined }));
jest.mock('~/utils', () => ({ getAskAnswerDraftId: (id: string) => `draft-${id}` }));
jest.mock('recoil', () => ({
atom: (cfg: unknown) => cfg,
- useRecoilState: () => [[], jest.fn()],
- useRecoilValue: () => false,
+ useRecoilState: (state: { key?: string }) => {
+ if (state.key === 'askAnswerModeSelection') {
+ return [null, mockSetSelected];
+ }
+ if (state.key === 'askAnswerModeChecked') {
+ return [[], mockSetChecked];
+ }
+ return [[], jest.fn()];
+ },
+ useRecoilValue: () => mockSaveDrafts,
}));
jest.mock('~/store', () => ({ __esModule: true, default: { saveDrafts: 'saveDrafts' } }));
@@ -25,7 +45,11 @@ const liveAsk = {
} as unknown as ReturnType<typeof findLiveAskUserQuestion>;
describe('useAskAnswerMode', () => {
- beforeEach(() => jest.clearAllMocks());
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockSaveDrafts = false;
+ mockGetComposerText.mockReturnValue('answer from A');
+ });
it('projects the live ask via the findLiveAskUserQuestion select over the conversation cache', () => {
mockUseGetMessages.mockReturnValue({ data: liveAsk });
@@ -71,4 +95,102 @@ describe('useAskAnswerMode', () => {
expect(result.current.liveAsk).toBeNull();
});
+
+ it('does not let a delayed answer success clear the composer or selection after navigation', () => {
+ let finishAnswer: (() => void) | undefined;
+ mockUseGetMessages.mockReturnValue({ data: liveAsk });
+ mockSubmitAskAnswer.mockImplementation(
+ (_actionId: string, _answer: string, options?: { onSuccess?: () => void }) => {
+ finishAnswer = options?.onSuccess;
+ },
+ );
+ const { result, rerender } = renderHook(
+ ({ conversationId }) => useAskAnswerMode(conversationId),
+ { initialProps: { conversationId: 'conversation-A' } },
+ );
+
+ expect(result.current.submitText('answer from A')).toBe(true);
+ expect(mockSubmitAskAnswer).toHaveBeenCalledWith(
+ 'a1',
+ 'answer from A',
+ expect.objectContaining({ onSuccess: expect.any(Function) }),
+ );
+
+ const nextAsk = {
+ ...liveAsk,
+ actionId: 'b1',
+ } as typeof liveAsk;
+ mockUseGetMessages.mockReturnValue({ data: nextAsk });
+ mockGetComposerText.mockReturnValue('draft typed in B');
+ rerender({ conversationId: 'conversation-B' });
+ mockSetSelected.mockClear();
+ mockSetChecked.mockClear();
+
+ finishAnswer?.();
+
+ expect(mockResetComposer).not.toHaveBeenCalled();
+ expect(mockSetSelected).not.toHaveBeenCalled();
+ expect(mockSetChecked).not.toHaveBeenCalled();
+ });
+
+ it('keeps newer composer text when the answer settles on the same question', () => {
+ let finishAnswer: (() => void) | undefined;
+ mockUseGetMessages.mockReturnValue({ data: liveAsk });
+ mockSubmitAskAnswer.mockImplementation(
+ (_actionId: string, _answer: string, options?: { onSuccess?: () => void }) => {
+ finishAnswer = options?.onSuccess;
+ },
+ );
+ const { result } = renderHook(() => useAskAnswerMode('conversation-A'));
+
+ expect(result.current.submitText('answer from A')).toBe(true);
+ mockGetComposerText.mockReturnValue('new text typed while resuming');
+ mockSetSelected.mockClear();
+ mockSetChecked.mockClear();
+
+ finishAnswer?.();
+
+ expect(mockResetComposer).not.toHaveBeenCalled();
+ expect(mockSetSelected).toHaveBeenCalledWith(null);
+ expect(mockSetChecked).toHaveBeenCalledWith([]);
+ });
+
+ it('clears the consumed answer when the same question and composer value still own it', () => {
+ let finishAnswer: (() => void) | undefined;
+ mockUseGetMessages.mockReturnValue({ data: liveAsk });
+ mockSubmitAskAnswer.mockImplementation(
+ (_actionId: string, _answer: string, options?: { onSuccess?: () => void }) => {
+ finishAnswer = options?.onSuccess;
+ },
+ );
+ const { result } = renderHook(() => useAskAnswerMode('conversation-A'));
+
+ expect(result.current.submitText('answer from A')).toBe(true);
+ finishAnswer?.();
+
+ expect(mockResetComposer).toHaveBeenCalledTimes(1);
+ expect(mockSetSelected).toHaveBeenCalledWith(null);
+ expect(mockSetChecked).toHaveBeenCalledWith([]);
+ });
+
+ it('ignores a delayed answer success after its answer-mode owner unmounts', () => {
+ let finishAnswer: (() => void) | undefined;
+ mockUseGetMessages.mockReturnValue({ data: liveAsk });
+ mockSubmitAskAnswer.mockImplementation(
+ (_actionId: string, _answer: string, options?: { onSuccess?: () => void }) => {
+ finishAnswer = options?.onSuccess;
+ },
+ );
+ const { result, unmount } = renderHook(() => useAskAnswerMode('conversation-A'));
+
+ expect(result.current.submitText('answer from A')).toBe(true);
+ unmount();
+ mockSetSelected.mockClear();
+ mockSetChecked.mockClear();
+ finishAnswer?.();
+
+ expect(mockResetComposer).not.toHaveBeenCalled();
+ expect(mockSetSelected).not.toHaveBeenCalled();
+ expect(mockSetChecked).not.toHaveBeenCalled();
+ });
});
diff --git a/client/src/hooks/Input/useAskAnswerMode.ts b/client/src/hooks/Input/useAskAnswerMode.ts
index db642258f6..d13d0205f2 100644
--- a/client/src/hooks/Input/useAskAnswerMode.ts
+++ b/client/src/hooks/Input/useAskAnswerMode.ts
@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useMemo } from 'react';
+import { useCallback, useEffect, useMemo, useRef } from 'react';
import { atom, useRecoilState, useRecoilValue } from 'recoil';
import {
useAskSubmitStatus,
@@ -81,7 +81,30 @@ export default function useAskAnswerMode(conversationId?: string | null) {
const { getAskStatus } = useAskSubmitStatus();
/** Absent outside ChatView (Share/search render the answer card without the
* composer form) — resets are simply skipped there. */
- const resetComposer = useOptionalChatFormContext()?.reset;
+ const formContext = useOptionalChatFormContext();
+ /** Resume callbacks may settle after this ChatForm has navigated to another
+ * conversation (the form instance is intentionally reused across routes).
+ * Keep the callback's current ownership observable without letting its old
+ * closure reset a newer draft or selection. */
+ const currentScopeRef = useRef({
+ conversationId,
+ actionId: liveAsk?.actionId,
+ formContext,
+ });
+ currentScopeRef.current = {
+ conversationId,
+ actionId: liveAsk?.actionId,
+ formContext,
+ };
+ const mountedRef = useRef(true);
+ useEffect(() => {
+ // Strict Mode runs setup, cleanup, then setup again in development; each
+ // live setup must reassert ownership before a success callback can clean.
+ mountedRef.current = true;
+ return () => {
+ mountedRef.current = false;
+ };
+ }, []);
/** The answer is in flight (or terminal): every submit path must become a
* no-op so a double-click or a stray Skip can't race a second resume. */
@@ -184,18 +207,42 @@ export default function useAskAnswerMode(conversationId?: string | null) {
return false;
}
const wasActive = active;
+ const submittedConversationId = conversationId;
+ const submittedActionId = liveAsk.actionId;
+ const submittedComposerText = formContext?.getValues('text') ?? '';
submitAskAnswer(liveAsk.actionId, values.join(', '), {
onSuccess: () => {
+ const currentScope = currentScopeRef.current;
+ if (
+ !mountedRef.current ||
+ currentScope.conversationId !== submittedConversationId ||
+ currentScope.actionId !== submittedActionId
+ ) {
+ return;
+ }
setSelected(null);
setChecked([]);
- if (consumedComposerText || (wasActive && saveDrafts)) {
- resetComposer?.();
+ if (
+ (consumedComposerText || (wasActive && saveDrafts)) &&
+ currentScope.formContext?.getValues('text') === submittedComposerText
+ ) {
+ currentScope.formContext.reset();
}
},
});
return true;
},
- [liveAsk, locked, active, saveDrafts, submitAskAnswer, setSelected, setChecked, resetComposer],
+ [
+ liveAsk,
+ locked,
+ active,
+ saveDrafts,
+ conversationId,
+ formContext,
+ submitAskAnswer,
+ setSelected,
+ setChecked,
+ ],
);
const checkedValues = useCallback(
diff --git a/client/src/hooks/Input/useComposerBindings.ts b/client/src/hooks/Input/useComposerBindings.ts
index 5f2d0a5dde..5da7acf476 100644
--- a/client/src/hooks/Input/useComposerBindings.ts
+++ b/client/src/hooks/Input/useComposerBindings.ts
@@ -1,8 +1,9 @@
import { useMemo } from 'react';
import { useRecoilValue } from 'recoil';
+import type { ShortcutActionId } from '~/hooks/useKeyboardShortcuts';
import type { ShortcutBinding } from '~/utils/shortcuts';
-import { parseBinding, bindingHash, isMacPlatform } from '~/utils/shortcuts';
-import { EDITING_ALLOWED_SHORTCUTS } from '~/hooks/useKeyboardShortcuts';
+import { EDITING_ALLOWED_SHORTCUTS, resolveShortcutBindings } from '~/hooks/useKeyboardShortcuts';
+import { bindingHash } from '~/utils/shortcuts';
import store from '~/store';
export type ComposerBindings = {
@@ -28,29 +29,33 @@ export type ComposerBindings = {
* composer keydown handler and the during-run hovercard hints. */
export default function useComposerBindings(): ComposerBindings {
const customShortcuts = useRecoilValue(store.customShortcuts);
+ const resolvedBindings = useMemo(
+ () => resolveShortcutBindings(customShortcuts),
+ [customShortcuts],
+ );
const submitOverride = useMemo(() => {
const override = customShortcuts['submitMessage'];
if (!override) {
- return undefined;
+ return resolvedBindings.get('submitMessage') == null ? null : undefined;
}
- return parseBinding(isMacPlatform ? override.mac : override.other);
- }, [customShortcuts]);
+ return resolvedBindings.get('submitMessage') ?? null;
+ }, [customShortcuts, resolvedBindings]);
const yieldedChords = useMemo(() => {
const editingAllowed: ReadonlySet<string> = EDITING_ALLOWED_SHORTCUTS;
const hashes = new Set<string>();
- for (const [actionId, override] of Object.entries(customShortcuts ?? {})) {
- if (actionId === 'submitMessage' || !editingAllowed.has(actionId)) {
+ for (const actionId of editingAllowed) {
+ if (actionId === 'submitMessage') {
continue;
}
- const binding = parseBinding(isMacPlatform ? override?.mac : override?.other);
+ const binding = resolvedBindings.get(actionId as ShortcutActionId);
if (binding) {
hashes.add(bindingHash(binding));
}
}
return hashes;
- }, [customShortcuts]);
+ }, [resolvedBindings]);
return useMemo(() => ({ submitOverride, yieldedChords }), [submitOverride, yieldedChords]);
}
diff --git a/client/src/hooks/Messages/__tests__/useSubmitMessage.spec.ts b/client/src/hooks/Messages/__tests__/useSubmitMessage.spec.ts
index 37015359a9..9525d4a622 100644
--- a/client/src/hooks/Messages/__tests__/useSubmitMessage.spec.ts
+++ b/client/src/hooks/Messages/__tests__/useSubmitMessage.spec.ts
@@ -130,4 +130,25 @@ describe('useSubmitMessage', () => {
expect(setMessages).not.toHaveBeenCalled();
expect(ask).toHaveBeenCalled();
});
+
+ it('uses the recovery source as the stable user row and forwards the attempt fields', () => {
+ ask.mockReturnValue(true);
+ const { result } = renderHook(() => useSubmitMessage());
+
+ act(() => {
+ result.current.submitMessage({
+ text: 'recover once',
+ overrideClientRequestId: 'attempt-uuid',
+ overrideRecoverySteerId: 'source-steer',
+ });
+ });
+
+ expect(ask).toHaveBeenCalledWith(
+ { text: 'recover once', overrideUserMessageId: 'source-steer' },
+ expect.objectContaining({
+ overrideClientRequestId: 'attempt-uuid',
+ overrideRecoverySteerId: 'source-steer',
+ }),
+ );
+ });
});
diff --git a/client/src/hooks/Messages/useSubmitMessage.ts b/client/src/hooks/Messages/useSubmitMessage.ts
index 57d9419029..17179e7f0b 100644
--- a/client/src/hooks/Messages/useSubmitMessage.ts
+++ b/client/src/hooks/Messages/useSubmitMessage.ts
@@ -24,6 +24,10 @@ export default function useSubmitMessage() {
overrideFiles?: TMessage['files'];
overrideQuotes?: string[];
overrideManualSkills?: string[];
+ overrideClientRequestId?: string;
+ overrideRecoverySteerId?: string;
+ overrideExpectedPredecessorCreatedAt?: number;
+ overrideQueuedMessageOrigin?: unknown;
}) => {
if (!data) {
return console.warn('No data provided to submitMessage');
@@ -40,6 +44,9 @@ export default function useSubmitMessage() {
const submitted = ask(
{
text: data.text,
+ ...(data.overrideRecoverySteerId != null && {
+ overrideUserMessageId: data.overrideRecoverySteerId,
+ }),
},
{
addedConvo: addedConvo ?? undefined,
@@ -48,6 +55,10 @@ export default function useSubmitMessage() {
overrideFiles: data.overrideFiles,
overrideQuotes: data.overrideQuotes,
overrideManualSkills: data.overrideManualSkills,
+ overrideClientRequestId: data.overrideClientRequestId,
+ overrideRecoverySteerId: data.overrideRecoverySteerId,
+ overrideExpectedPredecessorCreatedAt: data.overrideExpectedPredecessorCreatedAt,
+ overrideQueuedMessageOrigin: data.overrideQueuedMessageOrigin,
},
);
if (submitted === false) {
diff --git a/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts b/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts
index 2c94d25294..cc198a27a0 100644
--- a/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts
+++ b/client/src/hooks/SSE/__tests__/useResumableSSE.spec.ts
@@ -1,6 +1,7 @@
import { renderHook, act, waitFor } from '@testing-library/react';
import {
Constants,
+ ContentTypes,
LocalStorageKeys,
QueryKeys,
StepEvents,
@@ -11,6 +12,7 @@ import type { TMessage, TSubmission } from 'librechat-data-provider';
type SSEEventListener = (e: Partial<MessageEvent> & { responseCode?: number }) => void;
interface MockSSEInstance {
+ _url: string;
addEventListener: jest.Mock;
stream: jest.Mock;
close: jest.Mock;
@@ -22,31 +24,36 @@ interface MockSSEInstance {
const mockSSEInstances: MockSSEInstance[] = [];
jest.mock('sse.js', () => ({
- SSE: jest.fn().mockImplementation(() => {
- const listeners: Record<string, SSEEventListener> = {};
- const instance: MockSSEInstance = {
- addEventListener: jest.fn((event: string, cb: SSEEventListener) => {
- listeners[event] = cb;
- }),
- stream: jest.fn(),
- close: jest.fn(),
- headers: {},
- _listeners: listeners,
- _emit: (event, data = {}) => listeners[event]?.(data as MessageEvent),
- };
- mockSSEInstances.push(instance);
- return instance;
- }),
+ SSE: jest
+ .fn()
+ .mockImplementation((url: string, options?: { headers?: Record<string, string> }) => {
+ const listeners: Record<string, SSEEventListener> = {};
+ const instance: MockSSEInstance = {
+ _url: url,
+ addEventListener: jest.fn((event: string, cb: SSEEventListener) => {
+ listeners[event] = cb;
+ }),
+ stream: jest.fn(),
+ close: jest.fn(),
+ headers: { ...options?.headers },
+ _listeners: listeners,
+ _emit: (event, data = {}) => listeners[event]?.(data as MessageEvent),
+ };
+ mockSSEInstances.push(instance);
+ return instance;
+ }),
}));
const mockSetQueryData = jest.fn();
const mockGetQueryData = jest.fn();
+const mockFetchQuery = jest.fn();
const mockInvalidateQueries = jest.fn();
const mockRemoveQueries = jest.fn();
const mockFindAll = jest.fn((): Array<{ queryKey: unknown[] }> => []);
const mockQueryClient = {
setQueryData: mockSetQueryData,
getQueryData: mockGetQueryData,
+ fetchQuery: mockFetchQuery,
invalidateQueries: mockInvalidateQueries,
removeQueries: mockRemoveQueries,
getQueryCache: () => ({
@@ -59,6 +66,7 @@ const mockAbortScrollAtom = { key: 'abortScroll' };
const mockSubmissionAtom = { key: 'submission' };
const mockShowStopButtonAtom = { key: 'showStopButton' };
const mockRunEndAtom = { key: 'runEnd' };
+const mockDrainAfterAbortAtom = { key: 'drainAfterAbort' };
const mockPendingSteersAtom = { key: 'pendingSteers' };
const mockQueuedMessagesAtom = { key: 'queuedMessages' };
const mockSetActiveRun = jest.fn();
@@ -66,6 +74,27 @@ const mockSetAbortScroll = jest.fn();
const mockSetSubmission = jest.fn();
const mockSetShowStopButton = jest.fn();
const mockSetRunEnd = jest.fn();
+const mockSetDrainAfterAbort = jest.fn();
+const mockResolveSteerChip = jest.fn();
+const mockUpdateSteerChips = jest.fn();
+const mockSeedSteerChips = jest.fn();
+const mockSettleAppliedSteerParts = jest.fn();
+const mockConvertLocalSteersToQueued = jest.fn();
+const mockUpdateGenerationEpoch = jest.fn();
+const mockRestoreQueuedSubmission = jest.fn();
+let mockRecoilCallbackIndex = 0;
+const mockRecoilCallbacks = [
+ mockRestoreQueuedSubmission,
+ mockResolveSteerChip,
+ mockUpdateSteerChips,
+ mockSeedSteerChips,
+ mockSettleAppliedSteerParts,
+ mockConvertLocalSteersToQueued,
+ mockUpdateGenerationEpoch,
+];
+function mockUseRecoilCallback() {
+ return mockRecoilCallbacks[mockRecoilCallbackIndex++ % mockRecoilCallbacks.length];
+}
const mockUseSetRecoilStateMock = jest.fn((atom: unknown) => {
if (atom === mockActiveRunAtom) {
return mockSetActiveRun;
@@ -82,12 +111,34 @@ const mockUseSetRecoilStateMock = jest.fn((atom: unknown) => {
if (atom === mockRunEndAtom) {
return mockSetRunEnd;
}
+ if (atom === mockDrainAfterAbortAtom) {
+ return mockSetDrainAfterAbort;
+ }
return jest.fn();
});
function mockUseSetRecoilState(atom: unknown) {
return mockUseSetRecoilStateMock(atom);
}
+type DrainAfterAbortState =
+ | false
+ | {
+ conversationId: string;
+ generationCreatedAt: number;
+ };
+
+const applyLastDrainAfterAbortUpdate = (current: DrainAfterAbortState): DrainAfterAbortState => {
+ const calls = mockSetDrainAfterAbort.mock.calls;
+ const update = calls[calls.length - 1]?.[0] as
+ | DrainAfterAbortState
+ | ((armed: DrainAfterAbortState) => DrainAfterAbortState)
+ | undefined;
+ if (typeof update !== 'function') {
+ throw new Error('Expected a functional drain-after-abort update');
+ }
+ return update(current);
+};
+
jest.mock('@tanstack/react-query', () => ({
...jest.requireActual('@tanstack/react-query'),
useQueryClient: () => mockQueryClient,
@@ -96,9 +147,9 @@ jest.mock('@tanstack/react-query', () => ({
jest.mock('recoil', () => ({
...jest.requireActual('recoil'),
useSetRecoilState: mockUseSetRecoilState,
- // The hook's steer-chip/queue callbacks need a RecoilRoot; these tests
- // render bare, so return inert callbacks (chip state is not under test here).
- useRecoilCallback: () => jest.fn(),
+ // The hook's steer-chip/queue callbacks need a RecoilRoot; these tests render
+ // bare, so return stable named spies in the hook's declaration order.
+ useRecoilCallback: mockUseRecoilCallback,
}));
jest.mock('~/store', () => ({
@@ -109,6 +160,7 @@ jest.mock('~/store', () => ({
submissionByIndex: jest.fn(() => mockSubmissionAtom),
showStopButtonByIndex: jest.fn(() => mockShowStopButtonAtom),
runEndByIndex: jest.fn(() => mockRunEndAtom),
+ drainAfterAbortByIndex: jest.fn(() => mockDrainAfterAbortAtom),
pendingSteersByConvoId: jest.fn(() => mockPendingSteersAtom),
queuedMessagesByConvoId: jest.fn(() => mockQueuedMessagesAtom),
},
@@ -119,7 +171,9 @@ jest.mock('~/hooks/AuthContext', () => ({
}));
const mockFetchStreamStatus = jest.fn();
+const mockGetConversationById = jest.fn();
const mockConvertSteersToQueued = jest.fn();
+const mockPostGenerationRequest = jest.fn();
jest.mock('~/data-provider', () => ({
useGetStartupConfig: () => ({ data: { balance: { enabled: false } } }),
@@ -127,6 +181,19 @@ jest.mock('~/data-provider', () => ({
queueTitleGeneration: jest.fn(),
streamStatusQueryKey: (conversationId: string) => ['streamStatus', conversationId],
fetchStreamStatus: (conversationId: string) => mockFetchStreamStatus(conversationId),
+ GENERATION_PROTOCOL_VERSION: 2,
+ generationProtocolHeaders: () => ({ 'X-LibreChat-Generation-Protocol': '2' }),
+ getGenerationProtocolVersion: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2
+ ? 2
+ : 1,
+ supportsGenerationProtocolV2: (value: unknown) =>
+ value != null &&
+ typeof value === 'object' &&
+ (value as { generationProtocolVersion?: unknown }).generationProtocolVersion === 2,
+ postGenerationRequest: (...args: unknown[]) => mockPostGenerationRequest(...args),
}));
jest.mock('~/hooks/Chat/useSteerConvert', () => ({
@@ -170,6 +237,10 @@ jest.mock('librechat-data-provider', () => {
const actual = jest.requireActual('librechat-data-provider');
return {
...actual,
+ dataService: {
+ ...actual.dataService,
+ getConversationById: (...args: unknown[]) => mockGetConversationById(...args),
+ },
createPayload: jest.fn(() => ({
payload: { model: 'gpt-4o' },
server: '/api/agents/chat',
@@ -177,7 +248,10 @@ jest.mock('librechat-data-provider', () => {
removeNullishValues: jest.fn((v: unknown) => v),
apiBaseUrl: jest.fn(() => ''),
request: {
- post: jest.fn().mockResolvedValue({ streamId: 'stream-123' }),
+ post: jest.fn().mockResolvedValue({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ }),
refreshToken: jest.fn(),
dispatchTokenUpdatedEvent: jest.fn(),
},
@@ -190,6 +264,10 @@ const CONV_ID = 'conv-abc-123';
type PartialSubmission = {
conversation: { conversationId?: string };
+ clientRequestId?: string;
+ recoverySteerId?: string;
+ expectedPredecessorCreatedAt?: number;
+ queuedMessageOrigin?: unknown;
userMessage: Record<string, unknown>;
messages: TMessage[];
isTemporary: boolean;
@@ -271,7 +349,11 @@ describe('useResumableSSE', () => {
mockClearStepMaps.mockClear();
mockSetIsSubmitting.mockClear();
mockSetQueryData.mockClear();
- mockGetQueryData.mockClear();
+ mockGetQueryData.mockReset();
+ mockFetchQuery.mockReset();
+ mockFetchQuery.mockImplementation(({ queryKey }: { queryKey: unknown[] }) =>
+ Promise.resolve(mockGetQueryData(queryKey) ?? []),
+ );
mockInvalidateQueries.mockClear();
mockRemoveQueries.mockClear();
mockFindAll.mockClear();
@@ -281,15 +363,39 @@ describe('useResumableSSE', () => {
mockSetSubmission.mockClear();
mockSetShowStopButton.mockClear();
mockSetRunEnd.mockClear();
+ mockSetDrainAfterAbort.mockClear();
+ mockResolveSteerChip.mockClear();
+ mockUpdateSteerChips.mockClear();
+ mockSeedSteerChips.mockClear();
+ mockSettleAppliedSteerParts.mockClear();
+ mockConvertLocalSteersToQueued.mockClear();
+ mockUpdateGenerationEpoch.mockClear();
+ mockRestoreQueuedSubmission.mockClear();
+ mockRecoilCallbackIndex = 0;
mockConvertSteersToQueued.mockClear();
mockFetchStreamStatus.mockReset();
mockFetchStreamStatus.mockResolvedValue({ active: false });
+ mockGetConversationById.mockReset();
+ mockGetConversationById.mockResolvedValue({
+ conversationId: CONV_ID,
+ endpoint: 'agents',
+ });
(request.post as jest.Mock).mockReset();
- (request.post as jest.Mock).mockResolvedValue({ streamId: 'stream-123' });
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ });
+ (request.refreshToken as jest.Mock).mockReset();
+ (request.dispatchTokenUpdatedEvent as jest.Mock).mockReset();
+ mockPostGenerationRequest.mockReset();
+ mockPostGenerationRequest.mockImplementation((url: string, body: Record<string, unknown>) =>
+ request.post(url, { ...body, generationProtocolVersion: 2 }),
+ );
});
afterEach(() => {
jest.useRealTimers();
+ jest.restoreAllMocks();
});
const seedDraft = (conversationId: string) => {
@@ -297,8 +403,14 @@ describe('useResumableSSE', () => {
localStorage.setItem(`${LocalStorageKeys.FILES_DRAFT}${conversationId}`, '[]');
};
- const render404Scenario = async (conversationId = CONV_ID) => {
- const submission = buildSubmission({ conversation: { conversationId } });
+ const render404Scenario = async (
+ conversationId = CONV_ID,
+ submissionOverrides: Partial<PartialSubmission> = {},
+ ) => {
+ const submission = buildSubmission({
+ ...submissionOverrides,
+ conversation: { conversationId },
+ });
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
@@ -334,6 +446,7 @@ describe('useResumableSSE', () => {
expect(mockErrorHandler).not.toHaveBeenCalled();
expect(mockInvalidateQueries).toHaveBeenCalledWith({
queryKey: ['messages', CONV_ID],
+ refetchType: 'none',
});
expect(mockRemoveQueries).toHaveBeenCalledWith({
queryKey: ['streamStatus', CONV_ID],
@@ -403,6 +516,7 @@ describe('useResumableSSE', () => {
expect(mockInvalidateQueries).toHaveBeenCalledWith({
queryKey: [QueryKeys.messages, 'stream-123'],
+ refetchType: 'none',
});
expect(mockRemoveQueries).toHaveBeenCalledWith({
queryKey: ['streamStatus', 'stream-123'],
@@ -481,6 +595,47 @@ describe('useResumableSSE', () => {
unmount();
});
+ it('preserves and reattaches a generation when status proves a 404 stream is still active', async () => {
+ jest.useFakeTimers();
+ const pendingSteers = [{ steerId: 'still-active', text: 'keep me', createdAt: 1 }];
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: 'stream-123',
+ status: 'running',
+ createdAt: 1000,
+ resumeState: { pendingSteers, aggregatedContent: [] },
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+ const initialSSE = getLastSSE();
+
+ await act(async () => {
+ initialSSE._emit('error', { responseCode: 404 });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({ active: true, createdAt: 1000 }),
+ );
+ expect(mockSeedSteerChips).toHaveBeenCalledWith(CONV_ID, pendingSteers, 1000, 1);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetSubmission).not.toHaveBeenCalledWith(null);
+ expect(mockSetIsSubmitting).toHaveBeenLastCalledWith(true);
+ expect(mockSetShowStopButton).toHaveBeenLastCalledWith(true);
+
+ await advanceRetryTimer(1_000);
+ expect(mockSSEInstances).toHaveLength(2);
+ expect(getLastSSE()._url).toBe(
+ '/api/agents/chat/stream/stream-123?resume=true&generationCreatedAt=1000&generationProtocolVersion=2',
+ );
+ unmount();
+ });
+
it('claims parked steers and writes a non-completed run end on 404', async () => {
const parked = [{ steerId: 'p1', text: 'parked words', createdAt: 1 }];
mockFetchStreamStatus.mockResolvedValue({ active: false, unrecoveredSteers: parked });
@@ -492,7 +647,9 @@ describe('useResumableSSE', () => {
expect(mockFetchStreamStatus).toHaveBeenCalledTimes(1);
expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID);
- expect(mockConvertSteersToQueued).toHaveBeenCalledWith(CONV_ID, parked);
+ expect(mockConvertSteersToQueued).toHaveBeenCalledWith(CONV_ID, parked, {
+ generationProtocolVersion: 1,
+ });
// The true outcome is unknown — the run-end signal must release parked
// interrupt flags WITHOUT auto-sending queued messages.
expect(mockSetRunEnd).toHaveBeenCalledWith(
@@ -504,6 +661,80 @@ describe('useResumableSSE', () => {
unmount();
});
+ it('authorizes only the exact failed recovery source past the conversion tombstone', async () => {
+ const parked = [
+ { steerId: 'failed-source', text: 'retry these words', createdAt: 1 },
+ { steerId: 'unrelated-source', text: 'dedupe this snapshot', createdAt: 2 },
+ ];
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: false,
+ generationProtocolVersion: 2,
+ unrecoveredSteers: parked,
+ });
+
+ const { unmount } = await render404Scenario(CONV_ID, {
+ clientRequestId: 'steer-recovery:failed-source',
+ });
+ await flushMicrotasks();
+
+ expect(mockConvertSteersToQueued).toHaveBeenCalledWith(CONV_ID, parked, {
+ generationProtocolVersion: 2,
+ allowPreviouslyConvertedIds: ['failed-source'],
+ });
+ unmount();
+ });
+
+ it('suppresses persisted-applied and parked steer ids from local 404 recovery', async () => {
+ const persisted = [
+ {
+ messageId: 'persisted-response',
+ parentMessageId: 'persisted-user',
+ conversationId: CONV_ID,
+ isCreatedByUser: false,
+ text: 'already applied',
+ content: [
+ {
+ type: ContentTypes.STEER,
+ steerId: 'persisted-steer',
+ clientSteerId: 'persisted-client-steer',
+ steer: 'already applied',
+ },
+ ],
+ },
+ ] as TMessage[];
+ const parked = [
+ {
+ steerId: 'parked-steer',
+ clientSteerId: 'parked-client-steer',
+ text: 'recover exactly once',
+ createdAt: 1,
+ },
+ ];
+ mockGetQueryData.mockReturnValue(persisted);
+ mockFetchStreamStatus.mockResolvedValue({ active: false, unrecoveredSteers: parked });
+
+ const { unmount } = await render404Scenario(CONV_ID);
+ await flushMicrotasks();
+
+ expect(mockSettleAppliedSteerParts).toHaveBeenCalledWith(CONV_ID, persisted);
+ expect(mockConvertSteersToQueued).toHaveBeenCalledWith(CONV_ID, parked, {
+ generationProtocolVersion: 1,
+ });
+ expect(mockConvertLocalSteersToQueued).toHaveBeenCalledTimes(1);
+ const recoveryOptions = mockConvertLocalSteersToQueued.mock.calls[0][1] as {
+ excludeSteerIds: Iterable<string>;
+ };
+ expect(new Set(recoveryOptions.excludeSteerIds)).toEqual(
+ new Set(['persisted-steer', 'persisted-client-steer', 'parked-steer', 'parked-client-steer']),
+ );
+ unmount();
+ });
+
it('does not convert anything when the 404 status claim returns no parked steers', async () => {
const { unmount } = await render404Scenario(CONV_ID);
await act(async () => {
@@ -1039,151 +1270,156 @@ describe('useResumableSSE', () => {
unmount();
});
- it('continues retrying chat start while the server reports startup readiness pending', async () => {
+ it('carries the generation epoch on the initial SSE URL and every reconnect', async () => {
jest.useFakeTimers();
- for (let i = 0; i < 9; i++) {
- (request.post as jest.Mock).mockRejectedValueOnce(serverNotReadyError('1'));
- }
- (request.post as jest.Mock).mockResolvedValueOnce({ streamId: 'stream-ready' });
-
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: 'stream-epoch',
+ status: 'started',
+ generationCreatedAt: 1000,
+ });
const submission = buildSubmission();
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
-
await flushMicrotasks();
- for (let i = 0; i < 9; i++) {
- await advanceRetryTimer(1000);
- }
+ const initialSSE = getLastSSE();
+ expect(initialSSE._url).toBe(
+ '/api/agents/chat/stream/stream-epoch?generationCreatedAt=1000&generationProtocolVersion=2',
+ );
+ expect(initialSSE.headers).toEqual({
+ Authorization: 'Bearer test-token',
+ 'X-LibreChat-Generation-Protocol': '2',
+ });
+ expect(request.post).toHaveBeenCalledWith(
+ '/api/agents/chat',
+ expect.objectContaining({ generationProtocolVersion: 2 }),
+ );
- expect(request.post).toHaveBeenCalledTimes(10);
- expect(mockSSEInstances).toHaveLength(1);
- expect(mockSSEInstances[0].stream).toHaveBeenCalledTimes(1);
- expect(mockErrorHandler).not.toHaveBeenCalled();
+ await act(async () => {
+ initialSSE._emit('error', { responseCode: 0 });
+ });
+ await advanceRetryTimer(1000);
+
+ expect(getLastSSE()._url).toBe(
+ '/api/agents/chat/stream/stream-epoch?resume=true&generationCreatedAt=1000&generationProtocolVersion=2',
+ );
unmount();
- jest.useRealTimers();
});
- it('cancels startup readiness retries on cleanup before opening a stream', async () => {
- jest.useFakeTimers();
- (request.post as jest.Mock)
- .mockRejectedValueOnce(serverNotReadyError('1'))
- .mockResolvedValueOnce({ streamId: 'stale-stream' });
-
+ it('preserves the generation protocol header across a 401 token refresh', async () => {
+ (request.refreshToken as jest.Mock).mockResolvedValueOnce({ token: 'refreshed-token' });
const submission = buildSubmission();
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
-
await flushMicrotasks();
- unmount();
- await advanceRetryTimer(1000);
+ const sse = getLastSSE();
- expect(request.post).toHaveBeenCalledTimes(1);
- expect(mockSSEInstances).toHaveLength(0);
- expect(mockErrorHandler).not.toHaveBeenCalled();
- jest.useRealTimers();
- });
+ await act(async () => {
+ sse._emit('error', { responseCode: 401 });
+ await Promise.resolve();
+ });
- it('clears submission and stop state when starting generation fails', async () => {
- (request.post as jest.Mock).mockRejectedValueOnce({
- response: {
- status: 500,
- data: { message: 'failed to start' },
- },
+ expect(sse.headers).toEqual({
+ Authorization: 'Bearer refreshed-token',
+ 'X-LibreChat-Generation-Protocol': '2',
});
+ expect(request.dispatchTokenUpdatedEvent).toHaveBeenCalledWith('refreshed-token');
+ expect(sse.stream).toHaveBeenCalledTimes(2);
+ unmount();
+ });
+
+ it('withholds live Stop until the start response installs a generation epoch', async () => {
+ let resolveStart: ((value: { streamId: string; generationCreatedAt: number }) => void) | null =
+ null;
+ (request.post as jest.Mock).mockReturnValueOnce(
+ new Promise((resolve) => {
+ resolveStart = resolve;
+ }),
+ );
const submission = buildSubmission();
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
-
- await waitFor(() => {
- expect(mockSetSubmission).toHaveBeenCalledWith(null);
- });
+ await flushMicrotasks();
expect(mockSSEInstances).toHaveLength(0);
- expect(mockErrorHandler).toHaveBeenCalledWith(
- expect.objectContaining({
- data: {
- text: JSON.stringify({ message: 'failed to start' }),
- metadata: { streamStartFailed: true },
- },
- submission,
- }),
- );
- expect(mockSetIsSubmitting).toHaveBeenCalledWith(true);
- expect(mockSetIsSubmitting).toHaveBeenCalledWith(false);
- expect(mockSetShowStopButton).toHaveBeenCalledWith(true);
expect(mockSetShowStopButton).toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).not.toHaveBeenCalledWith(true);
+
+ await act(async () => {
+ resolveStart?.({ streamId: 'stream-epoch', generationCreatedAt: 1000 });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(getLastSSE()._url).toContain('generationCreatedAt=1000');
+ expect(mockUpdateGenerationEpoch).toHaveBeenCalledWith('stream-epoch', 1000, undefined, 1);
+ expect(mockSetShowStopButton).toHaveBeenLastCalledWith(true);
unmount();
});
- it('surfaces SSE error bodies returned while starting generation', async () => {
- (request.post as jest.Mock).mockResolvedValueOnce(
- 'event: error\ndata: {"text":"No model spec selected"}\n\n',
- );
+ it('keeps live controls hidden across reconnects when a malformed start omits the epoch', async () => {
+ jest.useFakeTimers();
+ (request.post as jest.Mock).mockResolvedValue({ streamId: 'stream-without-epoch' });
const submission = buildSubmission();
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
- await waitFor(() => {
- expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ await act(async () => {
+ getLastSSE()._emit('error');
});
- expect(mockSSEInstances).toHaveLength(0);
- expect(mockErrorHandler).toHaveBeenCalledWith(
- expect.objectContaining({
- data: {
- text: 'No model spec selected',
- metadata: { streamStartFailed: true },
- },
- submission,
- }),
- );
- expect(mockSetIsSubmitting).toHaveBeenCalledWith(false);
- expect(mockSetShowStopButton).toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).not.toHaveBeenCalledWith(true);
+ expect(mockSetShowStopButton).toHaveBeenLastCalledWith(false);
unmount();
});
- it('surfaces CRLF SSE error bodies returned while starting generation', async () => {
- (request.post as jest.Mock).mockResolvedValueOnce(
- 'event: error\r\ndata: {"text":"No model spec selected"}\r\n\r\n',
- );
+ it('adopts an authoritative replacement epoch for an epoch-less attachment', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: { pendingSteers: [], aggregatedContent: [] },
+ });
const submission = buildSubmission();
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+ const epochlessSSE = getLastSSE();
- await waitFor(() => {
- expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ await act(async () => {
+ epochlessSSE._emit('error', { responseCode: 409 });
+ await Promise.resolve();
});
+ await flushMicrotasks();
- expect(mockErrorHandler).toHaveBeenCalledWith(
- expect.objectContaining({
- data: {
- text: 'No model spec selected',
- metadata: { streamStartFailed: true },
- },
- submission,
- }),
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({ createdAt: 2000, generationHandoff: true }),
);
+ expect(mockUpdateGenerationEpoch).toHaveBeenCalledWith(CONV_ID, 2000, undefined, 2);
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
unmount();
});
- it('uses only the error event data from multi-event SSE start failures', async () => {
- (request.post as jest.Mock).mockResolvedValueOnce(
- [
- 'event: message',
- 'data: {"created":true,"message":{"messageId":"msg-1"}}',
- '',
- 'event: error',
- 'data: {"text":"Request was blocked"}',
- '',
- '',
- ].join('\n'),
- );
+ it('refetches a settled duplicate without opening an SSE stream', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ conversationId: CONV_ID,
+ status: 'settled',
+ generationProtocolVersion: 2,
+ });
const submission = buildSubmission();
const chatHelpers = buildChatHelpers();
@@ -1193,29 +1429,1407 @@ describe('useResumableSSE', () => {
expect(mockSetSubmission).toHaveBeenCalledWith(null);
});
- expect(mockErrorHandler).toHaveBeenCalledWith(
- expect.objectContaining({
- data: {
- text: 'Request was blocked',
- metadata: { streamStartFailed: true },
- },
- submission,
- }),
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockInvalidateQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.messages, CONV_ID],
+ refetchType: 'all',
+ });
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'aborted' }),
);
unmount();
});
- it('replays title events from resume state sync', async () => {
- const submission = buildSubmission();
+ it.each(['settled', 'replaced'] as const)(
+ 'rejects an unnegotiated %s start control response',
+ async (status) => {
+ (request.post as jest.Mock).mockResolvedValue({
+ status,
+ streamId: CONV_ID,
+ conversationId: CONV_ID,
+ generationCreatedAt: 1000,
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockErrorHandler).toHaveBeenCalledWith(expect.objectContaining({ submission }));
+ unmount();
+ },
+ );
+
+ it('returns a settled first turn to new chat only after an authoritative 404', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ conversationId: CONV_ID,
+ status: 'settled',
+ generationProtocolVersion: 2,
+ });
+ mockGetConversationById.mockRejectedValueOnce({ response: { status: 404 } });
+ const submission = buildSubmission({
+ conversation: {},
+ userMessage: {
+ messageId: 'msg-1',
+ conversationId: null,
+ text: 'Hello',
+ isCreatedByUser: true,
+ sender: 'User',
+ parentMessageId: Constants.NO_PARENT,
+ },
+ });
const chatHelpers = buildChatHelpers();
const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
- await act(async () => {
- await Promise.resolve();
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(chatHelpers.newConversation).toHaveBeenCalledWith({
+ template: { conversationId: String(Constants.NEW_CONVO) },
+ });
+ expect(mockRemoveQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.conversation, CONV_ID],
});
+ expect(mockRemoveQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.messages, CONV_ID],
+ });
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'aborted' }),
+ );
+ unmount();
+ });
- const titleEvent = {
+ it('preserves a settled first turn when conversation lookup fails transiently', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ conversationId: CONV_ID,
+ status: 'settled',
+ generationProtocolVersion: 2,
+ });
+ mockGetConversationById.mockRejectedValueOnce({ response: { status: 503 } });
+ const submission = buildSubmission({ conversation: {} });
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(chatHelpers.newConversation).not.toHaveBeenCalled();
+ expect(mockRemoveQueries).not.toHaveBeenCalled();
+ expect(mockInvalidateQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.messages, CONV_ID],
+ refetchType: 'all',
+ });
+ expect(mockInvalidateQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.allConversations],
+ });
+ /** The settled response carries no epoch, so it cannot authorize clearing
+ * whichever conversation/generation may now own this pane's arm. */
+ expect(mockSetDrainAfterAbort).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'error' }),
+ );
+ unmount();
+ });
+
+ it('drops an awaited settled-start callback after conversation B starts', async () => {
+ const conversationB = 'conv-b-settled-race';
+ (request.post as jest.Mock)
+ .mockResolvedValueOnce({
+ conversationId: CONV_ID,
+ status: 'settled',
+ generationProtocolVersion: 2,
+ })
+ .mockResolvedValueOnce({
+ streamId: conversationB,
+ status: 'started',
+ generationCreatedAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ let resolveConversation!: (value: { conversationId: string; endpoint: string }) => void;
+ mockGetConversationById.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveConversation = resolve;
+ }),
+ );
+ const submissionA = buildSubmission({ clientRequestId: 'settled-request-a' });
+ const submissionB = buildSubmission({
+ conversation: { conversationId: conversationB },
+ clientRequestId: 'request-b',
+ userMessage: {
+ ...submissionA.userMessage,
+ messageId: 'msg-b',
+ conversationId: conversationB,
+ },
+ initialResponse: {
+ ...submissionA.initialResponse,
+ messageId: 'resp-b',
+ conversationId: conversationB,
+ },
+ });
+ const chatHelpers = buildChatHelpers();
+ const { rerender, unmount } = renderHook(
+ ({ current }: { current: TSubmission }) => useResumableSSE(current, chatHelpers),
+ { initialProps: { current: submissionA } },
+ );
+ await waitFor(() => expect(mockGetConversationById).toHaveBeenCalledWith(CONV_ID));
+
+ rerender({ current: submissionB });
+ await waitFor(() => expect(mockSSEInstances).toHaveLength(1));
+ const generationB = getLastSSE();
+ chatHelpers.setConversation.mockClear();
+ mockSetSubmission.mockClear();
+ mockSetRunEnd.mockClear();
+ mockSetIsSubmitting.mockClear();
+ mockSetShowStopButton.mockClear();
+
+ await act(async () => {
+ resolveConversation({ conversationId: CONV_ID, endpoint: 'agents' });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(chatHelpers.setConversation).not.toHaveBeenCalled();
+ expect(mockSetSubmission).not.toHaveBeenCalledWith(null);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).not.toHaveBeenCalledWith(false);
+ expect(generationB.close).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('hands a stale SSE attachment to the active replacement generation', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: {
+ conversationId: CONV_ID,
+ runSteps: [],
+ aggregatedContent: [{ type: 'text', text: 'replacement content' }],
+ pendingSteers: [],
+ },
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const staleSSE = getLastSSE();
+ await act(async () => {
+ staleSSE._emit('error', { responseCode: 409 });
+ await Promise.resolve();
+ });
+
+ expect(staleSSE.close).toHaveBeenCalled();
+ expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID);
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({
+ active: true,
+ createdAt: 2000,
+ generationHandoff: true,
+ }),
+ );
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ unmount();
+ });
+
+ it('does not let a delayed FINAL from generation A idle active generation B', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: {
+ conversationId: CONV_ID,
+ runSteps: [],
+ aggregatedContent: [{ type: 'text', text: 'generation B content' }],
+ pendingSteers: [],
+ },
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const generationA = getLastSSE();
+ await act(async () => {
+ generationA._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ requestMessage: submission.userMessage,
+ responseMessage: submission.initialResponse,
+ }),
+ });
+ await Promise.resolve();
+ });
+
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+ expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID);
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({
+ active: true,
+ createdAt: 2000,
+ generationHandoff: true,
+ }),
+ );
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ unmount();
+ });
+
+ it('drops an awaited FINAL callback after navigation installs conversation B', async () => {
+ const conversationB = 'conv-b-456';
+ (request.post as jest.Mock)
+ .mockResolvedValueOnce({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ })
+ .mockResolvedValueOnce({
+ streamId: conversationB,
+ status: 'started',
+ generationCreatedAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ let resolveTerminalStatus!: (value: { active: false; generationProtocolVersion: 2 }) => void;
+ mockFetchStreamStatus.mockImplementationOnce(
+ () =>
+ new Promise((resolve) => {
+ resolveTerminalStatus = resolve;
+ }),
+ );
+ const submissionA = buildSubmission({ clientRequestId: 'request-a' });
+ const submissionB = buildSubmission({
+ conversation: { conversationId: conversationB },
+ clientRequestId: 'request-b',
+ userMessage: {
+ ...submissionA.userMessage,
+ messageId: 'msg-b',
+ conversationId: conversationB,
+ },
+ initialResponse: {
+ ...submissionA.initialResponse,
+ messageId: 'resp-b',
+ conversationId: conversationB,
+ },
+ });
+ const chatHelpers = buildChatHelpers();
+ const { rerender, unmount } = renderHook(
+ ({ current }: { current: TSubmission }) => useResumableSSE(current, chatHelpers),
+ { initialProps: { current: submissionA } },
+ );
+ await flushMicrotasks();
+
+ const generationA = getLastSSE();
+ act(() => {
+ generationA._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ requestMessage: submissionA.userMessage,
+ responseMessage: submissionA.initialResponse,
+ }),
+ });
+ });
+ await waitFor(() => expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID));
+
+ rerender({ current: submissionB });
+ await waitFor(() => expect(mockSSEInstances).toHaveLength(2));
+ const generationB = getLastSSE();
+ mockFinalHandler.mockClear();
+ mockSetSubmission.mockClear();
+ mockSetRunEnd.mockClear();
+ mockSetIsSubmitting.mockClear();
+ mockSetShowStopButton.mockClear();
+
+ await act(async () => {
+ resolveTerminalStatus({ active: false, generationProtocolVersion: 2 });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockSetSubmission).not.toHaveBeenCalledWith(null);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).not.toHaveBeenCalledWith(false);
+ expect(generationB.close).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('drops an awaited synthesized reconciliation after conversation B starts', async () => {
+ const conversationB = 'conv-b-789';
+ (request.post as jest.Mock)
+ .mockResolvedValueOnce({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ })
+ .mockResolvedValueOnce({
+ streamId: conversationB,
+ status: 'started',
+ generationCreatedAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ let resolveMessages!: (value: TMessage[]) => void;
+ mockFetchQuery.mockImplementationOnce(
+ () =>
+ new Promise<TMessage[]>((resolve) => {
+ resolveMessages = resolve;
+ }),
+ );
+ const submissionA = buildSubmission({ clientRequestId: 'request-a' });
+ const submissionB = buildSubmission({
+ conversation: { conversationId: conversationB },
+ clientRequestId: 'request-b',
+ userMessage: {
+ ...submissionA.userMessage,
+ messageId: 'msg-b',
+ conversationId: conversationB,
+ },
+ initialResponse: {
+ ...submissionA.initialResponse,
+ messageId: 'resp-b',
+ conversationId: conversationB,
+ },
+ });
+ const chatHelpers = buildChatHelpers();
+ const { rerender, unmount } = renderHook(
+ ({ current }: { current: TSubmission }) => useResumableSSE(current, chatHelpers),
+ { initialProps: { current: submissionA } },
+ );
+ await flushMicrotasks();
+
+ act(() => {
+ getLastSSE()._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ terminalStatus: 'aborted',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ }),
+ });
+ });
+ await waitFor(() => expect(mockFetchQuery).toHaveBeenCalled());
+
+ rerender({ current: submissionB });
+ await waitFor(() => expect(mockSSEInstances).toHaveLength(2));
+ const generationB = getLastSSE();
+ mockFetchStreamStatus.mockClear();
+ mockFinalHandler.mockClear();
+ mockSetSubmission.mockClear();
+ mockSetRunEnd.mockClear();
+ mockSetDrainAfterAbort.mockClear();
+ mockSetIsSubmitting.mockClear();
+ mockSetShowStopButton.mockClear();
+ mockSettleAppliedSteerParts.mockClear();
+
+ await act(async () => {
+ resolveMessages([]);
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockFetchStreamStatus).not.toHaveBeenCalled();
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockSettleAppliedSteerParts).not.toHaveBeenCalled();
+ expect(mockSetSubmission).not.toHaveBeenCalledWith(null);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetDrainAfterAbort).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).not.toHaveBeenCalledWith(false);
+ expect(generationB.close).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('does not let a delayed error from generation A idle active generation B', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: {
+ conversationId: CONV_ID,
+ runSteps: [],
+ aggregatedContent: [{ type: 'text', text: 'generation B content' }],
+ pendingSteers: [],
+ },
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const generationA = getLastSSE();
+ await act(async () => {
+ generationA._emit('error', {
+ data: JSON.stringify({
+ error: 'generation A failed late',
+ generationProtocolVersion: 2,
+ }),
+ });
+ await Promise.resolve();
+ });
+
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+ expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID);
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({
+ active: true,
+ createdAt: 2000,
+ generationHandoff: true,
+ }),
+ );
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ unmount();
+ });
+
+ it('retries the exact fenced generation when FINAL races an active status', async () => {
+ jest.useFakeTimers();
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const generationSSE = getLastSSE();
+ const sseCount = mockSSEInstances.length;
+ await act(async () => {
+ generationSSE._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ requestMessage: submission.userMessage,
+ responseMessage: submission.initialResponse,
+ }),
+ });
+ await Promise.resolve();
+ });
+
+ expect(generationSSE.close).toHaveBeenCalled();
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+
+ await advanceRetryTimer(250);
+
+ expect(mockSSEInstances).toHaveLength(sseCount + 1);
+ expect(getLastSSE()._url).toContain('resume=true');
+ expect(getLastSSE()._url).toContain('generationCreatedAt=1000');
+ unmount();
+ });
+
+ it('retries the exact fenced generation when error status is inconclusive', async () => {
+ jest.useFakeTimers();
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockRejectedValue(new Error('status unavailable'));
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const generationSSE = getLastSSE();
+ const sseCount = mockSSEInstances.length;
+ await act(async () => {
+ generationSSE._emit('error', {
+ data: JSON.stringify({
+ error: 'generation failed',
+ generationProtocolVersion: 2,
+ }),
+ });
+ await Promise.resolve();
+ });
+
+ expect(generationSSE.close).toHaveBeenCalled();
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+
+ await advanceRetryTimer(250);
+
+ expect(mockSSEInstances).toHaveLength(sseCount + 1);
+ expect(getLastSSE()._url).toContain('resume=true');
+ expect(getLastSSE()._url).toContain('generationCreatedAt=1000');
+ unmount();
+ });
+
+ it('hands a replaced start to the authoritative generation without attaching the stale submission', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ status: 'replaced',
+ streamId: CONV_ID,
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ const pendingSteers = [{ steerId: 'replacement-steer', text: 'new epoch', createdAt: 2 }];
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: {
+ conversationId: CONV_ID,
+ runSteps: [],
+ aggregatedContent: [{ type: 'text', text: 'replacement content' }],
+ pendingSteers,
+ },
+ });
+ mockGetQueryData.mockImplementation((queryKey: unknown[]) =>
+ queryKey[0] === QueryKeys.conversation
+ ? { conversationId: CONV_ID, endpoint: 'agents' }
+ : undefined,
+ );
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({
+ active: true,
+ createdAt: 2000,
+ generationHandoff: true,
+ }),
+ );
+ expect(mockSeedSteerChips).toHaveBeenCalledWith(CONV_ID, pendingSteers, 2000, 2);
+ expect(mockUpdateGenerationEpoch).toHaveBeenCalledWith(CONV_ID, 2000, undefined, 2);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('only disarms the replacement epoch when replacement status cannot be confirmed', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ status: 'replaced',
+ streamId: CONV_ID,
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: false,
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ });
+
+ const { unmount } = renderHook(() => useResumableSSE(buildSubmission(), buildChatHelpers()));
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockSetDrainAfterAbort).toHaveBeenCalledWith(expect.any(Function));
+ const conversationBArm = {
+ conversationId: 'conv-b',
+ generationCreatedAt: 3000,
+ };
+ const newerSameConversationArm = {
+ conversationId: CONV_ID,
+ generationCreatedAt: 3000,
+ };
+ expect(applyLastDrainAfterAbortUpdate(conversationBArm)).toBe(conversationBArm);
+ expect(applyLastDrainAfterAbortUpdate(newerSameConversationArm)).toBe(newerSameConversationArm);
+ expect(
+ applyLastDrainAfterAbortUpdate({
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ }),
+ ).toBe(false);
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, generationCreatedAt: 2000 }),
+ );
+ unmount();
+ });
+
+ it('never treats an incomplete replaced response as an ordinary stream start', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ status: 'replaced',
+ streamId: CONV_ID,
+ conversationId: CONV_ID,
+ generationProtocolVersion: 2,
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockErrorHandler).toHaveBeenCalledWith(expect.objectContaining({ submission }));
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('reconciles a synthesized terminal frame without rendering a bogus final or error', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ const persisted = [
+ {
+ messageId: 'persisted-response',
+ conversationId: CONV_ID,
+ isCreatedByUser: false,
+ text: 'Persisted answer',
+ },
+ ] as TMessage[];
+ mockGetQueryData.mockReturnValue(persisted);
+ mockFetchStreamStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const sse = getLastSSE();
+ await act(async () => {
+ sse._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'complete',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ }),
+ });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ expect(mockInvalidateQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.messages, CONV_ID],
+ refetchType: 'none',
+ });
+ expect(mockSettleAppliedSteerParts).toHaveBeenCalledWith(CONV_ID, persisted);
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'completed' }),
+ );
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ expect(mockUpdateGenerationEpoch).toHaveBeenCalledWith(CONV_ID, null, 1000);
+ unmount();
+ });
+
+ it('ignores v2-only lifecycle reconciliation on a legacy generation', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+ const sse = getLastSSE();
+ mockInvalidateQueries.mockClear();
+
+ await act(async () => {
+ sse._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ generationProtocolVersion: 2,
+ generationCreatedAt: 1000,
+ conversation: { conversationId: CONV_ID },
+ }),
+ });
+ await Promise.resolve();
+ });
+
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockInvalidateQueries).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(sse.close).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('does not publish a completed run when synthesized persistence cannot be fetched', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchQuery.mockRejectedValueOnce(new Error('message refetch failed'));
+ mockFetchStreamStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ await act(async () => {
+ getLastSSE()._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'complete',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ }),
+ });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'error' }),
+ );
+ expect(mockSetRunEnd).not.toHaveBeenCalledWith(
+ expect.objectContaining({ outcome: 'completed' }),
+ );
+ expect(mockConvertLocalSteersToQueued).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('disarms interrupt drain before publishing an abort persistence failure', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchQuery.mockResolvedValueOnce([]);
+ mockFetchStreamStatus.mockResolvedValue({ active: false, generationProtocolVersion: 2 });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ await act(async () => {
+ getLastSSE()._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ terminalStatus: 'aborted',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ }),
+ });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockSetDrainAfterAbort).toHaveBeenCalledWith(expect.any(Function));
+ const conversationBArm = {
+ conversationId: 'conv-b',
+ generationCreatedAt: 2000,
+ };
+ const newerSameConversationArm = {
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ };
+ expect(applyLastDrainAfterAbortUpdate(conversationBArm)).toBe(conversationBArm);
+ expect(applyLastDrainAfterAbortUpdate(newerSameConversationArm)).toBe(newerSameConversationArm);
+ expect(
+ applyLastDrainAfterAbortUpdate({
+ conversationId: CONV_ID,
+ generationCreatedAt: 1000,
+ }),
+ ).toBe(false);
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'aborted' }),
+ );
+ expect(mockSetDrainAfterAbort.mock.invocationCallOrder[0]).toBeLessThan(
+ mockSetRunEnd.mock.invocationCallOrder[0],
+ );
+ unmount();
+ });
+
+ it('hands a synthesized generation-replaced frame to the newer active epoch', async () => {
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: CONV_ID,
+ status: 'started',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ streamId: CONV_ID,
+ status: 'running',
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: { runSteps: [], aggregatedContent: [] },
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ const sse = getLastSSE();
+ await act(async () => {
+ sse._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ conversation: { conversationId: CONV_ID },
+ }),
+ });
+ await Promise.resolve();
+ });
+
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({ createdAt: 2000, generationHandoff: true }),
+ );
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockFinalHandler).not.toHaveBeenCalled();
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ expect(mockUpdateGenerationEpoch).toHaveBeenCalledWith(CONV_ID, 2000, undefined, 2);
+ unmount();
+ });
+
+ it('settles every applied steer immediately and cancels all render retries at terminal', async () => {
+ let nextFrameId = 0;
+ const requestFrame = jest
+ .spyOn(window, 'requestAnimationFrame')
+ .mockImplementation(() => ++nextFrameId);
+ const cancelFrame = jest.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {});
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+ const sse = getLastSSE();
+
+ for (const [steerId, clientSteerId] of [
+ ['server-1', 'client-1'],
+ ['server-2', 'client-2'],
+ ]) {
+ await act(async () => {
+ sse._emit('message', {
+ data: JSON.stringify({
+ event: 'on_steer_applied',
+ data: {
+ steerId,
+ clientSteerId,
+ conversationId: CONV_ID,
+ index: 0,
+ part: {
+ type: ContentTypes.STEER,
+ [ContentTypes.STEER]: steerId,
+ steerId,
+ clientSteerId,
+ },
+ },
+ }),
+ });
+ });
+ }
+
+ expect(mockResolveSteerChip).toHaveBeenNthCalledWith(1, CONV_ID, 'server-1', 'client-1');
+ expect(mockResolveSteerChip).toHaveBeenNthCalledWith(2, CONV_ID, 'server-2', 'client-2');
+ expect(requestFrame).toHaveBeenCalledTimes(2);
+
+ await act(async () => {
+ sse._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ conversation: { conversationId: CONV_ID },
+ requestMessage: { messageId: 'msg-1', conversationId: CONV_ID },
+ responseMessage: { messageId: 'resp-1', conversationId: CONV_ID },
+ }),
+ });
+ });
+
+ expect(cancelFrame).toHaveBeenCalledTimes(2);
+ expect(cancelFrame).toHaveBeenNthCalledWith(1, 1);
+ expect(cancelFrame).toHaveBeenNthCalledWith(2, 2);
+ unmount();
+ requestFrame.mockRestore();
+ cancelFrame.mockRestore();
+ });
+
+ it('does not publish a completed run when the ordinary final cannot be applied', async () => {
+ mockFinalHandler.mockImplementationOnce(() => {
+ throw new Error('final application failed');
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ await act(async () => {
+ getLastSSE()._emit('message', {
+ data: JSON.stringify({
+ final: true,
+ conversation: { conversationId: CONV_ID },
+ requestMessage: { messageId: 'msg-1', conversationId: CONV_ID },
+ responseMessage: {
+ messageId: 'resp-1',
+ conversationId: CONV_ID,
+ unfinished: false,
+ },
+ }),
+ });
+ });
+
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'error' }),
+ );
+ expect(mockSetRunEnd).not.toHaveBeenCalledWith(
+ expect.objectContaining({ outcome: 'completed' }),
+ );
+ unmount();
+ });
+
+ it('completes an applied authoritative final without requiring a client persistence refetch', async () => {
+ mockFetchQuery.mockRejectedValue(new Error('a reconciliation fetch would fail'));
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+ const finalPayload = {
+ final: true,
+ conversation: { conversationId: CONV_ID },
+ requestMessage: { messageId: 'msg-1', conversationId: CONV_ID },
+ responseMessage: {
+ messageId: 'resp-1',
+ conversationId: CONV_ID,
+ unfinished: false,
+ },
+ };
+
+ await act(async () => {
+ getLastSSE()._emit('message', { data: JSON.stringify(finalPayload) });
+ });
+
+ expect(mockFinalHandler).toHaveBeenCalledWith(
+ finalPayload,
+ expect.objectContaining(submission),
+ );
+ expect(mockFetchQuery).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'completed' }),
+ );
+ unmount();
+ });
+
+ it('continues retrying chat start while the server reports startup readiness pending', async () => {
+ jest.useFakeTimers();
+ for (let i = 0; i < 9; i++) {
+ (request.post as jest.Mock).mockRejectedValueOnce(serverNotReadyError('1'));
+ }
+ (request.post as jest.Mock).mockResolvedValueOnce({ streamId: 'stream-ready' });
+
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await flushMicrotasks();
+
+ for (let i = 0; i < 9; i++) {
+ await advanceRetryTimer(1000);
+ }
+
+ expect(request.post).toHaveBeenCalledTimes(10);
+ expect(mockSSEInstances).toHaveLength(1);
+ expect(mockSSEInstances[0].stream).toHaveBeenCalledTimes(1);
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ unmount();
+ jest.useRealTimers();
+ });
+
+ it('cancels startup readiness retries on cleanup before opening a stream', async () => {
+ jest.useFakeTimers();
+ (request.post as jest.Mock)
+ .mockRejectedValueOnce(serverNotReadyError('1'))
+ .mockResolvedValueOnce({ streamId: 'stale-stream' });
+
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await flushMicrotasks();
+ unmount();
+ await advanceRetryTimer(1000);
+
+ expect(request.post).toHaveBeenCalledTimes(1);
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ jest.useRealTimers();
+ });
+
+ it('clears submission and stop state when starting generation fails', async () => {
+ (request.post as jest.Mock).mockRejectedValueOnce({
+ response: {
+ status: 500,
+ data: { message: 'failed to start' },
+ },
+ });
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await waitFor(() => {
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ });
+
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockErrorHandler).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: {
+ text: JSON.stringify({ message: 'failed to start' }),
+ metadata: { streamStartFailed: true },
+ },
+ submission,
+ }),
+ );
+ expect(mockSetIsSubmitting).toHaveBeenCalledWith(true);
+ expect(mockSetIsSubmitting).toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).not.toHaveBeenCalledWith(true);
+ expect(mockSetShowStopButton).toHaveBeenCalledWith(false);
+ unmount();
+ });
+
+ it('restores an exact queued row after a definitive pre-create rejection', async () => {
+ (request.post as jest.Mock).mockRejectedValueOnce({
+ response: { status: 429, data: { message: 'too many requests' } },
+ });
+ const queuedMessageOrigin = {
+ item: { id: 'queued-1', text: 'try later', createdAt: 1 },
+ beforeIds: [],
+ afterIds: ['queued-2'],
+ };
+ const submission = buildSubmission({ queuedMessageOrigin });
+ const { unmount } = renderHook(() => useResumableSSE(submission, buildChatHelpers()));
+
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockRestoreQueuedSubmission).toHaveBeenCalledWith(submission);
+ expect(mockSSEInstances).toHaveLength(0);
+ unmount();
+ });
+
+ it('restores queued C and hands off to B when B wins the predecessor status window', async () => {
+ (request.post as jest.Mock).mockRejectedValueOnce({
+ response: {
+ status: 409,
+ data: {
+ status: 'predecessor_mismatch',
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ streamId: CONV_ID,
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ active: true,
+ },
+ },
+ });
+ // The atomic admission response can beat status publication. Its active
+ // replacement identity must still win over retrying C against stale A.
+ mockFetchStreamStatus.mockRejectedValueOnce(new Error('status publication lag'));
+ const queuedMessageOrigin = {
+ item: {
+ id: 'queued-c',
+ text: 'send C after B',
+ createdAt: 3,
+ expectedPredecessorCreatedAt: 1000,
+ },
+ beforeIds: [],
+ afterIds: ['queued-d'],
+ };
+ const submission = buildSubmission({
+ expectedPredecessorCreatedAt: 1000,
+ queuedMessageOrigin,
+ });
+ const { unmount } = renderHook(() => useResumableSSE(submission, buildChatHelpers()));
+
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(request.post).toHaveBeenCalledTimes(1);
+ expect(mockRestoreQueuedSubmission).toHaveBeenNthCalledWith(1, submission);
+ expect(mockRestoreQueuedSubmission).toHaveBeenLastCalledWith(submission, 2000);
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({
+ active: true,
+ streamId: CONV_ID,
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ generationHandoff: true,
+ }),
+ );
+ expect(mockInvalidateQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.messages, CONV_ID],
+ refetchType: 'all',
+ });
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).not.toHaveBeenCalledWith(false);
+ unmount();
+ });
+
+ it('keeps the rejected row queued when predecessor mismatch finds no active replacement', async () => {
+ (request.post as jest.Mock).mockRejectedValueOnce({
+ response: {
+ status: 409,
+ data: {
+ status: 'predecessor_mismatch',
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ streamId: CONV_ID,
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ active: false,
+ },
+ },
+ });
+ mockFetchStreamStatus.mockResolvedValueOnce({
+ active: false,
+ generationProtocolVersion: 2,
+ createdAt: 2000,
+ });
+ const queuedMessageOrigin = {
+ item: {
+ id: 'queued-c',
+ text: 'keep C queued',
+ createdAt: 3,
+ expectedPredecessorCreatedAt: 1000,
+ },
+ beforeIds: [],
+ afterIds: [],
+ };
+ const submission = buildSubmission({
+ expectedPredecessorCreatedAt: 1000,
+ queuedMessageOrigin,
+ });
+ const { unmount } = renderHook(() => useResumableSSE(submission, buildChatHelpers()));
+
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(request.post).toHaveBeenCalledTimes(1);
+ expect(mockRestoreQueuedSubmission).toHaveBeenNthCalledWith(1, submission);
+ expect(mockRestoreQueuedSubmission).toHaveBeenLastCalledWith(submission, 2000);
+ expect(mockInvalidateQueries).toHaveBeenCalledWith({
+ queryKey: [QueryKeys.messages, CONV_ID],
+ refetchType: 'all',
+ });
+ expect(mockErrorHandler).not.toHaveBeenCalled();
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ expect(mockSetIsSubmitting).toHaveBeenCalledWith(false);
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockSetDrainAfterAbort).toHaveBeenCalledWith(expect.any(Function));
+ const replacementArm = {
+ conversationId: CONV_ID,
+ generationCreatedAt: 2000,
+ };
+ expect(applyLastDrainAfterAbortUpdate(replacementArm)).toBe(replacementArm);
+ expect(
+ applyLastDrainAfterAbortUpdate({
+ conversationId: CONV_ID,
+ generationCreatedAt: 1000,
+ }),
+ ).toBe(false);
+ unmount();
+ });
+
+ it('re-queues a released recovery source after an ambiguous start failure', async () => {
+ (request.post as jest.Mock).mockRejectedValueOnce({
+ response: { status: 500, data: { message: 'failed to start' } },
+ });
+ const released = { steerId: 'source-steer', text: 'recover again', createdAt: 1 };
+ mockFetchStreamStatus.mockResolvedValue({
+ active: false,
+ generationProtocolVersion: 2,
+ unrecoveredSteers: [released],
+ });
+ const submission = buildSubmission({
+ clientRequestId: 'attempt-uuid',
+ recoverySteerId: 'source-steer',
+ });
+ const { unmount } = renderHook(() => useResumableSSE(submission, buildChatHelpers()));
+
+ await waitFor(() => expect(mockSetSubmission).toHaveBeenCalledWith(null));
+
+ expect(mockConvertSteersToQueued).toHaveBeenCalledWith(CONV_ID, [released], {
+ generationProtocolVersion: 2,
+ allowPreviouslyConvertedIds: ['source-steer'],
+ });
+ expect(mockRestoreQueuedSubmission).not.toHaveBeenCalled();
+ unmount();
+ });
+
+ it('surfaces SSE error bodies returned while starting generation', async () => {
+ (request.post as jest.Mock).mockResolvedValueOnce(
+ 'event: error\ndata: {"text":"No model spec selected"}\n\n',
+ );
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await waitFor(() => {
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ });
+
+ expect(mockSSEInstances).toHaveLength(0);
+ expect(mockErrorHandler).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: {
+ text: 'No model spec selected',
+ metadata: { streamStartFailed: true },
+ },
+ submission,
+ }),
+ );
+ expect(mockSetIsSubmitting).toHaveBeenCalledWith(false);
+ expect(mockSetShowStopButton).toHaveBeenCalledWith(false);
+ unmount();
+ });
+
+ it('surfaces CRLF SSE error bodies returned while starting generation', async () => {
+ (request.post as jest.Mock).mockResolvedValueOnce(
+ 'event: error\r\ndata: {"text":"No model spec selected"}\r\n\r\n',
+ );
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await waitFor(() => {
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ });
+
+ expect(mockErrorHandler).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: {
+ text: 'No model spec selected',
+ metadata: { streamStartFailed: true },
+ },
+ submission,
+ }),
+ );
+ unmount();
+ });
+
+ it('uses only the error event data from multi-event SSE start failures', async () => {
+ (request.post as jest.Mock).mockResolvedValueOnce(
+ [
+ 'event: message',
+ 'data: {"created":true,"message":{"messageId":"msg-1"}}',
+ '',
+ 'event: error',
+ 'data: {"text":"Request was blocked"}',
+ '',
+ '',
+ ].join('\n'),
+ );
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await waitFor(() => {
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ });
+
+ expect(mockErrorHandler).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: {
+ text: 'Request was blocked',
+ metadata: { streamStartFailed: true },
+ },
+ submission,
+ }),
+ );
+ unmount();
+ });
+
+ it('replays title events from resume state sync', async () => {
+ const submission = buildSubmission();
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ const titleEvent = {
event: 'title',
data: {
conversationId: CONV_ID,
@@ -1857,6 +3471,159 @@ describe('useResumableSSE', () => {
unmount();
});
+ it('keeps an active job locked and resubscribes after the reconnect retry ceiling', async () => {
+ jest.useFakeTimers();
+ const pendingSteers = [
+ { steerId: 'still-running-steer', text: 'wait for boundary', createdAt: 1 },
+ ];
+ const aggregatedContent = [{ type: 'text', text: 'partial answer' }];
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ status: 'running',
+ streamId: CONV_ID,
+ createdAt: 1000,
+ resumeState: { pendingSteers, aggregatedContent },
+ });
+ const submission = {
+ ...buildSubmission(),
+ resumeStreamId: CONV_ID,
+ resumeGenerationCreatedAt: 1000,
+ } as TSubmission & { resumeStreamId: string; resumeGenerationCreatedAt: number };
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ for (const delay of [1_000, 2_000, 4_000, 8_000, 16_000]) {
+ const current = getLastSSE();
+ await act(async () => {
+ current._emit('error');
+ await Promise.resolve();
+ });
+ await advanceRetryTimer(delay);
+ }
+ expect(mockSSEInstances).toHaveLength(6);
+
+ const retryCeilingSSE = getLastSSE();
+ const countAtCeiling = mockSSEInstances.length;
+ await act(async () => {
+ retryCeilingSSE._emit('error');
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID);
+ expect(mockSeedSteerChips).toHaveBeenCalledWith(CONV_ID, pendingSteers, 1000, 1);
+ expect(mockSettleAppliedSteerParts).toHaveBeenCalledWith(CONV_ID, aggregatedContent);
+ expect(mockSetIsSubmitting).toHaveBeenLastCalledWith(true);
+ expect(mockSetShowStopButton).toHaveBeenLastCalledWith(true);
+ expect(mockSetSubmission).not.toHaveBeenCalledWith(null);
+ expect(mockSSEInstances).toHaveLength(countAtCeiling);
+
+ await advanceRetryTimer(30_000);
+ expect(mockSSEInstances).toHaveLength(countAtCeiling + 1);
+ expect(getLastSSE().stream).toHaveBeenCalledTimes(1);
+ unmount();
+ });
+
+ it('hands off to a newer epoch discovered at the reconnect retry ceiling', async () => {
+ jest.useFakeTimers();
+ const pendingSteers = [
+ { steerId: 'replacement-steer', text: 'belongs to replacement', createdAt: 2 },
+ ];
+ mockFetchStreamStatus.mockResolvedValue({
+ active: true,
+ status: 'running',
+ streamId: CONV_ID,
+ createdAt: 2000,
+ generationProtocolVersion: 2,
+ resumeState: { pendingSteers, aggregatedContent: [] },
+ });
+ const submission = {
+ ...buildSubmission(),
+ resumeStreamId: CONV_ID,
+ resumeGenerationCreatedAt: 1000,
+ resumeGenerationProtocolVersion: 2,
+ } as TSubmission & {
+ resumeStreamId: string;
+ resumeGenerationCreatedAt: number;
+ resumeGenerationProtocolVersion: 2;
+ };
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ for (const delay of [1_000, 2_000, 4_000, 8_000, 16_000]) {
+ await act(async () => {
+ getLastSSE()._emit('error');
+ await Promise.resolve();
+ });
+ await advanceRetryTimer(delay);
+ }
+ const countAtCeiling = mockSSEInstances.length;
+
+ await act(async () => {
+ getLastSSE()._emit('error');
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockSetQueryData).toHaveBeenCalledWith(
+ ['streamStatus', CONV_ID],
+ expect.objectContaining({ createdAt: 2000, generationHandoff: true }),
+ );
+ expect(mockSeedSteerChips).toHaveBeenCalledWith(CONV_ID, pendingSteers, 2000, 2);
+ expect(mockSetSubmission).toHaveBeenCalledWith(null);
+ expect(mockSetRunEnd).not.toHaveBeenCalled();
+ await advanceRetryTimer(30_000);
+ expect(mockSSEInstances).toHaveLength(countAtCeiling);
+ unmount();
+ });
+
+ it('does not complete a terminal retry-ceiling status when persisted messages cannot be fetched', async () => {
+ jest.useFakeTimers();
+ mockFetchStreamStatus.mockResolvedValue({
+ active: false,
+ status: 'complete',
+ streamId: CONV_ID,
+ createdAt: 1000,
+ resumeState: { pendingSteers: [], aggregatedContent: [] },
+ });
+ mockFetchQuery.mockRejectedValueOnce(new Error('message refetch failed'));
+ const submission = {
+ ...buildSubmission(),
+ resumeStreamId: CONV_ID,
+ resumeGenerationCreatedAt: 1000,
+ } as TSubmission & { resumeStreamId: string; resumeGenerationCreatedAt: number };
+ const chatHelpers = buildChatHelpers();
+
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ for (const delay of [1_000, 2_000, 4_000, 8_000, 16_000]) {
+ await act(async () => {
+ getLastSSE()._emit('error');
+ await Promise.resolve();
+ });
+ await advanceRetryTimer(delay);
+ }
+ await act(async () => {
+ getLastSSE()._emit('error');
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'error' }),
+ );
+ expect(mockSetRunEnd).not.toHaveBeenCalledWith(
+ expect.objectContaining({ outcome: 'completed' }),
+ );
+ expect(mockConvertLocalSteersToQueued).not.toHaveBeenCalled();
+ unmount();
+ });
+
it.each([undefined, 500, 503])(
'does not call errorHandler for responseCode %s (reconnect path)',
async (responseCode) => {
@@ -1980,6 +3747,50 @@ describe('useResumableSSE', () => {
unmount();
});
+ it('re-queues the exact recovery source released by a post-start terminal error', async () => {
+ const released = { steerId: 'failed-source', text: 'retry these words', createdAt: 1 };
+ (request.post as jest.Mock).mockResolvedValue({
+ streamId: 'stream-123',
+ generationCreatedAt: 1000,
+ generationProtocolVersion: 2,
+ });
+ mockFetchStreamStatus.mockResolvedValue({
+ active: false,
+ generationProtocolVersion: 2,
+ unrecoveredSteers: [
+ released,
+ { steerId: 'unrelated-source', text: 'leave deduped', createdAt: 2 },
+ ],
+ });
+ const submission = buildSubmission({
+ clientRequestId: 'steer-recovery:failed-source',
+ });
+ const chatHelpers = buildChatHelpers();
+ const { unmount } = renderHook(() => useResumableSSE(submission, chatHelpers));
+ await flushMicrotasks();
+
+ await act(async () => {
+ getLastSSE()._emit('error', {
+ data: JSON.stringify({
+ error: 'initialization failed',
+ generationProtocolVersion: 2,
+ }),
+ });
+ await Promise.resolve();
+ });
+ await flushMicrotasks();
+
+ expect(mockFetchStreamStatus).toHaveBeenCalledWith(CONV_ID);
+ expect(mockConvertSteersToQueued).toHaveBeenCalledWith(CONV_ID, [released], {
+ generationProtocolVersion: 2,
+ allowPreviouslyConvertedIds: ['failed-source'],
+ });
+ expect(mockSetRunEnd).toHaveBeenCalledWith(
+ expect.objectContaining({ conversationId: CONV_ID, outcome: 'error' }),
+ );
+ unmount();
+ });
+
it('removes the optimistic sidebar row when a new conversation errors before created', async () => {
mockFindAll.mockReturnValue([{ queryKey: [QueryKeys.allConversations] }]);
const submission = buildSubmission({
diff --git a/client/src/hooks/SSE/__tests__/useResumeOnLoad.spec.tsx b/client/src/hooks/SSE/__tests__/useResumeOnLoad.spec.tsx
index 3f61f6f098..a776bd4f40 100644
--- a/client/src/hooks/SSE/__tests__/useResumeOnLoad.spec.tsx
+++ b/client/src/hooks/SSE/__tests__/useResumeOnLoad.spec.tsx
@@ -1,6 +1,6 @@
-import { RecoilRoot, useRecoilValue } from 'recoil';
-import { Constants } from 'librechat-data-provider';
import { renderHook, act } from '@testing-library/react';
+import { Constants, ContentTypes } from 'librechat-data-provider';
+import { RecoilRoot, useRecoilValue, useSetRecoilState } from 'recoil';
import type { TMessage, TConversation, TSubmission } from 'librechat-data-provider';
import type { MutableSnapshot } from 'recoil';
import type { ReactNode } from 'react';
@@ -85,6 +85,7 @@ function renderUseResumeOnLoad({
onQueuedMessages?: (queued: QueuedMessage[]) => void;
}) {
const getMessages = jest.fn(getMessagesOverride ?? (() => messages));
+ let setSubmissionState: ((submission: TSubmission | null) => void) | undefined;
const initializeState = (snapshot: MutableSnapshot) => {
snapshot.set(store.conversationByIndex(0), buildConversation(conversationId));
snapshot.set(store.submissionByIndex(0), submission);
@@ -95,6 +96,7 @@ function renderUseResumeOnLoad({
const SubmissionProbe = () => {
const currentSubmission = useRecoilValue(store.submissionByIndex(0));
+ setSubmissionState = useSetRecoilState(store.submissionByIndex(0));
onSubmission?.(currentSubmission);
return null;
};
@@ -128,6 +130,7 @@ function renderUseResumeOnLoad({
return {
getMessages,
+ setSubmission: (nextSubmission: TSubmission | null) => setSubmissionState?.(nextSubmission),
...renderHook(() => useResumeOnLoad(conversationId, getMessages, 0, messagesLoaded), {
wrapper,
}),
@@ -233,6 +236,8 @@ describe('useResumeOnLoad', () => {
active: true,
status: 'running',
streamId: CONVERSATION_ID,
+ createdAt: 1234,
+ generationProtocolVersion: 2,
resumeState: {
runSteps: [],
aggregatedContent: [{ type: 'text', text: 'Streaming...' }],
@@ -279,6 +284,82 @@ describe('useResumeOnLoad', () => {
model: 'gpt-4.1',
}),
);
+ expect(
+ (
+ observedSubmissions[observedSubmissions.length - 1] as TSubmission & {
+ resumeGenerationCreatedAt?: number;
+ resumeGenerationProtocolVersion?: number;
+ }
+ ).resumeGenerationCreatedAt,
+ ).toBe(1234);
+ expect(
+ (
+ observedSubmissions[observedSubmissions.length - 1] as TSubmission & {
+ resumeGenerationProtocolVersion?: number;
+ }
+ ).resumeGenerationProtocolVersion,
+ ).toBe(2);
+ });
+
+ it('reprocesses the same conversation when a stale attachment hands off to a newer epoch', async () => {
+ const observedSubmissions: Array<TSubmission | null> = [];
+ const staleSubmission = buildSubmission(CONVERSATION_ID);
+ mockUseStreamStatus.mockReturnValue({
+ isSuccess: true,
+ isFetching: false,
+ data: {
+ active: true,
+ generationHandoff: true,
+ generationProtocolVersion: 2,
+ status: 'running',
+ streamId: CONVERSATION_ID,
+ createdAt: 2000,
+ resumeState: {
+ runSteps: [],
+ aggregatedContent: [{ type: 'text', text: 'replacement content' }],
+ responseMessageId: 'replacement-response',
+ conversationId: CONVERSATION_ID,
+ userMessage: {
+ messageId: 'replacement-user',
+ parentMessageId: Constants.NO_PARENT,
+ conversationId: CONVERSATION_ID,
+ text: 'Replacement prompt',
+ },
+ },
+ },
+ });
+
+ const rendered = renderUseResumeOnLoad({
+ submission: staleSubmission,
+ messages: [buildUserMessage(CONVERSATION_ID)],
+ onSubmission: (currentSubmission) => observedSubmissions.push(currentSubmission),
+ });
+
+ await act(async () => {
+ rendered.setSubmission(null);
+ await Promise.resolve();
+ });
+
+ const replacement = observedSubmissions[observedSubmissions.length - 1] as TSubmission & {
+ resumeStreamId?: string;
+ resumeGenerationCreatedAt?: number;
+ };
+ expect(replacement.resumeStreamId).toBe(CONVERSATION_ID);
+ expect(replacement.resumeGenerationCreatedAt).toBe(2000);
+ expect(replacement.userMessage?.messageId).toBe('replacement-user');
+
+ await act(async () => {
+ rendered.setSubmission(null);
+ await Promise.resolve();
+ });
+
+ const replacementInstalls = observedSubmissions.filter(
+ (candidate) =>
+ (candidate as (TSubmission & { resumeGenerationCreatedAt?: number }) | null)
+ ?.resumeGenerationCreatedAt === 2000,
+ );
+ expect(replacementInstalls).toHaveLength(1);
+ expect(observedSubmissions[observedSubmissions.length - 1]).toBeNull();
});
it('strips the paused user/assistant rows from submission.messages (no duplicate on resume)', async () => {
@@ -337,6 +418,63 @@ describe('useResumeOnLoad', () => {
expect(submission?.initialResponse?.messageId).toBe(RESPONSE_MESSAGE_ID);
});
+ it('prefers the exact active response over an older assistant sibling', async () => {
+ const observedSubmissions: Array<TSubmission | null> = [];
+ const userMessage = buildUserMessage(CONVERSATION_ID);
+ const olderSibling = {
+ messageId: 'older-sibling-response',
+ parentMessageId: userMessage.messageId,
+ conversationId: CONVERSATION_ID,
+ text: 'Older sibling',
+ isCreatedByUser: false,
+ } as TMessage;
+ const activeResponse = {
+ messageId: 'active-response',
+ parentMessageId: userMessage.messageId,
+ conversationId: CONVERSATION_ID,
+ text: '',
+ isCreatedByUser: false,
+ unfinished: true,
+ } as TMessage;
+
+ mockUseStreamStatus.mockReturnValue({
+ isSuccess: true,
+ isFetching: false,
+ data: {
+ active: true,
+ status: 'running',
+ streamId: CONVERSATION_ID,
+ resumeState: {
+ runSteps: [],
+ aggregatedContent: [{ type: 'text', text: 'Active branch streaming' }],
+ responseMessageId: activeResponse.messageId,
+ conversationId: CONVERSATION_ID,
+ userMessage: {
+ messageId: userMessage.messageId,
+ parentMessageId: userMessage.parentMessageId,
+ conversationId: CONVERSATION_ID,
+ text: userMessage.text,
+ },
+ },
+ },
+ });
+
+ renderUseResumeOnLoad({
+ messages: [userMessage, olderSibling, activeResponse],
+ onSubmission: (currentSubmission) => observedSubmissions.push(currentSubmission),
+ });
+
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ const submission = observedSubmissions[observedSubmissions.length - 1];
+ expect(submission?.initialResponse?.messageId).toBe(activeResponse.messageId);
+ expect((submission?.messages ?? []).map((message) => message.messageId)).toEqual([
+ olderSibling.messageId,
+ ]);
+ });
+
it('restores the branch that owns a pending OAuth resume user message', async () => {
const rootUser = buildUserMessage(CONVERSATION_ID, 'root-user');
const branchOneResponse = {
@@ -417,6 +555,7 @@ describe('useResumeOnLoad', () => {
isCreatedByUser: false,
} as TMessage;
const observedSiblingIndexes: number[] = [];
+ const observedSubmissions: Array<TSubmission | null> = [];
mockUseStreamStatus.mockReturnValue({
isSuccess: true,
@@ -442,16 +581,23 @@ describe('useResumeOnLoad', () => {
});
renderUseResumeOnLoad({
- messages: [rootUser, olderResponse, newerResponse],
+ // Put the unrelated sibling first: parent-only fallback would select it.
+ messages: [rootUser, newerResponse, olderResponse],
siblingIndexParentId: rootUser.messageId,
onSiblingIndex: (siblingIndex) => observedSiblingIndexes.push(siblingIndex),
+ onSubmission: (currentSubmission) => observedSubmissions.push(currentSubmission),
});
await act(async () => {
await Promise.resolve();
});
- expect(observedSiblingIndexes[observedSiblingIndexes.length - 1]).toBe(1);
+ expect(observedSiblingIndexes[observedSiblingIndexes.length - 1]).toBe(0);
+ const submission = observedSubmissions[observedSubmissions.length - 1];
+ expect(submission?.initialResponse?.messageId).toBe(olderResponse.messageId);
+ expect((submission?.messages ?? []).map((message) => message.messageId)).toEqual([
+ newerResponse.messageId,
+ ]);
});
describe('steer chip restore', () => {
@@ -476,6 +622,8 @@ describe('useResumeOnLoad', () => {
active: true,
status: 'running',
streamId: CONVERSATION_ID,
+ createdAt: 1234,
+ generationProtocolVersion: 2,
resumeState: {
runSteps: [],
aggregatedContent: [],
@@ -529,10 +677,71 @@ describe('useResumeOnLoad', () => {
});
expect(observedSteers[observedSteers.length - 1]).toEqual([
- { steerId: 'queued-1', text: 'still queued', status: 'pending', createdAt: 5, files },
+ {
+ steerId: 'queued-1',
+ text: 'still queued',
+ status: 'pending',
+ createdAt: 5,
+ files,
+ generationCreatedAt: 1234,
+ generationProtocolVersion: 2,
+ },
]);
});
+ it('seeds the pending snapshot before settling an inline applied steer', async () => {
+ const observedSteers: PendingSteer[][] = [];
+ mockUseStreamStatus.mockReturnValue({
+ isSuccess: true,
+ isFetching: false,
+ data: {
+ active: true,
+ status: 'running',
+ streamId: CONVERSATION_ID,
+ resumeState: {
+ runSteps: [],
+ pendingSteers: [
+ {
+ steerId: 'steer-snapshot-race',
+ clientSteerId: 'local-snapshot-race',
+ text: 'applied at snapshot boundary',
+ createdAt: 5,
+ },
+ ],
+ aggregatedContent: [
+ {
+ type: ContentTypes.STEER,
+ steerId: 'steer-snapshot-race',
+ clientSteerId: 'local-snapshot-race',
+ steer: 'applied at snapshot boundary',
+ },
+ ],
+ responseMessageId: RESPONSE_MESSAGE_ID,
+ conversationId: CONVERSATION_ID,
+ userMessage: {
+ messageId: USER_MESSAGE_ID,
+ parentMessageId: Constants.NO_PARENT,
+ conversationId: CONVERSATION_ID,
+ text: 'Hello',
+ },
+ },
+ },
+ });
+
+ renderUseResumeOnLoad({
+ messages: [buildUserMessage(CONVERSATION_ID)],
+ onPendingSteers: (steers) => observedSteers.push(steers),
+ });
+
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ // The status snapshot and aggregated content can straddle the same apply
+ // boundary. Restore-then-settle must finish with no resurrected chip.
+ expect(observedSteers[observedSteers.length - 1]).toEqual([]);
+ });
+
it('clears stale pending chips when the run finished while away (no active job)', async () => {
const observedSteers: PendingSteer[][] = [];
mockUseStreamStatus.mockReturnValue({
diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.ts
index 472b705920..6f29a92599 100644
--- a/client/src/hooks/SSE/useResumableSSE.ts
+++ b/client/src/hooks/SSE/useResumableSSE.ts
@@ -11,6 +11,7 @@ import {
StepEvents,
apiBaseUrl,
SteerEvents,
+ dataService,
ActivityLabelEvents,
UsageEvents,
createPayload,
@@ -27,10 +28,13 @@ import type {
TPendingSteer,
EventSubmission,
TSteerAppliedEvent,
+ TSteerUpdatedEvent,
TActivityLabelEvent,
} from 'librechat-data-provider';
+import type { ActiveJobsResponse, StreamStatusResponse } from '~/data-provider';
+import type { DrainAfterAbort, QueuedMessageOrigin } from '~/store/families';
+import type { GenerationProtocolVersion } from '~/data-provider';
import type { EventHandlerParams } from './useEventHandlers';
-import type { ActiveJobsResponse } from '~/data-provider';
import type { TResData } from '~/common';
import {
logger,
@@ -43,12 +47,14 @@ import {
applyActivityLabelPart,
findActivityLabelMessageIndex,
appendAppliedSteerIds,
+ collectAppliedSteerIds,
removeConvoFromAllQueries,
upsertConvoInAllQueries,
countTaggedApprovalParts,
countTrailingOutputChars,
markStreamStartFailedMetadata,
findPendingActionMessageIndex,
+ insertQueuedOrigin,
} from '~/utils';
import {
useGetUserBalance,
@@ -56,6 +62,11 @@ import {
useGetStartupConfig,
queueTitleGeneration,
streamStatusQueryKey,
+ generationProtocolHeaders,
+ getGenerationProtocolVersion,
+ postGenerationRequest,
+ supportsGenerationProtocolV2,
+ GENERATION_PROTOCOL_VERSION,
} from '~/data-provider';
import useEventHandlers, { buildCreatedInitialResponse } from './useEventHandlers';
import useSteerConvert from '~/hooks/Chat/useSteerConvert';
@@ -68,10 +79,29 @@ type ChatHelpers = Pick<
'setMessages' | 'getMessages' | 'setConversation' | 'setIsSubmitting' | 'newConversation'
>;
+/**
+ * Reconciliation often crosses async query boundaries. By the time an old
+ * generation settles, this pane may have armed "interrupt & send" for a new
+ * conversation/generation. Recoil applies functional setters against the
+ * latest value, so only consume the exact intent owned by this terminal path.
+ */
+const clearMatchingDrainAfterAbort = (
+ armed: DrainAfterAbort | false,
+ conversationId: string,
+ generationCreatedAt: number,
+): DrainAfterAbort | false =>
+ armed !== false &&
+ armed.conversationId === conversationId &&
+ armed.generationCreatedAt === generationCreatedAt
+ ? false
+ : armed;
+
const MAX_RETRIES = 5;
const START_GENERATION_NETWORK_RETRIES = 3;
const START_GENERATION_READINESS_TIMEOUT_MS = 120000;
const SERVER_NOT_READY_CODE = 'SERVER_NOT_READY';
+const PREDECESSOR_MISMATCH_CODE = 'GENERATION_PREDECESSOR_MISMATCH';
+const STEER_RECOVERY_REQUEST_PREFIX = 'steer-recovery:';
type StartGenerationError = {
code?: string;
@@ -100,6 +130,133 @@ const getStartGenerationStreamId = (data: unknown): string | null => {
const isResumedStartResponse = (data: unknown): boolean =>
data != null && typeof data === 'object' && (data as { status?: unknown }).status === 'resumed';
+const isSettledStartResponse = (data: unknown): boolean =>
+ supportsGenerationProtocolV2(data) &&
+ data != null &&
+ typeof data === 'object' &&
+ (data as { status?: unknown }).status === 'settled';
+
+const isReplacedStartResponse = (data: unknown): boolean =>
+ supportsGenerationProtocolV2(data) &&
+ data != null &&
+ typeof data === 'object' &&
+ (data as { status?: unknown }).status === 'replaced';
+
+const isUnnegotiatedV2StartControl = (data: unknown): boolean => {
+ if (supportsGenerationProtocolV2(data) || data == null || typeof data !== 'object') {
+ return false;
+ }
+ const status = (data as { status?: unknown }).status;
+ return status === 'settled' || status === 'replaced';
+};
+
+const getStartGenerationConversationId = (data: unknown): string | null => {
+ if (data == null || typeof data !== 'object' || !('conversationId' in data)) {
+ return null;
+ }
+ const conversationId = (data as { conversationId?: unknown }).conversationId;
+ return typeof conversationId === 'string' && conversationId.length > 0 ? conversationId : null;
+};
+
+const getStartGenerationCreatedAt = (data: unknown): number | null => {
+ if (data == null || typeof data !== 'object' || !('generationCreatedAt' in data)) {
+ return null;
+ }
+ const createdAt = (data as { generationCreatedAt?: unknown }).generationCreatedAt;
+ return typeof createdAt === 'number' && Number.isSafeInteger(createdAt) && createdAt >= 0
+ ? createdAt
+ : null;
+};
+
+type PredecessorMismatchStartResponse = {
+ streamId: string;
+ conversationId: string;
+ generationCreatedAt: number;
+ active: boolean;
+};
+
+const getPredecessorMismatchStartResponse = (
+ error: unknown,
+): PredecessorMismatchStartResponse | null => {
+ const startError = toStartGenerationError(error);
+ const data = startError?.response?.data;
+ if (
+ startError?.response?.status !== 409 ||
+ data == null ||
+ typeof data !== 'object' ||
+ (data as { status?: unknown }).status !== 'predecessor_mismatch' ||
+ (data as { code?: unknown }).code !== PREDECESSOR_MISMATCH_CODE
+ ) {
+ return null;
+ }
+
+ const streamId = getStartGenerationStreamId(data);
+ const conversationId = getStartGenerationConversationId(data);
+ if (!streamId || !conversationId) {
+ return null;
+ }
+
+ const generationCreatedAt = getStartGenerationCreatedAt(data);
+ const active = (data as { active?: unknown }).active;
+ if (generationCreatedAt == null || typeof active !== 'boolean') {
+ return null;
+ }
+ return {
+ streamId,
+ conversationId,
+ generationCreatedAt,
+ active,
+ };
+};
+
+/** Recovery submissions use a deterministic request id tied to the parked
+ * source. A status read performed only after this exact submission is proven
+ * terminal may authorize that source past the client-side live-event
+ * tombstone; unrelated status snapshots must remain deduped. */
+const getRecoverySteerId = (submission: TSubmission): string | null => {
+ if (typeof submission.recoverySteerId === 'string' && submission.recoverySteerId.length > 0) {
+ return submission.recoverySteerId;
+ }
+ const clientRequestId = submission.clientRequestId;
+ if (
+ typeof clientRequestId !== 'string' ||
+ !clientRequestId.startsWith(STEER_RECOVERY_REQUEST_PREFIX)
+ ) {
+ return null;
+ }
+ const steerId = clientRequestId.slice(STEER_RECOVERY_REQUEST_PREFIX.length);
+ return steerId.length > 0 ? steerId : null;
+};
+
+type StartGenerationResult =
+ | {
+ status: 'stream';
+ streamId: string;
+ resumed: boolean;
+ generationCreatedAt?: number;
+ generationProtocolVersion: GenerationProtocolVersion;
+ }
+ | {
+ status: 'settled';
+ conversationId: string;
+ generationProtocolVersion: typeof GENERATION_PROTOCOL_VERSION;
+ }
+ | {
+ status: 'replaced';
+ streamId: string;
+ conversationId: string;
+ generationCreatedAt: number;
+ generationProtocolVersion: typeof GENERATION_PROTOCOL_VERSION;
+ }
+ | {
+ status: 'predecessor_mismatch';
+ streamId: string;
+ conversationId: string;
+ generationCreatedAt: number;
+ active: boolean;
+ generationProtocolVersion: typeof GENERATION_PROTOCOL_VERSION;
+ };
+
const parseSSEErrorData = (body: string): unknown | null => {
const blocks = body.split(/\r?\n\r?\n/);
for (const block of blocks) {
@@ -540,15 +697,19 @@ export default function useResumableSSE(
const reconnectAttemptRef = useRef(0);
const reconnectTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const submissionRef = useRef<TSubmission | null>(null);
+ /** Suppresses the normal close/cleanup idle transition while a stale
+ * generation is being atomically handed off to its live replacement. */
+ const replacementHandoffRef = useRef(false);
const optimisticStreamIdsRef = useRef(new Set<string>());
const createdStreamIdsRef = useRef(new Set<string>());
/** Pending action whose tool-call content part hasn't rendered yet — retried
* on the next frame so a fast pause-before-render race still attaches. */
const pendingActionRetryRef = useRef<number | null>(null);
- /** Steer event whose target response message hasn't rendered yet — same
- * bounded next-frame retry as pending actions, on its own handle so the
- * two retries can't cancel each other. */
- const steerRetryRef = useRef<number | null>(null);
+ /** EVERY steer retry frame, not only the newest one. Several steers can be
+ * applied before their shared response placeholder renders; retaining one
+ * handle let the older chains outlive navigation/terminal cleanup and
+ * mutate a later generation that reused the same response id. */
+ const steerRetryFramesRef = useRef<Set<number>>(new Set());
/** EVERY outstanding label-retry frame, not a single handle: labels fire
* two events per slot (reservation, then fill), so concurrent retry
* chains are the norm — a single ref would let cleanup cancel only the
@@ -569,49 +730,209 @@ export default function useResumableSSE(
* id and is therefore shared by every generation within it. */
const prefixStateGenerationIdRef = useRef<string | null>(null);
+ const restoreQueuedSubmission = useRecoilCallback(
+ ({ set }) =>
+ (failedSubmission: TSubmission, expectedPredecessorCreatedAt?: number) => {
+ const conversationId = failedSubmission.conversation?.conversationId;
+ const origin = failedSubmission.queuedMessageOrigin as QueuedMessageOrigin | undefined;
+ if (!conversationId || origin == null) {
+ return;
+ }
+ set(store.queuedMessagesByConvoId(conversationId), (prev) =>
+ insertQueuedOrigin(prev, origin, expectedPredecessorCreatedAt),
+ );
+ },
+ [],
+ );
+
/** Removes the pending chip once its steer is injected (the inline content
* part becomes the durable record), and records the id so a 202 ACK that
* arrives AFTER the applied event drops its chip instead of re-minting it. */
const resolveSteerChip = useRecoilCallback(
({ set }) =>
- (conversationId: string, steerId: string) => {
+ (conversationId: string, steerId: string, clientSteerId?: string) => {
+ const settledIds = clientSteerId ? [steerId, clientSteerId] : [steerId];
set(store.appliedSteerIdsByConvoId(conversationId), (prev) =>
- appendAppliedSteerIds(prev, [steerId]),
+ appendAppliedSteerIds(prev, settledIds),
);
set(store.pendingSteersByConvoId(conversationId), (prev) =>
- prev.some((steer) => steer.steerId === steerId)
- ? prev.filter((steer) => steer.steerId !== steerId)
+ prev.some((steer) => settledIds.includes(steer.steerId))
+ ? prev.filter((steer) => !settledIds.includes(steer.steerId))
+ : prev,
+ );
+ /** A terminal boundary may have re-homed this steer while its applied
+ * event was waiting for the response placeholder. The applied event
+ * is authoritative, so evict that recovery copy before it can be
+ * drained as a duplicate follow-up. */
+ set(store.queuedMessagesByConvoId(conversationId), (prev) =>
+ prev.some(
+ (item) =>
+ (item.recoverySteerId != null && settledIds.includes(item.recoverySteerId)) ||
+ (item.recoveryClientSteerId != null &&
+ settledIds.includes(item.recoveryClientSteerId)),
+ )
+ ? prev.filter(
+ (item) =>
+ !(
+ (item.recoverySteerId != null && settledIds.includes(item.recoverySteerId)) ||
+ (item.recoveryClientSteerId != null &&
+ settledIds.includes(item.recoveryClientSteerId))
+ ),
+ )
: prev,
);
},
[],
);
+ /** Capability can change while the same generation resumes after HITL.
+ * Patch labels in place; this event never removes/reorders a chip. */
+ const updateSteerChips = useRecoilCallback(
+ ({ set }) =>
+ (event: TSteerUpdatedEvent) => {
+ const byId = new Map<string, TSteerUpdatedEvent['steers'][number]>();
+ for (const steer of event.steers) {
+ byId.set(steer.steerId, steer);
+ if (steer.clientSteerId) {
+ byId.set(steer.clientSteerId, steer);
+ }
+ }
+ const acceptedClientIds = event.steers.flatMap((steer) =>
+ steer.clientSteerId ? [steer.clientSteerId] : [],
+ );
+ if (acceptedClientIds.length > 0) {
+ set(store.acceptedSteerClientIdsByConvoId(event.conversationId), (prev) =>
+ appendAppliedSteerIds(prev, acceptedClientIds),
+ );
+ }
+ set(store.pendingSteersByConvoId(event.conversationId), (prev) => {
+ let changed = false;
+ const next = prev.map((steer) => {
+ const update = byId.get(steer.steerId);
+ if (update == null) {
+ return steer;
+ }
+ const correlated =
+ update.clientSteerId === steer.steerId && steer.steerId !== update.steerId;
+ const base = correlated
+ ? {
+ ...steer,
+ steerId: update.steerId,
+ clientSteerId: update.clientSteerId,
+ status: 'pending' as const,
+ }
+ : steer;
+ if (update.preemptRevision < (base.preemptRevision ?? 0)) {
+ changed ||= base !== steer;
+ return base;
+ }
+ if (
+ Boolean(base.preempt) === update.preempt &&
+ base.preemptRevision === update.preemptRevision
+ ) {
+ changed ||= base !== steer;
+ return base;
+ }
+ changed = true;
+ if (update.preempt) {
+ return {
+ ...base,
+ preempt: true,
+ preemptRevision: update.preemptRevision,
+ };
+ }
+ const { preempt: _preempt, ...ordinary } = base;
+ return { ...ordinary, preemptRevision: update.preemptRevision };
+ });
+ return changed ? next : prev;
+ });
+ },
+ [],
+ );
+
/** Replaces the chip list with the server's still-queued steers (reconnect).
* Local `failed` entries are kept so their text stays recoverable, and a
* reseeded chip keeps its client-only quotes/skill picks. */
const seedSteerChips = useRecoilCallback(
({ set }) =>
- (conversationId: string, steers: TPendingSteer[]) => {
+ (
+ conversationId: string,
+ steers: TPendingSteer[],
+ generationCreatedAt?: number,
+ generationProtocolVersion: GenerationProtocolVersion = 1,
+ ) => {
+ const acceptedClientIds = steers.flatMap((steer) =>
+ steer.clientSteerId ? [steer.clientSteerId] : [],
+ );
+ if (acceptedClientIds.length > 0) {
+ set(store.acceptedSteerClientIdsByConvoId(conversationId), (prev) =>
+ appendAppliedSteerIds(prev, acceptedClientIds),
+ );
+ }
set(store.pendingSteersByConvoId(conversationId), (prev) => {
const chipById = new Map(prev.map((chip) => [chip.steerId, chip]));
+ const claimedIds = new Set(
+ steers.flatMap((steer) =>
+ steer.clientSteerId ? [steer.steerId, steer.clientSteerId] : [steer.steerId],
+ ),
+ );
return [
- ...steers.map((steer) => ({
- steerId: steer.steerId,
- text: steer.text,
- status: 'pending' as const,
- createdAt: steer.createdAt ?? Date.now(),
- ...(steer.files && steer.files.length > 0 && { files: steer.files }),
- ...(steer.preempt === true && { preempt: true }),
- ...carriedSteerContext(chipById.get(steer.steerId)),
- })),
- ...prev.filter((steer) => steer.status === 'failed'),
+ ...steers.map((steer) => {
+ const localChip =
+ chipById.get(steer.steerId) ??
+ (steer.clientSteerId ? chipById.get(steer.clientSteerId) : undefined);
+ const keepLocalPreempt =
+ (localChip?.preemptRevision ?? 0) > (steer.preemptRevision ?? 0);
+ const chipGenerationCreatedAt = generationCreatedAt ?? localChip?.generationCreatedAt;
+ return {
+ steerId: steer.steerId,
+ ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
+ text: steer.text,
+ status: 'pending' as const,
+ createdAt: steer.createdAt ?? Date.now(),
+ ...(steer.files && steer.files.length > 0 && { files: steer.files }),
+ ...((keepLocalPreempt ? localChip?.preempt : steer.preempt) === true && {
+ preempt: true,
+ }),
+ ...((keepLocalPreempt ? localChip?.preemptRevision : steer.preemptRevision) !=
+ null && {
+ preemptRevision: keepLocalPreempt
+ ? localChip?.preemptRevision
+ : steer.preemptRevision,
+ }),
+ ...(localChip?.queuedOrigin && { queuedOrigin: localChip.queuedOrigin }),
+ ...(chipGenerationCreatedAt != null && {
+ generationCreatedAt: chipGenerationCreatedAt,
+ }),
+ generationProtocolVersion,
+ ...carriedSteerContext(localChip),
+ };
+ }),
+ ...prev.filter((steer) => steer.status === 'failed' && !claimedIds.has(steer.steerId)),
];
});
},
[],
);
+ const settleAppliedSteerParts = useRecoilCallback(
+ ({ set }) =>
+ (conversationId: string, values: unknown[] | undefined) => {
+ const ids = collectAppliedSteerIds(values);
+ if (ids.length === 0) {
+ return;
+ }
+ const settled = new Set(ids);
+ set(store.appliedSteerIdsByConvoId(conversationId), (prev) =>
+ appendAppliedSteerIds(prev, ids),
+ );
+ set(store.pendingSteersByConvoId(conversationId), (prev) =>
+ prev.filter((steer) => !settled.has(steer.steerId)),
+ );
+ },
+ [],
+ );
+
/** Converts steers that never reached an injection boundary into queued
* follow-up messages (reported on the run's final/abort event; the abort
* HTTP response consumes the same data as a fallback in useChatHelpers). */
@@ -624,24 +945,79 @@ export default function useResumableSSE(
* queue/send) and `failed` chips keep their manual controls. */
const convertLocalSteersToQueued = useRecoilCallback(
({ snapshot }) =>
- (conversationId: string, options?: { claimParked?: boolean }) => {
+ (
+ conversationId: string,
+ options?: {
+ claimParked?: boolean;
+ excludeSteerIds?: Iterable<string>;
+ generationProtocolVersion?: GenerationProtocolVersion;
+ },
+ ) => {
const chips = snapshot.getLoadable(store.pendingSteersByConvoId(conversationId)).getValue();
+ const excluded = new Set(options?.excludeSteerIds ?? []);
const settled = chips
- .filter((steer) => steer.status === 'pending')
+ .filter(
+ (steer) =>
+ steer.status === 'pending' &&
+ !excluded.has(steer.steerId) &&
+ (steer.clientSteerId == null || !excluded.has(steer.clientSteerId)),
+ )
.map((steer) => ({
steerId: steer.steerId,
+ ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
text: steer.text,
createdAt: steer.createdAt,
...(steer.files && steer.files.length > 0 && { files: steer.files }),
+ ...(steer.queuedOrigin && { queuedOrigin: steer.queuedOrigin }),
}));
if (settled.length > 0) {
- convertSteersToQueued(conversationId, settled, options);
+ convertSteersToQueued(conversationId, settled, {
+ claimParked: options?.claimParked,
+ generationProtocolVersion: options?.generationProtocolVersion,
+ });
}
},
[convertSteersToQueued],
);
+ /** Conversation ids are reused by successive agent turns. Keep the exact
+ * live epoch beside the UI controls, and only clear it if the terminal event
+ * still belongs to the epoch that installed it. */
+ const updateActiveGenerationCreatedAt = useRecoilCallback(
+ ({ snapshot, set }) =>
+ (
+ conversationId: string,
+ next: number | null,
+ expected?: number,
+ generationProtocolVersion: GenerationProtocolVersion = 1,
+ ): boolean => {
+ const state = store.activeGenerationCreatedAtByConvoId(conversationId);
+ if (expected != null && snapshot.getLoadable(state).getValue() !== expected) {
+ return false;
+ }
+ set(state, next);
+ set(
+ store.activeGenerationProtocolVersionByConvoId(conversationId),
+ next == null ? 1 : generationProtocolVersion,
+ );
+ return true;
+ },
+ [],
+ );
+
const setRunEnd = useSetRecoilState(store.runEndByIndex(runIndex));
+ const setDrainAfterAbort = useSetRecoilState(store.drainAfterAbortByIndex(runIndex));
+ const clearDrainAfterAbort = useCallback(
+ (conversationId: string, generationCreatedAt?: number) => {
+ if (generationCreatedAt == null) {
+ return;
+ }
+ setDrainAfterAbort((armed) =>
+ clearMatchingDrainAfterAbort(armed, conversationId, generationCreatedAt),
+ );
+ },
+ [setDrainAfterAbort],
+ );
const {
stepHandler: rawStepHandler,
@@ -711,7 +1087,56 @@ export default function useResumableSSE(
* @param isResume - If true, adds ?resume=true to trigger sync event from server
*/
const subscribeToStream = useCallback(
- (currentStreamId: string, currentSubmission: TSubmission, isResume = false) => {
+ (
+ currentStreamId: string,
+ currentSubmission: TSubmission,
+ isResume = false,
+ generationCreatedAt?: number,
+ generationProtocolVersion: GenerationProtocolVersion = 1,
+ lifecycleSignal?: AbortSignal,
+ ) => {
+ /** Every effect owns one AbortSignal, while every reconnect owns one SSE
+ * instance. Both identities matter: cleanup can close an attachment after
+ * its async listener has already crossed an `await`, and a reconnect can
+ * supersede an older attachment without changing the submission object.
+ * The captured submission check is the final fence against a later
+ * conversation installing its own submission in this pane. */
+ if (lifecycleSignal?.aborted || submissionRef.current !== currentSubmission) {
+ return;
+ }
+ const startedAsNewConversation = optimisticStreamIdsRef.current.has(currentStreamId);
+ const clearAttachedGenerationCreatedAt = () => {
+ updateActiveGenerationCreatedAt(currentStreamId, null, generationCreatedAt);
+ if (startedAsNewConversation) {
+ updateActiveGenerationCreatedAt(Constants.NEW_CONVO, null, generationCreatedAt);
+ }
+ };
+ if (generationCreatedAt != null) {
+ updateActiveGenerationCreatedAt(
+ currentStreamId,
+ generationCreatedAt,
+ undefined,
+ generationProtocolVersion,
+ );
+ /** Until CREATED moves the conversation context off `/c/new`, Stop and
+ * approval controls still read the placeholder key. Mirror the exact
+ * epoch there briefly; the server's user-wide fallback plus this fence
+ * can safely resolve the just-started job. */
+ if (startedAsNewConversation) {
+ updateActiveGenerationCreatedAt(
+ Constants.NEW_CONVO,
+ generationCreatedAt,
+ undefined,
+ generationProtocolVersion,
+ );
+ }
+ setShowStopButton(true);
+ } else {
+ /** Generation-scoped mutations are unsafe until the server supplies
+ * the epoch they must fence against. Keep Stop hidden if an old or
+ * malformed response omitted it. */
+ setShowStopButton(false);
+ }
/**
* A NEW generation starts with its retained prefix intact, so the
* cleared-prefix state from a previous one must not carry over — the
@@ -746,7 +1171,7 @@ export default function useResumableSSE(
let finalReceived = false;
const preCreatedStepEvents: Array<Parameters<typeof stepHandler>[0]> = [];
const replayPreCreatedStepEvents = () => {
- if (preCreatedStepEvents.length === 0) {
+ if (!isCurrentSubscription() || preCreatedStepEvents.length === 0) {
return;
}
@@ -772,12 +1197,23 @@ export default function useResumableSSE(
// load, so a single retry would drop a valid pause and leave the run with no
// approval controls. Bounded so a genuinely-absent target can't spin forever.
const PENDING_ACTION_MAX_RETRY_FRAMES = 120;
+ const cancelSteerRetryFrames = () => {
+ for (const frameId of steerRetryFramesRef.current) {
+ cancelAnimationFrame(frameId);
+ }
+ steerRetryFramesRef.current.clear();
+ };
const applyPendingActionToMessages = (pendingAction: Agents.PendingAction, attempt = 0) => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
const retryNextFrame = () => {
if (attempt < PENDING_ACTION_MAX_RETRY_FRAMES) {
- pendingActionRetryRef.current = requestAnimationFrame(() =>
- applyPendingActionToMessages(pendingAction, attempt + 1),
- );
+ pendingActionRetryRef.current = requestAnimationFrame(() => {
+ if (isCurrentSubscription()) {
+ applyPendingActionToMessages(pendingAction, attempt + 1);
+ }
+ });
}
};
/** The pause card must attach to the same message state the stream
@@ -822,11 +1258,27 @@ export default function useResumableSSE(
* can land a few frames after the created event under load).
*/
const applySteerToMessages = (event: TSteerAppliedEvent, attempt = 0) => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const chipConvoId =
+ event.conversationId ?? currentSubmission.conversation?.conversationId ?? currentStreamId;
+ /** The server's applied event settles ownership immediately. Rendering
+ * the inline part can wait for React to mount the target, but leaving
+ * the chip pending during that wait lets an intervening error/final
+ * convert already-applied words into a duplicate queued message. */
+ if (attempt === 0) {
+ resolveSteerChip(chipConvoId, event.steerId, event.clientSteerId);
+ }
const retryNextFrame = () => {
if (attempt < PENDING_ACTION_MAX_RETRY_FRAMES) {
- steerRetryRef.current = requestAnimationFrame(() =>
- applySteerToMessages(event, attempt + 1),
- );
+ const frameId = requestAnimationFrame(() => {
+ steerRetryFramesRef.current.delete(frameId);
+ if (isCurrentSubscription()) {
+ applySteerToMessages(event, attempt + 1);
+ }
+ });
+ steerRetryFramesRef.current.add(frameId);
}
};
/** Same boundary as pending actions: land queued deltas before the
@@ -845,9 +1297,6 @@ export default function useResumableSSE(
setMessages(nextMessages);
syncStepMessage(updated);
}
- const chipConvoId =
- event.conversationId ?? currentSubmission.conversation?.conversationId ?? currentStreamId;
- resolveSteerChip(chipConvoId, event.steerId);
};
/**
@@ -859,11 +1308,16 @@ export default function useResumableSSE(
* bounded next-frame retry as steers for the inject-before-render race.
*/
const applyActivityLabelToMessages = (event: TActivityLabelEvent, attempt = 0) => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
const retryNextFrame = () => {
if (attempt < PENDING_ACTION_MAX_RETRY_FRAMES) {
const frameId = requestAnimationFrame(() => {
activityLabelRetryFramesRef.current.delete(frameId);
- applyActivityLabelToMessages(event, attempt + 1);
+ if (isCurrentSubscription()) {
+ applyActivityLabelToMessages(event, attempt + 1);
+ }
});
activityLabelRetryFramesRef.current.add(frameId);
}
@@ -911,30 +1365,83 @@ export default function useResumableSSE(
};
const baseUrl = `${apiBaseUrl()}/api/agents/chat/stream/${encodeURIComponent(currentStreamId)}`;
- const url = isResume ? `${baseUrl}?resume=true` : baseUrl;
+ const query = new URLSearchParams();
+ if (isResume) {
+ query.set('resume', 'true');
+ }
+ if (generationCreatedAt != null) {
+ query.set('generationCreatedAt', String(generationCreatedAt));
+ }
+ query.set('generationProtocolVersion', String(GENERATION_PROTOCOL_VERSION));
+ const queryString = query.toString();
+ const url = queryString ? `${baseUrl}?${queryString}` : baseUrl;
logger.log('ResumableSSE', 'Subscribing to stream:', url, { isResume });
const sse = new SSE(url, {
- headers: { Authorization: `Bearer ${token}` },
+ headers: {
+ Authorization: `Bearer ${token}`,
+ ...generationProtocolHeaders(),
+ },
method: 'GET',
});
sseRef.current = sse;
+ const isCurrentSubscription = () =>
+ lifecycleSignal?.aborted !== true &&
+ sseRef.current === sse &&
+ submissionRef.current === currentSubmission;
sse.addEventListener('open', () => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
logger.log('ResumableSSE', 'Stream connected');
setAbortScroll(false);
// Restore UI state on successful connection (including reconnection)
setIsSubmitting(true);
- setShowStopButton(true);
+ setShowStopButton(generationCreatedAt != null);
reconnectAttemptRef.current = 0;
});
- sse.addEventListener('message', (e: MessageEvent) => {
+ sse.addEventListener('message', async (e: MessageEvent) => {
try {
+ if (!isCurrentSubscription()) {
+ return;
+ }
const data = JSON.parse(e.data);
+ if (finalReceived) {
+ return;
+ }
+
+ if (data.final === true && data.reconcile === true) {
+ if (
+ generationProtocolVersion !== GENERATION_PROTOCOL_VERSION ||
+ !supportsGenerationProtocolV2(data)
+ ) {
+ logger.warn('ResumableSSE', 'Ignoring unnegotiated lifecycle reconciliation frame');
+ return;
+ }
+ // Synthesized lifecycle frames deliberately carry no ordinary
+ // response payload. Reconcile durable history (or hand off to a
+ // replacement epoch) instead of passing them to finalHandler.
+ finalReceived = true;
+ cancelSteerRetryFrames();
+ void reconcileGenerationLifecycle(data);
+ return;
+ }
if (data.final != null) {
finalReceived = true;
+ const finalConvoId =
+ data.conversation?.conversationId ??
+ currentSubmission.conversation?.conversationId ??
+ currentStreamId;
+ if (
+ !(await authorizeTerminalTeardown(finalConvoId, 'FINAL')) ||
+ !isCurrentSubscription()
+ ) {
+ return;
+ }
+ cancelSteerRetryFrames();
if (reconnectTimeoutRef.current) {
clearTimeout(reconnectTimeoutRef.current);
reconnectTimeoutRef.current = null;
@@ -949,22 +1456,42 @@ export default function useResumableSSE(
if (optimisticStreamIdsRef.current.has(currentStreamId)) {
clearAllDrafts(Constants.NEW_CONVO);
}
- const finalConvoId =
- data.conversation?.conversationId ??
- currentSubmission.conversation?.conversationId ??
- currentStreamId;
+ // A steer-applied event may still be waiting for its next-frame
+ // message target when FINAL arrives. Reconcile directly from the
+ // authoritative final message before converting leftovers so a
+ // late 202 ACK cannot resurrect (or queue) an already-applied steer.
+ const finalAppliedSteerIds = new Set(
+ collectAppliedSteerIds(data.responseMessage ? [data.responseMessage] : []),
+ );
+ settleAppliedSteerParts(
+ finalConvoId,
+ data.responseMessage ? [data.responseMessage] : undefined,
+ );
// Steers the run never injected ride the final event; convert them
// to queued follow-ups before the run-end signal fires the drain
// (also resets the applied-id set for the finished run).
- // `claimParked` clears the parked server copy of the same steers so
- // a later reload can't resurrect chips dismissed after this batch.
+ // `claimParked` reconciles the replayable parked copy of the same
+ // steers; explicit dismissal or a persisted recovery removes it.
convertSteersToQueued(
finalConvoId,
- Array.isArray(data.pendingSteers) ? (data.pendingSteers as TPendingSteer[]) : [],
- { claimParked: true },
+ (generationProtocolVersion === 1 || supportsGenerationProtocolV2(data)) &&
+ Array.isArray(data.pendingSteers)
+ ? (data.pendingSteers as TPendingSteer[]).filter(
+ (steer) =>
+ !finalAppliedSteerIds.has(steer.steerId) &&
+ (steer.clientSteerId == null ||
+ !finalAppliedSteerIds.has(steer.clientSteerId)),
+ )
+ : [],
+ {
+ claimParked: true,
+ generationProtocolVersion,
+ },
);
+ let finalHandled = false;
try {
finalHandler(data, currentSubmission as EventSubmission);
+ finalHandled = true;
finalizeUsage(data, { ...currentSubmission, userMessage });
} catch (error) {
logger.error('ResumableSSE', 'Error in finalHandler:', error);
@@ -981,22 +1508,27 @@ export default function useResumableSSE(
earlyAbort: data.earlyAbort === true,
startedAsNewConvo: optimisticStreamIdsRef.current.has(currentStreamId),
});
+ let finalOutcome: 'completed' | 'aborted' | 'error' = 'error';
+ if (data.aborted === true || data.responseMessage?.unfinished === true) {
+ finalOutcome = 'aborted';
+ } else if (finalHandled) {
+ finalOutcome = 'completed';
+ }
setRunEnd({
conversationId: runEndTarget.conversationId,
// A Stop that lands before completion can arrive as a final with
// `unfinished: true` and no `aborted` flag (request.js's
// wasAbortedBeforeComplete branch) — it must not auto-drain.
- outcome:
- data.aborted === true || data.responseMessage?.unfinished === true
- ? 'aborted'
- : 'completed',
+ outcome: finalOutcome,
startedAsNewConvo: runEndTarget.startedAsNewConvo,
endedAt: Date.now(),
+ generationCreatedAt,
});
// Clear handler maps on stream completion to prevent memory leaks
clearStepMaps();
// Optimistically remove from active jobs
removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt();
(startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
sse.close();
setStreamId(null);
@@ -1011,6 +1543,9 @@ export default function useResumableSSE(
conversationId: data.message?.conversationId,
});
createdStreamIdsRef.current.add(currentStreamId);
+ if (startedAsNewConversation) {
+ updateActiveGenerationCreatedAt(Constants.NEW_CONVO, null, generationCreatedAt);
+ }
const runId = v4();
setActiveRunId(runId);
userMessage = {
@@ -1068,6 +1603,11 @@ export default function useResumableSSE(
return;
}
+ if (data.event === SteerEvents.ON_STEER_UPDATED) {
+ updateSteerChips(data.data as TSteerUpdatedEvent);
+ return;
+ }
+
if (data.event === ActivityLabelEvents.ON_ACTIVITY_LABEL) {
applyActivityLabelToMessages(data.data as TActivityLabelEvent);
return;
@@ -1263,9 +1803,19 @@ export default function useResumableSSE(
// applied while this client was disconnected is absent here (its
// inline part rides aggregatedContent instead), so an EMPTY list
// must clear stale local pending chips, not leave them stranded.
+ const steerConvoId = currentSubmission.conversation?.conversationId ?? currentStreamId;
seedSteerChips(
- currentSubmission.conversation?.conversationId ?? currentStreamId,
+ steerConvoId,
(data.resumeState?.pendingSteers ?? []) as TPendingSteer[],
+ generationCreatedAt,
+ generationProtocolVersion,
+ );
+ // Seed first, then remove anything already represented in the
+ // authoritative content. Reversing this order can re-mint a stale
+ // server snapshot entry after it was just settled.
+ settleAppliedSteerParts(
+ steerConvoId,
+ data.resumeState?.aggregatedContent as unknown[] | undefined,
);
if (data.resumeState?.titleEvent) {
@@ -1288,6 +1838,8 @@ export default function useResumableSSE(
applyPendingActionToMessages(replayEvent.data as Agents.PendingAction);
} else if (replayEvent.event === SteerEvents.ON_STEER_APPLIED) {
applySteerToMessages(replayEvent.data as TSteerAppliedEvent);
+ } else if (replayEvent.event === SteerEvents.ON_STEER_UPDATED) {
+ updateSteerChips(replayEvent.data as TSteerUpdatedEvent);
} else if (replayEvent.event === ActivityLabelEvents.ON_ACTIVITY_LABEL) {
applyActivityLabelToMessages(replayEvent.data as TActivityLabelEvent);
} else if (replayEvent.event != null) {
@@ -1319,6 +1871,8 @@ export default function useResumableSSE(
applyPendingActionToMessages(pendingEvent.data as Agents.PendingAction);
} else if (pendingEvent.event === SteerEvents.ON_STEER_APPLIED) {
applySteerToMessages(pendingEvent.data as TSteerAppliedEvent);
+ } else if (pendingEvent.event === SteerEvents.ON_STEER_UPDATED) {
+ updateSteerChips(pendingEvent.data as TSteerUpdatedEvent);
} else if (pendingEvent.event === ActivityLabelEvents.ON_ACTIVITY_LABEL) {
applyActivityLabelToMessages(pendingEvent.data as TActivityLabelEvent);
} else if (pendingEvent.event != null) {
@@ -1339,7 +1893,7 @@ export default function useResumableSSE(
}
setIsSubmitting(true);
- setShowStopButton(true);
+ setShowStopButton(generationCreatedAt != null);
return;
}
@@ -1370,57 +1924,677 @@ export default function useResumableSSE(
}
});
+ async function handoffToReplacement(
+ conversationId: string,
+ knownStatus?: Awaited<ReturnType<typeof fetchStreamStatus>>,
+ ): Promise<boolean> {
+ if (!isCurrentSubscription() || generationProtocolVersion !== GENERATION_PROTOCOL_VERSION) {
+ return false;
+ }
+ let replacementStatus = knownStatus;
+ if (replacementStatus == null) {
+ try {
+ // Deliberately unfenced: this read discovers the newer generation
+ // after the epoch-fenced SSE endpoint rejects the stale attachment.
+ replacementStatus = await fetchStreamStatus(conversationId);
+ if (!isCurrentSubscription()) {
+ return false;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return false;
+ }
+ logger.warn('ResumableSSE', 'Could not inspect replacement generation', {
+ conversationId,
+ generationCreatedAt,
+ error,
+ });
+ return false;
+ }
+ }
+
+ if (
+ !supportsGenerationProtocolV2(replacementStatus) ||
+ replacementStatus.active !== true ||
+ !replacementStatus.streamId ||
+ replacementStatus.createdAt == null ||
+ replacementStatus.createdAt === generationCreatedAt
+ ) {
+ return false;
+ }
+
+ logger.log('ResumableSSE', 'Handing stale attachment to replacement generation', {
+ conversationId,
+ staleCreatedAt: generationCreatedAt,
+ replacementCreatedAt: replacementStatus.createdAt,
+ });
+ replacementHandoffRef.current = true;
+ cancelSteerRetryFrames();
+ sse.close();
+ if (reconnectTimeoutRef.current) {
+ clearTimeout(reconnectTimeoutRef.current);
+ reconnectTimeoutRef.current = null;
+ }
+ queryClient.setQueryData(streamStatusQueryKey(conversationId), {
+ ...replacementStatus,
+ generationHandoff: true,
+ });
+ updateActiveGenerationCreatedAt(
+ conversationId,
+ replacementStatus.createdAt,
+ undefined,
+ GENERATION_PROTOCOL_VERSION,
+ );
+ if (startedAsNewConversation) {
+ updateActiveGenerationCreatedAt(
+ Constants.NEW_CONVO,
+ replacementStatus.createdAt,
+ undefined,
+ GENERATION_PROTOCOL_VERSION,
+ );
+ }
+ addActiveJob(replacementStatus.streamId);
+ seedSteerChips(
+ conversationId,
+ (replacementStatus.resumeState?.pendingSteers ?? []) as TPendingSteer[],
+ replacementStatus.createdAt,
+ GENERATION_PROTOCOL_VERSION,
+ );
+ settleAppliedSteerParts(
+ conversationId,
+ replacementStatus.resumeState?.aggregatedContent ?? replacementStatus.aggregatedContent,
+ );
+ if (replacementStatus.pendingAction ?? replacementStatus.resumeState?.pendingAction) {
+ applyPendingActionToMessages(
+ (replacementStatus.pendingAction ??
+ replacementStatus.resumeState?.pendingAction) as Agents.PendingAction,
+ );
+ }
+ resetLive({ ...currentSubmission, userMessage });
+ clearStepMaps();
+ reconnectAttemptRef.current = 0;
+ submissionRef.current = null;
+ setIsSubmitting(true);
+ setShowStopButton(true);
+ setSubmission(null);
+ return true;
+ }
+
/**
- * Error event handler - handles BOTH:
- * 1. HTTP-level errors (responseCode present) - 404, 401, network failures
- * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
- *
- * Order matters: check responseCode first since HTTP errors may also include data
+ * A terminal frame proves only what happened on this SSE attachment. A
+ * fresh status read is the authority for whether terminal UI teardown is
+ * safe: a newer epoch may already own the conversation, while the exact
+ * epoch can still be active despite a racing FINAL/error frame.
*/
- sse.addEventListener('error', async (e: MessageEvent) => {
- /* @ts-ignore - sse.js types don't expose responseCode */
- const responseCode = e.responseCode;
+ function retryFencedTerminalAttachment(reason: string): void {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Deferring terminal teardown and retrying fenced attachment', {
+ conversationId: currentSubmission.conversation?.conversationId ?? currentStreamId,
+ generationCreatedAt,
+ reason,
+ });
+ reconnectAttemptRef.current = Math.max(reconnectAttemptRef.current, 1);
+ sse.close();
+ if (reconnectTimeoutRef.current) {
+ clearTimeout(reconnectTimeoutRef.current);
+ }
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 250);
+ }
- if (finalReceived) {
- logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
- responseCode,
- hasData: !!e.data,
+ async function authorizeTerminalTeardown(
+ conversationId: string,
+ terminalKind: 'FINAL' | 'error',
+ ): Promise<boolean> {
+ if (!isCurrentSubscription()) {
+ return false;
+ }
+ if (generationProtocolVersion !== GENERATION_PROTOCOL_VERSION) {
+ return true;
+ }
+
+ let status: Awaited<ReturnType<typeof fetchStreamStatus>>;
+ try {
+ status = await fetchStreamStatus(conversationId);
+ if (!isCurrentSubscription()) {
+ return false;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return false;
+ }
+ logger.warn('ResumableSSE', `Could not verify ${terminalKind} generation status`, {
+ conversationId,
+ generationCreatedAt,
+ error,
});
- return;
+ retryFencedTerminalAttachment(`${terminalKind.toLowerCase()} status unavailable`);
+ return false;
}
- (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
+ if (!supportsGenerationProtocolV2(status)) {
+ retryFencedTerminalAttachment(`${terminalKind.toLowerCase()} status was unnegotiated`);
+ return false;
+ }
- // 404 → job completed & was cleaned up; messages are persisted in DB.
- // Invalidate cache once so react-query refetches instead of showing an error.
- if (responseCode === 404) {
- const convoId = currentSubmission.conversation?.conversationId;
- logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
- sse.close();
- removeActiveJob(currentStreamId);
- /** Terminal: drop any in-flight live estimate so the gauge doesn't
- * keep counting stale streamed output after the stream ends */
- resetLive({ ...currentSubmission, userMessage });
- clearAllDrafts(convoId);
- if (optimisticStreamIdsRef.current.has(currentStreamId)) {
- clearAllDrafts(Constants.NEW_CONVO);
+ if (status.active === true) {
+ const handedOff =
+ status.createdAt != null &&
+ generationCreatedAt != null &&
+ status.createdAt !== generationCreatedAt &&
+ (await handoffToReplacement(conversationId, status));
+ if (!isCurrentSubscription()) {
+ return false;
}
- clearStepMaps();
- if (convoId) {
- queryClient.invalidateQueries({ queryKey: [QueryKeys.messages, convoId] });
- queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) });
+ if (handedOff) {
+ return false;
}
- if (
- !createdStreamIdsRef.current.has(currentStreamId) &&
- optimisticStreamIdsRef.current.has(currentStreamId)
- ) {
- if (isResume) {
- // A resumed subscribe attaches to an already-adopted stream (e.g. a deduped
- // start request). A 404 means the job is gone — but the conversation may be
- // persisted (the original completed and was cleaned up) or may never have
- // existed (the winner died before persisting). Don't guess: reconcile against
- // the server so a real conversation stays and a phantom is dropped.
- queryClient.invalidateQueries({ queryKey: [QueryKeys.allConversations] });
+ retryFencedTerminalAttachment(
+ status.createdAt == null || generationCreatedAt == null
+ ? `${terminalKind.toLowerCase()} status epoch was inconclusive`
+ : `${terminalKind.toLowerCase()} generation remains active`,
+ );
+ return false;
+ }
+
+ if (status.active !== false) {
+ retryFencedTerminalAttachment(`${terminalKind.toLowerCase()} status was inconclusive`);
+ return false;
+ }
+
+ return true;
+ }
+
+ const reconcileGenerationLifecycle = async (event: {
+ reconcileReason?: string;
+ terminalStatus?: 'complete' | 'error' | 'aborted';
+ generationCreatedAt?: number;
+ conversation?: { conversationId?: string };
+ }): Promise<void> => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const reconciliationConvoId =
+ event.conversation?.conversationId ??
+ currentSubmission.conversation?.conversationId ??
+ currentStreamId;
+ if (
+ generationCreatedAt != null &&
+ event.generationCreatedAt != null &&
+ event.generationCreatedAt !== generationCreatedAt
+ ) {
+ return;
+ }
+
+ if (event.reconcileReason === 'generation_replaced') {
+ const handedOff = await handoffToReplacement(reconciliationConvoId);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ // Replacement visibility can lag its notification by a round trip.
+ // Retry the stale epoch; the fenced HTTP endpoint will return the
+ // dedicated replacement response as soon as the new job is visible.
+ reconnectAttemptRef.current = Math.max(reconnectAttemptRef.current, 1);
+ sse.close();
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 250);
+ return;
+ }
+
+ reconnectAttemptRef.current = Math.max(reconnectAttemptRef.current, 1);
+ sse.close();
+ clearStepMaps();
+ let persistedMessages: TMessage[] | undefined;
+ const messageQueryKey = [QueryKeys.messages, reconciliationConvoId] as const;
+ try {
+ /** Force a fetch through the already-registered messages query and
+ * use its returned value. Merely awaiting invalidateQueries and
+ * reading the cache can mistake stale pre-run history for a
+ * successful reconciliation because React Query normally swallows
+ * refetch errors. */
+ await queryClient.invalidateQueries({
+ queryKey: messageQueryKey,
+ refetchType: 'none',
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const fetched = await queryClient.fetchQuery<TMessage[]>({
+ queryKey: messageQueryKey,
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (Array.isArray(fetched)) {
+ persistedMessages = fetched;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not refetch synthesized terminal state', {
+ conversationId: reconciliationConvoId,
+ error,
+ });
+ }
+ try {
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.conversation, reconciliationConvoId],
+ refetchType: 'all',
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ await queryClient.invalidateQueries({ queryKey: [QueryKeys.allConversations] });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not refresh synthesized conversation metadata', {
+ conversationId: reconciliationConvoId,
+ error,
+ });
+ }
+
+ let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;
+ try {
+ status = await fetchStreamStatus(reconciliationConvoId);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not inspect synthesized terminal status', {
+ conversationId: reconciliationConvoId,
+ error,
+ });
+ }
+ if (status != null && !supportsGenerationProtocolV2(status)) {
+ /** A v2 control frame cannot be reconciled with a legacy status
+ * snapshot (for example during a rolling deploy). Preserve the
+ * attachment and retry the exact fenced generation. */
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 250);
+ return;
+ }
+ if (
+ status?.active === true &&
+ status.createdAt != null &&
+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
+ ) {
+ const handedOff = await handoffToReplacement(reconciliationConvoId, status);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ }
+ if (
+ status?.active === true &&
+ (generationCreatedAt == null || status.createdAt === generationCreatedAt)
+ ) {
+ // Durable state still calls this exact epoch active. Prefer another
+ // fenced resume over terminalizing on a racing synthesized frame.
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 250);
+ return;
+ }
+
+ const authoritativeValues = [
+ ...(persistedMessages ?? []),
+ ...(status?.resumeState?.aggregatedContent ?? status?.aggregatedContent ?? []),
+ ];
+ const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));
+ settleAppliedSteerParts(reconciliationConvoId, authoritativeValues);
+ const unrecovered = status?.unrecoveredSteers ?? [];
+ if (unrecovered.length > 0) {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ convertSteersToQueued(reconciliationConvoId, unrecovered, {
+ ...(recoverySteerId != null && {
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ }),
+ });
+ for (const steer of unrecovered) {
+ reconciledIds.add(steer.steerId);
+ if (steer.clientSteerId) {
+ reconciledIds.add(steer.clientSteerId);
+ }
+ }
+ }
+ if (persistedMessages) {
+ convertLocalSteersToQueued(reconciliationConvoId, {
+ excludeSteerIds: reconciledIds,
+ generationProtocolVersion,
+ });
+ }
+
+ resetLive({ ...currentSubmission, userMessage });
+ removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt();
+ clearAllDrafts(reconciliationConvoId);
+ setIsSubmitting(false);
+ setShowStopButton(false);
+ if (event.reconcileReason === 'abort_persistence_failed') {
+ /** Interrupt & send arms an override that drains even an `aborted`
+ * run. This terminal frame explicitly says durable persistence is
+ * unknown, so consume that override before publishing runEnd; the
+ * queued words remain available for a deliberate retry. */
+ clearDrainAfterAbort(
+ currentSubmission.conversation?.conversationId || String(Constants.NEW_CONVO),
+ generationCreatedAt,
+ );
+ }
+ let reconciliationOutcome: 'completed' | 'aborted' | 'error' = 'aborted';
+ if (event.terminalStatus === 'complete') {
+ reconciliationOutcome = persistedMessages != null ? 'completed' : 'error';
+ } else if (event.terminalStatus === 'error') {
+ reconciliationOutcome = 'error';
+ }
+ setRunEnd({
+ conversationId: reconciliationConvoId,
+ outcome: reconciliationOutcome,
+ endedAt: Date.now(),
+ generationCreatedAt: status?.createdAt ?? generationCreatedAt,
+ });
+ setSubmission(null);
+ setStreamId(null);
+ optimisticStreamIdsRef.current.delete(currentStreamId);
+ createdStreamIdsRef.current.delete(currentStreamId);
+ reconnectAttemptRef.current = 0;
+ };
+
+ /**
+ * Error event handler - handles BOTH:
+ * 1. HTTP-level errors (responseCode present) - 404, 401, network failures
+ * 2. Server-sent error events (event: error with data) - known errors like ViolationTypes/ErrorTypes
+ *
+ * Order matters: check responseCode first since HTTP errors may also include data
+ */
+ sse.addEventListener('error', async (e: MessageEvent) => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const responseCode = (e as MessageEvent & { responseCode?: number }).responseCode;
+
+ if (finalReceived) {
+ logger.log('ResumableSSE', 'Ignoring error after FINAL event', {
+ responseCode,
+ hasData: !!e.data,
+ });
+ return;
+ }
+
+ (startupConfig?.balance?.enabled ?? false) && balanceQuery.refetch();
+
+ if (responseCode === 409) {
+ const replacementConvoId =
+ currentSubmission.conversation?.conversationId ?? currentStreamId;
+ const handedOff = await handoffToReplacement(replacementConvoId);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ }
+
+ // 404 → job completed & was cleaned up; messages are persisted in DB.
+ // Refetch and reconcile them before recovering local steers: the final
+ // response may contain an applied steer whose live event was missed.
+ if (responseCode === 404) {
+ const convoId = currentSubmission.conversation?.conversationId;
+ logger.log('ResumableSSE', 'Stream 404, invalidating messages for:', convoId);
+ // Keep the UI non-idle while the status read decides whether this is
+ // terminal cleanup or a handoff to a newer generation epoch.
+ reconnectAttemptRef.current = Math.max(reconnectAttemptRef.current, 1);
+ cancelSteerRetryFrames();
+ sse.close();
+ /** Terminal: drop any in-flight live estimate so the gauge doesn't
+ * keep counting stale streamed output after the stream ends */
+ resetLive({ ...currentSubmission, userMessage });
+ clearAllDrafts(convoId);
+ if (optimisticStreamIdsRef.current.has(currentStreamId)) {
+ clearAllDrafts(Constants.NEW_CONVO);
+ }
+ clearStepMaps();
+ let persistedMessages: TMessage[] | undefined;
+ if (convoId) {
+ try {
+ const messageQueryKey = [QueryKeys.messages, convoId] as const;
+ await queryClient.invalidateQueries({
+ queryKey: messageQueryKey,
+ refetchType: 'none',
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const fetched = await queryClient.fetchQuery<TMessage[]>({
+ queryKey: messageQueryKey,
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (Array.isArray(fetched)) {
+ persistedMessages = fetched;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not reconcile persisted messages after 404', {
+ conversationId: convoId,
+ error,
+ });
+ }
+ queryClient.removeQueries({ queryKey: streamStatusQueryKey(convoId) });
+ }
+ const recoveryConvoId = convoId ?? currentStreamId;
+ const reconciledIds = new Set(
+ persistedMessages ? collectAppliedSteerIds(persistedMessages) : [],
+ );
+ if (persistedMessages) {
+ settleAppliedSteerParts(recoveryConvoId, persistedMessages);
+ }
+
+ // The status replay is owner-gated and non-destructive; creating a
+ // recovered follow-up consumes its exact source. Prefer these
+ // authoritative leftovers over guessing from local chips.
+ let confirmedV2Terminal = false;
+ try {
+ const status = await fetchStreamStatus(recoveryConvoId);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ !supportsGenerationProtocolV2(status)
+ ) {
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 1_000);
+ return;
+ }
+ if (
+ status.active === true &&
+ status.createdAt != null &&
+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
+ ) {
+ const handedOff = await handoffToReplacement(recoveryConvoId, status);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ /** Legacy generations have no replacement-handoff contract.
+ * Never attach this submission to a different epoch. */
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 1_000);
+ return;
+ }
+ if (status.active === true) {
+ /** A stream lookup can briefly lose its routing record while the
+ * authoritative job status still proves the same generation is
+ * alive. Preserve ownership and retry the attachment instead of
+ * publishing a terminal run-end (which could auto-drain queued
+ * text into a still-running generation). */
+ const activeStreamId = status.streamId ?? currentStreamId;
+ queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
+ if (status.createdAt != null) {
+ updateActiveGenerationCreatedAt(
+ recoveryConvoId,
+ status.createdAt,
+ undefined,
+ generationProtocolVersion,
+ );
+ }
+ addActiveJob(activeStreamId);
+ seedSteerChips(
+ recoveryConvoId,
+ (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
+ status.createdAt,
+ generationProtocolVersion,
+ );
+ settleAppliedSteerParts(
+ recoveryConvoId,
+ status.resumeState?.aggregatedContent ?? status.aggregatedContent,
+ );
+ if (status.pendingAction ?? status.resumeState?.pendingAction) {
+ applyPendingActionToMessages(
+ (status.pendingAction ??
+ status.resumeState?.pendingAction) as Agents.PendingAction,
+ );
+ }
+ setIsSubmitting(true);
+ setShowStopButton(status.createdAt != null);
+ reconnectAttemptRef.current = Math.max(reconnectAttemptRef.current, 1);
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ activeStreamId,
+ submissionRef.current,
+ true,
+ status.createdAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 1_000);
+ return;
+ }
+ confirmedV2Terminal =
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ supportsGenerationProtocolV2(status);
+ const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
+ const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
+ if (unrecovered.length > 0) {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ convertSteersToQueued(recoveryConvoId, unrecovered, {
+ generationProtocolVersion,
+ ...(recoverySteerId != null && {
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ }),
+ });
+ for (const steer of unrecovered) {
+ reconciledIds.add(steer.steerId);
+ if (steer.clientSteerId) {
+ reconciledIds.add(steer.clientSteerId);
+ }
+ }
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not recover parked steers after 404', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+
+ removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt();
+ if (
+ !createdStreamIdsRef.current.has(currentStreamId) &&
+ optimisticStreamIdsRef.current.has(currentStreamId)
+ ) {
+ if (isResume) {
+ // A resumed subscribe attaches to an already-adopted stream (e.g. a deduped
+ // start request). A 404 means the job is gone — but the conversation may be
+ // persisted (the original completed and was cleaned up) or may never have
+ // existed (the winner died before persisting). Don't guess: reconcile against
+ // the server so a real conversation stays and a phantom is dropped.
+ queryClient.invalidateQueries({ queryKey: [QueryKeys.allConversations] });
} else {
// Fresh optimistic stream that never started: prune immediately.
removeConvoFromAllQueries(queryClient, currentStreamId);
@@ -1428,21 +2602,17 @@ export default function useResumableSSE(
}
setIsSubmitting(false);
setShowStopButton(false);
- const recoveryConvoId = convoId ?? currentStreamId;
- // Terminal for this run: the job is gone, so no event will ever
- // resolve an acknowledged chip — convert them to queued chips.
- convertLocalSteersToQueued(recoveryConvoId);
- // A terminal drain may have parked steers no subscriber received —
- // the status route claims them exactly once (same recovery as
- // useResumeOnLoad). Best-effort: chips are already converted above.
- fetchStreamStatus(recoveryConvoId)
- .then((status) => {
- const unrecovered = status.unrecoveredSteers ?? [];
- if (unrecovered.length > 0) {
- convertSteersToQueued(recoveryConvoId, unrecovered);
- }
- })
- .catch(() => undefined);
+
+ // Only infer recovery from local accepted chips after the persisted
+ // final was successfully inspected. If that fetch failed, their
+ // applied-vs-leftover outcome is unknown and converting could send
+ // already-applied words twice.
+ if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
+ convertLocalSteersToQueued(recoveryConvoId, {
+ excludeSteerIds: reconciledIds,
+ generationProtocolVersion,
+ });
+ }
// The true outcome is unknown here (job record already cleaned up):
// a non-'completed' outcome releases parked interrupt flags without
// auto-sending queued messages the user may not want fired.
@@ -1463,17 +2633,25 @@ export default function useResumableSSE(
if (responseCode === 401) {
try {
const refreshResponse = await request.refreshToken();
+ if (!isCurrentSubscription()) {
+ return;
+ }
const newToken = refreshResponse?.token ?? '';
if (!newToken) {
throw new Error('Token refresh failed.');
}
sse.headers = {
+ ...sse.headers,
+ ...generationProtocolHeaders(),
Authorization: `Bearer ${newToken}`,
};
request.dispatchTokenUpdatedEvent(newToken);
sse.stream();
return;
} catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
logger.log('ResumableSSE', 'Token refresh failed:', error);
}
}
@@ -1486,13 +2664,23 @@ export default function useResumableSSE(
* not a server-sent error payload. Use `== null` to only match undefined/null (no HTTP status).
*/
if (responseCode == null && e.data) {
+ finalReceived = true;
+ const recoveryConvoId = currentSubmission.conversation?.conversationId ?? currentStreamId;
+ if (
+ !(await authorizeTerminalTeardown(recoveryConvoId, 'error')) ||
+ !isCurrentSubscription()
+ ) {
+ return;
+ }
logger.log('ResumableSSE', 'Server-sent error event received:', e.data);
+ cancelSteerRetryFrames();
sse.close();
/** FLUSH (not cancel): the error card below is built from the cache
* tail, so queued tokens must land first — and a stale trailing
* frame must never overwrite the error write. */
flushPendingDeltas();
removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt();
resetLive({ ...currentSubmission, userMessage });
if (
!createdStreamIdsRef.current.has(currentStreamId) &&
@@ -1501,8 +2689,10 @@ export default function useResumableSSE(
removeConvoFromAllQueries(queryClient, currentStreamId);
}
+ let errorSupportsV2 = false;
try {
const errorData = JSON.parse(e.data);
+ errorSupportsV2 = supportsGenerationProtocolV2(errorData);
const errorString = errorData.error ?? errorData.message ?? JSON.stringify(errorData);
// Check if it's a known error type (ViolationTypes or ErrorTypes)
@@ -1538,16 +2728,57 @@ export default function useResumableSSE(
setIsSubmitting(false);
setShowStopButton(false);
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ if (
+ recoverySteerId != null &&
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ errorSupportsV2
+ ) {
+ try {
+ /** The error was received on the exact generation-fenced stream.
+ * Read status only now, after terminal was observed: a claim
+ * started before this recovery could return a stale parked copy
+ * after a successful commit and must never authorize redelivery. */
+ const terminalStatus = await fetchStreamStatus(recoveryConvoId);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const releasedSource = terminalStatus.unrecoveredSteers?.find(
+ (steer) => steer.steerId === recoverySteerId,
+ );
+ if (
+ terminalStatus.active === false &&
+ supportsGenerationProtocolV2(terminalStatus) &&
+ releasedSource != null
+ ) {
+ convertSteersToQueued(recoveryConvoId, [releasedSource], {
+ generationProtocolVersion,
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ });
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not recover source after generation error', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+ }
// The error terminal's backstop parks acked leftovers server-side;
// claim it now so a reload can't resurrect the chips converted here.
- convertLocalSteersToQueued(
- currentSubmission.conversation?.conversationId ?? currentStreamId,
- { claimParked: true },
- );
+ if (generationProtocolVersion === 1 || errorSupportsV2) {
+ convertLocalSteersToQueued(recoveryConvoId, {
+ claimParked: true,
+ generationProtocolVersion,
+ });
+ }
setRunEnd({
- conversationId: currentSubmission.conversation?.conversationId ?? currentStreamId,
+ conversationId: recoveryConvoId,
outcome: 'error',
endedAt: Date.now(),
+ generationCreatedAt,
});
setStreamId(null);
optimisticStreamIdsRef.current.delete(currentStreamId);
@@ -1575,26 +2806,210 @@ export default function useResumableSSE(
sse.close();
reconnectTimeoutRef.current = setTimeout(() => {
- if (submissionRef.current) {
+ if (isCurrentSubscription() && submissionRef.current) {
// Reconnect with isResume=true to get sync event with any missed content
- subscribeToStream(currentStreamId, submissionRef.current, true);
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
}
}, delay);
// Keep UI in "submitting" state during reconnection attempts
// so user knows we're still trying (abort handler may have reset these)
setIsSubmitting(true);
- setShowStopButton(true);
+ setShowStopButton(generationCreatedAt != null);
} else {
logger.error('ResumableSSE', 'Max reconnect attempts reached');
sse.close();
flushPendingDeltas();
- errorHandler({ data: undefined, submission: currentSubmission as EventSubmission });
- /** Terminal: clear the in-flight live estimate like the other
- * stop-reconnecting paths so the gauge doesn't show stale tokens */
+ const recoveryConvoId = currentSubmission.conversation?.conversationId ?? currentStreamId;
+ let status: Awaited<ReturnType<typeof fetchStreamStatus>> | undefined;
+ try {
+ status = await fetchStreamStatus(recoveryConvoId);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not determine job state after reconnect limit', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+
+ if (
+ status != null &&
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ !supportsGenerationProtocolV2(status)
+ ) {
+ // A legacy responder cannot adjudicate a v2 job during rollout.
+ status = undefined;
+ }
+
+ if (
+ status?.active === true &&
+ status.createdAt != null &&
+ (generationCreatedAt == null || status.createdAt !== generationCreatedAt)
+ ) {
+ const handedOff = await handoffToReplacement(recoveryConvoId, status);
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (handedOff) {
+ return;
+ }
+ reconnectAttemptRef.current = 0;
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 30_000);
+ return;
+ }
+
+ if (status?.active === true) {
+ // The transport gave up, not the generation. Keep the active-job
+ // marker and accepted chips intact, then release this stale
+ // submission so useResumeOnLoad can establish a fresh attachment.
+ queryClient.setQueryData(streamStatusQueryKey(recoveryConvoId), status);
+ seedSteerChips(
+ recoveryConvoId,
+ (status.resumeState?.pendingSteers ?? []) as TPendingSteer[],
+ status.createdAt,
+ generationProtocolVersion,
+ );
+ settleAppliedSteerParts(
+ recoveryConvoId,
+ status.resumeState?.aggregatedContent ?? status.aggregatedContent,
+ );
+ if (status.pendingAction ?? status.resumeState?.pendingAction) {
+ applyPendingActionToMessages(
+ (status.pendingAction ?? status.resumeState?.pendingAction) as Agents.PendingAction,
+ );
+ }
+ setIsSubmitting(true);
+ setShowStopButton(generationCreatedAt != null);
+ reconnectAttemptRef.current = 0;
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 30_000);
+ return;
+ }
+
+ if (status == null) {
+ // An inconclusive status read is still not proof of termination.
+ // Preserve server/client ownership and let the normal resume-on-load
+ // status query retry rather than duplicating accepted words.
+ queryClient.invalidateQueries({ queryKey: streamStatusQueryKey(recoveryConvoId) });
+ setIsSubmitting(true);
+ setShowStopButton(generationCreatedAt != null);
+ reconnectAttemptRef.current = 0;
+ reconnectTimeoutRef.current = setTimeout(() => {
+ if (isCurrentSubscription() && submissionRef.current) {
+ subscribeToStream(
+ currentStreamId,
+ submissionRef.current,
+ true,
+ generationCreatedAt,
+ generationProtocolVersion,
+ lifecycleSignal,
+ );
+ }
+ }, 30_000);
+ return;
+ }
+
+ // The status endpoint proved the job terminal. Reconcile persisted
+ // content before recovering leftovers, just like the 404 path.
+ cancelSteerRetryFrames();
+ let persistedMessages: TMessage[] | undefined;
+ try {
+ const messageQueryKey = [QueryKeys.messages, recoveryConvoId] as const;
+ await queryClient.invalidateQueries({
+ queryKey: messageQueryKey,
+ refetchType: 'none',
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ const fetched = await queryClient.fetchQuery<TMessage[]>({
+ queryKey: messageQueryKey,
+ });
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (Array.isArray(fetched)) {
+ persistedMessages = fetched;
+ }
+ } catch (error) {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not reconcile terminal messages after disconnect', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+
+ const authoritativeValues = [
+ ...(persistedMessages ?? []),
+ ...(status.resumeState?.aggregatedContent ?? status.aggregatedContent ?? []),
+ ];
+ const confirmedV2Terminal =
+ generationProtocolVersion === GENERATION_PROTOCOL_VERSION &&
+ supportsGenerationProtocolV2(status);
+ const reconciledIds = new Set(collectAppliedSteerIds(authoritativeValues));
+ settleAppliedSteerParts(recoveryConvoId, authoritativeValues);
+ const canRecoverTerminalSteers = generationProtocolVersion === 1 || confirmedV2Terminal;
+ const unrecovered = canRecoverTerminalSteers ? (status.unrecoveredSteers ?? []) : [];
+ if (unrecovered.length > 0) {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ convertSteersToQueued(recoveryConvoId, unrecovered, {
+ generationProtocolVersion,
+ ...(recoverySteerId != null && {
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ }),
+ });
+ for (const steer of unrecovered) {
+ reconciledIds.add(steer.steerId);
+ if (steer.clientSteerId) {
+ reconciledIds.add(steer.clientSteerId);
+ }
+ }
+ }
+ if (persistedMessages && (generationProtocolVersion === 1 || confirmedV2Terminal)) {
+ convertLocalSteersToQueued(recoveryConvoId, {
+ excludeSteerIds: reconciledIds,
+ generationProtocolVersion,
+ });
+ }
+
resetLive({ ...currentSubmission, userMessage });
- // Optimistically remove from active jobs on max retries
removeActiveJob(currentStreamId);
+ clearAttachedGenerationCreatedAt();
if (
!createdStreamIdsRef.current.has(currentStreamId) &&
optimisticStreamIdsRef.current.has(currentStreamId)
@@ -1603,18 +3018,23 @@ export default function useResumableSSE(
}
setIsSubmitting(false);
setShowStopButton(false);
- convertLocalSteersToQueued(
- currentSubmission.conversation?.conversationId ?? currentStreamId,
- { claimParked: true },
- );
+ let recoveryOutcome: 'completed' | 'aborted' | 'error' = 'error';
+ if (status.status === 'complete' && persistedMessages != null) {
+ recoveryOutcome = 'completed';
+ } else if (status.status === 'aborted') {
+ recoveryOutcome = 'aborted';
+ }
setRunEnd({
- conversationId: currentSubmission.conversation?.conversationId ?? currentStreamId,
- outcome: 'error',
+ conversationId: recoveryConvoId,
+ outcome: recoveryOutcome,
endedAt: Date.now(),
+ generationCreatedAt: status.createdAt ?? generationCreatedAt,
});
+ setSubmission(null);
setStreamId(null);
optimisticStreamIdsRef.current.delete(currentStreamId);
createdStreamIdsRef.current.delete(currentStreamId);
+ reconnectAttemptRef.current = 0;
}
});
@@ -1624,6 +3044,13 @@ export default function useResumableSSE(
* Only reset state if we're NOT in a reconnection cycle.
*/
sse.addEventListener('abort', () => {
+ if (!isCurrentSubscription()) {
+ return;
+ }
+ if (replacementHandoffRef.current) {
+ logger.log('ResumableSSE', 'Stream closed for generation handoff - preserving state');
+ return;
+ }
// If we're in a reconnection cycle, don't reset state
// (error handler will set up the reconnect timeout)
if (reconnectAttemptRef.current > 0) {
@@ -1632,6 +3059,7 @@ export default function useResumableSSE(
}
logger.log('ResumableSSE', 'Stream aborted (intentional close) - no reconnect');
+ cancelSteerRetryFrames();
// Clear any pending reconnect attempts
if (reconnectTimeoutRef.current) {
clearTimeout(reconnectTimeoutRef.current);
@@ -1707,16 +3135,22 @@ export default function useResumableSSE(
resetLive,
seedLive,
setRunEnd,
+ clearDrainAfterAbort,
resolveSteerChip,
+ updateSteerChips,
seedSteerChips,
+ settleAppliedSteerParts,
convertSteersToQueued,
convertLocalSteersToQueued,
+ addActiveJob,
+ setSubmission,
+ updateActiveGenerationCreatedAt,
],
);
/**
* Start generation (POST request that returns streamId)
- * Uses request.post which has axios interceptors for automatic token refresh.
+ * Uses the generation protocol request wrapper, including auth refresh.
* Retries transient network failures and startup readiness responses.
* Readiness retries honor Retry-After until cleanup or the readiness window expires.
*/
@@ -1724,7 +3158,7 @@ export default function useResumableSSE(
async (
currentSubmission: TSubmission,
signal?: AbortSignal,
- ): Promise<{ streamId: string; resumed: boolean } | null> => {
+ ): Promise<StartGenerationResult | null> => {
const payloadData = createPayload(currentSubmission);
let { payload } = payloadData;
payload = removeNullishValues(payload) as TPayload;
@@ -1742,16 +3176,63 @@ export default function useResumableSSE(
while (!signal?.aborted) {
requestAttempts += 1;
try {
- // Use request.post which handles auth token refresh via axios interceptors
- const data = await request.post(url, payload);
+ const data = await postGenerationRequest<unknown>(url, payload, { signal });
if (signal?.aborted) {
return null;
}
+ if (isSettledStartResponse(data)) {
+ const conversationId = getStartGenerationConversationId(data);
+ if (conversationId) {
+ logger.log('ResumableSSE', 'Generation already settled:', { conversationId });
+ return {
+ status: 'settled',
+ conversationId,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ }
+ lastError = { response: { data } };
+ break;
+ }
+ if (isReplacedStartResponse(data)) {
+ const streamId = getStartGenerationStreamId(data);
+ const conversationId = getStartGenerationConversationId(data);
+ const generationCreatedAt = getStartGenerationCreatedAt(data);
+ if (streamId && conversationId && generationCreatedAt != null) {
+ return {
+ status: 'replaced',
+ streamId,
+ conversationId,
+ generationCreatedAt,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ }
+ lastError = { response: { data } };
+ break;
+ }
+ if (isUnnegotiatedV2StartControl(data)) {
+ /** Never reinterpret a v2-only control word as a legacy stream
+ * response merely because it also carries a stream id. */
+ lastError = { response: { data } };
+ break;
+ }
const streamId = getStartGenerationStreamId(data);
if (streamId) {
const resumed = isResumedStartResponse(data);
- logger.log('ResumableSSE', 'Generation started:', { streamId, resumed });
- return { streamId, resumed };
+ const generationCreatedAt = getStartGenerationCreatedAt(data);
+ const generationProtocolVersion = getGenerationProtocolVersion(data);
+ logger.log('ResumableSSE', 'Generation started:', {
+ streamId,
+ resumed,
+ generationCreatedAt,
+ generationProtocolVersion,
+ });
+ return {
+ status: 'stream',
+ streamId,
+ resumed,
+ generationProtocolVersion,
+ ...(generationCreatedAt != null && { generationCreatedAt }),
+ };
}
lastError = { response: { data } };
@@ -1761,6 +3242,23 @@ export default function useResumableSSE(
return null;
}
+ const predecessorMismatch = getPredecessorMismatchStartResponse(error);
+ if (predecessorMismatch != null) {
+ /** Admission was atomically rejected before this queued turn
+ * created anything. Put the exact row back immediately and let
+ * the caller reconcile or hand off to the generation that won. */
+ restoreQueuedSubmission(currentSubmission);
+ logger.log('ResumableSSE', 'Queued generation lost predecessor fence', {
+ expectedPredecessorCreatedAt: currentSubmission.expectedPredecessorCreatedAt,
+ ...predecessorMismatch,
+ });
+ return {
+ status: 'predecessor_mismatch',
+ ...predecessorMismatch,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ }
+
lastError = error;
const isNetworkError = isRetryableNetworkError(error);
const isServerNotReady = isServerNotReadyError(error);
@@ -1803,7 +3301,50 @@ export default function useResumableSSE(
logger.error('ResumableSSE', 'Error starting generation:', lastError);
- const errorData = toStartGenerationError(lastError)?.response?.data;
+ const startError = toStartGenerationError(lastError);
+ const errorData = startError?.response?.data;
+ const responseStatus = startError?.response?.status;
+ if (responseStatus != null && responseStatus >= 400 && responseStatus < 500) {
+ // The server rejected admission before exposing a generation. Restore
+ // the exact queue row/position; ambiguous transport/5xx outcomes must
+ // first reconcile durable state instead of risking a duplicate start.
+ restoreQueuedSubmission(currentSubmission);
+ } else {
+ const recoverySteerId = getRecoverySteerId(currentSubmission);
+ const recoveryConvoId = currentSubmission.conversation?.conversationId;
+ if (recoverySteerId != null && recoveryConvoId) {
+ try {
+ const status = await fetchStreamStatus(recoveryConvoId);
+ if (signal?.aborted) {
+ return null;
+ }
+ const releasedSource = status.unrecoveredSteers?.find(
+ (steer) => steer.steerId === recoverySteerId,
+ );
+ if (
+ status.active === false &&
+ supportsGenerationProtocolV2(status) &&
+ releasedSource != null
+ ) {
+ convertSteersToQueued(recoveryConvoId, [releasedSource], {
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ allowPreviouslyConvertedIds: [recoverySteerId],
+ });
+ }
+ } catch (error) {
+ if (signal?.aborted) {
+ return null;
+ }
+ logger.warn('ResumableSSE', 'Could not recover source after start failure', {
+ conversationId: recoveryConvoId,
+ error,
+ });
+ }
+ }
+ }
+ if (signal?.aborted) {
+ return null;
+ }
errorHandler({
data: getStreamStartFailureData(errorData),
submission: currentSubmission as EventSubmission,
@@ -1813,7 +3354,15 @@ export default function useResumableSSE(
setSubmission(null);
return null;
},
- [clearStepMaps, errorHandler, setIsSubmitting, setShowStopButton, setSubmission],
+ [
+ clearStepMaps,
+ convertSteersToQueued,
+ errorHandler,
+ restoreQueuedSubmission,
+ setIsSubmitting,
+ setShowStopButton,
+ setSubmission,
+ ],
);
useEffect(() => {
@@ -1835,7 +3384,15 @@ export default function useResumableSSE(
return;
}
- const resumeStreamId = (submission as TSubmission & { resumeStreamId?: string }).resumeStreamId;
+ const resumableSubmission = submission as TSubmission & {
+ resumeStreamId?: string;
+ resumeGenerationCreatedAt?: number;
+ resumeGenerationProtocolVersion?: GenerationProtocolVersion;
+ };
+ const resumeStreamId = resumableSubmission.resumeStreamId;
+ const resumeGenerationCreatedAt = resumableSubmission.resumeGenerationCreatedAt;
+ const resumeGenerationProtocolVersion =
+ resumableSubmission.resumeGenerationProtocolVersion ?? 1;
logger.log('ResumableSSE', 'Effect triggered', {
conversationId: submission.conversation?.conversationId,
hasResumeStreamId: !!resumeStreamId,
@@ -1846,17 +3403,28 @@ export default function useResumableSSE(
submissionRef.current = submission;
const startController = new AbortController();
const { signal } = startController;
+ const isCurrentEffect = () => !signal.aborted && submissionRef.current === submission;
const initStream = async () => {
- if (signal.aborted) {
+ if (!isCurrentEffect()) {
return;
}
setIsSubmitting(true);
- setShowStopButton(true);
+ /** Starting a new generation is the one interval where `isSubmitting`
+ * is true but no generation epoch exists yet. Clear any prior epoch and
+ * withhold live Stop/steer controls; the composer can still queue input.
+ * subscribeToStream exposes controls once the POST/resume installs the
+ * server-provided epoch. */
+ const submissionConversationId =
+ submission.conversation?.conversationId ?? Constants.NEW_CONVO;
+ if (!resumeStreamId || resumeGenerationCreatedAt == null) {
+ updateActiveGenerationCreatedAt(resumeStreamId ?? submissionConversationId, null);
+ }
+ setShowStopButton(false);
if (resumeStreamId) {
- if (signal.aborted) {
+ if (!isCurrentEffect()) {
return;
}
// Resume: just subscribe to existing stream, don't start new generation
@@ -1864,16 +3432,387 @@ export default function useResumableSSE(
setStreamId(resumeStreamId);
// Optimistically add to active jobs (in case it's not already there)
addActiveJob(resumeStreamId);
- subscribeToStream(resumeStreamId, submission, true); // isResume=true
+ subscribeToStream(
+ resumeStreamId,
+ submission,
+ true,
+ resumeGenerationCreatedAt,
+ resumeGenerationProtocolVersion,
+ signal,
+ );
} else {
// New generation: start and then subscribe
logger.log('ResumableSSE', 'Starting NEW generation');
const startResult = await startGeneration(submission, signal);
- if (signal.aborted) {
+ if (!isCurrentEffect()) {
return;
}
if (startResult) {
- const { streamId: newStreamId, resumed } = startResult;
+ let replacementStart = startResult.status === 'replaced' ? startResult : null;
+ let knownReplacementStatus: StreamStatusResponse | null = null;
+
+ if (startResult.status === 'predecessor_mismatch') {
+ const expectedPredecessorCreatedAt = submission.expectedPredecessorCreatedAt;
+ let refreshedStatus: StreamStatusResponse | null = null;
+ try {
+ refreshedStatus = await fetchStreamStatus(startResult.conversationId);
+ if (!isCurrentEffect()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentEffect()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not refresh predecessor mismatch status', {
+ conversationId: startResult.conversationId,
+ error,
+ });
+ }
+ if (!isCurrentEffect()) {
+ return;
+ }
+
+ const reportedReplacement =
+ startResult.active === true &&
+ startResult.generationCreatedAt != null &&
+ startResult.generationCreatedAt !== expectedPredecessorCreatedAt;
+
+ if (
+ refreshedStatus?.active === true &&
+ supportsGenerationProtocolV2(refreshedStatus) &&
+ typeof refreshedStatus.streamId === 'string' &&
+ refreshedStatus.streamId.length > 0 &&
+ refreshedStatus.createdAt != null &&
+ refreshedStatus.createdAt !== expectedPredecessorCreatedAt
+ ) {
+ replacementStart = {
+ status: 'replaced',
+ streamId: refreshedStatus.streamId,
+ conversationId: startResult.conversationId,
+ generationCreatedAt: refreshedStatus.createdAt,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ knownReplacementStatus = refreshedStatus;
+ restoreQueuedSubmission(submission, refreshedStatus.createdAt);
+ } else if (reportedReplacement) {
+ /** The atomic 409 is itself authoritative at admission time.
+ * Status publication can lag that response, so retain its exact
+ * epoch as a minimal handoff snapshot; the fenced SSE attach can
+ * reconcile if the replacement finishes in the meantime. */
+ replacementStart = {
+ status: 'replaced',
+ streamId: startResult.streamId,
+ conversationId: startResult.conversationId,
+ generationCreatedAt: startResult.generationCreatedAt!,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ knownReplacementStatus = {
+ active: true,
+ status: 'running',
+ streamId: startResult.streamId,
+ createdAt: startResult.generationCreatedAt,
+ generationProtocolVersion: GENERATION_PROTOCOL_VERSION,
+ };
+ restoreQueuedSubmission(submission, startResult.generationCreatedAt!);
+ } else {
+ /** No different live epoch is proven. Reconcile away the
+ * optimistic C rows, but publish no run-end: doing so would
+ * immediately drain and re-send the just-restored queue item. */
+ const terminalWinnerCreatedAt =
+ refreshedStatus?.createdAt ?? startResult.generationCreatedAt;
+ if (terminalWinnerCreatedAt != null) {
+ restoreQueuedSubmission(submission, terminalWinnerCreatedAt);
+ }
+ if (refreshedStatus != null) {
+ queryClient.setQueryData(
+ streamStatusQueryKey(startResult.conversationId),
+ refreshedStatus,
+ );
+ } else {
+ queryClient.invalidateQueries({
+ queryKey: streamStatusQueryKey(startResult.conversationId),
+ });
+ }
+ try {
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.messages, startResult.conversationId],
+ refetchType: 'all',
+ });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.conversation, startResult.conversationId],
+ refetchType: 'all',
+ });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ await queryClient.invalidateQueries({ queryKey: [QueryKeys.allConversations] });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentEffect()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not reconcile predecessor mismatch history', {
+ conversationId: startResult.conversationId,
+ error,
+ });
+ }
+ clearDrainAfterAbort(startResult.conversationId, expectedPredecessorCreatedAt);
+ submissionRef.current = null;
+ setIsSubmitting(false);
+ setShowStopButton(false);
+ setSubmission(null);
+ return;
+ }
+ }
+
+ if (replacementStart != null) {
+ /** This POST belongs to an older optimistic submission, while the
+ * conversation-scoped stream id now belongs to a newer generation.
+ * Never attach A's submission directly to B. Discover B's
+ * authoritative resume snapshot, refresh durable history, then let
+ * useResumeOnLoad construct a new epoch-fenced submission. */
+ const replacementStatus =
+ knownReplacementStatus ??
+ (await fetchStreamStatus(replacementStart.conversationId).catch((error) => {
+ if (!isCurrentEffect()) {
+ return null;
+ }
+ logger.warn('ResumableSSE', 'Could not discover replacement start generation', {
+ conversationId: replacementStart.conversationId,
+ generationCreatedAt: replacementStart.generationCreatedAt,
+ error,
+ });
+ return null;
+ }));
+ if (!isCurrentEffect()) {
+ return;
+ }
+ if (
+ replacementStatus?.active === true &&
+ supportsGenerationProtocolV2(replacementStatus) &&
+ replacementStatus.streamId === replacementStart.streamId &&
+ replacementStatus.createdAt === replacementStart.generationCreatedAt
+ ) {
+ const startedAsNewConvo = isInitialNewConversation(submission);
+ try {
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.messages, replacementStart.conversationId],
+ refetchType: 'all',
+ });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.conversation, replacementStart.conversationId],
+ refetchType: 'all',
+ });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ await queryClient.invalidateQueries({ queryKey: [QueryKeys.allConversations] });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentEffect()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not refresh replacement generation history', {
+ conversationId: replacementStart.conversationId,
+ error,
+ });
+ }
+ const authoritativeConversation = queryClient.getQueryData<TConversation>([
+ QueryKeys.conversation,
+ replacementStart.conversationId,
+ ]);
+ if (authoritativeConversation?.conversationId === replacementStart.conversationId) {
+ setConversation?.(authoritativeConversation);
+ }
+ queryClient.setQueryData(streamStatusQueryKey(replacementStart.conversationId), {
+ ...replacementStatus,
+ generationHandoff: true,
+ });
+ updateActiveGenerationCreatedAt(
+ replacementStart.conversationId,
+ replacementStart.generationCreatedAt,
+ undefined,
+ replacementStart.generationProtocolVersion,
+ );
+ addActiveJob(replacementStart.streamId);
+ if (startedAsNewConvo) {
+ optimisticStreamIdsRef.current.add(replacementStart.streamId);
+ updateActiveGenerationCreatedAt(
+ Constants.NEW_CONVO,
+ replacementStart.generationCreatedAt,
+ undefined,
+ replacementStart.generationProtocolVersion,
+ );
+ replaceNewConversationUrl(replacementStart.conversationId);
+ }
+ seedSteerChips(
+ replacementStart.conversationId,
+ (replacementStatus.resumeState?.pendingSteers ?? []) as TPendingSteer[],
+ replacementStart.generationCreatedAt,
+ replacementStart.generationProtocolVersion,
+ );
+ settleAppliedSteerParts(
+ replacementStart.conversationId,
+ replacementStatus.resumeState?.aggregatedContent ??
+ replacementStatus.aggregatedContent,
+ );
+ replacementHandoffRef.current = true;
+ submissionRef.current = null;
+ setIsSubmitting(true);
+ setShowStopButton(true);
+ setSubmission(null);
+ return;
+ }
+
+ /** The server reported a replacement but its authoritative status
+ * could not be confirmed. Drop A without ever attaching it to B;
+ * leave history/status invalidated so navigation/reload can recover,
+ * and keep any queued text parked (non-completed outcome). */
+ queryClient.invalidateQueries({
+ queryKey: streamStatusQueryKey(replacementStart.conversationId),
+ });
+ queryClient.invalidateQueries({
+ queryKey: [QueryKeys.messages, replacementStart.conversationId],
+ });
+ clearDrainAfterAbort(
+ replacementStart.conversationId,
+ replacementStart.generationCreatedAt,
+ );
+ setIsSubmitting(false);
+ setShowStopButton(false);
+ setRunEnd({
+ conversationId: replacementStart.conversationId,
+ outcome: 'error',
+ endedAt: Date.now(),
+ generationCreatedAt: replacementStart.generationCreatedAt,
+ });
+ setSubmission(null);
+ return;
+ }
+ if (startResult.status === 'predecessor_mismatch') {
+ // Every mismatch path above either hands off or reconciles.
+ return;
+ }
+ if (startResult.status === 'settled') {
+ const settledConversationId = startResult.conversationId;
+ const startedAsNewConvo = isInitialNewConversation(submission);
+ let persistedConversation: TConversation | undefined;
+ let conversationLookup: 'found' | 'not_found' | 'inconclusive' = 'inconclusive';
+ try {
+ persistedConversation = await dataService.getConversationById(settledConversationId);
+ if (!isCurrentEffect()) {
+ return;
+ }
+ conversationLookup = 'found';
+ } catch (error) {
+ if (!isCurrentEffect()) {
+ return;
+ }
+ conversationLookup =
+ toStartGenerationError(error)?.response?.status === 404
+ ? 'not_found'
+ : 'inconclusive';
+ logger.warn('ResumableSSE', 'Could not load settled generation conversation', {
+ conversationId: settledConversationId,
+ conversationLookup,
+ error,
+ });
+ }
+
+ if (persistedConversation != null) {
+ queryClient.setQueryData(
+ [QueryKeys.conversation, settledConversationId],
+ persistedConversation,
+ );
+ upsertConvoInAllQueries(queryClient, persistedConversation);
+ setConversation?.(persistedConversation);
+ if (startedAsNewConvo) {
+ replaceNewConversationUrl(settledConversationId);
+ }
+ }
+
+ try {
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.messages, settledConversationId],
+ refetchType: 'all',
+ });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ await queryClient.invalidateQueries({
+ queryKey: [QueryKeys.conversation, settledConversationId],
+ refetchType: 'all',
+ });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ await queryClient.invalidateQueries({ queryKey: [QueryKeys.allConversations] });
+ if (!isCurrentEffect()) {
+ return;
+ }
+ } catch (error) {
+ if (!isCurrentEffect()) {
+ return;
+ }
+ logger.warn('ResumableSSE', 'Could not refresh settled generation history', {
+ conversationId: settledConversationId,
+ error,
+ });
+ }
+
+ if (conversationLookup === 'not_found' && startedAsNewConvo) {
+ /** A failed/early-aborted first turn can settle without ever
+ * creating a conversation. Never mint A's optimistic rows here:
+ * prune any partial cache and return to `/c/new` instead of
+ * leaving a phantom chat/URL. */
+ removeConvoFromAllQueries(queryClient, settledConversationId);
+ queryClient.removeQueries({
+ queryKey: [QueryKeys.conversation, settledConversationId],
+ });
+ queryClient.removeQueries({
+ queryKey: [QueryKeys.messages, settledConversationId],
+ });
+ newConversation?.({
+ template: { conversationId: String(Constants.NEW_CONVO) },
+ });
+ }
+ if (conversationLookup === 'inconclusive') {
+ /** A network/5xx lookup cannot prove the conversation is absent.
+ * Preserve its optimistic/cache state for a later refetch and do
+ * not navigate or mint a fresh `/c/new` conversation. */
+ clearDrainAfterAbort(settledConversationId, submission.expectedPredecessorCreatedAt);
+ }
+ clearStepMaps();
+ setIsSubmitting(false);
+ setShowStopButton(false);
+ setRunEnd({
+ conversationId: settledConversationId,
+ outcome: conversationLookup === 'inconclusive' ? 'error' : 'aborted',
+ startedAsNewConvo,
+ endedAt: Date.now(),
+ });
+ setSubmission(null);
+ return;
+ }
+ if (startResult.status !== 'stream') {
+ return;
+ }
+ const {
+ streamId: newStreamId,
+ resumed,
+ generationCreatedAt,
+ generationProtocolVersion,
+ } = startResult;
setStreamId(newStreamId);
// Optimistically add to active jobs
addActiveJob(newStreamId);
@@ -1892,7 +3831,14 @@ export default function useResumableSSE(
submissionRef.current = streamSubmission;
// A deduped retry (status: 'resumed') attaches to an already-running stream, so
// subscribe with resume=true to replay its state instead of only live events.
- subscribeToStream(newStreamId, streamSubmission, resumed);
+ subscribeToStream(
+ newStreamId,
+ streamSubmission,
+ resumed,
+ generationCreatedAt,
+ generationProtocolVersion,
+ signal,
+ );
} else {
logger.error('ResumableSSE', 'Failed to get streamId from startGeneration');
}
@@ -1905,6 +3851,7 @@ export default function useResumableSSE(
* LIVE frame ids at cleanup time (satisfies react-hooks/exhaustive-deps
* without changing behavior). */
const activityLabelRetryFrames = activityLabelRetryFramesRef.current;
+ const steerRetryFrames = steerRetryFramesRef.current;
return () => {
logger.log('ResumableSSE', 'Cleanup - closing SSE, resetting UI state');
startController.abort();
@@ -1923,10 +3870,10 @@ export default function useResumableSSE(
cancelAnimationFrame(frameId);
}
activityLabelRetryFrames.clear();
- if (steerRetryRef.current != null) {
- cancelAnimationFrame(steerRetryRef.current);
- steerRetryRef.current = null;
+ for (const frameId of steerRetryFrames) {
+ cancelAnimationFrame(frameId);
}
+ steerRetryFrames.clear();
// Reset reconnect counter before closing (so abort handler doesn't think we're reconnecting)
reconnectAttemptRef.current = 0;
if (sseRef.current) {
@@ -1935,9 +3882,14 @@ export default function useResumableSSE(
}
// Clear handler maps to prevent memory leaks and stale state
clearStepMaps();
- // Reset UI state on cleanup - useResumeOnLoad will restore if needed
- setIsSubmitting(false);
- setShowStopButton(false);
+ // Reset UI state on ordinary cleanup. A generation handoff already
+ // cached the replacement and must remain non-idle until resume-on-load
+ // installs its epoch-fenced submission.
+ if (!replacementHandoffRef.current) {
+ setIsSubmitting(false);
+ setShowStopButton(false);
+ }
+ replacementHandoffRef.current = false;
};
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [submission]);
diff --git a/client/src/hooks/SSE/useResumeOnLoad.ts b/client/src/hooks/SSE/useResumeOnLoad.ts
index f0df858e32..74a6df54ec 100644
--- a/client/src/hooks/SSE/useResumeOnLoad.ts
+++ b/client/src/hooks/SSE/useResumeOnLoad.ts
@@ -2,13 +2,20 @@ import { useEffect, useRef } from 'react';
import { useSetRecoilState, useRecoilValue, useRecoilCallback } from 'recoil';
import { Constants, tMessageSchema, isAssistantsEndpoint } from 'librechat-data-provider';
import type { TMessage, TConversation, TSubmission, Agents } from 'librechat-data-provider';
+import type { GenerationProtocolVersion } from '~/data-provider/SSE/protocol';
import type { StreamStatusResponse } from '~/data-provider';
import {
dedupeSteersById,
+ appendAppliedSteerIds,
+ collectAppliedSteerIds,
applyPendingAction,
carriedSteerContext,
getBranchSiblingIndexesForTarget,
} from '~/utils';
+import {
+ getGenerationProtocolVersion,
+ supportsGenerationProtocolV2,
+} from '~/data-provider/SSE/protocol';
import useSteerConvert from '~/hooks/Chat/useSteerConvert';
import { useStreamStatus } from '~/data-provider';
import store from '~/store';
@@ -97,6 +104,8 @@ function buildSubmissionFromResumeState(
streamId: string,
messages: TMessage[],
conversationId: string,
+ generationCreatedAt?: number,
+ generationProtocolVersion: GenerationProtocolVersion = 1,
): TSubmission {
const userMessageData = resumeState.userMessage;
const responseMessageId =
@@ -107,12 +116,15 @@ function buildSubmissionFromResumeState(
(m) => m.isCreatedByUser && m.messageId === userMessageData?.messageId,
);
- // Try to find existing response message in the messages array (from database)
- const existingResponseMessage = messages.find(
- (m) =>
- !m.isCreatedByUser &&
- (m.messageId === responseMessageId || m.parentMessageId === userMessageData?.messageId),
- );
+ // Try to find existing response message in the messages array (from database).
+ // Regeneration can expose the in-flight placeholder id with trailing underscores
+ // while the persisted sibling uses the unpadded id. Prefer both exact identities
+ // before falling back to the shared parent, where several branch siblings can match.
+ const unpaddedResponseMessageId = responseMessageId.replace(/_+$/, '');
+ const existingResponseMessage =
+ messages.find((m) => !m.isCreatedByUser && m.messageId === responseMessageId) ??
+ messages.find((m) => !m.isCreatedByUser && m.messageId === unpaddedResponseMessageId) ??
+ messages.find((m) => !m.isCreatedByUser && m.parentMessageId === userMessageData?.messageId);
// Create or use existing user message
const userMessage: TMessage =
@@ -185,7 +197,13 @@ function buildSubmissionFromResumeState(
endpointOption: {},
// Signal to useResumableSSE to subscribe to existing stream instead of starting new
resumeStreamId: streamId,
- } as TSubmission & { resumeStreamId: string };
+ ...(generationCreatedAt != null && { resumeGenerationCreatedAt: generationCreatedAt }),
+ resumeGenerationProtocolVersion: generationProtocolVersion,
+ } as TSubmission & {
+ resumeStreamId: string;
+ resumeGenerationCreatedAt?: number;
+ resumeGenerationProtocolVersion: GenerationProtocolVersion;
+ };
}
/**
@@ -214,6 +232,12 @@ export default function useResumeOnLoad(
const resumableEnabled = !isAssistantsEndpoint(actualEndpoint);
// Track conversations we've already processed (either resumed or skipped)
const processedConvoRef = useRef<string | null>(null);
+ /** `generationHandoff` lives in the React Query snapshot until a later
+ * status refetch. Remember the exact epoch already consumed so clearing the
+ * replacement submission on FINAL cannot re-install that stale snapshot and
+ * enter a resume→404→resume loop. A genuinely newer handoff has a different
+ * createdAt key and remains eligible. */
+ const consumedHandoffGenerationRef = useRef<string | null>(null);
const restoreResumeBranch = useRecoilCallback(
({ set }) =>
(resumeState: Agents.ResumeState, messages: TMessage[], activeConversationId: string) => {
@@ -237,7 +261,20 @@ export default function useResumeOnLoad(
const restoreSteerChips = useRecoilCallback(
({ set }) =>
- (activeConversationId: string, pendingSteers: Agents.ResumeState['pendingSteers']) => {
+ (
+ activeConversationId: string,
+ pendingSteers: Agents.ResumeState['pendingSteers'],
+ generationCreatedAt?: number,
+ generationProtocolVersion: GenerationProtocolVersion = 1,
+ ) => {
+ const acceptedClientIds = (pendingSteers ?? []).flatMap((steer) =>
+ steer.clientSteerId ? [steer.clientSteerId] : [],
+ );
+ if (acceptedClientIds.length > 0) {
+ set(store.acceptedSteerClientIdsByConvoId(activeConversationId), (prev) =>
+ appendAppliedSteerIds(prev, acceptedClientIds),
+ );
+ }
// Always reconcile against the server's still-queued list (mirrors the
// sync-path re-seed in useResumableSSE): a steer applied while this
// client was away is absent here (its inline part rides
@@ -245,23 +282,83 @@ export default function useResumeOnLoad(
// pending chips, not leave them stranded beside the applied part.
set(store.pendingSteersByConvoId(activeConversationId), (prev) => {
const chipById = new Map(prev.map((chip) => [chip.steerId, chip]));
+ const claimedIds = new Set(
+ (pendingSteers ?? []).flatMap((steer) =>
+ steer.clientSteerId ? [steer.steerId, steer.clientSteerId] : [steer.steerId],
+ ),
+ );
return [
- ...(pendingSteers ?? []).map((steer) => ({
- steerId: steer.steerId,
- text: steer.text,
- status: 'pending' as const,
- createdAt: steer.createdAt ?? Date.now(),
- ...(steer.files && steer.files.length > 0 && { files: steer.files }),
- ...(steer.preempt === true && { preempt: true }),
- ...carriedSteerContext(chipById.get(steer.steerId)),
- })),
- ...prev.filter((steer) => steer.status === 'failed'),
+ ...(pendingSteers ?? []).map((steer) => {
+ const localChip =
+ chipById.get(steer.steerId) ??
+ (steer.clientSteerId ? chipById.get(steer.clientSteerId) : undefined);
+ const keepLocalPreempt =
+ (localChip?.preemptRevision ?? 0) > (steer.preemptRevision ?? 0);
+ const chipGenerationCreatedAt = generationCreatedAt ?? localChip?.generationCreatedAt;
+ return {
+ steerId: steer.steerId,
+ ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
+ text: steer.text,
+ status: 'pending' as const,
+ createdAt: steer.createdAt ?? Date.now(),
+ ...(steer.files && steer.files.length > 0 && { files: steer.files }),
+ ...((keepLocalPreempt ? localChip?.preempt : steer.preempt) === true && {
+ preempt: true,
+ }),
+ ...((keepLocalPreempt ? localChip?.preemptRevision : steer.preemptRevision) !=
+ null && {
+ preemptRevision: keepLocalPreempt
+ ? localChip?.preemptRevision
+ : steer.preemptRevision,
+ }),
+ ...(localChip?.queuedOrigin && { queuedOrigin: localChip.queuedOrigin }),
+ ...(chipGenerationCreatedAt != null && {
+ generationCreatedAt: chipGenerationCreatedAt,
+ }),
+ generationProtocolVersion,
+ ...carriedSteerContext(localChip),
+ };
+ }),
+ ...prev.filter((steer) => steer.status === 'failed' && !claimedIds.has(steer.steerId)),
];
});
},
[],
);
+ const settleAppliedSteerParts = useRecoilCallback(
+ ({ set }) =>
+ (activeConversationId: string, values: unknown[] | undefined) => {
+ const ids = collectAppliedSteerIds(values);
+ if (ids.length === 0) {
+ return;
+ }
+ const settled = new Set(ids);
+ set(store.appliedSteerIdsByConvoId(activeConversationId), (prev) =>
+ appendAppliedSteerIds(prev, ids),
+ );
+ set(store.pendingSteersByConvoId(activeConversationId), (prev) =>
+ prev.filter((steer) => !settled.has(steer.steerId)),
+ );
+ },
+ [],
+ );
+ const setActiveGenerationCreatedAt = useRecoilCallback(
+ ({ set }) =>
+ (
+ activeConversationId: string,
+ createdAt: number,
+ generationProtocolVersion: GenerationProtocolVersion,
+ ) => {
+ set(store.activeGenerationCreatedAtByConvoId(activeConversationId), createdAt);
+ set(
+ store.activeGenerationProtocolVersionByConvoId(activeConversationId),
+ generationProtocolVersion,
+ );
+ },
+ [],
+ );
+
// Check for active stream when conversation changes
const submissionConvoId = currentSubmission?.conversation?.conversationId;
const loadedMessages = messagesLoaded ? getMessages() : undefined;
@@ -340,6 +437,28 @@ export default function useResumeOnLoad(
return;
}
+ /** useResumableSSE detected that this conversation-scoped stream now
+ * belongs to a newer generation. It cleared the stale submission and
+ * cached the replacement snapshot; allow the same conversation to be
+ * processed again so this epoch becomes the active resume submission. */
+ const generationProtocolVersion = getGenerationProtocolVersion(streamStatus);
+ const isGenerationProtocolV2 = supportsGenerationProtocolV2(streamStatus);
+ const handoffGenerationKey =
+ isGenerationProtocolV2 &&
+ streamStatus.generationHandoff === true &&
+ streamStatus.createdAt != null
+ ? `${conversationId}:${streamStatus.createdAt}`
+ : null;
+ if (
+ currentSubmission == null &&
+ handoffGenerationKey != null &&
+ consumedHandoffGenerationRef.current !== handoffGenerationKey &&
+ streamStatus.active &&
+ processedConvoRef.current === conversationId
+ ) {
+ processedConvoRef.current = null;
+ }
+
if (
streamStatus.active &&
streamStatus.streamId &&
@@ -374,17 +493,30 @@ export default function useResumeOnLoad(
streamStatus.resumeState?.pendingSteers,
);
if (conversationId && leftoverSteers.length > 0) {
- convertSteersToQueued(conversationId, leftoverSteers);
+ convertSteersToQueued(conversationId, leftoverSteers, {
+ generationProtocolVersion,
+ });
}
// The run is terminal, so any remaining local pending chip is stale:
// its steer either applied (inline part in the saved message) or rode
// `unrecoveredSteers` above — same empty-list reconcile as the resume path.
+ settleAppliedSteerParts(conversationId, getMessages());
restoreSteerChips(conversationId, undefined);
processedConvoRef.current = conversationId;
return;
}
processedConvoRef.current = conversationId;
+ if (handoffGenerationKey != null) {
+ consumedHandoffGenerationRef.current = handoffGenerationKey;
+ }
+ if (streamStatus.createdAt != null) {
+ setActiveGenerationCreatedAt(
+ conversationId,
+ streamStatus.createdAt,
+ generationProtocolVersion,
+ );
+ }
console.log('[ResumeOnLoad] Found active job, creating submission...', {
streamId: streamStatus.streamId,
@@ -397,12 +529,26 @@ export default function useResumeOnLoad(
// Build submission from resume state if available
if (streamStatus.resumeState) {
restoreResumeBranch(streamStatus.resumeState, messages, conversationId);
- restoreSteerChips(conversationId, streamStatus.resumeState.pendingSteers);
+ restoreSteerChips(
+ conversationId,
+ streamStatus.resumeState.pendingSteers,
+ streamStatus.createdAt,
+ generationProtocolVersion,
+ );
+ // Restore the server's pending snapshot before settling inline steer
+ // parts. A steer present in both views was applied during the snapshot
+ // boundary and must finish absent, never resurrected as a chip.
+ settleAppliedSteerParts(conversationId, [
+ ...messages,
+ ...(streamStatus.resumeState.aggregatedContent ?? []),
+ ]);
const submission = buildSubmissionFromResumeState(
streamStatus.resumeState,
streamStatus.streamId,
messages,
conversationId,
+ streamStatus.createdAt,
+ generationProtocolVersion,
);
setSubmission(submission);
} else {
@@ -424,7 +570,15 @@ export default function useResumeOnLoad(
endpointOption: {},
// Signal to useResumableSSE to subscribe to existing stream instead of starting new
resumeStreamId: streamStatus.streamId,
- } as TSubmission & { resumeStreamId: string };
+ ...(streamStatus.createdAt != null && {
+ resumeGenerationCreatedAt: streamStatus.createdAt,
+ }),
+ resumeGenerationProtocolVersion: generationProtocolVersion,
+ } as TSubmission & {
+ resumeStreamId: string;
+ resumeGenerationCreatedAt?: number;
+ resumeGenerationProtocolVersion: GenerationProtocolVersion;
+ };
setSubmission(submission);
}
}, [
@@ -442,7 +596,9 @@ export default function useResumeOnLoad(
setSubmission,
restoreResumeBranch,
restoreSteerChips,
+ settleAppliedSteerParts,
convertSteersToQueued,
+ setActiveGenerationCreatedAt,
]);
// Reset processedConvoRef when conversation changes to allow re-checking
@@ -454,6 +610,7 @@ export default function useResumeOnLoad(
new: conversationId,
});
processedConvoRef.current = null;
+ consumedHandoffGenerationRef.current = null;
}
}, [conversationId]);
}
diff --git a/client/src/hooks/useKeyboardShortcuts.spec.tsx b/client/src/hooks/useKeyboardShortcuts.spec.tsx
index 9826962b18..20bc2484ea 100644
--- a/client/src/hooks/useKeyboardShortcuts.spec.tsx
+++ b/client/src/hooks/useKeyboardShortcuts.spec.tsx
@@ -51,11 +51,16 @@ function Harness() {
);
}
-function renderHarness(conversation?: TConversation, route = '/c/test-convo') {
+function renderHarness(
+ conversation?: TConversation,
+ route = '/c/test-convo',
+ initialize?: (snapshot: MutableSnapshot) => void,
+) {
const initializeState = (snapshot: MutableSnapshot) => {
if (conversation) {
snapshot.set(store.conversationByIndex(0), conversation);
}
+ initialize?.(snapshot);
};
return render(<Harness />, {
wrapper: ({ children }: { children: ReactNode }) => (
@@ -96,6 +101,18 @@ function appendResponseCopyButton(onClick: () => void) {
document.body.appendChild(button);
}
+function appendEscalationButton(surface: 'bubble' | 'queued', active = false) {
+ const onClick = jest.fn();
+ const button = document.createElement('button');
+ button.dataset.escalateSteer = surface;
+ if (active) {
+ button.dataset.escalateSteerActive = 'true';
+ }
+ button.addEventListener('click', onClick);
+ document.body.appendChild(button);
+ return { button, onClick };
+}
+
describe('binding resolution helpers', () => {
it('falls back to the default binding when there is no override', () => {
const binding = effectiveBinding('newChat');
@@ -122,6 +139,38 @@ describe('binding resolution helpers', () => {
expect(getShortcutDisplay('newChat')).toBe('');
});
+ it('keeps a persisted custom chord when a new default later claims the same keys', () => {
+ window.localStorage.setItem(
+ STORAGE_KEY,
+ JSON.stringify({
+ toggleSidebar: { mac: 'Meta+Shift+.', other: 'Control+Shift+.' },
+ }),
+ );
+
+ expect(effectiveBinding('toggleSidebar')).toMatchObject({
+ ctrl: true,
+ shift: true,
+ key: '.',
+ });
+ expect(effectiveBinding('escalateSteer')).toBeNull();
+ expect(getShortcutDisplay('escalateSteer')).toBe('');
+ expect(getShortcutAriaKey('escalateSteer')).toBe('');
+ });
+
+ it("does not treat an other-platform edit's copied default as a custom claim", () => {
+ const overrides = {
+ newChat: { mac: 'Meta+Alt+N', other: 'Control+Shift+O' },
+ toggleSidebar: { mac: 'Meta+Shift+O', other: 'Control+Shift+O' },
+ };
+
+ expect(effectiveBinding('toggleSidebar', overrides)).toMatchObject({
+ ctrl: true,
+ shift: true,
+ key: 'O',
+ });
+ expect(effectiveBinding('newChat', overrides)).toBeNull();
+ });
+
it('detects whether an override diverges from the default', () => {
expect(isOverridden('newChat', undefined)).toBe(false);
expect(isOverridden('newChat', { mac: 'Meta+Shift+O', other: 'Control+Shift+O' })).toBe(false);
@@ -198,6 +247,54 @@ describe('global shortcut dispatch', () => {
expect(event.defaultPrevented).toBe(true);
expect(document.activeElement).toBe(textarea);
});
+
+ it('dispatches a persisted custom owner instead of a colliding new default', () => {
+ const overrides = {
+ toggleSidebar: { mac: 'Meta+Shift+.', other: 'Control+Shift+.' },
+ };
+ const { getByTestId } = renderHarness(undefined, '/c/test-convo', (snapshot) => {
+ snapshot.set(store.customShortcuts, overrides);
+ });
+ const before = getByTestId('sidebar').textContent;
+ const escalation = appendEscalationButton('bubble');
+
+ const event = dispatchKey({ key: '.', ctrlKey: true, shiftKey: true });
+
+ expect(event.defaultPrevented).toBe(true);
+ expect(getByTestId('sidebar').textContent).not.toBe(before);
+ expect(escalation.onClick).not.toHaveBeenCalled();
+ });
+
+ it('targets the active waiting row before falling back to the newest one', () => {
+ renderHarness();
+ const active = appendEscalationButton('bubble', true);
+ const newest = appendEscalationButton('bubble');
+
+ dispatchKey({ key: '.', ctrlKey: true, shiftKey: true });
+
+ expect(active.onClick).toHaveBeenCalledTimes(1);
+ expect(newest.onClick).not.toHaveBeenCalled();
+
+ active.button.removeAttribute('data-escalate-steer-active');
+ dispatchKey({ key: '.', ctrlKey: true, shiftKey: true });
+
+ expect(newest.onClick).toHaveBeenCalledTimes(1);
+ });
+
+ it('dispatches the escalation shortcut by physical key on a non-US layout', () => {
+ renderHarness();
+ const escalation = appendEscalationButton('bubble');
+
+ const event = dispatchKey({
+ key: ':',
+ code: 'Period',
+ ctrlKey: true,
+ shiftKey: true,
+ });
+
+ expect(escalation.onClick).toHaveBeenCalledTimes(1);
+ expect(event.defaultPrevented).toBe(true);
+ });
});
describe('clipboard shortcuts', () => {
diff --git a/client/src/hooks/useKeyboardShortcuts.ts b/client/src/hooks/useKeyboardShortcuts.ts
index 2b97f92702..290bc50b65 100644
--- a/client/src/hooks/useKeyboardShortcuts.ts
+++ b/client/src/hooks/useKeyboardShortcuts.ts
@@ -115,6 +115,14 @@ export const shortcutDefinitions = {
ariaMac: 'Meta+Shift+X',
ariaOther: 'Control+Shift+X',
},
+ escalateSteer: {
+ labelKey: 'com_ui_interrupt_steer_now',
+ groupKey: 'com_shortcut_group_chat',
+ displayMac: '⌘ ⇧ .',
+ displayOther: 'Ctrl+Shift+.',
+ ariaMac: 'Meta+Shift+.',
+ ariaOther: 'Control+Shift+.',
+ },
regenerateResponse: {
labelKey: 'com_shortcut_regenerate_response',
groupKey: 'com_shortcut_group_chat',
@@ -289,6 +297,7 @@ export const EDITING_ALLOWED_SHORTCUTS: ReadonlySet<ShortcutActionId> = new Set(
'focusSearch',
'showShortcuts',
'submitMessage',
+ 'escalateSteer',
]);
export type ShortcutAction = ShortcutDefinition & {
@@ -367,6 +376,70 @@ function defaultAria(actionId: ShortcutActionId): string {
return isMac ? def.ariaMac : def.ariaOther;
}
+/**
+ * Resolves one owner per chord, with persisted user choices ahead of defaults.
+ *
+ * This ordering matters when a release adds a new default chord that was
+ * previously free: an existing custom binding must keep working instead of
+ * being silently shadowed by the new action. Losers resolve to `null`, so the
+ * dispatcher, shortcut dialog, tooltips, and `aria-keyshortcuts` all describe
+ * the same effective ownership.
+ */
+export function resolveShortcutBindings(
+ overrides: Record<string, ShortcutOverride>,
+): Map<ShortcutActionId, ShortcutBinding | null> {
+ const resolved = new Map<ShortcutActionId, ShortcutBinding | null>();
+ const claimed = new Set<string>();
+ const explicit = new Set<ShortcutActionId>();
+
+ for (const id of shortcutActionIds) {
+ const override = overrides[id];
+ let platformValue: string | null | undefined;
+ if (override != null) {
+ platformValue = isMac ? override.mac : override.other;
+ }
+ /** Editing the other platform stores this platform's default alongside it.
+ * That copied default is not a user claim here and must still yield to a
+ * genuinely custom binding on the current platform. */
+ if (platformValue === undefined || platformValue === defaultAria(id)) {
+ continue;
+ }
+ explicit.add(id);
+ const binding = parseBinding(platformValue);
+ if (binding == null) {
+ resolved.set(id, null);
+ continue;
+ }
+ const hash = bindingHash(binding);
+ if (claimed.has(hash)) {
+ resolved.set(id, null);
+ continue;
+ }
+ claimed.add(hash);
+ resolved.set(id, binding);
+ }
+
+ for (const id of shortcutActionIds) {
+ if (explicit.has(id)) {
+ continue;
+ }
+ const binding = parseBinding(defaultAria(id));
+ if (binding == null) {
+ resolved.set(id, null);
+ continue;
+ }
+ const hash = bindingHash(binding);
+ if (claimed.has(hash)) {
+ resolved.set(id, null);
+ continue;
+ }
+ claimed.add(hash);
+ resolved.set(id, binding);
+ }
+
+ return resolved;
+}
+
function readOverridesFromStorage(): Record<string, ShortcutOverride> {
if (typeof window === 'undefined') {
return {};
@@ -381,29 +454,12 @@ function readOverridesFromStorage(): Record<string, ShortcutOverride> {
}
}
-function effectiveBindingString(
- actionId: ShortcutActionId,
- overrides: Record<string, ShortcutOverride>,
-): string | null {
- const override = overrides[actionId];
- if (override) {
- const platformValue = isMac ? override.mac : override.other;
- if (platformValue === null) {
- return null;
- }
- if (typeof platformValue === 'string') {
- return platformValue;
- }
- }
- return defaultAria(actionId);
-}
-
export function effectiveBinding(
actionId: ShortcutActionId,
overrides?: Record<string, ShortcutOverride>,
): ShortcutBinding | null {
const map = overrides ?? readOverridesFromStorage();
- return parseBinding(effectiveBindingString(actionId, map));
+ return resolveShortcutBindings(map).get(actionId) ?? null;
}
export function getShortcutDisplay(actionId: ShortcutActionId): string {
@@ -551,6 +607,30 @@ export function useShortcutActions(): ShortcutAction[] {
[],
);
+ /** Escalate the newest waiting message to an interrupt by pressing its own
+ * visible arrow control, so the shortcut can never diverge from the
+ * button's semantics. A waiting steer bubble beats a queued follow-up (it
+ * is closer to the run); newest-last matches how both stacks append. */
+ const handleEscalateSteer = useCallback(() => {
+ const active = document.querySelector<HTMLButtonElement>('[data-escalate-steer-active="true"]');
+ if (clickTarget(active)) {
+ return true;
+ }
+
+ const pick = (surface: string) => {
+ const list = document.querySelectorAll<HTMLButtonElement>(
+ `[data-escalate-steer="${surface}"]`,
+ );
+ for (let i = list.length - 1; i >= 0; i--) {
+ if (!isUnavailableElement(list[i])) {
+ return list[i];
+ }
+ }
+ return null;
+ };
+ return clickTarget(pick('bubble') ?? pick('queued'));
+ }, []);
+
const handleEditLastMessage = useCallback(() => {
const userTurns = document.querySelectorAll('.user-turn');
if (userTurns.length === 0) {
@@ -750,6 +830,7 @@ export function useShortcutActions(): ShortcutAction[] {
focusSearch: handleFocusSearch,
openSettings: handleOpenSettings,
stopGenerating: handleStopGenerating,
+ escalateSteer: handleEscalateSteer,
regenerateResponse: handleRegenerateResponse,
editLastMessage: handleEditLastMessage,
copyLastCode: handleCopyLastCode,
@@ -779,6 +860,7 @@ export function useShortcutActions(): ShortcutAction[] {
handleUploadFile,
handleToggleSidebar,
handleOpenModelSelector,
+ handleEscalateSteer,
handleFocusSearch,
handleOpenSettings,
handleStopGenerating,
@@ -820,7 +902,7 @@ export function useShortcutDisplay(actionId?: ShortcutActionId): string {
const overrides = useRecoilValue(store.customShortcuts);
return useMemo(() => {
if (!actionId) return '';
- const binding = parseBinding(effectiveBindingString(actionId, overrides));
+ const binding = resolveShortcutBindings(overrides).get(actionId) ?? null;
return binding ? bindingDisplayString(binding, isMac) : '';
}, [actionId, overrides]);
}
@@ -829,7 +911,7 @@ export function useShortcutAriaKey(actionId?: ShortcutActionId): string | undefi
const overrides = useRecoilValue(store.customShortcuts);
return useMemo(() => {
if (!actionId) return undefined;
- const binding = parseBinding(effectiveBindingString(actionId, overrides));
+ const binding = resolveShortcutBindings(overrides).get(actionId) ?? null;
return binding ? (bindingToString(binding) ?? undefined) : undefined;
}, [actionId, overrides]);
}
@@ -856,12 +938,14 @@ export function useShortcutBindings(): {
} {
const [overrides, setOverrides] = useRecoilState(store.customShortcuts);
+ const resolvedBindings = useMemo(() => resolveShortcutBindings(overrides), [overrides]);
+
const bindings = useMemo<ShortcutBindingInfo[]>(
() =>
shortcutActionIds.map((id) => {
const def = shortcutDefinitions[id];
const override = overrides[id];
- const binding = parseBinding(effectiveBindingString(id, overrides));
+ const binding = resolvedBindings.get(id) ?? null;
return {
id,
binding,
@@ -870,7 +954,7 @@ export function useShortcutBindings(): {
labelKey: def.labelKey,
};
}),
- [overrides],
+ [overrides, resolvedBindings],
);
const bindingMap = useMemo<Map<string, ShortcutActionId>>(() => {
@@ -912,18 +996,6 @@ export function useShortcutBindings(): {
if (!prev[id]) return prev;
const next = { ...prev };
delete next[id];
-
- const restored = parseBinding(effectiveBindingString(id, next));
- if (restored) {
- const restoredHash = bindingHash(restored);
- const platformKey: keyof ShortcutOverride = isMac ? 'mac' : 'other';
- for (const otherId of Object.keys(next) as ShortcutActionId[]) {
- const otherBinding = parseBinding(effectiveBindingString(otherId, next));
- if (otherBinding && bindingHash(otherBinding) === restoredHash) {
- next[otherId] = { ...next[otherId], [platformKey]: null };
- }
- }
- }
return next;
});
},
@@ -944,16 +1016,18 @@ export default function useKeyboardShortcuts() {
const actionMap = useMemo(() => new Map(actions.map((action) => [action.id, action])), [actions]);
+ const resolvedBindings = useMemo(() => resolveShortcutBindings(overrides), [overrides]);
+
const bindingMap = useMemo<Map<string, ShortcutActionId>>(() => {
const map = new Map<string, ShortcutActionId>();
for (const id of shortcutActionIds) {
- const binding = parseBinding(effectiveBindingString(id, overrides));
+ const binding = resolvedBindings.get(id) ?? null;
if (binding) {
map.set(bindingHash(binding), id);
}
}
return map;
- }, [overrides]);
+ }, [resolvedBindings]);
const handler = useCallback(
(e: KeyboardEvent) => {
diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json
index a666e4db13..80ad8ea141 100644
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@@ -821,6 +821,7 @@
"com_ui_all_projects": "All projects",
"com_ui_all_proper": "All",
"com_ui_all_turns": "All turns",
+ "com_ui_always_interrupt": "Always interrupt instead",
"com_ui_analyzing": "Analyzing",
"com_ui_analyzing_finished": "Finished analyzing",
"com_ui_api_key": "API Key",
@@ -1299,6 +1300,7 @@
"com_ui_interrupt_steer": "Interrupt & steer",
"com_ui_interrupt_steer_button": "Interrupt and steer the response",
"com_ui_interrupt_steer_desc": "Stops writing now and keeps what's written",
+ "com_ui_interrupt_steer_now": "Interrupt & steer now",
"com_ui_invalid_json": "Invalid JSON",
"com_ui_invocation_auto": "Auto",
"com_ui_invocation_auto_info": "The skill is automatically applied by the agent when relevant to the conversation",
@@ -1530,6 +1532,7 @@
"com_ui_pin_error": "Failed to pin conversation",
"com_ui_pinned": "Pinned",
"com_ui_plus_n_more": "+{{0}} more",
+ "com_ui_preferences": "Preferences",
"com_ui_preferences_updated": "Preferences updated successfully",
"com_ui_prev": "Prev",
"com_ui_prev_result": "Previous result",
@@ -1920,17 +1923,21 @@
"com_ui_status_prefix": "Status:",
"com_ui_steer": "Steer",
"com_ui_steer_already_applied": "That steering message already reached the agent, so it was left in the response",
+ "com_ui_steer_arm_unconfirmed": "Couldn't confirm whether the interrupt was armed. The steering message may interrupt or may wait for the next tool step",
+ "com_ui_steer_arm_lost_race": "The interrupt could not be armed. The steering message may already have moved on or may still be waiting for the next tool step",
"com_ui_steer_cancel": "Cancel steering message",
"com_ui_steer_cancel_failed": "Could not cancel the steering message — it may still reach the agent",
+ "com_ui_steer_delivery_unconfirmed": "Delivery unconfirmed",
"com_ui_steer_edit_queued": "Your composer already has a draft, so that steering message was queued for after the response instead",
"com_ui_steer_failed": "Steering failed",
"com_ui_steer_in_flight": "Steering",
"com_ui_steer_in_flight_preempt": "Interrupting",
"com_ui_steer_interrupts_default": "Steering interrupts generation",
"com_ui_steer_interrupts_default_info": "When on, Enter stops the response at the next safe point instead of waiting for the agent's next tool step. Either way the partial answer is kept and the response continues.",
+ "com_ui_steer_interrupts_enable_info": "Switches the during-run default to steering and stops the response at the next safe point. The partial answer is kept and the response continues.",
"com_ui_steer_paused_queued": "The agent is waiting for your review — your message was queued instead",
+ "com_ui_steer_preempt_unsupported": "This deployment can't interrupt mid-response, so that steering message stays queued for the next tool step",
"com_ui_steer_retry": "Retry steering",
- "com_ui_steer_run_ended_queued": "The response ended, so that steering message is queued as a follow-up",
"com_ui_steer_send": "Steer the current response",
"com_ui_steered_info": "You added this message while the response was generating, so it was inserted into the response at this point.",
"com_ui_stop": "Stop",
@@ -2085,6 +2092,7 @@
"com_ui_verify": "Verify",
"com_ui_version_var": "Version {{0}}",
"com_ui_versions": "Versions",
+ "com_ui_wait_for_tool_steps": "Wait for tool steps instead",
"com_ui_via_server": "in {{0}}",
"com_ui_view_memory": "View Memory",
"com_ui_web_search": "Web Search",
diff --git a/client/src/store/families.ts b/client/src/store/families.ts
index 63bc32dda2..18741a4cd8 100644
--- a/client/src/store/families.ts
+++ b/client/src/store/families.ts
@@ -18,6 +18,7 @@ import type {
TMessage,
TPreset,
} from 'librechat-data-provider';
+import type { GenerationProtocolVersion } from '~/data-provider/SSE/protocol';
import type { TOptionSettings, ExtendedFile } from '~/common';
import {
clearModelForNonEphemeralAgent,
@@ -316,8 +317,16 @@ const pendingQuotesByConvoId = atomFamily<string[], string>({
*/
export type PendingSteer = {
steerId: string;
+ /** Optimistic id echoed by server state when SYNC beats the POST callback. */
+ clientSteerId?: string;
text: string;
status: 'sending' | 'pending' | 'failed';
+ /** The transport failed without a definitive server rejection. The durable
+ * enqueue may have committed. Same-id Retry is safe only under protocol v2;
+ * edit/queue/remove stay hidden until ownership is resolved. */
+ deliveryUncertain?: boolean;
+ /** Protocol selected for the generation that owns this attempt. */
+ generationProtocolVersion?: GenerationProtocolVersion;
createdAt: number;
/** Attachments steered with the message (refs; already uploaded). */
files?: TMessage['files'];
@@ -330,6 +339,15 @@ export type PendingSteer = {
* wait for a tool step. Labelling only — the server owns the behaviour and
* echoes what it actually armed. */
preempt?: boolean;
+ /** Monotonic server revision; delayed ACKs cannot undo SSE corrections. */
+ preemptRevision?: number;
+ /** Exact server generation this steer belongs to. Conversation ids are
+ * reused by later turns, so retries/arm/cancel must retain this epoch rather
+ * than mutating whatever generation currently occupies the conversation. */
+ generationCreatedAt?: number;
+ /** Exact client queue identity/order to restore if this accepted steer is
+ * returned as a terminal leftover before injection. */
+ queuedOrigin?: QueuedMessageOrigin;
};
/**
@@ -351,6 +369,14 @@ export type QueuedMessage = {
id: string;
text: string;
createdAt: number;
+ /** Stable only for this queued recovery attempt and its transport retries.
+ * A failed generation re-converts the durable source with a fresh key. */
+ clientRequestId?: string;
+ /** Correlation used only to durably dismiss/reclaim the parked source. */
+ recoveryClientSteerId?: string;
+ recoverySteerId?: string;
+ /** Generation observed before this queued follow-up became eligible. */
+ expectedPredecessorCreatedAt?: number;
files?: TMessage['files'];
/** Quote chips consumed from the composer at enqueue time; passed to `ask`
* as `overrideQuotes` on drain so they pair with THIS message. */
@@ -363,6 +389,15 @@ export type QueuedMessage = {
priority?: boolean;
};
+/** Snapshot of a queued item's logical position while it is temporarily sent
+ * into a live run. Neighbour ids make restoration resilient to concurrent
+ * drains and sends without minting a replacement item. */
+export type QueuedMessageOrigin = {
+ item: QueuedMessage;
+ beforeIds: string[];
+ afterIds: string[];
+};
+
/**
* Per-conversation client-side queue of follow-up messages. Drained one per
* run completion by `useQueueDrain` (each dequeued message starts a normal
@@ -373,17 +408,6 @@ const queuedMessagesByConvoId = atomFamily<QueuedMessage[], string>({
default: [],
});
-/**
- * Run-end signals whose conversation was NOT active when they arrived —
- * parked here (keyed by conversation) so a later run finishing on the same
- * chat index cannot overwrite them; `useQueueDrain` consumes the parked
- * signal when the user returns to that conversation.
- */
-const pendingRunEndByConvoId = atomFamily<RunEnd | null, string>({
- key: 'pendingRunEndByConvoId',
- default: null,
-});
-
/**
* One-shot run-termination signal written by the SSE final/error handlers and
* consumed (reset to null) by `useQueueDrain`. Keyed by chat index like
@@ -396,22 +420,86 @@ export type RunEnd = {
outcome: 'completed' | 'aborted' | 'error';
startedAsNewConvo?: boolean;
endedAt: number;
+ /** Exact terminal epoch whose idle transition may release one queued start. */
+ generationCreatedAt?: number;
/** Armed "Interrupt & send" flag traveling with a PARKED signal, so
* another run on the same pane can neither consume nor clear it. */
interruptArmed?: boolean;
};
-const runEndByIndex = atomFamily<RunEnd | null, string | number>({
+/** A pane can receive A's terminal frame after the user has navigated to and
+ * started B. Keep each terminal epoch until the queue drain has either parked
+ * or consumed it; a single replaceable slot loses A when B finishes first. */
+const runEndsByIndex = atomFamily<RunEnd[], string | number>({
+ key: 'runEndsByIndex',
+ default: [],
+});
+
+/** Preserve the original nullable one-shot API for stream writers while the
+ * backing state retains every not-yet-consumed terminal epoch. Writing null
+ * consumes only the visible (oldest) signal. */
+const runEndByIndex = selectorFamily<RunEnd | null, string | number>({
key: 'runEndByIndex',
- default: null,
+ get:
+ (index) =>
+ ({ get }) =>
+ get(runEndsByIndex(index))[0] ?? null,
+ set:
+ (index) =>
+ ({ set }, value) => {
+ if (value instanceof DefaultValue) {
+ set(runEndsByIndex(index), []);
+ return;
+ }
+ if (value == null) {
+ set(runEndsByIndex(index), (prev) => prev.slice(1));
+ return;
+ }
+ set(runEndsByIndex(index), (prev) => [...prev, value]);
+ },
+});
+
+/** Foreign terminal epochs are moved off the shared pane immediately. This
+ * per-conversation carrier is queued for the same reason as the pane carrier:
+ * successive epochs cannot overwrite one another while the chat is hidden. */
+const pendingRunEndsByConvoId = atomFamily<RunEnd[], string>({
+ key: 'pendingRunEndsByConvoId',
+ default: [],
+});
+
+const pendingRunEndByConvoId = selectorFamily<RunEnd | null, string>({
+ key: 'pendingRunEndByConvoId',
+ get:
+ (conversationId) =>
+ ({ get }) =>
+ get(pendingRunEndsByConvoId(conversationId))[0] ?? null,
+ set:
+ (conversationId) =>
+ ({ set }, value) => {
+ if (value instanceof DefaultValue) {
+ set(pendingRunEndsByConvoId(conversationId), []);
+ return;
+ }
+ if (value == null) {
+ set(pendingRunEndsByConvoId(conversationId), (prev) => prev.slice(1));
+ return;
+ }
+ set(pendingRunEndsByConvoId(conversationId), (prev) => [...prev, value]);
+ },
});
+export type DrainAfterAbort = {
+ conversationId: string;
+ generationCreatedAt: number;
+};
+
/**
* One-shot override armed by "interrupt & send": the next `aborted` run-end
- * drains the queue exactly once (a plain Stop press leaves queued chips for
- * manual send).
+ * for the exact conversation generation drains the queue exactly once (a
+ * plain Stop press leaves queued chips for manual send). `false` remains the
+ * clear value used by stream reconciliation paths.
*/
-const drainAfterAbortByIndex = atomFamily<boolean, string | number>({
+const drainAfterAbortByIndex = atomFamily<DrainAfterAbort | false, string | number>({
key: 'drainAfterAbortByIndex',
default: false,
});
@@ -429,6 +517,30 @@ const appliedSteerIdsByConvoId = atomFamily<string[], string>({
default: [],
});
+/** Optimistic ids the server has proven accepted via ACK or SYNC. Separate
+ * from `appliedSteerIdsByConvoId`: accepted-but-still-queued steers must not
+ * be suppressed by terminal conversion, but a late POST error must not
+ * resurrect them after Cancel/Edit/Convert removes the visible chip. */
+const acceptedSteerClientIdsByConvoId = atomFamily<string[], string>({
+ key: 'acceptedSteerClientIdsByConvoId',
+ default: [],
+});
+
+/** Server generation epoch currently attached for each conversation. Stream
+ * ids are conversation-scoped and reused by later turns; every mutation that
+ * can affect a live run carries this value as an optimistic concurrency fence. */
+const activeGenerationCreatedAtByConvoId = atomFamily<number | null, string>({
+ key: 'activeGenerationCreatedAtByConvoId',
+ default: null,
+});
+
+/** Negotiated behavior contract for the active generation. Missing echoes are
+ * legacy by definition, so the safe default is always v1. */
+const activeGenerationProtocolVersionByConvoId = atomFamily<GenerationProtocolVersion, string>({
+ key: 'activeGenerationProtocolVersionByConvoId',
+ default: 1,
+});
+
const globalAudioURLFamily = atomFamily<string | null, string | number | null>({
key: 'globalAudioURLByIndex',
default: null,
@@ -606,5 +718,8 @@ export default {
pendingRunEndByConvoId,
drainAfterAbortByIndex,
appliedSteerIdsByConvoId,
+ acceptedSteerClientIdsByConvoId,
+ activeGenerationCreatedAtByConvoId,
+ activeGenerationProtocolVersionByConvoId,
updateConversationSelector,
};
diff --git a/client/src/store/steer.ts b/client/src/store/steer.ts
index b3fd63c451..d0c152f483 100644
--- a/client/src/store/steer.ts
+++ b/client/src/store/steer.ts
@@ -11,3 +11,12 @@ import { atomFamily } from 'jotai/utils';
* each holds a single number per visited conversation.
*/
export const steerOverlayHeightFamily = atomFamily((_conversationId: string) => atom<number>(0));
+
+/**
+ * Set synchronously before a bubble's arm request and cleared on settlement.
+ * Purely a UX gate: with the atomic in-place arm, a double-arm is harmless
+ * server-side (the run seals once and drains the whole queue in order), but
+ * every escalation control advertises "one interrupt at a time" by disabling,
+ * and the chip-derived check cannot see an arm until its response lands.
+ */
+export const escalatingSteerFamily = atomFamily((_conversationId: string) => atom<boolean>(false));
diff --git a/client/src/utils/__tests__/steer.spec.ts b/client/src/utils/__tests__/steer.spec.ts
index 37c283d27d..46dc8924fa 100644
--- a/client/src/utils/__tests__/steer.spec.ts
+++ b/client/src/utils/__tests__/steer.spec.ts
@@ -7,6 +7,7 @@ import {
findSteerMessageIndex,
appendAppliedSteerIds,
resolveAbortSteerTarget,
+ insertQueuedOrigin,
} from '../steer';
const buildEvent = (overrides: Partial<TSteerAppliedEvent> = {}): TSteerAppliedEvent => ({
@@ -180,11 +181,40 @@ describe('appendAppliedSteerIds', () => {
expect(appendAppliedSteerIds(prev, [])).toBe(prev);
});
- it('caps the set at 100 ids, dropping the oldest', () => {
- const prev = Array.from({ length: 100 }, (_, i) => `id-${i}`);
+ it('retains both correlation ids for all 100 server receipt slots', () => {
+ const prev = Array.from({ length: 200 }, (_, i) => `id-${i}`);
const next = appendAppliedSteerIds(prev, ['id-new']);
- expect(next).toHaveLength(100);
+ expect(next).toHaveLength(200);
expect(next[0]).toBe('id-1');
expect(next[next.length - 1]).toBe('id-new');
});
});
+
+describe('insertQueuedOrigin', () => {
+ const staleOrigin = {
+ item: {
+ id: 'queued-c',
+ text: 'send C',
+ createdAt: 3,
+ expectedPredecessorCreatedAt: 1000,
+ },
+ beforeIds: [],
+ afterIds: ['queued-d'],
+ };
+
+ it('rebases an already-restored row so its next resend targets the terminal winner', () => {
+ const restored = insertQueuedOrigin(
+ [{ id: 'queued-d', text: 'send D', createdAt: 4 }],
+ staleOrigin,
+ );
+ const rebased = insertQueuedOrigin(restored, staleOrigin, 2000);
+
+ expect(rebased).toEqual([
+ expect.objectContaining({
+ id: 'queued-c',
+ expectedPredecessorCreatedAt: 2000,
+ }),
+ expect.objectContaining({ id: 'queued-d' }),
+ ]);
+ });
+});
diff --git a/client/src/utils/shortcuts.spec.ts b/client/src/utils/shortcuts.spec.ts
index bd2c8be200..1c6e03cf89 100644
--- a/client/src/utils/shortcuts.spec.ts
+++ b/client/src/utils/shortcuts.spec.ts
@@ -75,6 +75,16 @@ describe('bindingFromEvent', () => {
makeBinding({ meta: true, shift: true, key: 'Backspace' }),
);
});
+
+ it('uses the physical punctuation key on non-US keyboard layouts', () => {
+ const event = new KeyboardEvent('keydown', {
+ key: ':',
+ code: 'Period',
+ ctrlKey: true,
+ shiftKey: true,
+ });
+ expect(bindingFromEvent(event)).toEqual(makeBinding({ ctrl: true, shift: true, key: '.' }));
+ });
});
describe('parseBinding', () => {
diff --git a/client/src/utils/shortcuts.ts b/client/src/utils/shortcuts.ts
index c6ec53b724..f4078cbfa4 100644
--- a/client/src/utils/shortcuts.ts
+++ b/client/src/utils/shortcuts.ts
@@ -70,18 +70,42 @@ export function isModifierKey(key: string): boolean {
export type KeyChordSource = Pick<
KeyboardEvent,
'key' | 'altKey' | 'ctrlKey' | 'metaKey' | 'shiftKey'
->;
+> &
+ Partial<Pick<KeyboardEvent, 'code'>>;
+
+/**
+ * Punctuation shortcuts should follow the physical key advertised by the UI.
+ * `KeyboardEvent.key` reports the character produced by the active layout, so
+ * the physical Period key can report `:` (or another character) on a non-US
+ * layout even though the displayed/ARIA shortcut is still `.`. `code` is
+ * layout-independent and keeps those bindings usable without changing how
+ * letter shortcuts follow the user's chosen layout.
+ */
+const PUNCTUATION_KEY_BY_CODE: Readonly<Record<string, string>> = {
+ Backquote: '`',
+ Minus: '-',
+ Equal: '=',
+ BracketLeft: '[',
+ BracketRight: ']',
+ Backslash: '\\',
+ Semicolon: ';',
+ Quote: "'",
+ Comma: ',',
+ Period: '.',
+ Slash: '/',
+};
export function bindingFromEvent(e: KeyChordSource): ShortcutBinding | null {
if (isModifierKey(e.key)) {
return null;
}
+ const key = (e.code && PUNCTUATION_KEY_BY_CODE[e.code]) || e.key;
return {
meta: e.metaKey,
ctrl: e.ctrlKey,
alt: e.altKey,
shift: e.shiftKey,
- key: normalizeKey(e.key, e.shiftKey),
+ key: normalizeKey(key, e.shiftKey),
};
}
diff --git a/client/src/utils/steer.ts b/client/src/utils/steer.ts
index 329b3eecfc..d6c4ed5c43 100644
--- a/client/src/utils/steer.ts
+++ b/client/src/utils/steer.ts
@@ -5,6 +5,7 @@ import type {
TSteerAppliedEvent,
TMessageContentParts,
} from 'librechat-data-provider';
+import type { QueuedMessage, QueuedMessageOrigin } from '~/store/families';
type SteerPart = Extract<TMessageContentParts, { type: ContentTypes.STEER }>;
@@ -13,6 +14,48 @@ export function getSteerPart(part: TMessageContentParts | undefined): SteerPart
return part?.type === ContentTypes.STEER ? (part as SteerPart) : undefined;
}
+/** Server/client ids embedded in applied steer parts, whether passed a raw
+ * content array or message objects. Used during reconnect to retire a failed
+ * optimistic chip whose POST ACK was lost before the steer applied offline. */
+export function collectAppliedSteerIds(values: unknown[] | undefined): string[] {
+ if (!values) {
+ return [];
+ }
+ const ids = new Set<string>();
+ for (const value of values) {
+ if (value == null || typeof value !== 'object') {
+ continue;
+ }
+ const object = value as {
+ type?: unknown;
+ steerId?: unknown;
+ clientSteerId?: unknown;
+ content?: unknown;
+ };
+ const parts = Array.isArray(object.content) ? object.content : [object];
+ for (const part of parts) {
+ if (part == null || typeof part !== 'object') {
+ continue;
+ }
+ const candidate = part as {
+ type?: unknown;
+ steerId?: unknown;
+ clientSteerId?: unknown;
+ };
+ if (candidate.type !== ContentTypes.STEER) {
+ continue;
+ }
+ if (typeof candidate.steerId === 'string') {
+ ids.add(candidate.steerId);
+ }
+ if (typeof candidate.clientSteerId === 'string') {
+ ids.add(candidate.clientSteerId);
+ }
+ }
+ }
+ return [...ids];
+}
+
/**
* Places an injected steer part at its absolute content index on the target
* response message. The server reserved that slot (subsequent SDK events were
@@ -80,8 +123,10 @@ export function resolveAbortSteerTarget(params: { conversationId: string; resolv
}
/** Bounds the per-conversation applied-steer id set. A late 202 ACK can land
- * after the run's final event, so the set is capped rather than cleared. */
-const APPLIED_STEER_IDS_CAP = 100;
+ * after the run's final event, so the set is capped rather than cleared.
+ * Modern steers contribute both a server and client correlation id; retain
+ * two ids for every one of the server's 100 durable receipt slots. */
+const APPLIED_STEER_IDS_CAP = 200;
/**
* Appends steer ids to an applied-id set, deduped and capped. Returns the
@@ -110,6 +155,65 @@ export function carriedSteerContext(source?: SteerCarriedContext): SteerCarriedC
};
}
+/** Restore a temporarily removed queue item using surviving original
+ * neighbours first, then the queue's durable priority/time ordering. */
+export function insertQueuedOrigin(
+ queue: QueuedMessage[],
+ origin: QueuedMessageOrigin,
+ expectedPredecessorCreatedAt?: number,
+): QueuedMessage[] {
+ const rebasePredecessor = (item: QueuedMessage): QueuedMessage => {
+ if (expectedPredecessorCreatedAt === undefined) {
+ return item;
+ }
+ return item.expectedPredecessorCreatedAt === expectedPredecessorCreatedAt
+ ? item
+ : { ...item, expectedPredecessorCreatedAt };
+ };
+
+ const existingIndex = queue.findIndex((queued) => queued.id === origin.item.id);
+ if (existingIndex >= 0) {
+ const rebased = rebasePredecessor(queue[existingIndex]);
+ if (rebased === queue[existingIndex]) {
+ return queue;
+ }
+ const next = [...queue];
+ next[existingIndex] = rebased;
+ return next;
+ }
+ const restoredItem = rebasePredecessor(origin.item);
+ let index = -1;
+ for (const id of origin.afterIds) {
+ index = queue.findIndex((queued) => queued.id === id);
+ if (index >= 0) {
+ break;
+ }
+ }
+ if (index < 0) {
+ for (let i = origin.beforeIds.length - 1; i >= 0; i -= 1) {
+ const beforeIndex = queue.findIndex((queued) => queued.id === origin.beforeIds[i]);
+ if (beforeIndex >= 0) {
+ index = beforeIndex + 1;
+ break;
+ }
+ }
+ }
+ if (index < 0) {
+ index = queue.findIndex((queued) => {
+ const itemPriority = Number(restoredItem.priority === true);
+ const queuedPriority = Number(queued.priority === true);
+ return (
+ itemPriority > queuedPriority ||
+ (itemPriority === queuedPriority && restoredItem.createdAt < queued.createdAt)
+ );
+ });
+ if (index < 0) {
+ index = queue.length;
+ }
+ }
+ return [...queue.slice(0, index), restoredItem, ...queue.slice(index)];
+}
+
/** Merges steer lists into one id-deduped conversion batch (first wins). */
export function dedupeSteersById(...lists: Array<TPendingSteer[] | undefined>): TPendingSteer[] {
const seen = new Set<string>();
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js
index 12f34a53de..ba65529786 100644
--- a/e2e/setup/fake-model.js
+++ b/e2e/setup/fake-model.js
@@ -28,6 +28,7 @@ const REPLY_MARKER = 'E2E_REPLY:';
const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';
const ORDERED_REPLY_MARKER = 'E2E_ORDERED_REPLY:';
const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';
+const EMPTY_SLOW_REPLY_MARKER = 'E2E_EMPTY_SLOW_REPLY:';
const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:';
const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';
@@ -56,12 +57,14 @@ const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';
const STEER_TOOL_FINAL_TEXT = 'E2E steer tool reply done';
const STEER_SPLIT_FINAL_TEXT = 'E2E steer split reply done';
const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';
+const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued';
const ACTIVITY_FINAL_TEXT = 'E2E activity reply done';
const STEER_TOOL_NAME_PREFIX = 'remember_fact';
const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
const ORDERED_CHUNK_DELAY_MS = 2;
const ORDERED_REPLY_PIECES = 64;
const SLOW_REPLY_CHUNKS = 160;
+const EMPTY_SLOW_REPLY_CHUNKS = 600;
const RESUME_ICON_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_RESUME_ICON_CHUNK_DELAY_MS) || 60;
const RESUME_ICON_REPLY_CHUNKS = 240;
const CREATE_FILE_TOOL_NAME = 'create_file';
@@ -424,12 +427,16 @@ function replyResponses(text) {
const slowName = getMarkerValue(text, SLOW_REPLY_MARKER);
if (slowName) {
- const chunks = Array.from(
- { length: SLOW_REPLY_CHUNKS },
- (_, index) => `chunk-${String(index).padStart(3, '0')}`,
- ).join(' ');
+ return slowReplyResponses(slowName);
+ }
+
+ /** Keep a generation live after `created` without producing any content
+ * that the abort persistence filter accepts. The browser regression waits
+ * for the user row, then interrupts this whitespace-only stream. */
+ const emptySlowName = getMarkerValue(text, EMPTY_SLOW_REPLY_MARKER);
+ if (emptySlowName) {
return {
- responses: [`E2E slow reply ${slowName} ${chunks}`],
+ responses: [' '.repeat(EMPTY_SLOW_REPLY_CHUNKS)],
sleep: SLOW_CHUNK_DELAY_MS,
};
}
@@ -922,6 +929,29 @@ function steerEchoSuffix(messages) {
return `[steers-seen=${steerTexts.length}] ${steerTexts.join(' | ')}`.trim();
}
+/**
+ * Pure-text stream used by the no-tool preemption specs. A cooperative seal
+ * self-loops through the same model instance, so a distinct second response
+ * proves both that generation resumed and that the injected steer reached the
+ * model. Without a seal, only the slow first response is ever requested.
+ */
+function slowReplyResponses(label) {
+ let invocation = 0;
+ return {
+ responses: [''],
+ sleep: SLOW_CHUNK_DELAY_MS,
+ resolveInvocation: async (messages) => {
+ invocation += 1;
+ if (invocation === 1) {
+ return { response: `E2E slow reply ${label} ${slowChunkPayload()}` };
+ }
+ return {
+ response: `${SLOW_REPLY_CONTINUATION_TEXT} ${label} ${steerEchoSuffix(messages)}`,
+ };
+ },
+ };
+}
+
/** Slow word-chunk payload shared by the steer scenarios. */
function slowChunkPayload() {
return Array.from(
diff --git a/e2e/specs/mock/message-tree.spec.ts b/e2e/specs/mock/message-tree.spec.ts
index f66c41dafa..2f5b46ca1a 100644
--- a/e2e/specs/mock/message-tree.spec.ts
+++ b/e2e/specs/mock/message-tree.spec.ts
@@ -275,7 +275,7 @@ async function mockActiveOAuthResumeStream({
);
}
- await page.route(`**/api/agents/chat/status/${conversationId}`, (route) =>
+ await page.route(`**/api/agents/chat/status/${conversationId}**`, (route) =>
route.fulfill({
status: 200,
contentType: 'application/json',
diff --git a/e2e/specs/mock/steering-escalation.spec.ts b/e2e/specs/mock/steering-escalation.spec.ts
new file mode 100644
index 0000000000..29550026e3
--- /dev/null
+++ b/e2e/specs/mock/steering-escalation.spec.ts
@@ -0,0 +1,294 @@
+import { expect, test } from '@playwright/test';
+import type { Page, Response } from '@playwright/test';
+import {
+ MOCK_ENDPOINTS,
+ NEW_CHAT_PATH,
+ messagesView,
+ replyPrompt,
+ replyText,
+ selectMockEndpoint,
+ sendMessage,
+} from './helpers';
+
+/** Last chunk streamed by the fake model's slow replies (160 chunks, 0-indexed). */
+const SLOW_REPLY_LAST_CHUNK = 'chunk-159';
+const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued';
+
+const uniqueLabel = (prefix: string) =>
+ `${prefix}-${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
+
+const messageInput = (page: Page) => page.getByRole('textbox', { name: 'Message input' });
+const duringRunSendButton = (page: Page) => page.getByTestId('during-run-send-button');
+const queuedRows = (page: Page) => page.getByTestId('queued-message-row');
+const messageTurns = (page: Page) => messagesView(page).locator('.message-render');
+const inFlightSteers = (page: Page) => page.getByTestId('in-flight-steer');
+const appliedSteerParts = (page: Page) => messagesView(page).getByTestId('steer-part');
+
+function isSteerRequest(response: Response) {
+ return (
+ response.request().method() === 'POST' &&
+ new URL(response.url()).pathname === '/api/agents/chat/steer'
+ );
+}
+
+function isArmRequest(response: Response) {
+ return (
+ response.request().method() === 'POST' &&
+ new URL(response.url()).pathname === '/api/agents/chat/steer/arm'
+ );
+}
+
+/** Establish a real conversation with a fast first turn so during-run actions
+ * target a persisted conversation id instead of racing new-convo creation. */
+async function establishConversation(page: Page, label: string) {
+ const setup = await sendMessage(page, replyPrompt(label));
+ expect(setup.ok()).toBeTruthy();
+ await expect(messagesView(page).getByText(replyText(label))).toBeVisible({ timeout: 30000 });
+ await expect(page).toHaveURL(/\/c\/[0-9a-fA-F-]{36}$/, { timeout: 15000 });
+}
+
+/** Fill the composer mid-run: the during-run send button must take the
+ * send/stop slot (it becomes the form submit target for Enter). */
+async function typeDuringRun(page: Page, text: string) {
+ const input = messageInput(page);
+ await input.click();
+ await input.fill(text);
+ await expect(duringRunSendButton(page)).toBeVisible({ timeout: 5000 });
+}
+
+/** Proves the post-seal model invocation both ran and received the steer. */
+async function expectModelContinuation(page: Page, label: string, steerText: string) {
+ await expect(messagesView(page).getByText(`[steers-seen=1] ${steerText}`)).toBeVisible({
+ timeout: 30000,
+ });
+ await expect(
+ messagesView(page).getByText(`${SLOW_REPLY_CONTINUATION_TEXT} ${label}`),
+ ).toBeVisible({ timeout: 30000 });
+}
+
+/**
+ * Escalation of WAITING messages (PR: interrupt-steer escalation controls).
+ * `E2E_SLOW_REPLY` streams pure text with no tool boundary, so nothing here
+ * can inject the ordinary way — an in-thread steer part can only come from a
+ * mid-stream seal, which makes it the behavioral proof that escalation armed
+ * a real interrupt rather than relabelling a chip.
+ */
+test.describe('escalating waiting messages to an interrupt', () => {
+ /** `steerInterruptsByDefault` is a localStorage preference; the toggle test
+ * flips it, and a mid-test failure must not leak preempt-by-default into
+ * the rest of the serial suite. */
+ test.afterEach(async ({ page }) => {
+ await page.evaluate(() => window.localStorage.removeItem('steerInterruptsByDefault'));
+ });
+
+ test('queued row escalates as an interrupt: the message seals mid-stream instead of waiting for run end', async ({
+ page,
+ }) => {
+ test.setTimeout(150000);
+ const label = uniqueLabel('queue-escalate');
+ const queueText = `Escalated queued message ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ await establishConversation(page, `queue-escalate-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
+
+ // Queue the message (Ctrl/Cmd+Enter routes to the non-default action).
+ await typeDuringRun(page, queueText);
+ await messageInput(page).press('ControlOrMeta+Enter');
+ const row = queuedRows(page).filter({ hasText: queueText });
+ await expect(row).toBeVisible({ timeout: 10000 });
+
+ // Escalate it: the row's ZapOff button submits the queued text as an
+ // interrupt steer (a preempt-armed POST /chat/steer).
+ const [steerResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ row.getByTestId('queued-interrupt-now').click(),
+ ]);
+ expect(steerResponse.status()).toBe(202);
+ expect(((await steerResponse.json()) as { preempt?: boolean }).preempt).toBe(true);
+ await expect(row).toHaveCount(0, { timeout: 10000 });
+
+ // Injected in-thread with no tool boundary available — only a mid-stream
+ // seal can put a steer part here. Without escalation this message would
+ // have waited for run end and auto-sent as its own follow-up turn.
+ await expect(appliedSteerParts(page).filter({ hasText: queueText })).toHaveCount(1, {
+ timeout: 90000,
+ });
+ await expect(inFlightSteers(page)).toHaveCount(0);
+
+ // Sealed, not run to completion, and the pre-seal text survives.
+ await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible();
+ await expectModelContinuation(page, label, queueText);
+
+ // Stayed INSIDE the response: no auto-sent follow-up pair.
+ await expect(messageTurns(page)).toHaveCount(4);
+ await expect(queuedRows(page)).toHaveCount(0);
+ });
+
+ test('waiting steer bubble arms in place via POST /chat/steer/arm and seals mid-stream', async ({
+ page,
+ }) => {
+ test.setTimeout(150000);
+ const label = uniqueLabel('bubble-arm');
+ const steerText = `Armed waiting steer ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ await establishConversation(page, `bubble-arm-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
+
+ // An ORDINARY steer (plain Enter, preference off): with no tool boundary
+ // in this stream it stays acknowledged-and-waiting as a bubble.
+ await typeDuringRun(page, steerText);
+ const [steerResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(steerResponse.status()).toBe(202);
+ expect(((await steerResponse.json()) as { preempt?: boolean }).preempt).toBeFalsy();
+ const bubble = inFlightSteers(page).filter({ hasText: steerText });
+ await expect(bubble).toBeVisible({ timeout: 10000 });
+
+ // Escalate via the bubble's always-visible arrow control: ONE atomic
+ // in-place arm.
+ const [armResponse] = await Promise.all([
+ page.waitForResponse(isArmRequest, { timeout: 15000 }),
+ bubble.getByTestId('steer-escalate-now').click(),
+ ]);
+ expect(armResponse.status()).toBe(200);
+ expect(((await armResponse.json()) as { armed?: boolean }).armed).toBe(true);
+
+ // Relabelled IN PLACE: still exactly one bubble with the same text, and
+ // an interrupting steer no longer offers its escalation control.
+ await expect(inFlightSteers(page)).toHaveCount(1);
+ await expect(bubble.getByTestId('steer-escalate-now')).toHaveCount(0);
+
+ // The armed steer seals mid-stream and injects with no tool boundary.
+ await expect(appliedSteerParts(page).filter({ hasText: steerText })).toHaveCount(1, {
+ timeout: 90000,
+ });
+ await expect(inFlightSteers(page)).toHaveCount(0);
+ await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible();
+ await expectModelContinuation(page, label, steerText);
+ await expect(messageTurns(page)).toHaveCount(4);
+ });
+
+ test('always-interrupt toggle in a waiting row menu makes plain Enter preempt', async ({
+ page,
+ }) => {
+ test.setTimeout(150000);
+ const label = uniqueLabel('toggle');
+ const queueText = `Queued while toggling ${label}`;
+ const steerText = `Enter now interrupts ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ await establishConversation(page, `toggle-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
+
+ // A queued row hosts the overflow menu carrying the preference toggle.
+ await typeDuringRun(page, queueText);
+ await messageInput(page).press('ControlOrMeta+Enter');
+ const row = queuedRows(page).filter({ hasText: queueText });
+ await expect(row).toBeVisible({ timeout: 10000 });
+
+ // The toggle lives in the row menu's separated Preferences section.
+ await row.getByRole('button', { name: 'More options' }).click();
+ await expect(page.getByText('Preferences', { exact: true })).toBeVisible({ timeout: 5000 });
+ await page.getByRole('menuitem', { name: 'Always interrupt instead', exact: true }).click();
+
+ // Verify the preference flips while this row is guaranteed to remain
+ // parked. After the interrupt is submitted the run may seal and auto-drain
+ // the row before another locator action can observe it.
+ await row.getByRole('button', { name: 'More options' }).click();
+ await expect(
+ page.getByRole('menuitem', { name: 'Wait for tool steps instead', exact: true }),
+ ).toBeVisible({ timeout: 5000 });
+ await page.keyboard.press('Escape');
+
+ // The toggle is live for the SAME run: plain Enter now routes the default
+ // steer through the preempt path (the 202 carries the armed flag).
+ await typeDuringRun(page, steerText);
+ const [steerResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(steerResponse.status()).toBe(202);
+ expect(((await steerResponse.json()) as { preempt?: boolean }).preempt).toBe(true);
+
+ // And the seal proves it end to end: injected with no boundary available.
+ await expect(appliedSteerParts(page).filter({ hasText: steerText })).toHaveCount(1, {
+ timeout: 90000,
+ });
+ await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
+ await expectModelContinuation(page, label, steerText);
+ });
+
+ test('the dedicated shortcut escalates the newest waiting steer from the keyboard', async ({
+ page,
+ }) => {
+ test.setTimeout(150000);
+ const label = uniqueLabel('shortcut');
+ const steerText = `Shortcut-armed steer ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ await establishConversation(page, `shortcut-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_SLOW_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+ await expect(messagesView(page).getByText('chunk-010')).toBeVisible({ timeout: 15000 });
+
+ await typeDuringRun(page, steerText);
+ const [steerResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(steerResponse.status()).toBe(202);
+ await expect(inFlightSteers(page).filter({ hasText: steerText })).toBeVisible({
+ timeout: 10000,
+ });
+
+ // The dedicated command works from the composer (it is editing-allowed),
+ // pressing the newest waiting bubble's own arrow control.
+ const escalationButton = inFlightSteers(page)
+ .filter({ hasText: steerText })
+ .getByTestId('steer-escalate-now');
+ await escalationButton.focus();
+ await expect(escalationButton).toHaveAttribute(
+ 'aria-keyshortcuts',
+ /^(Meta|Control)\+Shift\+\.$/,
+ );
+ const resolvedAriaKey = await escalationButton.getAttribute('aria-keyshortcuts');
+ expect(resolvedAriaKey).toBeTruthy();
+ await messageInput(page).click();
+ const [armResponse] = await Promise.all([
+ page.waitForResponse(isArmRequest, { timeout: 15000 }),
+ // Follow the browser-visible binding rather than Playwright's
+ // host-platform ControlOrMeta mapping: the emulated UA may differ from
+ // the machine running the test.
+ page.keyboard.press(resolvedAriaKey as string),
+ ]);
+ expect(armResponse.status()).toBe(200);
+ expect(((await armResponse.json()) as { armed?: boolean }).armed).toBe(true);
+
+ // And the armed steer seals mid-stream, same proof as the button path.
+ await expect(appliedSteerParts(page).filter({ hasText: steerText })).toHaveCount(1, {
+ timeout: 90000,
+ });
+ await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
+ await expectModelContinuation(page, label, steerText);
+ });
+});
diff --git a/e2e/specs/mock/steering.spec.ts b/e2e/specs/mock/steering.spec.ts
index 931d0081d4..1f67790064 100644
--- a/e2e/specs/mock/steering.spec.ts
+++ b/e2e/specs/mock/steering.spec.ts
@@ -1,5 +1,6 @@
import { expect, test } from '@playwright/test';
import type { Page, Response } from '@playwright/test';
+import type { CancelSteerParams } from '../../../client/src/data-provider/SSE/mutations';
import {
MOCK_ENDPOINTS,
MOCK_REPLY_TEXT,
@@ -7,6 +8,8 @@ import {
messagesView,
replyPrompt,
replyText,
+ getAccessToken,
+ requestJson,
selectMockEndpoint,
sendMessage,
} from './helpers';
@@ -17,6 +20,7 @@ const PROVIDER_C = { label: 'Mock Provider C', model: 'mock-model-c' };
const MCP_SERVER_TITLE = 'E2E Memory';
/** Last chunk streamed by the fake model's slow replies (160 chunks, 0-indexed). */
const SLOW_REPLY_LAST_CHUNK = 'chunk-159';
+const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued';
const uniqueLabel = (prefix: string) =>
`${prefix}-${Date.now()}-${Math.floor(Math.random() * 1e4)}`;
@@ -29,6 +33,19 @@ const messageTurns = (page: Page) => messagesView(page).locator('.message-render
const inFlightSteers = (page: Page) => page.getByTestId('in-flight-steer');
const appliedSteerParts = (page: Page) => messagesView(page).getByTestId('steer-part');
+type PersistedMessage = {
+ messageId: string;
+ parentMessageId?: string;
+ text?: string;
+ content?: unknown[];
+ unfinished?: boolean;
+ isCreatedByUser?: boolean;
+};
+
+type CancelSteerWirePayload = CancelSteerParams & {
+ generationProtocolVersion: 2;
+};
+
function isSteerRequest(response: Response) {
return (
response.request().method() === 'POST' &&
@@ -205,13 +222,13 @@ test.describe('mid-run steering and queuing', () => {
/**
* Human-cadence variant: the second steer is submitted while the FIRST
- * steer's 202 is still pending, so the two POSTs (and their chip upserts)
- * genuinely overlap in flight. The overlap is enforced, not raced: a route
- * intercept forwards the first POST to the server (the enqueue happens in
- * submission order) but holds its 202 from the client until the second POST
- * has been observed. Both must still inject.
+ * steer's 202 is still pending. The client must keep its second POST parked
+ * until that ACK settles so asynchronous route validation cannot reverse
+ * server admission order. Both optimistic submissions must still inject.
*/
- test('steers twice rapidly (second sent before the first ACK): both inject', async ({ page }) => {
+ test('steers twice rapidly: second POST waits for the first ACK and both inject', async ({
+ page,
+ }) => {
test.setTimeout(150000);
const label = uniqueLabel('steerrapid');
const firstSteer = `Rapid first steer ${label}`;
@@ -226,25 +243,23 @@ test.describe('mid-run steering and queuing', () => {
expect(run.ok()).toBeTruthy();
let releaseFirstAck!: () => void;
- const secondPosted = new Promise<void>((resolve) => (releaseFirstAck = resolve));
+ const firstAckGate = new Promise<void>((resolve) => (releaseFirstAck = resolve));
+ let markFirstForwarded!: () => void;
+ const firstForwarded = new Promise<void>((resolve) => (markFirstForwarded = resolve));
+ let markSecondPosted!: () => void;
+ const secondPosted = new Promise<void>((resolve) => (markSecondPosted = resolve));
let steersSeen = 0;
- let overlapProven = false;
await page.route('**/api/agents/chat/steer', async (route) => {
const ordinal = ++steersSeen;
if (ordinal === 2) {
- releaseFirstAck();
+ markSecondPosted();
}
- // Forward to the real server first: the enqueue lands in submission
- // order; only the CLIENT-side 202 delivery is held back.
const response = await route.fetch();
if (ordinal === 1) {
- // Bounded so a broken second submit fails on assertions, not a hang.
- await Promise.race([
- secondPosted.then(() => {
- overlapProven = true;
- }),
- new Promise<void>((resolve) => setTimeout(resolve, 10000)),
- ]);
+ // The server has accepted the first steer; only client-side delivery
+ // of its 202 remains held while the user submits the second.
+ markFirstForwarded();
+ await firstAckGate;
}
await route.fulfill({ response });
});
@@ -258,17 +273,28 @@ test.describe('mid-run steering and queuing', () => {
const responses: Promise<Response>[] = [steerResponseFor(firstSteer)];
await typeDuringRun(page, firstSteer);
await messageInput(page).press('Enter');
+ await firstForwarded;
responses.push(steerResponseFor(secondSteer));
await typeDuringRun(page, secondSteer);
await messageInput(page).press('Enter');
+ let secondPostedBeforeFirstAck = false;
+ try {
+ secondPostedBeforeFirstAck = await Promise.race([
+ secondPosted.then(() => true),
+ page.waitForTimeout(500).then(() => false),
+ ]);
+ expect(secondPostedBeforeFirstAck).toBe(false);
+ expect(steersSeen).toBe(1);
+ } finally {
+ releaseFirstAck();
+ }
+
const [firstResponse, secondResponse] = await Promise.all(responses);
expect(firstResponse.status()).toBe(202);
expect(secondResponse.status()).toBe(202);
- // The hold proves the overlap actually happened: the second POST was
- // observed while the first 202 was still parked in the intercept.
+ await secondPosted;
expect(steersSeen).toBe(2);
- expect(overlapProven).toBe(true);
await page.unroute('**/api/agents/chat/steer');
await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
@@ -413,6 +439,70 @@ test.describe('mid-run steering and queuing', () => {
await expect(queuedRows(page)).toHaveCount(0);
});
+ test('recovered queued follow-up exposes Edit and Remove and discards its parked source before editing', async ({
+ page,
+ }) => {
+ test.setTimeout(60000);
+ const label = uniqueLabel('recovered-controls');
+ const recoveredText = `Recovered follow-up ${label}`;
+ const serverSteerId = `server-${label}`;
+ const clientSteerId = `client-${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ await establishConversation(page, `recovered-controls-setup-${label}`);
+
+ const conversationId = new URL(page.url()).pathname.split('/').pop();
+ expect(conversationId).toBeTruthy();
+ await page.route(`**/api/agents/chat/status/${conversationId}**`, (route) =>
+ route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify({
+ active: false,
+ generationProtocolVersion: 2,
+ unrecoveredSteers: [
+ {
+ steerId: serverSteerId,
+ clientSteerId,
+ text: recoveredText,
+ createdAt: Date.now(),
+ },
+ ],
+ }),
+ }),
+ );
+
+ let cancelBody: CancelSteerWirePayload | undefined;
+ await page.route('**/api/agents/chat/steer/cancel**', async (route) => {
+ cancelBody = route.request().postDataJSON() as CancelSteerWirePayload;
+ await route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify({ removed: true, generationProtocolVersion: 2 }),
+ });
+ });
+
+ await page.reload({ waitUntil: 'domcontentloaded', timeout: 10000 });
+ const row = queuedRows(page).filter({ hasText: recoveredText });
+ await expect(row).toBeVisible({ timeout: 15000 });
+ await expect(row.getByRole('button', { name: 'Remove message', exact: true })).toBeVisible();
+
+ await row.getByRole('button', { name: 'More options', exact: true }).click();
+ const edit = page.getByRole('menuitem', { name: 'Edit message', exact: true });
+ await expect(edit).toBeVisible();
+ await edit.click();
+
+ await expect(row).toHaveCount(0, { timeout: 10000 });
+ await expect(messageInput(page)).toHaveValue(recoveredText);
+ expect(cancelBody).toEqual({
+ conversationId,
+ steerId: serverSteerId,
+ clientSteerId,
+ generationProtocolVersion: 2,
+ });
+ });
+
test('queues with Cmd/Ctrl+Enter during a run and auto-sends after clean completion', async ({
page,
}) => {
@@ -482,6 +572,88 @@ test.describe('mid-run steering and queuing', () => {
await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
});
+ test('interrupt & send drains after a created response with no persistable content', async ({
+ page,
+ }) => {
+ test.setTimeout(120000);
+ const label = uniqueLabel('interrupt-empty');
+ const emptyRunPrompt = `E2E_EMPTY_SLOW_REPLY:${label}`;
+ const interruptText = `Interrupt empty follow-up ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, MOCK_ENDPOINTS[0]);
+ await establishConversation(page, `interrupt-empty-setup-${label}`);
+
+ const conversationId = new URL(page.url()).pathname.split('/').pop();
+ expect(conversationId).toBeTruthy();
+ const accessToken = await getAccessToken(page);
+ const messagesPath = `/api/messages/${encodeURIComponent(conversationId as string)}`;
+
+ const run = await sendMessage(page, emptyRunPrompt);
+ expect(run.ok()).toBeTruthy();
+
+ /** BaseClient starts its user-row write only after `onStart` emitted
+ * `created`. Waiting for that row proves the server is in the exact
+ * created-but-still-whitespace state, without relying on a sleep. */
+ await expect
+ .poll(
+ async () => {
+ const persisted = await requestJson<PersistedMessage[]>(page, {
+ path: messagesPath,
+ token: accessToken,
+ });
+ return persisted.some(
+ (message) => message.isCreatedByUser === true && message.text === emptyRunPrompt,
+ );
+ },
+ { timeout: 30000 },
+ )
+ .toBe(true);
+
+ await typeDuringRun(page, interruptText);
+ const [abortResponse] = await Promise.all([
+ page.waitForResponse(
+ (response) =>
+ response.request().method() === 'POST' &&
+ new URL(response.url()).pathname === '/api/agents/chat/abort',
+ { timeout: 30000 },
+ ),
+ messageInput(page).press('Alt+Enter'),
+ ]);
+ expect(abortResponse.ok()).toBeTruthy();
+
+ // The abort FINAL releases the queued follow-up, which completes live.
+ await expect(messageTurns(page)).toHaveCount(6, { timeout: 60000 });
+ const followupTurn = messageTurns(page).nth(4);
+ await expect(followupTurn).toContainText(interruptText);
+ await expect(followupTurn.locator('.user-turn')).toBeVisible();
+ await expect(messageTurns(page).nth(5)).toContainText(MOCK_REPLY_TEXT, { timeout: 30000 });
+
+ /** The empty assistant is a durable parent, not merely the optimistic
+ * row that the created handler rendered. Without that row an underscore
+ * preliminary id can reject this same queued submission. */
+ const persisted = await requestJson<PersistedMessage[]>(page, {
+ path: messagesPath,
+ token: accessToken,
+ });
+ const interruptedUser = persisted.find(
+ (message) => message.isCreatedByUser === true && message.text === emptyRunPrompt,
+ );
+ expect(interruptedUser).toBeTruthy();
+ expect(
+ persisted.find(
+ (message) =>
+ message.isCreatedByUser === false &&
+ message.parentMessageId === interruptedUser?.messageId,
+ ),
+ ).toMatchObject({
+ content: [],
+ unfinished: true,
+ isCreatedByUser: false,
+ });
+ await expect(queuedRows(page)).toHaveCount(0);
+ });
+
/**
* Interrupt & steer is the only path that can inject with NO tool boundary
* ahead of it: the server asks the generating replica to seal the model
@@ -529,13 +701,15 @@ test.describe('mid-run steering and queuing', () => {
await expect(messagesView(page).getByText(SLOW_REPLY_LAST_CHUNK)).toHaveCount(0);
await expect(messagesView(page).getByText('chunk-010')).toBeVisible();
- // NOTE: an assertion that the run visibly RESUMES after the seal (the
- // continuation's reply appearing) fails here deterministically. Every
- // other assertion passes, so the seal and the injection are working; what
- // is unresolved is whether the mock harness surfaces the continuation at
- // all in a no-tool scenario, or whether generation genuinely stops. That
- // distinction matters and is tracked separately rather than asserted
- // loosely here — see the PR discussion.
+ // The fake model's second invocation is unique and echoes only messages
+ // stamped as steer injections. This proves the graph resumed after the
+ // seal and that the continuation actually received the instruction.
+ await expect(messagesView(page).getByText(`[steers-seen=1] ${steerText}`)).toBeVisible({
+ timeout: 30000,
+ });
+ await expect(
+ messagesView(page).getByText(`${SLOW_REPLY_CONTINUATION_TEXT} ${label}`),
+ ).toBeVisible({ timeout: 30000 });
// Stayed INSIDE the response: the setup pair plus this pair, with no
// auto-sent follow-up pair (which both degradation paths produce).
diff --git a/helm/librechat/readme.md b/helm/librechat/readme.md
index d8f72e0645..920709af15 100755
--- a/helm/librechat/readme.md
+++ b/helm/librechat/readme.md
@@ -57,6 +57,20 @@ also register this LibreChat callback URL with your identity provider:
https://<librechat-domain>/api/admin/oauth/openid/callback
```
+## Generation protocol rollout
+
+Redis-backed generation streams use protocol v1 by default during the first
+rollout of a v2-capable image. This keeps a rolling deployment compatible with
+replicas that still run the previous Redis queue, checkpoint, and recovery
+scripts.
+
+After every LibreChat replica is on the v2-capable image and all active
+generations owned by the old image have drained, set
+`librechat.configEnv.GENERATION_PROTOCOL_VERSION="2"` in a second rollout.
+Keep the new image in place until v2 generations have drained; an older image
+cannot safely operate on their Redis state. In-memory generation streams do not
+share state across replicas and negotiate v2 without this cutover.
+
## Langfuse Fanout
The chart can optionally deploy a Langfuse fanout gateway with an internal
diff --git a/packages/api/src/agents/checkpointer.integration.spec.ts b/packages/api/src/agents/checkpointer.integration.spec.ts
index 7e40d70675..a037475999 100644
--- a/packages/api/src/agents/checkpointer.integration.spec.ts
+++ b/packages/api/src/agents/checkpointer.integration.spec.ts
@@ -10,6 +10,7 @@ import {
deleteAgentCheckpoints,
LazyMongoSaver,
CheckpointTooLargeError,
+ LIBRECHAT_CHECKPOINT_NAMESPACE_KEY,
__resetCheckpointerForTests,
} from './checkpointer';
@@ -24,14 +25,16 @@ import {
const MONGO_CFG = { type: 'mongo' as const, ttl: 3600 };
/** Minimal LangGraph put() args for an empty checkpoint under a thread. */
-function putArgs(threadId: string) {
- const config = { configurable: { thread_id: threadId, checkpoint_ns: '' } };
+function putArgs(threadId: string, checkpointNamespace = '') {
+ const config = {
+ configurable: { thread_id: threadId, checkpoint_ns: checkpointNamespace },
+ };
const metadata = { source: 'input' as const, step: -1, writes: null, parents: {} };
return { config, checkpoint: emptyCheckpoint(), metadata };
}
-const readConfig = (threadId: string) => ({
- configurable: { thread_id: threadId, checkpoint_ns: '' },
+const readConfig = (threadId: string, checkpointNamespace = '') => ({
+ configurable: { thread_id: threadId, checkpoint_ns: checkpointNamespace },
});
/**
@@ -39,10 +42,20 @@ const readConfig = (threadId: string) => ({
* `INTERRUPT` channel for the checkpoint id, then `put` of that checkpoint. The lazy saver
* persists only checkpoints seeded this way (a bare `put` is a clean exit → discarded).
*/
-async function seedInterruptCheckpoint(saver: MongoDBSaver, threadId: string) {
- const { config, checkpoint, metadata } = putArgs(threadId);
+async function seedInterruptCheckpoint(
+ saver: MongoDBSaver,
+ threadId: string,
+ checkpointNamespace = '',
+) {
+ const { config, checkpoint, metadata } = putArgs(threadId, checkpointNamespace);
await saver.putWrites(
- { configurable: { thread_id: threadId, checkpoint_ns: '', checkpoint_id: checkpoint.id } },
+ {
+ configurable: {
+ thread_id: threadId,
+ checkpoint_ns: checkpointNamespace,
+ checkpoint_id: checkpoint.id,
+ },
+ },
[[INTERRUPT, 'approve?']],
'task-1',
);
@@ -108,6 +121,72 @@ describe('checkpointer (mongodb-memory-server integration)', () => {
expect(await saver!.getTuple(readConfig(threadId))).toBeUndefined();
});
+ it('captured legacy cleanup prunes nested graph checkpoints and writes without a thread-wide delete', async () => {
+ const { StateGraph, START, END, interrupt, Annotation } = await import('@langchain/langgraph');
+ const saver = await getAgentCheckpointer(MONGO_CFG);
+ const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
+
+ const State = Annotation.Root({
+ origin: Annotation,
+ approved: Annotation,
+ });
+ const child = new StateGraph(State)
+ .addNode('child-gate', (state: { origin?: string }) => ({
+ approved: interrupt(`approve-${state.origin}`),
+ }))
+ .addEdge(START, 'child-gate')
+ .addEdge('child-gate', END)
+ .compile();
+ const graph = new StateGraph(State)
+ .addNode('child', child)
+ .addEdge(START, 'child')
+ .addEdge('child', END)
+ .compile({ checkpointer: saver as never });
+ const config = {
+ configurable: { thread_id: threadId, checkpoint_ns: '' },
+ durability: 'exit' as const,
+ };
+
+ await graph.invoke({ origin: 'legacy', approved: null }, config);
+
+ const db = mongoose.connection.db!;
+ const storedNamespaces = await db
+ .collection('agent_checkpoints')
+ .distinct('checkpoint_ns', { thread_id: threadId });
+ expect(storedNamespaces).toContain('');
+ expect(storedNamespaces.some((namespace) => namespace.startsWith('child:'))).toBe(true);
+ expect(
+ await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
+ ).toBeGreaterThan(0);
+
+ const generation = await captureAgentCheckpointGeneration(threadId, MONGO_CFG, {
+ throwOnError: true,
+ });
+ const replacement = await seedInterruptCheckpoint(saver!, threadId, '2000');
+ await expect(
+ deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
+ throwOnError: true,
+ checkpointNamespace: '',
+ }),
+ ).rejects.toThrow('requires a captured checkpoint generation');
+ await deleteAgentCheckpoint(threadId, MONGO_CFG, generation, { throwOnError: true });
+
+ expect(await db.collection('agent_checkpoints').countDocuments({ thread_id: threadId })).toBe(
+ 1,
+ );
+ expect(
+ await db
+ .collection('agent_checkpoints')
+ .countDocuments({ thread_id: threadId, checkpoint_ns: { $ne: '2000' } }),
+ ).toBe(0);
+ expect(
+ await db.collection('agent_checkpoint_writes').countDocuments({ thread_id: threadId }),
+ ).toBe(1);
+ expect(await saver!.getTuple(readConfig(threadId, '2000'))).toMatchObject({
+ checkpoint: { id: replacement.id },
+ });
+ });
+
it('prunes only the targeted thread, leaving other conversations intact', async () => {
const saver = await getAgentCheckpointer(MONGO_CFG);
const threadA = `convo-${new mongoose.Types.ObjectId().toString()}`;
@@ -155,6 +234,110 @@ describe('checkpointer (mongodb-memory-server integration)', () => {
).toBe(0);
});
+ it('isolates a fresh generation from a predecessor checkpoint written after replacement', async () => {
+ const saver = await getAgentCheckpointer(MONGO_CFG);
+ const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
+ const predecessorNamespace = '1000';
+ const replacementNamespace = '2000';
+
+ await seedInterruptCheckpoint(saver!, threadId, predecessorNamespace);
+ // Fresh B prunes only its own namespace before graph construction. A is
+ // still running remotely and writes another interrupt after that barrier.
+ await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
+ checkpointNamespace: replacementNamespace,
+ });
+ const latePredecessor = await seedInterruptCheckpoint(saver!, threadId, predecessorNamespace);
+
+ expect(await saver!.getTuple(readConfig(threadId, replacementNamespace))).toBeUndefined();
+ const replacement = await seedInterruptCheckpoint(saver!, threadId, replacementNamespace);
+ expect(await saver!.getTuple(readConfig(threadId, replacementNamespace))).toMatchObject({
+ checkpoint: { id: replacement.id },
+ });
+ expect(await saver!.getTuple(readConfig(threadId, predecessorNamespace))).toMatchObject({
+ checkpoint: { id: latePredecessor.id },
+ });
+ });
+
+ it('adapts LangGraph root namespaces and prevents a late predecessor pause from hydrating into its replacement', async () => {
+ const { StateGraph, START, END, interrupt, Annotation, Command } = await import(
+ '@langchain/langgraph'
+ );
+ const saver = await getAgentCheckpointer(MONGO_CFG);
+ const threadId = `convo-${new mongoose.Types.ObjectId().toString()}`;
+
+ const State = Annotation.Root({
+ origin: Annotation,
+ approved: Annotation,
+ });
+ const graph = new StateGraph(State)
+ .addNode('gate', (state: { origin?: string }) => ({
+ approved: interrupt(`approve-${state.origin}`),
+ }))
+ .addEdge(START, 'gate')
+ .addEdge('gate', END)
+ .compile({ checkpointer: saver as never });
+
+ const scopedConfig = (generation: string, graphNamespace = '') => ({
+ configurable: {
+ thread_id: threadId,
+ checkpoint_ns: graphNamespace,
+ [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: generation,
+ },
+ durability: 'exit' as const,
+ });
+ // Deliberately nonempty on the initial root calls: PregelLoop.initialize
+ // resets these to ''. The ordinary client wiring uses the normalized
+ // configs below on both fresh and resume calls.
+ const replacementStartConfig = scopedConfig('generation-b', 'caller-root-generation-b');
+ const predecessorStartConfig = scopedConfig('generation-a', 'caller-root-generation-a');
+ const replacementConfig = scopedConfig('generation-b');
+ const predecessorConfig = scopedConfig('generation-a');
+
+ // B has started and paused. A's remote owner then lands a late pause on
+ // the same conversation thread; physical namespaces must keep it out of B.
+ await graph.invoke({ origin: 'B', approved: null }, replacementStartConfig);
+ await graph.invoke({ origin: 'A', approved: null }, predecessorStartConfig);
+
+ const storedNamespaces = await mongoose.connection
+ .db!.collection('agent_checkpoints')
+ .distinct('checkpoint_ns', { thread_id: threadId });
+ expect(new Set(storedNamespaces)).toEqual(new Set(['generation-a', 'generation-b']));
+ expect(storedNamespaces).not.toContain('');
+ expect(storedNamespaces).not.toContain('caller-root-generation-a');
+ expect(storedNamespaces).not.toContain('caller-root-generation-b');
+
+ const replacementState = await graph.getState(replacementConfig);
+ expect(replacementState.values).toMatchObject({ origin: 'B' });
+ const listedReplacementCheckpoints = [];
+ for await (const tuple of saver!.list(replacementConfig)) {
+ listedReplacementCheckpoints.push(tuple);
+ }
+ expect(listedReplacementCheckpoints.length).toBeGreaterThan(0);
+ expect(
+ listedReplacementCheckpoints.every(
+ (tuple) =>
+ tuple.config.configurable?.checkpoint_ns === '' &&
+ tuple.config.configurable?.[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY] === 'generation-b',
+ ),
+ ).toBe(true);
+ const replacementResult = await graph.invoke(
+ new Command({ resume: 'YES-B' }),
+ replacementConfig,
+ );
+ expect(replacementResult).toMatchObject({ origin: 'B', approved: 'YES-B' });
+
+ // Terminal B cleanup removes its whole namespace, including any write
+ // committed after an earlier snapshot, while late A remains resumable.
+ await deleteAgentCheckpoint(threadId, MONGO_CFG, undefined, {
+ checkpointNamespace: 'generation-b',
+ });
+ expect(await graph.getState(replacementConfig)).toMatchObject({ next: [] });
+ expect(await graph.getState(predecessorConfig)).toMatchObject({
+ values: expect.objectContaining({ origin: 'A' }),
+ next: expect.arrayContaining(['gate']),
+ });
+ });
+
it('deleteAgentCheckpoint is a no-op for an undefined threadId', async () => {
await expect(deleteAgentCheckpoint(undefined, MONGO_CFG)).resolves.toBeUndefined();
});
diff --git a/packages/api/src/agents/checkpointer.spec.ts b/packages/api/src/agents/checkpointer.spec.ts
index 3411b7202e..69ce7248c7 100644
--- a/packages/api/src/agents/checkpointer.spec.ts
+++ b/packages/api/src/agents/checkpointer.spec.ts
@@ -77,4 +77,15 @@ describe('deleteAgentCheckpoint', () => {
checkpointIds: [],
});
});
+
+ test('normalizes an explicit empty namespace to a thread-wide legacy capture', async () => {
+ await expect(
+ captureAgentCheckpointGeneration('conversation-1', undefined, {
+ checkpointNamespace: '',
+ }),
+ ).resolves.toEqual({
+ threadId: 'conversation-1',
+ checkpointIds: [],
+ });
+ });
});
diff --git a/packages/api/src/agents/checkpointer.ts b/packages/api/src/agents/checkpointer.ts
index 7ac8d39316..03739a84bf 100644
--- a/packages/api/src/agents/checkpointer.ts
+++ b/packages/api/src/agents/checkpointer.ts
@@ -2,10 +2,103 @@ import mongoose from 'mongoose';
import { logger } from '@librechat/data-schemas';
import { INTERRUPT } from '@langchain/langgraph-checkpoint';
import { MongoDBSaver } from '@langchain/langgraph-checkpoint-mongodb';
-import type { Checkpoint, CheckpointMetadata, PendingWrite } from '@langchain/langgraph-checkpoint';
+import type {
+ Checkpoint,
+ CheckpointListOptions,
+ CheckpointMetadata,
+ CheckpointTuple,
+ PendingWrite,
+} from '@langchain/langgraph-checkpoint';
import type { TCheckpointerConfig } from 'librechat-data-provider';
import type { RunnableConfig } from '@langchain/core/runnables';
+/**
+ * LangGraph reserves `checkpoint_ns` for nested graph namespaces and forcibly
+ * resets a non-empty value to `''` for every root invocation. Carry LibreChat's
+ * immutable generation scope on a private configurable key instead; the saver
+ * adapter below maps it into Mongo's storage namespace without changing the
+ * graph-visible conversation `thread_id`.
+ */
+export const LIBRECHAT_CHECKPOINT_NAMESPACE_KEY = '__librechat_checkpoint_ns';
+
+const CHECKPOINT_NAMESPACE_SEPARATOR = '|';
+
+function generationCheckpointNamespace(config: RunnableConfig): string | undefined {
+ const value = config.configurable?.[LIBRECHAT_CHECKPOINT_NAMESPACE_KEY];
+ return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Prefix every root/subgraph storage namespace with the immutable generation. */
+function toStorageCheckpointConfig(config: RunnableConfig): RunnableConfig {
+ const generationNamespace = generationCheckpointNamespace(config);
+ if (!generationNamespace) {
+ return config;
+ }
+ const graphNamespace =
+ typeof config.configurable?.checkpoint_ns === 'string' ? config.configurable.checkpoint_ns : '';
+ return {
+ ...config,
+ configurable: {
+ ...config.configurable,
+ checkpoint_ns:
+ graphNamespace === ''
+ ? generationNamespace
+ : `${generationNamespace}${CHECKPOINT_NAMESPACE_SEPARATOR}${graphNamespace}`,
+ },
+ };
+}
+
+/** Restore the namespace LangGraph supplied while retaining the private scope. */
+function fromStorageCheckpointConfig(
+ storedConfig: RunnableConfig,
+ requestedConfig: RunnableConfig,
+): RunnableConfig {
+ const generationNamespace = generationCheckpointNamespace(requestedConfig);
+ if (!generationNamespace) {
+ return storedConfig;
+ }
+ const graphNamespace =
+ typeof requestedConfig.configurable?.checkpoint_ns === 'string'
+ ? requestedConfig.configurable.checkpoint_ns
+ : '';
+ return {
+ ...storedConfig,
+ configurable: {
+ ...storedConfig.configurable,
+ thread_id: requestedConfig.configurable?.thread_id ?? storedConfig.configurable?.thread_id,
+ checkpoint_ns: graphNamespace,
+ [LIBRECHAT_CHECKPOINT_NAMESPACE_KEY]: generationNamespace,
+ },
+ };
+}
+
+function fromStorageCheckpointTuple(
+ tuple: CheckpointTuple,
+ requestedConfig: RunnableConfig,
+): CheckpointTuple {
+ return {
+ ...tuple,
+ config: fromStorageCheckpointConfig(tuple.config, requestedConfig),
+ ...(tuple.parentConfig && {
+ parentConfig: fromStorageCheckpointConfig(tuple.parentConfig, requestedConfig),
+ }),
+ };
+}
+
+function escapeRegExp(value: string): string {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/** Mongo filter matching a generation's root and every nested graph namespace. */
+function generationNamespaceFilter(checkpointNamespace: string): string | { $regex: string } {
+ if (checkpointNamespace === '') {
+ return '';
+ }
+ return {
+ $regex: `^${escapeRegExp(checkpointNamespace)}(?:\\${CHECKPOINT_NAMESPACE_SEPARATOR}|$)`,
+ };
+}
+
/**
* Durable checkpointing for human-in-the-loop (HITL) resume.
*
@@ -117,7 +210,8 @@ function hasResumableWrite(writes: PendingWrite[]): boolean {
* the writes themselves persist exactly as before, so resume is unchanged. The write-less clean
* checkpoint (and the now-discarded error-only checkpoint) was only ever written-then-pruned, so
* not writing it is observationally equivalent; the pre-run prune + Mongo TTL remain the
- * backstops. `getTuple`/`list`/`deleteThread`/`setup` are inherited.
+ * backstops. The saver overrides every config-bearing read/write path to apply
+ * generation storage scoping; `deleteThread` and `setup` remain inherited.
*/
/** A bookkeeping-only pending-write batch held until its checkpoint's fate is decided. */
interface BufferedWriteBatch {
@@ -209,15 +303,41 @@ export class LazyMongoSaver extends MongoDBSaver {
this.hardLimitBytes = hardLimitBytes ?? CHECKPOINT_HARD_LIMIT_BYTES;
}
+ /**
+ * LangGraph normalizes every root invocation to `checkpoint_ns: ''` before
+ * touching the saver. Map LibreChat's private generation key into Mongo's
+ * namespace at this storage boundary, then restore the graph-visible config
+ * on the way out. This keeps callbacks/tools on the real conversation
+ * `thread_id` while making replacement generations physically disjoint.
+ */
+ override async getTuple(config: RunnableConfig): Promise<CheckpointTuple | undefined> {
+ const tuple = await super.getTuple(toStorageCheckpointConfig(config));
+ return tuple ? fromStorageCheckpointTuple(tuple, config) : undefined;
+ }
+
+ override async *list(
+ config: RunnableConfig,
+ options?: CheckpointListOptions,
+ ): AsyncGenerator<CheckpointTuple> {
+ const storageConfig = toStorageCheckpointConfig(config);
+ const storageOptions = options?.before
+ ? { ...options, before: toStorageCheckpointConfig(options.before) }
+ : options;
+ for await (const tuple of super.list(storageConfig, storageOptions)) {
+ yield fromStorageCheckpointTuple(tuple, config);
+ }
+ }
+
override async putWrites(
config: RunnableConfig,
writes: PendingWrite[],
taskId: string,
): Promise<void> {
+ const storageConfig = toStorageCheckpointConfig(config);
const checkpointId = config.configurable?.checkpoint_id as string | undefined;
if (!checkpointId) {
// No checkpoint id to tie a fate to — forward untouched (the base saver's contract).
- return super.putWrites(config, writes, taskId);
+ return super.putWrites(storageConfig, writes, taskId);
}
if (!hasResumableWrite(writes)) {
// A bookkeeping-only batch (`__error__` from a failed turn, a completed Send-sibling's
@@ -227,16 +347,16 @@ export class LazyMongoSaver extends MongoDBSaver {
// sibling on resume), an orphan on a discarded one. Forward when the fate is already
// known to be "persist"; otherwise buffer until an anchoring batch or `put` decides.
if (this.writeAnchorIds.has(checkpointId) || this.persistedIds.has(checkpointId)) {
- return super.putWrites(config, writes, taskId);
+ return super.putWrites(storageConfig, writes, taskId);
}
const buffered = this.bufferedBookkeeping.get(checkpointId);
if (buffered) {
- buffered.batches.push({ config, writes, taskId });
+ buffered.batches.push({ config: storageConfig, writes, taskId });
} else {
sweepStale(this.bufferedBookkeeping, (b) => b.at);
this.bufferedBookkeeping.set(checkpointId, {
at: Date.now(),
- batches: [{ config, writes, taskId }],
+ batches: [{ config: storageConfig, writes, taskId }],
});
}
return;
@@ -253,7 +373,7 @@ export class LazyMongoSaver extends MongoDBSaver {
// The checkpoint's fate is now "persist" — flush the bookkeeping batches that
// arrived before this anchor so the stored pending writes are complete.
await this.flushBufferedBookkeeping(checkpointId);
- return await super.putWrites(config, writes, taskId);
+ return await super.putWrites(storageConfig, writes, taskId);
} catch (err) {
// The write batch never landed — best-effort un-anchor so the concurrent `put` doesn't
// persist a checkpoint whose pending writes are missing (an unresumable phantom pause).
@@ -276,14 +396,14 @@ export class LazyMongoSaver extends MongoDBSaver {
await this.assertCheckpointFitsDocument(config, checkpoint, metadata);
sweepStale(this.persistedIds, (t) => t);
this.persistedIds.set(checkpoint.id, Date.now());
- const persisted = await super.put(config, checkpoint, metadata);
+ const persisted = await super.put(toStorageCheckpointConfig(config), checkpoint, metadata);
// `assertCheckpointFitsDocument` awaits a (potentially slow) serialization AFTER the
// anchor was consumed above but BEFORE `persistedIds` was set — a bookkeeping-only
// `putWrites` dispatched in that window sees neither marker and parks its batch. Flush
// it now that the checkpoint is persisted; without this the marker is dropped and a
// resume can re-execute already-completed work.
await this.flushBufferedBookkeeping(checkpoint.id);
- return persisted;
+ return fromStorageCheckpointConfig(persisted, config);
}
// No resumable writes ⇒ a clean exit (a non-paused completion, a resumed turn's clean
// finish, or an error-only turn): discard, and drop the parked bookkeeping batches with
@@ -414,14 +534,18 @@ export interface ResolvedCheckpointerConfig {
}
/**
- * Exact checkpoint ids present before a resumed generation is claimed.
+ * Exact checkpoint ids present before a legacy, unscoped generation is claimed.
*
- * Terminal resume cleanup deletes only this immutable set. A replacement turn
- * that later pauses on the same `thread_id` receives fresh checkpoint ids and
- * therefore cannot be removed by the predecessor's delayed cleanup.
+ * New jobs delete their immutable saver scope wholesale at terminal ownership.
+ * Legacy jobs share storage, so cleanup deletes only this captured set; a later
+ * replacement's fresh checkpoint ids cannot be removed by the delayed cleanup.
*/
export interface AgentCheckpointGeneration {
threadId: string;
+ /** Nonempty saver-level generation scope. Missing means a legacy
+ * thread-wide capture; an empty string is invalid because it would omit the
+ * legacy generation's nested LangGraph namespaces during deletion. */
+ checkpointNamespace?: string;
checkpointIds: string[];
}
@@ -535,8 +659,22 @@ async function buildMongoSaver(
export async function captureAgentCheckpointGeneration(
threadId: string,
cfg?: TCheckpointerConfig,
+ options?: { throwOnError?: boolean; checkpointNamespace?: string },
): Promise<AgentCheckpointGeneration> {
- const generation: AgentCheckpointGeneration = { threadId, checkpointIds: [] };
+ const requestedNamespace = options?.checkpointNamespace ?? '';
+ /** Empty is the shared legacy namespace, whose nested subgraphs live under
+ * independent nonempty LangGraph namespaces. Treat it as a thread-wide id
+ * capture and omit the namespace marker so deletion cannot silently filter
+ * those child rows out. Only nonempty generation scopes are prefix-safe. */
+ const namespaceScoped =
+ options != null &&
+ Object.prototype.hasOwnProperty.call(options, 'checkpointNamespace') &&
+ requestedNamespace !== '';
+ const generation: AgentCheckpointGeneration = {
+ threadId,
+ ...(namespaceScoped && { checkpointNamespace: requestedNamespace }),
+ checkpointIds: [],
+ };
if (!threadId) {
return generation;
}
@@ -549,7 +687,15 @@ export async function captureAgentCheckpointGeneration(
const resolved = resolveCheckpointerConfig(cfg);
const checkpoints = await db
.collection<{ checkpoint_id?: string }>(resolved.checkpointCollectionName)
- .find({ thread_id: threadId }, { projection: { _id: 0, checkpoint_id: 1 } })
+ .find(
+ {
+ thread_id: threadId,
+ ...(namespaceScoped && {
+ checkpoint_ns: generationNamespaceFilter(requestedNamespace),
+ }),
+ },
+ { projection: { _id: 0, checkpoint_id: 1 } },
+ )
.toArray();
generation.checkpointIds = checkpoints.reduce<string[]>((ids, checkpoint) => {
if (typeof checkpoint.checkpoint_id === 'string') {
@@ -562,6 +708,9 @@ export async function captureAgentCheckpointGeneration(
`[checkpointer] Failed to capture checkpoint generation for thread ${threadId}:`,
err,
);
+ if (options?.throwOnError) {
+ throw err;
+ }
}
return generation;
}
@@ -581,6 +730,7 @@ export async function deleteAgentCheckpoint(
threadId: string | undefined,
cfg?: TCheckpointerConfig,
generation?: AgentCheckpointGeneration,
+ options?: { throwOnError?: boolean; checkpointNamespace?: string },
): Promise<void> {
if (!threadId) {
return;
@@ -594,6 +744,14 @@ export async function deleteAgentCheckpoint(
if (generation.threadId !== threadId || generation.checkpointIds.length === 0) {
return;
}
+ if (
+ Object.prototype.hasOwnProperty.call(generation, 'checkpointNamespace') &&
+ (generation.checkpointNamespace ?? '') === ''
+ ) {
+ throw new Error(
+ 'Legacy checkpoint cleanup requires a thread-wide captured generation without an empty namespace marker',
+ );
+ }
const db = mongoose.connection.db;
if (!db) {
return;
@@ -601,6 +759,9 @@ export async function deleteAgentCheckpoint(
const resolved = resolveCheckpointerConfig(cfg);
const filter = {
thread_id: threadId,
+ ...(Object.prototype.hasOwnProperty.call(generation, 'checkpointNamespace') && {
+ checkpoint_ns: generationNamespaceFilter(generation.checkpointNamespace ?? ''),
+ }),
checkpoint_id: { $in: generation.checkpointIds },
};
await Promise.all([
@@ -609,9 +770,40 @@ export async function deleteAgentCheckpoint(
]);
return;
}
+ if (Object.prototype.hasOwnProperty.call(options ?? {}, 'checkpointNamespace')) {
+ const checkpointNamespace = options?.checkpointNamespace ?? '';
+ /** An explicit empty namespace denotes a legacy/pre-isolation job, not
+ * an immutable storage scope. Thread-wide deletion could erase a newer
+ * v2 replacement, while filtering `checkpoint_ns: ''` would strand the
+ * legacy job's nested subgraphs. Such callers must capture a thread-wide
+ * immutable id set, verify their job epoch after that capture, and pass
+ * the resulting `generation` above. */
+ if (checkpointNamespace === '') {
+ throw new Error(
+ 'Legacy checkpoint cleanup requires a captured checkpoint generation, not an empty namespace',
+ );
+ }
+ const db = mongoose.connection.db;
+ if (!db) {
+ return;
+ }
+ const resolved = resolveCheckpointerConfig(cfg);
+ const filter = {
+ thread_id: threadId,
+ checkpoint_ns: generationNamespaceFilter(checkpointNamespace),
+ };
+ await Promise.all([
+ db.collection(resolved.checkpointCollectionName).deleteMany(filter),
+ db.collection(resolved.checkpointWritesCollectionName).deleteMany(filter),
+ ]);
+ return;
+ }
await saver.deleteThread(threadId);
} catch (err) {
logger.warn(`[checkpointer] Failed to delete checkpoints for thread ${threadId}:`, err);
+ if (options?.throwOnError) {
+ throw err;
+ }
}
}
diff --git a/packages/api/src/agents/steering/__tests__/request.spec.ts b/packages/api/src/agents/steering/__tests__/request.spec.ts
index 57301b991a..292728fe70 100644
--- a/packages/api/src/agents/steering/__tests__/request.spec.ts
+++ b/packages/api/src/agents/steering/__tests__/request.spec.ts
@@ -1,11 +1,11 @@
import type { IMongoFile } from '@librechat/data-schemas';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
+import { handleSteerRequest, handleSteerCancel, handleSteerArm } from '../request';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { isSteeringSupported, isSteerPreemptSupported } from '../runtime';
import { STEER_QUEUE_MAX_DEPTH } from '~/stream/interfaces/IJobStore';
import { GenerationJobManager } from '~/stream/GenerationJobManager';
-import { handleSteerRequest, handleSteerCancel } from '../request';
jest.mock('../runtime', () => ({
...jest.requireActual('../runtime'),
@@ -19,6 +19,12 @@ const mockIsSupported = isSteeringSupported as jest.Mock;
const mockIsPreemptSupported = isSteerPreemptSupported as jest.Mock;
const user = { id: 'user-1' };
+async function removeStoredJobOwner(streamId: string): Promise<void> {
+ const stored = await GenerationJobManager.getJobStore().getJob(streamId);
+ expect(stored).not.toBeNull();
+ delete (stored as { userId?: string }).userId;
+}
+
describe('handleSteerRequest (real in-memory job manager)', () => {
beforeEach(() => {
jest.clearAllMocks();
@@ -50,6 +56,42 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
expect(result.body.code).toBe('EMPTY_TEXT');
});
+ it('400s an invalid client correlation id', async () => {
+ const result = await handleSteerRequest(user, {
+ conversationId: 'c1',
+ text: 'valid text',
+ clientSteerId: 'not valid whitespace',
+ });
+ expect(result.status).toBe(400);
+ expect(result.body.code).toBe('INVALID_CLIENT_STEER_ID');
+ });
+
+ it('400s an invalid generation identity', async () => {
+ const result = await handleSteerRequest(user, {
+ conversationId: 'c1',
+ generationCreatedAt: '1000',
+ text: 'valid text',
+ });
+ expect(result).toEqual({
+ status: 400,
+ body: { code: 'INVALID_GENERATION_IDENTITY' },
+ });
+ });
+
+ it('409s a steer targeting a replaced generation without touching the live queue', async () => {
+ const streamId = 'steer-stale-generation';
+ const job = await GenerationJobManager.createJob(streamId, user.id);
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ generationCreatedAt: job.createdAt - 1,
+ text: 'belongs to the previous turn',
+ });
+
+ expect(result).toEqual({ status: 409, body: { code: 'RUN_REPLACED' } });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]);
+ });
+
it('413s past the length cap', async () => {
const result = await handleSteerRequest(user, {
conversationId: 'c1',
@@ -115,6 +157,17 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
expect(result.body.code).toBe('UNAUTHORIZED');
});
+ it('fails closed when the stored job owner is missing', async () => {
+ const streamId = 'steer-req-missing-owner';
+ await GenerationJobManager.createJob(streamId, user.id);
+ await removeStoredJobOwner(streamId);
+
+ const result = await handleSteerRequest(user, { conversationId: streamId, text: 'x' });
+
+ expect(result).toEqual({ status: 403, body: { code: 'UNAUTHORIZED' } });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]);
+ });
+
it('409s while the run is paused for human review', async () => {
const streamId = 'steer-req-paused';
await GenerationJobManager.createJob(streamId, user.id);
@@ -149,12 +202,45 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
expect(overflow.body.code).toBe('STEER_QUEUE_FULL');
});
+ it('429s a new receipt at the history cap while existing ids remain replayable', async () => {
+ const streamId = 'steer-req-receipt-full';
+ await GenerationJobManager.createJob(streamId, user.id);
+ for (let index = 0; index < 100; index++) {
+ const accepted = await handleSteerRequest(user, {
+ conversationId: streamId,
+ clientSteerId: `receipt-cap-${index}`,
+ text: `steer ${index}`,
+ });
+ expect(accepted.status).toBe(202);
+ await expect(
+ GenerationJobManager.steering.cancel(streamId, accepted.body.steerId as string),
+ ).resolves.toBe(true);
+ }
+
+ const overflow = await handleSteerRequest(user, {
+ conversationId: streamId,
+ clientSteerId: 'receipt-cap-overflow',
+ text: 'must not queue',
+ });
+ expect(overflow).toEqual({ status: 429, body: { code: 'STEER_RECEIPT_LIMIT' } });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]);
+
+ const replay = await handleSteerRequest(user, {
+ conversationId: streamId,
+ clientSteerId: 'receipt-cap-0',
+ text: 'steer 0',
+ });
+ expect(replay.status).toBe(202);
+ expect(replay.body).toMatchObject({ replayed: true, settled: true });
+ });
+
it('202s, sanitizes the text, and enqueues sanitized attachment refs', async () => {
const streamId = 'steer-req-accept';
await GenerationJobManager.createJob(streamId, user.id);
const result = await handleSteerRequest(user, {
conversationId: streamId,
+ clientSteerId: 'local-correlation-123',
text: ' focus on tests\0 ',
files: [
{
@@ -182,6 +268,7 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
const queued = await GenerationJobManager.steering.peek(streamId);
expect(queued).toHaveLength(1);
expect(queued[0].text).toBe('focus on tests');
+ expect(queued[0].clientSteerId).toBe('local-correlation-123');
expect(queued[0].userId).toBe(user.id);
expect(queued[0].files).toEqual([
{
@@ -285,6 +372,55 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
expect(result.status).toBe(202);
expect(getFiles).not.toHaveBeenCalled();
});
+
+ it('409s when attachment resolution races the same generation into a pause', async () => {
+ const streamId = 'steer-req-file-pause-race';
+ const job = await GenerationJobManager.createJob(streamId, user.id);
+ let signalResolveStarted: (() => void) | undefined;
+ const resolveStarted = new Promise<void>((resolve) => {
+ signalResolveStarted = resolve;
+ });
+ let releaseResolve: ((files: IMongoFile[]) => void) | undefined;
+ const getFiles = jest.fn(
+ () =>
+ new Promise<IMongoFile[]>((resolve) => {
+ releaseResolve = resolve;
+ signalResolveStarted?.();
+ }),
+ );
+
+ const requesting = handleSteerRequest(
+ user,
+ {
+ conversationId: streamId,
+ text: 'queue while approval is open',
+ files: [{ file_id: 'f1' }],
+ },
+ { getFiles },
+ );
+ await resolveStarted;
+ const action = buildPendingAction(
+ buildToolApprovalPayload([
+ { name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_pause_race' },
+ ]),
+ {
+ streamId,
+ conversationId: streamId,
+ runId: 'run-pause-race',
+ responseMessageId: 'msg-pause-race',
+ },
+ );
+ await expect(GenerationJobManager.approvals.pause(streamId, action)).resolves.toBe(true);
+ releaseResolve?.([dbDoc]);
+
+ await expect(requesting).resolves.toEqual({
+ status: 409,
+ body: { code: 'RUN_PAUSED' },
+ });
+ await expect(GenerationJobManager.steering.peek(streamId, job.createdAt)).resolves.toEqual(
+ [],
+ );
+ });
});
describe('injected updateFilesUsage (upload-window TTL parity)', () => {
@@ -294,7 +430,7 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
const streamId = 'steer-req-usage-ok';
await GenerationJobManager.createJob(streamId, user.id);
const getFiles = jest.fn(async () => [dbDoc]);
- const updateFilesUsage = jest.fn(async () => []);
+ const updateFilesUsage = jest.fn(async () => [dbDoc]);
const result = await handleSteerRequest(
user,
@@ -309,7 +445,7 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
});
});
- it('does not mark usage on a rejected steer and never fails the 202', async () => {
+ it('does not enqueue when resolved upload retention fails', async () => {
const streamId = 'steer-req-usage-deny';
await GenerationJobManager.createJob(streamId, user.id);
const getFiles = jest.fn(async () => [dbDoc]);
@@ -331,9 +467,62 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
{ conversationId: streamId, text: 'x', files: [{ file_id: 'f1' }] },
{ getFiles, updateFilesUsage: failing },
);
- expect(accepted.status).toBe(202);
- await new Promise(setImmediate);
+ expect(accepted).toEqual({
+ status: 503,
+ body: { code: 'STEER_FILE_RETENTION_FAILED' },
+ });
expect(failing).toHaveBeenCalledTimes(1);
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]);
+
+ const missing = await handleSteerRequest(
+ user,
+ { conversationId: streamId, text: 'x', files: [{ file_id: 'f1' }] },
+ { getFiles, updateFilesUsage: jest.fn(async () => []) },
+ );
+ expect(missing).toEqual({
+ status: 503,
+ body: { code: 'STEER_FILE_RETENTION_FAILED' },
+ });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([]);
+ });
+
+ it('withholds a receipt replay until an older accepted steer file is retained', async () => {
+ const streamId = 'steer-req-usage-replay';
+ const clientSteerId = 'steer-usage-replay-client';
+ await GenerationJobManager.createJob(streamId, user.id);
+ const getFiles = jest.fn(async () => [dbDoc]);
+ const retained = jest.fn(async () => [dbDoc]);
+ const body = {
+ conversationId: streamId,
+ clientSteerId,
+ text: 'x',
+ files: [{ file_id: 'f1' }],
+ };
+
+ const accepted = await handleSteerRequest(user, body, {
+ getFiles,
+ updateFilesUsage: retained,
+ });
+ expect(accepted.status).toBe(202);
+
+ const failedReplay = await handleSteerRequest(user, body, {
+ getFiles,
+ updateFilesUsage: jest.fn(async () => []),
+ });
+ expect(failedReplay).toEqual({
+ status: 503,
+ body: { code: 'STEER_FILE_RETENTION_FAILED' },
+ });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toHaveLength(1);
+
+ const healedReplay = await handleSteerRequest(user, body, {
+ getFiles,
+ updateFilesUsage: retained,
+ });
+ expect(healedReplay).toEqual({
+ status: 202,
+ body: expect.objectContaining({ replayed: true, steerId: accepted.body.steerId }),
+ });
});
});
@@ -379,6 +568,351 @@ describe('handleSteerRequest (real in-memory job manager)', () => {
expect(checkAgentAccess).toHaveBeenCalledWith({ agentId: undefined, endpoint: undefined });
expect(await GenerationJobManager.steering.peek(streamId)).toHaveLength(1);
});
+
+ it('409s when the run is replaced while the agent access check is in flight', async () => {
+ const streamId = 'steer-req-agent-replacement-race';
+ const original = await GenerationJobManager.createJob(streamId, user.id);
+ let signalAccessStarted: (() => void) | undefined;
+ const accessStarted = new Promise<void>((resolve) => {
+ signalAccessStarted = resolve;
+ });
+ let releaseAccess: ((allowed: boolean) => void) | undefined;
+ const checkAgentAccess = jest.fn(
+ () =>
+ new Promise<boolean>((resolve) => {
+ releaseAccess = resolve;
+ signalAccessStarted?.();
+ }),
+ );
+
+ const requesting = handleSteerRequest(
+ user,
+ { conversationId: streamId, text: 'belongs after the newer run' },
+ { checkAgentAccess },
+ );
+ await accessStarted;
+ const replacement = await GenerationJobManager.createJob(streamId, user.id);
+ expect(replacement.createdAt).not.toBe(original.createdAt);
+ releaseAccess?.(true);
+
+ await expect(requesting).resolves.toEqual({
+ status: 409,
+ body: { code: 'RUN_REPLACED' },
+ });
+ await expect(
+ GenerationJobManager.steering.peek(streamId, replacement.createdAt),
+ ).resolves.toEqual([]);
+ });
+ });
+});
+
+describe('generation protocol bridge for steering mutations', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockIsSupported.mockReturnValue(true);
+ GenerationJobManager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ GenerationJobManager.initialize();
+ });
+
+ afterEach(async () => {
+ await GenerationJobManager.destroy();
+ });
+
+ it('caps a v2 request to a v1 job and uses only legacy queue/update surfaces', async () => {
+ const streamId = 'steer-protocol-v1-job';
+ await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: {
+ generationProtocolVersion: 1,
+ preemptCapable: true,
+ },
+ });
+ const receiptRead = jest.spyOn(GenerationJobManager.steering, 'getReceipt');
+ const receiptEnqueue = jest.spyOn(GenerationJobManager.steering, 'enqueueWithReceipt');
+ const legacyEnqueue = jest.spyOn(GenerationJobManager.steering, 'enqueueVersioned');
+ const publishUpdate = jest.spyOn(GenerationJobManager, 'emitChunkFromAnyReplica');
+
+ const result = await handleSteerRequest(
+ user,
+ {
+ conversationId: streamId,
+ clientSteerId: 'client-v1-steer',
+ text: 'legacy-safe instruction',
+ preempt: true,
+ },
+ { generationProtocolVersion: 2 },
+ );
+
+ expect(result).toEqual({
+ status: 202,
+ body: {
+ status: 'queued',
+ steerId: expect.any(String),
+ position: 1,
+ conversationId: streamId,
+ preempt: true,
+ generationProtocolVersion: 1,
+ },
+ });
+ /** Host-v2 performs a read-only lookup before the current v1 marker can
+ * cap the request, because an older v2 receipt may outlive this job. */
+ expect(receiptRead).toHaveBeenCalledTimes(1);
+ expect(receiptEnqueue).not.toHaveBeenCalled();
+ expect(legacyEnqueue).toHaveBeenCalledTimes(1);
+ expect(publishUpdate).not.toHaveBeenCalled();
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toEqual([
+ expect.not.objectContaining({ clientSteerId: expect.anything() }),
+ ]);
+ });
+
+ it('keeps v2 receipt replay and correlation broadcasts behind an exact v2 job marker', async () => {
+ const streamId = 'steer-protocol-v2-job';
+ await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const receiptEnqueue = jest.spyOn(GenerationJobManager.steering, 'enqueueWithReceipt');
+ const publishUpdate = jest
+ .spyOn(GenerationJobManager, 'emitChunkFromAnyReplica')
+ .mockResolvedValue(true);
+ const requestBody = {
+ conversationId: streamId,
+ clientSteerId: 'client-v2-steer',
+ text: 'receipt-safe instruction',
+ };
+
+ const accepted = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ });
+ const replayed = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ });
+
+ expect(accepted.body).toMatchObject({
+ status: 'queued',
+ generationProtocolVersion: 2,
+ });
+ expect(replayed.body).toMatchObject({
+ steerId: accepted.body.steerId,
+ replayed: true,
+ generationProtocolVersion: 2,
+ });
+ expect(receiptEnqueue).toHaveBeenCalledTimes(1);
+ expect(publishUpdate).toHaveBeenCalledTimes(1);
+ });
+
+ it('replays a v2 receipt after terminal cleanup deletes the accepting job', async () => {
+ const streamId = 'steer-protocol-v2-replay-after-delete';
+ const job = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const requestBody = {
+ conversationId: streamId,
+ generationCreatedAt: job.createdAt,
+ clientSteerId: 'client-v2-after-delete',
+ text: 'accept exactly once',
+ };
+ const receiptEnqueue = jest.spyOn(GenerationJobManager.steering, 'enqueueWithReceipt');
+
+ const accepted = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ });
+ await GenerationJobManager.getJobStore().deleteJob(streamId, job.createdAt);
+ const replayed = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ });
+
+ expect(replayed).toEqual({
+ status: 202,
+ body: {
+ status: 'queued',
+ steerId: accepted.body.steerId,
+ position: 1,
+ conversationId: streamId,
+ preempt: false,
+ settled: true,
+ leftover: true,
+ replayed: true,
+ generationProtocolVersion: 2,
+ },
+ });
+ expect(receiptEnqueue).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not let a later v1 job hide an earlier v2 receipt', async () => {
+ const streamId = 'steer-protocol-v2-replay-over-v1';
+ const original = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const requestBody = {
+ conversationId: streamId,
+ generationCreatedAt: original.createdAt,
+ clientSteerId: 'client-v2-before-v1',
+ text: 'keep the original acceptance',
+ };
+ const receiptEnqueue = jest.spyOn(GenerationJobManager.steering, 'enqueueWithReceipt');
+ const accepted = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ });
+
+ const replacement = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 1 },
+ });
+ expect(replacement.createdAt).not.toBe(original.createdAt);
+ const replayed = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ });
+
+ expect(replayed.body).toMatchObject({
+ steerId: accepted.body.steerId,
+ replayed: true,
+ settled: true,
+ leftover: true,
+ generationProtocolVersion: 2,
+ });
+ expect(receiptEnqueue).toHaveBeenCalledTimes(1);
+ await expect(
+ GenerationJobManager.steering.peek(streamId, replacement.createdAt),
+ ).resolves.toEqual([]);
+ });
+
+ it('rejects receipt replay for a different explicit generation', async () => {
+ const streamId = 'steer-protocol-v2-replay-generation-fence';
+ const job = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const requestBody = {
+ conversationId: streamId,
+ generationCreatedAt: job.createdAt,
+ clientSteerId: 'client-v2-generation-fence',
+ text: 'generation-bound acceptance',
+ };
+ await handleSteerRequest(user, requestBody, { generationProtocolVersion: 2 });
+ await GenerationJobManager.getJobStore().deleteJob(streamId, job.createdAt);
+
+ const stale = await handleSteerRequest(
+ user,
+ { ...requestBody, generationCreatedAt: job.createdAt + 1 },
+ { generationProtocolVersion: 2 },
+ );
+
+ expect(stale).toEqual({
+ status: 409,
+ body: { code: 'RUN_REPLACED', generationProtocolVersion: 2 },
+ });
+ });
+
+ it('replays a committed receipt even if mutable agent access is later revoked', async () => {
+ const streamId = 'steer-protocol-v2-replay-after-revocation';
+ await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: {
+ generationProtocolVersion: 2,
+ agent_id: 'agent-replay',
+ endpoint: 'agents',
+ },
+ });
+ const requestBody = {
+ conversationId: streamId,
+ clientSteerId: 'client-v2-after-revocation',
+ text: 'already accepted instruction',
+ };
+ const checkAgentAccess = jest.fn().mockResolvedValueOnce(true).mockResolvedValue(false);
+
+ const accepted = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ checkAgentAccess,
+ });
+ const replayed = await handleSteerRequest(user, requestBody, {
+ generationProtocolVersion: 2,
+ checkAgentAccess,
+ });
+
+ expect(replayed).toEqual({
+ status: 202,
+ body: expect.objectContaining({
+ steerId: accepted.body.steerId,
+ replayed: true,
+ generationProtocolVersion: 2,
+ }),
+ });
+ expect(checkAgentAccess).toHaveBeenCalledTimes(1);
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toHaveLength(1);
+ });
+
+ it('treats a missing job marker and an explicitly malformed package cap as v1', async () => {
+ const streamId = 'steer-protocol-missing-marker';
+ await GenerationJobManager.createJob(streamId, user.id);
+ const stored = await GenerationJobManager.getJobStore().getJob(streamId);
+ expect(stored).not.toBeNull();
+ delete (stored as { generationProtocolVersion?: number }).generationProtocolVersion;
+ const receiptRead = jest.spyOn(GenerationJobManager.steering, 'getReceipt');
+
+ const missingMarker = await handleSteerRequest(
+ user,
+ { conversationId: streamId, clientSteerId: 'missing-marker-id', text: 'safe fallback' },
+ { generationProtocolVersion: 2 },
+ );
+ const malformedCap = await handleSteerRequest(
+ user,
+ { conversationId: 'invalid-cap-target', text: 'no live job' },
+ { generationProtocolVersion: 9 } as unknown as { generationProtocolVersion: 2 },
+ );
+
+ expect(missingMarker.body.generationProtocolVersion).toBe(1);
+ expect(malformedCap.body.generationProtocolVersion).toBe(1);
+ expect(receiptRead).toHaveBeenCalledTimes(1);
+ });
+
+ it('keeps v1 cancel and arm off receipt mutations and versioned updates', async () => {
+ const streamId = 'steer-protocol-v1-mutations';
+ await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 1, preemptCapable: true },
+ });
+ const queued = await handleSteerRequest(
+ user,
+ { conversationId: streamId, clientSteerId: 'v1-mutation-id', text: 'queue then arm' },
+ { generationProtocolVersion: 2 },
+ );
+ const steerId = queued.body.steerId as string;
+ const receiptRead = jest.spyOn(GenerationJobManager.steering, 'getReceipt');
+ const discardLeftover = jest.spyOn(GenerationJobManager.steering, 'discardLeftover');
+ const legacyArm = jest.spyOn(GenerationJobManager.steering, 'arm');
+ const versionedArm = jest.spyOn(GenerationJobManager.steering, 'armVersioned');
+ const publishUpdate = jest.spyOn(GenerationJobManager, 'emitChunkFromAnyReplica');
+ const requestPreempt = jest.spyOn(GenerationJobManager, 'requestPreempt');
+
+ const armed = await handleSteerArm(
+ user,
+ { conversationId: streamId, steerId },
+ { generationProtocolVersion: 2 },
+ );
+ const cancelled = await handleSteerCancel(
+ user,
+ { conversationId: streamId, steerId, clientSteerId: 'v1-mutation-id' },
+ { generationProtocolVersion: 2 },
+ );
+
+ expect(armed).toEqual({
+ status: 200,
+ body: { armed: true, generationProtocolVersion: 1 },
+ });
+ expect(cancelled).toEqual({
+ status: 200,
+ body: { removed: true, generationProtocolVersion: 1 },
+ });
+ /** The v2 host probes once for an older v2 receipt before the current
+ * v1 job caps mutation semantics. This v1 enqueue created no receipt, so
+ * the probe cannot mutate or replay anything. */
+ expect(receiptRead).toHaveBeenCalledTimes(1);
+ expect(receiptRead).toHaveBeenCalledWith(streamId, 'v1-mutation-id');
+ expect(discardLeftover).not.toHaveBeenCalled();
+ expect(legacyArm).not.toHaveBeenCalled();
+ expect(versionedArm).toHaveBeenCalledTimes(1);
+ expect(requestPreempt).toHaveBeenCalledWith(streamId, steerId, expect.any(Number), 1);
+ expect(publishUpdate).not.toHaveBeenCalled();
});
});
@@ -413,6 +947,21 @@ describe('handleSteerCancel (real in-memory job manager)', () => {
expect(badId.body.code).toBe('INVALID_STEER_ID');
});
+ it('409s a cancel targeting a replaced generation and leaves the live queue intact', async () => {
+ const streamId = 'cancel-stale-generation';
+ const steerId = await queueSteer(streamId);
+ const live = await GenerationJobManager.getJob(streamId);
+
+ const result = await handleSteerCancel(user, {
+ conversationId: streamId,
+ generationCreatedAt: (live?.createdAt as number) - 1,
+ steerId,
+ });
+
+ expect(result).toEqual({ status: 409, body: { code: 'RUN_REPLACED' } });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toHaveLength(1);
+ });
+
it('removes a queued steer and reports a lost race as removed:false', async () => {
const steerId = await queueSteer('cancel-ok');
const cancelled = await handleSteerCancel(user, { conversationId: 'cancel-ok', steerId });
@@ -423,6 +972,160 @@ describe('handleSteerCancel (real in-memory job manager)', () => {
expect(again).toEqual({ status: 200, body: { removed: false } });
});
+ it('replays a successful receipt-backed cancel when the first response was lost', async () => {
+ const streamId = 'cancel-replay';
+ const clientSteerId = 'cancel-replay-client';
+ await GenerationJobManager.createJob(streamId, 'user-1');
+ const queued = await handleSteerRequest(user, {
+ conversationId: streamId,
+ clientSteerId,
+ text: 'cancel exactly once',
+ });
+ const steerId = queued.body.steerId as string;
+
+ const first = await handleSteerCancel(user, {
+ conversationId: streamId,
+ steerId,
+ clientSteerId,
+ });
+ expect(first).toEqual({ status: 200, body: { removed: true } });
+
+ const replay = await handleSteerCancel(user, {
+ conversationId: streamId,
+ steerId,
+ clientSteerId,
+ });
+ expect(replay).toEqual({ status: 200, body: { removed: true, replayed: true } });
+ await expect(
+ GenerationJobManager.steering.getReceipt(streamId, clientSteerId),
+ ).resolves.toMatchObject({
+ state: 'cancelled',
+ });
+ });
+
+ it('reclaims a queued receipt that becomes leftover before the job lookup', async () => {
+ const streamId = 'cancel-terminal-race';
+ const clientSteerId = 'cancel-terminal-race-client';
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ const queued = await handleSteerRequest(user, {
+ conversationId: streamId,
+ clientSteerId,
+ text: 'do not revive me',
+ });
+ const steerId = queued.body.steerId as string;
+ const jobStore = GenerationJobManager.getJobStore();
+ const getJobSpy = jest
+ .spyOn(GenerationJobManager, 'getJob')
+ .mockImplementationOnce(async () => {
+ await jobStore.closeAndDrainSteers(streamId, job.createdAt);
+ await jobStore.deleteJob(streamId, job.createdAt);
+ return undefined;
+ });
+
+ try {
+ const cancelled = await handleSteerCancel(user, {
+ conversationId: streamId,
+ steerId,
+ clientSteerId,
+ });
+
+ expect(cancelled).toEqual({ status: 200, body: { removed: true, replayed: true } });
+ await expect(
+ GenerationJobManager.steering.getReceipt(streamId, clientSteerId),
+ ).resolves.toMatchObject({
+ state: 'cancelled',
+ });
+ await expect(
+ GenerationJobManager.steering.claim(streamId, { userId: 'user-1' }),
+ ).resolves.toEqual([]);
+ } finally {
+ getJobSpy.mockRestore();
+ }
+ });
+
+ it('does not settle a terminal receipt from a different explicit generation', async () => {
+ const streamId = 'cancel-terminal-replaced-receipt';
+ const clientSteerId = 'cancel-terminal-replaced-receipt-client';
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ const queued = await handleSteerRequest(user, {
+ conversationId: streamId,
+ generationCreatedAt: job.createdAt,
+ clientSteerId,
+ text: 'belongs to the newer generation',
+ });
+ const steerId = queued.body.steerId as string;
+ const jobStore = GenerationJobManager.getJobStore();
+ await jobStore.closeAndDrainSteers(streamId, job.createdAt);
+ await jobStore.deleteJob(streamId, job.createdAt);
+
+ const result = await handleSteerCancel(user, {
+ conversationId: streamId,
+ generationCreatedAt: job.createdAt - 1,
+ steerId,
+ clientSteerId,
+ });
+
+ expect(result).toEqual({ status: 409, body: { code: 'RUN_REPLACED' } });
+ await expect(
+ GenerationJobManager.steering.getReceipt(streamId, clientSteerId),
+ ).resolves.toMatchObject({ state: 'leftover', generationCreatedAt: job.createdAt });
+ await expect(
+ GenerationJobManager.steering.claim(streamId, { userId: 'user-1' }),
+ ).resolves.toEqual([expect.objectContaining({ steerId })]);
+ });
+
+ it('discards an older v2 leftover even when the current replacement job is v1', async () => {
+ const streamId = 'cancel-v2-leftover-after-v1-replacement';
+ const clientSteerId = 'cancel-v2-leftover-client';
+ const predecessor = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const queued = await handleSteerRequest(
+ user,
+ {
+ conversationId: streamId,
+ generationCreatedAt: predecessor.createdAt,
+ clientSteerId,
+ text: 'do not recover this',
+ },
+ { generationProtocolVersion: 2 },
+ );
+ const steerId = queued.body.steerId as string;
+ await expect(
+ GenerationJobManager.completeJob(streamId, undefined, predecessor.createdAt),
+ ).resolves.toBe(true);
+ const replacement = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 1 },
+ });
+
+ const result = await handleSteerCancel(
+ user,
+ {
+ conversationId: streamId,
+ generationCreatedAt: predecessor.createdAt,
+ steerId,
+ clientSteerId,
+ },
+ { generationProtocolVersion: 2 },
+ );
+
+ expect(result).toEqual({
+ status: 200,
+ body: { removed: true, replayed: true, generationProtocolVersion: 2 },
+ });
+ await expect(
+ GenerationJobManager.steering.getReceipt(streamId, clientSteerId),
+ ).resolves.toMatchObject({
+ state: 'cancelled',
+ generationCreatedAt: predecessor.createdAt,
+ });
+ await expect(GenerationJobManager.getJob(streamId)).resolves.toMatchObject({
+ status: 'running',
+ createdAt: replacement.createdAt,
+ metadata: { generationProtocolVersion: 1 },
+ });
+ });
+
it('treats a missing job as a lost race, not an error', async () => {
const result = await handleSteerCancel(user, { conversationId: 'gone', steerId: 's1' });
expect(result).toEqual({ status: 200, body: { removed: false } });
@@ -469,6 +1172,17 @@ describe('handleSteerCancel (real in-memory job manager)', () => {
expect(result.status).toBe(403);
expect((await GenerationJobManager.steering.peek('cancel-foreign')).length).toBe(1);
});
+
+ it('fails closed when the stored job owner is missing', async () => {
+ const streamId = 'cancel-missing-owner';
+ const steerId = await queueSteer(streamId);
+ await removeStoredJobOwner(streamId);
+
+ const result = await handleSteerCancel(user, { conversationId: streamId, steerId });
+
+ expect(result).toEqual({ status: 403, body: { code: 'UNAUTHORIZED' } });
+ await expect(GenerationJobManager.steering.peek(streamId)).resolves.toHaveLength(1);
+ });
});
describe('preempt flag on the steer request', () => {
@@ -561,8 +1275,8 @@ describe('preempt flag on the steer request', () => {
preempt: true,
});
- expect(result.status).toBe(404);
- expect(result.body.code).toBe('NO_ACTIVE_RUN');
+ expect(result.status).toBe(409);
+ expect(result.body.code).toBe('RUN_REPLACED');
/** Nothing reached the replacement's queue. */
expect(await GenerationJobManager.steering.peek(streamId)).toEqual([]);
} finally {
@@ -596,7 +1310,7 @@ describe('preempt flag on the steer request', () => {
expect(result.status).toBe(202);
expect(result.body.preempt).toBe(true);
- expect(spy).toHaveBeenCalledWith(streamId, expect.any(String), expect.any(Number));
+ expect(spy).toHaveBeenCalledWith(streamId, expect.any(String), expect.any(Number), 1);
/** Still durable, so the boundary drain will find it either way. */
expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);
} finally {
@@ -733,3 +1447,250 @@ describe('preempt flag on the steer request', () => {
expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
});
});
+
+/**
+ * Escalation of a waiting steer is ONE atomic in-place flag flip — the item
+ * keeps its FIFO position, id, and timestamp, so the whole queue still drains
+ * in the user's instruction order at the seal, and no reclaim window exists.
+ */
+describe('handleSteerArm (real in-memory job manager)', () => {
+ function createCapableJob(streamId: string) {
+ return GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
+ }
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockIsSupported.mockReturnValue(true);
+ mockIsPreemptSupported.mockReturnValue(true);
+ GenerationJobManager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ GenerationJobManager.initialize();
+ });
+
+ afterEach(async () => {
+ await GenerationJobManager.destroy();
+ });
+
+ it('400s on invalid input', async () => {
+ expect((await handleSteerArm(user, { steerId: 's1' })).status).toBe(400);
+ const badId = await handleSteerArm(user, { conversationId: 'c1', steerId: '' });
+ expect(badId.status).toBe(400);
+ expect(badId.body.code).toBe('INVALID_STEER_ID');
+ });
+
+ it('409s an arm targeting a replaced generation without relabelling the live steer', async () => {
+ const streamId = 'arm-stale-client-generation';
+ const job = await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'still an ordinary steer',
+ });
+
+ const result = await handleSteerArm(user, {
+ conversationId: streamId,
+ generationCreatedAt: job.createdAt - 1,
+ steerId: posted.body.steerId as string,
+ });
+
+ expect(result).toEqual({ status: 409, body: { code: 'RUN_REPLACED' } });
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ it('arms a queued steer in place: same item, same position, now preempting', async () => {
+ const streamId = 'arm-in-place';
+ await createCapableJob(streamId);
+ const first = await handleSteerRequest(user, { conversationId: streamId, text: 'first' });
+ const second = await handleSteerRequest(user, { conversationId: streamId, text: 'second' });
+ const firstId = first.body.steerId as string;
+
+ const armed = await handleSteerArm(user, { conversationId: streamId, steerId: firstId });
+ expect(armed).toEqual({ status: 200, body: { armed: true, preemptRevision: 1 } });
+
+ /** FIFO preserved: the escalated steer still drains FIRST at the seal. */
+ const queue = await GenerationJobManager.steering.peek(streamId);
+ expect(queue.map((item) => item.steerId)).toEqual([firstId, second.body.steerId]);
+ expect(queue[0].preempt).toBe(true);
+ expect(queue[1].preempt).toBeUndefined();
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+ });
+
+ it('is idempotent when a client retries after losing the first response', async () => {
+ const streamId = 'arm-idempotent-retry';
+ await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, { conversationId: streamId, text: 'once only' });
+ const steerId = posted.body.steerId as string;
+
+ await expect(handleSteerArm(user, { conversationId: streamId, steerId })).resolves.toEqual({
+ status: 200,
+ body: { armed: true, preemptRevision: 1 },
+ });
+ await expect(handleSteerArm(user, { conversationId: streamId, steerId })).resolves.toEqual({
+ status: 200,
+ body: { armed: true, preemptRevision: 2 },
+ });
+
+ const queue = await GenerationJobManager.steering.peek(streamId);
+ expect(queue).toHaveLength(1);
+ expect(queue[0]).toMatchObject({ steerId, text: 'once only', preempt: true });
+ });
+
+ it('uses the durable arm revision internally when a v1 job regains capability', async () => {
+ const streamId = 'arm-v1-revision-after-capability-handover';
+ const job = await GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { generationProtocolVersion: 1, preemptCapable: true },
+ });
+ const posted = await handleSteerRequest(
+ user,
+ { conversationId: streamId, text: 'interrupt after handover' },
+ { generationProtocolVersion: 2 },
+ );
+ const steerId = posted.body.steerId as string;
+
+ const firstArm = await GenerationJobManager.steering.armVersioned(
+ streamId,
+ steerId,
+ job.createdAt,
+ );
+ expect(firstArm).toMatchObject({ outcome: 'armed', revision: 1 });
+ await GenerationJobManager.requestPreempt(streamId, steerId, job.createdAt, 1);
+
+ await GenerationJobManager.updateMetadata(streamId, { preemptCapable: false });
+ await expect(GenerationJobManager.rearmQueuedPreempts(streamId, job.createdAt)).resolves.toBe(
+ 0,
+ );
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ await GenerationJobManager.updateMetadata(streamId, { preemptCapable: true });
+
+ const rearmed = await handleSteerArm(
+ user,
+ { conversationId: streamId, generationCreatedAt: job.createdAt, steerId },
+ { generationProtocolVersion: 2 },
+ );
+
+ expect(rearmed).toEqual({
+ status: 200,
+ body: { armed: true, generationProtocolVersion: 1 },
+ });
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+ await expect(GenerationJobManager.steering.peek(streamId, job.createdAt)).resolves.toEqual([
+ expect.objectContaining({ steerId, preempt: true, preemptRevision: 3 }),
+ ]);
+ });
+
+ it('refuses to relabel when the owner cannot seal', async () => {
+ const streamId = 'arm-incapable';
+ await GenerationJobManager.createJob(streamId, user.id);
+ const posted = await handleSteerRequest(user, { conversationId: streamId, text: 'plain' });
+ const steerId = posted.body.steerId as string;
+
+ const result = await handleSteerArm(user, { conversationId: streamId, steerId });
+ expect(result).toEqual({ status: 200, body: { armed: false, code: 'PREEMPT_UNSUPPORTED' } });
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ it('refuses to arm a queued steer after the run pauses', async () => {
+ const streamId = 'arm-paused';
+ await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, { conversationId: streamId, text: 'waiting' });
+ const payload = buildToolApprovalPayload([
+ { name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call_arm_paused' },
+ ]);
+ const action = buildPendingAction(payload, {
+ streamId,
+ conversationId: streamId,
+ runId: 'run-arm-paused',
+ responseMessageId: 'msg-arm-paused',
+ });
+ expect(await GenerationJobManager.approvals.pause(streamId, action)).toBe(true);
+
+ const result = await handleSteerArm(user, {
+ conversationId: streamId,
+ steerId: posted.body.steerId as string,
+ });
+
+ expect(result).toEqual({ status: 200, body: { armed: false } });
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ it('reports armed:false when the steer already left the queue', async () => {
+ const streamId = 'arm-too-late';
+ await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, { conversationId: streamId, text: 'gone soon' });
+ const steerId = posted.body.steerId as string;
+ await GenerationJobManager.steering.cancel(streamId, steerId);
+
+ const result = await handleSteerArm(user, { conversationId: streamId, steerId });
+ expect(result).toEqual({ status: 200, body: { armed: false } });
+ });
+
+ it('treats a missing job as a lost race, not an error', async () => {
+ const result = await handleSteerArm(user, { conversationId: 'gone', steerId: 's1' });
+ expect(result).toEqual({ status: 200, body: { armed: false } });
+ });
+
+ it("403s another user's run", async () => {
+ const streamId = 'arm-foreign';
+ await GenerationJobManager.createJob(streamId, 'someone-else');
+ const result = await handleSteerArm(user, { conversationId: streamId, steerId: 'x' });
+ expect(result.status).toBe(403);
+ });
+
+ it('fails closed when the stored job owner is missing', async () => {
+ const streamId = 'arm-missing-owner';
+ await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'do not arm from malformed state',
+ });
+ await removeStoredJobOwner(streamId);
+
+ const result = await handleSteerArm(user, {
+ conversationId: streamId,
+ steerId: posted.body.steerId as string,
+ });
+
+ expect(result).toEqual({ status: 403, body: { code: 'UNAUTHORIZED' } });
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ });
+
+ it('refuses when capability was rewritten for the same generation mid-flight', async () => {
+ /** A HITL resume on a rolling deploy rewrites `preemptCapable` for the
+ * SAME createdAt, so the capability must live inside the atomic store
+ * predicate — a value read before the arm is not trustworthy. */
+ const streamId = 'arm-capability-rewritten';
+ await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, { conversationId: streamId, text: 'waiting' });
+ await GenerationJobManager.updateMetadata(streamId, { preemptCapable: false });
+
+ const result = await handleSteerArm(user, {
+ conversationId: streamId,
+ steerId: posted.body.steerId as string,
+ });
+ expect(result).toEqual({ status: 200, body: { armed: false, code: 'PREEMPT_UNSUPPORTED' } });
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ });
+
+ it("never arms another generation's steer", async () => {
+ const streamId = 'arm-stale-generation';
+ await createCapableJob(streamId);
+ const posted = await handleSteerRequest(user, { conversationId: streamId, text: 'target' });
+ const live = await GenerationJobManager.getJob(streamId);
+
+ const armed = await GenerationJobManager.steering.arm(
+ streamId,
+ posted.body.steerId as string,
+ (live?.createdAt as number) + 999,
+ );
+ expect(armed).toBe('missing');
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ });
+});
diff --git a/packages/api/src/agents/steering/__tests__/runtime.spec.ts b/packages/api/src/agents/steering/__tests__/runtime.spec.ts
index a51242fd6e..f4c471cb58 100644
--- a/packages/api/src/agents/steering/__tests__/runtime.spec.ts
+++ b/packages/api/src/agents/steering/__tests__/runtime.spec.ts
@@ -143,7 +143,7 @@ describe('createSteerDrainHook', () => {
]);
});
- it('still injects when applySteer throws', async () => {
+ it('restores the claimed steer and does not inject when durable apply fails', async () => {
const streamId = `drain-apply-error-${Date.now()}`;
const job = await GenerationJobManager.createJob(streamId, 'user-1');
await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'survives'));
@@ -157,8 +157,9 @@ describe('createSteerDrainHook', () => {
});
const output: SteerDrainOutput = await hook(batchInput(), abortSignal);
- expect(output.injectedMessages).toEqual([
- { role: 'user', content: 'survives', source: 'steer' },
+ expect(output).toEqual({});
+ expect((await GenerationJobManager.steering.peek(streamId)).map((item) => item.text)).toEqual([
+ 'survives',
]);
});
@@ -196,7 +197,7 @@ describe('createSteerDrainHook', () => {
const output: SteerDrainOutput = await hook(batchInput(), abortSignal);
// buildMedia is consulted only for items that carry files.
expect(buildMedia).toHaveBeenCalledTimes(1);
- expect(calls).toEqual(['apply:s1', 'media:s1', 'apply:s2']);
+ expect(calls).toEqual(['apply:s1', 'apply:s2', 'media:s1']);
expect(output.injectedMessages).toEqual([
{ role: 'user', content: media.content, source: 'steer' },
{ role: 'user', content: 'text only', source: 'steer' },
@@ -467,7 +468,7 @@ describe('createSteerPreemptBoundaryHook', () => {
]);
});
- it('clears the request even when applySteer throws mid-drain', async () => {
+ it('restores the item and keeps its preempt request armed when durable apply fails', async () => {
const streamId = `preempt-clears-on-error-${Date.now()}`;
const job = await GenerationJobManager.createJob(streamId, 'user-1');
await GenerationJobManager.steering.enqueue(streamId, {
@@ -485,10 +486,11 @@ describe('createSteerPreemptBoundaryHook', () => {
});
const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);
- expect(output.injectedMessages).toEqual([
- { role: 'user', content: 'still injected', source: 'steer' },
+ expect(output).toEqual({});
+ expect((await GenerationJobManager.steering.peek(streamId)).map((item) => item.text)).toEqual([
+ 'still injected',
]);
- expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
});
});
diff --git a/packages/api/src/agents/steering/index.ts b/packages/api/src/agents/steering/index.ts
index 01c8a68fcb..35eb3fbd7f 100644
--- a/packages/api/src/agents/steering/index.ts
+++ b/packages/api/src/agents/steering/index.ts
@@ -9,6 +9,7 @@ export type { SteerDrainHookOptions, SteerMediaResult } from './runtime';
export {
handleSteerRequest,
handleSteerCancel,
+ handleSteerArm,
getSteerMaxLength,
STEER_MAX_FILES,
} from './request';
diff --git a/packages/api/src/agents/steering/request.ts b/packages/api/src/agents/steering/request.ts
index b37b1acef4..4a6f5625cc 100644
--- a/packages/api/src/agents/steering/request.ts
+++ b/packages/api/src/agents/steering/request.ts
@@ -1,8 +1,18 @@
-import { randomUUID } from 'crypto';
+import { createHash, randomUUID } from 'crypto';
import { logger } from '@librechat/data-schemas';
+import { SteerEvents } from 'librechat-data-provider';
import type { TFile } from 'librechat-data-provider';
+import type {
+ GenerationProtocolVersion,
+ SteerQueueItem,
+ SteerReceipt,
+} from '~/stream/interfaces/IJobStore';
import type { SteerFileFetcher } from './media';
-import { STEER_ENQUEUE_NOT_RUNNING, STEER_ENQUEUE_QUEUE_FULL } from '~/stream/interfaces/IJobStore';
+import {
+ STEER_ENQUEUE_NOT_RUNNING,
+ STEER_ENQUEUE_QUEUE_FULL,
+ STEER_ENQUEUE_RECEIPT_FULL,
+} from '~/stream/interfaces/IJobStore';
import { toSteerFileRef, collectFileIds, buildOwnerFilter } from './refs';
import { GenerationJobManager } from '~/stream/GenerationJobManager';
import { isSteeringSupported } from './runtime';
@@ -31,7 +41,9 @@ export interface SteerRequestUser {
export interface SteerRequestBody {
conversationId?: unknown;
+ generationCreatedAt?: unknown;
text?: unknown;
+ clientSteerId?: unknown;
files?: unknown;
/** Ask the generating replica to seal the live model stream at the next
* provider-safe boundary instead of waiting for a tool step. NEVER a
@@ -42,7 +54,9 @@ export interface SteerRequestBody {
export interface SteerCancelBody {
conversationId?: unknown;
+ generationCreatedAt?: unknown;
steerId?: unknown;
+ clientSteerId?: unknown;
}
/** HTTP-shaped outcome the thin route wrapper serializes verbatim. */
@@ -51,6 +65,47 @@ export interface SteerRequestResult {
body: Record<string, unknown>;
}
+/** Protocol-only options shared by cancel/arm, which do not need the steer
+ * request's file and agent-authorization dependencies. */
+export interface SteerProtocolOptions {
+ generationProtocolVersion?: GenerationProtocolVersion;
+}
+
+interface SteerProtocolContext {
+ value: GenerationProtocolVersion;
+ expose: boolean;
+}
+
+/** `undefined` means an older direct package caller, not an HTTP request. The
+ * LibreChat wrapper always supplies this field, so malformed/missing wire
+ * markers have already become an explicit v1 before entering the package. */
+function protocolFromOptions(options: SteerProtocolOptions): GenerationProtocolVersion {
+ if (!Object.prototype.hasOwnProperty.call(options, 'generationProtocolVersion')) {
+ return 2;
+ }
+ return options.generationProtocolVersion === 2 ? 2 : 1;
+}
+
+function capProtocolToJob(
+ protocol: SteerProtocolContext,
+ job: { metadata?: { generationProtocolVersion?: unknown } } | null | undefined,
+): void {
+ protocol.value = protocol.value === 2 && job?.metadata?.generationProtocolVersion === 2 ? 2 : 1;
+}
+
+function attachProtocol(
+ result: SteerRequestResult,
+ protocol: SteerProtocolContext,
+): SteerRequestResult {
+ if (!protocol.expose) {
+ return result;
+ }
+ return {
+ ...result,
+ body: { ...result.body, generationProtocolVersion: protocol.value },
+ };
+}
+
/** The originating run's identity, from job metadata (never the request). */
export interface SteerRunContext {
agentId?: string;
@@ -63,6 +118,10 @@ export interface SteerRunContext {
* behavior); the LibreChat route wires all three.
*/
export interface SteerRequestDeps {
+ /** Upper protocol bound selected by the HTTP host from the request markers
+ * and its rollout gate. Direct package callers that omit it retain the
+ * current-package (v2) behavior. */
+ generationProtocolVersion?: GenerationProtocolVersion;
/** Owner-scoped file fetch (`db.getFiles`-shaped). When present, every
* client-supplied ref must resolve to an owned DB doc at enqueue and the
* queued refs are replaced with DB-derived ones. */
@@ -84,6 +143,16 @@ interface SanitizedFilesResult {
error?: 'TOO_MANY_FILES' | 'INVALID_FILES';
}
+function parseExpectedGenerationCreatedAt(value: unknown): { value?: number; invalid?: true } {
+ if (value == null) {
+ return {};
+ }
+ if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) {
+ return { invalid: true };
+ }
+ return { value };
+}
+
/** Sanitizes client-supplied attachment refs via the shared ref picker;
* a single unusable entry rejects the whole list (fail-loud for the client). */
function sanitizeSteerFiles(rawFiles: unknown): SanitizedFilesResult {
@@ -148,6 +217,109 @@ function hasTenantMismatch(
return metadata?.tenantId != null && metadata.tenantId !== user.tenantId;
}
+function steerFingerprint(
+ text: string,
+ files: Partial<TFile>[] | undefined,
+ preempt: boolean,
+): string {
+ return createHash('sha256')
+ .update(JSON.stringify({ text, files: files ?? [], preempt }))
+ .digest('base64url');
+}
+
+function receiptResponse(conversationId: string, receipt: SteerReceipt): SteerRequestResult {
+ return {
+ status: 202,
+ body: {
+ status: 'queued',
+ steerId: receipt.item.steerId,
+ position: receipt.position,
+ conversationId,
+ preempt: receipt.item.preempt === true,
+ settled: receipt.state !== 'queued' && receipt.state !== 'claimed',
+ leftover: receipt.state === 'leftover',
+ replayed: true,
+ ...(receipt.item.preemptRevision != null && {
+ preemptRevision: receipt.item.preemptRevision,
+ }),
+ },
+ };
+}
+
+/**
+ * Makes every attachment durable before a fresh steer can enter the queue.
+ *
+ * This is also run for receipt replays: an older process may have committed
+ * the queue+receipt and disappeared before clearing the upload TTL. A 202 must
+ * therefore mean both the steer and every referenced file are durable, not
+ * merely that a best-effort background write was launched.
+ */
+async function markSteerFilesUsed(
+ streamId: string,
+ item: Pick<SteerQueueItem, 'files'>,
+ owner: SteerRequestUser,
+ deps: SteerRequestDeps,
+): Promise<boolean> {
+ if (!deps.updateFilesUsage) {
+ return true;
+ }
+ const fileIds = collectFileIds(item.files ?? []);
+ if (fileIds.length === 0) {
+ return true;
+ }
+
+ try {
+ const updated = await deps.updateFilesUsage(
+ fileIds.map((file_id) => ({ file_id })),
+ undefined,
+ { user: owner.id, tenantId: owner.tenantId },
+ );
+ const updatedIds = new Set(
+ updated.flatMap((file) =>
+ file != null &&
+ typeof file === 'object' &&
+ 'file_id' in file &&
+ typeof file.file_id === 'string'
+ ? [file.file_id]
+ : [],
+ ),
+ );
+ if (fileIds.every((fileId) => updatedIds.has(fileId))) {
+ return true;
+ }
+ logger.warn(
+ `[handleSteerRequest] Failed to retain every steer file: ${streamId} ` +
+ `expected=${fileIds.length} retained=${updatedIds.size}`,
+ );
+ } catch (error) {
+ logger.warn(`[handleSteerRequest] Failed to mark steer files used: ${streamId}`, error);
+ }
+ return false;
+}
+
+function publishSteerUpdate(streamId: string, createdAt: number, item: SteerQueueItem): void {
+ void GenerationJobManager.emitChunkFromAnyReplica(
+ streamId,
+ {
+ event: SteerEvents.ON_STEER_UPDATED,
+ data: {
+ conversationId: streamId,
+ steers: [
+ {
+ steerId: item.steerId,
+ ...(item.clientSteerId && { clientSteerId: item.clientSteerId }),
+ preempt: item.preempt === true,
+ preemptRevision: item.preemptRevision ?? 0,
+ },
+ ],
+ },
+ },
+ createdAt,
+ ).catch((error) =>
+ logger.warn(`[steering] Failed to publish steer update for ${streamId}`, error),
+ );
+}
+
/**
* The full steer-request guard ladder, typed and host-agnostic: validation,
* capability gate, job ownership/state checks, and the status-guarded enqueue.
@@ -162,11 +334,28 @@ export async function handleSteerRequest(
user: SteerRequestUser,
body: SteerRequestBody,
deps: SteerRequestDeps = {},
+): Promise<SteerRequestResult> {
+ const protocol: SteerProtocolContext = {
+ value: protocolFromOptions(deps),
+ expose: Object.prototype.hasOwnProperty.call(deps, 'generationProtocolVersion'),
+ };
+ return attachProtocol(await handleSteerRequestInternal(user, body, deps, protocol), protocol);
+}
+
+async function handleSteerRequestInternal(
+ user: SteerRequestUser,
+ body: SteerRequestBody,
+ deps: SteerRequestDeps,
+ protocol: SteerProtocolContext,
): Promise<SteerRequestResult> {
const conversationId = body.conversationId;
if (typeof conversationId !== 'string' || !conversationId || conversationId === 'new') {
return { status: 400, body: { code: 'INVALID_CONVERSATION' } };
}
+ const expectedGeneration = parseExpectedGenerationCreatedAt(body.generationCreatedAt);
+ if (expectedGeneration.invalid) {
+ return { status: 400, body: { code: 'INVALID_GENERATION_IDENTITY' } };
+ }
if (typeof body.text !== 'string') {
return { status: 400, body: { code: 'EMPTY_TEXT' } };
@@ -180,6 +369,17 @@ export async function handleSteerRequest(
return { status: 413, body: { code: 'STEER_TOO_LONG', maxLength } };
}
+ const clientSteerId = body.clientSteerId;
+ if (
+ clientSteerId != null &&
+ (typeof clientSteerId !== 'string' ||
+ clientSteerId.length === 0 ||
+ clientSteerId.length > 128 ||
+ !/^[A-Za-z0-9_-]+$/.test(clientSteerId))
+ ) {
+ return { status: 400, body: { code: 'INVALID_CLIENT_STEER_ID' } };
+ }
+
const { files, error: filesError } = sanitizeSteerFiles(body.files);
if (filesError) {
return { status: 400, body: { code: filesError } };
@@ -187,17 +387,74 @@ export async function handleSteerRequest(
/** streamId === conversationId for resumable agent jobs */
const streamId = conversationId;
+ const wantsPreempt = body.preempt === true;
+ const fingerprint = steerFingerprint(text, files, wantsPreempt);
+
+ /** A durable receipt is authoritative even after its accepting job was
+ * deleted or replaced. Read it before capping to the current job marker:
+ * otherwise a terminal cleanup (no job) or a later v1 generation hides the
+ * lost-ACK proof and the client can resend already-accepted text. The host
+ * and request still have to advertise v2 before any receipt surface is
+ * touched; the receipt itself then proves that its generation was v2. */
const job = await GenerationJobManager.getJob(streamId);
- if (!job || job.status === 'complete' || job.status === 'error' || job.status === 'aborted') {
+ if (protocol.value === 2 && typeof clientSteerId === 'string') {
+ const receipt = await GenerationJobManager.steering.getReceipt(streamId, clientSteerId);
+ if (receipt != null) {
+ if (receipt.userId !== (user.id ?? '') || hasTenantMismatch(receipt, user)) {
+ return { status: 403, body: { code: 'UNAUTHORIZED' } };
+ }
+ /** This is observation of an already-committed action, not a new agent
+ * mutation. Re-running mutable agent ACL here can turn a lost 202 into
+ * a definite 403 even though the queued words can still apply, causing
+ * the client to expose resend controls and duplicate them. Stable
+ * receipt ownership/tenant identity is the replay authorization. */
+ if (
+ expectedGeneration.value != null &&
+ receipt.generationCreatedAt !== expectedGeneration.value
+ ) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
+ if (receipt.fingerprint !== fingerprint) {
+ return { status: 409, body: { code: 'STEER_IDEMPOTENCY_CONFLICT' } };
+ }
+ if (
+ !(await markSteerFilesUsed(
+ streamId,
+ receipt.item,
+ { id: receipt.userId, tenantId: receipt.tenantId },
+ deps,
+ ))
+ ) {
+ return { status: 503, body: { code: 'STEER_FILE_RETENTION_FAILED' } };
+ }
+ return receiptResponse(conversationId, receipt);
+ }
+ }
+
+ if (job != null) {
+ capProtocolToJob(protocol, job);
+ }
+
+ if (!job) {
+ /** No live marker and no v2 receipt proof is the pre-rollout shape. */
+ protocol.value = 1;
return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
}
- if (job.metadata?.userId && job.metadata.userId !== user.id) {
+ if (job.status === 'complete' || job.status === 'error' || job.status === 'aborted') {
+ return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
+ }
+ // Job creation always records an owner. Missing/corrupt ownership must fail
+ // closed rather than authorizing anyone who knows the conversation id.
+ if (job.metadata?.userId !== user.id) {
logger.warn(`[handleSteerRequest] Unauthorized steer attempt for ${streamId} by ${user.id}`);
return { status: 403, body: { code: 'UNAUTHORIZED' } };
}
if (hasTenantMismatch(job.metadata, user)) {
return { status: 403, body: { code: 'UNAUTHORIZED' } };
}
+ if (expectedGeneration.value != null && job.createdAt !== expectedGeneration.value) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
/** A steer is model-bound instruction text for the ORIGINATING agent, so it
* must clear the same role/ACL gates the chat path runs — revoked access
@@ -225,17 +482,14 @@ export async function handleSteerRequest(
}
let queuedFiles = files;
- let resolvedFileIds: string[] | undefined;
if (files && deps.getFiles) {
const resolved = await resolveSteerFiles(files, user, deps.getFiles);
if (resolved.error) {
return { status: 400, body: { code: resolved.error } };
}
queuedFiles = resolved.files;
- resolvedFileIds = resolved.fileIds;
}
- const wantsPreempt = body.preempt === true;
/**
* The OWNER's recorded capability, not this replica's probe: a steer can
* land on any replica, so during a rolling deploy a local probe would
@@ -264,21 +518,30 @@ export async function handleSteerRequest(
* tenant, paused-state, agent ACL — was evaluated against `job`, so a
* different generation here is a run this request was never authorized
* against, and accepting into it would carry the wrong agent's metadata.
- * Refuse rather than re-derive: the run the caller targeted is gone, and
- * `NO_ACTIVE_RUN` is the code the client already turns into a queued
- * follow-up.
+ * Refuse rather than re-derive: the run the caller targeted is gone. A live
+ * replacement returns `RUN_REPLACED` so a stale tab queues behind it instead
+ * of normal-sending and replacing it again; a terminal replacement remains
+ * the ordinary `NO_ACTIVE_RUN` fallback.
*/
if (owner.createdAt !== job.createdAt) {
- return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
+ return owner.status === 'running' || owner.status === 'requires_action'
+ ? { status: 409, body: { code: 'RUN_REPLACED' } }
+ : { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
}
- const preemptCapable = wantsPreempt && owner.metadata?.preemptCapable === true;
- const item = {
+ /** Normal sends make resolved uploads durable before model execution. Do
+ * the same before enqueue: once a 202 is visible, neither an upload-window
+ * sweep nor a process crash may turn the accepted steer into text-only
+ * history. A failure here commits no fresh queue item, so retry is safe. */
+ if (!(await markSteerFilesUsed(streamId, { files: queuedFiles }, user, deps))) {
+ return { status: 503, body: { code: 'STEER_FILE_RETENTION_FAILED' } };
+ }
+ const item: SteerQueueItem = {
steerId: randomUUID(),
+ ...(protocol.value === 2 && typeof clientSteerId === 'string' && { clientSteerId }),
text,
userId: user.id ?? '',
createdAt: Date.now(),
...(queuedFiles && { files: queuedFiles }),
- ...(preemptCapable && { preempt: true }),
};
/**
* Fenced to the generation the capability decision was made against. The
@@ -290,13 +553,95 @@ export async function handleSteerRequest(
* the run the caller was told about is gone, and `NO_ACTIVE_RUN` is what
* the client already handles by converting to a queued follow-up.
*/
- const depth = await GenerationJobManager.steering.enqueue(streamId, item, owner.createdAt);
+ let depth: number;
+ let persistedItem: SteerQueueItem = item;
+ let preemptRevision: number | undefined;
+ if (protocol.value === 2 && typeof clientSteerId === 'string') {
+ const result = await GenerationJobManager.steering.enqueueWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId,
+ fingerprint,
+ userId: user.id ?? '',
+ ...(user.tenantId && { tenantId: user.tenantId }),
+ ...(job.metadata?.agent_id && { agentId: job.metadata.agent_id }),
+ ...(job.metadata?.endpoint && { endpoint: job.metadata.endpoint }),
+ generationCreatedAt: owner.createdAt,
+ },
+ wantsPreempt,
+ owner.createdAt,
+ );
+ if (typeof result === 'number') {
+ depth = result;
+ } else {
+ if (!('fingerprint' in result) || result.fingerprint !== fingerprint) {
+ return { status: 409, body: { code: 'STEER_IDEMPOTENCY_CONFLICT' } };
+ }
+ if (result.userId !== (user.id ?? '') || hasTenantMismatch(result, user)) {
+ return { status: 403, body: { code: 'UNAUTHORIZED' } };
+ }
+ if (
+ expectedGeneration.value != null &&
+ result.generationCreatedAt !== expectedGeneration.value
+ ) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
+ if (result.item.steerId !== item.steerId) {
+ return receiptResponse(conversationId, result);
+ }
+ depth = result.position;
+ persistedItem = result.item;
+ preemptRevision = result.item.preemptRevision;
+ }
+ } else {
+ const result = await GenerationJobManager.steering.enqueueVersioned(
+ streamId,
+ item,
+ wantsPreempt,
+ owner.createdAt,
+ );
+ if (typeof result === 'number') {
+ depth = result;
+ } else {
+ depth = result.position;
+ persistedItem = result.item;
+ preemptRevision = result.item.preemptRevision;
+ }
+ }
if (depth === STEER_ENQUEUE_NOT_RUNNING) {
+ /** The guarded enqueue may lose to a pause after the earlier job read
+ * (agent ACL and attachment resolution can both await). Preserve the
+ * paused contract for this same generation: clients queue on 409, while
+ * 404 intentionally converts terminal/replaced runs into normal sends. */
+ const liveJob = await GenerationJobManager.getJob(streamId);
+ if (liveJob?.createdAt === owner.createdAt && liveJob.status === 'requires_action') {
+ return { status: 409, body: { code: 'RUN_PAUSED' } };
+ }
+ if (
+ liveJob != null &&
+ liveJob.createdAt !== owner.createdAt &&
+ (liveJob.status === 'running' || liveJob.status === 'requires_action')
+ ) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
}
if (depth === STEER_ENQUEUE_QUEUE_FULL) {
return { status: 429, body: { code: 'STEER_QUEUE_FULL' } };
}
+ if (depth === STEER_ENQUEUE_RECEIPT_FULL) {
+ return { status: 429, body: { code: 'STEER_RECEIPT_LIMIT' } };
+ }
+
+ const preemptArmed = persistedItem.preempt === true;
+ /** Every modern enqueue publishes correlation, including ordinary steers
+ * and incapable preempt requests. A lost 202 then promotes the optimistic
+ * client id to the accepted server item instead of exposing unsafe local
+ * edit/queue/remove actions for words the server already owns. */
+ if (protocol.value === 2 && persistedItem.clientSteerId != null) {
+ publishSteerUpdate(streamId, owner.createdAt, persistedItem);
+ }
/**
* Strictly AFTER a successful enqueue: an armed request whose steer never
@@ -315,57 +660,46 @@ export async function handleSteerRequest(
* successful enqueue. Everything past that degrades to the documented
* fallback of injecting at the next tool boundary.
*/
- if (preemptCapable) {
+ if (preemptArmed) {
/**
* NOT awaited. The answer no longer depends on it — the 202 reports
* `preemptCapable`, not delivery — so awaiting only exposes the caller to
- * Redis latency after the queue item is already durable. A client that
- * times out on a stalled publish and retries mints a SECOND steer while
- * the first stays queued, injecting the same instruction twice; a lost
- * publish merely takes the documented tool-boundary fallback. The
- * asymmetry is the whole argument for detaching.
+ * Redis latency after the queue item is already durable. The durable
+ * clientSteerId receipt makes a lost-202 retry return this same item; a
+ * lost publish merely takes the documented tool-boundary fallback.
*/
- void GenerationJobManager.requestPreempt(streamId, item.steerId, owner.createdAt).then(
+ void GenerationJobManager.requestPreempt(
+ streamId,
+ persistedItem.steerId,
+ owner.createdAt,
+ protocol.value === 2 ? preemptRevision : undefined,
+ ).then(
(armed) => {
if (!armed) {
logger.warn(
- `[handleSteerRequest] Preempt arm not confirmed for ${streamId} steer=${item.steerId}; ` +
+ `[handleSteerRequest] Preempt arm not confirmed for ${streamId} steer=${persistedItem.steerId}; ` +
'the steer remains queued and will inject at the next boundary',
);
}
},
(error: unknown) => {
logger.error(
- `[handleSteerRequest] Preempt arm failed for ${streamId} steer=${item.steerId}:`,
+ `[handleSteerRequest] Preempt arm failed for ${streamId} steer=${persistedItem.steerId}:`,
error,
);
},
);
}
- /** Fire-and-forget: the persisted steer part references these uploads, so
- * mark them used (parity with `updateFilesUsage` on normal sends) or the
- * upload-window TTL could reap them before replay. Must not fail the 202. */
- if (resolvedFileIds?.length && deps.updateFilesUsage) {
- deps
- .updateFilesUsage(
- resolvedFileIds.map((file_id) => ({ file_id })),
- undefined,
- { user: user.id, tenantId: user.tenantId },
- )
- .catch((error) =>
- logger.warn(`[handleSteerRequest] Failed to mark steer files used: ${streamId}`, error),
- );
- }
-
return {
status: 202,
body: {
status: 'queued',
- steerId: item.steerId,
+ steerId: persistedItem.steerId,
position: depth,
conversationId,
- preempt: preemptCapable,
+ preempt: preemptArmed,
+ ...(protocol.value === 2 && preemptRevision != null && { preemptRevision }),
},
};
}
@@ -383,34 +717,151 @@ export async function handleSteerRequest(
export async function handleSteerCancel(
user: SteerRequestUser,
body: SteerCancelBody,
+ options: SteerProtocolOptions = {},
+): Promise<SteerRequestResult> {
+ const protocol: SteerProtocolContext = {
+ value: protocolFromOptions(options),
+ expose: Object.prototype.hasOwnProperty.call(options, 'generationProtocolVersion'),
+ };
+ return attachProtocol(await handleSteerCancelInternal(user, body, protocol), protocol);
+}
+
+async function handleSteerCancelInternal(
+ user: SteerRequestUser,
+ body: SteerCancelBody,
+ protocol: SteerProtocolContext,
): Promise<SteerRequestResult> {
const conversationId = body.conversationId;
if (typeof conversationId !== 'string' || !conversationId || conversationId === 'new') {
return { status: 400, body: { code: 'INVALID_CONVERSATION' } };
}
- if (typeof body.steerId !== 'string' || body.steerId.length === 0) {
+ const expectedGeneration = parseExpectedGenerationCreatedAt(body.generationCreatedAt);
+ if (expectedGeneration.invalid) {
+ return { status: 400, body: { code: 'INVALID_GENERATION_IDENTITY' } };
+ }
+ const steerId = body.steerId;
+ if (typeof steerId !== 'string' || steerId.length === 0) {
return { status: 400, body: { code: 'INVALID_STEER_ID' } };
}
+ if (
+ body.clientSteerId != null &&
+ (typeof body.clientSteerId !== 'string' ||
+ body.clientSteerId.length === 0 ||
+ body.clientSteerId.length > 128)
+ ) {
+ return { status: 400, body: { code: 'INVALID_CLIENT_STEER_ID' } };
+ }
const streamId = conversationId;
+ const clientSteerId = typeof body.clientSteerId === 'string' ? body.clientSteerId : undefined;
const job = await GenerationJobManager.getJob(streamId);
+ let sawReceipt = false;
+ const readOwnedReceipt = async (): Promise<SteerReceipt | null | 'unauthorized'> => {
+ if (protocol.value !== 2 || clientSteerId == null) {
+ return null;
+ }
+ const receipt = await GenerationJobManager.steering.getReceipt(streamId, clientSteerId);
+ if (receipt == null) {
+ return null;
+ }
+ sawReceipt = true;
+ if (
+ receipt.userId !== (user.id ?? '') ||
+ hasTenantMismatch(receipt, user) ||
+ receipt.item.steerId !== steerId
+ ) {
+ return 'unauthorized';
+ }
+ return receipt;
+ };
+ /** Settle receipt states that prove this exact steer was already cancelled
+ * or reached terminal leftover recovery. The second read covers concurrent
+ * cancel requests that both observed `leftover` before one won the atomic
+ * discard. */
+ const settleReceipt = async (): Promise<SteerRequestResult | null> => {
+ const receipt = await readOwnedReceipt();
+ if (receipt === 'unauthorized') {
+ return { status: 403, body: { code: 'UNAUTHORIZED' } };
+ }
+ if (
+ receipt != null &&
+ expectedGeneration.value != null &&
+ receipt.generationCreatedAt !== expectedGeneration.value
+ ) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
+ if (receipt?.state === 'cancelled') {
+ return { status: 200, body: { removed: true, replayed: true } };
+ }
+ if (receipt?.state !== 'leftover' || clientSteerId == null) {
+ return null;
+ }
+ const discarded = await GenerationJobManager.steering.discardLeftover(
+ streamId,
+ clientSteerId,
+ steerId,
+ { userId: user.id ?? '', tenantId: user.tenantId },
+ receipt.generationCreatedAt,
+ );
+ if (discarded) {
+ return { status: 200, body: { removed: true, replayed: true } };
+ }
+ const racedReceipt = await readOwnedReceipt();
+ if (racedReceipt === 'unauthorized') {
+ return { status: 403, body: { code: 'UNAUTHORIZED' } };
+ }
+ if (
+ racedReceipt != null &&
+ expectedGeneration.value != null &&
+ racedReceipt.generationCreatedAt !== expectedGeneration.value
+ ) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
+ return racedReceipt?.state === 'cancelled'
+ ? { status: 200, body: { removed: true, replayed: true } }
+ : null;
+ };
+ const replaySettlement = await settleReceipt();
+ if (replaySettlement != null) {
+ return replaySettlement;
+ }
+ /** A later v1 job must not hide an earlier v2 receipt. The receipt read
+ * above is owner/tenant/epoch-authorized evidence about an already-accepted
+ * action; only after settling it do we cap new mutations to the current
+ * job's marker. This mirrors the enqueue replay ordering. */
+ if (job != null) {
+ capProtocolToJob(protocol, job);
+ }
if (!job) {
+ /** The generation may have terminalized between the first receipt read
+ * and this job lookup. Re-read before declaring the cancel lost so its
+ * newly parked leftover cannot reappear after the client discarded it. */
+ const racedSettlement = await settleReceipt();
+ if (racedSettlement != null) {
+ return racedSettlement;
+ }
+ if (!sawReceipt) {
+ protocol.value = 1;
+ }
return { status: 200, body: { removed: false } };
}
- if (job.metadata?.userId && job.metadata.userId !== user.id) {
+ if (job.metadata?.userId !== user.id) {
logger.warn(`[handleSteerCancel] Unauthorized cancel attempt for ${streamId} by ${user.id}`);
return { status: 403, body: { code: 'UNAUTHORIZED' } };
}
if (hasTenantMismatch(job.metadata, user)) {
return { status: 403, body: { code: 'UNAUTHORIZED' } };
}
+ if (expectedGeneration.value != null && job.createdAt !== expectedGeneration.value) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
- const removed = await GenerationJobManager.steering.cancel(streamId, body.steerId);
+ const removed = await GenerationJobManager.steering.cancel(streamId, steerId, job.createdAt);
/** A cancelled steer must also disarm any preempt request it carried —
* cancel is live UI, and a request left armed would seal an unrelated
* stretch of generation, drain nothing, and end the run mid-sentence. */
if (!removed) {
- return { status: 200, body: { removed } };
+ return (await settleReceipt()) ?? { status: 200, body: { removed: false } };
}
/**
* Waited on so a failed disarm is retried and logged before the response,
@@ -439,14 +890,14 @@ export async function handleSteerCancel(
* queue, and inverting it would make the client re-show a chip for a steer
* that can never arrive.
*/
- const disarm = GenerationJobManager.noteSteersRemoved(streamId, [body.steerId], job.createdAt);
+ const disarm = GenerationJobManager.noteSteersRemoved(streamId, [steerId], job.createdAt);
let settleTimer: ReturnType<typeof setTimeout> | undefined;
await Promise.race([
disarm,
new Promise<void>((resolve) => {
settleTimer = setTimeout(() => {
logger.warn(
- `[handleSteerCancel] Disarm publish for ${streamId} steer=${body.steerId} still pending ` +
+ `[handleSteerCancel] Disarm publish for ${streamId} steer=${steerId} still pending ` +
`after ${STEER_DISARM_ACK_TIMEOUT_MS}ms; answering the cancel and letting it retry`,
);
resolve();
@@ -456,3 +907,120 @@ export async function handleSteerCancel(
clearTimeout(settleTimer);
return { status: 200, body: { removed } };
}
+
+/**
+ * Escalates a still-queued steer to an interrupt IN PLACE — one atomic flag
+ * flip on the existing item, so its FIFO position, id, and timestamp all
+ * survive and no reclaim window ever exists. `armed: false` is not an error:
+ * the steer already injected, was cancelled, or belongs to a run that ended
+ * (the client defers to the events it will receive). Mirrors the steer POST's
+ * preempt contract exactly: the durable flag is gated on the OWNER's recorded
+ * capability, the store op is fenced to the validated generation, and the
+ * volatile arm publish is fire-and-forget because the durable flag is the
+ * truth `rearmQueuedPreempts` trusts on resume and handover.
+ */
+export async function handleSteerArm(
+ user: SteerRequestUser,
+ body: SteerCancelBody,
+ options: SteerProtocolOptions = {},
+): Promise<SteerRequestResult> {
+ const protocol: SteerProtocolContext = {
+ value: protocolFromOptions(options),
+ expose: Object.prototype.hasOwnProperty.call(options, 'generationProtocolVersion'),
+ };
+ return attachProtocol(await handleSteerArmInternal(user, body, protocol), protocol);
+}
+
+async function handleSteerArmInternal(
+ user: SteerRequestUser,
+ body: SteerCancelBody,
+ protocol: SteerProtocolContext,
+): Promise<SteerRequestResult> {
+ const conversationId = body.conversationId;
+ if (typeof conversationId !== 'string' || !conversationId || conversationId === 'new') {
+ return { status: 400, body: { code: 'INVALID_CONVERSATION' } };
+ }
+ const expectedGeneration = parseExpectedGenerationCreatedAt(body.generationCreatedAt);
+ if (expectedGeneration.invalid) {
+ return { status: 400, body: { code: 'INVALID_GENERATION_IDENTITY' } };
+ }
+ if (typeof body.steerId !== 'string' || body.steerId.length === 0) {
+ return { status: 400, body: { code: 'INVALID_STEER_ID' } };
+ }
+
+ const streamId = conversationId;
+ const job = await GenerationJobManager.getJob(streamId);
+ if (!job) {
+ protocol.value = 1;
+ return { status: 200, body: { armed: false } };
+ }
+ capProtocolToJob(protocol, job);
+ if (job.metadata?.userId !== user.id) {
+ logger.warn(`[handleSteerArm] Unauthorized arm attempt for ${streamId} by ${user.id}`);
+ return { status: 403, body: { code: 'UNAUTHORIZED' } };
+ }
+ if (hasTenantMismatch(job.metadata, user)) {
+ return { status: 403, body: { code: 'UNAUTHORIZED' } };
+ }
+ if (expectedGeneration.value != null && job.createdAt !== expectedGeneration.value) {
+ return { status: 409, body: { code: 'RUN_REPLACED' } };
+ }
+
+ /**
+ * Capability is decided INSIDE the atomic store predicate, not from the job
+ * read above: a HITL resume on a rolling deploy rewrites `preemptCapable`
+ * for the SAME generation, so a value read here can be stale by the time
+ * the flag flips. An incapable owner answers `PREEMPT_UNSUPPORTED` (same
+ * honesty rule as the POST's echo — the chip must not read "interrupting"
+ * for a run that can only inject at a tool boundary), with the item left
+ * unflagged and still queued.
+ */
+ /** Always retain the store-assigned revision internally, including for a
+ * v1 wire response. A capable→incapable→capable handover raises the
+ * runtime's minimum accepted arm revision; publishing legacy revision 0
+ * after the durable flag advanced would leave the item marked preempting
+ * while the owner rejects the seal. v1 still exposes neither the revision
+ * nor the correlated update event, so its wire contract is unchanged. */
+ const {
+ outcome: armOutcome,
+ revision: preemptRevision,
+ item: armedItem,
+ } = await GenerationJobManager.steering.armVersioned(streamId, body.steerId, job.createdAt);
+ if (armOutcome === 'incapable') {
+ return { status: 200, body: { armed: false, code: 'PREEMPT_UNSUPPORTED' } };
+ }
+ if (armOutcome !== 'armed') {
+ return { status: 200, body: { armed: false } };
+ }
+ if (protocol.value === 2 && armedItem != null) {
+ publishSteerUpdate(streamId, job.createdAt, armedItem);
+ }
+ /** NOT awaited, exactly like the POST: the durable flag is already the
+ * truth, a lost publish degrades to the tool-boundary fallback, and
+ * resume/handover re-arm from the queue. */
+ void GenerationJobManager.requestPreempt(
+ streamId,
+ body.steerId,
+ job.createdAt,
+ preemptRevision,
+ ).then(
+ (confirmed) => {
+ if (!confirmed) {
+ logger.warn(
+ `[handleSteerArm] Preempt arm not confirmed for ${streamId} steer=${body.steerId}; ` +
+ 'the steer remains queued and will inject at the next boundary',
+ );
+ }
+ },
+ (error) => {
+ logger.error(`[handleSteerArm] Preempt arm publish failed for ${streamId}:`, error);
+ },
+ );
+ return {
+ status: 200,
+ body: {
+ armed: true,
+ ...(protocol.value === 2 && preemptRevision != null && { preemptRevision }),
+ },
+ };
+}
diff --git a/packages/api/src/agents/steering/runtime.ts b/packages/api/src/agents/steering/runtime.ts
index aafde3b84b..31cb0a9e93 100644
--- a/packages/api/src/agents/steering/runtime.ts
+++ b/packages/api/src/agents/steering/runtime.ts
@@ -76,10 +76,12 @@ export interface SteerDrainHookOptions {
* PreemptBoundary) — the provider-safety argument rests on the two sites
* emitting identical `InjectedMessage` shapes, so they must share one body.
*
- * The injections array builds incrementally under a swallow-all catch: once a
- * steer leaves the durable queue its part is persisted by `applySteer`
- * first, so a mid-loop throw must still inject whatever was already applied
- * rather than dropping the batch. `noteSteersRemoved` runs in `finally` —
+ * Every part is durably applied before ANY slow media encoding begins. A
+ * failed applied write restores that claimed item at the queue front; a crash
+ * before restoration leaves its receipt `claimed`, which repairs to leftover
+ * when the generation dies instead of claiming false delivery. The injections
+ * array then builds from only the durably applied items. `noteSteersRemoved`
+ * runs in `finally` —
* the drained items are out of the queue no matter what, so any armed
* preempt request for them must clear even on the failure path, or a
* satisfied preempt could seal a later, unrelated stretch of generation.
@@ -109,16 +111,52 @@ async function drainAndBuildInjections(opts: SteerDrainHookOptions): Promise<Inj
return [];
}
const injectedMessages: InjectedMessage[] = [];
+ const appliedSteers: SteerQueueItem[] = [];
+ const restoredIds = new Set<string>();
try {
- for (const item of steers) {
+ let failedSteers: SteerQueueItem[] = [];
+ for (let i = 0; i < steers.length; i++) {
+ const item = steers[i];
try {
await applySteer(item);
+ appliedSteers.push(item);
} catch (error) {
logger.error(
`[steering] Failed to apply steer part for ${streamId} steer=${item.steerId}:`,
error,
);
+ /** FIFO is semantic instruction order. Once one durable write fails,
+ * no later item may overtake it; restore the failed item plus the
+ * untouched suffix as one ordered front batch. */
+ failedSteers = steers.slice(i);
+ break;
}
+ }
+
+ if (failedSteers.length > 0) {
+ try {
+ if (
+ await GenerationJobManager.steering.restoreClaimed(streamId, failedSteers, jobCreatedAt)
+ ) {
+ for (const item of failedSteers) {
+ restoredIds.add(item.steerId);
+ }
+ } else {
+ logger.error(
+ `[steering] Could not restore ${failedSteers.length} claimed steer(s) for ${streamId}; ` +
+ 'their receipts remain recoverable after this generation ends',
+ );
+ }
+ } catch (error) {
+ logger.error(
+ `[steering] Failed to restore claimed steers for ${streamId}; ` +
+ 'their receipts remain recoverable after this generation ends:',
+ error,
+ );
+ }
+ }
+
+ for (const item of appliedSteers) {
let media: SteerMediaResult | undefined;
if (buildMedia != null && (item.files?.length ?? 0) > 0) {
try {
@@ -156,9 +194,11 @@ async function drainAndBuildInjections(opts: SteerDrainHookOptions): Promise<Inj
* synchronous and already effective — the publish only informs other
* replicas, and blocking a boundary drain on it would delay injection.
*/
- const spent = new Set(armedBeforeDrain);
+ const spent = new Set(armedBeforeDrain.filter((steerId) => !restoredIds.has(steerId)));
for (const item of steers) {
- spent.add(item.steerId);
+ if (!restoredIds.has(item.steerId)) {
+ spent.add(item.steerId);
+ }
}
void GenerationJobManager.noteSteersRemoved(streamId, [...spent], jobCreatedAt);
}
diff --git a/packages/api/src/stream/ApprovalLifecycle.ts b/packages/api/src/stream/ApprovalLifecycle.ts
index 7e6a2396b6..2bf081c666 100644
--- a/packages/api/src/stream/ApprovalLifecycle.ts
+++ b/packages/api/src/stream/ApprovalLifecycle.ts
@@ -1,18 +1,43 @@
import { logger } from '@librechat/data-schemas';
import type { Agents } from 'librechat-data-provider';
-import type { IJobStore } from '~/stream/interfaces/IJobStore';
-import { isPendingActionExpired, isPendingActionStale } from '~/stream/interfaces/IJobStore';
+import type { IJobStoreV2, JobMetadataPatch, SteerQueueItem } from '~/stream/interfaces/IJobStore';
+import {
+ isPendingActionExpired,
+ isPendingActionStale,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ PAUSE_PERSISTENCE_TIMEOUT_MS,
+} from '~/stream/interfaces/IJobStore';
export interface ApprovalLifecycleCallbacks {
onPaused?: (streamId: string, createdAt: number) => void;
onResumed?: (streamId: string, createdAt: number) => void;
onExpired?: (streamId: string, createdAt: number) => void;
+ onPausePersistenceFailed?: (
+ streamId: string,
+ createdAt: number,
+ error: string,
+ drainedSteers: SteerQueueItem[],
+ ) => void | Promise<void>;
}
export interface ApprovalPauseOptions {
discoveredTools?: string[];
/** Generation identity observed by the interrupted run. */
expectedCreatedAt?: number;
+ /** Hold Stop/resume until the paused assistant row is durably unfinished. */
+ persistencePending?: boolean;
+}
+
+const PAUSE_PERSISTENCE_ACTION_PREFIX = 'pause-persistence:';
+
+export function pausePersistenceActionId(actionId: string): string {
+ return `${PAUSE_PERSISTENCE_ACTION_PREFIX}${actionId}`;
+}
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolve) => {
+ setTimeout(resolve, ms);
+ });
}
/**
@@ -39,7 +64,7 @@ export interface ApprovalPauseOptions {
*/
export class ApprovalLifecycle {
constructor(
- private readonly store: IJobStore,
+ private readonly store: IJobStoreV2,
private readonly callbacks: ApprovalLifecycleCallbacks = {},
) {}
@@ -51,30 +76,49 @@ export class ApprovalLifecycle {
async pause(
streamId: string,
pendingAction: Agents.PendingAction,
- options?: ApprovalPauseOptions,
+ options: ApprovalPauseOptions = {},
): Promise<boolean> {
const job = await this.store.getJob(streamId);
if (
!job ||
job.status !== 'running' ||
- (options?.expectedCreatedAt != null && job.createdAt !== options.expectedCreatedAt)
+ (options.expectedCreatedAt != null && job.createdAt !== options.expectedCreatedAt)
) {
return false;
}
- const discoveredTools = options?.discoveredTools;
- const expectedCreatedAt = options?.expectedCreatedAt ?? job.createdAt;
+ const expectedCreatedAt = options.expectedCreatedAt ?? job.createdAt;
+ const discoveredTools = options.discoveredTools;
+ /** The normal receipt TTL matches a running job, but a review pause can
+ * live for 24h or an explicit later expiry. The store extends every
+ * receipt in the SAME CAS that closes running enqueues: a separate pass
+ * before or after the transition leaves either an enqueue or crash gap. */
+ const pauseReceiptTtl = Math.max(
+ 24 * 60 * 60,
+ pendingAction.expiresAt != null
+ ? Math.ceil((pendingAction.expiresAt - Date.now()) / 1000) + 60
+ : 0,
+ );
+ const persistenceStartedAt = Date.now();
const ok = await this.store.transitionStatus(streamId, {
from: 'running',
to: 'requires_action',
// pendingActionId is the flat mirror the atomic resolve/expire guard on.
patch: {
pendingAction,
- pendingActionId: pendingAction.actionId,
+ pendingActionId:
+ options.persistencePending === true
+ ? pausePersistenceActionId(pendingAction.actionId)
+ : pendingAction.actionId,
+ ...(options.persistencePending === true && {
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: persistenceStartedAt,
+ }),
...(discoveredTools != null && discoveredTools.length > 0
? { discoveredTools: [...discoveredTools] }
: {}),
},
expectCreatedAt: expectedCreatedAt,
+ steerReceiptTtlSeconds: pauseReceiptTtl,
});
if (ok) {
this.callbacks.onPaused?.(streamId, expectedCreatedAt);
@@ -85,6 +129,158 @@ export class ApprovalLifecycle {
return ok;
}
+ /**
+ * True only while this exact paused action still owns the durable response
+ * persistence barrier installed by {@link pause}. Terminal and resume CASes
+ * wait for the barrier, so a controller that observes `true` may safely write
+ * its `unfinished:true` response before releasing it.
+ */
+ async ownsPausePersistence(
+ streamId: string,
+ expectedActionId: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ const job = await this.store.getJob(streamId);
+ return (
+ job?.status === 'requires_action' &&
+ job.terminalPersistencePending === true &&
+ job.pendingAction?.actionId === expectedActionId &&
+ job.pendingActionId === pausePersistenceActionId(expectedActionId) &&
+ (expectedCreatedAt == null || job.createdAt === expectedCreatedAt)
+ );
+ }
+
+ /** Release the exact pause barrier after its required response write settles. */
+ async finishPausePersistence(
+ streamId: string,
+ expectedActionId: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ return this.store.transitionStatus(streamId, {
+ from: 'requires_action',
+ to: 'requires_action',
+ patch: {
+ pendingActionId: expectedActionId,
+ terminalPersistencePending: false,
+ },
+ clear: ['terminalPersistenceStartedAt'],
+ expectActionId: pausePersistenceActionId(expectedActionId),
+ expectCreatedAt: expectedCreatedAt,
+ });
+ }
+
+ /**
+ * Wait for a live pause writer, or recover its lock after the bounded owner
+ * deadline. Every requires_action→running/terminal path crosses this helper,
+ * making the pause response write linearize before either competing outcome.
+ */
+ async waitForPausePersistence(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<Awaited<ReturnType<IJobStoreV2['getJob']>>> {
+ for (;;) {
+ const job = await this.store.getJob(streamId);
+ if (
+ !job ||
+ job.status !== 'requires_action' ||
+ job.terminalPersistencePending !== true ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
+ return job;
+ }
+
+ const startedAt = job.terminalPersistenceStartedAt ?? job.createdAt;
+ const elapsed = Date.now() - startedAt;
+ if (elapsed >= PAUSE_PERSISTENCE_TIMEOUT_MS) {
+ const failed = await this.failStalePausePersistence(
+ streamId,
+ expectedCreatedAt ?? job.createdAt,
+ );
+ if (!failed) {
+ const current = await this.store.getJob(streamId);
+ if (
+ current?.createdAt === job.createdAt &&
+ current.status === 'requires_action' &&
+ current.terminalPersistencePending === true &&
+ current.pendingActionId === job.pendingActionId
+ ) {
+ throw new Error(
+ `Unable to safely terminalize stale pause-persistence barrier: ${streamId}`,
+ );
+ }
+ }
+ continue;
+ }
+
+ await delay(Math.min(25, PAUSE_PERSISTENCE_TIMEOUT_MS - elapsed));
+ }
+ }
+
+ /**
+ * Fail a pause writer whose bounded persistence lease expired. The terminal
+ * CAS is fenced to the exact action and generation and parks all accepted
+ * steers in the same store transaction. Returning the ordinary action id to
+ * callers here would be unsafe: the user, assistant, or conversation rows may
+ * never have committed before the writer disappeared.
+ */
+ async failStalePausePersistence(streamId: string, expectedCreatedAt?: number): Promise<boolean> {
+ const job = await this.store.getJob(streamId);
+ if (
+ !job ||
+ job.status !== 'requires_action' ||
+ job.terminalPersistencePending !== true ||
+ job.pendingActionId == null ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
+ return false;
+ }
+
+ const startedAt = job.terminalPersistenceStartedAt ?? job.createdAt;
+ if (Date.now() - startedAt < PAUSE_PERSISTENCE_TIMEOUT_MS) {
+ return false;
+ }
+
+ const completedAt = Date.now();
+ const drainedSteers = await this.store.transitionStatusAndDrainSteers(streamId, {
+ from: 'requires_action',
+ to: 'error',
+ expectActionId: job.pendingActionId,
+ expectCreatedAt: job.createdAt,
+ patch: {
+ completedAt,
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ },
+ clear: [
+ 'pendingAction',
+ 'pendingActionId',
+ 'terminalPersistencePending',
+ 'terminalPersistenceStartedAt',
+ ],
+ });
+ if (drainedSteers == null) {
+ return false;
+ }
+
+ logger.error(`[ApprovalLifecycle] Pause persistence timed out: ${streamId}`);
+ try {
+ await this.callbacks.onPausePersistenceFailed?.(
+ streamId,
+ job.createdAt,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ drainedSteers,
+ );
+ } catch (error) {
+ // The durable fail-closed transition already won. Notification/resource
+ // cleanup is best-effort here and must not turn resolve() into a retry of
+ // an action that is now terminal.
+ logger.error(
+ `[ApprovalLifecycle] Failed to retire timed-out pause runtime: ${streamId}`,
+ error,
+ );
+ }
+ return true;
+ }
+
/**
* The pending review record, or `null` when the job isn't awaiting review.
* A past-`expiresAt` record reads as `null` (lazy expiry) so a stale prompt
@@ -114,9 +310,18 @@ export class ApprovalLifecycle {
* The caller MUST treat `false` as "do not drive the run": only the `true`
* winner may re-enter the agent.
*/
- async resolve(streamId: string, expectedActionId?: string): Promise<boolean> {
- const job = await this.store.getJob(streamId);
- if (!job || job.status !== 'requires_action') {
+ async resolve(
+ streamId: string,
+ expectedActionId?: string,
+ resumePatch?: JobMetadataPatch,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ const job = await this.waitForPausePersistence(streamId, expectedCreatedAt);
+ if (
+ !job ||
+ job.status !== 'requires_action' ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
return false;
}
if (!job.pendingAction) {
@@ -140,7 +345,10 @@ export class ApprovalLifecycle {
clear: ['pendingAction', 'pendingActionId'],
// Refresh the liveness basis so a long-paused run isn't reaped as stale
// immediately after resuming (cleanup keys off lastActiveAt).
- patch: { lastActiveAt: Date.now() },
+ /** Ownership can move across replicas on resume. Owner-specific fields
+ * must change in this SAME CAS: once status is `running`, steering
+ * routes are live and may atomically inspect them. */
+ patch: { lastActiveAt: Date.now(), ...resumePatch },
expectActionId: expectedActionId,
expectCreatedAt: job.createdAt,
});
@@ -174,14 +382,15 @@ export class ApprovalLifecycle {
expectedActionId?: string,
expectedCreatedAt?: number,
): Promise<number | null> {
- let createdAt = expectedCreatedAt;
- if (createdAt == null) {
- const job = await this.store.getJob(streamId);
- if (!job || job.status !== 'requires_action') {
- return null;
- }
- createdAt = job.createdAt;
+ const job = await this.waitForPausePersistence(streamId, expectedCreatedAt);
+ if (
+ !job ||
+ job.status !== 'requires_action' ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
+ return null;
}
+ const createdAt = job.createdAt;
const ok = await this.store.transitionStatus(streamId, {
from: 'requires_action',
to: 'aborted',
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 9f390844c6..1d2e966c0e 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -1,8 +1,11 @@
+import { randomUUID } from 'crypto';
import { logger, getTenantId, SYSTEM_TENANT_ID } from '@librechat/data-schemas';
import {
Constants,
+ ContentTypes,
UsageEvents,
ApprovalEvents,
+ SteerEvents,
parseTextParts,
reconcileContextUsage,
promptTokensFromUsage,
@@ -16,27 +19,38 @@ import type {
import type { StandardGraph } from '@librechat/agents';
import type {
SerializableJobData,
+ CreatedJobData,
IEventTransport,
UsageMetadata,
AbortResult,
IJobStore,
+ IJobStoreV2,
IdempotencyClaimResult,
+ IdempotencyClaimValue,
+ PreemptMessage,
+ SteerQueueItem,
} from './interfaces/IJobStore';
import type { AgentStartupTelemetry } from '~/agents/startup';
+import type { RecoveredSteerPayload } from './SteerRecovery';
import type { SteerContentView } from './SteeringLifecycle';
import type { GenerationJobStore } from '~/app/metrics';
import type * as t from '~/types';
+import {
+ JobCreationSupersededError,
+ JobPredecessorMismatchError,
+ isPendingActionStale,
+ isPendingActionExpired,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ STEER_QUEUE_MAX_DEPTH,
+} from './interfaces/IJobStore';
import {
recordGenerationStreamResumePendingEvents,
recordGenerationStreamSubscription,
setGenerationJobsInFlight,
recordGenerationJob,
} from '~/app/metrics';
-import {
- isPendingActionStale,
- isPendingActionExpired,
- STEER_QUEUE_MAX_DEPTH,
-} from './interfaces/IJobStore';
+import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery';
+import { assertJobStoreV2 } from './jobStoreCapabilities';
/**
* Tombstone budget per generation. Twice the queue depth so a full queue's
@@ -55,7 +69,7 @@ import { normalizeResumeRunStepIndices } from '~/agents/hitl/resume';
import { emitChunkWithReceipt } from './internal/chunkPublication';
import { filterPersistableAbortContent } from './abortContent';
import { toClientPendingAction } from '~/agents/hitl/policy';
-import { ApprovalLifecycle } from './ApprovalLifecycle';
+import { ApprovalLifecycle, pausePersistenceActionId } from './ApprovalLifecycle';
import { sanitizeJobMetadata } from './metadata';
/** Terminal error surfaced to a client still attached when its approval window lapses. */
@@ -64,13 +78,192 @@ const APPROVAL_EXPIRED_ERROR = 'Approval expired before a decision was made';
/** Error surfaced to any client still attached when a stale/hung job is reaped. */
const REAPED_JOB_ERROR = 'Generation timed out';
-/** Lifetime of a start-generation idempotency claim (matches the running-job TTL: 20 min),
- * so a late retry still dedups for the whole generation window. */
-const IDEMPOTENCY_TTL_SECONDS = 1200;
+/** Bounded completed-request replay horizon. It exceeds the default 24-hour
+ * approval window; if a custom/live job outlasts it, `resumeClaimedGeneration`
+ * atomically adopts the reacquired claim instead of replacing that job. */
+const IDEMPOTENCY_TTL_SECONDS = 25 * 60 * 60;
+/** The legacy, user/request-scoped key deliberately outlives the same-slot
+ * primary. During a rolling upgrade this keeps old replicas fenced while a
+ * new replica can reconstruct an expired primary from the legacy tombstone. */
+const LEGACY_IDEMPOTENCY_TTL_SECONDS = 26 * 60 * 60;
const OAUTH_TOOL_CALL_PREFIX = `oauth${Constants.mcp_delimiter}`;
const SHUTDOWN_SUBSCRIBER_ERROR = 'Server is shutting down';
const SHUTDOWN_JOB_ERROR = 'Generation interrupted because its server shut down';
const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down';
+/** Internal transport signal: durable terminal state exists, but its DONE
+ * publication failed. HTTP subscribers must reconnect rather than treating
+ * this as an application-level generation error. */
+export const TERMINAL_PUBLICATION_RECONNECT_ERROR =
+ 'Terminal publication failed; reconnect to load the durable result';
+/** Upper bound for a terminal owner's required persistence barrier. A crashed
+ * owner leaves the durable pending bit behind; the next read or subscriber
+ * promotes it to conservative reconciliation after this window. */
+const TERMINAL_PERSISTENCE_TIMEOUT_MS = 30_000;
+const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
+type TokenIdempotencyClaim = IdempotencyClaimValue & {
+ claimedAt: number;
+ claimToken: string;
+ generationProtocolVersion: 1 | 2;
+};
+
+function normalizeClaimProtocol(
+ value: Pick<IdempotencyClaimValue, 'generationProtocolVersion'>,
+): 1 | 2 {
+ return value.generationProtocolVersion === 2 ? 2 : 1;
+}
+
+function assertClaimCoordinates(
+ value: IdempotencyClaimValue | undefined,
+ context: string,
+): asserts value is IdempotencyClaimValue {
+ if (
+ value == null ||
+ typeof value !== 'object' ||
+ typeof value.streamId !== 'string' ||
+ value.streamId.length === 0 ||
+ typeof value.conversationId !== 'string' ||
+ value.conversationId.length === 0 ||
+ !Number.isSafeInteger(value.claimedAt) ||
+ (value.claimedAt as number) < 0 ||
+ (value.startedAt != null && (!Number.isSafeInteger(value.startedAt) || value.startedAt < 0)) ||
+ (value.generationProtocolVersion != null &&
+ value.generationProtocolVersion !== 1 &&
+ value.generationProtocolVersion !== 2)
+ ) {
+ throw new Error(`Invalid ${context} generation idempotency claim`);
+ }
+}
+
+/** A token-bearing value was written by the bridge-capable manager. Its full
+ * coordinates are therefore mandatory; an empty/malformed token must never be
+ * reinterpreted as a harmless old-server value. */
+function normalizeTokenClaim(
+ value: IdempotencyClaimValue | undefined,
+ context: string,
+): TokenIdempotencyClaim {
+ assertClaimCoordinates(value, context);
+ if (
+ typeof value.claimToken !== 'string' ||
+ value.claimToken.length === 0 ||
+ value.claimToken.length > 128 ||
+ (value.previousClaimToken != null &&
+ (typeof value.previousClaimToken !== 'string' ||
+ value.previousClaimToken.length === 0 ||
+ value.previousClaimToken.length > 128 ||
+ value.previousClaimToken === value.claimToken))
+ ) {
+ throw new Error(`Invalid ${context} generation idempotency claim token`);
+ }
+ return {
+ ...value,
+ claimedAt: value.claimedAt as number,
+ claimToken: value.claimToken,
+ generationProtocolVersion: normalizeClaimProtocol(value),
+ };
+}
+
+/** Old replicas wrote a claimedAt/stream pointer but no lease token. They are
+ * authoritative duplicate evidence, not a lease a new replica may take over.
+ * Stream correlation is intentionally left to the controller: pre-upgrade new
+ * chats used a random stream while current retries derive a stable UUID. */
+function normalizeTokenlessLegacyClaim(
+ value: IdempotencyClaimValue | undefined,
+): IdempotencyClaimValue {
+ assertClaimCoordinates(value, 'legacy');
+ if (
+ value.claimToken != null ||
+ value.previousClaimToken != null ||
+ value.startedAt != null ||
+ normalizeClaimProtocol(value) !== 1
+ ) {
+ throw new Error('Invalid tokenless legacy generation idempotency claim');
+ }
+ return { ...value, generationProtocolVersion: 1 };
+}
+
+function assertClaimMatchesRequest(
+ value: TokenIdempotencyClaim,
+ streamId: string,
+ conversationId: string,
+): void {
+ if (value.streamId !== streamId || value.conversationId !== conversationId) {
+ throw new Error('Mismatched generation idempotency claim coordinates');
+ }
+}
+
+function claimsMirrorExactly(left: TokenIdempotencyClaim, right: TokenIdempotencyClaim): boolean {
+ return (
+ left.streamId === right.streamId &&
+ left.conversationId === right.conversationId &&
+ left.claimedAt === right.claimedAt &&
+ left.claimToken === right.claimToken &&
+ left.previousClaimToken === right.previousClaimToken &&
+ left.generationProtocolVersion === right.generationProtocolVersion &&
+ left.startedAt === right.startedAt
+ );
+}
+
+/** The legacy and primary values are mirrors. The only legitimate transient
+ * asymmetry is primary.startedAt being present before the follow-up legacy
+ * mark; the caller reconciles that one direction explicitly. */
+function assertClaimMirrors(legacy: TokenIdempotencyClaim, primary: TokenIdempotencyClaim): void {
+ const startedRepair = legacy.startedAt == null && primary.startedAt != null;
+ if (
+ legacy.streamId !== primary.streamId ||
+ legacy.conversationId !== primary.conversationId ||
+ legacy.claimedAt !== primary.claimedAt ||
+ legacy.claimToken !== primary.claimToken ||
+ legacy.previousClaimToken !== primary.previousClaimToken ||
+ (legacy.generationProtocolVersion !== primary.generationProtocolVersion && !startedRepair) ||
+ (legacy.startedAt != null && legacy.startedAt !== primary.startedAt)
+ ) {
+ throw new Error('Mismatched legacy and primary generation idempotency claims');
+ }
+}
+
+/** Proof of the only repairable unstarted token split: takeover wrote the
+ * primary with an explicit predecessor token, then its reply/legacy CAS was
+ * interrupted. Arbitrary token mismatches remain fail-closed. */
+function isClaimTakeoverOf(
+ predecessor: TokenIdempotencyClaim,
+ candidate: TokenIdempotencyClaim,
+): boolean {
+ return (
+ candidate.previousClaimToken === predecessor.claimToken &&
+ candidate.claimToken !== predecessor.claimToken &&
+ candidate.streamId === predecessor.streamId &&
+ candidate.conversationId === predecessor.conversationId &&
+ candidate.generationProtocolVersion === predecessor.generationProtocolVersion &&
+ candidate.claimedAt >= predecessor.claimedAt
+ );
+}
+
+function isRecoverableTakeoverSplit(
+ legacy: TokenIdempotencyClaim,
+ primary: TokenIdempotencyClaim,
+): boolean {
+ return (
+ legacy.startedAt == null && primary.startedAt == null && isClaimTakeoverOf(legacy, primary)
+ );
+}
+
+function buildTerminalPersistenceReconcile(
+ job: Pick<SerializableJobData, 'createdAt' | 'conversationId' | 'status'>,
+): t.FinalEvent {
+ const aborted = job.status === 'aborted';
+ return {
+ final: true,
+ reconcile: true,
+ reconcileReason: aborted ? 'abort_persistence_failed' : 'terminal_payload_missing',
+ /** A completion whose outcome-defining write is unknown must not trigger
+ * completed-run follow-up draining. Surface it conservatively as an error;
+ * the persisted history refetch remains authoritative if the write won
+ * before the owner disappeared. */
+ terminalStatus: aborted ? 'aborted' : 'error',
+ generationCreatedAt: job.createdAt,
+ conversation: { conversationId: job.conversationId },
+ };
+}
function getToolCallName(toolCall: unknown): unknown {
return toolCall != null && typeof toolCall === 'object' && 'name' in toolCall
@@ -88,6 +281,40 @@ function hasOAuthToolCall(toolCalls: unknown): boolean {
);
}
+/** One recovery projection for both still-queued and already-drained steers.
+ * Claimed items remain non-drainable, but a reconnect must still render them
+ * until their applied chunk is durably committed. */
+function mergeUnresolvedSteers(...groups: SteerQueueItem[][]): SteerQueueItem[] {
+ const byId = new Map<string, SteerQueueItem>();
+ for (const group of groups) {
+ for (const item of group) {
+ byId.set(item.steerId, item);
+ }
+ }
+ // Each group is already FIFO and callers pass the claimed prefix before the
+ // still-queued suffix. Map preserves first insertion position while still
+ // letting a later duplicate refresh its value; timestamps and random UUIDs
+ // are not safe ordering keys for two accepts in the same millisecond.
+ return [...byId.values()];
+}
+
+function omitAlreadyAppliedSteers(items: SteerQueueItem[], content: unknown[]): SteerQueueItem[] {
+ const appliedIds = new Set<string>();
+ for (const part of content) {
+ if (
+ part != null &&
+ typeof part === 'object' &&
+ 'type' in part &&
+ part.type === ContentTypes.STEER &&
+ 'steerId' in part &&
+ typeof part.steerId === 'string'
+ ) {
+ appliedIds.add(part.steerId);
+ }
+ }
+ return items.filter((item) => !appliedIds.has(item.steerId));
+}
+
function getReplayStepId(event: t.ServerSentEvent): unknown {
if (!('event' in event) || !event.data || typeof event.data !== 'object') {
return undefined;
@@ -189,6 +416,38 @@ export interface GenerationJobManagerOptions {
export interface CreateGenerationJobOptions {
startupTelemetry?: AgentStartupTelemetry;
initialMetadata?: Partial<t.GenerationJobMetadata>;
+ /** A terminal steer being handed off as this new normal turn. The store
+ * leases this exact parked item while the generation is active; persistence
+ * commits its removal through `consumeRecovered`. */
+ recoveredSteerId?: string;
+ /** Exact user-visible source proof checked inside the store's atomic create.
+ * Required whenever `recoveredSteerId` is present. */
+ recoveredSteerPayload?: RecoveredSteerPayload;
+ idempotencyClientRequestId?: string;
+ idempotencyClaimToken?: string;
+ /** Optional compare-and-set fence from the client's last authoritative
+ * status result. Creation may proceed only if that exact epoch is still
+ * current or the stream has no durable job. */
+ expectedPredecessorCreatedAt?: number;
+}
+
+/**
+ * Proof that this manager won the only legal `running -> terminal` transition
+ * for one generation. Controllers must publish their terminal event only after
+ * receiving this claim, then pass the same object to {@link finishTerminalJob}
+ * from a `finally` block.
+ */
+export interface TerminalJobClaim {
+ readonly streamId: string;
+ readonly createdAt: number;
+ readonly conversationId?: string;
+ readonly status: 'complete' | 'error' | 'aborted';
+ readonly error?: string;
+ /** The winner must durably publish either its normal FINAL or a
+ * reconciliation payload before cleanup may remove the job. */
+ readonly persistencePending?: true;
+ /** Exact queued + claimed steers atomically parked by the winning CAS. */
+ readonly drainedSteers: readonly SteerQueueItem[];
}
/**
@@ -216,6 +475,10 @@ interface RuntimeJobState {
abortController: AbortController;
/** Removes this generation's cross-replica abort listener without touching a replacement. */
abortUnsubscribe?: () => void;
+ /** A durable successor has stopped this provider but has not yet activated
+ * its own abort registration. Keep this runtime's transport callbacks as a
+ * channel handoff bridge until the successor explicitly releases them. */
+ replacementTransportHold?: boolean;
/**
* Cooperative-seal requests for THIS generation. Armed by `requestPreempt`
* (locally or via a fenced cross-replica publish), polled O(1) by the run's
@@ -238,6 +501,10 @@ interface RuntimeJobState {
* id can never legitimately be re-armed.
*/
cleared: Set<string>;
+ /** Lowest revision a still-queued steer may use after a non-terminal
+ * capability downgrade. Unlike `cleared`, this allows a later explicit
+ * arm whose store-assigned revision is newer. */
+ minArmRevision: Map<string, number>;
};
/** Removes this generation's cross-replica preempt listener without touching a replacement. */
preemptUnsubscribe?: () => void;
@@ -251,6 +518,9 @@ interface RuntimeJobState {
localErrorHandlers: Set<t.ErrorHandler>;
/** Prevents a repeated approval-expiry sweep from republishing the same terminal event. */
approvalExpiryPublished?: boolean;
+ /** Prevents a local or cross-replica stale-pause relay from retiring and
+ * republishing the same exact runtime more than once. */
+ pausePersistenceTimeoutRetired?: boolean;
syncSent: boolean;
earlyEventBuffer: t.ServerSentEvent[];
earlyEventSequencePromises: Array<Promise<void | number>>;
@@ -310,7 +580,7 @@ type DeferredDelivery =
*/
class GenerationJobManagerClass {
/** Job metadata + content state storage - swappable for Redis, etc. */
- private jobStore: IJobStore;
+ private jobStore: IJobStoreV2;
/** Guarded human-review lifecycle (pause / resolve / expire) over the store. */
private _approvals: ApprovalLifecycle;
/** FIFO steering queue (enqueue / drain / peek / clear) over the store. */
@@ -333,6 +603,17 @@ class GenerationJobManagerClass {
/** Partial-response and disconnect-state writes still draining during shutdown. */
private subscriberCleanupPromises = new Set<Promise<void>>();
+ /** Makes terminal cleanup idempotent while keeping claims opaque to callers. */
+ private terminalFinishPromises = new WeakMap<TerminalJobClaim, Promise<void>>();
+
+ /** Exact local runtime observed when a claim won; never clean a later runtime. */
+ private terminalClaimRuntimes = new WeakMap<TerminalJobClaim, RuntimeJobState | null>();
+
+ /** Claims whose durable final exists but whose DONE transport publication
+ * failed. Retain their job record through normal finish so the forced
+ * reconnect can replay that authoritative final payload. */
+ private terminalPublicationFailures = new WeakSet<TerminalJobClaim>();
+
private cleanupInterval: NodeJS.Timeout | null = null;
/** Rejects new jobs once graceful shutdown has started. */
@@ -345,8 +626,10 @@ class GenerationJobManagerClass {
private _cleanupOnComplete = true;
constructor(options?: GenerationJobManagerOptions) {
- this.jobStore =
+ const jobStore =
options?.jobStore ?? new InMemoryJobStore({ ttlAfterComplete: 0, maxJobs: 1000 });
+ assertJobStoreV2(jobStore);
+ this.jobStore = jobStore;
this._approvals = this.createApprovalLifecycle(this.jobStore);
this._steering = new SteeringLifecycle(this.jobStore);
this.eventTransport = options?.eventTransport ?? new InMemoryEventTransport();
@@ -395,6 +678,7 @@ class GenerationJobManagerClass {
isRedis?: boolean;
cleanupOnComplete?: boolean;
}): void {
+ assertJobStoreV2(services.jobStore);
const previousStore = this.storeLabel;
if (this.cleanupInterval) {
logger.warn(
@@ -409,7 +693,7 @@ class GenerationJobManagerClass {
for (const runtime of this.runtimeState.values()) {
runtime.startupTelemetry?.end('aborted');
runtime.startupTelemetry = undefined;
- this.releaseAbortSubscription(runtime);
+ this.releaseAbortSubscription(runtime, true);
runtime.abortController.abort();
}
@@ -465,12 +749,64 @@ class GenerationJobManagerClass {
setGenerationJobsInFlight(store, this.ownedJobs.size);
}
- private createApprovalLifecycle(store: IJobStore): ApprovalLifecycle {
+ private createApprovalLifecycle(store: IJobStoreV2): ApprovalLifecycle {
return new ApprovalLifecycle(store, {
onPaused: (streamId, createdAt) => this.releaseJobOwnership(streamId, createdAt),
onResumed: (streamId, createdAt) => this.acquireResumedJobOwnership(streamId, createdAt),
onExpired: (streamId, createdAt) => this.releaseJobOwnership(streamId, createdAt),
+ onPausePersistenceFailed: (streamId, createdAt, error, drainedSteers) =>
+ this.finishTimedOutPausePersistence(streamId, createdAt, error, drainedSteers, true),
+ });
+ }
+
+ /**
+ * The approval lifecycle already won the exact barrier-to-error CAS. Reuse
+ * the normal terminal finisher so attached clients receive the durable error
+ * and every runtime/ownership resource is retired with the same epoch guards
+ * as an ordinary generation failure.
+ */
+ private async finishTimedOutPausePersistence(
+ streamId: string,
+ createdAt: number,
+ error: string,
+ drainedSteers: SteerQueueItem[],
+ transitionWonLocally = false,
+ ): Promise<void> {
+ const observedRuntime = this.runtimeState.get(streamId);
+ const runtime = observedRuntime?.createdAt === createdAt ? observedRuntime : undefined;
+ if (runtime?.pausePersistenceTimeoutRetired === true) {
+ return;
+ }
+ const job = await this.jobStore.getJob(streamId);
+ if (job?.createdAt !== createdAt) {
+ /** A locally-won CAS may have used a zero completed TTL, or a
+ * replacement may have landed before this follow-up read. The claim's
+ * generation tag still makes publication/runtime cleanup safe. A relay,
+ * by contrast, needs the durable timeout row as its proof. */
+ if (!transitionWonLocally) {
+ this.reconcileInactiveGeneration(streamId, createdAt, job, runtime);
+ return;
+ }
+ }
+ if (job?.createdAt === createdAt && (job.status !== 'error' || job.error !== error)) {
+ return;
+ }
+
+ if (runtime) {
+ runtime.pausePersistenceTimeoutRetired = true;
+ }
+
+ const claim: TerminalJobClaim = Object.freeze({
+ streamId,
+ createdAt,
+ ...(job?.createdAt === createdAt &&
+ job.conversationId != null && { conversationId: job.conversationId }),
+ status: 'error' as const,
+ error,
+ drainedSteers: Object.freeze([...drainedSteers]),
});
+ this.terminalClaimRuntimes.set(claim, runtime ?? null);
+ await this.finishTerminalJob(claim);
}
private acquireJobOwnership(streamId: string, createdAt: number): void {
@@ -503,8 +839,14 @@ class GenerationJobManagerClass {
* runtime goes through here, so preempt state cannot outlive the
* generation it belongs to.
*/
- private releaseAbortSubscription(runtime: RuntimeJobState): void {
- this.releasePreemptSubscription(runtime);
+ private releaseAbortSubscription(runtime: RuntimeJobState, force = false): void {
+ if (runtime.replacementTransportHold === true && !force) {
+ return;
+ }
+ if (force) {
+ runtime.replacementTransportHold = false;
+ }
+ this.releasePreemptSubscription(runtime, force);
const unsubscribe = runtime.abortUnsubscribe;
runtime.abortUnsubscribe = undefined;
if (!unsubscribe) {
@@ -518,7 +860,10 @@ class GenerationJobManagerClass {
}
}
- private releasePreemptSubscription(runtime: RuntimeJobState): void {
+ private releasePreemptSubscription(runtime: RuntimeJobState, force = false): void {
+ if (runtime.replacementTransportHold === true && !force) {
+ return;
+ }
runtime.preempt = undefined;
const unsubscribe = runtime.preemptUnsubscribe;
runtime.preemptUnsubscribe = undefined;
@@ -568,6 +913,54 @@ class GenerationJobManagerClass {
this.reconcileInactiveGeneration(streamId, createdAt, currentJob, observedRuntime);
}
+ /** Promotes a terminal claim whose persistence owner disappeared to a conservative
+ * terminal payload. The store CAS races safely with a slow owner: whichever
+ * side finalizes first chooses the only payload subscribers may consume. */
+ private async recoverStaleTerminalPersistence(
+ jobData: SerializableJobData,
+ ): Promise<SerializableJobData | null> {
+ if (
+ jobData.terminalPersistencePending !== true ||
+ !['complete', 'error', 'aborted'].includes(jobData.status)
+ ) {
+ return jobData;
+ }
+ const startedAt =
+ jobData.terminalPersistenceStartedAt ?? jobData.completedAt ?? jobData.createdAt;
+ if (Date.now() - startedAt < TERMINAL_PERSISTENCE_TIMEOUT_MS) {
+ return jobData;
+ }
+
+ const reconcileEvent = buildTerminalPersistenceReconcile(jobData);
+ const serialized = JSON.stringify(reconcileEvent);
+ const recovered = await this.jobStore.finalizeTerminalPersistence(
+ jobData.streamId,
+ jobData.createdAt,
+ serialized,
+ );
+ if (!recovered) {
+ return this.jobStore.getJob(jobData.streamId);
+ }
+
+ const runtime = this.runtimeState.get(jobData.streamId);
+ if (runtime?.createdAt === jobData.createdAt) {
+ runtime.finalEvent = reconcileEvent;
+ }
+ try {
+ await this.eventTransport.emitDone(jobData.streamId, reconcileEvent, jobData.createdAt);
+ } catch (err) {
+ logger.error(
+ `[GenerationJobManager] Failed to publish stale terminal-persistence recovery ${jobData.streamId}:`,
+ err,
+ );
+ }
+ return {
+ ...jobData,
+ terminalPersistencePending: false,
+ finalEvent: serialized,
+ };
+ }
+
private async registerAbortSubscription(
streamId: string,
runtime: RuntimeJobState,
@@ -580,15 +973,18 @@ class GenerationJobManagerClass {
const currentRuntime = this.runtimeState.get(streamId);
if (
currentRuntime !== runtime ||
- (generationId != null && currentRuntime.createdAt !== generationId) ||
- currentRuntime.abortController.signal.aborted
+ (generationId != null && currentRuntime.createdAt !== generationId)
) {
- return;
+ return false;
}
- logger.debug(`[GenerationJobManager] Received cross-replica abort for ${streamId}`);
- currentRuntime.abortController.abort();
- this.releaseAbortSubscription(currentRuntime);
+ const ownsProvider = this.ownedJobs.get(streamId) === currentRuntime.createdAt;
+ if (!currentRuntime.abortController.signal.aborted) {
+ logger.debug(`[GenerationJobManager] Received cross-replica abort for ${streamId}`);
+ currentRuntime.abortController.abort();
+ this.releaseAbortSubscription(currentRuntime);
+ }
+ return ownsProvider;
});
if (typeof unsubscribe === 'function') {
@@ -603,7 +999,12 @@ class GenerationJobManagerClass {
runtime: RuntimeJobState,
createdAt: number,
): NonNullable<RuntimeJobState['preempt']> {
- runtime.preempt ??= { createdAt, ids: new Set(), cleared: new Set() };
+ runtime.preempt ??= {
+ createdAt,
+ ids: new Set(),
+ cleared: new Set(),
+ minArmRevision: new Map(),
+ };
return runtime.preempt;
}
@@ -613,11 +1014,21 @@ class GenerationJobManagerClass {
* ordinary boundary while this request was in flight) or an over-cap id is
* skipped, and the caller must not report those as armed.
*/
- private armPreemptIds(runtime: RuntimeJobState, createdAt: number, steerIds: string[]): number {
+ private armPreemptIds(
+ runtime: RuntimeJobState,
+ createdAt: number,
+ steerIds: string[],
+ revisions?: Record<string, number>,
+ ): number {
const state = this.ensurePreemptState(runtime, createdAt);
let accepted = 0;
for (const id of steerIds) {
- if (state.cleared.has(id) || state.ids.size >= STEER_QUEUE_MAX_DEPTH) {
+ const revision = revisions?.[id] ?? 0;
+ if (
+ state.cleared.has(id) ||
+ revision < (state.minArmRevision.get(id) ?? 0) ||
+ state.ids.size >= STEER_QUEUE_MAX_DEPTH
+ ) {
continue;
}
state.ids.add(id);
@@ -626,6 +1037,21 @@ class GenerationJobManagerClass {
return accepted;
}
+ /** Non-terminal disarm used by a capable→incapable owner handover. The
+ * queue item remains valid, so this records a revision floor instead of the
+ * permanent removal tombstone used by `clearPreemptIds`. */
+ private downgradePreemptIds(
+ runtime: RuntimeJobState,
+ createdAt: number,
+ steers: Array<{ steerId: string; preemptRevision?: number }>,
+ ): void {
+ const state = this.ensurePreemptState(runtime, createdAt);
+ for (const steer of steers) {
+ state.ids.delete(steer.steerId);
+ state.minArmRevision.set(steer.steerId, (steer.preemptRevision ?? 0) + 1);
+ }
+ }
+
/**
* Disarms ids and tombstones them against a late-arriving arm. The
* tombstone set is bounded by EVICTING its oldest entry rather than
@@ -651,6 +1077,68 @@ class GenerationJobManagerClass {
}
}
+ /**
+ * Pub/sub delivery is not durable: an ARM can sit in a subscriber buffer
+ * until after the steer drained and a handover reconciliation observed an
+ * empty queue. Re-check the generation-fenced queue on receipt so that late
+ * message cannot resurrect an interrupt with no payload behind it.
+ *
+ * The durable revision, rather than the publication's revision, is used for
+ * a still-preempting item. This preserves liveness when an older duplicate
+ * arrives after a later explicit arm, while `preempt: false` advances the
+ * local floor and keeps a pre-downgrade publication from re-arming it.
+ */
+ private async validateAndArmPreemptMessage(
+ streamId: string,
+ runtime: RuntimeJobState,
+ msg: PreemptMessage,
+ ): Promise<void> {
+ const queued = await this._steering.peek(streamId, msg.createdAt);
+ const currentRuntime = this.runtimeState.get(streamId);
+ if (currentRuntime !== runtime || currentRuntime.createdAt !== msg.createdAt) {
+ return;
+ }
+
+ const queuedById = new Map(queued.map((item) => [item.steerId, item]));
+ const backedIds: string[] = [];
+ const backedRevisions: Record<string, number> = {};
+ const removedIds: string[] = [];
+ const downgraded: SteerQueueItem[] = [];
+
+ for (const steerId of msg.steerIds) {
+ const item = queuedById.get(steerId);
+ if (item == null) {
+ removedIds.push(steerId);
+ continue;
+ }
+ if (item.preempt !== true) {
+ downgraded.push(item);
+ continue;
+ }
+
+ const durableRevision = item.preemptRevision ?? 0;
+ const publishedRevision = msg.revisions?.[steerId] ?? 0;
+ if (publishedRevision > durableRevision) {
+ /** The publication cannot be backed by this authoritative snapshot. */
+ continue;
+ }
+ backedIds.push(steerId);
+ backedRevisions[steerId] = durableRevision;
+ }
+
+ if (removedIds.length > 0) {
+ this.clearPreemptIds(currentRuntime, msg.createdAt, removedIds);
+ }
+ if (downgraded.length > 0) {
+ this.downgradePreemptIds(currentRuntime, msg.createdAt, downgraded);
+ }
+ if (backedIds.length > 0) {
+ /** A local drain/CLEAR during the queue read has already tombstoned the
+ * id, and `armPreemptIds` re-checks that state synchronously here. */
+ this.armPreemptIds(currentRuntime, msg.createdAt, backedIds, backedRevisions);
+ }
+ }
+
/**
* Mirrors {@link registerAbortSubscription} for cooperative preempts. The
* same double fence applies — runtime object identity plus the generation
@@ -688,7 +1176,13 @@ class GenerationJobManagerClass {
return;
}
- this.armPreemptIds(currentRuntime, msg.createdAt, msg.steerIds);
+ void this.validateAndArmPreemptMessage(streamId, runtime, msg).catch((error) => {
+ logger.error(
+ `[GenerationJobManager] Failed to validate preempt arm for ${streamId}; ` +
+ 'the queued steer will apply at the next tool boundary:',
+ error,
+ );
+ });
});
if (typeof unsubscribe === 'function') {
@@ -818,7 +1312,7 @@ class GenerationJobManagerClass {
/**
* Get the job store instance (for advanced use cases).
*/
- getJobStore(): IJobStore {
+ getJobStore(): IJobStoreV2 {
return this.jobStore;
}
@@ -838,83 +1332,828 @@ class GenerationJobManagerClass {
* @param conversationId - Optional conversation ID for lookup
* @returns A facade object for the GenerationJob
*/
- async createJob(
+ private async notifyReplacedGeneration(
streamId: string,
- userId: string,
- conversationId?: string,
- options: CreateGenerationJobOptions = {},
- ): Promise<t.GenerationJob> {
- if (this.shuttingDown) {
- throw new Error(SHUTTING_DOWN_ERROR);
+ predecessorCreatedAt: number,
+ predecessorConversationId?: string,
+ fallbackConversationId?: string,
+ replacementCreationAttemptId?: string,
+ stopProvider = true,
+ ): Promise<boolean> {
+ const reconcileEvent: t.ServerSentEvent = {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: predecessorCreatedAt,
+ conversation: {
+ conversationId: predecessorConversationId ?? fallbackConversationId ?? streamId,
+ },
+ } satisfies t.FinalEvent;
+ const localPredecessor = this.runtimeState.get(streamId);
+ const ownsExactLocalProvider =
+ localPredecessor?.createdAt === predecessorCreatedAt &&
+ this.ownedJobs.get(streamId) === predecessorCreatedAt;
+ const stoppedExactLocalProvider = stopProvider && ownsExactLocalProvider;
+ if (localPredecessor?.createdAt === predecessorCreatedAt) {
+ localPredecessor.finalEvent = reconcileEvent;
+ if (stopProvider) {
+ // The durable replacement transaction has already removed this active
+ // epoch. Retire its local provider immediately, before any fallible
+ // cross-node publication. A terminal receipt, however, has no live
+ // provider and may still be completing required persistence locally.
+ localPredecessor.startupTelemetry?.end('replaced');
+ localPredecessor.startupTelemetry = undefined;
+ localPredecessor.replacementTransportHold = true;
+ localPredecessor.abortController.abort();
+ // Keep its transport registrations until the successor's abort
+ // subscription is active. Removing the last callback while an older
+ // registration is still resolving can unsubscribe the shared Redis
+ // channel, then strand the successor in the resubscribe gap.
+ }
+ }
+ // Stop the provider before closing attached clients. Both signals are
+ // generation-tagged and serve different consumers; this ordering keeps a
+ // remote owner from emitting a last post-handoff chunk.
+ let delivered = true;
+ if (stopProvider && !stoppedExactLocalProvider) {
+ try {
+ if (!this._isRedis) {
+ // Multiple managers can share an in-memory transport in one process.
+ // Best-effort delivery stops an exact predecessor owned by another
+ // manager; only Redis handoffs require the durable owner proof below.
+ this.eventTransport.emitAbort?.(streamId, predecessorCreatedAt);
+ } else if (this.eventTransport.emitAbortConfirmed != null) {
+ const ownerAcknowledged = await this.eventTransport.emitAbortConfirmed(
+ streamId,
+ predecessorCreatedAt,
+ );
+ if (!ownerAcknowledged) {
+ throw new Error('Generation owner did not acknowledge the replacement abort');
+ }
+ } else {
+ throw new Error('Event transport cannot confirm generation-owner aborts');
+ }
+ } catch (err) {
+ delivered = false;
+ logger.error(
+ `[GenerationJobManager] Failed to stop replaced generation ${streamId}@${predecessorCreatedAt}:`,
+ err,
+ );
+ }
+ }
+ try {
+ if (
+ replacementCreationAttemptId != null &&
+ this.eventTransport.emitReplacedDoneConfirmed != null
+ ) {
+ await this.eventTransport.emitReplacedDoneConfirmed(
+ streamId,
+ reconcileEvent,
+ predecessorCreatedAt,
+ replacementCreationAttemptId,
+ );
+ } else {
+ await this.eventTransport.emitDone(streamId, reconcileEvent, predecessorCreatedAt);
+ }
+ } catch (err) {
+ delivered = false;
+ logger.error(
+ `[GenerationJobManager] Failed to notify replaced generation ${streamId}@${predecessorCreatedAt}:`,
+ err,
+ );
}
+ return delivered;
+ }
- const tenantId = getTenantId();
- const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;
- const initialMetadata = sanitizeJobMetadata(options.initialMetadata ?? {});
- const jobData = await this.jobStore.createJob(
- streamId,
- userId,
- conversationId,
- safeTenantId,
- initialMetadata,
- );
- const currentRuntimeBeforeInstall = this.runtimeState.get(streamId);
- const ownedCreatedAtBeforeInstall = this.ownedJobs.get(streamId);
+ /** Finish local resource cleanup when a replacement cannot be exposed after
+ * it already stopped the previous provider. The epoch and object checks keep
+ * this from touching a newer runtime installed during any preceding await. */
+ private retireStoppedPredecessorRuntime(streamId: string, replacementCreatedAt: number): void {
+ const predecessor = this.runtimeState.get(streamId);
if (
- (currentRuntimeBeforeInstall != null &&
- currentRuntimeBeforeInstall.createdAt > jobData.createdAt) ||
- (ownedCreatedAtBeforeInstall != null && ownedCreatedAtBeforeInstall > jobData.createdAt)
+ predecessor == null ||
+ predecessor.createdAt >= replacementCreatedAt ||
+ !predecessor.abortController.signal.aborted
) {
- throw new Error('Generation job was replaced during initialization');
+ return;
}
- if (this.shuttingDown) {
- await this.jobStore.transitionStatus(streamId, {
- from: 'running',
- to: 'error',
- patch: { completedAt: Date.now(), error: SHUTDOWN_JOB_ERROR },
- expectCreatedAt: jobData.createdAt,
- });
- throw new Error(SHUTTING_DOWN_ERROR);
+ if (this.runtimeState.get(streamId) === predecessor) {
+ this.runtimeState.delete(streamId);
+ this.releaseJobOwnership(streamId, predecessor.createdAt);
+ this.releaseAbortSubscription(predecessor, true);
+ this.runStepBuffers?.delete(streamId);
+ this.replayEventWriteQueues.delete(streamId);
+ this.tokenUsageWriteQueues.delete(streamId);
+ this.jobStore.clearContentState(streamId, predecessor.createdAt);
+ try {
+ // The replacement cannot be exposed, so neither generation can
+ // safely finish this connection. Delete runtime ownership before
+ // closing: allSubscribersLeft may run synchronously and must not
+ // persist a partial response for the already-replaced predecessor.
+ this.eventTransport.closeLocalSubscribers?.(streamId, TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ } catch (error) {
+ logger.error(
+ `[GenerationJobManager] Failed to recycle subscribers for stopped predecessor ${streamId}:`,
+ error,
+ );
+ }
}
- this.acquireJobOwnership(streamId, jobData.createdAt);
- recordGenerationJob(this.storeLabel, 'created');
+ }
- const replacedRuntime = this.runtimeState.get(streamId);
- if (replacedRuntime) {
- replacedRuntime.startupTelemetry?.end('replaced');
- replacedRuntime.startupTelemetry = undefined;
- this.releaseAbortSubscription(replacedRuntime);
- replacedRuntime.abortController.abort();
+ /** Deliver every transaction-time predecessor receipt retained by the
+ * current creation attempt, then epoch/attempt-fenced acknowledge only the
+ * receipts whose provider/client handoff completed. A replacement racing the
+ * acknowledgement inherits the still-unacknowledged chain and retries it. */
+ private async processDurableReplacementReceipts(
+ streamId: string,
+ job: CreatedJobData,
+ fallbackConversationId?: string,
+ acknowledgeOwnedReceipts = true,
+ ): Promise<boolean> {
+ const receipts = job.replacedJobs ?? (job.replacedJob != null ? [job.replacedJob] : []);
+ if (receipts.length === 0) {
+ return true;
}
- /**
- * Create runtime state with readyPromise.
- *
- * With the resumable stream architecture, we no longer need to wait for the
- * first subscriber before starting generation:
- * - Redis mode: Events are persisted and can be replayed via sync
- * - In-memory mode: Content is aggregated and sent via sync on connect
- *
- * We resolve readyPromise immediately to eliminate startup latency.
- * The sync mechanism handles late-connecting clients.
- */
- const readyPromise = Promise.resolve();
- const resolveReady = (): void => undefined;
+ const acknowledged: number[] = [];
+ let allReceiptsDelivered = true;
+ for (const receipt of receipts) {
+ // A paused run has already released provider ownership. If resume won
+ // before this atomic replacement, the receipt status is running and the
+ // resumed owner must acknowledge like any other live provider.
+ const hadRunningProvider =
+ receipt.status === 'running' && receipt.providerAbortReady !== false;
+ if (
+ await this.notifyReplacedGeneration(
+ streamId,
+ receipt.createdAt,
+ receipt.conversationId,
+ fallbackConversationId,
+ job.creationAttemptId,
+ hadRunningProvider,
+ )
+ ) {
+ acknowledged.push(receipt.createdAt);
+ } else {
+ allReceiptsDelivered = false;
+ }
+ }
- const runtime: RuntimeJobState = {
- createdAt: jobData.createdAt,
- abortController: new AbortController(),
- readyPromise,
- resolveReady,
- startupTelemetry: options.startupTelemetry,
- syncSent: false,
- earlyEventBuffer: [],
- earlyEventSequencePromises: [],
- earlyReplayHandlers: new Set(),
- resumeCaptureHandlers: new Set(),
- localErrorHandlers: new Set(),
- emissionSequence: 0,
- inFlightSnapshotEmissions: new Map(),
+ if (
+ acknowledged.length > 0 &&
+ acknowledgeOwnedReceipts &&
+ job.creationAttemptId != null &&
+ this.jobStore.acknowledgeReplacedJobs != null
+ ) {
+ try {
+ await this.jobStore.acknowledgeReplacedJobs(streamId, job.creationAttemptId, acknowledged);
+ } catch (error) {
+ // The acknowledgement is cleanup, not ownership proof. A lost reply
+ // either committed the trim or leaves the receipts for the next
+ // replacement to deliver again.
+ logger.error(
+ `[GenerationJobManager] Failed to acknowledge replacement receipts for ${streamId}:`,
+ error,
+ );
+ }
+ }
+ return allReceiptsDelivered;
+ }
+
+ /** A legacy started-mark EVAL can commit and lose its reply. Probe both
+ * mirrors before failing closed: the same-slot primary is authoritative for
+ * the full claim lineage, and the legacy key is safe only when it contains
+ * that exact lineage plus this job's started epoch/protocol. SET-NX may
+ * recreate an unexpectedly missing legacy mirror with the already-started
+ * primary value, which establishes the rollout fence before exposure. */
+ private async confirmLegacyStartedFence(
+ streamId: string,
+ userId: string,
+ conversationId: string,
+ clientRequestId: string,
+ claimToken: string,
+ job: Pick<SerializableJobData, 'createdAt' | 'generationProtocolVersion'>,
+ ): Promise<boolean> {
+ const fallback: TokenIdempotencyClaim = {
+ streamId,
+ conversationId,
+ claimedAt: Date.now(),
+ claimToken,
+ startedAt: job.createdAt,
+ generationProtocolVersion: normalizeClaimProtocol(job),
+ };
+ const primaryResult = await this.jobStore.claimIdempotencyKey(
+ this.generationClaimKey(userId, clientRequestId, streamId),
+ fallback,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ if (primaryResult.claimed) {
+ // Atomic job creation with an idempotency token must have marked this
+ // same-slot key. Recreating it is a conservative tombstone, not proof
+ // that the generation was safely admitted.
+ return false;
+ }
+ const primary = normalizeTokenClaim(primaryResult.existing, 'primary started fence');
+ assertClaimMatchesRequest(primary, streamId, conversationId);
+ if (
+ primary.claimToken !== claimToken ||
+ primary.startedAt !== job.createdAt ||
+ primary.generationProtocolVersion !== normalizeClaimProtocol(job)
+ ) {
+ return false;
+ }
+
+ const legacyResult = await this.jobStore.claimIdempotencyKey(
+ this.legacyGenerationClaimKey(userId, clientRequestId),
+ primary,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ const legacy = legacyResult.claimed
+ ? primary
+ : normalizeTokenClaim(legacyResult.existing, 'legacy started fence');
+ return claimsMirrorExactly(legacy, primary);
+ }
+
+ /**
+ * A Redis script can commit a job and its started idempotency tombstone, then
+ * lose the reply before the store repairs cross-slot membership. Recover only
+ * when the durable job's opaque attempt id (and, when present, its started
+ * claim) proves this invocation created this owner/tenant/epoch. The same-status transition is
+ * intentionally idempotent: besides refreshing the live TTL, Redis uses it to
+ * reconcile the running/user membership sets that live outside the stream's
+ * cluster slot.
+ */
+ private async recoverCommittedCreate(
+ streamId: string,
+ userId: string,
+ conversationId: string | undefined,
+ tenantId: string | undefined,
+ options: CreateGenerationJobOptions,
+ creationAttemptId: string,
+ ): Promise<CreatedJobData | null> {
+ const clientRequestId = options.idempotencyClientRequestId;
+ const claimToken = options.idempotencyClaimToken;
+ const expectedConversationId = conversationId ?? streamId;
+ const matchesCommittedCreate = (job: SerializableJobData | null): job is CreatedJobData =>
+ job != null &&
+ job.status === 'running' &&
+ (job as CreatedJobData).creationAttemptId === creationAttemptId &&
+ job.userId === userId &&
+ (job.tenantId ?? undefined) === tenantId &&
+ (job.conversationId ?? streamId) === expectedConversationId &&
+ (job.idempotencyClientRequestId ?? undefined) === clientRequestId;
+
+ let committedJob = (await this.jobStore.getJob(streamId)) as CreatedJobData | null;
+ if (
+ committedJob != null &&
+ committedJob.userId === userId &&
+ (committedJob.tenantId ?? undefined) === tenantId
+ ) {
+ // Keep a non-null snapshot across the awaited handoff. `committedJob` is
+ // mutable because later recovery may refresh it, so TypeScript cannot
+ // safely retain a narrowing on that outer binding across the await.
+ const currentCommittedJob = committedJob;
+ const ownsCommittedCreate = matchesCommittedCreate(currentCommittedJob);
+ const predecessorsDelivered = await this.processDurableReplacementReceipts(
+ streamId,
+ currentCommittedJob,
+ conversationId,
+ // A superseded/lost-reply helper may deliver the current creator's
+ // receipts, but only that exact opaque creation attempt may clear
+ // them. Otherwise it can race the owner's stale return value: the
+ // helper ACKs first, the owner's replacement-authorized publish is
+ // fenced, and the owner incorrectly terminalizes itself.
+ ownsCommittedCreate,
+ );
+ if (ownsCommittedCreate && !predecessorsDelivered) {
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ currentCommittedJob,
+ 'Generation predecessor handoff could not be confirmed',
+ );
+ return null;
+ }
+ }
+ if (!matchesCommittedCreate(committedJob)) {
+ return null;
+ }
+
+ // Same-slot creation proof comes first. Cross-slot legacy verification may
+ // be unavailable after the commit; in that case this exact epoch is
+ // terminalized below instead of leaving a provider-less running ghost.
+ if (clientRequestId != null && claimToken != null) {
+ try {
+ const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
+ const primaryFallback: TokenIdempotencyClaim = {
+ streamId,
+ conversationId: expectedConversationId,
+ claimedAt: Date.now(),
+ claimToken,
+ startedAt: committedJob.createdAt,
+ generationProtocolVersion: normalizeClaimProtocol(committedJob),
+ };
+ const primaryResult = await this.jobStore.claimIdempotencyKey(
+ primaryKey,
+ primaryFallback,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ const primary = primaryResult.claimed
+ ? primaryFallback
+ : normalizeTokenClaim(primaryResult.existing, 'primary create recovery');
+ assertClaimMatchesRequest(primary, streamId, expectedConversationId);
+ if (
+ primary.claimToken !== claimToken ||
+ primary.startedAt !== committedJob.createdAt ||
+ primary.generationProtocolVersion !== normalizeClaimProtocol(committedJob)
+ ) {
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ committedJob,
+ 'Generation idempotency proof did not match its ambiguous create',
+ );
+ return null;
+ }
+
+ const legacyResult = await this.jobStore.claimIdempotencyKey(
+ this.legacyGenerationClaimKey(userId, clientRequestId),
+ primary,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ const legacy = legacyResult.claimed
+ ? primary
+ : normalizeTokenClaim(legacyResult.existing, 'legacy create recovery');
+ assertClaimMatchesRequest(legacy, streamId, expectedConversationId);
+ if (legacy.claimToken !== claimToken) {
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ committedJob,
+ 'Generation legacy idempotency proof did not match its ambiguous create',
+ );
+ return null;
+ }
+ if (!claimsMirrorExactly(legacy, primary)) {
+ assertClaimMirrors(legacy, primary);
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, legacy, primary);
+ }
+
+ if (primaryResult.claimed) {
+ // JOB_CREATE with an idempotency token marks this same-slot key in
+ // its atomic script. Recreating a missing key is a conservative
+ // tombstone repair, not enough evidence to expose the generation.
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ committedJob,
+ 'Generation primary idempotency proof was missing after its ambiguous create',
+ );
+ return null;
+ }
+ } catch (error) {
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ committedJob,
+ 'Generation idempotency rollout fence could not be recovered',
+ );
+ throw error;
+ }
+ }
+
+ let repairError: unknown;
+ for (let attempt = 0; attempt < 2; attempt++) {
+ try {
+ const repaired = await this.jobStore.transitionStatus(streamId, {
+ from: 'running',
+ to: 'running',
+ expectCreatedAt: committedJob.createdAt,
+ });
+ if (repaired) {
+ const confirmed = await this.jobStore.getJob(streamId);
+ if (matchesCommittedCreate(confirmed)) {
+ return confirmed;
+ }
+ return null;
+ }
+ } catch (error) {
+ // The repair itself can commit and lose its reply. A second identical
+ // CAS is safe and gives us a positive, membership-reconciled result.
+ repairError = error;
+ }
+
+ const current = await this.jobStore.getJob(streamId);
+ if (!matchesCommittedCreate(current)) {
+ return null;
+ }
+ committedJob = current;
+ }
+
+ if (repairError != null) {
+ logger.error(
+ `[GenerationJobManager] Failed to confirm membership for recovered generation ${streamId}:`,
+ repairError,
+ );
+ }
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ committedJob,
+ 'Generation membership could not be recovered after an ambiguous create',
+ );
+ return null;
+ }
+
+ /** Persist a conservative FINAL for a job whose generation was never exposed
+ * to its controller. This prevents both a provider-less running ghost and a
+ * terminal hash that makes every duplicate retry wait until Redis TTL expiry. */
+ private async terminalizeUnexposedGeneration(
+ streamId: string,
+ job: Pick<SerializableJobData, 'createdAt' | 'conversationId'>,
+ message: string,
+ ): Promise<boolean> {
+ const finalEvent: t.FinalEvent = {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'error',
+ generationCreatedAt: job.createdAt,
+ conversation: { conversationId: job.conversationId ?? streamId },
+ };
+ let lastError: unknown;
+ for (let attempt = 0; attempt < 2; attempt++) {
+ try {
+ if (
+ await this.jobStore.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ patch: {
+ completedAt: Date.now(),
+ error: message,
+ finalEvent: JSON.stringify(finalEvent),
+ },
+ expectCreatedAt: job.createdAt,
+ })
+ ) {
+ return true;
+ }
+ } catch (error) {
+ // As with create, the terminal CAS may have committed before the reply
+ // was lost. Probe its exact epoch before retrying.
+ lastError = error;
+ }
+
+ try {
+ const current = await this.jobStore.getJob(streamId);
+ if (
+ current == null ||
+ current.createdAt !== job.createdAt ||
+ (current.status !== 'running' && current.status !== 'requires_action')
+ ) {
+ return true;
+ }
+ } catch (error) {
+ lastError = error;
+ }
+ }
+
+ if (lastError != null) {
+ logger.error(
+ `[GenerationJobManager] Failed to terminalize unexposed generation ${streamId}:`,
+ lastError,
+ );
+ }
+ return false;
+ }
+
+ async createJob(
+ streamId: string,
+ userId: string,
+ conversationId?: string,
+ options: CreateGenerationJobOptions = {},
+ ): Promise<t.GenerationJob> {
+ if (this.shuttingDown) {
+ throw new Error(SHUTTING_DOWN_ERROR);
+ }
+ if (typeof userId !== 'string' || userId.length === 0) {
+ throw new Error('Generation job requires a non-empty user id');
+ }
+ const hasClientRequestId = options.idempotencyClientRequestId != null;
+ const hasClaimToken = options.idempotencyClaimToken != null;
+ if (hasClientRequestId !== hasClaimToken) {
+ throw new Error(
+ 'Generation idempotency request id and claim token must be provided together',
+ );
+ }
+ if (
+ hasClientRequestId &&
+ (!CLIENT_REQUEST_ID_PATTERN.test(options.idempotencyClientRequestId!) ||
+ options.idempotencyClaimToken!.length === 0 ||
+ options.idempotencyClaimToken!.length > 128)
+ ) {
+ throw new Error('Generation idempotency request id or claim token is invalid');
+ }
+ if (
+ options.expectedPredecessorCreatedAt != null &&
+ (!Number.isSafeInteger(options.expectedPredecessorCreatedAt) ||
+ options.expectedPredecessorCreatedAt < 0)
+ ) {
+ throw new Error('Invalid expected generation predecessor');
+ }
+
+ const tenantId = getTenantId();
+ const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;
+ const initialMetadata = sanitizeJobMetadata(options.initialMetadata ?? {});
+ if (
+ (options.recoveredSteerId != null) !== (options.recoveredSteerPayload != null) ||
+ (options.recoveredSteerPayload != null &&
+ !isRecoveredSteerPayload(options.recoveredSteerPayload))
+ ) {
+ throw new RecoveredSteerPayloadMismatchError();
+ }
+ const creationAttemptId = randomUUID();
+ // Capture the active epoch before the store atomically replaces it. A
+ // subscriber attached to that predecessor filters events by generation,
+ // so replacement must explicitly close/handoff that old attachment.
+ const observedPredecessorJob = await this.jobStore.getJob(streamId);
+ let jobData: CreatedJobData;
+ try {
+ jobData = await this.jobStore.createJob(
+ streamId,
+ userId,
+ conversationId,
+ safeTenantId,
+ initialMetadata,
+ options.recoveredSteerId,
+ options.idempotencyClientRequestId != null
+ ? this.generationClaimKey(userId, options.idempotencyClientRequestId, streamId)
+ : undefined,
+ options.idempotencyClaimToken,
+ options.idempotencyClientRequestId,
+ options.recoveredSteerPayload,
+ creationAttemptId,
+ options.expectedPredecessorCreatedAt,
+ );
+ } catch (error) {
+ if (error instanceof JobPredecessorMismatchError) {
+ throw error;
+ }
+ if (error instanceof JobCreationSupersededError) {
+ try {
+ const current = (await this.jobStore.getJob(streamId)) as CreatedJobData | null;
+ if (
+ current != null &&
+ current.userId === userId &&
+ (current.tenantId ?? undefined) === safeTenantId
+ ) {
+ await this.processDurableReplacementReceipts(streamId, current, conversationId, false);
+ }
+ } catch (handoffError) {
+ logger.error(
+ `[GenerationJobManager] Failed to process superseded creation receipts ${streamId}:`,
+ handoffError,
+ );
+ }
+ throw error;
+ }
+
+ let recovered: CreatedJobData | null = null;
+ try {
+ recovered = await this.recoverCommittedCreate(
+ streamId,
+ userId,
+ conversationId,
+ safeTenantId,
+ options,
+ creationAttemptId,
+ );
+ } catch (recoveryError) {
+ logger.error(
+ `[GenerationJobManager] Failed to reconcile ambiguous job creation ${streamId}:`,
+ recoveryError,
+ );
+ }
+ if (recovered == null) {
+ throw error;
+ }
+ jobData = recovered;
+ }
+
+ if (options.idempotencyClientRequestId != null && options.idempotencyClaimToken != null) {
+ let legacyMarked = false;
+ let legacyMarkError: unknown;
+ try {
+ legacyMarked = await this.jobStore.markIdempotencyKeyStarted(
+ this.legacyGenerationClaimKey(userId, options.idempotencyClientRequestId),
+ options.idempotencyClaimToken,
+ jobData.createdAt,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ } catch (error) {
+ legacyMarkError = error;
+ logger.error(
+ '[GenerationJobManager] Failed to mark the legacy generation idempotency tombstone:',
+ error,
+ );
+ }
+ if (!legacyMarked) {
+ try {
+ legacyMarked = await this.confirmLegacyStartedFence(
+ streamId,
+ userId,
+ conversationId ?? streamId,
+ options.idempotencyClientRequestId,
+ options.idempotencyClaimToken,
+ jobData,
+ );
+ } catch (error) {
+ legacyMarkError = error;
+ logger.error(
+ '[GenerationJobManager] Failed to verify the legacy generation idempotency tombstone:',
+ error,
+ );
+ }
+ }
+ if (!legacyMarked) {
+ // The primary was committed atomically with this job, so generation
+ // ownership is no longer ambiguous on new replicas. Do not let the
+ // provider start while an old replica could still interpret an
+ // unmarked/mismatched legacy key as an abandoned request.
+ await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch(
+ (handoffError) => {
+ logger.error(
+ `[GenerationJobManager] Failed to hand off predecessor after legacy fence failure ${streamId}:`,
+ handoffError,
+ );
+ return false;
+ },
+ );
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ jobData,
+ 'Generation idempotency rollout fence could not be committed',
+ );
+ // If this create replaced a local provider, durable rollback is no
+ // longer possible. The receipt path below is skipped on this failure,
+ // so stop and retire that exact older runtime before returning.
+ const replacedLocal = this.runtimeState.get(streamId);
+ if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {
+ replacedLocal.abortController.abort();
+ this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);
+ }
+ if (legacyMarkError != null) {
+ logger.error(
+ '[GenerationJobManager] Legacy generation idempotency fence remained ambiguous:',
+ legacyMarkError,
+ );
+ }
+ throw new Error('Generation idempotency rollout fence could not be committed');
+ }
+ }
+ const currentRuntimeBeforeInstall = this.runtimeState.get(streamId);
+ const ownedCreatedAtBeforeInstall = this.ownedJobs.get(streamId);
+ if (
+ (currentRuntimeBeforeInstall != null &&
+ currentRuntimeBeforeInstall.createdAt > jobData.createdAt) ||
+ (ownedCreatedAtBeforeInstall != null && ownedCreatedAtBeforeInstall > jobData.createdAt)
+ ) {
+ throw new Error('Generation job was replaced during initialization');
+ }
+ if (this.shuttingDown) {
+ await this.processDurableReplacementReceipts(streamId, jobData, conversationId).catch(
+ (handoffError) => {
+ logger.error(
+ `[GenerationJobManager] Failed to hand off predecessor during shutdown ${streamId}:`,
+ handoffError,
+ );
+ return false;
+ },
+ );
+ await this.jobStore.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ patch: { completedAt: Date.now(), error: SHUTDOWN_JOB_ERROR },
+ expectCreatedAt: jobData.createdAt,
+ });
+ const replacedLocal = this.runtimeState.get(streamId);
+ if (replacedLocal != null && replacedLocal.createdAt < jobData.createdAt) {
+ replacedLocal.abortController.abort();
+ this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);
+ }
+ throw new Error(SHUTTING_DOWN_ERROR);
+ }
+
+ if (!(await this.processDurableReplacementReceipts(streamId, jobData, conversationId))) {
+ await this.terminalizeUnexposedGeneration(
+ streamId,
+ jobData,
+ 'Generation predecessor handoff could not be confirmed',
+ );
+ this.retireStoppedPredecessorRuntime(streamId, jobData.createdAt);
+ throw new Error('Generation predecessor handoff could not be confirmed');
+ }
+
+ const replacementEpochs = new Map<number, string | undefined>();
+ const exactPredecessors =
+ jobData.replacedJobs ?? (jobData.replacedJob != null ? [jobData.replacedJob] : []);
+ const exactPredecessorsByEpoch = new Map(
+ exactPredecessors.map((predecessor) => [predecessor.createdAt, predecessor]),
+ );
+ const observedPredecessorReceipt = observedPredecessorJob
+ ? exactPredecessorsByEpoch.get(observedPredecessorJob.createdAt)
+ : undefined;
+ if (
+ observedPredecessorJob &&
+ observedPredecessorJob.createdAt !== jobData.createdAt &&
+ (observedPredecessorJob.status === 'running' ||
+ observedPredecessorJob.status === 'requires_action') &&
+ // The transaction-time predecessor is authoritative for the same
+ // epoch. A pre-read may say running even though it terminalized before
+ // replacement; do not publish a contradictory replaced/abort pair.
+ observedPredecessorReceipt == null
+ ) {
+ replacementEpochs.set(
+ observedPredecessorJob.createdAt,
+ observedPredecessorJob.conversationId,
+ );
+ }
+ const localPredecessor = this.runtimeState.get(streamId);
+ const localPredecessorReceipt = localPredecessor
+ ? exactPredecessorsByEpoch.get(localPredecessor.createdAt)
+ : undefined;
+ if (
+ localPredecessor &&
+ localPredecessor.createdAt < jobData.createdAt &&
+ !localPredecessor.finalEvent &&
+ !localPredecessor.errorEvent &&
+ localPredecessorReceipt == null
+ ) {
+ replacementEpochs.set(localPredecessor.createdAt, observedPredecessorJob?.conversationId);
+ }
+ for (const [predecessorCreatedAt, predecessorConversationId] of replacementEpochs) {
+ await this.notifyReplacedGeneration(
+ streamId,
+ predecessorCreatedAt,
+ predecessorConversationId,
+ conversationId,
+ );
+ }
+
+ // Every predecessor handoff above can yield. A newer same-process create
+ // may have installed its runtime while this older invocation was waiting;
+ // never let the stale continuation overwrite ownership or abort that newer
+ // runtime. No await is permitted between this guard and the synchronous
+ // ownership/runtime installation below.
+ const runtimeImmediatelyBeforeInstall = this.runtimeState.get(streamId);
+ const ownedImmediatelyBeforeInstall = this.ownedJobs.get(streamId);
+ if (
+ (runtimeImmediatelyBeforeInstall != null &&
+ runtimeImmediatelyBeforeInstall.createdAt > jobData.createdAt) ||
+ (ownedImmediatelyBeforeInstall != null && ownedImmediatelyBeforeInstall > jobData.createdAt)
+ ) {
+ throw new Error('Generation job was replaced during initialization');
+ }
+
+ this.acquireJobOwnership(streamId, jobData.createdAt);
+ recordGenerationJob(this.storeLabel, 'created');
+
+ const replacedRuntime = this.runtimeState.get(streamId);
+ if (replacedRuntime) {
+ replacedRuntime.startupTelemetry?.end('replaced');
+ replacedRuntime.startupTelemetry = undefined;
+ const durableReceipt = exactPredecessorsByEpoch.get(replacedRuntime.createdAt);
+ if (
+ durableReceipt == null ||
+ durableReceipt.status === 'running' ||
+ durableReceipt.status === 'requires_action'
+ ) {
+ replacedRuntime.abortController.abort();
+ }
+ }
+
+ /**
+ * Create runtime state with readyPromise.
+ *
+ * With the resumable stream architecture, we no longer need to wait for the
+ * first subscriber before starting generation:
+ * - Redis mode: Events are persisted and can be replayed via sync
+ * - In-memory mode: Content is aggregated and sent via sync on connect
+ *
+ * We resolve readyPromise immediately to eliminate startup latency.
+ * The sync mechanism handles late-connecting clients.
+ */
+ const readyPromise = Promise.resolve();
+ const resolveReady = (): void => undefined;
+
+ const runtime: RuntimeJobState = {
+ createdAt: jobData.createdAt,
+ abortController: new AbortController(),
+ readyPromise,
+ resolveReady,
+ startupTelemetry: options.startupTelemetry,
+ syncSent: false,
+ earlyEventBuffer: [],
+ earlyEventSequencePromises: [],
+ earlyReplayHandlers: new Set(),
+ resumeCaptureHandlers: new Set(),
+ localErrorHandlers: new Set(),
+ emissionSequence: 0,
+ inFlightSnapshotEmissions: new Map(),
hasSubscriber: false,
attachmentGeneration: 0,
};
@@ -924,6 +2163,18 @@ class GenerationJobManagerClass {
this.registerAllSubscribersLeft(streamId);
await this.registerAbortSubscription(streamId, runtime);
+ if (replacedRuntime != null && replacedRuntime !== runtime) {
+ // Keep at least one abort registration on the shared Redis channel
+ // across same-stream replacement. This also lets a predecessor whose
+ // registration was still becoming ready dispose itself without
+ // briefly unsubscribing the successor.
+ this.releaseAbortSubscription(replacedRuntime, true);
+ }
+ // This epoch is not exposed to its controller until the durable bit and
+ // owner listener agree. A replacement that wins before this write sees
+ // explicit false and can safely skip an acknowledgement because no
+ // provider could have started; a lost write reply is confirmed below.
+ await this.jobStore.updateJob(streamId, { providerAbortReady: true }, runtime.createdAt);
/**
* NOT awaited, unlike abort. Abort must be deliverable before the job
* is exposed — a missed abort strands a run. A missed preempt only
@@ -943,7 +2194,8 @@ class GenerationJobManagerClass {
this.runtimeState.get(streamId) !== runtime ||
!confirmedJobData ||
confirmedJobData.createdAt !== runtime.createdAt ||
- confirmedJobData.status !== 'running'
+ confirmedJobData.status !== 'running' ||
+ confirmedJobData.providerAbortReady !== true
) {
throw new Error('Generation job was replaced during initialization');
}
@@ -951,6 +2203,9 @@ class GenerationJobManagerClass {
throw new Error(SHUTTING_DOWN_ERROR);
}
} catch (error) {
+ if (replacedRuntime != null && replacedRuntime !== runtime) {
+ this.releaseAbortSubscription(replacedRuntime, true);
+ }
// The durable job already exists, but the caller has not received its
// generation identity yet. Finalize that exact epoch here so a controller
// catch never needs to issue an unsafe unscoped terminal mutation.
@@ -964,7 +2219,10 @@ class GenerationJobManagerClass {
finalizeError,
);
});
- if (this.runtimeState.get(streamId) === runtime) {
+ if (
+ this.runtimeState.get(streamId) === runtime &&
+ runtime.replacementTransportHold !== true
+ ) {
this.releaseAbortSubscription(runtime);
runtime.abortController.abort();
this.runtimeState.delete(streamId);
@@ -1048,6 +2306,8 @@ class GenerationJobManagerClass {
userId: jobData.userId,
tenantId: jobData.tenantId,
conversationId: jobData.conversationId,
+ checkpointNamespace: jobData.checkpointNamespace,
+ generationProtocolVersion: jobData.generationProtocolVersion,
userMessage: jobData.userMessage,
responseMessageId: jobData.responseMessageId,
sender: jobData.sender,
@@ -1066,6 +2326,10 @@ class GenerationJobManagerClass {
// Surface the owning replica's seal capability so the steer route can
// honour it instead of probing its own (possibly older) SDK.
preemptCapable: jobData.preemptCapable,
+ steersClosed: jobData.steersClosed,
+ idempotencyClientRequestId: jobData.idempotencyClientRequestId,
+ terminalPersistencePending: jobData.terminalPersistencePending,
+ terminalPersistenceStartedAt: jobData.terminalPersistenceStartedAt,
// Surface the pending review so status/resume routes built on the
// facade can render the prompt for a `requires_action` job.
pendingAction: jobData.pendingAction,
@@ -1202,7 +2466,11 @@ class GenerationJobManagerClass {
* Get a job by streamId.
*/
async getJob(streamId: string): Promise<t.GenerationJob | undefined> {
- const jobData = await this.jobStore.getJob(streamId);
+ let jobData = await this.jobStore.getJob(streamId);
+ if (!jobData) {
+ return undefined;
+ }
+ jobData = await this.recoverStaleTerminalPersistence(jobData);
if (!jobData) {
return undefined;
}
@@ -1219,38 +2487,490 @@ class GenerationJobManagerClass {
return this.buildJobFacade(streamId, jobData, runtime);
}
- /**
- * Check if a job exists.
- */
- async hasJob(streamId: string): Promise<boolean> {
- return this.jobStore.hasJob(streamId);
+ /**
+ * Check if a job exists.
+ */
+ async hasJob(streamId: string): Promise<boolean> {
+ return this.jobStore.hasJob(streamId);
+ }
+
+ /**
+ * Atomically claim a start-generation request for `(userId, clientRequestId)`.
+ * The first caller wins (`claimed: true`) and should create the job; a retried
+ * request for the same submission loses and receives the original stream so it
+ * can attach to it instead of starting a second billed generation.
+ */
+ async claimGeneration(
+ userId: string,
+ clientRequestId: string,
+ streamId: string,
+ conversationId: string,
+ generationProtocolVersion?: 1 | 2,
+ ): Promise<IdempotencyClaimResult> {
+ const value: TokenIdempotencyClaim = {
+ streamId,
+ conversationId,
+ claimedAt: Date.now(),
+ claimToken: randomUUID(),
+ generationProtocolVersion: generationProtocolVersion === 2 ? 2 : 1,
+ };
+
+ // Old replicas know only this key. Claiming it first means a mixed fleet
+ // still has exactly one admission winner; the primary is a mirror/fence,
+ // never an independent chance to start a billed generation.
+ const legacyResult = await this.jobStore.claimIdempotencyKey(
+ this.legacyGenerationClaimKey(userId, clientRequestId),
+ value,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+
+ if (legacyResult.claimed) {
+ const primaryResult = await this.jobStore.claimIdempotencyKey(
+ this.generationClaimKey(userId, clientRequestId, streamId),
+ value,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ if (primaryResult.claimed) {
+ return { claimed: true, existing: value, source: 'primary' };
+ }
+
+ // This is a crash/expiry repair: an authoritative primary survived while
+ // its longer-lived legacy mirror was unexpectedly absent. Point the
+ // legacy key at that primary and remain a duplicate; never let the fresh
+ // legacy SET turn into a second owner.
+ const primary = normalizeTokenClaim(primaryResult.existing, 'primary');
+ assertClaimMatchesRequest(primary, streamId, conversationId);
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, value, primary);
+ return { claimed: false, existing: primary, source: 'primary' };
+ }
+
+ if (legacyResult.existing?.claimToken == null) {
+ return {
+ claimed: false,
+ existing: normalizeTokenlessLegacyClaim(legacyResult.existing),
+ source: 'legacy',
+ };
+ }
+
+ const legacy = normalizeTokenClaim(legacyResult.existing, 'legacy');
+ assertClaimMatchesRequest(legacy, streamId, conversationId);
+ const primaryResult = await this.jobStore.claimIdempotencyKey(
+ this.generationClaimKey(userId, clientRequestId, streamId),
+ legacy,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ if (primaryResult.claimed) {
+ // We merely repaired the mirror for the already-existing owner. This
+ // retry is still a duplicate and must not create a generation.
+ return { claimed: false, existing: legacy, source: 'primary' };
+ }
+
+ const primary = normalizeTokenClaim(primaryResult.existing, 'primary');
+ assertClaimMatchesRequest(primary, streamId, conversationId);
+ if (isRecoverableTakeoverSplit(legacy, primary)) {
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, legacy, primary);
+ return { claimed: false, existing: primary, source: 'primary' };
+ }
+ assertClaimMirrors(legacy, primary);
+ if (primary.startedAt != null && legacy.startedAt == null) {
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, legacy, primary);
+ }
+ return { claimed: false, existing: primary, source: 'primary' };
+ }
+
+ private generationClaimKey(userId: string, clientRequestId: string, streamId: string): string {
+ return `{${streamId}}:${userId}:${clientRequestId}`;
+ }
+
+ /** Exact pre-bridge physical key through RedisJobStore.KEYS.idempotency:
+ * `stream:idem:{userId:clientRequestId}`. The braces belong in the store key
+ * argument because current stores no longer add a second hash tag. */
+ private legacyGenerationClaimKey(userId: string, clientRequestId: string): string {
+ return `{${userId}:${clientRequestId}}`;
+ }
+
+ /** Finish one proven cross-slot bridge step. A failed CAS may mean another
+ * replica already completed the same value, so probe it before declaring an
+ * ambiguous outcome. The probe's SET-NX behavior also safely recreates an
+ * unexpectedly expired legacy mirror with the authoritative primary value. */
+ private async synchronizeLegacyGenerationClaim(
+ userId: string,
+ clientRequestId: string,
+ expectedLegacy: TokenIdempotencyClaim,
+ primary: TokenIdempotencyClaim,
+ ): Promise<void> {
+ const legacyKey = this.legacyGenerationClaimKey(userId, clientRequestId);
+ let synchronizationError: unknown;
+ let synchronized = false;
+ try {
+ synchronized = await this.jobStore.takeoverIdempotencyKey(
+ legacyKey,
+ expectedLegacy,
+ primary,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ } catch (error) {
+ // A single-key CAS can commit and then lose its reply. Probe below before
+ // treating the bridge as split or attempting any rollback.
+ synchronizationError = error;
+ }
+ if (synchronized) {
+ return;
+ }
+
+ const probe = await this.jobStore.claimIdempotencyKey(
+ legacyKey,
+ primary,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ const current = probe.claimed ? primary : normalizeTokenClaim(probe.existing, 'legacy');
+ if (claimsMirrorExactly(current, primary)) {
+ return;
+ }
+ if (synchronizationError != null) {
+ throw synchronizationError;
+ }
+ assertClaimMirrors(current, primary);
+ throw new Error('Legacy generation idempotency claim could not be synchronized');
+ }
+
+ /** Once an exact same-owner generation carrying this request id has been
+ * observed, its later completion/deletion cannot erase the historical fact
+ * that the request already started. Convert the freshly reacquired mirrors
+ * into started tombstones even when the live-job adoption CAS loses that
+ * race. The token CAS/probe never overwrites an unrelated claimant. */
+ private async tombstoneObservedGenerationClaim(
+ userId: string,
+ clientRequestId: string,
+ streamId: string,
+ claim: TokenIdempotencyClaim,
+ createdAt: number,
+ generationProtocolVersion: 1 | 2,
+ ): Promise<TokenIdempotencyClaim> {
+ const tombstone: TokenIdempotencyClaim = {
+ ...claim,
+ startedAt: createdAt,
+ generationProtocolVersion,
+ };
+ const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
+ let markError: unknown;
+ let marked = false;
+ try {
+ marked = await this.jobStore.takeoverIdempotencyKey(
+ primaryKey,
+ claim,
+ tombstone,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ } catch (error) {
+ // The CAS can commit and lose its reply. Resolve only the exact value we
+ // intended; a same-key unrelated token or coordinates remain fail-closed.
+ markError = error;
+ }
+
+ if (!marked) {
+ const probe = await this.jobStore.claimIdempotencyKey(
+ primaryKey,
+ tombstone,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ const current = probe.claimed
+ ? tombstone
+ : normalizeTokenClaim(probe.existing, 'observed generation tombstone');
+ if (!claimsMirrorExactly(current, tombstone)) {
+ if (markError != null) {
+ throw markError;
+ }
+ throw new Error('Observed generation idempotency claim changed before it was fenced');
+ }
+ }
+
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, claim, tombstone);
+ return tombstone;
+ }
+
+ async takeoverGeneration(
+ userId: string,
+ clientRequestId: string,
+ streamId: string,
+ expected: IdempotencyClaimValue,
+ ): Promise<IdempotencyClaimResult> {
+ if (expected.claimToken == null) {
+ return {
+ claimed: false,
+ existing: normalizeTokenlessLegacyClaim(expected),
+ source: 'legacy',
+ };
+ }
+
+ const expectedClaim = normalizeTokenClaim(expected, 'takeover');
+ assertClaimMatchesRequest(expectedClaim, streamId, expectedClaim.conversationId);
+ const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
+ const legacyKey = this.legacyGenerationClaimKey(userId, clientRequestId);
+
+ // Read-or-bind both mirrors before the CAS. `claimIdempotencyKey` is the
+ // store's cluster-safe single-key probe; if an expired mirror is absent it
+ // recreates the old owner rather than opening a second admission window.
+ const primaryProbe = await this.jobStore.claimIdempotencyKey(
+ primaryKey,
+ expectedClaim,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ const primary = primaryProbe.claimed
+ ? expectedClaim
+ : normalizeTokenClaim(primaryProbe.existing, 'primary');
+ const legacyProbe = await this.jobStore.claimIdempotencyKey(
+ legacyKey,
+ expectedClaim,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ const legacy = legacyProbe.claimed
+ ? expectedClaim
+ : normalizeTokenClaim(legacyProbe.existing, 'legacy');
+ assertClaimMatchesRequest(primary, streamId, expectedClaim.conversationId);
+ assertClaimMatchesRequest(legacy, streamId, expectedClaim.conversationId);
+ if (isRecoverableTakeoverSplit(legacy, primary)) {
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, legacy, primary);
+ return { claimed: false, existing: primary, source: 'primary' };
+ }
+ assertClaimMirrors(legacy, primary);
+
+ if (primary.startedAt != null) {
+ if (legacy.startedAt == null) {
+ await this.synchronizeLegacyGenerationClaim(userId, clientRequestId, legacy, primary);
+ }
+ return { claimed: false, existing: primary, source: 'primary' };
+ }
+
+ const value: TokenIdempotencyClaim = {
+ streamId,
+ conversationId: expectedClaim.conversationId,
+ claimedAt: Math.max(
+ Date.now(),
+ Math.min(Number.MAX_SAFE_INTEGER, expectedClaim.claimedAt + 1),
+ ),
+ claimToken: randomUUID(),
+ previousClaimToken: expectedClaim.claimToken,
+ generationProtocolVersion: expectedClaim.generationProtocolVersion,
+ };
+ let primaryTaken: boolean;
+ try {
+ primaryTaken = await this.jobStore.takeoverIdempotencyKey(
+ primaryKey,
+ expectedClaim,
+ value,
+ IDEMPOTENCY_TTL_SECONDS,
+ );
+ } catch (error) {
+ // A single-key CAS can commit and then lose its reply. Resolve only a
+ // split carrying this predecessor token; unrelated mismatches remain
+ // outcome-ambiguous and fail closed.
+ const [observedPrimaryResult, observedLegacyResult] = await Promise.all([
+ this.jobStore.claimIdempotencyKey(primaryKey, expectedClaim, IDEMPOTENCY_TTL_SECONDS),
+ this.jobStore.claimIdempotencyKey(legacyKey, expectedClaim, LEGACY_IDEMPOTENCY_TTL_SECONDS),
+ ]);
+ const observedPrimary = observedPrimaryResult.claimed
+ ? expectedClaim
+ : normalizeTokenClaim(observedPrimaryResult.existing, 'primary');
+ const observedLegacy = observedLegacyResult.claimed
+ ? expectedClaim
+ : normalizeTokenClaim(observedLegacyResult.existing, 'legacy');
+
+ if (isRecoverableTakeoverSplit(observedLegacy, observedPrimary)) {
+ await this.synchronizeLegacyGenerationClaim(
+ userId,
+ clientRequestId,
+ observedLegacy,
+ observedPrimary,
+ );
+ return claimsMirrorExactly(observedPrimary, value)
+ ? { claimed: true, existing: value, source: 'primary' }
+ : { claimed: false, existing: observedPrimary, source: 'primary' };
+ }
+ if (claimsMirrorExactly(observedLegacy, observedPrimary)) {
+ if (claimsMirrorExactly(observedPrimary, value)) {
+ return { claimed: true, existing: value, source: 'primary' };
+ }
+ if (isClaimTakeoverOf(expectedClaim, observedPrimary)) {
+ return { claimed: false, existing: observedPrimary, source: 'primary' };
+ }
+ }
+ throw error;
+ }
+ if (!primaryTaken) {
+ return { claimed: false, source: 'primary' };
+ }
+
+ try {
+ const legacyTaken = await this.jobStore.takeoverIdempotencyKey(
+ legacyKey,
+ expectedClaim,
+ value,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ if (!legacyTaken) {
+ // Do not strand a new primary token that is not mirrored to old
+ // replicas. Compare-release only the token installed above.
+ await this.jobStore.releaseIdempotencyKey(primaryKey, value);
+ return { claimed: false, source: 'primary' };
+ }
+ } catch (error) {
+ await this.jobStore.releaseIdempotencyKey(primaryKey, value).catch((releaseError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to roll back an unmirrored primary idempotency takeover:',
+ releaseError,
+ );
+ });
+ throw error;
+ }
+
+ return { claimed: true, existing: value, source: 'primary' };
+ }
+
+ /** A fixed replay lease can expire while a long generation is still active.
+ * If the identical client request reacquires that lease, atomically bind it
+ * back to the matching live job instead of letting createJob replace/rebill
+ * the generation. A different clientRequestId remains a genuinely new turn. */
+ async resumeClaimedGeneration(
+ userId: string,
+ clientRequestId: string,
+ streamId: string,
+ claim: IdempotencyClaimValue,
+ ): Promise<IdempotencyClaimValue | null> {
+ const normalizedClaim = normalizeTokenClaim(claim, 'reacquired');
+ assertClaimMatchesRequest(normalizedClaim, streamId, normalizedClaim.conversationId);
+ const tenantId = getTenantId();
+ const safeTenantId = tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;
+ const job = await this.jobStore.getJob(streamId);
+ if (
+ job == null ||
+ job.userId !== userId ||
+ (job.tenantId != null && job.tenantId !== safeTenantId) ||
+ (job.status !== 'running' && job.status !== 'requires_action')
+ ) {
+ return null;
+ }
+ const missingClientRequestId = job.idempotencyClientRequestId == null;
+ if (!missingClientRequestId && job.idempotencyClientRequestId !== clientRequestId) {
+ return null;
+ }
+ const conversationMatches = (job.conversationId ?? streamId) === normalizedClaim.conversationId;
+ const jobProtocol = normalizeClaimProtocol(job);
+ let adoptionError: unknown;
+ let adopted = false;
+ try {
+ adopted = await this.jobStore.adoptIdempotencyKeyForJob(
+ this.generationClaimKey(userId, clientRequestId, streamId),
+ normalizedClaim,
+ streamId,
+ userId,
+ clientRequestId,
+ safeTenantId,
+ job.createdAt,
+ IDEMPOTENCY_TTL_SECONDS,
+ missingClientRequestId,
+ );
+ } catch (error) {
+ adoptionError = error;
+ }
+ if (!adopted) {
+ try {
+ await this.tombstoneObservedGenerationClaim(
+ userId,
+ clientRequestId,
+ streamId,
+ normalizedClaim,
+ job.createdAt,
+ jobProtocol,
+ );
+ } catch (tombstoneError) {
+ if (adoptionError != null) {
+ throw adoptionError;
+ }
+ throw tombstoneError;
+ }
+ throw new Error(
+ 'Live generation settled while its idempotency claim was conservatively fenced',
+ );
+ }
+ const adoptedClaim: TokenIdempotencyClaim = {
+ ...normalizedClaim,
+ streamId,
+ conversationId: normalizedClaim.conversationId,
+ startedAt: job.createdAt,
+ generationProtocolVersion: jobProtocol,
+ };
+ await this.synchronizeLegacyGenerationClaim(
+ userId,
+ clientRequestId,
+ normalizedClaim,
+ adoptedClaim,
+ );
+
+ if (missingClientRequestId || !conversationMatches) {
+ // The exact same-owner/tenant/epoch job is active, but old metadata
+ // cannot prove that this retry should attach to its UI. Both fresh
+ // claims are now started tombstones: stay on the controller's 503 path
+ // while active, and settle after deletion without ever allowing a
+ // takeover/rebill.
+ throw new Error(
+ 'Active legacy generation was conservatively fenced without attaching the retry',
+ );
+ }
+ return adoptedClaim;
}
/**
- * Atomically claim a start-generation request for `(userId, clientRequestId)`.
- * The first caller wins (`claimed: true`) and should create the job; a retried
- * request for the same submission loses and receives the original stream so it
- * can attach to it instead of starting a second billed generation.
+ * Release a start-generation claim so the submission can be retried (e.g. the
+ * start failed before generation began).
*/
- async claimGeneration(
+ async releaseGeneration(
userId: string,
clientRequestId: string,
streamId: string,
- conversationId: string,
- ): Promise<IdempotencyClaimResult> {
- return this.jobStore.claimIdempotencyKey(
- `${userId}:${clientRequestId}`,
- { streamId, conversationId, claimedAt: Date.now() },
+ expected?: IdempotencyClaimValue,
+ ): Promise<void> {
+ // Releasing without a token would let a stale/old request delete another
+ // replica's owner. Tokenless legacy claims are duplicate-only and are
+ // never owned by this manager.
+ if (expected?.claimToken == null) {
+ return;
+ }
+ const expectedClaim = normalizeTokenClaim(expected, 'release');
+ assertClaimMatchesRequest(expectedClaim, streamId, expectedClaim.conversationId);
+
+ const primaryKey = this.generationClaimKey(userId, clientRequestId, streamId);
+ const primaryProbe = await this.jobStore.claimIdempotencyKey(
+ primaryKey,
+ expectedClaim,
IDEMPOTENCY_TTL_SECONDS,
);
- }
+ const primary = primaryProbe.claimed
+ ? expectedClaim
+ : normalizeTokenClaim(primaryProbe.existing, 'primary');
+ assertClaimMirrors(expectedClaim, primary);
+ if (primary.startedAt != null) {
+ // createJob may have committed the primary before a later initialization
+ // or legacy-mark failure. The caller still holds its pre-create value;
+ // deleting the now-started tombstone would make a retry bill again.
+ return;
+ }
+ await this.jobStore.releaseIdempotencyKey(primaryKey, expectedClaim);
- /**
- * Release a start-generation claim so the submission can be retried (e.g. the
- * start failed before generation began).
- */
- async releaseGeneration(userId: string, clientRequestId: string): Promise<void> {
- await this.jobStore.releaseIdempotencyKey(`${userId}:${clientRequestId}`);
+ const legacyKey = this.legacyGenerationClaimKey(userId, clientRequestId);
+ const legacyProbe = await this.jobStore.claimIdempotencyKey(
+ legacyKey,
+ expectedClaim,
+ LEGACY_IDEMPOTENCY_TTL_SECONDS,
+ );
+ const legacy = legacyProbe.claimed
+ ? expectedClaim
+ : normalizeTokenClaim(legacyProbe.existing, 'legacy');
+ assertClaimMirrors(legacy, expectedClaim);
+ if (legacy.startedAt != null) {
+ return;
+ }
+ await this.jobStore.releaseIdempotencyKey(legacyKey, expectedClaim);
}
/**
@@ -1262,148 +2982,427 @@ class GenerationJobManagerClass {
}
/**
- * Mark job as complete.
- * If cleanupOnComplete is true (default), immediately cleans up job resources.
- * Exception: Jobs with errors are NOT immediately deleted to allow late-connecting
- * clients to receive the error (race condition where error occurs before client connects).
- * Note: eventTransport is NOT cleaned up here to allow the final event to be
- * fully transmitted. It will be cleaned up when subscribers disconnect or
- * by the periodic cleanup job.
+ * Atomically win terminal ownership for a running generation and park the
+ * exact steer leftovers in the same transaction. A controller must not emit
+ * a terminal client event before this succeeds: abort, pause, and completion
+ * can all race on the same generation epoch.
*/
- async completeJob(streamId: string, error?: string, expectedCreatedAt?: number): Promise<void> {
+ async claimTerminalJob(
+ streamId: string,
+ status: TerminalJobClaim['status'],
+ error?: string,
+ expectedCreatedAt?: number,
+ options: { persistencePending?: boolean; failedPauseActionId?: string } = {},
+ ): Promise<TerminalJobClaim | null> {
+ if (
+ options.failedPauseActionId != null &&
+ (status !== 'error' || options.persistencePending === true)
+ ) {
+ throw new Error('A failed pause-persistence claim must be a non-pending error terminal');
+ }
const observedRuntime = this.runtimeState.get(streamId);
const targetCreatedAt = expectedCreatedAt ?? observedRuntime?.createdAt;
- const jobData = await this.jobStore.getJob(streamId);
+ let jobData = await this.jobStore.getJob(streamId);
if (!jobData || (targetCreatedAt != null && jobData.createdAt !== targetCreatedAt)) {
if (targetCreatedAt != null) {
this.reconcileInactiveGeneration(streamId, targetCreatedAt, jobData, observedRuntime);
}
logger.debug(
- `[GenerationJobManager] Skipping stale completion for replaced job: ${streamId}`,
+ `[GenerationJobManager] Skipping stale terminal claim for replaced job: ${streamId}`,
);
- return;
+ return null;
}
- if (jobData.status !== 'running') {
+ const failedPauseActionId = options.failedPauseActionId;
+ const failedPauseBarrierId =
+ failedPauseActionId != null ? pausePersistenceActionId(failedPauseActionId) : undefined;
+ if (failedPauseBarrierId != null) {
+ /** The required response write failed while this exact controller still
+ * owns the pause barrier. Terminalize from the barrier state itself;
+ * releasing it first would let a waiting approval resume win the next
+ * `requires_action -> running` CAS before the error transition. */
+ if (
+ jobData.status !== 'requires_action' ||
+ jobData.terminalPersistencePending !== true ||
+ jobData.pendingAction?.actionId !== failedPauseActionId ||
+ jobData.pendingActionId !== failedPauseBarrierId
+ ) {
+ return null;
+ }
+ } else if (
+ jobData.status === 'requires_action' &&
+ jobData.terminalPersistencePending === true
+ ) {
+ jobData = await this._approvals.waitForPausePersistence(streamId, jobData.createdAt);
+ if (!jobData || (targetCreatedAt != null && jobData.createdAt !== targetCreatedAt)) {
+ return null;
+ }
+ }
+ const sourceStatus = jobData.status;
+ const canClaim =
+ sourceStatus === 'running' || (status === 'error' && sourceStatus === 'requires_action');
+ if (!canClaim) {
this.reconcileInactiveGeneration(streamId, jobData.createdAt, jobData, observedRuntime);
logger.debug(
- `[GenerationJobManager] Skipping completion for non-running job ${streamId}: ${jobData.status}`,
+ `[GenerationJobManager] Skipping ${status} claim for job ${streamId}: ${jobData.status}`,
);
- return;
+ return null;
}
const createdAt = jobData.createdAt;
const runtime = observedRuntime?.createdAt === createdAt ? observedRuntime : undefined;
+ const terminalError = status === 'error' ? (error ?? 'Generation failed') : undefined;
+ const completedAt = Date.now();
+ const drainedSteers = await this.jobStore.transitionStatusAndDrainSteers(streamId, {
+ from: sourceStatus,
+ to: status,
+ expectCreatedAt: createdAt,
+ ...(sourceStatus === 'requires_action' && {
+ ...(failedPauseBarrierId != null
+ ? { expectActionId: failedPauseBarrierId }
+ : jobData.pendingActionId != null && { expectActionId: jobData.pendingActionId }),
+ }),
+ patch: {
+ completedAt,
+ ...(terminalError != null && { error: terminalError }),
+ ...(options.persistencePending === true && {
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: completedAt,
+ }),
+ },
+ ...(sourceStatus === 'requires_action' && {
+ clear: [
+ 'pendingAction',
+ 'pendingActionId',
+ ...(failedPauseBarrierId != null
+ ? ['terminalPersistencePending', 'terminalPersistenceStartedAt']
+ : []),
+ ] as (keyof SerializableJobData)[],
+ }),
+ });
+ if (drainedSteers == null) {
+ await this.reconcileLostTerminalTransition(streamId, createdAt, runtime);
+ return null;
+ }
+
+ const claim: TerminalJobClaim = Object.freeze({
+ streamId,
+ createdAt,
+ ...(jobData.conversationId != null && { conversationId: jobData.conversationId }),
+ status,
+ ...(terminalError != null && { error: terminalError }),
+ ...(options.persistencePending === true && { persistencePending: true as const }),
+ drainedSteers: Object.freeze([...drainedSteers]),
+ });
+ this.terminalClaimRuntimes.set(claim, runtime ?? null);
+
+ // An aborted claim must stop its exact generation immediately. Cleanup is
+ // still deferred to finishTerminalJob so terminal publication can happen
+ // first and is guaranteed a finally-paired resource release.
+ if (status === 'aborted') {
+ try {
+ this.eventTransport.emitAbort?.(streamId, createdAt);
+ } catch (abortError) {
+ logger.error(
+ `[GenerationJobManager] Failed to publish terminal abort for ${streamId}:`,
+ abortError,
+ );
+ }
+ if (runtime && this.runtimeState.get(streamId) === runtime) {
+ this.releaseAbortSubscription(runtime);
+ runtime.abortController.abort();
+ }
+ }
+
+ return claim;
+ }
+
+ /**
+ * Durably settle and publish a persistence-owning terminal claim. Passing a
+ * normal event means the controller's required writes succeeded; `null`
+ * publishes conservative reconciliation instead. The store CAS also races a
+ * stale-owner recovery, and whichever side wins supplies the only payload
+ * this method will publish.
+ */
+ async publishTerminalClaim(
+ claim: TerminalJobClaim,
+ finalEvent: t.ServerSentEvent | null,
+ ): Promise<{ finalEvent: t.ServerSentEvent; persistenceFailed: boolean }> {
+ if (!this.terminalClaimRuntimes.has(claim) || claim.persistencePending !== true) {
+ throw new Error('Terminal persistence claim was not issued by this manager');
+ }
+
+ const { streamId, createdAt } = claim;
+ const claimedRuntime = this.terminalClaimRuntimes.get(claim) ?? undefined;
+ const runtime =
+ claimedRuntime &&
+ claimedRuntime.createdAt === createdAt &&
+ this.runtimeState.get(streamId) === claimedRuntime
+ ? claimedRuntime
+ : undefined;
+ const reconcileEvent = buildTerminalPersistenceReconcile({
+ createdAt,
+ conversationId: claim.conversationId,
+ status: claim.status,
+ });
+ const desiredEvent = finalEvent ?? reconcileEvent;
+ let publicationEvent: t.ServerSentEvent | null = null;
+ let durable = false;
- // Backstop for direct terminal callers (init failures, unhandled errors)
- // that never ran the controllers' close-and-park. Every mutation is pinned
- // to this completion's immutable generation identity, so a predecessor can
- // never drain or terminalize a replacement that reuses the stream ID.
try {
- const leftovers = (await this.jobStore.closeAndDrainSteers(streamId, createdAt)).map(
- toPendingSteer,
- );
- await this._steering.park(
+ const finalized = await this.jobStore.finalizeTerminalPersistence(
streamId,
- leftovers,
- {
- userId: jobData.userId,
- tenantId: jobData.tenantId,
- },
createdAt,
+ JSON.stringify(desiredEvent),
+ );
+ if (finalized) {
+ publicationEvent = desiredEvent;
+ durable = true;
+ } else {
+ const settledJob = await this.jobStore.getJob(streamId);
+ if (
+ settledJob?.createdAt === createdAt &&
+ settledJob.terminalPersistencePending !== true &&
+ settledJob.finalEvent
+ ) {
+ publicationEvent = JSON.parse(settledJob.finalEvent) as t.ServerSentEvent;
+ durable = true;
+ }
+ }
+ } catch (settleError) {
+ logger.error(
+ `[GenerationJobManager] Failed to finalize terminal persistence for ${streamId}:`,
+ settleError,
);
- } catch (err) {
- logger.warn(`[GenerationJobManager] Failed to park leftover steers for ${streamId}:`, err);
}
- // Error jobs stay durable long enough for late subscribers to receive the
- // stored error. The status CAS also parks/cleans same-slot Redis state.
- if (error) {
- const finalized = await this.jobStore.transitionStatus(streamId, {
- from: 'running',
- to: 'error',
- expectCreatedAt: createdAt,
- patch: { completedAt: Date.now(), error },
- });
- if (!finalized) {
- await this.reconcileLostTerminalTransition(streamId, createdAt, runtime);
- return;
+ /** If durable finalization itself failed, leave the marker for bounded
+ * stale-owner recovery but still close an already-attached client with the
+ * conservative frame. Do not cache that best-effort frame on the runtime:
+ * late subscribers must honor the durable pending marker. */
+ if (!publicationEvent) {
+ publicationEvent = reconcileEvent;
+ } else if (runtime) {
+ runtime.finalEvent = publicationEvent;
+ }
+
+ try {
+ if (runtime?.createdEventPublication) {
+ await runtime.createdEventPublication;
+ }
+ await this.eventTransport.emitDone(streamId, publicationEvent, createdAt);
+ } catch (publicationError) {
+ logger.error(
+ `[GenerationJobManager] Failed to publish terminal persistence result ${streamId}:`,
+ publicationError,
+ );
+ if (durable) {
+ this.terminalPublicationFailures.add(claim);
}
+ /** Publication can yield long enough for a same-stream replacement to
+ * install its own runtime/subscriber. Close only while the exact runtime
+ * captured by this terminal claim is still current; the check and the
+ * synchronous close call deliberately have no await between them. */
if (runtime && this.runtimeState.get(streamId) === runtime) {
- runtime.errorEvent = error;
+ try {
+ this.eventTransport.closeLocalSubscribers?.(
+ streamId,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
+ );
+ } catch (closeError) {
+ logger.error(
+ `[GenerationJobManager] Failed to close local subscribers after terminal publication failure ${streamId}:`,
+ closeError,
+ );
+ }
}
- try {
- await this.eventTransport.emitError(streamId, error, createdAt);
- } catch (publishError) {
- logger.error(
- `[GenerationJobManager] Failed to publish terminal error for ${streamId}:`,
- publishError,
- );
- if (runtime && this.runtimeState.get(streamId) === runtime) {
- for (const notify of [...runtime.localErrorHandlers]) {
- try {
- notify(error);
- } catch (notifyError) {
- logger.error(
- `[GenerationJobManager] Failed to notify terminal error for ${streamId}:`,
- notifyError,
- );
+ throw publicationError;
+ }
+
+ const persistenceFailed =
+ finalEvent == null ||
+ !durable ||
+ publicationEvent !== finalEvent ||
+ ('reconcile' in publicationEvent && publicationEvent.reconcile === true);
+ if (!persistenceFailed && runtime?.startupTelemetry) {
+ this.recordStartupEvent(runtime, publicationEvent);
+ }
+ return { finalEvent: publicationEvent, persistenceFailed };
+ }
+
+ /**
+ * Release resources owned by a successful terminal claim. Reusing the same
+ * claim is idempotent, and every local mutation is pinned to the runtime
+ * object and generation epoch captured when the CAS won.
+ */
+ finishTerminalJob(claim: TerminalJobClaim): Promise<void> {
+ const inFlight = this.terminalFinishPromises.get(claim);
+ if (inFlight) {
+ return inFlight;
+ }
+ if (!this.terminalClaimRuntimes.has(claim)) {
+ return Promise.reject(new Error('Terminal claim was not issued by this manager'));
+ }
+
+ const finishing = this.finishTerminalJobInternal(claim);
+ this.terminalFinishPromises.set(claim, finishing);
+ return finishing;
+ }
+
+ private async finishTerminalJobInternal(claim: TerminalJobClaim): Promise<void> {
+ const { streamId, createdAt, status, error } = claim;
+ const retainForTerminalReplay = this.terminalPublicationFailures.has(claim);
+ const claimedRuntime = this.terminalClaimRuntimes.get(claim) ?? undefined;
+ const runtime =
+ claimedRuntime &&
+ claimedRuntime.createdAt === createdAt &&
+ this.runtimeState.get(streamId) === claimedRuntime
+ ? claimedRuntime
+ : undefined;
+ let cleanupError: unknown;
+
+ // Error jobs stay durable long enough for late subscribers to receive the
+ // stored error. A publication failure must never bypass the finally cleanup.
+ try {
+ if (status === 'error') {
+ const terminalError = error ?? 'Generation failed';
+ if (runtime) {
+ runtime.errorEvent = terminalError;
+ }
+ try {
+ if (runtime?.createdEventPublication) {
+ await runtime.createdEventPublication;
+ }
+ await this.eventTransport.emitError(streamId, terminalError, createdAt);
+ } catch (publishError) {
+ logger.error(
+ `[GenerationJobManager] Failed to publish terminal error for ${streamId}:`,
+ publishError,
+ );
+ if (runtime && this.runtimeState.get(streamId) === runtime) {
+ for (const notify of [...runtime.localErrorHandlers]) {
+ try {
+ notify(terminalError);
+ } catch (notifyError) {
+ logger.error(
+ `[GenerationJobManager] Failed to notify terminal error for ${streamId}:`,
+ notifyError,
+ );
+ }
}
}
}
+ } else if (this._cleanupOnComplete) {
+ // A failed finalization write deliberately leaves any terminal pending
+ // marker for bounded stale-owner recovery. Every normally finalized
+ // persistence-owning claim clears it before reaching this cleanup. A
+ // durable final whose DONE publish failed is retained for reconnect
+ // replay even though its pending marker is already clear.
+ const terminalJob = await this.jobStore.getJob(streamId);
+ if (
+ !retainForTerminalReplay &&
+ (terminalJob?.createdAt !== createdAt || terminalJob.terminalPersistencePending !== true)
+ ) {
+ // A same-stream replacement created after the claim makes this a safe
+ // no-op rather than deleting the replacement generation.
+ await this.jobStore.deleteJob(streamId, createdAt);
+ }
}
+ } catch (err) {
+ cleanupError = err;
+ } finally {
if (runtime && this.runtimeState.get(streamId) === runtime) {
this.releaseAbortSubscription(runtime);
runtime.abortController.abort();
- runtime.startupTelemetry?.end('error', new Error(error));
+ if (status === 'error') {
+ runtime.startupTelemetry?.end('error', new Error(error ?? 'Generation failed'));
+ } else if (status === 'aborted') {
+ runtime.startupTelemetry?.end('aborted');
+ } else {
+ runtime.startupTelemetry?.end('completed_without_delta');
+ }
runtime.startupTelemetry = undefined;
this.jobStore.clearContentState(streamId, createdAt);
this.runStepBuffers?.delete(streamId);
this.replayEventWriteQueues.delete(streamId);
this.tokenUsageWriteQueues.delete(streamId);
+ if (status !== 'error' && this._cleanupOnComplete) {
+ this.runtimeState.delete(streamId);
+ }
}
+
this.releaseJobOwnership(streamId, createdAt);
- recordGenerationJob(this.storeLabel, 'error');
- logger.debug(
- `[GenerationJobManager] Job completed with error (keeping for late subscribers): ${streamId}`,
- );
- return;
+ this.terminalPublicationFailures.delete(claim);
+ let metricStatus: 'completed' | 'error' | 'aborted' = 'aborted';
+ if (status === 'complete') {
+ metricStatus = 'completed';
+ } else if (status === 'error') {
+ metricStatus = 'error';
+ }
+ recordGenerationJob(this.storeLabel, metricStatus);
}
- // Successful completion deletes immediately by default. Both branches are
- // guarded at the store boundary, closing the read→delete replacement race.
- const completed = await this.jobStore.transitionStatus(streamId, {
- from: 'running',
- to: 'complete',
- expectCreatedAt: createdAt,
- patch: { completedAt: Date.now() },
- });
- if (!completed) {
- await this.reconcileLostTerminalTransition(streamId, createdAt, runtime);
- return;
- }
- if (this._cleanupOnComplete) {
- // A same-stream replacement created after the completion CAS makes this
- // a safe no-op rather than deleting the replacement generation.
- await this.jobStore.deleteJob(streamId, createdAt);
+ if (cleanupError != null) {
+ throw cleanupError;
}
+ logger.debug(
+ status === 'error'
+ ? `[GenerationJobManager] Job completed with error (keeping for late subscribers): ${streamId}`
+ : `[GenerationJobManager] Job ${status}: ${streamId}`,
+ );
+ }
- if (runtime && this.runtimeState.get(streamId) === runtime) {
- this.releaseAbortSubscription(runtime);
- runtime.abortController.abort();
- runtime.startupTelemetry?.end('completed_without_delta');
- runtime.startupTelemetry = undefined;
- this.jobStore.clearContentState(streamId, createdAt);
- this.runStepBuffers?.delete(streamId);
- this.replayEventWriteQueues.delete(streamId);
- this.tokenUsageWriteQueues.delete(streamId);
- if (this._cleanupOnComplete) {
- this.runtimeState.delete(streamId);
- }
+ /**
+ * Mark job as complete.
+ * Returns true only when this caller won the terminal CAS and finished it;
+ * controllers can safely gate generation-owned cleanup on that result.
+ * If cleanupOnComplete is true (default), immediately cleans up job resources.
+ * Exception: Jobs with errors are NOT immediately deleted to allow late-connecting
+ * clients to receive the error (race condition where error occurs before client connects).
+ * Note: eventTransport is NOT cleaned up here to allow the final event to be
+ * fully transmitted. It will be cleaned up when subscribers disconnect or
+ * by the periodic cleanup job.
+ */
+ async completeJob(
+ streamId: string,
+ error?: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ const claim = await this.claimTerminalJob(
+ streamId,
+ error ? 'error' : 'complete',
+ error,
+ expectedCreatedAt,
+ );
+ if (!claim) {
+ return false;
}
+ await this.finishTerminalJob(claim);
+ return true;
+ }
- this.releaseJobOwnership(streamId, createdAt);
- recordGenerationJob(this.storeLabel, 'completed');
- logger.debug(`[GenerationJobManager] Job completed: ${streamId}`);
+ /**
+ * Atomically fail the exact paused action whose required unfinished-response
+ * write did not succeed. Unlike `finishPausePersistence` followed by
+ * `completeJob`, this never exposes the ordinary action id between those two
+ * decisions, so a waiting/concurrent resume cannot drive an unpersisted turn.
+ */
+ async failPausePersistence(
+ streamId: string,
+ actionId: string,
+ error: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ if (actionId.length === 0) {
+ return false;
+ }
+ const claim = await this.claimTerminalJob(streamId, 'error', error, expectedCreatedAt, {
+ failedPauseActionId: actionId,
+ });
+ if (!claim) {
+ return false;
+ }
+ await this.finishTerminalJob(claim);
+ return true;
}
/**
@@ -1424,10 +3423,41 @@ class GenerationJobManagerClass {
streamId: string,
options?: {
transformAbortContent?: (content: TMessageContentParts[]) => TMessageContentParts[];
+ /** Required durable work (for example saving the partial response and
+ * pruning the paused checkpoint) that must finish before the normal
+ * FINAL lets the client submit against that history. Throwing emits a
+ * conservative reconciliation frame instead of the normal payload. */
+ beforePublish?: (result: AbortResult) => void | Promise<void>;
+ /** Epoch observed by the authenticated route. Without this fence, a
+ * same-stream replacement between authorization and the manager read
+ * could be stopped by a stale request intended for its predecessor. */
+ expectedCreatedAt?: number;
},
): Promise<AbortResult> {
const observedRuntime = this.runtimeState.get(streamId);
- const jobData = await this.jobStore.getJob(streamId);
+ let jobData = await this.jobStore.getJob(streamId);
+
+ if (options?.expectedCreatedAt != null && jobData?.createdAt !== options.expectedCreatedAt) {
+ this.reconcileInactiveGeneration(
+ streamId,
+ options.expectedCreatedAt,
+ jobData,
+ observedRuntime?.createdAt === options.expectedCreatedAt ? observedRuntime : undefined,
+ );
+ logger.debug(
+ `[GenerationJobManager] Refusing stale abort for replaced job: ${streamId}@${options.expectedCreatedAt}`,
+ );
+ recordGenerationJob(this.storeLabel, 'abort_failed');
+ return {
+ text: '',
+ content: [],
+ jobData,
+ success: false,
+ failureReason: 'generation_replaced',
+ finalEvent: null,
+ collectedUsage: [],
+ };
+ }
if (!jobData) {
if (observedRuntime) {
@@ -1450,6 +3480,25 @@ class GenerationJobManagerClass {
};
}
+ if (jobData.status === 'requires_action' && jobData.terminalPersistencePending === true) {
+ const unlockedJob = await this._approvals.waitForPausePersistence(
+ streamId,
+ jobData.createdAt,
+ );
+ if (!unlockedJob || unlockedJob.createdAt !== jobData.createdAt) {
+ this.reconcileInactiveGeneration(streamId, jobData.createdAt, unlockedJob, observedRuntime);
+ return {
+ text: '',
+ content: [],
+ jobData: unlockedJob,
+ success: false,
+ finalEvent: null,
+ collectedUsage: [],
+ };
+ }
+ jobData = unlockedJob;
+ }
+
const abortableStatus = jobData.status;
if (abortableStatus !== 'running' && abortableStatus !== 'requires_action') {
this.reconcileInactiveGeneration(streamId, jobData.createdAt, jobData, observedRuntime);
@@ -1468,170 +3517,244 @@ class GenerationJobManagerClass {
}
const runtime = observedRuntime?.createdAt === jobData.createdAt ? observedRuntime : undefined;
-
- /** Steers that never reached an injection boundary — reported on the abort
- * final event (and the abort route's JSON) so the client can restore them
- * as queued chips instead of silently dropping the user's words. The
- * close-and-drain rejects any steer POST racing this finalization, and
- * the createdAt guard keeps it off a replacement job's queue. Runs
- * BEFORE the content snapshot below: a drain hook that already popped a
- * steer and applied its part gets captured by the snapshot, so the text
- * surfaces either here (pendingSteers) or there (inline part) — an
- * encode still in flight across the abort remains inherently racy
- * cross-instance, but the window no longer includes completed applies. */
- const pendingSteers = (
- await this.jobStore.closeAndDrainSteers(streamId, jobData.createdAt)
- ).map(toPendingSteer);
- // No-subscriber recovery: the abort response/final are transient, so park
- // the leftovers for /chat/status claim-on-read within the recovery TTL.
- await this.steering.park(
- streamId,
- pendingSteers,
- {
- userId: jobData.userId,
- tenantId: jobData.tenantId,
- },
- jobData.createdAt,
- );
-
- /** Content before clearing state */
+ /** Snapshot before claiming terminal state. This is non-destructive: if a
+ * same-epoch approval resume wins the later CAS, its content and steer
+ * queue remain fully owned by that resumed run. */
const result = await this.jobStore.getContentParts(streamId, jobData.createdAt);
- const content = result?.content ?? [];
+ let content = result?.content ?? [];
let abortContent = filterPersistableAbortContent(content);
if (options?.transformAbortContent) {
abortContent = options.transformAbortContent(
abortContent as TMessageContentParts[],
) as typeof abortContent;
}
- const shouldPersistAbortContent = abortContent.length > 0;
+ let shouldPersistAbortContent = abortContent.length > 0;
/** Collected usage for all models */
const collectedUsage = this.jobStore.getCollectedUsage(streamId, jobData.createdAt);
/** Text from content parts for fallback token counting; the persisted
* abort record keeps steered words (they reached the model context). */
- const text = shouldPersistAbortContent
+ let text = shouldPersistAbortContent
? parseTextParts(abortContent as TMessageContentParts[], false, { includeSteer: true })
: '';
- /** Detect "early abort" - aborted before any generation happened (e.g., during tool loading)
- In this case, no messages were saved to DB, so frontend shouldn't navigate to conversation */
- const isEarlyAbort = !shouldPersistAbortContent && jobData.createdEventEmitted !== true;
+ /** Claim terminal ownership and drain steers in one store transaction. A
+ * destructive pre-CAS drain would corrupt a same-epoch run when approval
+ * resolution wins `requires_action -> running`. The returned batch is the
+ * exact queued/claimed set parked by the winning abort, so the final event
+ * can restore it without a second read or a recovery race. */
+ const terminalPersistenceStartedAt = Date.now();
+ let transitionFrom = abortableStatus;
+ let drainedSteers: SteerQueueItem[] | null = null;
+ let currentAfterConflict: SerializableJobData | null = jobData;
+ // An approval decision can move the same epoch between running and
+ // requires_action after our read. Retry that legal same-generation state
+ // change instead of telling the caller it stopped a run that is still live.
+ for (let attempt = 0; attempt < 3 && drainedSteers == null; attempt++) {
+ const expectedActionId =
+ transitionFrom === 'requires_action' ? currentAfterConflict?.pendingActionId : undefined;
+ drainedSteers = await this.jobStore.transitionStatusAndDrainSteers(streamId, {
+ from: transitionFrom,
+ to: 'aborted',
+ expectCreatedAt: jobData.createdAt,
+ ...(expectedActionId != null && { expectActionId: expectedActionId }),
+ patch: {
+ completedAt: terminalPersistenceStartedAt,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt,
+ },
+ });
+ if (drainedSteers != null) {
+ break;
+ }
+ currentAfterConflict = await this.jobStore.getJob(streamId);
+ if (
+ currentAfterConflict?.createdAt === jobData.createdAt &&
+ currentAfterConflict.status === 'requires_action' &&
+ currentAfterConflict.terminalPersistencePending === true
+ ) {
+ currentAfterConflict = await this._approvals.waitForPausePersistence(
+ streamId,
+ jobData.createdAt,
+ );
+ }
+ if (
+ currentAfterConflict?.createdAt !== jobData.createdAt ||
+ (currentAfterConflict.status !== 'running' &&
+ currentAfterConflict.status !== 'requires_action')
+ ) {
+ break;
+ }
+ if (
+ currentAfterConflict.status === transitionFrom &&
+ (transitionFrom !== 'requires_action' ||
+ currentAfterConflict.pendingActionId === expectedActionId)
+ ) {
+ break;
+ }
+ transitionFrom = currentAfterConflict.status;
+ }
+ if (drainedSteers == null) {
+ const currentJob = await this.jobStore.getJob(streamId);
+ this.reconcileInactiveGeneration(streamId, jobData.createdAt, currentJob, runtime);
+ const jobStillActive =
+ currentJob?.createdAt === jobData.createdAt &&
+ (currentJob.status === 'running' || currentJob.status === 'requires_action');
+ return {
+ success: false,
+ ...(jobStillActive
+ ? { failureReason: 'job_still_active' as const }
+ : options?.expectedCreatedAt != null &&
+ currentJob != null &&
+ currentJob?.createdAt !== options.expectedCreatedAt && {
+ failureReason: 'generation_replaced' as const,
+ }),
+ jobData,
+ content: abortContent,
+ finalEvent: null,
+ text,
+ collectedUsage,
+ };
+ }
+ const terminalClaim: TerminalJobClaim = Object.freeze({
+ streamId,
+ createdAt: jobData.createdAt,
+ ...(jobData.conversationId != null && { conversationId: jobData.conversationId }),
+ status: 'aborted',
+ persistencePending: true,
+ drainedSteers: Object.freeze([...drainedSteers]),
+ });
+ this.terminalClaimRuntimes.set(terminalClaim, runtime ?? null);
+
+ try {
+ const pendingSteers = drainedSteers.map(toPendingSteer);
- /** Final event for abort */
- const userMessageId = jobData.userMessage?.messageId;
+ // Signal only the generation whose terminal transaction won above. The
+ // transport tag prevents a delayed predecessor abort from reaching a
+ // same-stream replacement on another replica.
+ if (this.eventTransport.emitAbort) {
+ try {
+ this.eventTransport.emitAbort(streamId, jobData.createdAt);
+ } catch (abortError) {
+ logger.error(
+ `[GenerationJobManager] Failed to publish terminal abort for ${streamId}:`,
+ abortError,
+ );
+ }
+ }
+ if (runtime) {
+ this.releaseAbortSubscription(runtime);
+ }
+ runtime?.abortController.abort();
- const abortFinalEvent: t.ServerSentEvent = {
- final: true,
- // Don't include conversation for early aborts - it doesn't exist in DB
- conversation: isEarlyAbort ? null : { conversationId: jobData.conversationId },
- title: 'New Chat',
- requestMessage: jobData.userMessage
- ? {
- messageId: userMessageId,
- parentMessageId: jobData.userMessage.parentMessageId,
- conversationId: jobData.conversationId,
- text: jobData.userMessage.text ?? '',
- quotes: jobData.userMessage.quotes,
- isCreatedByUser: true,
+ // A chunk append racing the initial non-destructive snapshot either
+ // commits before the terminal CAS or loses its own running-status guard.
+ // Re-read after our CAS so the abort payload includes every committed
+ // part while never borrowing content from a replacement epoch.
+ try {
+ const committed = await this.jobStore.getContentParts(streamId, jobData.createdAt);
+ if (committed) {
+ content = committed.content;
+ abortContent = filterPersistableAbortContent(content);
+ if (options?.transformAbortContent) {
+ abortContent = options.transformAbortContent(
+ abortContent as TMessageContentParts[],
+ ) as typeof abortContent;
}
- : null,
- responseMessage: isEarlyAbort
- ? null
- : {
- messageId: jobData.responseMessageId ?? `${userMessageId ?? 'aborted'}_`,
- parentMessageId: userMessageId,
- conversationId: jobData.conversationId,
- content: abortContent,
- sender: jobData.sender ?? 'AI',
- endpoint: jobData.endpoint,
- iconURL: jobData.iconURL,
- model: jobData.model,
- unfinished: true,
- error: false,
- isCreatedByUser: false,
- },
- aborted: true,
- // Flag for early abort - no messages saved, frontend should go to new chat
- earlyAbort: isEarlyAbort,
- ...(pendingSteers.length > 0 && { pendingSteers }),
- } satisfies t.FinalEvent as t.ServerSentEvent;
-
- // Claim the terminal state before publishing anything client-visible. A
- // natural completion or approval resolution racing this abort can win the
- // CAS, in which case this stale abort must not signal or close that winner.
- const finalized = await this.jobStore.transitionStatus(streamId, {
- from: abortableStatus,
- to: 'aborted',
- expectCreatedAt: jobData.createdAt,
- patch: { completedAt: Date.now() },
- });
- if (!finalized) {
- await this.reconcileLostTerminalTransition(streamId, jobData.createdAt, runtime);
- return {
- success: false,
+ shouldPersistAbortContent = abortContent.length > 0;
+ text = shouldPersistAbortContent
+ ? parseTextParts(abortContent as TMessageContentParts[], false, {
+ includeSteer: true,
+ })
+ : '';
+ }
+ } catch (contentError) {
+ logger.warn(
+ `[GenerationJobManager] Failed to refresh committed abort content for ${streamId}:`,
+ contentError,
+ );
+ }
+
+ /** Detect "early abort" - aborted before any generation happened (e.g., during tool loading)
+ In this case, no messages were saved to DB, so frontend shouldn't navigate to conversation */
+ const isEarlyAbort = !shouldPersistAbortContent && jobData.createdEventEmitted !== true;
+
+ /** Final event for abort */
+ const userMessageId = jobData.userMessage?.messageId;
+
+ const abortFinalEvent: t.ServerSentEvent = {
+ final: true,
+ // Don't include conversation for early aborts - it doesn't exist in DB
+ conversation: isEarlyAbort ? null : { conversationId: jobData.conversationId },
+ title: 'New Chat',
+ requestMessage: jobData.userMessage
+ ? {
+ messageId: userMessageId,
+ parentMessageId: jobData.userMessage.parentMessageId,
+ conversationId: jobData.conversationId,
+ text: jobData.userMessage.text ?? '',
+ quotes: jobData.userMessage.quotes,
+ isCreatedByUser: true,
+ }
+ : null,
+ responseMessage: isEarlyAbort
+ ? null
+ : {
+ messageId: jobData.responseMessageId ?? `${userMessageId ?? 'aborted'}_`,
+ parentMessageId: userMessageId,
+ conversationId: jobData.conversationId,
+ content: abortContent,
+ sender: jobData.sender ?? 'AI',
+ endpoint: jobData.endpoint,
+ iconURL: jobData.iconURL,
+ model: jobData.model,
+ unfinished: true,
+ error: false,
+ isCreatedByUser: false,
+ },
+ aborted: true,
+ // Flag for early abort - no messages saved, frontend should go to new chat
+ earlyAbort: isEarlyAbort,
+ ...(pendingSteers.length > 0 && { pendingSteers }),
+ } satisfies t.FinalEvent as t.ServerSentEvent;
+
+ const abortResult: AbortResult = {
+ success: true,
jobData,
content: abortContent,
- finalEvent: null,
+ finalEvent: abortFinalEvent,
text,
collectedUsage,
...(pendingSteers.length > 0 && { pendingSteers }),
};
- }
-
- // Signal only the generation whose terminal transition won above. The
- // transport tag prevents a delayed predecessor abort from reaching a
- // same-stream replacement on another replica.
- if (this.eventTransport.emitAbort) {
- this.eventTransport.emitAbort(streamId, jobData.createdAt);
- }
- if (runtime) {
- this.releaseAbortSubscription(runtime);
- }
- runtime?.abortController.abort();
-
- if (runtime) {
- runtime.finalEvent = abortFinalEvent;
- }
- if (runtime?.createdEventPublication) {
- await runtime.createdEventPublication;
- }
- await this.eventTransport.emitDone(streamId, abortFinalEvent, jobData.createdAt);
- if (runtime?.startupTelemetry) {
- this.recordStartupEvent(runtime, abortFinalEvent);
- }
- runtime?.startupTelemetry?.end('aborted');
- if (runtime) {
- runtime.startupTelemetry = undefined;
- }
- if (this._cleanupOnComplete) {
- // A replacement created after the abort CAS makes this a safe no-op.
- await this.jobStore.deleteJob(streamId, jobData.createdAt);
- }
- if (runtime && this.runtimeState.get(streamId) === runtime) {
- this.jobStore.clearContentState(streamId, jobData.createdAt);
- this.runStepBuffers?.delete(streamId);
- this.replayEventWriteQueues.delete(streamId);
- this.tokenUsageWriteQueues.delete(streamId);
- if (this._cleanupOnComplete) {
- this.runtimeState.delete(streamId);
+ let requiredPersistenceFailed = false;
+ try {
+ if (options?.beforePublish) {
+ await options.beforePublish(abortResult);
+ }
+ } catch (persistenceError) {
+ requiredPersistenceFailed = true;
+ logger.error(
+ `[GenerationJobManager] Required abort persistence failed for ${streamId}:`,
+ persistenceError,
+ );
}
- }
- this.releaseJobOwnership(streamId, jobData.createdAt);
- recordGenerationJob(this.storeLabel, 'aborted');
- logger.debug(`[GenerationJobManager] Job aborted: ${streamId}`);
+ const publication = await this.publishTerminalClaim(
+ terminalClaim,
+ requiredPersistenceFailed ? null : abortFinalEvent,
+ );
- return {
- success: true,
- jobData,
- content: abortContent,
- finalEvent: abortFinalEvent,
- text,
- collectedUsage,
- ...(pendingSteers.length > 0 && { pendingSteers }),
- };
+ return {
+ ...abortResult,
+ finalEvent: publication.finalEvent,
+ ...(publication.persistenceFailed && { persistenceFailed: true }),
+ };
+ } finally {
+ await this.finishTerminalJob(terminalClaim);
+ }
}
/**
@@ -1686,7 +3809,10 @@ class GenerationJobManagerClass {
// Read the durable generation first, then reconcile any lazily-created
// runtime against it. This also avoids the historical second Redis lookup
// when a cross-replica subscriber has no local runtime yet.
- const jobData = prepared ? prepared.jobData : await this.jobStore.getJob(streamId);
+ let jobData = prepared ? prepared.jobData : await this.jobStore.getJob(streamId);
+ if (jobData) {
+ jobData = await this.recoverStaleTerminalPersistence(jobData);
+ }
if (options?.signal?.aborted) {
recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'error');
return null;
@@ -1694,6 +3820,10 @@ class GenerationJobManagerClass {
if (this.rejectSubscriptionDuringShutdown(subscriptionType, onError)) {
return null;
}
+ if (options?.expectedCreatedAt != null && jobData?.createdAt !== options.expectedCreatedAt) {
+ recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'not_found');
+ return null;
+ }
const runtime = prepared?.runtime ?? (await this.getOrCreateRuntimeState(streamId, jobData));
if (options?.signal?.aborted) {
@@ -1707,6 +3837,10 @@ class GenerationJobManagerClass {
recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'not_found');
return null;
}
+ if (options?.expectedCreatedAt != null && runtime.createdAt !== options.expectedCreatedAt) {
+ recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'not_found');
+ return null;
+ }
if (
this.runtimeState.get(streamId) !== runtime ||
@@ -1720,7 +3854,14 @@ class GenerationJobManagerClass {
let createdEventDelivered = options?.skipBufferReplay === true;
let terminalEventDelivered = false;
let terminalEventQueued = false;
- let deliveryActivated = prepared?.deferDeliveryUntilActivated !== true;
+ let terminalPersistenceTimer: NodeJS.Timeout | null = null;
+ /** Identity-fenced initial subscriptions remain paused until a final
+ * durable job check immediately before replay. This prevents a job
+ * replacement during attachment awaits from leaking either replayed or
+ * live events into the stale request. Resume subscriptions already use
+ * the same pause for their sync-first protocol. */
+ let deliveryActivated =
+ prepared?.deferDeliveryUntilActivated !== true && options?.expectedCreatedAt == null;
let deferredDeliveries: DeferredDelivery[] = [];
let subscription: {
ready?: Promise<void>;
@@ -1767,10 +3908,10 @@ class GenerationJobManagerClass {
return;
}
terminalEventDelivered = true;
- // The pre-drain shutdown error only closes this process's SSE response; it is not a
- // durable terminal job error. Leave errorEvent unset so all-subscribers-left cleanup
- // still persists partial progress before the job store is destroyed.
- if (error !== SHUTDOWN_SUBSCRIBER_ERROR) {
+ // These internal close signals only recycle this process's SSE response; they are not
+ // durable terminal job errors. Leave errorEvent unset so a reconnect can replay the
+ // authoritative store state instead of inheriting a process-local transport failure.
+ if (error !== SHUTDOWN_SUBSCRIBER_ERROR && error !== TERMINAL_PUBLICATION_RECONNECT_ERROR) {
runtime.errorEvent = error;
}
try {
@@ -1813,7 +3954,7 @@ class GenerationJobManagerClass {
}
if (!deliveryActivated) {
terminalEventQueued = true;
- if (error !== SHUTDOWN_SUBSCRIBER_ERROR) {
+ if (error !== SHUTDOWN_SUBSCRIBER_ERROR && error !== TERMINAL_PUBLICATION_RECONNECT_ERROR) {
runtime.errorEvent = error;
}
deferredDeliveries.push({ type: 'error', error });
@@ -1899,6 +4040,10 @@ class GenerationJobManagerClass {
return;
}
subscriptionActive = false;
+ if (terminalPersistenceTimer) {
+ clearTimeout(terminalPersistenceTimer);
+ terminalPersistenceTimer = null;
+ }
deferredDeliveries = [];
runtime.earlyReplayHandlers.delete(queueChunk);
runtime.localErrorHandlers.delete(queueError);
@@ -1921,6 +4066,18 @@ class GenerationJobManagerClass {
return subscriptionActive;
};
+ const stillMatchesExpectedGeneration = async (): Promise<boolean> => {
+ if (options?.expectedCreatedAt == null) {
+ return true;
+ }
+ const currentJob = await this.jobStore.getJob(streamId);
+ return (
+ currentJob?.createdAt === options.expectedCreatedAt &&
+ runtime.createdAt === options.expectedCreatedAt &&
+ this.runtimeState.get(streamId) === runtime
+ );
+ };
+
try {
if (subscription.ready && !(await waitWhileAttached(subscription.ready))) {
recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'error');
@@ -1930,7 +4087,11 @@ class GenerationJobManagerClass {
recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'error');
return null;
}
- recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'success');
+ if (!(await stillMatchesExpectedGeneration())) {
+ subscription.unsubscribe();
+ recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'not_found');
+ return null;
+ }
} catch (err) {
subscription.unsubscribe();
recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'error');
@@ -1957,6 +4118,11 @@ class GenerationJobManagerClass {
if (this.detachSubscriptionDuringShutdown(subscription)) {
return null;
}
+ if (!(await stillMatchesExpectedGeneration())) {
+ subscription.unsubscribe();
+ recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'not_found');
+ return null;
+ }
/**
* Redis sequences are conversation-scoped and may start above zero. Use the
@@ -2046,6 +4212,8 @@ class GenerationJobManagerClass {
return null;
}
+ recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'success');
+
if (isFirst) {
runtime.resolveReady();
logger.debug(
@@ -2053,24 +4221,46 @@ class GenerationJobManagerClass {
);
}
- // Only schedule stored terminal delivery after the attachment is fully prepared.
- // The async function resolves before setImmediate runs, giving the route its
- // unsubscribe handle before a terminal callback can end the response.
- setImmediate(() => {
- void (async () => {
+ if (prepared?.deferDeliveryUntilActivated !== true && options?.expectedCreatedAt != null) {
+ activateDelivery();
+ }
+
+ const scheduleTerminalPersistenceCheck = (delayMs: number): void => {
+ if (!subscriptionActive || terminalEventDelivered || terminalPersistenceTimer) {
+ return;
+ }
+ terminalPersistenceTimer = setTimeout(() => {
+ terminalPersistenceTimer = null;
+ void deliverStoredTerminal();
+ }, delayMs);
+ terminalPersistenceTimer.unref?.();
+ };
+
+ const deliverStoredTerminal = async (): Promise<void> => {
+ try {
if (this.shuttingDown || !subscriptionActive || terminalEventDelivered) {
return;
}
- let terminalJob = jobData;
- if (!terminalJob || !['complete', 'error', 'aborted'].includes(terminalJob.status)) {
- try {
- terminalJob = await this.jobStore.getJob(streamId);
- } catch (err) {
- logger.warn(
- `[GenerationJobManager] Failed to refresh terminal state for ${streamId}:`,
- err,
+ let terminalJob = await this.jobStore.getJob(streamId);
+ if (
+ terminalJob?.terminalPersistencePending === true &&
+ ['complete', 'error', 'aborted'].includes(terminalJob.status)
+ ) {
+ terminalJob = await this.recoverStaleTerminalPersistence(terminalJob);
+ if (terminalJob?.terminalPersistencePending === true) {
+ const startedAt =
+ terminalJob.terminalPersistenceStartedAt ??
+ terminalJob.completedAt ??
+ terminalJob.createdAt;
+ const remaining = Math.max(
+ 1,
+ TERMINAL_PERSISTENCE_TIMEOUT_MS - (Date.now() - startedAt),
);
+ // Poll modestly so a successfully persisted payload whose pub/sub
+ // frame was lost is delivered promptly, while the hard deadline
+ // still recovers a crashed owner.
+ scheduleTerminalPersistenceCheck(Math.min(1000, remaining));
return;
}
}
@@ -2079,7 +4269,9 @@ class GenerationJobManagerClass {
!subscriptionActive ||
terminalEventDelivered ||
!terminalJob ||
- !['complete', 'error', 'aborted'].includes(terminalJob.status)
+ (terminalJob.status !== 'complete' &&
+ terminalJob.status !== 'error' &&
+ terminalJob.status !== 'aborted')
) {
return;
}
@@ -2112,8 +4304,44 @@ class GenerationJobManagerClass {
if (finalEvent) {
runtime.finalEvent = finalEvent;
queueDone(finalEvent);
+ return;
}
- })();
+
+ /** The terminal CAS can commit just before its owner crashes, leaving
+ * no normal FINAL payload. Never strand a late SSE subscriber on that
+ * durable terminal row: send a control-only terminal event that tells
+ * the client to refetch messages/status and reconcile parked steers.
+ * This is deliberately not an error event, so no user-facing failure
+ * card is fabricated for a generation whose DB writes may be complete. */
+ const reconcileEvent: t.ServerSentEvent = {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: terminalJob.status,
+ generationCreatedAt: terminalJob.createdAt,
+ conversation: {
+ conversationId: terminalJob.conversationId ?? streamId,
+ },
+ } satisfies t.FinalEvent;
+ logger.warn(
+ `[GenerationJobManager] Terminal job has no final payload; requesting client reconciliation: ${streamId}`,
+ );
+ runtime.finalEvent = reconcileEvent;
+ queueDone(reconcileEvent);
+ } catch (err) {
+ logger.warn(
+ `[GenerationJobManager] Failed to refresh terminal state for ${streamId}:`,
+ err,
+ );
+ scheduleTerminalPersistenceCheck(1000);
+ }
+ };
+
+ // Only schedule stored terminal delivery after the attachment is fully prepared.
+ // The async function resolves before setImmediate runs, giving the route its
+ // unsubscribe handle before a terminal callback can end the response.
+ setImmediate(() => {
+ void deliverStoredTerminal();
});
return subscription;
@@ -2227,7 +4455,7 @@ class GenerationJobManagerClass {
onChunk: t.ChunkHandler,
onDone?: t.DoneHandler,
onError?: t.ErrorHandler,
- options?: Pick<t.SubscribeOptions, 'signal'>,
+ options?: Pick<t.SubscribeOptions, 'signal' | 'expectedCreatedAt'>,
): Promise<t.SubscribeWithResumeResult> {
if (options?.signal?.aborted) {
recordGenerationStreamSubscription(this.storeLabel, 'resume', 'error');
@@ -2250,6 +4478,11 @@ class GenerationJobManagerClass {
recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
return { subscription: null, resumeState: null, pendingEvents: [] };
}
+ if (options?.expectedCreatedAt != null && runtime.createdAt !== options.expectedCreatedAt) {
+ recordGenerationStreamSubscription(this.storeLabel, 'resume_state', 'missing');
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
const capturedPendingEvents: t.ServerSentEvent[] = [];
const pendingEvents: t.ServerSentEvent[] = [];
@@ -2310,7 +4543,7 @@ class GenerationJobManagerClass {
);
await Promise.all(preSnapshotEmissions.map(([, emission]) => emission.snapshotReady));
const [candidateState, candidateJob] = await Promise.all([
- this.getResumeState(streamId),
+ this.getResumeState(streamId, options?.expectedCreatedAt),
this.jobStore.getJob(streamId),
]);
if (
@@ -2337,7 +4570,7 @@ class GenerationJobManagerClass {
}
} else {
[resumeState, jobData] = await Promise.all([
- this.getResumeState(streamId),
+ this.getResumeState(streamId, options?.expectedCreatedAt),
this.jobStore.getJob(streamId),
]);
}
@@ -2351,6 +4584,17 @@ class GenerationJobManagerClass {
removeCaptureHandler();
return { subscription: null, resumeState, pendingEvents: [] };
}
+ if (
+ options?.expectedCreatedAt != null &&
+ (jobData?.createdAt !== options.expectedCreatedAt ||
+ runtime.createdAt !== options.expectedCreatedAt ||
+ this.runtimeState.get(streamId) !== runtime)
+ ) {
+ removeCaptureHandler();
+ recordGenerationStreamSubscription(this.storeLabel, 'resume_state', 'missing');
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
recordGenerationStreamSubscription(
this.storeLabel,
'resume_state',
@@ -2371,6 +4615,7 @@ class GenerationJobManagerClass {
{
skipBufferReplay: true,
signal: options?.signal,
+ expectedCreatedAt: options?.expectedCreatedAt,
},
{
runtime,
@@ -2455,32 +4700,48 @@ class GenerationJobManagerClass {
};
if (resumeState != null && jobActive) {
const snapshotSteers = resumeState.pendingSteers ?? [];
- const liveQueue = await this.jobStore.peekSteers(streamId, liveJob.createdAt);
+ const [liveQueue, liveClaims] = await Promise.all([
+ this.jobStore.peekSteers(streamId, liveJob.createdAt),
+ this.jobStore.peekClaimedSteers(streamId, liveJob.createdAt),
+ ]);
if (options?.signal?.aborted || this.detachSubscriptionDuringShutdown(subscription)) {
return cancelResumeSubscription();
}
- const liveIds = new Set(liveQueue.map((item) => item.steerId));
- const queueChanged =
- liveQueue.length !== snapshotSteers.length ||
- snapshotSteers.some((steer) => !liveIds.has(steer.steerId));
+ const liveUnresolved = mergeUnresolvedSteers(liveClaims, liveQueue);
+ const livePending = liveUnresolved.map(toPendingSteer);
+ /** Identity alone misses in-place mutations such as an interrupt flag
+ * downgrade in this exact gap. Compare the client-safe projections so
+ * revision/correlation/files/preempt changes replace stale snapshot
+ * truth even when FIFO membership is unchanged. */
+ let queueChanged = JSON.stringify(livePending) !== JSON.stringify(snapshotSteers);
if (queueChanged) {
- const livePending = liveQueue.map(toPendingSteer);
resumeState.pendingSteers = livePending.length > 0 ? livePending : undefined;
}
- if (queueChanged || liveQueue.length > 0) {
- const content = await readFreshContent();
- if (options?.signal?.aborted || this.detachSubscriptionDuringShutdown(subscription)) {
- return cancelResumeSubscription();
- }
- const gapEvents = synthesizeAppliedSteerEvents(
- (resumeState.aggregatedContent ?? []) as SteerContentView,
- liveQueue,
- (content ?? []) as SteerContentView,
- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },
- );
- if (gapEvents.length > 0) {
- pendingEvents.push(...gapEvents);
- }
+ // Always compare a fresh content frontier after attaching. A steer can
+ // be accepted+applied wholly between the initial XRANGE and queue
+ // reads, leaving both snapshots empty while its pub/sub event also
+ // predates the subscription.
+ const content = await readFreshContent();
+ if (options?.signal?.aborted || this.detachSubscriptionDuringShutdown(subscription)) {
+ return cancelResumeSubscription();
+ }
+ const freshUnresolved = omitAlreadyAppliedSteers(
+ liveUnresolved,
+ (content ?? []) as unknown[],
+ );
+ const freshPending = freshUnresolved.map(toPendingSteer);
+ if (JSON.stringify(freshPending) !== JSON.stringify(resumeState.pendingSteers ?? [])) {
+ queueChanged = true;
+ resumeState.pendingSteers = freshPending.length > 0 ? freshPending : undefined;
+ }
+ const gapEvents = synthesizeAppliedSteerEvents(
+ (resumeState.aggregatedContent ?? []) as SteerContentView,
+ liveQueue,
+ (content ?? []) as SteerContentView,
+ { conversationId: streamId, responseMessageId: resumeState.responseMessageId },
+ );
+ if (gapEvents.length > 0) {
+ pendingEvents.push(...gapEvents);
}
}
@@ -2600,18 +4861,65 @@ class GenerationJobManagerClass {
* snapshot->subscribe window, so content is not a reliable signal.
* Best-effort: the flag is an optimization hint, never correctness.
*/
- async markActivityLabels(streamId: string): Promise<void> {
+ async markActivityLabels(streamId: string, expectedCreatedAt?: number): Promise<void> {
/** Deliberately REJECTS on failure. This flag gates resume gap
* reconciliation, so the caller retries it; swallowing the error here
* resolved successfully and made that retry unreachable, leaving the
* flag absent after a transient write failure. */
- await this.jobStore.updateJob(streamId, { activityLabels: true });
+ await this.jobStore.updateJob(streamId, { activityLabels: true }, expectedCreatedAt);
+ }
+
+ /** Publish a durable non-streaming control event from any API replica.
+ * `emitChunk` intentionally requires a local runtime, but steer/arm routes
+ * may execute on a non-owner during horizontal deployment. When the owner
+ * is local we retain the normal ordering/buffering path; otherwise the
+ * generation-fenced chunk append commits before direct transport publish. */
+ async emitChunkFromAnyReplica(
+ streamId: string,
+ event: t.ServerSentEvent,
+ expectedCreatedAt: number,
+ ): Promise<boolean> {
+ const runtime = this.runtimeState.get(streamId);
+ if (
+ runtime != null &&
+ runtime.createdAt === expectedCreatedAt &&
+ this.isCurrentRuntime(streamId, runtime)
+ ) {
+ await this.emitChunk(streamId, event, {
+ durable: true,
+ expectedCreatedAt,
+ });
+ return true;
+ }
+ if (!('event' in event) || event.data === undefined) {
+ return false;
+ }
+ const appended = await this.jobStore.appendChunk(
+ streamId,
+ { event: event.event, data: event.data },
+ expectedCreatedAt,
+ );
+ if (appended === false) {
+ return false;
+ }
+ const published = await emitChunkWithReceipt(
+ this.eventTransport,
+ streamId,
+ event,
+ expectedCreatedAt,
+ );
+ return published !== false;
}
async emitChunk(
streamId: string,
event: t.ServerSentEvent,
- options?: { durable?: boolean; expectedCreatedAt?: number },
+ options?: {
+ durable?: boolean;
+ expectedCreatedAt?: number;
+ /** Atomically settles a claimed steer with its durable applied event. */
+ deliveredSteer?: SteerQueueItem;
+ },
): Promise<void> {
const runtime = this.runtimeState.get(streamId);
if (
@@ -2619,6 +4927,9 @@ class GenerationJobManagerClass {
(options?.expectedCreatedAt != null && runtime.createdAt !== options.expectedCreatedAt) ||
!this.isCurrentRuntime(streamId, runtime)
) {
+ if (options?.durable === true) {
+ throw new Error(`Durable chunk owner was fenced out for ${streamId}`);
+ }
return;
}
const sequence = ++runtime.emissionSequence;
@@ -2643,6 +4954,9 @@ class GenerationJobManagerClass {
if (pendingCreatedEvent) {
await pendingCreatedEvent;
if (!this.isCurrentRuntime(streamId, runtime)) {
+ if (options?.durable === true) {
+ throw new Error(`Durable chunk owner was fenced out for ${streamId}`);
+ }
return;
}
}
@@ -2653,6 +4967,9 @@ class GenerationJobManagerClass {
if (pendingCreatedEvent) {
await pendingCreatedEvent;
if (!this.isCurrentRuntime(streamId, runtime)) {
+ if (options?.durable === true) {
+ throw new Error(`Durable chunk owner was fenced out for ${streamId}`);
+ }
return;
}
}
@@ -2685,9 +5002,12 @@ class GenerationJobManagerClass {
runtime: RuntimeJobState,
sequence: number,
markSnapshotReady: () => void,
- options?: { durable?: boolean },
+ options?: { durable?: boolean; deliveredSteer?: SteerQueueItem },
): Promise<void> {
if (!this.isCurrentRuntime(streamId, runtime)) {
+ if (options?.durable === true) {
+ throw new Error(`Durable chunk owner was fenced out for ${streamId}`);
+ }
return;
}
@@ -2700,6 +5020,9 @@ class GenerationJobManagerClass {
if (eventTracking) {
await eventTracking;
if (!this.isCurrentRuntime(streamId, runtime)) {
+ if (options?.durable === true) {
+ throw new Error(`Durable chunk owner was fenced out for ${streamId}`);
+ }
return;
}
}
@@ -2725,19 +5048,57 @@ class GenerationJobManagerClass {
// The aggregator expects { event: string, data: unknown } where data is the payload
if (eventType && eventData !== undefined) {
// Store in format expected by aggregateContent: { event, data }
- const appendPromise = this.jobStore
- .appendChunk(streamId, { event: eventType, data: eventData }, runtime.createdAt)
- .catch((err) => {
- logger.error(`[GenerationJobManager] Failed to append chunk:`, err);
- });
+ const appendPromise = this.jobStore.appendChunk(
+ streamId,
+ { event: eventType, data: eventData },
+ runtime.createdAt,
+ options?.deliveredSteer,
+ );
if (options?.durable === true) {
- await appendPromise;
+ let appended: boolean;
+ try {
+ appended = await appendPromise;
+ } catch (error) {
+ if (options.deliveredSteer == null) {
+ this.retireRuntimeAfterDurableFence(streamId, runtime);
+ }
+ throw error;
+ }
+ if (appended === false) {
+ if (options.deliveredSteer == null) {
+ this.retireRuntimeAfterDurableFence(streamId, runtime);
+ }
+ throw new Error(`Durable chunk append was fenced out for ${streamId}`);
+ }
if (!this.isCurrentRuntime(streamId, runtime)) {
return;
}
+ } else {
+ void appendPromise
+ .then((appended) => {
+ if (appended === false && options?.deliveredSteer == null) {
+ this.retireRuntimeAfterDurableFence(streamId, runtime);
+ }
+ })
+ .catch((err) => {
+ logger.error(`[GenerationJobManager] Failed to append chunk:`, err);
+ if (options?.deliveredSteer == null) {
+ this.retireRuntimeAfterDurableFence(streamId, runtime);
+ }
+ });
}
}
+ } else if (options?.deliveredSteer != null) {
+ const settled = await this.jobStore.appendChunk(
+ streamId,
+ event,
+ runtime.createdAt,
+ options.deliveredSteer,
+ );
+ if (settled === false) {
+ throw new Error(`Steer receipt settlement was fenced out for ${streamId}`);
+ }
}
if (!this.isCurrentRuntime(streamId, runtime)) {
@@ -2762,7 +5123,33 @@ class GenerationJobManagerClass {
}
if (!buffered && !runtime.startupTelemetry) {
- await this.eventTransport.emitChunk(streamId, event, runtime.createdAt);
+ try {
+ const published = await emitChunkWithReceipt(
+ this.eventTransport,
+ streamId,
+ event,
+ runtime.createdAt,
+ );
+ if (published === false) {
+ this.retireRuntimeAfterDurableFence(streamId, runtime);
+ if (options?.durable === true && options.deliveredSteer == null) {
+ throw new Error(`Durable chunk publication was fenced out for ${streamId}`);
+ }
+ }
+ } catch (error) {
+ if (options?.deliveredSteer == null) {
+ throw error;
+ }
+ /** The applied part and receipt were committed together above. A
+ * subscriber/transport failure after that point is a delivery problem,
+ * not a storage failure: reconnect replay owns recovery, while
+ * rejecting here would make AgentClient roll back host content that
+ * the receipt already calls delivered. */
+ logger.error(
+ `[GenerationJobManager] Failed to publish committed steer ${options.deliveredSteer.steerId}:`,
+ error,
+ );
+ }
return;
}
@@ -2784,12 +5171,68 @@ class GenerationJobManagerClass {
);
}
- const published = await publication;
- if ((published !== false || buffered) && runtime.startupTelemetry) {
+ const published = await publication.catch((error) => {
+ if (options?.deliveredSteer == null) {
+ throw error;
+ }
+ /** Same post-commit rule as the direct publication path above. The
+ * buffered event and durable chunk remain replayable. */
+ logger.error(
+ `[GenerationJobManager] Failed to publish committed steer ${options.deliveredSteer.steerId}:`,
+ error,
+ );
+ return undefined;
+ });
+ if (published === false) {
+ this.retireRuntimeAfterDurableFence(streamId, runtime);
+ if (options?.durable === true && options.deliveredSteer == null) {
+ throw new Error(`Durable chunk publication was fenced out for ${streamId}`);
+ }
+ return;
+ }
+ // The false branch returned above, so any remaining receipt is either a
+ // sequence number or an unsequenced success.
+ if (runtime.startupTelemetry) {
this.recordStartupEvent(runtime, event);
}
}
+ /** A generation-fenced append returning false is same-slot proof that this
+ * epoch is no longer the durable owner. This is the provider's backstop when
+ * a replacement abort publication is lost. Retire only the exact captured
+ * runtime; a newer local successor may already occupy the same stream id. */
+ private retireRuntimeAfterDurableFence(streamId: string, runtime: RuntimeJobState): void {
+ if (this.runtimeState.get(streamId) !== runtime) {
+ return;
+ }
+ runtime.startupTelemetry?.end('replaced');
+ runtime.startupTelemetry = undefined;
+ runtime.abortController.abort();
+ if (runtime.replacementTransportHold === true) {
+ return;
+ }
+ this.releaseAbortSubscription(runtime);
+ if (this.runtimeState.get(streamId) === runtime) {
+ this.runtimeState.delete(streamId);
+ this.releaseJobOwnership(streamId, runtime.createdAt);
+ this.runStepBuffers?.delete(streamId);
+ this.replayEventWriteQueues.delete(streamId);
+ this.tokenUsageWriteQueues.delete(streamId);
+ this.jobStore.clearContentState(streamId, runtime.createdAt);
+ try {
+ // Delete runtime ownership before closing. The transport's
+ // all-subscribers-left callback may run synchronously; it must not
+ // persist a partial response for this already-replaced epoch.
+ this.eventTransport.closeLocalSubscribers?.(streamId, TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ } catch (error) {
+ logger.error(
+ `[GenerationJobManager] Failed to recycle subscribers for fenced generation ${streamId}:`,
+ error,
+ );
+ }
+ }
+ }
+
private isCurrentRuntime(streamId: string, runtime: RuntimeJobState): boolean {
return this.runtimeState.get(streamId) === runtime && !runtime.abortController.signal.aborted;
}
@@ -3255,7 +5698,12 @@ class GenerationJobManagerClass {
* will seal for it, so the caller must not acknowledge it as interrupting.
* The owner's own `createdAt` fence still decides whether to honour it.
*/
- async requestPreempt(streamId: string, steerId: string, jobCreatedAt: number): Promise<boolean> {
+ async requestPreempt(
+ streamId: string,
+ steerId: string,
+ jobCreatedAt: number,
+ revision = 0,
+ ): Promise<boolean> {
/**
* `ownedJobs`, NOT `runtimeState`: a cross-replica `getJob` installs a
* facade runtime on THIS replica via `getOrCreateRuntimeState`, so a
@@ -3268,7 +5716,11 @@ class GenerationJobManagerClass {
if (ownedHere && runtime != null && runtime.createdAt === jobCreatedAt) {
/** Zero accepted means the id was tombstoned (its steer drained at an
* ordinary boundary mid-request) — not an arm, so do not claim one. */
- return this.armPreemptIds(runtime, jobCreatedAt, [steerId]) > 0;
+ return (
+ this.armPreemptIds(runtime, jobCreatedAt, [steerId], {
+ [steerId]: revision,
+ }) > 0
+ );
}
if (this.eventTransport.emitPreempt == null) {
/** Single-process transport: the local arm is the whole mechanism. */
@@ -3279,6 +5731,7 @@ class GenerationJobManagerClass {
op: 'arm',
createdAt: jobCreatedAt,
steerIds: [steerId],
+ revisions: { [steerId]: revision },
});
/**
* Best effort, and deliberately not read as proof. The publisher's
@@ -3336,10 +5789,10 @@ class GenerationJobManagerClass {
* once, the empty-boundary self-clear disarms the generation, and the
* turn is persisted `unfinished: true` rather than silently truncated.
*/
- const publish = (): Promise<unknown> =>
- Promise.resolve(
- this.eventTransport.emitPreempt?.(streamId, { op: 'clear', createdAt, steerIds }),
- );
+ const publish = (): Promise<void> =>
+ Promise.resolve().then(async () => {
+ await this.eventTransport.emitPreempt?.(streamId, { op: 'clear', createdAt, steerIds });
+ });
return publish().then(
() => true,
() =>
@@ -3380,6 +5833,39 @@ class GenerationJobManagerClass {
* come back a second time.
*/
async rearmQueuedPreempts(streamId: string, jobCreatedAt: number): Promise<number> {
+ /** A capable→incapable HITL handover must rewrite durable truth before it
+ * rebuilds runtime arms. The store returns the exact ids it downgraded so
+ * they can be tombstoned against an arm publication that was already in
+ * flight when the approval transition changed capability. `[]` still
+ * proves the live owner is incapable; `null` means capable/missing/stale
+ * and falls through to the ordinary reconciliation below. */
+ const downgraded = await this._steering.downgradePreempts(streamId, jobCreatedAt);
+ if (downgraded != null) {
+ const runtime = this.runtimeState.get(streamId);
+ if (runtime != null && runtime.createdAt === jobCreatedAt) {
+ this.downgradePreemptIds(runtime, jobCreatedAt, downgraded);
+ }
+ if (downgraded.length > 0) {
+ await this.emitChunk(
+ streamId,
+ {
+ event: SteerEvents.ON_STEER_UPDATED,
+ data: {
+ conversationId: streamId,
+ steers: downgraded.map((steer) => ({
+ steerId: steer.steerId,
+ ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
+ preempt: false,
+ preemptRevision: steer.preemptRevision ?? 0,
+ })),
+ },
+ },
+ { durable: true, expectedCreatedAt: jobCreatedAt },
+ );
+ }
+ return 0;
+ }
+
/**
* The armed set is snapshotted BEFORE the queue is read, and that order is
* the entire safety argument. `approvals.resolve` has already reopened
@@ -3403,9 +5889,8 @@ class GenerationJobManagerClass {
: [];
const queued = await this._steering.peek(streamId, jobCreatedAt);
- const backed = new Set(
- queued.filter((item) => item.preempt === true).map((item) => item.steerId),
- );
+ const backedItems = queued.filter((item) => item.preempt === true);
+ const backed = new Set(backedItems.map((item) => item.steerId));
/** The generation can be replaced across the read; only disarm the runtime
* the snapshot was taken from. */
@@ -3421,8 +5906,8 @@ class GenerationJobManagerClass {
}
let rearmed = 0;
- for (const steerId of backed) {
- await this.requestPreempt(streamId, steerId, jobCreatedAt);
+ for (const item of backedItems) {
+ await this.requestPreempt(streamId, item.steerId, jobCreatedAt, item.preemptRevision ?? 0);
rearmed += 1;
}
return rearmed;
@@ -3471,19 +5956,23 @@ class GenerationJobManagerClass {
/**
* Get resume state for reconnecting clients.
*/
- async getResumeState(streamId: string): Promise<t.ResumeState | null> {
+ async getResumeState(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<t.ResumeState | null> {
const jobData = await this.jobStore.getJob(streamId);
- if (!jobData) {
+ if (!jobData || (expectedCreatedAt != null && jobData.createdAt !== expectedCreatedAt)) {
return null;
}
/** Independent reads (streamId-only): parallel to collapse 3 Redis round trips into 1.
* Safe despite readCachedGraph's cache-drop side effect — each call catches its own
* unusable-graph throw and falls back to reconstruction, so ordering cannot change the result. */
- const [result, runSteps, queuedSteers] = await Promise.all([
+ const [result, runSteps, queuedSteers, claimedSteers] = await Promise.all([
this.jobStore.getContentParts(streamId, jobData.createdAt),
this.jobStore.getRunSteps(streamId, jobData.createdAt),
this.jobStore.peekSteers(streamId, jobData.createdAt),
+ this.jobStore.peekClaimedSteers(streamId, jobData.createdAt),
]);
const aggregatedContent = result?.content ?? [];
const bufferState = this.runStepBuffers?.get(streamId);
@@ -3536,7 +6025,19 @@ class GenerationJobManagerClass {
}
/** Steers still queued (not yet injected); injected ones are already in aggregatedContent. */
- const pendingSteers = queuedSteers.map(toPendingSteer);
+ const pendingSteers = omitAlreadyAppliedSteers(
+ mergeUnresolvedSteers(claimedSteers, queuedSteers),
+ aggregatedContent as unknown[],
+ ).map(toPendingSteer);
+
+ /** The four component reads are generation-fenced, but replacement can
+ * land after the initial job read and make all of them return empty. Do
+ * not turn that into a plausible-looking snapshot with predecessor
+ * metadata; verify the same generation is still live after the snapshot. */
+ const verifiedJob = await this.jobStore.getJob(streamId);
+ if (!verifiedJob || verifiedJob.createdAt !== jobData.createdAt) {
+ return null;
+ }
logger.debug(`[GenerationJobManager] getResumeState:`, {
streamId,
@@ -3573,15 +6074,17 @@ class GenerationJobManagerClass {
* Mark that sync has been sent.
* Persists to Redis for cross-replica consistency.
*/
- markSyncSent(streamId: string): void {
+ markSyncSent(streamId: string, expectedCreatedAt?: number): void {
const runtime = this.runtimeState.get(streamId);
- if (runtime) {
+ if (runtime && (expectedCreatedAt == null || runtime.createdAt === expectedCreatedAt)) {
runtime.syncSent = true;
}
// Persist to Redis for cross-replica consistency
- this.jobStore.updateJob(streamId, { syncSent: true }, runtime?.createdAt).catch((err) => {
- logger.error(`[GenerationJobManager] Failed to persist syncSent flag:`, err);
- });
+ this.jobStore
+ .updateJob(streamId, { syncSent: true }, expectedCreatedAt ?? runtime?.createdAt)
+ .catch((err) => {
+ logger.error(`[GenerationJobManager] Failed to persist syncSent flag:`, err);
+ });
}
/**
@@ -3663,18 +6166,27 @@ class GenerationJobManagerClass {
* and by the resume route, which observes a just-expired action when the user submits a
* decision after the TTL lapsed. Returns true if this call expired the action.
*/
- async expireApproval(streamId: string, actionId?: string): Promise<boolean> {
+ async expireApproval(
+ streamId: string,
+ actionId?: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
const observedRuntime = this.runtimeState.get(streamId);
let observedJob: SerializableJobData | null = null;
- try {
- observedJob = await this.jobStore.getJob(streamId);
- } catch (err) {
- logger.warn(`[GenerationJobManager] Failed to read approval before expiry ${streamId}`, err);
+ if (expectedCreatedAt == null) {
+ try {
+ observedJob = await this.jobStore.getJob(streamId);
+ } catch (err) {
+ logger.warn(
+ `[GenerationJobManager] Failed to read approval before expiry ${streamId}`,
+ err,
+ );
+ }
}
const expiredCreatedAt = await this._approvals.expireWithIdentity(
streamId,
actionId,
- observedJob?.createdAt,
+ expectedCreatedAt ?? observedJob?.createdAt,
);
if (expiredCreatedAt == null) {
return false;
@@ -3756,7 +6268,11 @@ class GenerationJobManagerClass {
// as stale. Between this read and the CAS, the user could resolve it and the run
// re-pause on a fresh action; without the id, the CAS (status-only) would abort
// that valid new pause and leave it terminal.
- const didExpire = await this.expireApproval(streamId, job.pendingAction?.actionId);
+ const didExpire = await this.expireApproval(
+ streamId,
+ job.pendingAction?.actionId,
+ job.createdAt,
+ );
if (!didExpire) {
continue;
}
@@ -3768,7 +6284,41 @@ class GenerationJobManagerClass {
}
}
+ /**
+ * Let the manager (rather than store-only cleanup) win stale pause barriers
+ * whenever this replica has the runtime. That preserves the same atomic
+ * fail-closed store transition while also notifying an attached client and
+ * releasing process-local graph/listener resources.
+ */
+ private async failStalePausePersistenceBarriers(): Promise<void> {
+ for (const [streamId, runtime] of [...this.runtimeState]) {
+ try {
+ await this._approvals.failStalePausePersistence(streamId, runtime.createdAt);
+ /** Another replica's store-only cleanup may have won the same exact
+ * CAS first. Relay that durable timeout into this replica's attached
+ * runtime even though there is no barrier left for the lifecycle call
+ * above to transition. */
+ await this.finishTimedOutPausePersistence(
+ streamId,
+ runtime.createdAt,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ [],
+ );
+ } catch (error) {
+ logger.error(
+ `[GenerationJobManager] Failed to inspect stale pause persistence: ${streamId}`,
+ error,
+ );
+ }
+ }
+ }
+
private async cleanup(): Promise<void> {
+ // A crashed pause writer must never become resumable merely because its
+ // lease elapsed. Give a locally-observed runtime the first chance to fail
+ // closed so its attached subscriber receives the terminal error.
+ await this.failStalePausePersistenceBarriers();
+
// Finalize approvals whose window lapsed before the store's own cleanup, so a
// client still attached to a paused stream gets a terminal event instead of a
// connection that hangs open until it gives up.
@@ -4062,7 +6612,7 @@ class GenerationJobManagerClass {
for (const runtime of this.runtimeState.values()) {
runtime.startupTelemetry?.end('aborted');
runtime.startupTelemetry = undefined;
- this.releaseAbortSubscription(runtime);
+ this.releaseAbortSubscription(runtime, true);
runtime.abortController.abort();
}
diff --git a/packages/api/src/stream/SteerRecovery.ts b/packages/api/src/stream/SteerRecovery.ts
new file mode 100644
index 0000000000..f5a680ca81
--- /dev/null
+++ b/packages/api/src/stream/SteerRecovery.ts
@@ -0,0 +1,83 @@
+import type { TFile, TPendingSteer } from 'librechat-data-provider';
+
+/** Immutable user-visible payload a parked steer recovery is allowed to submit. */
+export interface RecoveredSteerPayload {
+ text: string;
+ /** Sorted, unique file ids. Display metadata is deliberately excluded: the
+ * normal send path re-resolves files by owner and only identity is binding. */
+ fileIds: string[];
+}
+
+/** A recovery-shaped request did not reproduce the parked source exactly. */
+export class RecoveredSteerPayloadMismatchError extends Error {
+ readonly code = 'RECOVERY_PAYLOAD_MISMATCH';
+
+ constructor() {
+ super('Recovered steer payload does not match its parked source');
+ this.name = 'RecoveredSteerPayloadMismatchError';
+ }
+}
+
+/** Canonical attachment identity: every entry must name a file; ordering,
+ * duplicates, and mutable display metadata do not affect identity. */
+export function canonicalRecoveryFileIds(files: unknown): string[] | null {
+ if (files == null) {
+ return [];
+ }
+ if (!Array.isArray(files)) {
+ return null;
+ }
+ const ids = new Set<string>();
+ for (const file of files) {
+ if (file == null || typeof file !== 'object') {
+ return null;
+ }
+ const fileId = (file as Partial<TFile>).file_id;
+ if (typeof fileId !== 'string' || fileId.length === 0) {
+ return null;
+ }
+ ids.add(fileId);
+ }
+ return [...ids].sort();
+}
+
+/** Build the trusted payload proof passed from the HTTP controller into the
+ * atomic job-creation transaction. `null` means the recovery request itself
+ * is malformed and must be rejected before claiming a generation. */
+export function buildRecoveredSteerPayload(
+ text: unknown,
+ files: unknown,
+): RecoveredSteerPayload | null {
+ if (typeof text !== 'string') {
+ return null;
+ }
+ const fileIds = canonicalRecoveryFileIds(files);
+ return fileIds == null ? null : { text, fileIds };
+}
+
+export function isRecoveredSteerPayload(value: unknown): value is RecoveredSteerPayload {
+ if (value == null || typeof value !== 'object') {
+ return false;
+ }
+ const payload = value as Partial<RecoveredSteerPayload>;
+ return (
+ typeof payload.text === 'string' &&
+ Array.isArray(payload.fileIds) &&
+ payload.fileIds.every((id) => typeof id === 'string' && id.length > 0) &&
+ payload.fileIds.length === new Set(payload.fileIds).size &&
+ payload.fileIds.every((id, index) => index === 0 || payload.fileIds![index - 1] < id)
+ );
+}
+
+export function recoveredSteerPayloadMatches(
+ item: Pick<TPendingSteer, 'text' | 'files'>,
+ expected: RecoveredSteerPayload,
+): boolean {
+ const fileIds = canonicalRecoveryFileIds(item.files);
+ return (
+ item.text === expected.text &&
+ fileIds != null &&
+ fileIds.length === expected.fileIds.length &&
+ fileIds.every((id, index) => id === expected.fileIds[index])
+ );
+}
diff --git a/packages/api/src/stream/SteeringLifecycle.ts b/packages/api/src/stream/SteeringLifecycle.ts
index f8cf62c717..33ac4c1660 100644
--- a/packages/api/src/stream/SteeringLifecycle.ts
+++ b/packages/api/src/stream/SteeringLifecycle.ts
@@ -1,17 +1,29 @@
import { logger } from '@librechat/data-schemas';
import { ContentTypes, SteerEvents } from 'librechat-data-provider';
import type { TPendingSteer } from 'librechat-data-provider';
-import type { IJobStore, SteerQueueItem } from '~/stream/interfaces/IJobStore';
+import type {
+ GenerationProtocolVersion,
+ IJobStoreV2,
+ SteerArmOutcome,
+ SteerArmResult,
+ SteerEnqueueReceiptResult,
+ SteerEnqueueVersionedResult,
+ SteerQueueItem,
+ SteerReceipt,
+ SteerReceiptInput,
+} from '~/stream/interfaces/IJobStore';
import type { ServerSentEvent } from '~/types';
/** Client-safe projection of a queued steer (drops the server-only userId). */
export function toPendingSteer(item: SteerQueueItem): TPendingSteer {
return {
steerId: item.steerId,
+ ...(item.clientSteerId && { clientSteerId: item.clientSteerId }),
text: item.text,
createdAt: item.createdAt,
...(item.files && item.files.length > 0 && { files: item.files }),
...(item.preempt === true && { preempt: true }),
+ ...(item.preemptRevision != null && { preemptRevision: item.preemptRevision }),
};
}
@@ -23,17 +35,19 @@ export interface SteerOwner {
}
interface ParkedSteers extends SteerOwner {
+ generationProtocolVersion?: GenerationProtocolVersion;
steers: TPendingSteer[];
}
-/** The exact JSON substring the stores gate the atomic claim on — matches the
- * `userId` member as {@link SteeringLifecycle.park} serializes it. */
-export function toOwnerFragment(userId: string): string {
- return `"userId":${JSON.stringify(userId)}`;
+export interface ParkedSteerRecovery {
+ generationProtocolVersion: GenerationProtocolVersion;
+ steers: TPendingSteer[];
}
/** Loosely-shaped content part for steer-id inspection across content views. */
-export type SteerContentView = Array<{ type?: string; steerId?: string } | undefined>;
+export type SteerContentView = Array<
+ { type?: string; steerId?: string; clientSteerId?: string } | undefined
+>;
/**
* Synthesize the `on_steer_applied` events a reconnecting subscriber missed in
@@ -69,6 +83,7 @@ export function synthesizeAppliedSteerEvents(
event: SteerEvents.ON_STEER_APPLIED,
data: {
steerId: part.steerId,
+ ...(part.clientSteerId && { clientSteerId: part.clientSteerId }),
index: i,
part,
conversationId: meta.conversationId,
@@ -95,7 +110,7 @@ export function synthesizeAppliedSteerEvents(
* to the client, which converts them to queued follow-up turns.
*/
export class SteeringLifecycle {
- constructor(private readonly store: IJobStore) {}
+ constructor(private readonly store: IJobStoreV2) {}
/**
* Append a steer, guarded on the job being `running`. Returns the new queue
@@ -116,12 +131,49 @@ export class SteeringLifecycle {
return depth;
}
+ enqueueVersioned(
+ streamId: string,
+ item: SteerQueueItem,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueVersionedResult> {
+ return this.store.enqueueSteerVersioned(streamId, item, wantsPreempt, expectedCreatedAt);
+ }
+
+ enqueueWithReceipt(
+ streamId: string,
+ item: SteerQueueItem,
+ receipt: SteerReceiptInput,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueReceiptResult> {
+ return this.store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ receipt,
+ wantsPreempt,
+ expectedCreatedAt,
+ );
+ }
+
+ getReceipt(streamId: string, clientSteerId: string): Promise<SteerReceipt | null> {
+ return this.store.getSteerReceipt(streamId, clientSteerId);
+ }
+
/** Atomically take ALL queued steers, FIFO. `expectedCreatedAt` refuses the
* drain inside the store when the job was replaced. */
drain(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]> {
return this.store.drainSteers(streamId, expectedCreatedAt);
}
+ restoreClaimed(
+ streamId: string,
+ items: SteerQueueItem[],
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ return this.store.restoreClaimedSteers(streamId, items, expectedCreatedAt);
+ }
+
/**
* Terminal drain: atomically CLOSE the queue to new steers, then take all
* queued items. Finalization paths use this so a steer POST racing the
@@ -146,8 +198,69 @@ export class SteeringLifecycle {
* inline part is authoritative) or the run ended (the terminal paths own
* delivery), and the cancel is simply too late.
*/
- cancel(streamId: string, steerId: string): Promise<boolean> {
- return this.store.removeSteer(streamId, steerId);
+ cancel(streamId: string, steerId: string, expectedCreatedAt?: number): Promise<boolean> {
+ return this.store.removeSteer(streamId, steerId, expectedCreatedAt);
+ }
+
+ discardLeftover(
+ streamId: string,
+ clientSteerId: string,
+ steerId: string,
+ owner: SteerOwner,
+ expectedGenerationCreatedAt?: number,
+ ): Promise<boolean> {
+ return this.store.discardSteerLeftover(
+ streamId,
+ clientSteerId,
+ steerId,
+ owner.userId,
+ owner.tenantId,
+ expectedGenerationCreatedAt,
+ );
+ }
+
+ consumeRecovered(
+ streamId: string,
+ steerId: string,
+ owner: SteerOwner,
+ expectedCreatedAt: number,
+ ): Promise<boolean> {
+ return this.store.consumeParkedSteer(
+ streamId,
+ steerId,
+ owner.userId,
+ owner.tenantId,
+ expectedCreatedAt,
+ );
+ }
+
+ /**
+ * Escalate a still-queued steer to an interrupt IN PLACE — the durable
+ * `preempt` flag flips on the existing item, so its FIFO position survives
+ * (the whole queue drains at the seal, in order). Races with a drain,
+ * cancel, or replacement run settle inside the store's atomic update as
+ * `missing`, and the owner's LIVE capability is part of the same predicate
+ * (`incapable`) — see {@link IJobStoreV2.armSteer}.
+ */
+ arm(streamId: string, steerId: string, expectedCreatedAt?: number): Promise<SteerArmOutcome> {
+ return this.store.armSteer(streamId, steerId, expectedCreatedAt);
+ }
+
+ armVersioned(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerArmResult> {
+ return this.store.armSteerVersioned(streamId, steerId, expectedCreatedAt);
+ }
+
+ /** Relabel durable interrupt requests as ordinary steers when HITL resume
+ * moves ownership to a replica that cannot seal mid-stream. */
+ downgradePreempts(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerQueueItem[] | null> {
+ return this.store.downgradeSteerPreempts(streamId, expectedCreatedAt);
}
/** Drop any queued steers (terminal cleanup backstop). */
@@ -187,23 +300,42 @@ export class SteeringLifecycle {
}
/**
- * Claim-on-read: returns parked leftovers and clears them, so a second
- * reload cannot re-mint chips the user already dismissed. The user check
- * runs INSIDE the store's atomic claim (substring gate on the payload), so
- * a non-owner probe never deletes the payload — not even transiently. The
- * parse below stays authoritative (a steer text could embed the fragment);
- * a tenant mismatch re-parks so it cannot destroy the owner's recovery.
+ * Owner-gated read of parked leftovers. The store keeps the payload until
+ * a recovered item starts its deterministic next generation, so losing this
+ * HTTP response cannot lose the user's words.
*/
async claim(streamId: string, requester: SteerOwner): Promise<TPendingSteer[]> {
+ return (await this.claimDetailed(streamId, requester, 2)).steers;
+ }
+
+ /** Protocol-aware recovery read used by the status route during the
+ * rolling-upgrade bridge. Legacy stores fall back to destructive v1. */
+ async claimDetailed(
+ streamId: string,
+ requester: SteerOwner,
+ requestedProtocolVersion: GenerationProtocolVersion = 1,
+ ): Promise<ParkedSteerRecovery> {
let raw: string | undefined;
+ let generationProtocolVersion: GenerationProtocolVersion = 1;
try {
- raw = await this.store.claimParkedSteers(streamId, toOwnerFragment(requester.userId));
+ const detailed = await this.store.claimParkedSteersDetailed?.(
+ streamId,
+ requester.userId,
+ requester.tenantId,
+ requestedProtocolVersion,
+ );
+ if (detailed != null) {
+ raw = detailed.payload;
+ generationProtocolVersion = detailed.generationProtocolVersion;
+ } else if (this.store.claimParkedSteersDetailed == null) {
+ raw = await this.store.claimParkedSteers(streamId, requester.userId, requester.tenantId);
+ }
} catch (error) {
logger.warn(`[SteeringLifecycle] Failed to claim leftover steers: ${streamId}`, error);
- return [];
+ return { generationProtocolVersion, steers: [] };
}
if (!raw) {
- return [];
+ return { generationProtocolVersion, steers: [] };
}
try {
const parsed = JSON.parse(raw) as ParkedSteers;
@@ -211,13 +343,19 @@ export class SteeringLifecycle {
parsed.userId === requester.userId &&
(parsed.tenantId == null || parsed.tenantId === requester.tenantId);
if (!ownerMatch) {
- await this.store.parkSteers(streamId, raw);
- return [];
+ return { generationProtocolVersion, steers: [] };
+ }
+ const payloadProtocol = parsed.generationProtocolVersion === 2 ? 2 : 1;
+ if (payloadProtocol !== generationProtocolVersion) {
+ return { generationProtocolVersion, steers: [] };
}
- return Array.isArray(parsed.steers) ? parsed.steers : [];
+ return {
+ generationProtocolVersion,
+ steers: Array.isArray(parsed.steers) ? parsed.steers : [],
+ };
} catch (error) {
logger.warn(`[SteeringLifecycle] Failed to parse leftover steers: ${streamId}`, error);
- return [];
+ return { generationProtocolVersion, steers: [] };
}
}
}
diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.resumeReplay.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.resumeReplay.spec.ts
index 82a726caae..360ed463c2 100644
--- a/packages/api/src/stream/__tests__/GenerationJobManager.resumeReplay.spec.ts
+++ b/packages/api/src/stream/__tests__/GenerationJobManager.resumeReplay.spec.ts
@@ -308,6 +308,37 @@ describe('GenerationJobManager resume replay events', () => {
expect(store.getCollectedUsage(streamId)).toEqual([]);
});
+ test('returns null when the generation is replaced during the resume snapshot', async () => {
+ const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ manager = createManagerWithStore(store);
+ const streamId = `resume-snapshot-replacement-${Date.now()}`;
+ const predecessor = await manager.createJob(streamId, 'user-1', streamId);
+ let releasePeek: (() => void) | undefined;
+ const peekGate = new Promise<void>((resolve) => {
+ releasePeek = resolve;
+ });
+ let signalPeekStarted: (() => void) | undefined;
+ const peekStarted = new Promise<void>((resolve) => {
+ signalPeekStarted = resolve;
+ });
+ const originalPeek = store.peekSteers.bind(store);
+ jest.spyOn(store, 'peekSteers').mockImplementationOnce(async (...args) => {
+ signalPeekStarted?.();
+ await peekGate;
+ return originalPeek(...args);
+ });
+
+ const reading = manager.getResumeState(streamId, predecessor.createdAt);
+ await peekStarted;
+ const replacement = await manager.createJob(streamId, 'user-1', streamId);
+ releasePeek?.();
+
+ await expect(reading).resolves.toBeNull();
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ });
+ });
+
test('rejects a delayed predecessor run step after the stream id is replaced', async () => {
manager = createInMemoryManager();
const streamId = `run-step-delayed-predecessor-${Date.now()}`;
diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
index c8e46cf094..aff86dec24 100644
--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
@@ -2431,12 +2431,18 @@ describe('GenerationJobManager Integration Tests', () => {
* duplicates a dedicated subscriber connection per call. Separate OS
* processes would exercise the same objects over the same Redis.
*/
- describeRedis('Cross-Replica Preempt (Redis, two manager instances)', () => {
+ describeRedis('Cross-Replica Runtime Signals (Redis, multiple manager instances)', () => {
const replicas: GenerationJobManagerClass[] = [];
- function createReplica(): GenerationJobManagerClass {
+ function createReplica(redisSubscriber?: Redis | Cluster): GenerationJobManagerClass {
const manager = new GenerationJobManagerClass();
- manager.configure(createStreamServices({ useRedis: true, redisClient: ioredisClient! }));
+ manager.configure(
+ createStreamServices({
+ useRedis: true,
+ redisClient: ioredisClient!,
+ redisSubscriber,
+ }),
+ );
manager.initialize();
replicas.push(manager);
return manager;
@@ -2479,6 +2485,61 @@ describe('GenerationJobManager Integration Tests', () => {
throw new Error(`Timed out waiting for ${what}`);
}
+ test('a replacement waits for the exact generation owner to acknowledge abort', async () => {
+ const owner = createReplica();
+ const router = createReplica();
+ const streamId = `${testPrefix}-replacement-owner-ack-${Date.now()}`;
+
+ const predecessor = await owner.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const replacement = await router.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ expect(predecessor.abortController.signal.aborted).toBe(true);
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ expect(await router.getJobStore().getJob(streamId)).toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ }, 40000);
+
+ test('a bystander subscriber cannot acknowledge for a disconnected generation owner', async () => {
+ const ownerSubscriber = (ioredisClient as Redis).duplicate();
+ const owner = createReplica(ownerSubscriber);
+ const bystander = createReplica();
+ const router = createReplica();
+ const streamId = `${testPrefix}-replacement-bystander-ack-${Date.now()}`;
+
+ const predecessor = await owner.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const bystanderJob = await bystander.getJob(streamId);
+ const bystanderSubscription = await bystander.subscribe(streamId, () => undefined);
+ expect(bystanderJob?.createdAt).toBe(predecessor.createdAt);
+ expect(bystanderSubscription).not.toBeNull();
+
+ ownerSubscriber.disconnect();
+
+ await expect(
+ router.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('predecessor handoff could not be confirmed');
+
+ expect(predecessor.abortController.signal.aborted).toBe(false);
+ expect(bystanderJob?.abortController.signal.aborted).toBe(true);
+ expect(await router.getJobStore().getJob(streamId)).toMatchObject({
+ status: 'error',
+ error: 'Generation predecessor handoff could not be confirmed',
+ replacedJobs: [
+ expect.objectContaining({ createdAt: predecessor.createdAt, status: 'running' }),
+ ],
+ });
+ bystanderSubscription?.unsubscribe();
+ }, 40000);
+
test('a steer routed to a non-owning replica arms the owner and flips its poll', async () => {
const owner = createReplica();
const router = createReplica();
@@ -2504,15 +2565,37 @@ describe('GenerationJobManager Integration Tests', () => {
*/
expect(seenByRouter?.metadata?.preemptCapable).toBe(true);
+ const enqueued = await router.steering.enqueueVersioned(
+ streamId,
+ {
+ steerId: 'steer-x',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ },
+ true,
+ job.createdAt,
+ );
+ if (typeof enqueued === 'number') {
+ throw new Error(`Unexpected enqueue rejection: ${enqueued}`);
+ }
+
/** The whole point: the owner's level-triggered poll flips from a
* request that was accepted on a different replica. */
await publishUntil(
- () => router.requestPreempt(streamId, 'steer-x', job.createdAt),
+ () =>
+ router.requestPreempt(
+ streamId,
+ enqueued.item.steerId,
+ job.createdAt,
+ enqueued.item.preemptRevision ?? 0,
+ ),
() => owner.isPreemptRequested(streamId),
'the owner to arm',
);
expect(owner.getArmedPreemptIds(streamId)).toContain('steer-x');
+ expect(await router.steering.cancel(streamId, 'steer-x', job.createdAt)).toBe(true);
await publishUntil(
() => router.noteSteersRemoved(streamId, ['steer-x'], job.createdAt),
() => !owner.isPreemptRequested(streamId),
@@ -2603,7 +2686,30 @@ describe('GenerationJobManager Integration Tests', () => {
const router = createReplica();
const streamId = `${testPrefix}-preempt-xreplica-stale-${Date.now()}`;
- const job = await owner.createJob(streamId, 'user-1');
+ const job = await owner.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
+
+ const enqueuePreempt = async (steerId: string) => {
+ const enqueued = await router.steering.enqueueVersioned(
+ streamId,
+ {
+ steerId,
+ text: steerId,
+ userId: 'user-1',
+ createdAt: Date.now(),
+ },
+ true,
+ job.createdAt,
+ );
+ if (typeof enqueued === 'number') {
+ throw new Error(`Unexpected enqueue rejection for ${steerId}: ${enqueued}`);
+ }
+ return enqueued.item;
+ };
+ const controlBefore = await enqueuePreempt('control-before');
+ const stale = await enqueuePreempt('steer-stale');
+ const controlAfter = await enqueuePreempt('control-after');
/**
* A negative assertion cannot be established by waiting: an undelivered
@@ -2614,15 +2720,32 @@ describe('GenerationJobManager Integration Tests', () => {
* behind — proves the stale arm has already had its chance to land.
*/
await publishUntil(
- () => router.requestPreempt(streamId, 'control-before', job.createdAt),
+ () =>
+ router.requestPreempt(
+ streamId,
+ controlBefore.steerId,
+ job.createdAt,
+ controlBefore.preemptRevision ?? 0,
+ ),
() => owner.getArmedPreemptIds(streamId).includes('control-before'),
'the first control arm',
);
- await router.requestPreempt(streamId, 'steer-stale', job.createdAt - 1);
+ await router.requestPreempt(
+ streamId,
+ stale.steerId,
+ job.createdAt - 1,
+ stale.preemptRevision ?? 0,
+ );
await publishUntil(
- () => router.requestPreempt(streamId, 'control-after', job.createdAt),
+ () =>
+ router.requestPreempt(
+ streamId,
+ controlAfter.steerId,
+ job.createdAt,
+ controlAfter.preemptRevision ?? 0,
+ ),
() => owner.getArmedPreemptIds(streamId).includes('control-after'),
'the second control arm',
);
diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
index e1c43cf097..6b9ae43f7f 100644
--- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
@@ -703,7 +703,7 @@ describe('RedisEventTransport', () => {
await manager.destroy();
});
- it('releases a registration that loses initialization without detaching its replacement', async () => {
+ it('resubscribes a replacement after an unexposed registration loses initialization', async () => {
const mockPublisher = createMockPublisher();
const mockSubscriber = createMockSubscriber();
let signalSubscriptionStarted: (() => void) | undefined;
@@ -740,7 +740,8 @@ describe('RedisEventTransport', () => {
);
const replacement = await replacementCreation;
- expect(mockSubscriber.unsubscribe).not.toHaveBeenCalled();
+ expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(1);
+ expect(mockSubscriber.subscribe).toHaveBeenCalledTimes(2);
getMessageHandler(mockSubscriber)(
`stream:{${streamId}}:events`,
@@ -748,7 +749,7 @@ describe('RedisEventTransport', () => {
);
expect(replacement.abortController.signal.aborted).toBe(true);
- expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(1);
+ expect(mockSubscriber.unsubscribe).toHaveBeenCalledTimes(2);
await manager.destroy();
});
@@ -848,10 +849,12 @@ describe('RedisEventTransport', () => {
await expect(emitChunkWithReceipt(transport, 'stream-1', { text: 'World' }, 777)).resolves.toBe(
1,
);
+ expect(mockPublisher.eval.mock.calls[0][12]).toBe('0');
const guardedPublish = mockPublisher.eval.mock.calls[1];
expect(guardedPublish[0]).toContain('local currentCreatedAt = redis.call("HGET", KEYS[2]');
expect(guardedPublish[9]).toBe('777');
expect(guardedPublish[10]).toBe('0');
+ expect(guardedPublish[12]).toBe('1');
expect(JSON.parse(`${guardedPublish[6]}1${guardedPublish[7]}`)).toMatchObject({
type: 'chunk',
data: { text: 'World' },
@@ -864,9 +867,9 @@ describe('RedisEventTransport', () => {
).resolves.toBe(false);
mockPublisher.eval.mockRejectedValue(new Error('publish failed'));
await expect(transport.emitChunk('failed-stream', { text: 'Hello' })).resolves.toBeUndefined();
- await expect(emitChunkWithReceipt(transport, 'failed-stream', { text: 'Hello' })).resolves.toBe(
- false,
- );
+ await expect(
+ emitChunkWithReceipt(transport, 'failed-stream', { text: 'Hello' }),
+ ).resolves.toBeUndefined();
transport.destroy();
});
@@ -895,12 +898,200 @@ describe('RedisEventTransport', () => {
expect(donePublish[9]).toBe('777');
expect(donePublish[10]).toBe('1');
expect(donePublish[11]).toBe('300');
+ expect(donePublish[12]).toBe('0');
expect(errorPublish[9]).toBe('777');
expect(errorPublish[10]).toBe('1');
expect(errorPublish[11]).toBe('300');
+ expect(errorPublish[12]).toBe('0');
expect(chunkPublish[9]).toBe('777');
expect(chunkPublish[10]).toBe('0');
expect(chunkPublish[11]).toBe('300');
+ expect(chunkPublish[12]).toBe('1');
+ expect(chunkPublish[0]).toContain('if ARGV[8] == "1" then');
+ expect(chunkPublish[0]).toContain('currentStatus ~= "running"');
+
+ transport.destroy();
+ });
+
+ it('publishes a receipt-authorized replacement DONE for an old epoch', async () => {
+ const mockPublisher = createMockPublisher();
+ const mockSubscriber = createMockSubscriber();
+ const transport = new RedisEventTransport(
+ mockPublisher as unknown as Redis,
+ mockSubscriber as unknown as Redis,
+ );
+ mockPublisher.eval.mockResolvedValueOnce(0);
+
+ await expect(
+ transport.emitReplacedDoneConfirmed(
+ 'replacement-done',
+ { final: true, reconcile: true },
+ 1234,
+ 'current-create-attempt',
+ ),
+ ).resolves.toBeUndefined();
+
+ const publish = mockPublisher.eval.mock.calls[0];
+ expect(publish[0]).toContain('HGET", KEYS[2], "__creationAttemptId"');
+ expect(publish[0]).toContain('__replacedGenerations');
+ expect(publish[0]).toContain('redis.call("PUBLISH"');
+ expect(publish[1]).toBe(2);
+ expect(publish[2]).toBe('stream:{replacement-done}:seq');
+ expect(publish[3]).toBe('stream:{replacement-done}:job');
+ expect(publish[8]).toBe('1234');
+ expect(publish[9]).toBe('current-create-attempt');
+ expect(JSON.parse(`${publish[5]}0${publish[6]}`)).toMatchObject({
+ type: 'done',
+ generationId: 1234,
+ data: { final: true, reconcile: true },
+ });
+
+ transport.destroy();
+ });
+
+ it('rejects stale replacement receipts and ordinary fenced terminal events', async () => {
+ const mockPublisher = createMockPublisher();
+ const mockSubscriber = createMockSubscriber();
+ const transport = new RedisEventTransport(
+ mockPublisher as unknown as Redis,
+ mockSubscriber as unknown as Redis,
+ );
+
+ mockPublisher.eval.mockResolvedValueOnce(-1);
+ await expect(
+ transport.emitReplacedDoneConfirmed('stale-replacement', { final: true }, 1234, 'wrong'),
+ ).rejects.toThrow('replacement DONE receipt is no longer current');
+ mockPublisher.eval.mockResolvedValueOnce(-1);
+ await expect(transport.emitDone('stale-done', { final: true }, 1234)).rejects.toThrow(
+ 'DONE publication was fenced',
+ );
+ mockPublisher.eval.mockResolvedValueOnce(-1);
+ await expect(transport.emitError('stale-error', 'failed', 1234)).rejects.toThrow(
+ 'error publication was fenced',
+ );
+
+ transport.destroy();
+ });
+
+ it('rejects a confirmed abort with no generation-owner acknowledgement', async () => {
+ jest.useFakeTimers();
+ const mockPublisher = createMockPublisher();
+ const mockSubscriber = createMockSubscriber();
+ const transport = new RedisEventTransport(
+ mockPublisher as unknown as Redis,
+ mockSubscriber as unknown as Redis,
+ );
+ let resolveAbortPublished!: () => void;
+ const abortPublished = new Promise<void>((resolve) => {
+ resolveAbortPublished = resolve;
+ });
+ mockPublisher.publish.mockImplementationOnce(async () => {
+ resolveAbortPublished();
+ return 0;
+ });
+ mockPublisher.eval.mockResolvedValueOnce(0);
+
+ try {
+ const confirmation = transport.emitAbortConfirmed('zero-listener', 1234);
+ await abortPublished;
+ await jest.advanceTimersByTimeAsync(3000);
+ await expect(confirmation).resolves.toBe(false);
+ await expect(
+ transport.emitReplacedDoneConfirmed('zero-listener', { final: true }, 1234, 'attempt'),
+ ).resolves.toBeUndefined();
+ } finally {
+ transport.destroy();
+ jest.useRealTimers();
+ }
+ });
+
+ it('waits for a delayed owner acknowledgement when cluster publish reports zero local receivers', async () => {
+ const mockPublisher = createMockPublisher();
+ const mockSubscriber = createMockSubscriber();
+ const transport = new RedisEventTransport(
+ mockPublisher as unknown as Redis,
+ mockSubscriber as unknown as Redis,
+ );
+ const streamId = 'cluster-zero-local-receivers';
+ const channel = `stream:{${streamId}}:events`;
+ const messageHandler = getMessageHandler(mockSubscriber);
+ let resolveAbortPublished!: (message: { abortRequestId: string }) => void;
+ const abortPublished = new Promise<{ abortRequestId: string }>((resolve) => {
+ resolveAbortPublished = resolve;
+ });
+ mockPublisher.publish.mockImplementation(async (_channel: string, payload: string) => {
+ const message = JSON.parse(payload) as { type: string; abortRequestId: string };
+ if (message.type === 'abort') {
+ resolveAbortPublished(message);
+ }
+ return 0;
+ });
+
+ const confirmation = transport.emitAbortConfirmed(streamId, 1234);
+ const abortMessage = await abortPublished;
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ messageHandler(
+ channel,
+ JSON.stringify({
+ type: 'abort_ack',
+ generationId: 1234,
+ abortRequestId: abortMessage.abortRequestId,
+ }),
+ );
+
+ await expect(confirmation).resolves.toBe(true);
+ transport.destroy();
+ });
+
+ it('correlates an abort acknowledgement to its request and generation', async () => {
+ const mockPublisher = createMockPublisher();
+ const mockSubscriber = createMockSubscriber();
+ const transport = new RedisEventTransport(
+ mockPublisher as unknown as Redis,
+ mockSubscriber as unknown as Redis,
+ );
+ const streamId = 'generation-correlated-abort-ack';
+ const channel = `stream:{${streamId}}:events`;
+ const messageHandler = getMessageHandler(mockSubscriber);
+ let resolveAbortMessage!: (message: { abortRequestId: string }) => void;
+ const abortPublished = new Promise<{ abortRequestId: string }>((resolve) => {
+ resolveAbortMessage = resolve;
+ });
+ mockPublisher.publish.mockImplementation(async (_channel: string, payload: string) => {
+ const message = JSON.parse(payload) as { type: string; abortRequestId: string };
+ if (message.type === 'abort') {
+ resolveAbortMessage(message);
+ }
+ return 4;
+ });
+
+ const confirmation = transport.emitAbortConfirmed(streamId, 1234);
+ const abortMessage = await abortPublished;
+ let settled = false;
+ void confirmation.then(() => {
+ settled = true;
+ });
+
+ messageHandler(
+ channel,
+ JSON.stringify({
+ type: 'abort_ack',
+ generationId: 5678,
+ abortRequestId: abortMessage.abortRequestId,
+ }),
+ );
+ await Promise.resolve();
+ expect(settled).toBe(false);
+
+ messageHandler(
+ channel,
+ JSON.stringify({
+ type: 'abort_ack',
+ generationId: 1234,
+ abortRequestId: abortMessage.abortRequestId,
+ }),
+ );
+ await expect(confirmation).resolves.toBe(true);
transport.destroy();
});
diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts
index 94c68ffe43..d5aa8867b5 100644
--- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts
@@ -150,7 +150,9 @@ describe('RedisEventTransport Integration Tests', () => {
expect(replacement.createdAt).toBe(terminalJob.createdAt + 1);
// A live replacement must reject the predecessor's delayed terminal event.
- await transport.emitDone(streamId, { final: true, stale: true }, terminalJob.createdAt);
+ await expect(
+ transport.emitDone(streamId, { final: true, stale: true }, terminalJob.createdAt),
+ ).rejects.toThrow('fenced by a replacement');
await new Promise((resolve) => setTimeout(resolve, 100));
expect(received).toEqual([finalEvent]);
@@ -165,11 +167,13 @@ describe('RedisEventTransport Integration Tests', () => {
}),
).resolves.toBe(true);
await expect(store.getJob(streamId)).resolves.toBeNull();
- await transport.emitDone(
- streamId,
- { final: true, staleAfterReplacement: true },
- terminalJob.createdAt,
- );
+ await expect(
+ transport.emitDone(
+ streamId,
+ { final: true, staleAfterReplacement: true },
+ terminalJob.createdAt,
+ ),
+ ).rejects.toThrow('fenced by a replacement');
await new Promise((resolve) => setTimeout(resolve, 100));
expect(received).toEqual([finalEvent]);
@@ -310,11 +314,13 @@ describe('RedisEventTransport Integration Tests', () => {
}
})();
expect(claimedReplacement.createdAt).toBe(claimFirstLegacy.createdAt + 1);
- await transport.emitError(
- claimBeforeCreateId,
- 'stale after replacement',
- claimFirstLegacy.createdAt,
- );
+ await expect(
+ transport.emitError(
+ claimBeforeCreateId,
+ 'stale after replacement',
+ claimFirstLegacy.createdAt,
+ ),
+ ).rejects.toThrow('fenced by a replacement');
await new Promise((resolve) => setTimeout(resolve, 100));
expect(claimedEvents).toEqual(['legacy claim won']);
@@ -340,18 +346,22 @@ describe('RedisEventTransport Integration Tests', () => {
}
})();
expect(createFirstReplacement.createdAt).toBe(createFirstLegacy.createdAt + 1000);
- await transport.emitError(
- createBeforeClaimId,
- 'legacy claim lost',
- createFirstLegacy.createdAt,
- );
+ await expect(
+ transport.emitError(
+ createBeforeClaimId,
+ 'legacy claim lost',
+ createFirstLegacy.createdAt,
+ ),
+ ).rejects.toThrow('fenced by a replacement');
await new Promise((resolve) => setTimeout(resolve, 100));
expect(createFirstEvents).toEqual([]);
// With two unknowable pre-marker epochs and no live hash, Redis ordering
// gives the marker to the first claimant and rejects the differing second.
await transport.emitError(competingClaimsId, 'first claimant', 100);
- await transport.emitError(competingClaimsId, 'second claimant', 200);
+ await expect(
+ transport.emitError(competingClaimsId, 'second claimant', 200),
+ ).rejects.toThrow('fenced by a replacement');
await new Promise((resolve) => setTimeout(resolve, 100));
expect(competingEvents).toEqual(['first claimant']);
await expect(
@@ -1458,9 +1468,9 @@ describe('RedisEventTransport Integration Tests', () => {
// Public callers retain Promise<void>; the internal manager capability receives failure
// without creating an unhandled rejection.
await expect(transport.emitChunk(streamId, { data: 'test' })).resolves.toBeUndefined();
- await expect(emitChunkWithReceipt(transport, streamId, { data: 'test' })).resolves.toBe(
- false,
- );
+ await expect(
+ emitChunkWithReceipt(transport, streamId, { data: 'test' }),
+ ).resolves.toBeUndefined();
transport.destroy();
});
@@ -1485,9 +1495,9 @@ describe('RedisEventTransport Integration Tests', () => {
const streamId = `error-prop-incr-${Date.now()}`;
await expect(transport.emitChunk(streamId, { data: 'test' })).resolves.toBeUndefined();
- await expect(emitChunkWithReceipt(transport, streamId, { data: 'test' })).resolves.toBe(
- false,
- );
+ await expect(
+ emitChunkWithReceipt(transport, streamId, { data: 'test' }),
+ ).resolves.toBeUndefined();
expect(mockPublisher.publish).not.toHaveBeenCalled();
transport.destroy();
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
index cbacf0b204..f7ddc2cb8c 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
@@ -32,13 +32,18 @@ async function waitFor(predicate: () => boolean): Promise<void> {
function jobHashFromCreationCall(call: unknown[]): Record<string, string> {
const keyCount = Number(call[1]);
- const fields = call.slice(5 + keyCount);
- return Object.fromEntries(
+ // JOB_CREATE_LUA receives twelve scalar arguments before its HSET pairs.
+ const fields = call.slice(14 + keyCount);
+ const hash = Object.fromEntries(
Array.from({ length: fields.length / 2 }, (_, index) => [
String(fields[index * 2]),
String(fields[index * 2 + 1]),
]),
);
+ if (hash.generationProtocolVersion === '2') {
+ hash.checkpointNamespace = hash.createdAt;
+ }
+ return hash;
}
describe('RedisJobStore', () => {
@@ -67,21 +72,25 @@ describe('RedisJobStore', () => {
chunksKey,
runStepsKey,
steersKey,
+ claimedSteersKey,
parkedSteersKey,
generationEpochKey,
+ receiptsKey,
+ receiptOrderKey,
from,
actionId,
createdAt,
] = evalTransition.mock.calls[0];
expect(script).toContain('HGET", KEYS[1], "createdAt"');
- expect(script).toContain('redis.call("DEL", KEYS[5])');
- expect(script).toContain('redis.call("SET", KEYS[6]');
+ expect(script).toContain('redis.call("DEL", KEYS[5], KEYS[6])');
+ expect(script).toContain('redis.call("SET", KEYS[7]');
expect(script).toContain(
- 'redis.call("SET", KEYS[7], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
+ 'redis.call("SET", KEYS[8], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
);
expect(script.indexOf('local ownerUserId')).toBeLessThan(
script.indexOf('redis.call("EXPIRE", KEYS[1], ttl)'),
);
+ expect(script).toContain('if currentTtl > ttl then ttl = currentTtl');
expect([
keyCount,
jobKey,
@@ -89,26 +98,65 @@ describe('RedisJobStore', () => {
chunksKey,
runStepsKey,
steersKey,
+ claimedSteersKey,
parkedSteersKey,
generationEpochKey,
+ receiptsKey,
+ receiptOrderKey,
from,
actionId,
createdAt,
]).toEqual([
- 7,
+ 10,
'stream:{stream-epoch}:job',
'stream:{stream-epoch}:seq',
'stream:{stream-epoch}:chunks',
'stream:{stream-epoch}:runsteps',
'stream:{stream-epoch}:steers',
+ 'stream:{stream-epoch}:steers-claimed',
'stream:{stream-epoch}:parked',
'stream:{stream-epoch}:generation-epoch',
+ 'stream:{stream-epoch}:steer-receipts',
+ 'stream:{stream-epoch}:steer-receipt-order',
'running',
'',
'123456',
]);
});
+ test('retains a terminal persistence barrier when completed TTL is zero', async () => {
+ const evalTransition = jest.fn().mockResolvedValue(0);
+ const redis = {
+ isCluster: true,
+ eval: evalTransition,
+ hgetall: jest.fn().mockResolvedValue({
+ streamId: 'stream-terminal-barrier',
+ userId: 'user-1',
+ status: 'running',
+ createdAt: '100',
+ syncSent: '0',
+ }),
+ } as unknown as Cluster;
+ const store = new RedisJobStore(redis, { completedTtl: 0 });
+
+ await expect(
+ store.transitionStatus('stream-terminal-barrier', {
+ from: 'running',
+ to: 'aborted',
+ expectCreatedAt: 100,
+ patch: {
+ completedAt: 200,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: 200,
+ },
+ }),
+ ).resolves.toBe(false);
+
+ // ARGV[4] is the terminal job TTL. The persistence owner/recovery path
+ // gets five minutes even when ordinary completed records are immediate.
+ expect(evalTransition.mock.calls[0][15]).toBe('300');
+ });
+
test('retains the generation epoch beyond the paused job TTL', async () => {
const evalTransition = jest.fn().mockResolvedValue(0);
const redis = {
@@ -125,11 +173,11 @@ describe('RedisJobStore', () => {
const transitionCall = evalTransition.mock.calls[0];
expect(transitionCall[0]).toContain(
- 'redis.call("SET", KEYS[7], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
+ 'redis.call("SET", KEYS[8], currentCreatedAt, "EX", ttl + generationEpochGraceTtl)',
);
- expect(transitionCall[12]).toBe('4321');
- expect(transitionCall[13]).toBe('0');
- expect(transitionCall[17]).toBe('300');
+ expect(transitionCall[15]).toBe('4621');
+ expect(transitionCall[16]).toBe('0');
+ expect(transitionCall[20]).toBe('300');
});
test('seeds the guarded epoch when reaping a legacy job without a marker', async () => {
@@ -160,15 +208,18 @@ describe('RedisJobStore', () => {
}
const reapCall = evalCommand.mock.calls[0];
- expect(reapCall[0]).toContain('redis.call("SET", KEYS[6], ARGV[1], "EX", tonumber(ARGV[5]))');
+ expect(reapCall[0]).toContain('redis.call("SET", KEYS[7], ARGV[1], "EX", tonumber(ARGV[5]))');
expect(reapCall.slice(1)).toEqual([
- 6,
+ 9,
'stream:{legacy-reap}:job',
'stream:{legacy-reap}:chunks',
'stream:{legacy-reap}:runsteps',
'stream:{legacy-reap}:steers',
+ 'stream:{legacy-reap}:steers-claimed',
'stream:{legacy-reap}:parked',
'stream:{legacy-reap}:generation-epoch',
+ 'stream:{legacy-reap}:steer-receipts',
+ 'stream:{legacy-reap}:steer-receipt-order',
'1',
'100000',
'60000',
@@ -237,6 +288,7 @@ describe('RedisJobStore', () => {
promptTokens: 0,
discoveredTools: [],
preemptCapable: true,
+ generationProtocolVersion: 2,
});
/**
@@ -246,6 +298,8 @@ describe('RedisJobStore', () => {
* degrading to ordinary steering in every Redis deployment.
*/
expect(job.preemptCapable).toBe(true);
+ expect(job.generationProtocolVersion).toBe(2);
+ expect(job.checkpointNamespace).toBe(String(job.createdAt));
expect(job).toMatchObject({
streamId: 'stream-metadata',
@@ -296,24 +350,33 @@ describe('RedisJobStore', () => {
await store.createJob('stream-replacement', 'user-1');
const [script, keyCount, ...args] = evalJobCreation.mock.calls[0];
- expect(script).toContain('local retainedEpoch = tonumber(redis.call("GET", KEYS[6]))');
+ expect(script).toContain(
+ 'local retainedEpochRaw = redis.call("GET", KEYS[7]) local retainedEpoch = tonumber(retainedEpochRaw)',
+ );
+ expect(script).toContain('if retainedEpochRaw and not isSafeEpoch(retainedEpoch) then');
expect(script).toContain(
'if previousCreatedAt and previousCreatedAt >= createdAt then createdAt = previousCreatedAt + 1 end',
);
+ expect(script).toContain('"checkpointNamespace", tostring(createdAt)');
expect(script).toContain('redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5])');
expect(script).toContain(
- 'redis.call("SET", KEYS[6], tostring(createdAt), "EX", ttl + generationEpochGraceTtl)',
+ 'redis.call("SET", KEYS[7], tostring(createdAt), "EX", ttl + generationEpochGraceTtl)',
);
- expect([keyCount, ...args.slice(0, 6)]).toEqual([
- 6,
+ expect([keyCount, ...args.slice(0, 10)]).toEqual([
+ 10,
'stream:{stream-replacement}:job',
'stream:{stream-replacement}:chunks',
'stream:{stream-replacement}:runsteps',
'stream:{stream-replacement}:steers',
+ 'stream:{stream-replacement}:steers-claimed',
'stream:{stream-replacement}:parked',
'stream:{stream-replacement}:generation-epoch',
+ 'stream:{stream-replacement}:steer-receipts',
+ 'stream:{stream-replacement}:steer-receipt-order',
+ 'stream:{stream-replacement}:job',
]);
- expect(args[8]).toBe('300');
+ expect(args[12]).toBe('300');
+ expect(args[13]).toBe('300');
expect(store.getCollectedUsage('stream-replacement')).toEqual([]);
});
@@ -605,11 +668,12 @@ describe('RedisJobStore', () => {
const deleteCall = evalCommand.mock.calls[1];
expect(deleteCall[0]).toContain('HGET", KEYS[1], "createdAt"');
expect(deleteCall.slice(1)).toEqual([
- 4,
+ 5,
'stream:{stream-guarded}:job',
'stream:{stream-guarded}:chunks',
'stream:{stream-guarded}:runsteps',
'stream:{stream-guarded}:steers',
+ 'stream:{stream-guarded}:steers-claimed',
'100',
'0',
]);
@@ -627,18 +691,33 @@ describe('RedisJobStore', () => {
await store.appendChunk('stream-chunk-guarded', event, 100);
const appendCall = evalCommand.mock.calls[0];
- expect(appendCall[0]).toContain('redis.call("HGET", KEYS[2], "createdAt") ~= ARGV[3]');
+ expect(appendCall[0]).toContain('currentCreatedAt ~= ARGV[3]');
+ expect(appendCall[0]).toContain('currentStatus ~= "running"');
+ expect(appendCall[0]).toContain('currentStatus ~= "requires_action"');
+ expect(appendCall[0]).toContain('retainedEpoch ~= currentCreatedAt');
+ expect(appendCall[0]).toContain('redis.call("EXPIRE", KEYS[8], epochTarget)');
expect(appendCall.slice(1)).toEqual([
- 2,
+ 8,
'stream:{stream-chunk-guarded}:chunks',
'stream:{stream-chunk-guarded}:job',
+ 'stream:{stream-chunk-guarded}:steer-receipts',
+ 'stream:{stream-chunk-guarded}:steer-receipt-order',
+ 'stream:{stream-chunk-guarded}:steers-claimed',
+ 'stream:{stream-chunk-guarded}:steers',
+ 'stream:{stream-chunk-guarded}:parked',
+ 'stream:{stream-chunk-guarded}:generation-epoch',
JSON.stringify(event),
- '1200',
+ '1500',
'100',
+ '',
+ '',
+ expect.any(String),
+ '300',
+ '300',
]);
});
- test('guards run-step saves with the expected creation epoch inside Redis Lua', async () => {
+ test('guards run-step saves with the expected epoch and active status inside Redis Lua', async () => {
const evalCommand = jest.fn().mockResolvedValue(0);
const redis = {
isCluster: true,
@@ -650,13 +729,15 @@ describe('RedisJobStore', () => {
await store.saveRunSteps?.('stream-runstep-guarded', runSteps as never, 100);
const saveCall = evalCommand.mock.calls[0];
- expect(saveCall[0]).toContain('redis.call("HGET", KEYS[2], "createdAt") ~= ARGV[3]');
+ expect(saveCall[0]).toContain('currentCreatedAt ~= ARGV[3]');
+ expect(saveCall[0]).toContain('currentStatus ~= "running"');
+ expect(saveCall[0]).toContain('currentStatus ~= "requires_action"');
expect(saveCall.slice(1)).toEqual([
2,
'stream:{stream-runstep-guarded}:runsteps',
'stream:{stream-runstep-guarded}:job',
JSON.stringify(runSteps),
- '1200',
+ '1500',
'100',
]);
});
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
index 2dabfb47d7..af4edb8203 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
@@ -2,6 +2,12 @@ import { StandardGraph } from '@librechat/agents';
import { StepTypes } from 'librechat-data-provider';
import type { Agents } from 'librechat-data-provider';
import type { Redis, Cluster } from 'ioredis';
+import type { SteerQueueItem, SteerReceipt } from '../interfaces/IJobStore';
+import {
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ STEER_ENQUEUE_RECEIPT_FULL,
+} from '../interfaces/IJobStore';
+import { clearRedisTestPrefix } from './helpers/redis';
/** Suppress winston Console transport output (survives jest.resetModules) */
jest.spyOn(console, 'log').mockImplementation();
@@ -45,6 +51,9 @@ describe('RedisJobStore Integration Tests', () => {
process.env.REDIS_KEY_PREFIX = testPrefix;
process.env.REDIS_PING_INTERVAL = '0';
process.env.REDIS_RETRY_MAX_ATTEMPTS = '5';
+ // This suite exercises the receipt-safe current behavior. Rollout-specific
+ // v1 defaults and mixed-client downgrade paths live in protocolRollout.
+ process.env.GENERATION_PROTOCOL_VERSION = '2';
jest.resetModules();
@@ -62,14 +71,9 @@ describe('RedisJobStore Integration Tests', () => {
return;
}
- // Clean up all test keys (delete individually for cluster compatibility)
try {
- const keys = await ioredisClient.keys(`${testPrefix}*`);
- // Also clean up stream keys which use hash tags
- const streamKeys = await ioredisClient.keys(`stream:*`);
- const allKeys = [...keys, ...streamKeys];
- // Delete individually to avoid CROSSSLOT errors in cluster mode
- await Promise.all(allKeys.map((key) => ioredisClient!.del(key)));
+ const keyPrefix = String(ioredisClient.options.keyPrefix ?? '');
+ await clearRedisTestPrefix(ioredisClient, keyPrefix);
} catch (error) {
console.warn('Error cleaning up test keys:', error);
}
@@ -125,6 +129,45 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
+ test('returns the exact predecessor captured by the atomic replacement script', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `test-replaced-owner-${Date.now()}`;
+ const predecessor = await store.createJob(streamId, 'user-1', streamId);
+ await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: predecessor.createdAt,
+ });
+
+ const replacement = await store.createJob(streamId, 'user-1', streamId);
+
+ expect(predecessor.checkpointNamespace).toBe(String(predecessor.createdAt));
+ expect(replacement.checkpointNamespace).toBe(String(replacement.createdAt));
+ expect(replacement.checkpointNamespace).not.toBe(predecessor.checkpointNamespace);
+ expect(replacement.replacedJob).toEqual({
+ createdAt: predecessor.createdAt,
+ status: 'requires_action',
+ conversationId: streamId,
+ });
+ // The receipt is durably reconstructed for lost-reply recovery, but it
+ // remains non-enumerable so ordinary job serializers cannot expose it.
+ const durableReplacement = await store.getJob(streamId);
+ expect((durableReplacement as typeof replacement).replacedJob).toEqual(
+ replacement.replacedJob,
+ );
+ expect(Object.getOwnPropertyDescriptor(durableReplacement!, 'replacedJob')).toMatchObject({
+ enumerable: false,
+ });
+ expect(Object.keys(durableReplacement!)).not.toContain('replacedJob');
+ await store.destroy();
+ });
+
test('should update job status', async () => {
if (!ioredisClient) {
return;
@@ -146,6 +189,62 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
+ test('atomically rewrites a requires_action barrier in the same status', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `same-status-pause-${Date.now()}`;
+ const job = await store.createJob(streamId, 'user-1', streamId);
+ const first = buildPendingAction(`${streamId}-first`);
+ const second = buildPendingAction(`${streamId}-second`);
+
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingAction: first,
+ pendingActionId: first.actionId,
+ terminalPersistenceStartedAt: 1234,
+ },
+ }),
+ ).resolves.toBe(true);
+
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'requires_action',
+ to: 'requires_action',
+ expectActionId: first.actionId,
+ expectCreatedAt: job.createdAt,
+ patch: { pendingAction: second, pendingActionId: second.actionId },
+ clear: ['terminalPersistenceStartedAt'],
+ }),
+ ).resolves.toBe(true);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ status: 'requires_action',
+ pendingActionId: second.actionId,
+ pendingAction: { actionId: second.actionId },
+ });
+ expect((await store.getJob(streamId))?.terminalPersistenceStartedAt).toBeUndefined();
+
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'requires_action',
+ to: 'requires_action',
+ expectActionId: first.actionId,
+ expectCreatedAt: job.createdAt,
+ patch: { pendingAction: first, pendingActionId: first.actionId },
+ }),
+ ).resolves.toBe(false);
+ expect((await store.getJob(streamId))?.pendingActionId).toBe(second.actionId);
+ await store.destroy();
+ });
+
test('should delete job and related data', async () => {
if (!ioredisClient) {
return;
@@ -374,9 +473,12 @@ describe('RedisJobStore Integration Tests', () => {
expect((await store.peekSteers(streamId)).map((steer) => steer.steerId)).toEqual([
'replacement-steer',
]);
- await expect(
- store.claimParkedSteers(streamId, `"userId":"${userId}"`),
- ).resolves.toBeUndefined();
+ const parked = await store.claimParkedSteers(streamId, userId);
+ expect(parked).toBeDefined();
+ expect(JSON.parse(parked as string)).toMatchObject({
+ userId,
+ steers: [{ steerId: 'old-steer', text: 'do not leak this' }],
+ });
expect(await ioredisClient.smembers('stream:running')).toContain(streamId);
expect(await store.getActiveJobIdsByUser(userId)).toContain(streamId);
} finally {
@@ -388,6 +490,169 @@ describe('RedisJobStore Integration Tests', () => {
});
});
+ describe('Steer queue arm (in-place escalation)', () => {
+ test('arms a queued steer in place, preserving FIFO order and every field', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `arm-steer-${Date.now()}`;
+
+ try {
+ const job = await store.createJob(streamId, 'user-1', streamId, undefined, {
+ preemptCapable: true,
+ });
+ await store.enqueueSteer(streamId, {
+ steerId: 'first',
+ text: 'earlier instruction',
+ userId: 'user-1',
+ createdAt: 1,
+ files: [{ file_id: 'f1' }] as never,
+ });
+ await store.enqueueSteer(streamId, {
+ steerId: 'second',
+ text: 'later instruction',
+ userId: 'user-1',
+ createdAt: 2,
+ });
+
+ await expect(store.armSteer(streamId, 'first', job.createdAt)).resolves.toBe('armed');
+
+ const queue = await store.peekSteers(streamId);
+ expect(queue.map((item) => item.steerId)).toEqual(['first', 'second']);
+ /** Whole-item decode/patch/encode: nothing but the flag changes. */
+ expect(queue[0]).toMatchObject({
+ steerId: 'first',
+ text: 'earlier instruction',
+ userId: 'user-1',
+ createdAt: 1,
+ preempt: true,
+ });
+ expect(queue[0].files).toEqual([{ file_id: 'f1' }]);
+ expect(queue[1].preempt).toBeUndefined();
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('matches the decoded steer id exactly instead of a caller-built JSON fragment', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `arm-steer-exact-id-${Date.now()}`;
+
+ try {
+ const job = await store.createJob(streamId, 'user-1', streamId, undefined, {
+ preemptCapable: true,
+ });
+ await store.enqueueSteer(streamId, {
+ steerId: 'target',
+ text: 'matching payload',
+ userId: 'user-1',
+ createdAt: 1,
+ });
+
+ /** The old optimization embedded this value in a JSON search fragment,
+ * which matched the real item's adjacent `steerId` + `text` fields. */
+ await expect(
+ store.armSteer(streamId, 'target","text":"matching payload', job.createdAt),
+ ).resolves.toBe('missing');
+ const queue = await store.peekSteers(streamId, job.createdAt);
+ expect(queue).toHaveLength(1);
+ expect(queue[0].steerId).toBe('target');
+ expect(queue[0].preempt).toBeUndefined();
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('refuses to arm a capable job after it enters requires_action', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `arm-steer-paused-${Date.now()}`;
+
+ try {
+ const job = await store.createJob(streamId, 'user-1', streamId, undefined, {
+ preemptCapable: true,
+ });
+ await store.enqueueSteer(streamId, {
+ steerId: 'kept',
+ text: 'wait until the run resumes',
+ userId: 'user-1',
+ createdAt: 1,
+ files: [{ file_id: 'f1' }] as never,
+ });
+ const before = await store.peekSteers(streamId, job.createdAt);
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBe(true);
+
+ await expect(store.armSteer(streamId, 'kept', job.createdAt)).resolves.toBe('missing');
+ await expect(store.peekSteers(streamId, job.createdAt)).resolves.toEqual(before);
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('refuses a missing steer, a stale generation, and a closed queue', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `arm-steer-guards-${Date.now()}`;
+
+ try {
+ const job = await store.createJob(streamId, 'user-1', streamId);
+ await store.enqueueSteer(streamId, {
+ steerId: 'kept',
+ text: 'still waiting',
+ userId: 'user-1',
+ createdAt: 1,
+ });
+
+ await expect(store.armSteer(streamId, 'absent', job.createdAt)).resolves.toBe('missing');
+ await expect(store.armSteer(streamId, 'kept', job.createdAt + 999)).resolves.toBe(
+ 'missing',
+ );
+ /** Live-capability predicate: the job above carries no preemptCapable,
+ * so an otherwise-valid arm answers `incapable` and leaves the item. */
+ await expect(store.armSteer(streamId, 'kept', job.createdAt)).resolves.toBe('incapable');
+ expect((await store.peekSteers(streamId))[0].preempt).toBeUndefined();
+
+ /** `enqueueSteer` refuses once closed, so plant a raw item directly to
+ * exercise the closed guard with something findable in the list. */
+ await store.closeAndDrainSteers(streamId, job.createdAt);
+ await ioredisClient.rpush(
+ `stream:{${streamId}}:steers`,
+ JSON.stringify({
+ steerId: 'kept',
+ text: 'still waiting',
+ userId: 'user-1',
+ createdAt: 1,
+ }),
+ );
+ await expect(store.armSteer(streamId, 'kept', job.createdAt)).resolves.toBe('missing');
+ } finally {
+ await store.destroy();
+ }
+ });
+ });
+
describe('Requires Action Status Tracking', () => {
test('should count requires_action jobs and remove them from the running set', async () => {
if (!ioredisClient) {
@@ -542,6 +807,217 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
+ test('releasing a pause-persistence barrier preserves an explicit long approval TTL', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient, { runningTtl: 60 });
+ await store.initialize();
+
+ const streamId = `long-pause-barrier-ttl-${Date.now()}`;
+ const jobKey = `stream:{${streamId}}:job`;
+ const chunkKey = `stream:{${streamId}}:chunks`;
+ const job = await store.createJob(streamId, 'user-1', streamId);
+ await store.appendChunk(
+ streamId,
+ { event: 'on_message_delta', data: { text: 'before approval' } },
+ job.createdAt,
+ );
+ const action = {
+ ...buildPendingAction(streamId),
+ expiresAt: Date.now() + 3 * 24 * 60 * 60 * 1000,
+ };
+ const barrierId = `pause-persistence:${action.actionId}`;
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingAction: action,
+ pendingActionId: barrierId,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: Date.now(),
+ },
+ }),
+ ).resolves.toBe(true);
+
+ const jobTtlBefore = await ioredisClient.ttl(jobKey);
+ const chunkTtlBefore = await ioredisClient.ttl(chunkKey);
+ expect(jobTtlBefore).toBeGreaterThan(2 * 24 * 60 * 60);
+ expect(chunkTtlBefore).toBeGreaterThan(2 * 24 * 60 * 60);
+
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'requires_action',
+ to: 'requires_action',
+ expectActionId: barrierId,
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingActionId: action.actionId,
+ terminalPersistencePending: false,
+ },
+ clear: ['terminalPersistenceStartedAt'],
+ }),
+ ).resolves.toBe(true);
+
+ expect(await ioredisClient.ttl(jobKey)).toBeGreaterThanOrEqual(jobTtlBefore - 2);
+ expect(await ioredisClient.ttl(chunkKey)).toBeGreaterThanOrEqual(chunkTtlBefore - 2);
+
+ await store.destroy();
+ });
+
+ test('failed pause persistence atomically terminalizes before a waiting Redis resume', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const [{ RedisJobStore }, { InMemoryEventTransport }, { GenerationJobManagerClass }] =
+ await Promise.all([
+ import('../implementations/RedisJobStore'),
+ import('../implementations/InMemoryEventTransport'),
+ import('../GenerationJobManager'),
+ ]);
+ const store = new RedisJobStore(ioredisClient, { runningTtl: 60 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+
+ try {
+ const streamId = `failed-pause-persistence-race-${Date.now()}`;
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const action = buildPendingAction(streamId);
+ const waitingSteer = {
+ steerId: 'redis-steer-waiting-on-failed-pause',
+ text: 'preserve the Redis steer',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ };
+ await expect(manager.steering.enqueue(streamId, waitingSteer, job.createdAt)).resolves.toBe(
+ 1,
+ );
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+
+ let resumeSettled = false;
+ const resuming = manager.approvals
+ .resolve(streamId, action.actionId, undefined, job.createdAt)
+ .then((result) => {
+ resumeSettled = true;
+ return result;
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(resumeSettled).toBe(false);
+
+ await expect(
+ manager.failPausePersistence(
+ streamId,
+ action.actionId,
+ 'Redis paused response was not persisted',
+ job.createdAt,
+ ),
+ ).resolves.toBe(true);
+ await expect(resuming).resolves.toBe(false);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ status: 'error',
+ error: 'Redis paused response was not persisted',
+ });
+ const failedStoredJob = await store.getJob(streamId);
+ expect(failedStoredJob?.pendingAction).toBeUndefined();
+ expect(failedStoredJob?.pendingActionId).toBeUndefined();
+ expect(failedStoredJob?.terminalPersistencePending).toBeUndefined();
+ expect(failedStoredJob?.terminalPersistenceStartedAt).toBeUndefined();
+ await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([]);
+ await expect(manager.steering.claim(streamId, { userId: 'user-1' })).resolves.toEqual([
+ expect.objectContaining({
+ steerId: waitingSteer.steerId,
+ text: waitingSteer.text,
+ }),
+ ]);
+ } finally {
+ await manager.destroy();
+ }
+ });
+
+ test('relays a Redis store-only pause timeout into the matching attached runtime once', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+ const [{ RedisJobStore }, { InMemoryEventTransport }, { GenerationJobManagerClass }] =
+ await Promise.all([
+ import('../implementations/RedisJobStore'),
+ import('../implementations/InMemoryEventTransport'),
+ import('../GenerationJobManager'),
+ ]);
+ const store = new RedisJobStore(ioredisClient, { runningTtl: 60 });
+ const transport = new InMemoryEventTransport();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: transport,
+ isRedis: true,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+
+ try {
+ const streamId = 'redis-remote-pause-timeout';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const action = buildPendingAction(streamId);
+ const onError = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+ const broadcast = jest.spyOn(transport, 'emitError');
+
+ now.mockReturnValue(31_001);
+ // Model cleanup on a different replica: it has the shared store but no
+ // access to this manager's runtime or attached local subscriber.
+ await store.cleanup();
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ status: 'error',
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ });
+ expect(job.abortController.signal.aborted).toBe(false);
+ expect(onError).not.toHaveBeenCalled();
+
+ const managerWithCleanup = manager as unknown as { cleanup(): Promise<void> };
+ await managerWithCleanup.cleanup();
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledTimes(1);
+ expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
+ expect(broadcast).toHaveBeenCalledWith(
+ streamId,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ job.createdAt,
+ );
+
+ await managerWithCleanup.cleanup();
+ expect(onError).toHaveBeenCalledTimes(1);
+ subscription?.unsubscribe();
+ } finally {
+ await manager.destroy();
+ now.mockRestore();
+ }
+ });
+
test('saveRunSteps preserves a paused job’s extended TTL (does not reset to running)', async () => {
if (!ioredisClient) {
return;
@@ -618,7 +1094,7 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
- test('appendChunk keeps a normally-running job on the short running TTL (no inflation)', async () => {
+ test('appendChunk keeps a normally-running job on the bounded running storage TTL', async () => {
if (!ioredisClient) {
return;
}
@@ -630,10 +1106,8 @@ describe('RedisJobStore Integration Tests', () => {
const streamId = `running-no-inflate-${Date.now()}`;
const chunkKey = `stream:{${streamId}}:chunks`;
- // A normal running job: the job key carries the running TTL (set by createJob),
- // NOT the long approval window. appendChunk must settle the chunk TTL on the
- // short running TTL — never max it against the job key — so a live stream is
- // not inflated to the 24h approval window.
+ // A normal running job uses the configured running TTL plus the five-minute
+ // publication grace window, not the much longer approval window.
await store.createJob(streamId, 'user-1', streamId);
await store.appendChunk(streamId, {
event: 'on_message_delta',
@@ -641,8 +1115,39 @@ describe('RedisJobStore Integration Tests', () => {
});
const ttl = await ioredisClient.ttl(chunkKey);
- expect(ttl).toBeGreaterThan(0);
- expect(ttl).toBeLessThanOrEqual(60);
+ expect(ttl).toBeGreaterThan(300);
+ expect(ttl).toBeLessThanOrEqual(360);
+
+ await store.destroy();
+ });
+
+ test('appendChunk refreshes the retained generation epoch beyond the live job TTL', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient, { runningTtl: 60 });
+ await store.initialize();
+
+ const streamId = `chunk-epoch-refresh-${Date.now()}`;
+ const jobKey = `stream:{${streamId}}:job`;
+ const epochKey = `stream:{${streamId}}:generation-epoch`;
+ const job = await store.createJob(streamId, 'user-1', streamId);
+ await ioredisClient.expire(epochKey, 1);
+
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_message_delta', data: { text: 'still alive' } },
+ job.createdAt,
+ ),
+ ).resolves.toBe(true);
+
+ const jobTtl = await ioredisClient.ttl(jobKey);
+ const epochTtl = await ioredisClient.ttl(epochKey);
+ expect(await ioredisClient.get(epochKey)).toBe(String(job.createdAt));
+ expect(epochTtl).toBeGreaterThan(jobTtl + 250);
await store.destroy();
});
@@ -1297,12 +1802,12 @@ describe('RedisJobStore Integration Tests', () => {
data: { id: 'step-1', type: 'text', text: 'test' },
});
- // Check that TTL was set on the stream key
+ // The storage TTL includes the five-minute terminal-publication grace.
// Note: ioredis client has keyPrefix, so we use the key WITHOUT the prefix
// Key uses hash tag format: stream:{streamId}:chunks
const ttl = await ioredisClient.ttl(`stream:{${streamId}}:chunks`);
- expect(ttl).toBeGreaterThan(0);
- expect(ttl).toBeLessThanOrEqual(60);
+ expect(ttl).toBeGreaterThan(300);
+ expect(ttl).toBeLessThanOrEqual(360);
await store.destroy();
});
@@ -1372,7 +1877,7 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
- test('same-stream replacement transfers active membership to the new owner', async () => {
+ test('same-stream replacement cannot transfer active membership to a different owner', async () => {
if (!ioredisClient) {
return;
}
@@ -1384,11 +1889,17 @@ describe('RedisJobStore Integration Tests', () => {
const streamId = `owner-replacement-${Date.now()}`;
const oldUserId = `old-owner-${Date.now()}`;
const newUserId = `new-owner-${Date.now()}`;
- await store.createJob(streamId, oldUserId, streamId);
- await store.createJob(streamId, newUserId, streamId);
+ const original = await store.createJob(streamId, oldUserId, streamId);
+ await expect(store.createJob(streamId, newUserId, streamId)).rejects.toThrow(
+ 'Generation job owner mismatch',
+ );
- await expect(store.getActiveJobIdsByUser(oldUserId)).resolves.not.toContain(streamId);
- await expect(store.getActiveJobIdsByUser(newUserId)).resolves.toContain(streamId);
+ await expect(store.getActiveJobIdsByUser(oldUserId)).resolves.toContain(streamId);
+ await expect(store.getActiveJobIdsByUser(newUserId)).resolves.not.toContain(streamId);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ userId: oldUserId,
+ createdAt: original.createdAt,
+ });
await store.destroy();
});
@@ -2130,8 +2641,8 @@ describe('RedisJobStore Integration Tests', () => {
const chunkKey = `stream:{${streamId}}:chunks`;
const ttl = await ioredisClient.ttl(chunkKey);
- expect(ttl).toBeGreaterThan(0);
- expect(ttl).toBeLessThanOrEqual(120);
+ expect(ttl).toBeGreaterThan(300);
+ expect(ttl).toBeLessThanOrEqual(420);
await store.destroy();
});
@@ -2163,16 +2674,17 @@ describe('RedisJobStore Integration Tests', () => {
const reducedTtl = await ioredisClient.ttl(chunkKey);
expect(reducedTtl).toBeLessThanOrEqual(30);
- // Append another chunk - TTL should be refreshed back to running TTL
+ // Append another chunk - TTL should be refreshed to the running storage
+ // window (configured running TTL plus publication grace).
await store.appendChunk(streamId, {
event: 'on_message_delta',
data: { id: 'step-1', type: 'text', text: 'second' },
});
const ttl2 = await ioredisClient.ttl(chunkKey);
- // Should be refreshed to ~120, not still ~30
+ // Should be refreshed to ~420, not still ~30.
expect(ttl2).toBeGreaterThan(30);
- expect(ttl2).toBeLessThanOrEqual(120);
+ expect(ttl2).toBeLessThanOrEqual(420);
await store.destroy();
});
@@ -2319,7 +2831,7 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
- test('createJob clears steers inherited from a replaced job', async () => {
+ test('terminal CAS atomically returns and parks claimed plus queued steers', async () => {
if (!ioredisClient) {
return;
}
@@ -2328,22 +2840,71 @@ describe('RedisJobStore Integration Tests', () => {
const store = new RedisJobStore(ioredisClient);
await store.initialize();
- const streamId = `steer-replace-${Date.now()}`;
- const predecessor = await store.createJob(streamId, 'steer-user', streamId);
- await store.enqueueSteer(streamId, buildSteer('s1', 'old run steer'));
+ const streamId = `steer-terminal-drain-${Date.now()}`;
+ const userId = 'terminal-drain-user';
+ const job = await store.createJob(streamId, userId, streamId);
+ const claimed = buildSteer('claimed-before-abort', 'claimed first');
+ const queued = buildSteer('queued-before-abort', 'queued second');
+ await store.enqueueSteer(streamId, claimed);
+ await expect(store.drainSteers(streamId, job.createdAt)).resolves.toEqual([claimed]);
+ await store.enqueueSteer(streamId, queued);
- const replacement = await store.createJob(streamId, 'steer-user', streamId);
- await store.enqueueSteer(streamId, buildSteer('s2', 'replacement steer'));
+ await expect(
+ store.transitionStatusAndDrainSteers(streamId, {
+ from: 'running',
+ to: 'aborted',
+ expectCreatedAt: job.createdAt,
+ patch: { completedAt: Date.now() },
+ }),
+ ).resolves.toEqual([claimed, queued]);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({ status: 'aborted' });
+ await expect(store.peekSteers(streamId, job.createdAt)).resolves.toEqual([]);
+ await expect(store.peekClaimedSteers(streamId, job.createdAt)).resolves.toEqual([]);
+ const parked = await store.claimParkedSteers(streamId, userId);
+ expect(JSON.parse(parked as string)).toMatchObject({
+ userId,
+ steers: [
+ { steerId: claimed.steerId, text: claimed.text },
+ { steerId: queued.steerId, text: queued.text },
+ ],
+ });
- expect(await store.peekSteers(streamId, predecessor.createdAt)).toEqual([]);
- expect(
- (await store.peekSteers(streamId, replacement.createdAt)).map((steer) => steer.text),
- ).toEqual(['replacement steer']);
+ await expect(
+ store.transitionStatusAndDrainSteers(streamId, {
+ from: 'running',
+ to: 'aborted',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBeNull();
await store.destroy();
});
- test('pausing for review extends the steers key TTL to the approval window', async () => {
+ test('createJob clears steers inherited from a replaced job', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+
+ const streamId = `steer-replace-${Date.now()}`;
+ const predecessor = await store.createJob(streamId, 'steer-user', streamId);
+ await store.enqueueSteer(streamId, buildSteer('s1', 'old run steer'));
+
+ const replacement = await store.createJob(streamId, 'steer-user', streamId);
+ await store.enqueueSteer(streamId, buildSteer('s2', 'replacement steer'));
+
+ expect(await store.peekSteers(streamId, predecessor.createdAt)).toEqual([]);
+ expect(
+ (await store.peekSteers(streamId, replacement.createdAt)).map((steer) => steer.text),
+ ).toEqual(['replacement steer']);
+
+ await store.destroy();
+ });
+
+ test('pausing for review extends the steers key TTL to the approval window', async () => {
if (!ioredisClient) {
return;
}
@@ -2401,7 +2962,7 @@ describe('RedisJobStore Integration Tests', () => {
const ttl = await ioredisClient.ttl(sequenceKey);
expect(ttl).toBeGreaterThan(24 * 60 * 60);
- expect(ttl).toBeLessThanOrEqual(approvalWindowSeconds + 60);
+ expect(ttl).toBeLessThanOrEqual(approvalWindowSeconds + 360);
await store.destroy();
});
@@ -2454,7 +3015,34 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
- test('parked steers survive deleteJob, claim exactly once, and reset on createJob', async () => {
+ test('removeSteer atomically refuses a generation replaced after authorization', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+
+ const streamId = `steer-remove-replaced-${Date.now()}`;
+ const predecessor = await store.createJob(streamId, 'steer-user', streamId);
+ const reusedId = buildSteer('reused-id', 'predecessor');
+ await store.enqueueSteer(streamId, reusedId, predecessor.createdAt);
+ const replacement = await store.createJob(streamId, 'steer-user', streamId);
+ const replacementItem = buildSteer(reusedId.steerId, 'replacement');
+ await store.enqueueSteer(streamId, replacementItem, replacement.createdAt);
+
+ await expect(
+ store.removeSteer(streamId, reusedId.steerId, predecessor.createdAt),
+ ).resolves.toBe(false);
+ await expect(store.peekSteers(streamId, replacement.createdAt)).resolves.toEqual([
+ replacementItem,
+ ]);
+
+ await store.destroy();
+ });
+
+ test('parked steers survive deleteJob, replay on read, and consume on exact recovery', async () => {
if (!ioredisClient) {
return;
}
@@ -2477,18 +3065,118 @@ describe('RedisJobStore Integration Tests', () => {
// …under its own bounded TTL, not the job's lifecycle.
expect(await ioredisClient.ttl(`stream:{${streamId}}:parked`)).toBeGreaterThan(0);
- // Claim-on-read: exactly once.
- expect(await store.claimParkedSteers(streamId, '"userId":"steer-user"')).toBe(payload);
- expect(await store.claimParkedSteers(streamId, '"userId":"steer-user"')).toBeUndefined();
+ // Owner reads are replayable so a lost status response cannot erase recovery.
+ expect(JSON.parse((await store.claimParkedSteers(streamId, 'steer-user')) as string)).toEqual(
+ { ...JSON.parse(payload), generationProtocolVersion: 2 },
+ );
+ expect(JSON.parse((await store.claimParkedSteers(streamId, 'steer-user')) as string)).toEqual(
+ { ...JSON.parse(payload), generationProtocolVersion: 2 },
+ );
- // A replacement run resets any parked payload atomically with creation.
- await store.parkSteers(streamId, payload);
- await store.createJob(streamId, 'steer-user', streamId);
- expect(await store.claimParkedSteers(streamId, '"userId":"steer-user"')).toBeUndefined();
+ // Starting the deterministic recovery leases (hides) the source without
+ // deleting it before the ordinary user message is durable.
+ const failedRecovery = await store.createJob(
+ streamId,
+ 'steer-user',
+ streamId,
+ undefined,
+ {},
+ 'p1',
+ undefined,
+ undefined,
+ undefined,
+ { text: 'kept', fileIds: [] },
+ );
+ expect(await store.claimParkedSteers(streamId, 'steer-user')).toBeUndefined();
+ expect(await ioredisClient.exists(`stream:{${streamId}}:parked`)).toBe(1);
+
+ // Initialization failure makes the lease visible again.
+ await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ expectCreatedAt: failedRecovery.createdAt,
+ patch: { error: 'provider init failed', completedAt: Date.now() },
+ });
+ expect(JSON.parse((await store.claimParkedSteers(streamId, 'steer-user')) as string)).toEqual(
+ { ...JSON.parse(payload), generationProtocolVersion: 2 },
+ );
+
+ // A persisted retry commits and removes the exact source.
+ const persistedRecovery = await store.createJob(
+ streamId,
+ 'steer-user',
+ streamId,
+ undefined,
+ {},
+ 'p1',
+ undefined,
+ undefined,
+ undefined,
+ { text: 'kept', fileIds: [] },
+ );
+ expect(
+ await store.consumeParkedSteer(
+ streamId,
+ 'p1',
+ 'steer-user',
+ undefined,
+ persistedRecovery.createdAt,
+ ),
+ ).toBe(true);
+ expect(await store.claimParkedSteers(streamId, 'steer-user')).toBeUndefined();
+ expect(await ioredisClient.exists(`stream:{${streamId}}:parked`)).toBe(0);
await store.destroy();
});
+ test.each([
+ ['changed text', { text: 'forged words', fileIds: ['file-a', 'file-b'] }],
+ ['changed files', { text: 'original words', fileIds: ['file-a', 'file-c'] }],
+ ])('atomically refuses parked recovery with %s', async (_label, proof) => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+
+ const streamId = `steer-recovery-mismatch-${_label.replace(' ', '-')}-${Date.now()}`;
+ const originalJob = await store.createJob(streamId, 'steer-user', streamId);
+ const source = {
+ steerId: `source-${_label.replace(' ', '-')}`,
+ text: 'original words',
+ createdAt: 1,
+ files: [{ file_id: 'file-b' }, { file_id: 'file-a' }],
+ };
+ await store.parkSteers(streamId, JSON.stringify({ userId: 'steer-user', steers: [source] }));
+
+ await expect(
+ store.createJob(
+ streamId,
+ 'steer-user',
+ streamId,
+ undefined,
+ {},
+ source.steerId,
+ undefined,
+ undefined,
+ undefined,
+ proof,
+ ),
+ ).rejects.toMatchObject({ code: 'RECOVERY_PAYLOAD_MISMATCH' });
+
+ expect((await store.getJob(streamId))?.createdAt).toBe(originalJob.createdAt);
+ expect(JSON.parse((await store.claimParkedSteers(streamId, 'steer-user')) as string)).toEqual(
+ {
+ userId: 'steer-user',
+ generationProtocolVersion: 2,
+ steers: [source],
+ },
+ );
+ await store.destroy();
+ });
+
test('a non-owner claim leaves the parked payload untouched (atomic owner gate)', async () => {
if (!ioredisClient) {
return;
@@ -2499,6 +3187,9 @@ describe('RedisJobStore Integration Tests', () => {
await store.initialize();
const streamId = `steer-parked-owner-${Date.now()}`;
+ await store.createJob(streamId, 'steer-user', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
const payload = JSON.stringify({
userId: 'steer-user',
steers: [{ steerId: 'p1', text: 'owner only', createdAt: 1 }],
@@ -2507,12 +3198,19 @@ describe('RedisJobStore Integration Tests', () => {
// The Lua gate rejects WITHOUT deleting: no delete-then-re-park window
// in which a concurrent owner claim would find nothing.
- expect(await store.claimParkedSteers(streamId, '"userId":"intruder"')).toBeUndefined();
- expect(await ioredisClient.get(`stream:{${streamId}}:parked`)).toBe(payload);
+ expect(await store.claimParkedSteers(streamId, 'intruder')).toBeUndefined();
+ expect(
+ JSON.parse((await ioredisClient.get(`stream:{${streamId}}:parked`)) as string),
+ ).toEqual({ ...JSON.parse(payload), generationProtocolVersion: 2 });
- // The owner still claims exactly once.
- expect(await store.claimParkedSteers(streamId, '"userId":"steer-user"')).toBe(payload);
- expect(await ioredisClient.exists(`stream:{${streamId}}:parked`)).toBe(0);
+ // The owner can safely replay the read without deleting the payload.
+ expect(JSON.parse((await store.claimParkedSteers(streamId, 'steer-user')) as string)).toEqual(
+ { ...JSON.parse(payload), generationProtocolVersion: 2 },
+ );
+ expect(JSON.parse((await store.claimParkedSteers(streamId, 'steer-user')) as string)).toEqual(
+ { ...JSON.parse(payload), generationProtocolVersion: 2 },
+ );
+ expect(await ioredisClient.exists(`stream:{${streamId}}:parked`)).toBe(1);
await store.destroy();
});
@@ -2548,7 +3246,7 @@ describe('RedisJobStore Integration Tests', () => {
expect(await ioredisClient.exists(`stream:{${streamId}}:steers`)).toBe(0);
// …but the 202-accepted steer is claimable by its owner.
- const claimed = await store.claimParkedSteers(streamId, '"userId":"steer-user"');
+ const claimed = await store.claimParkedSteers(streamId, 'steer-user', 'tenant-1');
expect(claimed).toBeDefined();
const parsed = JSON.parse(claimed as string) as {
userId: string;
@@ -2562,6 +3260,65 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
+ test('expiry cleanup waits for a fresh pause barrier then fails it closed', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ try {
+ const streamId = 'pause-barrier-expiry-cleanup';
+ const job = await store.createJob(streamId, 'steer-user', streamId, 'tenant-1');
+ await store.enqueueSteer(
+ streamId,
+ buildSteer('pause-barrier-steer', 'frozen while pause persists'),
+ job.createdAt,
+ );
+ const expiredAction = { ...buildPendingAction(streamId), expiresAt: 0 };
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingAction: expiredAction,
+ pendingActionId: `pause-persistence:${expiredAction.actionId}`,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: 1_000,
+ },
+ }),
+ ).resolves.toBe(true);
+
+ now.mockReturnValue(11_000);
+ await store.cleanup();
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ status: 'requires_action',
+ terminalPersistencePending: true,
+ });
+
+ now.mockReturnValue(31_001);
+ await store.cleanup();
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ status: 'error',
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ });
+ const failedJob = await store.getJob(streamId);
+ expect(failedJob?.pendingAction).toBeUndefined();
+ expect(failedJob?.pendingActionId).toBeUndefined();
+ expect(failedJob?.terminalPersistencePending).toBeUndefined();
+ expect(failedJob?.terminalPersistenceStartedAt).toBeUndefined();
+ await expect(store.claimParkedSteers(streamId, 'steer-user', 'tenant-1')).resolves.toEqual(
+ expect.any(String),
+ );
+ } finally {
+ await store.destroy();
+ now.mockRestore();
+ }
+ });
+
test('terminal CAS with zero completed TTL keeps parked steers owner-claimable', async () => {
if (!ioredisClient) {
return;
@@ -2589,7 +3346,7 @@ describe('RedisJobStore Integration Tests', () => {
expect(await ioredisClient.smembers('stream:running')).not.toContain(streamId);
expect(await store.getActiveJobIdsByUser(userId, 'tenant-1')).not.toContain(streamId);
- const claimed = await store.claimParkedSteers(streamId, `"userId":"${userId}"`);
+ const claimed = await store.claimParkedSteers(streamId, userId, 'tenant-1');
expect(claimed).toBeDefined();
expect(JSON.parse(claimed as string)).toMatchObject({
userId,
@@ -2600,7 +3357,47 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
- test('terminal CAS skips malformed steers while parking valid leftovers', async () => {
+ test('terminal persistence survives a zero completed TTL until it is finalized', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient, { completedTtl: 0 });
+ await store.initialize();
+
+ const streamId = `zero-terminal-persistence-${Date.now()}`;
+ const job = await store.createJob(streamId, 'terminal-user', streamId);
+ const completedAt = Date.now();
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'aborted',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ completedAt,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: completedAt,
+ },
+ }),
+ ).resolves.toBe(true);
+
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ status: 'aborted',
+ terminalPersistencePending: true,
+ });
+ await expect(
+ store.finalizeTerminalPersistence(streamId, job.createdAt, '{"event":"final"}'),
+ ).resolves.toBe(true);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ terminalPersistencePending: false,
+ finalEvent: '{"event":"final"}',
+ });
+
+ await store.destroy();
+ });
+
+ test('terminal CAS fails closed without deleting malformed recovery state', async () => {
if (!ioredisClient) {
return;
}
@@ -2622,23 +3419,20 @@ describe('RedisJobStore Integration Tests', () => {
expectCreatedAt: job.createdAt,
patch: { error: 'stopped', completedAt: Date.now() },
}),
- ).resolves.toBe(true);
+ ).resolves.toBe(false);
- await expect(store.getJob(streamId)).resolves.toMatchObject({ status: 'error' });
- expect(await ioredisClient.exists(`stream:{${streamId}}:steers`)).toBe(0);
-
- const claimed = await store.claimParkedSteers(streamId, `"userId":"${userId}"`);
- expect(claimed).toBeDefined();
- expect(JSON.parse(claimed as string)).toMatchObject({
- userId,
- tenantId: 'tenant-1',
- steers: [
- {
- steerId: 'valid-steer',
- text: 'preserve valid input',
- },
- ],
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ status: 'running',
+ createdAt: job.createdAt,
+ });
+ const preserved = await ioredisClient.lrange(`stream:{${streamId}}:steers`, 0, -1);
+ expect(preserved).toHaveLength(2);
+ expect(JSON.parse(preserved[0])).toMatchObject({
+ steerId: 'valid-steer',
+ text: 'preserve valid input',
});
+ expect(preserved[1]).toBe('{malformed-json');
+ await expect(store.claimParkedSteers(streamId, userId, 'tenant-1')).resolves.toBeUndefined();
await store.destroy();
});
@@ -2669,7 +3463,7 @@ describe('RedisJobStore Integration Tests', () => {
expect(await store.getJob(streamId)).toBeNull();
expect(await ioredisClient.exists(`stream:{${streamId}}:steers`)).toBe(0);
- const claimed = await store.claimParkedSteers(streamId, '"userId":"steer-user"');
+ const claimed = await store.claimParkedSteers(streamId, 'steer-user', 'tenant-1');
expect(claimed).toBeDefined();
const parsed = JSON.parse(claimed as string) as {
userId: string;
@@ -2761,9 +3555,12 @@ describe('RedisJobStore Integration Tests', () => {
expect((await store.peekSteers(streamId)).map((steer) => steer.steerId)).toEqual([
'replacement-steer',
]);
- await expect(
- store.claimParkedSteers(streamId, `"userId":"${userId}"`),
- ).resolves.toBeUndefined();
+ const parked = await store.claimParkedSteers(streamId, userId);
+ expect(parked).toBeDefined();
+ expect(JSON.parse(parked as string)).toMatchObject({
+ userId,
+ steers: [{ steerId: 'old-steer', text: 'old generation' }],
+ });
expect(await ioredisClient.smembers('stream:running')).toContain(streamId);
} finally {
releaseCleanup?.();
@@ -2960,7 +3757,10 @@ describe('RedisJobStore Integration Tests', () => {
{ streamId: 's1', conversationId: 'c1' },
1200,
);
- expect(first).toEqual({ claimed: true });
+ expect(first).toEqual({
+ claimed: true,
+ existing: { streamId: 's1', conversationId: 'c1' },
+ });
const second = await store.claimIdempotencyKey(
key,
@@ -2986,7 +3786,7 @@ describe('RedisJobStore Integration Tests', () => {
const key = `user-1:req-ttl-${Date.now()}`;
await store.claimIdempotencyKey(key, { streamId: 's1', conversationId: 'c1' }, 1200);
- const pttl = await ioredisClient.pttl(`stream:idem:{${key}}`);
+ const pttl = await ioredisClient.pttl(`stream:idem:${key}`);
expect(pttl).toBeGreaterThan(0);
expect(pttl).toBeLessThanOrEqual(1200 * 1000);
@@ -3010,7 +3810,10 @@ describe('RedisJobStore Integration Tests', () => {
{ streamId: 's2', conversationId: 'c2' },
1200,
);
- expect(reclaimed).toEqual({ claimed: true });
+ expect(reclaimed).toEqual({
+ claimed: true,
+ existing: { streamId: 's2', conversationId: 'c2' },
+ });
await store.destroy();
});
@@ -3043,4 +3846,541 @@ describe('RedisJobStore Integration Tests', () => {
await store.destroy();
});
});
+
+ describe('Steer receipt integrity', () => {
+ const corruptions = ['missing', 'malformed', 'wrong-state', 'wrong-generation'] as const;
+ type Corruption = (typeof corruptions)[number];
+
+ async function enqueueReceiptSteer(streamId: string, suffix: string) {
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient!);
+ await store.initialize();
+ const job = await store.createJob(streamId, 'receipt-user', streamId);
+ const item: SteerQueueItem = {
+ steerId: `steer-${suffix}`,
+ clientSteerId: `client-${suffix}`,
+ text: `instruction ${suffix}`,
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ const result = await store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${suffix}`,
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ );
+ expect(result).toMatchObject({ state: 'queued', item });
+ return { store, item, createdAt: job.createdAt };
+ }
+
+ async function corruptReceipt(
+ streamId: string,
+ clientSteerId: string,
+ corruption: Corruption,
+ ): Promise<void> {
+ const key = `stream:{${streamId}}:steer-receipts`;
+ if (corruption === 'missing') {
+ await ioredisClient!.hdel(key, clientSteerId);
+ return;
+ }
+ if (corruption === 'malformed') {
+ await ioredisClient!.hset(key, clientSteerId, '{not-json');
+ return;
+ }
+ const raw = await ioredisClient!.hget(key, clientSteerId);
+ expect(raw).not.toBeNull();
+ const receipt = JSON.parse(raw!) as SteerReceipt;
+ if (corruption === 'wrong-state') {
+ receipt.state = receipt.state === 'queued' ? 'claimed' : 'queued';
+ } else {
+ receipt.generationCreatedAt += 1;
+ }
+ await ioredisClient!.hset(key, clientSteerId, JSON.stringify(receipt));
+ }
+
+ test.each(['deleted', 'v1-replaced'] as const)(
+ 'replays an existing receipt after its accepting job is %s',
+ async (lifecycle) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `redis-receipt-replay-${lifecycle}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(streamId, lifecycle);
+ try {
+ if (lifecycle === 'deleted') {
+ await expect(store.deleteJob(streamId, createdAt)).resolves.toBe(true);
+ } else {
+ const replacement = await store.createJob(streamId, item.userId, streamId, undefined, {
+ generationProtocolVersion: 1,
+ });
+ expect(replacement.createdAt).not.toBe(createdAt);
+ await expect(store.peekSteers(streamId, replacement.createdAt)).resolves.toEqual([]);
+ }
+
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject(
+ {
+ state: 'leftover',
+ generationCreatedAt: createdAt,
+ item,
+ },
+ );
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ { ...item, steerId: `duplicate-${lifecycle}` },
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${lifecycle}`,
+ userId: item.userId,
+ generationCreatedAt: createdAt,
+ },
+ false,
+ createdAt,
+ ),
+ ).resolves.toMatchObject({
+ state: 'leftover',
+ generationCreatedAt: createdAt,
+ item,
+ });
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test('keeps an active recovery lease and receipt beyond job expiry, activity, and pause', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-recovery-lease-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(streamId, 'recovery-lease');
+ const parkedKey = `stream:{${streamId}}:parked`;
+ const receiptsKey = `stream:{${streamId}}:steer-receipts`;
+ const receiptOrderKey = `stream:{${streamId}}:steer-receipt-order`;
+ try {
+ await expect(store.closeAndDrainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ const recovery = await store.createJob(
+ streamId,
+ item.userId,
+ streamId,
+ undefined,
+ {},
+ item.steerId,
+ undefined,
+ undefined,
+ undefined,
+ {
+ text: item.text,
+ fileIds: (item.files ?? []).flatMap((file) => file.file_id ?? []).sort(),
+ },
+ );
+
+ // The default live-job storage horizon is 25m. A recovery lease gets a
+ // further normal 5m recovery window, rather than expiring with the job.
+ expect(await ioredisClient.ttl(parkedKey)).toBeGreaterThan(1500);
+ expect(await ioredisClient.ttl(receiptsKey)).toBeGreaterThan(1500);
+ expect(await ioredisClient.ttl(receiptOrderKey)).toBeGreaterThan(1500);
+
+ // Adding another steer to the recovery generation must not shorten the
+ // shared receipt/order lease back to the normal running horizon.
+ const followup: SteerQueueItem = {
+ steerId: 'steer-recovery-followup',
+ clientSteerId: 'client-recovery-followup',
+ text: 'follow-up while recovery is active',
+ userId: item.userId,
+ createdAt: Date.now(),
+ };
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ followup,
+ {
+ clientSteerId: followup.clientSteerId!,
+ fingerprint: 'fingerprint-recovery-followup',
+ userId: followup.userId,
+ generationCreatedAt: recovery.createdAt,
+ },
+ false,
+ recovery.createdAt,
+ ),
+ ).resolves.toMatchObject({ state: 'queued', item: followup });
+ expect(await ioredisClient.ttl(parkedKey)).toBeGreaterThan(1500);
+ expect(await ioredisClient.ttl(receiptsKey)).toBeGreaterThan(1500);
+ expect(await ioredisClient.ttl(receiptOrderKey)).toBeGreaterThan(1500);
+
+ // Simulate a long healthy run near the end of the prior lease. Activity
+ // refresh must move both the hidden source and its receipt back out.
+ await ioredisClient.expire(parkedKey, 30);
+ await ioredisClient.expire(receiptsKey, 30);
+ await ioredisClient.expire(receiptOrderKey, 30);
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_message_delta', data: { text: 'still healthy' } },
+ recovery.createdAt,
+ ),
+ ).resolves.toBe(true);
+ expect(await ioredisClient.ttl(parkedKey)).toBeGreaterThan(1500);
+ expect(await ioredisClient.ttl(receiptsKey)).toBeGreaterThan(1500);
+ expect(await ioredisClient.ttl(receiptOrderKey)).toBeGreaterThan(1500);
+
+ // A pause can outlive the running window; the same status CAS extends
+ // the uncommitted recovery source past the approval/job TTL as well.
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: recovery.createdAt,
+ patch: { pendingAction: buildPendingAction(streamId) },
+ }),
+ ).resolves.toBe(true);
+ expect(await ioredisClient.ttl(parkedKey)).toBeGreaterThan(24 * 60 * 60);
+ expect(await ioredisClient.ttl(receiptsKey)).toBeGreaterThan(24 * 60 * 60);
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('does not let a stale cancel discard a source leased to a live recovery', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-live-recovery-cancel-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(streamId, 'live-recovery');
+ try {
+ await expect(store.closeAndDrainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ const recovery = await store.createJob(
+ streamId,
+ item.userId,
+ streamId,
+ undefined,
+ {},
+ item.steerId,
+ undefined,
+ undefined,
+ undefined,
+ {
+ text: item.text,
+ fileIds: (item.files ?? []).flatMap((file) => file.file_id ?? []).sort(),
+ },
+ );
+ await expect(
+ store.discardSteerLeftover(streamId, item.clientSteerId!, item.steerId, item.userId),
+ ).resolves.toBe(false);
+ await expect(
+ store.consumeParkedSteer(
+ streamId,
+ item.steerId,
+ item.userId,
+ undefined,
+ recovery.createdAt,
+ ),
+ ).resolves.toBe(true);
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'recovered',
+ });
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test.each(['close', 'terminal'] as const)(
+ '%s recovery safely parks words when a receipt is malformed',
+ async (path) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-malformed-${path}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(streamId, `malformed-${path}`);
+ try {
+ await corruptReceipt(streamId, item.clientSteerId!, 'malformed');
+ if (path === 'close') {
+ await expect(store.closeAndDrainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } else {
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ expectCreatedAt: createdAt,
+ patch: { error: 'terminal failure', completedAt: Date.now() },
+ }),
+ ).resolves.toBe(true);
+ }
+
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([]);
+ const parked = await store.claimParkedSteers(streamId, item.userId);
+ expect(JSON.parse(parked!)).toMatchObject({
+ userId: item.userId,
+ steers: [{ steerId: item.steerId, clientSteerId: item.clientSteerId }],
+ });
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test('atomically settles a claimed receipt with its applied chunk', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-valid-delivery-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(streamId, 'valid');
+ try {
+ await expect(store.drainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_steer_applied', data: { steerId: item.steerId } },
+ createdAt,
+ item,
+ ),
+ ).resolves.toBe(true);
+
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'delivered',
+ item,
+ });
+ await expect(ioredisClient.xlen(`stream:{${streamId}}:chunks`)).resolves.toBe(1);
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test.each(corruptions)(
+ 'refuses to drain a queue with a %s receipt before moving any item',
+ async (corruption) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-drain-${corruption}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(
+ streamId,
+ `drain-${corruption}`,
+ );
+ try {
+ await corruptReceipt(streamId, item.clientSteerId!, corruption);
+
+ await expect(store.drainSteers(streamId, createdAt)).rejects.toThrow(/steer receipt/);
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(ioredisClient.xlen(`stream:{${streamId}}:chunks`)).resolves.toBe(0);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(corruptions)(
+ 'refuses to cancel with a %s receipt before mutating the queued item',
+ async (corruption) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-cancel-${corruption}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(
+ streamId,
+ `cancel-${corruption}`,
+ );
+ try {
+ await corruptReceipt(streamId, item.clientSteerId!, corruption);
+
+ await expect(store.removeSteer(streamId, item.steerId, createdAt)).rejects.toThrow(
+ /steer receipt/,
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(corruptions)(
+ 'refuses to settle a claimed steer with a %s receipt before mutating durable state',
+ async (corruption) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-append-${corruption}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(
+ streamId,
+ `append-${corruption}`,
+ );
+ try {
+ await expect(store.drainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await corruptReceipt(streamId, item.clientSteerId!, corruption);
+
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_steer_applied', data: { steerId: item.steerId } },
+ createdAt,
+ item,
+ ),
+ ).resolves.toBe(false);
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await expect(ioredisClient.xlen(`stream:{${streamId}}:chunks`)).resolves.toBe(0);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(corruptions)(
+ 'refuses to restore a claimed steer with a %s receipt before changing either list',
+ async (corruption) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-restore-${corruption}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(
+ streamId,
+ `restore-${corruption}`,
+ );
+ try {
+ await expect(store.drainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await corruptReceipt(streamId, item.clientSteerId!, corruption);
+
+ await expect(store.restoreClaimedSteers(streamId, [item], createdAt)).resolves.toBe(
+ false,
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await expect(ioredisClient.xlen(`stream:{${streamId}}:chunks`)).resolves.toBe(0);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(corruptions)(
+ 'refuses to arm with a %s receipt before mutating the queued item',
+ async (corruption) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-arm-${corruption}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(streamId, `arm-${corruption}`);
+ try {
+ await store.updateJob(streamId, { preemptCapable: true }, createdAt);
+ await corruptReceipt(streamId, item.clientSteerId!, corruption);
+
+ await expect(store.armSteerVersioned(streamId, item.steerId, createdAt)).rejects.toThrow(
+ /steer receipt/,
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(corruptions)(
+ 'refuses to downgrade with a %s receipt before mutating any queued item',
+ async (corruption) => {
+ if (!ioredisClient) {
+ return;
+ }
+ const streamId = `receipt-downgrade-${corruption}-${Date.now()}`;
+ const { store, item, createdAt } = await enqueueReceiptSteer(
+ streamId,
+ `downgrade-${corruption}`,
+ );
+ try {
+ await store.updateJob(streamId, { preemptCapable: true }, createdAt);
+ await expect(
+ store.armSteerVersioned(streamId, item.steerId, createdAt),
+ ).resolves.toMatchObject({ outcome: 'armed' });
+ const armed = await store.peekSteers(streamId, createdAt);
+ await store.updateJob(streamId, { preemptCapable: false }, createdAt);
+ await corruptReceipt(streamId, item.clientSteerId!, corruption);
+
+ await expect(store.downgradeSteerPreempts(streamId, createdAt)).rejects.toThrow(
+ /steer receipt/,
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual(armed);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test('refuses a 101st receipt without evicting replay evidence or queueing the item', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+ const streamId = `receipt-cap-${Date.now()}`;
+ try {
+ const job = await store.createJob(streamId, 'receipt-user', streamId);
+ for (let index = 0; index < 100; index++) {
+ const item: SteerQueueItem = {
+ steerId: `steer-cap-${index}`,
+ clientSteerId: `client-cap-${index}`,
+ text: `instruction ${index}`,
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${index}`,
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ ),
+ ).resolves.toMatchObject({ state: 'queued', item });
+ await expect(store.removeSteer(streamId, item.steerId)).resolves.toBe(true);
+ }
+
+ const overflow: SteerQueueItem = {
+ steerId: 'steer-cap-overflow',
+ clientSteerId: 'client-cap-overflow',
+ text: 'must not queue',
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ overflow,
+ {
+ clientSteerId: overflow.clientSteerId!,
+ fingerprint: 'fingerprint-overflow',
+ userId: overflow.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ ),
+ ).resolves.toBe(STEER_ENQUEUE_RECEIPT_FULL);
+
+ await expect(store.peekSteers(streamId, job.createdAt)).resolves.toEqual([]);
+ await expect(store.getSteerReceipt(streamId, 'client-cap-overflow')).resolves.toBeNull();
+ await expect(store.getSteerReceipt(streamId, 'client-cap-0')).resolves.toMatchObject({
+ state: 'cancelled',
+ item: { steerId: 'steer-cap-0' },
+ });
+ await expect(ioredisClient.zcard(`stream:{${streamId}}:steer-receipt-order`)).resolves.toBe(
+ 100,
+ );
+ } finally {
+ await store.destroy();
+ }
+ });
+ });
});
diff --git a/packages/api/src/stream/__tests__/createJobOwnership.spec.ts b/packages/api/src/stream/__tests__/createJobOwnership.spec.ts
new file mode 100644
index 0000000000..3c50889c7a
--- /dev/null
+++ b/packages/api/src/stream/__tests__/createJobOwnership.spec.ts
@@ -0,0 +1,125 @@
+import type { SteerQueueItem } from '../interfaces/IJobStore';
+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
+
+type ParkedMap = Map<string, { payload: string; expiresAt: number }>;
+
+function parkedMap(store: InMemoryJobStore): ParkedMap {
+ return (store as unknown as { parkedSteers: ParkedMap }).parkedSteers;
+}
+
+describe('InMemoryJobStore createJob ownership integrity', () => {
+ let store: InMemoryJobStore;
+
+ beforeEach(() => {
+ store = new InMemoryJobStore();
+ });
+
+ afterEach(async () => {
+ await store.destroy();
+ });
+
+ test('refuses a cross-user replacement without disturbing the live generation', async () => {
+ const streamId = 'create-owner-cross-user';
+ const original = await store.createJob(streamId, 'owner-a', streamId, 'tenant-a');
+ const steer: SteerQueueItem = {
+ steerId: 'keep-me',
+ text: 'must remain queued',
+ userId: 'owner-a',
+ createdAt: Date.now(),
+ };
+ await expect(store.enqueueSteer(streamId, steer, original.createdAt)).resolves.toBe(1);
+
+ await expect(store.createJob(streamId, 'owner-b', streamId, 'tenant-a')).rejects.toThrow(
+ /owner|ownership/i,
+ );
+
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ userId: 'owner-a',
+ tenantId: 'tenant-a',
+ createdAt: original.createdAt,
+ status: 'running',
+ });
+ await expect(store.peekSteers(streamId, original.createdAt)).resolves.toEqual([steer]);
+ });
+
+ test('refuses a cross-tenant replacement for the same user', async () => {
+ const streamId = 'create-owner-cross-tenant';
+ const original = await store.createJob(streamId, 'owner', streamId, 'tenant-a');
+
+ await expect(store.createJob(streamId, 'owner', streamId, 'tenant-b')).rejects.toThrow(
+ /owner|tenant|ownership/i,
+ );
+
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ userId: 'owner',
+ tenantId: 'tenant-a',
+ createdAt: original.createdAt,
+ });
+ });
+
+ test('allows a same-user legacy job to enter a tenant without leaking its membership', async () => {
+ const streamId = 'create-owner-legacy-tenant';
+ const legacy = await store.createJob(streamId, 'owner', streamId);
+ const replacement = await store.createJob(streamId, 'owner', streamId, 'tenant-a');
+
+ expect(replacement.createdAt).toBeGreaterThan(legacy.createdAt);
+ await expect(store.getActiveJobIdsByUser('owner')).resolves.not.toContain(streamId);
+ await expect(store.getActiveJobIdsByUser('owner', 'tenant-a')).resolves.toContain(streamId);
+ });
+
+ test.each([
+ ['malformed JSON', '{not-json'],
+ ['a scalar entry', JSON.stringify({ userId: 'owner', steers: [42] })],
+ ['an item without steerId', JSON.stringify({ userId: 'owner', steers: [{ text: 'lost' }] })],
+ ])('preserves parked recovery state containing %s', async (_name, payload) => {
+ const streamId = `create-corrupt-parked-${String(_name).replaceAll(' ', '-')}`;
+ parkedMap(store).set(streamId, { payload, expiresAt: Date.now() + 60_000 });
+
+ await expect(store.createJob(streamId, 'owner', streamId)).rejects.toThrow(
+ /recovery|parked|corrupt/i,
+ );
+
+ expect(parkedMap(store).get(streamId)?.payload).toBe(payload);
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ });
+
+ test('preserves and refuses recovery state owned by another user', async () => {
+ const streamId = 'create-foreign-parked';
+ const payload = JSON.stringify({
+ userId: 'owner-a',
+ tenantId: 'tenant-a',
+ steers: [{ steerId: 'recover-me', text: 'keep me', createdAt: Date.now() }],
+ });
+ parkedMap(store).set(streamId, { payload, expiresAt: Date.now() + 60_000 });
+
+ await expect(store.createJob(streamId, 'owner-b', streamId, 'tenant-a')).rejects.toThrow(
+ /owner|ownership/i,
+ );
+
+ expect(parkedMap(store).get(streamId)?.payload).toBe(payload);
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ });
+
+ test('refuses to create an ownerless generation', async () => {
+ await expect(store.createJob('create-ownerless', '', 'create-ownerless')).rejects.toThrow(
+ /owner|user/i,
+ );
+ await expect(store.getJob('create-ownerless')).resolves.toBeNull();
+ });
+
+ test('keeps the generation epoch monotonic after delete under a frozen clock', async () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+ try {
+ const streamId = 'create-monotonic-after-delete';
+ const first = await store.createJob(streamId, 'owner', streamId);
+ await expect(store.deleteJob(streamId, first.createdAt)).resolves.toBe(true);
+ const second = await store.createJob(streamId, 'owner', streamId);
+
+ expect(second.createdAt).toBeGreaterThan(first.createdAt);
+ expect(second.checkpointNamespace).not.toBe(first.checkpointNamespace);
+ expect(second.checkpointNamespace).toBe(String(second.createdAt));
+ } finally {
+ now.mockRestore();
+ }
+ });
+});
diff --git a/packages/api/src/stream/__tests__/helpers/publisher.ts b/packages/api/src/stream/__tests__/helpers/publisher.ts
index c749bfd83e..de10069e65 100644
--- a/packages/api/src/stream/__tests__/helpers/publisher.ts
+++ b/packages/api/src/stream/__tests__/helpers/publisher.ts
@@ -4,6 +4,7 @@ export interface MockPublisher {
expire: jest.Mock;
ttl: jest.Mock;
get: jest.Mock;
+ set: jest.Mock;
del: jest.Mock;
eval: jest.Mock;
}
@@ -11,6 +12,7 @@ export interface MockPublisher {
/** Mock publisher with Redis command simulation for atomic sequence counters */
export function createMockPublisher(): MockPublisher {
const counters = new Map<string, number>();
+ const values = new Map<string, string>();
const ttls = new Map<string, number>();
const publisher: MockPublisher = {
publish: jest.fn().mockResolvedValue(1),
@@ -33,12 +35,21 @@ export function createMockPublisher(): MockPublisher {
return Promise.resolve(ttls.get(key) ?? -1);
}),
get: jest.fn().mockImplementation((key: string) => {
+ const stored = values.get(key);
+ if (stored != null) {
+ return Promise.resolve(stored);
+ }
const val = counters.get(key);
return Promise.resolve(val != null ? String(val) : null);
}),
+ set: jest.fn().mockImplementation((key: string, value: string) => {
+ values.set(key, value);
+ return Promise.resolve('OK');
+ }),
del: jest.fn().mockImplementation((...keys: string[]) => {
for (const key of keys) {
counters.delete(key);
+ values.delete(key);
ttls.delete(key);
}
return Promise.resolve(keys.length);
diff --git a/packages/api/src/stream/__tests__/helpers/redis.ts b/packages/api/src/stream/__tests__/helpers/redis.ts
new file mode 100644
index 0000000000..694bc98d72
--- /dev/null
+++ b/packages/api/src/stream/__tests__/helpers/redis.ts
@@ -0,0 +1,53 @@
+import IoRedis from 'ioredis';
+import type { Redis, Cluster, RedisOptions } from 'ioredis';
+
+export type RedisTestClient = Redis | Cluster;
+
+/**
+ * Build the same kind of ioredis client selected by the cache integration
+ * workflow. In cluster mode REDIS_URI is a comma-separated seed list, which
+ * must not be passed to the single-node Redis constructor as one URL.
+ */
+export function createRedisTestClient(keyPrefix: string): RedisTestClient {
+ const urls = (process.env.REDIS_URI ?? 'redis://127.0.0.1:6379')
+ .split(',')
+ .map((uri) => new URL(uri));
+ const [primary] = urls;
+ const redisOptions: RedisOptions = {
+ keyPrefix,
+ maxRetriesPerRequest: 2,
+ ...(primary.username !== '' && { username: primary.username }),
+ ...(primary.password !== '' && { password: primary.password }),
+ ...(primary.protocol === 'rediss:' && { tls: {} }),
+ };
+
+ if (process.env.USE_REDIS_CLUSTER === 'true' || urls.length > 1) {
+ return new IoRedis.Cluster(
+ urls.map((url) => ({
+ host: url.hostname,
+ port: Number.parseInt(url.port, 10) || 6379,
+ })),
+ { lazyConnect: true, redisOptions },
+ );
+ }
+
+ return new IoRedis(primary.href, { ...redisOptions, lazyConnect: true });
+}
+
+/** Delete only this suite's keys, including keys spread across cluster masters. */
+export async function clearRedisTestPrefix(
+ redis: RedisTestClient,
+ keyPrefix: string,
+): Promise<void> {
+ const nodes = (redis as Cluster).isCluster
+ ? (redis as Cluster).nodes('master')
+ : [redis as Redis];
+ const physicalKeys = [
+ ...new Set((await Promise.all(nodes.map((node) => node.keys('*')))).flat()),
+ ];
+ const logicalKeys = physicalKeys
+ .filter((key) => key.startsWith(keyPrefix))
+ .map((key) => key.slice(keyPrefix.length));
+
+ await Promise.all(logicalKeys.map((key) => redis.del(key)));
+}
diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
index e33b74b57d..d8b4c407b8 100644
--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
@@ -1,6 +1,16 @@
+import {
+ JobCreationSupersededError,
+ type CreatedJobData,
+ type IEventTransport,
+} from '~/stream/interfaces/IJobStore';
+import {
+ GenerationJobManagerClass,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
+} from '~/stream/GenerationJobManager';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
+import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication';
+import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
-import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
jest.spyOn(console, 'log').mockImplementation();
@@ -22,7 +32,10 @@ describe('InMemoryJobStore.claimIdempotencyKey', () => {
{ streamId: 's1', conversationId: 'c1' },
1200,
);
- expect(first).toEqual({ claimed: true });
+ expect(first).toEqual({
+ claimed: true,
+ existing: { streamId: 's1', conversationId: 'c1' },
+ });
// A retry that computed a different streamId still gets the ORIGINAL stream back.
const second = await store.claimIdempotencyKey(
@@ -42,7 +55,10 @@ describe('InMemoryJobStore.claimIdempotencyKey', () => {
{ streamId: 's2', conversationId: 'c2' },
1200,
);
- expect(reclaimed).toEqual({ claimed: true });
+ expect(reclaimed).toEqual({
+ claimed: true,
+ existing: { streamId: 's2', conversationId: 'c2' },
+ });
});
it('clears claims on destroy so a reused store does not falsely dedup', async () => {
@@ -53,7 +69,10 @@ describe('InMemoryJobStore.claimIdempotencyKey', () => {
{ streamId: 's2', conversationId: 'c2' },
1200,
);
- expect(reclaimed).toEqual({ claimed: true });
+ expect(reclaimed).toEqual({
+ claimed: true,
+ existing: { streamId: 's2', conversationId: 'c2' },
+ });
});
it('treats distinct keys independently', async () => {
@@ -67,8 +86,14 @@ describe('InMemoryJobStore.claimIdempotencyKey', () => {
{ streamId: 's2', conversationId: 'c2' },
1200,
);
- expect(a).toEqual({ claimed: true });
- expect(b).toEqual({ claimed: true });
+ expect(a).toEqual({
+ claimed: true,
+ existing: { streamId: 's1', conversationId: 'c1' },
+ });
+ expect(b).toEqual({
+ claimed: true,
+ existing: { streamId: 's2', conversationId: 'c2' },
+ });
});
it('lets the key be reclaimed after its TTL elapses', async () => {
@@ -80,7 +105,10 @@ describe('InMemoryJobStore.claimIdempotencyKey', () => {
{ streamId: 's1', conversationId: 'c1' },
1,
);
- expect(first).toEqual({ claimed: true });
+ expect(first).toEqual({
+ claimed: true,
+ existing: { streamId: 's1', conversationId: 'c1' },
+ });
// Still held one moment before expiry.
jest.setSystemTime(new Date('2026-07-20T00:00:00.999Z'));
@@ -98,20 +126,41 @@ describe('InMemoryJobStore.claimIdempotencyKey', () => {
{ streamId: 's3', conversationId: 'c3' },
1,
);
- expect(expired).toEqual({ claimed: true });
+ expect(expired).toEqual({
+ claimed: true,
+ existing: { streamId: 's3', conversationId: 'c3' },
+ });
} finally {
jest.useRealTimers();
}
});
+
+ it('rejects partial low-level create idempotency arguments before mutation', async () => {
+ await expect(
+ store.createJob(
+ 'partial-store-idempotency',
+ 'user-1',
+ undefined,
+ undefined,
+ {},
+ undefined,
+ '{partial-store-idempotency}:user-1:req',
+ 'claim-token',
+ ),
+ ).rejects.toThrow('Invalid generation job idempotency arguments');
+ await expect(store.getJob('partial-store-idempotency')).resolves.toBeNull();
+ });
});
describe('GenerationJobManager start-generation claim', () => {
let manager: GenerationJobManagerClass;
+ let store: InMemoryJobStore;
beforeEach(() => {
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
manager = new GenerationJobManagerClass();
manager.configure({
- jobStore: new InMemoryJobStore({ ttlAfterComplete: 0 }),
+ jobStore: store,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
@@ -124,9 +173,18 @@ describe('GenerationJobManager start-generation claim', () => {
it('dedups a retry of the same submission to the original stream', async () => {
const first = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
- expect(first).toEqual({ claimed: true });
+ expect(first).toEqual({
+ claimed: true,
+ source: 'primary',
+ existing: expect.objectContaining({
+ streamId: 'stream-a',
+ conversationId: 'convo-a',
+ claimToken: expect.any(String),
+ claimedAt: expect.any(Number),
+ }),
+ });
- const retry = await manager.claimGeneration('user-1', 'req-1', 'stream-b', 'convo-b');
+ const retry = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
expect(retry.claimed).toBe(false);
expect(retry.existing).toEqual(
expect.objectContaining({ streamId: 'stream-a', conversationId: 'convo-a' }),
@@ -134,23 +192,1340 @@ describe('GenerationJobManager start-generation claim', () => {
expect(typeof retry.existing?.claimedAt).toBe('number');
});
+ it('claims the exact legacy key before the same-slot primary with staggered TTLs', async () => {
+ const claimSpy = jest.spyOn(store, 'claimIdempotencyKey');
+ const result = await manager.claimGeneration(
+ 'user-1',
+ 'req-order',
+ 'stream-order',
+ 'stream-order',
+ 2,
+ );
+
+ expect(result).toMatchObject({ claimed: true, source: 'primary' });
+ expect(claimSpy).toHaveBeenNthCalledWith(
+ 1,
+ '{user-1:req-order}',
+ expect.objectContaining({
+ streamId: 'stream-order',
+ generationProtocolVersion: 2,
+ claimToken: expect.any(String),
+ }),
+ 26 * 60 * 60,
+ );
+ expect(claimSpy).toHaveBeenNthCalledWith(
+ 2,
+ '{stream-order}:user-1:req-order',
+ expect.objectContaining({
+ streamId: 'stream-order',
+ generationProtocolVersion: 2,
+ claimToken: result.existing?.claimToken,
+ }),
+ 25 * 60 * 60,
+ );
+ });
+
+ it('keeps an old tokenless legacy claim duplicate-only without binding a primary', async () => {
+ const oldClaim = {
+ streamId: 'old-random-stream',
+ conversationId: 'old-random-stream',
+ claimedAt: Date.now(),
+ };
+ await store.claimIdempotencyKey('{user-1:req-legacy}', oldClaim, 1200);
+ const claimSpy = jest.spyOn(store, 'claimIdempotencyKey');
+ const takeoverSpy = jest.spyOn(store, 'takeoverIdempotencyKey');
+
+ const result = await manager.claimGeneration(
+ 'user-1',
+ 'req-legacy',
+ 'new-stable-stream',
+ 'new-stable-stream',
+ 2,
+ );
+ expect(result).toEqual({
+ claimed: false,
+ source: 'legacy',
+ existing: { ...oldClaim, generationProtocolVersion: 1 },
+ });
+ expect(claimSpy).toHaveBeenCalledTimes(1);
+
+ await expect(
+ manager.takeoverGeneration('user-1', 'req-legacy', 'new-stable-stream', result.existing!),
+ ).resolves.toEqual(result);
+ expect(takeoverSpy).not.toHaveBeenCalled();
+ });
+
+ it('elects exactly one manager winner while both durable keys converge', async () => {
+ const [first, second] = await Promise.all([
+ manager.claimGeneration('user-1', 'req-race', 'stream-race', 'stream-race', 2),
+ manager.claimGeneration('user-1', 'req-race', 'stream-race', 'stream-race', 2),
+ ]);
+
+ expect([first, second].filter((result) => result.claimed)).toHaveLength(1);
+ expect([first, second].filter((result) => !result.claimed)).toHaveLength(1);
+ expect(first.existing?.claimToken).toBe(second.existing?.claimToken);
+ expect(first.existing?.generationProtocolVersion).toBe(2);
+ expect(second.existing?.generationProtocolVersion).toBe(2);
+ });
+
+ it('marks primary and legacy claims started before createJob returns', async () => {
+ const claim = await manager.claimGeneration(
+ 'user-1',
+ 'req-started',
+ 'stream-started',
+ 'stream-started',
+ 2,
+ );
+ const job = await manager.createJob('stream-started', 'user-1', 'stream-started', {
+ idempotencyClientRequestId: 'req-started',
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ for (const key of ['{user-1:req-started}', '{stream-started}:user-1:req-started']) {
+ const stored = await store.claimIdempotencyKey(
+ key,
+ { streamId: 'wrong', conversationId: 'wrong' },
+ 60,
+ );
+ expect(stored).toMatchObject({
+ claimed: false,
+ existing: {
+ claimToken: claim.existing!.claimToken,
+ startedAt: job.createdAt,
+ generationProtocolVersion: 2,
+ },
+ });
+ }
+ });
+
+ it('accepts an exact legacy started mark whose committed reply was lost', async () => {
+ const streamId = 'stream-lost-legacy-mark-reply';
+ const clientRequestId = 'req-lost-legacy-mark-reply';
+ const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ const actualMark = store.markIdempotencyKeyStarted.bind(store);
+ jest.spyOn(store, 'markIdempotencyKeyStarted').mockImplementationOnce(async (...args) => {
+ expect(await actualMark(...args)).toBe(true);
+ throw new Error('simulated lost legacy mark reply');
+ });
+
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ expect(job).toMatchObject({ streamId, status: 'running' });
+ const legacy = await store.claimIdempotencyKey(
+ `{user-1:${clientRequestId}}`,
+ { streamId: 'wrong', conversationId: 'wrong' },
+ 60,
+ );
+ expect(legacy).toMatchObject({
+ claimed: false,
+ existing: {
+ streamId,
+ conversationId: streamId,
+ claimToken: claim.existing!.claimToken,
+ startedAt: job.createdAt,
+ generationProtocolVersion: 2,
+ },
+ });
+ });
+
+ it.each([
+ {
+ name: 'request id only',
+ options: { idempotencyClientRequestId: 'req-partial' },
+ },
+ {
+ name: 'claim token only',
+ options: { idempotencyClaimToken: 'claim-token' },
+ },
+ {
+ name: 'empty claim token',
+ options: { idempotencyClientRequestId: 'req-partial', idempotencyClaimToken: '' },
+ },
+ ])('rejects $name before reading or creating a job', async ({ options }) => {
+ const getSpy = jest.spyOn(store, 'getJob');
+ const createSpy = jest.spyOn(store, 'createJob');
+ await expect(
+ manager.createJob('stream-partial-idempotency', 'user-1', 'stream-partial-idempotency', {
+ ...options,
+ }),
+ ).rejects.toThrow(/provided together|invalid/);
+ expect(getSpy).not.toHaveBeenCalled();
+ expect(createSpy).not.toHaveBeenCalled();
+ });
+
+ it('recovers a job whose atomic create committed before its reply was lost', async () => {
+ const streamId = 'stream-lost-create-reply';
+ const clientRequestId = 'req-lost-create-reply';
+ const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ const actualCreate = store.createJob.bind(store);
+ let injectLostReply = true;
+ jest.spyOn(store, 'createJob').mockImplementation(async (...args) => {
+ const created = await actualCreate(...args);
+ if (injectLostReply) {
+ injectLostReply = false;
+ throw new Error('simulated lost atomic create reply');
+ }
+ return created;
+ });
+ const membershipRepair = jest.spyOn(store, 'transitionStatus');
+
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ expect(job).toMatchObject({ streamId, status: 'running' });
+ expect(membershipRepair).toHaveBeenCalledWith(streamId, {
+ from: 'running',
+ to: 'running',
+ expectCreatedAt: job.createdAt,
+ });
+ expect(await store.getActiveJobIdsByUser('user-1')).toContain(streamId);
+ expect(await manager.getJob(streamId)).toMatchObject({ createdAt: job.createdAt });
+
+ for (const key of [`{user-1:${clientRequestId}}`, `{${streamId}}:user-1:${clientRequestId}`]) {
+ const stored = await store.claimIdempotencyKey(key, claim.existing!, 60);
+ expect(stored).toMatchObject({
+ claimed: false,
+ existing: {
+ claimToken: claim.existing!.claimToken,
+ startedAt: job.createdAt,
+ generationProtocolVersion: 2,
+ },
+ });
+ }
+
+ const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: job.createdAt },
+ });
+ await expect(
+ manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ });
+
+ it('recovers a direct create with no public idempotency key after its committed reply is lost', async () => {
+ const streamId = 'stream-direct-lost-create-reply';
+ const actualCreate = store.createJob.bind(store);
+ let injectLostReply = true;
+ jest.spyOn(store, 'createJob').mockImplementation(async (...args) => {
+ const created = await actualCreate(...args);
+ if (injectLostReply) {
+ injectLostReply = false;
+ throw new Error('simulated lost direct create reply');
+ }
+ return created;
+ });
+ const membershipRepair = jest.spyOn(store, 'transitionStatus');
+
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ expect(job).toMatchObject({ streamId, status: 'running' });
+ expect(membershipRepair).toHaveBeenCalledWith(streamId, {
+ from: 'running',
+ to: 'running',
+ expectCreatedAt: job.createdAt,
+ });
+ const durable = await store.getJob(streamId);
+ expect(durable).toMatchObject({ createdAt: job.createdAt, status: 'running' });
+ expect(Object.getOwnPropertyDescriptor(durable!, 'creationAttemptId')?.value).toEqual(
+ expect.any(String),
+ );
+ expect(await store.getActiveJobIdsByUser('user-1')).toContain(streamId);
+ });
+
+ it('does not let an older create resume after handoff and overwrite a newer local runtime', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ const transport: IEventTransport = new InMemoryEventTransport();
+ const actualDone = transport.emitDone.bind(transport);
+ let releaseFirstDone!: () => void;
+ let enteredFirstDone!: () => void;
+ const firstDoneEntered = new Promise<void>((resolve) => {
+ enteredFirstDone = resolve;
+ });
+ const firstDoneGate = new Promise<void>((resolve) => {
+ releaseFirstDone = resolve;
+ });
+ let blockFirstDone = true;
+ transport.emitDone = (streamId, event, generationId) => {
+ const emitted = actualDone(streamId, event, generationId);
+ if (blockFirstDone) {
+ blockFirstDone = false;
+ enteredFirstDone();
+ return Promise.resolve(emitted).then(() => firstDoneGate);
+ }
+ return emitted;
+ };
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: false });
+ manager.initialize();
+
+ const streamId = 'stream-local-install-race';
+ const first = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const olderCreate = manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ await firstDoneEntered;
+
+ const newer = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(newer.createdAt).toBeGreaterThan(first.createdAt);
+ expect(newer.abortController.signal.aborted).toBe(false);
+
+ releaseFirstDone();
+ await expect(olderCreate).rejects.toThrow('replaced during initialization');
+ expect(newer.abortController.signal.aborted).toBe(false);
+ expect(await store.getJob(streamId)).toMatchObject({
+ createdAt: newer.createdAt,
+ status: 'running',
+ });
+ expect(await manager.getJob(streamId)).toMatchObject({
+ createdAt: newer.createdAt,
+ abortController: newer.abortController,
+ });
+ });
+
+ it('does not let a superseded helper acknowledge the current creator receipts', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ const transport = new InMemoryEventTransport() as InMemoryEventTransport & IEventTransport;
+ const actualDone = transport.emitDone.bind(transport);
+ transport.emitAbortConfirmed = jest.fn().mockResolvedValue(undefined);
+
+ let releaseOwnerPublish!: () => void;
+ let ownerPublishEntered!: () => void;
+ const ownerPublishGate = new Promise<void>((resolve) => {
+ releaseOwnerPublish = resolve;
+ });
+ const ownerPublishStarted = new Promise<void>((resolve) => {
+ ownerPublishEntered = resolve;
+ });
+ let replacementPublishCount = 0;
+ transport.emitReplacedDoneConfirmed = async (
+ streamId,
+ event,
+ replacedGenerationId,
+ creationAttemptId,
+ ) => {
+ replacementPublishCount += 1;
+ if (replacementPublishCount === 1) {
+ ownerPublishEntered();
+ await ownerPublishGate;
+ }
+ const current = (await store.getJob(streamId)) as CreatedJobData | null;
+ const receipts =
+ current?.replacedJobs ?? (current?.replacedJob != null ? [current.replacedJob] : []);
+ if (
+ current?.creationAttemptId !== creationAttemptId ||
+ !receipts.some((receipt) => receipt.createdAt === replacedGenerationId)
+ ) {
+ throw new Error('replacement receipt was already acknowledged');
+ }
+ actualDone(streamId, event, replacedGenerationId);
+ };
+
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: false });
+ manager.initialize();
+ const supersededManager = new GenerationJobManagerClass();
+ supersededManager.configure({ jobStore: store, eventTransport: transport, isRedis: false });
+ supersededManager.initialize();
+
+ try {
+ const streamId = 'stream-superseded-receipt-owner';
+ const initial = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const actualCreate = store.createJob.bind(store);
+ let releaseSupersededReply!: () => void;
+ let supersededCommitObserved!: () => void;
+ const supersededReplyGate = new Promise<void>((resolve) => {
+ releaseSupersededReply = resolve;
+ });
+ const supersededCommitted = new Promise<void>((resolve) => {
+ supersededCommitObserved = resolve;
+ });
+ let createCount = 0;
+ jest.spyOn(store, 'createJob').mockImplementation(async (...args) => {
+ const created = await actualCreate(...args);
+ createCount += 1;
+ if (createCount === 1) {
+ supersededCommitObserved();
+ await supersededReplyGate;
+ throw new JobCreationSupersededError(created);
+ }
+ return created;
+ });
+ const acknowledgeSpy = jest.spyOn(store, 'acknowledgeReplacedJobs');
+
+ const supersededCreate = supersededManager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ await supersededCommitted;
+ const currentCreate = manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ await ownerPublishStarted;
+
+ releaseSupersededReply();
+ await expect(supersededCreate).rejects.toThrow('replaced during creation');
+ expect(acknowledgeSpy).not.toHaveBeenCalled();
+
+ releaseOwnerPublish();
+ const current = await currentCreate;
+ expect(current.createdAt).toBeGreaterThan(initial.createdAt);
+ expect(await store.getJob(streamId)).toMatchObject({
+ createdAt: current.createdAt,
+ status: 'running',
+ });
+ expect(acknowledgeSpy).toHaveBeenCalledTimes(1);
+ } finally {
+ releaseOwnerPublish();
+ await supersededManager.destroy();
+ }
+ });
+
+ it('retires a local predecessor even when its confirmed handoff fails', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ const transport = new InMemoryEventTransport() as InMemoryEventTransport & IEventTransport;
+ transport.emitAbortConfirmed = jest
+ .fn()
+ .mockRejectedValue(new Error('simulated Redis abort publication failure'));
+ transport.emitReplacedDoneConfirmed = jest
+ .fn()
+ .mockRejectedValue(new Error('simulated Redis terminal publication failure'));
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: false });
+ manager.initialize();
+
+ const streamId = 'stream-failed-local-predecessor-handoff';
+ const predecessor = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const onError = jest.fn();
+ expect(await manager.subscribe(streamId, jest.fn(), jest.fn(), onError)).not.toBeNull();
+ expect(predecessor.abortController.signal.aborted).toBe(false);
+
+ await expect(
+ manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('predecessor handoff could not be confirmed');
+
+ expect(predecessor.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ expect(await store.getJob(streamId)).toMatchObject({
+ status: 'error',
+ error: 'Generation predecessor handoff could not be confirmed',
+ finalEvent: expect.stringContaining('terminal_payload_missing'),
+ });
+ });
+
+ it('closes a terminal predecessor with receipt-authorized DONE without aborting it', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const transport = new InMemoryEventTransport() as InMemoryEventTransport & IEventTransport;
+ transport.emitAbortConfirmed = jest.fn().mockResolvedValue(undefined);
+ transport.emitReplacedDoneConfirmed = jest.fn().mockResolvedValue(undefined);
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: false });
+ manager.initialize();
+
+ const streamId = 'stream-terminal-predecessor-receipt';
+ const predecessor = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(
+ await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: predecessor.createdAt,
+ patch: { completedAt: Date.now() },
+ }),
+ ).toBe(true);
+ const acknowledgeSpy = jest.spyOn(store, 'acknowledgeReplacedJobs');
+
+ const replacement = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ expect(transport.emitAbortConfirmed).not.toHaveBeenCalled();
+ expect(transport.emitReplacedDoneConfirmed).toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: predecessor.createdAt,
+ }),
+ predecessor.createdAt,
+ expect.any(String),
+ );
+ expect(predecessor.abortController.signal.aborted).toBe(false);
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ expect(acknowledgeSpy).toHaveBeenCalledTimes(1);
+ expect(Object.getOwnPropertyDescriptor((await store.getJob(streamId))!, 'replacedJobs')).toBe(
+ undefined,
+ );
+ });
+
+ it('fails closed and retains the receipt when no remote owner acknowledges the abort', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const transport = new InMemoryEventTransport() as InMemoryEventTransport & IEventTransport;
+ transport.emitAbortConfirmed = jest.fn().mockResolvedValue(false);
+ transport.emitReplacedDoneConfirmed = jest.fn().mockResolvedValue(undefined);
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: true });
+ manager.initialize();
+
+ const streamId = 'stream-zero-listener-remote-handoff';
+ const predecessor = await store.createJob(
+ streamId,
+ 'user-1',
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'remote-predecessor-attempt',
+ );
+ await store.updateJob(streamId, { providerAbortReady: true }, predecessor.createdAt);
+ const acknowledgeSpy = jest.spyOn(store, 'acknowledgeReplacedJobs');
+
+ await expect(
+ manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('predecessor handoff could not be confirmed');
+
+ const durable = (await store.getJob(streamId)) as CreatedJobData;
+ expect(durable).toMatchObject({
+ status: 'error',
+ error: 'Generation predecessor handoff could not be confirmed',
+ });
+ expect(durable.replacedJobs).toEqual([
+ expect.objectContaining({ createdAt: predecessor.createdAt, status: 'running' }),
+ ]);
+ expect(acknowledgeSpy).not.toHaveBeenCalled();
+ });
+
+ it('self-fences a provider when a missed replacement abort makes its Redis append stale', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ });
+ manager.initialize();
+
+ const streamId = 'stream-missed-abort-append-fence';
+ const predecessor = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const onError = jest.fn();
+ const subscription = await manager.subscribe(streamId, jest.fn(), jest.fn(), onError);
+ expect(subscription).not.toBeNull();
+ const replacement = await store.createJob(
+ streamId,
+ 'user-1',
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'remote-replacement-attempt',
+ );
+
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: { delta: 'stale provider output' },
+ });
+ await Promise.resolve();
+
+ expect(predecessor.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ expect(await store.getJob(streamId)).toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ });
+
+ it('self-fences an ordinary provider when its Redis append rejects', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ });
+ manager.initialize();
+
+ const streamId = 'stream-append-rejection-fence';
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const onError = jest.fn();
+ expect(await manager.subscribe(streamId, jest.fn(), jest.fn(), onError)).not.toBeNull();
+ jest.spyOn(store, 'appendChunk').mockRejectedValueOnce(new Error('simulated Redis outage'));
+
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: { delta: 'uncoordinated provider output' },
+ });
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ });
+
+ it('self-fences when a chunk append wins but its active-only publication is fenced', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ const transport = new InMemoryEventTransport();
+ manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: true });
+ manager.initialize();
+
+ const streamId = 'stream-publication-status-fence';
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const onError = jest.fn();
+ expect(await manager.subscribe(streamId, jest.fn(), jest.fn(), onError)).not.toBeNull();
+ registerChunkPublicationCapability(transport, async () => false);
+
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: { delta: 'publication lost its active-generation CAS' },
+ });
+
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ });
+
+ it('does not let a late stale-append result retire a newer local runtime', async () => {
+ await manager.destroy();
+ store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ });
+ manager.initialize();
+
+ const streamId = 'stream-late-append-fence';
+ const predecessor = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ let settleAppend!: (appended: boolean) => void;
+ const delayedAppend = new Promise<boolean>((resolve) => {
+ settleAppend = resolve;
+ });
+ jest.spyOn(store, 'appendChunk').mockImplementationOnce(() => delayedAppend);
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: { delta: 'in-flight predecessor output' },
+ });
+
+ const replacement = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(replacement.createdAt).toBeGreaterThan(predecessor.createdAt);
+ expect(replacement.abortController.signal.aborted).toBe(false);
+
+ settleAppend(false);
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ expect(await manager.getJob(streamId)).toMatchObject({
+ createdAt: replacement.createdAt,
+ abortController: replacement.abortController,
+ });
+ });
+
+ it('acknowledges predecessor receipts so ordinary stream reuse does not exhaust the chain cap', async () => {
+ const streamId = 'stream-replacement-receipt-ack';
+ let latestCreatedAt = 0;
+ for (let index = 0; index < 40; index++) {
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(job.createdAt).toBeGreaterThan(latestCreatedAt);
+ latestCreatedAt = job.createdAt;
+ }
+
+ const durable = await store.getJob(streamId);
+ expect(durable).toMatchObject({ createdAt: latestCreatedAt, status: 'running' });
+ expect(Object.getOwnPropertyDescriptor(durable!, 'replacedJobs')).toBeUndefined();
+ });
+
+ it('terminalizes the exact committed epoch when legacy verification fails during recovery', async () => {
+ const streamId = 'stream-lost-create-legacy-failure';
+ const clientRequestId = 'req-lost-create-legacy-failure';
+ const claim = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ const actualCreate = store.createJob.bind(store);
+ jest.spyOn(store, 'createJob').mockImplementationOnce(async (...args) => {
+ await actualCreate(...args);
+ throw new Error('simulated lost atomic create reply');
+ });
+ const actualClaim = store.claimIdempotencyKey.bind(store);
+ let failLegacyProbe = true;
+ jest.spyOn(store, 'claimIdempotencyKey').mockImplementation((key, value, ttlSeconds) => {
+ if (failLegacyProbe && key === `{user-1:${clientRequestId}}`) {
+ failLegacyProbe = false;
+ return Promise.reject(new Error('simulated legacy probe outage'));
+ }
+ return actualClaim(key, value, ttlSeconds);
+ });
+
+ await expect(
+ manager.createJob(streamId, 'user-1', streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('simulated lost atomic create reply');
+
+ const durable = await store.getJob(streamId);
+ expect(durable).toMatchObject({
+ status: 'error',
+ error: 'Generation idempotency rollout fence could not be recovered',
+ finalEvent: expect.stringContaining('terminal_payload_missing'),
+ });
+ const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: durable!.createdAt },
+ });
+ await expect(
+ manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ });
+
+ it('terminalizes and preserves the primary fence when the legacy started mark fails', async () => {
+ const claim = await manager.claimGeneration(
+ 'user-1',
+ 'req-mark-failure',
+ 'stream-mark-failure',
+ 'stream-mark-failure',
+ 2,
+ );
+ jest.spyOn(store, 'markIdempotencyKeyStarted').mockResolvedValue(false);
+
+ await expect(
+ manager.createJob('stream-mark-failure', 'user-1', 'stream-mark-failure', {
+ idempotencyClientRequestId: 'req-mark-failure',
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('rollout fence could not be committed');
+ expect(await store.getJob('stream-mark-failure')).toMatchObject({
+ status: 'error',
+ finalEvent: expect.stringContaining('terminal_payload_missing'),
+ });
+
+ await manager.releaseGeneration(
+ 'user-1',
+ 'req-mark-failure',
+ 'stream-mark-failure',
+ claim.existing,
+ );
+ const primary = await store.claimIdempotencyKey(
+ '{stream-mark-failure}:user-1:req-mark-failure',
+ { streamId: 'wrong', conversationId: 'wrong' },
+ 60,
+ );
+ expect(primary).toMatchObject({
+ claimed: false,
+ existing: { claimToken: claim.existing!.claimToken, startedAt: expect.any(Number) },
+ });
+ });
+
+ it('fails closed when primary and legacy token-bearing mirrors disagree', async () => {
+ const claimedAt = Date.now();
+ await store.claimIdempotencyKey(
+ '{user-1:req-mismatch}',
+ {
+ streamId: 'stream-mismatch',
+ conversationId: 'stream-mismatch',
+ claimedAt,
+ claimToken: 'legacy-token',
+ generationProtocolVersion: 2,
+ },
+ 1200,
+ );
+ await store.claimIdempotencyKey(
+ '{stream-mismatch}:user-1:req-mismatch',
+ {
+ streamId: 'stream-mismatch',
+ conversationId: 'stream-mismatch',
+ claimedAt,
+ claimToken: 'different-primary-token',
+ generationProtocolVersion: 2,
+ },
+ 1200,
+ );
+
+ await expect(
+ manager.claimGeneration('user-1', 'req-mismatch', 'stream-mismatch', 'stream-mismatch', 2),
+ ).rejects.toThrow('Mismatched legacy and primary');
+ });
+
it('does NOT dedup a distinct submission (e.g. regenerate reuses the user message id)', async () => {
await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
// A regenerate is a fresh ask() → fresh clientRequestId, so it must start its own generation.
const regenerate = await manager.claimGeneration('user-1', 'req-2', 'stream-a', 'convo-a');
- expect(regenerate).toEqual({ claimed: true });
+ expect(regenerate).toEqual(
+ expect.objectContaining({
+ claimed: true,
+ existing: expect.objectContaining({ claimToken: expect.any(String) }),
+ }),
+ );
});
it('scopes claims per user', async () => {
await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
const otherUser = await manager.claimGeneration('user-2', 'req-1', 'stream-z', 'convo-z');
- expect(otherUser).toEqual({ claimed: true });
+ expect(otherUser).toEqual(
+ expect.objectContaining({
+ claimed: true,
+ existing: expect.objectContaining({ claimToken: expect.any(String) }),
+ }),
+ );
});
it('allows a fresh claim after release', async () => {
- await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
- await manager.releaseGeneration('user-1', 'req-1');
- const again = await manager.claimGeneration('user-1', 'req-1', 'stream-c', 'convo-c');
- expect(again).toEqual({ claimed: true });
+ const first = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
+ await manager.releaseGeneration('user-1', 'req-1', 'stream-a', first.existing);
+ const again = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
+ expect(again).toEqual(
+ expect.objectContaining({
+ claimed: true,
+ existing: expect.objectContaining({ claimToken: expect.any(String) }),
+ }),
+ );
+ });
+
+ it('atomically fences an abandoned pre-create owner during takeover', async () => {
+ const original = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
+ expect(original.existing).toBeDefined();
+
+ const takeover = await manager.takeoverGeneration(
+ 'user-1',
+ 'req-1',
+ 'stream-a',
+ original.existing!,
+ );
+ expect(takeover).toEqual(
+ expect.objectContaining({
+ claimed: true,
+ existing: expect.objectContaining({
+ streamId: 'stream-a',
+ claimToken: expect.any(String),
+ }),
+ }),
+ );
+ expect(takeover.existing?.claimToken).not.toBe(original.existing?.claimToken);
+
+ await expect(
+ manager.createJob('stream-a', 'user-1', 'convo-a', {
+ idempotencyClientRequestId: 'req-1',
+ idempotencyClaimToken: original.existing!.claimToken,
+ }),
+ ).rejects.toThrow('claim was taken over');
+
+ await expect(
+ manager.createJob('stream-a', 'user-1', 'convo-a', {
+ idempotencyClientRequestId: 'req-1',
+ idempotencyClaimToken: takeover.existing!.claimToken,
+ }),
+ ).resolves.toEqual(expect.objectContaining({ streamId: 'stream-a' }));
+ });
+
+ it('rolls back the newly-taken primary if the legacy takeover loses', async () => {
+ const original = await manager.claimGeneration(
+ 'user-1',
+ 'req-rollback',
+ 'stream-rollback',
+ 'stream-rollback',
+ 2,
+ );
+ const actualTakeover = store.takeoverIdempotencyKey.bind(store);
+ jest
+ .spyOn(store, 'takeoverIdempotencyKey')
+ .mockImplementation((key, expected, value, ttlSeconds) =>
+ key === '{user-1:req-rollback}'
+ ? Promise.resolve(false)
+ : actualTakeover(key, expected, value, ttlSeconds),
+ );
+
+ const takeover = await manager.takeoverGeneration(
+ 'user-1',
+ 'req-rollback',
+ 'stream-rollback',
+ original.existing!,
+ );
+ expect(takeover).toEqual({ claimed: false, source: 'primary' });
+
+ const primaryAfterRollback = await store.claimIdempotencyKey(
+ '{stream-rollback}:user-1:req-rollback',
+ original.existing!,
+ 1200,
+ );
+ expect(primaryAfterRollback.claimed).toBe(true);
+ const legacy = await store.claimIdempotencyKey(
+ '{user-1:req-rollback}',
+ { streamId: 'wrong', conversationId: 'wrong' },
+ 1200,
+ );
+ expect(legacy.existing?.claimToken).toBe(original.existing?.claimToken);
+ });
+
+ it('finishes a primary takeover whose committed reply was lost', async () => {
+ jest.useFakeTimers();
+ try {
+ const originalTime = new Date('2026-07-20T00:00:00Z');
+ jest.setSystemTime(originalTime);
+ const original = await manager.claimGeneration(
+ 'user-1',
+ 'req-lost-takeover-reply',
+ 'stream-lost-takeover-reply',
+ 'stream-lost-takeover-reply',
+ 2,
+ );
+ const actualTakeover = store.takeoverIdempotencyKey.bind(store);
+ let injectLostReply = true;
+ jest
+ .spyOn(store, 'takeoverIdempotencyKey')
+ .mockImplementation(async (key, expected, value, ttlSeconds) => {
+ const result = await actualTakeover(key, expected, value, ttlSeconds);
+ if (
+ injectLostReply &&
+ key === '{stream-lost-takeover-reply}:user-1:req-lost-takeover-reply'
+ ) {
+ injectLostReply = false;
+ throw new Error('simulated lost primary CAS reply');
+ }
+ return result;
+ });
+
+ // A node clock may step backward between the original claim and its
+ // takeover. Lineage recovery must not reject that legitimate CAS split.
+ jest.setSystemTime(new Date(originalTime.getTime() - 60_000));
+ const takeover = await manager.takeoverGeneration(
+ 'user-1',
+ 'req-lost-takeover-reply',
+ 'stream-lost-takeover-reply',
+ original.existing!,
+ );
+ expect(takeover).toMatchObject({
+ claimed: true,
+ source: 'primary',
+ existing: {
+ previousClaimToken: original.existing!.claimToken,
+ claimToken: expect.any(String),
+ },
+ });
+ expect(takeover.existing!.claimedAt).toBeGreaterThanOrEqual(original.existing!.claimedAt!);
+ const retry = await manager.claimGeneration(
+ 'user-1',
+ 'req-lost-takeover-reply',
+ 'stream-lost-takeover-reply',
+ 'stream-lost-takeover-reply',
+ 2,
+ );
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { claimToken: takeover.existing!.claimToken },
+ });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('repairs only a lineage-proven unstarted takeover split', async () => {
+ const original = await manager.claimGeneration(
+ 'user-1',
+ 'req-split-repair',
+ 'stream-split-repair',
+ 'stream-split-repair',
+ 2,
+ );
+ const splitPrimary = {
+ ...original.existing!,
+ claimedAt: original.existing!.claimedAt! + 1,
+ claimToken: 'split-primary-token',
+ previousClaimToken: original.existing!.claimToken,
+ };
+ expect(
+ await store.takeoverIdempotencyKey(
+ '{stream-split-repair}:user-1:req-split-repair',
+ original.existing!,
+ splitPrimary,
+ 1200,
+ ),
+ ).toBe(true);
+
+ const repaired = await manager.claimGeneration(
+ 'user-1',
+ 'req-split-repair',
+ 'stream-split-repair',
+ 'stream-split-repair',
+ 2,
+ );
+ expect(repaired).toMatchObject({
+ claimed: false,
+ existing: { claimToken: 'split-primary-token' },
+ });
+ const legacy = await store.claimIdempotencyKey(
+ '{user-1:req-split-repair}',
+ original.existing!,
+ 1200,
+ );
+ expect(legacy.existing).toEqual(splitPrimary);
+ });
+
+ it('never takes over a claim already marked started with job creation', async () => {
+ const original = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a');
+ await manager.createJob('stream-a', 'user-1', 'convo-a', {
+ idempotencyClientRequestId: 'req-1',
+ idempotencyClaimToken: original.existing!.claimToken,
+ });
+
+ const takeover = await manager.takeoverGeneration(
+ 'user-1',
+ 'req-1',
+ 'stream-a',
+ original.existing!,
+ );
+ expect(takeover).toEqual(
+ expect.objectContaining({
+ claimed: false,
+ source: 'primary',
+ existing: expect.objectContaining({ startedAt: expect.any(Number) }),
+ }),
+ );
+ });
+
+ it('does not release either started tombstone with a stale pre-create value', async () => {
+ const original = await manager.claimGeneration(
+ 'user-1',
+ 'req-release-started',
+ 'stream-release-started',
+ 'stream-release-started',
+ 2,
+ );
+ const job = await manager.createJob(
+ 'stream-release-started',
+ 'user-1',
+ 'stream-release-started',
+ {
+ idempotencyClientRequestId: 'req-release-started',
+ idempotencyClaimToken: original.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ },
+ );
+
+ await manager.releaseGeneration(
+ 'user-1',
+ 'req-release-started',
+ 'stream-release-started',
+ original.existing,
+ );
+ for (const key of [
+ '{user-1:req-release-started}',
+ '{stream-release-started}:user-1:req-release-started',
+ ]) {
+ const stored = await store.claimIdempotencyKey(
+ key,
+ { streamId: 'wrong', conversationId: 'wrong' },
+ 60,
+ );
+ expect(stored.existing).toMatchObject({
+ claimToken: original.existing!.claimToken,
+ startedAt: job.createdAt,
+ });
+ }
+ });
+
+ it('tombstones an uncorrelated v1 legacy job before 503 so deletion cannot enable takeover', async () => {
+ const job = await manager.createJob('stream-legacy-live', 'user-1', 'stream-legacy-live', {
+ initialMetadata: { generationProtocolVersion: 1 },
+ });
+ const reacquired = await manager.claimGeneration(
+ 'user-1',
+ 'req-legacy-expired',
+ 'stream-legacy-live',
+ 'stream-legacy-live',
+ 2,
+ );
+ expect(reacquired.claimed).toBe(true);
+
+ await expect(
+ manager.resumeClaimedGeneration(
+ 'user-1',
+ 'req-legacy-expired',
+ 'stream-legacy-live',
+ reacquired.existing!,
+ ),
+ ).rejects.toThrow('conservatively fenced');
+
+ for (const key of [
+ '{user-1:req-legacy-expired}',
+ '{stream-legacy-live}:user-1:req-legacy-expired',
+ ]) {
+ const stored = await store.claimIdempotencyKey(key, reacquired.existing!, 60);
+ expect(stored.existing).toMatchObject({
+ claimToken: reacquired.existing!.claimToken,
+ startedAt: job.createdAt,
+ generationProtocolVersion: 1,
+ });
+ }
+
+ expect(await store.deleteJob('stream-legacy-live', job.createdAt)).toBe(true);
+ const retry = await manager.claimGeneration(
+ 'user-1',
+ 'req-legacy-expired',
+ 'stream-legacy-live',
+ 'stream-legacy-live',
+ 2,
+ );
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: job.createdAt, generationProtocolVersion: 1 },
+ });
+ await expect(
+ manager.takeoverGeneration(
+ 'user-1',
+ 'req-legacy-expired',
+ 'stream-legacy-live',
+ retry.existing!,
+ ),
+ ).resolves.toMatchObject({ claimed: false });
+ await expect(
+ manager.createJob('stream-legacy-live', 'user-1', 'stream-legacy-live', {
+ idempotencyClientRequestId: 'req-legacy-expired',
+ idempotencyClaimToken: reacquired.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('claim was taken over');
+ });
+
+ it.each([
+ { jobProtocol: 1 as const, retryProtocol: 2 as const },
+ { jobProtocol: 2 as const, retryProtocol: 1 as const },
+ ])(
+ 'adopts a correlated v$jobProtocol job with a v$retryProtocol fresh claim to the immutable job protocol',
+ async ({ jobProtocol, retryProtocol }) => {
+ const suffix = `${jobProtocol}-${retryProtocol}`;
+ const streamId = `stream-protocol-adopt-${suffix}`;
+ const clientRequestId = `req-protocol-adopt-${suffix}`;
+ const original = await manager.claimGeneration(
+ 'user-1',
+ clientRequestId,
+ streamId,
+ streamId,
+ jobProtocol,
+ );
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: original.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: jobProtocol },
+ });
+ await store.releaseIdempotencyKey(
+ `{${streamId}}:user-1:${clientRequestId}`,
+ original.existing,
+ );
+ await store.releaseIdempotencyKey(`{user-1:${clientRequestId}}`, original.existing);
+
+ const fresh = await manager.claimGeneration(
+ 'user-1',
+ clientRequestId,
+ streamId,
+ streamId,
+ retryProtocol,
+ );
+ expect(fresh.claimed).toBe(true);
+ const resumed = await manager.resumeClaimedGeneration(
+ 'user-1',
+ clientRequestId,
+ streamId,
+ fresh.existing!,
+ );
+ expect(resumed).toMatchObject({
+ startedAt: job.createdAt,
+ generationProtocolVersion: jobProtocol,
+ });
+
+ expect(await store.deleteJob(streamId, job.createdAt)).toBe(true);
+ const retry = await manager.claimGeneration(
+ 'user-1',
+ clientRequestId,
+ streamId,
+ streamId,
+ retryProtocol,
+ );
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: job.createdAt, generationProtocolVersion: jobProtocol },
+ });
+ await expect(
+ manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ },
+ );
+
+ it('tombstones a fresh claim when terminal deletion wins the live-adoption race', async () => {
+ jest.useFakeTimers();
+ try {
+ const startedAt = new Date('2026-07-20T00:00:00Z');
+ jest.setSystemTime(startedAt);
+ const streamId = 'stream-terminal-adopt-race';
+ const clientRequestId = 'req-terminal-adopt-race';
+ const original = await manager.claimGeneration(
+ 'user-1',
+ clientRequestId,
+ streamId,
+ streamId,
+ 2,
+ );
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: original.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ // Both mirrors have naturally expired, while a custom long-running job
+ // is still active and exactly correlated to this request.
+ jest.setSystemTime(new Date(startedAt.getTime() + 27 * 60 * 60 * 1000));
+ const fresh = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ expect(fresh.claimed).toBe(true);
+
+ jest.spyOn(store, 'adoptIdempotencyKeyForJob').mockImplementation(async () => {
+ expect(
+ await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).toBe(true);
+ expect(await store.deleteJob(streamId, job.createdAt)).toBe(true);
+ return false;
+ });
+
+ await expect(
+ manager.resumeClaimedGeneration('user-1', clientRequestId, streamId, fresh.existing!),
+ ).rejects.toThrow('conservatively fenced');
+
+ for (const key of [
+ `{user-1:${clientRequestId}}`,
+ `{${streamId}}:user-1:${clientRequestId}`,
+ ]) {
+ const stored = await store.claimIdempotencyKey(key, fresh.existing!, 60);
+ expect(stored).toMatchObject({
+ claimed: false,
+ existing: {
+ claimToken: fresh.existing!.claimToken,
+ startedAt: job.createdAt,
+ generationProtocolVersion: 2,
+ },
+ });
+ }
+
+ const retry = await manager.claimGeneration('user-1', clientRequestId, streamId, streamId, 2);
+ expect(retry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: job.createdAt },
+ });
+ await expect(
+ manager.takeoverGeneration('user-1', clientRequestId, streamId, retry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ it('adopts an expired lease for the same live paused generation instead of replacing it', async () => {
+ jest.useFakeTimers();
+ try {
+ const startedAt = new Date('2026-07-20T00:00:00Z');
+ jest.setSystemTime(startedAt);
+ const original = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a', 2);
+ const job = await manager.createJob('stream-a', 'user-1', 'convo-a', {
+ idempotencyClientRequestId: 'req-1',
+ idempotencyClaimToken: original.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const action = buildPendingAction(
+ buildToolApprovalPayload([
+ { name: 'shell', arguments: { command: 'ls' }, tool_call_id: 'call-1' },
+ ]),
+ {
+ streamId: 'stream-a',
+ conversationId: 'convo-a',
+ runId: 'run-1',
+ responseMessageId: 'message-1',
+ },
+ );
+ expect(await manager.approvals.pause('stream-a', action)).toBe(true);
+
+ jest.setSystemTime(new Date(startedAt.getTime() + 26 * 60 * 60 * 1000));
+ const reacquired = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a', 2);
+ expect(reacquired.claimed).toBe(true);
+
+ const resumed = await manager.resumeClaimedGeneration(
+ 'user-1',
+ 'req-1',
+ 'stream-a',
+ reacquired.existing!,
+ );
+ expect(resumed).toEqual(
+ expect.objectContaining({
+ streamId: 'stream-a',
+ conversationId: 'convo-a',
+ startedAt: job.createdAt,
+ }),
+ );
+ const retry = await manager.claimGeneration('user-1', 'req-1', 'stream-a', 'convo-a', 2);
+ expect(retry).toEqual(
+ expect.objectContaining({
+ claimed: false,
+ existing: expect.objectContaining({ startedAt: job.createdAt }),
+ }),
+ );
+ } finally {
+ jest.useRealTimers();
+ }
});
});
diff --git a/packages/api/src/stream/__tests__/jobStoreCapabilities.spec.ts b/packages/api/src/stream/__tests__/jobStoreCapabilities.spec.ts
new file mode 100644
index 0000000000..53a69183d5
--- /dev/null
+++ b/packages/api/src/stream/__tests__/jobStoreCapabilities.spec.ts
@@ -0,0 +1,136 @@
+import type { IJobStore } from '../interfaces/IJobStore';
+import {
+ assertJobStoreV2,
+ getMissingJobStoreV2Methods,
+ JOB_STORE_V2_REQUIRED_METHODS,
+} from '../jobStoreCapabilities';
+import { InMemoryEventTransport } from '../implementations/InMemoryEventTransport';
+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
+import { GenerationJobManagerClass } from '../GenerationJobManager';
+
+/**
+ * Deliberately implements only the public pre-v2 contract. In particular,
+ * appendChunk returns Promise<void>, proving older custom stores remain
+ * source-assignable to IJobStore.
+ */
+const legacyStore = {
+ async initialize() {},
+ async createJob(streamId, userId, conversationId, tenantId) {
+ return {
+ streamId,
+ userId,
+ ...(tenantId != null && { tenantId }),
+ status: 'running' as const,
+ createdAt: 1,
+ ...(conversationId != null && { conversationId }),
+ syncSent: false,
+ };
+ },
+ async getJob() {
+ return null;
+ },
+ async updateJob() {},
+ async transitionStatus() {
+ return false;
+ },
+ async claimIdempotencyKey() {
+ return { claimed: true };
+ },
+ async releaseIdempotencyKey() {},
+ async deleteJob() {
+ return false;
+ },
+ async hasJob() {
+ return false;
+ },
+ async getRunningJobs() {
+ return [];
+ },
+ async cleanup() {
+ return 0;
+ },
+ async getJobCount() {
+ return 0;
+ },
+ async getJobCountByStatus() {
+ return 0;
+ },
+ async destroy() {},
+ async getActiveJobIdsByUser() {
+ return [];
+ },
+ setGraph() {},
+ setContentParts() {},
+ async getContentParts() {
+ return null;
+ },
+ async getRunSteps() {
+ return [];
+ },
+ async appendChunk() {},
+ clearContentState() {},
+ setCollectedUsage() {},
+ getCollectedUsage() {
+ return [];
+ },
+ async enqueueSteer() {
+ return 1;
+ },
+ async drainSteers() {
+ return [];
+ },
+ async closeAndDrainSteers() {
+ return [];
+ },
+ async peekSteers() {
+ return [];
+ },
+ async removeSteer() {
+ return false;
+ },
+ async parkSteers() {},
+ async claimParkedSteers() {
+ return undefined;
+ },
+ async clearSteers() {},
+} satisfies IJobStore;
+
+describe('job-store v2 capability boundary', () => {
+ test('keeps a legacy-shaped custom store type-assignable', async () => {
+ const store: IJobStore = legacyStore;
+
+ await expect(store.appendChunk('legacy-stream', { event: 'chunk' })).resolves.toBeUndefined();
+ });
+
+ test('reports every v2 capability missing from a legacy store', () => {
+ expect(getMissingJobStoreV2Methods(legacyStore)).toEqual(JOB_STORE_V2_REQUIRED_METHODS);
+ expect(() => assertJobStoreV2(legacyStore)).toThrow(
+ `Invalid generation job store: missing required IJobStoreV2 method(s): ${JOB_STORE_V2_REQUIRED_METHODS.join(
+ ', ',
+ )}`,
+ );
+ });
+
+ test('accepts the built-in v2 store', () => {
+ expect(() => assertJobStoreV2(new InMemoryJobStore())).not.toThrow();
+ });
+
+ test('rejects a legacy store before mutating manager configuration', () => {
+ const manager = new GenerationJobManagerClass();
+ const originalStore = manager.getJobStore();
+
+ expect(() =>
+ manager.configure({
+ jobStore: legacyStore,
+ eventTransport: new InMemoryEventTransport(),
+ }),
+ ).toThrow('Invalid generation job store: missing required IJobStoreV2 method(s)');
+ expect(manager.getJobStore()).toBe(originalStore);
+ });
+
+ test('rejects a legacy store passed to the manager constructor', () => {
+ expect(() => new GenerationJobManagerClass({ jobStore: legacyStore })).toThrow(
+ 'Invalid generation job store: missing required IJobStoreV2 method(s)',
+ );
+ });
+});
diff --git a/packages/api/src/stream/__tests__/pendingAction.spec.ts b/packages/api/src/stream/__tests__/pendingAction.spec.ts
index deda3c9d75..ba7ce549a3 100644
--- a/packages/api/src/stream/__tests__/pendingAction.spec.ts
+++ b/packages/api/src/stream/__tests__/pendingAction.spec.ts
@@ -1,6 +1,7 @@
import type { Agents } from 'librechat-data-provider';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
+import { PAUSE_PERSISTENCE_TIMEOUT_ERROR } from '~/stream/interfaces/IJobStore';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { ApprovalLifecycle } from '~/stream/ApprovalLifecycle';
@@ -93,6 +94,316 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
expect(liveJob?.metadata.discoveredTools).toBeUndefined();
});
+ test('commits discoveries and the response barrier in the visible pause transition', async () => {
+ const streamId = 'stream-pause-discoveries-and-persistence';
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ discoveredTools: ['save_issue_mcp_linear'],
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ status: 'requires_action',
+ metadata: {
+ discoveredTools: ['save_issue_mcp_linear'],
+ terminalPersistencePending: true,
+ },
+ });
+ });
+
+ test('holds abort behind the paused-response persistence barrier', async () => {
+ const streamId = 'stream-pause-persistence-abort-race';
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+
+ let abortSettled = false;
+ const aborting = manager
+ .abortJob(streamId, { expectedCreatedAt: job.createdAt })
+ .then((result) => {
+ abortSettled = true;
+ return result;
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+
+ expect(abortSettled).toBe(false);
+ await expect(
+ manager.approvals.ownsPausePersistence(streamId, action.actionId, job.createdAt),
+ ).resolves.toBe(true);
+ await expect(
+ manager.approvals.finishPausePersistence(streamId, action.actionId, job.createdAt),
+ ).resolves.toBe(true);
+ await expect(aborting).resolves.toMatchObject({
+ success: true,
+ finalEvent: expect.objectContaining({ aborted: true }),
+ });
+ });
+
+ test('holds approval resume behind the paused-response persistence barrier', async () => {
+ const streamId = 'stream-pause-persistence-resume-race';
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+
+ let resumeSettled = false;
+ const resuming = manager.approvals
+ .resolve(streamId, action.actionId, undefined, job.createdAt)
+ .then((result) => {
+ resumeSettled = true;
+ return result;
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+
+ expect(resumeSettled).toBe(false);
+ await expect(
+ manager.approvals.finishPausePersistence(streamId, action.actionId, job.createdAt),
+ ).resolves.toBe(true);
+ await expect(resuming).resolves.toBe(true);
+ await expect(manager.getJobStatus(streamId)).resolves.toBe('running');
+ });
+
+ test('failed pause persistence atomically beats a waiting resume and parks steers', async () => {
+ const streamId = 'stream-pause-persistence-failure-race';
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+ const waitingSteer = {
+ steerId: 'steer-waiting-on-failed-pause',
+ text: 'preserve me after the failed pause',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ };
+ await expect(manager.steering.enqueue(streamId, waitingSteer, job.createdAt)).resolves.toBe(
+ 1,
+ );
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+
+ let resumeSettled = false;
+ const resuming = manager.approvals
+ .resolve(streamId, action.actionId, undefined, job.createdAt)
+ .then((result) => {
+ resumeSettled = true;
+ return result;
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(resumeSettled).toBe(false);
+
+ await expect(
+ manager.failPausePersistence(
+ streamId,
+ action.actionId,
+ 'paused response was not persisted',
+ job.createdAt,
+ ),
+ ).resolves.toBe(true);
+ await expect(resuming).resolves.toBe(false);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ status: 'error',
+ error: 'paused response was not persisted',
+ });
+ const failedStoredJob = await jobStore.getJob(streamId);
+ expect(failedStoredJob?.pendingAction).toBeUndefined();
+ expect(failedStoredJob?.pendingActionId).toBeUndefined();
+ expect(failedStoredJob?.terminalPersistencePending).toBeUndefined();
+ expect(failedStoredJob?.terminalPersistenceStartedAt).toBeUndefined();
+ await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([]);
+ await expect(manager.steering.claim(streamId, { userId: 'user-1' })).resolves.toEqual([
+ expect.objectContaining({
+ steerId: waitingSteer.steerId,
+ text: waitingSteer.text,
+ }),
+ ]);
+ });
+
+ test('failed pause persistence is fenced to the exact action and generation', async () => {
+ const streamId = 'stream-pause-persistence-failure-fence';
+ const predecessor = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId, { actionId: 'reused-pause-action' });
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: predecessor.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+
+ await expect(
+ manager.failPausePersistence(
+ streamId,
+ 'different-action',
+ 'must not win',
+ predecessor.createdAt,
+ ),
+ ).resolves.toBe(false);
+ await expect(
+ manager.approvals.ownsPausePersistence(streamId, action.actionId, predecessor.createdAt),
+ ).resolves.toBe(true);
+
+ await new Promise<void>((resolve) => setTimeout(resolve, 2));
+ const replacement = await manager.createJob(streamId, 'user-1');
+ await expect(
+ manager.failPausePersistence(
+ streamId,
+ action.actionId,
+ 'stale predecessor failure',
+ predecessor.createdAt,
+ ),
+ ).resolves.toBe(false);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ });
+
+ test('fails a crashed pause writer closed after the bounded barrier deadline', async () => {
+ const streamId = 'stream-stale-pause-persistence';
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+
+ try {
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+ const waitingSteer = {
+ steerId: 'stale-pause-steer',
+ text: 'recover me instead of resuming unsafe history',
+ userId: 'user-1',
+ createdAt: 1_000,
+ };
+ await expect(manager.steering.enqueue(streamId, waitingSteer, job.createdAt)).resolves.toBe(
+ 1,
+ );
+ const onError = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+
+ now.mockReturnValue(31_001);
+ await expect(
+ manager.approvals.resolve(streamId, action.actionId, undefined, job.createdAt),
+ ).resolves.toBe(false);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ status: 'error',
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ });
+ const failedJob = await jobStore.getJob(streamId);
+ expect(failedJob?.pendingAction).toBeUndefined();
+ expect(failedJob?.pendingActionId).toBeUndefined();
+ expect(failedJob?.terminalPersistencePending).toBeUndefined();
+ expect(failedJob?.terminalPersistenceStartedAt).toBeUndefined();
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
+ await expect(manager.steering.claim(streamId, { userId: 'user-1' })).resolves.toEqual([
+ expect.objectContaining({ steerId: waitingSteer.steerId, text: waitingSteer.text }),
+ ]);
+ subscription?.unsubscribe();
+ } finally {
+ now.mockRestore();
+ }
+ });
+
+ test('relays a store-only pause timeout into the matching attached runtime once', async () => {
+ const streamId = 'stream-remote-stale-pause-persistence';
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+
+ try {
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+ const onError = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+ const broadcast = jest.spyOn(eventTransport, 'emitError');
+ const clearContentState = jest.spyOn(jobStore, 'clearContentState');
+ expect(job.abortController.signal.aborted).toBe(false);
+
+ now.mockReturnValue(31_001);
+ // Model another replica/store cleanup winning the terminal CAS without
+ // access to this manager's in-process runtime.
+ await jobStore.cleanup();
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ status: 'error',
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ });
+ expect(onError).not.toHaveBeenCalled();
+ expect(clearContentState).not.toHaveBeenCalled();
+
+ const managerWithCleanup = manager as unknown as { cleanup(): Promise<void> };
+ await managerWithCleanup.cleanup();
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledTimes(1);
+ expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
+ expect(broadcast).toHaveBeenCalledWith(
+ streamId,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ job.createdAt,
+ );
+ expect(clearContentState).toHaveBeenCalledWith(streamId, job.createdAt);
+
+ await managerWithCleanup.cleanup();
+ expect(onError).toHaveBeenCalledTimes(1);
+ subscription?.unsubscribe();
+ } finally {
+ now.mockRestore();
+ }
+ });
+
+ test('notifies the attached runtime when a locally-won timeout row is immediately evicted', async () => {
+ const streamId = 'stream-zero-ttl-stale-pause-persistence';
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+
+ try {
+ const job = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId);
+ const onError = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
+ await expect(
+ manager.approvals.pause(streamId, action, {
+ expectedCreatedAt: job.createdAt,
+ persistencePending: true,
+ }),
+ ).resolves.toBe(true);
+ const originalGetJob = jobStore.getJob.bind(jobStore);
+ jest.spyOn(jobStore, 'getJob').mockImplementation(async (...args) => {
+ const stored = await originalGetJob(...args);
+ return stored?.status === 'error' ? null : stored;
+ });
+
+ now.mockReturnValue(31_001);
+ await expect(
+ manager.approvals.resolve(streamId, action.actionId, undefined, job.createdAt),
+ ).resolves.toBe(false);
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(PAUSE_PERSISTENCE_TIMEOUT_ERROR);
+ subscription?.unsubscribe();
+ } finally {
+ now.mockRestore();
+ }
+ });
+
test('returns false when the job is already terminal', async () => {
const streamId = 'stream-pause-dead';
await manager.createJob(streamId, 'user-1');
@@ -107,6 +418,22 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
expect(await manager.approvals.pause('nonexistent', buildAction('nonexistent'))).toBe(false);
});
+ test('a predecessor interrupt cannot pause a replacement generation', async () => {
+ const streamId = 'stream-pause-epoch-fence';
+ const predecessor = await manager.createJob(streamId, 'user-1');
+ const replacement = await manager.createJob(streamId, 'user-1');
+
+ await expect(
+ manager.approvals.pause(streamId, buildAction(streamId), {
+ expectedCreatedAt: predecessor.createdAt,
+ }),
+ ).resolves.toBe(false);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ });
+
test('client-facing copies omit resumeContext/requestFingerprint; the stored record keeps them', async () => {
const streamId = 'stream-redact';
await manager.createJob(streamId, 'user-1');
@@ -181,6 +508,32 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
expect(await manager.getJobStatus(streamId)).toBe('running');
});
+ test('publishes the new owner capability in the same CAS that reopens steering', async () => {
+ const streamId = 'stream-resolve-capability';
+ const job = await manager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { preemptCapable: true },
+ });
+ await manager.steering.enqueue(streamId, {
+ steerId: 'waiting-steer',
+ text: 'wait for the new owner',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ });
+ const action = buildAction(streamId);
+ await manager.approvals.pause(streamId, action);
+
+ expect(
+ await manager.approvals.resolve(streamId, action.actionId, {
+ preemptCapable: false,
+ }),
+ ).toBe(true);
+
+ expect((await manager.getJob(streamId))?.metadata.preemptCapable).toBe(false);
+ await expect(manager.steering.arm(streamId, 'waiting-steer', job.createdAt)).resolves.toBe(
+ 'incapable',
+ );
+ });
+
test('returns false when the job is not paused', async () => {
const streamId = 'stream-resolve-running';
await manager.createJob(streamId, 'user-1');
@@ -202,6 +555,35 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
expect(await manager.getJobStatus(streamId)).toBe('running');
});
+ test('does not resolve a replacement paused on the same action id', async () => {
+ const streamId = 'stream-resolve-epoch-fence';
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
+ try {
+ const predecessor = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId, { actionId: 'reused-action-id' });
+ await manager.approvals.pause(streamId, action);
+
+ now.mockReturnValue(2000);
+ const replacement = await manager.createJob(streamId, 'user-1');
+ await manager.approvals.pause(streamId, action);
+
+ expect(
+ await manager.approvals.resolve(
+ streamId,
+ action.actionId,
+ undefined,
+ predecessor.createdAt,
+ ),
+ ).toBe(false);
+ expect(await manager.getJob(streamId)).toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'requires_action',
+ });
+ } finally {
+ now.mockRestore();
+ }
+ });
+
test('an expired pending action expires instead of resuming', async () => {
const streamId = 'stream-resolve-expired';
await manager.createJob(streamId, 'user-1');
@@ -263,6 +645,30 @@ describe('ApprovalLifecycle via GenerationJobManager.approvals (in-memory)', ()
});
describe('expireApproval notification', () => {
+ test('does not expire a replacement paused on the same action id', async () => {
+ const streamId = 'stream-expire-epoch-fence';
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
+ try {
+ const predecessor = await manager.createJob(streamId, 'user-1');
+ const action = buildAction(streamId, { actionId: 'reused-action-id' });
+ await manager.approvals.pause(streamId, action);
+
+ now.mockReturnValue(2000);
+ const replacement = await manager.createJob(streamId, 'user-1');
+ await manager.approvals.pause(streamId, action);
+
+ expect(await manager.expireApproval(streamId, action.actionId, predecessor.createdAt)).toBe(
+ false,
+ );
+ expect(await manager.getJob(streamId)).toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'requires_action',
+ });
+ } finally {
+ now.mockRestore();
+ }
+ });
+
test('publishes a generation-tagged expiry to local and remote subscribers', async () => {
const streamId = 'stream-expire-local-notification';
const job = await manager.createJob(streamId, 'user-1');
@@ -472,6 +878,122 @@ describe('InMemoryJobStore — approval expiry cleanup', () => {
await store.cleanup();
expect(await store.getJob('s1')).toBeNull();
});
+
+ test('cleanup() preserves a fresh pause-persistence barrier before expiring the action', async () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ try {
+ const job = await store.createJob('fresh-pause-barrier', 'u1');
+ const action = buildPendingAction(
+ buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
+ { streamId: 'fresh-pause-barrier', ttlMs: -1_000 },
+ );
+ await store.transitionStatus('fresh-pause-barrier', {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingAction: action,
+ pendingActionId: `pause-persistence:${action.actionId}`,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: 1_000,
+ },
+ });
+
+ now.mockReturnValue(11_000);
+ await store.cleanup();
+ await expect(store.getJob('fresh-pause-barrier')).resolves.toMatchObject({
+ status: 'requires_action',
+ terminalPersistencePending: true,
+ });
+
+ now.mockReturnValue(31_001);
+ await store.cleanup();
+ await expect(store.getJob('fresh-pause-barrier')).resolves.toBeNull();
+ } finally {
+ now.mockRestore();
+ }
+ });
+
+ test('cleanup() fails a stale pause-persistence barrier closed and parks steers', async () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1_000);
+ const store = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ try {
+ const job = await store.createJob('stale-pause-barrier', 'u1');
+ await store.enqueueSteer(
+ 'stale-pause-barrier',
+ {
+ steerId: 'stale-cleanup-steer',
+ text: 'park me when the pause writer disappears',
+ userId: 'u1',
+ createdAt: 1_000,
+ },
+ job.createdAt,
+ );
+ const action = buildPendingAction(
+ buildToolApprovalPayload([{ name: 'shell', arguments: {}, tool_call_id: 'c1' }]),
+ { streamId: 'stale-pause-barrier' },
+ );
+ await store.transitionStatus('stale-pause-barrier', {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingAction: action,
+ pendingActionId: `pause-persistence:${action.actionId}`,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: 1_000,
+ },
+ });
+
+ now.mockReturnValue(31_001);
+ await store.cleanup();
+ await expect(store.getJob('stale-pause-barrier')).resolves.toMatchObject({
+ status: 'error',
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ });
+ const failedJob = await store.getJob('stale-pause-barrier');
+ expect(failedJob?.pendingAction).toBeUndefined();
+ expect(failedJob?.pendingActionId).toBeUndefined();
+ expect(failedJob?.terminalPersistencePending).toBeUndefined();
+ expect(failedJob?.terminalPersistenceStartedAt).toBeUndefined();
+ await expect(store.peekSteers('stale-pause-barrier', job.createdAt)).resolves.toEqual([]);
+ await expect(store.claimParkedSteers('stale-pause-barrier', 'u1')).resolves.toEqual(
+ expect.any(String),
+ );
+ } finally {
+ now.mockRestore();
+ }
+ });
+
+ test('cleanup() retains a terminal persistence barrier when normal retention is zero', async () => {
+ const store = new InMemoryJobStore({ ttlAfterComplete: 0 });
+ const completedAt = Date.now();
+ const job = await store.createJob('terminal-persistence-barrier', 'u1');
+ await store.transitionStatus('terminal-persistence-barrier', {
+ from: 'running',
+ to: 'aborted',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ completedAt,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: completedAt,
+ },
+ });
+
+ await store.cleanup();
+ await expect(store.getJob('terminal-persistence-barrier')).resolves.toMatchObject({
+ status: 'aborted',
+ terminalPersistencePending: true,
+ });
+ await expect(
+ store.finalizeTerminalPersistence('terminal-persistence-barrier', job.createdAt, 'final'),
+ ).resolves.toBe(true);
+ await expect(store.getJob('terminal-persistence-barrier')).resolves.toMatchObject({
+ terminalPersistencePending: false,
+ finalEvent: 'final',
+ });
+ });
});
describe('ApprovalLifecycle ownership callbacks', () => {
diff --git a/packages/api/src/stream/__tests__/predecessorFence.spec.ts b/packages/api/src/stream/__tests__/predecessorFence.spec.ts
new file mode 100644
index 0000000000..d5381dd2ec
--- /dev/null
+++ b/packages/api/src/stream/__tests__/predecessorFence.spec.ts
@@ -0,0 +1,259 @@
+import { InMemoryJobStore, PARKED_STEERS_TTL_MS } from '../implementations/InMemoryJobStore';
+import { InMemoryEventTransport } from '../implementations/InMemoryEventTransport';
+import { JobPredecessorMismatchError } from '../interfaces/IJobStore';
+import { GenerationJobManagerClass } from '../GenerationJobManager';
+
+function createConditionalJob(
+ store: InMemoryJobStore,
+ streamId: string,
+ expectedPredecessorCreatedAt: number,
+) {
+ return store.createJob(
+ streamId,
+ 'owner-1',
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'conditional-create-attempt',
+ expectedPredecessorCreatedAt,
+ );
+}
+
+describe('generation predecessor create fence', () => {
+ test('in-memory ordinary appends accept active states and reject retained terminal epochs', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'in-memory-terminal-append-fence';
+ try {
+ const job = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.appendChunk(streamId, { delta: 'running' }, job.createdAt)).resolves.toBe(
+ true,
+ );
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBe(true);
+ await expect(store.appendChunk(streamId, { delta: 'paused' }, job.createdAt)).resolves.toBe(
+ true,
+ );
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'requires_action',
+ to: 'aborted',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBe(true);
+ await expect(store.appendChunk(streamId, { delta: 'stale' }, job.createdAt)).resolves.toBe(
+ false,
+ );
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('in-memory mismatch rejects before replacing job, content, or queued work', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'predecessor-fence-no-mutation';
+ try {
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ const content = [{ type: 'text', text: 'keep predecessor content' }];
+ const queued = {
+ steerId: 'keep-predecessor-steer',
+ text: 'keep queued work',
+ userId: 'owner-1',
+ createdAt: Date.now(),
+ };
+ store.setContentParts(streamId, content, predecessor.createdAt);
+ await expect(store.enqueueSteer(streamId, queued, predecessor.createdAt)).resolves.toBe(1);
+
+ await expect(
+ createConditionalJob(store, streamId, predecessor.createdAt + 1),
+ ).rejects.toMatchObject({
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: {
+ createdAt: predecessor.createdAt,
+ active: true,
+ verified: true,
+ status: 'running',
+ conversationId: streamId,
+ },
+ });
+
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ createdAt: predecessor.createdAt,
+ status: 'running',
+ });
+ await expect(store.getContentParts(streamId, predecessor.createdAt)).resolves.toEqual({
+ content,
+ });
+ await expect(store.peekSteers(streamId, predecessor.createdAt)).resolves.toEqual([queued]);
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('a retained epoch matching the disappeared predecessor allows a new generation', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'predecessor-fence-absent-allowed';
+ try {
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, predecessor.createdAt)).resolves.toBe(true);
+
+ const replacement = await createConditionalJob(store, streamId, predecessor.createdAt);
+ expect(replacement.createdAt).toBeGreaterThan(predecessor.createdAt);
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('expired predecessor evidence rejects conditionally without creating state', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'predecessor-fence-expired-evidence';
+ const base = Date.now();
+ const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(base);
+ try {
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, predecessor.createdAt)).resolves.toBe(true);
+ nowSpy.mockReturnValue(base + PARKED_STEERS_TTL_MS + 1);
+
+ await expect(
+ createConditionalJob(store, streamId, predecessor.createdAt),
+ ).rejects.toMatchObject({
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: {
+ createdAt: predecessor.createdAt,
+ active: false,
+ verified: false,
+ },
+ });
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ } finally {
+ nowSpy.mockRestore();
+ await store.destroy();
+ }
+ });
+
+ test('a retained intervening epoch rejects a delayed create after its job disappeared', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'predecessor-fence-retained-intervening';
+ try {
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, predecessor.createdAt)).resolves.toBe(true);
+
+ const intervening = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, intervening.createdAt)).resolves.toBe(true);
+
+ await expect(
+ createConditionalJob(store, streamId, predecessor.createdAt),
+ ).rejects.toMatchObject({
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: {
+ createdAt: intervening.createdAt,
+ active: false,
+ verified: true,
+ },
+ });
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test.each([-1, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1])(
+ 'rejects invalid expected predecessor epoch %p without creating state',
+ async (invalidEpoch) => {
+ const store = new InMemoryJobStore();
+ const streamId = 'predecessor-fence-invalid';
+ try {
+ await expect(createConditionalJob(store, streamId, invalidEpoch)).rejects.toThrow(
+ 'Invalid expected generation predecessor',
+ );
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test('manager propagates the typed mismatch without ambiguous-create recovery', async () => {
+ const store = new InMemoryJobStore();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = 'predecessor-fence-manager-propagation';
+ try {
+ const predecessor = await manager.createJob(streamId, 'owner-1', streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const getJob = jest.spyOn(store, 'getJob');
+
+ await expect(
+ manager.createJob(streamId, 'owner-1', streamId, {
+ expectedPredecessorCreatedAt: predecessor.createdAt + 1,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toBeInstanceOf(JobPredecessorMismatchError);
+
+ // One pre-create observation is expected. A generic create failure would
+ // perform another read in recoverCommittedCreate and could misclassify a
+ // deliberate no-mutation CAS rejection as an ambiguous committed write.
+ expect(getJob).toHaveBeenCalledTimes(1);
+ expect(predecessor.abortController.signal.aborted).toBe(false);
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ createdAt: predecessor.createdAt,
+ abortController: predecessor.abortController,
+ });
+ } finally {
+ await manager.destroy();
+ }
+ });
+
+ test('manager validates the expected epoch before observing durable state', async () => {
+ const store = new InMemoryJobStore();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: store,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const getJob = jest.spyOn(store, 'getJob');
+ try {
+ await expect(
+ manager.createJob('predecessor-fence-manager-invalid', 'owner-1', undefined, {
+ expectedPredecessorCreatedAt: -1,
+ }),
+ ).rejects.toThrow('Invalid expected generation predecessor');
+ expect(getJob).not.toHaveBeenCalled();
+ } finally {
+ await manager.destroy();
+ }
+ });
+});
diff --git a/packages/api/src/stream/__tests__/predecessorFence.stream_integration.spec.ts b/packages/api/src/stream/__tests__/predecessorFence.stream_integration.spec.ts
new file mode 100644
index 0000000000..0bf6c8033e
--- /dev/null
+++ b/packages/api/src/stream/__tests__/predecessorFence.stream_integration.spec.ts
@@ -0,0 +1,308 @@
+import { clearRedisTestPrefix, createRedisTestClient, type RedisTestClient } from './helpers/redis';
+import { RedisEventTransport } from '../implementations/RedisEventTransport';
+import { JobPredecessorMismatchError } from '../interfaces/IJobStore';
+import { emitChunkWithReceipt } from '../internal/chunkPublication';
+import { RedisJobStore } from '../implementations/RedisJobStore';
+
+function createConditionalJob(
+ store: RedisJobStore,
+ streamId: string,
+ expectedPredecessorCreatedAt: number,
+ attempt: string,
+) {
+ return store.createJob(
+ streamId,
+ 'owner-1',
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ attempt,
+ expectedPredecessorCreatedAt,
+ );
+}
+
+describe('Redis generation predecessor create fence', () => {
+ const keyPrefix = `Predecessor-Fence-${process.pid}-${Date.now()}:`;
+ let redis: RedisTestClient;
+ let store: RedisJobStore;
+
+ beforeAll(async () => {
+ redis = createRedisTestClient(keyPrefix);
+ await redis.connect();
+ store = new RedisJobStore(redis, {
+ chunksAfterCompleteTtl: 300,
+ runStepsAfterCompleteTtl: 300,
+ });
+ await store.initialize();
+ });
+
+ afterEach(async () => {
+ await clearRedisTestPrefix(redis, keyPrefix);
+ });
+
+ afterAll(async () => {
+ await store.destroy();
+ await redis.quit();
+ });
+
+ test('mismatch returns the exact current generation without mutating durable state', async () => {
+ const streamId = 'redis-predecessor-fence-no-mutation';
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ const queued = {
+ steerId: 'redis-predecessor-queued',
+ text: 'keep queued work',
+ userId: 'owner-1',
+ createdAt: Date.now(),
+ };
+ await expect(store.enqueueSteer(streamId, queued, predecessor.createdAt)).resolves.toBe(1);
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_message_delta', data: { delta: 'keep persisted content' } },
+ predecessor.createdAt,
+ ),
+ ).resolves.toBe(true);
+ const chunksKey = `stream:{${streamId}}:chunks`;
+ const chunksBefore = await redis.xrange(chunksKey, '-', '+');
+ const epochBefore = await redis.get(`stream:{${streamId}}:generation-epoch`);
+
+ await expect(
+ createConditionalJob(store, streamId, predecessor.createdAt + 1, 'redis-mismatch-attempt'),
+ ).rejects.toMatchObject({
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: {
+ createdAt: predecessor.createdAt,
+ active: true,
+ verified: true,
+ status: 'running',
+ conversationId: streamId,
+ },
+ });
+
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ createdAt: predecessor.createdAt,
+ status: 'running',
+ });
+ await expect(store.peekSteers(streamId, predecessor.createdAt)).resolves.toEqual([queued]);
+ await expect(redis.xrange(chunksKey, '-', '+')).resolves.toEqual(chunksBefore);
+ await expect(redis.get(`stream:{${streamId}}:generation-epoch`)).resolves.toBe(epochBefore);
+ const durable = await store.getJob(streamId);
+ expect(Object.getOwnPropertyDescriptor(durable!, 'replacedJobs')).toBeUndefined();
+ });
+
+ test('matching expected predecessor atomically installs its replacement receipt', async () => {
+ const streamId = 'redis-predecessor-fence-match';
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+
+ const replacement = await createConditionalJob(
+ store,
+ streamId,
+ predecessor.createdAt,
+ 'redis-matching-attempt',
+ );
+
+ expect(replacement.createdAt).toBeGreaterThan(predecessor.createdAt);
+ expect(replacement.replacedJob).toEqual({
+ createdAt: predecessor.createdAt,
+ status: 'running',
+ conversationId: streamId,
+ });
+ await expect(store.getJob(streamId)).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ });
+
+ test('a retained epoch matching the disappeared predecessor permits a monotonic new generation', async () => {
+ const streamId = 'redis-predecessor-fence-absent';
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, predecessor.createdAt)).resolves.toBe(true);
+
+ const replacement = await createConditionalJob(
+ store,
+ streamId,
+ predecessor.createdAt,
+ 'redis-absent-attempt',
+ );
+
+ expect(replacement.createdAt).toBeGreaterThan(predecessor.createdAt);
+ expect(replacement.replacedJob).toBeUndefined();
+ });
+
+ test('missing predecessor evidence rejects conditionally without recreating state', async () => {
+ const streamId = 'redis-predecessor-fence-expired-evidence';
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, predecessor.createdAt)).resolves.toBe(true);
+ const retainedEpochKey = `stream:{${streamId}}:generation-epoch`;
+ await expect(redis.del(retainedEpochKey)).resolves.toBe(1);
+
+ await expect(
+ createConditionalJob(store, streamId, predecessor.createdAt, 'redis-expired-attempt'),
+ ).rejects.toMatchObject({
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: {
+ createdAt: predecessor.createdAt,
+ active: false,
+ verified: false,
+ },
+ });
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ await expect(redis.get(retainedEpochKey)).resolves.toBeNull();
+ });
+
+ test('a retained intervening epoch rejects a delayed create after its job disappeared', async () => {
+ const streamId = 'redis-predecessor-fence-retained-intervening';
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, predecessor.createdAt)).resolves.toBe(true);
+
+ const intervening = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(store.deleteJob(streamId, intervening.createdAt)).resolves.toBe(true);
+ const retainedEpochKey = `stream:{${streamId}}:generation-epoch`;
+ await expect(redis.get(retainedEpochKey)).resolves.toBe(String(intervening.createdAt));
+
+ await expect(
+ createConditionalJob(
+ store,
+ streamId,
+ predecessor.createdAt,
+ 'redis-retained-intervening-attempt',
+ ),
+ ).rejects.toMatchObject({
+ code: 'GENERATION_PREDECESSOR_MISMATCH',
+ currentJob: {
+ createdAt: intervening.createdAt,
+ active: false,
+ verified: true,
+ },
+ });
+ await expect(store.getJob(streamId)).resolves.toBeNull();
+ await expect(redis.get(retainedEpochKey)).resolves.toBe(String(intervening.createdAt));
+ });
+
+ test('exposes a typed mismatch error for callers that need a definitive 409', async () => {
+ const streamId = 'redis-predecessor-fence-error-type';
+ const predecessor = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+
+ await expect(
+ createConditionalJob(store, streamId, predecessor.createdAt + 1, 'redis-typed-attempt'),
+ ).rejects.toBeInstanceOf(JobPredecessorMismatchError);
+ });
+
+ test('same-epoch append is fenced after terminal transition without changing retained chunks', async () => {
+ const streamId = 'redis-terminal-append-fence';
+ const job = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_message_delta', data: { delta: 'committed before terminal' } },
+ job.createdAt,
+ ),
+ ).resolves.toBe(true);
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBe(true);
+
+ const chunksKey = `stream:{${streamId}}:chunks`;
+ const chunksAtTerminal = await redis.xrange(chunksKey, '-', '+');
+ expect(chunksAtTerminal).toHaveLength(1);
+
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_message_delta', data: { delta: 'must be rejected' } },
+ job.createdAt,
+ ),
+ ).resolves.toBe(false);
+ await expect(redis.xrange(chunksKey, '-', '+')).resolves.toEqual(chunksAtTerminal);
+ });
+
+ test('same-epoch run-step save is fenced after terminal transition without changing retained steps', async () => {
+ const streamId = 'redis-terminal-runsteps-fence';
+ const job = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ const committedSteps = [{ id: 'step-before-terminal', type: 'tool_call' }];
+ await store.saveRunSteps(streamId, committedSteps as never, job.createdAt);
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBe(true);
+
+ const runStepsKey = `stream:{${streamId}}:runsteps`;
+ const stepsAtTerminal = await redis.get(runStepsKey);
+ expect(stepsAtTerminal).toBe(JSON.stringify(committedSteps));
+
+ await store.saveRunSteps(
+ streamId,
+ [{ id: 'step-after-terminal', type: 'tool_call' }] as never,
+ job.createdAt,
+ );
+ await expect(redis.get(runStepsKey)).resolves.toBe(stepsAtTerminal);
+ });
+
+ test('terminal hash fences CHUNK before sequence allocation while permitting same-epoch DONE', async () => {
+ const streamId = 'redis-terminal-publication-fence';
+ const job = await store.createJob(streamId, 'owner-1', streamId, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: job.createdAt,
+ }),
+ ).resolves.toBe(true);
+
+ const subscriber = redis.duplicate();
+ const transport = new RedisEventTransport(redis, subscriber);
+ const sequenceKey = `stream:{${streamId}}:seq`;
+ try {
+ await expect(redis.get(sequenceKey)).resolves.toBeNull();
+ await expect(
+ emitChunkWithReceipt(
+ transport,
+ streamId,
+ { event: 'on_message_delta', data: { delta: 'must not publish' } },
+ job.createdAt,
+ ),
+ ).resolves.toBe(false);
+ await expect(redis.get(sequenceKey)).resolves.toBeNull();
+
+ await expect(
+ transport.emitDone(streamId, { final: true }, job.createdAt),
+ ).resolves.toBeUndefined();
+ await expect(redis.get(sequenceKey)).resolves.toBe('1');
+ } finally {
+ transport.destroy();
+ subscriber.disconnect();
+ }
+ });
+});
diff --git a/packages/api/src/stream/__tests__/protocolRollout.spec.ts b/packages/api/src/stream/__tests__/protocolRollout.spec.ts
new file mode 100644
index 0000000000..6f844ef3b6
--- /dev/null
+++ b/packages/api/src/stream/__tests__/protocolRollout.spec.ts
@@ -0,0 +1,215 @@
+import type { SteerQueueItem, SteerReceiptInput } from '../interfaces/IJobStore';
+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
+
+function steer(id: string): SteerQueueItem {
+ return {
+ steerId: id,
+ clientSteerId: `client-${id}`,
+ text: `text-${id}`,
+ userId: 'user-1',
+ createdAt: Date.now(),
+ };
+}
+
+function receipt(item: SteerQueueItem, createdAt: number): SteerReceiptInput {
+ return {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${item.steerId}`,
+ userId: item.userId,
+ generationCreatedAt: createdAt,
+ };
+}
+
+describe('generation protocol rollout storage', () => {
+ test('in-memory defaults to v2 while an explicit v1 omits checkpoint isolation', async () => {
+ const store = new InMemoryJobStore();
+ const v2 = await store.createJob('protocol-v2', 'user-1');
+ const v1 = await store.createJob('protocol-v1', 'user-1', undefined, undefined, {
+ generationProtocolVersion: 1,
+ });
+
+ expect(v2.generationProtocolVersion).toBe(2);
+ expect(v2.checkpointNamespace).toBe(String(v2.createdAt));
+ expect(v1.generationProtocolVersion).toBe(1);
+ expect(v1.checkpointNamespace).toBeUndefined();
+ });
+
+ test('v1 steering stays receiptless and uses the legacy destructive drain', async () => {
+ const store = new InMemoryJobStore();
+ const job = await store.createJob('protocol-v1-steer', 'user-1', undefined, undefined, {
+ generationProtocolVersion: 1,
+ preemptCapable: true,
+ });
+ const item = steer('v1');
+
+ const enqueued = await store.enqueueSteerWithReceipt(
+ job.streamId,
+ item,
+ receipt(item, job.createdAt),
+ true,
+ job.createdAt,
+ );
+ expect(enqueued).toMatchObject({ position: 1, item: { steerId: item.steerId } });
+ expect(enqueued).not.toHaveProperty('state');
+ expect(await store.getSteerReceipt(job.streamId, item.clientSteerId!)).toBeNull();
+
+ const drained = await store.drainSteers(job.streamId, job.createdAt);
+ expect(drained).toHaveLength(1);
+ expect(await store.peekClaimedSteers(job.streamId, job.createdAt)).toEqual([]);
+ expect(
+ await store.appendChunk(job.streamId, { event: 'chunk' }, job.createdAt, drained[0]),
+ ).toBe(true);
+ });
+
+ test('a v1 requester destructively downgrades even a v2 parked envelope', async () => {
+ const store = new InMemoryJobStore();
+ const job = await store.createJob('protocol-park-downgrade', 'user-1');
+ await store.parkSteers(
+ job.streamId,
+ JSON.stringify({
+ userId: 'user-1',
+ steers: [{ steerId: 'parked', text: 'keep me', createdAt: Date.now() }],
+ }),
+ job.createdAt,
+ );
+
+ const replayable = await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2);
+ expect(replayable?.generationProtocolVersion).toBe(2);
+ expect(
+ await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2),
+ ).toBeDefined();
+
+ const downgraded = await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 1);
+ expect(downgraded?.generationProtocolVersion).toBe(1);
+ expect(JSON.parse(downgraded!.payload)).toMatchObject({ generationProtocolVersion: 1 });
+ expect(
+ await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2),
+ ).toBeUndefined();
+ });
+
+ test('a v1 requester takes only unleased leftovers during an active v2 recovery', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'protocol-park-active-v2-lease';
+ const job = await store.createJob(streamId, 'user-1', undefined, undefined, {
+ generationProtocolVersion: 2,
+ });
+ const leased = { steerId: 'leased-v2', text: 'leased words', createdAt: 1 };
+ const legacy = { steerId: 'visible-v1', text: 'legacy words', createdAt: 2 };
+ await store.parkSteers(
+ streamId,
+ JSON.stringify({ userId: 'user-1', steers: [leased, legacy] }),
+ job.createdAt,
+ );
+ const recovery = await store.createJob(
+ streamId,
+ 'user-1',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ leased.steerId,
+ undefined,
+ undefined,
+ undefined,
+ { text: leased.text, fileIds: [] },
+ );
+
+ const downgraded = await store.claimParkedSteersDetailed(streamId, 'user-1', undefined, 1);
+ expect(downgraded?.generationProtocolVersion).toBe(1);
+ expect(JSON.parse(downgraded!.payload)).toEqual({
+ userId: 'user-1',
+ generationProtocolVersion: 1,
+ steers: [legacy],
+ });
+ await expect(
+ store.claimParkedSteersDetailed(streamId, 'user-1', undefined, 2),
+ ).resolves.toBeUndefined();
+
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ expectCreatedAt: recovery.createdAt,
+ patch: { error: 'recovery failed', completedAt: Date.now() },
+ }),
+ ).resolves.toBe(true);
+ const exposed = await store.claimParkedSteersDetailed(streamId, 'user-1', undefined, 2);
+ expect(JSON.parse(exposed!.payload)).toEqual({
+ userId: 'user-1',
+ generationProtocolVersion: 2,
+ steers: [leased],
+ });
+ });
+
+ test('a v1 parked envelope remains destructive for a v2 requester', async () => {
+ const store = new InMemoryJobStore();
+ const job = await store.createJob('protocol-park-v1', 'user-1', undefined, undefined, {
+ generationProtocolVersion: 1,
+ });
+ await store.parkSteers(
+ job.streamId,
+ JSON.stringify({
+ userId: 'user-1',
+ steers: [{ steerId: 'legacy', text: 'legacy words', createdAt: Date.now() }],
+ }),
+ job.createdAt,
+ );
+
+ expect(
+ (await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2))
+ ?.generationProtocolVersion,
+ ).toBe(1);
+ expect(
+ await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2),
+ ).toBeUndefined();
+ });
+
+ test('cannot upgrade a v1 parked item into the v2 lease protocol implicitly', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'protocol-v1-source';
+ const job = await store.createJob(streamId, 'user-1', undefined, undefined, {
+ generationProtocolVersion: 1,
+ });
+ await store.parkSteers(
+ streamId,
+ JSON.stringify({
+ userId: 'user-1',
+ steers: [{ steerId: 'legacy', text: 'legacy words', createdAt: Date.now() }],
+ }),
+ job.createdAt,
+ );
+
+ await expect(
+ store.createJob(
+ streamId,
+ 'user-1',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ 'legacy',
+ undefined,
+ undefined,
+ undefined,
+ { text: 'legacy words', fileIds: [] },
+ ),
+ ).rejects.toMatchObject({ code: 'RECOVERY_PAYLOAD_MISMATCH' });
+ });
+
+ test('marks a token-owned idempotency bridge key started idempotently', async () => {
+ const store = new InMemoryJobStore();
+ await store.claimIdempotencyKey(
+ 'legacy:user:req',
+ { streamId: 'stream-1', conversationId: 'stream-1', claimToken: 'token-1' },
+ 60,
+ );
+
+ expect(await store.markIdempotencyKeyStarted('legacy:user:req', 'token-1', 1234, 60)).toBe(
+ true,
+ );
+ expect(await store.markIdempotencyKeyStarted('legacy:user:req', 'token-1', 1234, 60)).toBe(
+ true,
+ );
+ expect(await store.markIdempotencyKeyStarted('legacy:user:req', 'token-1', 1235, 60)).toBe(
+ false,
+ );
+ });
+});
diff --git a/packages/api/src/stream/__tests__/protocolRollout.stream_integration.spec.ts b/packages/api/src/stream/__tests__/protocolRollout.stream_integration.spec.ts
new file mode 100644
index 0000000000..f75a478728
--- /dev/null
+++ b/packages/api/src/stream/__tests__/protocolRollout.stream_integration.spec.ts
@@ -0,0 +1,919 @@
+import type { SteerQueueItem } from '../interfaces/IJobStore';
+import { clearRedisTestPrefix, createRedisTestClient, type RedisTestClient } from './helpers/redis';
+import { InMemoryEventTransport } from '../implementations/InMemoryEventTransport';
+import { RedisEventTransport } from '../implementations/RedisEventTransport';
+import { GenerationJobManagerClass } from '../GenerationJobManager';
+import { RedisJobStore } from '../implementations/RedisJobStore';
+
+describe('Redis generation protocol rollout bridge', () => {
+ const keyPrefix = `Protocol-Rollout-${process.pid}-${Date.now()}:`;
+ const originalProtocol = process.env.GENERATION_PROTOCOL_VERSION;
+ let redis: RedisTestClient;
+ let store: RedisJobStore;
+
+ beforeAll(async () => {
+ delete process.env.GENERATION_PROTOCOL_VERSION;
+ redis = createRedisTestClient(keyPrefix);
+ await redis.connect();
+ store = new RedisJobStore(redis);
+ await store.initialize();
+ });
+
+ afterEach(async () => {
+ await clearRedisTestPrefix(redis, keyPrefix);
+ });
+
+ afterAll(async () => {
+ await store.destroy();
+ await redis.quit();
+ if (originalProtocol == null) {
+ delete process.env.GENERATION_PROTOCOL_VERSION;
+ } else {
+ process.env.GENERATION_PROTOCOL_VERSION = originalProtocol;
+ }
+ });
+
+ test('Redis defaults legacy jobs to v1 and isolates checkpoints only for v2', async () => {
+ const legacy = await store.createJob('redis-protocol-v1', 'user-1');
+ const current = await store.createJob('redis-protocol-v2', 'user-1', undefined, undefined, {
+ generationProtocolVersion: 2,
+ });
+
+ expect(legacy.generationProtocolVersion).toBe(1);
+ expect(legacy.checkpointNamespace).toBeUndefined();
+ expect(current.generationProtocolVersion).toBe(2);
+ expect(current.checkpointNamespace).toBe(String(current.createdAt));
+ expect((await store.getJob(current.streamId))?.generationProtocolVersion).toBe(2);
+ });
+
+ test('v1 receipt API falls back to a receiptless legacy queue and drain', async () => {
+ const job = await store.createJob('redis-protocol-v1-steer', 'user-1', undefined, undefined, {
+ generationProtocolVersion: 1,
+ preemptCapable: true,
+ });
+ const item: SteerQueueItem = {
+ steerId: 'legacy-steer',
+ clientSteerId: 'legacy-client-steer',
+ text: 'legacy words',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ };
+
+ const enqueued = await store.enqueueSteerWithReceipt(
+ job.streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: 'fingerprint',
+ userId: 'user-1',
+ generationCreatedAt: job.createdAt,
+ },
+ true,
+ job.createdAt,
+ );
+ expect(enqueued).toMatchObject({ position: 1, item: { steerId: item.steerId } });
+ expect(enqueued).not.toHaveProperty('state');
+ expect(await store.getSteerReceipt(job.streamId, item.clientSteerId!)).toBeNull();
+
+ const drained = await store.drainSteers(job.streamId, job.createdAt);
+ expect(drained).toHaveLength(1);
+ expect(await store.peekClaimedSteers(job.streamId, job.createdAt)).toEqual([]);
+ expect(
+ await store.appendChunk(job.streamId, { event: 'chunk' }, job.createdAt, drained[0]),
+ ).toBe(true);
+ });
+
+ test('requested v1 destructively downgrades a v2 parked payload', async () => {
+ const job = await store.createJob('redis-protocol-park', 'user-1', undefined, undefined, {
+ generationProtocolVersion: 2,
+ });
+ await store.parkSteers(
+ job.streamId,
+ JSON.stringify({
+ userId: 'user-1',
+ steers: [{ steerId: 'parked', text: 'parked words', createdAt: Date.now() }],
+ }),
+ job.createdAt,
+ );
+
+ expect(
+ (await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2))
+ ?.generationProtocolVersion,
+ ).toBe(2);
+ expect(
+ (await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2))
+ ?.generationProtocolVersion,
+ ).toBe(2);
+
+ const legacy = await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 1);
+ expect(legacy?.generationProtocolVersion).toBe(1);
+ expect(JSON.parse(legacy!.payload)).toMatchObject({ generationProtocolVersion: 1 });
+ expect(
+ await store.claimParkedSteersDetailed(job.streamId, 'user-1', undefined, 2),
+ ).toBeUndefined();
+ });
+
+ test('requested v1 preserves a source leased to an active v2 recovery', async () => {
+ const streamId = 'redis-protocol-park-active-v2-lease';
+ const job = await store.createJob(streamId, 'user-1', undefined, undefined, {
+ generationProtocolVersion: 2,
+ });
+ const leased = { steerId: 'redis-leased-v2', text: 'leased words', createdAt: 1 };
+ const legacy = { steerId: 'redis-visible-v1', text: 'legacy words', createdAt: 2 };
+ await store.parkSteers(
+ streamId,
+ JSON.stringify({ userId: 'user-1', steers: [leased, legacy] }),
+ job.createdAt,
+ );
+ const recovery = await store.createJob(
+ streamId,
+ 'user-1',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ leased.steerId,
+ undefined,
+ undefined,
+ undefined,
+ { text: leased.text, fileIds: [] },
+ );
+
+ const downgraded = await store.claimParkedSteersDetailed(streamId, 'user-1', undefined, 1);
+ expect(downgraded?.generationProtocolVersion).toBe(1);
+ expect(JSON.parse(downgraded!.payload)).toEqual({
+ userId: 'user-1',
+ generationProtocolVersion: 1,
+ steers: [legacy],
+ });
+ await expect(
+ store.claimParkedSteersDetailed(streamId, 'user-1', undefined, 2),
+ ).resolves.toBeUndefined();
+ await expect(redis.exists(`stream:{${streamId}}:parked`)).resolves.toBe(1);
+
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ expectCreatedAt: recovery.createdAt,
+ patch: { error: 'recovery failed', completedAt: Date.now() },
+ }),
+ ).resolves.toBe(true);
+ const exposed = await store.claimParkedSteersDetailed(streamId, 'user-1', undefined, 2);
+ expect(JSON.parse(exposed!.payload)).toEqual({
+ userId: 'user-1',
+ generationProtocolVersion: 2,
+ steers: [leased],
+ });
+ });
+
+ test('cannot lease a legacy parked item into a v2 generation', async () => {
+ const streamId = 'redis-protocol-v1-source';
+ const job = await store.createJob(streamId, 'user-1', undefined, undefined, {
+ generationProtocolVersion: 1,
+ });
+ await store.parkSteers(
+ streamId,
+ JSON.stringify({
+ userId: 'user-1',
+ steers: [{ steerId: 'legacy', text: 'legacy words', createdAt: Date.now() }],
+ }),
+ job.createdAt,
+ );
+
+ await expect(
+ store.createJob(
+ streamId,
+ 'user-1',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ 'legacy',
+ undefined,
+ undefined,
+ undefined,
+ { text: 'legacy words', fileIds: [] },
+ ),
+ ).rejects.toMatchObject({ code: 'RECOVERY_PAYLOAD_MISMATCH' });
+ });
+
+ test('marks the legacy bridge claim started only for its owner token', async () => {
+ await store.claimIdempotencyKey(
+ 'legacy:user:req',
+ { streamId: 'stream-1', conversationId: 'stream-1', claimToken: 'token-1' },
+ 60,
+ );
+ expect(await store.markIdempotencyKeyStarted('legacy:user:req', 'wrong', 1234, 60)).toBe(false);
+ expect(await store.markIdempotencyKeyStarted('legacy:user:req', 'token-1', 1234, 60)).toBe(
+ true,
+ );
+ expect(
+ (
+ await store.claimIdempotencyKey(
+ 'legacy:user:req',
+ { streamId: 'other', conversationId: 'other' },
+ 60,
+ )
+ ).existing,
+ ).toMatchObject({ startedAt: 1234 });
+ });
+
+ test('manager bridges the exact old physical key to the same-slot primary with one winner', async () => {
+ const managerStores = [new RedisJobStore(redis), new RedisJobStore(redis)];
+ const managers = managerStores.map((jobStore) => {
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ });
+ manager.initialize();
+ return manager;
+ });
+ try {
+ const userId = 'rollout-user';
+ const clientRequestId = `rollout-request-${Date.now()}`;
+ const streamId = `rollout-stream-${Date.now()}`;
+ const [first, second] = await Promise.all(
+ managers.map((manager) =>
+ manager.claimGeneration(userId, clientRequestId, streamId, streamId, 2),
+ ),
+ );
+
+ expect([first, second].filter((result) => result.claimed)).toHaveLength(1);
+ expect(first.existing?.claimToken).toBe(second.existing?.claimToken);
+ const legacyPhysicalKey = `stream:idem:{${userId}:${clientRequestId}}`;
+ const primaryPhysicalKey = `stream:idem:{${streamId}}:${userId}:${clientRequestId}`;
+ const legacy = JSON.parse((await redis.get(legacyPhysicalKey))!);
+ const primary = JSON.parse((await redis.get(primaryPhysicalKey))!);
+ expect(legacy).toEqual(primary);
+ expect(primary).toMatchObject({
+ streamId,
+ conversationId: streamId,
+ claimToken: first.existing?.claimToken,
+ generationProtocolVersion: 2,
+ });
+ expect(await redis.ttl(primaryPhysicalKey)).toBeGreaterThan(24 * 60 * 60);
+ expect(await redis.ttl(primaryPhysicalKey)).toBeLessThanOrEqual(25 * 60 * 60);
+ expect(await redis.ttl(legacyPhysicalKey)).toBeGreaterThan(25 * 60 * 60);
+ expect(await redis.ttl(legacyPhysicalKey)).toBeLessThanOrEqual(26 * 60 * 60);
+
+ const winner = first.claimed ? managers[0] : managers[1];
+ const claim = first.claimed ? first.existing! : second.existing!;
+ const job = await winner.createJob(streamId, userId, streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: claim.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(JSON.parse((await redis.get(primaryPhysicalKey))!)).toMatchObject({
+ startedAt: job.createdAt,
+ generationProtocolVersion: 2,
+ });
+ expect(JSON.parse((await redis.get(legacyPhysicalKey))!)).toMatchObject({
+ startedAt: job.createdAt,
+ generationProtocolVersion: 2,
+ });
+ } finally {
+ await Promise.all(managers.map((manager) => manager.destroy()));
+ }
+ });
+
+ test('recovers a lost create reply, repairs membership, and stops the exact raced predecessor', async () => {
+ const abortHandlers = new Map<string, Set<(generationId?: number) => void | boolean>>();
+ const acknowledgedAborts = new Map<string, Set<number>>();
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitAbort(streamId: string, generationId?: number) {
+ for (const handler of abortHandlers.get(streamId) ?? []) {
+ handler(generationId);
+ }
+ },
+ async onAbort(streamId: string, handler: (generationId?: number) => void) {
+ let handlers = abortHandlers.get(streamId);
+ if (handlers == null) {
+ handlers = new Set();
+ abortHandlers.set(streamId, handlers);
+ }
+ handlers.add(handler);
+ return () => handlers?.delete(handler);
+ },
+ async emitAbortConfirmed(streamId: string, generationId: number) {
+ let acknowledged = acknowledgedAborts.get(streamId)?.has(generationId) === true;
+ for (const handler of abortHandlers.get(streamId) ?? []) {
+ acknowledged = handler(generationId) === true || acknowledged;
+ }
+ if (acknowledged) {
+ let generations = acknowledgedAborts.get(streamId);
+ if (generations == null) {
+ generations = new Set();
+ acknowledgedAborts.set(streamId, generations);
+ }
+ generations.add(generationId);
+ }
+ return acknowledged;
+ },
+ });
+ const ownerStore = new RedisJobStore(redis);
+ const recoveringStore = new RedisJobStore(redis);
+ const owner = new GenerationJobManagerClass();
+ const recovering = new GenerationJobManagerClass();
+ owner.configure({ jobStore: ownerStore, eventTransport, isRedis: true });
+ recovering.configure({ jobStore: recoveringStore, eventTransport, isRedis: true });
+ owner.initialize();
+ recovering.initialize();
+
+ const streamId = `redis-lost-create-${Date.now()}`;
+ const userId = 'redis-lost-create-user';
+ const clientRequestId = `redis-lost-create-request-${Date.now()}`;
+ const doneEvents: Array<{ event: unknown; generationId?: number }> = [];
+ const subscription = eventTransport.subscribe(streamId, {
+ onChunk: () => undefined,
+ onDone: (event, generationId) => doneEvents.push({ event, generationId }),
+ });
+ const actualEval = redis.eval.bind(redis) as (...args: unknown[]) => Promise<unknown>;
+ const evalSpy = jest.spyOn(redis, 'eval');
+ let injectLostCreateReply = false;
+ evalSpy.mockImplementation((async (...args: unknown[]) => {
+ const result = await actualEval(...args);
+ const script = args[0];
+ if (
+ injectLostCreateReply &&
+ typeof script === 'string' &&
+ script.includes('previousJobExists') &&
+ script.includes('claim_lost')
+ ) {
+ injectLostCreateReply = false;
+ throw new Error('simulated lost JOB_CREATE_LUA reply');
+ }
+ return result;
+ }) as typeof redis.eval);
+
+ let racedPredecessor: Awaited<ReturnType<GenerationJobManagerClass['createJob']>> | undefined;
+ const actualGetJob = recoveringStore.getJob.bind(recoveringStore);
+ let injectRacedPredecessor = true;
+ const getJobSpy = jest.spyOn(recoveringStore, 'getJob').mockImplementation(async (id) => {
+ const observed = await actualGetJob(id);
+ if (id === streamId && injectRacedPredecessor) {
+ injectRacedPredecessor = false;
+ racedPredecessor = await owner.createJob(streamId, userId, streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ injectLostCreateReply = true;
+ }
+ return observed;
+ });
+
+ try {
+ // A is what the recovering manager pre-reads. B is installed by the
+ // getJob hook immediately afterward, so only Redis's persisted
+ // transaction-time receipt can identify B when C's eval reply is lost.
+ const stalePredecessor = await owner.createJob(streamId, userId, streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const claim = await recovering.claimGeneration(
+ userId,
+ clientRequestId,
+ streamId,
+ streamId,
+ 2,
+ );
+ const replacement = await recovering.createJob(streamId, userId, streamId, {
+ idempotencyClientRequestId: clientRequestId,
+ idempotencyClaimToken: claim.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+
+ expect(racedPredecessor).toBeDefined();
+ expect(racedPredecessor!.createdAt).toBeGreaterThan(stalePredecessor.createdAt);
+ expect(replacement.createdAt).toBeGreaterThan(racedPredecessor!.createdAt);
+ expect(racedPredecessor!.abortController.signal.aborted).toBe(true);
+ expect(doneEvents).toContainEqual({
+ event: expect.objectContaining({
+ final: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: racedPredecessor!.createdAt,
+ }),
+ generationId: racedPredecessor!.createdAt,
+ });
+ expect(await redis.sismember('stream:running', streamId)).toBe(1);
+ expect(await redis.sismember(`stream:user:{${userId}}:jobs`, streamId)).toBe(1);
+ expect(await recoveringStore.getJob(streamId)).toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ idempotencyClientRequestId: clientRequestId,
+ });
+ } finally {
+ getJobSpy.mockRestore();
+ evalSpy.mockRestore();
+ subscription.unsubscribe();
+ await Promise.all([recovering.destroy(), owner.destroy()]);
+ }
+ });
+
+ test('a superseded lost-reply helper delivers but does not acknowledge the crashed successor chain', async () => {
+ const abortHandlers = new Map<string, Set<(generationId?: number) => void | boolean>>();
+ const acknowledgedAborts = new Map<string, Set<number>>();
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitAbort(streamId: string, generationId?: number) {
+ for (const handler of abortHandlers.get(streamId) ?? []) {
+ handler(generationId);
+ }
+ },
+ async onAbort(streamId: string, handler: (generationId?: number) => void) {
+ let handlers = abortHandlers.get(streamId);
+ if (handlers == null) {
+ handlers = new Set();
+ abortHandlers.set(streamId, handlers);
+ }
+ handlers.add(handler);
+ return () => handlers?.delete(handler);
+ },
+ async emitAbortConfirmed(streamId: string, generationId: number) {
+ let acknowledged = acknowledgedAborts.get(streamId)?.has(generationId) === true;
+ for (const handler of abortHandlers.get(streamId) ?? []) {
+ acknowledged = handler(generationId) === true || acknowledged;
+ }
+ if (acknowledged) {
+ let generations = acknowledgedAborts.get(streamId);
+ if (generations == null) {
+ generations = new Set();
+ acknowledgedAborts.set(streamId, generations);
+ }
+ generations.add(generationId);
+ }
+ return acknowledged;
+ },
+ });
+ const ownerStore = new RedisJobStore(redis);
+ const recoveringStore = new RedisJobStore(redis);
+ const crashedStore = new RedisJobStore(redis);
+ const successorStore = new RedisJobStore(redis);
+ const owner = new GenerationJobManagerClass();
+ const recovering = new GenerationJobManagerClass();
+ const successor = new GenerationJobManagerClass();
+ owner.configure({ jobStore: ownerStore, eventTransport, isRedis: true });
+ recovering.configure({ jobStore: recoveringStore, eventTransport, isRedis: true });
+ successor.configure({ jobStore: successorStore, eventTransport, isRedis: true });
+ owner.initialize();
+ recovering.initialize();
+ successor.initialize();
+
+ const streamId = `redis-superseded-lost-create-${Date.now()}`;
+ const userId = 'redis-superseded-lost-create-user';
+ const doneEvents: Array<{ event: unknown; generationId?: number }> = [];
+ const subscription = eventTransport.subscribe(streamId, {
+ onChunk: () => undefined,
+ onDone: (event, generationId) => doneEvents.push({ event, generationId }),
+ });
+ const actualEval = redis.eval.bind(redis) as (...args: unknown[]) => Promise<unknown>;
+ const evalSpy = jest.spyOn(redis, 'eval');
+ let injectLostCreateReply = false;
+ evalSpy.mockImplementation((async (...args: unknown[]) => {
+ const result = await actualEval(...args);
+ const script = args[0];
+ if (
+ injectLostCreateReply &&
+ typeof script === 'string' &&
+ script.includes('previousJobExists') &&
+ script.includes('claim_lost')
+ ) {
+ injectLostCreateReply = false;
+ throw new Error('simulated lost C create reply');
+ }
+ return result;
+ }) as typeof redis.eval);
+
+ let getJobCalls = 0;
+ let crashedSuccessor: Awaited<ReturnType<RedisJobStore['createJob']>> | undefined;
+ const actualGetJob = recoveringStore.getJob.bind(recoveringStore);
+ const getJobSpy = jest.spyOn(recoveringStore, 'getJob').mockImplementation(async (id) => {
+ getJobCalls += 1;
+ if (id === streamId && getJobCalls === 2) {
+ crashedSuccessor = await crashedStore.createJob(
+ streamId,
+ userId,
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'crashed-successor-attempt',
+ );
+ }
+ return actualGetJob(id);
+ });
+
+ try {
+ const activePredecessor = await owner.createJob(streamId, userId, streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(await ownerStore.getJob(streamId)).toMatchObject({ providerAbortReady: true });
+ injectLostCreateReply = true;
+
+ await expect(
+ recovering.createJob(streamId, userId, streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('simulated lost C create reply');
+
+ expect(crashedSuccessor).toBeDefined();
+ expect(crashedSuccessor).toMatchObject({ providerAbortReady: false });
+ expect(activePredecessor.abortController.signal.aborted).toBe(true);
+ expect(doneEvents).toContainEqual({
+ event: expect.objectContaining({
+ final: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: activePredecessor.createdAt,
+ }),
+ generationId: activePredecessor.createdAt,
+ });
+ const jobKey = `stream:{${streamId}}:job`;
+ const retainedChain = JSON.parse((await redis.hget(jobKey, '__replacedGenerations'))!);
+ expect(retainedChain).toEqual([
+ expect.objectContaining({ createdAt: activePredecessor.createdAt, status: 'running' }),
+ expect.objectContaining({ status: 'running', providerAbortReady: false }),
+ ]);
+ expect(await crashedStore.getJob(streamId)).toMatchObject({
+ createdAt: crashedSuccessor!.createdAt,
+ status: 'running',
+ });
+
+ // The helper deliberately could not ACK another attempt's receipts. A
+ // later real successor inherits the chain, delivers it idempotently,
+ // and may then clear it under its own creation-attempt fence.
+ getJobSpy.mockRestore();
+ evalSpy.mockRestore();
+ const eventualSuccessor = await successor.createJob(streamId, userId, streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ expect(eventualSuccessor.createdAt).toBeGreaterThan(crashedSuccessor!.createdAt);
+ expect(await redis.hget(jobKey, '__replacedGenerations')).toBeNull();
+ } finally {
+ getJobSpy.mockRestore();
+ evalSpy.mockRestore();
+ subscription.unsubscribe();
+ await Promise.all([owner.destroy(), recovering.destroy(), successor.destroy()]);
+ await crashedStore.destroy();
+ }
+ });
+
+ test('rejects malformed replacement receipts before mutating the current job', async () => {
+ const corruptions: Array<{
+ name: string;
+ fields: (createdAt: number) => Record<string, string>;
+ }> = [
+ {
+ name: 'unsafe JSON epoch',
+ fields: () => ({
+ __replacedGenerations: JSON.stringify([
+ { createdAt: Number.MAX_SAFE_INTEGER + 1, status: 'running' },
+ ]),
+ __replacedCreatedAt: String(Number.MAX_SAFE_INTEGER + 1),
+ __replacedStatus: 'running',
+ }),
+ },
+ {
+ name: 'fractional JSON epoch',
+ fields: () => ({
+ __replacedGenerations: JSON.stringify([{ createdAt: 1.5, status: 'running' }]),
+ __replacedCreatedAt: '1.5',
+ __replacedStatus: 'running',
+ }),
+ },
+ {
+ name: 'unordered JSON epochs',
+ fields: () => ({
+ __replacedGenerations: JSON.stringify([
+ { createdAt: 2, status: 'running' },
+ { createdAt: 1, status: 'complete' },
+ ]),
+ __replacedCreatedAt: '1',
+ __replacedStatus: 'complete',
+ }),
+ },
+ {
+ name: 'receipt equal to current predecessor',
+ fields: (createdAt) => ({
+ __replacedGenerations: JSON.stringify([{ createdAt, status: 'complete' }]),
+ __replacedCreatedAt: String(createdAt),
+ __replacedStatus: 'complete',
+ }),
+ },
+ {
+ name: 'invalid provider abort readiness',
+ fields: () => ({
+ __replacedGenerations: JSON.stringify([
+ { createdAt: 1, status: 'running', providerAbortReady: 'yes' },
+ ]),
+ __replacedCreatedAt: '1',
+ __replacedStatus: 'running',
+ }),
+ },
+ {
+ name: 'invalid legacy scalar status',
+ fields: () => ({
+ __replacedCreatedAt: '1',
+ __replacedStatus: 'not-a-status',
+ }),
+ },
+ ];
+
+ for (const corruption of corruptions) {
+ const streamId = `redis-corrupt-receipt-${corruption.name.replaceAll(' ', '-')}-${Date.now()}`;
+ const current = await store.createJob(
+ streamId,
+ 'receipt-user',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'current-attempt',
+ );
+ const jobKey = `stream:{${streamId}}:job`;
+ const fields = corruption.fields(current.createdAt);
+ await redis.hset(jobKey, ...Object.entries(fields).flat());
+ const before = await redis.hgetall(jobKey);
+
+ await expect(
+ store.createJob(
+ streamId,
+ 'receipt-user',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ `replacement-attempt-${corruption.name}`,
+ ),
+ ).rejects.toThrow('replacement receipt is corrupt');
+ expect(await redis.hgetall(jobKey)).toEqual(before);
+ }
+ });
+
+ test('inherits a no-conversation replacement chain without corrupting its scalar mirror', async () => {
+ const streamId = `redis-no-conversation-chain-${Date.now()}`;
+ const first = await store.createJob(
+ streamId,
+ 'receipt-user',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'no-conversation-attempt-a',
+ );
+ const second = await store.createJob(
+ streamId,
+ 'receipt-user',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'no-conversation-attempt-b',
+ );
+ const third = await store.createJob(
+ streamId,
+ 'receipt-user',
+ undefined,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'no-conversation-attempt-c',
+ );
+
+ expect((third as typeof third & { replacedJobs?: unknown }).replacedJobs).toEqual([
+ { createdAt: first.createdAt, status: 'running' },
+ { createdAt: second.createdAt, status: 'running' },
+ ]);
+ });
+
+ test('replacement transport authorizes only the current receipt and tolerates zero DONE listeners', async () => {
+ const streamId = `redis-replacement-transport-${Date.now()}`;
+ const first = await store.createJob(
+ streamId,
+ 'transport-user',
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'transport-attempt-a',
+ );
+ expect(
+ await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ expectCreatedAt: first.createdAt,
+ patch: { completedAt: Date.now() },
+ }),
+ ).toBe(true);
+ await store.createJob(
+ streamId,
+ 'transport-user',
+ streamId,
+ undefined,
+ { generationProtocolVersion: 2 },
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ 'transport-attempt-b',
+ );
+ const subscriber = redis.duplicate();
+ const transport = new RedisEventTransport(redis, subscriber);
+ try {
+ await expect(
+ transport.emitReplacedDoneConfirmed(
+ streamId,
+ { final: true, reconcile: true },
+ first.createdAt,
+ 'transport-attempt-b',
+ ),
+ ).resolves.toBeUndefined();
+ await expect(
+ transport.emitReplacedDoneConfirmed(
+ streamId,
+ { final: true },
+ first.createdAt,
+ 'wrong-attempt',
+ ),
+ ).rejects.toThrow('no longer current');
+ await expect(
+ transport.emitReplacedDoneConfirmed(
+ streamId,
+ { final: true },
+ first.createdAt - 1,
+ 'transport-attempt-b',
+ ),
+ ).rejects.toThrow('no longer current');
+ await expect(transport.emitDone(streamId, { final: true }, first.createdAt)).rejects.toThrow(
+ 'fenced by a replacement',
+ );
+ await expect(transport.emitError(streamId, 'stale', first.createdAt)).rejects.toThrow(
+ 'fenced by a replacement',
+ );
+ } finally {
+ transport.destroy();
+ }
+ });
+
+ test('rejects partial low-level idempotency arguments without creating a Redis job', async () => {
+ const streamId = `redis-partial-create-idempotency-${Date.now()}`;
+ await expect(
+ store.createJob(
+ streamId,
+ 'user-1',
+ undefined,
+ undefined,
+ {},
+ undefined,
+ `{${streamId}}:user-1:req`,
+ 'claim-token',
+ ),
+ ).rejects.toThrow('Invalid generation job idempotency arguments');
+ expect(await redis.exists(`stream:{${streamId}}:job`)).toBe(0);
+ });
+
+ test('live Redis tombstones legacy/protocol-mismatch adoptions through job deletion', async () => {
+ const managerStore = new RedisJobStore(redis);
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: managerStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ });
+ manager.initialize();
+ try {
+ const userId = 'redis-adopt-user';
+
+ // Old v1 job: no durable clientRequestId, fresh upgraded request asks for
+ // v2. It cannot attach, but both mirrors must become v1 started
+ // tombstones before the controller returns 503.
+ const legacyStream = `redis-legacy-adopt-${Date.now()}`;
+ const legacyRequest = `redis-legacy-request-${Date.now()}`;
+ const legacyJob = await manager.createJob(legacyStream, userId, legacyStream, {
+ initialMetadata: { generationProtocolVersion: 1 },
+ });
+ const legacyFresh = await manager.claimGeneration(
+ userId,
+ legacyRequest,
+ legacyStream,
+ legacyStream,
+ 2,
+ );
+ await expect(
+ manager.resumeClaimedGeneration(userId, legacyRequest, legacyStream, legacyFresh.existing!),
+ ).rejects.toThrow('conservatively fenced');
+
+ const legacyKey = `stream:idem:{${userId}:${legacyRequest}}`;
+ const legacyPrimaryKey = `stream:idem:{${legacyStream}}:${userId}:${legacyRequest}`;
+ for (const key of [legacyKey, legacyPrimaryKey]) {
+ expect(JSON.parse((await redis.get(key))!)).toMatchObject({
+ startedAt: legacyJob.createdAt,
+ generationProtocolVersion: 1,
+ });
+ }
+ expect(await managerStore.deleteJob(legacyStream, legacyJob.createdAt)).toBe(true);
+ const legacyRetry = await manager.claimGeneration(
+ userId,
+ legacyRequest,
+ legacyStream,
+ legacyStream,
+ 2,
+ );
+ expect(legacyRetry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: legacyJob.createdAt, generationProtocolVersion: 1 },
+ });
+ await expect(
+ manager.takeoverGeneration(userId, legacyRequest, legacyStream, legacyRetry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ await expect(
+ manager.createJob(legacyStream, userId, legacyStream, {
+ idempotencyClientRequestId: legacyRequest,
+ idempotencyClaimToken: legacyFresh.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ }),
+ ).rejects.toThrow('claim was taken over');
+
+ // Correlated v2 job with expired mirrors and a fresh v1 request remains
+ // attachable, but the new tombstone is capped to immutable job protocol.
+ const currentStream = `redis-current-adopt-${Date.now()}`;
+ const currentRequest = `redis-current-request-${Date.now()}`;
+ const original = await manager.claimGeneration(
+ userId,
+ currentRequest,
+ currentStream,
+ currentStream,
+ 2,
+ );
+ const currentJob = await manager.createJob(currentStream, userId, currentStream, {
+ idempotencyClientRequestId: currentRequest,
+ idempotencyClaimToken: original.existing!.claimToken,
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ const currentLegacyKey = `stream:idem:{${userId}:${currentRequest}}`;
+ const currentPrimaryKey = `stream:idem:{${currentStream}}:${userId}:${currentRequest}`;
+ await redis.del(currentLegacyKey);
+ await redis.del(currentPrimaryKey);
+ const currentFresh = await manager.claimGeneration(
+ userId,
+ currentRequest,
+ currentStream,
+ currentStream,
+ 1,
+ );
+ await expect(
+ manager.resumeClaimedGeneration(
+ userId,
+ currentRequest,
+ currentStream,
+ currentFresh.existing!,
+ ),
+ ).resolves.toMatchObject({
+ startedAt: currentJob.createdAt,
+ generationProtocolVersion: 2,
+ });
+ expect(await managerStore.deleteJob(currentStream, currentJob.createdAt)).toBe(true);
+ const currentRetry = await manager.claimGeneration(
+ userId,
+ currentRequest,
+ currentStream,
+ currentStream,
+ 1,
+ );
+ expect(currentRetry).toMatchObject({
+ claimed: false,
+ existing: { startedAt: currentJob.createdAt, generationProtocolVersion: 2 },
+ });
+ await expect(
+ manager.takeoverGeneration(userId, currentRequest, currentStream, currentRetry.existing!),
+ ).resolves.toMatchObject({ claimed: false });
+ } finally {
+ await manager.destroy();
+ }
+ });
+});
diff --git a/packages/api/src/stream/__tests__/staleJobReaping.spec.ts b/packages/api/src/stream/__tests__/staleJobReaping.spec.ts
index 972683c97a..d6982ba4f8 100644
--- a/packages/api/src/stream/__tests__/staleJobReaping.spec.ts
+++ b/packages/api/src/stream/__tests__/staleJobReaping.spec.ts
@@ -170,7 +170,7 @@ describe('InMemoryJobStore - stale running-job failsafe', () => {
expect(await store.hasJob('s1')).toBe(false);
// No finalization ever ran — the crashed run's queue must be claimable.
- const claimed = await store.claimParkedSteers('s1', '"userId":"u1"');
+ const claimed = await store.claimParkedSteers('s1', 'u1', 'tenant-1');
expect(claimed).toBeDefined();
const parsed = JSON.parse(claimed as string) as {
userId: string;
diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.ts
index d1a578a2dc..a45e977331 100644
--- a/packages/api/src/stream/__tests__/startup.spec.ts
+++ b/packages/api/src/stream/__tests__/startup.spec.ts
@@ -1,10 +1,14 @@
+import type { AbortResult } from '~/stream/interfaces/IJobStore';
import type { AgentStartupTelemetry } from '~/agents/startup';
import type { ServerSentEvent } from '~/types';
+import {
+ GenerationJobManagerClass,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
+} from '~/stream/GenerationJobManager';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication';
import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
-import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
function createTelemetry(): jest.Mocked<AgentStartupTelemetry> {
return {
@@ -86,6 +90,7 @@ describe('GenerationJobManager startup telemetry', () => {
expect(job.metadata).toMatchObject({
userId: 'user-1',
conversationId: 'conversation-1',
+ checkpointNamespace: String(job.createdAt),
userMessage: {
messageId: 'message-1',
parentMessageId: 'parent-1',
@@ -102,7 +107,12 @@ describe('GenerationJobManager startup telemetry', () => {
});
expect(job.metadata.tenantId).toBeUndefined();
expect(job.metadata.pendingAction).toBeUndefined();
- expect(updateJob).not.toHaveBeenCalled();
+ expect(updateJob).toHaveBeenCalledTimes(1);
+ expect(updateJob).toHaveBeenCalledWith(
+ 'stream-initial-metadata',
+ { providerAbortReady: true },
+ job.createdAt,
+ );
await manager.destroy();
});
@@ -256,7 +266,7 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.destroy();
});
- it('bypasses publication receipts after startup telemetry completes', async () => {
+ it('keeps active-only publication receipts after startup telemetry completes', async () => {
const eventTransport = new InMemoryEventTransport();
const emitChunk = jest.spyOn(eventTransport, 'emitChunk');
const publishWithReceipt = jest.fn().mockResolvedValue(0);
@@ -286,10 +296,20 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.emitChunk('stream-hot-path', firstDelta);
await manager.emitChunk('stream-hot-path', laterDelta);
- expect(publishWithReceipt).toHaveBeenCalledTimes(1);
- expect(publishWithReceipt).toHaveBeenCalledWith('stream-hot-path', firstDelta, job.createdAt);
- expect(emitChunk).toHaveBeenCalledTimes(1);
- expect(emitChunk).toHaveBeenCalledWith('stream-hot-path', laterDelta, job.createdAt);
+ expect(publishWithReceipt).toHaveBeenCalledTimes(2);
+ expect(publishWithReceipt).toHaveBeenNthCalledWith(
+ 1,
+ 'stream-hot-path',
+ firstDelta,
+ job.createdAt,
+ );
+ expect(publishWithReceipt).toHaveBeenNthCalledWith(
+ 2,
+ 'stream-hot-path',
+ laterDelta,
+ job.createdAt,
+ );
+ expect(emitChunk).not.toHaveBeenCalled();
subscription?.unsubscribe();
await manager.destroy();
@@ -391,6 +411,109 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.destroy();
});
+ it('closes predecessor subscribers with a handoff event when createJob replaces the stream', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = new InMemoryEventTransport();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false });
+ manager.initialize();
+ const streamId = 'stream-active-replacement-handoff';
+ const predecessor = await manager.createJob(streamId, 'user-1', streamId);
+ const onDone = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, onDone);
+
+ const replacement = await manager.createJob(streamId, 'user-1', streamId);
+
+ expect(replacement.createdAt).not.toBe(predecessor.createdAt);
+ expect(onDone).toHaveBeenCalledWith({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: predecessor.createdAt,
+ conversation: { conversationId: streamId },
+ });
+ expect(predecessor.abortController.signal.aborted).toBe(true);
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ expect(eventTransport.getSubscriberCount(streamId)).toBe(0);
+ subscription?.unsubscribe();
+ await manager.destroy();
+ });
+
+ it('notifies and stops the exact predecessor installed after the pre-create lookup', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const abortHandlers = new Set<(generationId?: number) => void>();
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ onAbort: jest.fn(async (_streamId: string, handler: (generationId?: number) => void) => {
+ abortHandlers.add(handler);
+ return () => abortHandlers.delete(handler);
+ }),
+ emitAbort: jest.fn((_streamId: string, generationId?: number) => {
+ for (const handler of [...abortHandlers]) {
+ handler(generationId);
+ }
+ }),
+ });
+ const firstManager = new GenerationJobManagerClass();
+ const secondManager = new GenerationJobManagerClass();
+ firstManager.configure({ jobStore, eventTransport, isRedis: false });
+ secondManager.configure({ jobStore, eventTransport, isRedis: false });
+ firstManager.initialize();
+ secondManager.initialize();
+ const streamId = 'stream-atomic-replacement-predecessor';
+ await firstManager.createJob(streamId, 'user-1', streamId);
+
+ const originalCreateJob = jobStore.createJob.bind(jobStore);
+ let signalLastCreateReachedStore: (() => void) | undefined;
+ const lastCreateReachedStore = new Promise<void>((resolve) => {
+ signalLastCreateReachedStore = resolve;
+ });
+ let releaseLastCreate: (() => void) | undefined;
+ const lastCreateGate = new Promise<void>((resolve) => {
+ releaseLastCreate = resolve;
+ });
+ jest.spyOn(jobStore, 'createJob').mockImplementation(async (...args) => {
+ if (args[4]?.model === 'last-creator') {
+ signalLastCreateReachedStore?.();
+ await lastCreateGate;
+ }
+ return originalCreateJob(...args);
+ });
+
+ try {
+ // This manager has already pre-read the first epoch when its atomic
+ // store create is gated. A second replica installs the true predecessor
+ // in the gap.
+ const creatingLast = firstManager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { model: 'last-creator' },
+ });
+ await lastCreateReachedStore;
+
+ const exactPredecessor = await secondManager.createJob(streamId, 'user-1', streamId, {
+ initialMetadata: { model: 'middle-creator' },
+ });
+ const onDone = jest.fn();
+ const subscription = await secondManager.subscribe(streamId, () => undefined, onDone);
+
+ releaseLastCreate?.();
+ const replacement = await creatingLast;
+
+ expect(replacement.createdAt).toBeGreaterThan(exactPredecessor.createdAt);
+ expect(onDone).toHaveBeenCalledWith({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'generation_replaced',
+ generationCreatedAt: exactPredecessor.createdAt,
+ conversation: { conversationId: streamId },
+ });
+ expect(exactPredecessor.abortController.signal.aborted).toBe(true);
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ subscription?.unsubscribe();
+ } finally {
+ releaseLastCreate?.();
+ await Promise.all([firstManager.destroy(), secondManager.destroy()]);
+ }
+ });
+
it('shares one lazy runtime across concurrent first subscriptions', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
const eventTransport = new InMemoryEventTransport();
@@ -490,6 +613,34 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.destroy();
});
+ it('fences initial and resume subscriptions to the requested generation epoch', async () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
+ const manager = createManager();
+ const oldJob = await manager.createJob('stream-epoch-fence', 'user-1', 'conversation-1');
+ now.mockReturnValue(2000);
+ const replacement = await manager.createJob('stream-epoch-fence', 'user-1', 'conversation-1');
+ now.mockRestore();
+ const onChunk = jest.fn();
+
+ const initial = await manager.subscribe('stream-epoch-fence', onChunk, undefined, undefined, {
+ expectedCreatedAt: oldJob.createdAt,
+ });
+ const resumed = await manager.subscribeWithResume(
+ 'stream-epoch-fence',
+ onChunk,
+ undefined,
+ undefined,
+ { expectedCreatedAt: oldJob.createdAt },
+ );
+
+ expect(replacement.createdAt).not.toBe(oldJob.createdAt);
+ expect(initial).toBeNull();
+ expect(resumed).toEqual({ subscription: null, resumeState: null, pendingEvents: [] });
+ expect(onChunk).not.toHaveBeenCalled();
+
+ await manager.destroy();
+ });
+
it('does not return a lazy runtime replaced while its abort listener activates', async () => {
const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
@@ -559,7 +710,9 @@ describe('GenerationJobManager startup telemetry', () => {
};
await manager.emitDone('stream-late-complete', staleFinal, predecessor.createdAt);
- await manager.completeJob('stream-late-complete', undefined, predecessor.createdAt);
+ await expect(
+ manager.completeJob('stream-late-complete', undefined, predecessor.createdAt),
+ ).resolves.toBe(false);
await expect(jobStore.getJob('stream-late-complete')).resolves.toMatchObject({
createdAt: replacement.createdAt,
@@ -624,6 +777,33 @@ describe('GenerationJobManager startup telemetry', () => {
}
});
+ it('refuses to abort a replacement when the caller pins the predecessor epoch', async () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
+ const manager = createManager();
+ try {
+ const streamId = 'stream-abort-route-epoch-fence';
+ const predecessor = await manager.createJob(streamId, 'user-1');
+
+ now.mockReturnValue(2000);
+ const replacement = await manager.createJob(streamId, 'user-1');
+
+ await expect(
+ manager.abortJob(streamId, { expectedCreatedAt: predecessor.createdAt }),
+ ).resolves.toMatchObject({
+ success: false,
+ failureReason: 'generation_replaced',
+ });
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ } finally {
+ now.mockRestore();
+ await manager.destroy();
+ }
+ });
+
it('releases its exact runtime when completion loses to a terminal transition', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
const abortDisposer = jest.fn();
@@ -679,6 +859,203 @@ describe('GenerationJobManager startup telemetry', () => {
}
});
+ it('lets abort win while completion is blocked at the terminal CAS', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitDone: jest.fn(),
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: false });
+ manager.initialize();
+ const originalTransition = jobStore.transitionStatusAndDrainSteers.bind(jobStore);
+ let signalCompletionAtCas: (() => void) | undefined;
+ const completionAtCas = new Promise<void>((resolve) => {
+ signalCompletionAtCas = resolve;
+ });
+ let releaseCompletionCas: (() => void) | undefined;
+ const completionCasGate = new Promise<void>((resolve) => {
+ releaseCompletionCas = resolve;
+ });
+ let gateCompletion = true;
+ jest.spyOn(jobStore, 'transitionStatusAndDrainSteers').mockImplementation(async (...args) => {
+ if (gateCompletion && args[1].to === 'complete') {
+ gateCompletion = false;
+ signalCompletionAtCas?.();
+ await completionCasGate;
+ }
+ return originalTransition(...args);
+ });
+
+ try {
+ const streamId = 'stream-abort-wins-completion-persistence-race';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const completionClaimPromise = manager.claimTerminalJob(
+ streamId,
+ 'complete',
+ undefined,
+ job.createdAt,
+ { persistencePending: true },
+ );
+ await completionAtCas;
+
+ await expect(
+ manager.abortJob(streamId, { expectedCreatedAt: job.createdAt }),
+ ).resolves.toMatchObject({
+ success: true,
+ finalEvent: expect.objectContaining({ final: true, aborted: true }),
+ });
+ releaseCompletionCas?.();
+
+ await expect(completionClaimPromise).resolves.toBeNull();
+ expect(eventTransport.emitDone).toHaveBeenCalledTimes(1);
+ expect(eventTransport.emitDone).toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({ final: true, aborted: true }),
+ job.createdAt,
+ );
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'aborted',
+ terminalPersistencePending: false,
+ finalEvent: expect.any(String),
+ });
+ } finally {
+ releaseCompletionCas?.();
+ await manager.destroy();
+ }
+ });
+
+ it('lets completion persist and publish when abort is blocked at the terminal CAS', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitDone: jest.fn(),
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: false });
+ manager.initialize();
+ const originalTransition = jobStore.transitionStatusAndDrainSteers.bind(jobStore);
+ let signalAbortAtCas: (() => void) | undefined;
+ const abortAtCas = new Promise<void>((resolve) => {
+ signalAbortAtCas = resolve;
+ });
+ let releaseAbortCas: (() => void) | undefined;
+ const abortCasGate = new Promise<void>((resolve) => {
+ releaseAbortCas = resolve;
+ });
+ let gateAbort = true;
+ jest.spyOn(jobStore, 'transitionStatusAndDrainSteers').mockImplementation(async (...args) => {
+ if (gateAbort && args[1].to === 'aborted') {
+ gateAbort = false;
+ signalAbortAtCas?.();
+ await abortCasGate;
+ }
+ return originalTransition(...args);
+ });
+
+ try {
+ const streamId = 'stream-completion-wins-abort-persistence-race';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const aborting = manager.abortJob(streamId, { expectedCreatedAt: job.createdAt });
+ await abortAtCas;
+
+ const completionClaim = await manager.claimTerminalJob(
+ streamId,
+ 'complete',
+ undefined,
+ job.createdAt,
+ { persistencePending: true },
+ );
+ expect(completionClaim).not.toBeNull();
+ releaseAbortCas?.();
+ await expect(aborting).resolves.toMatchObject({ success: false, finalEvent: null });
+
+ const completionFinal = {
+ final: true,
+ conversation: { conversationId: streamId },
+ responseMessage: { messageId: 'response-completion-winner', unfinished: false },
+ } as const;
+ await expect(
+ manager.publishTerminalClaim(completionClaim!, completionFinal),
+ ).resolves.toEqual({ finalEvent: completionFinal, persistenceFailed: false });
+ await manager.finishTerminalJob(completionClaim!);
+
+ expect(eventTransport.emitDone).toHaveBeenCalledTimes(1);
+ expect(eventTransport.emitDone).toHaveBeenCalledWith(
+ streamId,
+ completionFinal,
+ job.createdAt,
+ );
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'complete',
+ terminalPersistencePending: false,
+ finalEvent: JSON.stringify(completionFinal),
+ });
+ } finally {
+ releaseAbortCas?.();
+ await manager.destroy();
+ }
+ });
+
+ it('wakes a live subscriber and retains the durable final when DONE publication fails', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = new InMemoryEventTransport();
+ jest.spyOn(eventTransport, 'emitDone').mockImplementation(() => {
+ throw new Error('DONE transport down');
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: true });
+ manager.initialize();
+ const streamId = 'stream-terminal-done-publication-fails';
+
+ try {
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const onDone = jest.fn();
+ const onError = jest.fn();
+ await manager.subscribe(streamId, () => undefined, onDone, onError, {
+ expectedCreatedAt: job.createdAt,
+ });
+ const claim = await manager.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, {
+ persistencePending: true,
+ });
+ expect(claim).not.toBeNull();
+ const finalEvent = {
+ final: true,
+ conversation: { conversationId: streamId },
+ responseMessage: { messageId: 'response-durable-final', unfinished: false },
+ } as const;
+
+ await expect(manager.publishTerminalClaim(claim!, finalEvent)).rejects.toThrow(
+ 'DONE transport down',
+ );
+
+ expect(onDone).not.toHaveBeenCalled();
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+
+ // Even a reconnect that wins the race with finishTerminalJob must see
+ // the durable final, not the process-local signal used to recycle SSE.
+ jest.spyOn(eventTransport, 'emitDone').mockRestore();
+ const replayDone = jest.fn();
+ const replayError = jest.fn();
+ await manager.subscribe(streamId, () => undefined, replayDone, replayError, {
+ expectedCreatedAt: job.createdAt,
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(replayDone).toHaveBeenCalledWith(finalEvent);
+ expect(replayError).not.toHaveBeenCalled();
+
+ await manager.finishTerminalJob(claim!);
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'complete',
+ terminalPersistencePending: false,
+ finalEvent: JSON.stringify(finalEvent),
+ });
+ } finally {
+ await manager.destroy();
+ }
+ });
+
it('releases its exact runtime when abort loses to a terminal transition', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
const abortDisposer = jest.fn();
@@ -734,7 +1111,51 @@ describe('GenerationJobManager startup telemetry', () => {
}
});
- it('retains a same-epoch run that resumes while abort is claiming terminal state', async () => {
+ it('does not misclassify a generation deleted during abort CAS as a replacement', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const originalTransition = jobStore.transitionStatus.bind(jobStore);
+ let signalTransitionStarted: (() => void) | undefined;
+ const transitionStarted = new Promise<void>((resolve) => {
+ signalTransitionStarted = resolve;
+ });
+ let releaseTransition: (() => void) | undefined;
+ const transitionGate = new Promise<void>((resolve) => {
+ releaseTransition = resolve;
+ });
+
+ try {
+ const streamId = 'stream-abort-deleted-race';
+ const job = await manager.createJob(streamId, 'user-1');
+ jest.spyOn(jobStore, 'transitionStatus').mockImplementationOnce(async (...args) => {
+ signalTransitionStarted?.();
+ await transitionGate;
+ return originalTransition(...args);
+ });
+
+ const aborting = manager.abortJob(streamId, { expectedCreatedAt: job.createdAt });
+ await transitionStarted;
+ await jobStore.deleteJob(streamId, job.createdAt);
+ releaseTransition?.();
+
+ const result = await aborting;
+ expect(result).toMatchObject({ success: false, finalEvent: null });
+ expect(result.failureReason).toBeUndefined();
+ expect(job.abortController.signal.aborted).toBe(true);
+ } finally {
+ releaseTransition?.();
+ await manager.destroy();
+ }
+ });
+
+ it('retries and stops a same-epoch run that resumes while abort is claiming terminal state', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
jest.spyOn(jobStore, 'destroy').mockResolvedValue();
const abortDisposer = jest.fn();
@@ -766,6 +1187,13 @@ describe('GenerationJobManager startup telemetry', () => {
try {
const streamId = 'stream-abort-resume-race';
const job = await manager.createJob(streamId, 'user-1');
+ const queuedSteer = {
+ steerId: 'steer-survives-abort-cas-loss',
+ text: 'keep this on the resumed run',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ };
+ await expect(manager.steering.enqueue(streamId, queuedSteer)).resolves.toBe(1);
const pendingAction = createPendingAction(streamId);
await expect(manager.approvals.pause(streamId, pendingAction)).resolves.toBe(true);
@@ -774,21 +1202,14 @@ describe('GenerationJobManager startup telemetry', () => {
await expect(manager.approvals.resolve(streamId, pendingAction.actionId)).resolves.toBe(true);
releaseAbortTransition?.();
- await expect(aborting).resolves.toMatchObject({ success: false, finalEvent: null });
- expect(job.abortController.signal.aborted).toBe(false);
- expect(abortDisposer).not.toHaveBeenCalled();
- await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
- createdAt: job.createdAt,
- status: 'running',
- });
-
- manager.prepareForShutdown();
- await manager.destroy();
- await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
- createdAt: job.createdAt,
- status: 'error',
- error: 'Generation interrupted because its server shut down',
+ await expect(aborting).resolves.toMatchObject({
+ success: true,
+ finalEvent: expect.objectContaining({ aborted: true }),
});
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(abortDisposer).toHaveBeenCalledTimes(1);
+ await expect(jobStore.getJob(streamId)).resolves.toBeNull();
+ await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([]);
} finally {
releaseAbortTransition?.();
if (manager.getRuntimeStats().runtimeStateSize > 0) {
@@ -797,6 +1218,81 @@ describe('GenerationJobManager startup telemetry', () => {
}
});
+ it.each([
+ { label: 'successful completion', error: undefined, terminalStatus: 'complete' as const },
+ { label: 'error completion', error: 'boom', terminalStatus: 'error' as const },
+ ])(
+ 'does not drain a queued steer when $label loses to a pause',
+ async ({ error, terminalStatus }) => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ jest.spyOn(jobStore, 'destroy').mockResolvedValue();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const originalTransition = jobStore.transitionStatus.bind(jobStore);
+ let signalCompletionTransitionStarted: (() => void) | undefined;
+ const completionTransitionStarted = new Promise<void>((resolve) => {
+ signalCompletionTransitionStarted = resolve;
+ });
+ let releaseCompletionTransition: (() => void) | undefined;
+ const completionTransitionGate = new Promise<void>((resolve) => {
+ releaseCompletionTransition = resolve;
+ });
+ jest.spyOn(jobStore, 'transitionStatus').mockImplementation(async (...args) => {
+ if (args[1].to === terminalStatus) {
+ signalCompletionTransitionStarted?.();
+ await completionTransitionGate;
+ }
+ return originalTransition(...args);
+ });
+
+ try {
+ const streamId = `stream-completion-pause-race-${terminalStatus}`;
+ const job = await manager.createJob(streamId, 'user-1');
+ const queuedSteer = {
+ steerId: `steer-survives-${terminalStatus}-cas-loss`,
+ text: 'keep this while paused',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ };
+ await expect(manager.steering.enqueue(streamId, queuedSteer)).resolves.toBe(1);
+
+ const completing = manager.completeJob(streamId, error, job.createdAt);
+ await completionTransitionStarted;
+ const pendingAction = createPendingAction(streamId);
+ await expect(
+ jobStore.transitionStatus(streamId, {
+ from: 'running',
+ to: 'requires_action',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ pendingAction,
+ pendingActionId: pendingAction.actionId,
+ },
+ }),
+ ).resolves.toBe(true);
+ releaseCompletionTransition?.();
+ await completing;
+
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'requires_action',
+ });
+ await expect(manager.steering.peek(streamId, job.createdAt)).resolves.toEqual([
+ queuedSteer,
+ ]);
+ } finally {
+ releaseCompletionTransition?.();
+ manager.prepareForShutdown();
+ await manager.destroy();
+ }
+ },
+ );
+
it('releases ownership but keeps the abort runtime when completion observes a pause', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
jest.spyOn(jobStore, 'destroy').mockResolvedValue();
@@ -859,12 +1355,12 @@ describe('GenerationJobManager startup telemetry', () => {
expect(job.abortController.signal.aborted).toBe(true);
expect(abortDisposer).toHaveBeenCalledTimes(1);
const restored = await jobStore.createJob(streamId, 'user-1');
- expect(restored.createdAt).toBe(job.createdAt);
+ expect(restored.createdAt).toBeGreaterThan(job.createdAt);
manager.prepareForShutdown();
await manager.destroy();
await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
- createdAt: job.createdAt,
+ createdAt: restored.createdAt,
status: 'running',
});
} finally {
@@ -900,12 +1396,12 @@ describe('GenerationJobManager startup telemetry', () => {
expect(job.abortController.signal.aborted).toBe(true);
expect(abortDisposer).toHaveBeenCalledTimes(1);
const restored = await jobStore.createJob(streamId, 'user-1');
- expect(restored.createdAt).toBe(job.createdAt);
+ expect(restored.createdAt).toBeGreaterThan(job.createdAt);
manager.prepareForShutdown();
await manager.destroy();
await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
- createdAt: job.createdAt,
+ createdAt: restored.createdAt,
status: 'running',
});
} finally {
@@ -1137,6 +1633,386 @@ describe('GenerationJobManager startup telemetry', () => {
await manager.destroy();
});
+ it('atomically terminalizes a paused job when post-HITL persistence fails', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = 'stream-paused-persistence-error';
+ const job = await manager.createJob(streamId, 'user-1');
+ const pendingAction = createPendingAction(streamId);
+ await expect(manager.approvals.pause(streamId, pendingAction)).resolves.toBe(true);
+
+ await manager.completeJob(streamId, 'paused user row unavailable', job.createdAt);
+
+ expect(job.abortController.signal.aborted).toBe(true);
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'error',
+ error: 'paused user row unavailable',
+ });
+ const terminalJob = await jobStore.getJob(streamId);
+ expect(terminalJob?.pendingAction).toBeUndefined();
+ expect(terminalJob?.pendingActionId).toBeUndefined();
+ await manager.destroy();
+ });
+
+ it('releases the claimed runtime when terminal error publication throws', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitError: jest.fn(async () => {
+ throw new Error('transport unavailable');
+ }),
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport,
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const job = await manager.createJob('stream-error-publish-fails', 'user-1');
+ const onError = jest.fn();
+ const subscription = await manager.subscribe(
+ 'stream-error-publish-fails',
+ () => undefined,
+ undefined,
+ onError,
+ );
+
+ await expect(
+ manager.completeJob('stream-error-publish-fails', 'provider failed', job.createdAt),
+ ).resolves.toBe(true);
+
+ expect(onError).toHaveBeenCalledWith('provider failed');
+ expect(job.abortController.signal.aborted).toBe(true);
+ await expect(jobStore.getJob('stream-error-publish-fails')).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'error',
+ error: 'provider failed',
+ });
+ subscription?.unsubscribe();
+ await manager.destroy();
+ });
+
+ it('finishes abort runtime cleanup but retains its durable final when publication throws', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitDone: jest.fn(async () => {
+ throw new Error('done transport unavailable');
+ }),
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: true });
+ manager.initialize();
+ const job = await manager.createJob('stream-abort-publish-fails', 'user-1');
+
+ await expect(manager.abortJob('stream-abort-publish-fails')).rejects.toThrow(
+ 'done transport unavailable',
+ );
+
+ expect(job.abortController.signal.aborted).toBe(true);
+ await expect(jobStore.getJob('stream-abort-publish-fails')).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ status: 'aborted',
+ terminalPersistencePending: false,
+ finalEvent: expect.any(String),
+ });
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ await manager.destroy();
+ });
+
+ it('awaits required persistence before publishing the normal abort FINAL', async () => {
+ const order: string[] = [];
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitDone: jest.fn(() => {
+ order.push('publish');
+ }),
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: true });
+ manager.initialize();
+ const streamId = 'stream-abort-persist-before-final';
+ const job = await manager.createJob(streamId, 'user-1');
+ const beforePublish = jest.fn(async (result: AbortResult) => {
+ order.push('persist');
+ const terminalJob = await jobStore.getJob(streamId);
+ expect(terminalJob).toMatchObject({
+ createdAt: job.createdAt,
+ status: 'aborted',
+ terminalPersistencePending: true,
+ });
+ expect(terminalJob?.finalEvent).toBeUndefined();
+ expect(result).toMatchObject({ success: true, finalEvent: { final: true, aborted: true } });
+ });
+
+ const result = await manager.abortJob(streamId, {
+ expectedCreatedAt: job.createdAt,
+ beforePublish,
+ });
+ expect(result).toMatchObject({ success: true });
+ expect(result.persistenceFailed).toBeUndefined();
+
+ expect(beforePublish).toHaveBeenCalledTimes(1);
+ expect(order).toEqual(['persist', 'publish']);
+ await manager.destroy();
+ });
+
+ it('holds a late subscriber until abort persistence finishes', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = new InMemoryEventTransport();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: true });
+ manager.initialize();
+ const streamId = 'stream-abort-persistence-late-subscriber';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ let signalPersistenceStarted: (() => void) | undefined;
+ const persistenceStarted = new Promise<void>((resolve) => {
+ signalPersistenceStarted = resolve;
+ });
+ let releasePersistence: (() => void) | undefined;
+ const persistenceGate = new Promise<void>((resolve) => {
+ releasePersistence = resolve;
+ });
+
+ try {
+ const aborting = manager.abortJob(streamId, {
+ expectedCreatedAt: job.createdAt,
+ beforePublish: async () => {
+ signalPersistenceStarted?.();
+ await persistenceGate;
+ },
+ });
+ await persistenceStarted;
+
+ const onDone = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, onDone, undefined, {
+ expectedCreatedAt: job.createdAt,
+ });
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ expect(onDone).not.toHaveBeenCalled();
+ await expect(manager.getJob(streamId)).resolves.toMatchObject({
+ metadata: { terminalPersistencePending: true },
+ });
+
+ releasePersistence?.();
+ await expect(aborting).resolves.toMatchObject({ success: true });
+ expect(onDone).toHaveBeenCalledWith(expect.objectContaining({ final: true, aborted: true }));
+ subscription?.unsubscribe();
+ } finally {
+ releasePersistence?.();
+ await manager.destroy();
+ }
+ });
+
+ it('recovers a stale abort-persistence marker to conservative reconciliation', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = 'stream-abort-persistence-owner-crashed';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ await jobStore.transitionStatusAndDrainSteers(streamId, {
+ from: 'running',
+ to: 'aborted',
+ expectCreatedAt: job.createdAt,
+ patch: {
+ completedAt: Date.now() - 60_000,
+ terminalPersistencePending: true,
+ terminalPersistenceStartedAt: Date.now() - 60_000,
+ },
+ });
+ const onDone = jest.fn();
+
+ const subscription = await manager.subscribe(streamId, () => undefined, onDone);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+
+ expect(onDone).toHaveBeenCalledWith({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ terminalStatus: 'aborted',
+ generationCreatedAt: job.createdAt,
+ conversation: { conversationId: streamId },
+ });
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ terminalPersistencePending: false,
+ finalEvent: expect.any(String),
+ });
+ subscription?.unsubscribe();
+ await manager.destroy();
+ });
+
+ it('closes an already-live subscriber when abort finalization storage fails', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: true,
+ });
+ manager.initialize();
+ const streamId = 'stream-abort-finalization-store-fails';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const onDone = jest.fn();
+ const subscription = await manager.subscribe(streamId, () => undefined, onDone);
+ jest
+ .spyOn(jobStore, 'finalizeTerminalPersistence')
+ .mockRejectedValueOnce(new Error('terminal store unavailable'));
+
+ const result = await manager.abortJob(streamId, { expectedCreatedAt: job.createdAt });
+
+ expect(result).toMatchObject({ success: true, persistenceFailed: true });
+ expect(onDone).toHaveBeenCalledWith({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ terminalStatus: 'aborted',
+ generationCreatedAt: job.createdAt,
+ conversation: { conversationId: streamId },
+ });
+ await expect(jobStore.getJob(streamId)).resolves.toMatchObject({
+ createdAt: job.createdAt,
+ terminalPersistencePending: true,
+ });
+ subscription?.unsubscribe();
+ await manager.destroy();
+ });
+
+ it('publishes reconciliation instead of a normal abort FINAL when required persistence fails', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = Object.assign(new InMemoryEventTransport(), {
+ emitDone: jest.fn(),
+ });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: false, cleanupOnComplete: true });
+ manager.initialize();
+ const streamId = 'stream-abort-persistence-fails';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+
+ await expect(
+ manager.abortJob(streamId, {
+ expectedCreatedAt: job.createdAt,
+ beforePublish: async () => {
+ throw new Error('partial response save failed');
+ },
+ }),
+ ).resolves.toMatchObject({
+ success: true,
+ persistenceFailed: true,
+ finalEvent: {
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ terminalStatus: 'aborted',
+ generationCreatedAt: job.createdAt,
+ },
+ });
+
+ expect(eventTransport.emitDone).toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'abort_persistence_failed',
+ }),
+ job.createdAt,
+ );
+ expect(eventTransport.emitDone).not.toHaveBeenCalledWith(
+ streamId,
+ expect.objectContaining({ aborted: true }),
+ job.createdAt,
+ );
+ await expect(jobStore.getJob(streamId)).resolves.toBeNull();
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ await manager.destroy();
+ });
+
+ it('a delayed terminal finish cannot clean up a same-stream replacement runtime', async () => {
+ const now = jest.spyOn(Date, 'now').mockReturnValue(1000);
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: true,
+ });
+ manager.initialize();
+
+ try {
+ const predecessor = await manager.createJob('stream-delayed-finish', 'user-1');
+ const claim = await manager.claimTerminalJob(
+ 'stream-delayed-finish',
+ 'complete',
+ undefined,
+ predecessor.createdAt,
+ );
+ expect(claim).not.toBeNull();
+
+ now.mockReturnValue(2000);
+ const replacement = await manager.createJob('stream-delayed-finish', 'user-1');
+ await manager.finishTerminalJob(claim!);
+
+ expect(replacement.abortController.signal.aborted).toBe(false);
+ await expect(jobStore.getJob('stream-delayed-finish')).resolves.toMatchObject({
+ createdAt: replacement.createdAt,
+ status: 'running',
+ });
+ } finally {
+ now.mockRestore();
+ await manager.destroy();
+ }
+ });
+
+ it('closes a late subscriber with a reconcile event when terminal payload publication was lost', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
+ const eventTransport = new InMemoryEventTransport();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport,
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = 'stream-terminal-payload-lost';
+ const job = await manager.createJob(streamId, 'user-1', streamId);
+ const claim = await manager.claimTerminalJob(streamId, 'complete', undefined, job.createdAt);
+ expect(claim).not.toBeNull();
+ const onDone = jest.fn();
+
+ const subscription = await manager.subscribe(streamId, () => undefined, onDone);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ await Promise.resolve();
+
+ expect(onDone).toHaveBeenCalledWith({
+ final: true,
+ reconcile: true,
+ reconcileReason: 'terminal_payload_missing',
+ terminalStatus: 'complete',
+ generationCreatedAt: job.createdAt,
+ conversation: { conversationId: streamId },
+ });
+ expect(subscription).not.toBeNull();
+ expect(eventTransport.getSubscriberCount(streamId)).toBe(0);
+ await manager.finishTerminalJob(claim!);
+ await manager.destroy();
+ });
+
it('finalizes the exact durable job when abort-listener setup fails before createJob returns', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
const eventTransport = Object.assign(new InMemoryEventTransport(), {
@@ -1454,7 +2330,7 @@ describe('GenerationJobManager startup telemetry', () => {
it('does not let late success overwrite or delete a shutdown error', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60_000 });
jest.spyOn(jobStore, 'destroy').mockResolvedValue();
- const originalCloseAndDrain = jobStore.closeAndDrainSteers.bind(jobStore);
+ const originalTransition = jobStore.transitionStatus.bind(jobStore);
let signalCompletionStarted: (() => void) | undefined;
const completionStarted = new Promise<void>((resolve) => {
signalCompletionStarted = resolve;
@@ -1463,10 +2339,12 @@ describe('GenerationJobManager startup telemetry', () => {
const completionGate = new Promise<void>((resolve) => {
releaseCompletion = resolve;
});
- jest.spyOn(jobStore, 'closeAndDrainSteers').mockImplementationOnce(async (...args) => {
- signalCompletionStarted?.();
- await completionGate;
- return originalCloseAndDrain(...args);
+ jest.spyOn(jobStore, 'transitionStatus').mockImplementation(async (...args) => {
+ if (args[1].to === 'complete') {
+ signalCompletionStarted?.();
+ await completionGate;
+ }
+ return originalTransition(...args);
});
const manager = new GenerationJobManagerClass();
manager.configure({
diff --git a/packages/api/src/stream/__tests__/steerReceiptIntegrity.spec.ts b/packages/api/src/stream/__tests__/steerReceiptIntegrity.spec.ts
new file mode 100644
index 0000000000..9602960389
--- /dev/null
+++ b/packages/api/src/stream/__tests__/steerReceiptIntegrity.spec.ts
@@ -0,0 +1,623 @@
+import type { SteerQueueItem, SteerReceipt } from '../interfaces/IJobStore';
+import {
+ GenerationJobManagerClass,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
+} from '../GenerationJobManager';
+import { InMemoryEventTransport } from '../implementations/InMemoryEventTransport';
+import { registerChunkPublicationCapability } from '../internal/chunkPublication';
+import { InMemoryJobStore } from '../implementations/InMemoryJobStore';
+import { STEER_ENQUEUE_RECEIPT_FULL } from '../interfaces/IJobStore';
+
+type ReceiptEntry = { receipt: SteerReceipt; expiresAt: number };
+type ReceiptMap = Map<string, ReceiptEntry>;
+
+interface ReceiptCorruption {
+ name: string;
+ apply: (receipts: ReceiptMap, clientSteerId: string) => void;
+}
+
+const receiptCorruptions: ReceiptCorruption[] = [
+ {
+ name: 'missing',
+ apply: (receipts, clientSteerId) => {
+ receipts.delete(clientSteerId);
+ },
+ },
+ {
+ name: 'malformed',
+ apply: (receipts, clientSteerId) => {
+ receipts.set(clientSteerId, {
+ receipt: 'not-a-receipt' as unknown as SteerReceipt,
+ expiresAt: Number.POSITIVE_INFINITY,
+ });
+ },
+ },
+ {
+ name: 'wrong-state',
+ apply: (receipts, clientSteerId) => {
+ const entry = receipts.get(clientSteerId);
+ if (entry != null) {
+ entry.receipt.state = entry.receipt.state === 'queued' ? 'claimed' : 'queued';
+ }
+ },
+ },
+ {
+ name: 'wrong-generation',
+ apply: (receipts, clientSteerId) => {
+ const entry = receipts.get(clientSteerId);
+ if (entry != null) {
+ entry.receipt.generationCreatedAt += 1;
+ }
+ },
+ },
+];
+
+function receiptMap(store: InMemoryJobStore, streamId: string): ReceiptMap {
+ return (
+ store as unknown as {
+ steerReceipts: Map<string, ReceiptMap>;
+ }
+ ).steerReceipts.get(streamId)!;
+}
+
+async function enqueueReceiptSteer(
+ store: InMemoryJobStore,
+ streamId: string,
+ suffix: string,
+): Promise<{ item: SteerQueueItem; createdAt: number }> {
+ const job = await store.createJob(streamId, 'receipt-user', streamId);
+ const item: SteerQueueItem = {
+ steerId: `steer-${suffix}`,
+ clientSteerId: `client-${suffix}`,
+ text: `instruction ${suffix}`,
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ const result = await store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${suffix}`,
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ );
+ expect(result).toMatchObject({ state: 'queued', item });
+ return { item, createdAt: job.createdAt };
+}
+
+describe('InMemoryJobStore steer receipt integrity', () => {
+ test.each(['deleted', 'v1-replaced'] as const)(
+ 'replays an existing receipt after its accepting job is %s',
+ async (lifecycle) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-replay-${lifecycle}`;
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, lifecycle);
+ if (lifecycle === 'deleted') {
+ await expect(store.deleteJob(streamId, createdAt)).resolves.toBe(true);
+ } else {
+ const replacement = await store.createJob(streamId, item.userId, streamId, undefined, {
+ generationProtocolVersion: 1,
+ });
+ expect(replacement.createdAt).not.toBe(createdAt);
+ await expect(store.peekSteers(streamId, replacement.createdAt)).resolves.toEqual([]);
+ }
+
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'leftover',
+ generationCreatedAt: createdAt,
+ item,
+ });
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ { ...item, steerId: `duplicate-${lifecycle}` },
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${lifecycle}`,
+ userId: item.userId,
+ generationCreatedAt: createdAt,
+ },
+ false,
+ createdAt,
+ ),
+ ).resolves.toMatchObject({
+ state: 'leftover',
+ generationCreatedAt: createdAt,
+ item,
+ });
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test('refreshes a recovery lease past the stale-job horizon and keeps its receipt', async () => {
+ const store = new InMemoryJobStore({ staleJobTimeout: 20 * 60 * 1000 });
+ const streamId = 'receipt-recovery-lease-activity';
+ const base = Date.now();
+ const nowSpy = jest.spyOn(Date, 'now').mockReturnValue(base);
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, 'recovery-lease');
+ await expect(store.closeAndDrainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ const recovery = await store.createJob(
+ streamId,
+ item.userId,
+ streamId,
+ undefined,
+ {},
+ item.steerId,
+ undefined,
+ undefined,
+ undefined,
+ {
+ text: item.text,
+ fileIds: (item.files ?? []).flatMap((file) => file.file_id ?? []).sort(),
+ },
+ );
+
+ // Generation remains healthy beyond the one-time create lease. Activity
+ // near its original horizon must move both parked source and receipt out.
+ nowSpy.mockReturnValue(base + 24 * 60 * 1000);
+ store.recordActivity(streamId, recovery.createdAt);
+ nowSpy.mockReturnValue(base + 30 * 60 * 1000);
+ await expect(
+ store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'error',
+ expectCreatedAt: recovery.createdAt,
+ patch: { error: 'crashed before persistence', completedAt: Date.now() },
+ }),
+ ).resolves.toBe(true);
+
+ const parked = await store.claimParkedSteers(streamId, item.userId);
+ expect(JSON.parse(parked!)).toMatchObject({
+ userId: item.userId,
+ steers: [{ steerId: item.steerId, clientSteerId: item.clientSteerId }],
+ });
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'leftover',
+ item: { steerId: item.steerId },
+ });
+ } finally {
+ nowSpy.mockRestore();
+ await store.destroy();
+ }
+ });
+
+ test('does not let a stale cancel discard a source leased to a live recovery', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'receipt-live-recovery-cancel';
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, 'live-recovery');
+ await expect(store.closeAndDrainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ const recovery = await store.createJob(
+ streamId,
+ item.userId,
+ streamId,
+ undefined,
+ {},
+ item.steerId,
+ undefined,
+ undefined,
+ undefined,
+ {
+ text: item.text,
+ fileIds: (item.files ?? []).flatMap((file) => file.file_id ?? []).sort(),
+ },
+ );
+
+ await expect(
+ store.discardSteerLeftover(streamId, item.clientSteerId!, item.steerId, item.userId),
+ ).resolves.toBe(false);
+ await expect(
+ store.consumeParkedSteer(
+ streamId,
+ item.steerId,
+ item.userId,
+ undefined,
+ recovery.createdAt,
+ ),
+ ).resolves.toBe(true);
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'recovered',
+ });
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test('keeps host content and a delivered receipt when publication fails after commit', async () => {
+ const store = new InMemoryJobStore();
+ const transport = new InMemoryEventTransport();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: false });
+ manager.initialize();
+
+ const streamId = 'receipt-post-commit-publish-failure';
+ const item: SteerQueueItem = {
+ steerId: 'steer-post-commit',
+ clientSteerId: 'client-post-commit',
+ text: 'keep this instruction',
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ const hostContent: Array<{ steerId: string; text: string }> = [];
+ let subscription: { unsubscribe: () => void } | null = null;
+
+ try {
+ const job = await manager.createJob(streamId, item.userId, streamId);
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: 'fingerprint-post-commit',
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ ),
+ ).resolves.toMatchObject({ state: 'queued', item });
+ await expect(store.drainSteers(streamId, job.createdAt)).resolves.toEqual([item]);
+
+ // A live subscriber forces the in-memory manager through transport
+ // publication instead of its no-subscriber replay buffer.
+ subscription = await manager.subscribe(streamId, () => undefined);
+ expect(subscription).not.toBeNull();
+ jest.spyOn(transport, 'emitChunk').mockImplementation(() => {
+ throw new Error('subscriber write failed');
+ });
+
+ const part = { steerId: item.steerId, text: item.text };
+ const applyLikeAgentClient = async (): Promise<void> => {
+ hostContent.push(part);
+ try {
+ await manager.emitChunk(
+ streamId,
+ {
+ event: 'on_steer_applied',
+ data: { steerId: item.steerId, index: 0, part },
+ },
+ {
+ durable: true,
+ expectedCreatedAt: job.createdAt,
+ deliveredSteer: item,
+ },
+ );
+ } catch (error) {
+ hostContent.pop();
+ throw error;
+ }
+ };
+
+ await expect(applyLikeAgentClient()).resolves.toBeUndefined();
+ expect(hostContent).toEqual([part]);
+ await expect(store.peekClaimedSteers(streamId, job.createdAt)).resolves.toEqual([]);
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'delivered',
+ item,
+ });
+ } finally {
+ subscription?.unsubscribe();
+ await manager.destroy();
+ }
+ });
+
+ test('keeps a committed steer but retires its provider when publication is durably fenced', async () => {
+ const store = new InMemoryJobStore();
+ const transport = new InMemoryEventTransport();
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore: store, eventTransport: transport, isRedis: true });
+ manager.initialize();
+
+ const streamId = 'receipt-post-commit-owner-fence';
+ const item: SteerQueueItem = {
+ steerId: 'steer-owner-fence',
+ clientSteerId: 'client-owner-fence',
+ text: 'commit before fencing',
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ const hostContent: Array<{ steerId: string; text: string }> = [];
+ let subscription: { unsubscribe: () => void } | null = null;
+
+ try {
+ const job = await manager.createJob(streamId, item.userId, streamId, {
+ initialMetadata: { generationProtocolVersion: 2 },
+ });
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: 'fingerprint-owner-fence',
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ ),
+ ).resolves.toMatchObject({ state: 'queued', item });
+ await expect(store.drainSteers(streamId, job.createdAt)).resolves.toEqual([item]);
+
+ const onError = jest.fn();
+ subscription = await manager.subscribe(streamId, () => undefined, undefined, onError);
+ expect(subscription).not.toBeNull();
+ registerChunkPublicationCapability(transport, async () => false);
+
+ const part = { steerId: item.steerId, text: item.text };
+ hostContent.push(part);
+ await expect(
+ manager.emitChunk(
+ streamId,
+ {
+ event: 'on_steer_applied',
+ data: { steerId: item.steerId, index: 0, part },
+ },
+ {
+ durable: true,
+ expectedCreatedAt: job.createdAt,
+ deliveredSteer: item,
+ },
+ ),
+ ).resolves.toBeUndefined();
+
+ expect(hostContent).toEqual([part]);
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'delivered',
+ item,
+ });
+ expect(job.abortController.signal.aborted).toBe(true);
+ expect(onError).toHaveBeenCalledWith(TERMINAL_PUBLICATION_RECONNECT_ERROR);
+ expect(manager.getRuntimeStats().runtimeStateSize).toBe(0);
+ } finally {
+ subscription?.unsubscribe();
+ await manager.destroy();
+ }
+ });
+
+ test('atomically settles a claimed receipt when its applied chunk commits', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'receipt-valid-delivery';
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, 'valid');
+
+ await expect(store.drainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_steer_applied', data: { steerId: item.steerId } },
+ createdAt,
+ item,
+ ),
+ ).resolves.toBe(true);
+
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(store.getSteerReceipt(streamId, item.clientSteerId!)).resolves.toMatchObject({
+ state: 'delivered',
+ item,
+ });
+ } finally {
+ await store.destroy();
+ }
+ });
+
+ test.each(receiptCorruptions)(
+ 'refuses to drain a queue with a $name receipt before moving any item',
+ async ({ name, apply }) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-drain-${name}`;
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, `drain-${name}`);
+ apply(receiptMap(store, streamId), item.clientSteerId!);
+
+ await expect(store.drainSteers(streamId, createdAt)).rejects.toThrow(
+ 'Invalid steer receipt while draining',
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([item]);
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(receiptCorruptions)(
+ 'refuses to arm with a $name receipt before mutating the queued item',
+ async ({ name, apply }) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-arm-${name}`;
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, `arm-${name}`);
+ await store.updateJob(streamId, { preemptCapable: true }, createdAt);
+ apply(receiptMap(store, streamId), item.clientSteerId!);
+
+ await expect(store.armSteerVersioned(streamId, item.steerId, createdAt)).rejects.toThrow(
+ 'Invalid steer receipt while arming',
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(receiptCorruptions)(
+ 'refuses to cancel with a $name receipt before mutating the queued item',
+ async ({ name, apply }) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-cancel-${name}`;
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, `cancel-${name}`);
+ apply(receiptMap(store, streamId), item.clientSteerId!);
+
+ await expect(store.removeSteer(streamId, item.steerId, createdAt)).rejects.toThrow(
+ 'Invalid steer receipt while cancelling',
+ );
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(receiptCorruptions)(
+ 'refuses to downgrade with a $name receipt before mutating any queued item',
+ async ({ name, apply }) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-downgrade-${name}`;
+ try {
+ const job = await store.createJob(streamId, 'receipt-user', streamId, undefined, {
+ preemptCapable: true,
+ });
+ const item: SteerQueueItem = {
+ steerId: `steer-downgrade-${name}`,
+ clientSteerId: `client-downgrade-${name}`,
+ text: `instruction downgrade-${name}`,
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-downgrade-${name}`,
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ true,
+ job.createdAt,
+ ),
+ ).resolves.toMatchObject({ state: 'queued', item: { preempt: true, preemptRevision: 1 } });
+ const armedItem = (await store.peekSteers(streamId, job.createdAt))[0];
+ await store.updateJob(streamId, { preemptCapable: false }, job.createdAt);
+ apply(receiptMap(store, streamId), item.clientSteerId!);
+
+ await expect(store.downgradeSteerPreempts(streamId, job.createdAt)).rejects.toThrow(
+ 'Invalid steer receipt while downgrading',
+ );
+ await expect(store.peekSteers(streamId, job.createdAt)).resolves.toEqual([armedItem]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(receiptCorruptions)(
+ 'refuses to settle a claimed steer with a $name receipt before removing its claim',
+ async ({ name, apply }) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-append-${name}`;
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, `append-${name}`);
+ await expect(store.drainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ apply(receiptMap(store, streamId), item.clientSteerId!);
+
+ await expect(
+ store.appendChunk(
+ streamId,
+ { event: 'on_steer_applied', data: { steerId: item.steerId } },
+ createdAt,
+ item,
+ ),
+ ).resolves.toBe(false);
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test.each(receiptCorruptions)(
+ 'refuses to restore a claimed steer with a $name receipt before changing either list',
+ async ({ name, apply }) => {
+ const store = new InMemoryJobStore();
+ const streamId = `receipt-restore-${name}`;
+ try {
+ const { item, createdAt } = await enqueueReceiptSteer(store, streamId, `restore-${name}`);
+ await expect(store.drainSteers(streamId, createdAt)).resolves.toEqual([item]);
+ apply(receiptMap(store, streamId), item.clientSteerId!);
+
+ await expect(store.restoreClaimedSteers(streamId, [item], createdAt)).resolves.toBe(false);
+ await expect(store.peekSteers(streamId, createdAt)).resolves.toEqual([]);
+ await expect(store.peekClaimedSteers(streamId, createdAt)).resolves.toEqual([item]);
+ } finally {
+ await store.destroy();
+ }
+ },
+ );
+
+ test('refuses a 101st receipt without evicting replay evidence or queueing the item', async () => {
+ const store = new InMemoryJobStore();
+ const streamId = 'receipt-cap';
+ try {
+ const job = await store.createJob(streamId, 'receipt-user', streamId);
+ for (let index = 0; index < 100; index++) {
+ const item: SteerQueueItem = {
+ steerId: `steer-cap-${index}`,
+ clientSteerId: `client-cap-${index}`,
+ text: `instruction ${index}`,
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ item,
+ {
+ clientSteerId: item.clientSteerId!,
+ fingerprint: `fingerprint-${index}`,
+ userId: item.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ ),
+ ).resolves.toMatchObject({ state: 'queued', item });
+ await expect(store.removeSteer(streamId, item.steerId)).resolves.toBe(true);
+ }
+
+ const overflow: SteerQueueItem = {
+ steerId: 'steer-cap-overflow',
+ clientSteerId: 'client-cap-overflow',
+ text: 'must not queue',
+ userId: 'receipt-user',
+ createdAt: Date.now(),
+ };
+ await expect(
+ store.enqueueSteerWithReceipt(
+ streamId,
+ overflow,
+ {
+ clientSteerId: overflow.clientSteerId!,
+ fingerprint: 'fingerprint-overflow',
+ userId: overflow.userId,
+ generationCreatedAt: job.createdAt,
+ },
+ false,
+ job.createdAt,
+ ),
+ ).resolves.toBe(STEER_ENQUEUE_RECEIPT_FULL);
+
+ await expect(store.peekSteers(streamId, job.createdAt)).resolves.toEqual([]);
+ await expect(store.getSteerReceipt(streamId, 'client-cap-overflow')).resolves.toBeNull();
+ await expect(store.getSteerReceipt(streamId, 'client-cap-0')).resolves.toMatchObject({
+ state: 'cancelled',
+ item: { steerId: 'steer-cap-0' },
+ });
+ } finally {
+ await store.destroy();
+ }
+ });
+});
diff --git a/packages/api/src/stream/__tests__/steering.spec.ts b/packages/api/src/stream/__tests__/steering.spec.ts
index 32edb26524..5f45d9a498 100644
--- a/packages/api/src/stream/__tests__/steering.spec.ts
+++ b/packages/api/src/stream/__tests__/steering.spec.ts
@@ -1,7 +1,11 @@
import { logger } from '@librechat/data-schemas';
import { SteerEvents } from 'librechat-data-provider';
import type { TPendingSteer, Agents } from 'librechat-data-provider';
-import type { SteerQueueItem, IEventTransport } from '~/stream/interfaces/IJobStore';
+import type {
+ SteerQueueItem,
+ IEventTransport,
+ PreemptMessage,
+} from '~/stream/interfaces/IJobStore';
import type { ResumeState, ServerSentEvent } from '~/types';
import {
STEER_ENQUEUE_NOT_RUNNING,
@@ -140,6 +144,18 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
expect(await manager.steering.drain(streamId)).toEqual([]);
});
+ test('destroy clears claimed steer state as well as the live queue', async () => {
+ const streamId = 'steer-destroy-claimed';
+ await manager.createJob(streamId, 'user-1');
+ await manager.steering.enqueue(streamId, buildSteer('claimed before destroy'));
+ await manager.steering.drain(streamId);
+ expect(await jobStore.peekClaimedSteers(streamId)).toHaveLength(1);
+
+ await jobStore.destroy();
+
+ expect(await jobStore.peekClaimedSteers(streamId)).toEqual([]);
+ });
+
test('peek is non-destructive', async () => {
const streamId = 'steer-peek';
await manager.createJob(streamId, 'user-1');
@@ -199,6 +215,24 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
expect(await manager.steering.cancel(streamId, steer.steerId)).toBe(false);
expect(await manager.steering.cancel('nonexistent', steer.steerId)).toBe(false);
});
+
+ test('a cancel authorized for generation A cannot remove generation B state', async () => {
+ const streamId = 'steer-cancel-replaced';
+ const predecessor = await manager.createJob(streamId, 'user-1');
+ const reusedId = buildSteer('predecessor');
+ await manager.steering.enqueue(streamId, reusedId, predecessor.createdAt);
+
+ const replacement = await manager.createJob(streamId, 'user-1');
+ const replacementItem = { ...buildSteer('replacement'), steerId: reusedId.steerId };
+ await manager.steering.enqueue(streamId, replacementItem, replacement.createdAt);
+
+ await expect(
+ manager.steering.cancel(streamId, reusedId.steerId, predecessor.createdAt),
+ ).resolves.toBe(false);
+ await expect(manager.steering.peek(streamId, replacement.createdAt)).resolves.toEqual([
+ replacementItem,
+ ]);
+ });
});
describe('closeAndDrain', () => {
@@ -255,7 +289,7 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
describe('park / claim (no-subscriber recovery)', () => {
const owner = { userId: 'user-1' };
- test('parked leftovers are claimable exactly once', async () => {
+ test('parked leftovers remain replayable until exact recovery starts', async () => {
const streamId = 'steer-park';
await manager.createJob(streamId, 'user-1');
const leftovers: TPendingSteer[] = [
@@ -264,8 +298,8 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
await manager.steering.park(streamId, leftovers, owner);
expect(await manager.steering.claim(streamId, owner)).toEqual(leftovers);
- // Claim-on-read: a second reload cannot re-mint dismissed chips.
- expect(await manager.steering.claim(streamId, owner)).toEqual([]);
+ // A lost status response must not erase the only recovery copy.
+ expect(await manager.steering.claim(streamId, owner)).toEqual(leftovers);
});
test('a stale generation cannot park leftovers onto a replacement', async () => {
@@ -373,16 +407,14 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
// Back inside the window: had the sweep NOT removed the entry, this
// claim would return it (expiry is otherwise only checked at read).
nowSpy.mockReturnValue(base);
- expect(await store.claimParkedSteers('steer-park-sweep', '"userId":"user-1"')).toBe(
- undefined,
- );
+ expect(await store.claimParkedSteers('steer-park-sweep', 'user-1')).toBe(undefined);
} finally {
nowSpy.mockRestore();
await store.destroy();
}
});
- test('a replacement run clears parked leftovers', async () => {
+ test('a replacement preserves leftovers until it starts their exact recovery', async () => {
const streamId = 'steer-park-replaced';
await manager.createJob(streamId, 'user-1');
await manager.steering.park(
@@ -392,9 +424,61 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
);
await manager.createJob(streamId, 'user-1');
+ expect((await manager.steering.claim(streamId, owner)).map((item) => item.steerId)).toEqual([
+ 'p3',
+ ]);
+
+ const failedRecovery = await manager.createJob(streamId, 'user-1', undefined, {
+ recoveredSteerId: 'p3',
+ recoveredSteerPayload: { text: 'stale', fileIds: [] },
+ });
+ expect(await manager.steering.claim(streamId, owner)).toEqual([]);
+
+ // Startup failed before the user message became durable: the lease is
+ // non-destructive, so terminal status exposes the source again.
+ await manager.completeJob(streamId, 'provider init failed', failedRecovery.createdAt);
+ expect((await manager.steering.claim(streamId, owner)).map((item) => item.steerId)).toEqual([
+ 'p3',
+ ]);
+
+ const persistedRecovery = await manager.createJob(streamId, 'user-1', undefined, {
+ recoveredSteerId: 'p3',
+ recoveredSteerPayload: { text: 'stale', fileIds: [] },
+ });
+ expect(
+ await manager.steering.consumeRecovered(streamId, 'p3', owner, persistedRecovery.createdAt),
+ ).toBe(true);
expect(await manager.steering.claim(streamId, owner)).toEqual([]);
});
+ test.each([
+ ['changed text', { text: 'forged words', fileIds: ['file-a', 'file-b'] }],
+ ['changed files', { text: 'original words', fileIds: ['file-a', 'file-c'] }],
+ ])(
+ 'refuses recovery with %s without leasing or consuming the source',
+ async (_label, proof) => {
+ const streamId = `steer-recovery-mismatch-${_label.replace(' ', '-')}`;
+ const originalJob = await manager.createJob(streamId, 'user-1');
+ const source = {
+ steerId: `source-${_label.replace(' ', '-')}`,
+ text: 'original words',
+ createdAt: Date.now(),
+ files: [{ file_id: 'file-b' }, { file_id: 'file-a' }],
+ };
+ await manager.steering.park(streamId, [source], owner);
+
+ await expect(
+ manager.createJob(streamId, 'user-1', undefined, {
+ recoveredSteerId: source.steerId,
+ recoveredSteerPayload: proof,
+ }),
+ ).rejects.toMatchObject({ code: 'RECOVERY_PAYLOAD_MISMATCH' });
+
+ expect((await manager.getJob(streamId))?.createdAt).toBe(originalJob.createdAt);
+ expect(await manager.steering.claim(streamId, owner)).toEqual([source]);
+ },
+ );
+
test('approval expiry parks queued steers instead of deleting them', async () => {
const streamId = 'steer-expire-park';
await manager.createJob(streamId, 'user-1');
@@ -666,15 +750,15 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
expect(event.data.index).toBe(0);
});
- test('an unchanged empty queue skips the content re-read', async () => {
+ test('an unchanged empty queue still checks the fresh content frontier', async () => {
const streamId = 'steer-gap-quiet';
- await manager.createJob(streamId, 'user-1');
+ const job = await manager.createJob(streamId, 'user-1');
jest.spyOn(manager, 'getResumeState').mockResolvedValue(staleSnapshot(streamId));
const readSpy = jest.spyOn(jobStore, 'getContentParts');
const result = await manager.subscribeWithResume(streamId, jest.fn());
- expect(readSpy).not.toHaveBeenCalled();
+ expect(readSpy).toHaveBeenCalledWith(streamId, job.createdAt);
expect(result.resumeState?.pendingSteers).toBeUndefined();
expect(result.pendingEvents).toEqual([]);
});
@@ -740,6 +824,34 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
expect(state?.pendingSteers?.every((s) => !('userId' in s))).toBe(true);
});
+ test('preserves claimed-prefix FIFO when equal timestamps sort UUIDs the other way', async () => {
+ const streamId = 'steer-resume-equal-time-fifo';
+ const createdAt = Date.now();
+ const job = await manager.createJob(streamId, 'user-1');
+ const claimedFirst: SteerQueueItem = {
+ steerId: 'z-accepted-first',
+ text: 'accepted first',
+ userId: 'user-1',
+ createdAt,
+ };
+ const queuedLater: SteerQueueItem = {
+ steerId: 'a-accepted-later',
+ text: 'accepted later',
+ userId: 'user-1',
+ createdAt,
+ };
+
+ await manager.steering.enqueue(streamId, claimedFirst, job.createdAt);
+ await manager.steering.drain(streamId, job.createdAt);
+ await manager.steering.enqueue(streamId, queuedLater, job.createdAt);
+
+ const state = await manager.getResumeState(streamId);
+ expect(state?.pendingSteers?.map((steer) => steer.steerId)).toEqual([
+ 'z-accepted-first',
+ 'a-accepted-later',
+ ]);
+ });
+
test('getResumeState omits pendingSteers when the queue is empty', async () => {
const streamId = 'steer-resume-empty';
await manager.createJob(streamId, 'user-1');
@@ -782,8 +894,8 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
const streamId = 'steer-durable';
const job = await redisModeManager.createJob(streamId, 'user-1');
- let resolveAppend!: () => void;
- const appendGate = new Promise<void>((resolve) => {
+ let resolveAppend!: (value: boolean) => void;
+ const appendGate = new Promise<boolean>((resolve) => {
resolveAppend = resolve;
});
jest.spyOn(store, 'appendChunk').mockReturnValue(appendGate);
@@ -798,7 +910,7 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
expect(settled).toBe(false);
expect(publishSpy).not.toHaveBeenCalled();
- resolveAppend();
+ resolveAppend(true);
await emit;
expect(settled).toBe(true);
expect(publishSpy).toHaveBeenCalledWith(streamId, steerEvent, job.createdAt);
@@ -816,7 +928,7 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
const job = await redisModeManager.createJob(streamId, 'user-1');
// Never resolves: the per-delta hot path must not gate on durability.
- jest.spyOn(store, 'appendChunk').mockReturnValue(new Promise<void>(() => undefined));
+ jest.spyOn(store, 'appendChunk').mockReturnValue(new Promise<boolean>(() => undefined));
const publishSpy = jest.spyOn(transport, 'emitChunk');
await redisModeManager.emitChunk(streamId, steerEvent);
@@ -831,7 +943,7 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const transport = new InMemoryEventTransport();
const redisModeManager = buildRedisModeManager(store, transport);
- let resolveAppend: (() => void) | undefined;
+ let resolveAppend: ((value: boolean) => void) | undefined;
try {
const streamId = 'steer-durable-replaced';
@@ -839,7 +951,7 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
const appendStarted = new Promise<void>((resolve) => {
jest.spyOn(store, 'appendChunk').mockImplementationOnce(
() =>
- new Promise<void>((resolveAppendPromise) => {
+ new Promise<boolean>((resolveAppendPromise) => {
resolveAppend = resolveAppendPromise;
resolve();
}),
@@ -851,14 +963,14 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
now.mockReturnValue(200);
const replacement = await redisModeManager.createJob(streamId, 'user-1');
- resolveAppend?.();
+ resolveAppend?.(true);
await staleEmission;
expect(predecessor.abortController.signal.aborted).toBe(true);
expect(replacement.abortController.signal.aborted).toBe(false);
expect(publishSpy).not.toHaveBeenCalled();
} finally {
- resolveAppend?.();
+ resolveAppend?.(true);
now.mockRestore();
await redisModeManager.destroy();
}
@@ -868,6 +980,40 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
describe('preempt request lifecycle (in-memory)', () => {
let manager: GenerationJobManagerClass;
+ function createControlledPreemptManager(): {
+ controlled: GenerationJobManagerClass;
+ deliver: (message: PreemptMessage) => void;
+ } {
+ let listener: ((message: PreemptMessage) => void) | undefined;
+ const eventTransport: IEventTransport = Object.assign(new InMemoryEventTransport(), {
+ onPreempt: (_streamId: string, callback: (message: PreemptMessage) => void) => {
+ listener = callback;
+ return () => {
+ if (listener === callback) {
+ listener = undefined;
+ }
+ };
+ },
+ });
+ const controlled = new GenerationJobManagerClass();
+ controlled.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport,
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ controlled.initialize();
+ return {
+ controlled,
+ deliver: (message) => {
+ if (listener == null) {
+ throw new Error('Preempt listener is not registered');
+ }
+ listener(message);
+ },
+ };
+ }
+
beforeEach(() => {
manager = new GenerationJobManagerClass();
manager.configure({
@@ -1010,15 +1156,17 @@ describe('preempt request lifecycle (in-memory)', () => {
expect(manager.isPreemptRequested(streamId)).toBe(false);
});
- test('toPendingSteer keeps the preempt label for parked/replayed chips', () => {
+ test('toPendingSteer keeps preempt and client correlation for parked/replayed chips', () => {
const item: SteerQueueItem = {
steerId: 'steer-1',
+ clientSteerId: 'local-steer-1',
text: 'interrupt me',
userId: 'user-1',
createdAt: Date.now(),
preempt: true,
};
expect(toPendingSteer(item).preempt).toBe(true);
+ expect(toPendingSteer(item).clientSteerId).toBe('local-steer-1');
expect(toPendingSteer({ ...item, preempt: undefined }).preempt).toBeUndefined();
});
@@ -1109,7 +1257,9 @@ describe('preempt request lifecycle (in-memory)', () => {
*/
test('rearmQueuedPreempts rebuilds the armed set from the durable queue', async () => {
const streamId = 'preempt-rearm';
- const job = await manager.createJob(streamId, 'user-1');
+ const job = await manager.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
await manager.steering.enqueue(streamId, {
steerId: 'steer-preempt',
text: 'interrupt me',
@@ -1144,7 +1294,9 @@ describe('preempt request lifecycle (in-memory)', () => {
*/
test('rearmQueuedPreempts drops an armed id the durable queue no longer backs', async () => {
const streamId = 'preempt-rearm-orphan';
- const job = await manager.createJob(streamId, 'user-1');
+ const job = await manager.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
/** Arm survived from before ownership moved; its steer is long drained. */
manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);
@@ -1172,7 +1324,9 @@ describe('preempt request lifecycle (in-memory)', () => {
*/
test('an arm that lands while the queue is being read is not tombstoned', async () => {
const streamId = 'preempt-rearm-race';
- const job = await manager.createJob(streamId, 'user-1');
+ const job = await manager.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
const peek = manager.steering.peek.bind(manager.steering);
const spy = jest
@@ -1204,7 +1358,9 @@ describe('preempt request lifecycle (in-memory)', () => {
test('an orphan dropped at handover is tombstoned against a late arm', async () => {
const streamId = 'preempt-rearm-orphan-tombstone';
- const job = await manager.createJob(streamId, 'user-1');
+ const job = await manager.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);
await manager.rearmQueuedPreempts(streamId, job.createdAt);
@@ -1215,6 +1371,103 @@ describe('preempt request lifecycle (in-memory)', () => {
expect(manager.isPreemptRequested(streamId)).toBe(false);
});
+ test('a delayed arm stays disarmed after its steer drained and reconciliation saw empty', async () => {
+ const { controlled, deliver } = createControlledPreemptManager();
+ const streamId = 'preempt-delayed-after-reconcile';
+ try {
+ const job = await controlled.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
+ const enqueued = await controlled.steering.enqueueVersioned(
+ streamId,
+ {
+ steerId: 'steer-delayed',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ },
+ true,
+ job.createdAt,
+ );
+ if (typeof enqueued === 'number') {
+ throw new Error(`Unexpected enqueue rejection: ${enqueued}`);
+ }
+
+ /** Simulate a remote drain whose CLEAR was lost, then ownership
+ * reconciliation completing before the buffered ARM is delivered. */
+ await controlled.steering.drain(streamId, job.createdAt);
+ expect(await controlled.rearmQueuedPreempts(streamId, job.createdAt)).toBe(0);
+
+ deliver({
+ op: 'arm',
+ createdAt: job.createdAt,
+ steerIds: [enqueued.item.steerId],
+ revisions: { [enqueued.item.steerId]: enqueued.item.preemptRevision ?? 0 },
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(controlled.getArmedPreemptIds(streamId, job.createdAt)).toEqual([]);
+ expect(controlled.isPreemptRequested(streamId)).toBe(false);
+ } finally {
+ await controlled.destroy();
+ }
+ });
+
+ test('a drain between arm validation and local arming leaves a tombstone', async () => {
+ const { controlled, deliver } = createControlledPreemptManager();
+ const streamId = 'preempt-drain-during-validation';
+ const peek = controlled.steering.peek.bind(controlled.steering);
+ let drainedDuringValidation = false;
+ const peekSpy = jest
+ .spyOn(controlled.steering, 'peek')
+ .mockImplementation(async (id: string, expectedCreatedAt?: number) => {
+ const snapshot = await peek(id, expectedCreatedAt);
+ const drained = await controlled.steering.drain(id, expectedCreatedAt);
+ drainedDuringValidation = drained.length > 0;
+ await controlled.noteSteersRemoved(
+ id,
+ drained.map((item) => item.steerId),
+ expectedCreatedAt,
+ );
+ return snapshot;
+ });
+
+ try {
+ const job = await controlled.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
+ const enqueued = await controlled.steering.enqueueVersioned(
+ streamId,
+ {
+ steerId: 'steer-validation-race',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ },
+ true,
+ job.createdAt,
+ );
+ if (typeof enqueued === 'number') {
+ throw new Error(`Unexpected enqueue rejection: ${enqueued}`);
+ }
+
+ deliver({
+ op: 'arm',
+ createdAt: job.createdAt,
+ steerIds: [enqueued.item.steerId],
+ revisions: { [enqueued.item.steerId]: enqueued.item.preemptRevision ?? 0 },
+ });
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(drainedDuringValidation).toBe(true);
+ expect(controlled.getArmedPreemptIds(streamId, job.createdAt)).toEqual([]);
+ expect(controlled.isPreemptRequested(streamId)).toBe(false);
+ } finally {
+ peekSpy.mockRestore();
+ await controlled.destroy();
+ }
+ });
+
test('rearmQueuedPreempts refuses a stale generation and arms nothing', async () => {
const streamId = 'preempt-rearm-stale';
const job = await manager.createJob(streamId, 'user-1');
@@ -1320,4 +1573,32 @@ describe('preempt request lifecycle (in-memory)', () => {
await failing.destroy();
}
});
+
+ test('retries a synchronous preempt-clear publish failure without throwing', async () => {
+ const streamId = 'preempt-clear-sync-throw';
+ let attempts = 0;
+ const retrying = new GenerationJobManagerClass();
+ retrying.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: Object.assign(new InMemoryEventTransport(), {
+ emitPreempt: () => {
+ attempts++;
+ if (attempts === 1) {
+ throw new Error('publisher not ready');
+ }
+ },
+ }),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ retrying.initialize();
+ try {
+ await expect(retrying.noteSteersRemoved(streamId, ['steer-1'], Date.now())).resolves.toBe(
+ true,
+ );
+ expect(attempts).toBe(2);
+ } finally {
+ await retrying.destroy();
+ }
+ });
});
diff --git a/packages/api/src/stream/createStreamServices.ts b/packages/api/src/stream/createStreamServices.ts
index 11fd535275..942b0e95a3 100644
--- a/packages/api/src/stream/createStreamServices.ts
+++ b/packages/api/src/stream/createStreamServices.ts
@@ -1,12 +1,12 @@
-import type { Redis, Cluster } from 'ioredis';
import { logger } from '@librechat/data-schemas';
-import type { IJobStore, IEventTransport } from './interfaces/IJobStore';
-import { InMemoryJobStore } from './implementations/InMemoryJobStore';
+import type { Redis, Cluster } from 'ioredis';
+import type { IJobStoreV2, IEventTransport } from './interfaces/IJobStore';
import { InMemoryEventTransport } from './implementations/InMemoryEventTransport';
-import { RedisJobStore } from './implementations/RedisJobStore';
import { RedisEventTransport } from './implementations/RedisEventTransport';
-import { cacheConfig } from '~/cache/cacheConfig';
+import { InMemoryJobStore } from './implementations/InMemoryJobStore';
+import { RedisJobStore } from './implementations/RedisJobStore';
import { ioredisClient } from '~/cache/redisClients';
+import { cacheConfig } from '~/cache/cacheConfig';
/**
* Configuration for stream services (optional overrides)
@@ -42,7 +42,7 @@ export interface StreamServicesConfig {
* Stream services result
*/
export interface StreamServices {
- jobStore: IJobStore;
+ jobStore: IJobStoreV2;
eventTransport: IEventTransport;
isRedis: boolean;
}
diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts
index fb1865dabb..fc6bb4c8ed 100644
--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts
+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts
@@ -3,25 +3,116 @@ import type { StandardGraph } from '@librechat/agents';
import type { Agents } from 'librechat-data-provider';
import type {
SerializableJobData,
+ CreatedJobData,
+ ReplacedGeneration,
+ SteerArmOutcome,
+ SteerArmResult,
+ SteerEnqueueReceiptResult,
+ SteerEnqueueVersionedResult,
SteerQueueItem,
+ SteerReceipt,
+ SteerReceiptInput,
UsageMetadata,
- IJobStore,
+ IJobStoreV2,
JobStatus,
JobMetadataPatch,
JobStatusTransition,
IdempotencyClaimValue,
IdempotencyClaimResult,
+ ParkedSteerClaim,
} from '~/stream/interfaces/IJobStore';
+import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';
import {
+ JobPredecessorMismatchError,
STEER_ENQUEUE_NOT_RUNNING,
STEER_ENQUEUE_QUEUE_FULL,
+ STEER_ENQUEUE_RECEIPT_FULL,
STEER_QUEUE_MAX_DEPTH,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ PAUSE_PERSISTENCE_TIMEOUT_MS,
isPendingActionStale,
} from '~/stream/interfaces/IJobStore';
+import {
+ isRecoveredSteerPayload,
+ recoveredSteerPayloadMatches,
+ RecoveredSteerPayloadMismatchError,
+} from '~/stream/SteerRecovery';
import { toPendingSteer } from '~/stream/SteeringLifecycle';
/** Recovery window for parked steers (mirrors Redis's completed-job TTL). */
export const PARKED_STEERS_TTL_MS: number = 5 * 60 * 1000;
+const STEER_RECEIPT_TTL_MS: number = 20 * 60 * 1000;
+const STEER_RECEIPT_MAX_PER_STREAM: number = 100;
+const CREATE_REPLACEMENT_RECEIPT_MAX = 32;
+const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
+const TERMINAL_PERSISTENCE_RETENTION_MS = 5 * 60 * 1000;
+
+function assertCreateIdempotencyArguments(
+ claimKey?: string,
+ claimToken?: string,
+ clientRequestId?: string,
+): void {
+ const supplied = [claimKey, claimToken, clientRequestId].filter((value) => value != null).length;
+ if (
+ (supplied !== 0 && supplied !== 3) ||
+ (supplied === 3 &&
+ (claimKey!.length === 0 ||
+ claimKey!.length > 1024 ||
+ claimToken!.length === 0 ||
+ claimToken!.length > 128 ||
+ !CLIENT_REQUEST_ID_PATTERN.test(clientRequestId!)))
+ ) {
+ throw new Error('Invalid generation job idempotency arguments');
+ }
+}
+
+function isValidParkedSteer(value: unknown): value is { steerId: string } {
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
+ return false;
+ }
+ const item = value as Record<string, unknown>;
+ if (typeof item.steerId !== 'string' || item.steerId.length === 0) {
+ return false;
+ }
+ if (
+ item.clientSteerId != null &&
+ (typeof item.clientSteerId !== 'string' || item.clientSteerId.length === 0)
+ ) {
+ return false;
+ }
+ if (item.text != null && typeof item.text !== 'string') {
+ return false;
+ }
+ if (
+ item.createdAt != null &&
+ (typeof item.createdAt !== 'number' ||
+ !Number.isSafeInteger(item.createdAt) ||
+ item.createdAt < 0)
+ ) {
+ return false;
+ }
+ if (
+ item.recoveringCreatedAt != null &&
+ (typeof item.recoveringCreatedAt !== 'number' ||
+ !Number.isSafeInteger(item.recoveringCreatedAt) ||
+ item.recoveringCreatedAt < 0)
+ ) {
+ return false;
+ }
+ return true;
+}
+
+function usesReceiptSafeProtocol(job: SerializableJobData | null | undefined): boolean {
+ return job?.generationProtocolVersion === 2;
+}
+
+function normalizeGenerationProtocol(value: unknown): 1 | 2 {
+ return value === 2 ? 2 : 1;
+}
+
+function isValidGenerationProtocolMarker(value: unknown): boolean {
+ return value == null || value === 1 || value === 2;
+}
/**
* Content state for a job - volatile, in-memory only.
@@ -34,7 +125,7 @@ interface ContentState {
}
/**
- * In-memory implementation of IJobStore.
+ * In-memory implementation of IJobStoreV2.
* Suitable for single-instance deployments.
* For horizontal scaling, use RedisJobStore.
*
@@ -42,7 +133,7 @@ interface ContentState {
* - Uses WeakRef to graph for live access to contentParts and contentData (run steps)
* - No chunk persistence needed - same instance handles generation and reconnects
*/
-export class InMemoryJobStore implements IJobStore {
+export class InMemoryJobStore implements IJobStoreV2 {
private jobs = new Map<string, SerializableJobData>();
private contentState = new Map<string, ContentState>();
private cleanupInterval: NodeJS.Timeout | null = null;
@@ -60,6 +151,11 @@ export class InMemoryJobStore implements IJobStore {
/** Maps streamId -> FIFO queue of pending steer messages. */
private steerQueues = new Map<string, SteerQueueItem[]>();
+ /** FIFO items removed from the pending queue but not yet durably applied.
+ * Keeping this independently of receipts also protects legacy steers that
+ * do not carry a clientSteerId. */
+ private claimedSteers = new Map<string, SteerQueueItem[]>();
+
/** Stream ids whose steer queue was closed by a terminal drain. Reopened by createJob. */
private closedSteerQueues = new Set<string>();
@@ -67,6 +163,12 @@ export class InMemoryJobStore implements IJobStore {
* default completeJob path deletes the job record immediately). */
private parkedSteers = new Map<string, { payload: string; expiresAt: number }>();
+ /** Conversation-scoped lost-ACK receipts; deliberately survives job replacement. */
+ private steerReceipts = new Map<
+ string,
+ Map<string, { receipt: SteerReceipt; expiresAt: number }>
+ >();
+
/** Maps idempotency key -> claimed stream + expiry, deduping retried start requests. */
private idempotencyClaims = new Map<
string,
@@ -79,6 +181,16 @@ export class InMemoryJobStore implements IJobStore {
/** Maximum number of concurrent jobs */
private maxJobs = 1000;
+ /** Store-lifetime generation clock. A completed job may be deleted before a
+ * same-millisecond replacement starts, so the current jobs map alone cannot
+ * guarantee a distinct checkpoint namespace. */
+ private lastGenerationEpoch = 0;
+
+ /** Latest per-stream epoch retained briefly after its job disappears. This
+ * is also the conditional-create fence: a delayed queue drain must not treat
+ * an empty job slot as proof that no intervening generation ran. */
+ private generationEpochs = new Map<string, { createdAt: number; expiresAt: number }>();
+
/**
* Failsafe timeout (ms) for jobs stuck in "running" status. Mirrors
* RedisJobStore's running-job TTL: a crashed or hung generation that never
@@ -116,45 +228,338 @@ export class InMemoryJobStore implements IJobStore {
logger.debug('[InMemoryJobStore] Initialized with cleanup interval');
}
+ private getRetainedGenerationEpoch(streamId: string, now = Date.now()): number | undefined {
+ const retained = this.generationEpochs.get(streamId);
+ if (retained == null) {
+ return undefined;
+ }
+ if (retained.expiresAt <= now) {
+ this.generationEpochs.delete(streamId);
+ return undefined;
+ }
+ return retained.createdAt;
+ }
+
+ private retainGenerationEpoch(streamId: string, createdAt: number, now = Date.now()): void {
+ const retained = this.generationEpochs.get(streamId);
+ if (retained != null && retained.createdAt > createdAt) {
+ return;
+ }
+ this.generationEpochs.set(streamId, {
+ createdAt,
+ expiresAt: now + PARKED_STEERS_TTL_MS,
+ });
+ }
+
async createJob(
streamId: string,
userId: string,
conversationId?: string,
tenantId?: string,
initialMetadata: JobMetadataPatch = {},
- ): Promise<SerializableJobData> {
- const previousCreatedAt = this.jobs.get(streamId)?.createdAt;
- if (previousCreatedAt == null && this.jobs.size >= this.maxJobs) {
- await this.evictOldest();
+ recoveredSteerId?: string,
+ idempotencyClaimKey?: string,
+ idempotencyClaimToken?: string,
+ idempotencyClientRequestId?: string,
+ recoveredSteerPayload?: RecoveredSteerPayload,
+ creationAttemptId?: string,
+ expectedPredecessorCreatedAt?: number,
+ ): Promise<CreatedJobData> {
+ if (typeof userId !== 'string' || userId.length === 0) {
+ throw new Error('Generation job requires a non-empty user id');
+ }
+ assertCreateIdempotencyArguments(
+ idempotencyClaimKey,
+ idempotencyClaimToken,
+ idempotencyClientRequestId,
+ );
+ if (
+ creationAttemptId != null &&
+ (creationAttemptId.length === 0 || creationAttemptId.length > 128)
+ ) {
+ throw new Error('Invalid generation creation attempt id');
+ }
+ if (
+ expectedPredecessorCreatedAt != null &&
+ (!Number.isSafeInteger(expectedPredecessorCreatedAt) || expectedPredecessorCreatedAt < 0)
+ ) {
+ throw new Error('Invalid expected generation predecessor');
}
+ const assertOwnerCompatible = (): void => {
+ const existingJob = this.jobs.get(streamId);
+ if (
+ existingJob != null &&
+ (typeof existingJob.userId !== 'string' ||
+ existingJob.userId.length === 0 ||
+ existingJob.userId !== userId ||
+ (existingJob.tenantId != null && existingJob.tenantId !== tenantId))
+ ) {
+ throw new Error('Generation job owner mismatch');
+ }
+
+ const parked = this.parkedSteers.get(streamId);
+ if (parked == null || parked.expiresAt <= Date.now()) {
+ return;
+ }
+ let parsed: {
+ userId?: unknown;
+ tenantId?: unknown;
+ generationProtocolVersion?: unknown;
+ steers?: unknown;
+ };
+ try {
+ parsed = JSON.parse(parked.payload) as typeof parsed;
+ } catch {
+ throw new Error('Generation recovery state is corrupt');
+ }
+ if (
+ parsed == null ||
+ typeof parsed !== 'object' ||
+ typeof parsed.userId !== 'string' ||
+ parsed.userId.length === 0 ||
+ !isValidGenerationProtocolMarker(parsed.generationProtocolVersion) ||
+ !Array.isArray(parsed.steers) ||
+ parsed.steers.length === 0 ||
+ !parsed.steers.every(isValidParkedSteer)
+ ) {
+ throw new Error('Generation recovery state is corrupt');
+ }
+ if (parsed.userId !== userId || (parsed.tenantId != null && parsed.tenantId !== tenantId)) {
+ throw new Error('Generation job owner mismatch');
+ }
+ };
+
+ const assertExpectedPredecessorCompatible = (): void => {
+ const current = this.jobs.get(streamId);
+ const currentCreatedAt = current?.createdAt ?? this.getRetainedGenerationEpoch(streamId);
+ if (
+ expectedPredecessorCreatedAt == null ||
+ currentCreatedAt === expectedPredecessorCreatedAt
+ ) {
+ return;
+ }
+ throw new JobPredecessorMismatchError({
+ createdAt: currentCreatedAt ?? expectedPredecessorCreatedAt,
+ active: current?.status === 'running' || current?.status === 'requires_action',
+ verified: currentCreatedAt != null,
+ ...(current !== undefined && { status: current.status }),
+ ...(current?.conversationId !== undefined && { conversationId: current.conversationId }),
+ });
+ };
+
+ /** Validate the source and payload before any predecessor/parked mutation.
+ * This closure is re-run after the only await below, making the validation
+ * and subsequent lease one synchronous in-memory transaction. */
+ const assertRecoveryCompatible = (): void => {
+ if (recoveredSteerId == null) {
+ if (recoveredSteerPayload != null) {
+ throw new RecoveredSteerPayloadMismatchError();
+ }
+ return;
+ }
+ if (!isRecoveredSteerPayload(recoveredSteerPayload)) {
+ throw new RecoveredSteerPayloadMismatchError();
+ }
+ if (initialMetadata.generationProtocolVersion === 1) {
+ throw new RecoveredSteerPayloadMismatchError();
+ }
+
+ let candidate: ReturnType<typeof toPendingSteer> | undefined;
+ let sourceProtocol: 1 | 2 = 1;
+ const parked = this.parkedSteers.get(streamId);
+ if (parked != null && parked.expiresAt > Date.now()) {
+ try {
+ const parsed = JSON.parse(parked.payload) as {
+ generationProtocolVersion?: unknown;
+ steers?: Array<ReturnType<typeof toPendingSteer>>;
+ };
+ candidate = parsed.steers?.find((item) => item.steerId === recoveredSteerId);
+ sourceProtocol = normalizeGenerationProtocol(parsed.generationProtocolVersion);
+ } catch {
+ // assertOwnerCompatible reports the more specific corruption error.
+ }
+ }
+ if (candidate == null) {
+ sourceProtocol = normalizeGenerationProtocol(
+ this.jobs.get(streamId)?.generationProtocolVersion,
+ );
+ const queued = [
+ ...this.claimedSteerItems(streamId, this.jobs.get(streamId)?.createdAt),
+ ...(this.steerQueues.get(streamId) ?? []),
+ ];
+ const item = queued.find((queuedItem) => queuedItem.steerId === recoveredSteerId);
+ candidate = item == null ? undefined : toPendingSteer(item);
+ }
+
+ if (
+ sourceProtocol !== 2 ||
+ candidate == null ||
+ !recoveredSteerPayloadMatches(candidate, recoveredSteerPayload)
+ ) {
+ throw new RecoveredSteerPayloadMismatchError();
+ }
+ };
+
+ // Validate before capacity eviction and repeat after its only await. This
+ // preserves both an existing foreign job and malformed/foreign recovery
+ // payload even when another creator races the eviction yield.
+ assertOwnerCompatible();
+ assertExpectedPredecessorCompatible();
+ assertRecoveryCompatible();
+ if (idempotencyClaimKey != null && idempotencyClaimToken != null) {
+ const claim = this.idempotencyClaims.get(idempotencyClaimKey);
+ if (
+ claim == null ||
+ claim.expiresAt <= Date.now() ||
+ claim.value.claimToken !== idempotencyClaimToken ||
+ claim.value.startedAt != null
+ ) {
+ throw new Error('Generation idempotency claim was taken over before job creation');
+ }
+ }
+ if (!this.jobs.has(streamId) && this.jobs.size >= this.maxJobs) {
+ await this.evictOldest();
+ }
+ assertOwnerCompatible();
+ assertExpectedPredecessorCompatible();
+ assertRecoveryCompatible();
+
+ // Re-read after the possible eviction await. A concurrent creator can
+ // install this stream while capacity eviction yields; this is the exact
+ // predecessor the synchronous replacement below will overwrite.
+ const previousJob = this.jobs.get(streamId);
+ const previousCreatedAt = previousJob?.createdAt ?? this.getRetainedGenerationEpoch(streamId);
const now = Date.now();
- const job: SerializableJobData = {
+ const createdAt = Math.max(
+ now,
+ previousCreatedAt == null ? 0 : previousCreatedAt + 1,
+ this.lastGenerationEpoch + 1,
+ );
+ const replacedJobs: ReplacedGeneration[] = [];
+ const replacedEpochs = new Set<number>();
+ const addReplacedJob = (replaced: ReplacedGeneration): void => {
+ if (replacedEpochs.has(replaced.createdAt)) {
+ return;
+ }
+ if (replacedJobs.length >= CREATE_REPLACEMENT_RECEIPT_MAX) {
+ throw new Error('Generation replacement receipt chain is full');
+ }
+ replacedEpochs.add(replaced.createdAt);
+ replacedJobs.push(replaced);
+ };
+ const previousReceipt = previousJob as CreatedJobData | undefined;
+ for (const inherited of previousReceipt?.replacedJobs ??
+ (previousReceipt?.replacedJob != null ? [previousReceipt.replacedJob] : [])) {
+ addReplacedJob(inherited);
+ }
+ if (previousJob != null) {
+ const replaced: ReplacedGeneration = {
+ createdAt: previousJob.createdAt,
+ status: previousJob.status,
+ ...(previousJob.conversationId !== undefined && {
+ conversationId: previousJob.conversationId,
+ }),
+ };
+ if (previousJob.providerAbortReady != null) {
+ Object.defineProperty(replaced, 'providerAbortReady', {
+ value: previousJob.providerAbortReady,
+ enumerable: false,
+ });
+ }
+ addReplacedJob(replaced);
+ }
+ this.lastGenerationEpoch = createdAt;
+ const job: CreatedJobData = {
...initialMetadata,
streamId,
userId,
...(tenantId && { tenantId }),
status: 'running',
- createdAt: previousCreatedAt == null ? now : Math.max(now, previousCreatedAt + 1),
+ createdAt,
+ generationProtocolVersion: initialMetadata.generationProtocolVersion === 1 ? 1 : 2,
+ ...(initialMetadata.generationProtocolVersion !== 1 && {
+ checkpointNamespace: String(createdAt),
+ }),
...(conversationId !== undefined && { conversationId }),
+ ...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),
+ ...(recoveredSteerId !== undefined && { recoveredSteerId }),
+ providerAbortReady: false,
syncSent: false,
};
+ if (creationAttemptId != null) {
+ Object.defineProperty(job, 'creationAttemptId', {
+ value: creationAttemptId,
+ enumerable: false,
+ configurable: true,
+ });
+ }
+ if (previousJob != null) {
+ Object.defineProperty(job, 'replacedJob', {
+ value: replacedJobs[replacedJobs.length - 1],
+ enumerable: false,
+ configurable: true,
+ });
+ }
+ if (replacedJobs.length > 0) {
+ Object.defineProperty(job, 'replacedJobs', {
+ value: replacedJobs,
+ enumerable: false,
+ configurable: true,
+ });
+ }
+
+ // Re-check after the possible eviction await. A stale creator must not
+ // park/replace state after another request took over its pre-create lease.
+ const liveClaim =
+ idempotencyClaimKey != null ? this.idempotencyClaims.get(idempotencyClaimKey) : undefined;
+ if (
+ idempotencyClaimKey != null &&
+ idempotencyClaimToken != null &&
+ (liveClaim == null ||
+ liveClaim.expiresAt <= Date.now() ||
+ liveClaim.value.claimToken !== idempotencyClaimToken ||
+ liveClaim.value.startedAt != null)
+ ) {
+ throw new Error('Generation idempotency claim was taken over before job creation');
+ }
+ if (previousJob != null) {
+ this.parkQueuedSteers(streamId, previousJob, now);
+ }
this.jobs.set(streamId, job);
+ this.retainGenerationEpoch(streamId, job.createdAt, now);
+ this.leaseParkedSteer(streamId, recoveredSteerId, userId, tenantId, job.createdAt);
this.contentState.delete(streamId);
// Clear any prior activity timestamp so a replacement reusing this streamId
// (the controller handles job replacement) falls back to the fresh createdAt
// and isn't reaped on the previous generation's stale last-activity time.
this.lastActivity.delete(streamId);
// Steer queues are keyed by streamId only, so a replacement must not
- // inherit the replaced run's undrained steers (or its closed flag), and
- // parked recovery belongs to the replaced run (a live client started this).
+ // inherit the replaced run's undrained steers or its closed flag. Any
+ // acknowledged items were copied into owner-gated parked recovery above.
this.steerQueues.delete(streamId);
+ this.claimedSteers.delete(streamId);
this.closedSteerQueues.delete(streamId);
- this.parkedSteers.delete(streamId);
+
+ if (liveClaim != null) {
+ liveClaim.value = { ...liveClaim.value, startedAt: job.createdAt };
+ }
// Track job by userId (tenant-qualified when available) for efficient user-scoped queries
const userKey = tenantId ? `${tenantId}:${userId}` : userId;
+ if (previousJob != null) {
+ const previousUserKey = previousJob.tenantId
+ ? `${previousJob.tenantId}:${previousJob.userId}`
+ : previousJob.userId;
+ if (previousUserKey !== userKey) {
+ const previousUserJobs = this.userJobMap.get(previousUserKey);
+ previousUserJobs?.delete(streamId);
+ if (previousUserJobs?.size === 0) {
+ this.userJobMap.delete(previousUserKey);
+ }
+ }
+ }
let userJobs = this.userJobMap.get(userKey);
if (!userJobs) {
userJobs = new Set();
@@ -162,15 +567,72 @@ export class InMemoryJobStore implements IJobStore {
}
userJobs.add(streamId);
+ const createdJob: CreatedJobData = previousJob == null ? job : { ...job };
+ if (previousJob != null) {
+ if (creationAttemptId != null) {
+ Object.defineProperty(createdJob, 'creationAttemptId', {
+ value: creationAttemptId,
+ enumerable: false,
+ configurable: true,
+ });
+ }
+ Object.defineProperty(createdJob, 'replacedJob', {
+ value: {
+ createdAt: previousJob.createdAt,
+ status: previousJob.status,
+ ...(previousJob.conversationId !== undefined && {
+ conversationId: previousJob.conversationId,
+ }),
+ },
+ enumerable: false,
+ configurable: true,
+ writable: false,
+ });
+ Object.defineProperty(createdJob, 'replacedJobs', {
+ value: replacedJobs,
+ enumerable: false,
+ configurable: true,
+ });
+ }
+
logger.debug(`[InMemoryJobStore] Created job: ${streamId}`);
- return job;
+ return createdJob;
}
async getJob(streamId: string): Promise<SerializableJobData | null> {
return this.jobs.get(streamId) ?? null;
}
+ async acknowledgeReplacedJobs(
+ streamId: string,
+ creationAttemptId: string,
+ replacedCreatedAts: readonly number[],
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId) as CreatedJobData | undefined;
+ if (job?.creationAttemptId !== creationAttemptId) {
+ return false;
+ }
+ const acknowledged = new Set(replacedCreatedAts);
+ const current = job.replacedJobs ?? (job.replacedJob != null ? [job.replacedJob] : []);
+ const retained = current.filter((receipt) => !acknowledged.has(receipt.createdAt));
+ delete job.replacedJobs;
+ delete job.replacedJob;
+ if (retained.length > 0) {
+ Object.defineProperty(job, 'replacedJobs', {
+ value: retained,
+ enumerable: false,
+ configurable: true,
+ });
+ Object.defineProperty(job, 'replacedJob', {
+ value: retained[retained.length - 1],
+ enumerable: false,
+ configurable: true,
+ });
+ }
+ return true;
+ }
+
async updateJob(
streamId: string,
updates: Partial<SerializableJobData>,
@@ -180,11 +642,39 @@ export class InMemoryJobStore implements IJobStore {
if (!job || (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)) {
return;
}
+ if (
+ updates.status != null &&
+ ['complete', 'error', 'aborted'].includes(updates.status) &&
+ (job.status === 'running' || job.status === 'requires_action')
+ ) {
+ const { status, ...patch } = updates;
+ await this.transitionStatus(streamId, {
+ from: job.status,
+ to: status,
+ patch,
+ expectCreatedAt: expectedCreatedAt ?? job.createdAt,
+ });
+ return;
+ }
// Plain field writer. Membership-aware status transitions
// (running ⇄ requires_action) go solely through transitionStatus.
Object.assign(job, updates);
}
+ async finalizeTerminalPersistence(
+ streamId: string,
+ expectedCreatedAt: number,
+ finalEvent: string,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (!job || job.createdAt !== expectedCreatedAt || job.terminalPersistencePending !== true) {
+ return false;
+ }
+ job.terminalPersistencePending = false;
+ job.finalEvent = finalEvent;
+ return true;
+ }
+
/**
* Atomic in-memory: the single-threaded event loop makes the
* read-check-write sequence indivisible, so the status guard is exact.
@@ -203,6 +693,9 @@ export class InMemoryJobStore implements IJobStore {
return false;
}
if (['complete', 'error', 'aborted'].includes(args.to)) {
+ if (!this.isParkedRecoveryCompatible(streamId, job)) {
+ return false;
+ }
this.parkQueuedSteers(streamId, job, Date.now());
}
job.status = args.to;
@@ -212,9 +705,94 @@ export class InMemoryJobStore implements IJobStore {
for (const field of args.clear ?? []) {
delete job[field];
}
+ const receiptEntries = this.steerReceipts.get(streamId)?.values() ?? [];
+ if (args.to === 'requires_action' && args.patch?.pendingAction?.expiresAt == null) {
+ // Unlike Redis, this store has no paused-job backstop eviction. Preserve
+ // the receipt for the full (potentially unbounded) approval lifetime.
+ for (const entry of receiptEntries) {
+ if (entry.receipt.generationCreatedAt === job.createdAt) {
+ entry.expiresAt = Infinity;
+ }
+ }
+ } else {
+ const normalExpiry = Date.now() + STEER_RECEIPT_TTL_MS;
+ const requestedExpiry =
+ args.steerReceiptTtlSeconds != null
+ ? Date.now() + Math.max(1, args.steerReceiptTtlSeconds) * 1000
+ : 0;
+ const terminalExpiry = ['complete', 'error', 'aborted'].includes(args.to)
+ ? Date.now() + PARKED_STEERS_TTL_MS
+ : 0;
+ const targetExpiry = Math.max(
+ requestedExpiry,
+ terminalExpiry,
+ args.from === 'requires_action' ? normalExpiry : 0,
+ );
+ if (targetExpiry > 0) {
+ for (const entry of receiptEntries) {
+ if (entry.receipt.generationCreatedAt !== job.createdAt) {
+ continue;
+ }
+ entry.expiresAt = Number.isFinite(entry.expiresAt)
+ ? Math.max(entry.expiresAt, targetExpiry)
+ : targetExpiry;
+ }
+ }
+ }
+ if (args.to === 'running') {
+ this.extendRecoveryLease(streamId, job, this.runningRecoveryLeaseExpiry(Date.now()));
+ } else if (args.to === 'requires_action') {
+ const approvalExpiry = args.patch?.pendingAction?.expiresAt;
+ this.extendRecoveryLease(
+ streamId,
+ job,
+ approvalExpiry == null
+ ? Number.POSITIVE_INFINITY
+ : Math.max(
+ this.runningRecoveryLeaseExpiry(Date.now()),
+ approvalExpiry + PARKED_STEERS_TTL_MS,
+ ),
+ );
+ }
return true;
}
+ async transitionStatusAndDrainSteers(
+ streamId: string,
+ args: JobStatusTransition,
+ ): Promise<SteerQueueItem[] | null> {
+ if (!['complete', 'error', 'aborted'].includes(args.to)) {
+ throw new Error('Steer-draining status transitions must be terminal');
+ }
+ const job = this.jobs.get(streamId);
+ if (
+ !job ||
+ job.status !== args.from ||
+ (args.expectActionId != null && job.pendingActionId !== args.expectActionId) ||
+ (args.expectCreatedAt != null && job.createdAt !== args.expectCreatedAt)
+ ) {
+ return null;
+ }
+ const seen = new Set<string>();
+ const drained = [
+ ...this.claimedSteerItems(streamId, job.createdAt),
+ ...(this.steerQueues.get(streamId) ?? []),
+ ]
+ .filter((item) => {
+ if (seen.has(item.steerId)) {
+ return false;
+ }
+ seen.add(item.steerId);
+ return true;
+ })
+ .map((item) => ({ ...item }));
+
+ // transitionStatus contains no await, so the validation, snapshot, and
+ // terminal mutation remain one indivisible event-loop operation.
+ const transitioned = await this.transitionStatus(streamId, args);
+ return transitioned ? drained : null;
+ }
+
async claimIdempotencyKey(
key: string,
value: IdempotencyClaimValue,
@@ -226,10 +804,97 @@ export class InMemoryJobStore implements IJobStore {
return { claimed: false, existing: existing.value };
}
this.idempotencyClaims.set(key, { value, expiresAt: now + ttlSeconds * 1000 });
- return { claimed: true };
+ return { claimed: true, existing: value };
+ }
+
+ async takeoverIdempotencyKey(
+ key: string,
+ expected: IdempotencyClaimValue,
+ value: IdempotencyClaimValue,
+ ttlSeconds: number,
+ ): Promise<boolean> {
+ const current = this.idempotencyClaims.get(key);
+ if (
+ current == null ||
+ current.expiresAt <= Date.now() ||
+ current.value.claimToken !== expected.claimToken ||
+ current.value.startedAt != null
+ ) {
+ return false;
+ }
+ this.idempotencyClaims.set(key, {
+ value,
+ expiresAt: Date.now() + ttlSeconds * 1000,
+ });
+ return true;
+ }
+
+ async markIdempotencyKeyStarted(
+ key: string,
+ claimToken: string,
+ startedAt: number,
+ ttlSeconds: number,
+ ): Promise<boolean> {
+ const current = this.idempotencyClaims.get(key);
+ if (
+ claimToken.length === 0 ||
+ current == null ||
+ current.expiresAt <= Date.now() ||
+ current.value.claimToken !== claimToken ||
+ (current.value.startedAt != null && current.value.startedAt !== startedAt)
+ ) {
+ return false;
+ }
+ current.value = { ...current.value, startedAt };
+ current.expiresAt = Date.now() + ttlSeconds * 1000;
+ return true;
}
- async releaseIdempotencyKey(key: string): Promise<void> {
+ async adoptIdempotencyKeyForJob(
+ key: string,
+ expected: IdempotencyClaimValue,
+ streamId: string,
+ userId: string,
+ clientRequestId: string,
+ tenantId: string | undefined,
+ expectedCreatedAt: number,
+ ttlSeconds: number,
+ allowMissingClientRequestId = false,
+ ): Promise<boolean> {
+ const claim = this.idempotencyClaims.get(key);
+ const job = this.jobs.get(streamId);
+ if (
+ claim == null ||
+ claim.expiresAt <= Date.now() ||
+ claim.value.claimToken !== expected.claimToken ||
+ claim.value.startedAt != null ||
+ job == null ||
+ job.userId !== userId ||
+ (job.tenantId != null && job.tenantId !== tenantId) ||
+ (job.idempotencyClientRequestId == null
+ ? !allowMissingClientRequestId
+ : job.idempotencyClientRequestId !== clientRequestId) ||
+ job.createdAt !== expectedCreatedAt ||
+ (job.status !== 'running' && job.status !== 'requires_action')
+ ) {
+ return false;
+ }
+ claim.value = {
+ ...claim.value,
+ startedAt: job.createdAt,
+ generationProtocolVersion: normalizeGenerationProtocol(job.generationProtocolVersion),
+ };
+ claim.expiresAt = Date.now() + ttlSeconds * 1000;
+ return true;
+ }
+
+ async releaseIdempotencyKey(key: string, expected?: IdempotencyClaimValue): Promise<void> {
+ if (
+ expected != null &&
+ this.idempotencyClaims.get(key)?.value.claimToken !== expected.claimToken
+ ) {
+ return;
+ }
this.idempotencyClaims.delete(key);
}
@@ -239,10 +904,12 @@ export class InMemoryJobStore implements IJobStore {
return false;
}
+ this.retainGenerationEpoch(streamId, job.createdAt);
this.jobs.delete(streamId);
this.contentState.delete(streamId);
this.lastActivity.delete(streamId);
this.steerQueues.delete(streamId);
+ this.claimedSteers.delete(streamId);
this.closedSteerQueues.delete(streamId);
const userKey = job.tenantId ? `${job.tenantId}:${job.userId}` : job.userId;
const userJobs = this.userJobMap.get(userKey);
@@ -262,7 +929,15 @@ export class InMemoryJobStore implements IJobStore {
recordActivity(streamId: string, expectedCreatedAt?: number): void {
const job = this.jobs.get(streamId);
if (job && (expectedCreatedAt == null || job.createdAt === expectedCreatedAt)) {
- this.lastActivity.set(streamId, Date.now());
+ const now = Date.now();
+ this.lastActivity.set(streamId, now);
+ const receiptExpiry = now + STEER_RECEIPT_TTL_MS;
+ for (const entry of this.steerReceipts.get(streamId)?.values() ?? []) {
+ if (entry.receipt.generationCreatedAt === job.createdAt) {
+ entry.expiresAt = Math.max(entry.expiresAt, receiptExpiry);
+ }
+ }
+ this.extendRecoveryLease(streamId, job, this.runningRecoveryLeaseExpiry(now));
}
}
@@ -300,11 +975,58 @@ export class InMemoryJobStore implements IJobStore {
}
}
+ for (const [streamId, retained] of this.generationEpochs) {
+ if (retained.expiresAt <= now) {
+ this.generationEpochs.delete(streamId);
+ }
+ }
+
for (const [streamId, job] of this.jobs) {
+ if (job.status === 'requires_action' && job.terminalPersistencePending === true) {
+ const startedAt = job.terminalPersistenceStartedAt ?? job.createdAt;
+ if (now - startedAt < PAUSE_PERSISTENCE_TIMEOUT_MS) {
+ // The pause owner is still within its response-write lease. Cleanup
+ // must not win the race and abort an action before that write settles.
+ continue;
+ }
+ if (job.pendingActionId == null) {
+ logger.error(
+ `[InMemoryJobStore] Refusing stale pause-persistence cleanup without an action fence: ${streamId}`,
+ );
+ continue;
+ }
+ const failed = await this.transitionStatusAndDrainSteers(streamId, {
+ from: 'requires_action',
+ to: 'error',
+ expectActionId: job.pendingActionId,
+ expectCreatedAt: job.createdAt,
+ patch: {
+ completedAt: now,
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ },
+ clear: [
+ 'pendingAction',
+ 'pendingActionId',
+ 'terminalPersistencePending',
+ 'terminalPersistenceStartedAt',
+ ],
+ });
+ if (failed == null) {
+ continue;
+ }
+ logger.error(`[InMemoryJobStore] Pause persistence timed out: ${streamId}`);
+ }
+
const isFinished = ['complete', 'error', 'aborted'].includes(job.status);
if (isFinished && job.completedAt) {
- // TTL of 0 means immediate cleanup, otherwise wait for TTL to expire
- if (this.ttlAfterComplete === 0 || now - job.completedAt > this.ttlAfterComplete) {
+ // A pending final event is a persistence barrier, not an ordinary
+ // completed row. Give its owner/recovery path a bounded window even
+ // when normal terminal retention is configured to zero.
+ const terminalTtl =
+ job.terminalPersistencePending === true
+ ? Math.max(this.ttlAfterComplete, TERMINAL_PERSISTENCE_RETENTION_MS)
+ : this.ttlAfterComplete;
+ if (terminalTtl === 0 || now - job.completedAt > terminalTtl) {
toDelete.push({ streamId, createdAt: job.createdAt });
}
} else if (job.status === 'requires_action' && isPendingActionStale(job)) {
@@ -314,6 +1036,12 @@ export class InMemoryJobStore implements IJobStore {
// Skipping it (active-list filter) alone would leave it resident.
// 202-accepted steers frozen across the pause are parked first —
// deleting the job would silently drop them otherwise.
+ if (!this.isParkedRecoveryCompatible(streamId, job)) {
+ logger.error(
+ `[InMemoryJobStore] Refusing stale approval cleanup with invalid recovery state: ${streamId}`,
+ );
+ continue;
+ }
this.parkQueuedSteers(streamId, job, now);
job.status = 'aborted';
job.completedAt = now;
@@ -339,6 +1067,12 @@ export class InMemoryJobStore implements IJobStore {
if (now - lastActive > this.staleJobTimeout) {
// A crashed/hung run never reached a finalization drain — park the
// 202-accepted queue before the delete drops it.
+ if (!this.isParkedRecoveryCompatible(streamId, job)) {
+ logger.error(
+ `[InMemoryJobStore] Refusing stale job cleanup with invalid recovery state: ${streamId}`,
+ );
+ continue;
+ }
this.parkQueuedSteers(streamId, job, now);
toDelete.push({ streamId, createdAt: job.createdAt });
staleRunning++;
@@ -346,6 +1080,20 @@ export class InMemoryJobStore implements IJobStore {
}
}
+ // Park/reaper processing above gets first chance to extend receipts and
+ // classify claimed items. Pruning first could erase the only durable copy
+ // of a drained steer immediately before terminal recovery snapshots it.
+ for (const [streamId, receipts] of this.steerReceipts) {
+ for (const [clientSteerId, entry] of receipts) {
+ if (entry.expiresAt <= now) {
+ receipts.delete(clientSteerId);
+ }
+ }
+ if (receipts.size === 0) {
+ this.steerReceipts.delete(streamId);
+ }
+ }
+
for (const { streamId, createdAt } of toDelete) {
await this.deleteJob(streamId, createdAt);
}
@@ -380,6 +1128,7 @@ export class InMemoryJobStore implements IJobStore {
if (!job) {
return;
}
+ this.parkQueuedSteers(oldestId, job, Date.now());
await this.deleteJob(oldestId, job.createdAt);
}
}
@@ -409,9 +1158,13 @@ export class InMemoryJobStore implements IJobStore {
this.contentState.clear();
this.userJobMap.clear();
this.steerQueues.clear();
+ this.claimedSteers.clear();
this.closedSteerQueues.clear();
this.parkedSteers.clear();
+ this.steerReceipts.clear();
this.idempotencyClaims.clear();
+ this.lastActivity.clear();
+ this.generationEpochs.clear();
logger.debug('[InMemoryJobStore] Destroyed');
}
@@ -431,11 +1184,19 @@ export class InMemoryJobStore implements IJobStore {
for (const streamId of trackedIds) {
const job = this.jobs.get(streamId);
+ const jobUserKey = job?.tenantId ? `${job.tenantId}:${job.userId}` : job?.userId;
+ if (job == null || jobUserKey !== userKey) {
+ // A same-stream replacement may move from a legacy user scope into a
+ // tenant-qualified scope. Never expose/count the replacement through
+ // stale predecessor membership.
+ trackedIds.delete(streamId);
+ continue;
+ }
// Include running jobs and jobs paused for human review (e.g. tool approval).
// A pending-approval job still occupies the user's conversation slot — but
// only while its prompt is live: a past-`expiresAt` approval no longer
// counts as active (cleanup/expiry will finalize it).
- if (job && (job.status === 'running' || job.status === 'requires_action')) {
+ if (job.status === 'running' || job.status === 'requires_action') {
if (job.status === 'requires_action' && isPendingActionStale(job)) {
continue;
}
@@ -569,11 +1330,63 @@ export class InMemoryJobStore implements IJobStore {
* No-op for in-memory - content available via graph reference.
*/
async appendChunk(
- _streamId: string,
+ streamId: string,
_event: unknown,
- _expectedCreatedAt?: number,
- ): Promise<void> {
- // No-op: content available via graph reference
+ expectedCreatedAt?: number,
+ deliveredSteer?: SteerQueueItem,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (!job || (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)) {
+ return false;
+ }
+ if (job.status !== 'running' && job.status !== 'requires_action') {
+ return false;
+ }
+ if (deliveredSteer != null) {
+ if (job.status !== 'running' || this.closedSteerQueues.has(streamId)) {
+ return false;
+ }
+ if (usesReceiptSafeProtocol(job)) {
+ const claimed = this.claimedSteers.get(streamId);
+ const claimedIndex =
+ claimed?.findIndex(
+ (item) =>
+ item.steerId === deliveredSteer.steerId &&
+ item.clientSteerId === deliveredSteer.clientSteerId,
+ ) ?? -1;
+ if (claimed == null || claimedIndex < 0) {
+ return false;
+ }
+ if (deliveredSteer.clientSteerId != null) {
+ const receipt = this.steerReceipts
+ .get(streamId)
+ ?.get(deliveredSteer.clientSteerId)?.receipt;
+ if (
+ receipt == null ||
+ receipt.clientSteerId !== deliveredSteer.clientSteerId ||
+ receipt.generationCreatedAt !== job.createdAt ||
+ receipt.state !== 'claimed' ||
+ receipt.item.steerId !== deliveredSteer.steerId ||
+ receipt.item.clientSteerId !== deliveredSteer.clientSteerId
+ ) {
+ return false;
+ }
+ }
+ claimed.splice(claimedIndex, 1);
+ if (claimed.length === 0) {
+ this.claimedSteers.delete(streamId);
+ }
+ }
+ }
+ if (usesReceiptSafeProtocol(job) && deliveredSteer?.clientSteerId != null) {
+ const receipt = this.steerReceipts.get(streamId)?.get(deliveredSteer.clientSteerId)?.receipt;
+ if (receipt != null) {
+ receipt.item = { ...deliveredSteer };
+ receipt.state = 'delivered';
+ }
+ }
+ // Content itself is available via the live graph reference.
+ return true;
}
/**
@@ -596,6 +1409,16 @@ export class InMemoryJobStore implements IJobStore {
item: SteerQueueItem,
expectedCreatedAt?: number,
): Promise<number> {
+ const result = await this.enqueueSteerVersioned(streamId, item, false, expectedCreatedAt);
+ return typeof result === 'number' ? result : result.position;
+ }
+
+ async enqueueSteerVersioned(
+ streamId: string,
+ item: SteerQueueItem,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueVersionedResult> {
const job = this.jobs.get(streamId);
if (!job || job.status !== 'running' || this.closedSteerQueues.has(streamId)) {
return STEER_ENQUEUE_NOT_RUNNING;
@@ -611,32 +1434,310 @@ export class InMemoryJobStore implements IJobStore {
if (queue.length >= STEER_QUEUE_MAX_DEPTH) {
return STEER_ENQUEUE_QUEUE_FULL;
}
- queue.push(item);
- return queue.length;
+ const persisted: SteerQueueItem = {
+ ...item,
+ ...(wantsPreempt && {
+ preemptRevision: 1,
+ ...(job.preemptCapable === true && { preempt: true }),
+ }),
+ };
+ queue.push(persisted);
+ return { item: { ...persisted }, position: queue.length };
+ }
+
+ private normalizeSteerReceipt(
+ streamId: string,
+ entry: { receipt: SteerReceipt; expiresAt: number },
+ ): void {
+ const receipt = entry.receipt;
+ const job = this.jobs.get(streamId);
+ const sameActiveGeneration =
+ usesReceiptSafeProtocol(job) &&
+ job?.createdAt === receipt.generationCreatedAt &&
+ (job.status === 'running' || job.status === 'requires_action');
+ if (receipt.state === 'claimed') {
+ if (!sameActiveGeneration) {
+ receipt.state = 'leftover';
+ }
+ return;
+ }
+ if (
+ receipt.state === 'queued' &&
+ (!sameActiveGeneration ||
+ !(this.steerQueues.get(streamId) ?? []).some(
+ (item) => item.clientSteerId === receipt.clientSteerId,
+ ))
+ ) {
+ /** New drains atomically move queued→claimed. A still-queued receipt
+ * without a list item is therefore either an older same-generation
+ * drain (delivered) or a terminal/replacement deletion (leftover). */
+ // Durable application explicitly writes `delivered`. Queue absence by
+ // itself is never proof of delivery (TTL loss/crash must recover text).
+ receipt.state = 'leftover';
+ }
+ }
+
+ private claimedSteerItems(streamId: string, generationCreatedAt?: number): SteerQueueItem[] {
+ const job = this.jobs.get(streamId);
+ if (
+ !usesReceiptSafeProtocol(job) ||
+ (generationCreatedAt != null && job?.createdAt !== generationCreatedAt)
+ ) {
+ return [];
+ }
+ return (this.claimedSteers.get(streamId) ?? []).map((item) => ({ ...item }));
+ }
+
+ async getSteerReceipt(streamId: string, clientSteerId: string): Promise<SteerReceipt | null> {
+ const entry = this.steerReceipts.get(streamId)?.get(clientSteerId);
+ if (entry == null) {
+ return null;
+ }
+ if (entry.expiresAt <= Date.now()) {
+ this.steerReceipts.get(streamId)?.delete(clientSteerId);
+ return null;
+ }
+ this.normalizeSteerReceipt(streamId, entry);
+ return { ...entry.receipt, item: { ...entry.receipt.item } };
+ }
+
+ async enqueueSteerWithReceipt(
+ streamId: string,
+ item: SteerQueueItem,
+ receiptInput: SteerReceiptInput,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueReceiptResult> {
+ /** Inline with no await: the event loop cannot interleave two identical
+ * claims between lookup and insert. */
+ const existingEntry = this.steerReceipts.get(streamId)?.get(receiptInput.clientSteerId);
+ if (existingEntry != null && existingEntry.expiresAt > Date.now()) {
+ this.normalizeSteerReceipt(streamId, existingEntry);
+ return { ...existingEntry.receipt, item: { ...existingEntry.receipt.item } };
+ }
+ if (existingEntry != null) {
+ this.steerReceipts.get(streamId)?.delete(receiptInput.clientSteerId);
+ }
+
+ const job = this.jobs.get(streamId);
+ if (
+ !job ||
+ job.status !== 'running' ||
+ this.closedSteerQueues.has(streamId) ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
+ return STEER_ENQUEUE_NOT_RUNNING;
+ }
+ if (!usesReceiptSafeProtocol(job)) {
+ return this.enqueueSteerVersioned(streamId, item, wantsPreempt, expectedCreatedAt);
+ }
+ let queue = this.steerQueues.get(streamId);
+ if (!queue) {
+ queue = [];
+ this.steerQueues.set(streamId, queue);
+ }
+ if (queue.length >= STEER_QUEUE_MAX_DEPTH) {
+ return STEER_ENQUEUE_QUEUE_FULL;
+ }
+ let receipts = this.steerReceipts.get(streamId);
+ if (receipts == null) {
+ receipts = new Map();
+ this.steerReceipts.set(streamId, receipts);
+ }
+ for (const [id, entry] of receipts) {
+ if (entry.expiresAt <= Date.now()) {
+ receipts.delete(id);
+ }
+ }
+ if (receipts.size >= STEER_RECEIPT_MAX_PER_STREAM) {
+ return STEER_ENQUEUE_RECEIPT_FULL;
+ }
+ const persisted: SteerQueueItem = {
+ ...item,
+ ...(wantsPreempt && {
+ preemptRevision: 1,
+ ...(job.preemptCapable === true && { preempt: true }),
+ }),
+ };
+ queue.push(persisted);
+ const receipt: SteerReceipt = {
+ ...receiptInput,
+ item: { ...persisted },
+ position: queue.length,
+ state: 'queued',
+ };
+ receipts.set(receipt.clientSteerId, {
+ receipt,
+ expiresAt: Date.now() + STEER_RECEIPT_TTL_MS,
+ });
+ return { ...receipt, item: { ...receipt.item } };
+ }
+
+ private settleSteerReceipts(
+ streamId: string,
+ items: SteerQueueItem[],
+ state: SteerReceipt['state'],
+ ): void {
+ const job = this.jobs.get(streamId);
+ if (job != null && !usesReceiptSafeProtocol(job)) {
+ return;
+ }
+ const receipts = this.steerReceipts.get(streamId);
+ if (receipts == null) {
+ return;
+ }
+ for (const item of items) {
+ if (!item.clientSteerId) {
+ continue;
+ }
+ const receipt = receipts.get(item.clientSteerId)?.receipt;
+ if (receipt != null) {
+ receipt.item = { ...item };
+ receipt.state = state;
+ }
+ }
}
/** With `expectedCreatedAt`, refuses when the job was replaced — a stale
* run's drain must never consume (or close) a replacement job's queue. */
async drainSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]> {
- if (expectedCreatedAt != null && this.jobs.get(streamId)?.createdAt !== expectedCreatedAt) {
+ const job = this.jobs.get(streamId);
+ if (
+ job?.status !== 'running' ||
+ this.closedSteerQueues.has(streamId) ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
return [];
}
const queue = this.steerQueues.get(streamId);
if (!queue || queue.length === 0) {
return [];
}
- return queue.splice(0);
+ if (!usesReceiptSafeProtocol(job)) {
+ return queue.splice(0);
+ }
+ for (const item of queue) {
+ if (item.clientSteerId == null) {
+ continue;
+ }
+ const receipt = this.steerReceipts.get(streamId)?.get(item.clientSteerId)?.receipt;
+ if (
+ receipt == null ||
+ receipt.clientSteerId !== item.clientSteerId ||
+ receipt.generationCreatedAt !== job.createdAt ||
+ receipt.state !== 'queued' ||
+ receipt.item.steerId !== item.steerId ||
+ receipt.item.clientSteerId !== item.clientSteerId
+ ) {
+ throw new Error(`Invalid steer receipt while draining ${streamId}`);
+ }
+ }
+ const drained = queue.splice(0);
+ const priorClaims = this.claimedSteers.get(streamId) ?? [];
+ this.claimedSteers.set(streamId, [...priorClaims, ...drained]);
+ this.settleSteerReceipts(streamId, drained, 'claimed');
+ const claimExpiry = Date.now() + STEER_RECEIPT_TTL_MS;
+ for (const item of drained) {
+ if (item.clientSteerId == null) {
+ continue;
+ }
+ const entry = this.steerReceipts.get(streamId)?.get(item.clientSteerId);
+ if (entry != null) {
+ entry.expiresAt = Math.max(entry.expiresAt, claimExpiry);
+ }
+ }
+ return drained;
+ }
+
+ async restoreClaimedSteers(
+ streamId: string,
+ items: SteerQueueItem[],
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (
+ !job ||
+ job.status !== 'running' ||
+ this.closedSteerQueues.has(streamId) ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)
+ ) {
+ return false;
+ }
+ if (!usesReceiptSafeProtocol(job)) {
+ const current = this.steerQueues.get(streamId) ?? [];
+ const currentIds = new Set(current.map((item) => item.steerId));
+ this.steerQueues.set(streamId, [
+ ...items.filter((item) => !currentIds.has(item.steerId)),
+ ...current,
+ ]);
+ return true;
+ }
+ const claimed = this.claimedSteers.get(streamId) ?? [];
+ const incomingIds = new Set(items.map((item) => item.steerId));
+ const actuallyClaimed = claimed.filter((item) => incomingIds.has(item.steerId));
+ if (actuallyClaimed.length !== items.length) {
+ return false;
+ }
+ for (const item of actuallyClaimed) {
+ if (item.clientSteerId == null) {
+ continue;
+ }
+ const receipt = this.steerReceipts.get(streamId)?.get(item.clientSteerId)?.receipt;
+ if (
+ receipt == null ||
+ receipt.generationCreatedAt !== job.createdAt ||
+ receipt.state !== 'claimed' ||
+ receipt.item.steerId !== item.steerId ||
+ receipt.item.clientSteerId !== item.clientSteerId
+ ) {
+ return false;
+ }
+ }
+ const remainingClaims = claimed.filter((item) => !incomingIds.has(item.steerId));
+ if (remainingClaims.length > 0) {
+ this.claimedSteers.set(streamId, remainingClaims);
+ } else {
+ this.claimedSteers.delete(streamId);
+ }
+ const current = this.steerQueues.get(streamId) ?? [];
+ const currentIds = new Set(current.map((item) => item.steerId));
+ const restored = actuallyClaimed.filter((item) => !currentIds.has(item.steerId));
+ if (restored.length > 0) {
+ this.steerQueues.set(streamId, [...restored, ...current]);
+ this.settleSteerReceipts(streamId, restored, 'queued');
+ }
+ return true;
}
async closeAndDrainSteers(
streamId: string,
expectedCreatedAt?: number,
): Promise<SteerQueueItem[]> {
- if (expectedCreatedAt != null && this.jobs.get(streamId)?.createdAt !== expectedCreatedAt) {
+ const job = this.jobs.get(streamId);
+ if (job == null || (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt)) {
return [];
}
+ if (!this.isParkedRecoveryCompatible(streamId, job)) {
+ throw new Error('Generation recovery state is corrupt or belongs to another owner');
+ }
this.closedSteerQueues.add(streamId);
- return this.drainSteers(streamId);
+ const queued = this.steerQueues.get(streamId)?.splice(0) ?? [];
+ const combined = [
+ ...(usesReceiptSafeProtocol(job) ? this.claimedSteerItems(streamId, expectedCreatedAt) : []),
+ ...queued,
+ ];
+ const seen = new Set<string>();
+ const leftovers = combined.filter((item) => {
+ if (seen.has(item.steerId)) {
+ return false;
+ }
+ seen.add(item.steerId);
+ return true;
+ });
+ this.settleSteerReceipts(streamId, leftovers, 'leftover');
+ this.claimedSteers.delete(streamId);
+ this.parkLeftovers(streamId, leftovers, job, Date.now());
+ return leftovers;
}
async peekSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]> {
@@ -647,55 +1748,682 @@ export class InMemoryJobStore implements IJobStore {
return queue ? [...queue] : [];
}
+ async peekClaimedSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]> {
+ const job = this.jobs.get(streamId);
+ if (
+ !usesReceiptSafeProtocol(job) ||
+ (expectedCreatedAt != null && job?.createdAt !== expectedCreatedAt)
+ ) {
+ return [];
+ }
+ return this.claimedSteerItems(streamId, expectedCreatedAt);
+ }
+
async clearSteers(streamId: string): Promise<void> {
+ this.settleSteerReceipts(
+ streamId,
+ [...this.claimedSteerItems(streamId), ...(this.steerQueues.get(streamId) ?? [])],
+ 'leftover',
+ );
this.steerQueues.delete(streamId);
+ this.claimedSteers.delete(streamId);
}
- async removeSteer(streamId: string, steerId: string): Promise<boolean> {
+ async removeSteer(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (expectedCreatedAt != null && job?.createdAt !== expectedCreatedAt) {
+ return false;
+ }
const queue = this.steerQueues.get(streamId);
- const index = queue?.findIndex((item) => item.steerId === steerId) ?? -1;
- if (queue == null || index < 0) {
+ if (queue == null) {
+ return false;
+ }
+ let index = -1;
+ for (let i = 0; i < queue.length; i++) {
+ const item = queue[i] as unknown;
+ if (
+ item == null ||
+ typeof item !== 'object' ||
+ !('steerId' in item) ||
+ typeof item.steerId !== 'string' ||
+ item.steerId.length === 0
+ ) {
+ throw new Error(`Invalid steer queue item while cancelling ${streamId}`);
+ }
+ if (item.steerId === steerId) {
+ if (index >= 0) {
+ throw new Error(`Duplicate steer id while cancelling ${streamId}`);
+ }
+ index = i;
+ }
+ }
+ if (index < 0) {
return false;
}
+ const target = queue[index];
+ if (target.clientSteerId != null && job == null) {
+ throw new Error(`Invalid steer receipt while cancelling ${streamId}`);
+ }
+ const receipt = job
+ ? this.queuedReceiptForMutation(streamId, job, target, 'cancelling')
+ : undefined;
queue.splice(index, 1);
+ if (receipt != null) {
+ receipt.item = { ...target };
+ receipt.state = 'cancelled';
+ }
return true;
}
+ async armSteer(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerArmOutcome> {
+ return (await this.armSteerVersioned(streamId, steerId, expectedCreatedAt)).outcome;
+ }
+
+ private queuedReceiptForMutation(
+ streamId: string,
+ job: SerializableJobData,
+ item: SteerQueueItem,
+ operation: string,
+ ): SteerReceipt | undefined {
+ if (!usesReceiptSafeProtocol(job)) {
+ return undefined;
+ }
+ if (item.clientSteerId == null) {
+ return undefined;
+ }
+ const receipt = this.steerReceipts.get(streamId)?.get(item.clientSteerId)?.receipt as unknown;
+ if (
+ receipt == null ||
+ typeof receipt !== 'object' ||
+ !('item' in receipt) ||
+ receipt.item == null ||
+ typeof receipt.item !== 'object' ||
+ !('clientSteerId' in receipt) ||
+ receipt.clientSteerId !== item.clientSteerId ||
+ !('generationCreatedAt' in receipt) ||
+ receipt.generationCreatedAt !== job.createdAt ||
+ !('state' in receipt) ||
+ receipt.state !== 'queued' ||
+ !('steerId' in receipt.item) ||
+ receipt.item.steerId !== item.steerId ||
+ !('clientSteerId' in receipt.item) ||
+ receipt.item.clientSteerId !== item.clientSteerId
+ ) {
+ throw new Error(`Invalid steer receipt while ${operation} ${streamId}`);
+ }
+ return receipt as SteerReceipt;
+ }
+
+ async armSteerVersioned(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerArmResult> {
+ const job = this.jobs.get(streamId);
+ if (!job || job.status !== 'running' || this.closedSteerQueues.has(streamId)) {
+ return { outcome: 'missing' };
+ }
+ if (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt) {
+ return { outcome: 'missing' };
+ }
+ const item = this.steerQueues.get(streamId)?.find((entry) => entry.steerId === steerId);
+ if (item == null) {
+ return { outcome: 'missing' };
+ }
+ if (job.preemptCapable !== true) {
+ return { outcome: 'incapable' };
+ }
+ const receipt = this.queuedReceiptForMutation(streamId, job, item, 'arming');
+ item.preemptRevision = (item.preemptRevision ?? 0) + 1;
+ item.preempt = true;
+ if (receipt != null) {
+ receipt.item = { ...item };
+ }
+ return { outcome: 'armed', revision: item.preemptRevision, item: { ...item } };
+ }
+
+ async downgradeSteerPreempts(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerQueueItem[] | null> {
+ const job = this.jobs.get(streamId);
+ if (
+ !job ||
+ (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt) ||
+ job.preemptCapable === true
+ ) {
+ return null;
+ }
+ const preempted = (this.steerQueues.get(streamId) ?? []).filter(
+ (item) => item.preempt === true,
+ );
+ const receipts = new Map<SteerQueueItem, SteerReceipt | undefined>();
+ for (const item of preempted) {
+ receipts.set(item, this.queuedReceiptForMutation(streamId, job, item, 'downgrading'));
+ }
+ const changed: SteerQueueItem[] = [];
+ for (const item of preempted) {
+ if (item.preempt === true) {
+ delete item.preempt;
+ item.preemptRevision = (item.preemptRevision ?? 0) + 1;
+ const receipt = receipts.get(item);
+ if (receipt != null) {
+ receipt.item = { ...item };
+ }
+ changed.push({ ...item });
+ }
+ }
+ return changed;
+ }
+
async parkSteers(streamId: string, payload: string, expectedCreatedAt?: number): Promise<void> {
- if (expectedCreatedAt != null && this.jobs.get(streamId)?.createdAt !== expectedCreatedAt) {
+ const job = this.jobs.get(streamId);
+ if (expectedCreatedAt != null && job?.createdAt !== expectedCreatedAt) {
return;
}
+ let normalizedPayload = payload;
+ try {
+ const parsed = JSON.parse(payload) as Record<string, unknown>;
+ parsed.generationProtocolVersion =
+ job != null
+ ? normalizeGenerationProtocol(job.generationProtocolVersion)
+ : normalizeGenerationProtocol(parsed.generationProtocolVersion);
+ normalizedPayload = JSON.stringify(parsed);
+ } catch {
+ // Keep malformed payloads bounded by the normal TTL; owner-gated reads
+ // fail closed rather than losing evidence needed for diagnosis.
+ }
this.parkedSteers.set(streamId, {
- payload,
+ payload: normalizedPayload,
expiresAt: Date.now() + PARKED_STEERS_TTL_MS,
});
}
- /** Non-owner fragments leave the payload untouched (mirrors the Redis Lua gate). */
- async claimParkedSteers(streamId: string, ownerFragment: string): Promise<string | undefined> {
+ async claimParkedSteers(
+ streamId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ ): Promise<string | undefined> {
+ return (await this.claimParkedSteersDetailed(streamId, ownerUserId, ownerTenantId, 2))?.payload;
+ }
+
+ async claimParkedSteersDetailed(
+ streamId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ requestedProtocolVersion: 1 | 2 = 1,
+ ): Promise<ParkedSteerClaim | undefined> {
const parked = this.parkedSteers.get(streamId);
- if (!parked || !parked.payload.includes(ownerFragment)) {
+ if (!parked || parked.expiresAt <= Date.now()) {
+ this.parkedSteers.delete(streamId);
+ return undefined;
+ }
+ let parsed: {
+ userId?: string;
+ tenantId?: string;
+ generationProtocolVersion?: unknown;
+ steers?: Array<
+ ReturnType<typeof toPendingSteer> & {
+ recoveringCreatedAt?: number;
+ }
+ >;
+ };
+ try {
+ parsed = JSON.parse(parked.payload) as typeof parsed;
+ } catch {
+ return undefined;
+ }
+ if (
+ parsed == null ||
+ typeof parsed !== 'object' ||
+ typeof parsed.userId !== 'string' ||
+ parsed.userId.length === 0 ||
+ !Array.isArray(parsed.steers) ||
+ parsed.steers.length === 0 ||
+ !parsed.steers.every(isValidParkedSteer) ||
+ !isValidGenerationProtocolMarker(parsed.generationProtocolVersion) ||
+ parsed.userId !== ownerUserId ||
+ (parsed.tenantId != null && parsed.tenantId !== ownerTenantId)
+ ) {
+ return undefined;
+ }
+ const job = this.jobs.get(streamId);
+ const activeRecovery =
+ job?.generationProtocolVersion === 2 &&
+ (job.status === 'running' || job.status === 'requires_action') &&
+ typeof job.recoveredSteerId === 'string' &&
+ job.recoveredSteerId.length > 0 &&
+ job.userId === ownerUserId &&
+ (job.tenantId == null || job.tenantId === ownerTenantId);
+ const leased: NonNullable<typeof parsed.steers> = [];
+ const visible: NonNullable<typeof parsed.steers> = [];
+ for (const item of parsed.steers) {
+ if (
+ activeRecovery &&
+ item.steerId === job.recoveredSteerId &&
+ item.recoveringCreatedAt === job.createdAt
+ ) {
+ leased.push(item);
+ continue;
+ }
+ const { recoveringCreatedAt: _recoveringCreatedAt, ...visibleItem } = item;
+ visible.push(visibleItem);
+ }
+ const generationProtocolVersion =
+ parsed.generationProtocolVersion === 2 && requestedProtocolVersion === 2 ? 2 : 1;
+ if (generationProtocolVersion === 1 && leased.length === 0) {
+ // Preserve the legacy GETDEL contract when no v2 recovery owns a source.
+ this.parkedSteers.delete(streamId);
+ return {
+ generationProtocolVersion,
+ payload: JSON.stringify({
+ ...parsed,
+ generationProtocolVersion,
+ steers: visible,
+ }),
+ };
+ }
+ if (visible.length === 0) {
return undefined;
}
- this.parkedSteers.delete(streamId);
- return parked.expiresAt > Date.now() ? parked.payload : undefined;
+ if (generationProtocolVersion === 1) {
+ parked.payload = JSON.stringify({
+ userId: parsed.userId,
+ ...(parsed.tenantId != null && { tenantId: parsed.tenantId }),
+ generationProtocolVersion: 2,
+ steers: leased,
+ });
+ }
+ return {
+ generationProtocolVersion,
+ payload: JSON.stringify({
+ userId: parsed.userId,
+ ...(parsed.tenantId != null && { tenantId: parsed.tenantId }),
+ generationProtocolVersion,
+ steers: visible,
+ }),
+ };
}
- /** Terminal cleanup (approval expiry, stale-running reap) must not drop
- * 202-accepted steers with the job. */
- private parkQueuedSteers(streamId: string, job: SerializableJobData, now: number): void {
- const queue = this.steerQueues.get(streamId);
- if (!queue || queue.length === 0) {
+ async discardSteerLeftover(
+ streamId: string,
+ clientSteerId: string,
+ steerId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ expectedGenerationCreatedAt?: number,
+ ): Promise<boolean> {
+ const entry = this.steerReceipts.get(streamId)?.get(clientSteerId);
+ if (
+ entry?.receipt.state !== 'leftover' ||
+ (expectedGenerationCreatedAt != null &&
+ entry.receipt.generationCreatedAt !== expectedGenerationCreatedAt) ||
+ entry.receipt.item.steerId !== steerId ||
+ entry.receipt.userId !== ownerUserId ||
+ (entry.receipt.tenantId != null && entry.receipt.tenantId !== ownerTenantId)
+ ) {
+ return false;
+ }
+ const activeJob = this.jobs.get(streamId);
+ if (
+ activeJob?.recoveredSteerId === steerId &&
+ activeJob.userId === ownerUserId &&
+ (activeJob.tenantId == null || activeJob.tenantId === ownerTenantId) &&
+ (activeJob.status === 'running' || activeJob.status === 'requires_action')
+ ) {
+ // A stale tab can still hold a leftover chip after another tab starts its
+ // recovery. Once createJob leases the source, that recovery owns the
+ // terminal action; cancelling it here would invalidate an in-flight turn
+ // after its ordinary user message may already have been persisted.
+ return false;
+ }
+ const parked = this.parkedSteers.get(streamId);
+ if (parked != null && parked.expiresAt > Date.now()) {
+ try {
+ const parsed = JSON.parse(parked.payload) as {
+ userId?: string;
+ tenantId?: string;
+ steers?: Array<{ steerId?: string }>;
+ };
+ if (
+ parsed.userId !== ownerUserId ||
+ (parsed.tenantId != null && parsed.tenantId !== ownerTenantId) ||
+ !Array.isArray(parsed.steers) ||
+ parsed.steers.length === 0 ||
+ !parsed.steers.every(isValidParkedSteer)
+ ) {
+ return false;
+ }
+ parsed.steers = parsed.steers.filter((item) => item.steerId !== steerId);
+ if (parsed.steers.length === 0) {
+ this.parkedSteers.delete(streamId);
+ } else {
+ parked.payload = JSON.stringify(parsed);
+ }
+ } catch {
+ return false;
+ }
+ }
+ entry.receipt.state = 'cancelled';
+ return true;
+ }
+
+ private parkLeftovers(
+ streamId: string,
+ items: SteerQueueItem[],
+ job: SerializableJobData | null | undefined,
+ now: number,
+ ): void {
+ if (items.length === 0 || job == null) {
return;
}
+ let prior: Array<ReturnType<typeof toPendingSteer>> = [];
+ let parkedProtocol = normalizeGenerationProtocol(job.generationProtocolVersion);
+ const existing = this.parkedSteers.get(streamId);
+ if (existing != null && existing.expiresAt > now) {
+ try {
+ const parsed = JSON.parse(existing.payload) as {
+ userId?: string;
+ tenantId?: string;
+ generationProtocolVersion?: unknown;
+ steers?: Array<ReturnType<typeof toPendingSteer>>;
+ };
+ if (
+ parsed.userId === job.userId &&
+ (parsed.tenantId == null || parsed.tenantId === job.tenantId) &&
+ isValidGenerationProtocolMarker(parsed.generationProtocolVersion) &&
+ Array.isArray(parsed.steers)
+ ) {
+ prior = parsed.steers;
+ parkedProtocol = Math.max(
+ parkedProtocol,
+ normalizeGenerationProtocol(parsed.generationProtocolVersion),
+ ) as 1 | 2;
+ }
+ } catch {
+ // Replace malformed legacy recovery state with the authoritative items.
+ }
+ }
+ const seen = new Set<string>();
+ const merged = [...prior, ...items.map(toPendingSteer)].filter((item) => {
+ if (seen.has(item.steerId)) {
+ return false;
+ }
+ seen.add(item.steerId);
+ return true;
+ });
this.parkedSteers.set(streamId, {
payload: JSON.stringify({
userId: job.userId,
...(job.tenantId != null && { tenantId: job.tenantId }),
- steers: queue.map(toPendingSteer),
+ generationProtocolVersion: parkedProtocol,
+ steers: merged,
}),
expiresAt: now + PARKED_STEERS_TTL_MS,
});
+ const receiptExpiry = now + PARKED_STEERS_TTL_MS;
+ for (const item of items) {
+ if (item.clientSteerId == null) {
+ continue;
+ }
+ const entry = this.steerReceipts.get(streamId)?.get(item.clientSteerId);
+ if (entry != null) {
+ entry.expiresAt = Number.isFinite(entry.expiresAt)
+ ? Math.max(entry.expiresAt, receiptExpiry)
+ : receiptExpiry;
+ }
+ }
+ }
+
+ /** Lease without deleting: an initialization failure makes the item visible
+ * again as soon as this generation becomes terminal. */
+ private leaseParkedSteer(
+ streamId: string,
+ steerId: string | undefined,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ createdAt?: number,
+ ): void {
+ if (!usesReceiptSafeProtocol(this.jobs.get(streamId))) {
+ return;
+ }
+ const parked = this.parkedSteers.get(streamId);
+ if (parked == null || parked.expiresAt <= Date.now()) {
+ return;
+ }
+ try {
+ const parsed = JSON.parse(parked.payload) as {
+ userId: string;
+ tenantId?: string;
+ steers?: Array<{ steerId: string; clientSteerId?: string; recoveringCreatedAt?: number }>;
+ };
+ if (!Array.isArray(parsed.steers)) {
+ return;
+ }
+ if (
+ parsed.userId !== ownerUserId ||
+ (parsed.tenantId != null && parsed.tenantId !== ownerTenantId)
+ ) {
+ return;
+ }
+ for (const item of parsed.steers) {
+ delete item.recoveringCreatedAt;
+ }
+ const leased =
+ steerId != null ? parsed.steers.find((item) => item.steerId === steerId) : null;
+ if (leased != null && createdAt != null) {
+ leased.recoveringCreatedAt = createdAt;
+ const job = this.jobs.get(streamId);
+ if (job?.createdAt === createdAt) {
+ this.extendRecoveryLease(
+ streamId,
+ job,
+ this.runningRecoveryLeaseExpiry(Date.now()),
+ parsed,
+ leased,
+ );
+ }
+ }
+ parked.payload = JSON.stringify(parsed);
+ } catch {
+ // Malformed recovery state remains bounded by its TTL.
+ }
+ }
+
+ /** A recovery source must outlive the generation that hides it, plus one
+ * normal parked-recovery window after a stale-job reap exposes it again. */
+ private runningRecoveryLeaseExpiry(now: number): number {
+ return this.staleJobTimeout > 0
+ ? now + this.staleJobTimeout + PARKED_STEERS_TTL_MS
+ : Number.POSITIVE_INFINITY;
+ }
+
+ private extendRecoveryLease(
+ streamId: string,
+ job: SerializableJobData,
+ expiresAt: number,
+ parsedPayload?: {
+ userId: string;
+ tenantId?: string;
+ steers?: Array<{ steerId: string; clientSteerId?: string; recoveringCreatedAt?: number }>;
+ },
+ parsedLeasedItem?: {
+ steerId: string;
+ clientSteerId?: string;
+ recoveringCreatedAt?: number;
+ },
+ ): void {
+ if (job.recoveredSteerId == null) {
+ return;
+ }
+ const parked = this.parkedSteers.get(streamId);
+ if (parked == null || parked.expiresAt <= Date.now()) {
+ return;
+ }
+ try {
+ const parsed =
+ parsedPayload ??
+ (JSON.parse(parked.payload) as {
+ userId: string;
+ tenantId?: string;
+ steers?: Array<{
+ steerId: string;
+ clientSteerId?: string;
+ recoveringCreatedAt?: number;
+ }>;
+ });
+ const leased =
+ parsedLeasedItem ??
+ parsed.steers?.find(
+ (item) =>
+ item.steerId === job.recoveredSteerId && item.recoveringCreatedAt === job.createdAt,
+ );
+ if (
+ leased == null ||
+ parsed.userId !== job.userId ||
+ (parsed.tenantId != null && parsed.tenantId !== job.tenantId)
+ ) {
+ return;
+ }
+ parked.expiresAt = Math.max(parked.expiresAt, expiresAt);
+ if (leased.clientSteerId != null) {
+ const receiptEntry = this.steerReceipts.get(streamId)?.get(leased.clientSteerId);
+ if (receiptEntry != null) {
+ receiptEntry.expiresAt = Math.max(receiptEntry.expiresAt, expiresAt);
+ }
+ }
+ } catch {
+ // Malformed recovery state remains bounded by its existing TTL.
+ }
+ }
+
+ async consumeParkedSteer(
+ streamId: string,
+ steerId: string,
+ ownerUserId: string,
+ ownerTenantId: string | undefined,
+ expectedCreatedAt: number,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (
+ job?.createdAt !== expectedCreatedAt ||
+ job.userId !== ownerUserId ||
+ job.recoveredSteerId !== steerId ||
+ (job.tenantId != null && job.tenantId !== ownerTenantId)
+ ) {
+ return false;
+ }
+ const parked = this.parkedSteers.get(streamId);
+ if (parked == null || parked.expiresAt <= Date.now()) {
+ return true;
+ }
+ try {
+ const parsed = JSON.parse(parked.payload) as {
+ userId: string;
+ tenantId?: string;
+ steers?: Array<{
+ steerId: string;
+ clientSteerId?: string;
+ recoveringCreatedAt?: number;
+ }>;
+ };
+ if (
+ parsed.userId !== ownerUserId ||
+ (parsed.tenantId != null && parsed.tenantId !== ownerTenantId) ||
+ !Array.isArray(parsed.steers) ||
+ parsed.steers.length === 0 ||
+ !parsed.steers.every(isValidParkedSteer)
+ ) {
+ return false;
+ }
+ const consumed = parsed.steers.find(
+ (item) => item.steerId === steerId && item.recoveringCreatedAt === expectedCreatedAt,
+ );
+ if (consumed == null) {
+ return false;
+ }
+ if (consumed.clientSteerId != null) {
+ const receipt = this.steerReceipts.get(streamId)?.get(consumed.clientSteerId)?.receipt;
+ if (
+ receipt?.state !== 'leftover' ||
+ receipt.item.steerId !== steerId ||
+ receipt.userId !== ownerUserId ||
+ (receipt.tenantId != null && receipt.tenantId !== ownerTenantId)
+ ) {
+ return false;
+ }
+ receipt.state = 'recovered';
+ }
+ parsed.steers = parsed.steers.filter((item) => item !== consumed);
+ if (parsed.steers.length === 0) {
+ this.parkedSteers.delete(streamId);
+ } else {
+ parked.payload = JSON.stringify(parsed);
+ }
+ return true;
+ } catch {
+ return false;
+ }
+ }
+
+ /** Terminal cleanup (approval expiry, stale-running reap) must not drop
+ * 202-accepted steers with the job. */
+ private isParkedRecoveryCompatible(streamId: string, job: SerializableJobData): boolean {
+ const parked = this.parkedSteers.get(streamId);
+ if (parked == null || parked.expiresAt <= Date.now()) {
+ return true;
+ }
+ try {
+ const parsed = JSON.parse(parked.payload) as {
+ userId?: unknown;
+ tenantId?: unknown;
+ generationProtocolVersion?: unknown;
+ steers?: unknown;
+ };
+ return (
+ parsed != null &&
+ typeof parsed === 'object' &&
+ typeof parsed.userId === 'string' &&
+ parsed.userId.length > 0 &&
+ parsed.userId === job.userId &&
+ (parsed.tenantId == null || parsed.tenantId === job.tenantId) &&
+ isValidGenerationProtocolMarker(parsed.generationProtocolVersion) &&
+ Array.isArray(parsed.steers) &&
+ parsed.steers.length > 0 &&
+ parsed.steers.every(isValidParkedSteer)
+ );
+ } catch {
+ return false;
+ }
+ }
+
+ private parkQueuedSteers(streamId: string, job: SerializableJobData, now: number): void {
+ const combined = [
+ ...this.claimedSteerItems(streamId, job.createdAt),
+ ...(this.steerQueues.get(streamId) ?? []),
+ ];
+ const seen = new Set<string>();
+ const queue = combined.filter((item) => {
+ if (seen.has(item.steerId)) {
+ return false;
+ }
+ seen.add(item.steerId);
+ return true;
+ });
+ if (queue.length === 0) {
+ return;
+ }
+ this.settleSteerReceipts(streamId, queue, 'leftover');
+ this.parkLeftovers(streamId, queue, job, now);
this.steerQueues.delete(streamId);
+ this.claimedSteers.delete(streamId);
}
}
diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.ts
index bce0b9bedc..8b6bc005e3 100644
--- a/packages/api/src/stream/implementations/RedisEventTransport.ts
+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts
@@ -1,3 +1,4 @@
+import { randomUUID } from 'crypto';
import { logger } from '@librechat/data-schemas';
import type { Redis, Cluster } from 'ioredis';
import type { IEventTransport, PreemptMessage } from '~/stream/interfaces/IJobStore';
@@ -22,6 +23,9 @@ const KEYS = {
job: (streamId: string) => `stream:{${streamId}}:job`,
/** Latest generation epoch, retained briefly beyond the live job hash. */
generationEpoch: (streamId: string) => `stream:{${streamId}}:generation-epoch`,
+ /** Owner-issued proof that this exact generation processed an abort. */
+ abortAck: (streamId: string, generationId: number) =>
+ `stream:{${streamId}}:abort-ack:${generationId}`,
};
/**
@@ -32,6 +36,7 @@ const EventTypes = {
DONE: 'done',
ERROR: 'error',
ABORT: 'abort',
+ ABORT_ACK: 'abort_ack',
PREEMPT: 'preempt',
} as const;
@@ -43,6 +48,8 @@ interface PubSubMessage {
error?: string;
/** Immutable identity of the generation that emitted the event. */
generationId?: number;
+ /** Opaque nonce linking a replacement abort to its owner acknowledgement. */
+ abortRequestId?: string;
/** Payload for PREEMPT messages; fenced by its own createdAt. */
preempt?: PreemptMessage;
}
@@ -63,7 +70,13 @@ interface ReorderBuffer {
}
interface AbortRegistration {
- callback: (generationId?: number) => void;
+ callback: (generationId?: number) => void | boolean;
+}
+
+interface AbortAckWaiter {
+ generationId: number;
+ resolve: (acknowledged: boolean) => void;
+ timeout: ReturnType<typeof setTimeout>;
}
interface PreemptRegistration {
@@ -95,7 +108,8 @@ interface PreemptRegistration {
* sequenceTtlSeconds,
* expectCreatedAt | "",
* allowRetainedEpoch ("0" | "1"),
- * generationEpochGraceTtl
+ * generationEpochGraceTtl,
+ * requireActiveJob ("0" | "1")
* ]
* RETURNS: the 0-indexed seq assigned to this event, or -1 when the generation guard fails
*
@@ -118,6 +132,10 @@ const PUBLISH_SEQ_LUA =
'if retainedEpoch ~= ARGV[5] then return -1 end ' +
'end ' +
'end ' +
+ 'if ARGV[8] == "1" then ' +
+ 'local currentStatus = redis.call("HGET", KEYS[2], "status") ' +
+ 'if currentStatus ~= "running" and currentStatus ~= "requires_action" then return -1 end ' +
+ 'end ' +
'local val = redis.call("INCR", KEYS[1]) ' +
'local ttl = tonumber(ARGV[4]) ' +
'local seqTtl = redis.call("TTL", KEYS[1]) ' +
@@ -130,12 +148,37 @@ const PUBLISH_SEQ_LUA =
'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq) .. ARGV[3]) ' +
'return seq';
+/** A normal generation guard correctly rejects events from an old epoch once
+ * its replacement exists. Replacement handoff is the one exception: the
+ * current create attempt must still carry the old epoch in its durable receipt
+ * chain. This script verifies that proof, assigns the shared sequence, and
+ * publishes the old-generation DONE in one same-slot decision. */
+const PUBLISH_REPLACED_DONE_LUA =
+ 'if redis.call("HGET", KEYS[2], "__creationAttemptId") ~= ARGV[6] then return -1 end ' +
+ 'local authorized = false local raw = redis.call("HGET", KEYS[2], "__replacedGenerations") ' +
+ 'if raw then local ok, receipts = pcall(cjson.decode, raw) if not ok or type(receipts) ~= "table" then return -1 end ' +
+ 'for i = 1, #receipts do local receipt = receipts[i] ' +
+ 'if type(receipt) == "table" and tostring(receipt.createdAt or "") == ARGV[5] then authorized = true break end end ' +
+ 'else authorized = redis.call("HGET", KEYS[2], "__replacedCreatedAt") == ARGV[5] end ' +
+ 'if not authorized then return -1 end ' +
+ 'local val = redis.call("INCR", KEYS[1]) local ttl = tonumber(ARGV[4]) ' +
+ 'local seqTtl = redis.call("TTL", KEYS[1]) if seqTtl < math.floor(ttl / 2) then ' +
+ 'local jobTtl = redis.call("TTL", KEYS[2]) if jobTtl > ttl then ttl = jobTtl end ' +
+ 'redis.call("EXPIRE", KEYS[1], ttl) end local seq = val - 1 ' +
+ 'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq) .. ARGV[3]) return seq';
+
/** Max time (ms) to wait for out-of-order messages before force-flushing */
const REORDER_TIMEOUT_MS = 500;
/** Max messages to buffer before force-flushing (prevents memory issues) */
const MAX_BUFFER_SIZE = 100;
/** Rolling-upgrade recovery window after a legacy job hash expires without an epoch marker. */
const GENERATION_EPOCH_GRACE_TTL_SECONDS = 300;
+/** A replacement remains fail-closed if its exact generation owner cannot
+ * acknowledge promptly. Redis pub/sub is local-network traffic; this budget
+ * tolerates reconnect jitter without holding an HTTP request indefinitely. */
+const ABORT_ACK_TIMEOUT_MS = 3000;
+/** Durable owner proof outlives receipt retries and process-local subscriptions. */
+const ABORT_ACK_TTL_SECONDS = 86400;
/**
* Subscriber state for a stream
@@ -154,6 +197,8 @@ interface StreamSubscribers {
allSubscribersLeftCallback?: () => void;
/** Abort callbacks - called when abort signal is received from any replica */
abortCallbacks: Set<AbortRegistration>;
+ /** Replacement aborts awaiting the exact generation owner's acknowledgement. */
+ abortAckWaiters: Map<string, AbortAckWaiter>;
preemptCallbacks: Set<PreemptRegistration>;
/** Reorder buffer for handling out-of-order delivery in Redis Cluster */
reorderBuffer: ReorderBuffer;
@@ -193,6 +238,32 @@ export class RedisEventTransport implements IEventTransport {
/** Counter for generating unique subscriber IDs */
private subscriberIdCounter = 0;
+ private createStreamState(): StreamSubscribers {
+ return {
+ count: 0,
+ handlers: new Map(),
+ abortCallbacks: new Set(),
+ abortAckWaiters: new Map(),
+ preemptCallbacks: new Set(),
+ reorderBuffer: {
+ nextSeq: 0,
+ pending: new Map(),
+ flushTimeout: null,
+ deliveryDeferred: false,
+ },
+ };
+ }
+
+ private getOrCreateStreamState(streamId: string): StreamSubscribers {
+ const existing = this.streams.get(streamId);
+ if (existing) {
+ return existing;
+ }
+ const state = this.createStreamState();
+ this.streams.set(streamId, state);
+ return state;
+ }
+
/**
* Create a new Redis event transport.
*
@@ -241,6 +312,7 @@ export class RedisEventTransport implements IEventTransport {
message: Omit<PubSubMessage, 'seq'>,
expectedGenerationId?: number,
allowRetainedEpoch = false,
+ requireActiveJob = false,
): Promise<number> {
const [prefix, suffix] = RedisEventTransport.buildPayloadParts(message);
const seq = await this.publisher.eval(
@@ -256,6 +328,7 @@ export class RedisEventTransport implements IEventTransport {
expectedGenerationId != null ? String(expectedGenerationId) : '',
allowRetainedEpoch ? '1' : '0',
String(GENERATION_EPOCH_GRACE_TTL_SECONDS),
+ requireActiveJob ? '1' : '0',
);
return seq as number;
}
@@ -264,7 +337,7 @@ export class RedisEventTransport implements IEventTransport {
streamId: string,
event: unknown,
generationId?: number,
- ): Promise<number | false> {
+ ): Promise<number | false | void> {
return this.publishWithSequence(
streamId,
{
@@ -273,11 +346,16 @@ export class RedisEventTransport implements IEventTransport {
...(generationId != null && { generationId }),
},
generationId,
+ false,
+ generationId != null,
)
.then((sequence) => (sequence === -1 ? false : sequence))
.catch((err) => {
logger.error(`[RedisEventTransport] Failed to publish chunk:`, err);
- return false;
+ /** `false` is reserved for an authoritative generation/status fence.
+ * An operational publication failure has no such ownership proof and
+ * remains replayable from the durable/local buffer. */
+ return undefined;
});
}
@@ -435,7 +513,7 @@ export class RedisEventTransport implements IEventTransport {
) {
this.handleTerminalEvent(streamId, streamState, parsed);
} else {
- this.deliverMessage(streamState, parsed);
+ this.deliverMessage(streamId, streamState, parsed);
}
} catch (err) {
logger.error(`[RedisEventTransport] Failed to parse message:`, err);
@@ -467,7 +545,7 @@ export class RedisEventTransport implements IEventTransport {
}
if (seq === buffer.nextSeq) {
- this.deliverMessage(streamState, message);
+ this.deliverMessage(streamId, streamState, message);
buffer.nextSeq++;
this.flushPendingMessages(streamId, streamState);
} else {
@@ -494,7 +572,7 @@ export class RedisEventTransport implements IEventTransport {
}
if (seq === buffer.nextSeq) {
- this.deliverMessage(streamState, message);
+ this.deliverMessage(streamId, streamState, message);
buffer.nextSeq++;
this.flushPendingMessages(streamId, streamState);
@@ -521,7 +599,7 @@ export class RedisEventTransport implements IEventTransport {
while (buffer.pending.has(buffer.nextSeq)) {
const message = buffer.pending.get(buffer.nextSeq)!;
buffer.pending.delete(buffer.nextSeq);
- this.deliverMessage(streamState, message);
+ this.deliverMessage(streamId, streamState, message);
buffer.nextSeq++;
}
@@ -556,7 +634,7 @@ export class RedisEventTransport implements IEventTransport {
for (const seq of sortedSeqs) {
const message = buffer.pending.get(seq)!;
buffer.pending.delete(seq);
- this.deliverMessage(streamState, message);
+ this.deliverMessage(streamId, streamState, message);
}
buffer.nextSeq = sortedSeqs[sortedSeqs.length - 1] + 1;
@@ -582,7 +660,11 @@ export class RedisEventTransport implements IEventTransport {
}
/** Deliver a message to all handlers */
- private deliverMessage(streamState: StreamSubscribers, message: PubSubMessage): void {
+ private deliverMessage(
+ streamId: string,
+ streamState: StreamSubscribers,
+ message: PubSubMessage,
+ ): void {
for (const [, handlers] of streamState.handlers) {
switch (message.type) {
case EventTypes.CHUNK:
@@ -607,24 +689,48 @@ export class RedisEventTransport implements IEventTransport {
}
break;
case EventTypes.ABORT:
+ case EventTypes.ABORT_ACK:
break;
case EventTypes.PREEMPT:
break;
}
}
+ if (message.type === EventTypes.ABORT_ACK) {
+ const requestId = message.abortRequestId;
+ const generationId = message.generationId;
+ if (requestId == null || generationId == null) {
+ return;
+ }
+ const waiter = streamState.abortAckWaiters.get(requestId);
+ if (waiter == null || waiter.generationId !== generationId) {
+ return;
+ }
+ this.settleAbortAck(streamId, streamState, requestId, true);
+ return;
+ }
+
if (message.type === EventTypes.ABORT) {
+ let ownerAcknowledged = false;
for (const registration of streamState.abortCallbacks) {
try {
if (message.generationId == null) {
- registration.callback();
+ ownerAcknowledged = registration.callback() === true || ownerAcknowledged;
} else {
- registration.callback(message.generationId);
+ ownerAcknowledged =
+ registration.callback(message.generationId) === true || ownerAcknowledged;
}
} catch (err) {
logger.error(`[RedisEventTransport] Error in abort callback:`, err);
}
}
+ if (ownerAcknowledged && message.abortRequestId != null && message.generationId != null) {
+ void this.publishAbortAcknowledgement(
+ streamId,
+ message.generationId,
+ message.abortRequestId,
+ );
+ }
}
if (message.type === EventTypes.PREEMPT && message.preempt != null) {
@@ -655,6 +761,7 @@ export class RedisEventTransport implements IEventTransport {
this.streams.get(streamId) !== state ||
state.count > 0 ||
state.abortCallbacks.size > 0 ||
+ state.abortAckWaiters.size > 0 ||
state.preemptCallbacks.size > 0
) {
return;
@@ -693,22 +800,7 @@ export class RedisEventTransport implements IEventTransport {
const subscriberId = `sub_${++this.subscriberIdCounter}`;
// Initialize stream state if needed
- if (!this.streams.has(streamId)) {
- this.streams.set(streamId, {
- count: 0,
- handlers: new Map(),
- abortCallbacks: new Set(),
- preemptCallbacks: new Set(),
- reorderBuffer: {
- nextSeq: 0,
- pending: new Map(),
- flushTimeout: null,
- deliveryDeferred: false,
- },
- });
- }
-
- const streamState = this.streams.get(streamId)!;
+ const streamState = this.getOrCreateStreamState(streamId);
// Internal listeners (for example cross-replica abort) can leave ordering
// state behind with no real SSE subscribers. A new subscriber is a fresh
// attachment and must not inherit that prior generation's expected seq.
@@ -778,7 +870,7 @@ export class RedisEventTransport implements IEventTransport {
*/
async emitDone(streamId: string, event: unknown, generationId?: number): Promise<void> {
try {
- await this.publishWithSequence(
+ const sequence = await this.publishWithSequence(
streamId,
{
type: EventTypes.DONE,
@@ -788,19 +880,50 @@ export class RedisEventTransport implements IEventTransport {
generationId,
true,
);
+ if (sequence === -1) {
+ throw new Error('Generation DONE publication was fenced by a replacement');
+ }
} catch (err) {
logger.error(`[RedisEventTransport] Failed to publish done:`, err);
throw err;
}
}
+ async emitReplacedDoneConfirmed(
+ streamId: string,
+ event: unknown,
+ replacedGenerationId: number,
+ creationAttemptId: string,
+ ): Promise<void> {
+ const [prefix, suffix] = RedisEventTransport.buildPayloadParts({
+ type: EventTypes.DONE,
+ data: event,
+ generationId: replacedGenerationId,
+ });
+ const result = await this.publisher.eval(
+ PUBLISH_REPLACED_DONE_LUA,
+ 2,
+ KEYS.sequence(streamId),
+ KEYS.job(streamId),
+ CHANNELS.events(streamId),
+ prefix,
+ suffix,
+ String(RedisEventTransport.SEQUENCE_TTL_SECONDS),
+ String(replacedGenerationId),
+ creationAttemptId,
+ );
+ if (result === -1) {
+ throw new Error('Generation replacement DONE receipt is no longer current');
+ }
+ }
+
/**
* Publish an error event to all subscribers.
* Includes sequence number to ensure delivery after all chunks.
*/
async emitError(streamId: string, error: string, generationId?: number): Promise<void> {
try {
- await this.publishWithSequence(
+ const sequence = await this.publishWithSequence(
streamId,
{
type: EventTypes.ERROR,
@@ -810,6 +933,9 @@ export class RedisEventTransport implements IEventTransport {
generationId,
true,
);
+ if (sequence === -1) {
+ throw new Error('Generation error publication was fenced by a replacement');
+ }
} catch (err) {
logger.error(`[RedisEventTransport] Failed to publish error:`, err);
throw err;
@@ -864,25 +990,7 @@ export class RedisEventTransport implements IEventTransport {
* Register callback for when all subscribers leave.
*/
onAllSubscribersLeft(streamId: string, callback: () => void): void {
- const state = this.streams.get(streamId);
- if (state) {
- state.allSubscribersLeftCallback = callback;
- } else {
- // Create state just for the callback
- this.streams.set(streamId, {
- count: 0,
- handlers: new Map(),
- allSubscribersLeftCallback: callback,
- abortCallbacks: new Set(),
- preemptCallbacks: new Set(),
- reorderBuffer: {
- nextSeq: 0,
- pending: new Map(),
- flushTimeout: null,
- deliveryDeferred: false,
- },
- });
- }
+ this.getOrCreateStreamState(streamId).allSubscribersLeftCallback = callback;
}
/**
@@ -891,15 +999,118 @@ export class RedisEventTransport implements IEventTransport {
* the generating Replica A receives the signal and stops.
*/
emitAbort(streamId: string, generationId?: number): void {
+ void this.publishAbort(streamId, generationId).catch((err) => {
+ logger.error(`[RedisEventTransport] Failed to publish abort:`, err);
+ });
+ }
+
+ private publishAbort(
+ streamId: string,
+ generationId?: number,
+ abortRequestId?: string,
+ ): Promise<number> {
const channel = CHANNELS.events(streamId);
const message: PubSubMessage = {
type: EventTypes.ABORT,
...(generationId != null && { generationId }),
+ ...(abortRequestId != null && { abortRequestId }),
};
+ return this.publisher.publish(channel, JSON.stringify(message));
+ }
- this.publisher.publish(channel, JSON.stringify(message)).catch((err) => {
- logger.error(`[RedisEventTransport] Failed to publish abort:`, err);
+ private settleAbortAck(
+ streamId: string,
+ state: StreamSubscribers,
+ abortRequestId: string,
+ acknowledged: boolean,
+ ): void {
+ const waiter = state.abortAckWaiters.get(abortRequestId);
+ if (!waiter) {
+ return;
+ }
+ state.abortAckWaiters.delete(abortRequestId);
+ clearTimeout(waiter.timeout);
+ waiter.resolve(acknowledged);
+ this.unsubscribeUnusedChannel(streamId, state);
+ }
+
+ private async hasDurableAbortAck(streamId: string, generationId: number): Promise<boolean> {
+ return (await this.publisher.get(KEYS.abortAck(streamId, generationId))) === '1';
+ }
+
+ private async publishAbortAcknowledgement(
+ streamId: string,
+ generationId: number,
+ abortRequestId: string,
+ ): Promise<void> {
+ try {
+ await this.publisher.set(
+ KEYS.abortAck(streamId, generationId),
+ '1',
+ 'EX',
+ ABORT_ACK_TTL_SECONDS,
+ );
+ } catch (error) {
+ logger.error(`[RedisEventTransport] Failed to persist generation abort proof:`, error);
+ // A live acknowledgement is only useful if a racing or inherited receipt
+ // can prove the same owner stop after this subscription disappears. A
+ // SET that committed despite a lost reply is recovered by the requester's
+ // timeout read, so do not publish an ephemeral success here.
+ return;
+ }
+
+ const acknowledgement: PubSubMessage = {
+ type: EventTypes.ABORT_ACK,
+ generationId,
+ abortRequestId,
+ };
+ try {
+ await this.publisher.publish(CHANNELS.events(streamId), JSON.stringify(acknowledgement));
+ } catch (error) {
+ logger.error(`[RedisEventTransport] Failed to acknowledge generation abort:`, error);
+ }
+ }
+
+ /** Awaitable variant for replacement handoff receipts. Success requires an
+ * acknowledgement from the callback that owns this exact generation; Redis
+ * PUBLISH receiver counts are deliberately not treated as proof. */
+ async emitAbortConfirmed(streamId: string, generationId: number): Promise<boolean> {
+ try {
+ if (await this.hasDurableAbortAck(streamId, generationId)) {
+ return true;
+ }
+ } catch (error) {
+ logger.error(`[RedisEventTransport] Failed to inspect generation abort proof:`, error);
+ }
+
+ const channel = CHANNELS.events(streamId);
+ const state = this.getOrCreateStreamState(streamId);
+ await this.ensureChannelSubscription(channel);
+ if (this.streams.get(streamId) !== state) {
+ return false;
+ }
+
+ const abortRequestId = randomUUID();
+ const acknowledgement = new Promise<boolean>((resolve) => {
+ const timeout = setTimeout(() => {
+ void this.hasDurableAbortAck(streamId, generationId).then(
+ (acknowledged) => this.settleAbortAck(streamId, state, abortRequestId, acknowledged),
+ () => this.settleAbortAck(streamId, state, abortRequestId, false),
+ );
+ }, ABORT_ACK_TIMEOUT_MS);
+ state.abortAckWaiters.set(abortRequestId, { generationId, resolve, timeout });
});
+
+ try {
+ // Redis Cluster may report zero receivers on the publishing node while a
+ // subscriber on another node is still processing the abort. Only the
+ // correlated acknowledgement (or its durable proof) can settle this.
+ await this.publishAbort(streamId, generationId, abortRequestId);
+ } catch (error) {
+ this.settleAbortAck(streamId, state, abortRequestId, false);
+ throw error;
+ }
+ return acknowledgement;
}
/**
@@ -910,25 +1121,12 @@ export class RedisEventTransport implements IEventTransport {
* @param streamId - The stream identifier
* @param callback - Called when abort signal is received
*/
- async onAbort(streamId: string, callback: (generationId?: number) => void): Promise<() => void> {
+ async onAbort(
+ streamId: string,
+ callback: (generationId?: number) => void | boolean,
+ ): Promise<() => void> {
const channel = CHANNELS.events(streamId);
- let state = this.streams.get(streamId);
-
- if (!state) {
- state = {
- count: 0,
- handlers: new Map(),
- abortCallbacks: new Set(),
- preemptCallbacks: new Set(),
- reorderBuffer: {
- nextSeq: 0,
- pending: new Map(),
- flushTimeout: null,
- deliveryDeferred: false,
- },
- };
- this.streams.set(streamId, state);
- }
+ const state = this.getOrCreateStreamState(streamId);
const registration = { callback };
state.abortCallbacks.add(registration);
@@ -982,23 +1180,7 @@ export class RedisEventTransport implements IEventTransport {
*/
async onPreempt(streamId: string, callback: (msg: PreemptMessage) => void): Promise<() => void> {
const channel = CHANNELS.events(streamId);
- let state = this.streams.get(streamId);
-
- if (!state) {
- state = {
- count: 0,
- handlers: new Map(),
- abortCallbacks: new Set(),
- preemptCallbacks: new Set(),
- reorderBuffer: {
- nextSeq: 0,
- pending: new Map(),
- flushTimeout: null,
- deliveryDeferred: false,
- },
- };
- this.streams.set(streamId, state);
- }
+ const state = this.getOrCreateStreamState(streamId);
const registration = { callback };
state.preemptCallbacks.add(registration);
@@ -1044,6 +1226,11 @@ export class RedisEventTransport implements IEventTransport {
state.handlers.clear();
state.allSubscribersLeftCallback = undefined;
state.abortCallbacks.clear();
+ for (const waiter of state.abortAckWaiters.values()) {
+ clearTimeout(waiter.timeout);
+ waiter.resolve(false);
+ }
+ state.abortAckWaiters.clear();
state.preemptCallbacks.clear();
}
@@ -1073,6 +1260,11 @@ export class RedisEventTransport implements IEventTransport {
state.reorderBuffer.flushTimeout = null;
}
state.reorderBuffer.pending.clear();
+ for (const waiter of state.abortAckWaiters.values()) {
+ clearTimeout(waiter.timeout);
+ waiter.resolve(false);
+ }
+ state.abortAckWaiters.clear();
}
for (const channel of this.channelSubscriptions.keys()) {
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index 4bc31f4500..ae252af620 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -5,21 +5,57 @@ import type { Agents } from 'librechat-data-provider';
import type { Redis, Cluster } from 'ioredis';
import type {
SerializableJobData,
+ CreatedJobData,
+ ReplacedGeneration,
SteerQueueItem,
UsageMetadata,
- IJobStore,
+ IJobStoreV2,
JobStatus,
JobMetadataPatch,
JobStatusTransition,
IdempotencyClaimValue,
IdempotencyClaimResult,
+ SteerArmOutcome,
+ SteerArmResult,
+ SteerEnqueueReceiptResult,
+ SteerEnqueueVersionedResult,
+ SteerReceipt,
+ SteerReceiptInput,
+ ParkedSteerClaim,
} from '~/stream/interfaces/IJobStore';
+import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';
import {
+ JobCreationSupersededError,
+ JobPredecessorMismatchError,
STEER_ENQUEUE_NOT_RUNNING,
STEER_QUEUE_MAX_DEPTH,
+ PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ PAUSE_PERSISTENCE_TIMEOUT_MS,
isPendingActionStale,
} from '~/stream/interfaces/IJobStore';
import { instrumentIORedisClient, RedisUseCases } from '~/cache/redisTelemetry';
+import { RecoveredSteerPayloadMismatchError } from '~/stream/SteerRecovery';
+
+const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
+
+function assertCreateIdempotencyArguments(
+ claimKey?: string,
+ claimToken?: string,
+ clientRequestId?: string,
+): void {
+ const supplied = [claimKey, claimToken, clientRequestId].filter((value) => value != null).length;
+ if (
+ (supplied !== 0 && supplied !== 3) ||
+ (supplied === 3 &&
+ (claimKey!.length === 0 ||
+ claimKey!.length > 1024 ||
+ claimToken!.length === 0 ||
+ claimToken!.length > 128 ||
+ !CLIENT_REQUEST_ID_PATTERN.test(clientRequestId!)))
+ ) {
+ throw new Error('Invalid generation job idempotency arguments');
+ }
+}
/**
* Atomic compare-and-set on the job hash — the single-winner decision for a
@@ -31,9 +67,11 @@ import { instrumentIORedisClient, RedisUseCases } from '~/cache/redisTelemetry';
* `pendingActionId` and `createdAt` fields — so a stale decision targeting a
* different action or replacement epoch loses. On success: removes `clear`
* fields, writes `status`+patch pairs, refreshes the job-hash TTL, and performs
- * terminal cleanup of same-slot stream state. Returns 1 if it fired, 0 otherwise.
+ * terminal cleanup of same-slot stream state. Returns 1 if it fired, 0 otherwise;
+ * the abort-only mode returns the JSON-encoded drained items on success.
*
- * KEYS: [job, eventSequence, chunks, runSteps, steers, parkedSteers, generationEpoch]
+ * KEYS: [job, eventSequence, chunks, runSteps, steers, claimedSteers,
+ * parkedSteers, generationEpoch, steerReceipts, steerReceiptOrder]
* ARGV: [
* from,
* expectActionId | "",
@@ -44,6 +82,8 @@ import { instrumentIORedisClient, RedisUseCases } from '~/cache/redisTelemetry';
* runStepsAfterComplete,
* parkedSteersTtl,
* generationEpochGraceTtl,
+ * steerReceiptTtl (0 to leave unchanged),
+ * returnDrainedSteers,
* hdelCount,
* ...hdelFields,
* ...hsetPairs
@@ -60,44 +100,107 @@ const JOB_CAS_LUA =
'local runStepsTtl = tonumber(ARGV[7]) ' +
'local parkedTtl = tonumber(ARGV[8]) ' +
'local generationEpochGraceTtl = tonumber(ARGV[9]) ' +
- 'local hdelCount = tonumber(ARGV[10]) ' +
- 'local idx = 11 ' +
+ 'local receiptTtl = tonumber(ARGV[10]) ' +
+ 'local ownerUserId = redis.call("HGET", KEYS[1], "userId") ' +
+ 'local ownerTenantId = redis.call("HGET", KEYS[1], "tenantId") ' +
+ 'local generationProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" and 2 or 1 ' +
+ 'local parkedProtocol = generationProtocol ' +
+ 'local function isDenseArray(value) if type(value) ~= "table" then return false end ' +
+ 'local count = 0 for key, _ in pairs(value) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return false end count = count + 1 end ' +
+ 'return count == #value end ' +
+ // A terminal transition is destructive. Validate every recovery source
+ // before changing status, receipts, queues, or parked state.
+ 'local validatedPrior = {} ' +
+ 'if terminal then ' +
+ 'local claimedRows = {} if generationProtocol == 2 then claimedRows = redis.call("LRANGE", KEYS[6], 0, -1) end ' +
+ 'local sources = { claimedRows, redis.call("LRANGE", KEYS[5], 0, -1) } ' +
+ 'for s = 1, #sources do for i = 1, #sources[s] do local ok, item = pcall(cjson.decode, sources[s][i]) ' +
+ 'if not ok or type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" then return 0 end end end ' +
+ 'local parkedRaw = redis.call("GET", KEYS[7]) ' +
+ 'if parkedRaw then local ok, parked = pcall(cjson.decode, parkedRaw) ' +
+ 'if not ok or type(parked) ~= "table" or type(parked.userId) ~= "string" or parked.userId == "" ' +
+ 'or not isDenseArray(parked.steers) or #parked.steers == 0 or parked.userId ~= ownerUserId ' +
+ 'or (parked.tenantId and parked.tenantId ~= ownerTenantId) then return 0 end ' +
+ 'if parked.generationProtocolVersion and parked.generationProtocolVersion ~= 1 ' +
+ 'and parked.generationProtocolVersion ~= 2 then return 0 end ' +
+ 'if parked.generationProtocolVersion == 2 then parkedProtocol = 2 end ' +
+ 'for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" ' +
+ 'or (item.clientSteerId and (type(item.clientSteerId) ~= "string" or item.clientSteerId == "")) ' +
+ 'or (item.text and type(item.text) ~= "string") ' +
+ 'or (item.createdAt and (type(item.createdAt) ~= "number" or item.createdAt < 0)) ' +
+ 'or (item.recoveringCreatedAt and (type(item.recoveringCreatedAt) ~= "number" or item.recoveringCreatedAt < 0)) then return 0 end ' +
+ 'validatedPrior[#validatedPrior + 1] = item end end end ' +
+ 'local hdelCount = tonumber(ARGV[12]) ' +
+ 'local idx = 13 ' +
'for i = 1, hdelCount do redis.call("HDEL", KEYS[1], ARGV[idx]) idx = idx + 1 end ' +
'local hset = {} ' +
'for i = idx, #ARGV do hset[#hset + 1] = ARGV[i] end ' +
'if #hset > 0 then redis.call("HSET", KEYS[1], unpack(hset)) end ' +
- 'local ownerUserId = redis.call("HGET", KEYS[1], "userId") ' +
- 'local ownerTenantId = redis.call("HGET", KEYS[1], "tenantId") ' +
+ 'if terminal then redis.call("HSET", KEYS[1], "steersClosed", "1") end ' +
+ // A same-status pause-barrier release does not carry pendingAction again.
+ // Preserve an explicit approval window that is longer than the default
+ // requires_action TTL instead of shortening the live job and its content.
+ 'if not terminal and redis.call("HGET", KEYS[1], "status") == "requires_action" then ' +
+ 'local currentTtl = redis.call("TTL", KEYS[1]) ' +
+ 'if currentTtl > ttl then ttl = currentTtl end end ' +
'redis.call("EXPIRE", KEYS[1], ttl) ' +
+ 'local effectiveReceiptTtl = receiptTtl ' +
+ 'if not terminal and ttl > effectiveReceiptTtl then effectiveReceiptTtl = ttl end ' +
+ 'if terminal and parkedTtl > effectiveReceiptTtl then effectiveReceiptTtl = parkedTtl end ' +
+ 'if not terminal and redis.call("HGET", KEYS[1], "recoveredSteerId") then ' +
+ 'local recoveryLeaseTtl = ttl + parkedTtl ' +
+ 'if recoveryLeaseTtl > effectiveReceiptTtl then effectiveReceiptTtl = recoveryLeaseTtl end ' +
+ 'local pt = redis.call("TTL", KEYS[7]) ' +
+ 'if pt >= 0 and pt < recoveryLeaseTtl then redis.call("EXPIRE", KEYS[7], recoveryLeaseTtl) end end ' +
+ 'if effectiveReceiptTtl > 0 then ' +
+ 'for i = 9, 10 do local rt = redis.call("TTL", KEYS[i]) ' +
+ 'if rt >= 0 and rt < effectiveReceiptTtl then redis.call("EXPIRE", KEYS[i], effectiveReceiptTtl) end end ' +
+ 'end ' +
'local seqTtl = redis.call("TTL", KEYS[2]) ' +
'if seqTtl >= 0 and seqTtl < ttl then redis.call("EXPIRE", KEYS[2], ttl) end ' +
- 'if currentCreatedAt then redis.call("SET", KEYS[7], currentCreatedAt, "EX", ttl + generationEpochGraceTtl) end ' +
+ 'if currentCreatedAt then redis.call("SET", KEYS[8], currentCreatedAt, "EX", ttl + generationEpochGraceTtl) end ' +
'if terminal then ' +
- 'local queued = redis.call("LRANGE", KEYS[5], 0, -1) ' +
- 'if #queued > 0 then ' +
- 'local steers = {} ' +
- 'for i = 1, #queued do ' +
- 'local decoded, item = pcall(cjson.decode, queued[i]) ' +
- 'if decoded and type(item) == "table" then ' +
- 'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
- 'if item.files then projected.files = item.files end ' +
- 'if item.preempt then projected.preempt = item.preempt end ' +
- 'steers[#steers + 1] = projected ' +
- 'end ' +
+ 'local items = {} local projected = {} local seen = {} ' +
+ 'local claimedRows = {} if generationProtocol == 2 then claimedRows = redis.call("LRANGE", KEYS[6], 0, -1) end ' +
+ 'local sources = { claimedRows, redis.call("LRANGE", KEYS[5], 0, -1) } ' +
+ 'for s = 1, #sources do for i = 1, #sources[s] do ' +
+ 'local decoded, item = pcall(cjson.decode, sources[s][i]) ' +
+ 'if decoded and type(item) == "table" and item.steerId and not seen[item.steerId] then ' +
+ 'seen[item.steerId] = true ' +
+ 'items[#items + 1] = item ' +
+ 'local clientItem = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
+ 'if item.clientSteerId then clientItem.clientSteerId = item.clientSteerId end ' +
+ 'if item.files then clientItem.files = item.files end ' +
+ 'if item.preempt then clientItem.preempt = item.preempt end ' +
+ 'if item.preemptRevision then clientItem.preemptRevision = item.preemptRevision end ' +
+ 'projected[#projected + 1] = clientItem ' +
+ 'if generationProtocol == 2 and item.clientSteerId then local raw = redis.call("HGET", KEYS[9], item.clientSteerId) ' +
+ 'if raw then local receiptOk, receipt = pcall(cjson.decode, raw) ' +
+ 'if receiptOk and type(receipt) == "table" then receipt.item = item receipt.state = "leftover" ' +
+ 'redis.call("HSET", KEYS[9], item.clientSteerId, cjson.encode(receipt)) end end end ' +
'end ' +
- 'if #steers > 0 then ' +
- 'local parked = { userId = ownerUserId, steers = steers } ' +
+ 'end end ' +
+ 'if #projected > 0 and ownerUserId then ' +
+ 'local merged = {} local parkedSeen = {} ' +
+ 'for i = 1, #validatedPrior do local item = validatedPrior[i] if not parkedSeen[item.steerId] then ' +
+ 'parkedSeen[item.steerId] = true merged[#merged + 1] = item end end ' +
+ 'for i = 1, #projected do local item = projected[i] if item.steerId and not parkedSeen[item.steerId] then ' +
+ 'parkedSeen[item.steerId] = true merged[#merged + 1] = item end end ' +
+ 'local parked = { userId = ownerUserId, generationProtocolVersion = parkedProtocol, steers = merged } ' +
'if ownerTenantId then parked.tenantId = ownerTenantId end ' +
- 'redis.call("SET", KEYS[6], cjson.encode(parked), "EX", parkedTtl) ' +
+ 'redis.call("SET", KEYS[7], cjson.encode(parked), "EX", parkedTtl) ' +
'end ' +
- 'end ' +
- 'redis.call("DEL", KEYS[5]) ' +
+ 'redis.call("DEL", KEYS[5], KEYS[6]) ' +
'if chunksTtl == 0 then redis.call("DEL", KEYS[3]) else redis.call("EXPIRE", KEYS[3], chunksTtl) end ' +
'if runStepsTtl == 0 then redis.call("DEL", KEYS[4]) else redis.call("EXPIRE", KEYS[4], runStepsTtl) end ' +
+ 'if ARGV[11] == "1" then if #items == 0 then return "[]" end return cjson.encode(items) end ' +
'else ' +
'redis.call("EXPIRE", KEYS[3], ttl) ' +
'redis.call("EXPIRE", KEYS[4], ttl) ' +
'redis.call("EXPIRE", KEYS[5], ttl) ' +
+ 'redis.call("EXPIRE", KEYS[6], ttl) ' +
'end ' +
'return 1';
@@ -113,42 +216,302 @@ const IDEMPOTENCY_CLAIM_LUA =
'if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", tonumber(ARGV[2])) then return false end ' +
'return redis.call("GET", KEYS[1])';
+const IDEMPOTENCY_TAKEOVER_LUA =
+ 'local raw = redis.call("GET", KEYS[1]) if not raw then return 0 end ' +
+ 'local ok, current = pcall(cjson.decode, raw) ' +
+ 'if not ok or current.claimToken ~= ARGV[1] or current.startedAt then return 0 end ' +
+ 'redis.call("SET", KEYS[1], ARGV[2], "PX", tonumber(ARGV[3])) return 1';
+
+const IDEMPOTENCY_MARK_STARTED_LUA =
+ 'if ARGV[1] == "" then return 0 end ' +
+ 'local raw = redis.call("GET", KEYS[1]) if not raw then return 0 end ' +
+ 'local ok, current = pcall(cjson.decode, raw) ' +
+ 'if not ok or current.claimToken ~= ARGV[1] then return 0 end ' +
+ 'if current.startedAt and tostring(current.startedAt) ~= ARGV[2] then return 0 end ' +
+ 'current.startedAt = tonumber(ARGV[2]) ' +
+ 'redis.call("SET", KEYS[1], cjson.encode(current), "PX", tonumber(ARGV[3])) return 1';
+
+/** Reacquire an expired claim without replacing the live generation it
+ * already started. Both keys carry the stream hash tag, so validation and the
+ * started tombstone are one atomic cluster-safe decision. */
+const IDEMPOTENCY_ADOPT_LIVE_JOB_LUA =
+ 'local raw = redis.call("GET", KEYS[1]) if not raw then return 0 end ' +
+ 'local ok, claim = pcall(cjson.decode, raw) ' +
+ 'if not ok or claim.claimToken ~= ARGV[1] or claim.startedAt then return 0 end ' +
+ 'if redis.call("HGET", KEYS[2], "createdAt") ~= ARGV[2] ' +
+ 'or redis.call("HGET", KEYS[2], "userId") ~= ARGV[3] then return 0 end ' +
+ 'local storedRequestId = redis.call("HGET", KEYS[2], "idempotencyClientRequestId") ' +
+ 'if storedRequestId then if storedRequestId ~= ARGV[4] then return 0 end ' +
+ 'elseif ARGV[7] ~= "1" then return 0 end ' +
+ 'local storedTenantId = redis.call("HGET", KEYS[2], "tenantId") ' +
+ 'if storedTenantId and storedTenantId ~= "" and storedTenantId ~= ARGV[5] then return 0 end ' +
+ 'local status = redis.call("HGET", KEYS[2], "status") ' +
+ 'if status ~= "running" and status ~= "requires_action" then return 0 end ' +
+ 'local protocol = redis.call("HGET", KEYS[2], "generationProtocolVersion") ' +
+ 'if protocol and protocol ~= "1" and protocol ~= "2" then return 0 end ' +
+ 'claim.startedAt = tonumber(ARGV[2]) ' +
+ 'claim.generationProtocolVersion = protocol == "2" and 2 or 1 ' +
+ 'redis.call("SET", KEYS[1], cjson.encode(claim), "PX", tonumber(ARGV[6])) return 1';
+
+const IDEMPOTENCY_RELEASE_LUA =
+ 'if ARGV[1] ~= "" then local raw = redis.call("GET", KEYS[1]) if not raw then return 0 end ' +
+ 'local ok, current = pcall(cjson.decode, raw) if not ok or current.claimToken ~= ARGV[1] then return 0 end end ' +
+ 'return redis.call("DEL", KEYS[1])';
+
+/** Attempt-fenced acknowledgement of transaction-time predecessor receipts.
+ * A replacement either inherits the untrimmed chain before this script or
+ * changes the attempt id first and makes this late acknowledgement a no-op. */
+const REPLACEMENT_RECEIPT_ACK_LUA =
+ 'if redis.call("HGET", KEYS[1], "__creationAttemptId") ~= ARGV[1] then return 0 end ' +
+ 'local acknowledged = {} for i = 2, #ARGV do acknowledged[ARGV[i]] = true end ' +
+ 'local raw = redis.call("HGET", KEYS[1], "__replacedGenerations") ' +
+ 'if raw then local ok, current = pcall(cjson.decode, raw) ' +
+ 'if not ok or type(current) ~= "table" then return 0 end ' +
+ 'local retained = {} for i = 1, #current do local item = current[i] ' +
+ 'if type(item) ~= "table" or type(item.createdAt) ~= "number" then return 0 end ' +
+ 'if not acknowledged[tostring(item.createdAt)] then retained[#retained + 1] = item end end ' +
+ 'if #retained == 0 then ' +
+ 'redis.call("HDEL", KEYS[1], "__replacedGenerations", "__replacedCreatedAt", "__replacedStatus", "__replacedConversationId") ' +
+ 'else local latest = retained[#retained] ' +
+ 'redis.call("HSET", KEYS[1], "__replacedGenerations", cjson.encode(retained), ' +
+ '"__replacedCreatedAt", tostring(latest.createdAt), "__replacedStatus", latest.status) ' +
+ 'if latest.conversationId then redis.call("HSET", KEYS[1], "__replacedConversationId", latest.conversationId) ' +
+ 'else redis.call("HDEL", KEYS[1], "__replacedConversationId") end end return 1 end ' +
+ 'local immediate = redis.call("HGET", KEYS[1], "__replacedCreatedAt") ' +
+ 'if immediate and acknowledged[immediate] then ' +
+ 'redis.call("HDEL", KEYS[1], "__replacedCreatedAt", "__replacedStatus", "__replacedConversationId") end ' +
+ 'return 1';
+
/**
* Atomic job (re)creation for all generation-scoped same-slot keys. The
- * predecessor hash, content, run steps, steer queue, and parked steers are
- * removed in the same script that installs the replacement hash. A reconnect
- * that observes the replacement can therefore never reconstruct predecessor
- * state, even when the predecessor's delayed completion loses its epoch guard.
+ * predecessor hash/content are removed in the same script that installs the
+ * replacement hash. Pending and claimed steers are first merged into the
+ * owner-scoped parked recovery payload, so replacement cannot discard an ACK.
*
- * KEYS: [job, chunks, runSteps, steers, parkedSteers, generationEpoch]
- * ARGV: [ttl, requestedCreatedAt, generationEpochGraceTtl, ...hsetPairs]
- * Returns: [previousUserId | "", previousTenantId | "", createdAt]
+ * KEYS: [job, chunks, runSteps, steers, claimedSteers, parkedSteers,
+ * generationEpoch, steerReceipts, steerReceiptOrder, idempotencyClaim]
+ * ARGV: [ttl, requestedCreatedAt, generationEpochGraceTtl, parkedTtl,
+ * recoveredSteerId | "", newOwnerUserId, newOwnerTenantId | "",
+ * idempotencyClaimToken | "",
+ * recoveredSteerPayloadJson | "",
+ * generationProtocolVersion,
+ * creationAttemptId | "",
+ * expectedPredecessorCreatedAt | "",
+ * ...hsetPairs]
+ * Returns: [previousUserId | "", previousTenantId | "", createdAt, "",
+ * replacedCreatedAt | "", replacedStatus | "", replacedConversationId | "",
+ * replacedProviderAbortReady | ""]
+ * Predecessor mismatch returns the latest retained epoch in the replaced
+ * position plus an eighth active flag and ninth verified flag ("1" | "0").
+ * Job-only metadata is empty when that epoch has outlived its hash. When all
+ * evidence expired, the finite expected epoch is echoed with verified=false.
*/
const JOB_CREATE_LUA =
+ 'if ARGV[8] ~= "" then local claimRaw = redis.call("GET", KEYS[10]) ' +
+ 'if not claimRaw then return { "", "", "0", "claim_lost" } end ' +
+ 'local ok, claim = pcall(cjson.decode, claimRaw) ' +
+ 'if not ok or claim.claimToken ~= ARGV[8] or claim.startedAt then ' +
+ 'return { "", "", "0", "claim_lost" } end end ' +
+ 'local previousJobExists = redis.call("EXISTS", KEYS[1]) ' +
'local previousUserId = redis.call("HGET", KEYS[1], "userId") ' +
'local previousTenantId = redis.call("HGET", KEYS[1], "tenantId") ' +
- 'local previousCreatedAt = tonumber(redis.call("HGET", KEYS[1], "createdAt")) ' +
- 'local retainedEpoch = tonumber(redis.call("GET", KEYS[6])) ' +
- 'if retainedEpoch and (not previousCreatedAt or retainedEpoch > previousCreatedAt) then previousCreatedAt = retainedEpoch end ' +
+ 'if previousJobExists == 1 and (not previousUserId or previousUserId == "" or previousUserId ~= ARGV[6] ' +
+ 'or (previousTenantId and previousTenantId ~= ARGV[7])) then ' +
+ 'return { "", "", "0", "owner_mismatch" } end ' +
+ 'local replacedCreatedAt = redis.call("HGET", KEYS[1], "createdAt") ' +
+ 'local replacedStatus = redis.call("HGET", KEYS[1], "status") ' +
+ 'local replacedConversationId = redis.call("HGET", KEYS[1], "conversationId") ' +
+ 'local replacedProviderAbortReady = redis.call("HGET", KEYS[1], "providerAbortReady") ' +
+ 'local replacedProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") ' +
+ 'local MAX_SAFE_EPOCH = 9007199254740991 ' +
+ 'local function isSafeEpoch(value) return type(value) == "number" and value >= 0 ' +
+ 'and value <= MAX_SAFE_EPOCH and value == math.floor(value) end ' +
+ 'local function isValidJobStatus(value) return value == "running" or value == "requires_action" ' +
+ 'or value == "complete" or value == "error" or value == "aborted" end ' +
+ 'local replacedEpoch = tonumber(replacedCreatedAt) local previousCreatedAt = replacedEpoch ' +
+ 'if previousJobExists == 1 and (not isSafeEpoch(replacedEpoch) or not isValidJobStatus(replacedStatus)) then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'local retainedEpochRaw = redis.call("GET", KEYS[7]) local retainedEpoch = tonumber(retainedEpochRaw) ' +
+ 'if retainedEpochRaw and not isSafeEpoch(retainedEpoch) then ' +
+ 'return { "", "", "0", "generation_epoch_corrupt" } end ' +
+ 'local observedCreatedAt = replacedCreatedAt local observedStatus = replacedStatus ' +
+ 'local observedConversationId = replacedConversationId ' +
+ 'local observedActive = previousJobExists == 1 and ' +
+ '(replacedStatus == "running" or replacedStatus == "requires_action") ' +
+ 'if retainedEpoch and (not previousCreatedAt or retainedEpoch > previousCreatedAt) then ' +
+ 'previousCreatedAt = retainedEpoch observedCreatedAt = retainedEpochRaw ' +
+ 'observedStatus = nil observedConversationId = nil observedActive = false end ' +
+ 'if ARGV[12] ~= "" and (not observedCreatedAt or observedCreatedAt ~= ARGV[12]) then ' +
+ 'return { previousUserId or "", previousTenantId or "", "0", "predecessor_mismatch", ' +
+ 'observedCreatedAt or ARGV[12], observedStatus or "", observedConversationId or "", ' +
+ 'observedActive and "1" or "0", observedCreatedAt and "1" or "0" } end ' +
'local createdAt = tonumber(ARGV[2]) ' +
+ 'if not isSafeEpoch(createdAt) then return { "", "", "0", "generation_epoch_corrupt" } end ' +
+ 'if previousCreatedAt and previousCreatedAt >= MAX_SAFE_EPOCH then ' +
+ 'return { "", "", "0", "generation_epoch_exhausted" } end ' +
'if previousCreatedAt and previousCreatedAt >= createdAt then createdAt = previousCreatedAt + 1 end ' +
+ 'local merged = {} local seen = {} local parkedUserId = previousUserId local parkedTenantId = previousTenantId ' +
+ 'local parkedProtocol = replacedProtocol == "2" and 2 or 1 ' +
+ 'local function isDenseArray(value) if type(value) ~= "table" then return false end ' +
+ 'local count = 0 for key, _ in pairs(value) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return false end count = count + 1 end ' +
+ 'return count == #value end ' +
+ // Carry every still-unacknowledged transaction-time predecessor forward.
+ // A later replacement can then stop providers skipped when an earlier
+ // create reply was lost. Reject before mutation rather than evicting an old
+ // active epoch when the bounded receipt chain is full.
+ 'local replacementChain = {} local replacementSeen = {} local lastReplacementEpoch = -1 ' +
+ 'local replacementRaw = redis.call("HGET", KEYS[1], "__replacedGenerations") ' +
+ 'local inheritedCreatedAtRaw = redis.call("HGET", KEYS[1], "__replacedCreatedAt") ' +
+ 'local inheritedStatus = redis.call("HGET", KEYS[1], "__replacedStatus") ' +
+ 'local inheritedConversationId = redis.call("HGET", KEYS[1], "__replacedConversationId") ' +
+ 'if (inheritedCreatedAtRaw and not inheritedStatus) or (inheritedStatus and not inheritedCreatedAtRaw) then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'if replacementRaw then local chainOk, inherited = pcall(cjson.decode, replacementRaw) ' +
+ 'if not chainOk or not isDenseArray(inherited) or #inherited == 0 or #inherited > 32 then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'for i = 1, #inherited do local item = inherited[i] ' +
+ 'local validStatus = type(item) == "table" and isValidJobStatus(item.status) ' +
+ 'if not validStatus or not isSafeEpoch(item.createdAt) ' +
+ 'or item.createdAt <= lastReplacementEpoch ' +
+ 'or (previousJobExists == 1 and item.createdAt >= replacedEpoch) ' +
+ 'or (item.conversationId and type(item.conversationId) ~= "string") ' +
+ 'or (item.providerAbortReady ~= nil and type(item.providerAbortReady) ~= "boolean") ' +
+ 'or replacementSeen[tostring(item.createdAt)] then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'lastReplacementEpoch = item.createdAt replacementSeen[tostring(item.createdAt)] = true ' +
+ 'replacementChain[#replacementChain + 1] = item end ' +
+ 'if not inheritedCreatedAtRaw then return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'local inheritedEpoch = tonumber(inheritedCreatedAtRaw) local latest = replacementChain[#replacementChain] ' +
+ 'if not isSafeEpoch(inheritedEpoch) or not isValidJobStatus(inheritedStatus) ' +
+ 'or latest.createdAt ~= inheritedEpoch or latest.status ~= inheritedStatus ' +
+ 'or (latest.conversationId or false) ~= inheritedConversationId then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'elseif inheritedCreatedAtRaw then local inheritedEpoch = tonumber(inheritedCreatedAtRaw) ' +
+ 'if not isSafeEpoch(inheritedEpoch) or not isValidJobStatus(inheritedStatus) ' +
+ 'or previousJobExists ~= 1 or inheritedEpoch >= replacedEpoch then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'local inherited = { createdAt = inheritedEpoch, status = inheritedStatus } ' +
+ 'if inheritedConversationId then inherited.conversationId = inheritedConversationId end ' +
+ 'replacementSeen[tostring(inheritedEpoch)] = true replacementChain[1] = inherited end ' +
+ 'if previousJobExists == 1 then ' +
+ 'if replacementSeen[tostring(replacedEpoch)] then ' +
+ 'return { "", "", "0", "replacement_receipt_corrupt" } end ' +
+ 'if #replacementChain >= 32 then return { "", "", "0", "replacement_chain_full" } end ' +
+ 'local replaced = { createdAt = replacedEpoch, status = replacedStatus } ' +
+ 'if replacedConversationId then replaced.conversationId = replacedConversationId end ' +
+ 'if replacedProviderAbortReady then replaced.providerAbortReady = replacedProviderAbortReady == "1" end ' +
+ 'replacementChain[#replacementChain + 1] = replaced replacementSeen[tostring(replacedEpoch)] = true end ' +
+ 'local recoveredSteerId = ARGV[5] local expectedRecovery = nil ' +
+ 'if recoveredSteerId ~= "" and ARGV[10] ~= "2" then return { "", "", "0", "recovery_payload_mismatch" } end ' +
+ 'if recoveredSteerId ~= "" then local ok, decoded = pcall(cjson.decode, ARGV[9]) ' +
+ 'if not ok or type(decoded) ~= "table" or type(decoded.text) ~= "string" ' +
+ 'or not isDenseArray(decoded.fileIds) then return { "", "", "0", "recovery_payload_mismatch" } end ' +
+ 'local expectedSeen = {} for i = 1, #decoded.fileIds do local fileId = decoded.fileIds[i] ' +
+ 'if type(fileId) ~= "string" or fileId == "" or expectedSeen[fileId] then ' +
+ 'return { "", "", "0", "recovery_payload_mismatch" } end expectedSeen[fileId] = true end ' +
+ 'expectedRecovery = decoded elseif ARGV[9] ~= "" then ' +
+ 'return { "", "", "0", "recovery_payload_mismatch" } end ' +
+ 'local function recoveryMatches(item, expected) ' +
+ 'if not expected or type(item.text) ~= "string" or item.text ~= expected.text then return false end ' +
+ 'local actualSeen = {} local actualCount = 0 local files = item.files ' +
+ 'if files then if not isDenseArray(files) then return false end ' +
+ 'for i = 1, #files do local file = files[i] ' +
+ 'if type(file) ~= "table" or type(file.file_id) ~= "string" or file.file_id == "" then return false end ' +
+ 'if not actualSeen[file.file_id] then actualSeen[file.file_id] = true actualCount = actualCount + 1 end end end ' +
+ 'if actualCount ~= #expected.fileIds then return false end ' +
+ 'for i = 1, #expected.fileIds do if not actualSeen[expected.fileIds[i]] then return false end end return true end ' +
+ 'local parkedRaw = redis.call("GET", KEYS[6]) ' +
+ 'if parkedRaw then local ok, parked = pcall(cjson.decode, parkedRaw) ' +
+ 'if not ok or type(parked) ~= "table" or type(parked.userId) ~= "string" ' +
+ 'or parked.userId == "" or not isDenseArray(parked.steers) or #parked.steers == 0 then ' +
+ 'return { "", "", "0", "recovery_corrupt" } end ' +
+ 'if parked.generationProtocolVersion and parked.generationProtocolVersion ~= 1 ' +
+ 'and parked.generationProtocolVersion ~= 2 then return { "", "", "0", "recovery_corrupt" } end ' +
+ 'if parked.generationProtocolVersion == 2 then parkedProtocol = 2 end ' +
+ 'for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" ' +
+ 'or (item.clientSteerId and (type(item.clientSteerId) ~= "string" or item.clientSteerId == "")) ' +
+ 'or (item.text and type(item.text) ~= "string") ' +
+ 'or (item.createdAt and (type(item.createdAt) ~= "number" or item.createdAt < 0)) ' +
+ 'or (item.recoveringCreatedAt and (type(item.recoveringCreatedAt) ~= "number" or item.recoveringCreatedAt < 0)) then ' +
+ 'return { "", "", "0", "recovery_corrupt" } end end ' +
+ 'if recoveredSteerId ~= "" and parked.generationProtocolVersion ~= 2 then ' +
+ 'for i = 1, #parked.steers do if parked.steers[i].steerId == recoveredSteerId then ' +
+ 'return { "", "", "0", "recovery_payload_mismatch" } end end end ' +
+ 'if parked.userId ~= ARGV[6] or (parked.tenantId and parked.tenantId ~= ARGV[7]) then ' +
+ 'return { "", "", "0", "owner_mismatch" } end ' +
+ 'parkedUserId = parked.userId parkedTenantId = parked.tenantId ' +
+ 'for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if item.steerId and not seen[item.steerId] then seen[item.steerId] = true merged[#merged + 1] = item end end end ' +
+ 'local receiptUpdates = {} local ownerMatches = not parkedUserId or not previousUserId or ' +
+ '(parkedUserId == previousUserId and (not parkedTenantId or parkedTenantId == previousTenantId)) ' +
+ 'if ownerMatches then if not parkedUserId then parkedUserId = previousUserId parkedTenantId = previousTenantId end ' +
+ 'local claimedRows = {} if replacedProtocol == "2" then claimedRows = redis.call("LRANGE", KEYS[5], 0, -1) end ' +
+ 'local sources = { claimedRows, redis.call("LRANGE", KEYS[4], 0, -1) } ' +
+ 'for s = 1, #sources do for i = 1, #sources[s] do local ok, item = pcall(cjson.decode, sources[s][i]) ' +
+ 'if ok and recoveredSteerId ~= "" and item.steerId == recoveredSteerId and replacedProtocol ~= "2" then ' +
+ 'return { "", "", "0", "recovery_payload_mismatch" } end ' +
+ 'if ok and item.steerId and not seen[item.steerId] then seen[item.steerId] = true ' +
+ 'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
+ 'if item.clientSteerId then projected.clientSteerId = item.clientSteerId end ' +
+ 'if item.files then projected.files = item.files end if item.preempt then projected.preempt = item.preempt end ' +
+ 'if item.preemptRevision then projected.preemptRevision = item.preemptRevision end ' +
+ 'merged[#merged + 1] = projected receiptUpdates[#receiptUpdates + 1] = item end end end end ' +
+ 'local recoveryOwnerMatches = parkedUserId == ARGV[6] and ' +
+ '(not parkedTenantId or parkedTenantId == ARGV[7]) ' +
+ 'local recoveryFound = recoveredSteerId == "" ' +
+ 'for i = 1, #merged do local item = merged[i] ' +
+ 'item.recoveringCreatedAt = nil ' +
+ 'if recoveredSteerId ~= "" and item.steerId == recoveredSteerId then ' +
+ 'if not recoveryOwnerMatches or not recoveryMatches(item, expectedRecovery) then ' +
+ 'return { "", "", "0", "recovery_payload_mismatch" } end ' +
+ 'item.recoveringCreatedAt = createdAt recoveryFound = true end end ' +
+ 'if not recoveryFound then return { "", "", "0", "recovery_payload_mismatch" } end ' +
+ 'for i = 1, #receiptUpdates do local item = receiptUpdates[i] ' +
+ 'if replacedProtocol == "2" and item.clientSteerId then local raw = redis.call("HGET", KEYS[8], item.clientSteerId) ' +
+ 'if raw then local receiptOk, receipt = pcall(cjson.decode, raw) ' +
+ 'if receiptOk and type(receipt) == "table" then receipt.item = item receipt.state = "leftover" ' +
+ 'redis.call("HSET", KEYS[8], item.clientSteerId, cjson.encode(receipt)) end end end end ' +
+ 'if #merged > 0 and parkedUserId then local parked = { userId = parkedUserId, generationProtocolVersion = parkedProtocol, steers = merged } ' +
+ 'if parkedTenantId then parked.tenantId = parkedTenantId end ' +
+ 'redis.call("SET", KEYS[6], cjson.encode(parked), "EX", ARGV[4]) else redis.call("DEL", KEYS[6]) end ' +
+ 'if #merged > 0 then for i = 8, 9 do local rt = redis.call("TTL", KEYS[i]) ' +
+ 'if rt >= 0 and rt < tonumber(ARGV[4]) then redis.call("EXPIRE", KEYS[i], ARGV[4]) end end end ' +
'redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5]) ' +
'local ttl = tonumber(ARGV[1]) ' +
'local generationEpochGraceTtl = tonumber(ARGV[3]) ' +
'local hset = {} ' +
- 'for i = 4, #ARGV do hset[#hset + 1] = ARGV[i] end ' +
+ 'for i = 13, #ARGV do hset[#hset + 1] = ARGV[i] end ' +
'redis.call("HSET", KEYS[1], unpack(hset)) ' +
'redis.call("HSET", KEYS[1], "createdAt", tostring(createdAt)) ' +
+ 'if ARGV[11] ~= "" then redis.call("HSET", KEYS[1], "__creationAttemptId", ARGV[11]) end ' +
+ // Keep the transaction-time predecessor receipt in the replacement hash.
+ // deserializeJob reconstructs it as non-enumerable return metadata, so an
+ // eval reply lost after commit can still stop the exact replaced provider.
+ 'if replacedCreatedAt and replacedStatus then ' +
+ 'redis.call("HSET", KEYS[1], "__replacedCreatedAt", replacedCreatedAt, "__replacedStatus", replacedStatus) ' +
+ 'if replacedConversationId then redis.call("HSET", KEYS[1], "__replacedConversationId", replacedConversationId) end end ' +
+ 'if #replacementChain > 0 then redis.call("HSET", KEYS[1], "__replacedGenerations", cjson.encode(replacementChain)) end ' +
+ 'if ARGV[10] == "2" then redis.call("HSET", KEYS[1], "checkpointNamespace", tostring(createdAt)) ' +
+ 'else redis.call("HDEL", KEYS[1], "checkpointNamespace") end ' +
'redis.call("EXPIRE", KEYS[1], ttl) ' +
- 'redis.call("SET", KEYS[6], tostring(createdAt), "EX", ttl + generationEpochGraceTtl) ' +
- 'return { previousUserId or "", previousTenantId or "", tostring(createdAt) }';
+ 'redis.call("SET", KEYS[7], tostring(createdAt), "EX", ttl + generationEpochGraceTtl) ' +
+ 'if ARGV[8] ~= "" then local claimRaw = redis.call("GET", KEYS[10]) ' +
+ 'local claimTtl = redis.call("PTTL", KEYS[10]) local ok, claim = pcall(cjson.decode, claimRaw) ' +
+ 'if not ok or claim.claimToken ~= ARGV[8] then return { "", "", "0", "claim_lost" } end ' +
+ 'claim.startedAt = createdAt redis.call("SET", KEYS[10], cjson.encode(claim)) ' +
+ 'if claimTtl > 0 then redis.call("PEXPIRE", KEYS[10], claimTtl) end end ' +
+ 'return { previousUserId or "", previousTenantId or "", tostring(createdAt), "", ' +
+ 'replacedCreatedAt or "", replacedStatus or "", replacedConversationId or "", ' +
+ 'replacedProviderAbortReady or "" }';
/**
* Epoch-guarded field update. Terminal writes reclaim same-slot content in the
* same atomic step, so a replacement cannot appear between the guarded write
* and content cleanup.
*
- * KEYS: [job, chunks, runSteps, steers]
+ * KEYS: [job, chunks, runSteps, steers, claimedSteers]
* ARGV: [
* expectCreatedAt | "",
* terminal ("0" | "1"),
@@ -175,6 +538,15 @@ const JOB_UPDATE_LUA =
'end ' +
'return 1';
+/** Single-winner promotion from abort-persistence pending to a consumable
+ * terminal payload. Owner success/failure and stale-owner recovery share this
+ * CAS, so a timeout cannot overwrite a normal FINAL (or vice versa). */
+const TERMINAL_PERSISTENCE_FINALIZE_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "terminalPersistencePending") ~= "1" then return 0 end ' +
+ 'redis.call("HSET", KEYS[1], "terminalPersistencePending", "0", "finalEvent", ARGV[2]) ' +
+ 'return 1';
+
/**
* Epoch-guarded hard deletion. `expectMissing` makes an unguarded cleanup of
* already-absent state safe against a replacement appearing after the read.
@@ -187,7 +559,7 @@ const JOB_DELETE_LUA =
'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
'local existed = redis.call("EXISTS", KEYS[1]) ' +
'if ARGV[2] == "1" and existed == 1 then return 0 end ' +
- 'redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4]) ' +
+ 'redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5]) ' +
'return existed';
/**
@@ -196,7 +568,8 @@ const JOB_DELETE_LUA =
* lands before the script and fails the guard, or lands afterward and clears
* the predecessor's parked payload in {@link JOB_CREATE_LUA}.
*
- * KEYS: [job, chunks, runSteps, steers, parkedSteers, generationEpoch]
+ * KEYS: [job, chunks, runSteps, steers, claimedSteers, parkedSteers,
+ * generationEpoch, steerReceipts, steerReceiptOrder]
* ARGV: [expectCreatedAt, nowMs, staleAfterMs, parkedSteersTtl, generationEpochGraceTtl]
*/
const STALE_JOB_DELETE_LUA =
@@ -207,26 +580,63 @@ const STALE_JOB_DELETE_LUA =
'if not liveSince or tonumber(ARGV[2]) - liveSince <= tonumber(ARGV[3]) then return 0 end ' +
'local ownerUserId = redis.call("HGET", KEYS[1], "userId") ' +
'local ownerTenantId = redis.call("HGET", KEYS[1], "tenantId") ' +
- 'local queued = redis.call("LRANGE", KEYS[4], 0, -1) ' +
- 'if #queued > 0 and ownerUserId then ' +
- 'local steers = {} ' +
- 'for i = 1, #queued do ' +
- 'local decoded, item = pcall(cjson.decode, queued[i]) ' +
+ 'local generationProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" and 2 or 1 ' +
+ 'local parkedProtocol = generationProtocol ' +
+ 'local function isDenseArray(value) if type(value) ~= "table" then return false end ' +
+ 'local count = 0 for key, _ in pairs(value) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return false end count = count + 1 end ' +
+ 'return count == #value end ' +
+ 'local prior = {} local parkedRaw = redis.call("GET", KEYS[6]) ' +
+ 'if parkedRaw then local ok, parked = pcall(cjson.decode, parkedRaw) ' +
+ 'if not ok or type(parked) ~= "table" or type(parked.userId) ~= "string" or parked.userId == "" ' +
+ 'or not isDenseArray(parked.steers) or #parked.steers == 0 or parked.userId ~= ownerUserId ' +
+ 'or (parked.tenantId and parked.tenantId ~= ownerTenantId) then return 0 end ' +
+ 'if parked.generationProtocolVersion and parked.generationProtocolVersion ~= 1 ' +
+ 'and parked.generationProtocolVersion ~= 2 then return 0 end ' +
+ 'if parked.generationProtocolVersion == 2 then parkedProtocol = 2 end ' +
+ 'for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" ' +
+ 'or (item.clientSteerId and (type(item.clientSteerId) ~= "string" or item.clientSteerId == "")) ' +
+ 'or (item.text and type(item.text) ~= "string") ' +
+ 'or (item.createdAt and (type(item.createdAt) ~= "number" or item.createdAt < 0)) ' +
+ 'or (item.recoveringCreatedAt and (type(item.recoveringCreatedAt) ~= "number" or item.recoveringCreatedAt < 0)) then return 0 end ' +
+ 'prior[#prior + 1] = item end end ' +
+ 'local claimedRows = {} if generationProtocol == 2 then claimedRows = redis.call("LRANGE", KEYS[5], 0, -1) end ' +
+ 'local sourceRows = { claimedRows, redis.call("LRANGE", KEYS[4], 0, -1) } ' +
+ 'for s = 1, #sourceRows do for i = 1, #sourceRows[s] do local ok, item = pcall(cjson.decode, sourceRows[s][i]) ' +
+ 'if not ok or type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" then return 0 end end end ' +
+ 'if (#sourceRows[1] > 0 or #sourceRows[2] > 0) and (not ownerUserId or ownerUserId == "") then return 0 end ' +
+ 'local fullItems = {} local projected = {} local seen = {} ' +
+ 'local sources = sourceRows ' +
+ 'for s = 1, #sources do for i = 1, #sources[s] do ' +
+ 'local decoded, item = pcall(cjson.decode, sources[s][i]) ' +
'if decoded and type(item) == "table" then ' +
- 'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
- 'if item.files then projected.files = item.files end ' +
- 'if item.preempt then projected.preempt = item.preempt end ' +
- 'steers[#steers + 1] = projected ' +
+ 'if item.steerId and not seen[item.steerId] then seen[item.steerId] = true fullItems[#fullItems + 1] = item ' +
+ 'local clientItem = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
+ 'if item.clientSteerId then clientItem.clientSteerId = item.clientSteerId end ' +
+ 'if item.files then clientItem.files = item.files end if item.preempt then clientItem.preempt = item.preempt end ' +
+ 'if item.preemptRevision then clientItem.preemptRevision = item.preemptRevision end ' +
+ 'projected[#projected + 1] = clientItem end ' +
+ 'if generationProtocol == 2 and item.clientSteerId then local raw = redis.call("HGET", KEYS[8], item.clientSteerId) ' +
+ 'if raw then local receiptOk, receipt = pcall(cjson.decode, raw) ' +
+ 'if receiptOk and type(receipt) == "table" then receipt.item = item receipt.state = "leftover" ' +
+ 'redis.call("HSET", KEYS[8], item.clientSteerId, cjson.encode(receipt)) end end end ' +
'end ' +
- 'end ' +
- 'if #steers > 0 then ' +
- 'local parked = { userId = ownerUserId, steers = steers } ' +
+ 'end end ' +
+ 'if #projected > 0 and ownerUserId then ' +
+ 'local merged = {} local parkedSeen = {} ' +
+ 'for i = 1, #prior do local item = prior[i] if not parkedSeen[item.steerId] then ' +
+ 'parkedSeen[item.steerId] = true merged[#merged + 1] = item end end ' +
+ 'for i = 1, #projected do local item = projected[i] if item.steerId and not parkedSeen[item.steerId] then ' +
+ 'parkedSeen[item.steerId] = true merged[#merged + 1] = item end end ' +
+ 'local parked = { userId = ownerUserId, generationProtocolVersion = parkedProtocol, steers = merged } ' +
'if ownerTenantId then parked.tenantId = ownerTenantId end ' +
- 'redis.call("SET", KEYS[5], cjson.encode(parked), "EX", tonumber(ARGV[4])) ' +
- 'end ' +
+ 'redis.call("SET", KEYS[6], cjson.encode(parked), "EX", tonumber(ARGV[4])) ' +
'end ' +
- 'redis.call("SET", KEYS[6], ARGV[1], "EX", tonumber(ARGV[5])) ' +
- 'redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4]) ' +
+ 'for i = 8, 9 do local ttl = redis.call("TTL", KEYS[i]) ' +
+ 'if ttl >= 0 and ttl < tonumber(ARGV[4]) then redis.call("EXPIRE", KEYS[i], ARGV[4]) end end ' +
+ 'redis.call("SET", KEYS[7], ARGV[1], "EX", tonumber(ARGV[5])) ' +
+ 'redis.call("DEL", KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5]) ' +
'return 1';
/**
@@ -255,20 +665,77 @@ const STALE_JOB_DELETE_LUA =
* window while paused; for a running job the job key carries the running TTL, so target
* stays `run`.
*
- * KEYS: [chunks, job]
- * ARGV: [eventJson, runningTtl, expectCreatedAt | ""]
+ * KEYS: [chunks, job, steerReceipts, steerReceiptOrder, claimedSteers, steers,
+ * parkedSteers, generationEpoch]
+ * ARGV: [eventJson, runningTtl, expectCreatedAt | "",
+ * deliveredClientSteerId | "", deliveredItemJson | "", nowMs,
+ * parkedSteersTtl, generationEpochGraceTtl]
*/
const CHUNK_APPEND_LUA =
- 'if ARGV[3] ~= "" and redis.call("HGET", KEYS[2], "createdAt") ~= ARGV[3] then return 0 end ' +
- 'redis.call("XADD", KEYS[1], "*", "event", ARGV[1]) ' +
+ 'local currentCreatedAt = redis.call("HGET", KEYS[2], "createdAt") ' +
+ 'if not currentCreatedAt then return 0 end ' +
+ 'if ARGV[3] ~= "" and currentCreatedAt ~= ARGV[3] then return 0 end ' +
+ 'local currentStatus = redis.call("HGET", KEYS[2], "status") ' +
+ 'if currentStatus ~= "running" and currentStatus ~= "requires_action" then return 0 end ' +
+ 'local retainedEpoch = redis.call("GET", KEYS[8]) ' +
+ 'if retainedEpoch and retainedEpoch ~= currentCreatedAt then return 0 end ' +
+ 'local protocolV2 = redis.call("HGET", KEYS[2], "generationProtocolVersion") == "2" ' +
+ 'local delivered = nil local kept = nil local receipt = nil ' +
+ 'if ARGV[5] ~= "" then ' +
+ 'if currentStatus ~= "running" ' +
+ 'or redis.call("HGET", KEYS[2], "steersClosed") == "1" then return 0 end ' +
+ 'local deliveredOk, deliveredValue = pcall(cjson.decode, ARGV[5]) ' +
+ 'if not deliveredOk or type(deliveredValue) ~= "table" or not deliveredValue.steerId then return 0 end ' +
+ 'delivered = deliveredValue ' +
+ 'if protocolV2 and delivered.clientSteerId then ' +
+ 'if ARGV[4] == "" or delivered.clientSteerId ~= ARGV[4] then return 0 end ' +
+ 'local raw = redis.call("HGET", KEYS[3], ARGV[4]) if not raw then return 0 end ' +
+ 'local receiptOk, receiptValue = pcall(cjson.decode, raw) ' +
+ 'if not receiptOk or type(receiptValue) ~= "table" ' +
+ 'or receiptValue.clientSteerId ~= ARGV[4] ' +
+ 'or tostring(receiptValue.generationCreatedAt or "") ~= currentCreatedAt ' +
+ 'or receiptValue.state ~= "claimed" or not receiptValue.item ' +
+ 'or receiptValue.item.steerId ~= delivered.steerId ' +
+ 'or receiptValue.item.clientSteerId ~= ARGV[4] then return 0 end ' +
+ 'receipt = receiptValue ' +
+ 'elseif protocolV2 and ARGV[4] ~= "" then return 0 end ' +
+ 'if protocolV2 then ' +
+ 'local claims = redis.call("LRANGE", KEYS[5], 0, -1) ' +
+ 'kept = {} local found = false ' +
+ 'for i = 1, #claims do local ok, item = pcall(cjson.decode, claims[i]) ' +
+ 'local sameClient = ok and item.clientSteerId == delivered.clientSteerId ' +
+ 'if not found and ok and item.steerId == delivered.steerId and sameClient then found = true ' +
+ 'else kept[#kept + 1] = claims[i] end end ' +
+ 'if not found then return 0 end end ' +
+ 'end ' +
'local run = tonumber(ARGV[2]) ' +
'local target = run ' +
- 'if redis.call("HGET", KEYS[2], "status") == "requires_action" then ' +
- 'local jt = redis.call("TTL", KEYS[2]) ' +
- 'if jt > target then target = jt end ' +
- 'end ' +
+ 'local jobTtl = redis.call("TTL", KEYS[2]) ' +
+ 'if jobTtl < target then redis.call("EXPIRE", KEYS[2], target) ' +
+ 'elseif jobTtl > target then target = jobTtl end ' +
+ 'local recoveryTarget = target ' +
+ 'if redis.call("HGET", KEYS[2], "recoveredSteerId") then ' +
+ 'recoveryTarget = target + tonumber(ARGV[7]) ' +
+ 'local pt = redis.call("TTL", KEYS[7]) ' +
+ 'if pt >= 0 and pt < recoveryTarget then redis.call("EXPIRE", KEYS[7], recoveryTarget) end end ' +
+ 'local epochTarget = target + tonumber(ARGV[8]) ' +
+ 'if retainedEpoch then local epochTtl = redis.call("TTL", KEYS[8]) ' +
+ 'if epochTtl >= 0 and epochTtl < epochTarget then redis.call("EXPIRE", KEYS[8], epochTarget) end ' +
+ 'else redis.call("SET", KEYS[8], currentCreatedAt, "EX", epochTarget) end ' +
+ 'if delivered and protocolV2 then local claimTtl = redis.call("PTTL", KEYS[5]) redis.call("DEL", KEYS[5]) ' +
+ 'if #kept > 0 then redis.call("RPUSH", KEYS[5], unpack(kept)) ' +
+ 'if claimTtl > 0 then redis.call("PEXPIRE", KEYS[5], claimTtl) end end end ' +
+ 'redis.call("XADD", KEYS[1], "*", "event", ARGV[1]) ' +
+ 'if currentStatus == "running" then ' +
+ 'redis.call("HSET", KEYS[2], "lastActiveAt", ARGV[6]) end ' +
'local cur = redis.call("TTL", KEYS[1]) ' +
'if cur < target then redis.call("EXPIRE", KEYS[1], target) end ' +
+ 'for i = 3, 4 do local rt = redis.call("TTL", KEYS[i]) ' +
+ 'if rt >= 0 and rt < recoveryTarget then redis.call("EXPIRE", KEYS[i], recoveryTarget) end end ' +
+ 'for i = 5, 6 do local qt = redis.call("TTL", KEYS[i]) ' +
+ 'if qt >= 0 and qt < target then redis.call("EXPIRE", KEYS[i], target) end end ' +
+ 'if receipt then receipt.item = delivered receipt.state = "delivered" ' +
+ 'redis.call("HSET", KEYS[3], ARGV[4], cjson.encode(receipt)) end ' +
'return 1';
/**
@@ -279,13 +746,19 @@ const CHUNK_APPEND_LUA =
* lands at/after a fast pause resets the key to the short running TTL, and a reload of a
* still-live approval after that window loses the tool/run-step timeline even though the
* approval remains resumable. Reads the paused window from the job key (which
- * `transitionStatus` set); a normally-running job keeps the short running TTL.
+ * `transitionStatus` set); a normally-running job keeps the short running TTL. The write
+ * also requires an active status so a late provider event cannot recreate run steps after
+ * a same-epoch terminal transition deleted or retained the final timeline.
*
* KEYS: [runSteps, job]
* ARGV: [runStepsJson, runningTtl, expectCreatedAt | ""]
*/
const RUNSTEPS_SAVE_LUA =
- 'if ARGV[3] ~= "" and redis.call("HGET", KEYS[2], "createdAt") ~= ARGV[3] then return 0 end ' +
+ 'local currentCreatedAt = redis.call("HGET", KEYS[2], "createdAt") ' +
+ 'if not currentCreatedAt then return 0 end ' +
+ 'if ARGV[3] ~= "" and currentCreatedAt ~= ARGV[3] then return 0 end ' +
+ 'local currentStatus = redis.call("HGET", KEYS[2], "status") ' +
+ 'if currentStatus ~= "running" and currentStatus ~= "requires_action" then return 0 end ' +
'redis.call("SET", KEYS[1], ARGV[1]) ' +
'local run = tonumber(ARGV[2]) ' +
'local target = run ' +
@@ -325,7 +798,7 @@ const RUNSTEPS_READ_LUA =
* too — a steer can never land on a completed/aborted/finalizing job, and the
* depth cap can't be raced past by concurrent enqueues.
*
- * KEYS: [job, steers]
+ * KEYS: [job, steers, receipts]
* ARGV: [itemJson, ttl, maxDepth]
* Returns: new depth, -1 (not running / closed), or -2 (queue full)
*/
@@ -338,6 +811,105 @@ const STEER_ENQUEUE_LUA =
'redis.call("EXPIRE", KEYS[2], tonumber(ARGV[2])) ' +
'return redis.call("LLEN", KEYS[2])';
+/** Atomic capability-normalized enqueue for callers without a receipt id.
+ * The persisted item and its queue position are returned from the same Lua
+ * step, eliminating the legacy enqueue→arm failure window. */
+const STEER_ENQUEUE_VERSIONED_LUA =
+ 'if ARGV[4] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[4] then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return -1 end ' +
+ 'if redis.call("LLEN", KEYS[2]) >= tonumber(ARGV[3]) then return -2 end ' +
+ 'local item = cjson.decode(ARGV[1]) ' +
+ 'if ARGV[5] == "1" then item.preemptRevision = 1 ' +
+ 'if redis.call("HGET", KEYS[1], "preemptCapable") == "1" then item.preempt = true end end ' +
+ 'local itemJson = cjson.encode(item) ' +
+ 'redis.call("RPUSH", KEYS[2], itemJson) ' +
+ 'redis.call("EXPIRE", KEYS[2], tonumber(ARGV[2])) ' +
+ 'return cjson.encode({ item = item, position = redis.call("LLEN", KEYS[2]) })';
+
+/** Lost-ACK-safe enqueue. The receipt hash deliberately outlives both the
+ * queue and job hash; all three keys share the stream hash slot. Existing
+ * receipts win before live-job guards, so retry after drain/terminal/replace
+ * returns the original ACK instead of re-injecting. */
+const STEER_ENQUEUE_RECEIPT_LUA =
+ 'local existing = redis.call("HGET", KEYS[3], ARGV[5]) ' +
+ 'if existing then ' +
+ 'local receipt = cjson.decode(existing) ' +
+ 'local epoch = redis.call("HGET", KEYS[1], "createdAt") ' +
+ 'local status = redis.call("HGET", KEYS[1], "status") ' +
+ 'local sameActive = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" ' +
+ 'and epoch and tostring(receipt.generationCreatedAt or "") == epoch ' +
+ 'and (status == "running" or status == "requires_action") ' +
+ 'if receipt.state == "queued" then ' +
+ 'if not sameActive then receipt.state = "leftover" existing = cjson.encode(receipt) ' +
+ 'redis.call("HSET", KEYS[3], ARGV[5], existing) else ' +
+ 'local found = false local queued = redis.call("LRANGE", KEYS[2], 0, -1) ' +
+ 'for i = 1, #queued do local ok, candidate = pcall(cjson.decode, queued[i]) ' +
+ 'if ok and candidate.clientSteerId == ARGV[5] then found = true break end end ' +
+ 'if not found then receipt.state = "leftover" existing = cjson.encode(receipt) ' +
+ 'redis.call("HSET", KEYS[3], ARGV[5], existing) end end ' +
+ 'elseif receipt.state == "claimed" and not sameActive then ' +
+ 'receipt.state = "leftover" existing = cjson.encode(receipt) ' +
+ 'redis.call("HSET", KEYS[3], ARGV[5], existing) end ' +
+ 'return existing end ' +
+ 'if redis.call("HGET", KEYS[1], "generationProtocolVersion") ~= "2" then ' +
+ 'if ARGV[4] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[4] then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return -1 end ' +
+ 'if redis.call("LLEN", KEYS[2]) >= tonumber(ARGV[3]) then return -2 end ' +
+ 'local legacyItem = cjson.decode(ARGV[1]) if ARGV[7] == "1" then legacyItem.preemptRevision = 1 ' +
+ 'if redis.call("HGET", KEYS[1], "preemptCapable") == "1" then legacyItem.preempt = true end end ' +
+ 'redis.call("RPUSH", KEYS[2], cjson.encode(legacyItem)) ' +
+ 'redis.call("EXPIRE", KEYS[2], tonumber(ARGV[2])) ' +
+ 'return cjson.encode({ item = legacyItem, position = redis.call("LLEN", KEYS[2]) }) end ' +
+ 'if ARGV[4] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[4] then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return -1 end ' +
+ 'if redis.call("LLEN", KEYS[2]) >= tonumber(ARGV[3]) then return -2 end ' +
+ 'if redis.call("ZCARD", KEYS[4]) >= tonumber(ARGV[9]) then return -3 end ' +
+ 'local item = cjson.decode(ARGV[1]) ' +
+ 'if ARGV[7] == "1" then ' +
+ 'item.preemptRevision = 1 ' +
+ 'if redis.call("HGET", KEYS[1], "preemptCapable") == "1" then item.preempt = true end ' +
+ 'end ' +
+ 'local itemJson = cjson.encode(item) ' +
+ 'redis.call("RPUSH", KEYS[2], itemJson) ' +
+ 'redis.call("EXPIRE", KEYS[2], tonumber(ARGV[2])) ' +
+ 'local receipt = cjson.decode(ARGV[6]) ' +
+ 'receipt.item = item receipt.position = redis.call("LLEN", KEYS[2]) receipt.state = "queued" ' +
+ 'local receiptJson = cjson.encode(receipt) ' +
+ 'redis.call("HSET", KEYS[3], ARGV[5], receiptJson) ' +
+ 'redis.call("ZADD", KEYS[4], item.createdAt, ARGV[5]) ' +
+ 'local receiptTtl = tonumber(ARGV[8]) ' +
+ 'for i = 3, 4 do local existingTtl = redis.call("TTL", KEYS[i]) ' +
+ 'if existingTtl == -1 or (existingTtl >= 0 and existingTtl < receiptTtl) then ' +
+ 'redis.call("EXPIRE", KEYS[i], receiptTtl) end end ' +
+ 'return receiptJson';
+
+/** Receipt read with lazy repair for terminal/reaper/replacement scripts that
+ * delete the queue directly. Generation identity distinguishes a same-run
+ * older drain from a replacement that discarded an undrained predecessor. */
+const STEER_RECEIPT_GET_LUA =
+ 'local raw = redis.call("HGET", KEYS[1], ARGV[1]) ' +
+ 'if not raw then return false end ' +
+ 'local receipt = cjson.decode(raw) ' +
+ 'local epoch = redis.call("HGET", KEYS[2], "createdAt") ' +
+ 'local status = redis.call("HGET", KEYS[2], "status") ' +
+ 'local sameActive = redis.call("HGET", KEYS[2], "generationProtocolVersion") == "2" ' +
+ 'and epoch and tostring(receipt.generationCreatedAt or "") == epoch ' +
+ 'and (status == "running" or status == "requires_action") ' +
+ 'if receipt.state == "claimed" then ' +
+ 'if not sameActive then receipt.state = "leftover" raw = cjson.encode(receipt) ' +
+ 'redis.call("HSET", KEYS[1], ARGV[1], raw) end return raw end ' +
+ 'if receipt.state ~= "queued" then return raw end ' +
+ 'if not sameActive then receipt.state = "leftover" raw = cjson.encode(receipt) ' +
+ 'redis.call("HSET", KEYS[1], ARGV[1], raw) return raw end ' +
+ 'local items = redis.call("LRANGE", KEYS[3], 0, -1) ' +
+ 'for i = 1, #items do local ok, item = pcall(cjson.decode, items[i]) ' +
+ 'if ok and item.clientSteerId == ARGV[1] then return raw end end ' +
+ 'receipt.state = "leftover" ' +
+ 'raw = cjson.encode(receipt) redis.call("HSET", KEYS[1], ARGV[1], raw) return raw';
+
/**
* Atomic take-all: read the whole queue FIFO and delete the key in one step,
* so two concurrent drains can never both deliver the same steer. When an
@@ -346,82 +918,382 @@ const STEER_ENQUEUE_LUA =
* replacement job's queue (the check-then-drain would otherwise race
* `createJob`).
*
- * KEYS: [job, steers]
- * ARGV: [expectedCreatedAt or ""]
+ * KEYS: [job, steers, claimedSteers, receipts, receiptOrder]
+ * ARGV: [expectedCreatedAt or "", runningTtl]
* Returns: array of item JSON strings (possibly empty)
*/
const STEER_DRAIN_LUA =
'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {} end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" ' +
+ 'or redis.call("HGET", KEYS[1], "steersClosed") == "1" then return {} end ' +
+ 'local currentCreatedAt = redis.call("HGET", KEYS[1], "createdAt") ' +
'local items = redis.call("LRANGE", KEYS[2], 0, -1) ' +
+ 'if redis.call("HGET", KEYS[1], "generationProtocolVersion") ~= "2" then ' +
+ 'redis.call("DEL", KEYS[2]) return items end ' +
+ 'local decodedItems = {} local decodedReceipts = {} ' +
+ 'for i = 1, #items do ' +
+ 'local ok, item = pcall(cjson.decode, items[i]) ' +
+ 'if not ok or type(item) ~= "table" or not item.steerId then ' +
+ 'return redis.error_reply("invalid steer queue item") end ' +
+ 'decodedItems[i] = item ' +
+ 'if item.clientSteerId then ' +
+ 'local raw = redis.call("HGET", KEYS[4], item.clientSteerId) ' +
+ 'if not raw then return redis.error_reply("missing steer receipt") end ' +
+ 'local receiptOk, receipt = pcall(cjson.decode, raw) ' +
+ 'if not receiptOk or type(receipt) ~= "table" ' +
+ 'or receipt.clientSteerId ~= item.clientSteerId ' +
+ 'or tostring(receipt.generationCreatedAt or "") ~= currentCreatedAt ' +
+ 'or receipt.state ~= "queued" or not receipt.item ' +
+ 'or receipt.item.steerId ~= item.steerId ' +
+ 'or receipt.item.clientSteerId ~= item.clientSteerId then ' +
+ 'return redis.error_reply("invalid steer receipt") end ' +
+ 'decodedReceipts[i] = receipt ' +
+ 'end end ' +
+ 'if #items > 0 then redis.call("RPUSH", KEYS[3], unpack(items)) redis.call("EXPIRE", KEYS[3], ARGV[2]) end ' +
+ 'for i = 1, #items do local item = decodedItems[i] local receipt = decodedReceipts[i] ' +
+ 'if receipt then receipt.item = item receipt.state = "claimed" ' +
+ 'redis.call("HSET", KEYS[4], item.clientSteerId, cjson.encode(receipt)) end end ' +
+ 'for i = 4, 5 do local ttl = redis.call("TTL", KEYS[i]) ' +
+ 'if ttl >= 0 and ttl < tonumber(ARGV[2]) then redis.call("EXPIRE", KEYS[i], ARGV[2]) end end ' +
'redis.call("DEL", KEYS[2]) ' +
'return items';
+/** Roll back claimed items whose durable applied-part write failed. New
+ * enqueues may have landed after the drain, so the failed accepted batch is
+ * prepended in its original order rather than replacing the live queue. */
+const STEER_RESTORE_CLAIMED_LUA =
+ 'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return 0 end ' +
+ 'local currentCreatedAt = redis.call("HGET", KEYS[1], "createdAt") ' +
+ 'local incoming = cjson.decode(ARGV[2]) local incomingIds = {} ' +
+ 'for i = 1, #incoming do if incoming[i].steerId then incomingIds[incoming[i].steerId] = true end end ' +
+ 'if redis.call("HGET", KEYS[1], "generationProtocolVersion") ~= "2" then ' +
+ 'local present = {} local current = redis.call("LRANGE", KEYS[2], 0, -1) ' +
+ 'for i = 1, #current do local ok, item = pcall(cjson.decode, current[i]) ' +
+ 'if not ok or type(item) ~= "table" or not item.steerId then return 0 end present[item.steerId] = true end ' +
+ 'for i = #incoming, 1, -1 do local item = incoming[i] if item.steerId and not present[item.steerId] then ' +
+ 'redis.call("LPUSH", KEYS[2], cjson.encode(item)) end end ' +
+ 'if redis.call("EXISTS", KEYS[2]) == 1 then redis.call("EXPIRE", KEYS[2], ARGV[3]) end return 1 end ' +
+ 'local claims = redis.call("LRANGE", KEYS[3], 0, -1) local keptClaims = {} local matched = 0 ' +
+ 'for i = 1, #claims do local ok, item = pcall(cjson.decode, claims[i]) ' +
+ 'if ok and item.steerId and incomingIds[item.steerId] then matched = matched + 1 ' +
+ 'else keptClaims[#keptClaims + 1] = claims[i] end end ' +
+ 'if matched ~= #incoming then return 0 end ' +
+ 'local receiptByClient = {} ' +
+ 'for i = 1, #incoming do local item = incoming[i] if item.clientSteerId then ' +
+ 'local raw = redis.call("HGET", KEYS[4], item.clientSteerId) if not raw then return 0 end ' +
+ 'local ok, receipt = pcall(cjson.decode, raw) ' +
+ 'if not ok or type(receipt) ~= "table" ' +
+ 'or receipt.clientSteerId ~= item.clientSteerId ' +
+ 'or tostring(receipt.generationCreatedAt or "") ~= currentCreatedAt ' +
+ 'or receipt.state ~= "claimed" or not receipt.item ' +
+ 'or receipt.item.steerId ~= item.steerId ' +
+ 'or receipt.item.clientSteerId ~= item.clientSteerId then return 0 end ' +
+ 'receiptByClient[item.clientSteerId] = receipt end end ' +
+ 'local present = {} local current = redis.call("LRANGE", KEYS[2], 0, -1) ' +
+ 'for i = 1, #current do local ok, item = pcall(cjson.decode, current[i]) ' +
+ 'if not ok or type(item) ~= "table" or not item.steerId then return 0 end ' +
+ 'present[item.steerId] = true end ' +
+ 'local claimsTtl = redis.call("PTTL", KEYS[3]) redis.call("DEL", KEYS[3]) ' +
+ 'if #keptClaims > 0 then redis.call("RPUSH", KEYS[3], unpack(keptClaims)) ' +
+ 'if claimsTtl > 0 then redis.call("PEXPIRE", KEYS[3], claimsTtl) end end ' +
+ 'for i = #incoming, 1, -1 do local item = incoming[i] ' +
+ 'if item.steerId and not present[item.steerId] then ' +
+ 'local itemJson = cjson.encode(item) redis.call("LPUSH", KEYS[2], itemJson) ' +
+ 'if item.clientSteerId then local receipt = receiptByClient[item.clientSteerId] ' +
+ 'receipt.item = item receipt.state = "queued" ' +
+ 'redis.call("HSET", KEYS[4], item.clientSteerId, cjson.encode(receipt)) end end end ' +
+ 'if redis.call("EXISTS", KEYS[2]) == 1 then redis.call("EXPIRE", KEYS[2], ARGV[3]) end ' +
+ 'return 1';
+
const STEER_PEEK_LUA =
'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {} end ' +
'return redis.call("LRANGE", KEYS[2], 0, -1)';
+const STEER_PEEK_CLAIMED_LUA =
+ 'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {} end ' +
+ 'if redis.call("HGET", KEYS[1], "generationProtocolVersion") ~= "2" then return {} end ' +
+ 'return redis.call("LRANGE", KEYS[2], 0, -1)';
+
/**
* Remove ONE queued steer by id without disturbing the rest: the list is
* rebuilt atomically, so a concurrent drain either delivers the steer or the
- * cancel wins — never a torn queue. ARGV[1] is the JSON fragment
- * `"steerId":"<id>"` matched as a plain substring against each serialized
- * item (ids are server-generated UUIDs, no escaping ambiguity). The list TTL
- * survives the rebuild.
+ * cancel wins — never a torn queue. The generation fence and receipt
+ * validation happen before either the queue or receipt is changed. The list
+ * TTL survives the rebuild.
*
- * KEYS: [steers]
- * ARGV: [steerIdFragment]
+ * KEYS: [job, steers, receipts]
+ * ARGV: [steerId, expectedCreatedAt or ""]
* Returns: 1 when removed, 0 when not found
*/
const STEER_REMOVE_LUA =
- 'local items = redis.call("LRANGE", KEYS[1], 0, -1) ' +
+ 'if ARGV[2] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[2] then return 0 end ' +
+ 'local currentCreatedAt = redis.call("HGET", KEYS[1], "createdAt") ' +
+ 'local protocolV2 = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" ' +
+ 'local items = redis.call("LRANGE", KEYS[2], 0, -1) ' +
'if #items == 0 then return 0 end ' +
- 'local kept = {} ' +
- 'local removed = 0 ' +
+ 'local target = 0 local targetItem = nil local targetReceipt = nil ' +
'for i = 1, #items do ' +
- 'if removed == 0 and string.find(items[i], ARGV[1], 1, true) then removed = 1 ' +
- 'else kept[#kept + 1] = items[i] end ' +
+ 'local ok, item = pcall(cjson.decode, items[i]) ' +
+ 'if not ok or type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" then ' +
+ 'return redis.error_reply("invalid steer queue item") end ' +
+ 'if item.steerId == ARGV[1] then ' +
+ 'if target ~= 0 then return redis.error_reply("duplicate steer id") end ' +
+ 'target = i targetItem = item ' +
+ 'if protocolV2 and item.clientSteerId then ' +
+ 'if type(item.clientSteerId) ~= "string" or item.clientSteerId == "" or not currentCreatedAt then ' +
+ 'return redis.error_reply("invalid steer receipt") end ' +
+ 'local raw = redis.call("HGET", KEYS[3], item.clientSteerId) ' +
+ 'if not raw then return redis.error_reply("missing steer receipt") end ' +
+ 'local receiptOk, receipt = pcall(cjson.decode, raw) ' +
+ 'if not receiptOk or type(receipt) ~= "table" ' +
+ 'or receipt.clientSteerId ~= item.clientSteerId ' +
+ 'or tostring(receipt.generationCreatedAt or "") ~= currentCreatedAt ' +
+ 'or receipt.state ~= "queued" or type(receipt.item) ~= "table" ' +
+ 'or receipt.item.steerId ~= item.steerId ' +
+ 'or receipt.item.clientSteerId ~= item.clientSteerId then ' +
+ 'return redis.error_reply("invalid steer receipt") end targetReceipt = receipt end end ' +
'end ' +
- 'if removed == 0 then return 0 end ' +
- 'local ttl = redis.call("PTTL", KEYS[1]) ' +
- 'redis.call("DEL", KEYS[1]) ' +
+ 'if target == 0 then return 0 end ' +
+ 'local ttl = redis.call("PTTL", KEYS[2]) ' +
+ 'redis.call("DEL", KEYS[2]) ' +
+ 'local kept = {} for i = 1, #items do if i ~= target then kept[#kept + 1] = items[i] end end ' +
'if #kept > 0 then ' +
- 'redis.call("RPUSH", KEYS[1], unpack(kept)) ' +
- 'if ttl > 0 then redis.call("PEXPIRE", KEYS[1], ttl) end ' +
+ 'redis.call("RPUSH", KEYS[2], unpack(kept)) ' +
+ 'if ttl > 0 then redis.call("PEXPIRE", KEYS[2], ttl) end ' +
'end ' +
+ 'if targetReceipt then targetReceipt.item = targetItem targetReceipt.state = "cancelled" ' +
+ 'redis.call("HSET", KEYS[3], targetItem.clientSteerId, cjson.encode(targetReceipt)) end ' +
'return 1';
/**
- * Claim-on-read for parked steers: return AND delete in one atomic step so a
- * second reload cannot re-mint chips the user already dismissed. The owner
- * gate runs INSIDE the same step — a payload not containing the requester's
- * `"userId":"…"` fragment (ARGV[1], plain-substring match) returns the empty
- * sentinel WITHOUT deleting, so a non-owner probe can never destroy the
- * owner's recovery, even transiently. Single key (cluster-safe); GETDEL is
- * avoided only for older-Redis compatibility.
+ * Escalate ONE queued steer to an interrupt IN PLACE: decode the whole item,
+ * set `preempt`, and LSET it back at its index, so its FIFO position is
+ * untouched (the entire queue drains at the seal, in order). Guarded like
+ * {@link STEER_ENQUEUE_LUA}: a non-running job, closed queue, or generation
+ * mismatch refuses, so an arm racing a pause cannot leak into the resumed
+ * segment and a stale request can never arm a replacement run's steer. The
+ * owner's LIVE `preemptCapable` is part of the same atomic predicate — a HITL
+ * resume on a rolling deploy rewrites it for the SAME generation, so a value
+ * the caller read earlier is not trustworthy.
+ *
+ * KEYS: [job, steers, receipts]
+ * ARGV: [steerId, expectedCreatedAt or ""]
+ * Returns: the updated item JSON, 0 not found / non-running / closed / fenced,
+ * or -1 when the owner cannot seal.
+ */
+const STEER_ARM_LUA =
+ 'if ARGV[2] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[2] then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "status") ~= "running" then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return 0 end ' +
+ 'local protocolV2 = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" ' +
+ 'local currentCreatedAt = redis.call("HGET", KEYS[1], "createdAt") ' +
+ 'local items = redis.call("LRANGE", KEYS[2], 0, -1) ' +
+ 'local target = 0 local item = nil ' +
+ 'for i = 1, #items do ' +
+ 'local decoded, candidate = pcall(cjson.decode, items[i]) ' +
+ 'if not decoded or type(candidate) ~= "table" or type(candidate.steerId) ~= "string" ' +
+ 'or candidate.steerId == "" then return redis.error_reply("invalid steer queue item") end ' +
+ 'if candidate.steerId == ARGV[1] then ' +
+ 'if target ~= 0 then return redis.error_reply("duplicate steer id") end ' +
+ 'target = i item = candidate end ' +
+ 'end ' +
+ 'if target == 0 then return 0 end ' +
+ 'if redis.call("HGET", KEYS[1], "preemptCapable") ~= "1" then return -1 end ' +
+ 'local receipt = nil ' +
+ 'if protocolV2 and item.clientSteerId then ' +
+ 'if type(item.clientSteerId) ~= "string" or item.clientSteerId == "" or not currentCreatedAt then ' +
+ 'return redis.error_reply("invalid steer receipt") end ' +
+ 'local receiptJson = redis.call("HGET", KEYS[3], item.clientSteerId) ' +
+ 'if not receiptJson then return redis.error_reply("missing steer receipt") end ' +
+ 'local receiptOk, receiptValue = pcall(cjson.decode, receiptJson) ' +
+ 'if not receiptOk or type(receiptValue) ~= "table" ' +
+ 'or receiptValue.clientSteerId ~= item.clientSteerId ' +
+ 'or tostring(receiptValue.generationCreatedAt or "") ~= currentCreatedAt ' +
+ 'or receiptValue.state ~= "queued" or not receiptValue.item ' +
+ 'or receiptValue.item.steerId ~= item.steerId ' +
+ 'or receiptValue.item.clientSteerId ~= item.clientSteerId then ' +
+ 'return redis.error_reply("invalid steer receipt") end receipt = receiptValue end ' +
+ 'item.preemptRevision = tonumber(item.preemptRevision or 0) + 1 ' +
+ 'item.preempt = true ' +
+ 'redis.call("LSET", KEYS[2], target - 1, cjson.encode(item)) ' +
+ 'if receipt then receipt.item = item ' +
+ 'redis.call("HSET", KEYS[3], item.clientSteerId, cjson.encode(receipt)) end ' +
+ 'return cjson.encode(item)';
+
+/** Downgrade durable interrupt labels during a capable→incapable owner
+ * handover. Capability and queue edits share one Lua transaction, so a later
+ * capable resume cannot race a stale cleanup from the previous owner. */
+const STEER_DOWNGRADE_PREEMPTS_LUA =
+ 'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return -1 end ' +
+ 'if redis.call("EXISTS", KEYS[1]) == 0 then return -1 end ' +
+ 'if redis.call("HGET", KEYS[1], "preemptCapable") == "1" then return -1 end ' +
+ 'local protocolV2 = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" ' +
+ 'local items = redis.call("LRANGE", KEYS[2], 0, -1) ' +
+ 'local decodedItems = {} local decodedReceipts = {} ' +
+ 'for i = 1, #items do ' +
+ 'local decoded, item = pcall(cjson.decode, items[i]) ' +
+ 'if decoded and item.preempt == true then ' +
+ 'decodedItems[i] = item ' +
+ 'if protocolV2 and item.clientSteerId then local receiptJson = redis.call("HGET", KEYS[3], item.clientSteerId) ' +
+ 'if not receiptJson then return redis.error_reply("missing steer receipt") end ' +
+ 'local receiptOk, receipt = pcall(cjson.decode, receiptJson) ' +
+ 'local epoch = redis.call("HGET", KEYS[1], "createdAt") ' +
+ 'if not receiptOk or type(receipt) ~= "table" ' +
+ 'or receipt.clientSteerId ~= item.clientSteerId ' +
+ 'or tostring(receipt.generationCreatedAt or "") ~= epoch ' +
+ 'or receipt.state ~= "queued" or not receipt.item ' +
+ 'or receipt.item.steerId ~= item.steerId ' +
+ 'or receipt.item.clientSteerId ~= item.clientSteerId then ' +
+ 'return redis.error_reply("invalid steer receipt") end decodedReceipts[i] = receipt end end end ' +
+ 'local changed = {} ' +
+ 'for i = 1, #items do local item = decodedItems[i] ' +
+ 'if item then ' +
+ 'item.preempt = nil ' +
+ 'item.preemptRevision = tonumber(item.preemptRevision or 0) + 1 ' +
+ 'redis.call("LSET", KEYS[2], i - 1, cjson.encode(item)) ' +
+ 'local receipt = decodedReceipts[i] if receipt then receipt.item = item ' +
+ 'redis.call("HSET", KEYS[3], item.clientSteerId, cjson.encode(receipt)) end ' +
+ 'changed[#changed + 1] = cjson.encode(item) ' +
+ 'end ' +
+ 'end ' +
+ 'return changed';
+
+/**
+ * Owner-gated replay read for parked steers. It intentionally does not delete:
+ * createJob only leases the exact recovered item while its deterministic next
+ * turn is active. Durable user-message persistence commits the removal, so a
+ * failed startup or lost status response cannot erase the only recovery copy.
*
- * KEYS: [parkedSteers]
- * ARGV: [ownerFragment]
- * Returns: the parked payload JSON, '' when not the owner, or nil
+ * KEYS: [parkedSteers, job]
+ * ARGV: [ownerUserId, ownerTenantId | "", requestedProtocolVersion]
+ * Returns: [parked payload JSON, protocol], '' when not the owner, or nil
*/
const CLAIM_PARKED_LUA =
'local v = redis.call("GET", KEYS[1]) ' +
'if not v then return v end ' +
- 'if not string.find(v, ARGV[1], 1, true) then return "" end ' +
- 'redis.call("DEL", KEYS[1]) ' +
- 'return v';
+ 'local function isDenseArray(value) if type(value) ~= "table" then return false end ' +
+ 'local count = 0 for key, _ in pairs(value) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return false end count = count + 1 end ' +
+ 'return count == #value end ' +
+ 'local ok, parked = pcall(cjson.decode, v) ' +
+ 'if not ok or type(parked) ~= "table" or type(parked.userId) ~= "string" or parked.userId == "" ' +
+ 'or not isDenseArray(parked.steers) or #parked.steers == 0 then return "" end ' +
+ 'if parked.generationProtocolVersion and parked.generationProtocolVersion ~= 1 ' +
+ 'and parked.generationProtocolVersion ~= 2 then return "" end ' +
+ 'for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" ' +
+ 'or (item.clientSteerId and (type(item.clientSteerId) ~= "string" or item.clientSteerId == "")) ' +
+ 'or (item.text and type(item.text) ~= "string") ' +
+ 'or (item.createdAt and (type(item.createdAt) ~= "number" or item.createdAt < 0)) ' +
+ 'or (item.recoveringCreatedAt and (type(item.recoveringCreatedAt) ~= "number" or item.recoveringCreatedAt < 0)) then return "" end end ' +
+ 'if parked.userId ~= ARGV[1] then return "" end ' +
+ 'if parked.tenantId and parked.tenantId ~= ARGV[2] then return "" end ' +
+ 'local createdAt = redis.call("HGET", KEYS[2], "createdAt") ' +
+ 'local status = redis.call("HGET", KEYS[2], "status") ' +
+ 'local recoveredSteerId = redis.call("HGET", KEYS[2], "recoveredSteerId") ' +
+ 'local jobTenantId = redis.call("HGET", KEYS[2], "tenantId") ' +
+ 'local activeRecovery = createdAt and (status == "running" or status == "requires_action") ' +
+ 'and redis.call("HGET", KEYS[2], "generationProtocolVersion") == "2" ' +
+ 'and recoveredSteerId and recoveredSteerId ~= "" ' +
+ 'and redis.call("HGET", KEYS[2], "userId") == ARGV[1] ' +
+ 'and (not jobTenantId or jobTenantId == ARGV[2]) ' +
+ 'local visible = {} local leased = {} for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if activeRecovery and item.steerId == recoveredSteerId ' +
+ 'and tostring(item.recoveringCreatedAt or "") == createdAt then leased[#leased + 1] = item ' +
+ 'else item.recoveringCreatedAt = nil visible[#visible + 1] = item end end ' +
+ 'local protocol = parked.generationProtocolVersion == 2 and ARGV[3] == "2" and 2 or 1 ' +
+ 'if protocol == 1 and #leased == 0 then parked.generationProtocolVersion = 1 ' +
+ 'parked.steers = visible redis.call("DEL", KEYS[1]) ' +
+ 'return { cjson.encode(parked), "1" } end ' +
+ 'if #visible == 0 then return "" end parked.steers = visible ' +
+ 'if protocol == 1 then local ttl = redis.call("PTTL", KEYS[1]) ' +
+ 'local retained = { userId = parked.userId, generationProtocolVersion = 2, steers = leased } ' +
+ 'if parked.tenantId then retained.tenantId = parked.tenantId end ' +
+ 'redis.call("SET", KEYS[1], cjson.encode(retained)) ' +
+ 'if ttl > 0 then redis.call("PEXPIRE", KEYS[1], ttl) end ' +
+ 'parked.generationProtocolVersion = 1 end ' +
+ 'return { cjson.encode(parked), tostring(protocol) }';
+
+/** Commit a leased recovery only after its ordinary user message is durable. */
+const CONSUME_PARKED_STEER_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] ' +
+ 'or redis.call("HGET", KEYS[1], "userId") ~= ARGV[3] ' +
+ 'or redis.call("HGET", KEYS[1], "recoveredSteerId") ~= ARGV[2] then return 0 end ' +
+ 'local jobTenant = redis.call("HGET", KEYS[1], "tenantId") ' +
+ 'if jobTenant and jobTenant ~= ARGV[4] then return 0 end ' +
+ 'local raw = redis.call("GET", KEYS[2]) if not raw then return 1 end ' +
+ 'local ok, parked = pcall(cjson.decode, raw) ' +
+ 'if not ok or type(parked) ~= "table" or type(parked.steers) ~= "table" then return 0 end ' +
+ 'local count = 0 for key, _ in pairs(parked.steers) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return 0 end count = count + 1 end ' +
+ 'if count == 0 or count ~= #parked.steers then return 0 end ' +
+ 'if parked.userId ~= ARGV[3] or (parked.tenantId and parked.tenantId ~= ARGV[4]) then return 0 end ' +
+ 'local consumed = nil local kept = {} for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" then return 0 end ' +
+ 'if not consumed and item.steerId == ARGV[2] ' +
+ 'and tostring(item.recoveringCreatedAt or "") == ARGV[1] then consumed = item ' +
+ 'else kept[#kept + 1] = item end end if not consumed then return 0 end ' +
+ 'local receipt = nil if consumed.clientSteerId then ' +
+ 'local receiptRaw = redis.call("HGET", KEYS[3], consumed.clientSteerId) if not receiptRaw then return 0 end ' +
+ 'local receiptOk, decoded = pcall(cjson.decode, receiptRaw) ' +
+ 'if not receiptOk or decoded.state ~= "leftover" or decoded.userId ~= ARGV[3] ' +
+ 'or (decoded.tenantId and decoded.tenantId ~= ARGV[4]) or not decoded.item ' +
+ 'or decoded.item.steerId ~= ARGV[2] then return 0 end receipt = decoded end ' +
+ 'local ttl = redis.call("PTTL", KEYS[2]) if #kept == 0 then redis.call("DEL", KEYS[2]) ' +
+ 'else parked.steers = kept redis.call("SET", KEYS[2], cjson.encode(parked)) ' +
+ 'if ttl > 0 then redis.call("PEXPIRE", KEYS[2], ttl) end end ' +
+ 'if receipt then receipt.state = "recovered" ' +
+ 'redis.call("HSET", KEYS[3], consumed.clientSteerId, cjson.encode(receipt)) end return 1';
+
+/** Idempotent terminal reclaim for Edit/Queue/dismiss. */
+const DISCARD_STEER_LEFTOVER_LUA =
+ 'local raw = redis.call("HGET", KEYS[1], ARGV[1]) if not raw then return 0 end ' +
+ 'local ok, receipt = pcall(cjson.decode, raw) if not ok or type(receipt) ~= "table" then return 0 end ' +
+ 'if ARGV[5] ~= "" and tostring(receipt.generationCreatedAt or "") ~= ARGV[5] then return 0 end ' +
+ 'local tenantMatches = not receipt.tenantId or receipt.tenantId == ARGV[4] ' +
+ 'if receipt.state ~= "leftover" or receipt.userId ~= ARGV[3] or not tenantMatches ' +
+ 'or not receipt.item or receipt.item.steerId ~= ARGV[2] then return 0 end ' +
+ 'local status = redis.call("HGET", KEYS[3], "status") ' +
+ 'local activeRecovery = (status == "running" or status == "requires_action") ' +
+ 'and redis.call("HGET", KEYS[3], "recoveredSteerId") == ARGV[2] ' +
+ 'and redis.call("HGET", KEYS[3], "userId") == ARGV[3] ' +
+ 'local jobTenant = redis.call("HGET", KEYS[3], "tenantId") ' +
+ 'if activeRecovery and (not jobTenant or jobTenant == ARGV[4]) then return 0 end ' +
+ 'local parkedRaw = redis.call("GET", KEYS[2]) ' +
+ 'if parkedRaw then local parsed, parked = pcall(cjson.decode, parkedRaw) ' +
+ 'if not parsed or type(parked) ~= "table" or type(parked.steers) ~= "table" then return 0 end ' +
+ 'local count = 0 for key, _ in pairs(parked.steers) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return 0 end count = count + 1 end ' +
+ 'if count == 0 or count ~= #parked.steers then return 0 end ' +
+ 'local parkedTenantMatches = not parked.tenantId or parked.tenantId == ARGV[4] ' +
+ 'if parked.userId ~= ARGV[3] or not parkedTenantMatches then return 0 end ' +
+ 'local kept = {} for i = 1, #parked.steers do local item = parked.steers[i] ' +
+ 'if type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" then return 0 end ' +
+ 'if item.steerId ~= ARGV[2] then kept[#kept + 1] = item end end ' +
+ 'if #kept == 0 then redis.call("DEL", KEYS[2]) else parked.steers = kept ' +
+ 'local ttl = redis.call("PTTL", KEYS[2]) redis.call("SET", KEYS[2], cjson.encode(parked)) ' +
+ 'if ttl > 0 then redis.call("PEXPIRE", KEYS[2], ttl) end end end ' +
+ 'receipt.state = "cancelled" redis.call("HSET", KEYS[1], ARGV[1], cjson.encode(receipt)) ' +
+ 'return 1';
/**
* Park leftovers only while the generation that drained them still owns the
* stream ID. If a replacement already exists, writing its parked key would
* leak predecessor state into the new run. A replacement created afterward
- * atomically clears this key in {@link JOB_CREATE_LUA}.
+ * retains the owner-gated payload and leases only an explicitly selected item.
*
* KEYS: [job, parkedSteers]
* ARGV: [expectedCreatedAt | "", payload, ttl]
*/
const PARK_STEERS_LUA =
'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
- 'redis.call("SET", KEYS[2], ARGV[2], "EX", ARGV[3]) ' +
+ 'local ok, parked = pcall(cjson.decode, ARGV[2]) if not ok or type(parked) ~= "table" then return 0 end ' +
+ 'local protocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") ' +
+ 'if protocol == "2" then parked.generationProtocolVersion = 2 ' +
+ 'else parked.generationProtocolVersion = 1 end ' +
+ 'redis.call("SET", KEYS[2], cjson.encode(parked), "EX", ARGV[3]) ' +
'return 1';
/**
@@ -433,16 +1305,77 @@ const PARK_STEERS_LUA =
* expected-`createdAt` guard as {@link STEER_DRAIN_LUA} keeps a stale run's
* finalization from closing (and stealing) a replacement job's queue.
*
- * KEYS: [job, steers]
- * ARGV: [expectedCreatedAt or ""]
- * Returns: array of item JSON strings (possibly empty)
+ * KEYS: [job, steers, claimedSteers, receipts, receiptOrder, parkedSteers]
+ * ARGV: [expectedCreatedAt or "", parkedTtl]
+ * Returns: array of item JSON strings (possibly empty), or the
+ * `recovery_corrupt` sentinel when destructive recovery is unsafe.
*/
const STEER_CLOSE_DRAIN_LUA =
'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {} end ' +
+ 'if redis.call("EXISTS", KEYS[1]) == 0 then return {} end ' +
+ 'local ownerUserId = redis.call("HGET", KEYS[1], "userId") ' +
+ 'local ownerTenantId = redis.call("HGET", KEYS[1], "tenantId") ' +
+ 'local generationProtocol = redis.call("HGET", KEYS[1], "generationProtocolVersion") == "2" and 2 or 1 ' +
+ 'local parkedProtocol = generationProtocol ' +
+ 'local function isDenseArray(value) if type(value) ~= "table" then return false end ' +
+ 'local count = 0 for key, _ in pairs(value) do ' +
+ 'if type(key) ~= "number" or key < 1 or key ~= math.floor(key) then return false end count = count + 1 end ' +
+ 'return count == #value end ' +
+ 'local claimedRows = {} if generationProtocol == 2 then claimedRows = redis.call("LRANGE", KEYS[3], 0, -1) end ' +
+ 'local sources = { claimedRows, redis.call("LRANGE", KEYS[2], 0, -1) } ' +
+ 'for s = 1, #sources do for i = 1, #sources[s] do local ok, item = pcall(cjson.decode, sources[s][i]) ' +
+ 'if not ok or type(item) ~= "table" or type(item.steerId) ~= "string" or item.steerId == "" then return "recovery_corrupt" end end end ' +
+ 'if (#sources[1] > 0 or #sources[2] > 0) and (not ownerUserId or ownerUserId == "") then return "recovery_corrupt" end ' +
+ 'local parkedSteers = {} local parkedRaw = redis.call("GET", KEYS[6]) ' +
+ 'if parkedRaw then local parsed, parked = pcall(cjson.decode, parkedRaw) ' +
+ 'if not parsed or type(parked) ~= "table" or type(parked.userId) ~= "string" or parked.userId == "" ' +
+ 'or not isDenseArray(parked.steers) or #parked.steers == 0 or parked.userId ~= ownerUserId ' +
+ 'or (parked.tenantId and parked.tenantId ~= ownerTenantId) then return "recovery_corrupt" end ' +
+ 'if parked.generationProtocolVersion and parked.generationProtocolVersion ~= 1 ' +
+ 'and parked.generationProtocolVersion ~= 2 then return "recovery_corrupt" end ' +
+ 'if parked.generationProtocolVersion == 2 then parkedProtocol = 2 end ' +
+ 'for i = 1, #parked.steers do local prior = parked.steers[i] ' +
+ 'if type(prior) ~= "table" or type(prior.steerId) ~= "string" or prior.steerId == "" ' +
+ 'or (prior.clientSteerId and (type(prior.clientSteerId) ~= "string" or prior.clientSteerId == "")) ' +
+ 'or (prior.text and type(prior.text) ~= "string") ' +
+ 'or (prior.createdAt and (type(prior.createdAt) ~= "number" or prior.createdAt < 0)) ' +
+ 'or (prior.recoveringCreatedAt and (type(prior.recoveringCreatedAt) ~= "number" or prior.recoveringCreatedAt < 0)) then return "recovery_corrupt" end ' +
+ 'parkedSteers[#parkedSteers + 1] = prior end end ' +
'if redis.call("EXISTS", KEYS[1]) == 1 then redis.call("HSET", KEYS[1], "steersClosed", "1") end ' +
- 'local items = redis.call("LRANGE", KEYS[2], 0, -1) ' +
- 'redis.call("DEL", KEYS[2]) ' +
- 'return items';
+ 'local combined = {} local seen = {} ' +
+ 'for s = 1, #sources do for i = 1, #sources[s] do ' +
+ 'local ok, item = pcall(cjson.decode, sources[s][i]) ' +
+ 'if ok and item.steerId and not seen[item.steerId] then ' +
+ 'seen[item.steerId] = true combined[#combined + 1] = item end ' +
+ 'if generationProtocol == 2 and ok and item.clientSteerId then ' +
+ 'local raw = redis.call("HGET", KEYS[4], item.clientSteerId) ' +
+ 'if raw then local receiptOk, receipt = pcall(cjson.decode, raw) ' +
+ 'if receiptOk and type(receipt) == "table" then receipt.item = item receipt.state = "leftover" ' +
+ 'redis.call("HSET", KEYS[4], item.clientSteerId, cjson.encode(receipt)) end end ' +
+ 'end end end ' +
+ 'if #combined > 0 and ownerUserId then ' +
+ 'local filteredParkedSteers = {} ' +
+ 'for i = 1, #parkedSteers do local prior = parkedSteers[i] ' +
+ 'if not seen[prior.steerId] then seen[prior.steerId] = true ' +
+ 'filteredParkedSteers[#filteredParkedSteers + 1] = prior end end ' +
+ 'local currentProjected = {} ' +
+ 'for i = 1, #combined do local item = combined[i] ' +
+ 'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
+ 'if item.clientSteerId then projected.clientSteerId = item.clientSteerId end ' +
+ 'if item.files then projected.files = item.files end ' +
+ 'if item.preempt then projected.preempt = item.preempt end ' +
+ 'if item.preemptRevision then projected.preemptRevision = item.preemptRevision end ' +
+ 'currentProjected[#currentProjected + 1] = projected end ' +
+ 'local merged = {} for i = 1, #filteredParkedSteers do merged[#merged + 1] = filteredParkedSteers[i] end ' +
+ 'for i = 1, #currentProjected do merged[#merged + 1] = currentProjected[i] end ' +
+ 'local parked = { userId = ownerUserId, generationProtocolVersion = parkedProtocol, steers = merged } ' +
+ 'if ownerTenantId then parked.tenantId = ownerTenantId end ' +
+ 'redis.call("SET", KEYS[6], cjson.encode(parked), "EX", ARGV[2]) end ' +
+ 'for i = 4, 5 do local ttl = redis.call("TTL", KEYS[i]) ' +
+ 'if ttl >= 0 and ttl < tonumber(ARGV[2]) then redis.call("EXPIRE", KEYS[i], ARGV[2]) end end ' +
+ 'redis.call("DEL", KEYS[2], KEYS[3]) ' +
+ 'local encoded = {} for i = 1, #combined do encoded[i] = cjson.encode(combined[i]) end ' +
+ 'return encoded';
/** Decision kinds the SDK can emit, used to sanity-check persisted records. */
const KNOWN_INTERRUPT_TYPES = new Set(['tool_approval', 'ask_user_question']);
@@ -451,8 +1384,12 @@ const KNOWN_INTERRUPT_TYPES = new Set(['tool_approval', 'ask_user_question']);
* configured to 0 — Redis rejects `EX 0`, which would silently kill
* park-based recovery. */
const PARKED_RECOVERY_TTL_S: number = 300;
+const STEER_RECEIPT_MAX_PER_STREAM: number = 100;
/** Grace window for publishing terminal/reaper events after the live job hash expires. */
const GENERATION_EPOCH_GRACE_TTL_S: number = 300;
+/** A terminal record carrying an uncommitted final event must survive the
+ * persistence-owner timeout even when completedTtl is configured to zero. */
+const TERMINAL_PERSISTENCE_RETENTION_TTL_S: number = 300;
/** Bound pathological replacement churn without leaving the request unbounded. */
const MEMBERSHIP_RECONCILE_MAX_ATTEMPTS: number = 8;
@@ -478,9 +1415,14 @@ const KEYS = {
runSteps: (streamId: string) => `stream:{${streamId}}:runsteps`,
/** Pending steer messages (FIFO list): stream:{streamId}:steers */
steers: (streamId: string) => `stream:{${streamId}}:steers`,
+ /** Drained steers awaiting an atomic durable applied-chunk commit. */
+ claimedSteers: (streamId: string) => `stream:{${streamId}}:steers-claimed`,
/** Parked terminally-drained steers (own TTL — must outlive the job hash,
* which the default completeJob path deletes immediately) */
parkedSteers: (streamId: string) => `stream:{${streamId}}:parked`,
+ /** Lost-ACK steer receipts (hash fields are clientSteerIds). */
+ steerReceipts: (streamId: string) => `stream:{${streamId}}:steer-receipts`,
+ steerReceiptOrder: (streamId: string) => `stream:{${streamId}}:steer-receipt-order`,
/** Latest generation epoch, retained briefly beyond the live job hash. */
generationEpoch: (streamId: string) => `stream:{${streamId}}:generation-epoch`,
/** Running jobs set for cleanup (global set - single slot) */
@@ -491,7 +1433,7 @@ const KEYS = {
userJobs: (userId: string, tenantId?: string) =>
tenantId ? `stream:user:{${tenantId}:${userId}}:jobs` : `stream:user:{${userId}}:jobs`,
/** Idempotency claim for a start-generation request: stream:idem:{userId:clientRequestId} */
- idempotency: (key: string) => `stream:idem:{${key}}`,
+ idempotency: (key: string) => `stream:idem:${key}`,
};
/**
@@ -520,7 +1462,7 @@ const DEFAULT_TTL = {
};
/**
- * Redis implementation of IJobStore.
+ * Redis implementation of IJobStoreV2.
* Enables horizontal scaling with multi-instance deployments.
*
* Storage strategy:
@@ -560,7 +1502,7 @@ interface LocalCacheEntry<T> {
value: T;
}
-export class RedisJobStore implements IJobStore {
+export class RedisJobStore implements IJobStoreV2 {
private redis: Redis | Cluster;
private cleanupInterval: NodeJS.Timeout | null = null;
private ttl: typeof DEFAULT_TTL;
@@ -696,17 +1638,64 @@ export class RedisJobStore implements IJobStore {
conversationId?: string,
tenantId?: string,
initialMetadata: JobMetadataPatch = {},
- ): Promise<SerializableJobData> {
- const job: SerializableJobData = {
+ recoveredSteerId?: string,
+ idempotencyClaimKey?: string,
+ idempotencyClaimToken?: string,
+ idempotencyClientRequestId?: string,
+ recoveredSteerPayload?: RecoveredSteerPayload,
+ creationAttemptId?: string,
+ expectedPredecessorCreatedAt?: number,
+ ): Promise<CreatedJobData> {
+ if (typeof userId !== 'string' || userId.length === 0) {
+ throw new Error('Generation job requires a non-empty user id');
+ }
+ assertCreateIdempotencyArguments(
+ idempotencyClaimKey,
+ idempotencyClaimToken,
+ idempotencyClientRequestId,
+ );
+ if (
+ creationAttemptId != null &&
+ (creationAttemptId.length === 0 || creationAttemptId.length > 128)
+ ) {
+ throw new Error('Invalid generation creation attempt id');
+ }
+ if (
+ expectedPredecessorCreatedAt != null &&
+ (!Number.isSafeInteger(expectedPredecessorCreatedAt) || expectedPredecessorCreatedAt < 0)
+ ) {
+ throw new Error('Invalid expected generation predecessor');
+ }
+ let generationProtocolVersion: 1 | 2 = 1;
+ if (
+ initialMetadata.generationProtocolVersion === 1 ||
+ initialMetadata.generationProtocolVersion === 2
+ ) {
+ generationProtocolVersion = initialMetadata.generationProtocolVersion;
+ } else if (process.env.GENERATION_PROTOCOL_VERSION === '2') {
+ generationProtocolVersion = 2;
+ }
+ const job: CreatedJobData = {
...initialMetadata,
streamId,
userId,
...(tenantId && { tenantId }),
status: 'running',
createdAt: Date.now(),
+ generationProtocolVersion,
...(conversationId !== undefined && { conversationId }),
+ ...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),
+ ...(recoveredSteerId !== undefined && { recoveredSteerId }),
+ providerAbortReady: false,
syncSent: false,
};
+ if (creationAttemptId != null) {
+ Object.defineProperty(job, 'creationAttemptId', {
+ value: creationAttemptId,
+ enumerable: false,
+ configurable: true,
+ });
+ }
const key = KEYS.job(streamId);
@@ -714,20 +1703,84 @@ export class RedisJobStore implements IJobStore {
// The job key uses hash tag {streamId}, runningJobs and userJobs are on different slots
// Generation-state reset + job-hash write happen ATOMICALLY (same-slot Lua).
const hsetPairs = Object.entries(this.serializeJob(job)).flat();
+ const parkedTtl = this.parkedRecoveryTtlSeconds();
+ /** A leased source is hidden while this job is active. Keep it for one
+ * normal recovery window beyond the live job's storage horizon so delayed
+ * cleanup or natural job expiry cannot make both keys disappear together. */
+ const createParkedTtl =
+ recoveredSteerId != null ? this.runningStorageTtlSeconds() + parkedTtl : parkedTtl;
const previousOwner = await this.redis.eval(
JOB_CREATE_LUA,
- 6,
+ 10,
key,
KEYS.chunks(streamId),
KEYS.runSteps(streamId),
KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
KEYS.parkedSteers(streamId),
KEYS.generationEpoch(streamId),
- String(this.ttl.running),
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
+ idempotencyClaimKey != null ? KEYS.idempotency(idempotencyClaimKey) : key,
+ String(this.runningStorageTtlSeconds()),
String(job.createdAt),
String(GENERATION_EPOCH_GRACE_TTL_S),
+ String(createParkedTtl),
+ recoveredSteerId ?? '',
+ userId,
+ tenantId ?? '',
+ idempotencyClaimToken ?? '',
+ recoveredSteerPayload == null ? '' : JSON.stringify(recoveredSteerPayload),
+ String(job.generationProtocolVersion),
+ creationAttemptId ?? '',
+ expectedPredecessorCreatedAt == null ? '' : String(expectedPredecessorCreatedAt),
...hsetPairs,
);
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'claim_lost') {
+ throw new Error('Generation idempotency claim was taken over before job creation');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'owner_mismatch') {
+ throw new Error('Generation job owner mismatch');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'recovery_corrupt') {
+ throw new Error('Generation recovery state is corrupt');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'recovery_payload_mismatch') {
+ throw new RecoveredSteerPayloadMismatchError();
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'replacement_receipt_corrupt') {
+ throw new Error('Generation replacement receipt is corrupt');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'generation_epoch_corrupt') {
+ throw new Error('Generation epoch is corrupt');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'generation_epoch_exhausted') {
+ throw new Error('Generation epoch is exhausted');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'replacement_chain_full') {
+ throw new Error('Generation replacement receipt chain is full');
+ }
+ if (Array.isArray(previousOwner) && previousOwner[3] === 'predecessor_mismatch') {
+ const currentCreatedAt = Number(previousOwner[4]);
+ const currentStatus =
+ typeof previousOwner[5] === 'string' && previousOwner[5] !== ''
+ ? (previousOwner[5] as JobStatus)
+ : undefined;
+ const currentConversationId =
+ typeof previousOwner[6] === 'string' && previousOwner[6] !== ''
+ ? previousOwner[6]
+ : undefined;
+ throw new JobPredecessorMismatchError({
+ createdAt: currentCreatedAt,
+ active:
+ previousOwner[7] === '1' ||
+ currentStatus === 'running' ||
+ currentStatus === 'requires_action',
+ verified: previousOwner[8] !== '0',
+ ...(currentStatus !== undefined && { status: currentStatus }),
+ ...(currentConversationId !== undefined && { conversationId: currentConversationId }),
+ });
+ }
const previousUserId =
Array.isArray(previousOwner) && typeof previousOwner[0] === 'string' ? previousOwner[0] : '';
const previousTenantId =
@@ -738,6 +1791,49 @@ export class RedisJobStore implements IJobStore {
? Number(previousOwner[2])
: job.createdAt;
job.createdAt = Number.isFinite(createdAt) ? createdAt : job.createdAt;
+ if (job.generationProtocolVersion === 2) {
+ job.checkpointNamespace = String(job.createdAt);
+ } else {
+ delete job.checkpointNamespace;
+ }
+ const replacedCreatedAt =
+ Array.isArray(previousOwner) &&
+ (typeof previousOwner[4] === 'string' || typeof previousOwner[4] === 'number') &&
+ String(previousOwner[4]) !== ''
+ ? Number(previousOwner[4])
+ : undefined;
+ const replacedStatus =
+ Array.isArray(previousOwner) && typeof previousOwner[5] === 'string'
+ ? (previousOwner[5] as JobStatus)
+ : undefined;
+ const replacedConversationId =
+ Array.isArray(previousOwner) &&
+ typeof previousOwner[6] === 'string' &&
+ previousOwner[6] !== ''
+ ? previousOwner[6]
+ : undefined;
+ const replacedProviderAbortReady =
+ Array.isArray(previousOwner) &&
+ typeof previousOwner[7] === 'string' &&
+ previousOwner[7] !== ''
+ ? previousOwner[7] === '1'
+ : undefined;
+ const replacedJob =
+ replacedCreatedAt != null && Number.isFinite(replacedCreatedAt) && replacedStatus != null
+ ? {
+ createdAt: replacedCreatedAt,
+ status: replacedStatus,
+ ...(replacedConversationId !== undefined && {
+ conversationId: replacedConversationId,
+ }),
+ }
+ : undefined;
+ if (replacedJob != null && replacedProviderAbortReady != null) {
+ Object.defineProperty(replacedJob, 'providerAbortReady', {
+ value: replacedProviderAbortReady,
+ enumerable: false,
+ });
+ }
const previousUserKeys =
previousUserId !== ''
? [KEYS.userJobs(previousUserId, previousTenantId || undefined)]
@@ -753,14 +1849,37 @@ export class RedisJobStore implements IJobStore {
currentJob !== undefined &&
(!currentJob || currentJob.createdAt !== job.createdAt || currentJob.status !== 'running')
) {
- throw new Error('Generation job was replaced during creation');
+ if (replacedJob != null) {
+ Object.defineProperty(job, 'replacedJob', {
+ value: replacedJob,
+ enumerable: false,
+ });
+ }
+ throw new JobCreationSupersededError(job);
}
if (currentJob === undefined) {
- logger.warn(`[RedisJobStore] Created job without verified membership: ${streamId}`);
- return job;
+ if (replacedJob != null) {
+ Object.defineProperty(job, 'replacedJob', {
+ value: replacedJob,
+ enumerable: false,
+ });
+ }
+ // The same-slot create may already be durable, but exposing a provider
+ // before cross-slot running/user membership is verified would strand an
+ // untracked generation. The manager's attempt-id recovery probes the
+ // durable job, repairs membership, and reconstructs the full predecessor
+ // chain before deciding whether it is safe to proceed.
+ throw new Error('Created job membership could not be verified');
}
this.clearPredecessorLocalState(streamId, currentJob.createdAt);
+ if (replacedJob != null) {
+ Object.defineProperty(currentJob, 'replacedJob', {
+ value: replacedJob,
+ enumerable: false,
+ });
+ }
+
logger.debug(`[RedisJobStore] Created job: ${streamId}`);
return currentJob;
}
@@ -773,12 +1892,49 @@ export class RedisJobStore implements IJobStore {
return this.deserializeJob(data);
}
+ async acknowledgeReplacedJobs(
+ streamId: string,
+ creationAttemptId: string,
+ replacedCreatedAts: readonly number[],
+ ): Promise<boolean> {
+ if (creationAttemptId.length === 0 || replacedCreatedAts.length === 0) {
+ return false;
+ }
+ const acknowledged = await this.redis.eval(
+ REPLACEMENT_RECEIPT_ACK_LUA,
+ 1,
+ KEYS.job(streamId),
+ creationAttemptId,
+ ...replacedCreatedAts.map(String),
+ );
+ return acknowledged === 1;
+ }
+
async updateJob(
streamId: string,
updates: Partial<SerializableJobData>,
expectedCreatedAt?: number,
): Promise<void> {
const key = KEYS.job(streamId);
+ const requestedTerminal =
+ updates.status != null && ['complete', 'error', 'aborted'].includes(updates.status);
+ if (requestedTerminal) {
+ const observed = await this.getJob(streamId);
+ if (
+ observed != null &&
+ (expectedCreatedAt == null || observed.createdAt === expectedCreatedAt) &&
+ (observed.status === 'running' || observed.status === 'requires_action')
+ ) {
+ const { status, ...patch } = updates;
+ await this.transitionStatus(streamId, {
+ from: observed.status,
+ to: status!,
+ patch,
+ expectCreatedAt: expectedCreatedAt ?? observed.createdAt,
+ });
+ return;
+ }
+ }
// Plain field writer. The membership-aware status transitions
// (running ⇄ requires_action — sets, TTLs, the actionId guard) go solely
@@ -789,9 +1945,13 @@ export class RedisJobStore implements IJobStore {
return;
}
- const terminal =
- updates.status != null && ['complete', 'error', 'aborted'].includes(updates.status);
+ const terminal = requestedTerminal;
const observedJob = terminal ? await this.getJob(streamId) : null;
+ const completedTtl =
+ updates.terminalPersistencePending === true ||
+ observedJob?.terminalPersistencePending === true
+ ? Math.max(this.ttl.completed, TERMINAL_PERSISTENCE_RETENTION_TTL_S)
+ : this.ttl.completed;
const fields = Object.entries(serialized).flat();
const updated = await this.redis.eval(
JOB_UPDATE_LUA,
@@ -802,7 +1962,7 @@ export class RedisJobStore implements IJobStore {
KEYS.steers(streamId),
expectedCreatedAt != null ? String(expectedCreatedAt) : '',
terminal ? '1' : '0',
- String(this.ttl.completed),
+ String(completedTtl),
String(this.ttl.chunksAfterComplete),
String(this.ttl.runStepsAfterComplete),
...fields,
@@ -823,6 +1983,24 @@ export class RedisJobStore implements IJobStore {
}
}
+ async finalizeTerminalPersistence(
+ streamId: string,
+ expectedCreatedAt: number,
+ finalEvent: string,
+ ): Promise<boolean> {
+ return (
+ Number(
+ await this.redis.eval(
+ TERMINAL_PERSISTENCE_FINALIZE_LUA,
+ 1,
+ KEYS.job(streamId),
+ String(expectedCreatedAt),
+ finalEvent,
+ ),
+ ) === 1
+ );
+ }
+
private sameMembershipSource(
left: SerializableJobData | null,
right: SerializableJobData | null,
@@ -990,10 +2168,20 @@ export class RedisJobStore implements IJobStore {
private pauseTtlSeconds(pendingAction?: Agents.PendingAction): number {
const exp = pendingAction?.expiresAt;
if (exp == null) {
- return this.ttl.requiresAction;
+ return this.ttl.requiresAction + GENERATION_EPOCH_GRACE_TTL_S;
}
const secondsUntilExpiry = Math.ceil((exp - Date.now()) / 1000) + 60;
- return Math.max(this.ttl.requiresAction, secondsUntilExpiry);
+ return Math.max(this.ttl.requiresAction, secondsUntilExpiry) + GENERATION_EPOCH_GRACE_TTL_S;
+ }
+
+ /** Same grace lets cleanup park accepted words before Redis expires the
+ * live hash/list keys at the semantic stale cutoff. */
+ private runningStorageTtlSeconds(): number {
+ return this.ttl.running + GENERATION_EPOCH_GRACE_TTL_S;
+ }
+
+ private parkedRecoveryTtlSeconds(): number {
+ return this.ttl.completed > 0 ? this.ttl.completed : PARKED_RECOVERY_TTL_S;
}
/** The membership set a status belongs to; terminal statuses have none. */
@@ -1008,6 +2196,25 @@ export class RedisJobStore implements IJobStore {
}
async transitionStatus(streamId: string, args: JobStatusTransition): Promise<boolean> {
+ return (await this.transitionStatusInternal(streamId, args, false)) === true;
+ }
+
+ async transitionStatusAndDrainSteers(
+ streamId: string,
+ args: JobStatusTransition,
+ ): Promise<SteerQueueItem[] | null> {
+ if (this.statusSetKey(args.to) !== null) {
+ throw new Error('Steer-draining status transitions must be terminal');
+ }
+ const result = await this.transitionStatusInternal(streamId, args, true);
+ return Array.isArray(result) ? result : null;
+ }
+
+ private async transitionStatusInternal(
+ streamId: string,
+ args: JobStatusTransition,
+ returnDrainedSteers: boolean,
+ ): Promise<true | SteerQueueItem[] | null> {
const { from, to, patch, clear, expectActionId, expectCreatedAt } = args;
const key = KEYS.job(streamId);
@@ -1019,7 +2226,10 @@ export class RedisJobStore implements IJobStore {
const clearFields = (clear ?? []).map(String);
const terminal = this.statusSetKey(to) === null;
- let ttl = terminal ? this.ttl.completed : this.ttl.running;
+ let ttl = terminal ? this.ttl.completed : this.runningStorageTtlSeconds();
+ if (terminal && patch?.terminalPersistencePending === true) {
+ ttl = Math.max(ttl, TERMINAL_PERSISTENCE_RETENTION_TTL_S);
+ }
if (to === 'requires_action') {
// A paused job must outlive its approval window, even when that window is
// longer than the running TTL — otherwise Redis evicts it before a
@@ -1031,16 +2241,19 @@ export class RedisJobStore implements IJobStore {
// 1) Single-winner decision: an atomic CAS on the single-slot job hash.
// Works identically on cluster and single-node, so two concurrent
// resolves can never both win (and drive the run twice).
- const won = await this.redis.eval(
+ const result = await this.redis.eval(
JOB_CAS_LUA,
- 7,
+ 10,
key,
KEYS.sequence(streamId),
KEYS.chunks(streamId),
KEYS.runSteps(streamId),
KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
KEYS.parkedSteers(streamId),
KEYS.generationEpoch(streamId),
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
from,
expectActionId ?? '',
expectCreatedAt != null ? String(expectCreatedAt) : '',
@@ -1048,14 +2261,16 @@ export class RedisJobStore implements IJobStore {
terminal ? '1' : '0',
String(this.ttl.chunksAfterComplete),
String(this.ttl.runStepsAfterComplete),
- String(this.ttl.completed > 0 ? this.ttl.completed : PARKED_RECOVERY_TTL_S),
+ String(this.parkedRecoveryTtlSeconds()),
String(GENERATION_EPOCH_GRACE_TTL_S),
+ String(args.steerReceiptTtlSeconds ?? 0),
+ returnDrainedSteers ? '1' : '0',
String(clearFields.length),
...clearFields,
...fields,
);
- if (won !== 1) {
- return false;
+ if (returnDrainedSteers ? typeof result !== 'string' : result !== 1) {
+ return null;
}
// 2) Same-slot TTL/content changes happened atomically in the CAS. Cross-slot
@@ -1070,7 +2285,14 @@ export class RedisJobStore implements IJobStore {
expectCreatedAt ?? terminalJob?.createdAt,
);
}
- return true;
+ if (!returnDrainedSteers) {
+ return true;
+ }
+ const drained = JSON.parse(result as string) as unknown;
+ if (!Array.isArray(drained)) {
+ throw new Error('Invalid terminal steer drain response');
+ }
+ return drained as SteerQueueItem[];
}
async claimIdempotencyKey(
@@ -1086,18 +2308,85 @@ export class RedisJobStore implements IJobStore {
String(ttlSeconds * 1000),
);
if (result == null) {
- return { claimed: true };
+ return { claimed: true, existing: value };
}
try {
return { claimed: false, existing: JSON.parse(result as string) as IdempotencyClaimValue };
} catch {
- // Unreachable in practice (we wrote the JSON); proceed rather than dedup to a broken target.
- return { claimed: false };
+ // An unreadable existing owner is outcome-ambiguous. Never turn store
+ // corruption into a duplicate generation by pretending the key is free.
+ throw new Error('Invalid generation idempotency claim');
}
}
- async releaseIdempotencyKey(key: string): Promise<void> {
- await this.redis.del(KEYS.idempotency(key));
+ async takeoverIdempotencyKey(
+ key: string,
+ expected: IdempotencyClaimValue,
+ value: IdempotencyClaimValue,
+ ttlSeconds: number,
+ ): Promise<boolean> {
+ const taken = await this.redis.eval(
+ IDEMPOTENCY_TAKEOVER_LUA,
+ 1,
+ KEYS.idempotency(key),
+ expected.claimToken ?? '',
+ JSON.stringify(value),
+ String(ttlSeconds * 1000),
+ );
+ return taken === 1;
+ }
+
+ async markIdempotencyKeyStarted(
+ key: string,
+ claimToken: string,
+ startedAt: number,
+ ttlSeconds: number,
+ ): Promise<boolean> {
+ const marked = await this.redis.eval(
+ IDEMPOTENCY_MARK_STARTED_LUA,
+ 1,
+ KEYS.idempotency(key),
+ claimToken,
+ String(startedAt),
+ String(ttlSeconds * 1000),
+ );
+ return marked === 1;
+ }
+
+ async adoptIdempotencyKeyForJob(
+ key: string,
+ expected: IdempotencyClaimValue,
+ streamId: string,
+ userId: string,
+ clientRequestId: string,
+ tenantId: string | undefined,
+ expectedCreatedAt: number,
+ ttlSeconds: number,
+ allowMissingClientRequestId = false,
+ ): Promise<boolean> {
+ const adopted = await this.redis.eval(
+ IDEMPOTENCY_ADOPT_LIVE_JOB_LUA,
+ 2,
+ KEYS.idempotency(key),
+ KEYS.job(streamId),
+ expected.claimToken ?? '',
+ String(expectedCreatedAt),
+ userId,
+ clientRequestId,
+ tenantId ?? '',
+ String(ttlSeconds * 1000),
+ allowMissingClientRequestId ? '1' : '0',
+ );
+ return adopted === 1;
+ }
+
+ async releaseIdempotencyKey(key: string, expected?: IdempotencyClaimValue): Promise<void> {
+ await this.redis.eval(
+ IDEMPOTENCY_RELEASE_LUA,
+ 1,
+ KEYS.idempotency(key),
+ expected?.claimToken ?? '',
+ );
}
async deleteJob(streamId: string, expectedCreatedAt?: number): Promise<boolean> {
@@ -1106,11 +2395,12 @@ export class RedisJobStore implements IJobStore {
const expectMissing = expectedCreatedAt == null && observedJob == null;
const deleted = await this.redis.eval(
JOB_DELETE_LUA,
- 4,
+ 5,
KEYS.job(streamId),
KEYS.chunks(streamId),
KEYS.runSteps(streamId),
KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
targetCreatedAt != null ? String(targetCreatedAt) : '',
expectMissing ? '1' : '0',
);
@@ -1134,13 +2424,16 @@ export class RedisJobStore implements IJobStore {
): Promise<boolean> {
const deleted = await this.redis.eval(
STALE_JOB_DELETE_LUA,
- 6,
+ 9,
KEYS.job(streamId),
KEYS.chunks(streamId),
KEYS.runSteps(streamId),
KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
KEYS.parkedSteers(streamId),
KEYS.generationEpoch(streamId),
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
String(observedJob.createdAt),
String(now),
String(this.ttl.running * 1000),
@@ -1271,6 +2564,7 @@ export class RedisJobStore implements IJobStore {
private async cleanupRequiresActionIndex(): Promise<number> {
const streamIds = await this.redis.smembers(KEYS.requiresActionJobs);
let cleaned = 0;
+ const now = Date.now();
const BATCH_SIZE = 50;
for (let i = 0; i < streamIds.length; i += BATCH_SIZE) {
@@ -1293,13 +2587,52 @@ export class RedisJobStore implements IJobStore {
return 1;
}
+ if (job.terminalPersistencePending === true) {
+ const startedAt = job.terminalPersistenceStartedAt ?? job.createdAt;
+ if (now - startedAt < PAUSE_PERSISTENCE_TIMEOUT_MS) {
+ // The pause owner is still within its response-write lease.
+ return 0;
+ }
+
+ if (job.pendingActionId == null) {
+ logger.error(
+ `[RedisJobStore] Refusing stale pause-persistence cleanup without an action fence: ${streamId}`,
+ );
+ return 0;
+ }
+ const failed = await this.transitionStatusAndDrainSteers(streamId, {
+ from: 'requires_action',
+ to: 'error',
+ expectActionId: job.pendingActionId,
+ expectCreatedAt: job.createdAt,
+ patch: {
+ completedAt: now,
+ error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
+ },
+ clear: [
+ 'pendingAction',
+ 'pendingActionId',
+ 'terminalPersistencePending',
+ 'terminalPersistenceStartedAt',
+ ],
+ });
+ if (failed == null) {
+ return 0;
+ }
+ logger.error(`[RedisJobStore] Pause persistence timed out: ${streamId}`);
+ return 1;
+ }
+
// Stale approval (expired, or missing/malformed pendingAction):
// finalize it (aborted) so it stops occupying the slot and its stream
// contents are reclaimed, mirroring ApprovalLifecycle.expire().
// transitionStatus atomically applies the terminal state and same-slot
// content cleanup. Cross-slot membership indexes self-heal on read or
// during the next cleanup pass.
- if (isPendingActionStale(job)) {
+ const exceededNoExpiryBackstop =
+ job.pendingAction?.expiresAt == null &&
+ now - (job.lastActiveAt ?? job.createdAt) > this.ttl.requiresAction * 1000;
+ if (isPendingActionStale(job) || exceededNoExpiryBackstop) {
const expired = await this.transitionStatus(streamId, {
from: 'requires_action',
to: 'aborted',
@@ -1816,7 +3149,7 @@ export class RedisJobStore implements IJobStore {
KEYS.job(streamId),
KEYS.steers(streamId),
JSON.stringify(item),
- String(this.ttl.running),
+ String(this.runningStorageTtlSeconds()),
String(STEER_QUEUE_MAX_DEPTH),
expectedCreatedAt != null ? String(expectedCreatedAt) : '',
);
@@ -1826,28 +3159,152 @@ export class RedisJobStore implements IJobStore {
return result;
}
+ async enqueueSteerVersioned(
+ streamId: string,
+ item: SteerQueueItem,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueVersionedResult> {
+ const result = await this.redis.eval(
+ STEER_ENQUEUE_VERSIONED_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.steers(streamId),
+ JSON.stringify(item),
+ String(this.runningStorageTtlSeconds()),
+ String(STEER_QUEUE_MAX_DEPTH),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ wantsPreempt ? '1' : '0',
+ );
+ if (typeof result === 'number') {
+ return result;
+ }
+ if (typeof result !== 'string') {
+ return STEER_ENQUEUE_NOT_RUNNING;
+ }
+ try {
+ return JSON.parse(result) as Exclude<SteerEnqueueVersionedResult, number>;
+ } catch {
+ logger.warn(`[RedisJobStore] Malformed atomic steer enqueue result for ${streamId}`);
+ return STEER_ENQUEUE_NOT_RUNNING;
+ }
+ }
+
+ async getSteerReceipt(streamId: string, clientSteerId: string): Promise<SteerReceipt | null> {
+ const raw = await this.redis.eval(
+ STEER_RECEIPT_GET_LUA,
+ 3,
+ KEYS.steerReceipts(streamId),
+ KEYS.job(streamId),
+ KEYS.steers(streamId),
+ clientSteerId,
+ );
+ if (typeof raw !== 'string') {
+ return null;
+ }
+ try {
+ return JSON.parse(raw) as SteerReceipt;
+ } catch {
+ logger.warn(`[RedisJobStore] Dropping malformed steer receipt for ${streamId}`);
+ await this.redis.hdel(KEYS.steerReceipts(streamId), clientSteerId);
+ return null;
+ }
+ }
+
+ async enqueueSteerWithReceipt(
+ streamId: string,
+ item: SteerQueueItem,
+ receipt: SteerReceiptInput,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueReceiptResult> {
+ const result = await this.redis.eval(
+ STEER_ENQUEUE_RECEIPT_LUA,
+ 4,
+ KEYS.job(streamId),
+ KEYS.steers(streamId),
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
+ JSON.stringify(item),
+ String(this.runningStorageTtlSeconds()),
+ String(STEER_QUEUE_MAX_DEPTH),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ receipt.clientSteerId,
+ JSON.stringify(receipt),
+ wantsPreempt ? '1' : '0',
+ String(this.runningStorageTtlSeconds()),
+ String(STEER_RECEIPT_MAX_PER_STREAM),
+ );
+ if (typeof result === 'number') {
+ return result;
+ }
+ if (typeof result !== 'string') {
+ return STEER_ENQUEUE_NOT_RUNNING;
+ }
+ try {
+ return JSON.parse(result) as Exclude<SteerEnqueueReceiptResult, number>;
+ } catch {
+ logger.warn(`[RedisJobStore] Malformed atomic steer receipt for ${streamId}`);
+ return STEER_ENQUEUE_NOT_RUNNING;
+ }
+ }
+
async drainSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]> {
const raw = await this.redis.eval(
STEER_DRAIN_LUA,
- 2,
+ 5,
KEYS.job(streamId),
KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ String(this.runningStorageTtlSeconds()),
);
return this.parseSteerItems(raw);
}
+ async restoreClaimedSteers(
+ streamId: string,
+ items: SteerQueueItem[],
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
+ if (items.length === 0) {
+ return true;
+ }
+ const restored = await this.redis.eval(
+ STEER_RESTORE_CLAIMED_LUA,
+ 4,
+ KEYS.job(streamId),
+ KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
+ KEYS.steerReceipts(streamId),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ JSON.stringify(items),
+ String(this.runningStorageTtlSeconds()),
+ );
+ return restored === 1;
+ }
+
async closeAndDrainSteers(
streamId: string,
expectedCreatedAt?: number,
): Promise<SteerQueueItem[]> {
const raw = await this.redis.eval(
STEER_CLOSE_DRAIN_LUA,
- 2,
+ 6,
KEYS.job(streamId),
KEYS.steers(streamId),
+ KEYS.claimedSteers(streamId),
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
+ KEYS.parkedSteers(streamId),
expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ String(this.ttl.completed > 0 ? this.ttl.completed : PARKED_RECOVERY_TTL_S),
);
+ if (raw === 'recovery_corrupt') {
+ throw new Error('Generation recovery state is corrupt or belongs to another owner');
+ }
return this.parseSteerItems(raw);
}
@@ -1865,20 +3322,89 @@ export class RedisJobStore implements IJobStore {
return this.parseSteerItems(raw);
}
+ async peekClaimedSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]> {
+ const raw = await this.redis.eval(
+ STEER_PEEK_CLAIMED_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.claimedSteers(streamId),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ );
+ return this.parseSteerItems(raw);
+ }
+
async clearSteers(streamId: string): Promise<void> {
- await this.redis.del(KEYS.steers(streamId));
+ await this.redis.del(KEYS.steers(streamId), KEYS.claimedSteers(streamId));
}
- async removeSteer(streamId: string, steerId: string): Promise<boolean> {
+ async removeSteer(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<boolean> {
const removed = (await this.redis.eval(
STEER_REMOVE_LUA,
- 1,
+ 3,
+ KEYS.job(streamId),
KEYS.steers(streamId),
- `"steerId":"${steerId}"`,
+ KEYS.steerReceipts(streamId),
+ steerId,
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
)) as number;
return removed === 1;
}
+ async armSteer(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerArmOutcome> {
+ return (await this.armSteerVersioned(streamId, steerId, expectedCreatedAt)).outcome;
+ }
+
+ async armSteerVersioned(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerArmResult> {
+ const result = await this.redis.eval(
+ STEER_ARM_LUA,
+ 3,
+ KEYS.job(streamId),
+ KEYS.steers(streamId),
+ KEYS.steerReceipts(streamId),
+ steerId,
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ );
+ if (typeof result === 'string') {
+ try {
+ const item = JSON.parse(result) as SteerQueueItem;
+ return { outcome: 'armed', revision: item.preemptRevision, item };
+ } catch {
+ return { outcome: 'missing' };
+ }
+ }
+ return { outcome: result === -1 ? 'incapable' : 'missing' };
+ }
+
+ async downgradeSteerPreempts(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerQueueItem[] | null> {
+ const changed = await this.redis.eval(
+ STEER_DOWNGRADE_PREEMPTS_LUA,
+ 3,
+ KEYS.job(streamId),
+ KEYS.steers(streamId),
+ KEYS.steerReceipts(streamId),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ );
+ if (changed === -1) {
+ return null;
+ }
+ return this.parseSteerItems(changed);
+ }
+
async parkSteers(streamId: string, payload: string, expectedCreatedAt?: number): Promise<void> {
const ttl = this.ttl.completed > 0 ? this.ttl.completed : PARKED_RECOVERY_TTL_S;
await this.redis.eval(
@@ -1892,15 +3418,85 @@ export class RedisJobStore implements IJobStore {
);
}
- async claimParkedSteers(streamId: string, ownerFragment: string): Promise<string | undefined> {
- const claimed = (await this.redis.eval(
+ async claimParkedSteers(
+ streamId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ ): Promise<string | undefined> {
+ return (await this.claimParkedSteersDetailed(streamId, ownerUserId, ownerTenantId, 2))?.payload;
+ }
+
+ async claimParkedSteersDetailed(
+ streamId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ requestedProtocolVersion: 1 | 2 = 1,
+ ): Promise<ParkedSteerClaim | undefined> {
+ const claimed = await this.redis.eval(
CLAIM_PARKED_LUA,
- 1,
+ 2,
+ KEYS.parkedSteers(streamId),
+ KEYS.job(streamId),
+ ownerUserId,
+ ownerTenantId ?? '',
+ String(requestedProtocolVersion),
+ );
+ if (
+ !Array.isArray(claimed) ||
+ typeof claimed[0] !== 'string' ||
+ (String(claimed[1]) !== '1' && String(claimed[1]) !== '2')
+ ) {
+ // '' is the non-owner/malformed sentinel (payload left in place).
+ return undefined;
+ }
+ return {
+ payload: claimed[0],
+ generationProtocolVersion: String(claimed[1]) === '2' ? 2 : 1,
+ };
+ }
+
+ async consumeParkedSteer(
+ streamId: string,
+ steerId: string,
+ ownerUserId: string,
+ ownerTenantId: string | undefined,
+ expectedCreatedAt: number,
+ ): Promise<boolean> {
+ const consumed = await this.redis.eval(
+ CONSUME_PARKED_STEER_LUA,
+ 3,
+ KEYS.job(streamId),
KEYS.parkedSteers(streamId),
- ownerFragment,
- )) as string | null;
- // '' is the non-owner sentinel (payload left in place) — reads as "nothing".
- return claimed ? claimed : undefined;
+ KEYS.steerReceipts(streamId),
+ String(expectedCreatedAt),
+ steerId,
+ ownerUserId,
+ ownerTenantId ?? '',
+ );
+ return consumed === 1;
+ }
+
+ async discardSteerLeftover(
+ streamId: string,
+ clientSteerId: string,
+ steerId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ expectedGenerationCreatedAt?: number,
+ ): Promise<boolean> {
+ const discarded = await this.redis.eval(
+ DISCARD_STEER_LEFTOVER_LUA,
+ 3,
+ KEYS.steerReceipts(streamId),
+ KEYS.parkedSteers(streamId),
+ KEYS.job(streamId),
+ clientSteerId,
+ steerId,
+ ownerUserId,
+ ownerTenantId ?? '',
+ expectedGenerationCreatedAt != null ? String(expectedGenerationCreatedAt) : '',
+ );
+ return discarded === 1;
}
/** A malformed entry is dropped (logged) rather than poisoning the drain. */
@@ -1927,7 +3523,12 @@ export class RedisJobStore implements IJobStore {
* Uses XADD for efficient append-only storage.
* Sets TTL on first chunk to ensure cleanup if job crashes.
*/
- async appendChunk(streamId: string, event: unknown, expectedCreatedAt?: number): Promise<void> {
+ async appendChunk(
+ streamId: string,
+ event: unknown,
+ expectedCreatedAt?: number,
+ deliveredSteer?: SteerQueueItem,
+ ): Promise<boolean> {
const key = KEYS.chunks(streamId);
const jobKey = KEYS.job(streamId);
// XADD + derive-and-extend-only EXPIRE in a single atomic eval. Refreshing the TTL on
@@ -1938,16 +3539,28 @@ export class RedisJobStore implements IJobStore {
// The script reads the paused window from the job key, so it bumps to the approval TTL
// even when the pause's own EXPIRE no-op'd because this key didn't exist yet, while a
// normally-running run still settles on the short running TTL. Both keys share the
- // {streamId} hash tag, so the 2-key eval stays on one slot under Redis Cluster.
- await this.redis.eval(
+ // {streamId} hash tag, so the multi-key eval stays on one slot under Redis Cluster.
+ const appended = await this.redis.eval(
CHUNK_APPEND_LUA,
- 2,
+ 8,
key,
jobKey,
+ KEYS.steerReceipts(streamId),
+ KEYS.steerReceiptOrder(streamId),
+ KEYS.claimedSteers(streamId),
+ KEYS.steers(streamId),
+ KEYS.parkedSteers(streamId),
+ KEYS.generationEpoch(streamId),
JSON.stringify(event),
- String(this.ttl.running),
+ String(this.runningStorageTtlSeconds()),
expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ deliveredSteer?.clientSteerId ?? '',
+ deliveredSteer != null ? JSON.stringify(deliveredSteer) : '',
+ String(Date.now()),
+ String(this.parkedRecoveryTtlSeconds()),
+ String(GENERATION_EPOCH_GRACE_TTL_S),
);
+ return appended === 1;
}
/**
@@ -1998,7 +3611,7 @@ export class RedisJobStore implements IJobStore {
KEYS.runSteps(streamId),
KEYS.job(streamId),
JSON.stringify(runSteps),
- String(this.ttl.running),
+ String(this.runningStorageTtlSeconds()),
expectedCreatedAt != null ? String(expectedCreatedAt) : '',
);
}
@@ -2222,21 +3835,32 @@ export class RedisJobStore implements IJobStore {
* Deserialize job data from Redis hash.
*/
private deserializeJob(data: Record<string, string>): SerializableJobData {
- return {
+ const job: CreatedJobData = {
streamId: data.streamId,
userId: data.userId,
tenantId: data.tenantId || undefined,
status: data.status as JobStatus,
createdAt: parseInt(data.createdAt, 10),
+ generationProtocolVersion: data.generationProtocolVersion === '2' ? 2 : 1,
+ checkpointNamespace: data.checkpointNamespace || undefined,
completedAt: data.completedAt ? parseInt(data.completedAt, 10) : undefined,
conversationId: data.conversationId || undefined,
error: data.error || undefined,
+ idempotencyClientRequestId: data.idempotencyClientRequestId || undefined,
+ recoveredSteerId: data.recoveredSteerId || undefined,
userMessage: data.userMessage ? JSON.parse(data.userMessage) : undefined,
responseMessageId: data.responseMessageId || undefined,
createdEventEmitted: data.createdEventEmitted === '1',
sender: data.sender || undefined,
syncSent: data.syncSent === '1',
finalEvent: data.finalEvent || undefined,
+ terminalPersistencePending:
+ data.terminalPersistencePending != null
+ ? data.terminalPersistencePending === '1'
+ : undefined,
+ terminalPersistenceStartedAt: data.terminalPersistenceStartedAt
+ ? parseInt(data.terminalPersistenceStartedAt, 10)
+ : undefined,
endpoint: data.endpoint || undefined,
iconURL: data.iconURL || undefined,
model: data.model || undefined,
@@ -2251,6 +3875,8 @@ export class RedisJobStore implements IJobStore {
* `preemptArmed: false` and silently degrade interrupt-steer to
* tool-boundary steering in EVERY Redis deployment. */
preemptCapable: data.preemptCapable != null ? data.preemptCapable === '1' : undefined,
+ providerAbortReady:
+ data.providerAbortReady != null ? data.providerAbortReady === '1' : undefined,
titleEvent: data.titleEvent || undefined,
replayEvents: data.replayEvents || undefined,
contextUsage: data.contextUsage || undefined,
@@ -2264,6 +3890,123 @@ export class RedisJobStore implements IJobStore {
* that resolved between the snapshot and subscriber attach. */
activityLabels: data.activityLabels != null ? data.activityLabels === '1' : undefined,
};
+
+ if (data.__creationAttemptId) {
+ Object.defineProperty(job, 'creationAttemptId', {
+ value: data.__creationAttemptId,
+ enumerable: false,
+ configurable: true,
+ });
+ }
+
+ let replacedJobs: ReplacedGeneration[] = [];
+ const validReplacementStatuses = new Set([
+ 'running',
+ 'requires_action',
+ 'complete',
+ 'error',
+ 'aborted',
+ ]);
+ const scalarEpochRaw = data.__replacedCreatedAt;
+ const scalarStatus = data.__replacedStatus;
+ if ((scalarEpochRaw != null) !== (scalarStatus != null)) {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ if (data.__replacedGenerations) {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(data.__replacedGenerations);
+ } catch {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ if (!Array.isArray(parsed) || parsed.length === 0 || parsed.length > 32) {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ const seen = new Set<number>();
+ let previousEpoch = -1;
+ for (const value of parsed) {
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ const candidate = value as Record<string, unknown>;
+ if (
+ !Number.isSafeInteger(candidate.createdAt) ||
+ (candidate.createdAt as number) < 0 ||
+ (candidate.createdAt as number) <= previousEpoch ||
+ (candidate.createdAt as number) >= job.createdAt ||
+ !validReplacementStatuses.has(candidate.status as string) ||
+ (candidate.conversationId != null && typeof candidate.conversationId !== 'string') ||
+ (candidate.providerAbortReady != null &&
+ typeof candidate.providerAbortReady !== 'boolean') ||
+ seen.has(candidate.createdAt as number)
+ ) {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ seen.add(candidate.createdAt as number);
+ previousEpoch = candidate.createdAt as number;
+ const receipt: ReplacedGeneration = {
+ createdAt: candidate.createdAt as number,
+ status: candidate.status as JobStatus,
+ ...(candidate.conversationId != null && {
+ conversationId: candidate.conversationId as string,
+ }),
+ };
+ if (candidate.providerAbortReady != null) {
+ Object.defineProperty(receipt, 'providerAbortReady', {
+ value: candidate.providerAbortReady,
+ enumerable: false,
+ });
+ }
+ replacedJobs.push(receipt);
+ }
+ const scalarEpoch = scalarEpochRaw != null ? Number(scalarEpochRaw) : undefined;
+ const latest = replacedJobs[replacedJobs.length - 1];
+ if (
+ !Number.isSafeInteger(scalarEpoch) ||
+ scalarEpoch !== latest.createdAt ||
+ scalarStatus !== latest.status ||
+ (data.__replacedConversationId || undefined) !== latest.conversationId
+ ) {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ } else {
+ const replacedCreatedAt = scalarEpochRaw != null ? Number(scalarEpochRaw) : undefined;
+ const replacedStatus = scalarStatus as JobStatus | undefined;
+ if (replacedCreatedAt != null || replacedStatus != null) {
+ if (
+ replacedCreatedAt == null ||
+ replacedStatus == null ||
+ !Number.isSafeInteger(replacedCreatedAt) ||
+ replacedCreatedAt < 0 ||
+ replacedCreatedAt >= job.createdAt ||
+ !validReplacementStatuses.has(replacedStatus)
+ ) {
+ throw new Error('Invalid generation replacement receipt');
+ }
+ replacedJobs = [
+ {
+ createdAt: replacedCreatedAt,
+ status: replacedStatus,
+ ...(data.__replacedConversationId && {
+ conversationId: data.__replacedConversationId,
+ }),
+ },
+ ];
+ }
+ }
+ if (replacedJobs.length > 0) {
+ Object.defineProperty(job, 'replacedJobs', {
+ value: replacedJobs,
+ enumerable: false,
+ configurable: true,
+ });
+ Object.defineProperty(job, 'replacedJob', {
+ value: replacedJobs[replacedJobs.length - 1],
+ enumerable: false,
+ configurable: true,
+ });
+ }
+ return job;
}
/**
diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts
index 378325d265..05995fd142 100644
--- a/packages/api/src/stream/index.ts
+++ b/packages/api/src/stream/index.ts
@@ -3,6 +3,8 @@ export {
GenerationJobManagerClass,
type CreateGenerationJobOptions,
type GenerationJobManagerOptions,
+ type TerminalJobClaim,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
} from './GenerationJobManager';
export type {
@@ -13,17 +15,38 @@ export type {
AbortResult,
JobStatus,
JobMetadataPatch,
+ GenerationProtocolVersion,
+ GenerationPredecessorState,
+ ParkedSteerClaim,
IJobStore,
+ IJobStoreV2,
} from './interfaces/IJobStore';
// Canonical "is this approval live?" predicate — one definition shared by the
// stores, the approval lifecycle, and the status route / message middleware.
-export { isPendingActionExpired, isPendingActionStale } from './interfaces/IJobStore';
+export {
+ JobPredecessorMismatchError,
+ isPendingActionExpired,
+ isPendingActionStale,
+} from './interfaces/IJobStore';
export {
STEER_ENQUEUE_NOT_RUNNING,
STEER_ENQUEUE_QUEUE_FULL,
+ STEER_ENQUEUE_RECEIPT_FULL,
STEER_QUEUE_MAX_DEPTH,
} from './interfaces/IJobStore';
export { SteeringLifecycle, toPendingSteer } from './SteeringLifecycle';
+export {
+ assertJobStoreV2,
+ getMissingJobStoreV2Methods,
+ JOB_STORE_V2_REQUIRED_METHODS,
+} from './jobStoreCapabilities';
+export type { JobStoreV2RequiredMethod } from './jobStoreCapabilities';
+export {
+ buildRecoveredSteerPayload,
+ canonicalRecoveryFileIds,
+ RecoveredSteerPayloadMismatchError,
+} from './SteerRecovery';
+export type { RecoveredSteerPayload } from './SteerRecovery';
export { createStreamServices } from './createStreamServices';
export type { StreamServicesConfig, StreamServices } from './createStreamServices';
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index 024c624faa..ac6452b318 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -1,5 +1,15 @@
import type { Agents, TFile, TPendingSteer } from 'librechat-data-provider';
import type { StandardGraph } from '@librechat/agents';
+import type { RecoveredSteerPayload } from '../SteerRecovery';
+
+/**
+ * A pause owner has this long to durably persist the interrupted turn before
+ * the barrier is considered abandoned. An abandoned barrier must fail closed:
+ * exposing its action id again would let a resume drive history that may never
+ * have reached the message database.
+ */
+export const PAUSE_PERSISTENCE_TIMEOUT_MS = 30_000;
+export const PAUSE_PERSISTENCE_TIMEOUT_ERROR = 'Paused response persistence timed out';
/**
* Job status enum.
@@ -10,6 +20,10 @@ import type { StandardGraph } from '@librechat/agents';
*/
export type JobStatus = 'running' | 'complete' | 'error' | 'aborted' | 'requires_action';
+/** Immutable wire/storage contract selected when a generation is created.
+ * Missing markers on pre-rollout records are interpreted as protocol v1. */
+export type GenerationProtocolVersion = 1 | 2;
+
/**
* Serializable job data - no object references, suitable for Redis/external storage
*/
@@ -19,10 +33,26 @@ export interface SerializableJobData {
tenantId?: string;
status: JobStatus;
createdAt: number;
+ generationProtocolVersion?: GenerationProtocolVersion;
+ /** Saver-level checkpoint scope for this exact generation. New jobs use
+ * their final store-assigned epoch; LangGraph still sees an empty root
+ * `checkpoint_ns`, which the saver adapter maps to this storage scope.
+ * Legacy paused jobs omit it and use the historical unscoped storage. */
+ checkpointNamespace?: string;
completedAt?: number;
conversationId?: string;
error?: string;
+ /** Stable identity of the HTTP submission that created this generation.
+ * Internal-only: lets an expired idempotency lease recognize the same live
+ * job instead of replacing and billing it again. */
+ idempotencyClientRequestId?: string;
+
+ /** Parked steer leased into this ordinary recovery turn. The source text is
+ * hidden while this exact generation is active and consumed only after the
+ * user message is durably persisted. */
+ recoveredSteerId?: string;
+
/** User message metadata */
userMessage?: {
messageId: string;
@@ -68,6 +98,11 @@ export interface SerializableJobData {
*/
preemptCapable?: boolean;
+ /** Explicitly false until the provider-owning replica has installed its
+ * generation-fenced abort subscription. Missing is conservative legacy
+ * evidence and must be treated like true by replacement handoff. */
+ providerAbortReady?: boolean;
+
/** Whether the user-message created event has been emitted */
createdEventEmitted?: boolean;
@@ -80,6 +115,13 @@ export interface SerializableJobData {
/** Serialized final event for replay */
finalEvent?: string;
+ /** Abort won its terminal CAS but the route is still saving the partial
+ * response / pruning HITL state. No normal terminal payload may be exposed
+ * while true. */
+ terminalPersistencePending?: boolean;
+ /** Crash-recovery deadline basis for `terminalPersistencePending`. */
+ terminalPersistenceStartedAt?: number;
+
/** Serialized title event for replay during active-stream resume */
titleEvent?: string;
@@ -143,6 +185,64 @@ export interface SerializableJobData {
steersClosed?: boolean;
}
+/** Exact active hash replaced by one atomic job creation. Built-in stores keep
+ * this as non-enumerable transaction metadata; Redis also retains a private
+ * receipt in the replacement hash so a committed create with a lost reply can
+ * reconstruct it without exposing it through normal job serialization. */
+export type ReplacedGeneration = Pick<
+ SerializableJobData,
+ 'createdAt' | 'status' | 'conversationId' | 'providerAbortReady'
+>;
+
+/** Latest generation epoch checked by a conditional create. A retained epoch
+ * can outlive its job hash, so inactive mismatches intentionally omit job-only
+ * metadata while still telling the caller which generation won the race.
+ * When all predecessor evidence has expired, `createdAt` safely echoes the
+ * caller's finite expected epoch and `verified` is false; the create is still
+ * rejected, but response consumers can preserve the queued turn without
+ * mistaking that fallback for an observed generation. */
+export type GenerationPredecessorState = Pick<SerializableJobData, 'createdAt'> &
+ Partial<Pick<SerializableJobData, 'status' | 'conversationId'>> & {
+ active: boolean;
+ /** False only when neither the job nor its retained epoch was observable.
+ * Missing values are compatible with pre-marker mismatch producers. */
+ verified?: boolean;
+ };
+
+export interface CreatedJobData extends SerializableJobData {
+ /** The predecessor observed inside the same transaction that installed this
+ * job. Non-enumerable in the built-in stores to keep it out of serializers. */
+ replacedJob?: ReplacedGeneration;
+ /** Transitive transaction receipts inherited from replacements whose
+ * managers may have lost their create replies. Ordered oldest to newest. */
+ replacedJobs?: readonly ReplacedGeneration[];
+ /** Opaque manager-minted proof for one create invocation. Non-enumerable in
+ * built-in stores and used only to reconcile a commit whose reply was lost. */
+ creationAttemptId?: string;
+}
+
+/** The store installed a generation, but another creator replaced it before
+ * cross-slot membership reconciliation completed. The exact predecessor
+ * metadata still has to reach the manager so its owner is stopped/notified. */
+export class JobCreationSupersededError extends Error {
+ constructor(readonly createdJob: CreatedJobData) {
+ super('Generation job was replaced during creation');
+ this.name = 'JobCreationSupersededError';
+ }
+}
+
+/** A conditional create observed a different current generation than the
+ * caller's last authoritative status read. The store rejects before replacing
+ * that generation, so queued client work can be restored safely. */
+export class JobPredecessorMismatchError extends Error {
+ readonly code = 'GENERATION_PREDECESSOR_MISMATCH';
+
+ constructor(readonly currentJob: GenerationPredecessorState) {
+ super('Generation predecessor changed before creation');
+ this.name = 'JobPredecessorMismatchError';
+ }
+}
+
export type JobMetadataPatch = Partial<
Pick<
SerializableJobData,
@@ -158,6 +258,7 @@ export type JobMetadataPatch = Partial<
| 'promptTokens'
| 'discoveredTools'
| 'preemptCapable'
+ | 'generationProtocolVersion'
>
>;
@@ -188,6 +289,9 @@ export function isPendingActionStale(job: Pick<SerializableJobData, 'pendingActi
*/
export interface SteerQueueItem {
steerId: string;
+ /** Client-generated correlation id. It lets a terminal event that beats the
+ * 202 ACK match the server item to its optimistic local chip. */
+ clientSteerId?: string;
text: string;
userId: string;
createdAt: number;
@@ -199,8 +303,50 @@ export interface SteerQueueItem {
* boundary instead of waiting for a tool step. Durable so a parked,
* claimed, or replayed chip keeps its "interrupting" label. */
preempt?: boolean;
+ /** Monotonic per-steer arm revision. A capability downgrade increments it
+ * so an older cross-replica arm cannot resurrect after the disarm, while a
+ * later explicit arm with a newer revision remains valid. */
+ preemptRevision?: number;
+}
+
+/** Durable acknowledgement keyed by `clientSteerId`. It outlives the queue
+ * and live job for a bounded window so a lost 202 cannot make Retry inject
+ * the same instruction twice after drain, terminal cleanup, or replacement. */
+export interface SteerReceipt {
+ clientSteerId: string;
+ fingerprint: string;
+ userId: string;
+ tenantId?: string;
+ agentId?: string;
+ endpoint?: string;
+ /** Generation epoch that accepted the item. Receipts intentionally survive
+ * replacement, so queue absence is only "delivered" while this exact
+ * generation remains active; a newer epoch means the predecessor dropped
+ * an undrained item and the client must recover it as a leftover. */
+ generationCreatedAt: number;
+ item: SteerQueueItem;
+ position: number;
+ /** `claimed` is the crash-recoverable window between an atomic queue drain
+ * and the durable applied-content record. It is not settled: a live client
+ * keeps the chip pending, and a dead/replaced generation repairs it to a
+ * leftover instead of silently dropping the instruction. */
+ /** `recovered` means one recovery surface already claimed a leftover. A
+ * later retry is settled without re-queuing the same follow-up. */
+ state: 'queued' | 'claimed' | 'delivered' | 'leftover' | 'recovered' | 'cancelled';
}
+export type SteerReceiptInput = Omit<SteerReceipt, 'item' | 'position' | 'state'>;
+export type SteerEnqueueReceiptResult = SteerReceipt | SteerEnqueueResult | number;
+
+/** Capability-normalized atomic enqueue result for callers without a stable
+ * client receipt id (legacy/API compatibility path). */
+export interface SteerEnqueueResult {
+ item: SteerQueueItem;
+ position: number;
+}
+
+export type SteerEnqueueVersionedResult = SteerEnqueueResult | number;
+
/**
* Cross-replica preempt signal. Unlike abort this does NOT stop the run — it
* asks the generating replica to seal its current model stream at the next
@@ -213,6 +359,20 @@ export interface PreemptMessage {
/** Generation identity (`SerializableJobData.createdAt`) this belongs to. */
createdAt: number;
steerIds: string[];
+ /** Per-id arm revision. Omitted by legacy senders (treated as revision 0). */
+ revisions?: Record<string, number>;
+}
+
+/** {@link IJobStoreV2.armSteer}: `armed` flipped the flag in place; `missing`
+ * covers every unavailable interleaving (paused, drained, cancelled, closed,
+ * replaced generation); `incapable` means the live owner cannot seal. */
+export type SteerArmOutcome = 'armed' | 'missing' | 'incapable';
+
+export interface SteerArmResult {
+ outcome: SteerArmOutcome;
+ /** Present only for an armed item. */
+ revision?: number;
+ item?: SteerQueueItem;
}
/** Maximum steers a single run can have queued at once. */
@@ -224,6 +384,11 @@ export const STEER_ENQUEUE_NOT_RUNNING = -1;
/** `enqueueSteer` rejection: the queue is at {@link STEER_QUEUE_MAX_DEPTH}. */
export const STEER_ENQUEUE_QUEUE_FULL = -2;
+/** `enqueueSteerWithReceipt` rejection: the bounded, unexpired receipt
+ * history is full. Existing ids still replay; only a new identity is refused
+ * so idempotency evidence is never evicted inside its recovery window. */
+export const STEER_ENQUEUE_RECEIPT_FULL = -3;
+
/**
* Arguments for an atomic {@link IJobStore.transitionStatus} compare-and-set.
*/
@@ -248,16 +413,37 @@ export interface JobStatusTransition {
* the same stream ID.
*/
expectCreatedAt?: number;
+ /** Extend all current steer receipts in the SAME atomic step as this
+ * transition. Used by running→requires_action so no enqueue can land between
+ * a pre-pause TTL pass and the status CAS. */
+ steerReceiptTtlSeconds?: number;
}
/** Value stored under an idempotency claim: the stream a retried request should attach to. */
export interface IdempotencyClaimValue {
streamId: string;
conversationId: string;
+ /** Wire/storage protocol selected by the request that first won this claim.
+ * Missing values are legacy v1. Keeping the marker in both idempotency
+ * tombstones lets a retry recover the original contract after its job hash
+ * has already been cleaned up. */
+ generationProtocolVersion?: GenerationProtocolVersion;
/** Epoch ms the claim was written — lets a losing duplicate tell a winner that is still
* starting (recent, no job yet → retry) from one that already finished and was cleaned
* up (old, no job → attach and let the client refetch). */
claimedAt?: number;
+ /** Lease owner verified atomically by createJob; stale-request takeover
+ * changes it so the abandoned winner can no longer create a generation. */
+ claimToken?: string;
+ /** Proof that a takeover token was derived by CAS from this predecessor.
+ * If a connection drops after the primary CAS but before the legacy CAS,
+ * the next replica can finish only that exact, same-coordinate split rather
+ * than treating an arbitrary token mismatch as repairable. */
+ previousClaimToken?: string;
+ /** Written atomically with job creation. Once present, a missing job is an
+ * already-started/cleaned generation and can never be taken over as an
+ * abandoned pre-create lease. */
+ startedAt?: number;
}
/** Result of an atomic {@link IJobStore.claimIdempotencyKey} attempt. */
@@ -266,6 +452,18 @@ export interface IdempotencyClaimResult {
claimed: boolean;
/** When `claimed` is false, the stream the original request is already driving. */
existing?: IdempotencyClaimValue;
+ /** The durable namespace that established the result. Manager-level claims
+ * use `legacy` only for an old, tokenless claim that must remain
+ * duplicate-only during a rolling upgrade. Store implementations may omit
+ * this field because they operate on one physical key at a time. */
+ source?: 'primary' | 'legacy';
+}
+
+/** Owner-authorized parked recovery payload plus the protocol that controls
+ * its delivery semantics. V1 is destructive (legacy GETDEL); v2 is leased. */
+export interface ParkedSteerClaim {
+ payload: string;
+ generationProtocolVersion: GenerationProtocolVersion;
}
/**
@@ -350,6 +548,14 @@ export interface UsageMetadata {
export interface AbortResult {
/** Whether the abort was successful */
success: boolean;
+ /** Distinguishes an epoch-fenced abort from ordinary not-found/terminal
+ * failures so an HTTP caller can return RUN_REPLACED instead of silently
+ * reporting success for a newer generation it deliberately did not stop. */
+ failureReason?: 'generation_replaced' | 'job_still_active';
+ /** The generation was stopped, but the caller's required durable side
+ * effects failed before normal FINAL publication. The manager emitted a
+ * conservative reconciliation frame instead. */
+ persistenceFailed?: boolean;
/** The job data at time of abort */
jobData: SerializableJobData | null;
/** Aggregated content from the stream */
@@ -391,16 +597,102 @@ export interface ResumeState {
}
/**
- * Interface for job storage backend.
- * Implementations can use in-memory Map, Redis, KV store, etc.
+ * Backward-compatible public job-store contract.
*
- * Content state is tied to jobs:
- * - In-memory: Holds WeakRef to graph for live content/run steps access
- * - Redis: Persists chunks, reconstructs content on reconnect
- *
- * This consolidates job metadata + content state into a single interface.
+ * This is the pre-v2 extension surface kept for third-party stores compiled
+ * against earlier `@librechat/api` releases. The generation manager validates
+ * the additional {@link IJobStoreV2} capabilities before accepting a custom
+ * store at runtime.
*/
export interface IJobStore {
+ initialize(): Promise<void>;
+
+ createJob(
+ streamId: string,
+ userId: string,
+ conversationId?: string,
+ tenantId?: string,
+ initialMetadata?: JobMetadataPatch,
+ ): Promise<SerializableJobData>;
+
+ getJob(streamId: string): Promise<SerializableJobData | null>;
+ updateJob(
+ streamId: string,
+ updates: Partial<SerializableJobData>,
+ expectedCreatedAt?: number,
+ ): Promise<void>;
+ transitionStatus(streamId: string, args: JobStatusTransition): Promise<boolean>;
+
+ claimIdempotencyKey(
+ key: string,
+ value: IdempotencyClaimValue,
+ ttlSeconds: number,
+ ): Promise<IdempotencyClaimResult>;
+ releaseIdempotencyKey(key: string): Promise<void>;
+
+ deleteJob(streamId: string, expectedCreatedAt?: number): Promise<boolean>;
+ hasJob(streamId: string): Promise<boolean>;
+ getRunningJobs(): Promise<SerializableJobData[]>;
+ cleanup(): Promise<number>;
+ recordActivity?(streamId: string, expectedCreatedAt?: number): void;
+ getJobCount(): Promise<number>;
+ getJobCountByStatus(status: JobStatus): Promise<number>;
+ destroy(): Promise<void>;
+ getActiveJobIdsByUser(userId: string, tenantId?: string): Promise<string[]>;
+
+ setGraph(streamId: string, graph: StandardGraph, expectedCreatedAt?: number): void;
+ setContentParts(
+ streamId: string,
+ contentParts: Agents.MessageContentComplex[],
+ expectedCreatedAt?: number,
+ ): void;
+ getContentParts(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<{ content: Agents.MessageContentComplex[] } | null>;
+ getRunSteps(streamId: string, expectedCreatedAt?: number): Promise<Agents.RunStep[]>;
+
+ /** Legacy stores returned `void`; v2 stores return whether the epoch-fenced
+ * append committed. Accept both here so existing implementations remain
+ * source-compatible. */
+ appendChunk(
+ streamId: string,
+ event: unknown,
+ expectedCreatedAt?: number,
+ ): Promise<void | boolean>;
+
+ clearContentState(streamId: string, expectedCreatedAt?: number): void;
+ saveRunSteps?(
+ streamId: string,
+ runSteps: Agents.RunStep[],
+ expectedCreatedAt?: number,
+ ): Promise<void>;
+ setCollectedUsage(
+ streamId: string,
+ collectedUsage: UsageMetadata[],
+ expectedCreatedAt?: number,
+ ): void;
+ getCollectedUsage(streamId: string, expectedCreatedAt?: number): UsageMetadata[];
+
+ enqueueSteer(streamId: string, item: SteerQueueItem, expectedCreatedAt?: number): Promise<number>;
+ drainSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]>;
+ closeAndDrainSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]>;
+ peekSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]>;
+ removeSteer(streamId: string, steerId: string): Promise<boolean>;
+ parkSteers(streamId: string, payload: string, expectedCreatedAt?: number): Promise<void>;
+ claimParkedSteers(streamId: string, ownerFragment: string): Promise<string | undefined>;
+ clearSteers(streamId: string): Promise<void>;
+}
+
+/**
+ * Full store contract required by the current generation runtime.
+ *
+ * Built-in stores implement this interface. Its overrides deliberately narrow
+ * the legacy return types while its extra methods provide the atomicity needed
+ * for terminal ownership, durable steering receipts, recovery, and idempotent
+ * generation startup.
+ */
+export interface IJobStoreV2 extends IJobStore {
/** Initialize the store (e.g., connect to Redis, start cleanup intervals) */
initialize(): Promise<void>;
@@ -411,7 +703,23 @@ export interface IJobStore {
conversationId?: string,
tenantId?: string,
initialMetadata?: JobMetadataPatch,
- ): Promise<SerializableJobData>;
+ recoveredSteerId?: string,
+ idempotencyClaimKey?: string,
+ idempotencyClaimToken?: string,
+ idempotencyClientRequestId?: string,
+ recoveredSteerPayload?: RecoveredSteerPayload,
+ creationAttemptId?: string,
+ expectedPredecessorCreatedAt?: number,
+ ): Promise<CreatedJobData>;
+
+ /** Remove transaction-time predecessor receipts after their handoff was
+ * delivered. The current creation attempt id fences a late acknowledgement
+ * from clearing receipts inherited by a replacement. */
+ acknowledgeReplacedJobs?(
+ streamId: string,
+ creationAttemptId: string,
+ replacedCreatedAts: readonly number[],
+ ): Promise<boolean>;
/** Get a job by streamId (streamId === conversationId) */
getJob(streamId: string): Promise<SerializableJobData | null>;
@@ -426,6 +734,15 @@ export interface IJobStore {
expectedCreatedAt?: number,
): Promise<void>;
+ /** Atomically replaces an abort persistence-pending marker with the one
+ * terminal payload that late subscribers may consume. Exactly one of the
+ * owner, its failure path, or stale-owner recovery wins. */
+ finalizeTerminalPersistence(
+ streamId: string,
+ expectedCreatedAt: number,
+ finalEvent: string,
+ ): Promise<boolean>;
+
/**
* Atomically transition a job's status, **only if** it is currently `from`.
* Returns `true` when the transition fired, `false` when the job was missing
@@ -448,6 +765,20 @@ export interface IJobStore {
*/
transitionStatus(streamId: string, args: JobStatusTransition): Promise<boolean>;
+ /**
+ * Terminal variant of {@link transitionStatus} that atomically captures and
+ * parks every queued/claimed steer owned by the winning generation. `null`
+ * means the status CAS lost; an array (including an empty one) means it won.
+ *
+ * Abort uses this primitive so it can include exact leftovers in its final
+ * event without destructively draining a run before terminal ownership is
+ * established.
+ */
+ transitionStatusAndDrainSteers(
+ streamId: string,
+ args: JobStatusTransition,
+ ): Promise<SteerQueueItem[] | null>;
+
/**
* Atomically claim an idempotency key so a retried start-generation request
* attaches to the original stream instead of starting a second billed
@@ -468,11 +799,47 @@ export interface IJobStore {
ttlSeconds: number,
): Promise<IdempotencyClaimResult>;
+ /** Compare-and-swap an abandoned pre-create claim. */
+ takeoverIdempotencyKey(
+ key: string,
+ expected: IdempotencyClaimValue,
+ value: IdempotencyClaimValue,
+ ttlSeconds: number,
+ ): Promise<boolean>;
+
+ /** Mark a token-owned claim as having created its generation. Used by the
+ * rolling-upgrade bridge to tombstone the legacy key after the same-slot
+ * primary key was committed by createJob. */
+ markIdempotencyKeyStarted(
+ key: string,
+ claimToken: string,
+ startedAt: number,
+ ttlSeconds: number,
+ ): Promise<boolean>;
+
+ /** Atomically attach a freshly reacquired claim to the still-live job that
+ * was created by the same client request. The job and claim keys share a
+ * Redis hash slot, so a replacement cannot cross this validation. */
+ adoptIdempotencyKeyForJob(
+ key: string,
+ expected: IdempotencyClaimValue,
+ streamId: string,
+ userId: string,
+ clientRequestId: string,
+ tenantId: string | undefined,
+ expectedCreatedAt: number,
+ ttlSeconds: number,
+ /** A pre-bridge job has no durable request id. This mode may tombstone a
+ * freshly reacquired claim against that exact active owner/epoch, but must
+ * still reject every different non-empty request id. */
+ allowMissingClientRequestId?: boolean,
+ ): Promise<boolean>;
+
/**
* Release a previously-claimed idempotency key so the submission can be retried
* (e.g. the start failed before generation began). No-op if the key is absent.
*/
- releaseIdempotencyKey(key: string): Promise<void>;
+ releaseIdempotencyKey(key: string, expected?: IdempotencyClaimValue): Promise<void>;
/**
* Delete a job, optionally only when the stream still belongs to the expected
@@ -592,7 +959,14 @@ export interface IJobStore {
* @param expectedCreatedAt - Optional generation identity. When supplied, the append is
* refused if the stream ID now belongs to a replacement generation.
*/
- appendChunk(streamId: string, event: unknown, expectedCreatedAt?: number): Promise<void>;
+ appendChunk(
+ streamId: string,
+ event: unknown,
+ expectedCreatedAt?: number,
+ /** When present, the same durable write records this drained steer as
+ * delivered. Redis performs both mutations in one same-slot Lua step. */
+ deliveredSteer?: SteerQueueItem,
+ ): Promise<boolean>;
/**
* Clear all content state for a job.
@@ -653,6 +1027,31 @@ export interface IJobStore {
*/
enqueueSteer(streamId: string, item: SteerQueueItem, expectedCreatedAt?: number): Promise<number>;
+ /** Atomic capability-normalized enqueue. Unlike enqueue→arm, no fallible
+ * mutation occurs after the item becomes durable and before its ACK. */
+ enqueueSteerVersioned(
+ streamId: string,
+ item: SteerQueueItem,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueVersionedResult>;
+
+ /** Atomic receipt lookup + capability-normalized enqueue. An existing
+ * receipt wins even if the live queue is now full, paused, drained, or the
+ * generation has ended/replaced. Numeric results are enqueue rejection
+ * codes (including {@link STEER_ENQUEUE_RECEIPT_FULL}); a receipt reports
+ * either the existing or newly inserted item. */
+ enqueueSteerWithReceipt(
+ streamId: string,
+ item: SteerQueueItem,
+ receipt: SteerReceiptInput,
+ wantsPreempt: boolean,
+ expectedCreatedAt?: number,
+ ): Promise<SteerEnqueueReceiptResult>;
+
+ /** Read a bounded receipt without requiring the live job/queue. */
+ getSteerReceipt(streamId: string, clientSteerId: string): Promise<SteerReceipt | null>;
+
/**
* Atomically take ALL queued steers, FIFO. Empty array when none. With
* `expectedCreatedAt`, the drain is refused (atomically, inside the store)
@@ -661,6 +1060,15 @@ export interface IJobStore {
*/
drainSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]>;
+ /** Put a failed claimed batch back at the FIFO front, preserving its order
+ * and receipt state. This is the rollback for a durable applied-part write
+ * that failed after the atomic drain. */
+ restoreClaimedSteers(
+ streamId: string,
+ items: SteerQueueItem[],
+ expectedCreatedAt?: number,
+ ): Promise<boolean>;
+
/**
* Atomically CLOSE the queue to new steers, then take all queued items
* FIFO. Used by the terminal paths (final event, abort) so a steer POST
@@ -679,32 +1087,98 @@ export interface IJobStore {
*/
peekSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]>;
+ /** Drained but not yet durably applied items. Kept separate from the live
+ * FIFO so a reconnect can render them without a second owner injecting them. */
+ peekClaimedSteers(streamId: string, expectedCreatedAt?: number): Promise<SteerQueueItem[]>;
+
/** Remove ONE queued steer by id (user-cancelled before injection).
- * False when it was no longer queued — already drained or run ended. */
- removeSteer(streamId: string, steerId: string): Promise<boolean>;
+ * False when it was no longer queued — already drained or run ended — or
+ * when `expectedCreatedAt` belongs to a replaced generation. */
+ removeSteer(streamId: string, steerId: string, expectedCreatedAt?: number): Promise<boolean>;
+
+ /**
+ * Atomically set `preempt: true` on ONE queued steer IN PLACE, preserving
+ * its FIFO position (the user escalated a waiting steer to an interrupt;
+ * the whole queue drains at the seal, so its order must not change).
+ * Guarded like {@link enqueueSteer}: `missing` when the job is not running,
+ * the steer is no longer queued, the queue is closed, or (with
+ * `expectedCreatedAt`) the stream belongs to another generation. The owner's
+ * LIVE `preemptCapable` is part of the same atomic predicate — a HITL resume
+ * on a rolling deploy can rewrite it for the SAME generation, so a value
+ * read before the call is not trustworthy — and an incapable owner answers
+ * `incapable` with the item left unflagged.
+ */
+ armSteer(streamId: string, steerId: string, expectedCreatedAt?: number): Promise<SteerArmOutcome>;
+
+ /** Versioned form used by the HTTP arm route so its publication carries
+ * the revision atomically assigned with the durable flag. */
+ armSteerVersioned(
+ streamId: string,
+ steerId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerArmResult>;
+
+ /** Atomically remove `preempt` from every queued steer only when the
+ * generation's current owner is incapable. Returns the changed steer ids
+ * (used to tombstone late arm publications), or `null` when the generation
+ * is missing/replaced or currently capable. */
+ downgradeSteerPreempts(
+ streamId: string,
+ expectedCreatedAt?: number,
+ ): Promise<SteerQueueItem[] | null>;
/**
* Persist terminally-drained steers under their OWN bounded-TTL key so a
* client with no live subscriber can recover them via the status route.
* Deliberately independent of the job record — the default `completeJob`
* path deletes the job immediately, and recovery must survive that.
- * Overwrites any prior payload; cleared by `createJob` (a replacement run
- * invalidates recovery — a live client had to start it).
+ * Merges with any prior payload. `createJob` leases the selected recovery
+ * item while that generation is active; the item is removed only after its
+ * ordinary user message is durably persisted.
* When `expectedCreatedAt` is supplied, the write is conditional on that
* generation still owning the stream ID.
*/
parkSteers(streamId: string, payload: string, expectedCreatedAt?: number): Promise<void>;
- /**
- * Claim-on-read: atomically return AND remove the parked payload, so a
- * second reload cannot re-mint chips the user already dismissed. The
- * removal is gated on `ownerFragment` (an opaque substring of the
- * serialized payload, e.g. `"userId":"u1"`) INSIDE the same atomic step —
- * a non-owner probe returns nothing and leaves the payload untouched
- * instead of deleting it ahead of the owner check. Stores stay
- * schema-free: the caller parses and authorizes the returned payload.
- */
- claimParkedSteers(streamId: string, ownerFragment: string): Promise<string | undefined>;
+ /** Owner-gated recovery read. A source leased to a matching active recovery
+ * generation is hidden but retained; it becomes visible again if startup
+ * fails, and is removed only after durable user-message persistence. */
+ claimParkedSteers(
+ streamId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ ): Promise<string | undefined>;
+
+ /** Protocol-aware form of {@link claimParkedSteers}. Existing callers may
+ * keep using the payload-only method. */
+ claimParkedSteersDetailed(
+ streamId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ requestedProtocolVersion?: GenerationProtocolVersion,
+ ): Promise<ParkedSteerClaim | undefined>;
+
+ /** Commit a recovery handoff after its ordinary user message is durable.
+ * Guarded by generation identity and owner, and idempotent for retries. */
+ consumeParkedSteer(
+ streamId: string,
+ steerId: string,
+ ownerUserId: string,
+ ownerTenantId: string | undefined,
+ expectedCreatedAt: number,
+ ): Promise<boolean>;
+
+ /** Owner-gated terminal reclaim. Atomically removes one leftover from the
+ * parked payload and marks its receipt cancelled, making edit/queue/dismiss
+ * retries idempotent without reviving an already-started recovery. */
+ discardSteerLeftover(
+ streamId: string,
+ clientSteerId: string,
+ steerId: string,
+ ownerUserId: string,
+ ownerTenantId?: string,
+ expectedGenerationCreatedAt?: number,
+ ): Promise<boolean>;
/** Drop any queued steers (terminal cleanup backstop). */
clearSteers(streamId: string): Promise<void>;
@@ -765,6 +1239,19 @@ export interface IEventTransport {
*/
emitAbort?(streamId: string, generationId?: number): void;
+ /** Awaitable, generation-correlated abort handoff. Resolves true only after
+ * the replica owning that generation processes the abort. */
+ emitAbortConfirmed?(streamId: string, generationId: number): Promise<boolean>;
+
+ /** Publish a predecessor DONE only while the current job's opaque creation
+ * attempt still carries that predecessor in its durable receipt chain. */
+ emitReplacedDoneConfirmed?(
+ streamId: string,
+ event: unknown,
+ replacedGenerationId: number,
+ creationAttemptId: string,
+ ): Promise<void>;
+
/**
* Register callback for abort signals from any replica (Redis mode).
* Called when abort is triggered from any replica.
@@ -775,7 +1262,8 @@ export interface IEventTransport {
*/
onAbort?(
streamId: string,
- callback: (generationId?: number) => void,
+ /** Return true only when this replica owns and stopped the tagged generation. */
+ callback: (generationId?: number) => void | boolean,
): void | (() => void) | Promise<void | (() => void)>;
/**
diff --git a/packages/api/src/stream/internal/chunkPublication.ts b/packages/api/src/stream/internal/chunkPublication.ts
index 0e66f13279..8fcced3fe0 100644
--- a/packages/api/src/stream/internal/chunkPublication.ts
+++ b/packages/api/src/stream/internal/chunkPublication.ts
@@ -2,7 +2,9 @@ import type { IEventTransport } from '../interfaces/IJobStore';
/**
* Internal publication result used to fence same-replica replay against Redis Pub/Sub.
- * `number` is the zero-based Redis sequence and `false` means publication failed.
+ * `number` is the zero-based Redis sequence, `false` is an authoritative
+ * generation/status fence, and `void` carries no ownership proof (including
+ * unsequenced transports and operational publication failure).
*/
export type ChunkPublicationReceipt = number | false | void;
diff --git a/packages/api/src/stream/jobStoreCapabilities.ts b/packages/api/src/stream/jobStoreCapabilities.ts
new file mode 100644
index 0000000000..f3e1de0caf
--- /dev/null
+++ b/packages/api/src/stream/jobStoreCapabilities.ts
@@ -0,0 +1,66 @@
+import type { IJobStore, IJobStoreV2 } from './interfaces/IJobStore';
+
+/**
+ * Methods added by the v2 generation-store contract.
+ *
+ * Keep this list in lockstep with {@link IJobStoreV2}: it is the runtime
+ * boundary that lets the public {@link IJobStore} type remain source-compatible
+ * while rejecting an implementation that cannot provide the atomic guarantees
+ * required by the current generation manager.
+ */
+export const JOB_STORE_V2_REQUIRED_METHODS = [
+ 'acknowledgeReplacedJobs',
+ 'finalizeTerminalPersistence',
+ 'transitionStatusAndDrainSteers',
+ 'takeoverIdempotencyKey',
+ 'markIdempotencyKeyStarted',
+ 'adoptIdempotencyKeyForJob',
+ 'enqueueSteerVersioned',
+ 'enqueueSteerWithReceipt',
+ 'getSteerReceipt',
+ 'restoreClaimedSteers',
+ 'peekClaimedSteers',
+ 'armSteer',
+ 'armSteerVersioned',
+ 'downgradeSteerPreempts',
+ 'claimParkedSteersDetailed',
+ 'consumeParkedSteer',
+ 'discardSteerLeftover',
+] as const satisfies ReadonlyArray<keyof IJobStoreV2>;
+
+export type JobStoreV2RequiredMethod = (typeof JOB_STORE_V2_REQUIRED_METHODS)[number];
+
+type MethodKeys<T> = {
+ [Key in keyof T]-?: NonNullable<T[Key]> extends (...args: never[]) => unknown ? Key : never;
+}[keyof T];
+type V2OnlyMethod = Exclude<MethodKeys<IJobStoreV2>, keyof IJobStore>;
+type SameUnion<Left, Right> = [Left] extends [Right]
+ ? [Right] extends [Left]
+ ? true
+ : false
+ : false;
+type AssertTrue<Value extends true> = Value;
+/** Compile-time tripwire: adding a v2-only method requires updating the runtime assertion. */
+type _AllV2MethodsHaveRuntimeChecks = AssertTrue<SameUnion<V2OnlyMethod, JobStoreV2RequiredMethod>>;
+
+/** Return the v2 capabilities absent from a legacy-compatible store. */
+export function getMissingJobStoreV2Methods(store: IJobStore): JobStoreV2RequiredMethod[] {
+ const candidate = store as unknown as Record<string, unknown>;
+ return JOB_STORE_V2_REQUIRED_METHODS.filter((method) => typeof candidate[method] !== 'function');
+}
+
+/**
+ * Narrow a public/legacy store to the contract the current runtime requires.
+ * Throws before the manager mutates any configuration when capabilities are
+ * missing, so a custom-store upgrade fails loudly and deterministically.
+ */
+export function assertJobStoreV2(store: IJobStore): asserts store is IJobStoreV2 {
+ const missing = getMissingJobStoreV2Methods(store);
+ if (missing.length === 0) {
+ return;
+ }
+
+ throw new TypeError(
+ `Invalid generation job store: missing required IJobStoreV2 method(s): ${missing.join(', ')}`,
+ );
+}
diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts
index e57a08330f..9524c67ab2 100644
--- a/packages/api/src/stream/metadata.ts
+++ b/packages/api/src/stream/metadata.ts
@@ -36,6 +36,9 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): J
if (metadata.preemptCapable !== undefined) {
patch.preemptCapable = metadata.preemptCapable;
}
+ if (metadata.generationProtocolVersion === 1 || metadata.generationProtocolVersion === 2) {
+ patch.generationProtocolVersion = metadata.generationProtocolVersion;
+ }
if (metadata.discoveredTools) {
patch.discoveredTools = metadata.discoveredTools;
}
diff --git a/packages/api/src/types/events.ts b/packages/api/src/types/events.ts
index c1e724ea75..90194b9e51 100644
--- a/packages/api/src/types/events.ts
+++ b/packages/api/src/types/events.ts
@@ -40,6 +40,13 @@ export type FinalMessageFields = {
/** Terminal event emitted when generation completes or is aborted */
export type FinalEvent = {
final: true;
+ /** The terminal status CAS committed, but its normal FINAL payload was not
+ * durably published (for example, the owner crashed in that narrow window).
+ * Clients close the stream and refetch authoritative message/status state. */
+ reconcile?: boolean;
+ reconcileReason?: 'terminal_payload_missing' | 'generation_replaced' | 'abort_persistence_failed';
+ terminalStatus?: 'complete' | 'error' | 'aborted';
+ generationCreatedAt?: number;
requestMessage?: FinalMessageFields | null;
responseMessage?: FinalMessageFields | null;
conversation?: { conversationId?: string; [key: string]: unknown } | null;
diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts
index 2684bb83aa..d9c1d22082 100644
--- a/packages/api/src/types/stream.ts
+++ b/packages/api/src/types/stream.ts
@@ -6,6 +6,11 @@ export interface GenerationJobMetadata {
userId: string;
tenantId?: string;
conversationId?: string;
+ /** Immutable per-generation saver scope. LangGraph's root `checkpoint_ns`
+ * remains empty; the checkpointer adapter maps this scope into storage. */
+ checkpointNamespace?: string;
+ /** Immutable generation protocol. Missing on legacy records means v1. */
+ generationProtocolVersion?: 1 | 2;
/** User message data for rebuilding submission on reconnect */
userMessage?: Agents.UserMessageMeta;
/** Response message ID for tracking */
@@ -32,6 +37,15 @@ export interface GenerationJobMetadata {
discoveredTools?: string[];
/** See `SerializableJobData.preemptCapable`. */
preemptCapable?: boolean;
+ /** Terminal close has atomically stopped new steer acceptance, even if the
+ * final status CAS has not yet run. */
+ steersClosed?: boolean;
+ /** Stable start-submission identity. Duplicate POSTs compare this with their
+ * claim before attaching to a conversation-scoped stream. */
+ idempotencyClientRequestId?: string;
+ /** Normal FINAL publication is waiting on required durable abort work. */
+ terminalPersistencePending?: boolean;
+ terminalPersistenceStartedAt?: number;
/** Set when the job is paused for human review (status === 'requires_action') */
pendingAction?: Agents.PendingAction;
}
@@ -82,6 +96,10 @@ export interface SubscribeOptions {
skipBufferReplay?: boolean;
/** Cancels attachment work when the HTTP client disconnects. */
signal?: AbortSignal;
+ /** Exact generation epoch the caller intends to observe. Conversation ids
+ * are reused by later turns, so a stale client must never attach to a
+ * replacement job that now occupies the same stream id. */
+ expectedCreatedAt?: number;
}
/** Result of an atomic subscribe-with-resume operation */
diff --git a/packages/data-provider/src/createPayload.ts b/packages/data-provider/src/createPayload.ts
index c31e22aef3..62271ac032 100644
--- a/packages/data-provider/src/createPayload.ts
+++ b/packages/data-provider/src/createPayload.ts
@@ -25,6 +25,8 @@ export default function createPayload(submission: t.TSubmission) {
endpointOption,
manualSkills,
clientRequestId,
+ recoverySteerId,
+ expectedPredecessorCreatedAt,
} = submission;
const { conversationId } = s.tConvoUpdateSchema.parse(conversation);
const { endpoint: _e, endpointType } = endpointOption as {
@@ -54,6 +56,8 @@ export default function createPayload(submission: t.TSubmission) {
manualSkills: s.isAssistantsEndpoint(endpoint) ? undefined : manualSkills,
timezone: getUserTimezone(),
clientRequestId,
+ recoverySteerId,
+ expectedPredecessorCreatedAt,
};
return { server, payload };
diff --git a/packages/data-provider/src/types.ts b/packages/data-provider/src/types.ts
index 6053f14cdb..ff7c18053f 100644
--- a/packages/data-provider/src/types.ts
+++ b/packages/data-provider/src/types.ts
@@ -152,6 +152,14 @@ export type TPayload = Partial<TMessage> &
* cannot trigger a second billed generation.
*/
clientRequestId?: string;
+ /** Parked steer source consumed only after this new turn's user message is
+ * durably saved. Separate from `clientRequestId`, which identifies one
+ * generation attempt and must rotate after a failed recovery. */
+ recoverySteerId?: string;
+ /** Optional optimistic-serialization fence for a queued follow-up. The
+ * server may create only if this observed predecessor is still current or
+ * the conversation is still idle after its cleanup. */
+ expectedPredecessorCreatedAt?: number;
};
export type TEditedContent =
@@ -211,6 +219,12 @@ export type TSubmission = {
manualSkills?: string[];
/** Stable per-submission idempotency key (uuid) forwarded to the server to dedup retried start-generation requests. */
clientRequestId?: string;
+ /** Client-only carry-through for a receipt-bound queued recovery. */
+ recoverySteerId?: string;
+ expectedPredecessorCreatedAt?: number;
+ /** Opaque client-only queue restoration metadata; intentionally omitted by
+ * `createPayload`. */
+ queuedMessageOrigin?: unknown;
};
export type EventSubmission = Omit<TSubmission, 'initialResponse'> & { initialResponse: TMessage };
diff --git a/packages/data-provider/src/types/assistants.ts b/packages/data-provider/src/types/assistants.ts
index 7f6710a705..4180eb8ad3 100644
--- a/packages/data-provider/src/types/assistants.ts
+++ b/packages/data-provider/src/types/assistants.ts
@@ -603,6 +603,8 @@ export type SteerContentPart = {
type: ContentTypes.STEER;
steer: string;
steerId?: string;
+ /** Stable optimistic-client id used to settle a POST whose response was lost. */
+ clientSteerId?: string;
createdAt?: number;
/** Attachments steered with the message; re-encoded per turn on replay
* like any other user-message media (refs only, never encoded data). */
diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.ts
index 651e84d0b6..e8a8bbe334 100644
--- a/packages/data-provider/src/types/runs.ts
+++ b/packages/data-provider/src/types/runs.ts
@@ -77,6 +77,8 @@ export enum ApprovalEvents {
*/
export enum SteerEvents {
ON_STEER_APPLIED = 'on_steer_applied',
+ /** Durable capability correction for queued steers after HITL handover. */
+ ON_STEER_UPDATED = 'on_steer_updated',
}
/**
@@ -115,23 +117,31 @@ export type TActivityLabelEvent = {
/** A steer message queued server-side but not yet injected into the run. */
export type TPendingSteer = {
steerId: string;
+ /** Correlates a server steer with its optimistic chip when terminal delivery
+ * races ahead of the POST response. */
+ clientSteerId?: string;
text: string;
createdAt?: number;
files?: Partial<TFile>[];
/** The steer asked to interrupt generation at the next safe boundary —
* kept on parked/replayed chips so the "interrupting" label survives. */
preempt?: boolean;
+ /** Monotonic server revision for last-writer-wins interrupt labels. */
+ preemptRevision?: number;
};
/** Payload of the `on_steer_applied` SSE event. */
export type TSteerAppliedEvent = {
steerId: string;
+ /** Correlates the applied event with the optimistic chip before the POST settles. */
+ clientSteerId?: string;
/** Absolute content index the steer part was injected at. */
index: number;
part: {
type: ContentTypes.STEER;
[ContentTypes.STEER]: string;
steerId?: string;
+ clientSteerId?: string;
createdAt?: number;
files?: Partial<TFile>[];
};
@@ -139,6 +149,17 @@ export type TSteerAppliedEvent = {
conversationId?: string;
};
+/** A queued steer's interrupt label changed without moving its FIFO slot. */
+export type TSteerUpdatedEvent = {
+ conversationId: string;
+ steers: Array<{
+ steerId: string;
+ clientSteerId?: string;
+ preempt: boolean;
+ preemptRevision: number;
+ }>;
+};
+
/** Mirrors TokenBudgetBreakdown from @librechat/agents (data-provider cannot import it). */
export type TTokenBudgetBreakdown = {
maxContextTokens: number;