Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🛑 feat: Preemptive Steer - Backend Interrupt & Steer (#14518)

Danny Avila7bb6651883a7
diff --git a/api/server/controllers/agents/__tests__/client.steerWiring.spec.js b/api/server/controllers/agents/__tests__/client.steerWiring.spec.js
new file mode 100644
index 0000000000..97efceb72f
--- /dev/null
+++ b/api/server/controllers/agents/__tests__/client.steerWiring.spec.js
@@ -0,0 +1,79 @@
+const AgentClient = require('../client');
+const { isSteeringSupported, isSteerPreemptSupported } = require('@librechat/api');
+
+jest.mock('@librechat/api', () => ({
+ ...jest.requireActual('@librechat/api'),
+ isSteeringSupported: jest.fn(() => true),
+ isSteerPreemptSupported: jest.fn(() => true),
+}));
+
+const mockIsSteeringSupported = isSteeringSupported;
+const mockIsPreemptSupported = isSteerPreemptSupported;
+
+/** Minimal `this` for the wiring builder — it only reads these three. */
+function buildWiring(streamId, { jobCreatedAt = 1700000000000 } = {}) {
+ const self = {
+ jobCreatedAt,
+ options: { req: { user: { id: 'user-1' } } },
+ applySteerPart: jest.fn(),
+ };
+ return AgentClient.prototype.buildSteerWiring.call(self, streamId);
+}
+
+describe('AgentClient.buildSteerWiring — preempt capability gating', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockIsSteeringSupported.mockReturnValue(true);
+ mockIsPreemptSupported.mockReturnValue(true);
+ });
+
+ it('returns both boundary hooks and the poll when preempt is supported', () => {
+ const wiring = buildWiring('stream-1');
+
+ expect(typeof wiring.hook).toBe('function');
+ expect(typeof wiring.preemptHook).toBe('function');
+ expect(typeof wiring.preemption?.shouldPreempt).toBe('function');
+ });
+
+ /**
+ * The separate capability probe is what keeps an interrupt affordance from
+ * arming against an SDK that can only inject at tool boundaries: steering
+ * still wires, preemption does not.
+ */
+ it('omits the preempt wiring when only tool-boundary steering is supported', () => {
+ mockIsPreemptSupported.mockReturnValue(false);
+ const wiring = buildWiring('stream-2');
+
+ expect(typeof wiring.hook).toBe('function');
+ expect(wiring.preemptHook).toBeUndefined();
+ expect(wiring.preemption).toBeUndefined();
+ });
+
+ it('returns undefined entirely when steering itself is unsupported', () => {
+ mockIsSteeringSupported.mockReturnValue(false);
+ expect(buildWiring('stream-3')).toBeUndefined();
+ });
+
+ it('returns undefined without a streamId (no resumable job surface)', () => {
+ expect(buildWiring(undefined)).toBeUndefined();
+ expect(buildWiring('')).toBeUndefined();
+ });
+
+ /**
+ * Both boundaries must drain through the same closures, or the two
+ * injection sites could persist steer parts differently — the SDK's
+ * provider-safety argument assumes identical shapes.
+ */
+ it('builds both hooks from one shared closures object', () => {
+ const applySteerPart = jest.fn();
+ const self = {
+ jobCreatedAt: 1700000000000,
+ options: { req: { user: { id: 'user-1' } } },
+ applySteerPart,
+ };
+ const wiring = AgentClient.prototype.buildSteerWiring.call(self, 'stream-4');
+
+ expect(wiring.hook).not.toBe(wiring.preemptHook);
+ expect(applySteerPart).not.toHaveBeenCalled();
+ });
+});
diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
index 1d00544db4..be73779101 100644
--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js
@@ -103,6 +103,9 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
sendEvent: jest.fn(),
+ /** Recorded onto the job so the steer route can honour the OWNING replica's
+ * seal capability rather than its own probe. */
+ isSteerPreemptSupported: jest.fn(() => true),
getViolationInfo: (...args) => mockGetViolationInfo(...args),
buildMessageFiles: jest.fn(() => []),
resolveTitleTiming: jest.fn(() => 'immediate'),
@@ -347,6 +350,8 @@ describe('ResumableAgentController resume metadata', () => {
endpoint: 'agents',
iconURL: 'https://example.com/spec-icon.png',
model: 'gpt-3.5-turbo',
+ /** The OWNING replica's seal capability, read by the steer route. */
+ preemptCapable: true,
agent_id: undefined,
isTemporary: true,
responseMessageId: 'follow-up-user_',
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js
index a52181feb0..56e0a7c8c9 100644
--- a/api/server/controllers/agents/__tests__/resume.spec.js
+++ b/api/server/controllers/agents/__tests__/resume.spec.js
@@ -50,6 +50,10 @@ const mockGenerationJobManager = {
getJobStore: jest.fn(() => mockJobStore),
getResumeState: jest.fn(),
setContentParts: jest.fn(),
+ /** Resume moves ownership: the new owner records its own seal capability
+ * and rebuilds armed interrupts from the durable queue. */
+ updateMetadata: jest.fn().mockResolvedValue(undefined),
+ rearmQueuedPreempts: jest.fn().mockResolvedValue(0),
emitChunk: jest.fn(),
emitDone: jest.fn(),
emitError: jest.fn(),
@@ -261,6 +265,40 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(capturedInit.conversationCreatedAt).toBe('2020-01-02T03:04:05.000Z');
});
+ /**
+ * `approvals.resolve` has already consumed the action and flipped the job
+ * to running by the time steering bookkeeping runs. ioredis queues rather
+ * than rejects during an outage, so `.catch` never fires and an unbounded
+ * await would strand the client: its retry gets a 409 for a spent action,
+ * and neither the continuation nor the failed-resume cleanup runs.
+ */
+ it('answers the resume even when steering bookkeeping never settles', async () => {
+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
+ let releaseRearm;
+ let releaseMetadata;
+ mockGenerationJobManager.rearmQueuedPreempts.mockReturnValue(
+ new Promise((resolve) => {
+ releaseRearm = () => resolve(0);
+ }),
+ );
+ mockGenerationJobManager.updateMetadata.mockReturnValue(
+ new Promise((resolve) => {
+ releaseMetadata = () => resolve(undefined);
+ }),
+ );
+
+ try {
+ const res = await post(approveBody());
+ expect(res.status).toBe(200);
+ expect(res.body.status).toBe('resuming');
+ } finally {
+ releaseRearm?.();
+ releaseMetadata?.();
+ mockGenerationJobManager.rearmQueuedPreempts.mockResolvedValue(0);
+ mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);
+ }
+ }, 15000);
+
it('leaves conversationCreatedAt unset when the convo lookup yields nothing', async () => {
mockGetConvo.mockResolvedValue(null);
mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index c450884165..ef4f34dd0e 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -48,7 +48,10 @@ const {
createContentIndexOffsetHandlers,
createSteerIndexOffsetHandlers,
createSteerDrainHook,
+ createSteerPreemptBoundaryHook,
+ createSteerPreemptPoll,
isSteeringSupported,
+ isSteerPreemptSupported,
buildSteerMedia,
stampSteerPartMedia,
createActivityLabelWiring,
@@ -323,8 +326,11 @@ class AgentClient extends BaseClient {
/**
* The `steering` fragment for `createRun`: the run-scoped PostToolBatch
- * drain hook, or `undefined` when there is no resumable job surface or the
- * installed SDK cannot inject hook messages (draining would drop them).
+ * drain hook — plus, when the SDK can seal mid-stream, the PreemptBoundary
+ * twin and the preempt poll built from the SAME drain closures, so both
+ * boundaries inject byte-identical shapes. `undefined` when there is no
+ * resumable job surface or the installed SDK cannot inject hook messages
+ * (draining would drop them).
*
* @param {string | undefined} streamId
*/
@@ -332,18 +338,23 @@ class AgentClient extends BaseClient {
if (!streamId || !isSteeringSupported()) {
return undefined;
}
+ const drainOptions = {
+ streamId,
+ jobCreatedAt: this.jobCreatedAt,
+ applySteer: (item) => this.applySteerPart(streamId, item),
+ buildMedia: (item) =>
+ buildSteerMedia({
+ client: this,
+ user: this.options.req?.user,
+ item,
+ getFiles: db.getFiles,
+ }),
+ };
return {
- hook: createSteerDrainHook({
- streamId,
- jobCreatedAt: this.jobCreatedAt,
- applySteer: (item) => this.applySteerPart(streamId, item),
- buildMedia: (item) =>
- buildSteerMedia({
- client: this,
- user: this.options.req?.user,
- item,
- getFiles: db.getFiles,
- }),
+ hook: createSteerDrainHook(drainOptions),
+ ...(isSteerPreemptSupported() && {
+ preemptHook: createSteerPreemptBoundaryHook(drainOptions),
+ preemption: createSteerPreemptPoll(streamId),
}),
};
}
diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js
index 172a895f4a..109581f38e 100644
--- a/api/server/controllers/agents/request.js
+++ b/api/server/controllers/agents/request.js
@@ -16,6 +16,7 @@ const {
resolveConversationAnchor,
getAgentStartupTelemetry,
acceptAgentStartupTelemetry,
+ isSteerPreemptSupported,
} = require('@librechat/api');
const { disposeClient, clientRegistry, requestDataMap } = require('~/server/cleanup');
const {
@@ -357,6 +358,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
endpoint: endpointOption.endpoint,
iconURL: endpointIconURL,
model: responseModel,
+ // Recorded HERE because this process owns the generation: the steer
+ // route may land on a different replica whose own SDK probe would
+ // answer for the wrong process during a rolling deploy.
+ preemptCapable: isSteerPreemptSupported(),
// Persist the originating agent so a HITL resume can refuse to rebuild this
// paused run on a different agent (see resume.js).
agent_id: endpointOption.agent_id ?? req.body?.agent_id,
@@ -835,6 +840,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// Check abort state BEFORE calling completeJob (which triggers abort signal for cleanup)
const wasAbortedBeforeComplete = job.abortController.signal.aborted;
+ /**
+ * A preempt boundary that had nothing to inject (cancelled/stale
+ * request) ends the turn with a genuinely truncated answer — the SDK
+ * reports it via preempt stats and the halt reason. Persist it with
+ * the same honest `unfinished` contract an abort gets, never as a
+ * silent completion.
+ */
+ const preemptStats = client?.run?.getPreemptStats?.();
+ const preemptIncomplete =
+ (preemptStats?.emptyBoundaries ?? 0) > 0 ||
+ client?.run?.getHaltReason?.() === 'preempt_incomplete';
const shouldGenerateTitle =
addTitle &&
parentMessageId === Constants.NO_PARENT &&
@@ -861,9 +877,27 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
if (client.savedMessageIds && !client.savedMessageIds.has(messageId)) {
await saveMessage(
reqCtx,
- { ...response, user: userId, unfinished: wasAbortedBeforeComplete },
+ {
+ ...response,
+ user: userId,
+ unfinished: wasAbortedBeforeComplete || preemptIncomplete,
+ },
{ context: 'api/server/controllers/agents/request.js - resumable response end' },
);
+ } else if (preemptIncomplete) {
+ /**
+ * A completed send already saved this row as `unfinished: false`
+ * from `BaseClient.sendMessage`, and registered it in
+ * `savedMessageIds` — so the branch above is skipped and the flag
+ * would never reach the database. An empty preempt boundary IS a
+ * completed send (the SDK halts and returns content), so re-mark
+ * it explicitly, the same way the HITL pause re-marks above.
+ */
+ await saveMessage(
+ reqCtx,
+ { ...response, user: userId, unfinished: true },
+ { context: 'api/server/controllers/agents/request.js - preempt incomplete' },
+ );
}
// Check if our job was replaced by a new request before emitting
@@ -955,7 +989,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
conversation,
title: conversation.title,
requestMessage: sanitizeMessageForTransmit(userMessage),
- responseMessage: { ...response },
+ responseMessage: { ...response, ...(preemptIncomplete && { unfinished: true }) },
...(pendingSteers && { pendingSteers }),
};
diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js
index a33a47be8d..86ab622f67 100644
--- a/api/server/controllers/agents/resume.js
+++ b/api/server/controllers/agents/resume.js
@@ -17,6 +17,7 @@ const {
filterMalformedContentParts,
decrementPendingRequest,
checkAndIncrementPendingRequest,
+ isSteerPreemptSupported,
toPendingSteer,
} = require('@librechat/api');
const { disposeClient } = require('~/server/cleanup');
@@ -33,6 +34,13 @@ const { saveMessage, getConvo, getMessages } = require('~/models');
*/
const MAX_ASK_ANSWER_LENGTH = 16_000;
+/**
+ * How long a resume waits on best-effort steering bookkeeping before answering
+ * anyway. The approval is already consumed by that point, so a stalled Redis
+ * must not strand the client behind a chip label and an arm.
+ */
+const STEER_RESUME_SETUP_TIMEOUT_MS = 1000;
+
/** De-duplicate a merged attachment list by a stable artifact identity. */
function mergeAttachments(existing, incoming) {
const seen = new Set();
@@ -246,6 +254,17 @@ async function finalizeResumedTurn({
// part that breaks reload/rendering.
const content = filterMalformedContentParts(rawContent);
+ /**
+ * A resumed segment can end on an empty preempt boundary just as a fresh
+ * one can — the boundary hook is re-registered by `buildSteerWiring` on
+ * resume. Persisting that as complete would contradict the honest contract
+ * the normal request path now keeps.
+ */
+ const preemptStats = client?.run?.getPreemptStats?.();
+ const preemptIncomplete =
+ (preemptStats?.emptyBoundaries ?? 0) > 0 ||
+ client?.run?.getHaltReason?.() === 'preempt_incomplete';
+
const responseMessage = {
messageId: responseMessageId,
parentMessageId,
@@ -255,7 +274,7 @@ async function finalizeResumedTurn({
endpoint: meta.endpoint,
iconURL: meta.iconURL,
model: meta.model,
- unfinished: false,
+ unfinished: preemptIncomplete,
error: false,
isCreatedByUser: false,
user: userId,
@@ -576,6 +595,70 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
return res.status(409).json({ error: 'This action was already resolved or has expired' });
}
+ /**
+ * Ownership moves on resume, so the job's recorded seal capability must
+ * describe THIS replica — otherwise a job created on a capable replica that
+ * resumes on an older one during a rolling deploy keeps acknowledging
+ * steers as interrupting. Written IMMEDIATELY after the claim, before
+ * client reconstruction: `approvals.resolve` has already flipped the job
+ * back to `running`, so the steer route is accepting requests from here on
+ * and every one of them reads this flag.
+ */
+ const capabilityRefresh = GenerationJobManager.updateMetadata(
+ streamId,
+ { preemptCapable: isSteerPreemptSupported() },
+ job.createdAt,
+ ).catch((error) => {
+ /**
+ * Logged, not fatal: this is a chip-label accuracy write, and failing the
+ * user's resume over it would be disproportionate. The write only fails
+ * when the job store is erroring, in which case the steer route's own
+ * `getJob` is degraded too — it cannot read a stale flag it cannot read.
+ */
+ logger.error('[ResumeAgentController] Failed to refresh preempt capability', error);
+ });
+
+ /**
+ * An interrupt steer enqueued just before the pause survives durably with
+ * its `preempt` flag, but the ARM lived only in the previous owner's
+ * runtime. Rebuild it from the queue so the resumed segment honours an
+ * interrupt the user already had acknowledged.
+ */
+ const preemptRearm = GenerationJobManager.rearmQueuedPreempts(streamId, job.createdAt).catch(
+ (error) => {
+ logger.error('[ResumeAgentController] Failed to re-arm queued preempts', error);
+ },
+ );
+
+ /**
+ * BOUNDED, and the bound is the point. `.catch` only fires on rejection,
+ * but ioredis queues commands while a connection is down instead of
+ * rejecting, so either of these can simply never settle. That would block
+ * here — after `approvals.resolve` has already consumed the action and
+ * flipped the job to `running`, and before both `res.json` and the resume
+ * lifecycle's own try/finally. The client times out, its retry gets a 409
+ * because the action is spent, and neither the continuation nor the
+ * failed-resume cleanup ever runs.
+ *
+ * Both writes are steering bookkeeping — a chip label and an arm that the
+ * next tool boundary would honour anyway — so they finish in the background
+ * rather than holding a resume the user is waiting on.
+ */
+ let steeringSetupTimer;
+ await Promise.race([
+ Promise.all([capabilityRefresh, preemptRearm]),
+ new Promise((resolve) => {
+ steeringSetupTimer = setTimeout(() => {
+ logger.warn(
+ `[ResumeAgentController] Steering setup for ${streamId} still pending after ` +
+ `${STEER_RESUME_SETUP_TIMEOUT_MS}ms; continuing the resume without it`,
+ );
+ resolve();
+ }, STEER_RESUME_SETUP_TIMEOUT_MS);
+ }),
+ ]);
+ clearTimeout(steeringSetupTimer);
+
// Seed the run-scoped MCP request-context store BEFORE the ACK: once `res.json`
// finishes the response, a later `getMCPRequestContext(req, res)` (from tool loading)
// sees `res` as ended and returns undefined, leaving the resumed run without its MCP
diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts
index e99cdff2a3..fb94d680e8 100644
--- a/packages/api/src/agents/run.ts
+++ b/packages/api/src/agents/run.ts
@@ -15,6 +15,7 @@ import type {
ContextPruningConfig,
OpenAIClientOptions,
StandardGraphConfig,
+ StreamPreemption,
LCToolRegistry,
SubagentConfig,
HookCallback,
@@ -49,10 +50,10 @@ import {
import { resolveToolApprovalPolicy, exemptAskUserQuestionFromApproval } from '~/agents/hitl/policy';
import { applyCustomHandoffPromptKeyCompatibility } from '~/agents/handoffPromptKeyCompatibility';
import { stripIntentFromToolRegistry, stripIntentFromToolDefinitions } from '~/agents/intent';
+import { isSteeringSupported, isSteerPreemptSupported } from '~/agents/steering/runtime';
import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm';
import { CREATE_FILE_TOOL_NAME, EDIT_FILE_TOOL_NAME } from '~/agents/tools';
import { getProviderConfig } from '~/endpoints/config/providers';
-import { isSteeringSupported } from '~/agents/steering/runtime';
import { extractDefaultParams } from '~/endpoints/openai/llm';
import { resolveHeaders, createSafeUser } from '~/utils/env';
import { getAgentCheckpointer } from '~/agents/checkpointer';
@@ -1116,7 +1117,21 @@ export async function createRun({
* node). Only the resumable agents controller passes this; the
* OpenAI-compatible and Responses controllers have no job/SSE surface.
*/
- steering?: { hook: HookCallback<'PostToolBatch'> };
+ steering?: {
+ hook: HookCallback<'PostToolBatch'>;
+ /**
+ * The PreemptBoundary twin of `hook`, built via
+ * `createSteerPreemptBoundaryHook` from the same drain closures. Fires
+ * when the SDK seals a model stream mid-generation on a preempt request.
+ */
+ preemptHook?: HookCallback<'PreemptBoundary'>;
+ /**
+ * Level-triggered O(1) poll over the job's armed preempt requests
+ * (`createSteerPreemptPoll`). Threaded into `RunConfig.preemption`, which
+ * also makes the SDK reserve recursion-limit headroom for its seals.
+ */
+ preemption?: StreamPreemption;
+ };
/**
* Run-scoped tool-batch summary hook (PostToolBatch). Like steering, it
* registers independently of the approval policy and needs no checkpointer;
@@ -1495,6 +1510,9 @@ export async function createRun({
if (steering != null && isSteeringSupported()) {
hooks = hooks ?? new HookRegistry();
hooks.register('PostToolBatch', { hooks: [steering.hook] });
+ if (steering.preemptHook != null && isSteerPreemptSupported()) {
+ hooks.register('PreemptBoundary', { hooks: [steering.preemptHook] });
+ }
}
/**
@@ -1595,6 +1613,11 @@ export async function createRun({
// (it gates on result-altering hooks, not registry presence).
...(hitl && { humanInTheLoop: hitl.humanInTheLoop }),
...(hooks && { hooks }),
+ // Preemption is observation-only like the boundary hooks: the poll never
+ // mutates and the SDK refuses to seal unless a PreemptBoundary matcher is
+ // live, so gating both on the same capability keeps them in lockstep.
+ ...(steering?.preemption != null &&
+ isSteerPreemptSupported() && { preemption: steering.preemption }),
};
const run = await Run.create(runConfig);
diff --git a/packages/api/src/agents/steering/__tests__/request.spec.ts b/packages/api/src/agents/steering/__tests__/request.spec.ts
index fd9ed846c5..57301b991a 100644
--- a/packages/api/src/agents/steering/__tests__/request.spec.ts
+++ b/packages/api/src/agents/steering/__tests__/request.spec.ts
@@ -2,25 +2,28 @@ import type { IMongoFile } from '@librechat/data-schemas';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
+import { isSteeringSupported, isSteerPreemptSupported } from '../runtime';
import { STEER_QUEUE_MAX_DEPTH } from '~/stream/interfaces/IJobStore';
import { GenerationJobManager } from '~/stream/GenerationJobManager';
import { handleSteerRequest, handleSteerCancel } from '../request';
-import { isSteeringSupported } from '../runtime';
jest.mock('../runtime', () => ({
...jest.requireActual('../runtime'),
isSteeringSupported: jest.fn(() => true),
+ isSteerPreemptSupported: jest.fn(() => true),
}));
jest.spyOn(console, 'log').mockImplementation();
const mockIsSupported = isSteeringSupported as jest.Mock;
+const mockIsPreemptSupported = isSteerPreemptSupported as jest.Mock;
const user = { id: 'user-1' };
describe('handleSteerRequest (real in-memory job manager)', () => {
beforeEach(() => {
jest.clearAllMocks();
mockIsSupported.mockReturnValue(true);
+ mockIsPreemptSupported.mockReturnValue(true);
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
@@ -425,6 +428,38 @@ describe('handleSteerCancel (real in-memory job manager)', () => {
expect(result).toEqual({ status: 200, body: { removed: false } });
});
+ /**
+ * ioredis queues commands during an outage instead of rejecting, so an
+ * unbounded wait on the disarm publish can hang for the length of the
+ * outage — with the steer already durably cancelled. A client that gives up
+ * treats the cancel as failed and restores a chip for a steer that can
+ * never produce an applied event.
+ */
+ it('answers the cancel even when the disarm publish never settles', async () => {
+ const steerId = await queueSteer('cancel-stalled-disarm');
+ let release: (() => void) | undefined;
+ const spy = jest.spyOn(GenerationJobManager, 'noteSteersRemoved').mockReturnValue(
+ new Promise<boolean>((resolve) => {
+ release = () => resolve(true);
+ }),
+ );
+
+ try {
+ const result = await handleSteerCancel(user, {
+ conversationId: 'cancel-stalled-disarm',
+ steerId,
+ });
+
+ expect(result).toEqual({ status: 200, body: { removed: true } });
+ expect(spy).toHaveBeenCalledWith('cancel-stalled-disarm', [steerId], expect.any(Number));
+ /** Durably gone regardless — the wait was only ever a head start. */
+ expect(await GenerationJobManager.steering.peek('cancel-stalled-disarm')).toEqual([]);
+ } finally {
+ release?.();
+ spy.mockRestore();
+ }
+ }, 10000);
+
it('403s another user and leaves the steer queued', async () => {
const steerId = await queueSteer('cancel-foreign');
const result = await handleSteerCancel(
@@ -435,3 +470,266 @@ describe('handleSteerCancel (real in-memory job manager)', () => {
expect((await GenerationJobManager.steering.peek('cancel-foreign')).length).toBe(1);
});
});
+
+describe('preempt flag on the steer request', () => {
+ /** The owning replica records its own seal capability at createJob; the
+ * route honours that rather than probing its own SDK. */
+ function createCapableJob(streamId: string) {
+ return GenerationJobManager.createJob(streamId, user.id, undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
+ }
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockIsSupported.mockReturnValue(true);
+ mockIsPreemptSupported.mockReturnValue(true);
+ GenerationJobManager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ GenerationJobManager.initialize();
+ });
+
+ afterEach(async () => {
+ await GenerationJobManager.destroy();
+ });
+
+ it('arms the request, marks the queued item, and echoes preempt: true', async () => {
+ const streamId = 'preempt-req-armed';
+ await createCapableJob(streamId);
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'stop and do this instead',
+ preempt: true,
+ });
+
+ expect(result.status).toBe(202);
+ expect(result.body.preempt).toBe(true);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);
+ });
+
+ it('omits the flag for an ordinary steer and never arms', async () => {
+ const streamId = 'preempt-req-plain';
+ await GenerationJobManager.createJob(streamId, user.id);
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'ordinary steer',
+ });
+
+ expect(result.status).toBe(202);
+ expect(result.body.preempt).toBe(false);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();
+ });
+
+ /**
+ * The guard ladder — ownership, tenant, paused-state, agent ACL — runs
+ * against the job read at the top. The owner re-read happens several awaits
+ * later and can cross a replacement, so a different generation there is a
+ * run this request was never authorized against. Accepting into it would
+ * carry the wrong agent's metadata.
+ */
+ it('refuses when the run is replaced between the guards and the owner re-read', async () => {
+ const streamId = 'preempt-req-replaced-midflight';
+ const original = await createCapableJob(streamId);
+ const originalSnapshot = await GenerationJobManager.getJob(streamId);
+ /** The run is genuinely replaced, so the owner re-read returns a REAL
+ * live generation — an enqueue fenced to it would succeed. */
+ const replacement = await createCapableJob(streamId);
+ expect(replacement.createdAt).not.toBe(original.createdAt);
+
+ const realGetJob = GenerationJobManager.getJob.bind(GenerationJobManager);
+ let calls = 0;
+ const spy = jest
+ .spyOn(GenerationJobManager, 'getJob')
+ .mockImplementation(async (id: string) => {
+ calls += 1;
+ /** The guard ladder ran before the replacement; the owner re-read after. */
+ return calls === 1 ? originalSnapshot : realGetJob(id);
+ });
+
+ try {
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'interrupt me',
+ preempt: true,
+ });
+
+ expect(result.status).toBe(404);
+ expect(result.body.code).toBe('NO_ACTIVE_RUN');
+ /** Nothing reached the replacement's queue. */
+ expect(await GenerationJobManager.steering.peek(streamId)).toEqual([]);
+ } finally {
+ spy.mockRestore();
+ }
+ });
+
+ /**
+ * The queue item is already durable by the time the arm is published, and
+ * the 202 reports capability rather than delivery — so waiting on a stalled
+ * Redis buys nothing and risks the caller timing out and retrying, which
+ * would mint a SECOND steer alongside the first and inject the same
+ * instruction twice. If this ever awaits again, this test times out.
+ */
+ it('does not hold the 202 behind a stalled preempt publish', async () => {
+ const streamId = 'preempt-req-stalled-publish';
+ await createCapableJob(streamId);
+
+ let release: (() => void) | undefined;
+ const neverSettles = new Promise<boolean>((resolve) => {
+ release = () => resolve(true);
+ });
+ const spy = jest.spyOn(GenerationJobManager, 'requestPreempt').mockReturnValue(neverSettles);
+
+ try {
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'interrupt me',
+ preempt: true,
+ });
+
+ expect(result.status).toBe(202);
+ expect(result.body.preempt).toBe(true);
+ expect(spy).toHaveBeenCalledWith(streamId, expect.any(String), expect.any(Number));
+ /** Still durable, so the boundary drain will find it either way. */
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);
+ } finally {
+ release?.();
+ spy.mockRestore();
+ }
+ });
+
+ /**
+ * The mirror of the owner-incapable case below, and the direction a local
+ * probe used to get wrong: during a rolling deploy the steer can land on an
+ * un-upgraded replica while a capable replica owns the generation. The
+ * route never seals — it enqueues and publishes an arm — so its own SDK is
+ * irrelevant and dropping the interrupt here would lose it for no reason.
+ */
+ it('honours a capable OWNER even when the routing replica cannot seal', async () => {
+ mockIsPreemptSupported.mockReturnValue(false);
+ const streamId = 'preempt-req-old-router';
+ await createCapableJob(streamId);
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'interrupt me',
+ preempt: true,
+ });
+
+ expect(result.status).toBe(202);
+ expect(result.body.preempt).toBe(true);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);
+ });
+
+ /**
+ * Rolling deploy: the route replica can seal but the replica that OWNS the
+ * generation cannot. Labelling it "interrupting" would lie.
+ */
+ it('degrades when the OWNING replica recorded no seal capability', async () => {
+ const streamId = 'preempt-req-old-owner';
+ await GenerationJobManager.createJob(streamId, user.id);
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'interrupt me',
+ preempt: true,
+ });
+
+ expect(result.status).toBe(202);
+ expect(result.body.preempt).toBe(false);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ it('does not arm when the enqueue itself is rejected', async () => {
+ const streamId = 'preempt-req-full';
+ await createCapableJob(streamId);
+ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH; i++) {
+ await GenerationJobManager.steering.enqueue(streamId, {
+ steerId: `filler-${i}`,
+ text: `filler ${i}`,
+ userId: user.id,
+ createdAt: Date.now(),
+ });
+ }
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'too late',
+ preempt: true,
+ });
+
+ expect(result.status).toBe(429);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * Cancel is live UI. Without this the request stays armed after its steer
+ * is gone and seals an unrelated stretch of generation.
+ */
+ /**
+ * Option A semantics: the 202's `preempt` mirrors the DURABLE queue flag,
+ * so the response and `SteerQueueItem.preempt` can never disagree — a
+ * resumed owner re-arming from the queue then honours exactly what the
+ * client was told.
+ */
+ it('the 202 flag and the durable queue item always agree', async () => {
+ const streamId = 'preempt-flag-agrees';
+ await createCapableJob(streamId);
+
+ const result = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'interrupt me',
+ preempt: true,
+ });
+ const queued = (await GenerationJobManager.steering.peek(streamId))[0];
+
+ expect(result.body.preempt).toBe(true);
+ expect(queued.preempt).toBe(true);
+ expect(result.body.preempt).toBe(queued.preempt === true);
+ });
+
+ it('cancelling a preempt steer disarms the request', async () => {
+ const streamId = 'preempt-req-cancel';
+ await createCapableJob(streamId);
+ const queued = await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'never mind',
+ preempt: true,
+ });
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+
+ const cancelled = await handleSteerCancel(user, {
+ conversationId: streamId,
+ steerId: queued.body.steerId,
+ });
+
+ expect(cancelled.body.removed).toBe(true);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ it('a lost cancel race leaves the surviving request armed', async () => {
+ const streamId = 'preempt-req-cancel-miss';
+ await createCapableJob(streamId);
+ await handleSteerRequest(user, {
+ conversationId: streamId,
+ text: 'still queued',
+ preempt: true,
+ });
+
+ const cancelled = await handleSteerCancel(user, {
+ conversationId: streamId,
+ steerId: 'never-existed',
+ });
+
+ expect(cancelled.body.removed).toBe(false);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+ });
+});
diff --git a/packages/api/src/agents/steering/__tests__/runtime.spec.ts b/packages/api/src/agents/steering/__tests__/runtime.spec.ts
index 2a32d52e3b..a51242fd6e 100644
--- a/packages/api/src/agents/steering/__tests__/runtime.spec.ts
+++ b/packages/api/src/agents/steering/__tests__/runtime.spec.ts
@@ -1,5 +1,9 @@
import * as agentsSdk from '@librechat/agents';
-import type { PostToolBatchHookInput, PostToolBatchHookOutput } from '@librechat/agents';
+import type {
+ PostToolBatchHookInput,
+ PostToolBatchHookOutput,
+ PreemptBoundaryHookInput,
+} from '@librechat/agents';
import type { SteerQueueItem } from '~/stream/interfaces/IJobStore';
/** Mirrors runtime.ts's local extension — the field predates the SDK pin bump. */
@@ -9,7 +13,13 @@ type SteerDrainOutput = PostToolBatchHookOutput & {
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { GenerationJobManager } from '~/stream/GenerationJobManager';
-import { createSteerDrainHook, isSteeringSupported } from '../runtime';
+import {
+ createSteerDrainHook,
+ createSteerPreemptBoundaryHook,
+ createSteerPreemptPoll,
+ isSteeringSupported,
+ isSteerPreemptSupported,
+} from '../runtime';
jest.spyOn(console, 'log').mockImplementation();
@@ -28,6 +38,17 @@ function buildSteer(steerId: string, text: string): SteerQueueItem {
return { steerId, text, userId: 'user-1', createdAt: Date.now() };
}
+function boundaryInput(
+ overrides: Partial<PreemptBoundaryHookInput> = {},
+): PreemptBoundaryHookInput {
+ return {
+ hook_event_name: 'PreemptBoundary',
+ runId: 'run-1',
+ sealCount: 1,
+ ...overrides,
+ };
+}
+
describe('isSteeringSupported', () => {
it('mirrors the installed SDK capability flag AND replay support', () => {
// CI runs against the published SDK pin (possibly pre-injectedMessages);
@@ -235,3 +256,278 @@ describe('createSteerDrainHook', () => {
]);
});
});
+
+describe('isSteerPreemptSupported', () => {
+ it('requires the preempt capability ON TOP of full steering support', () => {
+ const sdk = agentsSdk as {
+ HOOK_INJECTED_MESSAGES_CAPABLE?: boolean;
+ HOOK_PREEMPT_BOUNDARY_CAPABLE?: boolean;
+ ContentTypes?: { STEER?: string };
+ };
+ const expected = isSteeringSupported() && sdk.HOOK_PREEMPT_BOUNDARY_CAPABLE === true;
+ expect(isSteerPreemptSupported()).toBe(expected);
+ });
+});
+
+describe('createSteerPreemptBoundaryHook', () => {
+ beforeEach(() => {
+ GenerationJobManager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ GenerationJobManager.initialize();
+ });
+
+ afterEach(async () => {
+ await GenerationJobManager.destroy();
+ });
+
+ it('drains the same queue and emits the same shapes as the tool boundary', async () => {
+ const streamId = `preempt-drain-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'first'));
+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s2', 'second'));
+
+ const applied: string[] = [];
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: (item) => {
+ applied.push(item.text);
+ },
+ });
+
+ const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);
+ expect(applied).toEqual(['first', 'second']);
+ expect(output.injectedMessages).toEqual([
+ { role: 'user', content: 'first', source: 'steer' },
+ { role: 'user', content: 'second', source: 'steer' },
+ ]);
+ expect(await GenerationJobManager.steering.peek(streamId)).toEqual([]);
+ });
+
+ it('returns empty output when the queue drained before the seal landed', async () => {
+ const streamId = `preempt-empty-${Date.now()}`;
+ await GenerationJobManager.createJob(streamId, 'user-1');
+ const hook = createSteerPreemptBoundaryHook({ streamId, applySteer: jest.fn() });
+
+ expect(await hook(boundaryInput(), abortSignal)).toEqual({});
+ });
+
+ it('never drains inside a subagent scope', async () => {
+ const streamId = `preempt-subagent-${Date.now()}`;
+ await GenerationJobManager.createJob(streamId, 'user-1');
+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'kept'));
+
+ const hook = createSteerPreemptBoundaryHook({ streamId, applySteer: jest.fn() });
+ expect(await hook(boundaryInput({ agentId: 'child-agent' }), abortSignal)).toEqual({});
+ expect((await GenerationJobManager.steering.peek(streamId)).map((s) => s.text)).toEqual([
+ 'kept',
+ ]);
+ });
+
+ it('refuses to drain when the job was replaced', async () => {
+ const streamId = `preempt-replaced-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt - 1,
+ applySteer: jest.fn(),
+ });
+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'new job steer'));
+
+ expect(await hook(boundaryInput(), abortSignal)).toEqual({});
+ expect((await GenerationJobManager.steering.peek(streamId)).map((s) => s.text)).toEqual([
+ 'new job steer',
+ ]);
+ });
+
+ it('clears the armed preempt request after draining', async () => {
+ const streamId = `preempt-clears-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ await GenerationJobManager.steering.enqueue(streamId, {
+ ...buildSteer('s1', 'interrupt me'),
+ preempt: true,
+ });
+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: jest.fn(),
+ });
+ await hook(boundaryInput(), abortSignal);
+
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * The shared drain body is what makes this true: a preempt satisfied at an
+ * ordinary tool boundary must disarm there, or the request would seal a
+ * later, unrelated stretch of generation with nothing left to inject.
+ */
+ it('a tool-boundary drain also clears a pending preempt request', async () => {
+ const streamId = `preempt-tool-clears-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ await GenerationJobManager.steering.enqueue(streamId, {
+ ...buildSteer('s1', 'interrupt me'),
+ preempt: true,
+ });
+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);
+
+ const hook = createSteerDrainHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: jest.fn(),
+ });
+ await hook(batchInput(), abortSignal);
+
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * The empty-boundary self-clear: a seal was spent, so anything still armed
+ * points at a steer that already left the queue. Leaving it armed would
+ * seal again on the next chunk and truncate an unrelated answer.
+ */
+ it('disarms the generation when the boundary drains nothing', async () => {
+ const streamId = `preempt-empty-clears-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ GenerationJobManager.requestPreempt(streamId, 'orphaned', job.createdAt);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: jest.fn(),
+ });
+ expect(await hook(boundaryInput(), abortSignal)).toEqual({});
+
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * A cancel whose cross-replica clear was lost leaves a stale arm. If the
+ * next boundary drains a DIFFERENT steer, clearing only the drained id
+ * would leave the stale one level-triggered — it would immediately seal
+ * the continuation meant to answer the steer just injected, landing on an
+ * empty boundary as `preempt_incomplete`.
+ */
+ it('a nonempty drain also clears stale arms held since the snapshot', async () => {
+ const streamId = `preempt-stale-snapshot-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+
+ /** Stale: armed, but its steer never reaches the queue (cancelled). */
+ await GenerationJobManager.requestPreempt(streamId, 'steer-cancelled', job.createdAt);
+ /** Live: queued and armed, and this is what the boundary will drain. */
+ await GenerationJobManager.steering.enqueue(streamId, {
+ ...buildSteer('steer-live', 'interrupt me'),
+ preempt: true,
+ });
+ await GenerationJobManager.requestPreempt(streamId, 'steer-live', job.createdAt);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);
+
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: jest.fn(),
+ });
+ const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);
+
+ expect(output.injectedMessages).toHaveLength(1);
+ /** Both the drained id and the stale snapshot id are spent. */
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /** An arm that lands AFTER the snapshot is backed by a live queue item. */
+ it('a nonempty drain spares an arm that landed after the snapshot', async () => {
+ const streamId = `preempt-post-snapshot-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ await GenerationJobManager.steering.enqueue(streamId, {
+ ...buildSteer('steer-first', 'first'),
+ preempt: true,
+ });
+ await GenerationJobManager.requestPreempt(streamId, 'steer-first', job.createdAt);
+
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: async () => {
+ /** Arrives mid-drain, after the snapshot was taken. */
+ await GenerationJobManager.requestPreempt(streamId, 'steer-later', job.createdAt);
+ },
+ });
+ await hook(boundaryInput(), abortSignal);
+
+ expect(GenerationJobManager.getArmedPreemptIds(streamId, job.createdAt)).toEqual([
+ 'steer-later',
+ ]);
+ });
+
+ it('clears the request even when applySteer throws mid-drain', async () => {
+ const streamId = `preempt-clears-on-error-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ await GenerationJobManager.steering.enqueue(streamId, {
+ ...buildSteer('s1', 'still injected'),
+ preempt: true,
+ });
+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);
+
+ const hook = createSteerPreemptBoundaryHook({
+ streamId,
+ jobCreatedAt: job.createdAt,
+ applySteer: () => {
+ throw new Error('emit failed');
+ },
+ });
+
+ const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);
+ expect(output.injectedMessages).toEqual([
+ { role: 'user', content: 'still injected', source: 'steer' },
+ ]);
+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);
+ });
+});
+
+describe('createSteerPreemptPoll', () => {
+ beforeEach(() => {
+ GenerationJobManager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ GenerationJobManager.initialize();
+ });
+
+ afterEach(async () => {
+ await GenerationJobManager.destroy();
+ });
+
+ /**
+ * The SDK contract requires a LEVEL-triggered predicate: it may be polled
+ * many times before a chunk is safe to seal, and a self-clearing read would
+ * silently lose the request on the first unsafe chunk.
+ */
+ it('keeps returning true until the request is cleared', async () => {
+ const streamId = `preempt-poll-${Date.now()}`;
+ const job = await GenerationJobManager.createJob(streamId, 'user-1');
+ const { shouldPreempt } = createSteerPreemptPoll(streamId);
+
+ expect(shouldPreempt()).toBe(false);
+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);
+ expect(shouldPreempt()).toBe(true);
+ expect(shouldPreempt()).toBe(true);
+ expect(shouldPreempt()).toBe(true);
+
+ GenerationJobManager.noteSteersRemoved(streamId, ['s1'], job.createdAt);
+ expect(shouldPreempt()).toBe(false);
+ });
+
+ it('is false for a stream with no live generation', () => {
+ expect(createSteerPreemptPoll('no-such-stream').shouldPreempt()).toBe(false);
+ });
+});
diff --git a/packages/api/src/agents/steering/index.ts b/packages/api/src/agents/steering/index.ts
index a3a79c78f2..01c8a68fcb 100644
--- a/packages/api/src/agents/steering/index.ts
+++ b/packages/api/src/agents/steering/index.ts
@@ -1,4 +1,10 @@
-export { createSteerDrainHook, isSteeringSupported } from './runtime';
+export {
+ createSteerDrainHook,
+ createSteerPreemptBoundaryHook,
+ createSteerPreemptPoll,
+ isSteeringSupported,
+ isSteerPreemptSupported,
+} from './runtime';
export type { SteerDrainHookOptions, SteerMediaResult } from './runtime';
export {
handleSteerRequest,
diff --git a/packages/api/src/agents/steering/request.ts b/packages/api/src/agents/steering/request.ts
index 6acd9eaa33..b37b1acef4 100644
--- a/packages/api/src/agents/steering/request.ts
+++ b/packages/api/src/agents/steering/request.ts
@@ -12,6 +12,13 @@ export const STEER_MAX_FILES = 10;
const DEFAULT_STEER_MAX_LENGTH = 16000;
+/**
+ * How long a cancel waits for its disarm to publish before answering anyway.
+ * The steer is already durably removed by then, so this only buys the clear a
+ * head start; a stalled Redis must not hold the response open behind it.
+ */
+const STEER_DISARM_ACK_TIMEOUT_MS = 1000;
+
/** Character cap for a single steer message (env-overridable). */
export function getSteerMaxLength(): number {
return parseInt(process.env.STEER_MAX_LENGTH ?? '', 10) || DEFAULT_STEER_MAX_LENGTH;
@@ -26,6 +33,11 @@ export interface SteerRequestBody {
conversationId?: unknown;
text?: unknown;
files?: unknown;
+ /** Ask the generating replica to seal the live model stream at the next
+ * provider-safe boundary instead of waiting for a tool step. NEVER a
+ * rejection reason: on an SDK without the capability the steer still
+ * enqueues and the 202 echoes `preempt: false`. */
+ preempt?: unknown;
}
export interface SteerCancelBody {
@@ -223,14 +235,62 @@ export async function handleSteerRequest(
resolvedFileIds = resolved.fileIds;
}
+ const wantsPreempt = body.preempt === true;
+ /**
+ * The OWNER's recorded capability, not this replica's probe: a steer can
+ * land on any replica, so during a rolling deploy a local probe would
+ * answer for the wrong process and label a steer "interrupting" that the
+ * old owner can only inject at a tool boundary. Jobs created before
+ * preempt shipped carry no flag, which reads as incapable — the honest
+ * outcome, and the chip relabels to ordinary steering.
+ *
+ * Re-read rather than reused from the `job` fetched at the top of the
+ * ladder: `checkAgentAccess` and file resolution are awaits, so a request
+ * can span an entire HITL pause/resume that hands ownership to a replica
+ * with different capability and rewrites this very flag. Only paid for by
+ * requests that actually asked to interrupt.
+ *
+ * THIS replica's own SDK is deliberately not consulted. It never seals —
+ * it enqueues and publishes an arm, neither of which touches the SDK — so
+ * ANDing in a local probe would answer for the wrong process and silently
+ * drop interrupts during a rolling deploy whenever the request happened to
+ * land on an un-upgraded replica while a capable owner generated. When
+ * this replica IS the owner the probe is redundant anyway: the flag it
+ * would consult is the one this process already wrote at `createJob`.
+ */
+ const owner = wantsPreempt ? ((await GenerationJobManager.getJob(streamId)) ?? job) : job;
+ /**
+ * The re-read may have crossed a replacement. Every guard above — ownership,
+ * tenant, paused-state, agent ACL — was evaluated against `job`, so a
+ * different generation here is a run this request was never authorized
+ * against, and accepting into it would carry the wrong agent's metadata.
+ * Refuse rather than re-derive: the run the caller targeted is gone, and
+ * `NO_ACTIVE_RUN` is the code the client already turns into a queued
+ * follow-up.
+ */
+ if (owner.createdAt !== job.createdAt) {
+ return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
+ }
+ const preemptCapable = wantsPreempt && owner.metadata?.preemptCapable === true;
const item = {
steerId: randomUUID(),
text,
userId: user.id ?? '',
createdAt: Date.now(),
...(queuedFiles && { files: queuedFiles }),
+ ...(preemptCapable && { preempt: true }),
};
- const depth = await GenerationJobManager.steering.enqueue(streamId, item);
+ /**
+ * Fenced to the generation the capability decision was made against. The
+ * access checks, file resolution and owner re-read above are all awaits, so
+ * the run can be replaced before this line: without the fence the item
+ * lands on the REPLACEMENT queue while `preempt` and the arm below still
+ * describe the previous epoch, so the arm is fenced out at the owner and
+ * the 202 claims an interrupt that can never happen. Rejecting is honest —
+ * the run the caller was told about is gone, and `NO_ACTIVE_RUN` is what
+ * the client already handles by converting to a queued follow-up.
+ */
+ const depth = await GenerationJobManager.steering.enqueue(streamId, item, owner.createdAt);
if (depth === STEER_ENQUEUE_NOT_RUNNING) {
return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };
}
@@ -238,6 +298,51 @@ export async function handleSteerRequest(
return { status: 429, body: { code: 'STEER_QUEUE_FULL' } };
}
+ /**
+ * Strictly AFTER a successful enqueue: an armed request whose steer never
+ * made the durable queue could seal a generation with nothing to inject.
+ *
+ * `preempt` in the 202 means "queued as an interrupt request", NOT "a seal
+ * is guaranteed" — it mirrors the durable `item.preempt` exactly. A route
+ * cannot synchronously know whether another replica will seal: proving that
+ * needs a correlated request/response over pub-sub, and even then the owner
+ * may finish before the arm lands. Reporting delivery instead made the
+ * response disagree with the durable flag, which `rearmQueuedPreempts`
+ * trusts on resume — the two must agree or a resumed owner honours an
+ * interrupt the client was told had degraded.
+ *
+ * The gates that ARE knowable stay: the owner's recorded capability and a
+ * successful enqueue. Everything past that degrades to the documented
+ * fallback of injecting at the next tool boundary.
+ */
+ if (preemptCapable) {
+ /**
+ * NOT awaited. The answer no longer depends on it — the 202 reports
+ * `preemptCapable`, not delivery — so awaiting only exposes the caller to
+ * Redis latency after the queue item is already durable. A client that
+ * times out on a stalled publish and retries mints a SECOND steer while
+ * the first stays queued, injecting the same instruction twice; a lost
+ * publish merely takes the documented tool-boundary fallback. The
+ * asymmetry is the whole argument for detaching.
+ */
+ void GenerationJobManager.requestPreempt(streamId, item.steerId, owner.createdAt).then(
+ (armed) => {
+ if (!armed) {
+ logger.warn(
+ `[handleSteerRequest] Preempt arm not confirmed for ${streamId} steer=${item.steerId}; ` +
+ 'the steer remains queued and will inject at the next boundary',
+ );
+ }
+ },
+ (error: unknown) => {
+ logger.error(
+ `[handleSteerRequest] Preempt arm failed for ${streamId} steer=${item.steerId}:`,
+ error,
+ );
+ },
+ );
+ }
+
/** Fire-and-forget: the persisted steer part references these uploads, so
* mark them used (parity with `updateFilesUsage` on normal sends) or the
* upload-window TTL could reap them before replay. Must not fail the 202. */
@@ -255,7 +360,13 @@ export async function handleSteerRequest(
return {
status: 202,
- body: { status: 'queued', steerId: item.steerId, position: depth, conversationId },
+ body: {
+ status: 'queued',
+ steerId: item.steerId,
+ position: depth,
+ conversationId,
+ preempt: preemptCapable,
+ },
};
}
@@ -295,5 +406,53 @@ export async function handleSteerCancel(
}
const removed = await GenerationJobManager.steering.cancel(streamId, body.steerId);
+ /** A cancelled steer must also disarm any preempt request it carried —
+ * cancel is live UI, and a request left armed would seal an unrelated
+ * stretch of generation, drain nothing, and end the run mid-sentence. */
+ if (!removed) {
+ return { status: 200, body: { removed } };
+ }
+ /**
+ * Waited on so a failed disarm is retried and logged before the response,
+ * but BOUNDED, and its outcome is deliberately NOT reported to the client.
+ *
+ * The bound is the load-bearing part. ioredis queues commands while a
+ * connection is down rather than rejecting them, so an unbounded await here
+ * can hang for as long as the outage lasts — and the item is already
+ * durably cancelled at this point. A client that gives up then treats the
+ * cancel as failed and restores a chip for a steer that no longer exists
+ * and can never produce an applied event, which is strictly worse than the
+ * lost clear this wait was protecting against. Every successful cancel
+ * publishes, so ordinary steers are exposed to it too, not just preemptive
+ * ones. The publish keeps running with its retry and logging intact after
+ * the timeout — it is simply no longer in front of the response.
+ *
+ * A resolved publish is not proof the owner heard it — the delivery count
+ * includes this replica's own facade subscription — so any `disarmed` flag
+ * would claim a certainty the transport cannot provide, which is the same
+ * over-promise the `preempt` flag was corrected for. Disarm is best effort
+ * with a bounded, self-healing failure: if the clear is lost the owner
+ * seals once, its empty boundary self-clears, and the turn is persisted
+ * `unfinished: true` rather than silently truncated.
+ *
+ * `removed` stays true because it is true — the steer really did leave the
+ * queue, and inverting it would make the client re-show a chip for a steer
+ * that can never arrive.
+ */
+ const disarm = GenerationJobManager.noteSteersRemoved(streamId, [body.steerId], job.createdAt);
+ let settleTimer: ReturnType<typeof setTimeout> | undefined;
+ await Promise.race([
+ disarm,
+ new Promise<void>((resolve) => {
+ settleTimer = setTimeout(() => {
+ logger.warn(
+ `[handleSteerCancel] Disarm publish for ${streamId} steer=${body.steerId} still pending ` +
+ `after ${STEER_DISARM_ACK_TIMEOUT_MS}ms; answering the cancel and letting it retry`,
+ );
+ resolve();
+ }, STEER_DISARM_ACK_TIMEOUT_MS);
+ }),
+ ]);
+ clearTimeout(settleTimer);
return { status: 200, body: { removed } };
}
diff --git a/packages/api/src/agents/steering/runtime.ts b/packages/api/src/agents/steering/runtime.ts
index dbcdcf9367..aafde3b84b 100644
--- a/packages/api/src/agents/steering/runtime.ts
+++ b/packages/api/src/agents/steering/runtime.ts
@@ -1,6 +1,7 @@
import * as agentsSdk from '@librechat/agents';
import { logger } from '@librechat/data-schemas';
import type {
+ StreamPreemption,
HookCallback,
HookInputByEvent,
HookOutputByEvent,
@@ -71,31 +72,44 @@ export interface SteerDrainHookOptions {
}
/**
- * Build the run-scoped `PostToolBatch` hook that drains the job's steer queue
- * at each tool-batch boundary and injects each steer into graph state as its
- * own user message (`role: 'user'`, `source: 'steer'` — never consolidated
- * with hook context). Steers with attachments inject as multimodal content
- * arrays via `buildMedia`.
+ * Shared drain body for BOTH injection boundaries (PostToolBatch and
+ * PreemptBoundary) — the provider-safety argument rests on the two sites
+ * emitting identical `InjectedMessage` shapes, so they must share one body.
*
- * Subagent scopes are skipped (`input.agentId` set): a steer targets the
- * top-level conversation, not a child agent's context. Hook errors are
- * swallowed by the SDK's `executeHooks`, so a broken drain can never kill the
- * run.
+ * The injections array builds incrementally under a swallow-all catch: once a
+ * steer leaves the durable queue its part is persisted by `applySteer`
+ * first, so a mid-loop throw must still inject whatever was already applied
+ * rather than dropping the batch. `noteSteersRemoved` runs in `finally` —
+ * the drained items are out of the queue no matter what, so any armed
+ * preempt request for them must clear even on the failure path, or a
+ * satisfied preempt could seal a later, unrelated stretch of generation.
*/
-export function createSteerDrainHook(opts: SteerDrainHookOptions): HookCallback<'PostToolBatch'> {
+async function drainAndBuildInjections(opts: SteerDrainHookOptions): Promise<InjectedMessage[]> {
const { streamId, jobCreatedAt, applySteer, buildMedia } = opts;
- return async (input: HookInputByEvent['PostToolBatch']): Promise<SteerDrainOutput> => {
- if (input.agentId != null) {
- return {};
- }
- // The replacement guard lives INSIDE the store's atomic drain: a separate
- // check-then-drain could still consume a replacement job's queue if
- // createJob landed between the two steps.
- const steers = await GenerationJobManager.steering.drain(streamId, jobCreatedAt);
- if (steers.length === 0) {
- return {};
- }
- const injectedMessages: InjectedMessage[] = [];
+ // The replacement guard lives INSIDE the store's atomic drain: a separate
+ // check-then-drain could still consume a replacement job's queue if
+ // createJob landed between the two steps.
+ /**
+ * Snapshotted BEFORE the drain so an empty boundary disarms only the
+ * requests it was responsible for. A second steer can enqueue and arm while
+ * the drain is in flight; that arm is backed by a live queue item and must
+ * survive.
+ */
+ const armedBeforeDrain = GenerationJobManager.getArmedPreemptIds(streamId, jobCreatedAt);
+ const steers = await GenerationJobManager.steering.drain(streamId, jobCreatedAt);
+ if (steers.length === 0) {
+ /**
+ * Nothing to inject. The snapshotted ids point at steers that have
+ * already left the queue (drained at the other boundary, cancelled, or a
+ * late cross-replica arm), so disarm them — a level-triggered poll left
+ * true would seal again on the next chunk and truncate an unrelated
+ * answer.
+ */
+ GenerationJobManager.clearPreemptRequests(streamId, armedBeforeDrain, jobCreatedAt);
+ return [];
+ }
+ const injectedMessages: InjectedMessage[] = [];
+ try {
for (const item of steers) {
try {
await applySteer(item);
@@ -122,6 +136,104 @@ export function createSteerDrainHook(opts: SteerDrainHookOptions): HookCallback<
source: 'steer' as const,
});
}
+ } catch (error) {
+ logger.error(`[steering] Drain interrupted for ${streamId}; injecting applied items:`, error);
+ } finally {
+ /**
+ * Everything armed at snapshot time PLUS everything just drained. The
+ * boundary has spent its seal, so a snapshotted id that did NOT come back
+ * from the drain is stale — its steer left the queue by another route
+ * (typically a cancel whose cross-replica clear was lost). Clearing only
+ * the drained ids would leave that one level-triggered, sealing the very
+ * continuation meant to answer the steer we just injected and landing on
+ * an empty boundary as `preempt_incomplete`.
+ *
+ * Ids armed AFTER the snapshot are deliberately excluded: their queue
+ * items are live and uninjected, and disarming them would strand an
+ * interrupt the client was already told about.
+ *
+ * Not awaited: this runs on the OWNER, where the local disarm is
+ * synchronous and already effective — the publish only informs other
+ * replicas, and blocking a boundary drain on it would delay injection.
+ */
+ const spent = new Set(armedBeforeDrain);
+ for (const item of steers) {
+ spent.add(item.steerId);
+ }
+ void GenerationJobManager.noteSteersRemoved(streamId, [...spent], jobCreatedAt);
+ }
+ return injectedMessages;
+}
+
+/**
+ * Build the run-scoped `PostToolBatch` hook that drains the job's steer queue
+ * at each tool-batch boundary and injects each steer into graph state as its
+ * own user message (`role: 'user'`, `source: 'steer'` — never consolidated
+ * with hook context). Steers with attachments inject as multimodal content
+ * arrays via `buildMedia`.
+ *
+ * Subagent scopes are skipped (`input.agentId` set): a steer targets the
+ * top-level conversation, not a child agent's context. Hook errors are
+ * swallowed by the SDK's `executeHooks`, so a broken drain can never kill the
+ * run.
+ */
+export function createSteerDrainHook(opts: SteerDrainHookOptions): HookCallback<'PostToolBatch'> {
+ return async (input: HookInputByEvent['PostToolBatch']): Promise<SteerDrainOutput> => {
+ if (input.agentId != null) {
+ return {};
+ }
+ const injectedMessages = await drainAndBuildInjections(opts);
+ if (injectedMessages.length === 0) {
+ return {};
+ }
+ return { injectedMessages };
+ };
+}
+
+/**
+ * The PreemptBoundary twin of {@link createSteerDrainHook}: fires after the
+ * SDK seals a model stream mid-generation, drains the same durable queue
+ * through the same shared body, and injects the same message shapes. Because
+ * `noteSteersRemoved` lives in that shared body, a preempt satisfied at an
+ * ordinary tool boundary clears its request there and cannot cause a
+ * spurious seal later.
+ */
+export function createSteerPreemptBoundaryHook(
+ opts: SteerDrainHookOptions,
+): HookCallback<'PreemptBoundary'> {
+ return async (
+ input: HookInputByEvent['PreemptBoundary'],
+ ): Promise<HookOutputByEvent['PreemptBoundary']> => {
+ if (input.agentId != null) {
+ return {};
+ }
+ const injectedMessages = await drainAndBuildInjections(opts);
+ if (injectedMessages.length === 0) {
+ return {};
+ }
return { injectedMessages };
};
}
+
+/**
+ * The run's `RunConfig.preemption` — a level-triggered O(1) poll over the
+ * job's armed preempt requests, exactly as the SDK contract requires: it
+ * keeps returning true until the boundary drain (either boundary) clears the
+ * request via `noteSteersRemoved`. Never consumes on read.
+ */
+export function createSteerPreemptPoll(streamId: string): StreamPreemption {
+ return {
+ shouldPreempt: () => GenerationJobManager.isPreemptRequested(streamId),
+ };
+}
+
+/**
+ * Whether the installed SDK can seal a generation mid-stream and inject at
+ * the resulting boundary. A SEPARATE probe from `isSteeringSupported()`:
+ * reusing that flag would arm an interrupt affordance that silently does
+ * nothing on an SDK that can only inject at tool boundaries.
+ */
+export function isSteerPreemptSupported(): boolean {
+ const sdk = agentsSdk as { HOOK_PREEMPT_BOUNDARY_CAPABLE?: boolean };
+ return isSteeringSupported() && sdk.HOOK_PREEMPT_BOUNDARY_CAPABLE === true;
+}
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index d97318debb..9f390844c6 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -32,12 +32,22 @@ import {
setGenerationJobsInFlight,
recordGenerationJob,
} from '~/app/metrics';
+import {
+ isPendingActionStale,
+ isPendingActionExpired,
+ STEER_QUEUE_MAX_DEPTH,
+} from './interfaces/IJobStore';
+
+/**
+ * Tombstone budget per generation. Twice the queue depth so a full queue's
+ * worth of removals plus in-flight arms fit without eviction in practice.
+ */
+const PREEMPT_TOMBSTONE_MAX = STEER_QUEUE_MAX_DEPTH * 2;
import {
SteeringLifecycle,
toPendingSteer,
synthesizeAppliedSteerEvents,
} from './SteeringLifecycle';
-import { isPendingActionStale, isPendingActionExpired } from './interfaces/IJobStore';
import { synthesizeActivityLabelGapEvents } from '~/agents/activityLabels/wiring';
import { InMemoryEventTransport } from './implementations/InMemoryEventTransport';
import { InMemoryJobStore } from './implementations/InMemoryJobStore';
@@ -206,6 +216,31 @@ interface RuntimeJobState {
abortController: AbortController;
/** Removes this generation's cross-replica abort listener without touching a replacement. */
abortUnsubscribe?: () => void;
+ /**
+ * Cooperative-seal requests for THIS generation. Armed by `requestPreempt`
+ * (locally or via a fenced cross-replica publish), polled O(1) by the run's
+ * `shouldPreempt`, and cleared by `noteSteersRemoved` when the steer drains
+ * at any boundary or is cancelled. Lives and dies with the runtime object —
+ * no terminal bookkeeping needed beyond the listener release.
+ */
+ preempt?: {
+ /** Generation identity this request set belongs to. */
+ createdAt: number;
+ /** Server-minted steerIds requested but not yet drained/cancelled. */
+ ids: Set<string>;
+ /**
+ * steerIds whose removal was observed BEFORE their arm. Arm and clear are
+ * published by different replicas over different connections, so a steer
+ * drained at a tool boundary during the arming replica's enqueue round
+ * trip can have its clear land first — without this tombstone the late
+ * arm would resurrect a request no steer backs, sealing an unrelated
+ * stretch of generation. steerIds are single-use UUIDs, so a tombstoned
+ * id can never legitimately be re-armed.
+ */
+ cleared: Set<string>;
+ };
+ /** Removes this generation's cross-replica preempt listener without touching a replacement. */
+ preemptUnsubscribe?: () => void;
readyPromise: Promise<void>;
resolveReady: () => void;
startupTelemetry?: AgentStartupTelemetry;
@@ -462,7 +497,14 @@ class GenerationJobManagerClass {
return released;
}
+ /**
+ * Releases this generation's cross-replica listeners — abort AND preempt —
+ * and drops any armed preempt requests. Every terminal site that retires a
+ * runtime goes through here, so preempt state cannot outlive the
+ * generation it belongs to.
+ */
private releaseAbortSubscription(runtime: RuntimeJobState): void {
+ this.releasePreemptSubscription(runtime);
const unsubscribe = runtime.abortUnsubscribe;
runtime.abortUnsubscribe = undefined;
if (!unsubscribe) {
@@ -476,6 +518,21 @@ class GenerationJobManagerClass {
}
}
+ private releasePreemptSubscription(runtime: RuntimeJobState): void {
+ runtime.preempt = undefined;
+ const unsubscribe = runtime.preemptUnsubscribe;
+ runtime.preemptUnsubscribe = undefined;
+ if (!unsubscribe) {
+ return;
+ }
+
+ try {
+ unsubscribe();
+ } catch (err) {
+ logger.error('[GenerationJobManager] Failed to release preempt subscription:', err);
+ }
+ }
+
private reconcileInactiveGeneration(
streamId: string,
createdAt: number,
@@ -542,6 +599,113 @@ class GenerationJobManagerClass {
}
}
+ private ensurePreemptState(
+ runtime: RuntimeJobState,
+ createdAt: number,
+ ): NonNullable<RuntimeJobState['preempt']> {
+ runtime.preempt ??= { createdAt, ids: new Set(), cleared: new Set() };
+ return runtime.preempt;
+ }
+
+ /**
+ * Arms ids that have not already been observed as removed. Returns how many
+ * were actually accepted: a tombstoned id (its steer already drained at an
+ * ordinary boundary while this request was in flight) or an over-cap id is
+ * skipped, and the caller must not report those as armed.
+ */
+ private armPreemptIds(runtime: RuntimeJobState, createdAt: number, steerIds: string[]): number {
+ const state = this.ensurePreemptState(runtime, createdAt);
+ let accepted = 0;
+ for (const id of steerIds) {
+ if (state.cleared.has(id) || state.ids.size >= STEER_QUEUE_MAX_DEPTH) {
+ continue;
+ }
+ state.ids.add(id);
+ accepted += 1;
+ }
+ return accepted;
+ }
+
+ /**
+ * Disarms ids and tombstones them against a late-arriving arm. The
+ * tombstone set is bounded by EVICTING its oldest entry rather than
+ * refusing new ones: every drained or cancelled steer is tombstoned, not
+ * just preempting ones, so a refusing cap would silently stop recording
+ * after a long generation and let the late-arm race resurface. Set
+ * iteration is insertion-ordered, so the first key is the oldest.
+ */
+ private clearPreemptIds(runtime: RuntimeJobState, createdAt: number, steerIds: string[]): void {
+ const state = this.ensurePreemptState(runtime, createdAt);
+ for (const id of steerIds) {
+ state.ids.delete(id);
+ if (state.cleared.has(id)) {
+ continue;
+ }
+ if (state.cleared.size >= PREEMPT_TOMBSTONE_MAX) {
+ const oldest = state.cleared.values().next().value;
+ if (oldest != null) {
+ state.cleared.delete(oldest);
+ }
+ }
+ state.cleared.add(id);
+ }
+ }
+
+ /**
+ * Mirrors {@link registerAbortSubscription} for cooperative preempts. The
+ * same double fence applies — runtime object identity plus the generation
+ * `createdAt` carried by every {@link PreemptMessage} — so a stale
+ * cross-replica publish can never arm a replacement job on the same
+ * streamId. Arm requests cap at {@link STEER_QUEUE_MAX_DEPTH}, matching
+ * the durable queue they mirror.
+ *
+ * Never rejects. Every caller fires this without awaiting (see the
+ * createJob registration for why), so a propagating subscription error
+ * would surface as an unhandled rejection: an alarming stack trace in
+ * LibreChat's own server, which installs a global handler and keeps
+ * serving, and a dead process for any other consumer of this package,
+ * which under Node's default `--unhandled-rejections=throw` does not.
+ * Neither is warranted — a failed subscription degrades this generation's
+ * preemptive steers to the next tool boundary, the documented fallback.
+ */
+ private async registerPreemptSubscription(
+ streamId: string,
+ runtime: RuntimeJobState,
+ ): Promise<void> {
+ if (!this.eventTransport.onPreempt) {
+ return;
+ }
+
+ try {
+ const unsubscribe = await this.eventTransport.onPreempt(streamId, (msg) => {
+ const currentRuntime = this.runtimeState.get(streamId);
+ if (currentRuntime !== runtime || currentRuntime.createdAt !== msg.createdAt) {
+ return;
+ }
+
+ if (msg.op === 'clear') {
+ this.clearPreemptIds(currentRuntime, msg.createdAt, msg.steerIds);
+ return;
+ }
+
+ this.armPreemptIds(currentRuntime, msg.createdAt, msg.steerIds);
+ });
+
+ if (typeof unsubscribe === 'function') {
+ runtime.preemptUnsubscribe = unsubscribe;
+ }
+ if (this.runtimeState.get(streamId) !== runtime || runtime.abortController.signal.aborted) {
+ this.releasePreemptSubscription(runtime);
+ }
+ } catch (err) {
+ logger.error(
+ `[GenerationJobManager] Failed to subscribe to preempts for ${streamId}; ` +
+ 'steers on this generation will apply at the next tool boundary:',
+ err,
+ );
+ }
+ }
+
private rejectSubscriptionDuringShutdown(
subscriptionType: 'initial' | 'resume',
onError?: t.ErrorHandler,
@@ -760,6 +924,17 @@ class GenerationJobManagerClass {
this.registerAllSubscribersLeft(streamId);
await this.registerAbortSubscription(streamId, runtime);
+ /**
+ * NOT awaited, unlike abort. Abort must be deliverable before the job
+ * is exposed — a missed abort strands a run. A missed preempt only
+ * degrades that steer to the next tool boundary, which is the
+ * documented fallback, so blocking job creation on a second channel
+ * subscription would trade a real hang risk for a cosmetic guarantee.
+ * The registration's own lost-race tail releases it if the runtime is
+ * retired before the subscription resolves, and it swallows and logs
+ * its own failures, so this detached call cannot reject.
+ */
+ void this.registerPreemptSubscription(streamId, runtime);
if (this.runtimeState.get(streamId) !== runtime) {
throw new Error('Generation job was replaced during initialization');
}
@@ -888,6 +1063,9 @@ class GenerationJobManagerClass {
// Surface deferred tools discovered before the pause so the resume route can
// replay them into createRun (the rebuilt graph passes `messages: []`).
discoveredTools: jobData.discoveredTools,
+ // Surface the owning replica's seal capability so the steer route can
+ // honour it instead of probing its own (possibly older) SDK.
+ preemptCapable: jobData.preemptCapable,
// Surface the pending review so status/resume routes built on the
// facade can render the prompt for a `requires_action` job.
pendingAction: jobData.pendingAction,
@@ -982,6 +1160,8 @@ class GenerationJobManagerClass {
if (jobData.status === 'running' || jobData.status === 'requires_action') {
await this.registerAbortSubscription(streamId, runtime);
+ /** Best-effort, non-blocking — see the createJob registration. */
+ void this.registerPreemptSubscription(streamId, runtime);
}
const runtimeAfterAbortRegistration = this.runtimeState.get(streamId);
@@ -3064,6 +3244,230 @@ class GenerationJobManagerClass {
return this._steering;
}
+ /**
+ * Arms a cooperative-seal request for one queued steer and reports whether
+ * the arm actually reached an owner. Never touches job status.
+ *
+ * Returns true when this replica owns the generation (armed locally), or
+ * when the fenced publish was received by at least one subscriber. Returns
+ * FALSE when the publish failed or nobody was listening: the steer is still
+ * durably queued and will inject at the next tool boundary, but nothing
+ * will seal for it, so the caller must not acknowledge it as interrupting.
+ * The owner's own `createdAt` fence still decides whether to honour it.
+ */
+ async requestPreempt(streamId: string, steerId: string, jobCreatedAt: number): Promise<boolean> {
+ /**
+ * `ownedJobs`, NOT `runtimeState`: a cross-replica `getJob` installs a
+ * facade runtime on THIS replica via `getOrCreateRuntimeState`, so a
+ * matching `runtime.createdAt` proves only that we have looked at the
+ * job — not that we generate it. Arming that facade would satisfy
+ * nothing while reporting success.
+ */
+ const ownedHere = this.ownedJobs.get(streamId) === jobCreatedAt;
+ const runtime = this.runtimeState.get(streamId);
+ if (ownedHere && runtime != null && runtime.createdAt === jobCreatedAt) {
+ /** Zero accepted means the id was tombstoned (its steer drained at an
+ * ordinary boundary mid-request) — not an arm, so do not claim one. */
+ return this.armPreemptIds(runtime, jobCreatedAt, [steerId]) > 0;
+ }
+ if (this.eventTransport.emitPreempt == null) {
+ /** Single-process transport: the local arm is the whole mechanism. */
+ return false;
+ }
+ try {
+ await this.eventTransport.emitPreempt(streamId, {
+ op: 'arm',
+ createdAt: jobCreatedAt,
+ steerIds: [steerId],
+ });
+ /**
+ * Best effort, and deliberately not read as proof. The publisher's
+ * subscriber count includes THIS replica's own facade subscription, so
+ * it cannot distinguish "the owner heard" from "we heard ourselves";
+ * proving owner receipt would need a correlated request/response over
+ * pub-sub. See the acknowledgement-semantics note on the PR.
+ */
+ return true;
+ } catch (error) {
+ logger.error(`[GenerationJobManager] Failed to publish preempt arm for ${streamId}:`, error);
+ return false;
+ }
+ }
+
+ /** O(1) level-triggered poll consumed by the run's `shouldPreempt`. */
+ isPreemptRequested(streamId: string): boolean {
+ return (this.runtimeState.get(streamId)?.preempt?.ids.size ?? 0) > 0;
+ }
+
+ /**
+ * Clears preempt requests for steers that left the durable queue — drained
+ * at a boundary (tool or preempt), cancelled, or dropped. The fenced
+ * cross-replica `clear` keeps a request from outliving its steer on the
+ * generating replica; without a fence identity the publish is skipped and
+ * the empty-boundary path's self-clear bounds the damage to one seal.
+ */
+ noteSteersRemoved(streamId: string, steerIds: string[], jobCreatedAt?: number): Promise<boolean> {
+ if (steerIds.length === 0) {
+ return Promise.resolve(true);
+ }
+ const runtime = this.runtimeState.get(streamId);
+ if (runtime != null && (jobCreatedAt == null || runtime.createdAt === jobCreatedAt)) {
+ this.clearPreemptIds(runtime, jobCreatedAt ?? runtime.createdAt, steerIds);
+ }
+ const createdAt = jobCreatedAt ?? runtime?.createdAt;
+ if (createdAt == null || this.eventTransport.emitPreempt == null) {
+ /** Nothing to publish: the local disarm above is the whole mechanism. */
+ return Promise.resolve(true);
+ }
+ /**
+ * Awaitable so a CANCEL retries and logs before responding. A dropped
+ * clear is worse than a dropped arm: the owner keeps a level-triggered
+ * request for a steer that no longer exists, seals its next chunk,
+ * drains nothing, and truncates an unrelated answer. Retried once — a
+ * transient publish error is the common case and the retry is cheap.
+ *
+ * The returned boolean means "published without error", NOT "the owner
+ * disarmed": the delivery count includes this replica's own facade
+ * subscription, so publication cannot prove receipt. It is for logging
+ * only and must not be surfaced as a guarantee. Proving receipt needs a
+ * correlated request/response over pub-sub.
+ *
+ * Damage is bounded regardless: if the clear is lost the owner seals
+ * once, the empty-boundary self-clear disarms the generation, and the
+ * turn is persisted `unfinished: true` rather than silently truncated.
+ */
+ const publish = (): Promise<unknown> =>
+ Promise.resolve(
+ this.eventTransport.emitPreempt?.(streamId, { op: 'clear', createdAt, steerIds }),
+ );
+ return publish().then(
+ () => true,
+ () =>
+ publish().then(
+ () => true,
+ (error: unknown) => {
+ logger.error(
+ `[GenerationJobManager] Failed to publish preempt clear for ${streamId} after retry; ` +
+ 'the owner may seal once before its empty boundary self-clears:',
+ error,
+ );
+ return false;
+ },
+ ),
+ );
+ }
+
+ /**
+ * Rebuilds the armed set from the DURABLE queue for a generation whose
+ * ownership just moved (HITL resume landing on another replica).
+ *
+ * An arm lives only in the owning replica's runtime plus a transient
+ * pub/sub message, while the steer's `preempt` flag is durable on the queue
+ * item. A replica that never observed the original arm therefore starts
+ * with an empty set and its poll stays false, so an interrupt the user
+ * already had acknowledged would silently wait for an ordinary tool
+ * boundary.
+ *
+ * REPLACES the armed set rather than adding to it. A replica that only ever
+ * read this job still installed a facade runtime and subscribed, so it can
+ * have accepted an arm and then missed the best-effort clear that followed
+ * the drain. Promotion to owner makes that orphan live, and a union would
+ * keep it: the first resumed stream seals on a steer no longer in the
+ * queue, drains nothing, and truncates the resumed answer as
+ * `preempt_incomplete`. The durable queue is the only authority at a
+ * handover, so anything absent from it is disarmed and tombstoned here —
+ * tombstoned because an arm that outlived its steer must not be able to
+ * come back a second time.
+ */
+ async rearmQueuedPreempts(streamId: string, jobCreatedAt: number): Promise<number> {
+ /**
+ * The armed set is snapshotted BEFORE the queue is read, and that order is
+ * the entire safety argument. `approvals.resolve` has already reopened
+ * steering by the time this runs, so a concurrent replica can enqueue a
+ * preempt steer and publish its arm while the `peek` is in flight. Reading
+ * the queue first would leave that arm present locally but missing from a
+ * snapshot taken before it existed, and this method would tombstone a live
+ * interrupt the route already acknowledged — unrecoverable, since the
+ * tombstone also blocks the re-arm.
+ *
+ * Taking the arms first makes that impossible without a lock or a second
+ * round trip: a steer is durably enqueued BEFORE its arm is published, so
+ * any id in this snapshot was already queued when it was armed, and the
+ * later `peek` is guaranteed to observe it unless it has since drained —
+ * which is precisely the orphan this reconciliation exists to drop.
+ */
+ const runtime = this.runtimeState.get(streamId);
+ const armedBeforeRead =
+ runtime?.preempt != null && runtime.createdAt === jobCreatedAt
+ ? [...runtime.preempt.ids]
+ : [];
+
+ const queued = await this._steering.peek(streamId, jobCreatedAt);
+ const backed = new Set(
+ queued.filter((item) => item.preempt === true).map((item) => item.steerId),
+ );
+
+ /** The generation can be replaced across the read; only disarm the runtime
+ * the snapshot was taken from. */
+ if (runtime != null && this.runtimeState.get(streamId) === runtime) {
+ const orphaned = armedBeforeRead.filter((id) => !backed.has(id));
+ if (orphaned.length > 0) {
+ logger.warn(
+ `[GenerationJobManager] Dropping ${orphaned.length} preempt arm(s) with no queued steer ` +
+ `while ownership of ${streamId} moves`,
+ );
+ this.clearPreemptIds(runtime, jobCreatedAt, orphaned);
+ }
+ }
+
+ let rearmed = 0;
+ for (const steerId of backed) {
+ await this.requestPreempt(streamId, steerId, jobCreatedAt);
+ rearmed += 1;
+ }
+ return rearmed;
+ }
+
+ /**
+ * Snapshot of the ids armed right now, taken BEFORE a drain so the
+ * empty-boundary disarm can scope itself to them.
+ */
+ getArmedPreemptIds(streamId: string, jobCreatedAt?: number): string[] {
+ const runtime = this.runtimeState.get(streamId);
+ if (runtime?.preempt == null) {
+ return [];
+ }
+ if (jobCreatedAt != null && runtime.createdAt !== jobCreatedAt) {
+ return [];
+ }
+ return [...runtime.preempt.ids];
+ }
+
+ /**
+ * Disarms the requests a spent boundary was responsible for. Called when a
+ * boundary drains nothing: the seal is already spent and those ids refer to
+ * steers no longer in the queue, so leaving them armed would seal again on
+ * the next chunk and truncate an unrelated answer.
+ *
+ * Scoped to an explicit id list rather than wiping the set: a second steer
+ * can enqueue and arm between the atomic drain returning empty and this
+ * call, and that arm is backed by a live queue item that has not been
+ * injected yet.
+ */
+ clearPreemptRequests(streamId: string, steerIds: string[], jobCreatedAt?: number): void {
+ if (steerIds.length === 0) {
+ return;
+ }
+ const runtime = this.runtimeState.get(streamId);
+ if (runtime?.preempt == null) {
+ return;
+ }
+ if (jobCreatedAt != null && runtime.createdAt !== jobCreatedAt) {
+ return;
+ }
+ this.clearPreemptIds(runtime, runtime.createdAt, steerIds);
+ }
+
/**
* Get resume state for reconnecting clients.
*/
diff --git a/packages/api/src/stream/SteeringLifecycle.ts b/packages/api/src/stream/SteeringLifecycle.ts
index bc3a500239..f8cf62c717 100644
--- a/packages/api/src/stream/SteeringLifecycle.ts
+++ b/packages/api/src/stream/SteeringLifecycle.ts
@@ -11,6 +11,7 @@ export function toPendingSteer(item: SteerQueueItem): TPendingSteer {
text: item.text,
createdAt: item.createdAt,
...(item.files && item.files.length > 0 && { files: item.files }),
+ ...(item.preempt === true && { preempt: true }),
};
}
@@ -101,8 +102,12 @@ export class SteeringLifecycle {
* depth, or a rejection code ({@link STEER_ENQUEUE_NOT_RUNNING} /
* {@link STEER_ENQUEUE_QUEUE_FULL}).
*/
- async enqueue(streamId: string, item: SteerQueueItem): Promise<number> {
- const depth = await this.store.enqueueSteer(streamId, item);
+ async enqueue(
+ streamId: string,
+ item: SteerQueueItem,
+ expectedCreatedAt?: number,
+ ): Promise<number> {
+ const depth = await this.store.enqueueSteer(streamId, item, expectedCreatedAt);
if (depth > 0) {
logger.debug(
`[SteeringLifecycle] queued steer: ${streamId} steer=${item.steerId} depth=${depth}`,
diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
index 91f3d656d1..c8e46cf094 100644
--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
@@ -9,6 +9,7 @@ import {
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
+import { STEER_ENQUEUE_NOT_RUNNING } from '~/stream/interfaces/IJobStore';
import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';
import { RedisJobStore } from '~/stream/implementations/RedisJobStore';
import { createStreamServices } from '~/stream/createStreamServices';
@@ -2416,4 +2417,217 @@ describe('GenerationJobManager Integration Tests', () => {
expect(services.isRedis).toBe(false);
});
});
+
+ /**
+ * The production topology for interrupt & steer: the steer POST is routed to
+ * whichever replica the load balancer picks, which is usually NOT the replica
+ * running the generation. Everything else in the preempt suite runs against a
+ * single manager, so the hop that actually carries the request — non-owner
+ * publishes, owner arms, owner's SDK poll flips — has no coverage.
+ *
+ * Two `GenerationJobManagerClass` instances are a faithful pair of replicas
+ * here: `runtimeState` and `ownedJobs` are private instance fields, there is
+ * no module-level mutable state between them, and `createStreamServices`
+ * duplicates a dedicated subscriber connection per call. Separate OS
+ * processes would exercise the same objects over the same Redis.
+ */
+ describeRedis('Cross-Replica Preempt (Redis, two manager instances)', () => {
+ const replicas: GenerationJobManagerClass[] = [];
+
+ function createReplica(): GenerationJobManagerClass {
+ const manager = new GenerationJobManagerClass();
+ manager.configure(createStreamServices({ useRedis: true, redisClient: ioredisClient! }));
+ manager.initialize();
+ replicas.push(manager);
+ return manager;
+ }
+
+ afterEach(async () => {
+ /** `destroy()`, not `eventTransport.destroy()`: it also clears the
+ * manager's cleanup interval and disposes the job store and its timer.
+ * Dropping only the transport leaves each manager alive inside its own
+ * interval closure, still doing cleanup work against a dead transport. */
+ for (const manager of replicas.splice(0)) {
+ await manager.destroy().catch(() => {});
+ }
+ });
+
+ /**
+ * Redis pub/sub never replays and the owner's `SUBSCRIBE` is fired
+ * detached, so a one-shot publish can be dropped simply for arriving
+ * before anyone is listening — a fixed sleep only decides how often that
+ * happens on a loaded worker. Republishing until the owner's state
+ * converges is safe because both operations are idempotent set writes
+ * keyed by steerId, and it removes the timing assumption entirely.
+ */
+ async function publishUntil(
+ publish: () => Promise<unknown>,
+ settled: () => boolean,
+ what: string,
+ timeoutMs = 15000,
+ ): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ await publish();
+ for (let attempt = 0; attempt < 10; attempt++) {
+ if (settled()) {
+ return;
+ }
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+ }
+ throw new Error(`Timed out waiting for ${what}`);
+ }
+
+ test('a steer routed to a non-owning replica arms the owner and flips its poll', async () => {
+ const owner = createReplica();
+ const router = createReplica();
+ const streamId = `${testPrefix}-preempt-xreplica-${Date.now()}`;
+
+ const job = await owner.createJob(streamId, 'user-1', undefined, {
+ initialMetadata: { preemptCapable: true },
+ });
+
+ /**
+ * The routing replica reads the job, which installs a FACADE runtime
+ * entry on it. That facade must not make it look like an owner.
+ */
+ const seenByRouter = await router.getJob(streamId);
+ expect(seenByRouter?.createdAt).toBe(job.createdAt);
+ expect(router.isPreemptRequested(streamId)).toBe(false);
+
+ /**
+ * Guards the round-trip that shipped broken once: `preemptCapable` was
+ * serialized but never deserialized, so every Redis deployment read it
+ * back as undefined and the feature was a silent no-op. A same-replica
+ * read cannot catch that — this one crosses Redis.
+ */
+ expect(seenByRouter?.metadata?.preemptCapable).toBe(true);
+
+ /** The whole point: the owner's level-triggered poll flips from a
+ * request that was accepted on a different replica. */
+ await publishUntil(
+ () => router.requestPreempt(streamId, 'steer-x', job.createdAt),
+ () => owner.isPreemptRequested(streamId),
+ 'the owner to arm',
+ );
+ expect(owner.getArmedPreemptIds(streamId)).toContain('steer-x');
+
+ await publishUntil(
+ () => router.noteSteersRemoved(streamId, ['steer-x'], job.createdAt),
+ () => !owner.isPreemptRequested(streamId),
+ 'the owner to disarm',
+ );
+ expect(owner.getArmedPreemptIds(streamId)).not.toContain('steer-x');
+ }, 40000);
+
+ /**
+ * The in-memory store fences this in TypeScript; Redis fences it inside
+ * STEER_ENQUEUE_LUA, which only a real server can execute. That Lua path
+ * is where the `preemptCapable` P1 hid, so it gets its own coverage.
+ */
+ test('the enqueue Lua refuses an item fenced to a replaced generation', async () => {
+ const owner = createReplica();
+ const streamId = `${testPrefix}-enqueue-fenced-${Date.now()}`;
+
+ const first = await owner.createJob(streamId, 'user-1');
+ const replacement = await owner.createJob(streamId, 'user-1');
+ expect(replacement.createdAt).not.toBe(first.createdAt);
+
+ const stale = await owner.steering.enqueue(
+ streamId,
+ {
+ steerId: 'steer-stale-epoch',
+ text: 'belongs to the previous run',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ },
+ first.createdAt,
+ );
+ expect(stale).toBe(STEER_ENQUEUE_NOT_RUNNING);
+ expect(await owner.steering.peek(streamId)).toEqual([]);
+
+ const live = await owner.steering.enqueue(
+ streamId,
+ {
+ steerId: 'steer-live-epoch',
+ text: 'belongs to the live run',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ },
+ replacement.createdAt,
+ );
+ expect(live).toBe(1);
+ }, 30000);
+
+ /**
+ * Redis parks leftover steers inside its terminal-transition Lua, which
+ * projects each item field by field — so a field added to SteerQueueItem
+ * is silently dropped there unless the projection is updated too. A steer
+ * recovered from `/chat/status` would come back without its interrupting
+ * label even though every non-Redis path preserves it.
+ */
+ test('the terminal-transition Lua keeps preempt on parked steers', async () => {
+ const owner = createReplica();
+ const streamId = `${testPrefix}-park-preempt-${Date.now()}`;
+
+ const job = await owner.createJob(streamId, 'user-1');
+ await owner.steering.enqueue(
+ streamId,
+ {
+ steerId: 'steer-parked-preempt',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ },
+ job.createdAt,
+ );
+
+ const store = new RedisJobStore(ioredisClient!);
+ const moved = await store.transitionStatus(streamId, { from: 'running', to: 'complete' });
+ expect(moved).toBe(true);
+
+ const claimed = await store.claimParkedSteers(streamId, 'user-1');
+ expect(claimed).toBeDefined();
+ const parked = JSON.parse(claimed as string) as {
+ steers: Array<{ steerId: string; preempt?: boolean }>;
+ };
+ expect(parked.steers).toHaveLength(1);
+ expect(parked.steers[0].steerId).toBe('steer-parked-preempt');
+ expect(parked.steers[0].preempt).toBe(true);
+ }, 30000);
+
+ test('a stale generation id from another replica cannot arm the live job', async () => {
+ const owner = createReplica();
+ const router = createReplica();
+ const streamId = `${testPrefix}-preempt-xreplica-stale-${Date.now()}`;
+
+ const job = await owner.createJob(streamId, 'user-1');
+
+ /**
+ * A negative assertion cannot be established by waiting: an undelivered
+ * stale arm and a fenced one look identical. So bracket it between two
+ * control arms on the same channel. The first proves the owner is
+ * listening BEFORE the stale one is published, and the second — sent
+ * after it, over the same publisher connection, so Redis orders it
+ * behind — proves the stale arm has already had its chance to land.
+ */
+ await publishUntil(
+ () => router.requestPreempt(streamId, 'control-before', job.createdAt),
+ () => owner.getArmedPreemptIds(streamId).includes('control-before'),
+ 'the first control arm',
+ );
+
+ await router.requestPreempt(streamId, 'steer-stale', job.createdAt - 1);
+
+ await publishUntil(
+ () => router.requestPreempt(streamId, 'control-after', job.createdAt),
+ () => owner.getArmedPreemptIds(streamId).includes('control-after'),
+ 'the second control arm',
+ );
+
+ expect(owner.getArmedPreemptIds(streamId)).not.toContain('steer-stale');
+ }, 40000);
+ });
});
diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
index e02b3e0249..e1c43cf097 100644
--- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts
@@ -804,7 +804,10 @@ describe('RedisEventTransport', () => {
mockPublisher as unknown as Redis,
mockSubscriber as unknown as Redis,
);
+ /** Both manager-level channel registrations are suppressed so the gated
+ * `subscribe` below belongs to the SSE subscription under test. */
Object.defineProperty(transport, 'onAbort', { value: undefined });
+ Object.defineProperty(transport, 'onPreempt', { value: undefined });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }),
diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts
index 2e42a3f668..94c68ffe43 100644
--- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts
@@ -1626,4 +1626,102 @@ describe('RedisEventTransport Integration Tests', () => {
subscriber.disconnect();
});
});
+
+ describe('Cross-Replica Preempt Delivery', () => {
+ test('delivers a fenced arm/clear payload across instances', async () => {
+ if (!ioredisClient) {
+ console.warn('Redis not available, skipping test');
+ return;
+ }
+
+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');
+
+ const subscriber1 = (ioredisClient as Redis).duplicate();
+ const subscriber2 = (ioredisClient as Redis).duplicate();
+ const transport1 = new RedisEventTransport(ioredisClient, subscriber1);
+ const transport2 = new RedisEventTransport(ioredisClient, subscriber2);
+ const streamId = `preempt-cross-${Date.now()}`;
+ const received: Array<{ op: string; createdAt: number; steerIds: string[] }> = [];
+
+ try {
+ await transport1.onPreempt(streamId, (msg) => {
+ received.push(msg);
+ });
+
+ transport2.emitPreempt(streamId, {
+ op: 'arm',
+ createdAt: 1234,
+ steerIds: ['steer-a', 'steer-b'],
+ });
+ transport2.emitPreempt(streamId, { op: 'clear', createdAt: 1234, steerIds: ['steer-a'] });
+
+ await new Promise((resolve) => setTimeout(resolve, 300));
+
+ /**
+ * Payload fidelity and delivery, NOT ordering: two publishes from the
+ * same replica carry no cross-message ordering guarantee to a
+ * subscriber, which is exactly why the receiving side tombstones
+ * cleared ids rather than assuming arm-before-clear. Asserting order
+ * here would test something stronger than the protocol promises.
+ */
+ expect(received).toHaveLength(2);
+ expect(received).toContainEqual({
+ op: 'arm',
+ createdAt: 1234,
+ steerIds: ['steer-a', 'steer-b'],
+ });
+ expect(received).toContainEqual({
+ op: 'clear',
+ createdAt: 1234,
+ steerIds: ['steer-a'],
+ });
+ } finally {
+ transport1.cleanup(streamId);
+ transport1.destroy();
+ transport2.destroy();
+ subscriber1.disconnect();
+ subscriber2.disconnect();
+ }
+ });
+
+ test('unsubscribe removes only this registration and keeps the channel for a replacement', async () => {
+ if (!ioredisClient) {
+ console.warn('Redis not available, skipping test');
+ return;
+ }
+
+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');
+
+ const subscriber1 = (ioredisClient as Redis).duplicate();
+ const subscriber2 = (ioredisClient as Redis).duplicate();
+ const transport1 = new RedisEventTransport(ioredisClient, subscriber1);
+ const transport2 = new RedisEventTransport(ioredisClient, subscriber2);
+ const streamId = `preempt-unsub-${Date.now()}`;
+ const oldGeneration: string[] = [];
+ const newGeneration: string[] = [];
+
+ try {
+ const unsubscribeOld = await transport1.onPreempt(streamId, (msg) => {
+ oldGeneration.push(...msg.steerIds);
+ });
+ await transport1.onPreempt(streamId, (msg) => {
+ newGeneration.push(...msg.steerIds);
+ });
+
+ unsubscribeOld();
+ transport2.emitPreempt(streamId, { op: 'arm', createdAt: 99, steerIds: ['steer-live'] });
+
+ await new Promise((resolve) => setTimeout(resolve, 300));
+
+ expect(oldGeneration).toEqual([]);
+ expect(newGeneration).toEqual(['steer-live']);
+ } finally {
+ transport1.cleanup(streamId);
+ transport1.destroy();
+ transport2.destroy();
+ subscriber1.disconnect();
+ subscriber2.disconnect();
+ }
+ });
+ });
});
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
index 838fa49c23..cbacf0b204 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts
@@ -236,8 +236,17 @@ describe('RedisJobStore', () => {
isTemporary: false,
promptTokens: 0,
discoveredTools: [],
+ preemptCapable: true,
});
+ /**
+ * `serializeJob` writes every boolean generically but `deserializeJob` is
+ * an explicit field map, so a new flag silently vanishes on read unless
+ * it is added there too. For `preemptCapable` that meant interrupt-steer
+ * degrading to ordinary steering in every Redis deployment.
+ */
+ expect(job.preemptCapable).toBe(true);
+
expect(job).toMatchObject({
streamId: 'stream-metadata',
userId: 'user-1',
diff --git a/packages/api/src/stream/__tests__/steering.spec.ts b/packages/api/src/stream/__tests__/steering.spec.ts
index 3f39de295f..32edb26524 100644
--- a/packages/api/src/stream/__tests__/steering.spec.ts
+++ b/packages/api/src/stream/__tests__/steering.spec.ts
@@ -1,6 +1,7 @@
+import { logger } from '@librechat/data-schemas';
import { SteerEvents } from 'librechat-data-provider';
import type { TPendingSteer, Agents } from 'librechat-data-provider';
-import type { SteerQueueItem } from '~/stream/interfaces/IJobStore';
+import type { SteerQueueItem, IEventTransport } from '~/stream/interfaces/IJobStore';
import type { ResumeState, ServerSentEvent } from '~/types';
import {
STEER_ENQUEUE_NOT_RUNNING,
@@ -55,6 +56,30 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =
expect(await manager.steering.enqueue(streamId, buildSteer('two'))).toBe(2);
});
+ /**
+ * The steer route decides capability against a job it read several awaits
+ * earlier. If the run is replaced in between, an unfenced enqueue puts the
+ * item on the REPLACEMENT queue while the preempt flag and the arm still
+ * name the previous epoch — so the arm is fenced out at the owner and the
+ * 202 promises an interrupt that cannot happen. Rejecting is the honest
+ * outcome: the run the caller was told about is gone.
+ */
+ test('refuses an item fenced to a generation that has been replaced', async () => {
+ const streamId = 'steer-enqueue-fenced';
+ const first = await manager.createJob(streamId, 'user-1');
+ const replacement = await manager.createJob(streamId, 'user-1');
+ expect(replacement.createdAt).not.toBe(first.createdAt);
+
+ expect(await manager.steering.enqueue(streamId, buildSteer('stale'), first.createdAt)).toBe(
+ STEER_ENQUEUE_NOT_RUNNING,
+ );
+ expect(await manager.steering.peek(streamId)).toEqual([]);
+
+ expect(
+ await manager.steering.enqueue(streamId, buildSteer('live'), replacement.createdAt),
+ ).toBe(1);
+ });
+
test('rejects when the job does not exist', async () => {
expect(await manager.steering.enqueue('nonexistent', buildSteer('x'))).toBe(
STEER_ENQUEUE_NOT_RUNNING,
@@ -839,3 +864,460 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
}
});
});
+
+describe('preempt request lifecycle (in-memory)', () => {
+ let manager: GenerationJobManagerClass;
+
+ beforeEach(() => {
+ manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ });
+
+ afterEach(async () => {
+ await manager.destroy();
+ });
+
+ test('requestPreempt without a runtime is a no-op and the poll stays false', () => {
+ manager.requestPreempt('preempt-no-runtime', 'steer-1', Date.now());
+ expect(manager.isPreemptRequested('preempt-no-runtime')).toBe(false);
+ });
+
+ test('arms against the live generation and stays armed until cleared', async () => {
+ const streamId = 'preempt-arm';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+
+ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('refuses to arm when jobCreatedAt does not match the live generation', async () => {
+ const streamId = 'preempt-stale-arm';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt - 1);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('clears only the removed steerIds', async () => {
+ const streamId = 'preempt-partial-clear';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+ manager.requestPreempt(streamId, 'steer-2', job.createdAt);
+ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ manager.noteSteersRemoved(streamId, ['steer-2'], job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('caps armed requests at STEER_QUEUE_MAX_DEPTH', async () => {
+ const streamId = 'preempt-cap';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH + 5; i++) {
+ manager.requestPreempt(streamId, `steer-${i}`, job.createdAt);
+ }
+ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH; i++) {
+ manager.noteSteersRemoved(streamId, [`steer-${i}`], job.createdAt);
+ }
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('arming never changes job status', async () => {
+ const streamId = 'preempt-status';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+ expect((await manager.getJob(streamId))?.status).toBe('running');
+ });
+
+ test('a failing preempt subscription degrades the job instead of crashing the process', async () => {
+ const transport: IEventTransport = new InMemoryEventTransport();
+ transport.onPreempt = jest.fn().mockRejectedValue(new Error('SUBSCRIBE failed'));
+
+ /**
+ * The registration is deliberately detached, so a rejection propagating out
+ * of it would be unhandled — fatal under Node's default settings.
+ */
+ const unhandled: unknown[] = [];
+ const collect = (reason: unknown): number => unhandled.push(reason);
+ process.on('unhandledRejection', collect);
+ const logged = jest.spyOn(logger, 'error').mockImplementation(() => logger);
+
+ const degraded = new GenerationJobManagerClass();
+ degraded.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: transport,
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ degraded.initialize();
+
+ try {
+ const streamId = 'preempt-subscribe-fails';
+ const job = await degraded.createJob(streamId, 'user-1');
+ await new Promise((resolve) => setImmediate(resolve));
+
+ expect(unhandled).toEqual([]);
+ expect(job.status).toBe('running');
+ expect(logged).toHaveBeenCalledWith(
+ expect.stringContaining('Failed to subscribe to preempts'),
+ expect.any(Error),
+ );
+
+ /** Same-replica arming is runtime state, so it survives the lost channel. */
+ degraded.requestPreempt(streamId, 'steer-1', job.createdAt);
+ expect(degraded.isPreemptRequested(streamId)).toBe(true);
+ } finally {
+ process.off('unhandledRejection', collect);
+ logged.mockRestore();
+ await degraded.destroy();
+ }
+ });
+
+ test('abortJob retires the armed set with the runtime', async () => {
+ const streamId = 'preempt-abort';
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.steering.enqueue(streamId, {
+ steerId: 'steer-1',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ });
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+
+ await manager.abortJob(streamId);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('completeJob retires the armed set', async () => {
+ const streamId = 'preempt-complete';
+ const job = await manager.createJob(streamId, 'user-1');
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+
+ await manager.completeJob(streamId, undefined, job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('toPendingSteer keeps the preempt label for parked/replayed chips', () => {
+ const item: SteerQueueItem = {
+ steerId: 'steer-1',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ };
+ expect(toPendingSteer(item).preempt).toBe(true);
+ expect(toPendingSteer({ ...item, preempt: undefined }).preempt).toBeUndefined();
+ });
+
+ test('a late arm cannot resurrect an already-cleared steer', async () => {
+ const streamId = 'preempt-late-arm';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ /** The generating replica drained + cleared before this replica's arm. */
+ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('the tombstone is per-steer, not a blanket disarm', async () => {
+ const streamId = 'preempt-tombstone-scope';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+ manager.requestPreempt(streamId, 'steer-2', job.createdAt);
+
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ manager.noteSteersRemoved(streamId, ['steer-2'], job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('clearPreemptRequests disarms the ids it is given', async () => {
+ const streamId = 'preempt-clear-all';
+ const job = await manager.createJob(streamId, 'user-1');
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+ manager.requestPreempt(streamId, 'steer-2', job.createdAt);
+
+ const armed = manager.getArmedPreemptIds(streamId, job.createdAt);
+ expect(armed.sort()).toEqual(['steer-1', 'steer-2']);
+ manager.clearPreemptRequests(streamId, armed, job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * A steer can enqueue and arm while a drain is in flight. The empty-boundary
+ * disarm must not wipe it — its queue item is live and uninjected.
+ */
+ test('clearPreemptRequests spares an arm that landed after the snapshot', async () => {
+ const streamId = 'preempt-clear-scoped';
+ const job = await manager.createJob(streamId, 'user-1');
+ manager.requestPreempt(streamId, 'steer-old', job.createdAt);
+
+ const snapshot = manager.getArmedPreemptIds(streamId, job.createdAt);
+ manager.requestPreempt(streamId, 'steer-new', job.createdAt);
+ manager.clearPreemptRequests(streamId, snapshot, job.createdAt);
+
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ });
+
+ test('clearPreemptRequests refuses a stale generation', async () => {
+ const streamId = 'preempt-clear-stale';
+ const job = await manager.createJob(streamId, 'user-1');
+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);
+
+ manager.clearPreemptRequests(streamId, ['steer-1'], job.createdAt - 1);
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ });
+
+ /**
+ * Every drained or cancelled steer is tombstoned, not just preempting ones,
+ * so a refusing cap would stop recording after a long generation and let the
+ * late-arm race resurface. Eviction keeps the newest removals authoritative.
+ */
+ test('tombstones evict oldest-first instead of refusing new removals', async () => {
+ const streamId = 'preempt-tombstone-evict';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH * 2 + 5; i++) {
+ manager.noteSteersRemoved(streamId, [`bulk-${i}`], job.createdAt);
+ }
+ /** The most recent removal must still be remembered. */
+ const recent = `bulk-${STEER_QUEUE_MAX_DEPTH * 2 + 4}`;
+ manager.requestPreempt(streamId, recent, job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * An arm lives only in the owning replica's runtime; the steer's `preempt`
+ * flag is durable. A resume landing on another replica must rebuild the
+ * armed set from the queue or the acknowledged interrupt silently waits for
+ * an ordinary tool boundary.
+ */
+ test('rearmQueuedPreempts rebuilds the armed set from the durable queue', async () => {
+ const streamId = 'preempt-rearm';
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.steering.enqueue(streamId, {
+ steerId: 'steer-preempt',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ });
+ await manager.steering.enqueue(streamId, {
+ steerId: 'steer-plain',
+ text: 'ordinary steer',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ });
+
+ /** Fresh owner: nothing armed locally yet. */
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+
+ const rearmed = await manager.rearmQueuedPreempts(streamId, job.createdAt);
+
+ expect(rearmed).toBe(1);
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toEqual(['steer-preempt']);
+ });
+
+ /**
+ * A replica that only READ this job still installed a facade runtime and
+ * subscribed, so it can hold an arm whose clear it later missed. Promotion
+ * to owner on HITL resume would make that orphan live: the first resumed
+ * stream seals on a steer that is no longer queued, drains nothing, and
+ * truncates the resumed answer. The durable queue is the authority at a
+ * handover, so an arm it does not back must be dropped, not merged.
+ */
+ test('rearmQueuedPreempts drops an armed id the durable queue no longer backs', async () => {
+ const streamId = 'preempt-rearm-orphan';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ /** Arm survived from before ownership moved; its steer is long drained. */
+ manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);
+ await manager.steering.enqueue(streamId, {
+ steerId: 'steer-live',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ });
+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toContain('steer-orphan');
+
+ const rearmed = await manager.rearmQueuedPreempts(streamId, job.createdAt);
+
+ expect(rearmed).toBe(1);
+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toEqual(['steer-live']);
+ });
+
+ /**
+ * `approvals.resolve` reopens steering before reconciliation runs, so a
+ * steer enqueued on another replica can arm this one WHILE the queue read
+ * is in flight. Reading the queue first would see that arm as unbacked and
+ * tombstone a live interrupt the route already acknowledged — and the
+ * tombstone would block the re-arm, so it could never recover.
+ */
+ test('an arm that lands while the queue is being read is not tombstoned', async () => {
+ const streamId = 'preempt-rearm-race';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ const peek = manager.steering.peek.bind(manager.steering);
+ const spy = jest
+ .spyOn(manager.steering, 'peek')
+ .mockImplementation(async (id: string, expectedCreatedAt?: number) => {
+ const snapshot = await peek(id, expectedCreatedAt);
+ /** Another replica commits a steer and publishes its arm, both after
+ * this snapshot was taken. */
+ await manager.steering.enqueue(streamId, {
+ steerId: 'steer-inflight',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ });
+ manager.requestPreempt(streamId, 'steer-inflight', job.createdAt);
+ return snapshot;
+ });
+
+ try {
+ await manager.rearmQueuedPreempts(streamId, job.createdAt);
+ } finally {
+ spy.mockRestore();
+ }
+
+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toContain('steer-inflight');
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ });
+
+ test('an orphan dropped at handover is tombstoned against a late arm', async () => {
+ const streamId = 'preempt-rearm-orphan-tombstone';
+ const job = await manager.createJob(streamId, 'user-1');
+
+ manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);
+ await manager.rearmQueuedPreempts(streamId, job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+
+ /** A publish that was in flight while ownership moved must not revive it. */
+ manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('rearmQueuedPreempts refuses a stale generation and arms nothing', async () => {
+ const streamId = 'preempt-rearm-stale';
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.steering.enqueue(streamId, {
+ steerId: 'steer-preempt',
+ text: 'interrupt me',
+ userId: 'user-1',
+ createdAt: Date.now(),
+ preempt: true,
+ });
+
+ expect(await manager.rearmQueuedPreempts(streamId, job.createdAt - 1)).toBe(0);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ /**
+ * A non-owning replica can only publish. The subscriber count is NOT proof
+ * of owner receipt — this replica's own facade subscription is counted too
+ * — so a successful publish is reported as armed and only a rejected one
+ * is reported as unarmed. See the acknowledgement-semantics note on #14518.
+ */
+ test('a successful cross-replica publish reports armed', async () => {
+ const streamId = 'preempt-published';
+ const published = new GenerationJobManagerClass();
+ published.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: Object.assign(new InMemoryEventTransport(), {
+ emitPreempt: async () => 0,
+ }),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ published.initialize();
+ try {
+ expect(await published.requestPreempt(streamId, 'steer-1', Date.now())).toBe(true);
+ } finally {
+ await published.destroy();
+ }
+ });
+
+ /**
+ * A facade runtime exists on any replica that merely READ the job, so
+ * ownership must come from `ownedJobs` — arming a facade satisfies nothing
+ * while reporting success.
+ */
+ test('a facade runtime on a non-owning replica does not count as armed locally', async () => {
+ const streamId = 'preempt-facade';
+ const facade = new GenerationJobManagerClass();
+ facade.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ facade.initialize();
+ try {
+ /** No transport publish and no ownership: nothing can arm. */
+ expect(await facade.requestPreempt(streamId, 'steer-1', Date.now())).toBe(false);
+ expect(facade.isPreemptRequested(streamId)).toBe(false);
+ } finally {
+ await facade.destroy();
+ }
+ });
+
+ /**
+ * The steer drained at an ordinary boundary while the request was still in
+ * flight, so its id is tombstoned and no arm is accepted — the 202 must not
+ * claim an interrupt that cannot happen.
+ */
+ test('reports not-armed when the id was already tombstoned', async () => {
+ const streamId = 'preempt-tombstoned-arm';
+ const job = await manager.createJob(streamId, 'user-1');
+ await manager.noteSteersRemoved(streamId, ['steer-late'], job.createdAt);
+
+ expect(await manager.requestPreempt(streamId, 'steer-late', job.createdAt)).toBe(false);
+ expect(manager.isPreemptRequested(streamId)).toBe(false);
+ });
+
+ test('reports armed when this replica owns the generation', async () => {
+ const streamId = 'preempt-owned-here';
+ const job = await manager.createJob(streamId, 'user-1');
+ expect(await manager.requestPreempt(streamId, 'steer-1', job.createdAt)).toBe(true);
+ expect(manager.isPreemptRequested(streamId)).toBe(true);
+ });
+
+ test('a failed publish does not throw out of requestPreempt', async () => {
+ const streamId = 'preempt-publish-throws';
+ const failing = new GenerationJobManagerClass();
+ failing.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: Object.assign(new InMemoryEventTransport(), {
+ emitPreempt: async () => {
+ throw new Error('redis down');
+ },
+ }),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ failing.initialize();
+ try {
+ expect(await failing.requestPreempt(streamId, 'steer-1', Date.now())).toBe(false);
+ } finally {
+ await failing.destroy();
+ }
+ });
+});
diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts
index 9d3d55a7e6..fb1865dabb 100644
--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts
+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts
@@ -591,11 +591,18 @@ export class InMemoryJobStore implements IJobStore {
// Single-threaded event loop makes each read-check-write indivisible, so
// the status guard and depth cap are exact without extra locking.
- async enqueueSteer(streamId: string, item: SteerQueueItem): Promise<number> {
+ async enqueueSteer(
+ streamId: string,
+ item: SteerQueueItem,
+ expectedCreatedAt?: number,
+ ): Promise<number> {
const job = this.jobs.get(streamId);
if (!job || job.status !== 'running' || this.closedSteerQueues.has(streamId)) {
return STEER_ENQUEUE_NOT_RUNNING;
}
+ if (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt) {
+ return STEER_ENQUEUE_NOT_RUNNING;
+ }
let queue = this.steerQueues.get(streamId);
if (!queue) {
queue = [];
diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.ts
index 1cf32f28d1..bce0b9bedc 100644
--- a/packages/api/src/stream/implementations/RedisEventTransport.ts
+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts
@@ -1,6 +1,6 @@
import { logger } from '@librechat/data-schemas';
import type { Redis, Cluster } from 'ioredis';
-import type { IEventTransport } from '~/stream/interfaces/IJobStore';
+import type { IEventTransport, PreemptMessage } from '~/stream/interfaces/IJobStore';
import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication';
import { instrumentIORedisClient, RedisUseCases } from '~/cache/redisTelemetry';
@@ -32,6 +32,7 @@ const EventTypes = {
DONE: 'done',
ERROR: 'error',
ABORT: 'abort',
+ PREEMPT: 'preempt',
} as const;
interface PubSubMessage {
@@ -42,6 +43,8 @@ interface PubSubMessage {
error?: string;
/** Immutable identity of the generation that emitted the event. */
generationId?: number;
+ /** Payload for PREEMPT messages; fenced by its own createdAt. */
+ preempt?: PreemptMessage;
}
/**
@@ -63,6 +66,10 @@ interface AbortRegistration {
callback: (generationId?: number) => void;
}
+interface PreemptRegistration {
+ callback: (msg: PreemptMessage) => void;
+}
+
/**
* Allocate a sequence number and publish the event in a single round trip.
*
@@ -147,6 +154,7 @@ interface StreamSubscribers {
allSubscribersLeftCallback?: () => void;
/** Abort callbacks - called when abort signal is received from any replica */
abortCallbacks: Set<AbortRegistration>;
+ preemptCallbacks: Set<PreemptRegistration>;
/** Reorder buffer for handling out-of-order delivery in Redis Cluster */
reorderBuffer: ReorderBuffer;
}
@@ -600,6 +608,8 @@ export class RedisEventTransport implements IEventTransport {
break;
case EventTypes.ABORT:
break;
+ case EventTypes.PREEMPT:
+ break;
}
}
@@ -616,6 +626,16 @@ export class RedisEventTransport implements IEventTransport {
}
}
}
+
+ if (message.type === EventTypes.PREEMPT && message.preempt != null) {
+ for (const registration of streamState.preemptCallbacks) {
+ try {
+ registration.callback(message.preempt);
+ } catch (err) {
+ logger.error(`[RedisEventTransport] Error in preempt callback:`, err);
+ }
+ }
+ }
}
private detachStreamSubscribers(streamId: string, state: StreamSubscribers): void {
@@ -631,7 +651,12 @@ export class RedisEventTransport implements IEventTransport {
}
private unsubscribeUnusedChannel(streamId: string, state: StreamSubscribers): void {
- if (this.streams.get(streamId) !== state || state.count > 0 || state.abortCallbacks.size > 0) {
+ if (
+ this.streams.get(streamId) !== state ||
+ state.count > 0 ||
+ state.abortCallbacks.size > 0 ||
+ state.preemptCallbacks.size > 0
+ ) {
return;
}
@@ -673,6 +698,7 @@ export class RedisEventTransport implements IEventTransport {
count: 0,
handlers: new Map(),
abortCallbacks: new Set(),
+ preemptCallbacks: new Set(),
reorderBuffer: {
nextSeq: 0,
pending: new Map(),
@@ -848,6 +874,7 @@ export class RedisEventTransport implements IEventTransport {
handlers: new Map(),
allSubscribersLeftCallback: callback,
abortCallbacks: new Set(),
+ preemptCallbacks: new Set(),
reorderBuffer: {
nextSeq: 0,
pending: new Map(),
@@ -892,6 +919,7 @@ export class RedisEventTransport implements IEventTransport {
count: 0,
handlers: new Map(),
abortCallbacks: new Set(),
+ preemptCallbacks: new Set(),
reorderBuffer: {
nextSeq: 0,
pending: new Map(),
@@ -921,6 +949,76 @@ export class RedisEventTransport implements IEventTransport {
};
}
+ /**
+ * Publish a preempt arm/clear to all replicas. Unlike abort this does NOT
+ * stop the run — the generating replica seals its current model stream at
+ * the next provider-safe boundary. Same channel and subscription as every
+ * other stream event; fenced by `msg.createdAt` on the receiving side.
+ */
+ /**
+ * Resolves to the number of replicas that received the message, so an ARM
+ * can be acknowledged only once it actually reached someone. Unlike abort
+ * (fire-and-forget, because a failed abort is retried by the user hitting
+ * stop again) an unheard arm is invisible: the route would answer
+ * `preempt: true` for a seal that never happens. Rejects on publish
+ * failure; callers decide what to do.
+ */
+ async emitPreempt(streamId: string, msg: PreemptMessage): Promise<number> {
+ const channel = CHANNELS.events(streamId);
+ const message: PubSubMessage = {
+ type: EventTypes.PREEMPT,
+ preempt: msg,
+ };
+
+ return this.publisher.publish(channel, JSON.stringify(message));
+ }
+
+ /**
+ * Register callback for preempt signals from any replica.
+ * Resolves once the Redis channel is active so callers can safely arm.
+ * The returned function removes only this registration, so a terminal
+ * generation releases its channel without touching a same-stream
+ * replacement.
+ */
+ async onPreempt(streamId: string, callback: (msg: PreemptMessage) => void): Promise<() => void> {
+ const channel = CHANNELS.events(streamId);
+ let state = this.streams.get(streamId);
+
+ if (!state) {
+ state = {
+ count: 0,
+ handlers: new Map(),
+ abortCallbacks: new Set(),
+ preemptCallbacks: new Set(),
+ reorderBuffer: {
+ nextSeq: 0,
+ pending: new Map(),
+ flushTimeout: null,
+ deliveryDeferred: false,
+ },
+ };
+ this.streams.set(streamId, state);
+ }
+
+ const registration = { callback };
+ state.preemptCallbacks.add(registration);
+
+ try {
+ await this.ensureChannelSubscription(channel);
+ } catch (error) {
+ state.preemptCallbacks.delete(registration);
+ this.unsubscribeUnusedChannel(streamId, state);
+ throw error;
+ }
+
+ return () => {
+ if (this.streams.get(streamId) !== state || !state.preemptCallbacks.delete(registration)) {
+ return;
+ }
+ this.unsubscribeUnusedChannel(streamId, state);
+ };
+ }
+
/**
* Get all tracked stream IDs (for orphan cleanup)
*/
@@ -946,6 +1044,7 @@ export class RedisEventTransport implements IEventTransport {
state.handlers.clear();
state.allSubscribersLeftCallback = undefined;
state.abortCallbacks.clear();
+ state.preemptCallbacks.clear();
}
this.resetReorderBuffer(streamId);
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index 7f22304e49..4bc31f4500 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -81,6 +81,7 @@ const JOB_CAS_LUA =
'if decoded and type(item) == "table" then ' +
'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
'if item.files then projected.files = item.files end ' +
+ 'if item.preempt then projected.preempt = item.preempt end ' +
'steers[#steers + 1] = projected ' +
'end ' +
'end ' +
@@ -214,6 +215,7 @@ const STALE_JOB_DELETE_LUA =
'if decoded and type(item) == "table" then ' +
'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +
'if item.files then projected.files = item.files end ' +
+ 'if item.preempt then projected.preempt = item.preempt end ' +
'steers[#steers + 1] = projected ' +
'end ' +
'end ' +
@@ -328,6 +330,7 @@ const RUNSTEPS_READ_LUA =
* Returns: new depth, -1 (not running / closed), or -2 (queue full)
*/
const STEER_ENQUEUE_LUA =
+ 'if ARGV[4] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[4] then return -1 end ' +
'if redis.call("HGET", KEYS[1], "status") ~= "running" then return -1 end ' +
'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return -1 end ' +
'if redis.call("LLEN", KEYS[2]) >= tonumber(ARGV[3]) then return -2 end ' +
@@ -1802,7 +1805,11 @@ export class RedisJobStore implements IJobStore {
// ===== Steering Queue Methods =====
- async enqueueSteer(streamId: string, item: SteerQueueItem): Promise<number> {
+ async enqueueSteer(
+ streamId: string,
+ item: SteerQueueItem,
+ expectedCreatedAt?: number,
+ ): Promise<number> {
const result = await this.redis.eval(
STEER_ENQUEUE_LUA,
2,
@@ -1811,6 +1818,7 @@ export class RedisJobStore implements IJobStore {
JSON.stringify(item),
String(this.ttl.running),
String(STEER_QUEUE_MAX_DEPTH),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
);
if (typeof result !== 'number') {
return STEER_ENQUEUE_NOT_RUNNING;
@@ -2237,6 +2245,12 @@ export class RedisJobStore implements IJobStore {
isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,
// Deferred tools discovered before a HITL pause; replayed into createRun on resume.
discoveredTools: data.discoveredTools ? JSON.parse(data.discoveredTools) : undefined,
+ /** The owning replica's seal capability. `serializeJob` writes every
+ * boolean generically, but this mapper is explicit — omitting it here
+ * drops the flag on every read, so the steer route would compute
+ * `preemptArmed: false` and silently degrade interrupt-steer to
+ * tool-boundary steering in EVERY Redis deployment. */
+ preemptCapable: data.preemptCapable != null ? data.preemptCapable === '1' : undefined,
titleEvent: data.titleEvent || undefined,
replayEvents: data.replayEvents || undefined,
contextUsage: data.contextUsage || undefined,
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index 4d1f023685..fe5313f50a 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -59,6 +59,14 @@ export interface SerializableJobData {
* absent from the schema-only toolMap and resume would fail with "unknown tool".
*/
discoveredTools?: string[];
+ /**
+ * Whether the replica that OWNS this generation can seal mid-stream
+ * (`PreemptBoundary` wiring). Recorded at createJob because the steer route
+ * may land on a different replica — during a rolling deploy its own SDK
+ * probe would answer for the wrong process. Absent on jobs created before
+ * preempt shipped, which reads as incapable: the honest outcome.
+ */
+ preemptCapable?: boolean;
/** Whether the user-message created event has been emitted */
createdEventEmitted?: boolean;
@@ -149,6 +157,7 @@ export type JobMetadataPatch = Partial<
| 'isTemporary'
| 'promptTokens'
| 'discoveredTools'
+ | 'preemptCapable'
>
>;
@@ -186,6 +195,24 @@ export interface SteerQueueItem {
* drain re-fetches each file by id scoped to the run's user and encodes
* fresh, so nothing here is trusted beyond identifying the file. */
files?: Partial<TFile>[];
+ /** The steer asked to seal the live model stream at the next provider-safe
+ * boundary instead of waiting for a tool step. Durable so a parked,
+ * claimed, or replayed chip keeps its "interrupting" label. */
+ preempt?: boolean;
+}
+
+/**
+ * Cross-replica preempt signal. Unlike abort this does NOT stop the run — it
+ * asks the generating replica to seal its current model stream at the next
+ * provider-safe boundary so the queued steer can inject there. Fenced by
+ * `createdAt`: a stale publish must never arm a replacement job on the same
+ * streamId.
+ */
+export interface PreemptMessage {
+ op: 'arm' | 'clear';
+ /** Generation identity (`SerializableJobData.createdAt`) this belongs to. */
+ createdAt: number;
+ steerIds: string[];
}
/** Maximum steers a single run can have queued at once. */
@@ -624,7 +651,7 @@ export interface IJobStore {
* {@link STEER_ENQUEUE_NOT_RUNNING} when the job is missing, not running,
* or closed, or {@link STEER_ENQUEUE_QUEUE_FULL} at max depth.
*/
- enqueueSteer(streamId: string, item: SteerQueueItem): Promise<number>;
+ enqueueSteer(streamId: string, item: SteerQueueItem, expectedCreatedAt?: number): Promise<number>;
/**
* Atomically take ALL queued steers, FIFO. Empty array when none. With
@@ -751,6 +778,27 @@ export interface IEventTransport {
callback: (generationId?: number) => void,
): void | (() => void) | Promise<void | (() => void)>;
+ /**
+ * Publish a preempt arm/clear to all replicas (Redis mode). Unlike abort
+ * this does NOT stop the run — it asks the generating replica to seal its
+ * current model stream at the next provider-safe boundary. Fenced by
+ * {@link PreemptMessage.createdAt} against replacement jobs.
+ * Optional - only implemented in Redis transport.
+ */
+ emitPreempt?(streamId: string, msg: PreemptMessage): void | Promise<number>;
+
+ /**
+ * Register callback for preempt signals from any replica (Redis mode).
+ * An async implementation resolves only after it can receive messages.
+ * The returned function removes only this registration, allowing a terminal
+ * generation to release its channel without affecting a same-stream replacement.
+ * Optional - only implemented in Redis transport.
+ */
+ onPreempt?(
+ streamId: string,
+ callback: (msg: PreemptMessage) => void,
+ ): void | (() => void) | Promise<void | (() => void)>;
+
/** Get subscriber count for a stream */
getSubscriberCount(streamId: string): number;
diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.ts
index 856e9c6413..e57a08330f 100644
--- a/packages/api/src/stream/metadata.ts
+++ b/packages/api/src/stream/metadata.ts
@@ -33,6 +33,9 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): J
if (metadata.promptTokens !== undefined) {
patch.promptTokens = metadata.promptTokens;
}
+ if (metadata.preemptCapable !== undefined) {
+ patch.preemptCapable = metadata.preemptCapable;
+ }
if (metadata.discoveredTools) {
patch.discoveredTools = metadata.discoveredTools;
}
diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts
index 75afee28cb..30b8e6326b 100644
--- a/packages/api/src/types/stream.ts
+++ b/packages/api/src/types/stream.ts
@@ -30,6 +30,8 @@ export interface GenerationJobMetadata {
* without them the paused deferred tool would be missing from the schema-only toolMap.
*/
discoveredTools?: string[];
+ /** See `SerializableJobData.preemptCapable`. */
+ preemptCapable?: boolean;
/** Set when the job is paused for human review (status === 'requires_action') */
pendingAction?: Agents.PendingAction;
}
diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.ts
index 47362c94db..651e84d0b6 100644
--- a/packages/data-provider/src/types/runs.ts
+++ b/packages/data-provider/src/types/runs.ts
@@ -118,6 +118,9 @@ export type TPendingSteer = {
text: string;
createdAt?: number;
files?: Partial<TFile>[];
+ /** The steer asked to interrupt generation at the next safe boundary —
+ * kept on parked/replayed chips so the "interrupting" label survives. */
+ preempt?: boolean;
};
/** Payload of the `on_steer_applied` SSE event. */