Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🛑 feat: Preemptive Steer - Backend Interrupt & Steer (#14518)
Danny Avila7bb6651883a7
diff --git a/api/server/controllers/agents/__tests__/client.steerWiring.spec.js b/api/server/controllers/agents/__tests__/client.steerWiring.spec.jsnew file mode 100644index 0000000000..97efceb72f--- /dev/null+++ b/api/server/controllers/agents/__tests__/client.steerWiring.spec.js@@ -0,0 +1,79 @@+const AgentClient = require('../client');+const { isSteeringSupported, isSteerPreemptSupported } = require('@librechat/api');++jest.mock('@librechat/api', () => ({+ ...jest.requireActual('@librechat/api'),+ isSteeringSupported: jest.fn(() => true),+ isSteerPreemptSupported: jest.fn(() => true),+}));++const mockIsSteeringSupported = isSteeringSupported;+const mockIsPreemptSupported = isSteerPreemptSupported;++/** Minimal `this` for the wiring builder — it only reads these three. */+function buildWiring(streamId, { jobCreatedAt = 1700000000000 } = {}) {+ const self = {+ jobCreatedAt,+ options: { req: { user: { id: 'user-1' } } },+ applySteerPart: jest.fn(),+ };+ return AgentClient.prototype.buildSteerWiring.call(self, streamId);+}++describe('AgentClient.buildSteerWiring — preempt capability gating', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ mockIsSteeringSupported.mockReturnValue(true);+ mockIsPreemptSupported.mockReturnValue(true);+ });++ it('returns both boundary hooks and the poll when preempt is supported', () => {+ const wiring = buildWiring('stream-1');++ expect(typeof wiring.hook).toBe('function');+ expect(typeof wiring.preemptHook).toBe('function');+ expect(typeof wiring.preemption?.shouldPreempt).toBe('function');+ });++ /**+ * The separate capability probe is what keeps an interrupt affordance from+ * arming against an SDK that can only inject at tool boundaries: steering+ * still wires, preemption does not.+ */+ it('omits the preempt wiring when only tool-boundary steering is supported', () => {+ mockIsPreemptSupported.mockReturnValue(false);+ const wiring = buildWiring('stream-2');++ expect(typeof wiring.hook).toBe('function');+ expect(wiring.preemptHook).toBeUndefined();+ expect(wiring.preemption).toBeUndefined();+ });++ it('returns undefined entirely when steering itself is unsupported', () => {+ mockIsSteeringSupported.mockReturnValue(false);+ expect(buildWiring('stream-3')).toBeUndefined();+ });++ it('returns undefined without a streamId (no resumable job surface)', () => {+ expect(buildWiring(undefined)).toBeUndefined();+ expect(buildWiring('')).toBeUndefined();+ });++ /**+ * Both boundaries must drain through the same closures, or the two+ * injection sites could persist steer parts differently — the SDK's+ * provider-safety argument assumes identical shapes.+ */+ it('builds both hooks from one shared closures object', () => {+ const applySteerPart = jest.fn();+ const self = {+ jobCreatedAt: 1700000000000,+ options: { req: { user: { id: 'user-1' } } },+ applySteerPart,+ };+ const wiring = AgentClient.prototype.buildSteerWiring.call(self, 'stream-4');++ expect(wiring.hook).not.toBe(wiring.preemptHook);+ expect(applySteerPart).not.toHaveBeenCalled();+ });+});diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.jsindex 1d00544db4..be73779101 100644--- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js+++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js@@ -103,6 +103,9 @@ jest.mock('@librechat/data-schemas', () => ({jest.mock('@librechat/api', () => ({sendEvent: jest.fn(),+ /** Recorded onto the job so the steer route can honour the OWNING replica's+ * seal capability rather than its own probe. */+ isSteerPreemptSupported: jest.fn(() => true),getViolationInfo: (...args) => mockGetViolationInfo(...args),buildMessageFiles: jest.fn(() => []),resolveTitleTiming: jest.fn(() => 'immediate'),@@ -347,6 +350,8 @@ describe('ResumableAgentController resume metadata', () => {endpoint: 'agents',iconURL: 'https://example.com/spec-icon.png',model: 'gpt-3.5-turbo',+ /** The OWNING replica's seal capability, read by the steer route. */+ preemptCapable: true,agent_id: undefined,isTemporary: true,responseMessageId: 'follow-up-user_',diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.jsindex a52181feb0..56e0a7c8c9 100644--- a/api/server/controllers/agents/__tests__/resume.spec.js+++ b/api/server/controllers/agents/__tests__/resume.spec.js@@ -50,6 +50,10 @@ const mockGenerationJobManager = {getJobStore: jest.fn(() => mockJobStore),getResumeState: jest.fn(),setContentParts: jest.fn(),+ /** Resume moves ownership: the new owner records its own seal capability+ * and rebuilds armed interrupts from the durable queue. */+ updateMetadata: jest.fn().mockResolvedValue(undefined),+ rearmQueuedPreempts: jest.fn().mockResolvedValue(0),emitChunk: jest.fn(),emitDone: jest.fn(),emitError: jest.fn(),@@ -261,6 +265,40 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(capturedInit.conversationCreatedAt).toBe('2020-01-02T03:04:05.000Z');});+ /**+ * `approvals.resolve` has already consumed the action and flipped the job+ * to running by the time steering bookkeeping runs. ioredis queues rather+ * than rejects during an outage, so `.catch` never fires and an unbounded+ * await would strand the client: its retry gets a 409 for a spent action,+ * and neither the continuation nor the failed-resume cleanup runs.+ */+ it('answers the resume even when steering bookkeeping never settles', async () => {+ mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());+ let releaseRearm;+ let releaseMetadata;+ mockGenerationJobManager.rearmQueuedPreempts.mockReturnValue(+ new Promise((resolve) => {+ releaseRearm = () => resolve(0);+ }),+ );+ mockGenerationJobManager.updateMetadata.mockReturnValue(+ new Promise((resolve) => {+ releaseMetadata = () => resolve(undefined);+ }),+ );++ try {+ const res = await post(approveBody());+ expect(res.status).toBe(200);+ expect(res.body.status).toBe('resuming');+ } finally {+ releaseRearm?.();+ releaseMetadata?.();+ mockGenerationJobManager.rearmQueuedPreempts.mockResolvedValue(0);+ mockGenerationJobManager.updateMetadata.mockResolvedValue(undefined);+ }+ }, 15000);+it('leaves conversationCreatedAt unset when the convo lookup yields nothing', async () => {mockGetConvo.mockResolvedValue(null);mockGenerationJobManager.getJob.mockResolvedValue(makeToolApprovalJob());diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex c450884165..ef4f34dd0e 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -48,7 +48,10 @@ const {createContentIndexOffsetHandlers,createSteerIndexOffsetHandlers,createSteerDrainHook,+ createSteerPreemptBoundaryHook,+ createSteerPreemptPoll,isSteeringSupported,+ isSteerPreemptSupported,buildSteerMedia,stampSteerPartMedia,createActivityLabelWiring,@@ -323,8 +326,11 @@ class AgentClient extends BaseClient {/*** The `steering` fragment for `createRun`: the run-scoped PostToolBatch- * drain hook, or `undefined` when there is no resumable job surface or the- * installed SDK cannot inject hook messages (draining would drop them).+ * drain hook — plus, when the SDK can seal mid-stream, the PreemptBoundary+ * twin and the preempt poll built from the SAME drain closures, so both+ * boundaries inject byte-identical shapes. `undefined` when there is no+ * resumable job surface or the installed SDK cannot inject hook messages+ * (draining would drop them).** @param {string | undefined} streamId*/@@ -332,18 +338,23 @@ class AgentClient extends BaseClient {if (!streamId || !isSteeringSupported()) {return undefined;}+ const drainOptions = {+ streamId,+ jobCreatedAt: this.jobCreatedAt,+ applySteer: (item) => this.applySteerPart(streamId, item),+ buildMedia: (item) =>+ buildSteerMedia({+ client: this,+ user: this.options.req?.user,+ item,+ getFiles: db.getFiles,+ }),+ };return {- hook: createSteerDrainHook({- streamId,- jobCreatedAt: this.jobCreatedAt,- applySteer: (item) => this.applySteerPart(streamId, item),- buildMedia: (item) =>- buildSteerMedia({- client: this,- user: this.options.req?.user,- item,- getFiles: db.getFiles,- }),+ hook: createSteerDrainHook(drainOptions),+ ...(isSteerPreemptSupported() && {+ preemptHook: createSteerPreemptBoundaryHook(drainOptions),+ preemption: createSteerPreemptPoll(streamId),}),};}diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex 172a895f4a..109581f38e 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -16,6 +16,7 @@ const {resolveConversationAnchor,getAgentStartupTelemetry,acceptAgentStartupTelemetry,+ isSteerPreemptSupported,} = require('@librechat/api');const { disposeClient, clientRegistry, requestDataMap } = require('~/server/cleanup');const {@@ -357,6 +358,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitendpoint: endpointOption.endpoint,iconURL: endpointIconURL,model: responseModel,+ // Recorded HERE because this process owns the generation: the steer+ // route may land on a different replica whose own SDK probe would+ // answer for the wrong process during a rolling deploy.+ preemptCapable: isSteerPreemptSupported(),// Persist the originating agent so a HITL resume can refuse to rebuild this// paused run on a different agent (see resume.js).agent_id: endpointOption.agent_id ?? req.body?.agent_id,@@ -835,6 +840,17 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit// Check abort state BEFORE calling completeJob (which triggers abort signal for cleanup)const wasAbortedBeforeComplete = job.abortController.signal.aborted;+ /**+ * A preempt boundary that had nothing to inject (cancelled/stale+ * request) ends the turn with a genuinely truncated answer — the SDK+ * reports it via preempt stats and the halt reason. Persist it with+ * the same honest `unfinished` contract an abort gets, never as a+ * silent completion.+ */+ const preemptStats = client?.run?.getPreemptStats?.();+ const preemptIncomplete =+ (preemptStats?.emptyBoundaries ?? 0) > 0 ||+ client?.run?.getHaltReason?.() === 'preempt_incomplete';const shouldGenerateTitle =addTitle &&parentMessageId === Constants.NO_PARENT &&@@ -861,9 +877,27 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitif (client.savedMessageIds && !client.savedMessageIds.has(messageId)) {await saveMessage(reqCtx,- { ...response, user: userId, unfinished: wasAbortedBeforeComplete },+ {+ ...response,+ user: userId,+ unfinished: wasAbortedBeforeComplete || preemptIncomplete,+ },{ context: 'api/server/controllers/agents/request.js - resumable response end' },);+ } else if (preemptIncomplete) {+ /**+ * A completed send already saved this row as `unfinished: false`+ * from `BaseClient.sendMessage`, and registered it in+ * `savedMessageIds` — so the branch above is skipped and the flag+ * would never reach the database. An empty preempt boundary IS a+ * completed send (the SDK halts and returns content), so re-mark+ * it explicitly, the same way the HITL pause re-marks above.+ */+ await saveMessage(+ reqCtx,+ { ...response, user: userId, unfinished: true },+ { context: 'api/server/controllers/agents/request.js - preempt incomplete' },+ );}// Check if our job was replaced by a new request before emitting@@ -955,7 +989,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitconversation,title: conversation.title,requestMessage: sanitizeMessageForTransmit(userMessage),- responseMessage: { ...response },+ responseMessage: { ...response, ...(preemptIncomplete && { unfinished: true }) },...(pendingSteers && { pendingSteers }),};diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.jsindex a33a47be8d..86ab622f67 100644--- a/api/server/controllers/agents/resume.js+++ b/api/server/controllers/agents/resume.js@@ -17,6 +17,7 @@ const {filterMalformedContentParts,decrementPendingRequest,checkAndIncrementPendingRequest,+ isSteerPreemptSupported,toPendingSteer,} = require('@librechat/api');const { disposeClient } = require('~/server/cleanup');@@ -33,6 +34,13 @@ const { saveMessage, getConvo, getMessages } = require('~/models');*/const MAX_ASK_ANSWER_LENGTH = 16_000;+/**+ * How long a resume waits on best-effort steering bookkeeping before answering+ * anyway. The approval is already consumed by that point, so a stalled Redis+ * must not strand the client behind a chip label and an arm.+ */+const STEER_RESUME_SETUP_TIMEOUT_MS = 1000;+/** De-duplicate a merged attachment list by a stable artifact identity. */function mergeAttachments(existing, incoming) {const seen = new Set();@@ -246,6 +254,17 @@ async function finalizeResumedTurn({// part that breaks reload/rendering.const content = filterMalformedContentParts(rawContent);+ /**+ * A resumed segment can end on an empty preempt boundary just as a fresh+ * one can — the boundary hook is re-registered by `buildSteerWiring` on+ * resume. Persisting that as complete would contradict the honest contract+ * the normal request path now keeps.+ */+ const preemptStats = client?.run?.getPreemptStats?.();+ const preemptIncomplete =+ (preemptStats?.emptyBoundaries ?? 0) > 0 ||+ client?.run?.getHaltReason?.() === 'preempt_incomplete';+const responseMessage = {messageId: responseMessageId,parentMessageId,@@ -255,7 +274,7 @@ async function finalizeResumedTurn({endpoint: meta.endpoint,iconURL: meta.iconURL,model: meta.model,- unfinished: false,+ unfinished: preemptIncomplete,error: false,isCreatedByUser: false,user: userId,@@ -576,6 +595,70 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)return res.status(409).json({ error: 'This action was already resolved or has expired' });}+ /**+ * Ownership moves on resume, so the job's recorded seal capability must+ * describe THIS replica — otherwise a job created on a capable replica that+ * resumes on an older one during a rolling deploy keeps acknowledging+ * steers as interrupting. Written IMMEDIATELY after the claim, before+ * client reconstruction: `approvals.resolve` has already flipped the job+ * back to `running`, so the steer route is accepting requests from here on+ * and every one of them reads this flag.+ */+ const capabilityRefresh = GenerationJobManager.updateMetadata(+ streamId,+ { preemptCapable: isSteerPreemptSupported() },+ job.createdAt,+ ).catch((error) => {+ /**+ * Logged, not fatal: this is a chip-label accuracy write, and failing the+ * user's resume over it would be disproportionate. The write only fails+ * when the job store is erroring, in which case the steer route's own+ * `getJob` is degraded too — it cannot read a stale flag it cannot read.+ */+ logger.error('[ResumeAgentController] Failed to refresh preempt capability', error);+ });++ /**+ * An interrupt steer enqueued just before the pause survives durably with+ * its `preempt` flag, but the ARM lived only in the previous owner's+ * runtime. Rebuild it from the queue so the resumed segment honours an+ * interrupt the user already had acknowledged.+ */+ const preemptRearm = GenerationJobManager.rearmQueuedPreempts(streamId, job.createdAt).catch(+ (error) => {+ logger.error('[ResumeAgentController] Failed to re-arm queued preempts', error);+ },+ );++ /**+ * BOUNDED, and the bound is the point. `.catch` only fires on rejection,+ * but ioredis queues commands while a connection is down instead of+ * rejecting, so either of these can simply never settle. That would block+ * here — after `approvals.resolve` has already consumed the action and+ * flipped the job to `running`, and before both `res.json` and the resume+ * lifecycle's own try/finally. The client times out, its retry gets a 409+ * because the action is spent, and neither the continuation nor the+ * failed-resume cleanup ever runs.+ *+ * Both writes are steering bookkeeping — a chip label and an arm that the+ * next tool boundary would honour anyway — so they finish in the background+ * rather than holding a resume the user is waiting on.+ */+ let steeringSetupTimer;+ await Promise.race([+ Promise.all([capabilityRefresh, preemptRearm]),+ new Promise((resolve) => {+ steeringSetupTimer = setTimeout(() => {+ logger.warn(+ `[ResumeAgentController] Steering setup for ${streamId} still pending after ` ++ `${STEER_RESUME_SETUP_TIMEOUT_MS}ms; continuing the resume without it`,+ );+ resolve();+ }, STEER_RESUME_SETUP_TIMEOUT_MS);+ }),+ ]);+ clearTimeout(steeringSetupTimer);+// Seed the run-scoped MCP request-context store BEFORE the ACK: once `res.json`// finishes the response, a later `getMCPRequestContext(req, res)` (from tool loading)// sees `res` as ended and returns undefined, leaving the resumed run without its MCPdiff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.tsindex e99cdff2a3..fb94d680e8 100644--- a/packages/api/src/agents/run.ts+++ b/packages/api/src/agents/run.ts@@ -15,6 +15,7 @@ import type {ContextPruningConfig,OpenAIClientOptions,StandardGraphConfig,+ StreamPreemption,LCToolRegistry,SubagentConfig,HookCallback,@@ -49,10 +50,10 @@ import {import { resolveToolApprovalPolicy, exemptAskUserQuestionFromApproval } from '~/agents/hitl/policy';import { applyCustomHandoffPromptKeyCompatibility } from '~/agents/handoffPromptKeyCompatibility';import { stripIntentFromToolRegistry, stripIntentFromToolDefinitions } from '~/agents/intent';+import { isSteeringSupported, isSteerPreemptSupported } from '~/agents/steering/runtime';import { getLLMConfig as getAnthropicLLMConfig } from '~/endpoints/anthropic/llm';import { CREATE_FILE_TOOL_NAME, EDIT_FILE_TOOL_NAME } from '~/agents/tools';import { getProviderConfig } from '~/endpoints/config/providers';-import { isSteeringSupported } from '~/agents/steering/runtime';import { extractDefaultParams } from '~/endpoints/openai/llm';import { resolveHeaders, createSafeUser } from '~/utils/env';import { getAgentCheckpointer } from '~/agents/checkpointer';@@ -1116,7 +1117,21 @@ export async function createRun({* node). Only the resumable agents controller passes this; the* OpenAI-compatible and Responses controllers have no job/SSE surface.*/- steering?: { hook: HookCallback<'PostToolBatch'> };+ steering?: {+ hook: HookCallback<'PostToolBatch'>;+ /**+ * The PreemptBoundary twin of `hook`, built via+ * `createSteerPreemptBoundaryHook` from the same drain closures. Fires+ * when the SDK seals a model stream mid-generation on a preempt request.+ */+ preemptHook?: HookCallback<'PreemptBoundary'>;+ /**+ * Level-triggered O(1) poll over the job's armed preempt requests+ * (`createSteerPreemptPoll`). Threaded into `RunConfig.preemption`, which+ * also makes the SDK reserve recursion-limit headroom for its seals.+ */+ preemption?: StreamPreemption;+ };/*** Run-scoped tool-batch summary hook (PostToolBatch). Like steering, it* registers independently of the approval policy and needs no checkpointer;@@ -1495,6 +1510,9 @@ export async function createRun({if (steering != null && isSteeringSupported()) {hooks = hooks ?? new HookRegistry();hooks.register('PostToolBatch', { hooks: [steering.hook] });+ if (steering.preemptHook != null && isSteerPreemptSupported()) {+ hooks.register('PreemptBoundary', { hooks: [steering.preemptHook] });+ }}/**@@ -1595,6 +1613,11 @@ export async function createRun({// (it gates on result-altering hooks, not registry presence)....(hitl && { humanInTheLoop: hitl.humanInTheLoop }),...(hooks && { hooks }),+ // Preemption is observation-only like the boundary hooks: the poll never+ // mutates and the SDK refuses to seal unless a PreemptBoundary matcher is+ // live, so gating both on the same capability keeps them in lockstep.+ ...(steering?.preemption != null &&+ isSteerPreemptSupported() && { preemption: steering.preemption }),};const run = await Run.create(runConfig);diff --git a/packages/api/src/agents/steering/__tests__/request.spec.ts b/packages/api/src/agents/steering/__tests__/request.spec.tsindex fd9ed846c5..57301b991a 100644--- a/packages/api/src/agents/steering/__tests__/request.spec.ts+++ b/packages/api/src/agents/steering/__tests__/request.spec.ts@@ -2,25 +2,28 @@ import type { IMongoFile } from '@librechat/data-schemas';import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';import { buildPendingAction, buildToolApprovalPayload } from '~/agents/hitl/policy';import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';+import { isSteeringSupported, isSteerPreemptSupported } from '../runtime';import { STEER_QUEUE_MAX_DEPTH } from '~/stream/interfaces/IJobStore';import { GenerationJobManager } from '~/stream/GenerationJobManager';import { handleSteerRequest, handleSteerCancel } from '../request';-import { isSteeringSupported } from '../runtime';jest.mock('../runtime', () => ({...jest.requireActual('../runtime'),isSteeringSupported: jest.fn(() => true),+ isSteerPreemptSupported: jest.fn(() => true),}));jest.spyOn(console, 'log').mockImplementation();const mockIsSupported = isSteeringSupported as jest.Mock;+const mockIsPreemptSupported = isSteerPreemptSupported as jest.Mock;const user = { id: 'user-1' };describe('handleSteerRequest (real in-memory job manager)', () => {beforeEach(() => {jest.clearAllMocks();mockIsSupported.mockReturnValue(true);+ mockIsPreemptSupported.mockReturnValue(true);GenerationJobManager.configure({jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),eventTransport: new InMemoryEventTransport(),@@ -425,6 +428,38 @@ describe('handleSteerCancel (real in-memory job manager)', () => {expect(result).toEqual({ status: 200, body: { removed: false } });});+ /**+ * ioredis queues commands during an outage instead of rejecting, so an+ * unbounded wait on the disarm publish can hang for the length of the+ * outage — with the steer already durably cancelled. A client that gives up+ * treats the cancel as failed and restores a chip for a steer that can+ * never produce an applied event.+ */+ it('answers the cancel even when the disarm publish never settles', async () => {+ const steerId = await queueSteer('cancel-stalled-disarm');+ let release: (() => void) | undefined;+ const spy = jest.spyOn(GenerationJobManager, 'noteSteersRemoved').mockReturnValue(+ new Promise<boolean>((resolve) => {+ release = () => resolve(true);+ }),+ );++ try {+ const result = await handleSteerCancel(user, {+ conversationId: 'cancel-stalled-disarm',+ steerId,+ });++ expect(result).toEqual({ status: 200, body: { removed: true } });+ expect(spy).toHaveBeenCalledWith('cancel-stalled-disarm', [steerId], expect.any(Number));+ /** Durably gone regardless — the wait was only ever a head start. */+ expect(await GenerationJobManager.steering.peek('cancel-stalled-disarm')).toEqual([]);+ } finally {+ release?.();+ spy.mockRestore();+ }+ }, 10000);+it('403s another user and leaves the steer queued', async () => {const steerId = await queueSteer('cancel-foreign');const result = await handleSteerCancel(@@ -435,3 +470,266 @@ describe('handleSteerCancel (real in-memory job manager)', () => {expect((await GenerationJobManager.steering.peek('cancel-foreign')).length).toBe(1);});});++describe('preempt flag on the steer request', () => {+ /** The owning replica records its own seal capability at createJob; the+ * route honours that rather than probing its own SDK. */+ function createCapableJob(streamId: string) {+ return GenerationJobManager.createJob(streamId, user.id, undefined, {+ initialMetadata: { preemptCapable: true },+ });+ }++ beforeEach(() => {+ jest.clearAllMocks();+ mockIsSupported.mockReturnValue(true);+ mockIsPreemptSupported.mockReturnValue(true);+ GenerationJobManager.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ GenerationJobManager.initialize();+ });++ afterEach(async () => {+ await GenerationJobManager.destroy();+ });++ it('arms the request, marks the queued item, and echoes preempt: true', async () => {+ const streamId = 'preempt-req-armed';+ await createCapableJob(streamId);++ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'stop and do this instead',+ preempt: true,+ });++ expect(result.status).toBe(202);+ expect(result.body.preempt).toBe(true);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);+ });++ it('omits the flag for an ordinary steer and never arms', async () => {+ const streamId = 'preempt-req-plain';+ await GenerationJobManager.createJob(streamId, user.id);++ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'ordinary steer',+ });++ expect(result.status).toBe(202);+ expect(result.body.preempt).toBe(false);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBeUndefined();+ });++ /**+ * The guard ladder — ownership, tenant, paused-state, agent ACL — runs+ * against the job read at the top. The owner re-read happens several awaits+ * later and can cross a replacement, so a different generation there is a+ * run this request was never authorized against. Accepting into it would+ * carry the wrong agent's metadata.+ */+ it('refuses when the run is replaced between the guards and the owner re-read', async () => {+ const streamId = 'preempt-req-replaced-midflight';+ const original = await createCapableJob(streamId);+ const originalSnapshot = await GenerationJobManager.getJob(streamId);+ /** The run is genuinely replaced, so the owner re-read returns a REAL+ * live generation — an enqueue fenced to it would succeed. */+ const replacement = await createCapableJob(streamId);+ expect(replacement.createdAt).not.toBe(original.createdAt);++ const realGetJob = GenerationJobManager.getJob.bind(GenerationJobManager);+ let calls = 0;+ const spy = jest+ .spyOn(GenerationJobManager, 'getJob')+ .mockImplementation(async (id: string) => {+ calls += 1;+ /** The guard ladder ran before the replacement; the owner re-read after. */+ return calls === 1 ? originalSnapshot : realGetJob(id);+ });++ try {+ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'interrupt me',+ preempt: true,+ });++ expect(result.status).toBe(404);+ expect(result.body.code).toBe('NO_ACTIVE_RUN');+ /** Nothing reached the replacement's queue. */+ expect(await GenerationJobManager.steering.peek(streamId)).toEqual([]);+ } finally {+ spy.mockRestore();+ }+ });++ /**+ * The queue item is already durable by the time the arm is published, and+ * the 202 reports capability rather than delivery — so waiting on a stalled+ * Redis buys nothing and risks the caller timing out and retrying, which+ * would mint a SECOND steer alongside the first and inject the same+ * instruction twice. If this ever awaits again, this test times out.+ */+ it('does not hold the 202 behind a stalled preempt publish', async () => {+ const streamId = 'preempt-req-stalled-publish';+ await createCapableJob(streamId);++ let release: (() => void) | undefined;+ const neverSettles = new Promise<boolean>((resolve) => {+ release = () => resolve(true);+ });+ const spy = jest.spyOn(GenerationJobManager, 'requestPreempt').mockReturnValue(neverSettles);++ try {+ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'interrupt me',+ preempt: true,+ });++ expect(result.status).toBe(202);+ expect(result.body.preempt).toBe(true);+ expect(spy).toHaveBeenCalledWith(streamId, expect.any(String), expect.any(Number));+ /** Still durable, so the boundary drain will find it either way. */+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);+ } finally {+ release?.();+ spy.mockRestore();+ }+ });++ /**+ * The mirror of the owner-incapable case below, and the direction a local+ * probe used to get wrong: during a rolling deploy the steer can land on an+ * un-upgraded replica while a capable replica owns the generation. The+ * route never seals — it enqueues and publishes an arm — so its own SDK is+ * irrelevant and dropping the interrupt here would lose it for no reason.+ */+ it('honours a capable OWNER even when the routing replica cannot seal', async () => {+ mockIsPreemptSupported.mockReturnValue(false);+ const streamId = 'preempt-req-old-router';+ await createCapableJob(streamId);++ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'interrupt me',+ preempt: true,+ });++ expect(result.status).toBe(202);+ expect(result.body.preempt).toBe(true);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);+ expect((await GenerationJobManager.steering.peek(streamId))[0].preempt).toBe(true);+ });++ /**+ * Rolling deploy: the route replica can seal but the replica that OWNS the+ * generation cannot. Labelling it "interrupting" would lie.+ */+ it('degrades when the OWNING replica recorded no seal capability', async () => {+ const streamId = 'preempt-req-old-owner';+ await GenerationJobManager.createJob(streamId, user.id);++ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'interrupt me',+ preempt: true,+ });++ expect(result.status).toBe(202);+ expect(result.body.preempt).toBe(false);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ it('does not arm when the enqueue itself is rejected', async () => {+ const streamId = 'preempt-req-full';+ await createCapableJob(streamId);+ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH; i++) {+ await GenerationJobManager.steering.enqueue(streamId, {+ steerId: `filler-${i}`,+ text: `filler ${i}`,+ userId: user.id,+ createdAt: Date.now(),+ });+ }++ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'too late',+ preempt: true,+ });++ expect(result.status).toBe(429);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * Cancel is live UI. Without this the request stays armed after its steer+ * is gone and seals an unrelated stretch of generation.+ */+ /**+ * Option A semantics: the 202's `preempt` mirrors the DURABLE queue flag,+ * so the response and `SteerQueueItem.preempt` can never disagree — a+ * resumed owner re-arming from the queue then honours exactly what the+ * client was told.+ */+ it('the 202 flag and the durable queue item always agree', async () => {+ const streamId = 'preempt-flag-agrees';+ await createCapableJob(streamId);++ const result = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'interrupt me',+ preempt: true,+ });+ const queued = (await GenerationJobManager.steering.peek(streamId))[0];++ expect(result.body.preempt).toBe(true);+ expect(queued.preempt).toBe(true);+ expect(result.body.preempt).toBe(queued.preempt === true);+ });++ it('cancelling a preempt steer disarms the request', async () => {+ const streamId = 'preempt-req-cancel';+ await createCapableJob(streamId);+ const queued = await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'never mind',+ preempt: true,+ });+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);++ const cancelled = await handleSteerCancel(user, {+ conversationId: streamId,+ steerId: queued.body.steerId,+ });++ expect(cancelled.body.removed).toBe(true);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ it('a lost cancel race leaves the surviving request armed', async () => {+ const streamId = 'preempt-req-cancel-miss';+ await createCapableJob(streamId);+ await handleSteerRequest(user, {+ conversationId: streamId,+ text: 'still queued',+ preempt: true,+ });++ const cancelled = await handleSteerCancel(user, {+ conversationId: streamId,+ steerId: 'never-existed',+ });++ expect(cancelled.body.removed).toBe(false);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);+ });+});diff --git a/packages/api/src/agents/steering/__tests__/runtime.spec.ts b/packages/api/src/agents/steering/__tests__/runtime.spec.tsindex 2a32d52e3b..a51242fd6e 100644--- a/packages/api/src/agents/steering/__tests__/runtime.spec.ts+++ b/packages/api/src/agents/steering/__tests__/runtime.spec.ts@@ -1,5 +1,9 @@import * as agentsSdk from '@librechat/agents';-import type { PostToolBatchHookInput, PostToolBatchHookOutput } from '@librechat/agents';+import type {+ PostToolBatchHookInput,+ PostToolBatchHookOutput,+ PreemptBoundaryHookInput,+} from '@librechat/agents';import type { SteerQueueItem } from '~/stream/interfaces/IJobStore';/** Mirrors runtime.ts's local extension — the field predates the SDK pin bump. */@@ -9,7 +13,13 @@ type SteerDrainOutput = PostToolBatchHookOutput & {import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';import { GenerationJobManager } from '~/stream/GenerationJobManager';-import { createSteerDrainHook, isSteeringSupported } from '../runtime';+import {+ createSteerDrainHook,+ createSteerPreemptBoundaryHook,+ createSteerPreemptPoll,+ isSteeringSupported,+ isSteerPreemptSupported,+} from '../runtime';jest.spyOn(console, 'log').mockImplementation();@@ -28,6 +38,17 @@ function buildSteer(steerId: string, text: string): SteerQueueItem {return { steerId, text, userId: 'user-1', createdAt: Date.now() };}+function boundaryInput(+ overrides: Partial<PreemptBoundaryHookInput> = {},+): PreemptBoundaryHookInput {+ return {+ hook_event_name: 'PreemptBoundary',+ runId: 'run-1',+ sealCount: 1,+ ...overrides,+ };+}+describe('isSteeringSupported', () => {it('mirrors the installed SDK capability flag AND replay support', () => {// CI runs against the published SDK pin (possibly pre-injectedMessages);@@ -235,3 +256,278 @@ describe('createSteerDrainHook', () => {]);});});++describe('isSteerPreemptSupported', () => {+ it('requires the preempt capability ON TOP of full steering support', () => {+ const sdk = agentsSdk as {+ HOOK_INJECTED_MESSAGES_CAPABLE?: boolean;+ HOOK_PREEMPT_BOUNDARY_CAPABLE?: boolean;+ ContentTypes?: { STEER?: string };+ };+ const expected = isSteeringSupported() && sdk.HOOK_PREEMPT_BOUNDARY_CAPABLE === true;+ expect(isSteerPreemptSupported()).toBe(expected);+ });+});++describe('createSteerPreemptBoundaryHook', () => {+ beforeEach(() => {+ GenerationJobManager.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ GenerationJobManager.initialize();+ });++ afterEach(async () => {+ await GenerationJobManager.destroy();+ });++ it('drains the same queue and emits the same shapes as the tool boundary', async () => {+ const streamId = `preempt-drain-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'first'));+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s2', 'second'));++ const applied: string[] = [];+ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: (item) => {+ applied.push(item.text);+ },+ });++ const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);+ expect(applied).toEqual(['first', 'second']);+ expect(output.injectedMessages).toEqual([+ { role: 'user', content: 'first', source: 'steer' },+ { role: 'user', content: 'second', source: 'steer' },+ ]);+ expect(await GenerationJobManager.steering.peek(streamId)).toEqual([]);+ });++ it('returns empty output when the queue drained before the seal landed', async () => {+ const streamId = `preempt-empty-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');+ const hook = createSteerPreemptBoundaryHook({ streamId, applySteer: jest.fn() });++ expect(await hook(boundaryInput(), abortSignal)).toEqual({});+ });++ it('never drains inside a subagent scope', async () => {+ const streamId = `preempt-subagent-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'kept'));++ const hook = createSteerPreemptBoundaryHook({ streamId, applySteer: jest.fn() });+ expect(await hook(boundaryInput({ agentId: 'child-agent' }), abortSignal)).toEqual({});+ expect((await GenerationJobManager.steering.peek(streamId)).map((s) => s.text)).toEqual([+ 'kept',+ ]);+ });++ it('refuses to drain when the job was replaced', async () => {+ const streamId = `preempt-replaced-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt - 1,+ applySteer: jest.fn(),+ });+ await GenerationJobManager.steering.enqueue(streamId, buildSteer('s1', 'new job steer'));++ expect(await hook(boundaryInput(), abortSignal)).toEqual({});+ expect((await GenerationJobManager.steering.peek(streamId)).map((s) => s.text)).toEqual([+ 'new job steer',+ ]);+ });++ it('clears the armed preempt request after draining', async () => {+ const streamId = `preempt-clears-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ await GenerationJobManager.steering.enqueue(streamId, {+ ...buildSteer('s1', 'interrupt me'),+ preempt: true,+ });+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);++ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: jest.fn(),+ });+ await hook(boundaryInput(), abortSignal);++ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * The shared drain body is what makes this true: a preempt satisfied at an+ * ordinary tool boundary must disarm there, or the request would seal a+ * later, unrelated stretch of generation with nothing left to inject.+ */+ it('a tool-boundary drain also clears a pending preempt request', async () => {+ const streamId = `preempt-tool-clears-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ await GenerationJobManager.steering.enqueue(streamId, {+ ...buildSteer('s1', 'interrupt me'),+ preempt: true,+ });+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);++ const hook = createSteerDrainHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: jest.fn(),+ });+ await hook(batchInput(), abortSignal);++ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * The empty-boundary self-clear: a seal was spent, so anything still armed+ * points at a steer that already left the queue. Leaving it armed would+ * seal again on the next chunk and truncate an unrelated answer.+ */+ it('disarms the generation when the boundary drains nothing', async () => {+ const streamId = `preempt-empty-clears-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ GenerationJobManager.requestPreempt(streamId, 'orphaned', job.createdAt);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);++ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: jest.fn(),+ });+ expect(await hook(boundaryInput(), abortSignal)).toEqual({});++ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * A cancel whose cross-replica clear was lost leaves a stale arm. If the+ * next boundary drains a DIFFERENT steer, clearing only the drained id+ * would leave the stale one level-triggered — it would immediately seal+ * the continuation meant to answer the steer just injected, landing on an+ * empty boundary as `preempt_incomplete`.+ */+ it('a nonempty drain also clears stale arms held since the snapshot', async () => {+ const streamId = `preempt-stale-snapshot-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');++ /** Stale: armed, but its steer never reaches the queue (cancelled). */+ await GenerationJobManager.requestPreempt(streamId, 'steer-cancelled', job.createdAt);+ /** Live: queued and armed, and this is what the boundary will drain. */+ await GenerationJobManager.steering.enqueue(streamId, {+ ...buildSteer('steer-live', 'interrupt me'),+ preempt: true,+ });+ await GenerationJobManager.requestPreempt(streamId, 'steer-live', job.createdAt);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(true);++ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: jest.fn(),+ });+ const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);++ expect(output.injectedMessages).toHaveLength(1);+ /** Both the drained id and the stale snapshot id are spent. */+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });++ /** An arm that lands AFTER the snapshot is backed by a live queue item. */+ it('a nonempty drain spares an arm that landed after the snapshot', async () => {+ const streamId = `preempt-post-snapshot-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ await GenerationJobManager.steering.enqueue(streamId, {+ ...buildSteer('steer-first', 'first'),+ preempt: true,+ });+ await GenerationJobManager.requestPreempt(streamId, 'steer-first', job.createdAt);++ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: async () => {+ /** Arrives mid-drain, after the snapshot was taken. */+ await GenerationJobManager.requestPreempt(streamId, 'steer-later', job.createdAt);+ },+ });+ await hook(boundaryInput(), abortSignal);++ expect(GenerationJobManager.getArmedPreemptIds(streamId, job.createdAt)).toEqual([+ 'steer-later',+ ]);+ });++ it('clears the request even when applySteer throws mid-drain', async () => {+ const streamId = `preempt-clears-on-error-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ await GenerationJobManager.steering.enqueue(streamId, {+ ...buildSteer('s1', 'still injected'),+ preempt: true,+ });+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);++ const hook = createSteerPreemptBoundaryHook({+ streamId,+ jobCreatedAt: job.createdAt,+ applySteer: () => {+ throw new Error('emit failed');+ },+ });++ const output: SteerDrainOutput = await hook(boundaryInput(), abortSignal);+ expect(output.injectedMessages).toEqual([+ { role: 'user', content: 'still injected', source: 'steer' },+ ]);+ expect(GenerationJobManager.isPreemptRequested(streamId)).toBe(false);+ });+});++describe('createSteerPreemptPoll', () => {+ beforeEach(() => {+ GenerationJobManager.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ GenerationJobManager.initialize();+ });++ afterEach(async () => {+ await GenerationJobManager.destroy();+ });++ /**+ * The SDK contract requires a LEVEL-triggered predicate: it may be polled+ * many times before a chunk is safe to seal, and a self-clearing read would+ * silently lose the request on the first unsafe chunk.+ */+ it('keeps returning true until the request is cleared', async () => {+ const streamId = `preempt-poll-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');+ const { shouldPreempt } = createSteerPreemptPoll(streamId);++ expect(shouldPreempt()).toBe(false);+ GenerationJobManager.requestPreempt(streamId, 's1', job.createdAt);+ expect(shouldPreempt()).toBe(true);+ expect(shouldPreempt()).toBe(true);+ expect(shouldPreempt()).toBe(true);++ GenerationJobManager.noteSteersRemoved(streamId, ['s1'], job.createdAt);+ expect(shouldPreempt()).toBe(false);+ });++ it('is false for a stream with no live generation', () => {+ expect(createSteerPreemptPoll('no-such-stream').shouldPreempt()).toBe(false);+ });+});diff --git a/packages/api/src/agents/steering/index.ts b/packages/api/src/agents/steering/index.tsindex a3a79c78f2..01c8a68fcb 100644--- a/packages/api/src/agents/steering/index.ts+++ b/packages/api/src/agents/steering/index.ts@@ -1,4 +1,10 @@-export { createSteerDrainHook, isSteeringSupported } from './runtime';+export {+ createSteerDrainHook,+ createSteerPreemptBoundaryHook,+ createSteerPreemptPoll,+ isSteeringSupported,+ isSteerPreemptSupported,+} from './runtime';export type { SteerDrainHookOptions, SteerMediaResult } from './runtime';export {handleSteerRequest,diff --git a/packages/api/src/agents/steering/request.ts b/packages/api/src/agents/steering/request.tsindex 6acd9eaa33..b37b1acef4 100644--- a/packages/api/src/agents/steering/request.ts+++ b/packages/api/src/agents/steering/request.ts@@ -12,6 +12,13 @@ export const STEER_MAX_FILES = 10;const DEFAULT_STEER_MAX_LENGTH = 16000;+/**+ * How long a cancel waits for its disarm to publish before answering anyway.+ * The steer is already durably removed by then, so this only buys the clear a+ * head start; a stalled Redis must not hold the response open behind it.+ */+const STEER_DISARM_ACK_TIMEOUT_MS = 1000;+/** Character cap for a single steer message (env-overridable). */export function getSteerMaxLength(): number {return parseInt(process.env.STEER_MAX_LENGTH ?? '', 10) || DEFAULT_STEER_MAX_LENGTH;@@ -26,6 +33,11 @@ export interface SteerRequestBody {conversationId?: unknown;text?: unknown;files?: unknown;+ /** Ask the generating replica to seal the live model stream at the next+ * provider-safe boundary instead of waiting for a tool step. NEVER a+ * rejection reason: on an SDK without the capability the steer still+ * enqueues and the 202 echoes `preempt: false`. */+ preempt?: unknown;}export interface SteerCancelBody {@@ -223,14 +235,62 @@ export async function handleSteerRequest(resolvedFileIds = resolved.fileIds;}+ const wantsPreempt = body.preempt === true;+ /**+ * The OWNER's recorded capability, not this replica's probe: a steer can+ * land on any replica, so during a rolling deploy a local probe would+ * answer for the wrong process and label a steer "interrupting" that the+ * old owner can only inject at a tool boundary. Jobs created before+ * preempt shipped carry no flag, which reads as incapable — the honest+ * outcome, and the chip relabels to ordinary steering.+ *+ * Re-read rather than reused from the `job` fetched at the top of the+ * ladder: `checkAgentAccess` and file resolution are awaits, so a request+ * can span an entire HITL pause/resume that hands ownership to a replica+ * with different capability and rewrites this very flag. Only paid for by+ * requests that actually asked to interrupt.+ *+ * THIS replica's own SDK is deliberately not consulted. It never seals —+ * it enqueues and publishes an arm, neither of which touches the SDK — so+ * ANDing in a local probe would answer for the wrong process and silently+ * drop interrupts during a rolling deploy whenever the request happened to+ * land on an un-upgraded replica while a capable owner generated. When+ * this replica IS the owner the probe is redundant anyway: the flag it+ * would consult is the one this process already wrote at `createJob`.+ */+ const owner = wantsPreempt ? ((await GenerationJobManager.getJob(streamId)) ?? job) : job;+ /**+ * The re-read may have crossed a replacement. Every guard above — ownership,+ * tenant, paused-state, agent ACL — was evaluated against `job`, so a+ * different generation here is a run this request was never authorized+ * against, and accepting into it would carry the wrong agent's metadata.+ * Refuse rather than re-derive: the run the caller targeted is gone, and+ * `NO_ACTIVE_RUN` is the code the client already turns into a queued+ * follow-up.+ */+ if (owner.createdAt !== job.createdAt) {+ return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };+ }+ const preemptCapable = wantsPreempt && owner.metadata?.preemptCapable === true;const item = {steerId: randomUUID(),text,userId: user.id ?? '',createdAt: Date.now(),...(queuedFiles && { files: queuedFiles }),+ ...(preemptCapable && { preempt: true }),};- const depth = await GenerationJobManager.steering.enqueue(streamId, item);+ /**+ * Fenced to the generation the capability decision was made against. The+ * access checks, file resolution and owner re-read above are all awaits, so+ * the run can be replaced before this line: without the fence the item+ * lands on the REPLACEMENT queue while `preempt` and the arm below still+ * describe the previous epoch, so the arm is fenced out at the owner and+ * the 202 claims an interrupt that can never happen. Rejecting is honest —+ * the run the caller was told about is gone, and `NO_ACTIVE_RUN` is what+ * the client already handles by converting to a queued follow-up.+ */+ const depth = await GenerationJobManager.steering.enqueue(streamId, item, owner.createdAt);if (depth === STEER_ENQUEUE_NOT_RUNNING) {return { status: 404, body: { code: 'NO_ACTIVE_RUN' } };}@@ -238,6 +298,51 @@ export async function handleSteerRequest(return { status: 429, body: { code: 'STEER_QUEUE_FULL' } };}+ /**+ * Strictly AFTER a successful enqueue: an armed request whose steer never+ * made the durable queue could seal a generation with nothing to inject.+ *+ * `preempt` in the 202 means "queued as an interrupt request", NOT "a seal+ * is guaranteed" — it mirrors the durable `item.preempt` exactly. A route+ * cannot synchronously know whether another replica will seal: proving that+ * needs a correlated request/response over pub-sub, and even then the owner+ * may finish before the arm lands. Reporting delivery instead made the+ * response disagree with the durable flag, which `rearmQueuedPreempts`+ * trusts on resume — the two must agree or a resumed owner honours an+ * interrupt the client was told had degraded.+ *+ * The gates that ARE knowable stay: the owner's recorded capability and a+ * successful enqueue. Everything past that degrades to the documented+ * fallback of injecting at the next tool boundary.+ */+ if (preemptCapable) {+ /**+ * NOT awaited. The answer no longer depends on it — the 202 reports+ * `preemptCapable`, not delivery — so awaiting only exposes the caller to+ * Redis latency after the queue item is already durable. A client that+ * times out on a stalled publish and retries mints a SECOND steer while+ * the first stays queued, injecting the same instruction twice; a lost+ * publish merely takes the documented tool-boundary fallback. The+ * asymmetry is the whole argument for detaching.+ */+ void GenerationJobManager.requestPreempt(streamId, item.steerId, owner.createdAt).then(+ (armed) => {+ if (!armed) {+ logger.warn(+ `[handleSteerRequest] Preempt arm not confirmed for ${streamId} steer=${item.steerId}; ` ++ 'the steer remains queued and will inject at the next boundary',+ );+ }+ },+ (error: unknown) => {+ logger.error(+ `[handleSteerRequest] Preempt arm failed for ${streamId} steer=${item.steerId}:`,+ error,+ );+ },+ );+ }+/** Fire-and-forget: the persisted steer part references these uploads, so* mark them used (parity with `updateFilesUsage` on normal sends) or the* upload-window TTL could reap them before replay. Must not fail the 202. */@@ -255,7 +360,13 @@ export async function handleSteerRequest(return {status: 202,- body: { status: 'queued', steerId: item.steerId, position: depth, conversationId },+ body: {+ status: 'queued',+ steerId: item.steerId,+ position: depth,+ conversationId,+ preempt: preemptCapable,+ },};}@@ -295,5 +406,53 @@ export async function handleSteerCancel(}const removed = await GenerationJobManager.steering.cancel(streamId, body.steerId);+ /** A cancelled steer must also disarm any preempt request it carried —+ * cancel is live UI, and a request left armed would seal an unrelated+ * stretch of generation, drain nothing, and end the run mid-sentence. */+ if (!removed) {+ return { status: 200, body: { removed } };+ }+ /**+ * Waited on so a failed disarm is retried and logged before the response,+ * but BOUNDED, and its outcome is deliberately NOT reported to the client.+ *+ * The bound is the load-bearing part. ioredis queues commands while a+ * connection is down rather than rejecting them, so an unbounded await here+ * can hang for as long as the outage lasts — and the item is already+ * durably cancelled at this point. A client that gives up then treats the+ * cancel as failed and restores a chip for a steer that no longer exists+ * and can never produce an applied event, which is strictly worse than the+ * lost clear this wait was protecting against. Every successful cancel+ * publishes, so ordinary steers are exposed to it too, not just preemptive+ * ones. The publish keeps running with its retry and logging intact after+ * the timeout — it is simply no longer in front of the response.+ *+ * A resolved publish is not proof the owner heard it — the delivery count+ * includes this replica's own facade subscription — so any `disarmed` flag+ * would claim a certainty the transport cannot provide, which is the same+ * over-promise the `preempt` flag was corrected for. Disarm is best effort+ * with a bounded, self-healing failure: if the clear is lost the owner+ * seals once, its empty boundary self-clears, and the turn is persisted+ * `unfinished: true` rather than silently truncated.+ *+ * `removed` stays true because it is true — the steer really did leave the+ * queue, and inverting it would make the client re-show a chip for a steer+ * that can never arrive.+ */+ const disarm = GenerationJobManager.noteSteersRemoved(streamId, [body.steerId], job.createdAt);+ let settleTimer: ReturnType<typeof setTimeout> | undefined;+ await Promise.race([+ disarm,+ new Promise<void>((resolve) => {+ settleTimer = setTimeout(() => {+ logger.warn(+ `[handleSteerCancel] Disarm publish for ${streamId} steer=${body.steerId} still pending ` ++ `after ${STEER_DISARM_ACK_TIMEOUT_MS}ms; answering the cancel and letting it retry`,+ );+ resolve();+ }, STEER_DISARM_ACK_TIMEOUT_MS);+ }),+ ]);+ clearTimeout(settleTimer);return { status: 200, body: { removed } };}diff --git a/packages/api/src/agents/steering/runtime.ts b/packages/api/src/agents/steering/runtime.tsindex dbcdcf9367..aafde3b84b 100644--- a/packages/api/src/agents/steering/runtime.ts+++ b/packages/api/src/agents/steering/runtime.ts@@ -1,6 +1,7 @@import * as agentsSdk from '@librechat/agents';import { logger } from '@librechat/data-schemas';import type {+ StreamPreemption,HookCallback,HookInputByEvent,HookOutputByEvent,@@ -71,31 +72,44 @@ export interface SteerDrainHookOptions {}/**- * Build the run-scoped `PostToolBatch` hook that drains the job's steer queue- * at each tool-batch boundary and injects each steer into graph state as its- * own user message (`role: 'user'`, `source: 'steer'` — never consolidated- * with hook context). Steers with attachments inject as multimodal content- * arrays via `buildMedia`.+ * Shared drain body for BOTH injection boundaries (PostToolBatch and+ * PreemptBoundary) — the provider-safety argument rests on the two sites+ * emitting identical `InjectedMessage` shapes, so they must share one body.*- * Subagent scopes are skipped (`input.agentId` set): a steer targets the- * top-level conversation, not a child agent's context. Hook errors are- * swallowed by the SDK's `executeHooks`, so a broken drain can never kill the- * run.+ * The injections array builds incrementally under a swallow-all catch: once a+ * steer leaves the durable queue its part is persisted by `applySteer`+ * first, so a mid-loop throw must still inject whatever was already applied+ * rather than dropping the batch. `noteSteersRemoved` runs in `finally` —+ * the drained items are out of the queue no matter what, so any armed+ * preempt request for them must clear even on the failure path, or a+ * satisfied preempt could seal a later, unrelated stretch of generation.*/-export function createSteerDrainHook(opts: SteerDrainHookOptions): HookCallback<'PostToolBatch'> {+async function drainAndBuildInjections(opts: SteerDrainHookOptions): Promise<InjectedMessage[]> {const { streamId, jobCreatedAt, applySteer, buildMedia } = opts;- return async (input: HookInputByEvent['PostToolBatch']): Promise<SteerDrainOutput> => {- if (input.agentId != null) {- return {};- }- // The replacement guard lives INSIDE the store's atomic drain: a separate- // check-then-drain could still consume a replacement job's queue if- // createJob landed between the two steps.- const steers = await GenerationJobManager.steering.drain(streamId, jobCreatedAt);- if (steers.length === 0) {- return {};- }- const injectedMessages: InjectedMessage[] = [];+ // The replacement guard lives INSIDE the store's atomic drain: a separate+ // check-then-drain could still consume a replacement job's queue if+ // createJob landed between the two steps.+ /**+ * Snapshotted BEFORE the drain so an empty boundary disarms only the+ * requests it was responsible for. A second steer can enqueue and arm while+ * the drain is in flight; that arm is backed by a live queue item and must+ * survive.+ */+ const armedBeforeDrain = GenerationJobManager.getArmedPreemptIds(streamId, jobCreatedAt);+ const steers = await GenerationJobManager.steering.drain(streamId, jobCreatedAt);+ if (steers.length === 0) {+ /**+ * Nothing to inject. The snapshotted ids point at steers that have+ * already left the queue (drained at the other boundary, cancelled, or a+ * late cross-replica arm), so disarm them — a level-triggered poll left+ * true would seal again on the next chunk and truncate an unrelated+ * answer.+ */+ GenerationJobManager.clearPreemptRequests(streamId, armedBeforeDrain, jobCreatedAt);+ return [];+ }+ const injectedMessages: InjectedMessage[] = [];+ try {for (const item of steers) {try {await applySteer(item);@@ -122,6 +136,104 @@ export function createSteerDrainHook(opts: SteerDrainHookOptions): HookCallback<source: 'steer' as const,});}+ } catch (error) {+ logger.error(`[steering] Drain interrupted for ${streamId}; injecting applied items:`, error);+ } finally {+ /**+ * Everything armed at snapshot time PLUS everything just drained. The+ * boundary has spent its seal, so a snapshotted id that did NOT come back+ * from the drain is stale — its steer left the queue by another route+ * (typically a cancel whose cross-replica clear was lost). Clearing only+ * the drained ids would leave that one level-triggered, sealing the very+ * continuation meant to answer the steer we just injected and landing on+ * an empty boundary as `preempt_incomplete`.+ *+ * Ids armed AFTER the snapshot are deliberately excluded: their queue+ * items are live and uninjected, and disarming them would strand an+ * interrupt the client was already told about.+ *+ * Not awaited: this runs on the OWNER, where the local disarm is+ * synchronous and already effective — the publish only informs other+ * replicas, and blocking a boundary drain on it would delay injection.+ */+ const spent = new Set(armedBeforeDrain);+ for (const item of steers) {+ spent.add(item.steerId);+ }+ void GenerationJobManager.noteSteersRemoved(streamId, [...spent], jobCreatedAt);+ }+ return injectedMessages;+}++/**+ * Build the run-scoped `PostToolBatch` hook that drains the job's steer queue+ * at each tool-batch boundary and injects each steer into graph state as its+ * own user message (`role: 'user'`, `source: 'steer'` — never consolidated+ * with hook context). Steers with attachments inject as multimodal content+ * arrays via `buildMedia`.+ *+ * Subagent scopes are skipped (`input.agentId` set): a steer targets the+ * top-level conversation, not a child agent's context. Hook errors are+ * swallowed by the SDK's `executeHooks`, so a broken drain can never kill the+ * run.+ */+export function createSteerDrainHook(opts: SteerDrainHookOptions): HookCallback<'PostToolBatch'> {+ return async (input: HookInputByEvent['PostToolBatch']): Promise<SteerDrainOutput> => {+ if (input.agentId != null) {+ return {};+ }+ const injectedMessages = await drainAndBuildInjections(opts);+ if (injectedMessages.length === 0) {+ return {};+ }+ return { injectedMessages };+ };+}++/**+ * The PreemptBoundary twin of {@link createSteerDrainHook}: fires after the+ * SDK seals a model stream mid-generation, drains the same durable queue+ * through the same shared body, and injects the same message shapes. Because+ * `noteSteersRemoved` lives in that shared body, a preempt satisfied at an+ * ordinary tool boundary clears its request there and cannot cause a+ * spurious seal later.+ */+export function createSteerPreemptBoundaryHook(+ opts: SteerDrainHookOptions,+): HookCallback<'PreemptBoundary'> {+ return async (+ input: HookInputByEvent['PreemptBoundary'],+ ): Promise<HookOutputByEvent['PreemptBoundary']> => {+ if (input.agentId != null) {+ return {};+ }+ const injectedMessages = await drainAndBuildInjections(opts);+ if (injectedMessages.length === 0) {+ return {};+ }return { injectedMessages };};}++/**+ * The run's `RunConfig.preemption` — a level-triggered O(1) poll over the+ * job's armed preempt requests, exactly as the SDK contract requires: it+ * keeps returning true until the boundary drain (either boundary) clears the+ * request via `noteSteersRemoved`. Never consumes on read.+ */+export function createSteerPreemptPoll(streamId: string): StreamPreemption {+ return {+ shouldPreempt: () => GenerationJobManager.isPreemptRequested(streamId),+ };+}++/**+ * Whether the installed SDK can seal a generation mid-stream and inject at+ * the resulting boundary. A SEPARATE probe from `isSteeringSupported()`:+ * reusing that flag would arm an interrupt affordance that silently does+ * nothing on an SDK that can only inject at tool boundaries.+ */+export function isSteerPreemptSupported(): boolean {+ const sdk = agentsSdk as { HOOK_PREEMPT_BOUNDARY_CAPABLE?: boolean };+ return isSteeringSupported() && sdk.HOOK_PREEMPT_BOUNDARY_CAPABLE === true;+}diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex d97318debb..9f390844c6 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -32,12 +32,22 @@ import {setGenerationJobsInFlight,recordGenerationJob,} from '~/app/metrics';+import {+ isPendingActionStale,+ isPendingActionExpired,+ STEER_QUEUE_MAX_DEPTH,+} from './interfaces/IJobStore';++/**+ * Tombstone budget per generation. Twice the queue depth so a full queue's+ * worth of removals plus in-flight arms fit without eviction in practice.+ */+const PREEMPT_TOMBSTONE_MAX = STEER_QUEUE_MAX_DEPTH * 2;import {SteeringLifecycle,toPendingSteer,synthesizeAppliedSteerEvents,} from './SteeringLifecycle';-import { isPendingActionStale, isPendingActionExpired } from './interfaces/IJobStore';import { synthesizeActivityLabelGapEvents } from '~/agents/activityLabels/wiring';import { InMemoryEventTransport } from './implementations/InMemoryEventTransport';import { InMemoryJobStore } from './implementations/InMemoryJobStore';@@ -206,6 +216,31 @@ interface RuntimeJobState {abortController: AbortController;/** Removes this generation's cross-replica abort listener without touching a replacement. */abortUnsubscribe?: () => void;+ /**+ * Cooperative-seal requests for THIS generation. Armed by `requestPreempt`+ * (locally or via a fenced cross-replica publish), polled O(1) by the run's+ * `shouldPreempt`, and cleared by `noteSteersRemoved` when the steer drains+ * at any boundary or is cancelled. Lives and dies with the runtime object —+ * no terminal bookkeeping needed beyond the listener release.+ */+ preempt?: {+ /** Generation identity this request set belongs to. */+ createdAt: number;+ /** Server-minted steerIds requested but not yet drained/cancelled. */+ ids: Set<string>;+ /**+ * steerIds whose removal was observed BEFORE their arm. Arm and clear are+ * published by different replicas over different connections, so a steer+ * drained at a tool boundary during the arming replica's enqueue round+ * trip can have its clear land first — without this tombstone the late+ * arm would resurrect a request no steer backs, sealing an unrelated+ * stretch of generation. steerIds are single-use UUIDs, so a tombstoned+ * id can never legitimately be re-armed.+ */+ cleared: Set<string>;+ };+ /** Removes this generation's cross-replica preempt listener without touching a replacement. */+ preemptUnsubscribe?: () => void;readyPromise: Promise<void>;resolveReady: () => void;startupTelemetry?: AgentStartupTelemetry;@@ -462,7 +497,14 @@ class GenerationJobManagerClass {return released;}+ /**+ * Releases this generation's cross-replica listeners — abort AND preempt —+ * and drops any armed preempt requests. Every terminal site that retires a+ * runtime goes through here, so preempt state cannot outlive the+ * generation it belongs to.+ */private releaseAbortSubscription(runtime: RuntimeJobState): void {+ this.releasePreemptSubscription(runtime);const unsubscribe = runtime.abortUnsubscribe;runtime.abortUnsubscribe = undefined;if (!unsubscribe) {@@ -476,6 +518,21 @@ class GenerationJobManagerClass {}}+ private releasePreemptSubscription(runtime: RuntimeJobState): void {+ runtime.preempt = undefined;+ const unsubscribe = runtime.preemptUnsubscribe;+ runtime.preemptUnsubscribe = undefined;+ if (!unsubscribe) {+ return;+ }++ try {+ unsubscribe();+ } catch (err) {+ logger.error('[GenerationJobManager] Failed to release preempt subscription:', err);+ }+ }+private reconcileInactiveGeneration(streamId: string,createdAt: number,@@ -542,6 +599,113 @@ class GenerationJobManagerClass {}}+ private ensurePreemptState(+ runtime: RuntimeJobState,+ createdAt: number,+ ): NonNullable<RuntimeJobState['preempt']> {+ runtime.preempt ??= { createdAt, ids: new Set(), cleared: new Set() };+ return runtime.preempt;+ }++ /**+ * Arms ids that have not already been observed as removed. Returns how many+ * were actually accepted: a tombstoned id (its steer already drained at an+ * ordinary boundary while this request was in flight) or an over-cap id is+ * skipped, and the caller must not report those as armed.+ */+ private armPreemptIds(runtime: RuntimeJobState, createdAt: number, steerIds: string[]): number {+ const state = this.ensurePreemptState(runtime, createdAt);+ let accepted = 0;+ for (const id of steerIds) {+ if (state.cleared.has(id) || state.ids.size >= STEER_QUEUE_MAX_DEPTH) {+ continue;+ }+ state.ids.add(id);+ accepted += 1;+ }+ return accepted;+ }++ /**+ * Disarms ids and tombstones them against a late-arriving arm. The+ * tombstone set is bounded by EVICTING its oldest entry rather than+ * refusing new ones: every drained or cancelled steer is tombstoned, not+ * just preempting ones, so a refusing cap would silently stop recording+ * after a long generation and let the late-arm race resurface. Set+ * iteration is insertion-ordered, so the first key is the oldest.+ */+ private clearPreemptIds(runtime: RuntimeJobState, createdAt: number, steerIds: string[]): void {+ const state = this.ensurePreemptState(runtime, createdAt);+ for (const id of steerIds) {+ state.ids.delete(id);+ if (state.cleared.has(id)) {+ continue;+ }+ if (state.cleared.size >= PREEMPT_TOMBSTONE_MAX) {+ const oldest = state.cleared.values().next().value;+ if (oldest != null) {+ state.cleared.delete(oldest);+ }+ }+ state.cleared.add(id);+ }+ }++ /**+ * Mirrors {@link registerAbortSubscription} for cooperative preempts. The+ * same double fence applies — runtime object identity plus the generation+ * `createdAt` carried by every {@link PreemptMessage} — so a stale+ * cross-replica publish can never arm a replacement job on the same+ * streamId. Arm requests cap at {@link STEER_QUEUE_MAX_DEPTH}, matching+ * the durable queue they mirror.+ *+ * Never rejects. Every caller fires this without awaiting (see the+ * createJob registration for why), so a propagating subscription error+ * would surface as an unhandled rejection: an alarming stack trace in+ * LibreChat's own server, which installs a global handler and keeps+ * serving, and a dead process for any other consumer of this package,+ * which under Node's default `--unhandled-rejections=throw` does not.+ * Neither is warranted — a failed subscription degrades this generation's+ * preemptive steers to the next tool boundary, the documented fallback.+ */+ private async registerPreemptSubscription(+ streamId: string,+ runtime: RuntimeJobState,+ ): Promise<void> {+ if (!this.eventTransport.onPreempt) {+ return;+ }++ try {+ const unsubscribe = await this.eventTransport.onPreempt(streamId, (msg) => {+ const currentRuntime = this.runtimeState.get(streamId);+ if (currentRuntime !== runtime || currentRuntime.createdAt !== msg.createdAt) {+ return;+ }++ if (msg.op === 'clear') {+ this.clearPreemptIds(currentRuntime, msg.createdAt, msg.steerIds);+ return;+ }++ this.armPreemptIds(currentRuntime, msg.createdAt, msg.steerIds);+ });++ if (typeof unsubscribe === 'function') {+ runtime.preemptUnsubscribe = unsubscribe;+ }+ if (this.runtimeState.get(streamId) !== runtime || runtime.abortController.signal.aborted) {+ this.releasePreemptSubscription(runtime);+ }+ } catch (err) {+ logger.error(+ `[GenerationJobManager] Failed to subscribe to preempts for ${streamId}; ` ++ 'steers on this generation will apply at the next tool boundary:',+ err,+ );+ }+ }+private rejectSubscriptionDuringShutdown(subscriptionType: 'initial' | 'resume',onError?: t.ErrorHandler,@@ -760,6 +924,17 @@ class GenerationJobManagerClass {this.registerAllSubscribersLeft(streamId);await this.registerAbortSubscription(streamId, runtime);+ /**+ * NOT awaited, unlike abort. Abort must be deliverable before the job+ * is exposed — a missed abort strands a run. A missed preempt only+ * degrades that steer to the next tool boundary, which is the+ * documented fallback, so blocking job creation on a second channel+ * subscription would trade a real hang risk for a cosmetic guarantee.+ * The registration's own lost-race tail releases it if the runtime is+ * retired before the subscription resolves, and it swallows and logs+ * its own failures, so this detached call cannot reject.+ */+ void this.registerPreemptSubscription(streamId, runtime);if (this.runtimeState.get(streamId) !== runtime) {throw new Error('Generation job was replaced during initialization');}@@ -888,6 +1063,9 @@ class GenerationJobManagerClass {// Surface deferred tools discovered before the pause so the resume route can// replay them into createRun (the rebuilt graph passes `messages: []`).discoveredTools: jobData.discoveredTools,+ // Surface the owning replica's seal capability so the steer route can+ // honour it instead of probing its own (possibly older) SDK.+ preemptCapable: jobData.preemptCapable,// Surface the pending review so status/resume routes built on the// facade can render the prompt for a `requires_action` job.pendingAction: jobData.pendingAction,@@ -982,6 +1160,8 @@ class GenerationJobManagerClass {if (jobData.status === 'running' || jobData.status === 'requires_action') {await this.registerAbortSubscription(streamId, runtime);+ /** Best-effort, non-blocking — see the createJob registration. */+ void this.registerPreemptSubscription(streamId, runtime);}const runtimeAfterAbortRegistration = this.runtimeState.get(streamId);@@ -3064,6 +3244,230 @@ class GenerationJobManagerClass {return this._steering;}+ /**+ * Arms a cooperative-seal request for one queued steer and reports whether+ * the arm actually reached an owner. Never touches job status.+ *+ * Returns true when this replica owns the generation (armed locally), or+ * when the fenced publish was received by at least one subscriber. Returns+ * FALSE when the publish failed or nobody was listening: the steer is still+ * durably queued and will inject at the next tool boundary, but nothing+ * will seal for it, so the caller must not acknowledge it as interrupting.+ * The owner's own `createdAt` fence still decides whether to honour it.+ */+ async requestPreempt(streamId: string, steerId: string, jobCreatedAt: number): Promise<boolean> {+ /**+ * `ownedJobs`, NOT `runtimeState`: a cross-replica `getJob` installs a+ * facade runtime on THIS replica via `getOrCreateRuntimeState`, so a+ * matching `runtime.createdAt` proves only that we have looked at the+ * job — not that we generate it. Arming that facade would satisfy+ * nothing while reporting success.+ */+ const ownedHere = this.ownedJobs.get(streamId) === jobCreatedAt;+ const runtime = this.runtimeState.get(streamId);+ if (ownedHere && runtime != null && runtime.createdAt === jobCreatedAt) {+ /** Zero accepted means the id was tombstoned (its steer drained at an+ * ordinary boundary mid-request) — not an arm, so do not claim one. */+ return this.armPreemptIds(runtime, jobCreatedAt, [steerId]) > 0;+ }+ if (this.eventTransport.emitPreempt == null) {+ /** Single-process transport: the local arm is the whole mechanism. */+ return false;+ }+ try {+ await this.eventTransport.emitPreempt(streamId, {+ op: 'arm',+ createdAt: jobCreatedAt,+ steerIds: [steerId],+ });+ /**+ * Best effort, and deliberately not read as proof. The publisher's+ * subscriber count includes THIS replica's own facade subscription, so+ * it cannot distinguish "the owner heard" from "we heard ourselves";+ * proving owner receipt would need a correlated request/response over+ * pub-sub. See the acknowledgement-semantics note on the PR.+ */+ return true;+ } catch (error) {+ logger.error(`[GenerationJobManager] Failed to publish preempt arm for ${streamId}:`, error);+ return false;+ }+ }++ /** O(1) level-triggered poll consumed by the run's `shouldPreempt`. */+ isPreemptRequested(streamId: string): boolean {+ return (this.runtimeState.get(streamId)?.preempt?.ids.size ?? 0) > 0;+ }++ /**+ * Clears preempt requests for steers that left the durable queue — drained+ * at a boundary (tool or preempt), cancelled, or dropped. The fenced+ * cross-replica `clear` keeps a request from outliving its steer on the+ * generating replica; without a fence identity the publish is skipped and+ * the empty-boundary path's self-clear bounds the damage to one seal.+ */+ noteSteersRemoved(streamId: string, steerIds: string[], jobCreatedAt?: number): Promise<boolean> {+ if (steerIds.length === 0) {+ return Promise.resolve(true);+ }+ const runtime = this.runtimeState.get(streamId);+ if (runtime != null && (jobCreatedAt == null || runtime.createdAt === jobCreatedAt)) {+ this.clearPreemptIds(runtime, jobCreatedAt ?? runtime.createdAt, steerIds);+ }+ const createdAt = jobCreatedAt ?? runtime?.createdAt;+ if (createdAt == null || this.eventTransport.emitPreempt == null) {+ /** Nothing to publish: the local disarm above is the whole mechanism. */+ return Promise.resolve(true);+ }+ /**+ * Awaitable so a CANCEL retries and logs before responding. A dropped+ * clear is worse than a dropped arm: the owner keeps a level-triggered+ * request for a steer that no longer exists, seals its next chunk,+ * drains nothing, and truncates an unrelated answer. Retried once — a+ * transient publish error is the common case and the retry is cheap.+ *+ * The returned boolean means "published without error", NOT "the owner+ * disarmed": the delivery count includes this replica's own facade+ * subscription, so publication cannot prove receipt. It is for logging+ * only and must not be surfaced as a guarantee. Proving receipt needs a+ * correlated request/response over pub-sub.+ *+ * Damage is bounded regardless: if the clear is lost the owner seals+ * once, the empty-boundary self-clear disarms the generation, and the+ * turn is persisted `unfinished: true` rather than silently truncated.+ */+ const publish = (): Promise<unknown> =>+ Promise.resolve(+ this.eventTransport.emitPreempt?.(streamId, { op: 'clear', createdAt, steerIds }),+ );+ return publish().then(+ () => true,+ () =>+ publish().then(+ () => true,+ (error: unknown) => {+ logger.error(+ `[GenerationJobManager] Failed to publish preempt clear for ${streamId} after retry; ` ++ 'the owner may seal once before its empty boundary self-clears:',+ error,+ );+ return false;+ },+ ),+ );+ }++ /**+ * Rebuilds the armed set from the DURABLE queue for a generation whose+ * ownership just moved (HITL resume landing on another replica).+ *+ * An arm lives only in the owning replica's runtime plus a transient+ * pub/sub message, while the steer's `preempt` flag is durable on the queue+ * item. A replica that never observed the original arm therefore starts+ * with an empty set and its poll stays false, so an interrupt the user+ * already had acknowledged would silently wait for an ordinary tool+ * boundary.+ *+ * REPLACES the armed set rather than adding to it. A replica that only ever+ * read this job still installed a facade runtime and subscribed, so it can+ * have accepted an arm and then missed the best-effort clear that followed+ * the drain. Promotion to owner makes that orphan live, and a union would+ * keep it: the first resumed stream seals on a steer no longer in the+ * queue, drains nothing, and truncates the resumed answer as+ * `preempt_incomplete`. The durable queue is the only authority at a+ * handover, so anything absent from it is disarmed and tombstoned here —+ * tombstoned because an arm that outlived its steer must not be able to+ * come back a second time.+ */+ async rearmQueuedPreempts(streamId: string, jobCreatedAt: number): Promise<number> {+ /**+ * The armed set is snapshotted BEFORE the queue is read, and that order is+ * the entire safety argument. `approvals.resolve` has already reopened+ * steering by the time this runs, so a concurrent replica can enqueue a+ * preempt steer and publish its arm while the `peek` is in flight. Reading+ * the queue first would leave that arm present locally but missing from a+ * snapshot taken before it existed, and this method would tombstone a live+ * interrupt the route already acknowledged — unrecoverable, since the+ * tombstone also blocks the re-arm.+ *+ * Taking the arms first makes that impossible without a lock or a second+ * round trip: a steer is durably enqueued BEFORE its arm is published, so+ * any id in this snapshot was already queued when it was armed, and the+ * later `peek` is guaranteed to observe it unless it has since drained —+ * which is precisely the orphan this reconciliation exists to drop.+ */+ const runtime = this.runtimeState.get(streamId);+ const armedBeforeRead =+ runtime?.preempt != null && runtime.createdAt === jobCreatedAt+ ? [...runtime.preempt.ids]+ : [];++ const queued = await this._steering.peek(streamId, jobCreatedAt);+ const backed = new Set(+ queued.filter((item) => item.preempt === true).map((item) => item.steerId),+ );++ /** The generation can be replaced across the read; only disarm the runtime+ * the snapshot was taken from. */+ if (runtime != null && this.runtimeState.get(streamId) === runtime) {+ const orphaned = armedBeforeRead.filter((id) => !backed.has(id));+ if (orphaned.length > 0) {+ logger.warn(+ `[GenerationJobManager] Dropping ${orphaned.length} preempt arm(s) with no queued steer ` ++ `while ownership of ${streamId} moves`,+ );+ this.clearPreemptIds(runtime, jobCreatedAt, orphaned);+ }+ }++ let rearmed = 0;+ for (const steerId of backed) {+ await this.requestPreempt(streamId, steerId, jobCreatedAt);+ rearmed += 1;+ }+ return rearmed;+ }++ /**+ * Snapshot of the ids armed right now, taken BEFORE a drain so the+ * empty-boundary disarm can scope itself to them.+ */+ getArmedPreemptIds(streamId: string, jobCreatedAt?: number): string[] {+ const runtime = this.runtimeState.get(streamId);+ if (runtime?.preempt == null) {+ return [];+ }+ if (jobCreatedAt != null && runtime.createdAt !== jobCreatedAt) {+ return [];+ }+ return [...runtime.preempt.ids];+ }++ /**+ * Disarms the requests a spent boundary was responsible for. Called when a+ * boundary drains nothing: the seal is already spent and those ids refer to+ * steers no longer in the queue, so leaving them armed would seal again on+ * the next chunk and truncate an unrelated answer.+ *+ * Scoped to an explicit id list rather than wiping the set: a second steer+ * can enqueue and arm between the atomic drain returning empty and this+ * call, and that arm is backed by a live queue item that has not been+ * injected yet.+ */+ clearPreemptRequests(streamId: string, steerIds: string[], jobCreatedAt?: number): void {+ if (steerIds.length === 0) {+ return;+ }+ const runtime = this.runtimeState.get(streamId);+ if (runtime?.preempt == null) {+ return;+ }+ if (jobCreatedAt != null && runtime.createdAt !== jobCreatedAt) {+ return;+ }+ this.clearPreemptIds(runtime, runtime.createdAt, steerIds);+ }+/*** Get resume state for reconnecting clients.*/diff --git a/packages/api/src/stream/SteeringLifecycle.ts b/packages/api/src/stream/SteeringLifecycle.tsindex bc3a500239..f8cf62c717 100644--- a/packages/api/src/stream/SteeringLifecycle.ts+++ b/packages/api/src/stream/SteeringLifecycle.ts@@ -11,6 +11,7 @@ export function toPendingSteer(item: SteerQueueItem): TPendingSteer {text: item.text,createdAt: item.createdAt,...(item.files && item.files.length > 0 && { files: item.files }),+ ...(item.preempt === true && { preempt: true }),};}@@ -101,8 +102,12 @@ export class SteeringLifecycle {* depth, or a rejection code ({@link STEER_ENQUEUE_NOT_RUNNING} /* {@link STEER_ENQUEUE_QUEUE_FULL}).*/- async enqueue(streamId: string, item: SteerQueueItem): Promise<number> {- const depth = await this.store.enqueueSteer(streamId, item);+ async enqueue(+ streamId: string,+ item: SteerQueueItem,+ expectedCreatedAt?: number,+ ): Promise<number> {+ const depth = await this.store.enqueueSteer(streamId, item, expectedCreatedAt);if (depth > 0) {logger.debug(`[SteeringLifecycle] queued steer: ${streamId} steer=${item.steerId} depth=${depth}`,diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.tsindex 91f3d656d1..c8e46cf094 100644--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts@@ -9,6 +9,7 @@ import {import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';+import { STEER_ENQUEUE_NOT_RUNNING } from '~/stream/interfaces/IJobStore';import { GenerationJobManagerClass } from '~/stream/GenerationJobManager';import { RedisJobStore } from '~/stream/implementations/RedisJobStore';import { createStreamServices } from '~/stream/createStreamServices';@@ -2416,4 +2417,217 @@ describe('GenerationJobManager Integration Tests', () => {expect(services.isRedis).toBe(false);});});++ /**+ * The production topology for interrupt & steer: the steer POST is routed to+ * whichever replica the load balancer picks, which is usually NOT the replica+ * running the generation. Everything else in the preempt suite runs against a+ * single manager, so the hop that actually carries the request — non-owner+ * publishes, owner arms, owner's SDK poll flips — has no coverage.+ *+ * Two `GenerationJobManagerClass` instances are a faithful pair of replicas+ * here: `runtimeState` and `ownedJobs` are private instance fields, there is+ * no module-level mutable state between them, and `createStreamServices`+ * duplicates a dedicated subscriber connection per call. Separate OS+ * processes would exercise the same objects over the same Redis.+ */+ describeRedis('Cross-Replica Preempt (Redis, two manager instances)', () => {+ const replicas: GenerationJobManagerClass[] = [];++ function createReplica(): GenerationJobManagerClass {+ const manager = new GenerationJobManagerClass();+ manager.configure(createStreamServices({ useRedis: true, redisClient: ioredisClient! }));+ manager.initialize();+ replicas.push(manager);+ return manager;+ }++ afterEach(async () => {+ /** `destroy()`, not `eventTransport.destroy()`: it also clears the+ * manager's cleanup interval and disposes the job store and its timer.+ * Dropping only the transport leaves each manager alive inside its own+ * interval closure, still doing cleanup work against a dead transport. */+ for (const manager of replicas.splice(0)) {+ await manager.destroy().catch(() => {});+ }+ });++ /**+ * Redis pub/sub never replays and the owner's `SUBSCRIBE` is fired+ * detached, so a one-shot publish can be dropped simply for arriving+ * before anyone is listening — a fixed sleep only decides how often that+ * happens on a loaded worker. Republishing until the owner's state+ * converges is safe because both operations are idempotent set writes+ * keyed by steerId, and it removes the timing assumption entirely.+ */+ async function publishUntil(+ publish: () => Promise<unknown>,+ settled: () => boolean,+ what: string,+ timeoutMs = 15000,+ ): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ await publish();+ for (let attempt = 0; attempt < 10; attempt++) {+ if (settled()) {+ return;+ }+ await new Promise((resolve) => setTimeout(resolve, 25));+ }+ }+ throw new Error(`Timed out waiting for ${what}`);+ }++ test('a steer routed to a non-owning replica arms the owner and flips its poll', async () => {+ const owner = createReplica();+ const router = createReplica();+ const streamId = `${testPrefix}-preempt-xreplica-${Date.now()}`;++ const job = await owner.createJob(streamId, 'user-1', undefined, {+ initialMetadata: { preemptCapable: true },+ });++ /**+ * The routing replica reads the job, which installs a FACADE runtime+ * entry on it. That facade must not make it look like an owner.+ */+ const seenByRouter = await router.getJob(streamId);+ expect(seenByRouter?.createdAt).toBe(job.createdAt);+ expect(router.isPreemptRequested(streamId)).toBe(false);++ /**+ * Guards the round-trip that shipped broken once: `preemptCapable` was+ * serialized but never deserialized, so every Redis deployment read it+ * back as undefined and the feature was a silent no-op. A same-replica+ * read cannot catch that — this one crosses Redis.+ */+ expect(seenByRouter?.metadata?.preemptCapable).toBe(true);++ /** The whole point: the owner's level-triggered poll flips from a+ * request that was accepted on a different replica. */+ await publishUntil(+ () => router.requestPreempt(streamId, 'steer-x', job.createdAt),+ () => owner.isPreemptRequested(streamId),+ 'the owner to arm',+ );+ expect(owner.getArmedPreemptIds(streamId)).toContain('steer-x');++ await publishUntil(+ () => router.noteSteersRemoved(streamId, ['steer-x'], job.createdAt),+ () => !owner.isPreemptRequested(streamId),+ 'the owner to disarm',+ );+ expect(owner.getArmedPreemptIds(streamId)).not.toContain('steer-x');+ }, 40000);++ /**+ * The in-memory store fences this in TypeScript; Redis fences it inside+ * STEER_ENQUEUE_LUA, which only a real server can execute. That Lua path+ * is where the `preemptCapable` P1 hid, so it gets its own coverage.+ */+ test('the enqueue Lua refuses an item fenced to a replaced generation', async () => {+ const owner = createReplica();+ const streamId = `${testPrefix}-enqueue-fenced-${Date.now()}`;++ const first = await owner.createJob(streamId, 'user-1');+ const replacement = await owner.createJob(streamId, 'user-1');+ expect(replacement.createdAt).not.toBe(first.createdAt);++ const stale = await owner.steering.enqueue(+ streamId,+ {+ steerId: 'steer-stale-epoch',+ text: 'belongs to the previous run',+ userId: 'user-1',+ createdAt: Date.now(),+ },+ first.createdAt,+ );+ expect(stale).toBe(STEER_ENQUEUE_NOT_RUNNING);+ expect(await owner.steering.peek(streamId)).toEqual([]);++ const live = await owner.steering.enqueue(+ streamId,+ {+ steerId: 'steer-live-epoch',+ text: 'belongs to the live run',+ userId: 'user-1',+ createdAt: Date.now(),+ },+ replacement.createdAt,+ );+ expect(live).toBe(1);+ }, 30000);++ /**+ * Redis parks leftover steers inside its terminal-transition Lua, which+ * projects each item field by field — so a field added to SteerQueueItem+ * is silently dropped there unless the projection is updated too. A steer+ * recovered from `/chat/status` would come back without its interrupting+ * label even though every non-Redis path preserves it.+ */+ test('the terminal-transition Lua keeps preempt on parked steers', async () => {+ const owner = createReplica();+ const streamId = `${testPrefix}-park-preempt-${Date.now()}`;++ const job = await owner.createJob(streamId, 'user-1');+ await owner.steering.enqueue(+ streamId,+ {+ steerId: 'steer-parked-preempt',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ },+ job.createdAt,+ );++ const store = new RedisJobStore(ioredisClient!);+ const moved = await store.transitionStatus(streamId, { from: 'running', to: 'complete' });+ expect(moved).toBe(true);++ const claimed = await store.claimParkedSteers(streamId, 'user-1');+ expect(claimed).toBeDefined();+ const parked = JSON.parse(claimed as string) as {+ steers: Array<{ steerId: string; preempt?: boolean }>;+ };+ expect(parked.steers).toHaveLength(1);+ expect(parked.steers[0].steerId).toBe('steer-parked-preempt');+ expect(parked.steers[0].preempt).toBe(true);+ }, 30000);++ test('a stale generation id from another replica cannot arm the live job', async () => {+ const owner = createReplica();+ const router = createReplica();+ const streamId = `${testPrefix}-preempt-xreplica-stale-${Date.now()}`;++ const job = await owner.createJob(streamId, 'user-1');++ /**+ * A negative assertion cannot be established by waiting: an undelivered+ * stale arm and a fenced one look identical. So bracket it between two+ * control arms on the same channel. The first proves the owner is+ * listening BEFORE the stale one is published, and the second — sent+ * after it, over the same publisher connection, so Redis orders it+ * behind — proves the stale arm has already had its chance to land.+ */+ await publishUntil(+ () => router.requestPreempt(streamId, 'control-before', job.createdAt),+ () => owner.getArmedPreemptIds(streamId).includes('control-before'),+ 'the first control arm',+ );++ await router.requestPreempt(streamId, 'steer-stale', job.createdAt - 1);++ await publishUntil(+ () => router.requestPreempt(streamId, 'control-after', job.createdAt),+ () => owner.getArmedPreemptIds(streamId).includes('control-after'),+ 'the second control arm',+ );++ expect(owner.getArmedPreemptIds(streamId)).not.toContain('steer-stale');+ }, 40000);+ });});diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.spec.tsindex e02b3e0249..e1c43cf097 100644--- a/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts+++ b/packages/api/src/stream/__tests__/RedisEventTransport.spec.ts@@ -804,7 +804,10 @@ describe('RedisEventTransport', () => {mockPublisher as unknown as Redis,mockSubscriber as unknown as Redis,);+ /** Both manager-level channel registrations are suppressed so the gated+ * `subscribe` below belongs to the SSE subscription under test. */Object.defineProperty(transport, 'onAbort', { value: undefined });+ Object.defineProperty(transport, 'onPreempt', { value: undefined });const manager = new GenerationJobManagerClass();manager.configure({jobStore: new InMemoryJobStore({ ttlAfterComplete: 60_000 }),diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.tsindex 2e42a3f668..94c68ffe43 100644--- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts@@ -1626,4 +1626,102 @@ describe('RedisEventTransport Integration Tests', () => {subscriber.disconnect();});});++ describe('Cross-Replica Preempt Delivery', () => {+ test('delivers a fenced arm/clear payload across instances', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber1 = (ioredisClient as Redis).duplicate();+ const subscriber2 = (ioredisClient as Redis).duplicate();+ const transport1 = new RedisEventTransport(ioredisClient, subscriber1);+ const transport2 = new RedisEventTransport(ioredisClient, subscriber2);+ const streamId = `preempt-cross-${Date.now()}`;+ const received: Array<{ op: string; createdAt: number; steerIds: string[] }> = [];++ try {+ await transport1.onPreempt(streamId, (msg) => {+ received.push(msg);+ });++ transport2.emitPreempt(streamId, {+ op: 'arm',+ createdAt: 1234,+ steerIds: ['steer-a', 'steer-b'],+ });+ transport2.emitPreempt(streamId, { op: 'clear', createdAt: 1234, steerIds: ['steer-a'] });++ await new Promise((resolve) => setTimeout(resolve, 300));++ /**+ * Payload fidelity and delivery, NOT ordering: two publishes from the+ * same replica carry no cross-message ordering guarantee to a+ * subscriber, which is exactly why the receiving side tombstones+ * cleared ids rather than assuming arm-before-clear. Asserting order+ * here would test something stronger than the protocol promises.+ */+ expect(received).toHaveLength(2);+ expect(received).toContainEqual({+ op: 'arm',+ createdAt: 1234,+ steerIds: ['steer-a', 'steer-b'],+ });+ expect(received).toContainEqual({+ op: 'clear',+ createdAt: 1234,+ steerIds: ['steer-a'],+ });+ } finally {+ transport1.cleanup(streamId);+ transport1.destroy();+ transport2.destroy();+ subscriber1.disconnect();+ subscriber2.disconnect();+ }+ });++ test('unsubscribe removes only this registration and keeps the channel for a replacement', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber1 = (ioredisClient as Redis).duplicate();+ const subscriber2 = (ioredisClient as Redis).duplicate();+ const transport1 = new RedisEventTransport(ioredisClient, subscriber1);+ const transport2 = new RedisEventTransport(ioredisClient, subscriber2);+ const streamId = `preempt-unsub-${Date.now()}`;+ const oldGeneration: string[] = [];+ const newGeneration: string[] = [];++ try {+ const unsubscribeOld = await transport1.onPreempt(streamId, (msg) => {+ oldGeneration.push(...msg.steerIds);+ });+ await transport1.onPreempt(streamId, (msg) => {+ newGeneration.push(...msg.steerIds);+ });++ unsubscribeOld();+ transport2.emitPreempt(streamId, { op: 'arm', createdAt: 99, steerIds: ['steer-live'] });++ await new Promise((resolve) => setTimeout(resolve, 300));++ expect(oldGeneration).toEqual([]);+ expect(newGeneration).toEqual(['steer-live']);+ } finally {+ transport1.cleanup(streamId);+ transport1.destroy();+ transport2.destroy();+ subscriber1.disconnect();+ subscriber2.disconnect();+ }+ });+ });});diff --git a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.spec.tsindex 838fa49c23..cbacf0b204 100644--- a/packages/api/src/stream/__tests__/RedisJobStore.spec.ts+++ b/packages/api/src/stream/__tests__/RedisJobStore.spec.ts@@ -236,8 +236,17 @@ describe('RedisJobStore', () => {isTemporary: false,promptTokens: 0,discoveredTools: [],+ preemptCapable: true,});+ /**+ * `serializeJob` writes every boolean generically but `deserializeJob` is+ * an explicit field map, so a new flag silently vanishes on read unless+ * it is added there too. For `preemptCapable` that meant interrupt-steer+ * degrading to ordinary steering in every Redis deployment.+ */+ expect(job.preemptCapable).toBe(true);+expect(job).toMatchObject({streamId: 'stream-metadata',userId: 'user-1',diff --git a/packages/api/src/stream/__tests__/steering.spec.ts b/packages/api/src/stream/__tests__/steering.spec.tsindex 3f39de295f..32edb26524 100644--- a/packages/api/src/stream/__tests__/steering.spec.ts+++ b/packages/api/src/stream/__tests__/steering.spec.ts@@ -1,6 +1,7 @@+import { logger } from '@librechat/data-schemas';import { SteerEvents } from 'librechat-data-provider';import type { TPendingSteer, Agents } from 'librechat-data-provider';-import type { SteerQueueItem } from '~/stream/interfaces/IJobStore';+import type { SteerQueueItem, IEventTransport } from '~/stream/interfaces/IJobStore';import type { ResumeState, ServerSentEvent } from '~/types';import {STEER_ENQUEUE_NOT_RUNNING,@@ -55,6 +56,30 @@ describe('SteeringLifecycle via GenerationJobManager.steering (in-memory)', () =expect(await manager.steering.enqueue(streamId, buildSteer('two'))).toBe(2);});+ /**+ * The steer route decides capability against a job it read several awaits+ * earlier. If the run is replaced in between, an unfenced enqueue puts the+ * item on the REPLACEMENT queue while the preempt flag and the arm still+ * name the previous epoch — so the arm is fenced out at the owner and the+ * 202 promises an interrupt that cannot happen. Rejecting is the honest+ * outcome: the run the caller was told about is gone.+ */+ test('refuses an item fenced to a generation that has been replaced', async () => {+ const streamId = 'steer-enqueue-fenced';+ const first = await manager.createJob(streamId, 'user-1');+ const replacement = await manager.createJob(streamId, 'user-1');+ expect(replacement.createdAt).not.toBe(first.createdAt);++ expect(await manager.steering.enqueue(streamId, buildSteer('stale'), first.createdAt)).toBe(+ STEER_ENQUEUE_NOT_RUNNING,+ );+ expect(await manager.steering.peek(streamId)).toEqual([]);++ expect(+ await manager.steering.enqueue(streamId, buildSteer('live'), replacement.createdAt),+ ).toBe(1);+ });+test('rejects when the job does not exist', async () => {expect(await manager.steering.enqueue('nonexistent', buildSteer('x'))).toBe(STEER_ENQUEUE_NOT_RUNNING,@@ -839,3 +864,460 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {}});});++describe('preempt request lifecycle (in-memory)', () => {+ let manager: GenerationJobManagerClass;++ beforeEach(() => {+ manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ manager.initialize();+ });++ afterEach(async () => {+ await manager.destroy();+ });++ test('requestPreempt without a runtime is a no-op and the poll stays false', () => {+ manager.requestPreempt('preempt-no-runtime', 'steer-1', Date.now());+ expect(manager.isPreemptRequested('preempt-no-runtime')).toBe(false);+ });++ test('arms against the live generation and stays armed until cleared', async () => {+ const streamId = 'preempt-arm';+ const job = await manager.createJob(streamId, 'user-1');++ manager.requestPreempt(streamId, 'steer-1', job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(true);+ expect(manager.isPreemptRequested(streamId)).toBe(true);++ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('refuses to arm when jobCreatedAt does not match the live generation', async () => {+ const streamId = 'preempt-stale-arm';+ const job = await manager.createJob(streamId, 'user-1');++ manager.requestPreempt(streamId, 'steer-1', job.createdAt - 1);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('clears only the removed steerIds', async () => {+ const streamId = 'preempt-partial-clear';+ const job = await manager.createJob(streamId, 'user-1');++ manager.requestPreempt(streamId, 'steer-1', job.createdAt);+ manager.requestPreempt(streamId, 'steer-2', job.createdAt);+ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(true);+ manager.noteSteersRemoved(streamId, ['steer-2'], job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('caps armed requests at STEER_QUEUE_MAX_DEPTH', async () => {+ const streamId = 'preempt-cap';+ const job = await manager.createJob(streamId, 'user-1');++ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH + 5; i++) {+ manager.requestPreempt(streamId, `steer-${i}`, job.createdAt);+ }+ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH; i++) {+ manager.noteSteersRemoved(streamId, [`steer-${i}`], job.createdAt);+ }+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('arming never changes job status', async () => {+ const streamId = 'preempt-status';+ const job = await manager.createJob(streamId, 'user-1');++ manager.requestPreempt(streamId, 'steer-1', job.createdAt);+ expect((await manager.getJob(streamId))?.status).toBe('running');+ });++ test('a failing preempt subscription degrades the job instead of crashing the process', async () => {+ const transport: IEventTransport = new InMemoryEventTransport();+ transport.onPreempt = jest.fn().mockRejectedValue(new Error('SUBSCRIBE failed'));++ /**+ * The registration is deliberately detached, so a rejection propagating out+ * of it would be unhandled — fatal under Node's default settings.+ */+ const unhandled: unknown[] = [];+ const collect = (reason: unknown): number => unhandled.push(reason);+ process.on('unhandledRejection', collect);+ const logged = jest.spyOn(logger, 'error').mockImplementation(() => logger);++ const degraded = new GenerationJobManagerClass();+ degraded.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: transport,+ isRedis: false,+ cleanupOnComplete: false,+ });+ degraded.initialize();++ try {+ const streamId = 'preempt-subscribe-fails';+ const job = await degraded.createJob(streamId, 'user-1');+ await new Promise((resolve) => setImmediate(resolve));++ expect(unhandled).toEqual([]);+ expect(job.status).toBe('running');+ expect(logged).toHaveBeenCalledWith(+ expect.stringContaining('Failed to subscribe to preempts'),+ expect.any(Error),+ );++ /** Same-replica arming is runtime state, so it survives the lost channel. */+ degraded.requestPreempt(streamId, 'steer-1', job.createdAt);+ expect(degraded.isPreemptRequested(streamId)).toBe(true);+ } finally {+ process.off('unhandledRejection', collect);+ logged.mockRestore();+ await degraded.destroy();+ }+ });++ test('abortJob retires the armed set with the runtime', async () => {+ const streamId = 'preempt-abort';+ const job = await manager.createJob(streamId, 'user-1');+ await manager.steering.enqueue(streamId, {+ steerId: 'steer-1',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ });+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);++ await manager.abortJob(streamId);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('completeJob retires the armed set', async () => {+ const streamId = 'preempt-complete';+ const job = await manager.createJob(streamId, 'user-1');+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);++ await manager.completeJob(streamId, undefined, job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('toPendingSteer keeps the preempt label for parked/replayed chips', () => {+ const item: SteerQueueItem = {+ steerId: 'steer-1',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ };+ expect(toPendingSteer(item).preempt).toBe(true);+ expect(toPendingSteer({ ...item, preempt: undefined }).preempt).toBeUndefined();+ });++ test('a late arm cannot resurrect an already-cleared steer', async () => {+ const streamId = 'preempt-late-arm';+ const job = await manager.createJob(streamId, 'user-1');++ /** The generating replica drained + cleared before this replica's arm. */+ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);++ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('the tombstone is per-steer, not a blanket disarm', async () => {+ const streamId = 'preempt-tombstone-scope';+ const job = await manager.createJob(streamId, 'user-1');++ manager.noteSteersRemoved(streamId, ['steer-1'], job.createdAt);+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);+ manager.requestPreempt(streamId, 'steer-2', job.createdAt);++ expect(manager.isPreemptRequested(streamId)).toBe(true);+ manager.noteSteersRemoved(streamId, ['steer-2'], job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('clearPreemptRequests disarms the ids it is given', async () => {+ const streamId = 'preempt-clear-all';+ const job = await manager.createJob(streamId, 'user-1');+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);+ manager.requestPreempt(streamId, 'steer-2', job.createdAt);++ const armed = manager.getArmedPreemptIds(streamId, job.createdAt);+ expect(armed.sort()).toEqual(['steer-1', 'steer-2']);+ manager.clearPreemptRequests(streamId, armed, job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * A steer can enqueue and arm while a drain is in flight. The empty-boundary+ * disarm must not wipe it — its queue item is live and uninjected.+ */+ test('clearPreemptRequests spares an arm that landed after the snapshot', async () => {+ const streamId = 'preempt-clear-scoped';+ const job = await manager.createJob(streamId, 'user-1');+ manager.requestPreempt(streamId, 'steer-old', job.createdAt);++ const snapshot = manager.getArmedPreemptIds(streamId, job.createdAt);+ manager.requestPreempt(streamId, 'steer-new', job.createdAt);+ manager.clearPreemptRequests(streamId, snapshot, job.createdAt);++ expect(manager.isPreemptRequested(streamId)).toBe(true);+ });++ test('clearPreemptRequests refuses a stale generation', async () => {+ const streamId = 'preempt-clear-stale';+ const job = await manager.createJob(streamId, 'user-1');+ manager.requestPreempt(streamId, 'steer-1', job.createdAt);++ manager.clearPreemptRequests(streamId, ['steer-1'], job.createdAt - 1);+ expect(manager.isPreemptRequested(streamId)).toBe(true);+ });++ /**+ * Every drained or cancelled steer is tombstoned, not just preempting ones,+ * so a refusing cap would stop recording after a long generation and let the+ * late-arm race resurface. Eviction keeps the newest removals authoritative.+ */+ test('tombstones evict oldest-first instead of refusing new removals', async () => {+ const streamId = 'preempt-tombstone-evict';+ const job = await manager.createJob(streamId, 'user-1');++ for (let i = 0; i < STEER_QUEUE_MAX_DEPTH * 2 + 5; i++) {+ manager.noteSteersRemoved(streamId, [`bulk-${i}`], job.createdAt);+ }+ /** The most recent removal must still be remembered. */+ const recent = `bulk-${STEER_QUEUE_MAX_DEPTH * 2 + 4}`;+ manager.requestPreempt(streamId, recent, job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * An arm lives only in the owning replica's runtime; the steer's `preempt`+ * flag is durable. A resume landing on another replica must rebuild the+ * armed set from the queue or the acknowledged interrupt silently waits for+ * an ordinary tool boundary.+ */+ test('rearmQueuedPreempts rebuilds the armed set from the durable queue', async () => {+ const streamId = 'preempt-rearm';+ const job = await manager.createJob(streamId, 'user-1');+ await manager.steering.enqueue(streamId, {+ steerId: 'steer-preempt',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ });+ await manager.steering.enqueue(streamId, {+ steerId: 'steer-plain',+ text: 'ordinary steer',+ userId: 'user-1',+ createdAt: Date.now(),+ });++ /** Fresh owner: nothing armed locally yet. */+ expect(manager.isPreemptRequested(streamId)).toBe(false);++ const rearmed = await manager.rearmQueuedPreempts(streamId, job.createdAt);++ expect(rearmed).toBe(1);+ expect(manager.isPreemptRequested(streamId)).toBe(true);+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toEqual(['steer-preempt']);+ });++ /**+ * A replica that only READ this job still installed a facade runtime and+ * subscribed, so it can hold an arm whose clear it later missed. Promotion+ * to owner on HITL resume would make that orphan live: the first resumed+ * stream seals on a steer that is no longer queued, drains nothing, and+ * truncates the resumed answer. The durable queue is the authority at a+ * handover, so an arm it does not back must be dropped, not merged.+ */+ test('rearmQueuedPreempts drops an armed id the durable queue no longer backs', async () => {+ const streamId = 'preempt-rearm-orphan';+ const job = await manager.createJob(streamId, 'user-1');++ /** Arm survived from before ownership moved; its steer is long drained. */+ manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);+ await manager.steering.enqueue(streamId, {+ steerId: 'steer-live',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ });+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toContain('steer-orphan');++ const rearmed = await manager.rearmQueuedPreempts(streamId, job.createdAt);++ expect(rearmed).toBe(1);+ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toEqual(['steer-live']);+ });++ /**+ * `approvals.resolve` reopens steering before reconciliation runs, so a+ * steer enqueued on another replica can arm this one WHILE the queue read+ * is in flight. Reading the queue first would see that arm as unbacked and+ * tombstone a live interrupt the route already acknowledged — and the+ * tombstone would block the re-arm, so it could never recover.+ */+ test('an arm that lands while the queue is being read is not tombstoned', async () => {+ const streamId = 'preempt-rearm-race';+ const job = await manager.createJob(streamId, 'user-1');++ const peek = manager.steering.peek.bind(manager.steering);+ const spy = jest+ .spyOn(manager.steering, 'peek')+ .mockImplementation(async (id: string, expectedCreatedAt?: number) => {+ const snapshot = await peek(id, expectedCreatedAt);+ /** Another replica commits a steer and publishes its arm, both after+ * this snapshot was taken. */+ await manager.steering.enqueue(streamId, {+ steerId: 'steer-inflight',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ });+ manager.requestPreempt(streamId, 'steer-inflight', job.createdAt);+ return snapshot;+ });++ try {+ await manager.rearmQueuedPreempts(streamId, job.createdAt);+ } finally {+ spy.mockRestore();+ }++ expect(manager.getArmedPreemptIds(streamId, job.createdAt)).toContain('steer-inflight');+ expect(manager.isPreemptRequested(streamId)).toBe(true);+ });++ test('an orphan dropped at handover is tombstoned against a late arm', async () => {+ const streamId = 'preempt-rearm-orphan-tombstone';+ const job = await manager.createJob(streamId, 'user-1');++ manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);+ await manager.rearmQueuedPreempts(streamId, job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);++ /** A publish that was in flight while ownership moved must not revive it. */+ manager.requestPreempt(streamId, 'steer-orphan', job.createdAt);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('rearmQueuedPreempts refuses a stale generation and arms nothing', async () => {+ const streamId = 'preempt-rearm-stale';+ const job = await manager.createJob(streamId, 'user-1');+ await manager.steering.enqueue(streamId, {+ steerId: 'steer-preempt',+ text: 'interrupt me',+ userId: 'user-1',+ createdAt: Date.now(),+ preempt: true,+ });++ expect(await manager.rearmQueuedPreempts(streamId, job.createdAt - 1)).toBe(0);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ /**+ * A non-owning replica can only publish. The subscriber count is NOT proof+ * of owner receipt — this replica's own facade subscription is counted too+ * — so a successful publish is reported as armed and only a rejected one+ * is reported as unarmed. See the acknowledgement-semantics note on #14518.+ */+ test('a successful cross-replica publish reports armed', async () => {+ const streamId = 'preempt-published';+ const published = new GenerationJobManagerClass();+ published.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: Object.assign(new InMemoryEventTransport(), {+ emitPreempt: async () => 0,+ }),+ isRedis: false,+ cleanupOnComplete: false,+ });+ published.initialize();+ try {+ expect(await published.requestPreempt(streamId, 'steer-1', Date.now())).toBe(true);+ } finally {+ await published.destroy();+ }+ });++ /**+ * A facade runtime exists on any replica that merely READ the job, so+ * ownership must come from `ownedJobs` — arming a facade satisfies nothing+ * while reporting success.+ */+ test('a facade runtime on a non-owning replica does not count as armed locally', async () => {+ const streamId = 'preempt-facade';+ const facade = new GenerationJobManagerClass();+ facade.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ facade.initialize();+ try {+ /** No transport publish and no ownership: nothing can arm. */+ expect(await facade.requestPreempt(streamId, 'steer-1', Date.now())).toBe(false);+ expect(facade.isPreemptRequested(streamId)).toBe(false);+ } finally {+ await facade.destroy();+ }+ });++ /**+ * The steer drained at an ordinary boundary while the request was still in+ * flight, so its id is tombstoned and no arm is accepted — the 202 must not+ * claim an interrupt that cannot happen.+ */+ test('reports not-armed when the id was already tombstoned', async () => {+ const streamId = 'preempt-tombstoned-arm';+ const job = await manager.createJob(streamId, 'user-1');+ await manager.noteSteersRemoved(streamId, ['steer-late'], job.createdAt);++ expect(await manager.requestPreempt(streamId, 'steer-late', job.createdAt)).toBe(false);+ expect(manager.isPreemptRequested(streamId)).toBe(false);+ });++ test('reports armed when this replica owns the generation', async () => {+ const streamId = 'preempt-owned-here';+ const job = await manager.createJob(streamId, 'user-1');+ expect(await manager.requestPreempt(streamId, 'steer-1', job.createdAt)).toBe(true);+ expect(manager.isPreemptRequested(streamId)).toBe(true);+ });++ test('a failed publish does not throw out of requestPreempt', async () => {+ const streamId = 'preempt-publish-throws';+ const failing = new GenerationJobManagerClass();+ failing.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: Object.assign(new InMemoryEventTransport(), {+ emitPreempt: async () => {+ throw new Error('redis down');+ },+ }),+ isRedis: false,+ cleanupOnComplete: false,+ });+ failing.initialize();+ try {+ expect(await failing.requestPreempt(streamId, 'steer-1', Date.now())).toBe(false);+ } finally {+ await failing.destroy();+ }+ });+});diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.tsindex 9d3d55a7e6..fb1865dabb 100644--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts@@ -591,11 +591,18 @@ export class InMemoryJobStore implements IJobStore {// Single-threaded event loop makes each read-check-write indivisible, so// the status guard and depth cap are exact without extra locking.- async enqueueSteer(streamId: string, item: SteerQueueItem): Promise<number> {+ async enqueueSteer(+ streamId: string,+ item: SteerQueueItem,+ expectedCreatedAt?: number,+ ): Promise<number> {const job = this.jobs.get(streamId);if (!job || job.status !== 'running' || this.closedSteerQueues.has(streamId)) {return STEER_ENQUEUE_NOT_RUNNING;}+ if (expectedCreatedAt != null && job.createdAt !== expectedCreatedAt) {+ return STEER_ENQUEUE_NOT_RUNNING;+ }let queue = this.steerQueues.get(streamId);if (!queue) {queue = [];diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.tsindex 1cf32f28d1..bce0b9bedc 100644--- a/packages/api/src/stream/implementations/RedisEventTransport.ts+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts@@ -1,6 +1,6 @@import { logger } from '@librechat/data-schemas';import type { Redis, Cluster } from 'ioredis';-import type { IEventTransport } from '~/stream/interfaces/IJobStore';+import type { IEventTransport, PreemptMessage } from '~/stream/interfaces/IJobStore';import { registerChunkPublicationCapability } from '~/stream/internal/chunkPublication';import { instrumentIORedisClient, RedisUseCases } from '~/cache/redisTelemetry';@@ -32,6 +32,7 @@ const EventTypes = {DONE: 'done',ERROR: 'error',ABORT: 'abort',+ PREEMPT: 'preempt',} as const;interface PubSubMessage {@@ -42,6 +43,8 @@ interface PubSubMessage {error?: string;/** Immutable identity of the generation that emitted the event. */generationId?: number;+ /** Payload for PREEMPT messages; fenced by its own createdAt. */+ preempt?: PreemptMessage;}/**@@ -63,6 +66,10 @@ interface AbortRegistration {callback: (generationId?: number) => void;}+interface PreemptRegistration {+ callback: (msg: PreemptMessage) => void;+}+/*** Allocate a sequence number and publish the event in a single round trip.*@@ -147,6 +154,7 @@ interface StreamSubscribers {allSubscribersLeftCallback?: () => void;/** Abort callbacks - called when abort signal is received from any replica */abortCallbacks: Set<AbortRegistration>;+ preemptCallbacks: Set<PreemptRegistration>;/** Reorder buffer for handling out-of-order delivery in Redis Cluster */reorderBuffer: ReorderBuffer;}@@ -600,6 +608,8 @@ export class RedisEventTransport implements IEventTransport {break;case EventTypes.ABORT:break;+ case EventTypes.PREEMPT:+ break;}}@@ -616,6 +626,16 @@ export class RedisEventTransport implements IEventTransport {}}}++ if (message.type === EventTypes.PREEMPT && message.preempt != null) {+ for (const registration of streamState.preemptCallbacks) {+ try {+ registration.callback(message.preempt);+ } catch (err) {+ logger.error(`[RedisEventTransport] Error in preempt callback:`, err);+ }+ }+ }}private detachStreamSubscribers(streamId: string, state: StreamSubscribers): void {@@ -631,7 +651,12 @@ export class RedisEventTransport implements IEventTransport {}private unsubscribeUnusedChannel(streamId: string, state: StreamSubscribers): void {- if (this.streams.get(streamId) !== state || state.count > 0 || state.abortCallbacks.size > 0) {+ if (+ this.streams.get(streamId) !== state ||+ state.count > 0 ||+ state.abortCallbacks.size > 0 ||+ state.preemptCallbacks.size > 0+ ) {return;}@@ -673,6 +698,7 @@ export class RedisEventTransport implements IEventTransport {count: 0,handlers: new Map(),abortCallbacks: new Set(),+ preemptCallbacks: new Set(),reorderBuffer: {nextSeq: 0,pending: new Map(),@@ -848,6 +874,7 @@ export class RedisEventTransport implements IEventTransport {handlers: new Map(),allSubscribersLeftCallback: callback,abortCallbacks: new Set(),+ preemptCallbacks: new Set(),reorderBuffer: {nextSeq: 0,pending: new Map(),@@ -892,6 +919,7 @@ export class RedisEventTransport implements IEventTransport {count: 0,handlers: new Map(),abortCallbacks: new Set(),+ preemptCallbacks: new Set(),reorderBuffer: {nextSeq: 0,pending: new Map(),@@ -921,6 +949,76 @@ export class RedisEventTransport implements IEventTransport {};}+ /**+ * Publish a preempt arm/clear to all replicas. Unlike abort this does NOT+ * stop the run — the generating replica seals its current model stream at+ * the next provider-safe boundary. Same channel and subscription as every+ * other stream event; fenced by `msg.createdAt` on the receiving side.+ */+ /**+ * Resolves to the number of replicas that received the message, so an ARM+ * can be acknowledged only once it actually reached someone. Unlike abort+ * (fire-and-forget, because a failed abort is retried by the user hitting+ * stop again) an unheard arm is invisible: the route would answer+ * `preempt: true` for a seal that never happens. Rejects on publish+ * failure; callers decide what to do.+ */+ async emitPreempt(streamId: string, msg: PreemptMessage): Promise<number> {+ const channel = CHANNELS.events(streamId);+ const message: PubSubMessage = {+ type: EventTypes.PREEMPT,+ preempt: msg,+ };++ return this.publisher.publish(channel, JSON.stringify(message));+ }++ /**+ * Register callback for preempt signals from any replica.+ * Resolves once the Redis channel is active so callers can safely arm.+ * The returned function removes only this registration, so a terminal+ * generation releases its channel without touching a same-stream+ * replacement.+ */+ async onPreempt(streamId: string, callback: (msg: PreemptMessage) => void): Promise<() => void> {+ const channel = CHANNELS.events(streamId);+ let state = this.streams.get(streamId);++ if (!state) {+ state = {+ count: 0,+ handlers: new Map(),+ abortCallbacks: new Set(),+ preemptCallbacks: new Set(),+ reorderBuffer: {+ nextSeq: 0,+ pending: new Map(),+ flushTimeout: null,+ deliveryDeferred: false,+ },+ };+ this.streams.set(streamId, state);+ }++ const registration = { callback };+ state.preemptCallbacks.add(registration);++ try {+ await this.ensureChannelSubscription(channel);+ } catch (error) {+ state.preemptCallbacks.delete(registration);+ this.unsubscribeUnusedChannel(streamId, state);+ throw error;+ }++ return () => {+ if (this.streams.get(streamId) !== state || !state.preemptCallbacks.delete(registration)) {+ return;+ }+ this.unsubscribeUnusedChannel(streamId, state);+ };+ }+/*** Get all tracked stream IDs (for orphan cleanup)*/@@ -946,6 +1044,7 @@ export class RedisEventTransport implements IEventTransport {state.handlers.clear();state.allSubscribersLeftCallback = undefined;state.abortCallbacks.clear();+ state.preemptCallbacks.clear();}this.resetReorderBuffer(streamId);diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.tsindex 7f22304e49..4bc31f4500 100644--- a/packages/api/src/stream/implementations/RedisJobStore.ts+++ b/packages/api/src/stream/implementations/RedisJobStore.ts@@ -81,6 +81,7 @@ const JOB_CAS_LUA ='if decoded and type(item) == "table" then ' +'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +'if item.files then projected.files = item.files end ' ++ 'if item.preempt then projected.preempt = item.preempt end ' +'steers[#steers + 1] = projected ' +'end ' +'end ' +@@ -214,6 +215,7 @@ const STALE_JOB_DELETE_LUA ='if decoded and type(item) == "table" then ' +'local projected = { steerId = item.steerId, text = item.text, createdAt = item.createdAt } ' +'if item.files then projected.files = item.files end ' ++ 'if item.preempt then projected.preempt = item.preempt end ' +'steers[#steers + 1] = projected ' +'end ' +'end ' +@@ -328,6 +330,7 @@ const RUNSTEPS_READ_LUA =* Returns: new depth, -1 (not running / closed), or -2 (queue full)*/const STEER_ENQUEUE_LUA =+ 'if ARGV[4] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[4] then return -1 end ' +'if redis.call("HGET", KEYS[1], "status") ~= "running" then return -1 end ' +'if redis.call("HGET", KEYS[1], "steersClosed") == "1" then return -1 end ' +'if redis.call("LLEN", KEYS[2]) >= tonumber(ARGV[3]) then return -2 end ' +@@ -1802,7 +1805,11 @@ export class RedisJobStore implements IJobStore {// ===== Steering Queue Methods =====- async enqueueSteer(streamId: string, item: SteerQueueItem): Promise<number> {+ async enqueueSteer(+ streamId: string,+ item: SteerQueueItem,+ expectedCreatedAt?: number,+ ): Promise<number> {const result = await this.redis.eval(STEER_ENQUEUE_LUA,2,@@ -1811,6 +1818,7 @@ export class RedisJobStore implements IJobStore {JSON.stringify(item),String(this.ttl.running),String(STEER_QUEUE_MAX_DEPTH),+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',);if (typeof result !== 'number') {return STEER_ENQUEUE_NOT_RUNNING;@@ -2237,6 +2245,12 @@ export class RedisJobStore implements IJobStore {isTemporary: data.isTemporary != null ? data.isTemporary === '1' : undefined,// Deferred tools discovered before a HITL pause; replayed into createRun on resume.discoveredTools: data.discoveredTools ? JSON.parse(data.discoveredTools) : undefined,+ /** The owning replica's seal capability. `serializeJob` writes every+ * boolean generically, but this mapper is explicit — omitting it here+ * drops the flag on every read, so the steer route would compute+ * `preemptArmed: false` and silently degrade interrupt-steer to+ * tool-boundary steering in EVERY Redis deployment. */+ preemptCapable: data.preemptCapable != null ? data.preemptCapable === '1' : undefined,titleEvent: data.titleEvent || undefined,replayEvents: data.replayEvents || undefined,contextUsage: data.contextUsage || undefined,diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex 4d1f023685..fe5313f50a 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -59,6 +59,14 @@ export interface SerializableJobData {* absent from the schema-only toolMap and resume would fail with "unknown tool".*/discoveredTools?: string[];+ /**+ * Whether the replica that OWNS this generation can seal mid-stream+ * (`PreemptBoundary` wiring). Recorded at createJob because the steer route+ * may land on a different replica — during a rolling deploy its own SDK+ * probe would answer for the wrong process. Absent on jobs created before+ * preempt shipped, which reads as incapable: the honest outcome.+ */+ preemptCapable?: boolean;/** Whether the user-message created event has been emitted */createdEventEmitted?: boolean;@@ -149,6 +157,7 @@ export type JobMetadataPatch = Partial<| 'isTemporary'| 'promptTokens'| 'discoveredTools'+ | 'preemptCapable'>>;@@ -186,6 +195,24 @@ export interface SteerQueueItem {* drain re-fetches each file by id scoped to the run's user and encodes* fresh, so nothing here is trusted beyond identifying the file. */files?: Partial<TFile>[];+ /** The steer asked to seal the live model stream at the next provider-safe+ * boundary instead of waiting for a tool step. Durable so a parked,+ * claimed, or replayed chip keeps its "interrupting" label. */+ preempt?: boolean;+}++/**+ * Cross-replica preempt signal. Unlike abort this does NOT stop the run — it+ * asks the generating replica to seal its current model stream at the next+ * provider-safe boundary so the queued steer can inject there. Fenced by+ * `createdAt`: a stale publish must never arm a replacement job on the same+ * streamId.+ */+export interface PreemptMessage {+ op: 'arm' | 'clear';+ /** Generation identity (`SerializableJobData.createdAt`) this belongs to. */+ createdAt: number;+ steerIds: string[];}/** Maximum steers a single run can have queued at once. */@@ -624,7 +651,7 @@ export interface IJobStore {* {@link STEER_ENQUEUE_NOT_RUNNING} when the job is missing, not running,* or closed, or {@link STEER_ENQUEUE_QUEUE_FULL} at max depth.*/- enqueueSteer(streamId: string, item: SteerQueueItem): Promise<number>;+ enqueueSteer(streamId: string, item: SteerQueueItem, expectedCreatedAt?: number): Promise<number>;/*** Atomically take ALL queued steers, FIFO. Empty array when none. With@@ -751,6 +778,27 @@ export interface IEventTransport {callback: (generationId?: number) => void,): void | (() => void) | Promise<void | (() => void)>;+ /**+ * Publish a preempt arm/clear to all replicas (Redis mode). Unlike abort+ * this does NOT stop the run — it asks the generating replica to seal its+ * current model stream at the next provider-safe boundary. Fenced by+ * {@link PreemptMessage.createdAt} against replacement jobs.+ * Optional - only implemented in Redis transport.+ */+ emitPreempt?(streamId: string, msg: PreemptMessage): void | Promise<number>;++ /**+ * Register callback for preempt signals from any replica (Redis mode).+ * An async implementation resolves only after it can receive messages.+ * The returned function removes only this registration, allowing a terminal+ * generation to release its channel without affecting a same-stream replacement.+ * Optional - only implemented in Redis transport.+ */+ onPreempt?(+ streamId: string,+ callback: (msg: PreemptMessage) => void,+ ): void | (() => void) | Promise<void | (() => void)>;+/** Get subscriber count for a stream */getSubscriberCount(streamId: string): number;diff --git a/packages/api/src/stream/metadata.ts b/packages/api/src/stream/metadata.tsindex 856e9c6413..e57a08330f 100644--- a/packages/api/src/stream/metadata.ts+++ b/packages/api/src/stream/metadata.ts@@ -33,6 +33,9 @@ export function sanitizeJobMetadata(metadata: Partial<GenerationJobMetadata>): Jif (metadata.promptTokens !== undefined) {patch.promptTokens = metadata.promptTokens;}+ if (metadata.preemptCapable !== undefined) {+ patch.preemptCapable = metadata.preemptCapable;+ }if (metadata.discoveredTools) {patch.discoveredTools = metadata.discoveredTools;}diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.tsindex 75afee28cb..30b8e6326b 100644--- a/packages/api/src/types/stream.ts+++ b/packages/api/src/types/stream.ts@@ -30,6 +30,8 @@ export interface GenerationJobMetadata {* without them the paused deferred tool would be missing from the schema-only toolMap.*/discoveredTools?: string[];+ /** See `SerializableJobData.preemptCapable`. */+ preemptCapable?: boolean;/** Set when the job is paused for human review (status === 'requires_action') */pendingAction?: Agents.PendingAction;}diff --git a/packages/data-provider/src/types/runs.ts b/packages/data-provider/src/types/runs.tsindex 47362c94db..651e84d0b6 100644--- a/packages/data-provider/src/types/runs.ts+++ b/packages/data-provider/src/types/runs.ts@@ -118,6 +118,9 @@ export type TPendingSteer = {text: string;createdAt?: number;files?: Partial<TFile>[];+ /** The steer asked to interrupt generation at the next safe boundary —+ * kept on parked/replayed chips so the "interrupting" label survives. */+ preempt?: boolean;};/** Payload of the `on_steer_applied` SSE event. */