Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧫 fix: Verify Early Stream Recovery (#15676)
Danny Avila0b9dc209b021
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.jsindex 79f31815c8..062edf6639 100644--- a/api/server/controllers/agents/__tests__/resume.spec.js+++ b/api/server/controllers/agents/__tests__/resume.spec.js@@ -127,6 +127,7 @@ jest.mock('@librechat/data-schemas', () => ({jest.mock('@librechat/api', () => ({...jest.requireActual('@librechat/api'),GenerationJobManager: mockGenerationJobManager,+ GENERATION_RECOVERY_FAILED_ERROR: 'generation_recovery_failed',captureAgentCheckpointGeneration: (...args) => mockCaptureAgentCheckpointGeneration(...args),deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),@@ -1070,6 +1071,39 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {},});+ it('settles and prunes a scheduled occurrence terminalized by recovery validation', async () => {+ const scheduledJob = makeScheduledJob();+ mockGenerationJobManager.getJob.mockResolvedValueOnce(scheduledJob).mockResolvedValueOnce({+ ...scheduledJob,+ status: 'error',+ error: 'generation_recovery_failed',+ });+ mockGenerationJobManager.getResumeState.mockRejectedValueOnce(+ new Error('terminal cleanup read failed'),+ );++ const res = await post(approveBody());++ expect(res.status).toBe(500);+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({+ scheduleId: 'schedule-1',+ scheduledFor,+ streamId: CONVO_ID,+ jobCreatedAt: 1000,+ status: 'error',+ conversationId: CONVO_ID,+ error: 'generation_recovery_failed',+ });+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(+ CONVO_ID,+ { type: 'mongo' },+ undefined,+ { checkpointNamespace: '1000' },+ );+ expect(mockClaimScheduleResume).not.toHaveBeenCalled();+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();+ });+it('stops and settles an occurrence that became inactive while awaiting approval', async () => {mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());mockIsScheduleLive.mockResolvedValue(false);@@ -2014,6 +2048,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();+ expect(mockGenerationJobManager.getResumeState).toHaveBeenCalledWith(+ CONVO_ID,+ job.createdAt,+ { validateEarlyBufferRecovery: true },+ );});it('blocks a user-authored respond decision before consuming the action', async () => {diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.jsindex 2edec4f3ac..d6ec8c54fe 100644--- a/api/server/controllers/agents/resume.js+++ b/api/server/controllers/agents/resume.js@@ -11,6 +11,7 @@ const {const {checkAccess,GenerationJobManager,+ GENERATION_RECOVERY_FAILED_ERROR,isPendingActionStale,mapToolApprovalResolutions,resolveAskUserQuestionResume,@@ -1031,7 +1032,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)let resumeState;let preparedContent;try {- resumeState = await GenerationJobManager.getResumeState(streamId, job.createdAt);+ resumeState = await GenerationJobManager.getResumeState(streamId, job.createdAt, {+ validateEarlyBufferRecovery: true,+ });const batchedAnswer =mapped.resumeValue?.answers != null &&typeof mapped.resumeValue.answers === 'object' &&@@ -1086,6 +1089,40 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)'[ResumeAgentController] Resume content preflight failed',getSafeErrorMetadata(err),);+ if (scheduleId) {+ const terminalJob = await GenerationJobManager.getJob(streamId).catch(() => null);+ if (+ terminalJob?.createdAt === job.createdAt &&+ terminalJob.status === 'error' &&+ terminalJob.error === GENERATION_RECOVERY_FAILED_ERROR+ ) {+ try {+ await recordScheduleOutcome({+ scheduleId,+ scheduledFor,+ streamId,+ jobCreatedAt: job.createdAt,+ status: 'error',+ conversationId,+ error: GENERATION_RECOVERY_FAILED_ERROR,+ });+ } catch (scheduleError) {+ logger.error(+ '[ResumeAgentController] Failed to settle scheduled recovery failure',+ getSafeErrorMetadata(scheduleError),+ );+ }+ await deleteFailedResumeCheckpoint(+ {+ conversationId,+ checkpointerCfg,+ job,+ checkpointGeneration: await checkpointGenerationPromise,+ },+ 'scheduled recovery validation failure',+ );+ }+ }if (isContentFilterError(err)) {return sendGenerationJson(res, err.statusCode, err.body, generationProtocolVersion);}diff --git a/packages/api/src/app/metrics.spec.ts b/packages/api/src/app/metrics.spec.tsindex 59267d36ed..3b4ab585d5 100644--- a/packages/api/src/app/metrics.spec.ts+++ b/packages/api/src/app/metrics.spec.ts@@ -11,6 +11,9 @@ import {recordAgentStartupMilestone,recordAgentStartupResult,recordGenerationJob,+ recordGenerationStreamAttachment,+ recordGenerationStreamEarlyBufferOverflow,+ recordGenerationStreamRecovery,recordGenerationStreamResumePendingEvents,recordGenerationStreamSubscription,recordOpenIDUserLookup,@@ -527,6 +530,10 @@ describe('createMetrics', () => {recordGenerationStreamSubscription('redis', 'resume', 'not_found');recordGenerationStreamSubscription('redis', 'resume_state', 'missing');recordGenerationStreamResumePendingEvents('memory', 3);+ recordGenerationStreamEarlyBufferOverflow('redis');+ recordGenerationStreamRecovery('redis', 'redis', 'success', 0.25, 5001, 12);+ recordGenerationStreamAttachment('redis', 'attached', 4.5);+ recordGenerationStreamAttachment('redis', 'bootstrap_slow');const response = await request(app).get('/metrics')@@ -544,6 +551,24 @@ describe('createMetrics', () => {expect(response.text).toMatch(/generation_stream_resume_pending_events_total\{store="memory"\} 3/,);+ expect(response.text).toMatch(+ /generation_stream_early_buffer_overflows_total\{store="redis"\} 1/,+ );+ expect(response.text).toMatch(+ /generation_stream_recoveries_total\{store="redis",method="redis",outcome="success"\} 1/,+ );+ expect(response.text).toMatch(+ /generation_stream_recovery_duration_seconds_sum\{store="redis",method="redis",outcome="success"\} 0.25/,+ );+ expect(response.text).toMatch(+ /generation_stream_attachment_outcomes_total\{store="redis",outcome="attached"\} 1/,+ );+ expect(response.text).toMatch(+ /generation_stream_attachment_outcomes_total\{store="redis",outcome="bootstrap_slow"\} 1/,+ );+ expect(response.text).toMatch(+ /generation_stream_first_attachment_delay_seconds_sum\{store="redis"\} 4.5/,+ );});it('tracks cumulative agent startup milestones and terminal results', async () => {diff --git a/packages/api/src/app/metrics.ts b/packages/api/src/app/metrics.tsindex d83a64ceb4..f44ae1ff59 100644--- a/packages/api/src/app/metrics.ts+++ b/packages/api/src/app/metrics.ts@@ -157,6 +157,13 @@ export type GenerationStreamSubscriptionResult =| 'error'| 'found'| 'missing';+export type GenerationStreamRecoveryMethod = 'redis' | 'snapshot';+export type GenerationStreamRecoveryOutcome = 'success' | 'failed' | 'not_required';+export type GenerationStreamAttachmentOutcome =+ | 'attached'+ | 'bootstrap_slow'+ | 'disconnected'+ | 'never_attached';export type RumProxyEndpoint = 'traces' | 'logs' | 'unknown';export type RumProxyResult =| 'success'@@ -206,6 +213,19 @@ type GenerationJobMetrics = {) => void;recordResumePendingEvents: (store: GenerationJobStore, count: number) => void;recordEarlyBufferOverflow: (store: GenerationJobStore) => void;+ recordRecovery: (+ store: GenerationJobStore,+ method: GenerationStreamRecoveryMethod,+ outcome: GenerationStreamRecoveryOutcome,+ durationSeconds: number,+ reconstructedEvents: number,+ reconstructedContent: number,+ ) => void;+ recordAttachment: (+ store: GenerationJobStore,+ outcome: GenerationStreamAttachmentOutcome,+ delaySeconds?: number,+ ) => void;};let generationJobMetrics: GenerationJobMetrics = {@@ -214,6 +234,8 @@ let generationJobMetrics: GenerationJobMetrics = {recordSubscription: () => undefined,recordResumePendingEvents: () => undefined,recordEarlyBufferOverflow: () => undefined,+ recordRecovery: () => undefined,+ recordAttachment: () => undefined,};type AgentStartupMetrics = {@@ -272,6 +294,8 @@ const resetMetricRecorders = (): void => {recordSubscription: () => undefined,recordResumePendingEvents: () => undefined,recordEarlyBufferOverflow: () => undefined,+ recordRecovery: () => undefined,+ recordAttachment: () => undefined,};agentStartupMetrics = {recordMilestone: () => undefined,@@ -316,6 +340,32 @@ export function recordGenerationStreamEarlyBufferOverflow(store: GenerationJobStgenerationJobMetrics.recordEarlyBufferOverflow(store);}+export function recordGenerationStreamRecovery(+ store: GenerationJobStore,+ method: GenerationStreamRecoveryMethod,+ outcome: GenerationStreamRecoveryOutcome,+ durationSeconds: number,+ reconstructedEvents: number,+ reconstructedContent: number,+): void {+ generationJobMetrics.recordRecovery(+ store,+ method,+ outcome,+ durationSeconds,+ reconstructedEvents,+ reconstructedContent,+ );+}++export function recordGenerationStreamAttachment(+ store: GenerationJobStore,+ outcome: GenerationStreamAttachmentOutcome,+ delaySeconds?: number,+): void {+ generationJobMetrics.recordAttachment(store, outcome, delaySeconds);+}+export function recordAgentStartupMilestone(milestone: AgentStartupMilestone,durationSeconds: number,@@ -641,6 +691,52 @@ export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {registers: [registry],});+ const generationStreamRecoveries = new Counter({+ name: 'generation_stream_recoveries_total',+ help: 'Early buffer recovery attempts by backing store, source, and outcome',+ labelNames: ['store', 'method', 'outcome'] as const,+ registers: [registry],+ });++ const generationStreamRecoveryDuration = new Histogram({+ name: 'generation_stream_recovery_duration_seconds',+ help: 'Time spent reconstructing an overflowed early generation stream',+ labelNames: ['store', 'method', 'outcome'] as const,+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],+ registers: [registry],+ });++ const generationStreamRecoveryEvents = new Histogram({+ name: 'generation_stream_recovery_events',+ help: 'Event count reconstructed during early buffer recovery',+ labelNames: ['store', 'method', 'outcome'] as const,+ buckets: [1, 10, 100, 1_000, 5_000, 10_000, 50_000],+ registers: [registry],+ });++ const generationStreamRecoveryContent = new Histogram({+ name: 'generation_stream_recovery_content_parts',+ help: 'Content part count reconstructed during early buffer recovery',+ labelNames: ['store', 'method', 'outcome'] as const,+ buckets: [1, 5, 10, 25, 50, 100, 500, 1_000],+ registers: [registry],+ });++ const generationStreamAttachments = new Counter({+ name: 'generation_stream_attachment_outcomes_total',+ help: 'Generation stream attachment lifecycle outcomes',+ labelNames: ['store', 'outcome'] as const,+ registers: [registry],+ });++ const generationStreamFirstAttachmentDelay = new Histogram({+ name: 'generation_stream_first_attachment_delay_seconds',+ help: 'Time from generation creation to its first subscriber attachment',+ labelNames: ['store'] as const,+ buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],+ registers: [registry],+ });+const agentStartupMilestoneDuration = new Histogram({name: 'agent_startup_milestone_duration_seconds',help: 'Cumulative agent chat startup latency from request ingress to each milestone',@@ -765,6 +861,26 @@ export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {recordResumePendingEvents: (store, count) =>generationStreamResumePendingEvents.inc({ store }, count),recordEarlyBufferOverflow: (store) => generationStreamEarlyBufferOverflows.inc({ store }),+ recordRecovery: (+ store,+ method,+ outcome,+ durationSeconds,+ reconstructedEvents,+ reconstructedContent,+ ) => {+ const labels = { store, method, outcome };+ generationStreamRecoveries.inc(labels);+ generationStreamRecoveryDuration.observe(labels, durationSeconds);+ generationStreamRecoveryEvents.observe(labels, reconstructedEvents);+ generationStreamRecoveryContent.observe(labels, reconstructedContent);+ },+ recordAttachment: (store, outcome, delaySeconds) => {+ generationStreamAttachments.inc({ store, outcome });+ if (outcome === 'attached' && delaySeconds != null) {+ generationStreamFirstAttachmentDelay.observe({ store }, delaySeconds);+ }+ },};agentStartupMetrics = {@@ -841,7 +957,10 @@ export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {if (completed) return;completed = true;- const requestLabels = { ...labels, status: completedBy === 'close' ? 499 : res.statusCode };+ const requestLabels = {+ ...labels,+ status: completedBy === 'close' ? 499 : res.statusCode,+ };httpRequests.inc(requestLabels);end(requestLabels);httpRequestsInFlight.dec(labels);diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 5fe8ff6d75..4322bb31b2 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -32,11 +32,25 @@ import type {SteerQueueItem,DetachedAgentEventActionStoreMode,} from './interfaces/IJobStore';+import type {+ EarlyBufferOverflowState,+ EarlyBufferRecoveryFailureReason,+ EarlyBufferRecoveryOutcome,+} from '../types/earlyBufferRecovery';import type { AgentStartupTelemetry } from '~/agents/startup';import type { RecoveredSteerPayload } from './SteerRecovery';import type { SteerContentView } from './SteeringLifecycle';import type { GenerationJobStore } from '~/app/metrics';import type * as t from '~/types';+import {+ recordGenerationStreamEarlyBufferOverflow,+ recordGenerationStreamResumePendingEvents,+ recordGenerationStreamSubscription,+ recordGenerationStreamAttachment,+ recordGenerationStreamRecovery,+ setGenerationJobsInFlight,+ recordGenerationJob,+} from '~/app/metrics';import {GenerationPublicationFencedError,JobCreationSupersededError,@@ -47,13 +61,6 @@ import {PROVIDER_DRAIN_TIMEOUT_MS,STEER_QUEUE_MAX_DEPTH,} from './interfaces/IJobStore';-import {- recordGenerationStreamEarlyBufferOverflow,- recordGenerationStreamResumePendingEvents,- recordGenerationStreamSubscription,- setGenerationJobsInFlight,- recordGenerationJob,-} from '~/app/metrics';import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery';import { assertJobStoreV2 } from './jobStoreCapabilities';@@ -144,6 +151,7 @@ const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down';* this as an application-level generation error. */export const TERMINAL_PUBLICATION_RECONNECT_ERROR ='Terminal publication failed; reconnect to load the durable result';+export const GENERATION_RECOVERY_FAILED_ERROR = 'generation_recovery_failed';/** Upper bound for a terminal owner's required persistence barrier. A crashed* owner leaves the durable pending bit behind; the next read or subscriber* promotes it to conservative reconciliation after this window. */@@ -155,6 +163,10 @@ const TERMINAL_PERSISTENCE_TIMEOUT_MS = 30_000;* durable chunk log, in-memory reconnects recover from the resume snapshot. */const EARLY_EVENT_BUFFER_MAX_EVENTS = 5_000;const EARLY_EVENT_BUFFER_MAX_BYTES = 8 * 1024 * 1024;+const SLOW_ATTACHMENT_BOOTSTRAP_MS = 3_000;+const SUBSCRIBER_LEASE_TTL_MS = 30_000;+const SUBSCRIBER_LEASE_REFRESH_MS = 10_000;+const EARLY_BUFFER_OVERFLOW_PERSISTENCE_TIMEOUT_MS = 3_000;const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;type TokenIdempotencyClaim = IdempotencyClaimValue & {claimedAt: number;@@ -664,6 +676,7 @@ interface RuntimeJobState {* consumed it. Non-resume attachments are redirected to the resume path,* which reconstructs the discarded output from durable/snapshot state. */earlyEventBufferOverflowed?: true;+ earlyBufferOverflow?: EarlyBufferOverflowState;earlyEventSequencePromises: Array<Promise<void | number>>;/** Initial subscribers eligible to receive the local pre-attachment replay. */earlyReplayHandlers: Set<t.ChunkHandler>;@@ -671,6 +684,8 @@ interface RuntimeJobState {resumeCaptureHandlers: Set<(event: t.ServerSentEvent, sequence: number) => void>;/** Monotonic local emission sequence used to establish an exact resume snapshot frontier. */emissionSequence: number;+ /** Count of emitted Redis envelopes that are eligible for durable append. */+ durableEventSequence: number;/** Emissions that started before an in-memory resume snapshot and must become snapshot-visible* before the graph/job state is read. The event identity also suppresses their later publish. */inFlightSnapshotEmissions: Map<@@ -682,6 +697,16 @@ interface RuntimeJobState {/** Coalesced delta publications emitted but not yet settled by a window flush. */outstandingCoalescedReceipts?: number;hasSubscriber: boolean;+ everHadSubscriber: boolean;+ /** Ordered, non-blocking durable subscriber lifecycle writes. */+ subscriberStateWrite?: Promise<unknown>;+ /** Whether this runtime's local subscriber group is represented durably. */+ subscriberStateAttached: boolean;+ subscriberLeaseId: string;+ subscriberLeaseTimer?: ReturnType<typeof setInterval>;+ subscriberLeaseRenewalPending?: boolean;+ /** One-shot telemetry callback retained until a durable first-subscriber claim succeeds. */+ onFirstSubscriberLeaseClaim?: (firstSubscriber: boolean) => void;/** Advances whenever every local SSE subscriber for one attachment generation leaves. */attachmentGeneration: number;/** Attachment generation whose partial-response disconnect cleanup was most recently started. */@@ -752,6 +777,9 @@ class GenerationJobManagerClass {/** Serializes whole-array run-step snapshots so an older save cannot overwrite completion. */private runStepWriteQueues = new Map<string, Promise<void>>();+ /** Coalesces reconnects waiting on the same owner-side overflow flush. */+ private earlyBufferOverflowWaiters = new Map<string, Promise<SerializableJobData | null>>();+/** Partial-response and disconnect-state writes still draining during shutdown. */private subscriberCleanupPromises = new Set<Promise<void>>();@@ -880,6 +908,7 @@ class GenerationJobManagerClass {for (const runtime of this.runtimeState.values()) {runtime.startupTelemetry?.end('aborted');runtime.startupTelemetry = undefined;+ this.stopSubscriberLease(runtime);this.releaseAbortSubscription(runtime, true);runtime.abortController.abort();}@@ -903,6 +932,7 @@ class GenerationJobManagerClass {this.replayEventWriteQueues.clear();this.tokenUsageWriteQueues.clear();this.runStepWriteQueues.clear();+ this.earlyBufferOverflowWaiters.clear();}this.ownedJobs.clear();@@ -1599,6 +1629,26 @@ class GenerationJobManagerClass {runtime.hasSubscriber = false;runtime.attachmentGeneration++;runtime.lastSubscriberCleanupGeneration = runtime.attachmentGeneration;+ this.stopSubscriberLease(runtime);++ runtime.subscriberStateWrite = (runtime.subscriberStateWrite ?? Promise.resolve())+ .then(async () => {+ if (!runtime.subscriberStateAttached) {+ return;+ }+ await this.jobStore.detachSubscriber(+ streamId,+ runtime.createdAt,+ runtime.subscriberLeaseId,+ );+ runtime.subscriberStateAttached = false;+ })+ .catch((detachError) => {+ logger.error(+ '[GenerationJobManager] Failed to persist subscriber detachment',+ detachError,+ );+ });// Terminal delivery closes the SSE subscription too, but it is not a user// disconnect. Running partial-response handlers here can overwrite the@@ -1607,7 +1657,14 @@ class GenerationJobManagerClass {return;}- const cleanup = this.persistSubscriberCleanup(streamId, runtime);+ if (runtime.everHadSubscriber) {+ recordGenerationStreamAttachment(this.storeLabel, 'disconnected');+ }++ const cleanup = Promise.all([+ runtime.subscriberStateWrite,+ this.persistSubscriberCleanup(streamId, runtime),+ ]).then(() => undefined);this.subscriberCleanupPromises.add(cleanup);void cleanup.then(() => this.subscriberCleanupPromises.delete(cleanup),@@ -1619,6 +1676,49 @@ class GenerationJobManagerClass {});}+ private stopSubscriberLease(runtime: RuntimeJobState): void {+ if (runtime.subscriberLeaseTimer != null) {+ clearInterval(runtime.subscriberLeaseTimer);+ runtime.subscriberLeaseTimer = undefined;+ }+ }++ private startSubscriberLease(streamId: string, runtime: RuntimeJobState): void {+ if (runtime.subscriberLeaseTimer != null) {+ return;+ }+ runtime.subscriberLeaseTimer = setInterval(() => {+ if (!runtime.hasSubscriber || runtime.subscriberLeaseRenewalPending === true) {+ return;+ }+ runtime.subscriberLeaseRenewalPending = true;+ runtime.subscriberStateWrite = (runtime.subscriberStateWrite ?? Promise.resolve())+ .then(() => {+ const refreshedAt = Date.now();+ return this.jobStore.claimFirstSubscriber(+ streamId,+ runtime.createdAt,+ refreshedAt,+ runtime.subscriberLeaseId,+ refreshedAt + SUBSCRIBER_LEASE_TTL_MS,+ );+ })+ .then((firstSubscriber) => {+ runtime.subscriberStateAttached = true;+ runtime.onFirstSubscriberLeaseClaim?.(firstSubscriber);+ return firstSubscriber;+ })+ .catch((leaseError) => {+ logger.error('[GenerationJobManager] Failed to renew subscriber lease', leaseError);+ return false;+ })+ .finally(() => {+ runtime.subscriberLeaseRenewalPending = false;+ });+ }, SUBSCRIBER_LEASE_REFRESH_MS);+ runtime.subscriberLeaseTimer.unref?.();+ }+private async persistSubscriberCleanup(streamId: string,runtime: RuntimeJobState,@@ -1804,6 +1904,7 @@ class GenerationJobManagerClass {return;}if (this.runtimeState.get(streamId) === predecessor) {+ this.stopSubscriberLease(predecessor);this.runtimeState.delete(streamId);this.releaseJobOwnership(streamId, predecessor.createdAt);this.releaseAbortSubscription(predecessor, true);@@ -2283,7 +2384,9 @@ class GenerationJobManagerClass {const { steerQuotesCapable, ...storedMetadata } = sanitizedMetadata;const initialMetadata = {...storedMetadata,- ...(steerQuotesCapable === true && { steerQuotesExecutionId: creationAttemptId }),+ ...(steerQuotesCapable === true && {+ steerQuotesExecutionId: creationAttemptId,+ }),providerExecutionId: creationAttemptId,providerDrained: true,};@@ -2572,6 +2675,7 @@ class GenerationJobManagerClass {if (replacedRuntime) {replacedRuntime.startupTelemetry?.end('replaced');replacedRuntime.startupTelemetry = undefined;+ this.stopSubscriberLease(replacedRuntime);const durableReceipt = exactPredecessorsByEpoch.get(replacedRuntime.createdAt);if (durableReceipt == null ||@@ -2611,8 +2715,12 @@ class GenerationJobManagerClass {resumeCaptureHandlers: new Set(),localErrorHandlers: new Set(),emissionSequence: 0,+ durableEventSequence: 0,inFlightSnapshotEmissions: new Map(),hasSubscriber: false,+ everHadSubscriber: jobData.firstSubscriberAttachedAt != null,+ subscriberStateAttached: false,+ subscriberLeaseId: randomUUID(),attachmentGeneration: 0,};this.runtimeState.set(streamId, runtime);@@ -2695,6 +2803,7 @@ class GenerationJobManagerClass {) {this.releaseAbortSubscription(runtime);runtime.abortController.abort();+ this.stopSubscriberLease(runtime);this.runtimeState.delete(streamId);this.releaseJobOwnership(streamId, runtime.createdAt);}@@ -2778,6 +2887,7 @@ class GenerationJobManagerClass {conversationId: jobData.conversationId,checkpointNamespace: jobData.checkpointNamespace,generationProtocolVersion: jobData.generationProtocolVersion,+ earlyBufferOverflow: jobData.earlyBufferOverflow,userMessage: jobData.userMessage,responseMessageId: jobData.responseMessageId,isRegenerate: jobData.isRegenerate,@@ -2869,6 +2979,12 @@ class GenerationJobManagerClass {const concurrentRuntime = this.runtimeState.get(streamId);if (concurrentRuntime?.createdAt === jobData.createdAt) {+ if (jobData.earlyBufferOverflow != null) {+ concurrentRuntime.earlyBufferOverflow = jobData.earlyBufferOverflow;+ concurrentRuntime.earlyEventBufferClosed = true;+ concurrentRuntime.earlyEventBufferOverflowed = true;+ this.resetEarlyEventBuffer(concurrentRuntime);+ }this.reconcileInactiveGeneration(streamId, jobData.createdAt, jobData, concurrentRuntime);return concurrentRuntime;}@@ -2878,6 +2994,7 @@ class GenerationJobManagerClass {if (concurrentRuntime) {concurrentRuntime.startupTelemetry?.end('replaced');concurrentRuntime.startupTelemetry = undefined;+ this.stopSubscriberLease(concurrentRuntime);this.releaseAbortSubscription(concurrentRuntime);concurrentRuntime.abortController.abort();}@@ -2913,11 +3030,20 @@ class GenerationJobManagerClass {resumeCaptureHandlers: new Set(),localErrorHandlers: new Set(),emissionSequence: 0,+ durableEventSequence: jobData.durableEventCount ?? 0,inFlightSnapshotEmissions: new Map(),hasSubscriber: false,+ everHadSubscriber: false,+ subscriberStateAttached: false,+ subscriberLeaseId: randomUUID(),attachmentGeneration: 0,finalEvent,errorEvent: jobData.error,+ earlyBufferOverflow: jobData.earlyBufferOverflow,+ ...(jobData.earlyBufferOverflow != null && {+ earlyEventBufferClosed: true,+ earlyEventBufferOverflowed: true,+ }),};this.runtimeState.set(streamId, runtime);@@ -2948,6 +3074,7 @@ class GenerationJobManagerClass {if (!confirmedJobData) {this.releaseAbortSubscription(runtime);runtime.abortController.abort();+ this.stopSubscriberLease(runtime);this.runtimeState.delete(streamId);return null;}@@ -3403,7 +3530,11 @@ class GenerationJobManagerClass {return { claimed: true, existing: value, source: 'primary' };}if (isClaimTakeoverOf(expectedClaim, observedPrimary)) {- return { claimed: false, existing: observedPrimary, source: 'primary' };+ return {+ claimed: false,+ existing: observedPrimary,+ source: 'primary',+ };}}throw error;@@ -3634,7 +3765,10 @@ class GenerationJobManagerClass {status: TerminalJobClaim['status'],error?: string,expectedCreatedAt?: number,- options: { persistencePending?: boolean; failedPauseActionId?: string } = {},+ options: {+ persistencePending?: boolean;+ failedPauseActionId?: string;+ } = {},): Promise<TerminalJobClaim | null> {if (options.failedPauseActionId != null &&@@ -3703,7 +3837,9 @@ class GenerationJobManagerClass {...(sourceStatus === 'requires_action' && {...(failedPauseBarrierId != null? { expectActionId: failedPauseBarrierId }- : jobData.pendingActionId != null && { expectActionId: jobData.pendingActionId }),+ : jobData.pendingActionId != null && {+ expectActionId: jobData.pendingActionId,+ }),}),patch: {completedAt,@@ -3735,10 +3871,14 @@ class GenerationJobManagerClass {const claim: TerminalJobClaim = Object.freeze({streamId,createdAt,- ...(jobData.conversationId != null && { conversationId: jobData.conversationId }),+ ...(jobData.conversationId != null && {+ conversationId: jobData.conversationId,+ }),status,...(terminalError != null && { error: terminalError }),- ...(options.persistencePending === true && { persistencePending: true as const }),+ ...(options.persistencePending === true && {+ persistencePending: true as const,+ }),drainedSteers: Object.freeze([...drainedSteers]),});this.terminalClaimRuntimes.set(claim, runtime ?? null);@@ -3917,8 +4057,68 @@ class GenerationJobManagerClass {this.runtimeState.get(streamId) === claimedRuntime? claimedRuntime: undefined;+ await runtime?.subscriberStateWrite?.catch(() => undefined);let cleanupError: unknown;let retainTerminalHostEvidence = false;+ let persistedLifecycle: SerializableJobData | null | undefined = null;+ if (runtime != null) {+ persistedLifecycle = await this.jobStore.getJob(streamId).catch((lifecycleError) => {+ logger.error('[GenerationJobManager] Failed to read terminal subscriber lifecycle', {+ streamId,+ error: lifecycleError instanceof Error ? lifecycleError.message : String(lifecycleError),+ });+ return undefined;+ });+ }+ const generationHadSubscriber =+ runtime?.everHadSubscriber === true ||+ (persistedLifecycle?.createdAt === createdAt &&+ persistedLifecycle.firstSubscriberAttachedAt != null);+ const unresolvedOverflow =+ runtime?.earlyBufferOverflow?.recoveryOutcome == null+ ? runtime?.earlyBufferOverflow+ : undefined;+ if (runtime != null && unresolvedOverflow != null) {+ let remoteSubscriberActive: boolean | undefined =+ persistedLifecycle === undefined ? undefined : false;+ if (persistedLifecycle?.createdAt === createdAt) {+ remoteSubscriberActive = await this.jobStore+ .hasActiveSubscriber(streamId, createdAt, Date.now())+ .catch((leaseError) => {+ logger.error(+ '[GenerationJobManager] Failed to read active subscriber leases',+ leaseError,+ );+ return undefined;+ });+ }+ /** A failed lease read is not evidence of an active subscriber. Leave the+ * overflow unresolved so a later resume must still validate recovery. */+ if (runtime.hasSubscriber || remoteSubscriberActive != null) {+ const subscriberActive = runtime.hasSubscriber || remoteSubscriberActive === true;+ let unobservedFailureReason: EarlyBufferRecoveryFailureReason | undefined;+ if (!subscriberActive) {+ unobservedFailureReason = generationHadSubscriber+ ? 'subscriber_disconnected'+ : 'subscriber_never_attached';+ }+ await this.settleEarlyBufferRecovery(+ streamId,+ runtime,+ unresolvedOverflow,+ subscriberActive ? 'not_required' : 'failed',+ 0,+ 0,+ 0,+ unobservedFailureReason,+ ).catch((recoveryError) => {+ logger.error(+ '[GenerationJobManager] Failed to settle unobserved early buffer recovery',+ recoveryError,+ );+ });+ }+ }// Error jobs stay durable long enough for late subscribers to receive the// stored error. A publication failure must never bypass the finally cleanup.@@ -3986,6 +4186,9 @@ class GenerationJobManagerClass {retainTerminalHostEvidence = true;} finally {if (runtime && this.runtimeState.get(streamId) === runtime) {+ if (!generationHadSubscriber) {+ recordGenerationStreamAttachment(this.storeLabel, 'never_attached');+ }this.releaseAbortSubscription(runtime);runtime.abortController.abort();if (status === 'error') {@@ -4004,6 +4207,7 @@ class GenerationJobManagerClass {this.tokenUsageWriteQueues.delete(streamId);this.runStepWriteQueues.delete(streamId);if (status !== 'error' && this._cleanupOnComplete) {+ this.stopSubscriberLease(runtime);this.runtimeState.delete(streamId);}}@@ -4275,7 +4479,9 @@ class GenerationJobManagerClass {/** Text from content parts for fallback token counting; the persisted* abort record keeps steered words (they reached the model context). */let text = shouldPersistAbortContent- ? parseTextParts(abortContent as TMessageContentParts[], false, { includeSteer: true })+ ? parseTextParts(abortContent as TMessageContentParts[], false, {+ includeSteer: true,+ }): '';/** Claim terminal ownership and drain steers in one store transaction. A@@ -4369,7 +4575,9 @@ class GenerationJobManagerClass {const terminalClaim: TerminalJobClaim = Object.freeze({streamId,createdAt: jobData.createdAt,- ...(jobData.conversationId != null && { conversationId: jobData.conversationId }),+ ...(jobData.conversationId != null && {+ conversationId: jobData.conversationId,+ }),status: 'aborted',persistencePending: true,drainedSteers: Object.freeze([...drainedSteers]),@@ -4440,7 +4648,9 @@ class GenerationJobManagerClass {abortContent = filterPersistableAbortContent(content);shouldPersistAbortContent = abortContent.length > 0;text = shouldPersistAbortContent- ? parseTextParts(abortContent as TMessageContentParts[], false, { includeSteer: true })+ ? parseTextParts(abortContent as TMessageContentParts[], false, {+ includeSteer: true,+ }): '';/** Detect "early abort" - aborted before any generation happened (e.g., during tool loading)@@ -4583,6 +4793,7 @@ class GenerationJobManagerClass {options?: t.SubscribeOptions,prepared?: PreparedSubscription,): Promise<(t.StreamSubscription & { activate?: () => void }) | null> {+ const attachmentStartedAt = Date.now();const subscriptionType = options?.skipBufferReplay ? 'resume' : 'initial';if (options?.signal?.aborted) {recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'error');@@ -4823,7 +5034,9 @@ class GenerationJobManagerClass {};subscription = {ready: transportSubscription.ready,- ...(prepared?.deferDeliveryUntilActivated === true && { activate: activateDelivery }),+ ...(prepared?.deferDeliveryUntilActivated === true && {+ activate: activateDelivery,+ }),unsubscribe: (): void => {if (!subscriptionActive) {return;@@ -4913,6 +5126,49 @@ class GenerationJobManagerClass {if (!runtime.hasSubscriber) {runtime.hasSubscriber = true;+ const attachedAt = Date.now();+ if (!runtime.everHadSubscriber) {+ const bootstrapSlow = attachedAt - attachmentStartedAt >= SLOW_ATTACHMENT_BOOTSTRAP_MS;+ runtime.everHadSubscriber = true;+ runtime.onFirstSubscriberLeaseClaim = (firstSubscriber) => {+ if (!firstSubscriber) {+ return;+ }+ runtime.onFirstSubscriberLeaseClaim = undefined;+ recordGenerationStreamAttachment(+ this.storeLabel,+ 'attached',+ Math.max(0, attachedAt - runtime.createdAt) / 1000,+ );+ if (bootstrapSlow) {+ recordGenerationStreamAttachment(this.storeLabel, 'bootstrap_slow');+ }+ };+ }+ this.startSubscriberLease(streamId, runtime);+ const subscriberClaim = (runtime.subscriberStateWrite ?? Promise.resolve())+ .then(async () => {+ const firstSubscriber = await this.jobStore.claimFirstSubscriber(+ streamId,+ runtime.createdAt,+ attachedAt,+ runtime.subscriberLeaseId,+ Date.now() + SUBSCRIBER_LEASE_TTL_MS,+ );+ runtime.subscriberStateAttached = true;+ return firstSubscriber;+ })+ .catch((claimError) => {+ logger.error(+ '[GenerationJobManager] Failed to persist subscriber attachment',+ claimError,+ );+ return false;+ });+ runtime.subscriberStateWrite = subscriberClaim;+ void subscriberClaim.then((firstSubscriber) =>+ runtime.onFirstSubscriberLeaseClaim?.(firstSubscriber),+ );const attachmentGeneration = runtime.attachmentGeneration;const earlyPublicationFence = this.waitForEarlyEventPublications(runtime);if (!(await waitWhileAttached(earlyPublicationFence))) {@@ -5225,11 +5481,11 @@ class GenerationJobManagerClass {** @returns whether the event was accepted into the buffer.*/- private bufferEarlyEvent(+ private async bufferEarlyEvent(streamId: string,runtime: RuntimeJobState,event: t.ServerSentEvent,- ): boolean {+ ): Promise<boolean> {if (runtime.earlyEventBufferClosed) {return false;}@@ -5238,7 +5494,7 @@ class GenerationJobManagerClass {runtime.earlyEventBuffer.length >= EARLY_EVENT_BUFFER_MAX_EVENTS ||runtime.earlyEventBufferBytes + estimatedBytes > EARLY_EVENT_BUFFER_MAX_BYTES) {- this.overflowEarlyEventBuffer(streamId, runtime);+ await this.overflowEarlyEventBuffer(streamId, runtime, 1, estimatedBytes);return false;}runtime.earlyEventBuffer.push(event);@@ -5246,18 +5502,119 @@ class GenerationJobManagerClass {return true;}- private overflowEarlyEventBuffer(streamId: string, runtime: RuntimeJobState): void {- const droppedEvents = runtime.earlyEventBuffer.length;- const droppedBytes = runtime.earlyEventBufferBytes;- this.resetEarlyEventBuffer(runtime);+ private async overflowEarlyEventBuffer(+ streamId: string,+ runtime: RuntimeJobState,+ rejectedEvents = 0,+ rejectedBytes = 0,+ ): Promise<void> {+ const droppedEvents = runtime.earlyEventBuffer.length + rejectedEvents;+ const droppedBytes = runtime.earlyEventBufferBytes + rejectedBytes;+ const overflow: EarlyBufferOverflowState = {+ id: randomUUID(),+ occurredAt: Date.now(),+ durableEvents: this._isRedis ? runtime.durableEventSequence : runtime.emissionSequence,+ droppedEvents,+ droppedBytes,+ persistencePending: true,+ };runtime.earlyEventBufferClosed = true;runtime.earlyEventBufferOverflowed = true;+ runtime.earlyBufferOverflow = overflow;recordGenerationStreamEarlyBufferOverflow(this.storeLabel);logger.warn(- `[GenerationJobManager] Early event buffer overflow for ${streamId}; ` +- `discarded ${droppedEvents} buffered events (~${droppedBytes} bytes); ` +- 'late subscribers will recover from durable/resume state',+ '[GenerationJobManager] Early event buffer overflow; late subscriber recovery required',+ {+ recoveryId: overflow.id,+ store: this.storeLabel,+ droppedEvents,+ droppedBytes,+ },);+ try {+ /** Publish an admission fence before releasing the local recovery copy.+ * Detached Redis events are durably appended before publication below,+ * so a replica that races before this write includes every earlier+ * publication in its snapshot. A replica that observes this marker waits+ * for the finalized frontier before it can activate. */+ await this.jobStore.updateJob(streamId, { earlyBufferOverflow: overflow }, runtime.createdAt);+ await this.jobStore.flushPendingAppends?.(streamId);+ const frontierJob = await this.jobStore.getJob(streamId);+ if (frontierJob?.createdAt !== runtime.createdAt) {+ throw new Error('Early buffer overflow generation was replaced before persistence');+ }+ const finalizedOverflow: EarlyBufferOverflowState = {+ ...overflow,+ durableEvents: this._isRedis+ ? (frontierJob.durableEventCount ?? runtime.durableEventSequence)+ : runtime.emissionSequence,+ persistencePending: false,+ };+ const finalized = await this.jobStore.finalizeEarlyBufferOverflow(+ streamId,+ runtime.createdAt,+ overflow.id,+ finalizedOverflow,+ );+ const persistedJob = await this.jobStore.getJob(streamId);+ if (+ !finalized ||+ persistedJob?.createdAt !== runtime.createdAt ||+ persistedJob.earlyBufferOverflow?.id !== overflow.id ||+ persistedJob.earlyBufferOverflow.persistencePending === true+ ) {+ if (+ persistedJob?.createdAt === runtime.createdAt &&+ persistedJob.earlyBufferOverflow?.id === overflow.id &&+ persistedJob.earlyBufferOverflow.recoveryOutcome != null+ ) {+ runtime.earlyBufferOverflow = persistedJob.earlyBufferOverflow;+ this.resetEarlyEventBuffer(runtime);+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR);+ }+ throw new Error('Early buffer overflow marker was not durably persisted');+ }+ runtime.earlyBufferOverflow = finalizedOverflow;+ this.resetEarlyEventBuffer(runtime);+ } catch (err) {+ this.resetEarlyEventBuffer(runtime);+ if (+ err instanceof Error &&+ err.message === GENERATION_RECOVERY_FAILED_ERROR &&+ runtime.earlyBufferOverflow?.recoveryOutcome != null+ ) {+ throw err;+ }+ logger.error('[GenerationJobManager] Failed to persist early buffer overflow identity', {+ recoveryId: overflow.id,+ store: this.storeLabel,+ error: err instanceof Error ? err.message : String(err),+ });+ const settlement = {+ recoveryMethod: this._isRedis ? ('redis' as const) : ('snapshot' as const),+ recoveryOutcome: 'failed' as const,+ recoveryCompletedAt: Date.now(),+ recoveryFailureReason: 'overflow_marker_persistence_failed' as const,+ };+ runtime.earlyBufferOverflow = { ...overflow, ...settlement };+ recordGenerationStreamRecovery(+ this.storeLabel,+ settlement.recoveryMethod,+ settlement.recoveryOutcome,+ Math.max(0, settlement.recoveryCompletedAt - overflow.occurredAt) / 1000,+ 0,+ 0,+ );+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt).catch(+ (terminalError) => {+ logger.error(+ '[GenerationJobManager] Failed to terminalize after overflow marker persistence failure',+ terminalError,+ );+ },+ );+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR, { cause: err });+ }}/**@@ -5345,6 +5702,106 @@ class GenerationJobManagerClass {});}+ private async waitForFinalizedEarlyBufferOverflow(+ streamId: string,+ jobData: SerializableJobData,+ ): Promise<SerializableJobData | null> {+ const overflow = jobData.earlyBufferOverflow;+ if (overflow?.persistencePending !== true) {+ return jobData;+ }++ const waiterKey = `${streamId}:${jobData.createdAt}:${overflow.id}`;+ const existing = this.earlyBufferOverflowWaiters.get(waiterKey);+ if (existing) {+ return existing;+ }++ const waiter = (async (): Promise<SerializableJobData | null> => {+ const deadline = Date.now() + EARLY_BUFFER_OVERFLOW_PERSISTENCE_TIMEOUT_MS;+ let delayMs = 100;+ let current: SerializableJobData | null = jobData;+ while (+ current?.createdAt === jobData.createdAt &&+ current.earlyBufferOverflow?.id === overflow.id &&+ current.earlyBufferOverflow.persistencePending === true &&+ Date.now() < deadline+ ) {+ const remainingMs = deadline - Date.now();+ await new Promise<void>((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs)));+ current = await this.jobStore.getJob(streamId);+ delayMs = Math.min(delayMs * 2, 1000);+ }+ return current;+ })();+ this.earlyBufferOverflowWaiters.set(waiterKey, waiter);+ try {+ return await waiter;+ } finally {+ if (this.earlyBufferOverflowWaiters.get(waiterKey) === waiter) {+ this.earlyBufferOverflowWaiters.delete(waiterKey);+ }+ }+ }++ private async settleEarlyBufferRecovery(+ streamId: string,+ runtime: RuntimeJobState,+ overflow: EarlyBufferOverflowState,+ outcome: EarlyBufferRecoveryOutcome,+ durationMs: number,+ reconstructedEvents: number,+ reconstructedContent: number,+ failureReason?: EarlyBufferRecoveryFailureReason,+ ): Promise<EarlyBufferRecoveryOutcome | undefined> {+ const recoveryMethod = this._isRedis ? 'redis' : 'snapshot';+ const settlement = {+ recoveryMethod,+ recoveryOutcome: outcome,+ recoveryCompletedAt: Date.now(),+ ...(failureReason != null && { recoveryFailureReason: failureReason }),+ } as const;+ const settled = await this.jobStore.settleEarlyBufferRecovery(+ streamId,+ runtime.createdAt,+ overflow.id,+ settlement,+ );+ if (!settled) {+ const winningJob = await this.jobStore.getJob(streamId);+ const winningOverflow = winningJob?.earlyBufferOverflow;+ if (+ winningJob?.createdAt === runtime.createdAt &&+ winningOverflow?.id === overflow.id &&+ winningOverflow.recoveryOutcome != null+ ) {+ runtime.earlyBufferOverflow = winningOverflow;+ return winningOverflow.recoveryOutcome;+ }+ return undefined;+ }+ runtime.earlyBufferOverflow = { ...overflow, ...settlement };+ recordGenerationStreamRecovery(+ this.storeLabel,+ recoveryMethod,+ outcome,+ Math.max(0, durationMs) / 1000,+ reconstructedEvents,+ reconstructedContent,+ );+ logger.info('[GenerationJobManager] Early buffer recovery completed', {+ recoveryId: overflow.id,+ store: this.storeLabel,+ method: recoveryMethod,+ outcome,+ durationMs,+ reconstructedEvents,+ reconstructedContent,+ ...(failureReason != null && { failureReason }),+ });+ return outcome;+ }+/*** Snapshots resume state and attaches a paused live subscription.*@@ -5363,6 +5820,7 @@ class GenerationJobManagerClass {recordGenerationStreamSubscription(this.storeLabel, 'resume', 'error');return { subscription: null, resumeState: null, pendingEvents: [] };}+if (this.rejectSubscriptionDuringShutdown('resume', onError)) {return { subscription: null, resumeState: null, pendingEvents: [] };}@@ -5386,12 +5844,76 @@ class GenerationJobManagerClass {return { subscription: null, resumeState: null, pendingEvents: [] };}+ if (runtime.earlyBufferOverflow?.persistencePending === true) {+ const pendingMarker = runtime.earlyBufferOverflow;+ const currentJob = await this.jobStore.getJob(streamId);+ if (+ currentJob?.createdAt !== runtime.createdAt ||+ currentJob.earlyBufferOverflow?.id !== pendingMarker.id+ ) {+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ const finalizedJob = await this.waitForFinalizedEarlyBufferOverflow(streamId, currentJob);+ if (finalizedJob?.createdAt !== runtime.createdAt) {+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ await this.getOrCreateRuntimeState(streamId, finalizedJob);+ if (finalizedJob.earlyBufferOverflow?.persistencePending === true) {+ await this.settleEarlyBufferRecovery(+ streamId,+ runtime,+ pendingMarker,+ 'failed',+ EARLY_BUFFER_OVERFLOW_PERSISTENCE_TIMEOUT_MS,+ 0,+ 0,+ 'overflow_marker_persistence_failed',+ );+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ }++ if (+ runtime.earlyBufferOverflow?.recoveryOutcome === 'failed' &&+ !runtime.finalEvent &&+ !runtime.errorEvent+ ) {+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }++ const snapshotOverflowId = runtime.earlyBufferOverflow?.id;+ const pendingOverflow =+ runtime.earlyBufferOverflow?.recoveryOutcome == null &&+ !runtime.finalEvent &&+ !runtime.errorEvent+ ? runtime.earlyBufferOverflow+ : undefined;+ const recoveryStartedAt = pendingOverflow == null ? undefined : Date.now();+ let recoveryContentSnapshot: Awaited<ReturnType<IJobStore['getContentParts']>> | undefined;+ let recoverySettled = false;+ if (pendingOverflow != null) {+ logger.info('[GenerationJobManager] Early buffer recovery started', {+ recoveryId: pendingOverflow.id,+ store: this.storeLabel,+ method: this._isRedis ? 'redis' : 'snapshot',+ });+ }+const capturedPendingEvents: t.ServerSentEvent[] = [];const pendingEvents: t.ServerSentEvent[] = [];const capturedEventSet = new Set<t.ServerSentEvent>();const snapshotCoveredEventSet = new Set<t.ServerSentEvent>();const seenEmissionEvents = new Set<t.ServerSentEvent>();- const unclassifiedEmissions: Array<{ event: t.ServerSentEvent; sequence: number }> = [];+ const unclassifiedEmissions: Array<{+ event: t.ServerSentEvent;+ sequence: number;+ }> = [];let snapshotFrontier = 0;let snapshotClassified = this._isRedis;const classifyEmission = (event: t.ServerSentEvent, sequence: number): void => {@@ -5419,6 +5941,14 @@ class GenerationJobManagerClass {const removeCaptureHandler = (): void => {runtime.resumeCaptureHandlers.delete(capturePendingEvent);};+ const discardCapturedEvents = (): void => {+ removeCaptureHandler();+ capturedPendingEvents.length = 0;+ pendingEvents.length = 0;+ unclassifiedEmissions.length = 0;+ capturedEventSet.clear();+ snapshotCoveredEventSet.clear();+ };const restoreCapturedEvents = (): void => {if (capturedPendingEvents.length === 0) {return;@@ -5441,7 +5971,14 @@ class GenerationJobManagerClass {EARLY_EVENT_BUFFER_MAX_EVENTS ||currentRuntime.earlyEventBufferBytes + restoredBytes > EARLY_EVENT_BUFFER_MAX_BYTES;if (overflows) {- this.overflowEarlyEventBuffer(streamId, currentRuntime);+ void this.overflowEarlyEventBuffer(+ streamId,+ currentRuntime,+ missingEvents.length,+ restoredBytes,+ ).catch((err) => {+ logger.error('[GenerationJobManager] Failed to persist early buffer overflow', err);+ });} else {currentRuntime.earlyEventBuffer = [...missingEvents,@@ -5465,7 +6002,12 @@ class GenerationJobManagerClass {);await Promise.all(preSnapshotEmissions.map(([, emission]) => emission.snapshotReady));const [candidateState, candidateJob] = await Promise.all([- this.getResumeState(streamId, options?.expectedCreatedAt),+ this.getResumeState(streamId, options?.expectedCreatedAt, {+ durableOnly: false,+ onContentSnapshot: (snapshot) => {+ recoveryContentSnapshot = snapshot;+ },+ }),this.jobStore.getJob(streamId),]);if (@@ -5492,11 +6034,108 @@ class GenerationJobManagerClass {}} else {[resumeState, jobData] = await Promise.all([- this.getResumeState(streamId, options?.expectedCreatedAt),+ this.getResumeState(streamId, options?.expectedCreatedAt, {+ durableOnly: pendingOverflow != null,+ onContentSnapshot: (snapshot) => {+ recoveryContentSnapshot = snapshot;+ },+ }),this.jobStore.getJob(streamId),]);}+ if (pendingOverflow != null && recoveryStartedAt != null) {+ const contentSnapshot = recoveryContentSnapshot;+ const reconstructedEvents = this._isRedis+ ? (contentSnapshot?.reconstructedEventCount ?? 0)+ : snapshotFrontier;+ const durableEvents = this._isRedis+ ? (contentSnapshot?.durableEventCount ?? 0)+ : snapshotFrontier;+ const reconstructedContent = resumeState?.aggregatedContent?.length ?? 0;+ let failureReason: EarlyBufferRecoveryFailureReason | undefined;+ if (resumeState == null || contentSnapshot == null) {+ failureReason = this._isRedis ? 'durable_state_missing' : 'snapshot_missing';+ } else if (+ durableEvents < pendingOverflow.durableEvents ||+ reconstructedEvents !== durableEvents+ ) {+ failureReason = 'durable_frontier_gap';+ }+ const durationMs = Date.now() - recoveryStartedAt;+ if (failureReason == null) {+ const winningOutcome = await this.settleEarlyBufferRecovery(+ streamId,+ runtime,+ pendingOverflow,+ 'success',+ durationMs,+ reconstructedEvents,+ reconstructedContent,+ );+ recoverySettled = winningOutcome != null;+ if (winningOutcome == null) {+ const currentJob = await this.jobStore.getJob(streamId);+ if (currentJob?.createdAt !== runtime.createdAt) {+ removeCaptureHandler();+ if (currentJob != null) {+ await this.reconcileFencedRuntimeHandoff(streamId, runtime, currentJob);+ }+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ }+ if (winningOutcome === 'not_required') {+ discardCapturedEvents();+ return { subscription: null, resumeState, pendingEvents: [] };+ }+ if (winningOutcome !== 'success') {+ discardCapturedEvents();+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ } else {+ const winningOutcome = await this.settleEarlyBufferRecovery(+ streamId,+ runtime,+ pendingOverflow,+ 'failed',+ durationMs,+ reconstructedEvents,+ reconstructedContent,+ failureReason,+ );+ recoverySettled = winningOutcome != null;+ if (winningOutcome == null) {+ const currentJob = await this.jobStore.getJob(streamId);+ if (currentJob?.createdAt !== runtime.createdAt) {+ removeCaptureHandler();+ if (currentJob != null) {+ await this.reconcileFencedRuntimeHandoff(streamId, runtime, currentJob);+ }+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ }+ if (winningOutcome === 'not_required') {+ discardCapturedEvents();+ return { subscription: null, resumeState, pendingEvents: [] };+ }+ if (winningOutcome !== 'success') {+ discardCapturedEvents();+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ /** Another replica proved the generation recoverable, but this+ * attachment's snapshot is still incomplete. Keep the generation+ * healthy and fail this request closed so its normal retry can read+ * the durable winner's now-established frontier. */+ throw new Error('Concurrent recovery succeeded; retry this attachment');+ }+ }+if (options?.signal?.aborted) {removeCaptureHandler();recordGenerationStreamSubscription(this.storeLabel, 'resume', 'error');@@ -5580,6 +6219,25 @@ class GenerationJobManagerClass {if (!liveJob || liveJob.createdAt !== runtime.createdAt) {return cancelResumeSubscription();}+ const liveOverflow = liveJob.earlyBufferOverflow;+ if (liveOverflow != null) {+ runtime.earlyBufferOverflow = liveOverflow;+ runtime.earlyEventBufferOverflowed = true;+ runtime.earlyEventBufferClosed = true;+ this.resetEarlyEventBuffer(runtime);+ if (+ !runtime.finalEvent &&+ !runtime.errorEvent &&+ (liveOverflow.recoveryOutcome === 'failed' || snapshotOverflowId !== liveOverflow.id)+ ) {+ /** The owner can discard its local buffer before the durable marker+ * write becomes visible here. This final pre-activation read closes+ * that window: retire the paused attachment and rebuild its snapshot+ * through the durable-only recovery path now reflected in `runtime`. */+ cancelResumeSubscription();+ return this.subscribeWithResume(streamId, onChunk, onDone, onError, options);+ }+ }if (!resumeState?.pendingAction) {if (liveJob?.status === 'requires_action' &&@@ -5660,7 +6318,10 @@ class GenerationJobManagerClass {(resumeState.aggregatedContent ?? []) as SteerContentView,liveQueue,(content ?? []) as SteerContentView,- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },+ {+ conversationId: streamId,+ responseMessageId: resumeState.responseMessageId,+ },);if (gapEvents.length > 0) {pendingEvents.push(...gapEvents);@@ -5696,8 +6357,12 @@ class GenerationJobManagerClass {const snapshotHasReasoningLabels =resumeState?.aggregatedContent?.some((part) =>- (part as { type?: string; reasoning_label_revision?: unknown } | null)?.type ===- 'think' &&+ (+ part as {+ type?: string;+ reasoning_label_revision?: unknown;+ } | null+ )?.type === 'think' &&typeof (part as { reasoning_label_revision?: unknown }).reasoning_label_revision ==='number',) === true;@@ -5718,7 +6383,10 @@ class GenerationJobManagerClass {typeof synthesizeActivityLabelGapEvents>[0],labelContent as Parameters<typeof synthesizeActivityLabelGapEvents>[1],- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },+ {+ conversationId: streamId,+ responseMessageId: resumeState.responseMessageId,+ },);if (labelGapEvents.length > 0) {pendingEvents.push(...(labelGapEvents as t.ServerSentEvent[]));@@ -5728,7 +6396,10 @@ class GenerationJobManagerClass {typeof synthesizeReasoningLabelGapEvents>[0],labelContent as Parameters<typeof synthesizeReasoningLabelGapEvents>[1],- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },+ {+ conversationId: streamId,+ responseMessageId: resumeState.responseMessageId,+ },);if (reasoningGapEvents.length > 0) {pendingEvents.push(...(reasoningGapEvents as t.ServerSentEvent[]));@@ -5775,6 +6446,51 @@ class GenerationJobManagerClass {subscription?.unsubscribe();restoreCapturedEvents();snapshotCoveredEventSet.clear();+ if (pendingOverflow != null && recoveryStartedAt != null && !recoverySettled) {+ const winningOutcome = await this.settleEarlyBufferRecovery(+ streamId,+ runtime,+ pendingOverflow,+ 'failed',+ Date.now() - recoveryStartedAt,+ 0,+ 0,+ 'reconstruction_error',+ ).catch((settlementError) => {+ logger.error(+ '[GenerationJobManager] Failed to settle early buffer recovery error',+ settlementError,+ );+ return undefined;+ });+ if (winningOutcome == null) {+ const currentJob = await this.jobStore.getJob(streamId);+ if (currentJob?.createdAt !== runtime.createdAt) {+ if (currentJob != null) {+ await this.reconcileFencedRuntimeHandoff(streamId, runtime, currentJob);+ }+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ }+ if (winningOutcome === 'not_required') {+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ if (winningOutcome !== 'success') {+ await this.completeJob(+ streamId,+ GENERATION_RECOVERY_FAILED_ERROR,+ runtime.createdAt,+ ).catch((terminalError) => {+ logger.error(+ '[GenerationJobManager] Failed to terminalize early buffer recovery error',+ terminalError,+ );+ });+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);+ return { subscription: null, resumeState: null, pendingEvents: [] };+ }+ }throw err;}}@@ -6004,11 +6720,24 @@ class GenerationJobManagerClass {options?.deliveredSteer == null &&isCoalescableDeltaEvent(eventType);+ const detachedAtAppend = !runtime.hasSubscriber;+ const admissionFenceRequired = detachedAtAppend && !runtime.everHadSubscriber;+ const subscriberAdmission = admissionFenceRequired+ ? this.jobStore.hasSubscriberAttached(streamId, runtime.createdAt).catch((error) => {+ logger.warn(+ '[GenerationJobManager] Failed to read generation-wide subscriber admission',+ error,+ );+ return false;+ })+ : undefined;+// For Redis mode, persist chunk for later reconstruction (fire-and-forget for resumability)if (this._isRedis) {// The SSE event structure is { event: string, data: unknown, ... }// The aggregator expects { event: string, data: unknown } where data is the payloadif (eventType && eventData !== undefined) {+ runtime.durableEventSequence += 1;// Store in format expected by aggregateContent: { event, data }const appendPromise = this.jobStore.appendChunk(streamId,@@ -6018,18 +6747,23 @@ class GenerationJobManagerClass {coalescableDelta ? { coalesce: true } : undefined,);- if (options?.durable === true) {+ /** Before the generation-wide first-subscriber admission, publication+ * is the cross-replica boundary: its durable record must already exist+ * before another replica can snapshot. The admission probe runs beside+ * the append, so the first event after a remote attachment remains+ * fenced and later events recover the normal fire-and-forget path. */+ if (options?.durable === true || admissionFenceRequired) {let appended: boolean;try {appended = await appendPromise;} catch (error) {- if (options.deliveredSteer == null) {+ if (options?.deliveredSteer == null) {this.retireRuntimeAfterDurableFence(streamId, runtime);}throw error;}if (appended === false) {- if (options.deliveredSteer == null) {+ if (options?.deliveredSteer == null) {this.retireRuntimeAfterDurableFence(streamId, runtime);}throw new Error(`Durable chunk append was fenced out for ${streamId}`);@@ -6074,8 +6808,12 @@ class GenerationJobManagerClass {}}+ if ((await subscriberAdmission) === true) {+ runtime.everHadSubscriber = true;+ }+const detached = !runtime.hasSubscriber;- const buffered = detached && this.bufferEarlyEvent(streamId, runtime, event);+ const buffered = detached && (await this.bufferEarlyEvent(streamId, runtime, event));if (detached && !this._isRedis) {if (runtime.startupTelemetry) {this.recordStartupEvent(runtime, event);@@ -6471,6 +7209,7 @@ class GenerationJobManagerClass {this.jobStore.clearContentState(streamId, runtime.createdAt);if (this.runtimeState.get(streamId) === runtime) {+ this.stopSubscriberLease(runtime);this.runtimeState.delete(streamId);this.runStepBuffers?.delete(streamId);this.replayEventWriteQueues.delete(streamId);@@ -6541,7 +7280,9 @@ class GenerationJobManagerClass {this.cleanupFencedRuntime(streamId, runtime);return;}- const retirement: FencedRuntimeRetirementContext = { controller: new AbortController() };+ const retirement: FencedRuntimeRetirementContext = {+ controller: new AbortController(),+ };this.fencedRuntimeRetirements.set(runtime, retirement);this.scheduleFencedRuntimeRetirement(streamId, runtime, Date.now(), retirement);}@@ -6727,7 +7468,10 @@ class GenerationJobManagerClass {} as ToolCallWithExecutionStatus): { ...existingCall, ...completedToolCall, executionStatus };} else {- completedCalls.push({ ...completedToolCall, executionStatus } as ToolCallWithExecutionStatus);+ completedCalls.push({+ ...completedToolCall,+ executionStatus,+ } as ToolCallWithExecutionStatus);}this.accumulateRunStep(@@ -6929,7 +7673,9 @@ class GenerationJobManagerClass {}tokenUsage.push(event.data);- const update: Partial<SerializableJobData> = { tokenUsage: JSON.stringify(tokenUsage) };+ const update: Partial<SerializableJobData> = {+ tokenUsage: JSON.stringify(tokenUsage),+ };/** Reconcile the resume snapshot to this call's ACTUAL prompt tokens. A primary* usage is the post-invoke truth for the call the latest stored snapshot@@ -7040,7 +7786,9 @@ class GenerationJobManagerClass {// its pills — this is the authoritative writer of job.metadata.userMessage and// would otherwise drop them (the emitted created message includes them)....(Array.isArray(extra.manualSkills) &&- extra.manualSkills.length > 0 && { manualSkills: extra.manualSkills }),+ extra.manualSkills.length > 0 && {+ manualSkills: extra.manualSkills,+ }),...(Array.isArray(extra.alwaysAppliedSkills) &&extra.alwaysAppliedSkills.length > 0 && {alwaysAppliedSkills: extra.alwaysAppliedSkills,@@ -7541,7 +8289,11 @@ class GenerationJobManagerClass {*/const publish = (): Promise<void> =>Promise.resolve().then(async () => {- await this.eventTransport.emitPreempt?.(streamId, { op: 'clear', createdAt, steerIds });+ await this.eventTransport.emitPreempt?.(streamId, {+ op: 'clear',+ createdAt,+ steerIds,+ });});return publish().then(() => true,@@ -7604,7 +8356,9 @@ class GenerationJobManagerClass {conversationId: streamId,steers: downgraded.map((steer) => ({steerId: steer.steerId,- ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),+ ...(steer.clientSteerId && {+ clientSteerId: steer.clientSteerId,+ }),preempt: false,preemptRevision: steer.preemptRevision ?? 0,})),@@ -7709,21 +8463,112 @@ class GenerationJobManagerClass {async getResumeState(streamId: string,expectedCreatedAt?: number,+ options?: {+ durableOnly?: boolean;+ validateEarlyBufferRecovery?: boolean;+ onContentSnapshot?: (snapshot: Awaited<ReturnType<IJobStore['getContentParts']>>) => void;+ },): Promise<t.ResumeState | null> {- const jobData = await this.jobStore.getJob(streamId);+ const recoveryStartedAt =+ options?.validateEarlyBufferRecovery === true ? Date.now() : undefined;+ let jobData = await this.jobStore.getJob(streamId);if (!jobData || (expectedCreatedAt != null && jobData.createdAt !== expectedCreatedAt)) {return null;}+ if (jobData.earlyBufferOverflow?.persistencePending === true) {+ const initialCreatedAt = jobData.createdAt;+ jobData = await this.waitForFinalizedEarlyBufferOverflow(streamId, jobData);+ if (!jobData || jobData.createdAt !== initialCreatedAt) {+ return null;+ }+ if (+ jobData.earlyBufferOverflow?.persistencePending === true &&+ options?.validateEarlyBufferRecovery !== true+ ) {+ return null;+ }+ }+ const unresolvedOverflowValidation =+ options?.validateEarlyBufferRecovery === true &&+ jobData.earlyBufferOverflow != null &&+ jobData.earlyBufferOverflow.recoveryOutcome == null;+ const validationRuntime =+ options?.validateEarlyBufferRecovery === true+ ? await this.getOrCreateRuntimeState(streamId, jobData)+ : undefined;+ if (+ options?.validateEarlyBufferRecovery === true &&+ (validationRuntime == null || validationRuntime.createdAt !== jobData.createdAt)+ ) {+ throw new Error('Generation changed during HITL recovery validation');+ }+ if (+ options?.validateEarlyBufferRecovery === true &&+ jobData.earlyBufferOverflow?.recoveryOutcome === 'failed'+ ) {+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, jobData.createdAt);+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR);+ }/** Independent reads (streamId-only): parallel to collapse 3 Redis round trips into 1.* Safe despite readCachedGraph's cache-drop side effect — each call catches its own* unusable-graph throw and falls back to reconstruction, so ordering cannot change the result. */const [result, runSteps, queuedSteers, claimedSteers] = await Promise.all([- this.jobStore.getContentParts(streamId, jobData.createdAt),+ this.jobStore.getContentParts(streamId, jobData.createdAt, {+ durableOnly: options?.durableOnly === true || unresolvedOverflowValidation,+ }),this.jobStore.getRunSteps(streamId, jobData.createdAt),this.jobStore.peekSteers(streamId, jobData.createdAt),this.jobStore.peekClaimedSteers(streamId, jobData.createdAt),]);+ options?.onContentSnapshot?.(result);+ const pendingOverflow = unresolvedOverflowValidation ? jobData.earlyBufferOverflow : undefined;+ if (pendingOverflow != null) {+ const runtime = validationRuntime!;+ const reconstructedEvents = this._isRedis+ ? (result?.reconstructedEventCount ?? 0)+ : runtime.emissionSequence;+ const durableEvents = this._isRedis+ ? (result?.durableEventCount ?? 0)+ : runtime.emissionSequence;+ let failureReason: EarlyBufferRecoveryFailureReason | undefined;+ if (pendingOverflow.persistencePending === true) {+ failureReason = 'overflow_marker_persistence_failed';+ } else if (result == null) {+ failureReason = 'durable_state_missing';+ } else if (+ durableEvents < pendingOverflow.durableEvents ||+ reconstructedEvents !== durableEvents+ ) {+ failureReason = 'durable_frontier_gap';+ }+ const winningOutcome = await this.settleEarlyBufferRecovery(+ streamId,+ runtime,+ pendingOverflow,+ failureReason == null ? 'success' : 'failed',+ recoveryStartedAt == null ? 0 : Date.now() - recoveryStartedAt,+ reconstructedEvents,+ result?.content.length ?? 0,+ failureReason,+ );+ if (winningOutcome == null) {+ const currentJob = await this.jobStore.getJob(streamId);+ if (currentJob?.createdAt !== jobData.createdAt) {+ throw new Error('Generation changed during HITL recovery validation');+ }+ }+ if (failureReason != null && winningOutcome === 'success') {+ throw new Error('Concurrent recovery succeeded; retry HITL resume');+ }+ if (winningOutcome === 'not_required') {+ throw new Error('HITL recovery no longer required for this generation');+ }+ if (winningOutcome !== 'success') {+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR);+ }+ }const reconstructedContent = result?.content ?? [];const bufferState = this.runStepBuffers?.get(streamId);const bufferedRunSteps = bufferState?.createdAt === jobData.createdAt ? bufferState.steps : [];@@ -8263,6 +9108,7 @@ class GenerationJobManagerClass {currentJob,observedRuntime,);+ this.stopSubscriberLease(observedRuntime);this.runtimeState.delete(streamId);this.runStepBuffers?.delete(streamId);this.replayEventWriteQueues.delete(streamId);@@ -8320,6 +9166,7 @@ class GenerationJobManagerClass {observedRuntime.startupTelemetry?.end('error', new Error(REAPED_JOB_ERROR));observedRuntime.startupTelemetry = undefined;this.releaseAbortSubscription(observedRuntime);+ this.stopSubscriberLease(observedRuntime);this.runtimeState.delete(streamId);runningJobsChanged = this.ownedJobs.delete(streamId) || runningJobsChanged;this.runStepBuffers?.delete(streamId);@@ -8552,6 +9399,7 @@ class GenerationJobManagerClass {for (const runtime of this.runtimeState.values()) {runtime.startupTelemetry?.end('aborted');runtime.startupTelemetry = undefined;+ this.stopSubscriberLease(runtime);this.releaseAbortSubscription(runtime, true);runtime.abortController.abort();}@@ -8572,6 +9420,7 @@ class GenerationJobManagerClass {this.replayEventWriteQueues.clear();this.tokenUsageWriteQueues.clear();this.runStepWriteQueues.clear();+ this.earlyBufferOverflowWaiters.clear();logger.debug('[GenerationJobManager] Destroyed');}diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.tsindex 868dcab480..90c64e626f 100644--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts@@ -1,15 +1,17 @@/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */import type { Redis, Cluster } from 'ioredis';import type { ServerSentEvent, StreamEvent, CreatedEvent } from '~/types';+import type { IJobStoreV2 } from '~/stream/interfaces/IJobStore';+import {+ GenerationJobManagerClass,+ GENERATION_RECOVERY_FAILED_ERROR,+ TERMINAL_PUBLICATION_RECONNECT_ERROR,+} from '~/stream/GenerationJobManager';import {ioredisClient as staticRedisClient,keyvRedisClient as staticKeyvClient,keyvRedisClientReady,} from '~/cache/redisClients';-import {- GenerationJobManagerClass,- TERMINAL_PUBLICATION_RECONNECT_ERROR,-} from '~/stream/GenerationJobManager';import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';@@ -128,11 +130,19 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_run_step',- data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },+ data: {+ id: 'step-1',+ runId: 'run-1',+ index: 0,+ stepDetails: { type: 'message_creation' },+ },});await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: 'Hello' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'Hello' } },+ },});await new Promise((resolve) => setTimeout(resolve, delay));@@ -143,7 +153,10 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: ' world' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: ' world' } },+ },});await manager.emitChunk(streamId, {event: 'on_message_delta',@@ -185,7 +198,9 @@ describe('GenerationJobManager Integration Tests', () => {expect(retrieved?.streamId).toBe(streamId);// Update job- await GenerationJobManager.updateMetadata(streamId, { sender: 'TestAgent' });+ await GenerationJobManager.updateMetadata(streamId, {+ sender: 'TestAgent',+ });const updated = await GenerationJobManager.getJob(streamId);expect(updated?.metadata?.sender).toBe('TestAgent');@@ -269,7 +284,9 @@ describe('GenerationJobManager Integration Tests', () => {expect(hasJob).toBe(true);// Update and verify- await GenerationJobManager.updateMetadata(streamId, { sender: 'RedisAgent' });+ await GenerationJobManager.updateMetadata(streamId, {+ sender: 'RedisAgent',+ });const updated = await GenerationJobManager.getJob(streamId);expect(updated?.metadata?.sender).toBe('RedisAgent');@@ -1049,12 +1066,18 @@ describe('GenerationJobManager Integration Tests', () => {await GenerationJobManager.emitChunk(streamId, {event: 'on_run_step_delta',- data: { id: 'step-1', delta: { type: 'tool_calls', tool_calls: [{ args: '{' }] } },+ data: {+ id: 'step-1',+ delta: { type: 'tool_calls', tool_calls: [{ args: '{' }] },+ },});await GenerationJobManager.emitChunk(streamId, {event: 'on_run_step_delta',- data: { id: 'step-1', delta: { type: 'tool_calls', tool_calls: [{ args: '}' }] } },+ data: {+ id: 'step-1',+ delta: { type: 'tool_calls', tool_calls: [{ args: '}' }] },+ },});await GenerationJobManager.emitChunk(streamId, {@@ -1116,10 +1139,16 @@ describe('GenerationJobManager Integration Tests', () => {const emitPromises: Promise<void>[] = [];for (let i = 0; i < 10; i++) {emitPromises.push(- GenerationJobManager.emitChunk(streamId1, { event: 'test', data: { index: i } }),+ GenerationJobManager.emitChunk(streamId1, {+ event: 'test',+ data: { index: i },+ }),);emitPromises.push(- GenerationJobManager.emitChunk(streamId2, { event: 'test', data: { index: i * 100 } }),+ GenerationJobManager.emitChunk(streamId2, {+ event: 'test',+ data: { index: i * 100 },+ }),);}await Promise.all(emitPromises);@@ -1275,7 +1304,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 10; i++) {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `word${i} ` } }, index: i },+ data: {+ delta: { content: { type: 'text', text: `word${i} ` } },+ index: i,+ },});}@@ -1359,7 +1391,10 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: ' Live!' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: ' Live!' } },+ },});await new Promise((resolve) => setTimeout(resolve, 20));@@ -1403,7 +1438,12 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_run_step',- data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },+ data: {+ id: 'step-1',+ runId: 'run-1',+ index: 0,+ stepDetails: { type: 'message_creation' },+ },});await new Promise((resolve) => setTimeout(resolve, 10));@@ -1412,7 +1452,10 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: 'buffered' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'buffered' } },+ },});const sub2Events: ServerSentEvent[] = [];@@ -1580,7 +1623,10 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: ' Live!' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: ' Live!' } },+ },});await new Promise((resolve) => setTimeout(resolve, 200));@@ -1619,7 +1665,10 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: 'detached-redis' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'detached-redis' } },+ },});await new Promise((resolve) => setTimeout(resolve, 100));@@ -1636,7 +1685,10 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: ' live' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: ' live' } },+ },});await new Promise((resolve) => setTimeout(resolve, 200));@@ -1657,6 +1709,22 @@ describe('GenerationJobManager Integration Tests', () => {* detached run.*/+ async function forceEarlyBufferOverflow(+ manager: GenerationJobManagerClass,+ streamId: string,+ ): Promise<void> {+ const runtime = (+ manager as unknown as {+ runtimeState: Map<string, unknown>;+ }+ ).runtimeState.get(streamId)!;+ await (+ manager as unknown as {+ overflowEarlyEventBuffer: (id: string, runtime: unknown) => Promise<void>;+ }+ ).overflowEarlyEventBuffer(streamId, runtime);+ }+testRedis('detached generation keeps the local buffer empty after first attachment (Redis)',async () => {@@ -1696,17 +1764,26 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 5; i++) {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: bigText } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: bigText } },+ },});}const stats = manager.getRuntimeStats();expect(stats.earlyBufferedEvents).toBe(0);expect(stats.earlyBufferedBytes).toBe(0);+ const overflow = (await manager.getJob(streamId))?.metadata.earlyBufferOverflow;+ expect(overflow?.droppedEvents).toBe(4);+ expect(overflow?.droppedBytes).toBeGreaterThan(8 * 1024 * 1024);await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: 'after-overflow' } } },+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'after-overflow' } },+ },});expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);@@ -1730,64 +1807,84 @@ describe('GenerationJobManager Integration Tests', () => {await manager.destroy();});- testRedis('redirects a post-overflow first attachment to resume recovery (Redis)', async () => {- const manager = createRedisManager();- const streamId = `overflow-redirect-${Date.now()}`;- await manager.createJob(streamId, 'user-1');-- await manager.emitChunk(streamId, {- event: 'on_run_step',- data: {- id: 'step-1',- runId: 'run-1',- index: 0,- stepDetails: { type: 'message_creation' },- },+ test('replays a completed result after the overflow reconnect was forced', async () => {+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,});- const bigText = 'y'.repeat(2 * 1024 * 1024);- for (let i = 0; i < 5; i++) {- await manager.emitChunk(streamId, {- event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: bigText } } },- });- }- expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);+ manager.initialize();+ const streamId = `overflow-complete-reconnect-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(manager, streamId);- const errors: string[] = [];- const sub = await manager.subscribe(+ const reconnectErrors: string[] = [];+ await manager.subscribe(streamId,() => {},undefined,- (error) => errors.push(error),+ (error) => reconnectErrors.push(error),);- await new Promise((resolve) => setTimeout(resolve, 100));- expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);- sub?.unsubscribe();- await new Promise((resolve) => setTimeout(resolve, 100));+ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(reconnectErrors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);- /** The resume path the client falls back to reconstructs the- * discarded output from the durable chunk log. */- const resumeState = await manager.getResumeState(streamId);- expect(JSON.stringify(resumeState?.aggregatedContent ?? [])).toContain('yyyy');+ const finalEvent = {+ final: true,+ conversation: { conversationId: streamId },+ responseMessage: { text: 'complete' },+ } as ServerSentEvent;+ await manager.emitDone(streamId, finalEvent);+ await manager.completeJob(streamId);+ const terminalEvents: ServerSentEvent[] = [];+ const resumed = await manager.subscribeWithResume(+ streamId,+ () => {},+ (event) => terminalEvents.push(event),+ );+ resumed.subscription?.activate();+ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(terminalEvents).toEqual([finalEvent]);+ resumed.subscription?.unsubscribe();await manager.destroy();});- test('buffers detached events until the cap in in-memory mode', async () => {- const manager = createInMemoryManager();- const streamId = `buf-below-cap-${Date.now()}`;+ test('reconciles a generation replaced while recovery settles', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `overflow-replaced-during-settlement-${Date.now()}`;await manager.createJob(streamId, 'user-1');+ const originalCreatedAt = (await jobStore.getJob(streamId))!.createdAt;+ await forceEarlyBufferOverflow(manager, streamId);+ jest.spyOn(jobStore, 'settleEarlyBufferRecovery').mockImplementationOnce(async () => {+ await jobStore.createJob(streamId, 'user-1');+ return false;+ });- await setupDisconnectedStream(manager, streamId, 10);-- const stats = manager.getRuntimeStats();- expect(stats.earlyBufferedEvents).toBe(2);- expect(stats.earlyBufferedBytes).toBeGreaterThan(0);+ const errors: string[] = [];+ const result = await manager.subscribeWithResume(+ streamId,+ () => {},+ undefined,+ (error) => errors.push(error),+ { expectedCreatedAt: originalCreatedAt },+ );+ expect(result.subscription).toBeNull();+ expect(errors).not.toContain(GENERATION_RECOVERY_FAILED_ERROR);+ expect((await jobStore.getJob(streamId))!.createdAt).toBeGreaterThan(originalCreatedAt);await manager.destroy();});- test('caps captured events restored by a resume canceled before activation', async () => {+ test('reconciles a generation replaced while a recovery error settles', async () => {const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });const manager = new GenerationJobManagerClass();manager.configure({@@ -1796,150 +1893,1047 @@ describe('GenerationJobManager Integration Tests', () => {isRedis: false,});manager.initialize();- const streamId = `restore-cap-${Date.now()}`;+ const streamId = `overflow-replaced-during-error-settlement-${Date.now()}`;await manager.createJob(streamId, 'user-1');-- await setupDisconnectedStream(manager, streamId, 10);-- /** Arm only after the snapshot completes so the gate parks the resume- * in its post-attachment steer reconciliation, the window where- * emissions are captured per-resume instead of buffered. */- let armed = false;- const originalGetResumeState = manager.getResumeState.bind(manager);+ const originalCreatedAt = (await jobStore.getJob(streamId))!.createdAt;+ await forceEarlyBufferOverflow(manager, streamId);jest.spyOn(manager, 'getResumeState')- .mockImplementation(- async (...args: Parameters<GenerationJobManagerClass['getResumeState']>) => {- const result = await originalGetResumeState(...args);- armed = true;- return result;- },- );- let releaseGate!: () => void;- const gate = new Promise<void>((resolve) => (releaseGate = resolve));- let gateReached!: () => void;- const reached = new Promise<void>((resolve) => (gateReached = resolve));- const originalPeek = jobStore.peekSteers.bind(jobStore);- let gated = true;- jest- .spyOn(jobStore, 'peekSteers')- .mockImplementation(async (...args: Parameters<InMemoryJobStore['peekSteers']>) => {- if (armed && gated) {- gated = false;- gateReached();- await gate;- }- return originalPeek(...args);- });-- const resumePromise = manager.subscribeWithResume(streamId, () => {});-- await reached;- const bigText = 'z'.repeat(2 * 1024 * 1024);- for (let i = 0; i < 5; i++) {- await manager.emitChunk(streamId, {- event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: bigText } } },- });- }- releaseGate();-- const { subscription } = await resumePromise;- expect(subscription).not.toBeNull();- subscription!.unsubscribe();-- /** The ~10MB of captured events must not survive restoration. */- const stats = manager.getRuntimeStats();- expect(stats.earlyBufferedEvents).toBe(0);- expect(stats.earlyBufferedBytes).toBe(0);+ .mockRejectedValueOnce(new Error('snapshot unavailable'));+ jest.spyOn(jobStore, 'settleEarlyBufferRecovery').mockImplementationOnce(async () => {+ await jobStore.createJob(streamId, 'user-1');+ return false;+ });- /** Restoration overflowed, so a non-resume attach takes the redirect. */const errors: string[] = [];- const probe = await manager.subscribe(+ const result = await manager.subscribeWithResume(streamId,() => {},undefined,(error) => errors.push(error),+ { expectedCreatedAt: originalCreatedAt },);- await new Promise((resolve) => setTimeout(resolve, 20));- expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);- probe?.unsubscribe();+ expect(result.subscription).toBeNull();+ expect(errors).not.toContain(GENERATION_RECOVERY_FAILED_ERROR);+ expect((await jobStore.getJob(streamId))!.createdAt).toBeGreaterThan(originalCreatedAt);await manager.destroy();});- });-- describe('Atomic subscribeWithResume', () => {- test('should return empty pendingEvents for pre-snapshot buffer events (in-memory)', async () => {- const manager = createInMemoryManager();- const streamId = `atomic-drain-${Date.now()}`;- await manager.createJob(streamId, 'user-1');-- const sub1 = await manager.subscribe(streamId, () => {});- await new Promise((resolve) => setTimeout(resolve, 10));- sub1?.unsubscribe();- await new Promise((resolve) => setTimeout(resolve, 20));- await manager.emitChunk(streamId, {- event: 'on_run_step',- data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },+ test('does not overwrite recovery settlement when the overflow owner finishes flushing', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,});- await manager.emitChunk(streamId, {- event: 'on_message_delta',- data: { id: 'step-1', delta: { content: { type: 'text', text: 'buffered' } } },+ manager.initialize();+ const streamId = `overflow-finalize-settlement-race-${Date.now()}`;+ const job = await manager.createJob(streamId, 'user-1');+ const originalFinalize = jobStore.finalizeEarlyBufferOverflow.bind(jobStore);+ jest+ .spyOn(jobStore, 'finalizeEarlyBufferOverflow')+ .mockImplementationOnce(async (id, createdAt, overflowId, finalizedOverflow) => {+ await jobStore.settleEarlyBufferRecovery(id, createdAt, overflowId, {+ recoveryMethod: 'snapshot',+ recoveryOutcome: 'failed',+ recoveryCompletedAt: Date.now(),+ recoveryFailureReason: 'overflow_marker_persistence_failed',+ });+ return originalFinalize(id, createdAt, overflowId, finalizedOverflow);+ });++ await expect(forceEarlyBufferOverflow(manager, streamId)).rejects.toThrow(+ GENERATION_RECOVERY_FAILED_ERROR,+ );+ expect((await jobStore.getJob(streamId))?.earlyBufferOverflow).toMatchObject({+ recoveryOutcome: 'failed',+ recoveryFailureReason: 'overflow_marker_persistence_failed',});+ expect((await jobStore.getJob(streamId))?.createdAt).toBe(job.createdAt);- const liveEvents: ServerSentEvent[] = [];- const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(+ await manager.destroy();+ });++ test('coalesces pending overflow waiters and backs off their store read', async () => {+ jest.useFakeTimers();+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({ jobStore, eventTransport: new InMemoryEventTransport(), isRedis: false });+ manager.initialize();+ const streamId = `overflow-shared-waiter-${Date.now()}`;+ const job = await manager.createJob(streamId, 'user-1');+ const pendingOverflow = {+ id: 'pending-overflow',+ occurredAt: Date.now(),+ durableEvents: 1,+ droppedEvents: 1,+ droppedBytes: 1,+ persistencePending: true,+ };+ await jobStore.updateJob(streamId, { earlyBufferOverflow: pendingOverflow }, job.createdAt);+ const pendingJob = (await jobStore.getJob(streamId))!;+ const getJob = jest.spyOn(jobStore, 'getJob');+ const waitForFinalized = (+ manager as unknown as {+ waitForFinalizedEarlyBufferOverflow: (+ id: string,+ data: typeof pendingJob,+ ) => Promise<typeof pendingJob | null>;+ }+ ).waitForFinalizedEarlyBufferOverflow.bind(manager);++ const first = waitForFinalized(streamId, pendingJob);+ const second = waitForFinalized(streamId, pendingJob);+ await jest.advanceTimersByTimeAsync(99);+ expect(getJob).not.toHaveBeenCalled();+ await jobStore.updateJob(streamId,- (event) => liveEvents.push(event),+ { earlyBufferOverflow: { ...pendingOverflow, persistencePending: false } },+ job.createdAt,);+ await jest.advanceTimersByTimeAsync(1);- expect(resumeState).not.toBeNull();- expect(pendingEvents.length).toBe(0);- expect(liveEvents.length).toBe(0);+ await expect(Promise.all([first, second])).resolves.toEqual([+ expect.objectContaining({+ earlyBufferOverflow: expect.objectContaining({ persistencePending: false }),+ }),+ expect.objectContaining({+ earlyBufferOverflow: expect.objectContaining({ persistencePending: false }),+ }),+ ]);+ expect(getJob).toHaveBeenCalledTimes(1);- subscription?.unsubscribe();await manager.destroy();+ jest.useRealTimers();});- test('should return empty pendingEvents when buffer is empty', async () => {- const manager = createInMemoryManager();- const streamId = `atomic-empty-${Date.now()}`;+ test('fences publication only until generation-wide subscriber admission', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const eventTransport = new InMemoryEventTransport();+ let releaseAppend!: (appended: boolean) => void;+ const pendingAppend = new Promise<boolean>((resolve) => {+ releaseAppend = resolve;+ });+ const append = jest.spyOn(jobStore, 'appendChunk').mockReturnValueOnce(pendingAppend);+ jest.spyOn(jobStore, 'hasSubscriberAttached').mockResolvedValue(true);+ const publish = jest.spyOn(eventTransport, 'emitChunk');+ const manager = new GenerationJobManagerClass();+ manager.configure({ jobStore, eventTransport, isRedis: true });+ manager.initialize();+ const streamId = `detached-append-before-publish-${Date.now()}`;await manager.createJob(streamId, 'user-1');- const sub1 = await manager.subscribe(streamId, () => {});- await new Promise((resolve) => setTimeout(resolve, 10));-- await manager.emitChunk(streamId, {+ const emission = manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: 'delivered' } } },+ data: { delta: { content: { type: 'text', text: 'durable first' } } },});- await new Promise((resolve) => setTimeout(resolve, 10));+ await Promise.resolve();+ expect(publish).not.toHaveBeenCalled();+ releaseAppend(true);+ await emission;+ expect(publish).toHaveBeenCalledTimes(1);- sub1?.unsubscribe();- await new Promise((resolve) => setTimeout(resolve, 20));-- const { pendingEvents } = await manager.subscribeWithResume(streamId, () => {});-- expect(pendingEvents.length).toBe(0);+ append.mockReturnValueOnce(new Promise<boolean>(() => {}));+ await expect(+ manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'publish without append wait' } } },+ }),+ ).resolves.toBeUndefined();+ expect(publish).toHaveBeenCalledTimes(2);+ expect(jobStore.hasSubscriberAttached).toHaveBeenCalledTimes(1);await manager.destroy();});- test('should defer live events until a resumed subscription is activated', async () => {+ test('validates an in-memory HITL snapshot with the event frontier', async () => {const manager = createInMemoryManager();- const streamId = `atomic-live-${Date.now()}`;+ const streamId = `overflow-hitl-memory-${Date.now()}`;await manager.createJob(streamId, 'user-1');+ const createdAt = (await manager.getJob(streamId))!.createdAt;+ await forceEarlyBufferOverflow(manager, streamId);+ manager.setContentParts(+ streamId,+ [{ type: 'text', text: 'complete snapshot' }] as never,+ createdAt,+ );+ const getRuntimeSpy = jest.spyOn(+ manager as unknown as {+ getOrCreateRuntimeState: (...args: unknown[]) => Promise<unknown>;+ },+ 'getOrCreateRuntimeState',+ );- const sub1 = await manager.subscribe(streamId, () => {});+ await expect(+ manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),+ ).resolves.toMatchObject({ aggregatedContent: expect.any(Array) });+ expect(getRuntimeSpy).toHaveBeenCalledWith(streamId, expect.objectContaining({ createdAt }));+ expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).toMatchObject({+ durableEvents: 0,+ recoveryOutcome: 'success',+ });++ await manager.destroy();+ });++ test('rejects a durable failed outcome before exposing HITL state', async () => {+ const manager = createInMemoryManager();+ const streamId = `overflow-hitl-failed-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ const createdAt = (await manager.getJob(streamId))!.createdAt;+ await forceEarlyBufferOverflow(manager, streamId);+ const overflow = (await manager.getJob(streamId))!.metadata.earlyBufferOverflow!;+ await (manager.getJobStore() as IJobStoreV2).settleEarlyBufferRecovery(+ streamId,+ createdAt,+ overflow.id,+ {+ recoveryMethod: 'snapshot',+ recoveryOutcome: 'failed',+ recoveryCompletedAt: Date.now(),+ recoveryFailureReason: 'snapshot_missing',+ },+ );++ await expect(+ manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),+ ).rejects.toThrow(GENERATION_RECOVERY_FAILED_ERROR);++ await manager.destroy();+ });++ test('contains subscriber lease lookup failure during terminal cleanup', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ manager.initialize();+ const streamId = `overflow-lease-read-failure-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(manager, streamId);+ jest.spyOn(jobStore, 'hasActiveSubscriber').mockRejectedValueOnce(new Error('unavailable'));++ await expect(manager.completeJob(streamId)).resolves.toBe(true);+ expect((await manager.getJob(streamId))?.status).toBe('complete');+ expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(+ 'recoveryOutcome',+ );++ await manager.destroy();+ });++ test('leaves recovery unresolved when terminal lifecycle lookup fails', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ manager.initialize();+ const streamId = `overflow-lifecycle-read-failure-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(manager, streamId);+ const originalGetJob = jobStore.getJob.bind(jobStore);+ jest+ .spyOn(jobStore, 'getJob')+ .mockImplementationOnce(originalGetJob)+ .mockRejectedValueOnce(new Error('unavailable'))+ .mockImplementation(originalGetJob);++ await expect(manager.completeJob(streamId)).resolves.toBe(true);+ expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(+ 'recoveryOutcome',+ );++ await manager.destroy();+ });++ test('removes in-memory capture handlers when another attachment wins recovery', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ cleanupOnComplete: false,+ });+ manager.initialize();+ const streamId = `overflow-recovery-loser-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(manager, streamId);+ const originalSettle = jobStore.settleEarlyBufferRecovery.bind(jobStore);+ jest+ .spyOn(jobStore, 'settleEarlyBufferRecovery')+ .mockImplementationOnce(async (id, createdAt, recoveryId) => {+ await originalSettle(id, createdAt, recoveryId, {+ recoveryMethod: 'snapshot',+ recoveryOutcome: 'not_required',+ recoveryCompletedAt: Date.now(),+ });+ return false;+ });++ const result = await manager.subscribeWithResume(streamId, () => {});++ expect(result.subscription).toBeNull();+ const runtime = (+ manager as unknown as {+ runtimeState: Map<string, { resumeCaptureHandlers: Set<unknown> }>;+ }+ ).runtimeState.get(streamId);+ expect(runtime?.resumeCaptureHandlers.size).toBe(0);++ await manager.destroy();+ });++ testRedis('redirects a post-overflow first attachment to resume recovery (Redis)', async () => {+ const manager = createRedisManager();+ const streamId = `overflow-redirect-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ await manager.emitChunk(streamId, {+ event: 'on_run_step',+ data: {+ id: 'step-1',+ runId: 'run-1',+ index: 0,+ stepDetails: { type: 'message_creation' },+ },+ });+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'durable-overflow-content' } },+ },+ });+ await forceEarlyBufferOverflow(manager, streamId);+ expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);++ const errors: string[] = [];+ const sub = await manager.subscribe(+ streamId,+ () => {},+ undefined,+ (error) => errors.push(error),+ );+ await new Promise((resolve) => setTimeout(resolve, 100));+ expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);+ sub?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 100));++ /** The resume path the client falls back to reconstructs the+ * discarded output from the durable chunk log. */+ const { subscription, resumeState } = await manager.subscribeWithResume(streamId, () => {});+ expect(JSON.stringify(resumeState?.aggregatedContent ?? [])).toContain(+ 'durable-overflow-content',+ );+ subscription?.unsubscribe();+ const recoveredJob = await manager.getJob(streamId);+ expect(recoveredJob?.metadata.earlyBufferOverflow).toMatchObject({+ recoveryMethod: 'redis',+ recoveryOutcome: 'success',+ });++ await manager.destroy();+ });++ testRedis('refreshes an overflow marker before activating a resume attachment', async () => {+ const owner = createRedisManager();+ const streamId = `overflow-marker-attachment-race-${Date.now()}`;+ await owner.createJob(streamId, 'user-1');+ await owner.emitChunk(streamId, {+ event: 'on_run_step',+ data: {+ id: 'step-1',+ runId: 'run-1',+ index: 0,+ stepDetails: { type: 'message_creation' },+ },+ });+ await owner.emitChunk(streamId, {+ event: 'on_message_delta',+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'durable-before-marker' } },+ },+ });+ await forceEarlyBufferOverflow(owner, streamId);++ const replica = createRedisManager();+ const replicaStore = replica.getJobStore();+ const originalGetJob = replicaStore.getJob.bind(replicaStore);+ let jobReads = 0;+ jest.spyOn(replicaStore, 'getJob').mockImplementation(async (id) => {+ const job = await originalGetJob(id);+ jobReads++;+ if (jobReads <= 4 && job != null) {+ return { ...job, earlyBufferOverflow: undefined };+ }+ return job;+ });++ const result = await replica.subscribeWithResume(streamId, () => {});++ expect(jobReads).toBeGreaterThan(4);+ expect(JSON.stringify(result.resumeState?.aggregatedContent ?? [])).toContain(+ 'durable-before-marker',+ );+ expect((await originalGetJob(streamId))?.earlyBufferOverflow).toMatchObject({+ recoveryMethod: 'redis',+ recoveryOutcome: 'success',+ });+ result.subscription?.unsubscribe();++ const followup = await replica.subscribeWithResume(streamId, () => {});+ expect(JSON.stringify(followup.resumeState?.aggregatedContent ?? [])).toContain(+ 'durable-before-marker',+ );+ followup.subscription?.unsubscribe();++ await Promise.all([owner.destroy(), replica.destroy()]);+ });++ testRedis(+ 'reconstructs more than 5,000 pre-attachment events across replica reconnects (Redis)',+ async () => {+ const owner = createRedisManager();+ const streamId = `overflow-cross-replica-${Date.now()}`;+ await owner.createJob(streamId, 'user-1');+ const createdAt = (await owner.getJob(streamId))!.createdAt;+ const firstReplica = createRedisManager();+ /** Cache a pre-overflow runtime so resume must refresh the later+ * durable marker rather than trust process-local state. */+ await firstReplica.getJob(streamId);+ /** Created envelopes are published but not durably appendable, so they+ * must not advance the durable recovery frontier. */+ await owner.emitChunk(streamId, {+ created: true,+ message: { text: 'hello' },+ streamId,+ } as CreatedEvent);+ await owner.emitChunk(streamId, {+ event: 'on_run_step',+ data: {+ id: 'step-1',+ runId: 'run-1',+ index: 0,+ stepDetails: { type: 'message_creation' },+ },+ });+ for (let i = 0; i < 5_001; i++) {+ await firstReplica.emitChunk(streamId, {+ event: 'on_message_delta',+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: `${i},` } },+ },+ });+ }++ const first = await firstReplica.subscribeWithResume(streamId, () => {});+ expect(first.resumeState?.aggregatedContent).toHaveLength(1);+ expect(JSON.stringify(first.resumeState?.aggregatedContent)).toContain('5000,');+ first.subscription?.activate();+ first.subscription?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 50));+ const firstAttachedAt = await ioredisClient!.hget(+ `stream:{${streamId}}:job`,+ 'firstSubscriberAttachedAt',+ );+ expect(firstAttachedAt).not.toBeNull();+ expect(+ await (firstReplica.getJobStore() as IJobStoreV2).hasActiveSubscriber(+ streamId,+ createdAt,+ Date.now(),+ ),+ ).toBe(false);++ await owner.emitChunk(streamId, {+ event: 'on_message_delta',+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'after-disconnect' } },+ },+ });++ const secondReplica = createRedisManager();+ const second = await secondReplica.subscribeWithResume(streamId, () => {});+ expect(JSON.stringify(second.resumeState?.aggregatedContent)).toContain('after-disconnect');+ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(+ await (secondReplica.getJobStore() as IJobStoreV2).hasActiveSubscriber(+ streamId,+ createdAt,+ Date.now(),+ ),+ ).toBe(true);+ second.subscription?.unsubscribe();+ expect(+ await ioredisClient!.hget(`stream:{${streamId}}:job`, 'firstSubscriberAttachedAt'),+ ).toBe(firstAttachedAt);++ const recoveredJob = await secondReplica.getJob(streamId);+ expect(recoveredJob?.metadata.earlyBufferOverflow).toMatchObject({+ durableEvents: 5_002,+ recoveryMethod: 'redis',+ recoveryOutcome: 'success',+ });++ await Promise.all([owner.destroy(), firstReplica.destroy(), secondReplica.destroy()]);+ },+ 60_000,+ );++ testRedis(+ 'terminalizes an overflow when durable reconstruction is missing (Redis)',+ async () => {+ const owner = createRedisManager();+ const streamId = `overflow-missing-durable-${Date.now()}`;+ await owner.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(owner, streamId);+ await ioredisClient!.del(`stream:{${streamId}}:chunks`);++ const replica = createRedisManager();+ const errors: string[] = [];+ const result = await replica.subscribeWithResume(+ streamId,+ () => {},+ undefined,+ (error) => errors.push(error),+ );++ expect(result.subscription).toBeNull();+ expect(errors).toEqual([GENERATION_RECOVERY_FAILED_ERROR]);+ const failedJob = await replica.getJob(streamId);+ expect(failedJob?.status).toBe('error');+ expect(failedJob?.error).toBe(GENERATION_RECOVERY_FAILED_ERROR);+ expect(failedJob?.metadata.earlyBufferOverflow).toMatchObject({+ recoveryMethod: 'redis',+ recoveryOutcome: 'failed',+ recoveryFailureReason: 'durable_state_missing',+ });++ await Promise.all([owner.destroy(), replica.destroy()]);+ },+ );++ testRedis('replays a durable terminal payload with unresolved overflow (Redis)', async () => {+ const owner = createRedisManager();+ const streamId = `overflow-durable-terminal-${Date.now()}`;+ const job = await owner.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(owner, streamId);+ const claim = await owner.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, {+ persistencePending: true,+ });+ expect(claim).not.toBeNull();+ const finalEvent = {+ final: true,+ conversation: { conversationId: streamId },+ responseMessage: { text: 'complete' },+ } as ServerSentEvent;+ await owner.publishTerminalClaim(claim!, finalEvent);+ expect((await owner.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(+ 'recoveryOutcome',+ );++ const replica = createRedisManager();+ const terminalEvents: ServerSentEvent[] = [];+ const errors: string[] = [];+ const resumed = await replica.subscribeWithResume(+ streamId,+ () => {},+ (event) => terminalEvents.push(event),+ (error) => errors.push(error),+ { expectedCreatedAt: job.createdAt },+ );+ resumed.subscription?.activate();+ await new Promise((resolve) => setTimeout(resolve, 20));++ expect(terminalEvents).toEqual([finalEvent]);+ expect(errors).toEqual([]);+ expect((await replica.getJob(streamId))?.status).toBe('complete');++ resumed.subscription?.unsubscribe();+ await Promise.all([owner.destroy(), replica.destroy()]);+ });++ testRedis('validates overflow recovery before exposing HITL resume state', async () => {+ const manager = createRedisManager();+ const streamId = `overflow-hitl-validation-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(manager, streamId);+ const createdAt = (await manager.getJob(streamId))!.createdAt;+ await ioredisClient!.del(`stream:{${streamId}}:chunks`);++ await expect(+ manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),+ ).rejects.toThrow(GENERATION_RECOVERY_FAILED_ERROR);+ expect((await manager.getJob(streamId))?.status).toBe('error');++ await manager.destroy();+ });++ testRedis(+ 'terminalizes a durable failed outcome left running by a crashed replica',+ async () => {+ const owner = createRedisManager();+ const streamId = `overflow-failed-outcome-${Date.now()}`;+ await owner.createJob(streamId, 'user-1');+ await forceEarlyBufferOverflow(owner, streamId);++ const overflowJob = await owner.getJob(streamId);+ const overflow = overflowJob?.metadata.earlyBufferOverflow;+ expect(overflow).toBeDefined();+ await (owner.getJobStore() as IJobStoreV2).settleEarlyBufferRecovery(+ streamId,+ overflowJob!.createdAt,+ overflow!.id,+ {+ recoveryMethod: 'redis',+ recoveryOutcome: 'failed',+ recoveryCompletedAt: Date.now(),+ recoveryFailureReason: 'reconstruction_error',+ },+ );++ const replica = createRedisManager();+ const errors: string[] = [];+ const result = await replica.subscribeWithResume(+ streamId,+ () => {},+ undefined,+ (error) => errors.push(error),+ );+ expect(result.subscription).toBeNull();+ expect(errors).toEqual([GENERATION_RECOVERY_FAILED_ERROR]);+ expect((await replica.getJob(streamId))?.status).toBe('error');++ await Promise.all([owner.destroy(), replica.destroy()]);+ },+ );++ test('buffers detached events until the cap in in-memory mode', async () => {+ const manager = createInMemoryManager();+ const streamId = `buf-below-cap-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ await setupDisconnectedStream(manager, streamId, 10);++ const stats = manager.getRuntimeStats();+ expect(stats.earlyBufferedEvents).toBe(2);+ expect(stats.earlyBufferedBytes).toBeGreaterThan(0);++ await manager.destroy();+ });++ test('caps captured events restored by a resume canceled before activation', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `restore-cap-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ await setupDisconnectedStream(manager, streamId, 10);++ /** Arm only after the snapshot completes so the gate parks the resume+ * in its post-attachment steer reconciliation, the window where+ * emissions are captured per-resume instead of buffered. */+ let armed = false;+ const originalGetResumeState = manager.getResumeState.bind(manager);+ jest+ .spyOn(manager, 'getResumeState')+ .mockImplementation(+ async (...args: Parameters<GenerationJobManagerClass['getResumeState']>) => {+ const result = await originalGetResumeState(...args);+ armed = true;+ return result;+ },+ );+ let releaseGate!: () => void;+ const gate = new Promise<void>((resolve) => (releaseGate = resolve));+ let gateReached!: () => void;+ const reached = new Promise<void>((resolve) => (gateReached = resolve));+ const originalPeek = jobStore.peekSteers.bind(jobStore);+ let gated = true;+ jest+ .spyOn(jobStore, 'peekSteers')+ .mockImplementation(async (...args: Parameters<InMemoryJobStore['peekSteers']>) => {+ if (armed && gated) {+ gated = false;+ gateReached();+ await gate;+ }+ return originalPeek(...args);+ });++ const resumePromise = manager.subscribeWithResume(streamId, () => {});++ await reached;+ const bigText = 'z'.repeat(2 * 1024 * 1024);+ for (let i = 0; i < 5; i++) {+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: bigText } },+ },+ });+ }+ releaseGate();++ const { subscription } = await resumePromise;+ expect(subscription).not.toBeNull();+ subscription!.unsubscribe();++ /** The ~10MB of captured events must not survive restoration. */+ const stats = manager.getRuntimeStats();+ expect(stats.earlyBufferedEvents).toBe(0);+ expect(stats.earlyBufferedBytes).toBe(0);++ /** Restoration overflowed, so a non-resume attach takes the redirect. */+ const errors: string[] = [];+ const probe = await manager.subscribe(+ streamId,+ () => {},+ undefined,+ (error) => errors.push(error),+ );+ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);+ probe?.unsubscribe();++ await manager.destroy();+ });+ });++ describe('Atomic subscribeWithResume', () => {+ test('retries a failed first-subscriber lease claim while attached', async () => {+ jest.useFakeTimers();+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const originalClaim = jobStore.claimFirstSubscriber.bind(jobStore);+ const claim = jest+ .spyOn(jobStore, 'claimFirstSubscriber')+ .mockRejectedValueOnce(new Error('store unavailable'))+ .mockImplementation(originalClaim);+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `subscriber-claim-retry-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ const createdAt = (await jobStore.getJob(streamId))!.createdAt;+ const subscription = await manager.subscribe(streamId, () => {});+ await jest.advanceTimersByTimeAsync(0);+ expect(claim).toHaveBeenCalledTimes(1);+ const runtime = (+ manager as unknown as {+ runtimeState: Map<string, { onFirstSubscriberLeaseClaim?: (first: boolean) => void }>;+ }+ ).runtimeState.get(streamId)!;+ expect(runtime.onFirstSubscriberLeaseClaim).toBeDefined();++ await jest.advanceTimersByTimeAsync(10_000);++ expect(claim).toHaveBeenCalledTimes(2);+ expect(runtime.onFirstSubscriberLeaseClaim).toBeUndefined();+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(+ true,+ );+ subscription?.unsubscribe();+ await manager.destroy();+ jest.useRealTimers();+ });++ test('coalesces subscriber lease renewals while a store write is pending', async () => {+ jest.useFakeTimers();+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const originalClaim = jobStore.claimFirstSubscriber.bind(jobStore);+ let releaseRenewal!: (claimed: boolean) => void;+ const pendingRenewal = new Promise<boolean>((resolve) => {+ releaseRenewal = resolve;+ });+ const claim = jest+ .spyOn(jobStore, 'claimFirstSubscriber')+ .mockImplementationOnce(originalClaim)+ .mockReturnValueOnce(pendingRenewal)+ .mockImplementation(originalClaim);+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `subscriber-renewal-coalesce-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ const subscription = await manager.subscribe(streamId, () => {});+ await jest.advanceTimersByTimeAsync(30_000);++ expect(claim).toHaveBeenCalledTimes(2);+ releaseRenewal(false);+ await jest.advanceTimersByTimeAsync(10_000);+ expect(claim).toHaveBeenCalledTimes(3);+ const lastCall = claim.mock.calls[2];+ expect(lastCall[4] - lastCall[2]).toBe(30_000);++ subscription?.unsubscribe();+ await manager.destroy();+ jest.useRealTimers();+ });++ test('stops the predecessor lease timer when a durable generation replaces it', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: true,+ });+ manager.initialize();+ const streamId = `subscriber-runtime-replacement-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ const subscription = await manager.subscribe(streamId, () => {});+ const predecessor = (+ manager as unknown as {+ runtimeState: Map<string, { subscriberLeaseTimer?: ReturnType<typeof setInterval> }>;+ }+ ).runtimeState.get(streamId)!;+ expect(predecessor.subscriberLeaseTimer).toBeDefined();++ await jobStore.createJob(streamId, 'user-1');+ await manager.getJob(streamId);++ expect(predecessor.subscriberLeaseTimer).toBeUndefined();+ subscription?.unsubscribe();+ await manager.destroy();+ });++ test('keeps ordinary HITL resume reads on the cached content path', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `hitl-cached-content-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ const createdAt = (await jobStore.getJob(streamId))!.createdAt;+ const getContentParts = jest.spyOn(jobStore, 'getContentParts');++ await manager.getResumeState(streamId, createdAt, {+ validateEarlyBufferRecovery: true,+ });++ expect(getContentParts).toHaveBeenCalledWith(streamId, createdAt, {+ durableOnly: false,+ });+ await manager.destroy();+ });++ test('renews the subscriber lease after an ambiguous detach failure', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const claim = jest.spyOn(jobStore, 'claimFirstSubscriber');+ jest+ .spyOn(jobStore, 'detachSubscriber')+ .mockRejectedValueOnce(new Error('store unavailable'));+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `subscriber-detach-retry-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const first = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 20));+ first?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));+ const second = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(claim).toHaveBeenCalledTimes(2);++ second?.unsubscribe();+ await manager.destroy();+ });++ test('tracks active subscriber groups across local reattachments', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `subscriber-lifecycle-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');+ const createdAt = (await jobStore.getJob(streamId))!.createdAt;++ const first = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 20));+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(+ true,+ );+ first?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(+ false,+ );++ const second = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 20));+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(+ true,+ );+ second?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));+ await jobStore.claimFirstSubscriber(+ streamId,+ createdAt,+ Date.now(),+ 'crashed-subscriber',+ Date.now() - 1,+ );+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(+ false,+ );+ await manager.destroy();+ });++ test('keeps the first-subscriber telemetry claim off the attachment path', async () => {+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });+ let releaseClaim!: (claimed: boolean) => void;+ const pendingClaim = new Promise<boolean>((resolve) => {+ releaseClaim = resolve;+ });+ jest.spyOn(jobStore, 'claimFirstSubscriber').mockReturnValue(pendingClaim);+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore,+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ const streamId = `nonblocking-attachment-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const attachment = manager.subscribe(streamId, () => {});+ const result = await Promise.race([+ attachment,+ new Promise<'timed_out'>((resolve) => setTimeout(() => resolve('timed_out'), 50)),+ ]);+ expect(result).not.toBe('timed_out');+ releaseClaim(true);+ await pendingClaim;+ if (result !== 'timed_out') {+ result?.unsubscribe();+ }+ await manager.destroy();+ });++ test('should return empty pendingEvents for pre-snapshot buffer events (in-memory)', async () => {+ const manager = createInMemoryManager();+ const streamId = `atomic-drain-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));+ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_run_step',+ data: {+ id: 'step-1',+ runId: 'run-1',+ index: 0,+ stepDetails: { type: 'message_creation' },+ },+ });+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: {+ id: 'step-1',+ delta: { content: { type: 'text', text: 'buffered' } },+ },+ });++ const liveEvents: ServerSentEvent[] = [];+ const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(+ streamId,+ (event) => liveEvents.push(event),+ );++ expect(resumeState).not.toBeNull();+ expect(pendingEvents.length).toBe(0);+ expect(liveEvents.length).toBe(0);++ subscription?.unsubscribe();+ await manager.destroy();+ });++ test('should return empty pendingEvents when buffer is empty', async () => {+ const manager = createInMemoryManager();+ const streamId = `atomic-empty-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'delivered' } } },+ });+ await new Promise((resolve) => setTimeout(resolve, 10));++ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ const { pendingEvents } = await manager.subscribeWithResume(streamId, () => {});++ expect(pendingEvents.length).toBe(0);++ await manager.destroy();+ });++ test('should defer live events until a resumed subscription is activated', async () => {+ const manager = createInMemoryManager();+ const streamId = `atomic-live-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});await new Promise((resolve) => setTimeout(resolve, 10));sub1?.unsubscribe();await new Promise((resolve) => setTimeout(resolve, 20));await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: 'buffered-pre-snapshot' } } },+ data: {+ delta: { content: { type: 'text', text: 'buffered-pre-snapshot' } },+ },});const liveEvents: ServerSentEvent[] = [];@@ -1984,7 +2978,9 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: 'buffered-redis' } } },+ data: {+ delta: { content: { type: 'text', text: 'buffered-redis' } },+ },});await new Promise((resolve) => setTimeout(resolve, 100));@@ -2290,7 +3286,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 5; i++) {await replicaA.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `token${i} ` } }, index: i },+ data: {+ delta: { content: { type: 'text', text: `token${i} ` } },+ index: i,+ },});}@@ -2334,7 +3333,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 3; i++) {await replicaA.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `pre-local-${i}` } }, index: i },+ data: {+ delta: { content: { type: 'text', text: `pre-local-${i}` } },+ index: i,+ },});}@@ -2350,7 +3352,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 3; i++) {await replicaA.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `post-local-${i}` } }, index: i + 3 },+ data: {+ delta: { content: { type: 'text', text: `post-local-${i}` } },+ index: i + 3,+ },});}@@ -2423,7 +3428,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 3; i++) {await replicaA.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `word${i} ` } }, index: i },+ data: {+ delta: { content: { type: 'text', text: `word${i} ` } },+ index: i,+ },});}@@ -2486,11 +3494,17 @@ describe('GenerationJobManager Integration Tests', () => {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: 'pre-sub-0' } }, index: 0 },+ data: {+ delta: { content: { type: 'text', text: 'pre-sub-0' } },+ index: 0,+ },});await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: 'pre-sub-1' } }, index: 1 },+ data: {+ delta: { content: { type: 'text', text: 'pre-sub-1' } },+ index: 1,+ },});const receivedEvents: unknown[] = [];@@ -2506,7 +3520,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 5; i++) {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `post-sub-${i}` } }, index: i + 2 },+ data: {+ delta: { content: { type: 'text', text: `post-sub-${i}` } },+ index: i + 2,+ },});}@@ -2553,7 +3570,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 0; i < 3; i++) {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `chunk-${i}` } }, index: i },+ data: {+ delta: { content: { type: 'text', text: `chunk-${i}` } },+ index: i,+ },});}@@ -2567,7 +3587,10 @@ describe('GenerationJobManager Integration Tests', () => {for (let i = 3; i < 6; i++) {await manager.emitChunk(streamId, {event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `chunk-${i}` } }, index: i },+ data: {+ delta: { content: { type: 'text', text: `chunk-${i}` } },+ index: i,+ },});}@@ -2631,7 +3654,10 @@ describe('GenerationJobManager Integration Tests', () => {await sub2.ready;expect(callCount).toBe(2);- await transport.emitChunk(streamId, { event: 'test', data: { value: 'hello' } });+ await transport.emitChunk(streamId, {+ event: 'test',+ data: { value: 'hello' },+ });await new Promise((resolve) => setTimeout(resolve, 100));expect(receivedEvents.length).toBe(1);@@ -2782,7 +3808,10 @@ describe('GenerationJobManager Integration Tests', () => {status: 'error',error: 'Generation predecessor handoff could not be confirmed',replacedJobs: [- expect.objectContaining({ createdAt: predecessor.createdAt, status: 'running' }),+ expect.objectContaining({+ createdAt: predecessor.createdAt,+ status: 'running',+ }),],});bystanderSubscription?.unsubscribe();@@ -2916,7 +3945,10 @@ describe('GenerationJobManager Integration Tests', () => {);const store = new RedisJobStore(ioredisClient!);- const moved = await store.transitionStatus(streamId, { from: 'running', to: 'complete' });+ const moved = await store.transitionStatus(streamId, {+ from: 'running',+ to: 'complete',+ });expect(moved).toBe(true);const claimed = await store.claimParkedSteers(streamId, 'user-1');diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.tsindex 6805b63db1..b7bb784812 100644--- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts@@ -1757,6 +1757,82 @@ describe('RedisJobStore Integration Tests', () => {await instance2.destroy();});+ test('reports the generation-wide durable count after chunk trimming', async () => {+ if (!ioredisClient) {+ return;+ }++ const { RedisJobStore } = await import('../implementations/RedisJobStore');+ const store = new RedisJobStore(ioredisClient);+ await store.initialize();++ const streamId = `trimmed-chunk-frontier-${Date.now()}`;+ const job = await store.createJob(streamId, 'user-1', streamId);+ for (let i = 0; i < 3; i++) {+ await store.appendChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: String(i) } } },+ });+ }+ await ioredisClient.xtrim(`stream:{${streamId}}:chunks`, 'MAXLEN', 1);+ for (let i = 3; i < 5; i++) {+ await store.appendChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: String(i) } } },+ });+ }++ const result = await store.getContentParts(streamId, job.createdAt, { durableOnly: true });++ expect(result).toMatchObject({+ reconstructedEventCount: 3,+ durableEventCount: 5,+ });++ await store.destroy();+ });++ test('repairs the durable count after an append from a legacy writer', async () => {+ if (!ioredisClient) {+ return;+ }++ const { RedisJobStore } = await import('../implementations/RedisJobStore');+ const store = new RedisJobStore(ioredisClient);+ await store.initialize();++ const streamId = `mixed-writer-frontier-${Date.now()}`;+ const job = await store.createJob(streamId, 'user-1', streamId);+ for (let i = 0; i < 3; i++) {+ await store.appendChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: String(i) } } },+ });+ }+ await ioredisClient.xadd(+ `stream:{${streamId}}:chunks`,+ '*',+ 'event',+ JSON.stringify({+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'legacy' } } },+ }),+ );+ await store.appendChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'current' } } },+ });++ const result = await store.getContentParts(streamId, job.createdAt, { durableOnly: true });++ expect(result).toMatchObject({+ reconstructedEventCount: 5,+ durableEventCount: 5,+ });++ await store.destroy();+ });+test('should share run steps between instances', async () => {if (!ioredisClient) {return;diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.tsindex f8c30b4b8d..3b5d964d9d 100644--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts@@ -1030,10 +1030,11 @@ describe('GenerationJobManager start-generation claim', () => {settleAppend = resolve;});jest.spyOn(store, 'appendChunk').mockImplementationOnce(() => delayedAppend);- await manager.emitChunk(streamId, {+ const predecessorEmission = manager.emitChunk(streamId, {event: 'on_message_delta',data: { delta: 'in-flight predecessor output' },});+ await Promise.resolve();const replacement = await manager.createJob(streamId, 'user-1', streamId, {initialMetadata: { generationProtocolVersion: 2 },@@ -1042,6 +1043,7 @@ describe('GenerationJobManager start-generation claim', () => {expect(replacement.abortController.signal.aborted).toBe(false);settleAppend(false);+ await expect(predecessorEmission).rejects.toThrow('Durable chunk append was fenced out');await Promise.resolve();await Promise.resolve();diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.tsindex 3afb487751..1a388dd853 100644--- a/packages/api/src/stream/__tests__/startup.spec.ts+++ b/packages/api/src/stream/__tests__/startup.spec.ts@@ -675,6 +675,12 @@ describe('GenerationJobManager startup telemetry', () => {const predecessor = await manager.createJob(streamId, 'user-1', streamId);const onDone = jest.fn();const subscription = await manager.subscribe(streamId, () => undefined, onDone);+ const predecessorRuntime = (+ manager as unknown as {+ runtimeState: Map<string, { subscriberLeaseTimer?: ReturnType<typeof setInterval> }>;+ }+ ).runtimeState.get(streamId)!;+ expect(predecessorRuntime.subscriberLeaseTimer).toBeDefined();const replacement = await manager.createJob(streamId, 'user-1', streamId);@@ -687,6 +693,7 @@ describe('GenerationJobManager startup telemetry', () => {conversation: { conversationId: streamId },});expect(predecessor.abortController.signal.aborted).toBe(true);+ expect(predecessorRuntime.subscriberLeaseTimer).toBeUndefined();expect(replacement.abortController.signal.aborted).toBe(false);expect(eventTransport.getSubscriberCount(streamId)).toBe(0);subscription?.unsubscribe();@@ -1404,7 +1411,7 @@ describe('GenerationJobManager startup telemetry', () => {expect(getJob).toHaveBeenCalledTimes(1);expect(onError).not.toHaveBeenCalled();expect(onAllSubscribersLeft).not.toHaveBeenCalled();- expect(jest.getTimerCount()).toBe(timerCountBeforeFence);+ expect(jest.getTimerCount()).toBe(timerCountBeforeFence - 1);releaseLookup?.();await jest.advanceTimersByTimeAsync(0);diff --git a/packages/api/src/stream/__tests__/steering.spec.ts b/packages/api/src/stream/__tests__/steering.spec.tsindex b444d0248e..c57539610b 100644--- a/packages/api/src/stream/__tests__/steering.spec.ts+++ b/packages/api/src/stream/__tests__/steering.spec.ts@@ -1288,17 +1288,21 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });const transport = new InMemoryEventTransport();const redisModeManager = buildRedisModeManager(store, transport);+ let subscription: Awaited<ReturnType<typeof redisModeManager.subscribe>> | undefined;try {const streamId = 'steer-fire-and-forget';const job = await redisModeManager.createJob(streamId, 'user-1');+ subscription = await redisModeManager.subscribe(streamId, () => undefined);- // Never resolves: the per-delta hot path must not gate on durability.+ // After subscriber admission, the per-delta hot path must not gate on durability.+ // Pre-admission events intentionally await the append so recovery cannot miss them.jest.spyOn(store, 'appendChunk').mockReturnValue(new Promise<boolean>(() => undefined));const publishSpy = jest.spyOn(transport, 'emitChunk');await redisModeManager.emitChunk(streamId, steerEvent);expect(publishSpy).toHaveBeenCalledWith(streamId, steerEvent, job.createdAt);} finally {+ subscription?.unsubscribe();await redisModeManager.destroy();}});diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.tsindex 2d059e47ff..3d17d392a2 100644--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts@@ -23,6 +23,7 @@ import type {IdempotencyClaimResult,ParkedSteerClaim,} from '~/stream/interfaces/IJobStore';+import type { EarlyBufferOverflowState } from '../../types/earlyBufferRecovery';import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';import {JobStatusTransitionDeadlineError,@@ -368,7 +369,9 @@ export class InMemoryJobStore implements IJobStoreV2 {active: true,verified: true,status: current.status,- ...(current.conversationId !== undefined && { conversationId: current.conversationId }),+ ...(current.conversationId !== undefined && {+ conversationId: current.conversationId,+ }),});}if (@@ -382,7 +385,9 @@ export class InMemoryJobStore implements IJobStoreV2 {active: current?.status === 'running' || current?.status === 'requires_action',verified: currentCreatedAt != null,...(current !== undefined && { status: current.status }),- ...(current?.conversationId !== undefined && { conversationId: current.conversationId }),+ ...(current?.conversationId !== undefined && {+ conversationId: current.conversationId,+ }),});};@@ -533,7 +538,9 @@ export class InMemoryJobStore implements IJobStoreV2 {checkpointNamespace: String(createdAt),}),...(conversationId !== undefined && { conversationId }),- ...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),+ ...(idempotencyClientRequestId !== undefined && {+ idempotencyClientRequestId,+ }),...(recoveredSteerId !== undefined && { recoveredSteerId }),providerAbortReady: false,...(providerExecutionId != null && { providerDrained: true }),@@ -890,7 +897,10 @@ export class InMemoryJobStore implements IJobStoreV2 {if (existing && existing.expiresAt > now) {return { claimed: false, existing: existing.value };}- this.idempotencyClaims.set(key, { value, expiresAt: now + ttlSeconds * 1000 });+ this.idempotencyClaims.set(key, {+ value,+ expiresAt: now + ttlSeconds * 1000,+ });return { claimed: true, existing: value };}@@ -1162,7 +1172,9 @@ export class InMemoryJobStore implements IJobStoreV2 {patch: {completedAt: now,error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,- ...(job.agentEventDeliveryKey != null && { terminalHostActionPending: true }),+ ...(job.agentEventDeliveryKey != null && {+ terminalHostActionPending: true,+ }),},clear: ['pendingAction',@@ -1446,7 +1458,11 @@ export class InMemoryJobStore implements IJobStoreV2 {if (existing) {existing.contentParts = contentParts;} else {- this.contentState.set(streamId, { contentParts, graphRef: null, collectedUsage: [] });+ this.contentState.set(streamId, {+ contentParts,+ graphRef: null,+ collectedUsage: [],+ });}}@@ -1465,7 +1481,11 @@ export class InMemoryJobStore implements IJobStoreV2 {if (existing) {existing.collectedUsage = collectedUsage;} else {- this.contentState.set(streamId, { contentParts: [], graphRef: null, collectedUsage });+ this.contentState.set(streamId, {+ contentParts: [],+ graphRef: null,+ collectedUsage,+ });}}@@ -1487,8 +1507,11 @@ export class InMemoryJobStore implements IJobStoreV2 {async getContentParts(streamId: string,expectedCreatedAt?: number,+ _options?: { durableOnly?: boolean },): Promise<{content: Agents.MessageContentComplex[];+ reconstructedEventCount?: number;+ durableEventCount?: number;} | null> {if (expectedCreatedAt != null && this.jobs.get(streamId)?.createdAt !== expectedCreatedAt) {return null;@@ -1499,9 +1522,107 @@ export class InMemoryJobStore implements IJobStoreV2 {}return {content: state.contentParts,+ ...(_options?.durableOnly === true && {+ reconstructedEventCount: state.contentParts.length,+ durableEventCount: state.contentParts.length,+ }),};}+ async settleEarlyBufferRecovery(+ streamId: string,+ expectedCreatedAt: number,+ overflowId: string,+ settlement: Pick<+ EarlyBufferOverflowState,+ 'recoveryMethod' | 'recoveryOutcome' | 'recoveryCompletedAt' | 'recoveryFailureReason'+ >,+ ): Promise<boolean> {+ const job = this.jobs.get(streamId);+ const overflow = job?.earlyBufferOverflow;+ if (+ job?.createdAt !== expectedCreatedAt ||+ overflow?.id !== overflowId ||+ overflow.recoveryOutcome != null+ ) {+ return false;+ }+ job.earlyBufferOverflow = { ...overflow, ...settlement };+ return true;+ }++ async finalizeEarlyBufferOverflow(+ streamId: string,+ expectedCreatedAt: number,+ overflowId: string,+ finalizedOverflow: EarlyBufferOverflowState,+ ): Promise<boolean> {+ const job = this.jobs.get(streamId);+ const overflow = job?.earlyBufferOverflow;+ if (+ job?.createdAt !== expectedCreatedAt ||+ overflow?.id !== overflowId ||+ overflow.persistencePending !== true ||+ overflow.recoveryOutcome != null+ ) {+ return false;+ }+ job.earlyBufferOverflow = finalizedOverflow;+ return true;+ }++ async hasSubscriberAttached(streamId: string, expectedCreatedAt: number): Promise<boolean> {+ const job = this.jobs.get(streamId);+ return job?.createdAt === expectedCreatedAt && job.firstSubscriberAttachedAt != null;+ }++ async claimFirstSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ attachedAt: number,+ subscriberId: string,+ leaseExpiresAt: number,+ ): Promise<boolean> {+ const job = this.jobs.get(streamId);+ if (job?.createdAt !== expectedCreatedAt) {+ return false;+ }+ job.activeSubscriberLeases ??= {};+ job.activeSubscriberLeases[subscriberId] = leaseExpiresAt;+ const firstSubscriber = job.firstSubscriberAttachedAt == null;+ job.firstSubscriberAttachedAt ??= attachedAt;+ return firstSubscriber;+ }++ async detachSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ subscriberId: string,+ ): Promise<void> {+ const job = this.jobs.get(streamId);+ if (job?.createdAt !== expectedCreatedAt) {+ return;+ }+ delete job.activeSubscriberLeases?.[subscriberId];+ }++ async hasActiveSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ observedAt: number,+ ): Promise<boolean> {+ const job = this.jobs.get(streamId);+ if (job?.createdAt !== expectedCreatedAt) {+ return false;+ }+ for (const [subscriberId, expiresAt] of Object.entries(job.activeSubscriberLeases ?? {})) {+ if (expiresAt <= observedAt) {+ delete job.activeSubscriberLeases?.[subscriberId];+ }+ }+ return Object.keys(job.activeSubscriberLeases ?? {}).length > 0;+ }+/*** Get run steps for a job from graph.contentData.* Uses WeakRef - may return empty if graph has been GC'd.@@ -1685,7 +1806,9 @@ export class InMemoryJobStore implements IJobStoreV2 {) {return [];}- return (this.claimedSteers.get(streamId) ?? []).map((item) => ({ ...item }));+ return (this.claimedSteers.get(streamId) ?? []).map((item) => ({+ ...item,+ }));}async getSteerReceipt(streamId: string, clientSteerId: string): Promise<SteerReceipt | null> {@@ -1713,7 +1836,10 @@ export class InMemoryJobStore implements IJobStoreV2 {const existingEntry = this.steerReceipts.get(streamId)?.get(receiptInput.clientSteerId);if (existingEntry != null && existingEntry.expiresAt > Date.now()) {this.normalizeSteerReceipt(streamId, existingEntry);- return { ...existingEntry.receipt, item: { ...existingEntry.receipt.item } };+ return {+ ...existingEntry.receipt,+ item: { ...existingEntry.receipt.item },+ };}if (existingEntry != null) {this.steerReceipts.get(streamId)?.delete(receiptInput.clientSteerId);@@ -2126,7 +2252,11 @@ export class InMemoryJobStore implements IJobStoreV2 {if (receipt != null) {receipt.item = { ...item };}- return { outcome: 'armed', revision: item.preemptRevision, item: { ...item } };+ return {+ outcome: 'armed',+ revision: item.preemptRevision,+ item: { ...item },+ };}async downgradeSteerPreempts(@@ -2442,7 +2572,11 @@ export class InMemoryJobStore implements IJobStoreV2 {const parsed = JSON.parse(parked.payload) as {userId: string;tenantId?: string;- steers?: Array<{ steerId: string; clientSteerId?: string; recoveringCreatedAt?: number }>;+ steers?: Array<{+ steerId: string;+ clientSteerId?: string;+ recoveringCreatedAt?: number;+ }>;};if (!Array.isArray(parsed.steers)) {return;@@ -2492,7 +2626,11 @@ export class InMemoryJobStore implements IJobStoreV2 {parsedPayload?: {userId: string;tenantId?: string;- steers?: Array<{ steerId: string; clientSteerId?: string; recoveringCreatedAt?: number }>;+ steers?: Array<{+ steerId: string;+ clientSteerId?: string;+ recoveringCreatedAt?: number;+ }>;},parsedLeasedItem?: {steerId: string;diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.tsindex b582e40e72..b34f180055 100644--- a/packages/api/src/stream/implementations/RedisJobStore.ts+++ b/packages/api/src/stream/implementations/RedisJobStore.ts@@ -26,6 +26,7 @@ import type {SteerReceiptInput,ParkedSteerClaim,} from '~/stream/interfaces/IJobStore';+import type { EarlyBufferOverflowState } from '../../types/earlyBufferRecovery';import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';import {@@ -617,6 +618,47 @@ const JOB_UPDATE_LUA ='end ' +'return 1';+/** Single-winner recovery outcome for one generation-scoped overflow. */+const SETTLE_EARLY_BUFFER_RECOVERY_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'local raw = redis.call("HGET", KEYS[1], "earlyBufferOverflow") ' ++ 'if not raw then return 0 end local ok, overflow = pcall(cjson.decode, raw) ' ++ 'if not ok or type(overflow) ~= "table" or overflow.id ~= ARGV[2] ' ++ 'or overflow.recoveryOutcome ~= nil then return 0 end ' ++ 'local settlementOk, settlement = pcall(cjson.decode, ARGV[3]) ' ++ 'if not settlementOk or type(settlement) ~= "table" then return 0 end ' ++ 'for key, value in pairs(settlement) do overflow[key] = value end ' ++ 'redis.call("HSET", KEYS[1], "earlyBufferOverflow", cjson.encode(overflow)) return 1';++/** Finalizes only the unresolved pending marker installed by this owner. */+const FINALIZE_EARLY_BUFFER_OVERFLOW_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'local raw = redis.call("HGET", KEYS[1], "earlyBufferOverflow") ' ++ 'if not raw then return 0 end local ok, overflow = pcall(cjson.decode, raw) ' ++ 'if not ok or type(overflow) ~= "table" or overflow.id ~= ARGV[2] ' ++ 'or overflow.persistencePending ~= true or overflow.recoveryOutcome ~= nil then return 0 end ' ++ 'redis.call("HSET", KEYS[1], "earlyBufferOverflow", ARGV[3]) return 1';++const HAS_SUBSCRIBER_ATTACHED_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'if redis.call("HEXISTS", KEYS[1], "firstSubscriberAttachedAt") == 1 then return 1 end return 0';++/** Generation-scoped single-winner first-subscriber claim. */+const CLAIM_FIRST_SUBSCRIBER_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'redis.call("ZADD", KEYS[2], ARGV[4], ARGV[3]) redis.call("PEXPIRE", KEYS[2], 60000) ' ++ 'if redis.call("HEXISTS", KEYS[1], "firstSubscriberAttachedAt") == 1 then return 0 end ' ++ 'redis.call("HSET", KEYS[1], "firstSubscriberAttachedAt", ARGV[2]) return 1';++const DETACH_SUBSCRIBER_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'redis.call("ZREM", KEYS[2], ARGV[2]) return 1';++const HAS_ACTIVE_SUBSCRIBER_LUA =+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' ++ 'redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[2]) ' ++ 'if redis.call("ZCARD", KEYS[2]) > 0 then return 1 end return 0';+/** Exact provider-segment completion fence. A paused segment finishing after a* resume cannot mark the resumed provider drained because its opaque id differs. */const PROVIDER_DRAIN_LUA =@@ -834,6 +876,10 @@ const CHUNK_APPEND_LUA ='if #kept > 0 then redis.call("RPUSH", KEYS[5], unpack(kept)) ' +'if claimTtl > 0 then redis.call("PEXPIRE", KEYS[5], claimTtl) end end end ' +'redis.call("XADD", KEYS[1], "*", "event", ARGV[1]) ' ++ 'local durable = redis.call("HGET", KEYS[2], "durableEventCount") ' ++ 'local priorLength = redis.call("XLEN", KEYS[1]) - 1 ' ++ 'if not durable or tonumber(durable) < priorLength then durable = priorLength end ' ++ 'redis.call("HSET", KEYS[2], "durableEventCount", tonumber(durable) + 1) ' +'if currentStatus == "running" then ' +'redis.call("HSET", KEYS[2], "lastActiveAt", ARGV[6]) end ' +'local cur = redis.call("TTL", KEYS[1]) ' +@@ -880,6 +926,10 @@ const CHUNK_APPEND_BATCH_LUA ='if epochTtl >= 0 and epochTtl < epochTarget then redis.call("EXPIRE", KEYS[8], epochTarget) end ' +'else redis.call("SET", KEYS[8], currentCreatedAt, "EX", epochTarget) end ' +'for i = 6, #ARGV do redis.call("XADD", KEYS[1], "*", "event", ARGV[i]) end ' ++ 'local durable = redis.call("HGET", KEYS[2], "durableEventCount") ' ++ 'local priorLength = redis.call("XLEN", KEYS[1]) - (#ARGV - 5) ' ++ 'if not durable or tonumber(durable) < priorLength then durable = priorLength end ' ++ 'redis.call("HSET", KEYS[2], "durableEventCount", tonumber(durable) + (#ARGV - 5)) ' +'if currentStatus == "running" then ' +'redis.call("HSET", KEYS[2], "lastActiveAt", ARGV[3]) end ' +'local cur = redis.call("TTL", KEYS[1]) ' +@@ -940,6 +990,12 @@ const CHUNKS_READ_LUA ='if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {} end ' +'return redis.call("XRANGE", KEYS[2], "-", "+")';+const CHUNKS_RECOVERY_READ_LUA =+ 'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {{}, false} end ' ++ 'local entries = redis.call("XRANGE", KEYS[2], "-", "+") ' ++ 'local durable = redis.call("HGET", KEYS[1], "durableEventCount") ' ++ 'return {entries, durable or false}';+const RUNSTEPS_READ_LUA ='if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return false end ' +'return redis.call("GET", KEYS[2])';@@ -1621,6 +1677,9 @@ const KEYS = {steerReceiptOrder: (streamId: string) => `stream:{${streamId}}:steer-receipt-order`,/** Latest generation epoch, retained briefly beyond the live job hash. */generationEpoch: (streamId: string) => `stream:{${streamId}}:generation-epoch`,+ /** Expiring, generation-scoped subscriber-group leases. */+ subscriberLeases: (streamId: string, createdAt: number) =>+ `stream:{${streamId}}:subscriber-leases:${createdAt}`,/** Running jobs set for cleanup (global set - single slot) */runningJobs: 'stream:running',/** Jobs paused for human review (global set - single slot) */@@ -1756,7 +1815,10 @@ interface LocalCacheEntry<T> {interface PendingChunkAppendBatch {expectedCreatedAt?: number;events: string[];- settlers: Array<{ resolve: (appended: boolean) => void; reject: (err: unknown) => void }>;+ settlers: Array<{+ resolve: (appended: boolean) => void;+ reject: (err: unknown) => void;+ }>;bytes: number;timer: ReturnType<typeof setTimeout> | null;}@@ -1961,7 +2023,9 @@ export class RedisJobStore implements IJobStoreV2 {createdAt: Date.now(),generationProtocolVersion,...(conversationId !== undefined && { conversationId }),- ...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),+ ...(idempotencyClientRequestId !== undefined && {+ idempotencyClientRequestId,+ }),...(recoveredSteerId !== undefined && { recoveredSteerId }),providerAbortReady: false,...(providerExecutionId != null && { providerDrained: true }),@@ -2057,7 +2121,9 @@ export class RedisJobStore implements IJobStoreV2 {currentStatus === 'requires_action',verified: previousOwner[8] !== '0',...(currentStatus !== undefined && { status: currentStatus }),- ...(currentConversationId !== undefined && { conversationId: currentConversationId }),+ ...(currentConversationId !== undefined && {+ conversationId: currentConversationId,+ }),});}const previousUserId =@@ -2286,6 +2352,113 @@ export class RedisJobStore implements IJobStoreV2 {}}+ async settleEarlyBufferRecovery(+ streamId: string,+ expectedCreatedAt: number,+ overflowId: string,+ settlement: Pick<+ EarlyBufferOverflowState,+ 'recoveryMethod' | 'recoveryOutcome' | 'recoveryCompletedAt' | 'recoveryFailureReason'+ >,+ ): Promise<boolean> {+ const settled = await this.redis.eval(+ SETTLE_EARLY_BUFFER_RECOVERY_LUA,+ 1,+ KEYS.job(streamId),+ String(expectedCreatedAt),+ overflowId,+ JSON.stringify(settlement),+ );+ return settled === 1;+ }++ async finalizeEarlyBufferOverflow(+ streamId: string,+ expectedCreatedAt: number,+ overflowId: string,+ finalizedOverflow: EarlyBufferOverflowState,+ ): Promise<boolean> {+ const finalized = await this.redis.eval(+ FINALIZE_EARLY_BUFFER_OVERFLOW_LUA,+ 1,+ KEYS.job(streamId),+ String(expectedCreatedAt),+ overflowId,+ JSON.stringify(finalizedOverflow),+ );+ return finalized === 1;+ }++ async hasSubscriberAttached(streamId: string, expectedCreatedAt: number): Promise<boolean> {+ return (+ Number(+ await this.redis.eval(+ HAS_SUBSCRIBER_ATTACHED_LUA,+ 1,+ KEYS.job(streamId),+ String(expectedCreatedAt),+ ),+ ) === 1+ );+ }++ async claimFirstSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ attachedAt: number,+ subscriberId: string,+ leaseExpiresAt: number,+ ): Promise<boolean> {+ return (+ Number(+ await this.redis.eval(+ CLAIM_FIRST_SUBSCRIBER_LUA,+ 2,+ KEYS.job(streamId),+ KEYS.subscriberLeases(streamId, expectedCreatedAt),+ String(expectedCreatedAt),+ String(attachedAt),+ subscriberId,+ String(leaseExpiresAt),+ ),+ ) === 1+ );+ }++ async detachSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ subscriberId: string,+ ): Promise<void> {+ await this.redis.eval(+ DETACH_SUBSCRIBER_LUA,+ 2,+ KEYS.job(streamId),+ KEYS.subscriberLeases(streamId, expectedCreatedAt),+ String(expectedCreatedAt),+ subscriberId,+ );+ }++ async hasActiveSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ observedAt: number,+ ): Promise<boolean> {+ return (+ Number(+ await this.redis.eval(+ HAS_ACTIVE_SUBSCRIBER_LUA,+ 2,+ KEYS.job(streamId),+ KEYS.subscriberLeases(streamId, expectedCreatedAt),+ String(expectedCreatedAt),+ String(observedAt),+ ),+ ) === 1+ );+ }+async markProviderExecutionDrained(streamId: string,expectedCreatedAt: number,@@ -2624,7 +2797,10 @@ export class RedisJobStore implements IJobStoreV2 {// status + patch become HSET pairs; serializeJob skips undefined, so// cleared fields go through HDEL (`clear`) instead.const fields = Object.entries(- this.serializeJob({ status: to, ...(persistedPatch ?? {}) } as SerializableJobData),+ this.serializeJob({+ status: to,+ ...(persistedPatch ?? {}),+ } as SerializableJobData),).flat();const clearFields = (clear ?? []).map(String);@@ -2743,7 +2919,10 @@ export class RedisJobStore implements IJobStoreV2 {return { claimed: true, existing: value };}try {- return { claimed: false, existing: JSON.parse(result as string) as IdempotencyClaimValue };+ return {+ claimed: false,+ existing: JSON.parse(result as string) as IdempotencyClaimValue,+ };} catch {// An unreadable existing owner is outcome-ambiguous. Never turn store// corruption into a duplicate generation by pretending the key is free.@@ -2951,7 +3130,10 @@ export class RedisJobStore implements IJobStoreV2 {if (members.length === 0) {return [];}- const indexed = members.map((member) => ({ member, ...parseTerminalHostActionMember(member) }));+ const indexed = members.map((member) => ({+ member,+ ...parseTerminalHostActionMember(member),+ }));const jobs = await Promise.all(indexed.map(({ streamId }) => this.getJob(streamId)));// The durable hash field is the source of truth; a stale set entry (job reaped, or the// marker/generation already replaced) is filtered out. Bare legacy members@@ -3123,13 +3305,17 @@ export class RedisJobStore implements IJobStoreV2 {// Job no longer exists (TTL expired) - remove from setif (!job) {- const currentJob = await this.reconcileJobMembership(streamId, { initialJob: null });+ const currentJob = await this.reconcileJobMembership(streamId, {+ initialJob: null,+ });this.clearLocalStateUnlessActive(streamId, currentJob);return 1;}if (job.status === 'requires_action') {- const currentJob = await this.reconcileJobMembership(streamId, { initialJob: job });+ const currentJob = await this.reconcileJobMembership(streamId, {+ initialJob: job,+ });if (currentJob !== undefined &&(!currentJob || currentJob.createdAt === job.createdAt)@@ -3202,7 +3388,9 @@ export class RedisJobStore implements IJobStoreV2 {const job = await this.getJob(streamId);if (!job) {- const currentJob = await this.reconcileJobMembership(streamId, { initialJob: null });+ const currentJob = await this.reconcileJobMembership(streamId, {+ initialJob: null,+ });this.clearLocalStateUnlessActive(streamId, currentJob);return 1;}@@ -3236,7 +3424,9 @@ export class RedisJobStore implements IJobStoreV2 {patch: {completedAt: now,error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,- ...(job.agentEventDeliveryKey != null && { terminalHostActionPending: true }),+ ...(job.agentEventDeliveryKey != null && {+ terminalHostActionPending: true,+ }),},clear: ['pendingAction',@@ -3520,14 +3710,20 @@ export class RedisJobStore implements IJobStoreV2 {continue;}if (event.event === 'on_steer_applied') {- const steerData = event.data as { index?: number; part?: Agents.MessageContentComplex };+ const steerData = event.data as {+ index?: number;+ part?: Agents.MessageContentComplex;+ };if (typeof steerData.index === 'number' && steerData.part != null) {steers.push({ index: steerData.index, part: steerData.part });}continue;}if (event.event === 'on_activity_label') {- const labelData = event.data as { index?: number; part?: Agents.MessageContentComplex };+ const labelData = event.data as {+ index?: number;+ part?: Agents.MessageContentComplex;+ };if (typeof labelData.index === 'number' && labelData.part != null) {labelsByIndex.set(labelData.index, labelData.part);}@@ -3727,12 +3923,17 @@ export class RedisJobStore implements IJobStoreV2 {async getContentParts(streamId: string,expectedCreatedAt?: number,+ options?: { durableOnly?: boolean },): Promise<{content: Agents.MessageContentComplex[];+ reconstructedEventCount?: number;+ durableEventCount?: number;} | null> {// 1. Prefer the HOST content array (same-instance fast path): it already// contains host-authored steer parts the SDK graph never sees.- const hostEntry = this.getLocalEntry(this.localContentParts, streamId, expectedCreatedAt);+ const hostEntry = options?.durableOnly+ ? undefined+ : this.getLocalEntry(this.localContentParts, streamId, expectedCreatedAt);if (hostEntry) {const hostParts = hostEntry.value.deref();if (hostParts && hostParts.length > 0) {@@ -3747,7 +3948,9 @@ export class RedisJobStore implements IJobStoreV2 {// lacks host-authored steer parts, so overlay them from the chunk log —// insert (not assign): the graph array is UNSHIFTED, while recorded steer// indices are host-view positions that already account for prior steers.- const graphEntry = this.getLocalEntry(this.localGraphCache, streamId, expectedCreatedAt);+ const graphEntry = options?.durableOnly+ ? undefined+ : this.getLocalEntry(this.localGraphCache, streamId, expectedCreatedAt);if (graphEntry) {const graph = graphEntry.value.deref();if (graph) {@@ -3766,8 +3969,13 @@ export class RedisJobStore implements IJobStoreV2 {}// 2. Fall back to Redis chunk reconstruction (cross-instance reconnect)- const chunks = await this.getChunks(streamId, expectedCreatedAt);- if (chunks.length === 0) {+ const chunkSnapshot = await this.getChunkSnapshot(+ streamId,+ expectedCreatedAt,+ options?.durableOnly === true,+ );+ const { chunks } = chunkSnapshot;+ if (chunkSnapshot.durableEventCount === 0) {return null;}@@ -3807,7 +4015,10 @@ export class RedisJobStore implements IJobStoreV2 {// injection were emitted with already-shifted indices, so both sources// land disjoint.if (event.event === 'on_steer_applied') {- const steerData = event.data as { index?: number; part?: Agents.MessageContentComplex };+ const steerData = event.data as {+ index?: number;+ part?: Agents.MessageContentComplex;+ };if (typeof steerData.index === 'number' && steerData.part != null) {contentParts[steerData.index] = steerData.part;}@@ -3818,7 +4029,10 @@ export class RedisJobStore implements IJobStoreV2 {// fixed index. The event fires twice per slot (counts placeholder,// then resolved label); chronological replay makes the last write win.if (event.event === 'on_activity_label') {- const labelData = event.data as { index?: number; part?: Agents.MessageContentComplex };+ const labelData = event.data as {+ index?: number;+ part?: Agents.MessageContentComplex;+ };if (typeof labelData.index === 'number' && labelData.part != null) {contentParts[labelData.index] = labelData.part;}@@ -3977,6 +4191,10 @@ export class RedisJobStore implements IJobStoreV2 {return {content: filtered,+ ...(options?.durableOnly === true && {+ reconstructedEventCount: chunks.length,+ durableEventCount: chunkSnapshot.durableEventCount,+ }),};}@@ -4549,7 +4767,13 @@ export class RedisJobStore implements IJobStoreV2 {pending = undefined;}if (!pending) {- pending = { expectedCreatedAt, events: [], settlers: [], bytes: 0, timer: null };+ pending = {+ expectedCreatedAt,+ events: [],+ settlers: [],+ bytes: 0,+ timer: null,+ };this.pendingAppends.set(streamId, pending);}@@ -4626,24 +4850,45 @@ export class RedisJobStore implements IJobStoreV2 {* Get all chunks from Redis Stream.*/private async getChunks(streamId: string, expectedCreatedAt?: number): Promise<unknown[]> {+ return (await this.getChunkSnapshot(streamId, expectedCreatedAt)).chunks;+ }++ private async getChunkSnapshot(+ streamId: string,+ expectedCreatedAt?: number,+ includeDurableEventCount = false,+ ): Promise<{ chunks: unknown[]; durableEventCount: number }> {/** A same-replica snapshot read must observe the appends this process has* already accepted, or a resume during an active window reconstructs* without the buffered tail. Cross-replica readers keep today's contract:* the log may trail live emission by up to one window. */await this.flushCoalescedAppends(streamId);- const rawEntries =- expectedCreatedAt == null- ? await this.redis.xrange(KEYS.chunks(streamId), '-', '+')- : await this.redis.eval(- CHUNKS_READ_LUA,- 2,- KEYS.job(streamId),- KEYS.chunks(streamId),- String(expectedCreatedAt),- );+ let rawEntries: unknown;+ let rawDurableEventCount: unknown;+ if (includeDurableEventCount) {+ const rawSnapshot = await this.redis.eval(+ CHUNKS_RECOVERY_READ_LUA,+ 2,+ KEYS.job(streamId),+ KEYS.chunks(streamId),+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',+ );+ [rawEntries, rawDurableEventCount] = Array.isArray(rawSnapshot) ? rawSnapshot : [];+ } else {+ rawEntries =+ expectedCreatedAt == null+ ? await this.redis.xrange(KEYS.chunks(streamId), '-', '+')+ : await this.redis.eval(+ CHUNKS_READ_LUA,+ 2,+ KEYS.job(streamId),+ KEYS.chunks(streamId),+ String(expectedCreatedAt),+ );+ }const entries = Array.isArray(rawEntries) ? (rawEntries as Array<[string, string[]]>) : [];- return entries+ const chunks = entries.map(([, fields]) => {const eventIdx = fields.indexOf('event');if (eventIdx >= 0 && eventIdx + 1 < fields.length) {@@ -4656,6 +4901,16 @@ export class RedisJobStore implements IJobStoreV2 {return null;}).filter(Boolean);+ const parsedDurableEventCount =+ typeof rawDurableEventCount === 'string' || typeof rawDurableEventCount === 'number'+ ? Number(rawDurableEventCount)+ : Number.NaN;+ return {+ chunks,+ durableEventCount: Number.isFinite(parsedDurableEventCount)+ ? parsedDurableEventCount+ : entries.length,+ };}/**@@ -4916,6 +5171,13 @@ export class RedisJobStore implements IJobStoreV2 {completedAt: data.completedAt ? parseInt(data.completedAt, 10) : undefined,conversationId: data.conversationId || undefined,error: data.error || undefined,+ earlyBufferOverflow: data.earlyBufferOverflow+ ? (JSON.parse(data.earlyBufferOverflow) as EarlyBufferOverflowState)+ : undefined,+ firstSubscriberAttachedAt: data.firstSubscriberAttachedAt+ ? parseInt(data.firstSubscriberAttachedAt, 10)+ : undefined,+ durableEventCount: data.durableEventCount ? parseInt(data.durableEventCount, 10) : undefined,idempotencyClientRequestId: data.idempotencyClientRequestId || undefined,recoveredSteerId: data.recoveredSteerId || undefined,userMessage: data.userMessage ? JSON.parse(data.userMessage) : undefined,diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.tsindex cef396978e..752c454148 100644--- a/packages/api/src/stream/index.ts+++ b/packages/api/src/stream/index.ts@@ -4,6 +4,7 @@ export {type CreateGenerationJobOptions,type GenerationJobManagerOptions,type TerminalJobClaim,+ GENERATION_RECOVERY_FAILED_ERROR,TERMINAL_PUBLICATION_RECONNECT_ERROR,} from './GenerationJobManager';diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex 18064da049..ff128e1bda 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -10,6 +10,7 @@ import type {} from 'librechat-data-provider';import type { RunStep, StandardGraph } from '@librechat/agents';import type { AgentEventDetachedTerminalEvidence } from '~/agents/triggers/types';+import type { EarlyBufferOverflowState } from '../../types/earlyBufferRecovery';import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';import type { RecoveredSteerPayload } from '../SteerRecovery';@@ -121,6 +122,17 @@ export interface SerializableJobData {conversationId?: string;error?: string;+ /** Durable, non-sensitive identity and one-shot outcome for an early replay+ * buffer overflow. This lets another replica account for recovery. */+ earlyBufferOverflow?: EarlyBufferOverflowState;++ /** Generation-level first subscriber claim shared across replicas. */+ firstSubscriberAttachedAt?: number;+ /** Expiring local subscriber-group leases (in-memory store only). */+ activeSubscriberLeases?: Record<string, number>;+ /** Generation-wide durable chunk frontier maintained by the store. */+ durableEventCount?: number;+/** Stable identity of the HTTP submission that created this generation.* Internal-only: lets an expired idempotency lease recognize the same live* job instead of replacing and billing it again. */@@ -950,7 +962,13 @@ export interface IJobStore {getContentParts(streamId: string,expectedCreatedAt?: number,- ): Promise<{ content: Agents.MessageContentComplex[] } | null>;+ options?: { durableOnly?: boolean },+ ): Promise<{+ content: Agents.MessageContentComplex[];+ reconstructedEventCount?: number;+ durableEventCount?: number;+ } | null>;+getRunSteps(streamId: string, expectedCreatedAt?: number): Promise<Agents.RunStep[]>;/** Legacy stores returned `void`; v2 stores return whether the epoch-fenced@@ -1269,10 +1287,58 @@ export interface IJobStoreV2 extends IJobStore {getContentParts(streamId: string,expectedCreatedAt?: number,+ options?: { durableOnly?: boolean },): Promise<{content: Agents.MessageContentComplex[];+ reconstructedEventCount?: number;+ durableEventCount?: number;} | null>;+ /** Atomically records the only recovery outcome for one overflow identity. */+ settleEarlyBufferRecovery(+ streamId: string,+ expectedCreatedAt: number,+ overflowId: string,+ settlement: Pick<+ EarlyBufferOverflowState,+ 'recoveryMethod' | 'recoveryOutcome' | 'recoveryCompletedAt' | 'recoveryFailureReason'+ >,+ ): Promise<boolean>;++ /** Atomically replaces an unresolved pending overflow marker with its+ * finalized durable frontier. A concurrent recovery settlement wins over+ * this owner-side finalization. */+ finalizeEarlyBufferOverflow(+ streamId: string,+ expectedCreatedAt: number,+ overflowId: string,+ overflow: EarlyBufferOverflowState,+ ): Promise<boolean>;++ /** Whether this generation has admitted any subscriber on any replica. */+ hasSubscriberAttached(streamId: string, expectedCreatedAt: number): Promise<boolean>;++ /** Atomically claims the first subscriber for one generation epoch. */+ claimFirstSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ attachedAt: number,+ subscriberId: string,+ leaseExpiresAt: number,+ ): Promise<boolean>;++ detachSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ subscriberId: string,+ ): Promise<void>;++ hasActiveSubscriber(+ streamId: string,+ expectedCreatedAt: number,+ observedAt: number,+ ): Promise<boolean>;+/*** Get run steps for a job (for resume state).*diff --git a/packages/api/src/stream/jobStoreCapabilities.ts b/packages/api/src/stream/jobStoreCapabilities.tsindex 92a6f915cb..7603c983f0 100644--- a/packages/api/src/stream/jobStoreCapabilities.ts+++ b/packages/api/src/stream/jobStoreCapabilities.ts@@ -30,6 +30,12 @@ export const JOB_STORE_V2_REQUIRED_METHODS = ['claimParkedSteersDetailed','consumeParkedSteer','discardSteerLeftover',+ 'settleEarlyBufferRecovery',+ 'finalizeEarlyBufferOverflow',+ 'hasSubscriberAttached',+ 'claimFirstSubscriber',+ 'detachSubscriber',+ 'hasActiveSubscriber',] as const satisfies ReadonlyArray<keyof IJobStoreV2>;export type JobStoreV2RequiredMethod = (typeof JOB_STORE_V2_REQUIRED_METHODS)[number];diff --git a/packages/api/src/types/earlyBufferRecovery.ts b/packages/api/src/types/earlyBufferRecovery.tsnew file mode 100644index 0000000000..fe83386c7d--- /dev/null+++ b/packages/api/src/types/earlyBufferRecovery.ts@@ -0,0 +1,25 @@+export type EarlyBufferRecoveryMethod = 'redis' | 'snapshot';+export type EarlyBufferRecoveryOutcome = 'success' | 'failed' | 'not_required';+export type EarlyBufferRecoveryFailureReason =+ | 'durable_state_missing'+ | 'durable_frontier_gap'+ | 'snapshot_missing'+ | 'subscriber_never_attached'+ | 'subscriber_disconnected'+ | 'reconstruction_error'+ | 'overflow_marker_persistence_failed';++export interface EarlyBufferOverflowState {+ id: string;+ occurredAt: number;+ durableEvents: number;+ droppedEvents: number;+ droppedBytes: number;+ /** True while the owner is flushing accepted durable appends and has not+ * published the final recovery frontier yet. */+ persistencePending?: boolean;+ recoveryMethod?: EarlyBufferRecoveryMethod;+ recoveryOutcome?: EarlyBufferRecoveryOutcome;+ recoveryCompletedAt?: number;+ recoveryFailureReason?: EarlyBufferRecoveryFailureReason;+}diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.tsindex d6ec148c87..c7571c557f 100644--- a/packages/api/src/types/stream.ts+++ b/packages/api/src/types/stream.ts@@ -10,6 +10,7 @@ import type {AgentTriggerExpectedAction,} from '../agents/triggers/types';import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';+import type { EarlyBufferOverflowState } from './earlyBufferRecovery';import type { ResolvedAskUserQuestion } from '../agents/hitl/resume';import type { MCPRuntimeRequestBody } from '../mcp/types';import type { ServerSentEvent } from './events';@@ -23,6 +24,7 @@ export interface GenerationJobMetadata {checkpointNamespace?: string;/** Immutable generation protocol. Missing on legacy records means v1. */generationProtocolVersion?: 1 | 2;+ earlyBufferOverflow?: EarlyBufferOverflowState;/** User message data for rebuilding submission on reconnect */userMessage?: Agents.UserMessageMeta;/** Response message ID for tracking */