Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧫 fix: Verify Early Stream Recovery (#15676)

Danny Avila0b9dc209b021
diff --git a/api/server/controllers/agents/__tests__/resume.spec.js b/api/server/controllers/agents/__tests__/resume.spec.js
index 79f31815c8..062edf6639 100644
--- a/api/server/controllers/agents/__tests__/resume.spec.js
+++ b/api/server/controllers/agents/__tests__/resume.spec.js
@@ -127,6 +127,7 @@ jest.mock('@librechat/data-schemas', () => ({
jest.mock('@librechat/api', () => ({
...jest.requireActual('@librechat/api'),
GenerationJobManager: mockGenerationJobManager,
+ GENERATION_RECOVERY_FAILED_ERROR: 'generation_recovery_failed',
captureAgentCheckpointGeneration: (...args) => mockCaptureAgentCheckpointGeneration(...args),
deleteAgentCheckpoint: (...args) => mockDeleteAgentCheckpoint(...args),
decrementPendingRequest: (...args) => mockDecrementPendingRequest(...args),
@@ -1070,6 +1071,39 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
},
});
+ it('settles and prunes a scheduled occurrence terminalized by recovery validation', async () => {
+ const scheduledJob = makeScheduledJob();
+ mockGenerationJobManager.getJob.mockResolvedValueOnce(scheduledJob).mockResolvedValueOnce({
+ ...scheduledJob,
+ status: 'error',
+ error: 'generation_recovery_failed',
+ });
+ mockGenerationJobManager.getResumeState.mockRejectedValueOnce(
+ new Error('terminal cleanup read failed'),
+ );
+
+ const res = await post(approveBody());
+
+ expect(res.status).toBe(500);
+ expect(mockRecordScheduleOutcome).toHaveBeenCalledWith({
+ scheduleId: 'schedule-1',
+ scheduledFor,
+ streamId: CONVO_ID,
+ jobCreatedAt: 1000,
+ status: 'error',
+ conversationId: CONVO_ID,
+ error: 'generation_recovery_failed',
+ });
+ expect(mockDeleteAgentCheckpoint).toHaveBeenCalledWith(
+ CONVO_ID,
+ { type: 'mongo' },
+ undefined,
+ { checkpointNamespace: '1000' },
+ );
+ expect(mockClaimScheduleResume).not.toHaveBeenCalled();
+ expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
+ });
+
it('stops and settles an occurrence that became inactive while awaiting approval', async () => {
mockGenerationJobManager.getJob.mockResolvedValue(makeScheduledJob());
mockIsScheduleLive.mockResolvedValue(false);
@@ -2014,6 +2048,11 @@ describe('ResumeAgentController (POST /agents/chat/resume)', () => {
expect(mockCheckAndIncrementPendingRequest).not.toHaveBeenCalled();
expect(mockGenerationJobManager.approvals.resolve).not.toHaveBeenCalled();
expect(mockGenerationJobManager.emitError).not.toHaveBeenCalled();
+ expect(mockGenerationJobManager.getResumeState).toHaveBeenCalledWith(
+ CONVO_ID,
+ job.createdAt,
+ { validateEarlyBufferRecovery: true },
+ );
});
it('blocks a user-authored respond decision before consuming the action', async () => {
diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js
index 2edec4f3ac..d6ec8c54fe 100644
--- a/api/server/controllers/agents/resume.js
+++ b/api/server/controllers/agents/resume.js
@@ -11,6 +11,7 @@ const {
const {
checkAccess,
GenerationJobManager,
+ GENERATION_RECOVERY_FAILED_ERROR,
isPendingActionStale,
mapToolApprovalResolutions,
resolveAskUserQuestionResume,
@@ -1031,7 +1032,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
let resumeState;
let preparedContent;
try {
- resumeState = await GenerationJobManager.getResumeState(streamId, job.createdAt);
+ resumeState = await GenerationJobManager.getResumeState(streamId, job.createdAt, {
+ validateEarlyBufferRecovery: true,
+ });
const batchedAnswer =
mapped.resumeValue?.answers != null &&
typeof mapped.resumeValue.answers === 'object' &&
@@ -1086,6 +1089,40 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
'[ResumeAgentController] Resume content preflight failed',
getSafeErrorMetadata(err),
);
+ if (scheduleId) {
+ const terminalJob = await GenerationJobManager.getJob(streamId).catch(() => null);
+ if (
+ terminalJob?.createdAt === job.createdAt &&
+ terminalJob.status === 'error' &&
+ terminalJob.error === GENERATION_RECOVERY_FAILED_ERROR
+ ) {
+ try {
+ await recordScheduleOutcome({
+ scheduleId,
+ scheduledFor,
+ streamId,
+ jobCreatedAt: job.createdAt,
+ status: 'error',
+ conversationId,
+ error: GENERATION_RECOVERY_FAILED_ERROR,
+ });
+ } catch (scheduleError) {
+ logger.error(
+ '[ResumeAgentController] Failed to settle scheduled recovery failure',
+ getSafeErrorMetadata(scheduleError),
+ );
+ }
+ await deleteFailedResumeCheckpoint(
+ {
+ conversationId,
+ checkpointerCfg,
+ job,
+ checkpointGeneration: await checkpointGenerationPromise,
+ },
+ 'scheduled recovery validation failure',
+ );
+ }
+ }
if (isContentFilterError(err)) {
return sendGenerationJson(res, err.statusCode, err.body, generationProtocolVersion);
}
diff --git a/packages/api/src/app/metrics.spec.ts b/packages/api/src/app/metrics.spec.ts
index 59267d36ed..3b4ab585d5 100644
--- a/packages/api/src/app/metrics.spec.ts
+++ b/packages/api/src/app/metrics.spec.ts
@@ -11,6 +11,9 @@ import {
recordAgentStartupMilestone,
recordAgentStartupResult,
recordGenerationJob,
+ recordGenerationStreamAttachment,
+ recordGenerationStreamEarlyBufferOverflow,
+ recordGenerationStreamRecovery,
recordGenerationStreamResumePendingEvents,
recordGenerationStreamSubscription,
recordOpenIDUserLookup,
@@ -527,6 +530,10 @@ describe('createMetrics', () => {
recordGenerationStreamSubscription('redis', 'resume', 'not_found');
recordGenerationStreamSubscription('redis', 'resume_state', 'missing');
recordGenerationStreamResumePendingEvents('memory', 3);
+ recordGenerationStreamEarlyBufferOverflow('redis');
+ recordGenerationStreamRecovery('redis', 'redis', 'success', 0.25, 5001, 12);
+ recordGenerationStreamAttachment('redis', 'attached', 4.5);
+ recordGenerationStreamAttachment('redis', 'bootstrap_slow');
const response = await request(app)
.get('/metrics')
@@ -544,6 +551,24 @@ describe('createMetrics', () => {
expect(response.text).toMatch(
/generation_stream_resume_pending_events_total\{store="memory"\} 3/,
);
+ expect(response.text).toMatch(
+ /generation_stream_early_buffer_overflows_total\{store="redis"\} 1/,
+ );
+ expect(response.text).toMatch(
+ /generation_stream_recoveries_total\{store="redis",method="redis",outcome="success"\} 1/,
+ );
+ expect(response.text).toMatch(
+ /generation_stream_recovery_duration_seconds_sum\{store="redis",method="redis",outcome="success"\} 0.25/,
+ );
+ expect(response.text).toMatch(
+ /generation_stream_attachment_outcomes_total\{store="redis",outcome="attached"\} 1/,
+ );
+ expect(response.text).toMatch(
+ /generation_stream_attachment_outcomes_total\{store="redis",outcome="bootstrap_slow"\} 1/,
+ );
+ expect(response.text).toMatch(
+ /generation_stream_first_attachment_delay_seconds_sum\{store="redis"\} 4.5/,
+ );
});
it('tracks cumulative agent startup milestones and terminal results', async () => {
diff --git a/packages/api/src/app/metrics.ts b/packages/api/src/app/metrics.ts
index d83a64ceb4..f44ae1ff59 100644
--- a/packages/api/src/app/metrics.ts
+++ b/packages/api/src/app/metrics.ts
@@ -157,6 +157,13 @@ export type GenerationStreamSubscriptionResult =
| 'error'
| 'found'
| 'missing';
+export type GenerationStreamRecoveryMethod = 'redis' | 'snapshot';
+export type GenerationStreamRecoveryOutcome = 'success' | 'failed' | 'not_required';
+export type GenerationStreamAttachmentOutcome =
+ | 'attached'
+ | 'bootstrap_slow'
+ | 'disconnected'
+ | 'never_attached';
export type RumProxyEndpoint = 'traces' | 'logs' | 'unknown';
export type RumProxyResult =
| 'success'
@@ -206,6 +213,19 @@ type GenerationJobMetrics = {
) => void;
recordResumePendingEvents: (store: GenerationJobStore, count: number) => void;
recordEarlyBufferOverflow: (store: GenerationJobStore) => void;
+ recordRecovery: (
+ store: GenerationJobStore,
+ method: GenerationStreamRecoveryMethod,
+ outcome: GenerationStreamRecoveryOutcome,
+ durationSeconds: number,
+ reconstructedEvents: number,
+ reconstructedContent: number,
+ ) => void;
+ recordAttachment: (
+ store: GenerationJobStore,
+ outcome: GenerationStreamAttachmentOutcome,
+ delaySeconds?: number,
+ ) => void;
};
let generationJobMetrics: GenerationJobMetrics = {
@@ -214,6 +234,8 @@ let generationJobMetrics: GenerationJobMetrics = {
recordSubscription: () => undefined,
recordResumePendingEvents: () => undefined,
recordEarlyBufferOverflow: () => undefined,
+ recordRecovery: () => undefined,
+ recordAttachment: () => undefined,
};
type AgentStartupMetrics = {
@@ -272,6 +294,8 @@ const resetMetricRecorders = (): void => {
recordSubscription: () => undefined,
recordResumePendingEvents: () => undefined,
recordEarlyBufferOverflow: () => undefined,
+ recordRecovery: () => undefined,
+ recordAttachment: () => undefined,
};
agentStartupMetrics = {
recordMilestone: () => undefined,
@@ -316,6 +340,32 @@ export function recordGenerationStreamEarlyBufferOverflow(store: GenerationJobSt
generationJobMetrics.recordEarlyBufferOverflow(store);
}
+export function recordGenerationStreamRecovery(
+ store: GenerationJobStore,
+ method: GenerationStreamRecoveryMethod,
+ outcome: GenerationStreamRecoveryOutcome,
+ durationSeconds: number,
+ reconstructedEvents: number,
+ reconstructedContent: number,
+): void {
+ generationJobMetrics.recordRecovery(
+ store,
+ method,
+ outcome,
+ durationSeconds,
+ reconstructedEvents,
+ reconstructedContent,
+ );
+}
+
+export function recordGenerationStreamAttachment(
+ store: GenerationJobStore,
+ outcome: GenerationStreamAttachmentOutcome,
+ delaySeconds?: number,
+): void {
+ generationJobMetrics.recordAttachment(store, outcome, delaySeconds);
+}
+
export function recordAgentStartupMilestone(
milestone: AgentStartupMilestone,
durationSeconds: number,
@@ -641,6 +691,52 @@ export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {
registers: [registry],
});
+ const generationStreamRecoveries = new Counter({
+ name: 'generation_stream_recoveries_total',
+ help: 'Early buffer recovery attempts by backing store, source, and outcome',
+ labelNames: ['store', 'method', 'outcome'] as const,
+ registers: [registry],
+ });
+
+ const generationStreamRecoveryDuration = new Histogram({
+ name: 'generation_stream_recovery_duration_seconds',
+ help: 'Time spent reconstructing an overflowed early generation stream',
+ labelNames: ['store', 'method', 'outcome'] as const,
+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30],
+ registers: [registry],
+ });
+
+ const generationStreamRecoveryEvents = new Histogram({
+ name: 'generation_stream_recovery_events',
+ help: 'Event count reconstructed during early buffer recovery',
+ labelNames: ['store', 'method', 'outcome'] as const,
+ buckets: [1, 10, 100, 1_000, 5_000, 10_000, 50_000],
+ registers: [registry],
+ });
+
+ const generationStreamRecoveryContent = new Histogram({
+ name: 'generation_stream_recovery_content_parts',
+ help: 'Content part count reconstructed during early buffer recovery',
+ labelNames: ['store', 'method', 'outcome'] as const,
+ buckets: [1, 5, 10, 25, 50, 100, 500, 1_000],
+ registers: [registry],
+ });
+
+ const generationStreamAttachments = new Counter({
+ name: 'generation_stream_attachment_outcomes_total',
+ help: 'Generation stream attachment lifecycle outcomes',
+ labelNames: ['store', 'outcome'] as const,
+ registers: [registry],
+ });
+
+ const generationStreamFirstAttachmentDelay = new Histogram({
+ name: 'generation_stream_first_attachment_delay_seconds',
+ help: 'Time from generation creation to its first subscriber attachment',
+ labelNames: ['store'] as const,
+ buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],
+ registers: [registry],
+ });
+
const agentStartupMilestoneDuration = new Histogram({
name: 'agent_startup_milestone_duration_seconds',
help: 'Cumulative agent chat startup latency from request ingress to each milestone',
@@ -765,6 +861,26 @@ export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {
recordResumePendingEvents: (store, count) =>
generationStreamResumePendingEvents.inc({ store }, count),
recordEarlyBufferOverflow: (store) => generationStreamEarlyBufferOverflows.inc({ store }),
+ recordRecovery: (
+ store,
+ method,
+ outcome,
+ durationSeconds,
+ reconstructedEvents,
+ reconstructedContent,
+ ) => {
+ const labels = { store, method, outcome };
+ generationStreamRecoveries.inc(labels);
+ generationStreamRecoveryDuration.observe(labels, durationSeconds);
+ generationStreamRecoveryEvents.observe(labels, reconstructedEvents);
+ generationStreamRecoveryContent.observe(labels, reconstructedContent);
+ },
+ recordAttachment: (store, outcome, delaySeconds) => {
+ generationStreamAttachments.inc({ store, outcome });
+ if (outcome === 'attached' && delaySeconds != null) {
+ generationStreamFirstAttachmentDelay.observe({ store }, delaySeconds);
+ }
+ },
};
agentStartupMetrics = {
@@ -841,7 +957,10 @@ export function createMetrics(options: MetricsOptions = {}): PrometheusMetrics {
if (completed) return;
completed = true;
- const requestLabels = { ...labels, status: completedBy === 'close' ? 499 : res.statusCode };
+ const requestLabels = {
+ ...labels,
+ status: completedBy === 'close' ? 499 : res.statusCode,
+ };
httpRequests.inc(requestLabels);
end(requestLabels);
httpRequestsInFlight.dec(labels);
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.ts
index 5fe8ff6d75..4322bb31b2 100644
--- a/packages/api/src/stream/GenerationJobManager.ts
+++ b/packages/api/src/stream/GenerationJobManager.ts
@@ -32,11 +32,25 @@ import type {
SteerQueueItem,
DetachedAgentEventActionStoreMode,
} from './interfaces/IJobStore';
+import type {
+ EarlyBufferOverflowState,
+ EarlyBufferRecoveryFailureReason,
+ EarlyBufferRecoveryOutcome,
+} from '../types/earlyBufferRecovery';
import type { AgentStartupTelemetry } from '~/agents/startup';
import type { RecoveredSteerPayload } from './SteerRecovery';
import type { SteerContentView } from './SteeringLifecycle';
import type { GenerationJobStore } from '~/app/metrics';
import type * as t from '~/types';
+import {
+ recordGenerationStreamEarlyBufferOverflow,
+ recordGenerationStreamResumePendingEvents,
+ recordGenerationStreamSubscription,
+ recordGenerationStreamAttachment,
+ recordGenerationStreamRecovery,
+ setGenerationJobsInFlight,
+ recordGenerationJob,
+} from '~/app/metrics';
import {
GenerationPublicationFencedError,
JobCreationSupersededError,
@@ -47,13 +61,6 @@ import {
PROVIDER_DRAIN_TIMEOUT_MS,
STEER_QUEUE_MAX_DEPTH,
} from './interfaces/IJobStore';
-import {
- recordGenerationStreamEarlyBufferOverflow,
- recordGenerationStreamResumePendingEvents,
- recordGenerationStreamSubscription,
- setGenerationJobsInFlight,
- recordGenerationJob,
-} from '~/app/metrics';
import { isRecoveredSteerPayload, RecoveredSteerPayloadMismatchError } from './SteerRecovery';
import { assertJobStoreV2 } from './jobStoreCapabilities';
@@ -144,6 +151,7 @@ const SHUTTING_DOWN_ERROR = 'Generation job manager is shutting down';
* this as an application-level generation error. */
export const TERMINAL_PUBLICATION_RECONNECT_ERROR =
'Terminal publication failed; reconnect to load the durable result';
+export const GENERATION_RECOVERY_FAILED_ERROR = 'generation_recovery_failed';
/** Upper bound for a terminal owner's required persistence barrier. A crashed
* owner leaves the durable pending bit behind; the next read or subscriber
* promotes it to conservative reconciliation after this window. */
@@ -155,6 +163,10 @@ const TERMINAL_PERSISTENCE_TIMEOUT_MS = 30_000;
* durable chunk log, in-memory reconnects recover from the resume snapshot. */
const EARLY_EVENT_BUFFER_MAX_EVENTS = 5_000;
const EARLY_EVENT_BUFFER_MAX_BYTES = 8 * 1024 * 1024;
+const SLOW_ATTACHMENT_BOOTSTRAP_MS = 3_000;
+const SUBSCRIBER_LEASE_TTL_MS = 30_000;
+const SUBSCRIBER_LEASE_REFRESH_MS = 10_000;
+const EARLY_BUFFER_OVERFLOW_PERSISTENCE_TIMEOUT_MS = 3_000;
const CLIENT_REQUEST_ID_PATTERN = /^[A-Za-z0-9:_-]{1,128}$/;
type TokenIdempotencyClaim = IdempotencyClaimValue & {
claimedAt: number;
@@ -664,6 +676,7 @@ interface RuntimeJobState {
* consumed it. Non-resume attachments are redirected to the resume path,
* which reconstructs the discarded output from durable/snapshot state. */
earlyEventBufferOverflowed?: true;
+ earlyBufferOverflow?: EarlyBufferOverflowState;
earlyEventSequencePromises: Array<Promise<void | number>>;
/** Initial subscribers eligible to receive the local pre-attachment replay. */
earlyReplayHandlers: Set<t.ChunkHandler>;
@@ -671,6 +684,8 @@ interface RuntimeJobState {
resumeCaptureHandlers: Set<(event: t.ServerSentEvent, sequence: number) => void>;
/** Monotonic local emission sequence used to establish an exact resume snapshot frontier. */
emissionSequence: number;
+ /** Count of emitted Redis envelopes that are eligible for durable append. */
+ durableEventSequence: number;
/** Emissions that started before an in-memory resume snapshot and must become snapshot-visible
* before the graph/job state is read. The event identity also suppresses their later publish. */
inFlightSnapshotEmissions: Map<
@@ -682,6 +697,16 @@ interface RuntimeJobState {
/** Coalesced delta publications emitted but not yet settled by a window flush. */
outstandingCoalescedReceipts?: number;
hasSubscriber: boolean;
+ everHadSubscriber: boolean;
+ /** Ordered, non-blocking durable subscriber lifecycle writes. */
+ subscriberStateWrite?: Promise<unknown>;
+ /** Whether this runtime's local subscriber group is represented durably. */
+ subscriberStateAttached: boolean;
+ subscriberLeaseId: string;
+ subscriberLeaseTimer?: ReturnType<typeof setInterval>;
+ subscriberLeaseRenewalPending?: boolean;
+ /** One-shot telemetry callback retained until a durable first-subscriber claim succeeds. */
+ onFirstSubscriberLeaseClaim?: (firstSubscriber: boolean) => void;
/** Advances whenever every local SSE subscriber for one attachment generation leaves. */
attachmentGeneration: number;
/** Attachment generation whose partial-response disconnect cleanup was most recently started. */
@@ -752,6 +777,9 @@ class GenerationJobManagerClass {
/** Serializes whole-array run-step snapshots so an older save cannot overwrite completion. */
private runStepWriteQueues = new Map<string, Promise<void>>();
+ /** Coalesces reconnects waiting on the same owner-side overflow flush. */
+ private earlyBufferOverflowWaiters = new Map<string, Promise<SerializableJobData | null>>();
+
/** Partial-response and disconnect-state writes still draining during shutdown. */
private subscriberCleanupPromises = new Set<Promise<void>>();
@@ -880,6 +908,7 @@ class GenerationJobManagerClass {
for (const runtime of this.runtimeState.values()) {
runtime.startupTelemetry?.end('aborted');
runtime.startupTelemetry = undefined;
+ this.stopSubscriberLease(runtime);
this.releaseAbortSubscription(runtime, true);
runtime.abortController.abort();
}
@@ -903,6 +932,7 @@ class GenerationJobManagerClass {
this.replayEventWriteQueues.clear();
this.tokenUsageWriteQueues.clear();
this.runStepWriteQueues.clear();
+ this.earlyBufferOverflowWaiters.clear();
}
this.ownedJobs.clear();
@@ -1599,6 +1629,26 @@ class GenerationJobManagerClass {
runtime.hasSubscriber = false;
runtime.attachmentGeneration++;
runtime.lastSubscriberCleanupGeneration = runtime.attachmentGeneration;
+ this.stopSubscriberLease(runtime);
+
+ runtime.subscriberStateWrite = (runtime.subscriberStateWrite ?? Promise.resolve())
+ .then(async () => {
+ if (!runtime.subscriberStateAttached) {
+ return;
+ }
+ await this.jobStore.detachSubscriber(
+ streamId,
+ runtime.createdAt,
+ runtime.subscriberLeaseId,
+ );
+ runtime.subscriberStateAttached = false;
+ })
+ .catch((detachError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to persist subscriber detachment',
+ detachError,
+ );
+ });
// Terminal delivery closes the SSE subscription too, but it is not a user
// disconnect. Running partial-response handlers here can overwrite the
@@ -1607,7 +1657,14 @@ class GenerationJobManagerClass {
return;
}
- const cleanup = this.persistSubscriberCleanup(streamId, runtime);
+ if (runtime.everHadSubscriber) {
+ recordGenerationStreamAttachment(this.storeLabel, 'disconnected');
+ }
+
+ const cleanup = Promise.all([
+ runtime.subscriberStateWrite,
+ this.persistSubscriberCleanup(streamId, runtime),
+ ]).then(() => undefined);
this.subscriberCleanupPromises.add(cleanup);
void cleanup.then(
() => this.subscriberCleanupPromises.delete(cleanup),
@@ -1619,6 +1676,49 @@ class GenerationJobManagerClass {
});
}
+ private stopSubscriberLease(runtime: RuntimeJobState): void {
+ if (runtime.subscriberLeaseTimer != null) {
+ clearInterval(runtime.subscriberLeaseTimer);
+ runtime.subscriberLeaseTimer = undefined;
+ }
+ }
+
+ private startSubscriberLease(streamId: string, runtime: RuntimeJobState): void {
+ if (runtime.subscriberLeaseTimer != null) {
+ return;
+ }
+ runtime.subscriberLeaseTimer = setInterval(() => {
+ if (!runtime.hasSubscriber || runtime.subscriberLeaseRenewalPending === true) {
+ return;
+ }
+ runtime.subscriberLeaseRenewalPending = true;
+ runtime.subscriberStateWrite = (runtime.subscriberStateWrite ?? Promise.resolve())
+ .then(() => {
+ const refreshedAt = Date.now();
+ return this.jobStore.claimFirstSubscriber(
+ streamId,
+ runtime.createdAt,
+ refreshedAt,
+ runtime.subscriberLeaseId,
+ refreshedAt + SUBSCRIBER_LEASE_TTL_MS,
+ );
+ })
+ .then((firstSubscriber) => {
+ runtime.subscriberStateAttached = true;
+ runtime.onFirstSubscriberLeaseClaim?.(firstSubscriber);
+ return firstSubscriber;
+ })
+ .catch((leaseError) => {
+ logger.error('[GenerationJobManager] Failed to renew subscriber lease', leaseError);
+ return false;
+ })
+ .finally(() => {
+ runtime.subscriberLeaseRenewalPending = false;
+ });
+ }, SUBSCRIBER_LEASE_REFRESH_MS);
+ runtime.subscriberLeaseTimer.unref?.();
+ }
+
private async persistSubscriberCleanup(
streamId: string,
runtime: RuntimeJobState,
@@ -1804,6 +1904,7 @@ class GenerationJobManagerClass {
return;
}
if (this.runtimeState.get(streamId) === predecessor) {
+ this.stopSubscriberLease(predecessor);
this.runtimeState.delete(streamId);
this.releaseJobOwnership(streamId, predecessor.createdAt);
this.releaseAbortSubscription(predecessor, true);
@@ -2283,7 +2384,9 @@ class GenerationJobManagerClass {
const { steerQuotesCapable, ...storedMetadata } = sanitizedMetadata;
const initialMetadata = {
...storedMetadata,
- ...(steerQuotesCapable === true && { steerQuotesExecutionId: creationAttemptId }),
+ ...(steerQuotesCapable === true && {
+ steerQuotesExecutionId: creationAttemptId,
+ }),
providerExecutionId: creationAttemptId,
providerDrained: true,
};
@@ -2572,6 +2675,7 @@ class GenerationJobManagerClass {
if (replacedRuntime) {
replacedRuntime.startupTelemetry?.end('replaced');
replacedRuntime.startupTelemetry = undefined;
+ this.stopSubscriberLease(replacedRuntime);
const durableReceipt = exactPredecessorsByEpoch.get(replacedRuntime.createdAt);
if (
durableReceipt == null ||
@@ -2611,8 +2715,12 @@ class GenerationJobManagerClass {
resumeCaptureHandlers: new Set(),
localErrorHandlers: new Set(),
emissionSequence: 0,
+ durableEventSequence: 0,
inFlightSnapshotEmissions: new Map(),
hasSubscriber: false,
+ everHadSubscriber: jobData.firstSubscriberAttachedAt != null,
+ subscriberStateAttached: false,
+ subscriberLeaseId: randomUUID(),
attachmentGeneration: 0,
};
this.runtimeState.set(streamId, runtime);
@@ -2695,6 +2803,7 @@ class GenerationJobManagerClass {
) {
this.releaseAbortSubscription(runtime);
runtime.abortController.abort();
+ this.stopSubscriberLease(runtime);
this.runtimeState.delete(streamId);
this.releaseJobOwnership(streamId, runtime.createdAt);
}
@@ -2778,6 +2887,7 @@ class GenerationJobManagerClass {
conversationId: jobData.conversationId,
checkpointNamespace: jobData.checkpointNamespace,
generationProtocolVersion: jobData.generationProtocolVersion,
+ earlyBufferOverflow: jobData.earlyBufferOverflow,
userMessage: jobData.userMessage,
responseMessageId: jobData.responseMessageId,
isRegenerate: jobData.isRegenerate,
@@ -2869,6 +2979,12 @@ class GenerationJobManagerClass {
const concurrentRuntime = this.runtimeState.get(streamId);
if (concurrentRuntime?.createdAt === jobData.createdAt) {
+ if (jobData.earlyBufferOverflow != null) {
+ concurrentRuntime.earlyBufferOverflow = jobData.earlyBufferOverflow;
+ concurrentRuntime.earlyEventBufferClosed = true;
+ concurrentRuntime.earlyEventBufferOverflowed = true;
+ this.resetEarlyEventBuffer(concurrentRuntime);
+ }
this.reconcileInactiveGeneration(streamId, jobData.createdAt, jobData, concurrentRuntime);
return concurrentRuntime;
}
@@ -2878,6 +2994,7 @@ class GenerationJobManagerClass {
if (concurrentRuntime) {
concurrentRuntime.startupTelemetry?.end('replaced');
concurrentRuntime.startupTelemetry = undefined;
+ this.stopSubscriberLease(concurrentRuntime);
this.releaseAbortSubscription(concurrentRuntime);
concurrentRuntime.abortController.abort();
}
@@ -2913,11 +3030,20 @@ class GenerationJobManagerClass {
resumeCaptureHandlers: new Set(),
localErrorHandlers: new Set(),
emissionSequence: 0,
+ durableEventSequence: jobData.durableEventCount ?? 0,
inFlightSnapshotEmissions: new Map(),
hasSubscriber: false,
+ everHadSubscriber: false,
+ subscriberStateAttached: false,
+ subscriberLeaseId: randomUUID(),
attachmentGeneration: 0,
finalEvent,
errorEvent: jobData.error,
+ earlyBufferOverflow: jobData.earlyBufferOverflow,
+ ...(jobData.earlyBufferOverflow != null && {
+ earlyEventBufferClosed: true,
+ earlyEventBufferOverflowed: true,
+ }),
};
this.runtimeState.set(streamId, runtime);
@@ -2948,6 +3074,7 @@ class GenerationJobManagerClass {
if (!confirmedJobData) {
this.releaseAbortSubscription(runtime);
runtime.abortController.abort();
+ this.stopSubscriberLease(runtime);
this.runtimeState.delete(streamId);
return null;
}
@@ -3403,7 +3530,11 @@ class GenerationJobManagerClass {
return { claimed: true, existing: value, source: 'primary' };
}
if (isClaimTakeoverOf(expectedClaim, observedPrimary)) {
- return { claimed: false, existing: observedPrimary, source: 'primary' };
+ return {
+ claimed: false,
+ existing: observedPrimary,
+ source: 'primary',
+ };
}
}
throw error;
@@ -3634,7 +3765,10 @@ class GenerationJobManagerClass {
status: TerminalJobClaim['status'],
error?: string,
expectedCreatedAt?: number,
- options: { persistencePending?: boolean; failedPauseActionId?: string } = {},
+ options: {
+ persistencePending?: boolean;
+ failedPauseActionId?: string;
+ } = {},
): Promise<TerminalJobClaim | null> {
if (
options.failedPauseActionId != null &&
@@ -3703,7 +3837,9 @@ class GenerationJobManagerClass {
...(sourceStatus === 'requires_action' && {
...(failedPauseBarrierId != null
? { expectActionId: failedPauseBarrierId }
- : jobData.pendingActionId != null && { expectActionId: jobData.pendingActionId }),
+ : jobData.pendingActionId != null && {
+ expectActionId: jobData.pendingActionId,
+ }),
}),
patch: {
completedAt,
@@ -3735,10 +3871,14 @@ class GenerationJobManagerClass {
const claim: TerminalJobClaim = Object.freeze({
streamId,
createdAt,
- ...(jobData.conversationId != null && { conversationId: jobData.conversationId }),
+ ...(jobData.conversationId != null && {
+ conversationId: jobData.conversationId,
+ }),
status,
...(terminalError != null && { error: terminalError }),
- ...(options.persistencePending === true && { persistencePending: true as const }),
+ ...(options.persistencePending === true && {
+ persistencePending: true as const,
+ }),
drainedSteers: Object.freeze([...drainedSteers]),
});
this.terminalClaimRuntimes.set(claim, runtime ?? null);
@@ -3917,8 +4057,68 @@ class GenerationJobManagerClass {
this.runtimeState.get(streamId) === claimedRuntime
? claimedRuntime
: undefined;
+ await runtime?.subscriberStateWrite?.catch(() => undefined);
let cleanupError: unknown;
let retainTerminalHostEvidence = false;
+ let persistedLifecycle: SerializableJobData | null | undefined = null;
+ if (runtime != null) {
+ persistedLifecycle = await this.jobStore.getJob(streamId).catch((lifecycleError) => {
+ logger.error('[GenerationJobManager] Failed to read terminal subscriber lifecycle', {
+ streamId,
+ error: lifecycleError instanceof Error ? lifecycleError.message : String(lifecycleError),
+ });
+ return undefined;
+ });
+ }
+ const generationHadSubscriber =
+ runtime?.everHadSubscriber === true ||
+ (persistedLifecycle?.createdAt === createdAt &&
+ persistedLifecycle.firstSubscriberAttachedAt != null);
+ const unresolvedOverflow =
+ runtime?.earlyBufferOverflow?.recoveryOutcome == null
+ ? runtime?.earlyBufferOverflow
+ : undefined;
+ if (runtime != null && unresolvedOverflow != null) {
+ let remoteSubscriberActive: boolean | undefined =
+ persistedLifecycle === undefined ? undefined : false;
+ if (persistedLifecycle?.createdAt === createdAt) {
+ remoteSubscriberActive = await this.jobStore
+ .hasActiveSubscriber(streamId, createdAt, Date.now())
+ .catch((leaseError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to read active subscriber leases',
+ leaseError,
+ );
+ return undefined;
+ });
+ }
+ /** A failed lease read is not evidence of an active subscriber. Leave the
+ * overflow unresolved so a later resume must still validate recovery. */
+ if (runtime.hasSubscriber || remoteSubscriberActive != null) {
+ const subscriberActive = runtime.hasSubscriber || remoteSubscriberActive === true;
+ let unobservedFailureReason: EarlyBufferRecoveryFailureReason | undefined;
+ if (!subscriberActive) {
+ unobservedFailureReason = generationHadSubscriber
+ ? 'subscriber_disconnected'
+ : 'subscriber_never_attached';
+ }
+ await this.settleEarlyBufferRecovery(
+ streamId,
+ runtime,
+ unresolvedOverflow,
+ subscriberActive ? 'not_required' : 'failed',
+ 0,
+ 0,
+ 0,
+ unobservedFailureReason,
+ ).catch((recoveryError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to settle unobserved early buffer recovery',
+ recoveryError,
+ );
+ });
+ }
+ }
// Error jobs stay durable long enough for late subscribers to receive the
// stored error. A publication failure must never bypass the finally cleanup.
@@ -3986,6 +4186,9 @@ class GenerationJobManagerClass {
retainTerminalHostEvidence = true;
} finally {
if (runtime && this.runtimeState.get(streamId) === runtime) {
+ if (!generationHadSubscriber) {
+ recordGenerationStreamAttachment(this.storeLabel, 'never_attached');
+ }
this.releaseAbortSubscription(runtime);
runtime.abortController.abort();
if (status === 'error') {
@@ -4004,6 +4207,7 @@ class GenerationJobManagerClass {
this.tokenUsageWriteQueues.delete(streamId);
this.runStepWriteQueues.delete(streamId);
if (status !== 'error' && this._cleanupOnComplete) {
+ this.stopSubscriberLease(runtime);
this.runtimeState.delete(streamId);
}
}
@@ -4275,7 +4479,9 @@ class GenerationJobManagerClass {
/** Text from content parts for fallback token counting; the persisted
* abort record keeps steered words (they reached the model context). */
let text = shouldPersistAbortContent
- ? parseTextParts(abortContent as TMessageContentParts[], false, { includeSteer: true })
+ ? parseTextParts(abortContent as TMessageContentParts[], false, {
+ includeSteer: true,
+ })
: '';
/** Claim terminal ownership and drain steers in one store transaction. A
@@ -4369,7 +4575,9 @@ class GenerationJobManagerClass {
const terminalClaim: TerminalJobClaim = Object.freeze({
streamId,
createdAt: jobData.createdAt,
- ...(jobData.conversationId != null && { conversationId: jobData.conversationId }),
+ ...(jobData.conversationId != null && {
+ conversationId: jobData.conversationId,
+ }),
status: 'aborted',
persistencePending: true,
drainedSteers: Object.freeze([...drainedSteers]),
@@ -4440,7 +4648,9 @@ class GenerationJobManagerClass {
abortContent = filterPersistableAbortContent(content);
shouldPersistAbortContent = abortContent.length > 0;
text = shouldPersistAbortContent
- ? parseTextParts(abortContent as TMessageContentParts[], false, { includeSteer: true })
+ ? parseTextParts(abortContent as TMessageContentParts[], false, {
+ includeSteer: true,
+ })
: '';
/** Detect "early abort" - aborted before any generation happened (e.g., during tool loading)
@@ -4583,6 +4793,7 @@ class GenerationJobManagerClass {
options?: t.SubscribeOptions,
prepared?: PreparedSubscription,
): Promise<(t.StreamSubscription & { activate?: () => void }) | null> {
+ const attachmentStartedAt = Date.now();
const subscriptionType = options?.skipBufferReplay ? 'resume' : 'initial';
if (options?.signal?.aborted) {
recordGenerationStreamSubscription(this.storeLabel, subscriptionType, 'error');
@@ -4823,7 +5034,9 @@ class GenerationJobManagerClass {
};
subscription = {
ready: transportSubscription.ready,
- ...(prepared?.deferDeliveryUntilActivated === true && { activate: activateDelivery }),
+ ...(prepared?.deferDeliveryUntilActivated === true && {
+ activate: activateDelivery,
+ }),
unsubscribe: (): void => {
if (!subscriptionActive) {
return;
@@ -4913,6 +5126,49 @@ class GenerationJobManagerClass {
if (!runtime.hasSubscriber) {
runtime.hasSubscriber = true;
+ const attachedAt = Date.now();
+ if (!runtime.everHadSubscriber) {
+ const bootstrapSlow = attachedAt - attachmentStartedAt >= SLOW_ATTACHMENT_BOOTSTRAP_MS;
+ runtime.everHadSubscriber = true;
+ runtime.onFirstSubscriberLeaseClaim = (firstSubscriber) => {
+ if (!firstSubscriber) {
+ return;
+ }
+ runtime.onFirstSubscriberLeaseClaim = undefined;
+ recordGenerationStreamAttachment(
+ this.storeLabel,
+ 'attached',
+ Math.max(0, attachedAt - runtime.createdAt) / 1000,
+ );
+ if (bootstrapSlow) {
+ recordGenerationStreamAttachment(this.storeLabel, 'bootstrap_slow');
+ }
+ };
+ }
+ this.startSubscriberLease(streamId, runtime);
+ const subscriberClaim = (runtime.subscriberStateWrite ?? Promise.resolve())
+ .then(async () => {
+ const firstSubscriber = await this.jobStore.claimFirstSubscriber(
+ streamId,
+ runtime.createdAt,
+ attachedAt,
+ runtime.subscriberLeaseId,
+ Date.now() + SUBSCRIBER_LEASE_TTL_MS,
+ );
+ runtime.subscriberStateAttached = true;
+ return firstSubscriber;
+ })
+ .catch((claimError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to persist subscriber attachment',
+ claimError,
+ );
+ return false;
+ });
+ runtime.subscriberStateWrite = subscriberClaim;
+ void subscriberClaim.then((firstSubscriber) =>
+ runtime.onFirstSubscriberLeaseClaim?.(firstSubscriber),
+ );
const attachmentGeneration = runtime.attachmentGeneration;
const earlyPublicationFence = this.waitForEarlyEventPublications(runtime);
if (!(await waitWhileAttached(earlyPublicationFence))) {
@@ -5225,11 +5481,11 @@ class GenerationJobManagerClass {
*
* @returns whether the event was accepted into the buffer.
*/
- private bufferEarlyEvent(
+ private async bufferEarlyEvent(
streamId: string,
runtime: RuntimeJobState,
event: t.ServerSentEvent,
- ): boolean {
+ ): Promise<boolean> {
if (runtime.earlyEventBufferClosed) {
return false;
}
@@ -5238,7 +5494,7 @@ class GenerationJobManagerClass {
runtime.earlyEventBuffer.length >= EARLY_EVENT_BUFFER_MAX_EVENTS ||
runtime.earlyEventBufferBytes + estimatedBytes > EARLY_EVENT_BUFFER_MAX_BYTES
) {
- this.overflowEarlyEventBuffer(streamId, runtime);
+ await this.overflowEarlyEventBuffer(streamId, runtime, 1, estimatedBytes);
return false;
}
runtime.earlyEventBuffer.push(event);
@@ -5246,18 +5502,119 @@ class GenerationJobManagerClass {
return true;
}
- private overflowEarlyEventBuffer(streamId: string, runtime: RuntimeJobState): void {
- const droppedEvents = runtime.earlyEventBuffer.length;
- const droppedBytes = runtime.earlyEventBufferBytes;
- this.resetEarlyEventBuffer(runtime);
+ private async overflowEarlyEventBuffer(
+ streamId: string,
+ runtime: RuntimeJobState,
+ rejectedEvents = 0,
+ rejectedBytes = 0,
+ ): Promise<void> {
+ const droppedEvents = runtime.earlyEventBuffer.length + rejectedEvents;
+ const droppedBytes = runtime.earlyEventBufferBytes + rejectedBytes;
+ const overflow: EarlyBufferOverflowState = {
+ id: randomUUID(),
+ occurredAt: Date.now(),
+ durableEvents: this._isRedis ? runtime.durableEventSequence : runtime.emissionSequence,
+ droppedEvents,
+ droppedBytes,
+ persistencePending: true,
+ };
runtime.earlyEventBufferClosed = true;
runtime.earlyEventBufferOverflowed = true;
+ runtime.earlyBufferOverflow = overflow;
recordGenerationStreamEarlyBufferOverflow(this.storeLabel);
logger.warn(
- `[GenerationJobManager] Early event buffer overflow for ${streamId}; ` +
- `discarded ${droppedEvents} buffered events (~${droppedBytes} bytes); ` +
- 'late subscribers will recover from durable/resume state',
+ '[GenerationJobManager] Early event buffer overflow; late subscriber recovery required',
+ {
+ recoveryId: overflow.id,
+ store: this.storeLabel,
+ droppedEvents,
+ droppedBytes,
+ },
);
+ try {
+ /** Publish an admission fence before releasing the local recovery copy.
+ * Detached Redis events are durably appended before publication below,
+ * so a replica that races before this write includes every earlier
+ * publication in its snapshot. A replica that observes this marker waits
+ * for the finalized frontier before it can activate. */
+ await this.jobStore.updateJob(streamId, { earlyBufferOverflow: overflow }, runtime.createdAt);
+ await this.jobStore.flushPendingAppends?.(streamId);
+ const frontierJob = await this.jobStore.getJob(streamId);
+ if (frontierJob?.createdAt !== runtime.createdAt) {
+ throw new Error('Early buffer overflow generation was replaced before persistence');
+ }
+ const finalizedOverflow: EarlyBufferOverflowState = {
+ ...overflow,
+ durableEvents: this._isRedis
+ ? (frontierJob.durableEventCount ?? runtime.durableEventSequence)
+ : runtime.emissionSequence,
+ persistencePending: false,
+ };
+ const finalized = await this.jobStore.finalizeEarlyBufferOverflow(
+ streamId,
+ runtime.createdAt,
+ overflow.id,
+ finalizedOverflow,
+ );
+ const persistedJob = await this.jobStore.getJob(streamId);
+ if (
+ !finalized ||
+ persistedJob?.createdAt !== runtime.createdAt ||
+ persistedJob.earlyBufferOverflow?.id !== overflow.id ||
+ persistedJob.earlyBufferOverflow.persistencePending === true
+ ) {
+ if (
+ persistedJob?.createdAt === runtime.createdAt &&
+ persistedJob.earlyBufferOverflow?.id === overflow.id &&
+ persistedJob.earlyBufferOverflow.recoveryOutcome != null
+ ) {
+ runtime.earlyBufferOverflow = persistedJob.earlyBufferOverflow;
+ this.resetEarlyEventBuffer(runtime);
+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR);
+ }
+ throw new Error('Early buffer overflow marker was not durably persisted');
+ }
+ runtime.earlyBufferOverflow = finalizedOverflow;
+ this.resetEarlyEventBuffer(runtime);
+ } catch (err) {
+ this.resetEarlyEventBuffer(runtime);
+ if (
+ err instanceof Error &&
+ err.message === GENERATION_RECOVERY_FAILED_ERROR &&
+ runtime.earlyBufferOverflow?.recoveryOutcome != null
+ ) {
+ throw err;
+ }
+ logger.error('[GenerationJobManager] Failed to persist early buffer overflow identity', {
+ recoveryId: overflow.id,
+ store: this.storeLabel,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ const settlement = {
+ recoveryMethod: this._isRedis ? ('redis' as const) : ('snapshot' as const),
+ recoveryOutcome: 'failed' as const,
+ recoveryCompletedAt: Date.now(),
+ recoveryFailureReason: 'overflow_marker_persistence_failed' as const,
+ };
+ runtime.earlyBufferOverflow = { ...overflow, ...settlement };
+ recordGenerationStreamRecovery(
+ this.storeLabel,
+ settlement.recoveryMethod,
+ settlement.recoveryOutcome,
+ Math.max(0, settlement.recoveryCompletedAt - overflow.occurredAt) / 1000,
+ 0,
+ 0,
+ );
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt).catch(
+ (terminalError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to terminalize after overflow marker persistence failure',
+ terminalError,
+ );
+ },
+ );
+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR, { cause: err });
+ }
}
/**
@@ -5345,6 +5702,106 @@ class GenerationJobManagerClass {
});
}
+ private async waitForFinalizedEarlyBufferOverflow(
+ streamId: string,
+ jobData: SerializableJobData,
+ ): Promise<SerializableJobData | null> {
+ const overflow = jobData.earlyBufferOverflow;
+ if (overflow?.persistencePending !== true) {
+ return jobData;
+ }
+
+ const waiterKey = `${streamId}:${jobData.createdAt}:${overflow.id}`;
+ const existing = this.earlyBufferOverflowWaiters.get(waiterKey);
+ if (existing) {
+ return existing;
+ }
+
+ const waiter = (async (): Promise<SerializableJobData | null> => {
+ const deadline = Date.now() + EARLY_BUFFER_OVERFLOW_PERSISTENCE_TIMEOUT_MS;
+ let delayMs = 100;
+ let current: SerializableJobData | null = jobData;
+ while (
+ current?.createdAt === jobData.createdAt &&
+ current.earlyBufferOverflow?.id === overflow.id &&
+ current.earlyBufferOverflow.persistencePending === true &&
+ Date.now() < deadline
+ ) {
+ const remainingMs = deadline - Date.now();
+ await new Promise<void>((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs)));
+ current = await this.jobStore.getJob(streamId);
+ delayMs = Math.min(delayMs * 2, 1000);
+ }
+ return current;
+ })();
+ this.earlyBufferOverflowWaiters.set(waiterKey, waiter);
+ try {
+ return await waiter;
+ } finally {
+ if (this.earlyBufferOverflowWaiters.get(waiterKey) === waiter) {
+ this.earlyBufferOverflowWaiters.delete(waiterKey);
+ }
+ }
+ }
+
+ private async settleEarlyBufferRecovery(
+ streamId: string,
+ runtime: RuntimeJobState,
+ overflow: EarlyBufferOverflowState,
+ outcome: EarlyBufferRecoveryOutcome,
+ durationMs: number,
+ reconstructedEvents: number,
+ reconstructedContent: number,
+ failureReason?: EarlyBufferRecoveryFailureReason,
+ ): Promise<EarlyBufferRecoveryOutcome | undefined> {
+ const recoveryMethod = this._isRedis ? 'redis' : 'snapshot';
+ const settlement = {
+ recoveryMethod,
+ recoveryOutcome: outcome,
+ recoveryCompletedAt: Date.now(),
+ ...(failureReason != null && { recoveryFailureReason: failureReason }),
+ } as const;
+ const settled = await this.jobStore.settleEarlyBufferRecovery(
+ streamId,
+ runtime.createdAt,
+ overflow.id,
+ settlement,
+ );
+ if (!settled) {
+ const winningJob = await this.jobStore.getJob(streamId);
+ const winningOverflow = winningJob?.earlyBufferOverflow;
+ if (
+ winningJob?.createdAt === runtime.createdAt &&
+ winningOverflow?.id === overflow.id &&
+ winningOverflow.recoveryOutcome != null
+ ) {
+ runtime.earlyBufferOverflow = winningOverflow;
+ return winningOverflow.recoveryOutcome;
+ }
+ return undefined;
+ }
+ runtime.earlyBufferOverflow = { ...overflow, ...settlement };
+ recordGenerationStreamRecovery(
+ this.storeLabel,
+ recoveryMethod,
+ outcome,
+ Math.max(0, durationMs) / 1000,
+ reconstructedEvents,
+ reconstructedContent,
+ );
+ logger.info('[GenerationJobManager] Early buffer recovery completed', {
+ recoveryId: overflow.id,
+ store: this.storeLabel,
+ method: recoveryMethod,
+ outcome,
+ durationMs,
+ reconstructedEvents,
+ reconstructedContent,
+ ...(failureReason != null && { failureReason }),
+ });
+ return outcome;
+ }
+
/**
* Snapshots resume state and attaches a paused live subscription.
*
@@ -5363,6 +5820,7 @@ class GenerationJobManagerClass {
recordGenerationStreamSubscription(this.storeLabel, 'resume', 'error');
return { subscription: null, resumeState: null, pendingEvents: [] };
}
+
if (this.rejectSubscriptionDuringShutdown('resume', onError)) {
return { subscription: null, resumeState: null, pendingEvents: [] };
}
@@ -5386,12 +5844,76 @@ class GenerationJobManagerClass {
return { subscription: null, resumeState: null, pendingEvents: [] };
}
+ if (runtime.earlyBufferOverflow?.persistencePending === true) {
+ const pendingMarker = runtime.earlyBufferOverflow;
+ const currentJob = await this.jobStore.getJob(streamId);
+ if (
+ currentJob?.createdAt !== runtime.createdAt ||
+ currentJob.earlyBufferOverflow?.id !== pendingMarker.id
+ ) {
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ const finalizedJob = await this.waitForFinalizedEarlyBufferOverflow(streamId, currentJob);
+ if (finalizedJob?.createdAt !== runtime.createdAt) {
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ await this.getOrCreateRuntimeState(streamId, finalizedJob);
+ if (finalizedJob.earlyBufferOverflow?.persistencePending === true) {
+ await this.settleEarlyBufferRecovery(
+ streamId,
+ runtime,
+ pendingMarker,
+ 'failed',
+ EARLY_BUFFER_OVERFLOW_PERSISTENCE_TIMEOUT_MS,
+ 0,
+ 0,
+ 'overflow_marker_persistence_failed',
+ );
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);
+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ }
+
+ if (
+ runtime.earlyBufferOverflow?.recoveryOutcome === 'failed' &&
+ !runtime.finalEvent &&
+ !runtime.errorEvent
+ ) {
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);
+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+
+ const snapshotOverflowId = runtime.earlyBufferOverflow?.id;
+ const pendingOverflow =
+ runtime.earlyBufferOverflow?.recoveryOutcome == null &&
+ !runtime.finalEvent &&
+ !runtime.errorEvent
+ ? runtime.earlyBufferOverflow
+ : undefined;
+ const recoveryStartedAt = pendingOverflow == null ? undefined : Date.now();
+ let recoveryContentSnapshot: Awaited<ReturnType<IJobStore['getContentParts']>> | undefined;
+ let recoverySettled = false;
+ if (pendingOverflow != null) {
+ logger.info('[GenerationJobManager] Early buffer recovery started', {
+ recoveryId: pendingOverflow.id,
+ store: this.storeLabel,
+ method: this._isRedis ? 'redis' : 'snapshot',
+ });
+ }
+
const capturedPendingEvents: t.ServerSentEvent[] = [];
const pendingEvents: t.ServerSentEvent[] = [];
const capturedEventSet = new Set<t.ServerSentEvent>();
const snapshotCoveredEventSet = new Set<t.ServerSentEvent>();
const seenEmissionEvents = new Set<t.ServerSentEvent>();
- const unclassifiedEmissions: Array<{ event: t.ServerSentEvent; sequence: number }> = [];
+ const unclassifiedEmissions: Array<{
+ event: t.ServerSentEvent;
+ sequence: number;
+ }> = [];
let snapshotFrontier = 0;
let snapshotClassified = this._isRedis;
const classifyEmission = (event: t.ServerSentEvent, sequence: number): void => {
@@ -5419,6 +5941,14 @@ class GenerationJobManagerClass {
const removeCaptureHandler = (): void => {
runtime.resumeCaptureHandlers.delete(capturePendingEvent);
};
+ const discardCapturedEvents = (): void => {
+ removeCaptureHandler();
+ capturedPendingEvents.length = 0;
+ pendingEvents.length = 0;
+ unclassifiedEmissions.length = 0;
+ capturedEventSet.clear();
+ snapshotCoveredEventSet.clear();
+ };
const restoreCapturedEvents = (): void => {
if (capturedPendingEvents.length === 0) {
return;
@@ -5441,7 +5971,14 @@ class GenerationJobManagerClass {
EARLY_EVENT_BUFFER_MAX_EVENTS ||
currentRuntime.earlyEventBufferBytes + restoredBytes > EARLY_EVENT_BUFFER_MAX_BYTES;
if (overflows) {
- this.overflowEarlyEventBuffer(streamId, currentRuntime);
+ void this.overflowEarlyEventBuffer(
+ streamId,
+ currentRuntime,
+ missingEvents.length,
+ restoredBytes,
+ ).catch((err) => {
+ logger.error('[GenerationJobManager] Failed to persist early buffer overflow', err);
+ });
} else {
currentRuntime.earlyEventBuffer = [
...missingEvents,
@@ -5465,7 +6002,12 @@ class GenerationJobManagerClass {
);
await Promise.all(preSnapshotEmissions.map(([, emission]) => emission.snapshotReady));
const [candidateState, candidateJob] = await Promise.all([
- this.getResumeState(streamId, options?.expectedCreatedAt),
+ this.getResumeState(streamId, options?.expectedCreatedAt, {
+ durableOnly: false,
+ onContentSnapshot: (snapshot) => {
+ recoveryContentSnapshot = snapshot;
+ },
+ }),
this.jobStore.getJob(streamId),
]);
if (
@@ -5492,11 +6034,108 @@ class GenerationJobManagerClass {
}
} else {
[resumeState, jobData] = await Promise.all([
- this.getResumeState(streamId, options?.expectedCreatedAt),
+ this.getResumeState(streamId, options?.expectedCreatedAt, {
+ durableOnly: pendingOverflow != null,
+ onContentSnapshot: (snapshot) => {
+ recoveryContentSnapshot = snapshot;
+ },
+ }),
this.jobStore.getJob(streamId),
]);
}
+ if (pendingOverflow != null && recoveryStartedAt != null) {
+ const contentSnapshot = recoveryContentSnapshot;
+ const reconstructedEvents = this._isRedis
+ ? (contentSnapshot?.reconstructedEventCount ?? 0)
+ : snapshotFrontier;
+ const durableEvents = this._isRedis
+ ? (contentSnapshot?.durableEventCount ?? 0)
+ : snapshotFrontier;
+ const reconstructedContent = resumeState?.aggregatedContent?.length ?? 0;
+ let failureReason: EarlyBufferRecoveryFailureReason | undefined;
+ if (resumeState == null || contentSnapshot == null) {
+ failureReason = this._isRedis ? 'durable_state_missing' : 'snapshot_missing';
+ } else if (
+ durableEvents < pendingOverflow.durableEvents ||
+ reconstructedEvents !== durableEvents
+ ) {
+ failureReason = 'durable_frontier_gap';
+ }
+ const durationMs = Date.now() - recoveryStartedAt;
+ if (failureReason == null) {
+ const winningOutcome = await this.settleEarlyBufferRecovery(
+ streamId,
+ runtime,
+ pendingOverflow,
+ 'success',
+ durationMs,
+ reconstructedEvents,
+ reconstructedContent,
+ );
+ recoverySettled = winningOutcome != null;
+ if (winningOutcome == null) {
+ const currentJob = await this.jobStore.getJob(streamId);
+ if (currentJob?.createdAt !== runtime.createdAt) {
+ removeCaptureHandler();
+ if (currentJob != null) {
+ await this.reconcileFencedRuntimeHandoff(streamId, runtime, currentJob);
+ }
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ }
+ if (winningOutcome === 'not_required') {
+ discardCapturedEvents();
+ return { subscription: null, resumeState, pendingEvents: [] };
+ }
+ if (winningOutcome !== 'success') {
+ discardCapturedEvents();
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);
+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ } else {
+ const winningOutcome = await this.settleEarlyBufferRecovery(
+ streamId,
+ runtime,
+ pendingOverflow,
+ 'failed',
+ durationMs,
+ reconstructedEvents,
+ reconstructedContent,
+ failureReason,
+ );
+ recoverySettled = winningOutcome != null;
+ if (winningOutcome == null) {
+ const currentJob = await this.jobStore.getJob(streamId);
+ if (currentJob?.createdAt !== runtime.createdAt) {
+ removeCaptureHandler();
+ if (currentJob != null) {
+ await this.reconcileFencedRuntimeHandoff(streamId, runtime, currentJob);
+ }
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ }
+ if (winningOutcome === 'not_required') {
+ discardCapturedEvents();
+ return { subscription: null, resumeState, pendingEvents: [] };
+ }
+ if (winningOutcome !== 'success') {
+ discardCapturedEvents();
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);
+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ /** Another replica proved the generation recoverable, but this
+ * attachment's snapshot is still incomplete. Keep the generation
+ * healthy and fail this request closed so its normal retry can read
+ * the durable winner's now-established frontier. */
+ throw new Error('Concurrent recovery succeeded; retry this attachment');
+ }
+ }
+
if (options?.signal?.aborted) {
removeCaptureHandler();
recordGenerationStreamSubscription(this.storeLabel, 'resume', 'error');
@@ -5580,6 +6219,25 @@ class GenerationJobManagerClass {
if (!liveJob || liveJob.createdAt !== runtime.createdAt) {
return cancelResumeSubscription();
}
+ const liveOverflow = liveJob.earlyBufferOverflow;
+ if (liveOverflow != null) {
+ runtime.earlyBufferOverflow = liveOverflow;
+ runtime.earlyEventBufferOverflowed = true;
+ runtime.earlyEventBufferClosed = true;
+ this.resetEarlyEventBuffer(runtime);
+ if (
+ !runtime.finalEvent &&
+ !runtime.errorEvent &&
+ (liveOverflow.recoveryOutcome === 'failed' || snapshotOverflowId !== liveOverflow.id)
+ ) {
+ /** The owner can discard its local buffer before the durable marker
+ * write becomes visible here. This final pre-activation read closes
+ * that window: retire the paused attachment and rebuild its snapshot
+ * through the durable-only recovery path now reflected in `runtime`. */
+ cancelResumeSubscription();
+ return this.subscribeWithResume(streamId, onChunk, onDone, onError, options);
+ }
+ }
if (!resumeState?.pendingAction) {
if (
liveJob?.status === 'requires_action' &&
@@ -5660,7 +6318,10 @@ class GenerationJobManagerClass {
(resumeState.aggregatedContent ?? []) as SteerContentView,
liveQueue,
(content ?? []) as SteerContentView,
- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },
+ {
+ conversationId: streamId,
+ responseMessageId: resumeState.responseMessageId,
+ },
);
if (gapEvents.length > 0) {
pendingEvents.push(...gapEvents);
@@ -5696,8 +6357,12 @@ class GenerationJobManagerClass {
const snapshotHasReasoningLabels =
resumeState?.aggregatedContent?.some(
(part) =>
- (part as { type?: string; reasoning_label_revision?: unknown } | null)?.type ===
- 'think' &&
+ (
+ part as {
+ type?: string;
+ reasoning_label_revision?: unknown;
+ } | null
+ )?.type === 'think' &&
typeof (part as { reasoning_label_revision?: unknown }).reasoning_label_revision ===
'number',
) === true;
@@ -5718,7 +6383,10 @@ class GenerationJobManagerClass {
typeof synthesizeActivityLabelGapEvents
>[0],
labelContent as Parameters<typeof synthesizeActivityLabelGapEvents>[1],
- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },
+ {
+ conversationId: streamId,
+ responseMessageId: resumeState.responseMessageId,
+ },
);
if (labelGapEvents.length > 0) {
pendingEvents.push(...(labelGapEvents as t.ServerSentEvent[]));
@@ -5728,7 +6396,10 @@ class GenerationJobManagerClass {
typeof synthesizeReasoningLabelGapEvents
>[0],
labelContent as Parameters<typeof synthesizeReasoningLabelGapEvents>[1],
- { conversationId: streamId, responseMessageId: resumeState.responseMessageId },
+ {
+ conversationId: streamId,
+ responseMessageId: resumeState.responseMessageId,
+ },
);
if (reasoningGapEvents.length > 0) {
pendingEvents.push(...(reasoningGapEvents as t.ServerSentEvent[]));
@@ -5775,6 +6446,51 @@ class GenerationJobManagerClass {
subscription?.unsubscribe();
restoreCapturedEvents();
snapshotCoveredEventSet.clear();
+ if (pendingOverflow != null && recoveryStartedAt != null && !recoverySettled) {
+ const winningOutcome = await this.settleEarlyBufferRecovery(
+ streamId,
+ runtime,
+ pendingOverflow,
+ 'failed',
+ Date.now() - recoveryStartedAt,
+ 0,
+ 0,
+ 'reconstruction_error',
+ ).catch((settlementError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to settle early buffer recovery error',
+ settlementError,
+ );
+ return undefined;
+ });
+ if (winningOutcome == null) {
+ const currentJob = await this.jobStore.getJob(streamId);
+ if (currentJob?.createdAt !== runtime.createdAt) {
+ if (currentJob != null) {
+ await this.reconcileFencedRuntimeHandoff(streamId, runtime, currentJob);
+ }
+ recordGenerationStreamSubscription(this.storeLabel, 'resume', 'not_found');
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ }
+ if (winningOutcome === 'not_required') {
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ if (winningOutcome !== 'success') {
+ await this.completeJob(
+ streamId,
+ GENERATION_RECOVERY_FAILED_ERROR,
+ runtime.createdAt,
+ ).catch((terminalError) => {
+ logger.error(
+ '[GenerationJobManager] Failed to terminalize early buffer recovery error',
+ terminalError,
+ );
+ });
+ onError?.(GENERATION_RECOVERY_FAILED_ERROR);
+ return { subscription: null, resumeState: null, pendingEvents: [] };
+ }
+ }
throw err;
}
}
@@ -6004,11 +6720,24 @@ class GenerationJobManagerClass {
options?.deliveredSteer == null &&
isCoalescableDeltaEvent(eventType);
+ const detachedAtAppend = !runtime.hasSubscriber;
+ const admissionFenceRequired = detachedAtAppend && !runtime.everHadSubscriber;
+ const subscriberAdmission = admissionFenceRequired
+ ? this.jobStore.hasSubscriberAttached(streamId, runtime.createdAt).catch((error) => {
+ logger.warn(
+ '[GenerationJobManager] Failed to read generation-wide subscriber admission',
+ error,
+ );
+ return false;
+ })
+ : undefined;
+
// For Redis mode, persist chunk for later reconstruction (fire-and-forget for resumability)
if (this._isRedis) {
// The SSE event structure is { event: string, data: unknown, ... }
// The aggregator expects { event: string, data: unknown } where data is the payload
if (eventType && eventData !== undefined) {
+ runtime.durableEventSequence += 1;
// Store in format expected by aggregateContent: { event, data }
const appendPromise = this.jobStore.appendChunk(
streamId,
@@ -6018,18 +6747,23 @@ class GenerationJobManagerClass {
coalescableDelta ? { coalesce: true } : undefined,
);
- if (options?.durable === true) {
+ /** Before the generation-wide first-subscriber admission, publication
+ * is the cross-replica boundary: its durable record must already exist
+ * before another replica can snapshot. The admission probe runs beside
+ * the append, so the first event after a remote attachment remains
+ * fenced and later events recover the normal fire-and-forget path. */
+ if (options?.durable === true || admissionFenceRequired) {
let appended: boolean;
try {
appended = await appendPromise;
} catch (error) {
- if (options.deliveredSteer == null) {
+ if (options?.deliveredSteer == null) {
this.retireRuntimeAfterDurableFence(streamId, runtime);
}
throw error;
}
if (appended === false) {
- if (options.deliveredSteer == null) {
+ if (options?.deliveredSteer == null) {
this.retireRuntimeAfterDurableFence(streamId, runtime);
}
throw new Error(`Durable chunk append was fenced out for ${streamId}`);
@@ -6074,8 +6808,12 @@ class GenerationJobManagerClass {
}
}
+ if ((await subscriberAdmission) === true) {
+ runtime.everHadSubscriber = true;
+ }
+
const detached = !runtime.hasSubscriber;
- const buffered = detached && this.bufferEarlyEvent(streamId, runtime, event);
+ const buffered = detached && (await this.bufferEarlyEvent(streamId, runtime, event));
if (detached && !this._isRedis) {
if (runtime.startupTelemetry) {
this.recordStartupEvent(runtime, event);
@@ -6471,6 +7209,7 @@ class GenerationJobManagerClass {
this.jobStore.clearContentState(streamId, runtime.createdAt);
if (this.runtimeState.get(streamId) === runtime) {
+ this.stopSubscriberLease(runtime);
this.runtimeState.delete(streamId);
this.runStepBuffers?.delete(streamId);
this.replayEventWriteQueues.delete(streamId);
@@ -6541,7 +7280,9 @@ class GenerationJobManagerClass {
this.cleanupFencedRuntime(streamId, runtime);
return;
}
- const retirement: FencedRuntimeRetirementContext = { controller: new AbortController() };
+ const retirement: FencedRuntimeRetirementContext = {
+ controller: new AbortController(),
+ };
this.fencedRuntimeRetirements.set(runtime, retirement);
this.scheduleFencedRuntimeRetirement(streamId, runtime, Date.now(), retirement);
}
@@ -6727,7 +7468,10 @@ class GenerationJobManagerClass {
} as ToolCallWithExecutionStatus)
: { ...existingCall, ...completedToolCall, executionStatus };
} else {
- completedCalls.push({ ...completedToolCall, executionStatus } as ToolCallWithExecutionStatus);
+ completedCalls.push({
+ ...completedToolCall,
+ executionStatus,
+ } as ToolCallWithExecutionStatus);
}
this.accumulateRunStep(
@@ -6929,7 +7673,9 @@ class GenerationJobManagerClass {
}
tokenUsage.push(event.data);
- const update: Partial<SerializableJobData> = { tokenUsage: JSON.stringify(tokenUsage) };
+ const update: Partial<SerializableJobData> = {
+ tokenUsage: JSON.stringify(tokenUsage),
+ };
/** Reconcile the resume snapshot to this call's ACTUAL prompt tokens. A primary
* usage is the post-invoke truth for the call the latest stored snapshot
@@ -7040,7 +7786,9 @@ class GenerationJobManagerClass {
// its pills — this is the authoritative writer of job.metadata.userMessage and
// would otherwise drop them (the emitted created message includes them).
...(Array.isArray(extra.manualSkills) &&
- extra.manualSkills.length > 0 && { manualSkills: extra.manualSkills }),
+ extra.manualSkills.length > 0 && {
+ manualSkills: extra.manualSkills,
+ }),
...(Array.isArray(extra.alwaysAppliedSkills) &&
extra.alwaysAppliedSkills.length > 0 && {
alwaysAppliedSkills: extra.alwaysAppliedSkills,
@@ -7541,7 +8289,11 @@ class GenerationJobManagerClass {
*/
const publish = (): Promise<void> =>
Promise.resolve().then(async () => {
- await this.eventTransport.emitPreempt?.(streamId, { op: 'clear', createdAt, steerIds });
+ await this.eventTransport.emitPreempt?.(streamId, {
+ op: 'clear',
+ createdAt,
+ steerIds,
+ });
});
return publish().then(
() => true,
@@ -7604,7 +8356,9 @@ class GenerationJobManagerClass {
conversationId: streamId,
steers: downgraded.map((steer) => ({
steerId: steer.steerId,
- ...(steer.clientSteerId && { clientSteerId: steer.clientSteerId }),
+ ...(steer.clientSteerId && {
+ clientSteerId: steer.clientSteerId,
+ }),
preempt: false,
preemptRevision: steer.preemptRevision ?? 0,
})),
@@ -7709,21 +8463,112 @@ class GenerationJobManagerClass {
async getResumeState(
streamId: string,
expectedCreatedAt?: number,
+ options?: {
+ durableOnly?: boolean;
+ validateEarlyBufferRecovery?: boolean;
+ onContentSnapshot?: (snapshot: Awaited<ReturnType<IJobStore['getContentParts']>>) => void;
+ },
): Promise<t.ResumeState | null> {
- const jobData = await this.jobStore.getJob(streamId);
+ const recoveryStartedAt =
+ options?.validateEarlyBufferRecovery === true ? Date.now() : undefined;
+ let jobData = await this.jobStore.getJob(streamId);
if (!jobData || (expectedCreatedAt != null && jobData.createdAt !== expectedCreatedAt)) {
return null;
}
+ if (jobData.earlyBufferOverflow?.persistencePending === true) {
+ const initialCreatedAt = jobData.createdAt;
+ jobData = await this.waitForFinalizedEarlyBufferOverflow(streamId, jobData);
+ if (!jobData || jobData.createdAt !== initialCreatedAt) {
+ return null;
+ }
+ if (
+ jobData.earlyBufferOverflow?.persistencePending === true &&
+ options?.validateEarlyBufferRecovery !== true
+ ) {
+ return null;
+ }
+ }
+ const unresolvedOverflowValidation =
+ options?.validateEarlyBufferRecovery === true &&
+ jobData.earlyBufferOverflow != null &&
+ jobData.earlyBufferOverflow.recoveryOutcome == null;
+ const validationRuntime =
+ options?.validateEarlyBufferRecovery === true
+ ? await this.getOrCreateRuntimeState(streamId, jobData)
+ : undefined;
+ if (
+ options?.validateEarlyBufferRecovery === true &&
+ (validationRuntime == null || validationRuntime.createdAt !== jobData.createdAt)
+ ) {
+ throw new Error('Generation changed during HITL recovery validation');
+ }
+ if (
+ options?.validateEarlyBufferRecovery === true &&
+ jobData.earlyBufferOverflow?.recoveryOutcome === 'failed'
+ ) {
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, jobData.createdAt);
+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR);
+ }
/** Independent reads (streamId-only): parallel to collapse 3 Redis round trips into 1.
* Safe despite readCachedGraph's cache-drop side effect — each call catches its own
* unusable-graph throw and falls back to reconstruction, so ordering cannot change the result. */
const [result, runSteps, queuedSteers, claimedSteers] = await Promise.all([
- this.jobStore.getContentParts(streamId, jobData.createdAt),
+ this.jobStore.getContentParts(streamId, jobData.createdAt, {
+ durableOnly: options?.durableOnly === true || unresolvedOverflowValidation,
+ }),
this.jobStore.getRunSteps(streamId, jobData.createdAt),
this.jobStore.peekSteers(streamId, jobData.createdAt),
this.jobStore.peekClaimedSteers(streamId, jobData.createdAt),
]);
+ options?.onContentSnapshot?.(result);
+ const pendingOverflow = unresolvedOverflowValidation ? jobData.earlyBufferOverflow : undefined;
+ if (pendingOverflow != null) {
+ const runtime = validationRuntime!;
+ const reconstructedEvents = this._isRedis
+ ? (result?.reconstructedEventCount ?? 0)
+ : runtime.emissionSequence;
+ const durableEvents = this._isRedis
+ ? (result?.durableEventCount ?? 0)
+ : runtime.emissionSequence;
+ let failureReason: EarlyBufferRecoveryFailureReason | undefined;
+ if (pendingOverflow.persistencePending === true) {
+ failureReason = 'overflow_marker_persistence_failed';
+ } else if (result == null) {
+ failureReason = 'durable_state_missing';
+ } else if (
+ durableEvents < pendingOverflow.durableEvents ||
+ reconstructedEvents !== durableEvents
+ ) {
+ failureReason = 'durable_frontier_gap';
+ }
+ const winningOutcome = await this.settleEarlyBufferRecovery(
+ streamId,
+ runtime,
+ pendingOverflow,
+ failureReason == null ? 'success' : 'failed',
+ recoveryStartedAt == null ? 0 : Date.now() - recoveryStartedAt,
+ reconstructedEvents,
+ result?.content.length ?? 0,
+ failureReason,
+ );
+ if (winningOutcome == null) {
+ const currentJob = await this.jobStore.getJob(streamId);
+ if (currentJob?.createdAt !== jobData.createdAt) {
+ throw new Error('Generation changed during HITL recovery validation');
+ }
+ }
+ if (failureReason != null && winningOutcome === 'success') {
+ throw new Error('Concurrent recovery succeeded; retry HITL resume');
+ }
+ if (winningOutcome === 'not_required') {
+ throw new Error('HITL recovery no longer required for this generation');
+ }
+ if (winningOutcome !== 'success') {
+ await this.completeJob(streamId, GENERATION_RECOVERY_FAILED_ERROR, runtime.createdAt);
+ throw new Error(GENERATION_RECOVERY_FAILED_ERROR);
+ }
+ }
const reconstructedContent = result?.content ?? [];
const bufferState = this.runStepBuffers?.get(streamId);
const bufferedRunSteps = bufferState?.createdAt === jobData.createdAt ? bufferState.steps : [];
@@ -8263,6 +9108,7 @@ class GenerationJobManagerClass {
currentJob,
observedRuntime,
);
+ this.stopSubscriberLease(observedRuntime);
this.runtimeState.delete(streamId);
this.runStepBuffers?.delete(streamId);
this.replayEventWriteQueues.delete(streamId);
@@ -8320,6 +9166,7 @@ class GenerationJobManagerClass {
observedRuntime.startupTelemetry?.end('error', new Error(REAPED_JOB_ERROR));
observedRuntime.startupTelemetry = undefined;
this.releaseAbortSubscription(observedRuntime);
+ this.stopSubscriberLease(observedRuntime);
this.runtimeState.delete(streamId);
runningJobsChanged = this.ownedJobs.delete(streamId) || runningJobsChanged;
this.runStepBuffers?.delete(streamId);
@@ -8552,6 +9399,7 @@ class GenerationJobManagerClass {
for (const runtime of this.runtimeState.values()) {
runtime.startupTelemetry?.end('aborted');
runtime.startupTelemetry = undefined;
+ this.stopSubscriberLease(runtime);
this.releaseAbortSubscription(runtime, true);
runtime.abortController.abort();
}
@@ -8572,6 +9420,7 @@ class GenerationJobManagerClass {
this.replayEventWriteQueues.clear();
this.tokenUsageWriteQueues.clear();
this.runStepWriteQueues.clear();
+ this.earlyBufferOverflowWaiters.clear();
logger.debug('[GenerationJobManager] Destroyed');
}
diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
index 868dcab480..90c64e626f 100644
--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts
@@ -1,15 +1,17 @@
/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */
import type { Redis, Cluster } from 'ioredis';
import type { ServerSentEvent, StreamEvent, CreatedEvent } from '~/types';
+import type { IJobStoreV2 } from '~/stream/interfaces/IJobStore';
+import {
+ GenerationJobManagerClass,
+ GENERATION_RECOVERY_FAILED_ERROR,
+ TERMINAL_PUBLICATION_RECONNECT_ERROR,
+} from '~/stream/GenerationJobManager';
import {
ioredisClient as staticRedisClient,
keyvRedisClient as staticKeyvClient,
keyvRedisClientReady,
} from '~/cache/redisClients';
-import {
- GenerationJobManagerClass,
- TERMINAL_PUBLICATION_RECONNECT_ERROR,
-} from '~/stream/GenerationJobManager';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
@@ -128,11 +130,19 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_run_step',
- data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },
+ data: {
+ id: 'step-1',
+ runId: 'run-1',
+ index: 0,
+ stepDetails: { type: 'message_creation' },
+ },
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: 'Hello' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'Hello' } },
+ },
});
await new Promise((resolve) => setTimeout(resolve, delay));
@@ -143,7 +153,10 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: ' world' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: ' world' } },
+ },
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
@@ -185,7 +198,9 @@ describe('GenerationJobManager Integration Tests', () => {
expect(retrieved?.streamId).toBe(streamId);
// Update job
- await GenerationJobManager.updateMetadata(streamId, { sender: 'TestAgent' });
+ await GenerationJobManager.updateMetadata(streamId, {
+ sender: 'TestAgent',
+ });
const updated = await GenerationJobManager.getJob(streamId);
expect(updated?.metadata?.sender).toBe('TestAgent');
@@ -269,7 +284,9 @@ describe('GenerationJobManager Integration Tests', () => {
expect(hasJob).toBe(true);
// Update and verify
- await GenerationJobManager.updateMetadata(streamId, { sender: 'RedisAgent' });
+ await GenerationJobManager.updateMetadata(streamId, {
+ sender: 'RedisAgent',
+ });
const updated = await GenerationJobManager.getJob(streamId);
expect(updated?.metadata?.sender).toBe('RedisAgent');
@@ -1049,12 +1066,18 @@ describe('GenerationJobManager Integration Tests', () => {
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step_delta',
- data: { id: 'step-1', delta: { type: 'tool_calls', tool_calls: [{ args: '{' }] } },
+ data: {
+ id: 'step-1',
+ delta: { type: 'tool_calls', tool_calls: [{ args: '{' }] },
+ },
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step_delta',
- data: { id: 'step-1', delta: { type: 'tool_calls', tool_calls: [{ args: '}' }] } },
+ data: {
+ id: 'step-1',
+ delta: { type: 'tool_calls', tool_calls: [{ args: '}' }] },
+ },
});
await GenerationJobManager.emitChunk(streamId, {
@@ -1116,10 +1139,16 @@ describe('GenerationJobManager Integration Tests', () => {
const emitPromises: Promise<void>[] = [];
for (let i = 0; i < 10; i++) {
emitPromises.push(
- GenerationJobManager.emitChunk(streamId1, { event: 'test', data: { index: i } }),
+ GenerationJobManager.emitChunk(streamId1, {
+ event: 'test',
+ data: { index: i },
+ }),
);
emitPromises.push(
- GenerationJobManager.emitChunk(streamId2, { event: 'test', data: { index: i * 100 } }),
+ GenerationJobManager.emitChunk(streamId2, {
+ event: 'test',
+ data: { index: i * 100 },
+ }),
);
}
await Promise.all(emitPromises);
@@ -1275,7 +1304,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 10; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `word${i} ` } }, index: i },
+ data: {
+ delta: { content: { type: 'text', text: `word${i} ` } },
+ index: i,
+ },
});
}
@@ -1359,7 +1391,10 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: ' Live!' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: ' Live!' } },
+ },
});
await new Promise((resolve) => setTimeout(resolve, 20));
@@ -1403,7 +1438,12 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_run_step',
- data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },
+ data: {
+ id: 'step-1',
+ runId: 'run-1',
+ index: 0,
+ stepDetails: { type: 'message_creation' },
+ },
});
await new Promise((resolve) => setTimeout(resolve, 10));
@@ -1412,7 +1452,10 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: 'buffered' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'buffered' } },
+ },
});
const sub2Events: ServerSentEvent[] = [];
@@ -1580,7 +1623,10 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: ' Live!' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: ' Live!' } },
+ },
});
await new Promise((resolve) => setTimeout(resolve, 200));
@@ -1619,7 +1665,10 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: 'detached-redis' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'detached-redis' } },
+ },
});
await new Promise((resolve) => setTimeout(resolve, 100));
@@ -1636,7 +1685,10 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: ' live' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: ' live' } },
+ },
});
await new Promise((resolve) => setTimeout(resolve, 200));
@@ -1657,6 +1709,22 @@ describe('GenerationJobManager Integration Tests', () => {
* detached run.
*/
+ async function forceEarlyBufferOverflow(
+ manager: GenerationJobManagerClass,
+ streamId: string,
+ ): Promise<void> {
+ const runtime = (
+ manager as unknown as {
+ runtimeState: Map<string, unknown>;
+ }
+ ).runtimeState.get(streamId)!;
+ await (
+ manager as unknown as {
+ overflowEarlyEventBuffer: (id: string, runtime: unknown) => Promise<void>;
+ }
+ ).overflowEarlyEventBuffer(streamId, runtime);
+ }
+
testRedis(
'detached generation keeps the local buffer empty after first attachment (Redis)',
async () => {
@@ -1696,17 +1764,26 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 5; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: bigText } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: bigText } },
+ },
});
}
const stats = manager.getRuntimeStats();
expect(stats.earlyBufferedEvents).toBe(0);
expect(stats.earlyBufferedBytes).toBe(0);
+ const overflow = (await manager.getJob(streamId))?.metadata.earlyBufferOverflow;
+ expect(overflow?.droppedEvents).toBe(4);
+ expect(overflow?.droppedBytes).toBeGreaterThan(8 * 1024 * 1024);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: 'after-overflow' } } },
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'after-overflow' } },
+ },
});
expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);
@@ -1730,64 +1807,84 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.destroy();
});
- testRedis('redirects a post-overflow first attachment to resume recovery (Redis)', async () => {
- const manager = createRedisManager();
- const streamId = `overflow-redirect-${Date.now()}`;
- await manager.createJob(streamId, 'user-1');
-
- await manager.emitChunk(streamId, {
- event: 'on_run_step',
- data: {
- id: 'step-1',
- runId: 'run-1',
- index: 0,
- stepDetails: { type: 'message_creation' },
- },
+ test('replays a completed result after the overflow reconnect was forced', async () => {
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
});
- const bigText = 'y'.repeat(2 * 1024 * 1024);
- for (let i = 0; i < 5; i++) {
- await manager.emitChunk(streamId, {
- event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: bigText } } },
- });
- }
- expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);
+ manager.initialize();
+ const streamId = `overflow-complete-reconnect-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(manager, streamId);
- const errors: string[] = [];
- const sub = await manager.subscribe(
+ const reconnectErrors: string[] = [];
+ await manager.subscribe(
streamId,
() => {},
undefined,
- (error) => errors.push(error),
+ (error) => reconnectErrors.push(error),
);
- await new Promise((resolve) => setTimeout(resolve, 100));
- expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
- sub?.unsubscribe();
- await new Promise((resolve) => setTimeout(resolve, 100));
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(reconnectErrors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
- /** The resume path the client falls back to reconstructs the
- * discarded output from the durable chunk log. */
- const resumeState = await manager.getResumeState(streamId);
- expect(JSON.stringify(resumeState?.aggregatedContent ?? [])).toContain('yyyy');
+ const finalEvent = {
+ final: true,
+ conversation: { conversationId: streamId },
+ responseMessage: { text: 'complete' },
+ } as ServerSentEvent;
+ await manager.emitDone(streamId, finalEvent);
+ await manager.completeJob(streamId);
+ const terminalEvents: ServerSentEvent[] = [];
+ const resumed = await manager.subscribeWithResume(
+ streamId,
+ () => {},
+ (event) => terminalEvents.push(event),
+ );
+ resumed.subscription?.activate();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(terminalEvents).toEqual([finalEvent]);
+ resumed.subscription?.unsubscribe();
await manager.destroy();
});
- test('buffers detached events until the cap in in-memory mode', async () => {
- const manager = createInMemoryManager();
- const streamId = `buf-below-cap-${Date.now()}`;
+ test('reconciles a generation replaced while recovery settles', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `overflow-replaced-during-settlement-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
+ const originalCreatedAt = (await jobStore.getJob(streamId))!.createdAt;
+ await forceEarlyBufferOverflow(manager, streamId);
+ jest.spyOn(jobStore, 'settleEarlyBufferRecovery').mockImplementationOnce(async () => {
+ await jobStore.createJob(streamId, 'user-1');
+ return false;
+ });
- await setupDisconnectedStream(manager, streamId, 10);
-
- const stats = manager.getRuntimeStats();
- expect(stats.earlyBufferedEvents).toBe(2);
- expect(stats.earlyBufferedBytes).toBeGreaterThan(0);
+ const errors: string[] = [];
+ const result = await manager.subscribeWithResume(
+ streamId,
+ () => {},
+ undefined,
+ (error) => errors.push(error),
+ { expectedCreatedAt: originalCreatedAt },
+ );
+ expect(result.subscription).toBeNull();
+ expect(errors).not.toContain(GENERATION_RECOVERY_FAILED_ERROR);
+ expect((await jobStore.getJob(streamId))!.createdAt).toBeGreaterThan(originalCreatedAt);
await manager.destroy();
});
- test('caps captured events restored by a resume canceled before activation', async () => {
+ test('reconciles a generation replaced while a recovery error settles', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
@@ -1796,150 +1893,1047 @@ describe('GenerationJobManager Integration Tests', () => {
isRedis: false,
});
manager.initialize();
- const streamId = `restore-cap-${Date.now()}`;
+ const streamId = `overflow-replaced-during-error-settlement-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
-
- await setupDisconnectedStream(manager, streamId, 10);
-
- /** Arm only after the snapshot completes so the gate parks the resume
- * in its post-attachment steer reconciliation, the window where
- * emissions are captured per-resume instead of buffered. */
- let armed = false;
- const originalGetResumeState = manager.getResumeState.bind(manager);
+ const originalCreatedAt = (await jobStore.getJob(streamId))!.createdAt;
+ await forceEarlyBufferOverflow(manager, streamId);
jest
.spyOn(manager, 'getResumeState')
- .mockImplementation(
- async (...args: Parameters<GenerationJobManagerClass['getResumeState']>) => {
- const result = await originalGetResumeState(...args);
- armed = true;
- return result;
- },
- );
- let releaseGate!: () => void;
- const gate = new Promise<void>((resolve) => (releaseGate = resolve));
- let gateReached!: () => void;
- const reached = new Promise<void>((resolve) => (gateReached = resolve));
- const originalPeek = jobStore.peekSteers.bind(jobStore);
- let gated = true;
- jest
- .spyOn(jobStore, 'peekSteers')
- .mockImplementation(async (...args: Parameters<InMemoryJobStore['peekSteers']>) => {
- if (armed && gated) {
- gated = false;
- gateReached();
- await gate;
- }
- return originalPeek(...args);
- });
-
- const resumePromise = manager.subscribeWithResume(streamId, () => {});
-
- await reached;
- const bigText = 'z'.repeat(2 * 1024 * 1024);
- for (let i = 0; i < 5; i++) {
- await manager.emitChunk(streamId, {
- event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: bigText } } },
- });
- }
- releaseGate();
-
- const { subscription } = await resumePromise;
- expect(subscription).not.toBeNull();
- subscription!.unsubscribe();
-
- /** The ~10MB of captured events must not survive restoration. */
- const stats = manager.getRuntimeStats();
- expect(stats.earlyBufferedEvents).toBe(0);
- expect(stats.earlyBufferedBytes).toBe(0);
+ .mockRejectedValueOnce(new Error('snapshot unavailable'));
+ jest.spyOn(jobStore, 'settleEarlyBufferRecovery').mockImplementationOnce(async () => {
+ await jobStore.createJob(streamId, 'user-1');
+ return false;
+ });
- /** Restoration overflowed, so a non-resume attach takes the redirect. */
const errors: string[] = [];
- const probe = await manager.subscribe(
+ const result = await manager.subscribeWithResume(
streamId,
() => {},
undefined,
(error) => errors.push(error),
+ { expectedCreatedAt: originalCreatedAt },
);
- await new Promise((resolve) => setTimeout(resolve, 20));
- expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
- probe?.unsubscribe();
+ expect(result.subscription).toBeNull();
+ expect(errors).not.toContain(GENERATION_RECOVERY_FAILED_ERROR);
+ expect((await jobStore.getJob(streamId))!.createdAt).toBeGreaterThan(originalCreatedAt);
await manager.destroy();
});
- });
-
- describe('Atomic subscribeWithResume', () => {
- test('should return empty pendingEvents for pre-snapshot buffer events (in-memory)', async () => {
- const manager = createInMemoryManager();
- const streamId = `atomic-drain-${Date.now()}`;
- await manager.createJob(streamId, 'user-1');
-
- const sub1 = await manager.subscribe(streamId, () => {});
- await new Promise((resolve) => setTimeout(resolve, 10));
- sub1?.unsubscribe();
- await new Promise((resolve) => setTimeout(resolve, 20));
- await manager.emitChunk(streamId, {
- event: 'on_run_step',
- data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },
+ test('does not overwrite recovery settlement when the overflow owner finishes flushing', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
});
- await manager.emitChunk(streamId, {
- event: 'on_message_delta',
- data: { id: 'step-1', delta: { content: { type: 'text', text: 'buffered' } } },
+ manager.initialize();
+ const streamId = `overflow-finalize-settlement-race-${Date.now()}`;
+ const job = await manager.createJob(streamId, 'user-1');
+ const originalFinalize = jobStore.finalizeEarlyBufferOverflow.bind(jobStore);
+ jest
+ .spyOn(jobStore, 'finalizeEarlyBufferOverflow')
+ .mockImplementationOnce(async (id, createdAt, overflowId, finalizedOverflow) => {
+ await jobStore.settleEarlyBufferRecovery(id, createdAt, overflowId, {
+ recoveryMethod: 'snapshot',
+ recoveryOutcome: 'failed',
+ recoveryCompletedAt: Date.now(),
+ recoveryFailureReason: 'overflow_marker_persistence_failed',
+ });
+ return originalFinalize(id, createdAt, overflowId, finalizedOverflow);
+ });
+
+ await expect(forceEarlyBufferOverflow(manager, streamId)).rejects.toThrow(
+ GENERATION_RECOVERY_FAILED_ERROR,
+ );
+ expect((await jobStore.getJob(streamId))?.earlyBufferOverflow).toMatchObject({
+ recoveryOutcome: 'failed',
+ recoveryFailureReason: 'overflow_marker_persistence_failed',
});
+ expect((await jobStore.getJob(streamId))?.createdAt).toBe(job.createdAt);
- const liveEvents: ServerSentEvent[] = [];
- const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(
+ await manager.destroy();
+ });
+
+ test('coalesces pending overflow waiters and backs off their store read', async () => {
+ jest.useFakeTimers();
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport: new InMemoryEventTransport(), isRedis: false });
+ manager.initialize();
+ const streamId = `overflow-shared-waiter-${Date.now()}`;
+ const job = await manager.createJob(streamId, 'user-1');
+ const pendingOverflow = {
+ id: 'pending-overflow',
+ occurredAt: Date.now(),
+ durableEvents: 1,
+ droppedEvents: 1,
+ droppedBytes: 1,
+ persistencePending: true,
+ };
+ await jobStore.updateJob(streamId, { earlyBufferOverflow: pendingOverflow }, job.createdAt);
+ const pendingJob = (await jobStore.getJob(streamId))!;
+ const getJob = jest.spyOn(jobStore, 'getJob');
+ const waitForFinalized = (
+ manager as unknown as {
+ waitForFinalizedEarlyBufferOverflow: (
+ id: string,
+ data: typeof pendingJob,
+ ) => Promise<typeof pendingJob | null>;
+ }
+ ).waitForFinalizedEarlyBufferOverflow.bind(manager);
+
+ const first = waitForFinalized(streamId, pendingJob);
+ const second = waitForFinalized(streamId, pendingJob);
+ await jest.advanceTimersByTimeAsync(99);
+ expect(getJob).not.toHaveBeenCalled();
+ await jobStore.updateJob(
streamId,
- (event) => liveEvents.push(event),
+ { earlyBufferOverflow: { ...pendingOverflow, persistencePending: false } },
+ job.createdAt,
);
+ await jest.advanceTimersByTimeAsync(1);
- expect(resumeState).not.toBeNull();
- expect(pendingEvents.length).toBe(0);
- expect(liveEvents.length).toBe(0);
+ await expect(Promise.all([first, second])).resolves.toEqual([
+ expect.objectContaining({
+ earlyBufferOverflow: expect.objectContaining({ persistencePending: false }),
+ }),
+ expect.objectContaining({
+ earlyBufferOverflow: expect.objectContaining({ persistencePending: false }),
+ }),
+ ]);
+ expect(getJob).toHaveBeenCalledTimes(1);
- subscription?.unsubscribe();
await manager.destroy();
+ jest.useRealTimers();
});
- test('should return empty pendingEvents when buffer is empty', async () => {
- const manager = createInMemoryManager();
- const streamId = `atomic-empty-${Date.now()}`;
+ test('fences publication only until generation-wide subscriber admission', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const eventTransport = new InMemoryEventTransport();
+ let releaseAppend!: (appended: boolean) => void;
+ const pendingAppend = new Promise<boolean>((resolve) => {
+ releaseAppend = resolve;
+ });
+ const append = jest.spyOn(jobStore, 'appendChunk').mockReturnValueOnce(pendingAppend);
+ jest.spyOn(jobStore, 'hasSubscriberAttached').mockResolvedValue(true);
+ const publish = jest.spyOn(eventTransport, 'emitChunk');
+ const manager = new GenerationJobManagerClass();
+ manager.configure({ jobStore, eventTransport, isRedis: true });
+ manager.initialize();
+ const streamId = `detached-append-before-publish-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
- const sub1 = await manager.subscribe(streamId, () => {});
- await new Promise((resolve) => setTimeout(resolve, 10));
-
- await manager.emitChunk(streamId, {
+ const emission = manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: 'delivered' } } },
+ data: { delta: { content: { type: 'text', text: 'durable first' } } },
});
- await new Promise((resolve) => setTimeout(resolve, 10));
+ await Promise.resolve();
+ expect(publish).not.toHaveBeenCalled();
+ releaseAppend(true);
+ await emission;
+ expect(publish).toHaveBeenCalledTimes(1);
- sub1?.unsubscribe();
- await new Promise((resolve) => setTimeout(resolve, 20));
-
- const { pendingEvents } = await manager.subscribeWithResume(streamId, () => {});
-
- expect(pendingEvents.length).toBe(0);
+ append.mockReturnValueOnce(new Promise<boolean>(() => {}));
+ await expect(
+ manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: { delta: { content: { type: 'text', text: 'publish without append wait' } } },
+ }),
+ ).resolves.toBeUndefined();
+ expect(publish).toHaveBeenCalledTimes(2);
+ expect(jobStore.hasSubscriberAttached).toHaveBeenCalledTimes(1);
await manager.destroy();
});
- test('should defer live events until a resumed subscription is activated', async () => {
+ test('validates an in-memory HITL snapshot with the event frontier', async () => {
const manager = createInMemoryManager();
- const streamId = `atomic-live-${Date.now()}`;
+ const streamId = `overflow-hitl-memory-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
+ const createdAt = (await manager.getJob(streamId))!.createdAt;
+ await forceEarlyBufferOverflow(manager, streamId);
+ manager.setContentParts(
+ streamId,
+ [{ type: 'text', text: 'complete snapshot' }] as never,
+ createdAt,
+ );
+ const getRuntimeSpy = jest.spyOn(
+ manager as unknown as {
+ getOrCreateRuntimeState: (...args: unknown[]) => Promise<unknown>;
+ },
+ 'getOrCreateRuntimeState',
+ );
- const sub1 = await manager.subscribe(streamId, () => {});
+ await expect(
+ manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),
+ ).resolves.toMatchObject({ aggregatedContent: expect.any(Array) });
+ expect(getRuntimeSpy).toHaveBeenCalledWith(streamId, expect.objectContaining({ createdAt }));
+ expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).toMatchObject({
+ durableEvents: 0,
+ recoveryOutcome: 'success',
+ });
+
+ await manager.destroy();
+ });
+
+ test('rejects a durable failed outcome before exposing HITL state', async () => {
+ const manager = createInMemoryManager();
+ const streamId = `overflow-hitl-failed-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ const createdAt = (await manager.getJob(streamId))!.createdAt;
+ await forceEarlyBufferOverflow(manager, streamId);
+ const overflow = (await manager.getJob(streamId))!.metadata.earlyBufferOverflow!;
+ await (manager.getJobStore() as IJobStoreV2).settleEarlyBufferRecovery(
+ streamId,
+ createdAt,
+ overflow.id,
+ {
+ recoveryMethod: 'snapshot',
+ recoveryOutcome: 'failed',
+ recoveryCompletedAt: Date.now(),
+ recoveryFailureReason: 'snapshot_missing',
+ },
+ );
+
+ await expect(
+ manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),
+ ).rejects.toThrow(GENERATION_RECOVERY_FAILED_ERROR);
+
+ await manager.destroy();
+ });
+
+ test('contains subscriber lease lookup failure during terminal cleanup', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = `overflow-lease-read-failure-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(manager, streamId);
+ jest.spyOn(jobStore, 'hasActiveSubscriber').mockRejectedValueOnce(new Error('unavailable'));
+
+ await expect(manager.completeJob(streamId)).resolves.toBe(true);
+ expect((await manager.getJob(streamId))?.status).toBe('complete');
+ expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(
+ 'recoveryOutcome',
+ );
+
+ await manager.destroy();
+ });
+
+ test('leaves recovery unresolved when terminal lifecycle lookup fails', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = `overflow-lifecycle-read-failure-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(manager, streamId);
+ const originalGetJob = jobStore.getJob.bind(jobStore);
+ jest
+ .spyOn(jobStore, 'getJob')
+ .mockImplementationOnce(originalGetJob)
+ .mockRejectedValueOnce(new Error('unavailable'))
+ .mockImplementation(originalGetJob);
+
+ await expect(manager.completeJob(streamId)).resolves.toBe(true);
+ expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(
+ 'recoveryOutcome',
+ );
+
+ await manager.destroy();
+ });
+
+ test('removes in-memory capture handlers when another attachment wins recovery', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ cleanupOnComplete: false,
+ });
+ manager.initialize();
+ const streamId = `overflow-recovery-loser-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(manager, streamId);
+ const originalSettle = jobStore.settleEarlyBufferRecovery.bind(jobStore);
+ jest
+ .spyOn(jobStore, 'settleEarlyBufferRecovery')
+ .mockImplementationOnce(async (id, createdAt, recoveryId) => {
+ await originalSettle(id, createdAt, recoveryId, {
+ recoveryMethod: 'snapshot',
+ recoveryOutcome: 'not_required',
+ recoveryCompletedAt: Date.now(),
+ });
+ return false;
+ });
+
+ const result = await manager.subscribeWithResume(streamId, () => {});
+
+ expect(result.subscription).toBeNull();
+ const runtime = (
+ manager as unknown as {
+ runtimeState: Map<string, { resumeCaptureHandlers: Set<unknown> }>;
+ }
+ ).runtimeState.get(streamId);
+ expect(runtime?.resumeCaptureHandlers.size).toBe(0);
+
+ await manager.destroy();
+ });
+
+ testRedis('redirects a post-overflow first attachment to resume recovery (Redis)', async () => {
+ const manager = createRedisManager();
+ const streamId = `overflow-redirect-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ await manager.emitChunk(streamId, {
+ event: 'on_run_step',
+ data: {
+ id: 'step-1',
+ runId: 'run-1',
+ index: 0,
+ stepDetails: { type: 'message_creation' },
+ },
+ });
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'durable-overflow-content' } },
+ },
+ });
+ await forceEarlyBufferOverflow(manager, streamId);
+ expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);
+
+ const errors: string[] = [];
+ const sub = await manager.subscribe(
+ streamId,
+ () => {},
+ undefined,
+ (error) => errors.push(error),
+ );
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
+ sub?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 100));
+
+ /** The resume path the client falls back to reconstructs the
+ * discarded output from the durable chunk log. */
+ const { subscription, resumeState } = await manager.subscribeWithResume(streamId, () => {});
+ expect(JSON.stringify(resumeState?.aggregatedContent ?? [])).toContain(
+ 'durable-overflow-content',
+ );
+ subscription?.unsubscribe();
+ const recoveredJob = await manager.getJob(streamId);
+ expect(recoveredJob?.metadata.earlyBufferOverflow).toMatchObject({
+ recoveryMethod: 'redis',
+ recoveryOutcome: 'success',
+ });
+
+ await manager.destroy();
+ });
+
+ testRedis('refreshes an overflow marker before activating a resume attachment', async () => {
+ const owner = createRedisManager();
+ const streamId = `overflow-marker-attachment-race-${Date.now()}`;
+ await owner.createJob(streamId, 'user-1');
+ await owner.emitChunk(streamId, {
+ event: 'on_run_step',
+ data: {
+ id: 'step-1',
+ runId: 'run-1',
+ index: 0,
+ stepDetails: { type: 'message_creation' },
+ },
+ });
+ await owner.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'durable-before-marker' } },
+ },
+ });
+ await forceEarlyBufferOverflow(owner, streamId);
+
+ const replica = createRedisManager();
+ const replicaStore = replica.getJobStore();
+ const originalGetJob = replicaStore.getJob.bind(replicaStore);
+ let jobReads = 0;
+ jest.spyOn(replicaStore, 'getJob').mockImplementation(async (id) => {
+ const job = await originalGetJob(id);
+ jobReads++;
+ if (jobReads <= 4 && job != null) {
+ return { ...job, earlyBufferOverflow: undefined };
+ }
+ return job;
+ });
+
+ const result = await replica.subscribeWithResume(streamId, () => {});
+
+ expect(jobReads).toBeGreaterThan(4);
+ expect(JSON.stringify(result.resumeState?.aggregatedContent ?? [])).toContain(
+ 'durable-before-marker',
+ );
+ expect((await originalGetJob(streamId))?.earlyBufferOverflow).toMatchObject({
+ recoveryMethod: 'redis',
+ recoveryOutcome: 'success',
+ });
+ result.subscription?.unsubscribe();
+
+ const followup = await replica.subscribeWithResume(streamId, () => {});
+ expect(JSON.stringify(followup.resumeState?.aggregatedContent ?? [])).toContain(
+ 'durable-before-marker',
+ );
+ followup.subscription?.unsubscribe();
+
+ await Promise.all([owner.destroy(), replica.destroy()]);
+ });
+
+ testRedis(
+ 'reconstructs more than 5,000 pre-attachment events across replica reconnects (Redis)',
+ async () => {
+ const owner = createRedisManager();
+ const streamId = `overflow-cross-replica-${Date.now()}`;
+ await owner.createJob(streamId, 'user-1');
+ const createdAt = (await owner.getJob(streamId))!.createdAt;
+ const firstReplica = createRedisManager();
+ /** Cache a pre-overflow runtime so resume must refresh the later
+ * durable marker rather than trust process-local state. */
+ await firstReplica.getJob(streamId);
+ /** Created envelopes are published but not durably appendable, so they
+ * must not advance the durable recovery frontier. */
+ await owner.emitChunk(streamId, {
+ created: true,
+ message: { text: 'hello' },
+ streamId,
+ } as CreatedEvent);
+ await owner.emitChunk(streamId, {
+ event: 'on_run_step',
+ data: {
+ id: 'step-1',
+ runId: 'run-1',
+ index: 0,
+ stepDetails: { type: 'message_creation' },
+ },
+ });
+ for (let i = 0; i < 5_001; i++) {
+ await firstReplica.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: `${i},` } },
+ },
+ });
+ }
+
+ const first = await firstReplica.subscribeWithResume(streamId, () => {});
+ expect(first.resumeState?.aggregatedContent).toHaveLength(1);
+ expect(JSON.stringify(first.resumeState?.aggregatedContent)).toContain('5000,');
+ first.subscription?.activate();
+ first.subscription?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ const firstAttachedAt = await ioredisClient!.hget(
+ `stream:{${streamId}}:job`,
+ 'firstSubscriberAttachedAt',
+ );
+ expect(firstAttachedAt).not.toBeNull();
+ expect(
+ await (firstReplica.getJobStore() as IJobStoreV2).hasActiveSubscriber(
+ streamId,
+ createdAt,
+ Date.now(),
+ ),
+ ).toBe(false);
+
+ await owner.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'after-disconnect' } },
+ },
+ });
+
+ const secondReplica = createRedisManager();
+ const second = await secondReplica.subscribeWithResume(streamId, () => {});
+ expect(JSON.stringify(second.resumeState?.aggregatedContent)).toContain('after-disconnect');
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ expect(
+ await (secondReplica.getJobStore() as IJobStoreV2).hasActiveSubscriber(
+ streamId,
+ createdAt,
+ Date.now(),
+ ),
+ ).toBe(true);
+ second.subscription?.unsubscribe();
+ expect(
+ await ioredisClient!.hget(`stream:{${streamId}}:job`, 'firstSubscriberAttachedAt'),
+ ).toBe(firstAttachedAt);
+
+ const recoveredJob = await secondReplica.getJob(streamId);
+ expect(recoveredJob?.metadata.earlyBufferOverflow).toMatchObject({
+ durableEvents: 5_002,
+ recoveryMethod: 'redis',
+ recoveryOutcome: 'success',
+ });
+
+ await Promise.all([owner.destroy(), firstReplica.destroy(), secondReplica.destroy()]);
+ },
+ 60_000,
+ );
+
+ testRedis(
+ 'terminalizes an overflow when durable reconstruction is missing (Redis)',
+ async () => {
+ const owner = createRedisManager();
+ const streamId = `overflow-missing-durable-${Date.now()}`;
+ await owner.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(owner, streamId);
+ await ioredisClient!.del(`stream:{${streamId}}:chunks`);
+
+ const replica = createRedisManager();
+ const errors: string[] = [];
+ const result = await replica.subscribeWithResume(
+ streamId,
+ () => {},
+ undefined,
+ (error) => errors.push(error),
+ );
+
+ expect(result.subscription).toBeNull();
+ expect(errors).toEqual([GENERATION_RECOVERY_FAILED_ERROR]);
+ const failedJob = await replica.getJob(streamId);
+ expect(failedJob?.status).toBe('error');
+ expect(failedJob?.error).toBe(GENERATION_RECOVERY_FAILED_ERROR);
+ expect(failedJob?.metadata.earlyBufferOverflow).toMatchObject({
+ recoveryMethod: 'redis',
+ recoveryOutcome: 'failed',
+ recoveryFailureReason: 'durable_state_missing',
+ });
+
+ await Promise.all([owner.destroy(), replica.destroy()]);
+ },
+ );
+
+ testRedis('replays a durable terminal payload with unresolved overflow (Redis)', async () => {
+ const owner = createRedisManager();
+ const streamId = `overflow-durable-terminal-${Date.now()}`;
+ const job = await owner.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(owner, streamId);
+ const claim = await owner.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, {
+ persistencePending: true,
+ });
+ expect(claim).not.toBeNull();
+ const finalEvent = {
+ final: true,
+ conversation: { conversationId: streamId },
+ responseMessage: { text: 'complete' },
+ } as ServerSentEvent;
+ await owner.publishTerminalClaim(claim!, finalEvent);
+ expect((await owner.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(
+ 'recoveryOutcome',
+ );
+
+ const replica = createRedisManager();
+ const terminalEvents: ServerSentEvent[] = [];
+ const errors: string[] = [];
+ const resumed = await replica.subscribeWithResume(
+ streamId,
+ () => {},
+ (event) => terminalEvents.push(event),
+ (error) => errors.push(error),
+ { expectedCreatedAt: job.createdAt },
+ );
+ resumed.subscription?.activate();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ expect(terminalEvents).toEqual([finalEvent]);
+ expect(errors).toEqual([]);
+ expect((await replica.getJob(streamId))?.status).toBe('complete');
+
+ resumed.subscription?.unsubscribe();
+ await Promise.all([owner.destroy(), replica.destroy()]);
+ });
+
+ testRedis('validates overflow recovery before exposing HITL resume state', async () => {
+ const manager = createRedisManager();
+ const streamId = `overflow-hitl-validation-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(manager, streamId);
+ const createdAt = (await manager.getJob(streamId))!.createdAt;
+ await ioredisClient!.del(`stream:{${streamId}}:chunks`);
+
+ await expect(
+ manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),
+ ).rejects.toThrow(GENERATION_RECOVERY_FAILED_ERROR);
+ expect((await manager.getJob(streamId))?.status).toBe('error');
+
+ await manager.destroy();
+ });
+
+ testRedis(
+ 'terminalizes a durable failed outcome left running by a crashed replica',
+ async () => {
+ const owner = createRedisManager();
+ const streamId = `overflow-failed-outcome-${Date.now()}`;
+ await owner.createJob(streamId, 'user-1');
+ await forceEarlyBufferOverflow(owner, streamId);
+
+ const overflowJob = await owner.getJob(streamId);
+ const overflow = overflowJob?.metadata.earlyBufferOverflow;
+ expect(overflow).toBeDefined();
+ await (owner.getJobStore() as IJobStoreV2).settleEarlyBufferRecovery(
+ streamId,
+ overflowJob!.createdAt,
+ overflow!.id,
+ {
+ recoveryMethod: 'redis',
+ recoveryOutcome: 'failed',
+ recoveryCompletedAt: Date.now(),
+ recoveryFailureReason: 'reconstruction_error',
+ },
+ );
+
+ const replica = createRedisManager();
+ const errors: string[] = [];
+ const result = await replica.subscribeWithResume(
+ streamId,
+ () => {},
+ undefined,
+ (error) => errors.push(error),
+ );
+ expect(result.subscription).toBeNull();
+ expect(errors).toEqual([GENERATION_RECOVERY_FAILED_ERROR]);
+ expect((await replica.getJob(streamId))?.status).toBe('error');
+
+ await Promise.all([owner.destroy(), replica.destroy()]);
+ },
+ );
+
+ test('buffers detached events until the cap in in-memory mode', async () => {
+ const manager = createInMemoryManager();
+ const streamId = `buf-below-cap-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ await setupDisconnectedStream(manager, streamId, 10);
+
+ const stats = manager.getRuntimeStats();
+ expect(stats.earlyBufferedEvents).toBe(2);
+ expect(stats.earlyBufferedBytes).toBeGreaterThan(0);
+
+ await manager.destroy();
+ });
+
+ test('caps captured events restored by a resume canceled before activation', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `restore-cap-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ await setupDisconnectedStream(manager, streamId, 10);
+
+ /** Arm only after the snapshot completes so the gate parks the resume
+ * in its post-attachment steer reconciliation, the window where
+ * emissions are captured per-resume instead of buffered. */
+ let armed = false;
+ const originalGetResumeState = manager.getResumeState.bind(manager);
+ jest
+ .spyOn(manager, 'getResumeState')
+ .mockImplementation(
+ async (...args: Parameters<GenerationJobManagerClass['getResumeState']>) => {
+ const result = await originalGetResumeState(...args);
+ armed = true;
+ return result;
+ },
+ );
+ let releaseGate!: () => void;
+ const gate = new Promise<void>((resolve) => (releaseGate = resolve));
+ let gateReached!: () => void;
+ const reached = new Promise<void>((resolve) => (gateReached = resolve));
+ const originalPeek = jobStore.peekSteers.bind(jobStore);
+ let gated = true;
+ jest
+ .spyOn(jobStore, 'peekSteers')
+ .mockImplementation(async (...args: Parameters<InMemoryJobStore['peekSteers']>) => {
+ if (armed && gated) {
+ gated = false;
+ gateReached();
+ await gate;
+ }
+ return originalPeek(...args);
+ });
+
+ const resumePromise = manager.subscribeWithResume(streamId, () => {});
+
+ await reached;
+ const bigText = 'z'.repeat(2 * 1024 * 1024);
+ for (let i = 0; i < 5; i++) {
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: bigText } },
+ },
+ });
+ }
+ releaseGate();
+
+ const { subscription } = await resumePromise;
+ expect(subscription).not.toBeNull();
+ subscription!.unsubscribe();
+
+ /** The ~10MB of captured events must not survive restoration. */
+ const stats = manager.getRuntimeStats();
+ expect(stats.earlyBufferedEvents).toBe(0);
+ expect(stats.earlyBufferedBytes).toBe(0);
+
+ /** Restoration overflowed, so a non-resume attach takes the redirect. */
+ const errors: string[] = [];
+ const probe = await manager.subscribe(
+ streamId,
+ () => {},
+ undefined,
+ (error) => errors.push(error),
+ );
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
+ probe?.unsubscribe();
+
+ await manager.destroy();
+ });
+ });
+
+ describe('Atomic subscribeWithResume', () => {
+ test('retries a failed first-subscriber lease claim while attached', async () => {
+ jest.useFakeTimers();
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const originalClaim = jobStore.claimFirstSubscriber.bind(jobStore);
+ const claim = jest
+ .spyOn(jobStore, 'claimFirstSubscriber')
+ .mockRejectedValueOnce(new Error('store unavailable'))
+ .mockImplementation(originalClaim);
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `subscriber-claim-retry-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ const createdAt = (await jobStore.getJob(streamId))!.createdAt;
+ const subscription = await manager.subscribe(streamId, () => {});
+ await jest.advanceTimersByTimeAsync(0);
+ expect(claim).toHaveBeenCalledTimes(1);
+ const runtime = (
+ manager as unknown as {
+ runtimeState: Map<string, { onFirstSubscriberLeaseClaim?: (first: boolean) => void }>;
+ }
+ ).runtimeState.get(streamId)!;
+ expect(runtime.onFirstSubscriberLeaseClaim).toBeDefined();
+
+ await jest.advanceTimersByTimeAsync(10_000);
+
+ expect(claim).toHaveBeenCalledTimes(2);
+ expect(runtime.onFirstSubscriberLeaseClaim).toBeUndefined();
+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
+ true,
+ );
+ subscription?.unsubscribe();
+ await manager.destroy();
+ jest.useRealTimers();
+ });
+
+ test('coalesces subscriber lease renewals while a store write is pending', async () => {
+ jest.useFakeTimers();
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const originalClaim = jobStore.claimFirstSubscriber.bind(jobStore);
+ let releaseRenewal!: (claimed: boolean) => void;
+ const pendingRenewal = new Promise<boolean>((resolve) => {
+ releaseRenewal = resolve;
+ });
+ const claim = jest
+ .spyOn(jobStore, 'claimFirstSubscriber')
+ .mockImplementationOnce(originalClaim)
+ .mockReturnValueOnce(pendingRenewal)
+ .mockImplementation(originalClaim);
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `subscriber-renewal-coalesce-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ const subscription = await manager.subscribe(streamId, () => {});
+ await jest.advanceTimersByTimeAsync(30_000);
+
+ expect(claim).toHaveBeenCalledTimes(2);
+ releaseRenewal(false);
+ await jest.advanceTimersByTimeAsync(10_000);
+ expect(claim).toHaveBeenCalledTimes(3);
+ const lastCall = claim.mock.calls[2];
+ expect(lastCall[4] - lastCall[2]).toBe(30_000);
+
+ subscription?.unsubscribe();
+ await manager.destroy();
+ jest.useRealTimers();
+ });
+
+ test('stops the predecessor lease timer when a durable generation replaces it', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: true,
+ });
+ manager.initialize();
+ const streamId = `subscriber-runtime-replacement-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ const subscription = await manager.subscribe(streamId, () => {});
+ const predecessor = (
+ manager as unknown as {
+ runtimeState: Map<string, { subscriberLeaseTimer?: ReturnType<typeof setInterval> }>;
+ }
+ ).runtimeState.get(streamId)!;
+ expect(predecessor.subscriberLeaseTimer).toBeDefined();
+
+ await jobStore.createJob(streamId, 'user-1');
+ await manager.getJob(streamId);
+
+ expect(predecessor.subscriberLeaseTimer).toBeUndefined();
+ subscription?.unsubscribe();
+ await manager.destroy();
+ });
+
+ test('keeps ordinary HITL resume reads on the cached content path', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `hitl-cached-content-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ const createdAt = (await jobStore.getJob(streamId))!.createdAt;
+ const getContentParts = jest.spyOn(jobStore, 'getContentParts');
+
+ await manager.getResumeState(streamId, createdAt, {
+ validateEarlyBufferRecovery: true,
+ });
+
+ expect(getContentParts).toHaveBeenCalledWith(streamId, createdAt, {
+ durableOnly: false,
+ });
+ await manager.destroy();
+ });
+
+ test('renews the subscriber lease after an ambiguous detach failure', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const claim = jest.spyOn(jobStore, 'claimFirstSubscriber');
+ jest
+ .spyOn(jobStore, 'detachSubscriber')
+ .mockRejectedValueOnce(new Error('store unavailable'));
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `subscriber-detach-retry-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ const first = await manager.subscribe(streamId, () => {});
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ first?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ const second = await manager.subscribe(streamId, () => {});
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ expect(claim).toHaveBeenCalledTimes(2);
+
+ second?.unsubscribe();
+ await manager.destroy();
+ });
+
+ test('tracks active subscriber groups across local reattachments', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `subscriber-lifecycle-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+ const createdAt = (await jobStore.getJob(streamId))!.createdAt;
+
+ const first = await manager.subscribe(streamId, () => {});
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
+ true,
+ );
+ first?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
+ false,
+ );
+
+ const second = await manager.subscribe(streamId, () => {});
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
+ true,
+ );
+ second?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ await jobStore.claimFirstSubscriber(
+ streamId,
+ createdAt,
+ Date.now(),
+ 'crashed-subscriber',
+ Date.now() - 1,
+ );
+ await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
+ false,
+ );
+ await manager.destroy();
+ });
+
+ test('keeps the first-subscriber telemetry claim off the attachment path', async () => {
+ const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
+ let releaseClaim!: (claimed: boolean) => void;
+ const pendingClaim = new Promise<boolean>((resolve) => {
+ releaseClaim = resolve;
+ });
+ jest.spyOn(jobStore, 'claimFirstSubscriber').mockReturnValue(pendingClaim);
+ const manager = new GenerationJobManagerClass();
+ manager.configure({
+ jobStore,
+ eventTransport: new InMemoryEventTransport(),
+ isRedis: false,
+ });
+ manager.initialize();
+ const streamId = `nonblocking-attachment-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ const attachment = manager.subscribe(streamId, () => {});
+ const result = await Promise.race([
+ attachment,
+ new Promise<'timed_out'>((resolve) => setTimeout(() => resolve('timed_out'), 50)),
+ ]);
+ expect(result).not.toBe('timed_out');
+ releaseClaim(true);
+ await pendingClaim;
+ if (result !== 'timed_out') {
+ result?.unsubscribe();
+ }
+ await manager.destroy();
+ });
+
+ test('should return empty pendingEvents for pre-snapshot buffer events (in-memory)', async () => {
+ const manager = createInMemoryManager();
+ const streamId = `atomic-drain-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ const sub1 = await manager.subscribe(streamId, () => {});
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ sub1?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ await manager.emitChunk(streamId, {
+ event: 'on_run_step',
+ data: {
+ id: 'step-1',
+ runId: 'run-1',
+ index: 0,
+ stepDetails: { type: 'message_creation' },
+ },
+ });
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: {
+ id: 'step-1',
+ delta: { content: { type: 'text', text: 'buffered' } },
+ },
+ });
+
+ const liveEvents: ServerSentEvent[] = [];
+ const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(
+ streamId,
+ (event) => liveEvents.push(event),
+ );
+
+ expect(resumeState).not.toBeNull();
+ expect(pendingEvents.length).toBe(0);
+ expect(liveEvents.length).toBe(0);
+
+ subscription?.unsubscribe();
+ await manager.destroy();
+ });
+
+ test('should return empty pendingEvents when buffer is empty', async () => {
+ const manager = createInMemoryManager();
+ const streamId = `atomic-empty-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ const sub1 = await manager.subscribe(streamId, () => {});
+ await new Promise((resolve) => setTimeout(resolve, 10));
+
+ await manager.emitChunk(streamId, {
+ event: 'on_message_delta',
+ data: { delta: { content: { type: 'text', text: 'delivered' } } },
+ });
+ await new Promise((resolve) => setTimeout(resolve, 10));
+
+ sub1?.unsubscribe();
+ await new Promise((resolve) => setTimeout(resolve, 20));
+
+ const { pendingEvents } = await manager.subscribeWithResume(streamId, () => {});
+
+ expect(pendingEvents.length).toBe(0);
+
+ await manager.destroy();
+ });
+
+ test('should defer live events until a resumed subscription is activated', async () => {
+ const manager = createInMemoryManager();
+ const streamId = `atomic-live-${Date.now()}`;
+ await manager.createJob(streamId, 'user-1');
+
+ const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: 'buffered-pre-snapshot' } } },
+ data: {
+ delta: { content: { type: 'text', text: 'buffered-pre-snapshot' } },
+ },
});
const liveEvents: ServerSentEvent[] = [];
@@ -1984,7 +2978,9 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: 'buffered-redis' } } },
+ data: {
+ delta: { content: { type: 'text', text: 'buffered-redis' } },
+ },
});
await new Promise((resolve) => setTimeout(resolve, 100));
@@ -2290,7 +3286,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 5; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `token${i} ` } }, index: i },
+ data: {
+ delta: { content: { type: 'text', text: `token${i} ` } },
+ index: i,
+ },
});
}
@@ -2334,7 +3333,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 3; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `pre-local-${i}` } }, index: i },
+ data: {
+ delta: { content: { type: 'text', text: `pre-local-${i}` } },
+ index: i,
+ },
});
}
@@ -2350,7 +3352,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 3; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `post-local-${i}` } }, index: i + 3 },
+ data: {
+ delta: { content: { type: 'text', text: `post-local-${i}` } },
+ index: i + 3,
+ },
});
}
@@ -2423,7 +3428,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 3; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `word${i} ` } }, index: i },
+ data: {
+ delta: { content: { type: 'text', text: `word${i} ` } },
+ index: i,
+ },
});
}
@@ -2486,11 +3494,17 @@ describe('GenerationJobManager Integration Tests', () => {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: 'pre-sub-0' } }, index: 0 },
+ data: {
+ delta: { content: { type: 'text', text: 'pre-sub-0' } },
+ index: 0,
+ },
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: 'pre-sub-1' } }, index: 1 },
+ data: {
+ delta: { content: { type: 'text', text: 'pre-sub-1' } },
+ index: 1,
+ },
});
const receivedEvents: unknown[] = [];
@@ -2506,7 +3520,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 5; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `post-sub-${i}` } }, index: i + 2 },
+ data: {
+ delta: { content: { type: 'text', text: `post-sub-${i}` } },
+ index: i + 2,
+ },
});
}
@@ -2553,7 +3570,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 0; i < 3; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `chunk-${i}` } }, index: i },
+ data: {
+ delta: { content: { type: 'text', text: `chunk-${i}` } },
+ index: i,
+ },
});
}
@@ -2567,7 +3587,10 @@ describe('GenerationJobManager Integration Tests', () => {
for (let i = 3; i < 6; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
- data: { delta: { content: { type: 'text', text: `chunk-${i}` } }, index: i },
+ data: {
+ delta: { content: { type: 'text', text: `chunk-${i}` } },
+ index: i,
+ },
});
}
@@ -2631,7 +3654,10 @@ describe('GenerationJobManager Integration Tests', () => {
await sub2.ready;
expect(callCount).toBe(2);
- await transport.emitChunk(streamId, { event: 'test', data: { value: 'hello' } });
+ await transport.emitChunk(streamId, {
+ event: 'test',
+ data: { value: 'hello' },
+ });
await new Promise((resolve) => setTimeout(resolve, 100));
expect(receivedEvents.length).toBe(1);
@@ -2782,7 +3808,10 @@ describe('GenerationJobManager Integration Tests', () => {
status: 'error',
error: 'Generation predecessor handoff could not be confirmed',
replacedJobs: [
- expect.objectContaining({ createdAt: predecessor.createdAt, status: 'running' }),
+ expect.objectContaining({
+ createdAt: predecessor.createdAt,
+ status: 'running',
+ }),
],
});
bystanderSubscription?.unsubscribe();
@@ -2916,7 +3945,10 @@ describe('GenerationJobManager Integration Tests', () => {
);
const store = new RedisJobStore(ioredisClient!);
- const moved = await store.transitionStatus(streamId, { from: 'running', to: 'complete' });
+ const moved = await store.transitionStatus(streamId, {
+ from: 'running',
+ to: 'complete',
+ });
expect(moved).toBe(true);
const claimed = await store.claimParkedSteers(streamId, 'user-1');
diff --git a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
index 6805b63db1..b7bb784812 100644
--- a/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
+++ b/packages/api/src/stream/__tests__/RedisJobStore.stream_integration.spec.ts
@@ -1757,6 +1757,82 @@ describe('RedisJobStore Integration Tests', () => {
await instance2.destroy();
});
+ test('reports the generation-wide durable count after chunk trimming', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+
+ const streamId = `trimmed-chunk-frontier-${Date.now()}`;
+ const job = await store.createJob(streamId, 'user-1', streamId);
+ for (let i = 0; i < 3; i++) {
+ await store.appendChunk(streamId, {
+ event: 'on_message_delta',
+ data: { id: 'step-1', delta: { content: { type: 'text', text: String(i) } } },
+ });
+ }
+ await ioredisClient.xtrim(`stream:{${streamId}}:chunks`, 'MAXLEN', 1);
+ for (let i = 3; i < 5; i++) {
+ await store.appendChunk(streamId, {
+ event: 'on_message_delta',
+ data: { id: 'step-1', delta: { content: { type: 'text', text: String(i) } } },
+ });
+ }
+
+ const result = await store.getContentParts(streamId, job.createdAt, { durableOnly: true });
+
+ expect(result).toMatchObject({
+ reconstructedEventCount: 3,
+ durableEventCount: 5,
+ });
+
+ await store.destroy();
+ });
+
+ test('repairs the durable count after an append from a legacy writer', async () => {
+ if (!ioredisClient) {
+ return;
+ }
+
+ const { RedisJobStore } = await import('../implementations/RedisJobStore');
+ const store = new RedisJobStore(ioredisClient);
+ await store.initialize();
+
+ const streamId = `mixed-writer-frontier-${Date.now()}`;
+ const job = await store.createJob(streamId, 'user-1', streamId);
+ for (let i = 0; i < 3; i++) {
+ await store.appendChunk(streamId, {
+ event: 'on_message_delta',
+ data: { id: 'step-1', delta: { content: { type: 'text', text: String(i) } } },
+ });
+ }
+ await ioredisClient.xadd(
+ `stream:{${streamId}}:chunks`,
+ '*',
+ 'event',
+ JSON.stringify({
+ event: 'on_message_delta',
+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'legacy' } } },
+ }),
+ );
+ await store.appendChunk(streamId, {
+ event: 'on_message_delta',
+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'current' } } },
+ });
+
+ const result = await store.getContentParts(streamId, job.createdAt, { durableOnly: true });
+
+ expect(result).toMatchObject({
+ reconstructedEventCount: 5,
+ durableEventCount: 5,
+ });
+
+ await store.destroy();
+ });
+
test('should share run steps between instances', async () => {
if (!ioredisClient) {
return;
diff --git a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
index f8c30b4b8d..3b5d964d9d 100644
--- a/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
+++ b/packages/api/src/stream/__tests__/idempotencyClaim.spec.ts
@@ -1030,10 +1030,11 @@ describe('GenerationJobManager start-generation claim', () => {
settleAppend = resolve;
});
jest.spyOn(store, 'appendChunk').mockImplementationOnce(() => delayedAppend);
- await manager.emitChunk(streamId, {
+ const predecessorEmission = manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: 'in-flight predecessor output' },
});
+ await Promise.resolve();
const replacement = await manager.createJob(streamId, 'user-1', streamId, {
initialMetadata: { generationProtocolVersion: 2 },
@@ -1042,6 +1043,7 @@ describe('GenerationJobManager start-generation claim', () => {
expect(replacement.abortController.signal.aborted).toBe(false);
settleAppend(false);
+ await expect(predecessorEmission).rejects.toThrow('Durable chunk append was fenced out');
await Promise.resolve();
await Promise.resolve();
diff --git a/packages/api/src/stream/__tests__/startup.spec.ts b/packages/api/src/stream/__tests__/startup.spec.ts
index 3afb487751..1a388dd853 100644
--- a/packages/api/src/stream/__tests__/startup.spec.ts
+++ b/packages/api/src/stream/__tests__/startup.spec.ts
@@ -675,6 +675,12 @@ describe('GenerationJobManager startup telemetry', () => {
const predecessor = await manager.createJob(streamId, 'user-1', streamId);
const onDone = jest.fn();
const subscription = await manager.subscribe(streamId, () => undefined, onDone);
+ const predecessorRuntime = (
+ manager as unknown as {
+ runtimeState: Map<string, { subscriberLeaseTimer?: ReturnType<typeof setInterval> }>;
+ }
+ ).runtimeState.get(streamId)!;
+ expect(predecessorRuntime.subscriberLeaseTimer).toBeDefined();
const replacement = await manager.createJob(streamId, 'user-1', streamId);
@@ -687,6 +693,7 @@ describe('GenerationJobManager startup telemetry', () => {
conversation: { conversationId: streamId },
});
expect(predecessor.abortController.signal.aborted).toBe(true);
+ expect(predecessorRuntime.subscriberLeaseTimer).toBeUndefined();
expect(replacement.abortController.signal.aborted).toBe(false);
expect(eventTransport.getSubscriberCount(streamId)).toBe(0);
subscription?.unsubscribe();
@@ -1404,7 +1411,7 @@ describe('GenerationJobManager startup telemetry', () => {
expect(getJob).toHaveBeenCalledTimes(1);
expect(onError).not.toHaveBeenCalled();
expect(onAllSubscribersLeft).not.toHaveBeenCalled();
- expect(jest.getTimerCount()).toBe(timerCountBeforeFence);
+ expect(jest.getTimerCount()).toBe(timerCountBeforeFence - 1);
releaseLookup?.();
await jest.advanceTimersByTimeAsync(0);
diff --git a/packages/api/src/stream/__tests__/steering.spec.ts b/packages/api/src/stream/__tests__/steering.spec.ts
index b444d0248e..c57539610b 100644
--- a/packages/api/src/stream/__tests__/steering.spec.ts
+++ b/packages/api/src/stream/__tests__/steering.spec.ts
@@ -1288,17 +1288,21 @@ describe('emitChunk durability (Redis-mode chunk log)', () => {
const store = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const transport = new InMemoryEventTransport();
const redisModeManager = buildRedisModeManager(store, transport);
+ let subscription: Awaited<ReturnType<typeof redisModeManager.subscribe>> | undefined;
try {
const streamId = 'steer-fire-and-forget';
const job = await redisModeManager.createJob(streamId, 'user-1');
+ subscription = await redisModeManager.subscribe(streamId, () => undefined);
- // Never resolves: the per-delta hot path must not gate on durability.
+ // After subscriber admission, the per-delta hot path must not gate on durability.
+ // Pre-admission events intentionally await the append so recovery cannot miss them.
jest.spyOn(store, 'appendChunk').mockReturnValue(new Promise<boolean>(() => undefined));
const publishSpy = jest.spyOn(transport, 'emitChunk');
await redisModeManager.emitChunk(streamId, steerEvent);
expect(publishSpy).toHaveBeenCalledWith(streamId, steerEvent, job.createdAt);
} finally {
+ subscription?.unsubscribe();
await redisModeManager.destroy();
}
});
diff --git a/packages/api/src/stream/implementations/InMemoryJobStore.ts b/packages/api/src/stream/implementations/InMemoryJobStore.ts
index 2d059e47ff..3d17d392a2 100644
--- a/packages/api/src/stream/implementations/InMemoryJobStore.ts
+++ b/packages/api/src/stream/implementations/InMemoryJobStore.ts
@@ -23,6 +23,7 @@ import type {
IdempotencyClaimResult,
ParkedSteerClaim,
} from '~/stream/interfaces/IJobStore';
+import type { EarlyBufferOverflowState } from '../../types/earlyBufferRecovery';
import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';
import {
JobStatusTransitionDeadlineError,
@@ -368,7 +369,9 @@ export class InMemoryJobStore implements IJobStoreV2 {
active: true,
verified: true,
status: current.status,
- ...(current.conversationId !== undefined && { conversationId: current.conversationId }),
+ ...(current.conversationId !== undefined && {
+ conversationId: current.conversationId,
+ }),
});
}
if (
@@ -382,7 +385,9 @@ export class InMemoryJobStore implements IJobStoreV2 {
active: current?.status === 'running' || current?.status === 'requires_action',
verified: currentCreatedAt != null,
...(current !== undefined && { status: current.status }),
- ...(current?.conversationId !== undefined && { conversationId: current.conversationId }),
+ ...(current?.conversationId !== undefined && {
+ conversationId: current.conversationId,
+ }),
});
};
@@ -533,7 +538,9 @@ export class InMemoryJobStore implements IJobStoreV2 {
checkpointNamespace: String(createdAt),
}),
...(conversationId !== undefined && { conversationId }),
- ...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),
+ ...(idempotencyClientRequestId !== undefined && {
+ idempotencyClientRequestId,
+ }),
...(recoveredSteerId !== undefined && { recoveredSteerId }),
providerAbortReady: false,
...(providerExecutionId != null && { providerDrained: true }),
@@ -890,7 +897,10 @@ export class InMemoryJobStore implements IJobStoreV2 {
if (existing && existing.expiresAt > now) {
return { claimed: false, existing: existing.value };
}
- this.idempotencyClaims.set(key, { value, expiresAt: now + ttlSeconds * 1000 });
+ this.idempotencyClaims.set(key, {
+ value,
+ expiresAt: now + ttlSeconds * 1000,
+ });
return { claimed: true, existing: value };
}
@@ -1162,7 +1172,9 @@ export class InMemoryJobStore implements IJobStoreV2 {
patch: {
completedAt: now,
error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
- ...(job.agentEventDeliveryKey != null && { terminalHostActionPending: true }),
+ ...(job.agentEventDeliveryKey != null && {
+ terminalHostActionPending: true,
+ }),
},
clear: [
'pendingAction',
@@ -1446,7 +1458,11 @@ export class InMemoryJobStore implements IJobStoreV2 {
if (existing) {
existing.contentParts = contentParts;
} else {
- this.contentState.set(streamId, { contentParts, graphRef: null, collectedUsage: [] });
+ this.contentState.set(streamId, {
+ contentParts,
+ graphRef: null,
+ collectedUsage: [],
+ });
}
}
@@ -1465,7 +1481,11 @@ export class InMemoryJobStore implements IJobStoreV2 {
if (existing) {
existing.collectedUsage = collectedUsage;
} else {
- this.contentState.set(streamId, { contentParts: [], graphRef: null, collectedUsage });
+ this.contentState.set(streamId, {
+ contentParts: [],
+ graphRef: null,
+ collectedUsage,
+ });
}
}
@@ -1487,8 +1507,11 @@ export class InMemoryJobStore implements IJobStoreV2 {
async getContentParts(
streamId: string,
expectedCreatedAt?: number,
+ _options?: { durableOnly?: boolean },
): Promise<{
content: Agents.MessageContentComplex[];
+ reconstructedEventCount?: number;
+ durableEventCount?: number;
} | null> {
if (expectedCreatedAt != null && this.jobs.get(streamId)?.createdAt !== expectedCreatedAt) {
return null;
@@ -1499,9 +1522,107 @@ export class InMemoryJobStore implements IJobStoreV2 {
}
return {
content: state.contentParts,
+ ...(_options?.durableOnly === true && {
+ reconstructedEventCount: state.contentParts.length,
+ durableEventCount: state.contentParts.length,
+ }),
};
}
+ async settleEarlyBufferRecovery(
+ streamId: string,
+ expectedCreatedAt: number,
+ overflowId: string,
+ settlement: Pick<
+ EarlyBufferOverflowState,
+ 'recoveryMethod' | 'recoveryOutcome' | 'recoveryCompletedAt' | 'recoveryFailureReason'
+ >,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ const overflow = job?.earlyBufferOverflow;
+ if (
+ job?.createdAt !== expectedCreatedAt ||
+ overflow?.id !== overflowId ||
+ overflow.recoveryOutcome != null
+ ) {
+ return false;
+ }
+ job.earlyBufferOverflow = { ...overflow, ...settlement };
+ return true;
+ }
+
+ async finalizeEarlyBufferOverflow(
+ streamId: string,
+ expectedCreatedAt: number,
+ overflowId: string,
+ finalizedOverflow: EarlyBufferOverflowState,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ const overflow = job?.earlyBufferOverflow;
+ if (
+ job?.createdAt !== expectedCreatedAt ||
+ overflow?.id !== overflowId ||
+ overflow.persistencePending !== true ||
+ overflow.recoveryOutcome != null
+ ) {
+ return false;
+ }
+ job.earlyBufferOverflow = finalizedOverflow;
+ return true;
+ }
+
+ async hasSubscriberAttached(streamId: string, expectedCreatedAt: number): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ return job?.createdAt === expectedCreatedAt && job.firstSubscriberAttachedAt != null;
+ }
+
+ async claimFirstSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ attachedAt: number,
+ subscriberId: string,
+ leaseExpiresAt: number,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (job?.createdAt !== expectedCreatedAt) {
+ return false;
+ }
+ job.activeSubscriberLeases ??= {};
+ job.activeSubscriberLeases[subscriberId] = leaseExpiresAt;
+ const firstSubscriber = job.firstSubscriberAttachedAt == null;
+ job.firstSubscriberAttachedAt ??= attachedAt;
+ return firstSubscriber;
+ }
+
+ async detachSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ subscriberId: string,
+ ): Promise<void> {
+ const job = this.jobs.get(streamId);
+ if (job?.createdAt !== expectedCreatedAt) {
+ return;
+ }
+ delete job.activeSubscriberLeases?.[subscriberId];
+ }
+
+ async hasActiveSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ observedAt: number,
+ ): Promise<boolean> {
+ const job = this.jobs.get(streamId);
+ if (job?.createdAt !== expectedCreatedAt) {
+ return false;
+ }
+ for (const [subscriberId, expiresAt] of Object.entries(job.activeSubscriberLeases ?? {})) {
+ if (expiresAt <= observedAt) {
+ delete job.activeSubscriberLeases?.[subscriberId];
+ }
+ }
+ return Object.keys(job.activeSubscriberLeases ?? {}).length > 0;
+ }
+
/**
* Get run steps for a job from graph.contentData.
* Uses WeakRef - may return empty if graph has been GC'd.
@@ -1685,7 +1806,9 @@ export class InMemoryJobStore implements IJobStoreV2 {
) {
return [];
}
- return (this.claimedSteers.get(streamId) ?? []).map((item) => ({ ...item }));
+ return (this.claimedSteers.get(streamId) ?? []).map((item) => ({
+ ...item,
+ }));
}
async getSteerReceipt(streamId: string, clientSteerId: string): Promise<SteerReceipt | null> {
@@ -1713,7 +1836,10 @@ export class InMemoryJobStore implements IJobStoreV2 {
const existingEntry = this.steerReceipts.get(streamId)?.get(receiptInput.clientSteerId);
if (existingEntry != null && existingEntry.expiresAt > Date.now()) {
this.normalizeSteerReceipt(streamId, existingEntry);
- return { ...existingEntry.receipt, item: { ...existingEntry.receipt.item } };
+ return {
+ ...existingEntry.receipt,
+ item: { ...existingEntry.receipt.item },
+ };
}
if (existingEntry != null) {
this.steerReceipts.get(streamId)?.delete(receiptInput.clientSteerId);
@@ -2126,7 +2252,11 @@ export class InMemoryJobStore implements IJobStoreV2 {
if (receipt != null) {
receipt.item = { ...item };
}
- return { outcome: 'armed', revision: item.preemptRevision, item: { ...item } };
+ return {
+ outcome: 'armed',
+ revision: item.preemptRevision,
+ item: { ...item },
+ };
}
async downgradeSteerPreempts(
@@ -2442,7 +2572,11 @@ export class InMemoryJobStore implements IJobStoreV2 {
const parsed = JSON.parse(parked.payload) as {
userId: string;
tenantId?: string;
- steers?: Array<{ steerId: string; clientSteerId?: string; recoveringCreatedAt?: number }>;
+ steers?: Array<{
+ steerId: string;
+ clientSteerId?: string;
+ recoveringCreatedAt?: number;
+ }>;
};
if (!Array.isArray(parsed.steers)) {
return;
@@ -2492,7 +2626,11 @@ export class InMemoryJobStore implements IJobStoreV2 {
parsedPayload?: {
userId: string;
tenantId?: string;
- steers?: Array<{ steerId: string; clientSteerId?: string; recoveringCreatedAt?: number }>;
+ steers?: Array<{
+ steerId: string;
+ clientSteerId?: string;
+ recoveringCreatedAt?: number;
+ }>;
},
parsedLeasedItem?: {
steerId: string;
diff --git a/packages/api/src/stream/implementations/RedisJobStore.ts b/packages/api/src/stream/implementations/RedisJobStore.ts
index b582e40e72..b34f180055 100644
--- a/packages/api/src/stream/implementations/RedisJobStore.ts
+++ b/packages/api/src/stream/implementations/RedisJobStore.ts
@@ -26,6 +26,7 @@ import type {
SteerReceiptInput,
ParkedSteerClaim,
} from '~/stream/interfaces/IJobStore';
+import type { EarlyBufferOverflowState } from '../../types/earlyBufferRecovery';
import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';
import type { RecoveredSteerPayload } from '~/stream/SteerRecovery';
import {
@@ -617,6 +618,47 @@ const JOB_UPDATE_LUA =
'end ' +
'return 1';
+/** Single-winner recovery outcome for one generation-scoped overflow. */
+const SETTLE_EARLY_BUFFER_RECOVERY_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'local raw = redis.call("HGET", KEYS[1], "earlyBufferOverflow") ' +
+ 'if not raw then return 0 end local ok, overflow = pcall(cjson.decode, raw) ' +
+ 'if not ok or type(overflow) ~= "table" or overflow.id ~= ARGV[2] ' +
+ 'or overflow.recoveryOutcome ~= nil then return 0 end ' +
+ 'local settlementOk, settlement = pcall(cjson.decode, ARGV[3]) ' +
+ 'if not settlementOk or type(settlement) ~= "table" then return 0 end ' +
+ 'for key, value in pairs(settlement) do overflow[key] = value end ' +
+ 'redis.call("HSET", KEYS[1], "earlyBufferOverflow", cjson.encode(overflow)) return 1';
+
+/** Finalizes only the unresolved pending marker installed by this owner. */
+const FINALIZE_EARLY_BUFFER_OVERFLOW_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'local raw = redis.call("HGET", KEYS[1], "earlyBufferOverflow") ' +
+ 'if not raw then return 0 end local ok, overflow = pcall(cjson.decode, raw) ' +
+ 'if not ok or type(overflow) ~= "table" or overflow.id ~= ARGV[2] ' +
+ 'or overflow.persistencePending ~= true or overflow.recoveryOutcome ~= nil then return 0 end ' +
+ 'redis.call("HSET", KEYS[1], "earlyBufferOverflow", ARGV[3]) return 1';
+
+const HAS_SUBSCRIBER_ATTACHED_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'if redis.call("HEXISTS", KEYS[1], "firstSubscriberAttachedAt") == 1 then return 1 end return 0';
+
+/** Generation-scoped single-winner first-subscriber claim. */
+const CLAIM_FIRST_SUBSCRIBER_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'redis.call("ZADD", KEYS[2], ARGV[4], ARGV[3]) redis.call("PEXPIRE", KEYS[2], 60000) ' +
+ 'if redis.call("HEXISTS", KEYS[1], "firstSubscriberAttachedAt") == 1 then return 0 end ' +
+ 'redis.call("HSET", KEYS[1], "firstSubscriberAttachedAt", ARGV[2]) return 1';
+
+const DETACH_SUBSCRIBER_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'redis.call("ZREM", KEYS[2], ARGV[2]) return 1';
+
+const HAS_ACTIVE_SUBSCRIBER_LUA =
+ 'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return 0 end ' +
+ 'redis.call("ZREMRANGEBYSCORE", KEYS[2], "-inf", ARGV[2]) ' +
+ 'if redis.call("ZCARD", KEYS[2]) > 0 then return 1 end return 0';
+
/** Exact provider-segment completion fence. A paused segment finishing after a
* resume cannot mark the resumed provider drained because its opaque id differs. */
const PROVIDER_DRAIN_LUA =
@@ -834,6 +876,10 @@ const CHUNK_APPEND_LUA =
'if #kept > 0 then redis.call("RPUSH", KEYS[5], unpack(kept)) ' +
'if claimTtl > 0 then redis.call("PEXPIRE", KEYS[5], claimTtl) end end end ' +
'redis.call("XADD", KEYS[1], "*", "event", ARGV[1]) ' +
+ 'local durable = redis.call("HGET", KEYS[2], "durableEventCount") ' +
+ 'local priorLength = redis.call("XLEN", KEYS[1]) - 1 ' +
+ 'if not durable or tonumber(durable) < priorLength then durable = priorLength end ' +
+ 'redis.call("HSET", KEYS[2], "durableEventCount", tonumber(durable) + 1) ' +
'if currentStatus == "running" then ' +
'redis.call("HSET", KEYS[2], "lastActiveAt", ARGV[6]) end ' +
'local cur = redis.call("TTL", KEYS[1]) ' +
@@ -880,6 +926,10 @@ const CHUNK_APPEND_BATCH_LUA =
'if epochTtl >= 0 and epochTtl < epochTarget then redis.call("EXPIRE", KEYS[8], epochTarget) end ' +
'else redis.call("SET", KEYS[8], currentCreatedAt, "EX", epochTarget) end ' +
'for i = 6, #ARGV do redis.call("XADD", KEYS[1], "*", "event", ARGV[i]) end ' +
+ 'local durable = redis.call("HGET", KEYS[2], "durableEventCount") ' +
+ 'local priorLength = redis.call("XLEN", KEYS[1]) - (#ARGV - 5) ' +
+ 'if not durable or tonumber(durable) < priorLength then durable = priorLength end ' +
+ 'redis.call("HSET", KEYS[2], "durableEventCount", tonumber(durable) + (#ARGV - 5)) ' +
'if currentStatus == "running" then ' +
'redis.call("HSET", KEYS[2], "lastActiveAt", ARGV[3]) end ' +
'local cur = redis.call("TTL", KEYS[1]) ' +
@@ -940,6 +990,12 @@ const CHUNKS_READ_LUA =
'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {} end ' +
'return redis.call("XRANGE", KEYS[2], "-", "+")';
+const CHUNKS_RECOVERY_READ_LUA =
+ 'if ARGV[1] ~= "" and redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return {{}, false} end ' +
+ 'local entries = redis.call("XRANGE", KEYS[2], "-", "+") ' +
+ 'local durable = redis.call("HGET", KEYS[1], "durableEventCount") ' +
+ 'return {entries, durable or false}';
+
const RUNSTEPS_READ_LUA =
'if redis.call("HGET", KEYS[1], "createdAt") ~= ARGV[1] then return false end ' +
'return redis.call("GET", KEYS[2])';
@@ -1621,6 +1677,9 @@ const KEYS = {
steerReceiptOrder: (streamId: string) => `stream:{${streamId}}:steer-receipt-order`,
/** Latest generation epoch, retained briefly beyond the live job hash. */
generationEpoch: (streamId: string) => `stream:{${streamId}}:generation-epoch`,
+ /** Expiring, generation-scoped subscriber-group leases. */
+ subscriberLeases: (streamId: string, createdAt: number) =>
+ `stream:{${streamId}}:subscriber-leases:${createdAt}`,
/** Running jobs set for cleanup (global set - single slot) */
runningJobs: 'stream:running',
/** Jobs paused for human review (global set - single slot) */
@@ -1756,7 +1815,10 @@ interface LocalCacheEntry<T> {
interface PendingChunkAppendBatch {
expectedCreatedAt?: number;
events: string[];
- settlers: Array<{ resolve: (appended: boolean) => void; reject: (err: unknown) => void }>;
+ settlers: Array<{
+ resolve: (appended: boolean) => void;
+ reject: (err: unknown) => void;
+ }>;
bytes: number;
timer: ReturnType<typeof setTimeout> | null;
}
@@ -1961,7 +2023,9 @@ export class RedisJobStore implements IJobStoreV2 {
createdAt: Date.now(),
generationProtocolVersion,
...(conversationId !== undefined && { conversationId }),
- ...(idempotencyClientRequestId !== undefined && { idempotencyClientRequestId }),
+ ...(idempotencyClientRequestId !== undefined && {
+ idempotencyClientRequestId,
+ }),
...(recoveredSteerId !== undefined && { recoveredSteerId }),
providerAbortReady: false,
...(providerExecutionId != null && { providerDrained: true }),
@@ -2057,7 +2121,9 @@ export class RedisJobStore implements IJobStoreV2 {
currentStatus === 'requires_action',
verified: previousOwner[8] !== '0',
...(currentStatus !== undefined && { status: currentStatus }),
- ...(currentConversationId !== undefined && { conversationId: currentConversationId }),
+ ...(currentConversationId !== undefined && {
+ conversationId: currentConversationId,
+ }),
});
}
const previousUserId =
@@ -2286,6 +2352,113 @@ export class RedisJobStore implements IJobStoreV2 {
}
}
+ async settleEarlyBufferRecovery(
+ streamId: string,
+ expectedCreatedAt: number,
+ overflowId: string,
+ settlement: Pick<
+ EarlyBufferOverflowState,
+ 'recoveryMethod' | 'recoveryOutcome' | 'recoveryCompletedAt' | 'recoveryFailureReason'
+ >,
+ ): Promise<boolean> {
+ const settled = await this.redis.eval(
+ SETTLE_EARLY_BUFFER_RECOVERY_LUA,
+ 1,
+ KEYS.job(streamId),
+ String(expectedCreatedAt),
+ overflowId,
+ JSON.stringify(settlement),
+ );
+ return settled === 1;
+ }
+
+ async finalizeEarlyBufferOverflow(
+ streamId: string,
+ expectedCreatedAt: number,
+ overflowId: string,
+ finalizedOverflow: EarlyBufferOverflowState,
+ ): Promise<boolean> {
+ const finalized = await this.redis.eval(
+ FINALIZE_EARLY_BUFFER_OVERFLOW_LUA,
+ 1,
+ KEYS.job(streamId),
+ String(expectedCreatedAt),
+ overflowId,
+ JSON.stringify(finalizedOverflow),
+ );
+ return finalized === 1;
+ }
+
+ async hasSubscriberAttached(streamId: string, expectedCreatedAt: number): Promise<boolean> {
+ return (
+ Number(
+ await this.redis.eval(
+ HAS_SUBSCRIBER_ATTACHED_LUA,
+ 1,
+ KEYS.job(streamId),
+ String(expectedCreatedAt),
+ ),
+ ) === 1
+ );
+ }
+
+ async claimFirstSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ attachedAt: number,
+ subscriberId: string,
+ leaseExpiresAt: number,
+ ): Promise<boolean> {
+ return (
+ Number(
+ await this.redis.eval(
+ CLAIM_FIRST_SUBSCRIBER_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.subscriberLeases(streamId, expectedCreatedAt),
+ String(expectedCreatedAt),
+ String(attachedAt),
+ subscriberId,
+ String(leaseExpiresAt),
+ ),
+ ) === 1
+ );
+ }
+
+ async detachSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ subscriberId: string,
+ ): Promise<void> {
+ await this.redis.eval(
+ DETACH_SUBSCRIBER_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.subscriberLeases(streamId, expectedCreatedAt),
+ String(expectedCreatedAt),
+ subscriberId,
+ );
+ }
+
+ async hasActiveSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ observedAt: number,
+ ): Promise<boolean> {
+ return (
+ Number(
+ await this.redis.eval(
+ HAS_ACTIVE_SUBSCRIBER_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.subscriberLeases(streamId, expectedCreatedAt),
+ String(expectedCreatedAt),
+ String(observedAt),
+ ),
+ ) === 1
+ );
+ }
+
async markProviderExecutionDrained(
streamId: string,
expectedCreatedAt: number,
@@ -2624,7 +2797,10 @@ export class RedisJobStore implements IJobStoreV2 {
// status + patch become HSET pairs; serializeJob skips undefined, so
// cleared fields go through HDEL (`clear`) instead.
const fields = Object.entries(
- this.serializeJob({ status: to, ...(persistedPatch ?? {}) } as SerializableJobData),
+ this.serializeJob({
+ status: to,
+ ...(persistedPatch ?? {}),
+ } as SerializableJobData),
).flat();
const clearFields = (clear ?? []).map(String);
@@ -2743,7 +2919,10 @@ export class RedisJobStore implements IJobStoreV2 {
return { claimed: true, existing: value };
}
try {
- return { claimed: false, existing: JSON.parse(result as string) as IdempotencyClaimValue };
+ return {
+ claimed: false,
+ existing: JSON.parse(result as string) as IdempotencyClaimValue,
+ };
} catch {
// An unreadable existing owner is outcome-ambiguous. Never turn store
// corruption into a duplicate generation by pretending the key is free.
@@ -2951,7 +3130,10 @@ export class RedisJobStore implements IJobStoreV2 {
if (members.length === 0) {
return [];
}
- const indexed = members.map((member) => ({ member, ...parseTerminalHostActionMember(member) }));
+ const indexed = members.map((member) => ({
+ member,
+ ...parseTerminalHostActionMember(member),
+ }));
const jobs = await Promise.all(indexed.map(({ streamId }) => this.getJob(streamId)));
// The durable hash field is the source of truth; a stale set entry (job reaped, or the
// marker/generation already replaced) is filtered out. Bare legacy members
@@ -3123,13 +3305,17 @@ export class RedisJobStore implements IJobStoreV2 {
// Job no longer exists (TTL expired) - remove from set
if (!job) {
- const currentJob = await this.reconcileJobMembership(streamId, { initialJob: null });
+ const currentJob = await this.reconcileJobMembership(streamId, {
+ initialJob: null,
+ });
this.clearLocalStateUnlessActive(streamId, currentJob);
return 1;
}
if (job.status === 'requires_action') {
- const currentJob = await this.reconcileJobMembership(streamId, { initialJob: job });
+ const currentJob = await this.reconcileJobMembership(streamId, {
+ initialJob: job,
+ });
if (
currentJob !== undefined &&
(!currentJob || currentJob.createdAt === job.createdAt)
@@ -3202,7 +3388,9 @@ export class RedisJobStore implements IJobStoreV2 {
const job = await this.getJob(streamId);
if (!job) {
- const currentJob = await this.reconcileJobMembership(streamId, { initialJob: null });
+ const currentJob = await this.reconcileJobMembership(streamId, {
+ initialJob: null,
+ });
this.clearLocalStateUnlessActive(streamId, currentJob);
return 1;
}
@@ -3236,7 +3424,9 @@ export class RedisJobStore implements IJobStoreV2 {
patch: {
completedAt: now,
error: PAUSE_PERSISTENCE_TIMEOUT_ERROR,
- ...(job.agentEventDeliveryKey != null && { terminalHostActionPending: true }),
+ ...(job.agentEventDeliveryKey != null && {
+ terminalHostActionPending: true,
+ }),
},
clear: [
'pendingAction',
@@ -3520,14 +3710,20 @@ export class RedisJobStore implements IJobStoreV2 {
continue;
}
if (event.event === 'on_steer_applied') {
- const steerData = event.data as { index?: number; part?: Agents.MessageContentComplex };
+ const steerData = event.data as {
+ index?: number;
+ part?: Agents.MessageContentComplex;
+ };
if (typeof steerData.index === 'number' && steerData.part != null) {
steers.push({ index: steerData.index, part: steerData.part });
}
continue;
}
if (event.event === 'on_activity_label') {
- const labelData = event.data as { index?: number; part?: Agents.MessageContentComplex };
+ const labelData = event.data as {
+ index?: number;
+ part?: Agents.MessageContentComplex;
+ };
if (typeof labelData.index === 'number' && labelData.part != null) {
labelsByIndex.set(labelData.index, labelData.part);
}
@@ -3727,12 +3923,17 @@ export class RedisJobStore implements IJobStoreV2 {
async getContentParts(
streamId: string,
expectedCreatedAt?: number,
+ options?: { durableOnly?: boolean },
): Promise<{
content: Agents.MessageContentComplex[];
+ reconstructedEventCount?: number;
+ durableEventCount?: number;
} | null> {
// 1. Prefer the HOST content array (same-instance fast path): it already
// contains host-authored steer parts the SDK graph never sees.
- const hostEntry = this.getLocalEntry(this.localContentParts, streamId, expectedCreatedAt);
+ const hostEntry = options?.durableOnly
+ ? undefined
+ : this.getLocalEntry(this.localContentParts, streamId, expectedCreatedAt);
if (hostEntry) {
const hostParts = hostEntry.value.deref();
if (hostParts && hostParts.length > 0) {
@@ -3747,7 +3948,9 @@ export class RedisJobStore implements IJobStoreV2 {
// lacks host-authored steer parts, so overlay them from the chunk log —
// insert (not assign): the graph array is UNSHIFTED, while recorded steer
// indices are host-view positions that already account for prior steers.
- const graphEntry = this.getLocalEntry(this.localGraphCache, streamId, expectedCreatedAt);
+ const graphEntry = options?.durableOnly
+ ? undefined
+ : this.getLocalEntry(this.localGraphCache, streamId, expectedCreatedAt);
if (graphEntry) {
const graph = graphEntry.value.deref();
if (graph) {
@@ -3766,8 +3969,13 @@ export class RedisJobStore implements IJobStoreV2 {
}
// 2. Fall back to Redis chunk reconstruction (cross-instance reconnect)
- const chunks = await this.getChunks(streamId, expectedCreatedAt);
- if (chunks.length === 0) {
+ const chunkSnapshot = await this.getChunkSnapshot(
+ streamId,
+ expectedCreatedAt,
+ options?.durableOnly === true,
+ );
+ const { chunks } = chunkSnapshot;
+ if (chunkSnapshot.durableEventCount === 0) {
return null;
}
@@ -3807,7 +4015,10 @@ export class RedisJobStore implements IJobStoreV2 {
// injection were emitted with already-shifted indices, so both sources
// land disjoint.
if (event.event === 'on_steer_applied') {
- const steerData = event.data as { index?: number; part?: Agents.MessageContentComplex };
+ const steerData = event.data as {
+ index?: number;
+ part?: Agents.MessageContentComplex;
+ };
if (typeof steerData.index === 'number' && steerData.part != null) {
contentParts[steerData.index] = steerData.part;
}
@@ -3818,7 +4029,10 @@ export class RedisJobStore implements IJobStoreV2 {
// fixed index. The event fires twice per slot (counts placeholder,
// then resolved label); chronological replay makes the last write win.
if (event.event === 'on_activity_label') {
- const labelData = event.data as { index?: number; part?: Agents.MessageContentComplex };
+ const labelData = event.data as {
+ index?: number;
+ part?: Agents.MessageContentComplex;
+ };
if (typeof labelData.index === 'number' && labelData.part != null) {
contentParts[labelData.index] = labelData.part;
}
@@ -3977,6 +4191,10 @@ export class RedisJobStore implements IJobStoreV2 {
return {
content: filtered,
+ ...(options?.durableOnly === true && {
+ reconstructedEventCount: chunks.length,
+ durableEventCount: chunkSnapshot.durableEventCount,
+ }),
};
}
@@ -4549,7 +4767,13 @@ export class RedisJobStore implements IJobStoreV2 {
pending = undefined;
}
if (!pending) {
- pending = { expectedCreatedAt, events: [], settlers: [], bytes: 0, timer: null };
+ pending = {
+ expectedCreatedAt,
+ events: [],
+ settlers: [],
+ bytes: 0,
+ timer: null,
+ };
this.pendingAppends.set(streamId, pending);
}
@@ -4626,24 +4850,45 @@ export class RedisJobStore implements IJobStoreV2 {
* Get all chunks from Redis Stream.
*/
private async getChunks(streamId: string, expectedCreatedAt?: number): Promise<unknown[]> {
+ return (await this.getChunkSnapshot(streamId, expectedCreatedAt)).chunks;
+ }
+
+ private async getChunkSnapshot(
+ streamId: string,
+ expectedCreatedAt?: number,
+ includeDurableEventCount = false,
+ ): Promise<{ chunks: unknown[]; durableEventCount: number }> {
/** A same-replica snapshot read must observe the appends this process has
* already accepted, or a resume during an active window reconstructs
* without the buffered tail. Cross-replica readers keep today's contract:
* the log may trail live emission by up to one window. */
await this.flushCoalescedAppends(streamId);
- const rawEntries =
- expectedCreatedAt == null
- ? await this.redis.xrange(KEYS.chunks(streamId), '-', '+')
- : await this.redis.eval(
- CHUNKS_READ_LUA,
- 2,
- KEYS.job(streamId),
- KEYS.chunks(streamId),
- String(expectedCreatedAt),
- );
+ let rawEntries: unknown;
+ let rawDurableEventCount: unknown;
+ if (includeDurableEventCount) {
+ const rawSnapshot = await this.redis.eval(
+ CHUNKS_RECOVERY_READ_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.chunks(streamId),
+ expectedCreatedAt != null ? String(expectedCreatedAt) : '',
+ );
+ [rawEntries, rawDurableEventCount] = Array.isArray(rawSnapshot) ? rawSnapshot : [];
+ } else {
+ rawEntries =
+ expectedCreatedAt == null
+ ? await this.redis.xrange(KEYS.chunks(streamId), '-', '+')
+ : await this.redis.eval(
+ CHUNKS_READ_LUA,
+ 2,
+ KEYS.job(streamId),
+ KEYS.chunks(streamId),
+ String(expectedCreatedAt),
+ );
+ }
const entries = Array.isArray(rawEntries) ? (rawEntries as Array<[string, string[]]>) : [];
- return entries
+ const chunks = entries
.map(([, fields]) => {
const eventIdx = fields.indexOf('event');
if (eventIdx >= 0 && eventIdx + 1 < fields.length) {
@@ -4656,6 +4901,16 @@ export class RedisJobStore implements IJobStoreV2 {
return null;
})
.filter(Boolean);
+ const parsedDurableEventCount =
+ typeof rawDurableEventCount === 'string' || typeof rawDurableEventCount === 'number'
+ ? Number(rawDurableEventCount)
+ : Number.NaN;
+ return {
+ chunks,
+ durableEventCount: Number.isFinite(parsedDurableEventCount)
+ ? parsedDurableEventCount
+ : entries.length,
+ };
}
/**
@@ -4916,6 +5171,13 @@ export class RedisJobStore implements IJobStoreV2 {
completedAt: data.completedAt ? parseInt(data.completedAt, 10) : undefined,
conversationId: data.conversationId || undefined,
error: data.error || undefined,
+ earlyBufferOverflow: data.earlyBufferOverflow
+ ? (JSON.parse(data.earlyBufferOverflow) as EarlyBufferOverflowState)
+ : undefined,
+ firstSubscriberAttachedAt: data.firstSubscriberAttachedAt
+ ? parseInt(data.firstSubscriberAttachedAt, 10)
+ : undefined,
+ durableEventCount: data.durableEventCount ? parseInt(data.durableEventCount, 10) : undefined,
idempotencyClientRequestId: data.idempotencyClientRequestId || undefined,
recoveredSteerId: data.recoveredSteerId || undefined,
userMessage: data.userMessage ? JSON.parse(data.userMessage) : undefined,
diff --git a/packages/api/src/stream/index.ts b/packages/api/src/stream/index.ts
index cef396978e..752c454148 100644
--- a/packages/api/src/stream/index.ts
+++ b/packages/api/src/stream/index.ts
@@ -4,6 +4,7 @@ export {
type CreateGenerationJobOptions,
type GenerationJobManagerOptions,
type TerminalJobClaim,
+ GENERATION_RECOVERY_FAILED_ERROR,
TERMINAL_PUBLICATION_RECONNECT_ERROR,
} from './GenerationJobManager';
diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.ts
index 18064da049..ff128e1bda 100644
--- a/packages/api/src/stream/interfaces/IJobStore.ts
+++ b/packages/api/src/stream/interfaces/IJobStore.ts
@@ -10,6 +10,7 @@ import type {
} from 'librechat-data-provider';
import type { RunStep, StandardGraph } from '@librechat/agents';
import type { AgentEventDetachedTerminalEvidence } from '~/agents/triggers/types';
+import type { EarlyBufferOverflowState } from '../../types/earlyBufferRecovery';
import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';
import type { ResolvedAskUserQuestion } from '~/agents/hitl/resume';
import type { RecoveredSteerPayload } from '../SteerRecovery';
@@ -121,6 +122,17 @@ export interface SerializableJobData {
conversationId?: string;
error?: string;
+ /** Durable, non-sensitive identity and one-shot outcome for an early replay
+ * buffer overflow. This lets another replica account for recovery. */
+ earlyBufferOverflow?: EarlyBufferOverflowState;
+
+ /** Generation-level first subscriber claim shared across replicas. */
+ firstSubscriberAttachedAt?: number;
+ /** Expiring local subscriber-group leases (in-memory store only). */
+ activeSubscriberLeases?: Record<string, number>;
+ /** Generation-wide durable chunk frontier maintained by the store. */
+ durableEventCount?: number;
+
/** Stable identity of the HTTP submission that created this generation.
* Internal-only: lets an expired idempotency lease recognize the same live
* job instead of replacing and billing it again. */
@@ -950,7 +962,13 @@ export interface IJobStore {
getContentParts(
streamId: string,
expectedCreatedAt?: number,
- ): Promise<{ content: Agents.MessageContentComplex[] } | null>;
+ options?: { durableOnly?: boolean },
+ ): Promise<{
+ content: Agents.MessageContentComplex[];
+ reconstructedEventCount?: number;
+ durableEventCount?: number;
+ } | null>;
+
getRunSteps(streamId: string, expectedCreatedAt?: number): Promise<Agents.RunStep[]>;
/** Legacy stores returned `void`; v2 stores return whether the epoch-fenced
@@ -1269,10 +1287,58 @@ export interface IJobStoreV2 extends IJobStore {
getContentParts(
streamId: string,
expectedCreatedAt?: number,
+ options?: { durableOnly?: boolean },
): Promise<{
content: Agents.MessageContentComplex[];
+ reconstructedEventCount?: number;
+ durableEventCount?: number;
} | null>;
+ /** Atomically records the only recovery outcome for one overflow identity. */
+ settleEarlyBufferRecovery(
+ streamId: string,
+ expectedCreatedAt: number,
+ overflowId: string,
+ settlement: Pick<
+ EarlyBufferOverflowState,
+ 'recoveryMethod' | 'recoveryOutcome' | 'recoveryCompletedAt' | 'recoveryFailureReason'
+ >,
+ ): Promise<boolean>;
+
+ /** Atomically replaces an unresolved pending overflow marker with its
+ * finalized durable frontier. A concurrent recovery settlement wins over
+ * this owner-side finalization. */
+ finalizeEarlyBufferOverflow(
+ streamId: string,
+ expectedCreatedAt: number,
+ overflowId: string,
+ overflow: EarlyBufferOverflowState,
+ ): Promise<boolean>;
+
+ /** Whether this generation has admitted any subscriber on any replica. */
+ hasSubscriberAttached(streamId: string, expectedCreatedAt: number): Promise<boolean>;
+
+ /** Atomically claims the first subscriber for one generation epoch. */
+ claimFirstSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ attachedAt: number,
+ subscriberId: string,
+ leaseExpiresAt: number,
+ ): Promise<boolean>;
+
+ detachSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ subscriberId: string,
+ ): Promise<void>;
+
+ hasActiveSubscriber(
+ streamId: string,
+ expectedCreatedAt: number,
+ observedAt: number,
+ ): Promise<boolean>;
+
/**
* Get run steps for a job (for resume state).
*
diff --git a/packages/api/src/stream/jobStoreCapabilities.ts b/packages/api/src/stream/jobStoreCapabilities.ts
index 92a6f915cb..7603c983f0 100644
--- a/packages/api/src/stream/jobStoreCapabilities.ts
+++ b/packages/api/src/stream/jobStoreCapabilities.ts
@@ -30,6 +30,12 @@ export const JOB_STORE_V2_REQUIRED_METHODS = [
'claimParkedSteersDetailed',
'consumeParkedSteer',
'discardSteerLeftover',
+ 'settleEarlyBufferRecovery',
+ 'finalizeEarlyBufferOverflow',
+ 'hasSubscriberAttached',
+ 'claimFirstSubscriber',
+ 'detachSubscriber',
+ 'hasActiveSubscriber',
] as const satisfies ReadonlyArray<keyof IJobStoreV2>;
export type JobStoreV2RequiredMethod = (typeof JOB_STORE_V2_REQUIRED_METHODS)[number];
diff --git a/packages/api/src/types/earlyBufferRecovery.ts b/packages/api/src/types/earlyBufferRecovery.ts
new file mode 100644
index 0000000000..fe83386c7d
--- /dev/null
+++ b/packages/api/src/types/earlyBufferRecovery.ts
@@ -0,0 +1,25 @@
+export type EarlyBufferRecoveryMethod = 'redis' | 'snapshot';
+export type EarlyBufferRecoveryOutcome = 'success' | 'failed' | 'not_required';
+export type EarlyBufferRecoveryFailureReason =
+ | 'durable_state_missing'
+ | 'durable_frontier_gap'
+ | 'snapshot_missing'
+ | 'subscriber_never_attached'
+ | 'subscriber_disconnected'
+ | 'reconstruction_error'
+ | 'overflow_marker_persistence_failed';
+
+export interface EarlyBufferOverflowState {
+ id: string;
+ occurredAt: number;
+ durableEvents: number;
+ droppedEvents: number;
+ droppedBytes: number;
+ /** True while the owner is flushing accepted durable appends and has not
+ * published the final recovery frontier yet. */
+ persistencePending?: boolean;
+ recoveryMethod?: EarlyBufferRecoveryMethod;
+ recoveryOutcome?: EarlyBufferRecoveryOutcome;
+ recoveryCompletedAt?: number;
+ recoveryFailureReason?: EarlyBufferRecoveryFailureReason;
+}
diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.ts
index d6ec148c87..c7571c557f 100644
--- a/packages/api/src/types/stream.ts
+++ b/packages/api/src/types/stream.ts
@@ -10,6 +10,7 @@ import type {
AgentTriggerExpectedAction,
} from '../agents/triggers/types';
import type { ActivityPhaseSnapshot } from '~/agents/activityPhases/runtime';
+import type { EarlyBufferOverflowState } from './earlyBufferRecovery';
import type { ResolvedAskUserQuestion } from '../agents/hitl/resume';
import type { MCPRuntimeRequestBody } from '../mcp/types';
import type { ServerSentEvent } from './events';
@@ -23,6 +24,7 @@ export interface GenerationJobMetadata {
checkpointNamespace?: string;
/** Immutable generation protocol. Missing on legacy records means v1. */
generationProtocolVersion?: 1 | 2;
+ earlyBufferOverflow?: EarlyBufferOverflowState;
/** User message data for rebuilding submission on reconnect */
userMessage?: Agents.UserMessageMeta;
/** Response message ID for tracking */