Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🌊 fix: Prevent Buffered Event Duplication on SSE Resume Connections (#12225)
Danny Avila7bc793b18db3
diff --git a/api/server/routes/agents/index.js b/api/server/routes/agents/index.jsindex f8d39cb4d8..a99fdca592 100644--- a/api/server/routes/agents/index.js+++ b/api/server/routes/agents/index.js@@ -76,52 +76,62 @@ router.get('/chat/stream/:streamId', async (req, res) => {logger.debug(`[AgentStream] Client subscribed to ${streamId}, resume: ${isResume}`);- // Send sync event with resume state for ALL reconnecting clients- // This supports multi-tab scenarios where each tab needs run step data- if (isResume) {- const resumeState = await GenerationJobManager.getResumeState(streamId);- if (resumeState && !res.writableEnded) {- // Send sync event with run steps AND aggregatedContent- // Client will use aggregatedContent to initialize message state- res.write(`event: message\ndata: ${JSON.stringify({ sync: true, resumeState })}\n\n`);+ const writeEvent = (event) => {+ if (!res.writableEnded) {+ res.write(`event: message\ndata: ${JSON.stringify(event)}\n\n`);if (typeof res.flush === 'function') {res.flush();}- logger.debug(- `[AgentStream] Sent sync event for ${streamId} with ${resumeState.runSteps.length} run steps`,- );}- }+ };- const result = await GenerationJobManager.subscribe(- streamId,- (event) => {- if (!res.writableEnded) {- res.write(`event: message\ndata: ${JSON.stringify(event)}\n\n`);- if (typeof res.flush === 'function') {- res.flush();- }+ const onDone = (event) => {+ writeEvent(event);+ res.end();+ };++ const onError = (error) => {+ if (!res.writableEnded) {+ res.write(`event: error\ndata: ${JSON.stringify({ error })}\n\n`);+ if (typeof res.flush === 'function') {+ res.flush();}- },- (event) => {- if (!res.writableEnded) {- res.write(`event: message\ndata: ${JSON.stringify(event)}\n\n`);+ res.end();+ }+ };++ let result;++ if (isResume) {+ const { subscription, resumeState, pendingEvents } =+ await GenerationJobManager.subscribeWithResume(streamId, writeEvent, onDone, onError);++ if (!res.writableEnded) {+ if (resumeState) {+ res.write(+ `event: message\ndata: ${JSON.stringify({ sync: true, resumeState, pendingEvents })}\n\n`,+ );if (typeof res.flush === 'function') {res.flush();}- res.end();- }- },- (error) => {- if (!res.writableEnded) {- res.write(`event: error\ndata: ${JSON.stringify({ error })}\n\n`);- if (typeof res.flush === 'function') {- res.flush();+ GenerationJobManager.markSyncSent(streamId);+ logger.debug(+ `[AgentStream] Sent sync event for ${streamId} with ${resumeState.runSteps.length} run steps, ${pendingEvents.length} pending events`,+ );+ } else if (pendingEvents.length > 0) {+ for (const event of pendingEvents) {+ writeEvent(event);}- res.end();+ logger.warn(+ `[AgentStream] Resume state null for ${streamId}, replayed ${pendingEvents.length} gap events directly`,+ );}- },- );+ }++ result = subscription;+ } else {+ result = await GenerationJobManager.subscribe(streamId, writeEvent, onDone, onError);+ }if (!result) {return res.status(404).json({ error: 'Failed to subscribe to stream' });diff --git a/client/src/hooks/SSE/useResumableSSE.ts b/client/src/hooks/SSE/useResumableSSE.tsindex 831bf042ad..4d4cb4841a 100644--- a/client/src/hooks/SSE/useResumableSSE.ts+++ b/client/src/hooks/SSE/useResumableSSE.ts@@ -226,12 +226,12 @@ export default function useResumableSSE(if (data.sync != null) {console.log('[ResumableSSE] SYNC received', {runSteps: data.resumeState?.runSteps?.length ?? 0,+ pendingEvents: data.pendingEvents?.length ?? 0,});const runId = v4();setActiveRunId(runId);- // Replay run stepsif (data.resumeState?.runSteps) {for (const runStep of data.resumeState.runSteps) {stepHandler({ event: 'on_run_step', data: runStep }, {@@ -241,19 +241,15 @@ export default function useResumableSSE(}}- // Set message content from aggregatedContentif (data.resumeState?.aggregatedContent && userMessage?.messageId) {const messages = getMessages() ?? [];const userMsgId = userMessage.messageId;const serverResponseId = data.resumeState.responseMessageId;- // Find the EXACT response message - prioritize responseMessageId from server- // This is critical when there are multiple responses to the same user messagelet responseIdx = -1;if (serverResponseId) {responseIdx = messages.findIndex((m) => m.messageId === serverResponseId);}- // Fallback: find by parentMessageId pattern (for new messages)if (responseIdx < 0) {responseIdx = messages.findIndex((m) =>@@ -272,7 +268,6 @@ export default function useResumableSSE(});if (responseIdx >= 0) {- // Update existing response message with aggregatedContentconst updated = [...messages];const oldContent = updated[responseIdx]?.content;updated[responseIdx] = {@@ -285,25 +280,34 @@ export default function useResumableSSE(newContentLength: data.resumeState.aggregatedContent?.length,});setMessages(updated);- // Sync both content handler and step handler with the updated message- // so subsequent deltas build on synced content, not stale contentresetContentHandler();syncStepMessage(updated[responseIdx]);console.log('[ResumableSSE] SYNC complete, handlers synced');} else {- // Add new response messageconst responseId = serverResponseId ?? `${userMsgId}_`;- setMessages([- ...messages,- {- messageId: responseId,- parentMessageId: userMsgId,- conversationId: currentSubmission.conversation?.conversationId ?? '',- text: '',- content: data.resumeState.aggregatedContent,- isCreatedByUser: false,- } as TMessage,- ]);+ const newMessage = {+ messageId: responseId,+ parentMessageId: userMsgId,+ conversationId: currentSubmission.conversation?.conversationId ?? '',+ text: '',+ content: data.resumeState.aggregatedContent,+ isCreatedByUser: false,+ } as TMessage;+ setMessages([...messages, newMessage]);+ resetContentHandler();+ syncStepMessage(newMessage);+ }+ }++ if (data.pendingEvents?.length > 0) {+ console.log(`[ResumableSSE] Replaying ${data.pendingEvents.length} pending events`);+ const submission = { ...currentSubmission, userMessage } as EventSubmission;+ for (const pendingEvent of data.pendingEvents) {+ if (pendingEvent.event != null) {+ stepHandler(pendingEvent, submission);+ } else if (pendingEvent.type != null) {+ contentHandler({ data: pendingEvent, submission });+ }}}diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex cd5ff04eb0..1b612dcb8f 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -707,6 +707,10 @@ class GenerationJobManagerClass {* @param onChunk - Handler for chunk events (streamed tokens, run steps, etc.)* @param onDone - Handler for completion event (includes final message)* @param onError - Handler for error events+ * @param options - Subscription configuration+ * @param options.skipBufferReplay - When true, skips replaying the earlyEventBuffer.+ * Use this when a sync event was already sent (resume), since the sync's+ * aggregatedContent already includes all buffered events.* @returns Subscription object with unsubscribe function, or null if job not found*/async subscribe(@@ -714,6 +718,7 @@ class GenerationJobManagerClass {onChunk: t.ChunkHandler,onDone?: t.DoneHandler,onError?: t.ErrorHandler,+ options?: t.SubscribeOptions,): Promise<{ unsubscribe: t.UnsubscribeFn } | null> {// Use lazy initialization to support cross-replica subscriptionsconst runtime = await this.getOrCreateRuntimeState(streamId);@@ -763,11 +768,17 @@ class GenerationJobManagerClass {runtime.hasSubscriber = true;if (runtime.earlyEventBuffer.length > 0) {- logger.debug(- `[GenerationJobManager] Replaying ${runtime.earlyEventBuffer.length} buffered events for ${streamId}`,- );- for (const bufferedEvent of runtime.earlyEventBuffer) {- onChunk(bufferedEvent);+ if (options?.skipBufferReplay) {+ logger.debug(+ `[GenerationJobManager] Skipping ${runtime.earlyEventBuffer.length} buffered events for ${streamId} (skipBufferReplay)`,+ );+ } else {+ logger.debug(+ `[GenerationJobManager] Replaying ${runtime.earlyEventBuffer.length} buffered events for ${streamId}`,+ );+ for (const bufferedEvent of runtime.earlyEventBuffer) {+ onChunk(bufferedEvent);+ }}runtime.earlyEventBuffer = [];}@@ -785,6 +796,52 @@ class GenerationJobManagerClass {return subscription;}+ /**+ * Atomic resume + subscribe: snapshots resume state and drains the early event buffer+ * in one synchronous step, then subscribes with skipBufferReplay.+ *+ * Closes the timing gap between separate `getResumeState()` and `subscribe()` calls+ * where events could arrive in earlyEventBuffer after the snapshot but before subscribe+ * clears the buffer.+ *+ * In-memory mode: drained buffer events are returned as `pendingEvents` since+ * they exist nowhere else. The caller must deliver them after the sync payload.+ * Redis mode: `pendingEvents` is empty — chunks are persisted via appendChunk+ * and will appear in aggregatedContent on the next resume.+ */+ async subscribeWithResume(+ streamId: string,+ onChunk: t.ChunkHandler,+ onDone?: t.DoneHandler,+ onError?: t.ErrorHandler,+ ): Promise<t.SubscribeWithResumeResult> {+ const bufferLengthAtSnapshot = !this._isRedis+ ? (this.runtimeState.get(streamId)?.earlyEventBuffer.length ?? 0)+ : 0;++ const resumeState = await this.getResumeState(streamId);++ let pendingEvents: t.ServerSentEvent[] = [];+ if (!this._isRedis) {+ const runtime = this.runtimeState.get(streamId);+ if (runtime) {+ pendingEvents = runtime.earlyEventBuffer.slice(bufferLengthAtSnapshot);+ runtime.earlyEventBuffer = [];+ if (pendingEvents.length > 0) {+ logger.debug(+ `[GenerationJobManager] Captured ${pendingEvents.length} gap events for ${streamId}`,+ );+ }+ }+ }++ const subscription = await this.subscribe(streamId, onChunk, onDone, onError, {+ skipBufferReplay: true,+ });++ return { subscription, resumeState, pendingEvents };+ }+/*** Emit a chunk event to all subscribers.* Uses runtime state check for performance (avoids async job store lookup per token).diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.tsindex 59fe32e4e5..2f23510018 100644--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts@@ -1,3 +1,4 @@+/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */import type { Redis, Cluster } from 'ioredis';import type { ServerSentEvent } from '~/types/events';import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';@@ -27,6 +28,9 @@ describe('GenerationJobManager Integration Tests', () => {let dynamicKeyvClient: unknown = null;let dynamicKeyvReady: Promise<unknown> | null = null;const testPrefix = 'JobManager-Integration-Test';+ const redisConfigured = process.env.USE_REDIS === 'true';+ const describeRedis = redisConfigured ? describe : describe.skip;+ const testRedis = redisConfigured ? test : test.skip;beforeAll(async () => {originalEnv = { ...process.env };@@ -82,6 +86,68 @@ describe('GenerationJobManager Integration Tests', () => {process.env = originalEnv;});+ function createInMemoryManager(): GenerationJobManagerClass {+ const manager = new GenerationJobManagerClass();+ manager.configure({+ jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),+ eventTransport: new InMemoryEventTransport(),+ isRedis: false,+ });+ manager.initialize();+ return manager;+ }++ function createRedisManager(): GenerationJobManagerClass {+ const manager = new GenerationJobManagerClass();+ manager.configure(+ createStreamServices({+ useRedis: true,+ redisClient: ioredisClient!,+ }),+ );+ manager.initialize();+ return manager;+ }++ async function setupDisconnectedStream(+ manager: GenerationJobManagerClass,+ streamId: string,+ delay: number,+ ): Promise<ServerSentEvent[]> {+ const firstEvents: ServerSentEvent[] = [];+ const sub = await manager.subscribe(streamId, (event) => firstEvents.push(event));++ await new Promise((resolve) => setTimeout(resolve, delay));++ await manager.emitChunk(streamId, {+ event: 'on_run_step',+ data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },+ });+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'Hello' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, delay));+ expect(firstEvents.length).toBe(2);++ sub?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, delay));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: ' world' } } },+ });+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: '!' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, delay));++ return firstEvents;+ }+describe('In-Memory Mode', () => {test('should create and manage jobs', async () => {// Configure with in-memory@@ -171,13 +237,8 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Redis Mode', () => {+ describeRedis('Redis Mode', () => {test('should create and manage jobs via Redis', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-// Create Redis servicesconst services = createStreamServices({useRedis: true,@@ -209,11 +270,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should persist chunks for cross-instance resume', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const services = createStreamServices({useRedis: true,redisClient: ioredisClient,@@ -264,11 +320,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should handle abort and return content', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const services = createStreamServices({useRedis: true,redisClient: ioredisClient,@@ -374,7 +425,7 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Cross-Replica Support (Redis)', () => {+ describeRedis('Cross-Replica Support (Redis)', () => {/*** Problem: In k8s with Redis and multiple replicas, when a user sends a message:* 1. POST /api/agents/chat hits Replica A, creates job@@ -387,15 +438,10 @@ describe('GenerationJobManager Integration Tests', () => {* when the job exists in Redis but not in local memory.*/test('should NOT return 404 when stream endpoint hits different replica than job creator', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-// === REPLICA A: Creates the job ===// Simulate Replica A creating the job directly in Redis// (In real scenario, this happens via GenerationJobManager.createJob on Replica A)- const replicaAJobStore = new RedisJobStore(ioredisClient);+ const replicaAJobStore = new RedisJobStore(ioredisClient!);await replicaAJobStore.initialize();const streamId = `cross-replica-404-test-${Date.now()}`;@@ -452,13 +498,8 @@ describe('GenerationJobManager Integration Tests', () => {});test('should lazily create runtime state for jobs created on other replicas', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-// Instance 1: Create the job directly in Redis (simulating another replica)- const jobStore = new RedisJobStore(ioredisClient);+ const jobStore = new RedisJobStore(ioredisClient!);await jobStore.initialize();const streamId = `cross-replica-${Date.now()}`;@@ -500,11 +541,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should persist syncSent to Redis for cross-replica consistency', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const services = createStreamServices({useRedis: true,redisClient: ioredisClient,@@ -539,11 +575,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should persist finalEvent to Redis for cross-replica access', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const services = createStreamServices({useRedis: true,redisClient: ioredisClient,@@ -581,11 +612,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should emit cross-replica abort signal via Redis pub/sub', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const services = createStreamServices({useRedis: true,redisClient: ioredisClient,@@ -620,16 +646,11 @@ describe('GenerationJobManager Integration Tests', () => {});test('should handle abort for lazily-initialized cross-replica jobs', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-// This test validates that jobs created on Replica A and lazily-initialized// on Replica B can still receive and handle abort signals.// === Replica A: Create job directly in Redis ===- const replicaAJobStore = new RedisJobStore(ioredisClient);+ const replicaAJobStore = new RedisJobStore(ioredisClient!);await replicaAJobStore.initialize();const streamId = `lazy-abort-${Date.now()}`;@@ -675,11 +696,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should abort generation when abort signal received from another replica', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-// This test simulates:// 1. Replica A creates a job and starts generation// 2. Replica B receives abort request and emits abort signal@@ -729,13 +745,8 @@ describe('GenerationJobManager Integration Tests', () => {});test('should handle wasSyncSent for cross-replica scenarios', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-// Create job directly in Redis with syncSent: true- const jobStore = new RedisJobStore(ioredisClient);+ const jobStore = new RedisJobStore(ioredisClient!);await jobStore.initialize();const streamId = `cross-sync-${Date.now()}`;@@ -762,7 +773,7 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Sequential Event Ordering (Redis)', () => {+ describeRedis('Sequential Event Ordering (Redis)', () => {/*** These tests verify that events are delivered in strict sequential order* when using Redis mode. This is critical because:@@ -773,11 +784,6 @@ describe('GenerationJobManager Integration Tests', () => {* The fix: emitChunk now awaits Redis publish to ensure ordered delivery.*/test('should maintain strict order for rapid sequential emits', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-jest.resetModules();const services = createStreamServices({@@ -823,11 +829,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should maintain order for tool call argument deltas', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-jest.resetModules();const services = createStreamServices({@@ -882,11 +883,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should maintain order: on_run_step before on_run_step_delta', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-jest.resetModules();const services = createStreamServices({@@ -945,11 +941,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should not block other streams when awaiting emitChunk', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-jest.resetModules();const services = createStreamServices({@@ -1069,12 +1060,7 @@ describe('GenerationJobManager Integration Tests', () => {await manager.destroy();});- test('should buffer and replay events emitted before subscribe (Redis)', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-+ testRedis('should buffer and replay events emitted before subscribe (Redis)', async () => {const manager = new GenerationJobManagerClass();const services = createStreamServices({useRedis: true,@@ -1118,67 +1104,60 @@ describe('GenerationJobManager Integration Tests', () => {await manager.destroy();});- test('should not lose events when emitting before and after subscribe (Redis)', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-- const manager = new GenerationJobManagerClass();- const services = createStreamServices({- useRedis: true,- redisClient: ioredisClient,- });-- manager.configure(services);- manager.initialize();-- const streamId = `no-loss-${Date.now()}`;- await manager.createJob(streamId, 'user-1');-- await manager.emitChunk(streamId, {- created: true,- message: { text: 'hello' },- streamId,- } as unknown as ServerSentEvent);- await manager.emitChunk(streamId, {- event: 'on_run_step',- data: { id: 'step-1', type: 'message_creation', index: 0 },- });+ testRedis(+ 'should not lose events when emitting before and after subscribe (Redis)',+ async () => {+ const manager = new GenerationJobManagerClass();+ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });- const receivedEvents: unknown[] = [];- const subscription = await manager.subscribe(streamId, (event: unknown) =>- receivedEvents.push(event),- );+ manager.configure(services);+ manager.initialize();- await new Promise((resolve) => setTimeout(resolve, 100));+ const streamId = `no-loss-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');- for (let i = 0; i < 10; i++) {await manager.emitChunk(streamId, {- event: 'on_message_delta',- data: { delta: { content: { type: 'text', text: `word${i} ` } }, index: i },+ created: true,+ message: { text: 'hello' },+ streamId,+ } as unknown as ServerSentEvent);+ await manager.emitChunk(streamId, {+ event: 'on_run_step',+ data: { id: 'step-1', type: 'message_creation', index: 0 },});- }- await new Promise((resolve) => setTimeout(resolve, 300));+ const receivedEvents: unknown[] = [];+ const subscription = await manager.subscribe(streamId, (event: unknown) =>+ receivedEvents.push(event),+ );- expect(receivedEvents.length).toBe(12);- expect((receivedEvents[0] as Record<string, unknown>).created).toBe(true);- expect((receivedEvents[1] as Record<string, unknown>).event).toBe('on_run_step');- for (let i = 0; i < 10; i++) {- expect((receivedEvents[i + 2] as Record<string, unknown>).event).toBe('on_message_delta');- }+ await new Promise((resolve) => setTimeout(resolve, 100));- subscription?.unsubscribe();- await manager.destroy();- });+ for (let i = 0; i < 10; i++) {+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: `word${i} ` } }, index: i },+ });+ }- test('RedisEventTransport.subscribe() should return a ready promise', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }+ await new Promise((resolve) => setTimeout(resolve, 300));++ expect(receivedEvents.length).toBe(12);+ expect((receivedEvents[0] as Record<string, unknown>).created).toBe(true);+ expect((receivedEvents[1] as Record<string, unknown>).event).toBe('on_run_step');+ for (let i = 0; i < 10; i++) {+ expect((receivedEvents[i + 2] as Record<string, unknown>).event).toBe('on_message_delta');+ }+ subscription?.unsubscribe();+ await manager.destroy();+ },+ );++ testRedis('RedisEventTransport.subscribe() should return a ready promise', async () => {const subscriber = (ioredisClient as unknown as { duplicate: () => unknown }).duplicate();const transport = new RedisEventTransport(ioredisClient as never, subscriber as never);@@ -1211,6 +1190,421 @@ describe('GenerationJobManager Integration Tests', () => {});});+ describe('Resume: skipBufferReplay prevents duplication', () => {+ /**+ * Verifies the fix for duplicated content when navigating away from an+ * in-progress conversation and back. Events accumulate in earlyEventBuffer+ * while the subscriber is absent. On resume, the sync event delivers all+ * accumulated content via aggregatedContent, so buffer replay must be+ * skipped to prevent duplication.+ */++ test('should NOT replay buffer when skipBufferReplay is true (resume scenario)', async () => {+ const manager = createInMemoryManager();+ const streamId = `skip-buf-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ await setupDisconnectedStream(manager, streamId, 10);++ const resumeState = await manager.getResumeState(streamId);+ expect(resumeState).not.toBeNull();++ const resumeEvents: ServerSentEvent[] = [];+ const sub2 = await manager.subscribe(+ streamId,+ (event) => resumeEvents.push(event),+ undefined,+ undefined,+ { skipBufferReplay: true },+ );++ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(resumeEvents.length).toBe(0);++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: ' Live!' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(resumeEvents.length).toBe(1);+ expect(resumeEvents[0].event).toBe('on_message_delta');++ sub2?.unsubscribe();+ await manager.destroy();+ });++ test('should replay buffer by default when no options are passed', async () => {+ const manager = createInMemoryManager();+ const streamId = `replay-buf-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1Events: ServerSentEvent[] = [];+ const sub1 = await manager.subscribe(streamId, (event) => sub1Events.push(event));+ await new Promise((resolve) => setTimeout(resolve, 10));++ await manager.emitChunk(streamId, {+ event: 'on_run_step',+ data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },+ });+ await new Promise((resolve) => setTimeout(resolve, 10));++ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'buffered' } } },+ });++ const sub2Events: ServerSentEvent[] = [];+ const sub2 = await manager.subscribe(streamId, (event) => sub2Events.push(event));+ await new Promise((resolve) => setTimeout(resolve, 20));++ expect(sub2Events.length).toBe(1);+ expect(sub2Events[0].event).toBe('on_message_delta');++ sub2?.unsubscribe();+ await manager.destroy();+ });++ test('should clear earlyEventBuffer even when skipping replay (no memory leak)', async () => {+ const manager = createInMemoryManager();+ const streamId = `buf-clear-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));+ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buf1' } } },+ });+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buf2' } } },+ });++ const sub2Events: ServerSentEvent[] = [];+ const sub2 = await manager.subscribe(+ streamId,+ (event) => sub2Events.push(event),+ undefined,+ undefined,+ { skipBufferReplay: true },+ );+ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(sub2Events.length).toBe(0);++ sub2?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'new-event' } } },+ });++ const sub3Events: ServerSentEvent[] = [];+ const sub3 = await manager.subscribe(streamId, (event) => sub3Events.push(event));+ await new Promise((resolve) => setTimeout(resolve, 20));++ expect(sub3Events.length).toBe(1);+ const event = sub3Events[0] as {+ event: string;+ data: { delta: { content: { text: string } } };+ };+ expect(event.data.delta.content.text).toBe('new-event');++ sub3?.unsubscribe();+ await manager.destroy();+ });++ test('should handle multiple disconnect/reconnect cycles with skipBufferReplay', async () => {+ const manager = createInMemoryManager();+ const streamId = `multi-reconnect-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'initial' } } },+ });+ await new Promise((resolve) => setTimeout(resolve, 10));++ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buffered-1' } } },+ });++ const resumeState1 = await manager.getResumeState(streamId);+ expect(resumeState1).not.toBeNull();++ const sub2Events: ServerSentEvent[] = [];+ const sub2 = await manager.subscribe(+ streamId,+ (event) => sub2Events.push(event),+ undefined,+ undefined,+ { skipBufferReplay: true },+ );+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(sub2Events.length).toBe(0);++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'live-1' } } },+ });+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(sub2Events.length).toBe(1);++ sub2?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buffered-2' } } },+ });++ const resumeState2 = await manager.getResumeState(streamId);+ expect(resumeState2).not.toBeNull();++ const sub3Events: ServerSentEvent[] = [];+ const sub3 = await manager.subscribe(+ streamId,+ (event) => sub3Events.push(event),+ undefined,+ undefined,+ { skipBufferReplay: true },+ );+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(sub3Events.length).toBe(0);++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'live-2' } } },+ });+ await new Promise((resolve) => setTimeout(resolve, 10));+ expect(sub3Events.length).toBe(1);++ sub3?.unsubscribe();+ await manager.destroy();+ });++ testRedis('should NOT replay buffer when skipBufferReplay is true (Redis)', async () => {+ const manager = createRedisManager();+ const streamId = `skip-buf-redis-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ await setupDisconnectedStream(manager, streamId, 100);++ const resumeState = await manager.getResumeState(streamId);+ expect(resumeState).not.toBeNull();+ expect(resumeState!.aggregatedContent?.length).toBeGreaterThan(0);++ const resumeEvents: ServerSentEvent[] = [];+ const sub2 = await manager.subscribe(+ streamId,+ (event) => resumeEvents.push(event),+ undefined,+ undefined,+ { skipBufferReplay: true },+ );++ await new Promise((resolve) => setTimeout(resolve, 200));+ expect(resumeEvents.length).toBe(0);++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: ' Live!' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, 200));+ expect(resumeEvents.length).toBe(1);+ expect(resumeEvents[0].event).toBe('on_message_delta');++ sub2?.unsubscribe();+ await manager.destroy();+ });++ testRedis(+ 'should replay buffer without skipBufferReplay after disconnect (Redis)',+ async () => {+ const manager = createRedisManager();+ const streamId = `replay-buf-redis-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 100));+ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 100));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buffered-redis' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ const sub2Events: ServerSentEvent[] = [];+ const sub2 = await manager.subscribe(streamId, (event) => sub2Events.push(event));++ await new Promise((resolve) => setTimeout(resolve, 200));++ expect(sub2Events.length).toBe(1);+ expect(sub2Events[0].event).toBe('on_message_delta');++ sub2?.unsubscribe();+ await manager.destroy();+ },+ );+ });++ describe('Atomic subscribeWithResume', () => {+ test('should return empty pendingEvents for pre-snapshot buffer events (in-memory)', async () => {+ const manager = createInMemoryManager();+ const streamId = `atomic-drain-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));+ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_run_step',+ data: { id: 'step-1', runId: 'run-1', index: 0, stepDetails: { type: 'message_creation' } },+ });+ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { id: 'step-1', delta: { content: { type: 'text', text: 'buffered' } } },+ });++ const liveEvents: ServerSentEvent[] = [];+ const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(+ streamId,+ (event) => liveEvents.push(event),+ );++ expect(resumeState).not.toBeNull();+ expect(pendingEvents.length).toBe(0);+ expect(liveEvents.length).toBe(0);++ subscription?.unsubscribe();+ await manager.destroy();+ });++ test('should return empty pendingEvents when buffer is empty', async () => {+ const manager = createInMemoryManager();+ const streamId = `atomic-empty-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'delivered' } } },+ });+ await new Promise((resolve) => setTimeout(resolve, 10));++ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ const { pendingEvents } = await manager.subscribeWithResume(streamId, () => {});++ expect(pendingEvents.length).toBe(0);++ await manager.destroy();+ });++ test('should deliver live events after subscribeWithResume', async () => {+ const manager = createInMemoryManager();+ const streamId = `atomic-live-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 10));+ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 20));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buffered-pre-snapshot' } } },+ });++ const liveEvents: ServerSentEvent[] = [];+ const { subscription, pendingEvents } = await manager.subscribeWithResume(streamId, (event) =>+ liveEvents.push(event),+ );++ expect(pendingEvents.length).toBe(0);+ expect(liveEvents.length).toBe(0);++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'live-after' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, 20));+ expect(liveEvents.length).toBe(1);+ const liveEvent = liveEvents[0] as {+ event: string;+ data: { delta: { content: { text: string } } };+ };+ expect(liveEvent.data.delta.content.text).toBe('live-after');++ subscription?.unsubscribe();+ await manager.destroy();+ });++ testRedis(+ 'should return empty pendingEvents in Redis mode (chunks already persisted)',+ async () => {+ const manager = createRedisManager();+ const streamId = `atomic-redis-${Date.now()}`;+ await manager.createJob(streamId, 'user-1');++ const sub1 = await manager.subscribe(streamId, () => {});+ await new Promise((resolve) => setTimeout(resolve, 100));+ sub1?.unsubscribe();+ await new Promise((resolve) => setTimeout(resolve, 100));++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'buffered-redis' } } },+ });+ await new Promise((resolve) => setTimeout(resolve, 100));++ const liveEvents: ServerSentEvent[] = [];+ const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(+ streamId,+ (event) => liveEvents.push(event),+ );++ expect(resumeState).not.toBeNull();+ expect(pendingEvents.length).toBe(0);++ await manager.emitChunk(streamId, {+ event: 'on_message_delta',+ data: { delta: { content: { type: 'text', text: 'live-redis' } } },+ });++ await new Promise((resolve) => setTimeout(resolve, 200));+ expect(liveEvents.length).toBe(1);++ subscription?.unsubscribe();+ await manager.destroy();+ },+ );+ });+describe('Error Preservation for Late Subscribers', () => {/*** These tests verify the fix for the race condition where errors@@ -1369,14 +1763,9 @@ describe('GenerationJobManager Integration Tests', () => {await GenerationJobManager.destroy();});- test('should handle error preservation in Redis mode (cross-replica)', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-+ testRedis('should handle error preservation in Redis mode (cross-replica)', async () => {// === Replica A: Creates job and emits error ===- const replicaAJobStore = new RedisJobStore(ioredisClient);+ const replicaAJobStore = new RedisJobStore(ioredisClient!);await replicaAJobStore.initialize();const streamId = `redis-error-${Date.now()}`;@@ -1463,13 +1852,8 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Cross-Replica Live Streaming (Redis)', () => {+ describeRedis('Cross-Replica Live Streaming (Redis)', () => {test('should publish events to Redis even when no local subscriber exists', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const replicaA = new GenerationJobManagerClass();const servicesA = createStreamServices({useRedis: true,@@ -1489,7 +1873,7 @@ describe('GenerationJobManager Integration Tests', () => {const streamId = `cross-live-${Date.now()}`;await replicaA.createJob(streamId, 'user-1');- const replicaBJobStore = new RedisJobStore(ioredisClient);+ const replicaBJobStore = new RedisJobStore(ioredisClient!);await replicaBJobStore.initialize();await replicaBJobStore.createJob(streamId, 'user-1');@@ -1519,11 +1903,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should not cause data loss on cross-replica subscribers when local subscriber joins', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const replicaA = new GenerationJobManagerClass();const servicesA = createStreamServices({useRedis: true,@@ -1543,7 +1922,7 @@ describe('GenerationJobManager Integration Tests', () => {const streamId = `cross-seq-safe-${Date.now()}`;await replicaA.createJob(streamId, 'user-1');- const replicaBJobStore = new RedisJobStore(ioredisClient);+ const replicaBJobStore = new RedisJobStore(ioredisClient!);await replicaBJobStore.initialize();await replicaBJobStore.createJob(streamId, 'user-1');@@ -1603,11 +1982,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should deliver buffered events locally AND publish live events cross-replica', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const replicaA = new GenerationJobManagerClass();const servicesA = createStreamServices({useRedis: true,@@ -1641,7 +2015,7 @@ describe('GenerationJobManager Integration Tests', () => {replicaB.configure(servicesB);replicaB.initialize();- const replicaBJobStore = new RedisJobStore(ioredisClient);+ const replicaBJobStore = new RedisJobStore(ioredisClient!);await replicaBJobStore.initialize();await replicaBJobStore.createJob(streamId, 'user-1');@@ -1671,13 +2045,8 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Concurrent Subscriber Readiness (Redis)', () => {+ describeRedis('Concurrent Subscriber Readiness (Redis)', () => {test('should return ready promise to all concurrent subscribers for same stream', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const subscriber = (ioredisClient as unknown as { duplicate: () => typeof ioredisClient }).duplicate()!;@@ -1706,13 +2075,8 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Sequence Reset Safety (Redis)', () => {+ describeRedis('Sequence Reset Safety (Redis)', () => {test('should not receive stale pre-subscribe events via Redis after sequence reset', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const manager = new GenerationJobManagerClass();const services = createStreamServices({useRedis: true,@@ -1774,11 +2138,6 @@ describe('GenerationJobManager Integration Tests', () => {});test('should not reset sequence when second subscriber joins mid-stream', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const manager = new GenerationJobManagerClass();const services = createStreamServices({useRedis: true,@@ -1837,13 +2196,8 @@ describe('GenerationJobManager Integration Tests', () => {});});- describe('Subscribe Error Recovery (Redis)', () => {+ describeRedis('Subscribe Error Recovery (Redis)', () => {test('should allow resubscription after Redis subscribe failure', async () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-const subscriber = (ioredisClient as unknown as { duplicate: () => typeof ioredisClient }).duplicate()!;@@ -1892,12 +2246,7 @@ describe('GenerationJobManager Integration Tests', () => {});describe('createStreamServices Auto-Detection', () => {- test('should use Redis when useRedis is true and client is available', () => {- if (!ioredisClient) {- console.warn('Redis not available, skipping test');- return;- }-+ testRedis('should use Redis when useRedis is true and client is available', () => {const services = createStreamServices({useRedis: true,redisClient: ioredisClient,diff --git a/packages/api/src/types/stream.ts b/packages/api/src/types/stream.tsindex 79b29d774f..068d9c8db8 100644--- a/packages/api/src/types/stream.ts+++ b/packages/api/src/types/stream.ts@@ -47,3 +47,24 @@ export type ChunkHandler = (event: ServerSentEvent) => void;export type DoneHandler = (event: ServerSentEvent) => void;export type ErrorHandler = (error: string) => void;export type UnsubscribeFn = () => void;++/** Options for subscribing to a job event stream */+export interface SubscribeOptions {+ /**+ * When true, skips replaying the earlyEventBuffer.+ * Use for resume connections after a sync event has been sent.+ */+ skipBufferReplay?: boolean;+}++/** Result of an atomic subscribe-with-resume operation */+export interface SubscribeWithResumeResult {+ subscription: { unsubscribe: UnsubscribeFn } | null;+ resumeState: ResumeState | null;+ /**+ * Events that arrived between the resume snapshot and the subscribe call.+ * In-memory mode: drained from earlyEventBuffer (only place they exist).+ * Redis mode: empty — chunks are persisted to the store and appear in aggregatedContent on next resume.+ */+ pendingEvents: ServerSentEvent[];+}