Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🔄 refactor: Sequential Event Ordering in Redis Streaming Mode (#11650)
Danny Avilafeb72ad2dce3
diff --git a/.gitignore b/.gitignoreindex d173d26b60..d0c87ff03d 100644--- a/.gitignore+++ b/.gitignore@@ -15,6 +15,7 @@ pids# CI/CD datatest-image*+dump.rdb# Directory for instrumented libs generated by jscoverage/JSCoverlib-covdiff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.jsindex 5d706875ff..867e7f53af 100644--- a/api/server/controllers/agents/callbacks.js+++ b/api/server/controllers/agents/callbacks.js@@ -152,13 +152,15 @@ function checkIfLastAgent(last_agent_id, langgraph_node) {/*** Helper to emit events either to res (standard mode) or to job emitter (resumable mode).+ * In Redis mode, awaits the emit to guarantee event ordering (critical for streaming deltas).* @param {ServerResponse} res - The server response object* @param {string | null} streamId - The stream ID for resumable mode, or null for standard mode* @param {Object} eventData - The event data to send+ * @returns {Promise<void>}*/-function emitEvent(res, streamId, eventData) {+async function emitEvent(res, streamId, eventData) {if (streamId) {- GenerationJobManager.emitChunk(streamId, eventData);+ await GenerationJobManager.emitChunk(streamId, eventData);} else {sendEvent(res, eventData);}@@ -206,18 +208,18 @@ function getDefaultHandlers({* @param {StreamEventData} data - The event data.* @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata.*/- handle: (event, data, metadata) => {+ handle: async (event, data, metadata) => {if (data?.stepDetails.type === StepTypes.TOOL_CALLS) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (checkIfLastAgent(metadata?.last_agent_id, metadata?.langgraph_node)) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (!metadata?.hide_sequential_outputs) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else {const agentName = metadata?.name ?? 'Agent';const isToolCall = data?.stepDetails.type === StepTypes.TOOL_CALLS;const action = isToolCall ? 'performing a task...' : 'thinking...';- emitEvent(res, streamId, {+ await emitEvent(res, streamId, {event: 'on_agent_update',data: {runId: metadata?.run_id,@@ -235,13 +237,13 @@ function getDefaultHandlers({* @param {StreamEventData} data - The event data.* @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata.*/- handle: (event, data, metadata) => {+ handle: async (event, data, metadata) => {if (data?.delta.type === StepTypes.TOOL_CALLS) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (checkIfLastAgent(metadata?.last_agent_id, metadata?.langgraph_node)) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (!metadata?.hide_sequential_outputs) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });}aggregateContent({ event, data });},@@ -253,13 +255,13 @@ function getDefaultHandlers({* @param {StreamEventData & { result: ToolEndData }} data - The event data.* @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata.*/- handle: (event, data, metadata) => {+ handle: async (event, data, metadata) => {if (data?.result != null) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (checkIfLastAgent(metadata?.last_agent_id, metadata?.langgraph_node)) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (!metadata?.hide_sequential_outputs) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });}aggregateContent({ event, data });},@@ -271,11 +273,11 @@ function getDefaultHandlers({* @param {StreamEventData} data - The event data.* @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata.*/- handle: (event, data, metadata) => {+ handle: async (event, data, metadata) => {if (checkIfLastAgent(metadata?.last_agent_id, metadata?.langgraph_node)) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (!metadata?.hide_sequential_outputs) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });}aggregateContent({ event, data });},@@ -287,11 +289,11 @@ function getDefaultHandlers({* @param {StreamEventData} data - The event data.* @param {GraphRunnableConfig['configurable']} [metadata] The runnable metadata.*/- handle: (event, data, metadata) => {+ handle: async (event, data, metadata) => {if (checkIfLastAgent(metadata?.last_agent_id, metadata?.langgraph_node)) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });} else if (!metadata?.hide_sequential_outputs) {- emitEvent(res, streamId, { event, data });+ await emitEvent(res, streamId, { event, data });}aggregateContent({ event, data });},@@ -307,6 +309,7 @@ function getDefaultHandlers({/*** Helper to write attachment events either to res or to job emitter.+ * Note: Attachments are not order-sensitive like deltas, so fire-and-forget is acceptable.* @param {ServerResponse} res - The server response object* @param {string | null} streamId - The stream ID for resumable mode, or null for standard mode* @param {Object} attachment - The attachment datadiff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.jsindex eb8fd5aec6..79387b6e89 100644--- a/api/server/controllers/agents/request.js+++ b/api/server/controllers/agents/request.js@@ -324,7 +324,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitconversationId: conversation?.conversationId,});- GenerationJobManager.emitDone(streamId, finalEvent);+ await GenerationJobManager.emitDone(streamId, finalEvent);GenerationJobManager.completeJob(streamId);await decrementPendingRequest(userId);} else {@@ -344,7 +344,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitconversationId: conversation?.conversationId,});- GenerationJobManager.emitDone(streamId, finalEvent);+ await GenerationJobManager.emitDone(streamId, finalEvent);GenerationJobManager.completeJob(streamId, 'Request aborted');await decrementPendingRequest(userId);}@@ -377,7 +377,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit// abortJob already handled emitDone and completeJob} else {logger.error(`[ResumableAgentController] Generation error for ${streamId}:`, error);- GenerationJobManager.emitError(streamId, error.message || 'Generation failed');+ await GenerationJobManager.emitError(streamId, error.message || 'Generation failed');GenerationJobManager.completeJob(streamId, error.message);}@@ -406,7 +406,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTitres.status(500).json({ error: error.message || 'Failed to start generation' });} else {// JSON already sent, emit error to stream so client can receive it- GenerationJobManager.emitError(streamId, error.message || 'Failed to start generation');+ await GenerationJobManager.emitError(streamId, error.message || 'Failed to start generation');}GenerationJobManager.completeJob(streamId, error.message);await decrementPendingRequest(userId);diff --git a/api/server/services/ActionService.js b/api/server/services/ActionService.jsindex a2a515d14a..132f6f4686 100644--- a/api/server/services/ActionService.js+++ b/api/server/services/ActionService.js@@ -201,7 +201,7 @@ async function createActionTool({async () => {const eventData = { event: GraphEvents.ON_RUN_STEP_DELTA, data };if (streamId) {- GenerationJobManager.emitChunk(streamId, eventData);+ await GenerationJobManager.emitChunk(streamId, eventData);} else {sendEvent(res, eventData);}@@ -231,7 +231,7 @@ async function createActionTool({data.delta.expires_at = undefined;const successEventData = { event: GraphEvents.ON_RUN_STEP_DELTA, data };if (streamId) {- GenerationJobManager.emitChunk(streamId, successEventData);+ await GenerationJobManager.emitChunk(streamId, successEventData);} else {sendEvent(res, successEventData);}diff --git a/api/server/services/MCP.js b/api/server/services/MCP.jsindex e5133eaca5..8cb9932097 100644--- a/api/server/services/MCP.js+++ b/api/server/services/MCP.js@@ -53,9 +53,9 @@ function isEmptyObjectSchema(jsonSchema) {function createRunStepDeltaEmitter({ res, stepId, toolCall, streamId = null }) {/*** @param {string} authURL - The URL to redirect the user for OAuth authentication.- * @returns {void}+ * @returns {Promise<void>}*/- return function (authURL) {+ return async function (authURL) {/** @type {{ id: string; delta: AgentToolCallDelta }} */const data = {id: stepId,@@ -68,7 +68,7 @@ function createRunStepDeltaEmitter({ res, stepId, toolCall, streamId = null }) {};const eventData = { event: GraphEvents.ON_RUN_STEP_DELTA, data };if (streamId) {- GenerationJobManager.emitChunk(streamId, eventData);+ await GenerationJobManager.emitChunk(streamId, eventData);} else {sendEvent(res, eventData);}@@ -83,9 +83,10 @@ function createRunStepDeltaEmitter({ res, stepId, toolCall, streamId = null }) {* @param {ToolCallChunk} params.toolCall - The tool call object containing tool information.* @param {number} [params.index]* @param {string | null} [params.streamId] - The stream ID for resumable mode.+ * @returns {() => Promise<void>}*/function createRunStepEmitter({ res, runId, stepId, toolCall, index, streamId = null }) {- return function () {+ return async function () {/** @type {import('@librechat/agents').RunStep} */const data = {runId: runId ?? Constants.USE_PRELIM_RESPONSE_MESSAGE_ID,@@ -99,7 +100,7 @@ function createRunStepEmitter({ res, runId, stepId, toolCall, index, streamId =};const eventData = { event: GraphEvents.ON_RUN_STEP, data };if (streamId) {- GenerationJobManager.emitChunk(streamId, eventData);+ await GenerationJobManager.emitChunk(streamId, eventData);} else {sendEvent(res, eventData);}@@ -147,7 +148,7 @@ function createOAuthEnd({ res, stepId, toolCall, streamId = null }) {};const eventData = { event: GraphEvents.ON_RUN_STEP_DELTA, data };if (streamId) {- GenerationJobManager.emitChunk(streamId, eventData);+ await GenerationJobManager.emitChunk(streamId, eventData);} else {sendEvent(res, eventData);}diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.jsindex 21fba1dc39..fe7a0f40c2 100644--- a/api/server/services/ToolService.js+++ b/api/server/services/ToolService.js@@ -526,8 +526,8 @@ async function loadToolDefinitionsWrapper({ req, res, agent, streamId = null, toconst runStepDeltaEvent = { event: GraphEvents.ON_RUN_STEP_DELTA, data: runStepDeltaData };if (streamId) {- GenerationJobManager.emitChunk(streamId, runStepEvent);- GenerationJobManager.emitChunk(streamId, runStepDeltaEvent);+ await GenerationJobManager.emitChunk(streamId, runStepEvent);+ await GenerationJobManager.emitChunk(streamId, runStepDeltaEvent);} else if (res && !res.writableEnded) {sendEvent(res, runStepEvent);sendEvent(res, runStepDeltaEvent);diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex d4b9b97eda..fefb0dd207 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -662,7 +662,7 @@ class GenerationJobManagerClass {runtime.finalEvent = abortFinalEvent;}- this.eventTransport.emitDone(streamId, abortFinalEvent);+ await this.eventTransport.emitDone(streamId, abortFinalEvent);this.jobStore.clearContentState(streamId);this.runStepBuffers?.delete(streamId);@@ -789,8 +789,11 @@ class GenerationJobManagerClass {** If no subscriber has connected yet, buffers the event for replay when they do.* This ensures early events (like 'created') aren't lost due to race conditions.+ *+ * In Redis mode, awaits the publish to guarantee event ordering.+ * This is critical for streaming deltas (tool args, message content) to arrive in order.*/- emitChunk(streamId: string, event: t.ServerSentEvent): void {+ async emitChunk(streamId: string, event: t.ServerSentEvent): Promise<void> {const runtime = this.runtimeState.get(streamId);if (!runtime || runtime.abortController.signal.aborted) {return;@@ -799,7 +802,7 @@ class GenerationJobManagerClass {// Track user message from created eventthis.trackUserMessage(streamId, event);- // For Redis mode, persist chunk for later reconstruction+ // For Redis mode, persist chunk for later reconstruction (fire-and-forget for resumability)if (this._isRedis) {// The SSE event structure is { event: string, data: unknown, ... }// The aggregator expects { event: string, data: unknown } where data is the payload@@ -825,7 +828,8 @@ class GenerationJobManagerClass {runtime.earlyEventBuffer.push(event);}- this.eventTransport.emitChunk(streamId, event);+ // Await the transport emit - critical for Redis mode to maintain event order+ await this.eventTransport.emitChunk(streamId, event);}/**@@ -1035,7 +1039,7 @@ class GenerationJobManagerClass {* Emit a done event.* Persists finalEvent to Redis for cross-replica access.*/- emitDone(streamId: string, event: t.ServerSentEvent): void {+ async emitDone(streamId: string, event: t.ServerSentEvent): Promise<void> {const runtime = this.runtimeState.get(streamId);if (runtime) {runtime.finalEvent = event;@@ -1044,7 +1048,7 @@ class GenerationJobManagerClass {this.jobStore.updateJob(streamId, { finalEvent: JSON.stringify(event) }).catch((err) => {logger.error(`[GenerationJobManager] Failed to persist finalEvent:`, err);});- this.eventTransport.emitDone(streamId, event);+ await this.eventTransport.emitDone(streamId, event);}/**@@ -1052,7 +1056,7 @@ class GenerationJobManagerClass {* Stores the error for late-connecting subscribers (race condition where error* occurs before client connects to SSE stream).*/- emitError(streamId: string, error: string): void {+ async emitError(streamId: string, error: string): Promise<void> {const runtime = this.runtimeState.get(streamId);if (runtime) {runtime.errorEvent = error;@@ -1061,7 +1065,7 @@ class GenerationJobManagerClass {this.jobStore.updateJob(streamId, { error }).catch((err) => {logger.error(`[GenerationJobManager] Failed to persist error:`, err);});- this.eventTransport.emitError(streamId, error);+ await this.eventTransport.emitError(streamId, error);}/**diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.tsindex e3ea16c8f0..8723f3f000 100644--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts@@ -134,12 +134,12 @@ describe('GenerationJobManager Integration Tests', () => {// Wait for first subscriber to be registeredawait new Promise((resolve) => setTimeout(resolve, 10));- // Emit chunks (emitChunk takes { event, data } format)- GenerationJobManager.emitChunk(streamId, {+ // Emit chunks (emitChunk takes { event, data } format, now async for Redis ordering)+ await GenerationJobManager.emitChunk(streamId, {event: 'on_message_delta',data: { type: 'text', text: 'Hello' },});- GenerationJobManager.emitChunk(streamId, {+ await GenerationJobManager.emitChunk(streamId, {event: 'on_message_delta',data: { type: 'text', text: ' world' },});@@ -219,8 +219,8 @@ describe('GenerationJobManager Integration Tests', () => {await GenerationJobManager.createJob(streamId, 'user-1');// Emit chunks (these should be persisted to Redis)- // emitChunk takes { event, data } format- GenerationJobManager.emitChunk(streamId, {+ // emitChunk takes { event, data } format, now async for Redis ordering+ await GenerationJobManager.emitChunk(streamId, {event: 'on_run_step',data: {id: 'step-1',@@ -229,14 +229,14 @@ describe('GenerationJobManager Integration Tests', () => {stepDetails: { type: 'message_creation' },},});- GenerationJobManager.emitChunk(streamId, {+ await GenerationJobManager.emitChunk(streamId, {event: 'on_message_delta',data: {id: 'step-1',delta: { content: { type: 'text', text: 'Persisted ' } },},});- GenerationJobManager.emitChunk(streamId, {+ await GenerationJobManager.emitChunk(streamId, {event: 'on_message_delta',data: {id: 'step-1',@@ -276,8 +276,8 @@ describe('GenerationJobManager Integration Tests', () => {const streamId = `redis-abort-${Date.now()}`;await GenerationJobManager.createJob(streamId, 'user-1');- // Emit some content (emitChunk takes { event, data } format)- GenerationJobManager.emitChunk(streamId, {+ // Emit some content (emitChunk takes { event, data } format, now async)+ await GenerationJobManager.emitChunk(streamId, {event: 'on_run_step',data: {id: 'step-1',@@ -286,7 +286,7 @@ describe('GenerationJobManager Integration Tests', () => {stepDetails: { type: 'message_creation' },},});- GenerationJobManager.emitChunk(streamId, {+ await GenerationJobManager.emitChunk(streamId, {event: 'on_message_delta',data: {id: 'step-1',@@ -582,7 +582,7 @@ describe('GenerationJobManager Integration Tests', () => {conversation: { conversationId: streamId },responseMessage: { text: 'Hello world' },};- GenerationJobManager.emitDone(streamId, finalEventData as never);+ await GenerationJobManager.emitDone(streamId, finalEventData as never);await new Promise((resolve) => setTimeout(resolve, 200));@@ -796,6 +796,267 @@ describe('GenerationJobManager Integration Tests', () => {});});+ describe('Sequential Event Ordering (Redis)', () => {+ /**+ * These tests verify that events are delivered in strict sequential order+ * when using Redis mode. This is critical because:+ * 1. LLM streaming tokens must arrive in order for coherent output+ * 2. Tool call argument deltas must be concatenated in order+ * 3. Run step events must precede their deltas+ *+ * The fix: emitChunk now awaits Redis publish to ensure ordered delivery.+ */+ test('should maintain strict order for rapid sequential emits', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId = `order-rapid-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');++ const receivedIndices: number[] = [];++ const subscription = await GenerationJobManager.subscribe(streamId, (event) => {+ const data = event as { event: string; data: { index: number } };+ if (data.event === 'test') {+ receivedIndices.push(data.data.index);+ }+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Emit 30 events rapidly - with await, they must arrive in order+ for (let i = 0; i < 30; i++) {+ await GenerationJobManager.emitChunk(streamId, {+ event: 'test',+ data: { index: i },+ });+ }++ await new Promise((resolve) => setTimeout(resolve, 300));++ // Verify all events arrived in correct order+ expect(receivedIndices.length).toBe(30);+ for (let i = 0; i < 30; i++) {+ expect(receivedIndices[i]).toBe(i);+ }++ subscription?.unsubscribe();+ await GenerationJobManager.destroy();+ });++ test('should maintain order for tool call argument deltas', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId = `tool-args-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');++ const receivedArgs: string[] = [];++ const subscription = await GenerationJobManager.subscribe(streamId, (event) => {+ const data = event as {+ event: string;+ data: { delta: { tool_calls: { args: string }[] } };+ };+ if (data.event === 'on_run_step_delta') {+ receivedArgs.push(data.data.delta.tool_calls[0].args);+ }+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Simulate streaming JSON args: {"code": "print('hello')"}+ const argChunks = ['{"', 'code', '": "', 'print', "('", 'hello', "')", '"}'];++ for (const chunk of argChunks) {+ await GenerationJobManager.emitChunk(streamId, {+ event: 'on_run_step_delta',+ data: {+ id: 'step-1',+ delta: {+ type: 'tool_calls',+ tool_calls: [{ index: 0, args: chunk }],+ },+ },+ });+ }++ await new Promise((resolve) => setTimeout(resolve, 300));++ // This was the original bug - args would arrive scrambled without await+ expect(receivedArgs).toEqual(argChunks);+ expect(receivedArgs.join('')).toBe(`{"code": "print('hello')"}`);++ subscription?.unsubscribe();+ await GenerationJobManager.destroy();+ });++ test('should maintain order: on_run_step before on_run_step_delta', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId = `step-order-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');++ const receivedEvents: string[] = [];++ const subscription = await GenerationJobManager.subscribe(streamId, (event) => {+ const data = event as { event: string };+ receivedEvents.push(data.event);+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Emit in correct order: step first, then deltas+ await GenerationJobManager.emitChunk(streamId, {+ event: 'on_run_step',+ data: { id: 'step-1', type: 'tool_calls', index: 0 },+ });++ await GenerationJobManager.emitChunk(streamId, {+ event: 'on_run_step_delta',+ data: { id: 'step-1', delta: { type: 'tool_calls', tool_calls: [{ args: '{' }] } },+ });++ await GenerationJobManager.emitChunk(streamId, {+ event: 'on_run_step_delta',+ data: { id: 'step-1', delta: { type: 'tool_calls', tool_calls: [{ args: '}' }] } },+ });++ await GenerationJobManager.emitChunk(streamId, {+ event: 'on_run_step_completed',+ data: { id: 'step-1', result: { content: '{}' } },+ });++ await new Promise((resolve) => setTimeout(resolve, 300));++ // Verify ordering: step -> deltas -> completed+ expect(receivedEvents).toEqual([+ 'on_run_step',+ 'on_run_step_delta',+ 'on_run_step_delta',+ 'on_run_step_completed',+ ]);++ subscription?.unsubscribe();+ await GenerationJobManager.destroy();+ });++ test('should not block other streams when awaiting emitChunk', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId1 = `concurrent-1-${Date.now()}`;+ const streamId2 = `concurrent-2-${Date.now()}`;++ await GenerationJobManager.createJob(streamId1, 'user-1');+ await GenerationJobManager.createJob(streamId2, 'user-2');++ const stream1Events: number[] = [];+ const stream2Events: number[] = [];++ const sub1 = await GenerationJobManager.subscribe(streamId1, (event) => {+ const data = event as { event: string; data: { index: number } };+ if (data.event === 'test') {+ stream1Events.push(data.data.index);+ }+ });++ const sub2 = await GenerationJobManager.subscribe(streamId2, (event) => {+ const data = event as { event: string; data: { index: number } };+ if (data.event === 'test') {+ stream2Events.push(data.data.index);+ }+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Emit to both streams concurrently (simulating two LLM responses)+ const emitPromises: Promise<void>[] = [];+ for (let i = 0; i < 10; i++) {+ emitPromises.push(+ GenerationJobManager.emitChunk(streamId1, { event: 'test', data: { index: i } }),+ );+ emitPromises.push(+ GenerationJobManager.emitChunk(streamId2, { event: 'test', data: { index: i * 100 } }),+ );+ }+ await Promise.all(emitPromises);++ await new Promise((resolve) => setTimeout(resolve, 300));++ // Each stream should have all events, in order within their stream+ expect(stream1Events.length).toBe(10);+ expect(stream2Events.length).toBe(10);++ // Verify each stream's internal order+ for (let i = 0; i < 10; i++) {+ expect(stream1Events[i]).toBe(i);+ expect(stream2Events[i]).toBe(i * 100);+ }++ sub1?.unsubscribe();+ sub2?.unsubscribe();+ await GenerationJobManager.destroy();+ });+ });+describe('Error Preservation for Late Subscribers', () => {/*** These tests verify the fix for the race condition where errors@@ -825,7 +1086,7 @@ describe('GenerationJobManager Integration Tests', () => {const errorMessage = '{ "type": "INPUT_LENGTH", "info": "234856 / 172627" }';// Emit error (no subscribers yet - simulates race condition)- GenerationJobManager.emitError(streamId, errorMessage);+ await GenerationJobManager.emitError(streamId, errorMessage);// Wait for async job store updateawait new Promise((resolve) => setTimeout(resolve, 50));@@ -891,7 +1152,7 @@ describe('GenerationJobManager Integration Tests', () => {const errorMessage = '{ "type": "INPUT_LENGTH", "info": "234856 / 172627" }';// Simulate race condition: error occurs before client connects- GenerationJobManager.emitError(streamId, errorMessage);+ await GenerationJobManager.emitError(streamId, errorMessage);await GenerationJobManager.completeJob(streamId, errorMessage);// Wait for async operations@@ -940,7 +1201,7 @@ describe('GenerationJobManager Integration Tests', () => {const errorMessage = 'Error should take priority';// Emit error and complete with error- GenerationJobManager.emitError(streamId, errorMessage);+ await GenerationJobManager.emitError(streamId, errorMessage);await GenerationJobManager.completeJob(streamId, errorMessage);await new Promise((resolve) => setTimeout(resolve, 50));diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.tsindex 31266b3e11..d1f9467cd0 100644--- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts@@ -70,16 +70,16 @@ describe('RedisEventTransport Integration Tests', () => {},});- // Wait for subscription to be established- await new Promise((resolve) => setTimeout(resolve, 100));+ // Wait for subscription to be established (increased for CI)+ await new Promise((resolve) => setTimeout(resolve, 500));- // Emit events- transport.emitChunk(streamId, { type: 'text', text: 'Hello' });- transport.emitChunk(streamId, { type: 'text', text: ' World' });- transport.emitDone(streamId, { finished: true });+ // Emit events (emitChunk/emitDone are async for ordered delivery)+ await transport.emitChunk(streamId, { type: 'text', text: 'Hello' });+ await transport.emitChunk(streamId, { type: 'text', text: ' World' });+ await transport.emitDone(streamId, { finished: true });- // Wait for events to propagate- await new Promise((resolve) => setTimeout(resolve, 200));+ // Wait for events to propagate (increased for CI)+ await new Promise((resolve) => setTimeout(resolve, 500));expect(receivedChunks.length).toBe(2);expect(doneEvent).toEqual({ finished: true });@@ -117,7 +117,7 @@ describe('RedisEventTransport Integration Tests', () => {await new Promise((resolve) => setTimeout(resolve, 100));// Emit from transport 1 (producer on different instance)- transport1.emitChunk(streamId, { data: 'from-instance-1' });+ await transport1.emitChunk(streamId, { data: 'from-instance-1' });// Wait for cross-instance deliveryawait new Promise((resolve) => setTimeout(resolve, 200));@@ -160,7 +160,7 @@ describe('RedisEventTransport Integration Tests', () => {await new Promise((resolve) => setTimeout(resolve, 100));- transport.emitChunk(streamId, { data: 'broadcast' });+ await transport.emitChunk(streamId, { data: 'broadcast' });await new Promise((resolve) => setTimeout(resolve, 200));@@ -175,6 +175,425 @@ describe('RedisEventTransport Integration Tests', () => {});});+ describe('Sequential Event Ordering', () => {+ test('should maintain strict order when emitChunk is awaited', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `order-test-${Date.now()}`;+ const receivedEvents: number[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => receivedEvents.push((event as { index: number }).index),+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Emit 20 events rapidly with await - they should arrive in order+ for (let i = 0; i < 20; i++) {+ await transport.emitChunk(streamId, { index: i });+ }++ // Wait for all events to propagate+ await new Promise((resolve) => setTimeout(resolve, 300));++ // Verify all events arrived in correct order+ expect(receivedEvents.length).toBe(20);+ for (let i = 0; i < 20; i++) {+ expect(receivedEvents[i]).toBe(i);+ }++ transport.destroy();+ subscriber.disconnect();+ });++ test('should maintain order for tool call delta chunks (simulates streaming args)', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `tool-delta-order-${Date.now()}`;+ const receivedArgs: string[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => {+ const data = event as {+ event: string;+ data: { delta: { tool_calls: { args: string }[] } };+ };+ if (data.event === 'on_run_step_delta') {+ receivedArgs.push(data.data.delta.tool_calls[0].args);+ }+ },+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Simulate streaming tool call arguments like: {"code": "# First line\n..."+ const argChunks = ['{"code"', ': "', '# First', ' line', '\\n', '..."', '}'];++ for (const chunk of argChunks) {+ await transport.emitChunk(streamId, {+ event: 'on_run_step_delta',+ data: {+ id: 'step-1',+ delta: {+ type: 'tool_calls',+ tool_calls: [{ index: 0, args: chunk }],+ },+ },+ });+ }++ await new Promise((resolve) => setTimeout(resolve, 300));++ // Verify chunks arrived in correct order - this was the bug we fixed+ expect(receivedArgs).toEqual(argChunks);+ expect(receivedArgs.join('')).toBe('{"code": "# First line\\n..."}');++ transport.destroy();+ subscriber.disconnect();+ });++ test('should maintain order across multiple concurrent streams (no cross-contamination)', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId1 = `concurrent-stream-1-${Date.now()}`;+ const streamId2 = `concurrent-stream-2-${Date.now()}`;++ const stream1Events: number[] = [];+ const stream2Events: number[] = [];++ transport.subscribe(streamId1, {+ onChunk: (event) => stream1Events.push((event as { index: number }).index),+ });+ transport.subscribe(streamId2, {+ onChunk: (event) => stream2Events.push((event as { index: number }).index),+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Interleave events from both streams+ for (let i = 0; i < 10; i++) {+ await transport.emitChunk(streamId1, { index: i });+ await transport.emitChunk(streamId2, { index: i * 10 });+ }++ await new Promise((resolve) => setTimeout(resolve, 300));++ // Each stream should have its own ordered events+ expect(stream1Events).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);+ expect(stream2Events).toEqual([0, 10, 20, 30, 40, 50, 60, 70, 80, 90]);++ transport.destroy();+ subscriber.disconnect();+ });+ });++ describe('Reorder Buffer (Redis Cluster Fix)', () => {+ test('should reorder out-of-sequence messages', async () => {+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const mockPublisher = {+ publish: jest.fn().mockResolvedValue(1),+ };+ const mockSubscriber = {+ on: jest.fn(),+ subscribe: jest.fn().mockResolvedValue(undefined),+ unsubscribe: jest.fn().mockResolvedValue(undefined),+ };++ const transport = new RedisEventTransport(+ mockPublisher as unknown as Redis,+ mockSubscriber as unknown as Redis,+ );++ const streamId = 'reorder-test';+ const receivedEvents: number[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => receivedEvents.push((event as { index: number }).index),+ });++ const messageHandler = mockSubscriber.on.mock.calls.find(+ (call) => call[0] === 'message',+ )?.[1] as (channel: string, message: string) => void;++ const channel = `stream:{${streamId}}:events`;++ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 0, data: { index: 0 } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 2, data: { index: 2 } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 1, data: { index: 1 } }));++ await new Promise((resolve) => setTimeout(resolve, 50));++ expect(receivedEvents).toEqual([0, 1, 2]);++ transport.destroy();+ });++ test('should buffer early messages and deliver when gaps are filled', async () => {+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const mockPublisher = {+ publish: jest.fn().mockResolvedValue(1),+ };+ const mockSubscriber = {+ on: jest.fn(),+ subscribe: jest.fn().mockResolvedValue(undefined),+ unsubscribe: jest.fn().mockResolvedValue(undefined),+ };++ const transport = new RedisEventTransport(+ mockPublisher as unknown as Redis,+ mockSubscriber as unknown as Redis,+ );++ const streamId = 'buffer-test';+ const receivedEvents: number[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => receivedEvents.push((event as { index: number }).index),+ });++ const messageHandler = mockSubscriber.on.mock.calls.find(+ (call) => call[0] === 'message',+ )?.[1] as (channel: string, message: string) => void;++ const channel = `stream:{${streamId}}:events`;++ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 2, data: { index: 2 } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 4, data: { index: 4 } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 3, data: { index: 3 } }));++ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(receivedEvents).toEqual([]);++ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 0, data: { index: 0 } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 1, data: { index: 1 } }));++ await new Promise((resolve) => setTimeout(resolve, 50));+ expect(receivedEvents).toEqual([0, 1, 2, 3, 4]);++ transport.destroy();+ });++ test('should force-flush on timeout when gaps are not filled', async () => {+ jest.useFakeTimers();++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const mockPublisher = {+ publish: jest.fn().mockResolvedValue(1),+ };+ const mockSubscriber = {+ on: jest.fn(),+ subscribe: jest.fn().mockResolvedValue(undefined),+ unsubscribe: jest.fn().mockResolvedValue(undefined),+ };++ const transport = new RedisEventTransport(+ mockPublisher as unknown as Redis,+ mockSubscriber as unknown as Redis,+ );++ const streamId = 'timeout-test';+ const receivedEvents: number[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => receivedEvents.push((event as { index: number }).index),+ });++ const messageHandler = mockSubscriber.on.mock.calls.find(+ (call) => call[0] === 'message',+ )?.[1] as (channel: string, message: string) => void;++ const channel = `stream:{${streamId}}:events`;++ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 2, data: { index: 2 } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 3, data: { index: 3 } }));++ expect(receivedEvents).toEqual([]);++ jest.advanceTimersByTime(600);++ expect(receivedEvents).toEqual([2, 3]);++ transport.destroy();+ jest.useRealTimers();+ });++ test('should handle messages without sequence numbers (backward compatibility)', async () => {+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const mockPublisher = {+ publish: jest.fn().mockResolvedValue(1),+ };+ const mockSubscriber = {+ on: jest.fn(),+ subscribe: jest.fn().mockResolvedValue(undefined),+ unsubscribe: jest.fn().mockResolvedValue(undefined),+ };++ const transport = new RedisEventTransport(+ mockPublisher as unknown as Redis,+ mockSubscriber as unknown as Redis,+ );++ const streamId = 'compat-test';+ const receivedEvents: string[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => receivedEvents.push((event as { msg: string }).msg),+ onDone: (event) => receivedEvents.push(`done:${(event as { msg: string }).msg}`),+ });++ const messageHandler = mockSubscriber.on.mock.calls.find(+ (call) => call[0] === 'message',+ )?.[1] as (channel: string, message: string) => void;++ const channel = `stream:{${streamId}}:events`;++ messageHandler(channel, JSON.stringify({ type: 'chunk', data: { msg: 'no-seq-1' } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', data: { msg: 'no-seq-2' } }));+ messageHandler(channel, JSON.stringify({ type: 'done', data: { msg: 'finished' } }));++ await new Promise((resolve) => setTimeout(resolve, 50));++ expect(receivedEvents).toEqual(['no-seq-1', 'no-seq-2', 'done:finished']);++ transport.destroy();+ });++ test('should deliver done event after all pending chunks (terminal event ordering)', async () => {+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const mockPublisher = {+ publish: jest.fn().mockResolvedValue(1),+ };+ const mockSubscriber = {+ subscribe: jest.fn().mockResolvedValue(undefined),+ unsubscribe: jest.fn().mockResolvedValue(undefined),+ on: jest.fn(),+ };++ const transport = new RedisEventTransport(mockPublisher as never, mockSubscriber as never);+ const streamId = `terminal-order-${Date.now()}`;++ const receivedEvents: string[] = [];+ let doneReceived = false;++ transport.subscribe(streamId, {+ onChunk: (event: unknown) => {+ const e = event as { msg?: string };+ receivedEvents.push(e.msg ?? 'unknown');+ },+ onDone: (event: unknown) => {+ const e = event as { msg?: string };+ receivedEvents.push(`done:${e.msg ?? 'finished'}`);+ doneReceived = true;+ },+ });++ await new Promise((resolve) => setTimeout(resolve, 50));++ const messageHandler = mockSubscriber.on.mock.calls.find(+ (call) => call[0] === 'message',+ )?.[1];+ expect(messageHandler).toBeDefined();++ const channel = `stream:{${streamId}}:events`;++ // Simulate out-of-order delivery in Redis Cluster:+ // Done event (seq=3) arrives before chunk seq=2+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 0, data: { msg: 'chunk-0' } }));+ messageHandler(channel, JSON.stringify({ type: 'done', seq: 3, data: { msg: 'complete' } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 2, data: { msg: 'chunk-2' } }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 1, data: { msg: 'chunk-1' } }));++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Done event should be delivered AFTER all chunks despite arriving early+ expect(doneReceived).toBe(true);+ expect(receivedEvents).toEqual(['chunk-0', 'chunk-1', 'chunk-2', 'done:complete']);++ transport.destroy();+ });++ test('should deliver error event after all pending chunks (terminal event ordering)', async () => {+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const mockPublisher = {+ publish: jest.fn().mockResolvedValue(1),+ };+ const mockSubscriber = {+ subscribe: jest.fn().mockResolvedValue(undefined),+ unsubscribe: jest.fn().mockResolvedValue(undefined),+ on: jest.fn(),+ };++ const transport = new RedisEventTransport(mockPublisher as never, mockSubscriber as never);+ const streamId = `terminal-error-${Date.now()}`;++ const receivedEvents: string[] = [];+ let errorReceived: string | undefined;++ transport.subscribe(streamId, {+ onChunk: (event: unknown) => {+ const e = event as { msg?: string };+ receivedEvents.push(e.msg ?? 'unknown');+ },+ onError: (error: string) => {+ receivedEvents.push(`error:${error}`);+ errorReceived = error;+ },+ });++ await new Promise((resolve) => setTimeout(resolve, 50));++ const messageHandler = mockSubscriber.on.mock.calls.find(+ (call) => call[0] === 'message',+ )?.[1];+ expect(messageHandler).toBeDefined();++ const channel = `stream:{${streamId}}:events`;++ // Simulate out-of-order delivery: error arrives before final chunks+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 0, data: { msg: 'chunk-0' } }));+ messageHandler(channel, JSON.stringify({ type: 'error', seq: 2, error: 'Something failed' }));+ messageHandler(channel, JSON.stringify({ type: 'chunk', seq: 1, data: { msg: 'chunk-1' } }));++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Error event should be delivered AFTER all preceding chunks+ expect(errorReceived).toBe('Something failed');+ expect(receivedEvents).toEqual(['chunk-0', 'chunk-1', 'error:Something failed']);++ transport.destroy();+ });+ });+describe('Subscriber Management', () => {test('should track first subscriber correctly', async () => {if (!ioredisClient) {@@ -283,7 +702,7 @@ describe('RedisEventTransport Integration Tests', () => {await new Promise((resolve) => setTimeout(resolve, 100));- transport.emitError(streamId, 'Test error message');+ await transport.emitError(streamId, 'Test error message');await new Promise((resolve) => setTimeout(resolve, 200));diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.tsindex 02d4cb69ed..78f545c18e 100644--- a/packages/api/src/stream/implementations/RedisEventTransport.ts+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts@@ -22,10 +22,30 @@ const EventTypes = {interface PubSubMessage {type: (typeof EventTypes)[keyof typeof EventTypes];+ /** Sequence number for ordering (critical for Redis Cluster) */+ seq?: number;data?: unknown;error?: string;}+/**+ * Reorder buffer state for a stream subscription.+ * Handles out-of-order message delivery in Redis Cluster mode.+ */+interface ReorderBuffer {+ /** Next expected sequence number */+ nextSeq: number;+ /** Buffered messages waiting for earlier sequences */+ pending: Map<number, PubSubMessage>;+ /** Timeout handle for flushing stale messages */+ flushTimeout: ReturnType<typeof setTimeout> | null;+}++/** Max time (ms) to wait for out-of-order messages before force-flushing */+const REORDER_TIMEOUT_MS = 500;+/** Max messages to buffer before force-flushing (prevents memory issues) */+const MAX_BUFFER_SIZE = 100;+/*** Subscriber state for a stream*/@@ -42,6 +62,8 @@ interface StreamSubscribers {allSubscribersLeftCallbacks: Array<() => void>;/** Abort callbacks - called when abort signal is received from any replica */abortCallbacks: Array<() => void>;+ /** Reorder buffer for handling out-of-order delivery in Redis Cluster */+ reorderBuffer: ReorderBuffer;}/**@@ -74,6 +96,8 @@ export class RedisEventTransport implements IEventTransport {private subscribedChannels = new Set<string>();/** Counter for generating unique subscriber IDs */private subscriberIdCounter = 0;+ /** Sequence counters per stream for publishing (ensures ordered delivery in cluster mode) */+ private sequenceCounters = new Map<string, number>();/*** Create a new Redis event transport.@@ -91,12 +115,22 @@ export class RedisEventTransport implements IEventTransport {});}+ /** Get next sequence number for a stream (0-indexed) */+ private getNextSequence(streamId: string): number {+ const current = this.sequenceCounters.get(streamId) ?? 0;+ this.sequenceCounters.set(streamId, current + 1);+ return current;+ }++ /** Reset sequence counter for a stream */+ private resetSequence(streamId: string): void {+ this.sequenceCounters.delete(streamId);+ }+/**- * Handle incoming pub/sub message+ * Handle incoming pub/sub message with reordering support for Redis Cluster*/private handleMessage(channel: string, message: string): void {- // Extract streamId from channel name: stream:{streamId}:events- // Use regex to extract the hash tag contentconst match = channel.match(/^stream:\{([^}]+)\}:events$/);if (!match) {return;@@ -111,35 +145,176 @@ export class RedisEventTransport implements IEventTransport {try {const parsed = JSON.parse(message) as PubSubMessage;- for (const [, handlers] of streamState.handlers) {- switch (parsed.type) {- case EventTypes.CHUNK:- handlers.onChunk(parsed.data);- break;- case EventTypes.DONE:- handlers.onDone?.(parsed.data);- break;- case EventTypes.ERROR:- handlers.onError?.(parsed.error ?? 'Unknown error');- break;- case EventTypes.ABORT:- // Abort is handled at stream level, not per-handler- break;- }+ if (parsed.type === EventTypes.CHUNK && parsed.seq != null) {+ this.handleOrderedChunk(streamId, streamState, parsed);+ } else if (+ (parsed.type === EventTypes.DONE || parsed.type === EventTypes.ERROR) &&+ parsed.seq != null+ ) {+ this.handleTerminalEvent(streamId, streamState, parsed);+ } else {+ this.deliverMessage(streamState, parsed);}+ } catch (err) {+ logger.error(`[RedisEventTransport] Failed to parse message:`, err);+ }+ }- // Handle abort signals at stream level (not per-handler)- if (parsed.type === EventTypes.ABORT) {- for (const callback of streamState.abortCallbacks) {- try {- callback();- } catch (err) {- logger.error(`[RedisEventTransport] Error in abort callback:`, err);- }+ /**+ * Handle terminal events (done/error) with sequence-based ordering.+ * Buffers the terminal event and delivers after all preceding chunks arrive.+ */+ private handleTerminalEvent(+ streamId: string,+ streamState: StreamSubscribers,+ message: PubSubMessage,+ ): void {+ const buffer = streamState.reorderBuffer;+ const seq = message.seq!;++ if (seq < buffer.nextSeq) {+ logger.debug(+ `[RedisEventTransport] Dropping duplicate terminal event for stream ${streamId}: seq=${seq}, expected=${buffer.nextSeq}`,+ );+ return;+ }++ if (seq === buffer.nextSeq) {+ this.deliverMessage(streamState, message);+ buffer.nextSeq++;+ this.flushPendingMessages(streamId, streamState);+ } else {+ buffer.pending.set(seq, message);+ this.scheduleFlushTimeout(streamId, streamState);+ }+ }++ /**+ * Handle chunk messages with sequence-based reordering.+ * Buffers out-of-order messages and delivers them in sequence.+ */+ private handleOrderedChunk(+ streamId: string,+ streamState: StreamSubscribers,+ message: PubSubMessage,+ ): void {+ const buffer = streamState.reorderBuffer;+ const seq = message.seq!;++ if (seq === buffer.nextSeq) {+ this.deliverMessage(streamState, message);+ buffer.nextSeq++;++ this.flushPendingMessages(streamId, streamState);+ } else if (seq > buffer.nextSeq) {+ buffer.pending.set(seq, message);++ if (buffer.pending.size >= MAX_BUFFER_SIZE) {+ logger.warn(`[RedisEventTransport] Buffer overflow for stream ${streamId}, force-flushing`);+ this.forceFlushBuffer(streamId, streamState);+ } else {+ this.scheduleFlushTimeout(streamId, streamState);+ }+ } else {+ logger.debug(+ `[RedisEventTransport] Dropping duplicate/old message for stream ${streamId}: seq=${seq}, expected=${buffer.nextSeq}`,+ );+ }+ }++ /** Deliver consecutive pending messages */+ private flushPendingMessages(streamId: string, streamState: StreamSubscribers): void {+ const buffer = streamState.reorderBuffer;++ while (buffer.pending.has(buffer.nextSeq)) {+ const message = buffer.pending.get(buffer.nextSeq)!;+ buffer.pending.delete(buffer.nextSeq);+ this.deliverMessage(streamState, message);+ buffer.nextSeq++;+ }++ if (buffer.pending.size === 0 && buffer.flushTimeout) {+ clearTimeout(buffer.flushTimeout);+ buffer.flushTimeout = null;+ }+ }++ /** Force-flush all pending messages in order (used on timeout or overflow) */+ private forceFlushBuffer(streamId: string, streamState: StreamSubscribers): void {+ const buffer = streamState.reorderBuffer;++ if (buffer.flushTimeout) {+ clearTimeout(buffer.flushTimeout);+ buffer.flushTimeout = null;+ }++ if (buffer.pending.size === 0) {+ return;+ }++ const sortedSeqs = [...buffer.pending.keys()].sort((a, b) => a - b);+ const skipped = sortedSeqs[0] - buffer.nextSeq;++ if (skipped > 0) {+ logger.warn(+ `[RedisEventTransport] Stream ${streamId}: skipping ${skipped} missing messages (seq ${buffer.nextSeq}-${sortedSeqs[0] - 1})`,+ );+ }++ for (const seq of sortedSeqs) {+ const message = buffer.pending.get(seq)!;+ buffer.pending.delete(seq);+ this.deliverMessage(streamState, message);+ }++ buffer.nextSeq = sortedSeqs[sortedSeqs.length - 1] + 1;+ }++ /** Schedule a timeout to force-flush if gaps aren't filled */+ private scheduleFlushTimeout(streamId: string, streamState: StreamSubscribers): void {+ const buffer = streamState.reorderBuffer;++ if (buffer.flushTimeout) {+ return;+ }++ buffer.flushTimeout = setTimeout(() => {+ buffer.flushTimeout = null;+ if (buffer.pending.size > 0) {+ logger.warn(+ `[RedisEventTransport] Stream ${streamId}: timeout waiting for seq ${buffer.nextSeq}, force-flushing ${buffer.pending.size} messages`,+ );+ this.forceFlushBuffer(streamId, streamState);+ }+ }, REORDER_TIMEOUT_MS);+ }++ /** Deliver a message to all handlers */+ private deliverMessage(streamState: StreamSubscribers, message: PubSubMessage): void {+ for (const [, handlers] of streamState.handlers) {+ switch (message.type) {+ case EventTypes.CHUNK:+ handlers.onChunk(message.data);+ break;+ case EventTypes.DONE:+ handlers.onDone?.(message.data);+ break;+ case EventTypes.ERROR:+ handlers.onError?.(message.error ?? 'Unknown error');+ break;+ case EventTypes.ABORT:+ break;+ }+ }++ if (message.type === EventTypes.ABORT) {+ for (const callback of streamState.abortCallbacks) {+ try {+ callback();+ } catch (err) {+ logger.error(`[RedisEventTransport] Error in abort callback:`, err);}}- } catch (err) {- logger.error(`[RedisEventTransport] Failed to parse message:`, err);}}@@ -167,6 +342,11 @@ export class RedisEventTransport implements IEventTransport {handlers: new Map(),allSubscribersLeftCallbacks: [],abortCallbacks: [],+ reorderBuffer: {+ nextSeq: 0,+ pending: new Map(),+ flushTimeout: null,+ },});}@@ -195,6 +375,13 @@ export class RedisEventTransport implements IEventTransport {// If last subscriber left, unsubscribe from Redis and notifyif (state.count === 0) {+ // Clear any pending flush timeout and buffered messages+ if (state.reorderBuffer.flushTimeout) {+ clearTimeout(state.reorderBuffer.flushTimeout);+ state.reorderBuffer.flushTimeout = null;+ }+ state.reorderBuffer.pending.clear();+this.subscriber.unsubscribe(channel).catch((err) => {logger.error(`[RedisEventTransport] Failed to unsubscribe from ${channel}:`, err);});@@ -217,38 +404,50 @@ export class RedisEventTransport implements IEventTransport {/*** Publish a chunk event to all subscribers across all instances.+ * Includes sequence number for ordered delivery in Redis Cluster mode.*/- emitChunk(streamId: string, event: unknown): void {+ async emitChunk(streamId: string, event: unknown): Promise<void> {const channel = CHANNELS.events(streamId);- const message: PubSubMessage = { type: EventTypes.CHUNK, data: event };+ const seq = this.getNextSequence(streamId);+ const message: PubSubMessage = { type: EventTypes.CHUNK, seq, data: event };- this.publisher.publish(channel, JSON.stringify(message)).catch((err) => {+ try {+ await this.publisher.publish(channel, JSON.stringify(message));+ } catch (err) {logger.error(`[RedisEventTransport] Failed to publish chunk:`, err);- });+ }}/*** Publish a done event to all subscribers.+ * Includes sequence number to ensure delivery after all chunks.*/- emitDone(streamId: string, event: unknown): void {+ async emitDone(streamId: string, event: unknown): Promise<void> {const channel = CHANNELS.events(streamId);- const message: PubSubMessage = { type: EventTypes.DONE, data: event };+ const seq = this.getNextSequence(streamId);+ const message: PubSubMessage = { type: EventTypes.DONE, seq, data: event };- this.publisher.publish(channel, JSON.stringify(message)).catch((err) => {+ try {+ await this.publisher.publish(channel, JSON.stringify(message));+ } catch (err) {logger.error(`[RedisEventTransport] Failed to publish done:`, err);- });+ }}/*** Publish an error event to all subscribers.+ * Includes sequence number to ensure delivery after all chunks.*/- emitError(streamId: string, error: string): void {+ async emitError(streamId: string, error: string): Promise<void> {const channel = CHANNELS.events(streamId);- const message: PubSubMessage = { type: EventTypes.ERROR, error };+ const seq = this.getNextSequence(streamId);+ const message: PubSubMessage = { type: EventTypes.ERROR, seq, error };- this.publisher.publish(channel, JSON.stringify(message)).catch((err) => {+ try {+ await this.publisher.publish(channel, JSON.stringify(message));+ } catch (err) {logger.error(`[RedisEventTransport] Failed to publish error:`, err);- });+ }}/**@@ -282,6 +481,11 @@ export class RedisEventTransport implements IEventTransport {handlers: new Map(),allSubscribersLeftCallbacks: [callback],abortCallbacks: [],+ reorderBuffer: {+ nextSeq: 0,+ pending: new Map(),+ flushTimeout: null,+ },});}}@@ -317,6 +521,11 @@ export class RedisEventTransport implements IEventTransport {handlers: new Map(),allSubscribersLeftCallbacks: [],abortCallbacks: [],+ reorderBuffer: {+ nextSeq: 0,+ pending: new Map(),+ flushTimeout: null,+ },};this.streams.set(streamId, state);}@@ -347,12 +556,21 @@ export class RedisEventTransport implements IEventTransport {const state = this.streams.get(streamId);if (state) {+ // Clear flush timeout+ if (state.reorderBuffer.flushTimeout) {+ clearTimeout(state.reorderBuffer.flushTimeout);+ state.reorderBuffer.flushTimeout = null;+ }// Clear all handlers and callbacksstate.handlers.clear();state.allSubscribersLeftCallbacks = [];state.abortCallbacks = [];+ state.reorderBuffer.pending.clear();}+ // Reset sequence counter for this stream+ this.resetSequence(streamId);+// Unsubscribe from Redis channelif (this.subscribedChannels.has(channel)) {this.subscriber.unsubscribe(channel).catch((err) => {@@ -368,6 +586,15 @@ export class RedisEventTransport implements IEventTransport {* Destroy all resources.*/destroy(): void {+ // Clear all flush timeouts and buffered messages+ for (const [, state] of this.streams) {+ if (state.reorderBuffer.flushTimeout) {+ clearTimeout(state.reorderBuffer.flushTimeout);+ state.reorderBuffer.flushTimeout = null;+ }+ state.reorderBuffer.pending.clear();+ }+// Unsubscribe from all channelsfor (const channel of this.subscribedChannels) {this.subscriber.unsubscribe(channel).catch(() => {@@ -377,6 +604,7 @@ export class RedisEventTransport implements IEventTransport {this.subscribedChannels.clear();this.streams.clear();+ this.sequenceCounters.clear();// Note: Don't close Redis connections - they may be sharedlogger.info('[RedisEventTransport] Destroyed');diff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex af681fb2e9..d990283925 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -296,14 +296,14 @@ export interface IEventTransport {},): { unsubscribe: () => void };- /** Publish a chunk event */- emitChunk(streamId: string, event: unknown): void;+ /** Publish a chunk event - returns Promise in Redis mode for ordered delivery */+ emitChunk(streamId: string, event: unknown): void | Promise<void>;- /** Publish a done event */- emitDone(streamId: string, event: unknown): void;+ /** Publish a done event - returns Promise in Redis mode for ordered delivery */+ emitDone(streamId: string, event: unknown): void | Promise<void>;- /** Publish an error event */- emitError(streamId: string, error: string): void;+ /** Publish an error event - returns Promise in Redis mode for ordered delivery */+ emitError(streamId: string, error: string): void | Promise<void>;/*** Publish an abort signal to all replicas (Redis mode).diff --git a/packages/api/src/tools/toolkits/imageContext.ts b/packages/api/src/tools/toolkits/imageContext.tsindex 0485ed815a..723f173104 100644--- a/packages/api/src/tools/toolkits/imageContext.ts+++ b/packages/api/src/tools/toolkits/imageContext.ts@@ -35,4 +35,3 @@ export function buildImageToolContext({}return toolContext;}-