Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🪐 feat: Cross-replica support in `GenerationJobManager` for Redis mode (#11233)
Danny Avilaa7645f47057a
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex 61646b3aa5..44b38f48f6 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -227,13 +227,17 @@ class GenerationJobManagerClass {/*** Set up all-subscribers-left callback.* When all SSE clients disconnect, this:- * 1. Resets syncSent so reconnecting clients get sync event+ * 1. Resets syncSent so reconnecting clients get sync event (persisted to Redis)* 2. Calls any registered allSubscribersLeft handlers (e.g., to save partial responses)*/this.eventTransport.onAllSubscribersLeft(streamId, () => {const currentRuntime = this.runtimeState.get(streamId);if (currentRuntime) {currentRuntime.syncSent = false;+ // Persist syncSent=false to Redis for cross-replica consistency+ this.jobStore.updateJob(streamId, { syncSent: false }).catch((err) => {+ logger.error(`[GenerationJobManager] Failed to persist syncSent=false:`, err);+ });// Call registered handlers (from job.emitter.on('allSubscribersLeft', ...))if (currentRuntime.allSubscribersLeftHandlers) {this.jobStore@@ -258,6 +262,21 @@ class GenerationJobManagerClass {}});+ /**+ * Set up cross-replica abort listener (Redis mode only).+ * When abort is triggered on ANY replica, this replica receives the signal+ * and aborts its local AbortController (if it's the one running generation).+ */+ if (this.eventTransport.onAbort) {+ this.eventTransport.onAbort(streamId, () => {+ const currentRuntime = this.runtimeState.get(streamId);+ if (currentRuntime && !currentRuntime.abortController.signal.aborted) {+ logger.debug(`[GenerationJobManager] Received cross-replica abort for ${streamId}`);+ currentRuntime.abortController.abort();+ }+ });+ }+logger.debug(`[GenerationJobManager] Created job: ${streamId}`);// Return facade for backwards compatibility@@ -343,15 +362,133 @@ class GenerationJobManagerClass {};}+ /**+ * Get or create runtime state for a job.+ *+ * This enables cross-replica support in Redis mode:+ * - If runtime exists locally (same replica), return it+ * - If job exists in Redis but not locally (cross-replica), create minimal runtime+ *+ * The lazily-created runtime state is sufficient for:+ * - Subscribing to events (via Redis pub/sub)+ * - Getting resume state+ * - Handling reconnections+ * - Receiving cross-replica abort signals (via Redis pub/sub)+ *+ * @param streamId - The stream identifier+ * @returns Runtime state or null if job doesn't exist anywhere+ */+ private async getOrCreateRuntimeState(streamId: string): Promise<RuntimeJobState | null> {+ const existingRuntime = this.runtimeState.get(streamId);+ if (existingRuntime) {+ return existingRuntime;+ }++ // Job doesn't exist locally - check Redis+ const jobData = await this.jobStore.getJob(streamId);+ if (!jobData) {+ return null;+ }++ // Cross-replica scenario: job exists in Redis but not locally+ // Create minimal runtime state for handling reconnection/subscription+ logger.debug(`[GenerationJobManager] Creating cross-replica runtime for ${streamId}`);++ let resolveReady: () => void;+ const readyPromise = new Promise<void>((resolve) => {+ resolveReady = resolve;+ });++ // For jobs created on other replicas, readyPromise should be pre-resolved+ // since generation has already started+ resolveReady!();++ // Parse finalEvent from Redis if available+ let finalEvent: t.ServerSentEvent | undefined;+ if (jobData.finalEvent) {+ try {+ finalEvent = JSON.parse(jobData.finalEvent) as t.ServerSentEvent;+ } catch {+ // Ignore parse errors+ }+ }++ const runtime: RuntimeJobState = {+ abortController: new AbortController(),+ readyPromise,+ resolveReady: resolveReady!,+ syncSent: jobData.syncSent ?? false,+ earlyEventBuffer: [],+ hasSubscriber: false,+ finalEvent,+ };++ this.runtimeState.set(streamId, runtime);++ // Set up all-subscribers-left callback for this replica+ this.eventTransport.onAllSubscribersLeft(streamId, () => {+ const currentRuntime = this.runtimeState.get(streamId);+ if (currentRuntime) {+ currentRuntime.syncSent = false;+ // Persist syncSent=false to Redis+ this.jobStore.updateJob(streamId, { syncSent: false }).catch((err) => {+ logger.error(`[GenerationJobManager] Failed to persist syncSent=false:`, err);+ });+ // Call registered handlers+ if (currentRuntime.allSubscribersLeftHandlers) {+ this.jobStore+ .getContentParts(streamId)+ .then((result) => {+ const parts = result?.content ?? [];+ for (const handler of currentRuntime.allSubscribersLeftHandlers ?? []) {+ try {+ handler(parts);+ } catch (err) {+ logger.error(`[GenerationJobManager] Error in allSubscribersLeft handler:`, err);+ }+ }+ })+ .catch((err) => {+ logger.error(+ `[GenerationJobManager] Failed to get content parts for allSubscribersLeft handlers:`,+ err,+ );+ });+ }+ }+ });++ // Set up cross-replica abort listener (Redis mode only)+ // This ensures lazily-initialized jobs can receive abort signals+ if (this.eventTransport.onAbort) {+ this.eventTransport.onAbort(streamId, () => {+ const currentRuntime = this.runtimeState.get(streamId);+ if (currentRuntime && !currentRuntime.abortController.signal.aborted) {+ logger.debug(+ `[GenerationJobManager] Received cross-replica abort for lazily-init job ${streamId}`,+ );+ currentRuntime.abortController.abort();+ }+ });+ }++ return runtime;+ }+/*** Get a job by streamId.*/async getJob(streamId: string): Promise<t.GenerationJob | undefined> {const jobData = await this.jobStore.getJob(streamId);- const runtime = this.runtimeState.get(streamId);- if (!jobData || !runtime) {+ if (!jobData) {+ return undefined;+ }++ const runtime = await this.getOrCreateRuntimeState(streamId);+ if (!runtime) {return undefined;}+return this.buildJobFacade(streamId, jobData, runtime);}@@ -411,6 +548,10 @@ class GenerationJobManagerClass {/*** Abort a job (user-initiated).* Returns all data needed for token spending and message saving.+ *+ * Cross-replica support (Redis mode):+ * - Emits abort signal via Redis pub/sub+ * - The replica running generation receives signal and aborts its AbortController*/async abortJob(streamId: string): Promise<AbortResult> {const jobData = await this.jobStore.getJob(streamId);@@ -421,6 +562,13 @@ class GenerationJobManagerClass {return { success: false, jobData: null, content: [], finalEvent: null };}+ // Emit abort signal for cross-replica support (Redis mode)+ // This ensures the generating replica receives the abort signal+ if (this.eventTransport.emitAbort) {+ this.eventTransport.emitAbort(streamId);+ }++ // Also abort local controller if we have it (same-replica abort)if (runtime) {runtime.abortController.abort();}@@ -506,6 +654,10 @@ class GenerationJobManagerClass {* - Resolves readyPromise (legacy, for API compatibility)* - Replays any buffered early events (e.g., 'created' event)*+ * Supports cross-replica reconnection in Redis mode:+ * - If job exists in Redis but not locally, creates minimal runtime state+ * - Events are delivered via Redis pub/sub, not in-memory EventEmitter+ ** @param streamId - The stream to subscribe to* @param onChunk - Handler for chunk events (streamed tokens, run steps, etc.)* @param onDone - Handler for completion event (includes final message)@@ -518,7 +670,8 @@ class GenerationJobManagerClass {onDone?: t.DoneHandler,onError?: t.ErrorHandler,): Promise<{ unsubscribe: t.UnsubscribeFn } | null> {- const runtime = this.runtimeState.get(streamId);+ // Use lazy initialization to support cross-replica subscriptions+ const runtime = await this.getOrCreateRuntimeState(streamId);if (!runtime) {return null;}@@ -788,29 +941,46 @@ class GenerationJobManagerClass {/*** Mark that sync has been sent.+ * Persists to Redis for cross-replica consistency.*/markSyncSent(streamId: string): void {const runtime = this.runtimeState.get(streamId);if (runtime) {runtime.syncSent = true;}+ // Persist to Redis for cross-replica consistency+ this.jobStore.updateJob(streamId, { syncSent: true }).catch((err) => {+ logger.error(`[GenerationJobManager] Failed to persist syncSent flag:`, err);+ });}/*** Check if sync has been sent.+ * Checks local runtime first, then falls back to Redis for cross-replica scenarios.*/- wasSyncSent(streamId: string): boolean {- return this.runtimeState.get(streamId)?.syncSent ?? false;+ async wasSyncSent(streamId: string): Promise<boolean> {+ const localSyncSent = this.runtimeState.get(streamId)?.syncSent;+ if (localSyncSent !== undefined) {+ return localSyncSent;+ }+ // Cross-replica: check Redis+ const jobData = await this.jobStore.getJob(streamId);+ return jobData?.syncSent ?? false;}/*** Emit a done event.+ * Persists finalEvent to Redis for cross-replica access.*/emitDone(streamId: string, event: t.ServerSentEvent): void {const runtime = this.runtimeState.get(streamId);if (runtime) {runtime.finalEvent = event;}+ // Persist finalEvent to Redis for cross-replica consistency+ this.jobStore.updateJob(streamId, { finalEvent: JSON.stringify(event) }).catch((err) => {+ logger.error(`[GenerationJobManager] Failed to persist finalEvent:`, err);+ });this.eventTransport.emitDone(streamId, event);}diff --git a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.tsindex c593d3d15a..d95376f782 100644--- a/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/GenerationJobManager.stream_integration.spec.ts@@ -377,6 +377,426 @@ describe('GenerationJobManager Integration Tests', () => {});});+ describe('Cross-Replica Support (Redis)', () => {+ /**+ * Problem: In k8s with Redis and multiple replicas, when a user sends a message:+ * 1. POST /api/agents/chat hits Replica A, creates job+ * 2. GET /api/agents/chat/stream/:streamId hits Replica B+ * 3. Replica B calls getJob() which returned undefined because runtimeState+ * was only in Replica A's memory+ * 4. Stream endpoint returns 404+ *+ * Fix: getJob() and subscribe() now lazily create runtime state from Redis+ * when the job exists in Redis but not in local memory.+ */+ test('should NOT return 404 when stream endpoint hits different replica than job creator', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisJobStore } = await import('../implementations/RedisJobStore');++ // === REPLICA A: Creates the job ===+ // Simulate Replica A creating the job directly in Redis+ // (In real scenario, this happens via GenerationJobManager.createJob on Replica A)+ const replicaAJobStore = new RedisJobStore(ioredisClient);+ await replicaAJobStore.initialize();++ const streamId = `cross-replica-404-test-${Date.now()}`;+ const userId = 'test-user';++ // Create job in Redis (simulates Replica A's createJob)+ await replicaAJobStore.createJob(streamId, userId);++ // === REPLICA B: Receives the stream request ===+ // Fresh GenerationJobManager that does NOT have this job in its local runtimeState+ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ // This is what the stream endpoint does:+ // const job = await GenerationJobManager.getJob(streamId);+ // if (!job) return res.status(404).json({ error: 'Stream not found' });++ const job = await GenerationJobManager.getJob(streamId);++ // BEFORE FIX: job would be undefined → 404+ // AFTER FIX: job should exist via lazy runtime state creation+ expect(job).not.toBeNull();+ expect(job).toBeDefined();+ expect(job?.streamId).toBe(streamId);++ // The stream endpoint then calls subscribe:+ // const result = await GenerationJobManager.subscribe(streamId, onChunk, onDone, onError);+ // if (!result) return res.status(404).json({ error: 'Failed to subscribe' });++ const subscription = await GenerationJobManager.subscribe(+ streamId,+ () => {}, // onChunk+ () => {}, // onDone+ () => {}, // onError+ );++ // BEFORE FIX: subscription would be null → 404+ // AFTER FIX: subscription should succeed+ expect(subscription).not.toBeNull();+ expect(subscription).toBeDefined();+ expect(typeof subscription?.unsubscribe).toBe('function');++ // Cleanup+ subscription?.unsubscribe();+ await GenerationJobManager.destroy();+ await replicaAJobStore.destroy();+ });++ test('should lazily create runtime state for jobs created on other replicas', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ // Simulate two instances - one creates job, other tries to get it+ const { createStreamServices } = await import('../createStreamServices');+ const { RedisJobStore } = await import('../implementations/RedisJobStore');++ // Instance 1: Create the job directly in Redis (simulating another replica)+ const jobStore = new RedisJobStore(ioredisClient);+ await jobStore.initialize();++ const streamId = `cross-replica-${Date.now()}`;+ const userId = 'test-user';++ // Create job data directly in jobStore (as if from another instance)+ await jobStore.createJob(streamId, userId);++ // Instance 2: Fresh GenerationJobManager that doesn't have this job in memory+ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ // This should work even though the job was created by "another instance"+ // The manager should lazily create runtime state from Redis data+ const job = await GenerationJobManager.getJob(streamId);++ expect(job).not.toBeNull();+ expect(job?.streamId).toBe(streamId);+ expect(job?.status).toBe('running');++ // Should also be able to subscribe+ const chunks: unknown[] = [];+ const subscription = await GenerationJobManager.subscribe(streamId, (event) => {+ chunks.push(event);+ });++ expect(subscription).not.toBeNull();++ subscription?.unsubscribe();+ await GenerationJobManager.destroy();+ await jobStore.destroy();+ });++ test('should persist syncSent to Redis for cross-replica consistency', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId = `sync-sent-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');++ // Initially syncSent should be false+ let wasSent = await GenerationJobManager.wasSyncSent(streamId);+ expect(wasSent).toBe(false);++ // Mark sync sent+ GenerationJobManager.markSyncSent(streamId);++ // Wait for async Redis update+ await new Promise((resolve) => setTimeout(resolve, 50));++ // Should now be true+ wasSent = await GenerationJobManager.wasSyncSent(streamId);+ expect(wasSent).toBe(true);++ // Verify it's actually in Redis by checking via jobStore+ const jobStore = services.jobStore;+ const jobData = await jobStore.getJob(streamId);+ expect(jobData?.syncSent).toBe(true);++ await GenerationJobManager.destroy();+ });++ test('should persist finalEvent to Redis for cross-replica access', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure({+ ...services,+ cleanupOnComplete: false, // Keep job for verification+ });+ await GenerationJobManager.initialize();++ const streamId = `final-event-${Date.now()}`;+ await GenerationJobManager.createJob(streamId, 'user-1');++ // Emit done event with final data+ const finalEventData = {+ final: true,+ conversation: { conversationId: streamId },+ responseMessage: { text: 'Hello world' },+ };+ GenerationJobManager.emitDone(streamId, finalEventData as never);++ // Wait for async Redis update+ await new Promise((resolve) => setTimeout(resolve, 50));++ // Verify finalEvent is in Redis+ const jobStore = services.jobStore;+ const jobData = await jobStore.getJob(streamId);+ expect(jobData?.finalEvent).toBeDefined();++ const storedFinalEvent = JSON.parse(jobData!.finalEvent!);+ expect(storedFinalEvent.final).toBe(true);+ expect(storedFinalEvent.conversation.conversationId).toBe(streamId);++ await GenerationJobManager.destroy();+ });++ test('should emit cross-replica abort signal via Redis pub/sub', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { GenerationJobManager } = await import('../GenerationJobManager');+ const { createStreamServices } = await import('../createStreamServices');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId = `abort-signal-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');++ // Track if abort controller was signaled+ let abortSignaled = false;+ job.abortController.signal.addEventListener('abort', () => {+ abortSignaled = true;+ });++ // Wait for abort listener setup+ await new Promise((resolve) => setTimeout(resolve, 100));++ // Abort the job - this should emit abort signal via Redis+ await GenerationJobManager.abortJob(streamId);++ // Wait for signal propagation+ await new Promise((resolve) => setTimeout(resolve, 100));++ // The local abort controller should be signaled+ expect(abortSignaled).toBe(true);+ expect(job.abortController.signal.aborted).toBe(true);++ await GenerationJobManager.destroy();+ });++ test('should handle abort for lazily-initialized cross-replica jobs', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ // This test validates that jobs created on Replica A and lazily-initialized+ // on Replica B can still receive and handle abort signals.++ const { createStreamServices } = await import('../createStreamServices');+ const { RedisJobStore } = await import('../implementations/RedisJobStore');++ // === Replica A: Create job directly in Redis ===+ const replicaAJobStore = new RedisJobStore(ioredisClient);+ await replicaAJobStore.initialize();++ const streamId = `lazy-abort-${Date.now()}`;+ await replicaAJobStore.createJob(streamId, 'user-1');++ // === Replica B: Fresh manager that lazily initializes the job ===+ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ // Get job triggers lazy initialization of runtime state+ const job = await GenerationJobManager.getJob(streamId);+ expect(job).not.toBeNull();++ // Track abort signal+ let abortSignaled = false;+ job!.abortController.signal.addEventListener('abort', () => {+ abortSignaled = true;+ });++ // Wait for abort listener to be set up via Redis subscription+ await new Promise((resolve) => setTimeout(resolve, 150));++ // Abort the job - this should emit abort signal via Redis pub/sub+ // The lazily-initialized runtime should receive it+ await GenerationJobManager.abortJob(streamId);++ // Wait for signal propagation+ await new Promise((resolve) => setTimeout(resolve, 150));++ // Verify the lazily-initialized job received the abort signal+ expect(abortSignaled).toBe(true);+ expect(job!.abortController.signal.aborted).toBe(true);++ await GenerationJobManager.destroy();+ await replicaAJobStore.destroy();+ });++ test('should abort generation when abort signal received from another replica', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ // This test simulates:+ // 1. Replica A creates a job and starts generation+ // 2. Replica B receives abort request and emits abort signal+ // 3. Replica A receives signal and aborts its AbortController++ const { createStreamServices } = await import('../createStreamServices');+ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ // Create the job on "Replica A"+ const { GenerationJobManager } = await import('../GenerationJobManager');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ const streamId = `cross-abort-${Date.now()}`;+ const job = await GenerationJobManager.createJob(streamId, 'user-1');++ let abortSignaled = false;+ job.abortController.signal.addEventListener('abort', () => {+ abortSignaled = true;+ });++ // Wait for abort listener to be set up via Redis subscription+ await new Promise((resolve) => setTimeout(resolve, 150));++ // Simulate "Replica B" emitting abort signal directly via Redis+ // This is what would happen if abortJob was called on a different replica+ const subscriber2 = (ioredisClient as unknown as { duplicate: () => unknown }).duplicate();+ const replicaBTransport = new RedisEventTransport(+ ioredisClient as never,+ subscriber2 as never,+ );++ // Emit abort signal (as if from Replica B)+ replicaBTransport.emitAbort(streamId);++ // Wait for cross-replica signal propagation+ await new Promise((resolve) => setTimeout(resolve, 200));++ // Replica A's abort controller should be signaled+ expect(abortSignaled).toBe(true);+ expect(job.abortController.signal.aborted).toBe(true);++ replicaBTransport.destroy();+ (subscriber2 as { disconnect: () => void }).disconnect();+ await GenerationJobManager.destroy();+ });++ test('should handle wasSyncSent for cross-replica scenarios', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { createStreamServices } = await import('../createStreamServices');+ const { RedisJobStore } = await import('../implementations/RedisJobStore');++ // Create job directly in Redis with syncSent: true+ const jobStore = new RedisJobStore(ioredisClient);+ await jobStore.initialize();++ const streamId = `cross-sync-${Date.now()}`;+ await jobStore.createJob(streamId, 'user-1');+ await jobStore.updateJob(streamId, { syncSent: true });++ // Fresh manager that doesn't have this job locally+ jest.resetModules();+ const { GenerationJobManager } = await import('../GenerationJobManager');++ const services = createStreamServices({+ useRedis: true,+ redisClient: ioredisClient,+ });++ GenerationJobManager.configure(services);+ await GenerationJobManager.initialize();++ // wasSyncSent should check Redis even without local runtime+ const wasSent = await GenerationJobManager.wasSyncSent(streamId);+ expect(wasSent).toBe(true);++ await GenerationJobManager.destroy();+ await jobStore.destroy();+ });+ });+describe('createStreamServices Auto-Detection', () => {test('should auto-detect Redis when USE_REDIS is true', async () => {if (!ioredisClient) {diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.tsindex b70e53012e..31266b3e11 100644--- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts@@ -294,6 +294,154 @@ describe('RedisEventTransport Integration Tests', () => {});});+ describe('Cross-Replica Abort', () => {+ test('should emit and receive abort signals on same instance', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `abort-same-${Date.now()}`;+ let abortReceived = false;++ // Register abort callback+ transport.onAbort(streamId, () => {+ abortReceived = true;+ });++ // Wait for subscription to be established+ await new Promise((resolve) => setTimeout(resolve, 100));++ // Emit abort+ transport.emitAbort(streamId);++ // Wait for signal to propagate+ await new Promise((resolve) => setTimeout(resolve, 200));++ expect(abortReceived).toBe(true);++ transport.destroy();+ subscriber.disconnect();+ });++ test('should deliver abort signals across transport instances', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ // Two separate instances (simulating two servers)+ const subscriber1 = (ioredisClient as Redis).duplicate();+ const subscriber2 = (ioredisClient as Redis).duplicate();++ const transport1 = new RedisEventTransport(ioredisClient, subscriber1);+ const transport2 = new RedisEventTransport(ioredisClient, subscriber2);++ const streamId = `abort-cross-${Date.now()}`;+ let instance1AbortReceived = false;++ // Instance 1 registers abort callback (simulates server running generation)+ transport1.onAbort(streamId, () => {+ instance1AbortReceived = true;+ });++ // Wait for subscription+ await new Promise((resolve) => setTimeout(resolve, 100));++ // Instance 2 emits abort (simulates server receiving abort request)+ transport2.emitAbort(streamId);++ // Wait for cross-instance delivery+ await new Promise((resolve) => setTimeout(resolve, 200));++ // Instance 1 should receive abort signal+ expect(instance1AbortReceived).toBe(true);++ transport1.destroy();+ transport2.destroy();+ subscriber1.disconnect();+ subscriber2.disconnect();+ });++ test('should call multiple abort callbacks', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `abort-multi-${Date.now()}`;+ let callback1Called = false;+ let callback2Called = false;++ // Multiple abort callbacks+ transport.onAbort(streamId, () => {+ callback1Called = true;+ });+ transport.onAbort(streamId, () => {+ callback2Called = true;+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ transport.emitAbort(streamId);++ await new Promise((resolve) => setTimeout(resolve, 200));++ expect(callback1Called).toBe(true);+ expect(callback2Called).toBe(true);++ transport.destroy();+ subscriber.disconnect();+ });++ test('should cleanup abort callbacks on stream cleanup', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `abort-cleanup-${Date.now()}`;+ let abortReceived = false;++ transport.onAbort(streamId, () => {+ abortReceived = true;+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ // Cleanup the stream+ transport.cleanup(streamId);++ // Emit abort after cleanup+ transport.emitAbort(streamId);++ await new Promise((resolve) => setTimeout(resolve, 200));++ // Should NOT receive abort since stream was cleaned up+ expect(abortReceived).toBe(false);++ transport.destroy();+ subscriber.disconnect();+ });+ });+describe('Cleanup', () => {test('should clean up stream resources', async () => {if (!ioredisClient) {diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.tsindex 79aa05699a..02d4cb69ed 100644--- a/packages/api/src/stream/implementations/RedisEventTransport.ts+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts@@ -17,6 +17,7 @@ const EventTypes = {CHUNK: 'chunk',DONE: 'done',ERROR: 'error',+ ABORT: 'abort',} as const;interface PubSubMessage {@@ -39,6 +40,8 @@ interface StreamSubscribers {}>;allSubscribersLeftCallbacks: Array<() => void>;+ /** Abort callbacks - called when abort signal is received from any replica */+ abortCallbacks: Array<() => void>;}/**@@ -119,6 +122,20 @@ export class RedisEventTransport implements IEventTransport {case EventTypes.ERROR:handlers.onError?.(parsed.error ?? 'Unknown error');break;+ case EventTypes.ABORT:+ // Abort is handled at stream level, not per-handler+ break;+ }+ }++ // Handle abort signals at stream level (not per-handler)+ if (parsed.type === EventTypes.ABORT) {+ for (const callback of streamState.abortCallbacks) {+ try {+ callback();+ } catch (err) {+ logger.error(`[RedisEventTransport] Error in abort callback:`, err);+ }}}} catch (err) {@@ -149,6 +166,7 @@ export class RedisEventTransport implements IEventTransport {count: 0,handlers: new Map(),allSubscribersLeftCallbacks: [],+ abortCallbacks: [],});}@@ -263,6 +281,53 @@ export class RedisEventTransport implements IEventTransport {count: 0,handlers: new Map(),allSubscribersLeftCallbacks: [callback],+ abortCallbacks: [],+ });+ }+ }++ /**+ * Publish an abort signal to all replicas.+ * This enables cross-replica abort: when a user aborts on Replica B,+ * the generating Replica A receives the signal and stops.+ */+ emitAbort(streamId: string): void {+ const channel = CHANNELS.events(streamId);+ const message: PubSubMessage = { type: EventTypes.ABORT };++ this.publisher.publish(channel, JSON.stringify(message)).catch((err) => {+ logger.error(`[RedisEventTransport] Failed to publish abort:`, err);+ });+ }++ /**+ * Register callback for abort signals from any replica.+ * Called when abort is triggered on any replica (including this one).+ *+ * @param streamId - The stream identifier+ * @param callback - Called when abort signal is received+ */+ onAbort(streamId: string, callback: () => void): void {+ const channel = CHANNELS.events(streamId);+ let state = this.streams.get(streamId);++ if (!state) {+ state = {+ count: 0,+ handlers: new Map(),+ allSubscribersLeftCallbacks: [],+ abortCallbacks: [],+ };+ this.streams.set(streamId, state);+ }++ state.abortCallbacks.push(callback);++ // Subscribe to Redis channel if not already subscribed+ if (!this.subscribedChannels.has(channel)) {+ this.subscribedChannels.add(channel);+ this.subscriber.subscribe(channel).catch((err) => {+ logger.error(`[RedisEventTransport] Failed to subscribe to ${channel}:`, err);});}}@@ -282,9 +347,10 @@ export class RedisEventTransport implements IEventTransport {const state = this.streams.get(streamId);if (state) {- // Clear all handlers+ // Clear all handlers and callbacksstate.handlers.clear();state.allSubscribersLeftCallbacks = [];+ state.abortCallbacks = [];}// Unsubscribe from Redis channeldiff --git a/packages/api/src/stream/interfaces/IJobStore.ts b/packages/api/src/stream/interfaces/IJobStore.tsindex 80cdd30d83..14611a7fad 100644--- a/packages/api/src/stream/interfaces/IJobStore.ts+++ b/packages/api/src/stream/interfaces/IJobStore.ts@@ -236,6 +236,21 @@ export interface IEventTransport {/** Publish an error event */emitError(streamId: string, error: string): void;+ /**+ * Publish an abort signal to all replicas (Redis mode).+ * Enables cross-replica abort: user aborts on Replica B,+ * generating Replica A receives signal and stops.+ * Optional - only implemented in Redis transport.+ */+ emitAbort?(streamId: string): void;++ /**+ * Register callback for abort signals from any replica (Redis mode).+ * Called when abort is triggered from any replica.+ * Optional - only implemented in Redis transport.+ */+ onAbort?(streamId: string, callback: () => void): void;+/** Get subscriber count for a stream */getSubscriberCount(streamId: string): number;