Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
/* eslint jest/no-standalone-expect: ["error", { "additionalTestBlockFunctions": ["testRedis"] }] */
import type { Redis, Cluster } from 'ioredis';
import type { ServerSentEvent, StreamEvent, CreatedEvent } from '~/types';
import type { IJobStoreV2 } from '~/stream/interfaces/IJobStore';
import {
GenerationJobManagerClass,
GENERATION_RECOVERY_FAILED_ERROR,
TERMINAL_PUBLICATION_RECONNECT_ERROR,
} from '~/stream/GenerationJobManager';
import {
ioredisClient as staticRedisClient,
keyvRedisClient as staticKeyvClient,
keyvRedisClientReady,
} from '~/cache/redisClients';
import { InMemoryEventTransport } from '~/stream/implementations/InMemoryEventTransport';
import { RedisEventTransport } from '~/stream/implementations/RedisEventTransport';
import { InMemoryJobStore } from '~/stream/implementations/InMemoryJobStore';
import { STEER_ENQUEUE_NOT_RUNNING } from '~/stream/interfaces/IJobStore';
import { RedisJobStore } from '~/stream/implementations/RedisJobStore';
import { createStreamServices } from '~/stream/createStreamServices';
import { GenerationJobManager } from '~/stream/GenerationJobManager';
/** Suppress winston Console transport output (survives jest.resetModules) */
jest.spyOn(console, 'log').mockImplementation();
/**
* Integration tests for GenerationJobManager.
*
* Tests the job manager with both in-memory and Redis backends
* to ensure consistent behavior across deployment modes.
*
* Run with: USE_REDIS=true npx jest GenerationJobManager.stream_integration
*/
describe('GenerationJobManager Integration Tests', () => {
let originalEnv: NodeJS.ProcessEnv;
let ioredisClient: Redis | Cluster | null = null;
let dynamicKeyvClient: unknown = null;
let dynamicKeyvReady: Promise<unknown> | null = null;
const testPrefix = 'JobManager-Integration-Test';
const redisConfigured = process.env.USE_REDIS === 'true';
const describeRedis = redisConfigured ? describe : describe.skip;
const testRedis = redisConfigured ? test : test.skip;
beforeAll(async () => {
originalEnv = { ...process.env };
process.env.USE_REDIS = process.env.USE_REDIS ?? 'true';
process.env.REDIS_URI = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379';
process.env.REDIS_KEY_PREFIX = testPrefix;
jest.resetModules();
const redisModule = await import('~/cache/redisClients');
ioredisClient = redisModule.ioredisClient;
dynamicKeyvClient = redisModule.keyvRedisClient;
dynamicKeyvReady = redisModule.keyvRedisClientReady;
});
afterEach(async () => {
// Clean up module state
jest.resetModules();
// Clean up Redis keys (delete individually for cluster compatibility)
if (ioredisClient) {
try {
const keys = await ioredisClient.keys(`${testPrefix}*`);
const streamKeys = await ioredisClient.keys(`stream:*`);
const allKeys = [...keys, ...streamKeys];
await Promise.all(allKeys.map((key) => ioredisClient!.del(key)));
} catch {
// Ignore cleanup errors
}
}
});
afterAll(async () => {
for (const ready of [keyvRedisClientReady, dynamicKeyvReady]) {
if (ready) {
await ready.catch(() => {});
}
}
const clients = [ioredisClient, staticRedisClient, staticKeyvClient, dynamicKeyvClient];
for (const client of clients) {
if (!client) {
continue;
}
try {
await (client as { disconnect: () => void | Promise<void> }).disconnect();
} catch {
/* ignore */
}
}
process.env = originalEnv;
});
function createInMemoryManager(): GenerationJobManagerClass {
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
return manager;
}
function createRedisManager(): GenerationJobManagerClass {
const manager = new GenerationJobManagerClass();
manager.configure(
createStreamServices({
useRedis: true,
redisClient: ioredisClient!,
}),
);
manager.initialize();
return manager;
}
async function setupDisconnectedStream(
manager: GenerationJobManagerClass,
streamId: string,
delay: number,
): Promise<ServerSentEvent[]> {
const firstEvents: ServerSentEvent[] = [];
const sub = await manager.subscribe(streamId, (event) => firstEvents.push(event));
await new Promise((resolve) => setTimeout(resolve, delay));
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'Hello' } },
},
});
await new Promise((resolve) => setTimeout(resolve, delay));
expect(firstEvents.length).toBe(2);
sub?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, delay));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: ' world' } },
},
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { id: 'step-1', delta: { content: { type: 'text', text: '!' } } },
});
await new Promise((resolve) => setTimeout(resolve, delay));
return firstEvents;
}
describe('In-Memory Mode', () => {
test('should create and manage jobs', async () => {
// Configure with in-memory
// cleanupOnComplete: false so we can verify completed status
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
GenerationJobManager.initialize();
const streamId = `inmem-job-${Date.now()}`;
const userId = 'test-user-1';
// Create job (async)
const job = await GenerationJobManager.createJob(streamId, userId);
expect(job.streamId).toBe(streamId);
expect(job.status).toBe('running');
// Check job exists
const hasJob = await GenerationJobManager.hasJob(streamId);
expect(hasJob).toBe(true);
// Get job
const retrieved = await GenerationJobManager.getJob(streamId);
expect(retrieved?.streamId).toBe(streamId);
// Update job
await GenerationJobManager.updateMetadata(streamId, {
sender: 'TestAgent',
});
const updated = await GenerationJobManager.getJob(streamId);
expect(updated?.metadata?.sender).toBe('TestAgent');
// Complete job
await GenerationJobManager.completeJob(streamId);
const completed = await GenerationJobManager.getJob(streamId);
expect(completed?.status).toBe('complete');
await GenerationJobManager.destroy();
});
test('should handle event streaming', async () => {
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
GenerationJobManager.initialize();
const streamId = `inmem-events-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const receivedChunks: unknown[] = [];
// Subscribe to events (subscribe takes separate args, not an object)
const subscription = await GenerationJobManager.subscribe(streamId, (event) =>
receivedChunks.push(event),
);
const { unsubscribe } = subscription!;
// Wait for first subscriber to be registered
await new Promise((resolve) => setTimeout(resolve, 10));
// Emit chunks (emitChunk takes { event, data } format, now async for Redis ordering)
await GenerationJobManager.emitChunk(streamId, {
event: 'on_message_delta',
data: { type: 'text', text: 'Hello' },
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_message_delta',
data: { type: 'text', text: ' world' },
});
// Give time for events to propagate
await new Promise((resolve) => setTimeout(resolve, 50));
// Verify chunks were received
expect(receivedChunks.length).toBeGreaterThan(0);
// Complete the job (this cleans up resources)
await GenerationJobManager.completeJob(streamId);
unsubscribe();
await GenerationJobManager.destroy();
});
});
describeRedis('Redis Mode', () => {
test('should create and manage jobs via Redis', async () => {
// Create Redis services
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
expect(services.isRedis).toBe(true);
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `redis-job-${Date.now()}`;
const userId = 'test-user-redis';
// Create job (async)
const job = await GenerationJobManager.createJob(streamId, userId);
expect(job.streamId).toBe(streamId);
// Verify in Redis
const hasJob = await GenerationJobManager.hasJob(streamId);
expect(hasJob).toBe(true);
// Update and verify
await GenerationJobManager.updateMetadata(streamId, {
sender: 'RedisAgent',
});
const updated = await GenerationJobManager.getJob(streamId);
expect(updated?.metadata?.sender).toBe('RedisAgent');
await GenerationJobManager.destroy();
});
test('should persist chunks for cross-instance resume', async () => {
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `redis-chunks-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
// Emit chunks (these should be persisted to Redis)
// emitChunk takes { event, data } format, now async for Redis ordering
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'Persisted ' } },
},
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'content' } },
},
});
// Wait for async operations
await new Promise((resolve) => setTimeout(resolve, 100));
// Simulate getting resume state (as if from different instance)
const resumeState = await GenerationJobManager.getResumeState(streamId);
expect(resumeState).not.toBeNull();
expect(resumeState!.aggregatedContent?.length).toBeGreaterThan(0);
await GenerationJobManager.destroy();
});
test('should handle abort and return content', async () => {
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `redis-abort-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
// Emit some content (emitChunk takes { event, data } format, now async)
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'Partial response...' } },
},
});
await new Promise((resolve) => setTimeout(resolve, 100));
// Abort the job
const abortResult = await GenerationJobManager.abortJob(streamId);
expect(abortResult.success).toBe(true);
expect(abortResult.content.length).toBeGreaterThan(0);
await GenerationJobManager.destroy();
});
});
describe('Cross-Mode Consistency', () => {
test('should have consistent API between in-memory and Redis modes', async () => {
// This test verifies that the same operations work identically
// regardless of backend mode
const runTestWithMode = async (isRedis: boolean) => {
jest.resetModules();
if (isRedis && ioredisClient) {
GenerationJobManager.configure({
...createStreamServices({
useRedis: true,
redisClient: ioredisClient,
}),
cleanupOnComplete: false, // Keep job for verification
});
} else {
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
}
GenerationJobManager.initialize();
const streamId = `consistency-${isRedis ? 'redis' : 'inmem'}-${Date.now()}`;
// Test sequence
const job = await GenerationJobManager.createJob(streamId, 'user-1');
expect(job.streamId).toBe(streamId);
expect(job.status).toBe('running');
const hasJob = await GenerationJobManager.hasJob(streamId);
expect(hasJob).toBe(true);
await GenerationJobManager.updateMetadata(streamId, {
sender: 'ConsistencyAgent',
responseMessageId: 'resp-123',
iconURL: 'https://example.com/spec-icon.png',
model: 'gpt-4.1',
});
const updated = await GenerationJobManager.getJob(streamId);
expect(updated?.metadata?.sender).toBe('ConsistencyAgent');
expect(updated?.metadata?.responseMessageId).toBe('resp-123');
expect(updated?.metadata?.iconURL).toBe('https://example.com/spec-icon.png');
expect(updated?.metadata?.model).toBe('gpt-4.1');
const resumeState = await GenerationJobManager.getResumeState(streamId);
expect(resumeState?.sender).toBe('ConsistencyAgent');
expect(resumeState?.responseMessageId).toBe('resp-123');
expect(resumeState?.iconURL).toBe('https://example.com/spec-icon.png');
expect(resumeState?.model).toBe('gpt-4.1');
await GenerationJobManager.completeJob(streamId);
const completed = await GenerationJobManager.getJob(streamId);
expect(completed?.status).toBe('complete');
await GenerationJobManager.destroy();
};
// Test in-memory mode
await runTestWithMode(false);
// Test Redis mode if available
if (ioredisClient) {
await runTestWithMode(true);
}
});
});
describeRedis('Cross-Replica Support (Redis)', () => {
/**
* Problem: In k8s with Redis and multiple replicas, when a user sends a message:
* 1. POST /api/agents/chat hits Replica A, creates job
* 2. GET /api/agents/chat/stream/:streamId hits Replica B
* 3. Replica B calls getJob() which returned undefined because runtimeState
* was only in Replica A's memory
* 4. Stream endpoint returns 404
*
* Fix: getJob() and subscribe() now lazily create runtime state from Redis
* when the job exists in Redis but not in local memory.
*/
test('should NOT return 404 when stream endpoint hits different replica than job creator', async () => {
// === REPLICA A: Creates the job ===
// Simulate Replica A creating the job directly in Redis
// (In real scenario, this happens via GenerationJobManager.createJob on Replica A)
const replicaAJobStore = new RedisJobStore(ioredisClient!);
await replicaAJobStore.initialize();
const streamId = `cross-replica-404-test-${Date.now()}`;
const userId = 'test-user';
// Create job in Redis (simulates Replica A's createJob)
await replicaAJobStore.createJob(streamId, userId);
// === REPLICA B: Receives the stream request ===
// Fresh GenerationJobManager that does NOT have this job in its local runtimeState
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
// This is what the stream endpoint does:
// const job = await GenerationJobManager.getJob(streamId);
// if (!job) return res.status(404).json({ error: 'Stream not found' });
const job = await GenerationJobManager.getJob(streamId);
// BEFORE FIX: job would be undefined → 404
// AFTER FIX: job should exist via lazy runtime state creation
expect(job).not.toBeNull();
expect(job).toBeDefined();
expect(job?.streamId).toBe(streamId);
// The stream endpoint then calls subscribe:
// const result = await GenerationJobManager.subscribe(streamId, onChunk, onDone, onError);
// if (!result) return res.status(404).json({ error: 'Failed to subscribe' });
const subscription = await GenerationJobManager.subscribe(
streamId,
() => {}, // onChunk
() => {}, // onDone
() => {}, // onError
);
// BEFORE FIX: subscription would be null → 404
// AFTER FIX: subscription should succeed
expect(subscription).not.toBeNull();
expect(subscription).toBeDefined();
expect(typeof subscription?.unsubscribe).toBe('function');
// Cleanup
subscription?.unsubscribe();
await GenerationJobManager.destroy();
await replicaAJobStore.destroy();
});
test('should lazily create runtime state for jobs created on other replicas', async () => {
// Instance 1: Create the job directly in Redis (simulating another replica)
const jobStore = new RedisJobStore(ioredisClient!);
await jobStore.initialize();
const streamId = `cross-replica-${Date.now()}`;
const userId = 'test-user';
// Create job data directly in jobStore (as if from another instance)
await jobStore.createJob(streamId, userId);
// Instance 2: Fresh GenerationJobManager that doesn't have this job in memory
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
// This should work even though the job was created by "another instance"
// The manager should lazily create runtime state from Redis data
const job = await GenerationJobManager.getJob(streamId);
expect(job).not.toBeNull();
expect(job?.streamId).toBe(streamId);
expect(job?.status).toBe('running');
// Should also be able to subscribe
const chunks: unknown[] = [];
const subscription = await GenerationJobManager.subscribe(streamId, (event) => {
chunks.push(event);
});
expect(subscription).not.toBeNull();
subscription?.unsubscribe();
await GenerationJobManager.destroy();
await jobStore.destroy();
});
test('should persist syncSent to Redis for cross-replica consistency', async () => {
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `sync-sent-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
// Initially syncSent should be false
let wasSent = await GenerationJobManager.wasSyncSent(streamId);
expect(wasSent).toBe(false);
// Mark sync sent
GenerationJobManager.markSyncSent(streamId);
// Wait for async Redis update
await new Promise((resolve) => setTimeout(resolve, 50));
// Should now be true
wasSent = await GenerationJobManager.wasSyncSent(streamId);
expect(wasSent).toBe(true);
// Verify it's actually in Redis by checking via jobStore
const jobStore = services.jobStore;
const jobData = await jobStore.getJob(streamId);
expect(jobData?.syncSent).toBe(true);
await GenerationJobManager.destroy();
});
test('should persist finalEvent to Redis for cross-replica access', async () => {
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure({
...services,
cleanupOnComplete: false, // Keep job for verification
});
GenerationJobManager.initialize();
const streamId = `final-event-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
// Emit done event with final data
const finalEventData = {
final: true,
conversation: { conversationId: streamId },
responseMessage: { text: 'Hello world' },
};
await GenerationJobManager.emitDone(streamId, finalEventData as never);
await new Promise((resolve) => setTimeout(resolve, 200));
// Verify finalEvent is in Redis
const jobStore = services.jobStore;
const jobData = await jobStore.getJob(streamId);
expect(jobData?.finalEvent).toBeDefined();
const storedFinalEvent = JSON.parse(jobData!.finalEvent!);
expect(storedFinalEvent.final).toBe(true);
expect(storedFinalEvent.conversation.conversationId).toBe(streamId);
await GenerationJobManager.destroy();
});
test('should emit cross-replica abort signal via Redis pub/sub', async () => {
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `abort-signal-${Date.now()}`;
const job = await GenerationJobManager.createJob(streamId, 'user-1');
// Track if abort controller was signaled
let abortSignaled = false;
job.abortController.signal.addEventListener('abort', () => {
abortSignaled = true;
});
// Wait for abort listener setup
await new Promise((resolve) => setTimeout(resolve, 100));
// Abort the job - this should emit abort signal via Redis
await GenerationJobManager.abortJob(streamId);
// Wait for signal propagation
await new Promise((resolve) => setTimeout(resolve, 100));
// The local abort controller should be signaled
expect(abortSignaled).toBe(true);
expect(job.abortController.signal.aborted).toBe(true);
await GenerationJobManager.destroy();
});
test('should handle abort for lazily-initialized cross-replica jobs', async () => {
// This test validates that jobs created on Replica A and lazily-initialized
// on Replica B can still receive and handle abort signals.
// === Replica A: Create job directly in Redis ===
const replicaAJobStore = new RedisJobStore(ioredisClient!);
await replicaAJobStore.initialize();
const streamId = `lazy-abort-${Date.now()}`;
await replicaAJobStore.createJob(streamId, 'user-1');
// === Replica B: Fresh manager that lazily initializes the job ===
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
// Get job triggers lazy initialization of runtime state
const job = await GenerationJobManager.getJob(streamId);
expect(job).not.toBeNull();
// Track abort signal
let abortSignaled = false;
job!.abortController.signal.addEventListener('abort', () => {
abortSignaled = true;
});
// Wait for abort listener to be set up via Redis subscription
await new Promise((resolve) => setTimeout(resolve, 150));
// Abort the job - this should emit abort signal via Redis pub/sub
// The lazily-initialized runtime should receive it
await GenerationJobManager.abortJob(streamId);
// Wait for signal propagation
await new Promise((resolve) => setTimeout(resolve, 150));
// Verify the lazily-initialized job received the abort signal
expect(abortSignaled).toBe(true);
expect(job!.abortController.signal.aborted).toBe(true);
await GenerationJobManager.destroy();
await replicaAJobStore.destroy();
});
test('should abort generation when abort signal received from another replica', async () => {
// This test simulates:
// 1. Replica A creates a job and starts generation
// 2. Replica B receives abort request and emits abort signal
// 3. Replica A receives signal and aborts its AbortController
// Create the job on "Replica A"
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `cross-abort-${Date.now()}`;
const job = await GenerationJobManager.createJob(streamId, 'user-1');
let abortSignaled = false;
job.abortController.signal.addEventListener('abort', () => {
abortSignaled = true;
});
// Wait for abort listener to be set up via Redis subscription
await new Promise((resolve) => setTimeout(resolve, 150));
// Simulate "Replica B" emitting abort signal directly via Redis
// This is what would happen if abortJob was called on a different replica
const subscriber2 = (ioredisClient as unknown as { duplicate: () => unknown }).duplicate();
const replicaBTransport = new RedisEventTransport(
ioredisClient as never,
subscriber2 as never,
);
// Emit abort signal (as if from Replica B)
replicaBTransport.emitAbort(streamId);
// Wait for cross-replica signal propagation
await new Promise((resolve) => setTimeout(resolve, 200));
// Replica A's abort controller should be signaled
expect(abortSignaled).toBe(true);
expect(job.abortController.signal.aborted).toBe(true);
replicaBTransport.destroy();
(subscriber2 as { disconnect: () => void }).disconnect();
await GenerationJobManager.destroy();
});
test('should handle wasSyncSent for cross-replica scenarios', async () => {
// Create job directly in Redis with syncSent: true
const jobStore = new RedisJobStore(ioredisClient!);
await jobStore.initialize();
const streamId = `cross-sync-${Date.now()}`;
await jobStore.createJob(streamId, 'user-1');
await jobStore.updateJob(streamId, { syncSent: true });
// Fresh manager that doesn't have this job locally
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
// wasSyncSent should check Redis even without local runtime
const wasSent = await GenerationJobManager.wasSyncSent(streamId);
expect(wasSent).toBe(true);
await GenerationJobManager.destroy();
await jobStore.destroy();
});
test('should emit created event from metadata on cross-replica subscribe', async () => {
const replicaAJobStore = new RedisJobStore(ioredisClient!);
await replicaAJobStore.initialize();
const streamId = `cross-created-${Date.now()}`;
const userId = 'test-user';
await replicaAJobStore.createJob(streamId, userId);
await replicaAJobStore.updateJob(streamId, {
userMessage: {
messageId: 'msg-123',
parentMessageId: '00000000-0000-0000-0000-000000000000',
conversationId: streamId,
text: 'hello world',
},
});
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const received: unknown[] = [];
const subscription = await GenerationJobManager.subscribe(streamId, (event) =>
received.push(event),
);
expect(subscription).not.toBeNull();
expect(received.length).toBe(1);
const created = received[0] as CreatedEvent;
expect(created.created).toBe(true);
expect(created.streamId).toBe(streamId);
expect(created.message.messageId).toBe('msg-123');
expect(created.message.conversationId).toBe(streamId);
expect(created.message.sender).toBe('User');
expect(created.message.isCreatedByUser).toBe(true);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
await replicaAJobStore.destroy();
});
test('should NOT emit created event from metadata when userMessage is not set', async () => {
const replicaAJobStore = new RedisJobStore(ioredisClient!);
await replicaAJobStore.initialize();
const streamId = `cross-no-created-${Date.now()}`;
await replicaAJobStore.createJob(streamId, 'test-user');
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const received: unknown[] = [];
const subscription = await GenerationJobManager.subscribe(streamId, (event) =>
received.push(event),
);
expect(subscription).not.toBeNull();
expect(received.length).toBe(0);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
await replicaAJobStore.destroy();
});
test('should NOT emit created event when skipBufferReplay is true (resume path)', async () => {
const replicaAJobStore = new RedisJobStore(ioredisClient!);
await replicaAJobStore.initialize();
const streamId = `cross-no-replay-${Date.now()}`;
await replicaAJobStore.createJob(streamId, 'test-user');
await replicaAJobStore.updateJob(streamId, {
userMessage: {
messageId: 'msg-456',
conversationId: streamId,
text: 'hi',
},
});
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const received: unknown[] = [];
const subscription = await GenerationJobManager.subscribe(
streamId,
(event) => received.push(event),
undefined,
undefined,
{ skipBufferReplay: true },
);
expect(subscription).not.toBeNull();
expect(received.length).toBe(0);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
await replicaAJobStore.destroy();
});
});
describeRedis('Sequential Event Ordering (Redis)', () => {
/**
* These tests verify that events are delivered in strict sequential order
* when using Redis mode. This is critical because:
* 1. LLM streaming tokens must arrive in order for coherent output
* 2. Tool call argument deltas must be concatenated in order
* 3. Run step events must precede their deltas
*
* The fix: emitChunk now awaits Redis publish to ensure ordered delivery.
*/
test('should maintain strict order for rapid sequential emits', async () => {
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `order-rapid-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const receivedIndices: number[] = [];
const subscription = await GenerationJobManager.subscribe(streamId, (event) => {
const data = event as { event: string; data: { index: number } };
if (data.event === 'test') {
receivedIndices.push(data.data.index);
}
});
await new Promise((resolve) => setTimeout(resolve, 100));
// Emit 30 events rapidly - with await, they must arrive in order
for (let i = 0; i < 30; i++) {
await GenerationJobManager.emitChunk(streamId, {
event: 'test',
data: { index: i },
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
// Verify all events arrived in correct order
expect(receivedIndices.length).toBe(30);
for (let i = 0; i < 30; i++) {
expect(receivedIndices[i]).toBe(i);
}
subscription?.unsubscribe();
await GenerationJobManager.destroy();
});
test('should maintain order for tool call argument deltas', async () => {
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `tool-args-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const receivedArgs: string[] = [];
const subscription = await GenerationJobManager.subscribe(streamId, (event) => {
const data = event as {
event: string;
data: { delta: { tool_calls: { args: string }[] } };
};
if (data.event === 'on_run_step_delta') {
receivedArgs.push(data.data.delta.tool_calls[0].args);
}
});
await new Promise((resolve) => setTimeout(resolve, 100));
// Simulate streaming JSON args: {"code": "print('hello')"}
const argChunks = ['{"', 'code', '": "', 'print', "('", 'hello', "')", '"}'];
for (const chunk of argChunks) {
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step_delta',
data: {
id: 'step-1',
delta: {
type: 'tool_calls',
tool_calls: [{ index: 0, args: chunk }],
},
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
// This was the original bug - args would arrive scrambled without await
expect(receivedArgs).toEqual(argChunks);
expect(receivedArgs.join('')).toBe(`{"code": "print('hello')"}`);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
});
test('should maintain order: on_run_step before on_run_step_delta', async () => {
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId = `step-order-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const receivedEvents: string[] = [];
const subscription = await GenerationJobManager.subscribe(streamId, (event) => {
const data = event as { event: string };
receivedEvents.push(data.event);
});
await new Promise((resolve) => setTimeout(resolve, 100));
// Emit in correct order: step first, then deltas
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step',
data: { id: 'step-1', type: 'tool_calls', index: 0 },
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step_delta',
data: {
id: 'step-1',
delta: { type: 'tool_calls', tool_calls: [{ args: '{' }] },
},
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step_delta',
data: {
id: 'step-1',
delta: { type: 'tool_calls', tool_calls: [{ args: '}' }] },
},
});
await GenerationJobManager.emitChunk(streamId, {
event: 'on_run_step_completed',
data: { id: 'step-1', result: { content: '{}' } },
});
await new Promise((resolve) => setTimeout(resolve, 300));
// Verify ordering: step -> deltas -> completed
expect(receivedEvents).toEqual([
'on_run_step',
'on_run_step_delta',
'on_run_step_delta',
'on_run_step_completed',
]);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
});
test('should not block other streams when awaiting emitChunk', async () => {
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure(services);
GenerationJobManager.initialize();
const streamId1 = `concurrent-1-${Date.now()}`;
const streamId2 = `concurrent-2-${Date.now()}`;
await GenerationJobManager.createJob(streamId1, 'user-1');
await GenerationJobManager.createJob(streamId2, 'user-2');
const stream1Events: number[] = [];
const stream2Events: number[] = [];
const sub1 = await GenerationJobManager.subscribe(streamId1, (event) => {
const data = event as { event: string; data: { index: number } };
if (data.event === 'test') {
stream1Events.push(data.data.index);
}
});
const sub2 = await GenerationJobManager.subscribe(streamId2, (event) => {
const data = event as { event: string; data: { index: number } };
if (data.event === 'test') {
stream2Events.push(data.data.index);
}
});
await new Promise((resolve) => setTimeout(resolve, 100));
// Emit to both streams concurrently (simulating two LLM responses)
const emitPromises: Promise<void>[] = [];
for (let i = 0; i < 10; i++) {
emitPromises.push(
GenerationJobManager.emitChunk(streamId1, {
event: 'test',
data: { index: i },
}),
);
emitPromises.push(
GenerationJobManager.emitChunk(streamId2, {
event: 'test',
data: { index: i * 100 },
}),
);
}
await Promise.all(emitPromises);
await new Promise((resolve) => setTimeout(resolve, 300));
// Each stream should have all events, in order within their stream
expect(stream1Events.length).toBe(10);
expect(stream2Events.length).toBe(10);
// Verify each stream's internal order
for (let i = 0; i < 10; i++) {
expect(stream1Events[i]).toBe(i);
expect(stream2Events[i]).toBe(i * 100);
}
sub1?.unsubscribe();
sub2?.unsubscribe();
await GenerationJobManager.destroy();
});
});
describe('Race Condition: Events Before Subscriber Ready', () => {
/**
* These tests verify the fix for the race condition where early events
* (like the 'created' event at seq 0) are lost because the Redis SUBSCRIBE
* command hasn't completed when events are published.
*
* Symptom: "[RedisEventTransport] Stream <id>: timeout waiting for seq 0"
* followed by truncated responses in the UI.
*
* Root cause: RedisEventTransport.subscribe() fired Redis SUBSCRIBE as
* fire-and-forget. GenerationJobManager set hasSubscriber=true immediately,
* disabling the earlyEventBuffer before Redis was actually listening.
*
* Fix: subscribe() now returns a `ready` promise that resolves when the
* Redis subscription is confirmed. earlyEventBuffer stays active until then.
*/
test('should buffer and replay events emitted before subscribe (in-memory)', async () => {
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `early-buf-inmem-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await manager.emitChunk(streamId, {
created: true,
message: { text: 'hello' },
streamId,
} as CreatedEvent);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'First chunk' } } },
});
const receivedEvents: unknown[] = [];
const subscription = await manager.subscribe(streamId, (event: unknown) =>
receivedEvents.push(event),
);
await new Promise((resolve) => setTimeout(resolve, 50));
expect(receivedEvents.length).toBe(2);
expect((receivedEvents[0] as Record<string, unknown>).created).toBe(true);
subscription?.unsubscribe();
await manager.destroy();
});
testRedis('should buffer and replay events emitted before subscribe (Redis)', async () => {
const manager = new GenerationJobManagerClass();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
manager.configure(services);
manager.initialize();
const streamId = `early-buf-redis-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await manager.emitChunk(streamId, {
created: true,
message: { text: 'hello' },
streamId,
} as CreatedEvent);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'First' } } },
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: ' chunk' } } },
});
const receivedEvents: unknown[] = [];
const subscription = await manager.subscribe(streamId, (event: unknown) =>
receivedEvents.push(event),
);
await new Promise((resolve) => setTimeout(resolve, 100));
expect(receivedEvents.length).toBe(3);
expect((receivedEvents[0] as Record<string, unknown>).created).toBe(true);
expect(
((receivedEvents[1] as Record<string, unknown>).data as Record<string, unknown>).delta,
).toBeDefined();
subscription?.unsubscribe();
await manager.destroy();
});
testRedis(
'should not lose events when emitting before and after subscribe (Redis)',
async () => {
const manager = new GenerationJobManagerClass();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
manager.configure(services);
manager.initialize();
const streamId = `no-loss-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await manager.emitChunk(streamId, {
created: true,
message: { text: 'hello' },
streamId,
} as CreatedEvent);
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: { id: 'step-1', type: 'message_creation', index: 0 },
});
const receivedEvents: unknown[] = [];
const subscription = await manager.subscribe(streamId, (event: unknown) =>
receivedEvents.push(event),
);
await new Promise((resolve) => setTimeout(resolve, 100));
for (let i = 0; i < 10; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `word${i} ` } },
index: i,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
expect(receivedEvents.length).toBe(12);
expect((receivedEvents[0] as Record<string, unknown>).created).toBe(true);
expect((receivedEvents[1] as Record<string, unknown>).event).toBe('on_run_step');
for (let i = 0; i < 10; i++) {
expect((receivedEvents[i + 2] as Record<string, unknown>).event).toBe('on_message_delta');
}
subscription?.unsubscribe();
await manager.destroy();
},
);
testRedis('RedisEventTransport.subscribe() should return a ready promise', async () => {
const subscriber = (ioredisClient as unknown as { duplicate: () => unknown }).duplicate();
const transport = new RedisEventTransport(ioredisClient as never, subscriber as never);
const streamId = `ready-promise-${Date.now()}`;
const result = transport.subscribe(streamId, {
onChunk: () => {},
});
expect(result.ready).toBeDefined();
expect(result.ready).toBeInstanceOf(Promise);
await result.ready;
result.unsubscribe();
transport.destroy();
(subscriber as { disconnect: () => void }).disconnect();
});
test('InMemoryEventTransport.subscribe() should not have a ready promise', () => {
const transport = new InMemoryEventTransport();
const streamId = `no-ready-${Date.now()}`;
const result = transport.subscribe(streamId, {
onChunk: () => {},
});
expect(result.ready).toBeUndefined();
result.unsubscribe();
transport.destroy();
});
});
describe('Resume: skipBufferReplay prevents duplication', () => {
/**
* Verifies the fix for duplicated content when navigating away from an
* in-progress conversation and back. Events accumulate in earlyEventBuffer
* while the subscriber is absent. On resume, the sync event delivers all
* accumulated content via aggregatedContent, so buffer replay must be
* skipped to prevent duplication.
*/
test('should NOT replay buffer when skipBufferReplay is true (resume scenario)', async () => {
const manager = createInMemoryManager();
const streamId = `skip-buf-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await setupDisconnectedStream(manager, streamId, 10);
const resumeState = await manager.getResumeState(streamId);
expect(resumeState).not.toBeNull();
const resumeEvents: ServerSentEvent[] = [];
const sub2 = await manager.subscribe(
streamId,
(event) => resumeEvents.push(event),
undefined,
undefined,
{ skipBufferReplay: true },
);
await new Promise((resolve) => setTimeout(resolve, 20));
expect(resumeEvents.length).toBe(0);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: ' Live!' } },
},
});
await new Promise((resolve) => setTimeout(resolve, 20));
expect(resumeEvents.length).toBe(1);
expect((resumeEvents[0] as StreamEvent).event).toBe('on_message_delta');
sub2?.unsubscribe();
await manager.destroy();
});
test('should include emitted title event in resume state', async () => {
const manager = createInMemoryManager();
const streamId = `title-resume-${Date.now()}`;
await manager.createJob(streamId, 'user-1', streamId);
const titleEvent = {
event: 'title',
data: {
conversationId: streamId,
title: 'Resumed Title',
},
} satisfies ServerSentEvent;
await manager.emitChunk(streamId, titleEvent);
const resumeState = await manager.getResumeState(streamId);
expect(resumeState?.titleEvent).toEqual(titleEvent);
await manager.destroy();
});
test('should replay buffer by default when no options are passed', async () => {
const manager = createInMemoryManager();
const streamId = `replay-buf-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1Events: ServerSentEvent[] = [];
const sub1 = await manager.subscribe(streamId, (event) => sub1Events.push(event));
await new Promise((resolve) => setTimeout(resolve, 10));
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'buffered' } },
},
});
const sub2Events: ServerSentEvent[] = [];
const sub2 = await manager.subscribe(streamId, (event) => sub2Events.push(event));
await new Promise((resolve) => setTimeout(resolve, 20));
expect(sub2Events.length).toBe(1);
expect((sub2Events[0] as StreamEvent).event).toBe('on_message_delta');
sub2?.unsubscribe();
await manager.destroy();
});
test('should clear earlyEventBuffer even when skipping replay (no memory leak)', async () => {
const manager = createInMemoryManager();
const streamId = `buf-clear-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'buf1' } } },
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'buf2' } } },
});
const sub2Events: ServerSentEvent[] = [];
const sub2 = await manager.subscribe(
streamId,
(event) => sub2Events.push(event),
undefined,
undefined,
{ skipBufferReplay: true },
);
await new Promise((resolve) => setTimeout(resolve, 20));
expect(sub2Events.length).toBe(0);
sub2?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'new-event' } } },
});
const sub3Events: ServerSentEvent[] = [];
const sub3 = await manager.subscribe(streamId, (event) => sub3Events.push(event));
await new Promise((resolve) => setTimeout(resolve, 20));
expect(sub3Events.length).toBe(1);
const event = sub3Events[0] as {
event: string;
data: { delta: { content: { text: string } } };
};
expect(event.data.delta.content.text).toBe('new-event');
sub3?.unsubscribe();
await manager.destroy();
});
test('should handle multiple disconnect/reconnect cycles with skipBufferReplay', async () => {
const manager = createInMemoryManager();
const streamId = `multi-reconnect-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 10));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'initial' } } },
});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'buffered-1' } } },
});
const resumeState1 = await manager.getResumeState(streamId);
expect(resumeState1).not.toBeNull();
const sub2Events: ServerSentEvent[] = [];
const sub2 = await manager.subscribe(
streamId,
(event) => sub2Events.push(event),
undefined,
undefined,
{ skipBufferReplay: true },
);
await new Promise((resolve) => setTimeout(resolve, 10));
expect(sub2Events.length).toBe(0);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'live-1' } } },
});
await new Promise((resolve) => setTimeout(resolve, 10));
expect(sub2Events.length).toBe(1);
sub2?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'buffered-2' } } },
});
const resumeState2 = await manager.getResumeState(streamId);
expect(resumeState2).not.toBeNull();
const sub3Events: ServerSentEvent[] = [];
const sub3 = await manager.subscribe(
streamId,
(event) => sub3Events.push(event),
undefined,
undefined,
{ skipBufferReplay: true },
);
await new Promise((resolve) => setTimeout(resolve, 10));
expect(sub3Events.length).toBe(0);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'live-2' } } },
});
await new Promise((resolve) => setTimeout(resolve, 10));
expect(sub3Events.length).toBe(1);
sub3?.unsubscribe();
await manager.destroy();
});
testRedis('should NOT replay buffer when skipBufferReplay is true (Redis)', async () => {
const manager = createRedisManager();
const streamId = `skip-buf-redis-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await setupDisconnectedStream(manager, streamId, 100);
const resumeState = await manager.getResumeState(streamId);
expect(resumeState).not.toBeNull();
expect(resumeState!.aggregatedContent?.length).toBeGreaterThan(0);
const resumeEvents: ServerSentEvent[] = [];
const sub2 = await manager.subscribe(
streamId,
(event) => resumeEvents.push(event),
undefined,
undefined,
{ skipBufferReplay: true },
);
await new Promise((resolve) => setTimeout(resolve, 200));
expect(resumeEvents.length).toBe(0);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: ' Live!' } },
},
});
await new Promise((resolve) => setTimeout(resolve, 200));
expect(resumeEvents.length).toBe(1);
expect((resumeEvents[0] as StreamEvent).event).toBe('on_message_delta');
sub2?.unsubscribe();
await manager.destroy();
});
testRedis('should not re-buffer detached events after first attachment (Redis)', async () => {
/**
* After the first attachment, the durable chunk log owns recovery for
* detached events; re-buffering them locally grew without bound for
* long detached generations. A late subscriber gets live events only,
* and resume reconstructs the detached content from the chunk log.
*/
const manager = createRedisManager();
const streamId = `no-rebuffer-redis-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 100));
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await new Promise((resolve) => setTimeout(resolve, 100));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 100));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'detached-redis' } },
},
});
await new Promise((resolve) => setTimeout(resolve, 100));
expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);
const sub2Events: ServerSentEvent[] = [];
const sub2 = await manager.subscribe(streamId, (event) => sub2Events.push(event));
await new Promise((resolve) => setTimeout(resolve, 200));
expect(sub2Events.length).toBe(0);
const resumeState = await manager.getResumeState(streamId);
expect(JSON.stringify(resumeState?.aggregatedContent ?? [])).toContain('detached-redis');
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: ' live' } },
},
});
await new Promise((resolve) => setTimeout(resolve, 200));
expect(sub2Events.length).toBe(1);
expect((sub2Events[0] as StreamEvent).event).toBe('on_message_delta');
sub2?.unsubscribe();
await manager.destroy();
});
});
describe('Early event buffer bounds', () => {
/**
* Regression tests for a production incident: a model streamed a malformed
* 150k-character tool argument for ~26 minutes after the browser
* disconnected (~40 events/sec, ~58,800 publications). Every event was
* retained in earlyEventBuffer, so heap and GC cost climbed for the whole
* detached run.
*/
async function forceEarlyBufferOverflow(
manager: GenerationJobManagerClass,
streamId: string,
): Promise<void> {
const runtime = (
manager as unknown as {
runtimeState: Map<string, unknown>;
}
).runtimeState.get(streamId)!;
await (
manager as unknown as {
overflowEarlyEventBuffer: (id: string, runtime: unknown) => Promise<void>;
}
).overflowEarlyEventBuffer(streamId, runtime);
}
testRedis(
'detached generation keeps the local buffer empty after first attachment (Redis)',
async () => {
const manager = createRedisManager();
const streamId = `detached-flat-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 100));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 100));
for (let i = 0; i < 200; i++) {
await manager.emitChunk(streamId, {
event: 'on_run_step_delta',
data: {
id: 'step-1',
delta: { type: 'tool_call_delta', args: `"table_${i}", ` },
},
});
}
const stats = manager.getRuntimeStats();
expect(stats.earlyBufferedEvents).toBe(0);
expect(stats.earlyBufferedBytes).toBe(0);
await manager.destroy();
},
);
test('discards and closes the buffer when the byte budget is exceeded (never attached)', async () => {
const manager = createInMemoryManager();
const streamId = `buf-byte-cap-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const bigText = 'x'.repeat(2 * 1024 * 1024);
for (let i = 0; i < 5; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: bigText } },
},
});
}
const stats = manager.getRuntimeStats();
expect(stats.earlyBufferedEvents).toBe(0);
expect(stats.earlyBufferedBytes).toBe(0);
const overflow = (await manager.getJob(streamId))?.metadata.earlyBufferOverflow;
expect(overflow?.droppedEvents).toBe(4);
expect(overflow?.droppedBytes).toBeGreaterThan(8 * 1024 * 1024);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'after-overflow' } },
},
});
expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);
const errors: string[] = [];
const events: ServerSentEvent[] = [];
const sub = await manager.subscribe(
streamId,
(event) => events.push(event),
undefined,
(error) => errors.push(error),
);
await new Promise((resolve) => setTimeout(resolve, 20));
/** A non-resume attachment cannot be made whole once the buffer was
* discarded, so it is closed with the reconnect signal; the client then
* re-attaches with resume=true and syncs from snapshot state. */
expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
expect(events).toEqual([]);
sub?.unsubscribe();
await manager.destroy();
});
test('replays a completed result after the overflow reconnect was forced', async () => {
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
const streamId = `overflow-complete-reconnect-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(manager, streamId);
const reconnectErrors: string[] = [];
await manager.subscribe(
streamId,
() => {},
undefined,
(error) => reconnectErrors.push(error),
);
await new Promise((resolve) => setTimeout(resolve, 20));
expect(reconnectErrors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
const finalEvent = {
final: true,
conversation: { conversationId: streamId },
responseMessage: { text: 'complete' },
} as ServerSentEvent;
await manager.emitDone(streamId, finalEvent);
await manager.completeJob(streamId);
const terminalEvents: ServerSentEvent[] = [];
const resumed = await manager.subscribeWithResume(
streamId,
() => {},
(event) => terminalEvents.push(event),
);
resumed.subscription?.activate();
await new Promise((resolve) => setTimeout(resolve, 20));
expect(terminalEvents).toEqual([finalEvent]);
resumed.subscription?.unsubscribe();
await manager.destroy();
});
test('reconciles a generation replaced while recovery settles', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `overflow-replaced-during-settlement-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const originalCreatedAt = (await jobStore.getJob(streamId))!.createdAt;
await forceEarlyBufferOverflow(manager, streamId);
jest.spyOn(jobStore, 'settleEarlyBufferRecovery').mockImplementationOnce(async () => {
await jobStore.createJob(streamId, 'user-1');
return false;
});
const errors: string[] = [];
const result = await manager.subscribeWithResume(
streamId,
() => {},
undefined,
(error) => errors.push(error),
{ expectedCreatedAt: originalCreatedAt },
);
expect(result.subscription).toBeNull();
expect(errors).not.toContain(GENERATION_RECOVERY_FAILED_ERROR);
expect((await jobStore.getJob(streamId))!.createdAt).toBeGreaterThan(originalCreatedAt);
await manager.destroy();
});
test('reconciles a generation replaced while a recovery error settles', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `overflow-replaced-during-error-settlement-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const originalCreatedAt = (await jobStore.getJob(streamId))!.createdAt;
await forceEarlyBufferOverflow(manager, streamId);
jest
.spyOn(manager, 'getResumeState')
.mockRejectedValueOnce(new Error('snapshot unavailable'));
jest.spyOn(jobStore, 'settleEarlyBufferRecovery').mockImplementationOnce(async () => {
await jobStore.createJob(streamId, 'user-1');
return false;
});
const errors: string[] = [];
const result = await manager.subscribeWithResume(
streamId,
() => {},
undefined,
(error) => errors.push(error),
{ expectedCreatedAt: originalCreatedAt },
);
expect(result.subscription).toBeNull();
expect(errors).not.toContain(GENERATION_RECOVERY_FAILED_ERROR);
expect((await jobStore.getJob(streamId))!.createdAt).toBeGreaterThan(originalCreatedAt);
await manager.destroy();
});
test('does not overwrite recovery settlement when the overflow owner finishes flushing', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
const streamId = `overflow-finalize-settlement-race-${Date.now()}`;
const job = await manager.createJob(streamId, 'user-1');
const originalFinalize = jobStore.finalizeEarlyBufferOverflow.bind(jobStore);
jest
.spyOn(jobStore, 'finalizeEarlyBufferOverflow')
.mockImplementationOnce(async (id, createdAt, overflowId, finalizedOverflow) => {
await jobStore.settleEarlyBufferRecovery(id, createdAt, overflowId, {
recoveryMethod: 'snapshot',
recoveryOutcome: 'failed',
recoveryCompletedAt: Date.now(),
recoveryFailureReason: 'overflow_marker_persistence_failed',
});
return originalFinalize(id, createdAt, overflowId, finalizedOverflow);
});
await expect(forceEarlyBufferOverflow(manager, streamId)).rejects.toThrow(
GENERATION_RECOVERY_FAILED_ERROR,
);
expect((await jobStore.getJob(streamId))?.earlyBufferOverflow).toMatchObject({
recoveryOutcome: 'failed',
recoveryFailureReason: 'overflow_marker_persistence_failed',
});
expect((await jobStore.getJob(streamId))?.createdAt).toBe(job.createdAt);
await manager.destroy();
});
test('coalesces pending overflow waiters and backs off their store read', async () => {
jest.useFakeTimers();
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({ jobStore, eventTransport: new InMemoryEventTransport(), isRedis: false });
manager.initialize();
const streamId = `overflow-shared-waiter-${Date.now()}`;
const job = await manager.createJob(streamId, 'user-1');
const pendingOverflow = {
id: 'pending-overflow',
occurredAt: Date.now(),
durableEvents: 1,
droppedEvents: 1,
droppedBytes: 1,
persistencePending: true,
};
await jobStore.updateJob(streamId, { earlyBufferOverflow: pendingOverflow }, job.createdAt);
const pendingJob = (await jobStore.getJob(streamId))!;
const getJob = jest.spyOn(jobStore, 'getJob');
const waitForFinalized = (
manager as unknown as {
waitForFinalizedEarlyBufferOverflow: (
id: string,
data: typeof pendingJob,
) => Promise<typeof pendingJob | null>;
}
).waitForFinalizedEarlyBufferOverflow.bind(manager);
const first = waitForFinalized(streamId, pendingJob);
const second = waitForFinalized(streamId, pendingJob);
await jest.advanceTimersByTimeAsync(99);
expect(getJob).not.toHaveBeenCalled();
await jobStore.updateJob(
streamId,
{ earlyBufferOverflow: { ...pendingOverflow, persistencePending: false } },
job.createdAt,
);
await jest.advanceTimersByTimeAsync(1);
await expect(Promise.all([first, second])).resolves.toEqual([
expect.objectContaining({
earlyBufferOverflow: expect.objectContaining({ persistencePending: false }),
}),
expect.objectContaining({
earlyBufferOverflow: expect.objectContaining({ persistencePending: false }),
}),
]);
expect(getJob).toHaveBeenCalledTimes(1);
await manager.destroy();
jest.useRealTimers();
});
test('fences publication only until generation-wide subscriber admission', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const eventTransport = new InMemoryEventTransport();
let releaseAppend!: (appended: boolean) => void;
const pendingAppend = new Promise<boolean>((resolve) => {
releaseAppend = resolve;
});
const append = jest.spyOn(jobStore, 'appendChunk').mockReturnValueOnce(pendingAppend);
jest.spyOn(jobStore, 'hasSubscriberAttached').mockResolvedValue(true);
const publish = jest.spyOn(eventTransport, 'emitChunk');
const manager = new GenerationJobManagerClass();
manager.configure({ jobStore, eventTransport, isRedis: true });
manager.initialize();
const streamId = `detached-append-before-publish-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const emission = manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'durable first' } } },
});
await Promise.resolve();
expect(publish).not.toHaveBeenCalled();
releaseAppend(true);
await emission;
expect(publish).toHaveBeenCalledTimes(1);
append.mockReturnValueOnce(new Promise<boolean>(() => {}));
await expect(
manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'publish without append wait' } } },
}),
).resolves.toBeUndefined();
expect(publish).toHaveBeenCalledTimes(2);
expect(jobStore.hasSubscriberAttached).toHaveBeenCalledTimes(1);
await manager.destroy();
});
test('validates an in-memory HITL snapshot with the event frontier', async () => {
const manager = createInMemoryManager();
const streamId = `overflow-hitl-memory-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const createdAt = (await manager.getJob(streamId))!.createdAt;
await forceEarlyBufferOverflow(manager, streamId);
manager.setContentParts(
streamId,
[{ type: 'text', text: 'complete snapshot' }] as never,
createdAt,
);
const getRuntimeSpy = jest.spyOn(
manager as unknown as {
getOrCreateRuntimeState: (...args: unknown[]) => Promise<unknown>;
},
'getOrCreateRuntimeState',
);
await expect(
manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),
).resolves.toMatchObject({ aggregatedContent: expect.any(Array) });
expect(getRuntimeSpy).toHaveBeenCalledWith(streamId, expect.objectContaining({ createdAt }));
expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).toMatchObject({
durableEvents: 0,
recoveryOutcome: 'success',
});
await manager.destroy();
});
test('rejects a durable failed outcome before exposing HITL state', async () => {
const manager = createInMemoryManager();
const streamId = `overflow-hitl-failed-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const createdAt = (await manager.getJob(streamId))!.createdAt;
await forceEarlyBufferOverflow(manager, streamId);
const overflow = (await manager.getJob(streamId))!.metadata.earlyBufferOverflow!;
await (manager.getJobStore() as IJobStoreV2).settleEarlyBufferRecovery(
streamId,
createdAt,
overflow.id,
{
recoveryMethod: 'snapshot',
recoveryOutcome: 'failed',
recoveryCompletedAt: Date.now(),
recoveryFailureReason: 'snapshot_missing',
},
);
await expect(
manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),
).rejects.toThrow(GENERATION_RECOVERY_FAILED_ERROR);
await manager.destroy();
});
test('contains subscriber lease lookup failure during terminal cleanup', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
const streamId = `overflow-lease-read-failure-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(manager, streamId);
jest.spyOn(jobStore, 'hasActiveSubscriber').mockRejectedValueOnce(new Error('unavailable'));
await expect(manager.completeJob(streamId)).resolves.toBe(true);
expect((await manager.getJob(streamId))?.status).toBe('complete');
expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(
'recoveryOutcome',
);
await manager.destroy();
});
test('leaves recovery unresolved when terminal lifecycle lookup fails', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
const streamId = `overflow-lifecycle-read-failure-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(manager, streamId);
const originalGetJob = jobStore.getJob.bind(jobStore);
jest
.spyOn(jobStore, 'getJob')
.mockImplementationOnce(originalGetJob)
.mockRejectedValueOnce(new Error('unavailable'))
.mockImplementation(originalGetJob);
await expect(manager.completeJob(streamId)).resolves.toBe(true);
expect((await manager.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(
'recoveryOutcome',
);
await manager.destroy();
});
test('removes in-memory capture handlers when another attachment wins recovery', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
manager.initialize();
const streamId = `overflow-recovery-loser-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(manager, streamId);
const originalSettle = jobStore.settleEarlyBufferRecovery.bind(jobStore);
jest
.spyOn(jobStore, 'settleEarlyBufferRecovery')
.mockImplementationOnce(async (id, createdAt, recoveryId) => {
await originalSettle(id, createdAt, recoveryId, {
recoveryMethod: 'snapshot',
recoveryOutcome: 'not_required',
recoveryCompletedAt: Date.now(),
});
return false;
});
const result = await manager.subscribeWithResume(streamId, () => {});
expect(result.subscription).toBeNull();
const runtime = (
manager as unknown as {
runtimeState: Map<string, { resumeCaptureHandlers: Set<unknown> }>;
}
).runtimeState.get(streamId);
expect(runtime?.resumeCaptureHandlers.size).toBe(0);
await manager.destroy();
});
testRedis('redirects a post-overflow first attachment to resume recovery (Redis)', async () => {
const manager = createRedisManager();
const streamId = `overflow-redirect-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'durable-overflow-content' } },
},
});
await forceEarlyBufferOverflow(manager, streamId);
expect(manager.getRuntimeStats().earlyBufferedEvents).toBe(0);
const errors: string[] = [];
const sub = await manager.subscribe(
streamId,
() => {},
undefined,
(error) => errors.push(error),
);
await new Promise((resolve) => setTimeout(resolve, 100));
expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
sub?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 100));
/** The resume path the client falls back to reconstructs the
* discarded output from the durable chunk log. */
const { subscription, resumeState } = await manager.subscribeWithResume(streamId, () => {});
expect(JSON.stringify(resumeState?.aggregatedContent ?? [])).toContain(
'durable-overflow-content',
);
subscription?.unsubscribe();
const recoveredJob = await manager.getJob(streamId);
expect(recoveredJob?.metadata.earlyBufferOverflow).toMatchObject({
recoveryMethod: 'redis',
recoveryOutcome: 'success',
});
await manager.destroy();
});
testRedis('refreshes an overflow marker before activating a resume attachment', async () => {
const owner = createRedisManager();
const streamId = `overflow-marker-attachment-race-${Date.now()}`;
await owner.createJob(streamId, 'user-1');
await owner.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await owner.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'durable-before-marker' } },
},
});
await forceEarlyBufferOverflow(owner, streamId);
const replica = createRedisManager();
const replicaStore = replica.getJobStore();
const originalGetJob = replicaStore.getJob.bind(replicaStore);
let jobReads = 0;
jest.spyOn(replicaStore, 'getJob').mockImplementation(async (id) => {
const job = await originalGetJob(id);
jobReads++;
if (jobReads <= 4 && job != null) {
return { ...job, earlyBufferOverflow: undefined };
}
return job;
});
const result = await replica.subscribeWithResume(streamId, () => {});
expect(jobReads).toBeGreaterThan(4);
expect(JSON.stringify(result.resumeState?.aggregatedContent ?? [])).toContain(
'durable-before-marker',
);
expect((await originalGetJob(streamId))?.earlyBufferOverflow).toMatchObject({
recoveryMethod: 'redis',
recoveryOutcome: 'success',
});
result.subscription?.unsubscribe();
const followup = await replica.subscribeWithResume(streamId, () => {});
expect(JSON.stringify(followup.resumeState?.aggregatedContent ?? [])).toContain(
'durable-before-marker',
);
followup.subscription?.unsubscribe();
await Promise.all([owner.destroy(), replica.destroy()]);
});
testRedis(
'reconstructs more than 5,000 pre-attachment events across replica reconnects (Redis)',
async () => {
const owner = createRedisManager();
const streamId = `overflow-cross-replica-${Date.now()}`;
await owner.createJob(streamId, 'user-1');
const createdAt = (await owner.getJob(streamId))!.createdAt;
const firstReplica = createRedisManager();
/** Cache a pre-overflow runtime so resume must refresh the later
* durable marker rather than trust process-local state. */
await firstReplica.getJob(streamId);
/** Created envelopes are published but not durably appendable, so they
* must not advance the durable recovery frontier. */
await owner.emitChunk(streamId, {
created: true,
message: { text: 'hello' },
streamId,
} as CreatedEvent);
await owner.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
for (let i = 0; i < 5_001; i++) {
await firstReplica.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: `${i},` } },
},
});
}
const first = await firstReplica.subscribeWithResume(streamId, () => {});
expect(first.resumeState?.aggregatedContent).toHaveLength(1);
expect(JSON.stringify(first.resumeState?.aggregatedContent)).toContain('5000,');
first.subscription?.activate();
first.subscription?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 50));
const firstAttachedAt = await ioredisClient!.hget(
`stream:{${streamId}}:job`,
'firstSubscriberAttachedAt',
);
expect(firstAttachedAt).not.toBeNull();
expect(
await (firstReplica.getJobStore() as IJobStoreV2).hasActiveSubscriber(
streamId,
createdAt,
Date.now(),
),
).toBe(false);
await owner.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'after-disconnect' } },
},
});
const secondReplica = createRedisManager();
const second = await secondReplica.subscribeWithResume(streamId, () => {});
expect(JSON.stringify(second.resumeState?.aggregatedContent)).toContain('after-disconnect');
await new Promise((resolve) => setTimeout(resolve, 50));
expect(
await (secondReplica.getJobStore() as IJobStoreV2).hasActiveSubscriber(
streamId,
createdAt,
Date.now(),
),
).toBe(true);
second.subscription?.unsubscribe();
expect(
await ioredisClient!.hget(`stream:{${streamId}}:job`, 'firstSubscriberAttachedAt'),
).toBe(firstAttachedAt);
const recoveredJob = await secondReplica.getJob(streamId);
expect(recoveredJob?.metadata.earlyBufferOverflow).toMatchObject({
durableEvents: 5_002,
recoveryMethod: 'redis',
recoveryOutcome: 'success',
});
await Promise.all([owner.destroy(), firstReplica.destroy(), secondReplica.destroy()]);
},
60_000,
);
testRedis(
'terminalizes an overflow when durable reconstruction is missing (Redis)',
async () => {
const owner = createRedisManager();
const streamId = `overflow-missing-durable-${Date.now()}`;
await owner.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(owner, streamId);
await ioredisClient!.del(`stream:{${streamId}}:chunks`);
const replica = createRedisManager();
const errors: string[] = [];
const result = await replica.subscribeWithResume(
streamId,
() => {},
undefined,
(error) => errors.push(error),
);
expect(result.subscription).toBeNull();
expect(errors).toEqual([GENERATION_RECOVERY_FAILED_ERROR]);
const failedJob = await replica.getJob(streamId);
expect(failedJob?.status).toBe('error');
expect(failedJob?.error).toBe(GENERATION_RECOVERY_FAILED_ERROR);
expect(failedJob?.metadata.earlyBufferOverflow).toMatchObject({
recoveryMethod: 'redis',
recoveryOutcome: 'failed',
recoveryFailureReason: 'durable_state_missing',
});
await Promise.all([owner.destroy(), replica.destroy()]);
},
);
testRedis('replays a durable terminal payload with unresolved overflow (Redis)', async () => {
const owner = createRedisManager();
const streamId = `overflow-durable-terminal-${Date.now()}`;
const job = await owner.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(owner, streamId);
const claim = await owner.claimTerminalJob(streamId, 'complete', undefined, job.createdAt, {
persistencePending: true,
});
expect(claim).not.toBeNull();
const finalEvent = {
final: true,
conversation: { conversationId: streamId },
responseMessage: { text: 'complete' },
} as ServerSentEvent;
await owner.publishTerminalClaim(claim!, finalEvent);
expect((await owner.getJob(streamId))?.metadata.earlyBufferOverflow).not.toHaveProperty(
'recoveryOutcome',
);
const replica = createRedisManager();
const terminalEvents: ServerSentEvent[] = [];
const errors: string[] = [];
const resumed = await replica.subscribeWithResume(
streamId,
() => {},
(event) => terminalEvents.push(event),
(error) => errors.push(error),
{ expectedCreatedAt: job.createdAt },
);
resumed.subscription?.activate();
await new Promise((resolve) => setTimeout(resolve, 20));
expect(terminalEvents).toEqual([finalEvent]);
expect(errors).toEqual([]);
expect((await replica.getJob(streamId))?.status).toBe('complete');
resumed.subscription?.unsubscribe();
await Promise.all([owner.destroy(), replica.destroy()]);
});
testRedis('validates overflow recovery before exposing HITL resume state', async () => {
const manager = createRedisManager();
const streamId = `overflow-hitl-validation-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(manager, streamId);
const createdAt = (await manager.getJob(streamId))!.createdAt;
await ioredisClient!.del(`stream:{${streamId}}:chunks`);
await expect(
manager.getResumeState(streamId, createdAt, { validateEarlyBufferRecovery: true }),
).rejects.toThrow(GENERATION_RECOVERY_FAILED_ERROR);
expect((await manager.getJob(streamId))?.status).toBe('error');
await manager.destroy();
});
testRedis(
'terminalizes a durable failed outcome left running by a crashed replica',
async () => {
const owner = createRedisManager();
const streamId = `overflow-failed-outcome-${Date.now()}`;
await owner.createJob(streamId, 'user-1');
await forceEarlyBufferOverflow(owner, streamId);
const overflowJob = await owner.getJob(streamId);
const overflow = overflowJob?.metadata.earlyBufferOverflow;
expect(overflow).toBeDefined();
await (owner.getJobStore() as IJobStoreV2).settleEarlyBufferRecovery(
streamId,
overflowJob!.createdAt,
overflow!.id,
{
recoveryMethod: 'redis',
recoveryOutcome: 'failed',
recoveryCompletedAt: Date.now(),
recoveryFailureReason: 'reconstruction_error',
},
);
const replica = createRedisManager();
const errors: string[] = [];
const result = await replica.subscribeWithResume(
streamId,
() => {},
undefined,
(error) => errors.push(error),
);
expect(result.subscription).toBeNull();
expect(errors).toEqual([GENERATION_RECOVERY_FAILED_ERROR]);
expect((await replica.getJob(streamId))?.status).toBe('error');
await Promise.all([owner.destroy(), replica.destroy()]);
},
);
test('buffers detached events until the cap in in-memory mode', async () => {
const manager = createInMemoryManager();
const streamId = `buf-below-cap-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await setupDisconnectedStream(manager, streamId, 10);
const stats = manager.getRuntimeStats();
expect(stats.earlyBufferedEvents).toBe(2);
expect(stats.earlyBufferedBytes).toBeGreaterThan(0);
await manager.destroy();
});
test('caps captured events restored by a resume canceled before activation', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `restore-cap-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await setupDisconnectedStream(manager, streamId, 10);
/** Arm only after the snapshot completes so the gate parks the resume
* in its post-attachment steer reconciliation, the window where
* emissions are captured per-resume instead of buffered. */
let armed = false;
const originalGetResumeState = manager.getResumeState.bind(manager);
jest
.spyOn(manager, 'getResumeState')
.mockImplementation(
async (...args: Parameters<GenerationJobManagerClass['getResumeState']>) => {
const result = await originalGetResumeState(...args);
armed = true;
return result;
},
);
let releaseGate!: () => void;
const gate = new Promise<void>((resolve) => (releaseGate = resolve));
let gateReached!: () => void;
const reached = new Promise<void>((resolve) => (gateReached = resolve));
const originalPeek = jobStore.peekSteers.bind(jobStore);
let gated = true;
jest
.spyOn(jobStore, 'peekSteers')
.mockImplementation(async (...args: Parameters<InMemoryJobStore['peekSteers']>) => {
if (armed && gated) {
gated = false;
gateReached();
await gate;
}
return originalPeek(...args);
});
const resumePromise = manager.subscribeWithResume(streamId, () => {});
await reached;
const bigText = 'z'.repeat(2 * 1024 * 1024);
for (let i = 0; i < 5; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: bigText } },
},
});
}
releaseGate();
const { subscription } = await resumePromise;
expect(subscription).not.toBeNull();
subscription!.unsubscribe();
/** The ~10MB of captured events must not survive restoration. */
const stats = manager.getRuntimeStats();
expect(stats.earlyBufferedEvents).toBe(0);
expect(stats.earlyBufferedBytes).toBe(0);
/** Restoration overflowed, so a non-resume attach takes the redirect. */
const errors: string[] = [];
const probe = await manager.subscribe(
streamId,
() => {},
undefined,
(error) => errors.push(error),
);
await new Promise((resolve) => setTimeout(resolve, 20));
expect(errors).toEqual([TERMINAL_PUBLICATION_RECONNECT_ERROR]);
probe?.unsubscribe();
await manager.destroy();
});
});
describe('Atomic subscribeWithResume', () => {
test('retries a failed first-subscriber lease claim while attached', async () => {
jest.useFakeTimers();
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const originalClaim = jobStore.claimFirstSubscriber.bind(jobStore);
const claim = jest
.spyOn(jobStore, 'claimFirstSubscriber')
.mockRejectedValueOnce(new Error('store unavailable'))
.mockImplementation(originalClaim);
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `subscriber-claim-retry-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const createdAt = (await jobStore.getJob(streamId))!.createdAt;
const subscription = await manager.subscribe(streamId, () => {});
await jest.advanceTimersByTimeAsync(0);
expect(claim).toHaveBeenCalledTimes(1);
const runtime = (
manager as unknown as {
runtimeState: Map<string, { onFirstSubscriberLeaseClaim?: (first: boolean) => void }>;
}
).runtimeState.get(streamId)!;
expect(runtime.onFirstSubscriberLeaseClaim).toBeDefined();
await jest.advanceTimersByTimeAsync(10_000);
expect(claim).toHaveBeenCalledTimes(2);
expect(runtime.onFirstSubscriberLeaseClaim).toBeUndefined();
await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
true,
);
subscription?.unsubscribe();
await manager.destroy();
jest.useRealTimers();
});
test('coalesces subscriber lease renewals while a store write is pending', async () => {
jest.useFakeTimers();
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const originalClaim = jobStore.claimFirstSubscriber.bind(jobStore);
let releaseRenewal!: (claimed: boolean) => void;
const pendingRenewal = new Promise<boolean>((resolve) => {
releaseRenewal = resolve;
});
const claim = jest
.spyOn(jobStore, 'claimFirstSubscriber')
.mockImplementationOnce(originalClaim)
.mockReturnValueOnce(pendingRenewal)
.mockImplementation(originalClaim);
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `subscriber-renewal-coalesce-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const subscription = await manager.subscribe(streamId, () => {});
await jest.advanceTimersByTimeAsync(30_000);
expect(claim).toHaveBeenCalledTimes(2);
releaseRenewal(false);
await jest.advanceTimersByTimeAsync(10_000);
expect(claim).toHaveBeenCalledTimes(3);
const lastCall = claim.mock.calls[2];
expect(lastCall[4] - lastCall[2]).toBe(30_000);
subscription?.unsubscribe();
await manager.destroy();
jest.useRealTimers();
});
test('stops the predecessor lease timer when a durable generation replaces it', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: true,
});
manager.initialize();
const streamId = `subscriber-runtime-replacement-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const subscription = await manager.subscribe(streamId, () => {});
const predecessor = (
manager as unknown as {
runtimeState: Map<string, { subscriberLeaseTimer?: ReturnType<typeof setInterval> }>;
}
).runtimeState.get(streamId)!;
expect(predecessor.subscriberLeaseTimer).toBeDefined();
await jobStore.createJob(streamId, 'user-1');
await manager.getJob(streamId);
expect(predecessor.subscriberLeaseTimer).toBeUndefined();
subscription?.unsubscribe();
await manager.destroy();
});
test('keeps ordinary HITL resume reads on the cached content path', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `hitl-cached-content-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const createdAt = (await jobStore.getJob(streamId))!.createdAt;
const getContentParts = jest.spyOn(jobStore, 'getContentParts');
await manager.getResumeState(streamId, createdAt, {
validateEarlyBufferRecovery: true,
});
expect(getContentParts).toHaveBeenCalledWith(streamId, createdAt, {
durableOnly: false,
});
await manager.destroy();
});
test('renews the subscriber lease after an ambiguous detach failure', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const claim = jest.spyOn(jobStore, 'claimFirstSubscriber');
jest
.spyOn(jobStore, 'detachSubscriber')
.mockRejectedValueOnce(new Error('store unavailable'));
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `subscriber-detach-retry-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const first = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 20));
first?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
const second = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 20));
expect(claim).toHaveBeenCalledTimes(2);
second?.unsubscribe();
await manager.destroy();
});
test('tracks active subscriber groups across local reattachments', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `subscriber-lifecycle-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const createdAt = (await jobStore.getJob(streamId))!.createdAt;
const first = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 20));
await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
true,
);
first?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
false,
);
const second = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 20));
await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
true,
);
second?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await jobStore.claimFirstSubscriber(
streamId,
createdAt,
Date.now(),
'crashed-subscriber',
Date.now() - 1,
);
await expect(jobStore.hasActiveSubscriber(streamId, createdAt, Date.now())).resolves.toBe(
false,
);
await manager.destroy();
});
test('keeps the first-subscriber telemetry claim off the attachment path', async () => {
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 60000 });
let releaseClaim!: (claimed: boolean) => void;
const pendingClaim = new Promise<boolean>((resolve) => {
releaseClaim = resolve;
});
jest.spyOn(jobStore, 'claimFirstSubscriber').mockReturnValue(pendingClaim);
const manager = new GenerationJobManagerClass();
manager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
});
manager.initialize();
const streamId = `nonblocking-attachment-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const attachment = manager.subscribe(streamId, () => {});
const result = await Promise.race([
attachment,
new Promise<'timed_out'>((resolve) => setTimeout(() => resolve('timed_out'), 50)),
]);
expect(result).not.toBe('timed_out');
releaseClaim(true);
await pendingClaim;
if (result !== 'timed_out') {
result?.unsubscribe();
}
await manager.destroy();
});
test('should return empty pendingEvents for pre-snapshot buffer events (in-memory)', async () => {
const manager = createInMemoryManager();
const streamId = `atomic-drain-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_run_step',
data: {
id: 'step-1',
runId: 'run-1',
index: 0,
stepDetails: { type: 'message_creation' },
},
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
id: 'step-1',
delta: { content: { type: 'text', text: 'buffered' } },
},
});
const liveEvents: ServerSentEvent[] = [];
const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(
streamId,
(event) => liveEvents.push(event),
);
expect(resumeState).not.toBeNull();
expect(pendingEvents.length).toBe(0);
expect(liveEvents.length).toBe(0);
subscription?.unsubscribe();
await manager.destroy();
});
test('should return empty pendingEvents when buffer is empty', async () => {
const manager = createInMemoryManager();
const streamId = `atomic-empty-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 10));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'delivered' } } },
});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
const { pendingEvents } = await manager.subscribeWithResume(streamId, () => {});
expect(pendingEvents.length).toBe(0);
await manager.destroy();
});
test('should defer live events until a resumed subscription is activated', async () => {
const manager = createInMemoryManager();
const streamId = `atomic-live-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 10));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 20));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: 'buffered-pre-snapshot' } },
},
});
const liveEvents: ServerSentEvent[] = [];
const { subscription, pendingEvents } = await manager.subscribeWithResume(streamId, (event) =>
liveEvents.push(event),
);
expect(pendingEvents.length).toBe(0);
expect(liveEvents.length).toBe(0);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'live-after' } } },
});
await new Promise((resolve) => setTimeout(resolve, 20));
expect(liveEvents.length).toBe(0);
subscription?.activate();
expect(liveEvents.length).toBe(1);
const liveEvent = liveEvents[0] as {
event: string;
data: { delta: { content: { text: string } } };
};
expect(liveEvent.data.delta.content.text).toBe('live-after');
subscription?.unsubscribe();
await manager.destroy();
});
testRedis(
'should return empty pendingEvents in Redis mode (chunks already persisted)',
async () => {
const manager = createRedisManager();
const streamId = `atomic-redis-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const sub1 = await manager.subscribe(streamId, () => {});
await new Promise((resolve) => setTimeout(resolve, 100));
sub1?.unsubscribe();
await new Promise((resolve) => setTimeout(resolve, 100));
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: 'buffered-redis' } },
},
});
await new Promise((resolve) => setTimeout(resolve, 100));
const liveEvents: ServerSentEvent[] = [];
const { subscription, resumeState, pendingEvents } = await manager.subscribeWithResume(
streamId,
(event) => liveEvents.push(event),
);
expect(resumeState).not.toBeNull();
expect(pendingEvents.length).toBe(0);
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: { delta: { content: { type: 'text', text: 'live-redis' } } },
});
await new Promise((resolve) => setTimeout(resolve, 200));
expect(liveEvents.length).toBe(0);
subscription?.activate();
expect(liveEvents.length).toBe(1);
subscription?.unsubscribe();
await manager.destroy();
},
);
});
describe('Error Preservation for Late Subscribers', () => {
/**
* These tests verify the fix for the race condition where errors
* (like INPUT_LENGTH) occur before the SSE client connects.
*
* Problem: Error → emitError → completeJob → job deleted → client connects → 404
* Fix: Store error, don't delete job immediately, send error to late subscriber
*/
test('should store error in emitError for late-connecting subscribers', async () => {
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
GenerationJobManager.initialize();
const streamId = `error-store-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const errorMessage = '{ "type": "INPUT_LENGTH", "info": "234856 / 172627" }';
// Emit error (no subscribers yet - simulates race condition)
await GenerationJobManager.emitError(streamId, errorMessage);
// Wait for async job store update
await new Promise((resolve) => setTimeout(resolve, 50));
// Verify error is stored in job store
const job = await GenerationJobManager.getJob(streamId);
expect(job?.error).toBe(errorMessage);
await GenerationJobManager.destroy();
});
test('should NOT delete job immediately when completeJob is called with error', async () => {
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: true, // Default behavior
});
GenerationJobManager.initialize();
const streamId = `error-no-delete-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const errorMessage = 'Test error message';
// Complete with error
await GenerationJobManager.completeJob(streamId, errorMessage);
// Job should still exist (not deleted)
const hasJob = await GenerationJobManager.hasJob(streamId);
expect(hasJob).toBe(true);
// Job should have error status
const job = await GenerationJobManager.getJob(streamId);
expect(job?.status).toBe('error');
expect(job?.error).toBe(errorMessage);
await GenerationJobManager.destroy();
});
test('should send stored error to late-connecting subscriber', async () => {
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: true,
});
GenerationJobManager.initialize();
const streamId = `error-late-sub-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const errorMessage = '{ "type": "INPUT_LENGTH", "info": "234856 / 172627" }';
// Simulate race condition: error occurs before client connects
await GenerationJobManager.emitError(streamId, errorMessage);
await GenerationJobManager.completeJob(streamId, errorMessage);
// Wait for async operations
await new Promise((resolve) => setTimeout(resolve, 50));
// Now client connects (late subscriber)
let receivedError: string | undefined;
const subscription = await GenerationJobManager.subscribe(
streamId,
() => {}, // onChunk
() => {}, // onDone
(error) => {
receivedError = error;
}, // onError
);
expect(subscription).not.toBeNull();
// Wait for setImmediate in subscribe to fire
await new Promise((resolve) => setTimeout(resolve, 50));
// Late subscriber should receive the stored error
expect(receivedError).toBe(errorMessage);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
});
test('should prioritize error status over finalEvent in subscribe', async () => {
GenerationJobManager.configure({
jobStore: new InMemoryJobStore({ ttlAfterComplete: 60000 }),
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: false,
});
GenerationJobManager.initialize();
const streamId = `error-priority-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
const errorMessage = 'Error should take priority';
// Emit error and complete with error
await GenerationJobManager.emitError(streamId, errorMessage);
await GenerationJobManager.completeJob(streamId, errorMessage);
await new Promise((resolve) => setTimeout(resolve, 50));
// Subscribe and verify error is received (not a done event)
let receivedError: string | undefined;
let receivedDone = false;
const subscription = await GenerationJobManager.subscribe(
streamId,
() => {},
() => {
receivedDone = true;
},
(error) => {
receivedError = error;
},
);
await new Promise((resolve) => setTimeout(resolve, 50));
// Error should be received, not done
expect(receivedError).toBe(errorMessage);
expect(receivedDone).toBe(false);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
});
testRedis('should handle error preservation in Redis mode (cross-replica)', async () => {
// === Replica A: Creates job and emits error ===
const replicaAJobStore = new RedisJobStore(ioredisClient!);
await replicaAJobStore.initialize();
const streamId = `redis-error-${Date.now()}`;
const errorMessage = '{ "type": "INPUT_LENGTH", "info": "234856 / 172627" }';
await replicaAJobStore.createJob(streamId, 'user-1');
await replicaAJobStore.updateJob(streamId, {
status: 'error',
error: errorMessage,
completedAt: Date.now(),
});
// === Replica B: Fresh manager receives client connection ===
jest.resetModules();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
GenerationJobManager.configure({
...services,
cleanupOnComplete: false,
});
GenerationJobManager.initialize();
// Client connects to Replica B (job created on Replica A)
let receivedError: string | undefined;
const subscription = await GenerationJobManager.subscribe(
streamId,
() => {},
() => {},
(error) => {
receivedError = error;
},
);
expect(subscription).not.toBeNull();
await new Promise((resolve) => setTimeout(resolve, 100));
// Error should be loaded from Redis and sent to subscriber
expect(receivedError).toBe(errorMessage);
subscription?.unsubscribe();
await GenerationJobManager.destroy();
await replicaAJobStore.destroy();
});
test('error jobs should be cleaned up by periodic cleanup after TTL', async () => {
// Use a very short TTL for testing
const jobStore = new InMemoryJobStore({ ttlAfterComplete: 100 });
GenerationJobManager.configure({
jobStore,
eventTransport: new InMemoryEventTransport(),
isRedis: false,
cleanupOnComplete: true,
});
GenerationJobManager.initialize();
const streamId = `error-cleanup-${Date.now()}`;
await GenerationJobManager.createJob(streamId, 'user-1');
// Complete with error
await GenerationJobManager.completeJob(streamId, 'Test error');
// Job should exist immediately after error
let hasJob = await GenerationJobManager.hasJob(streamId);
expect(hasJob).toBe(true);
// Wait for TTL to expire
await new Promise((resolve) => setTimeout(resolve, 150));
// Trigger cleanup
await jobStore.cleanup();
// Job should be cleaned up after TTL
hasJob = await GenerationJobManager.hasJob(streamId);
expect(hasJob).toBe(false);
await GenerationJobManager.destroy();
});
});
describeRedis('Cross-Replica Live Streaming (Redis)', () => {
test('should publish events to Redis even when no local subscriber exists', async () => {
const replicaA = new GenerationJobManagerClass();
const servicesA = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
replicaA.configure(servicesA);
replicaA.initialize();
const replicaB = new GenerationJobManagerClass();
const servicesB = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
replicaB.configure(servicesB);
replicaB.initialize();
const streamId = `cross-live-${Date.now()}`;
await replicaA.createJob(streamId, 'user-1');
const receivedOnB: unknown[] = [];
const subB = await replicaB.subscribe(streamId, (event: unknown) => receivedOnB.push(event));
await new Promise((resolve) => setTimeout(resolve, 100));
for (let i = 0; i < 5; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `token${i} ` } },
index: i,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
expect(receivedOnB.length).toBe(5);
for (let i = 0; i < 5; i++) {
expect((receivedOnB[i] as Record<string, unknown>).event).toBe('on_message_delta');
}
subB?.unsubscribe();
await replicaA.destroy();
await replicaB.destroy();
});
test('should not cause data loss on cross-replica subscribers when local subscriber joins', async () => {
const replicaA = new GenerationJobManagerClass();
const servicesA = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
replicaA.configure(servicesA);
replicaA.initialize();
const replicaB = new GenerationJobManagerClass();
const servicesB = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
replicaB.configure(servicesB);
replicaB.initialize();
const streamId = `cross-seq-safe-${Date.now()}`;
await replicaA.createJob(streamId, 'user-1');
const receivedOnB: unknown[] = [];
const subB = await replicaB.subscribe(streamId, (event: unknown) => receivedOnB.push(event));
await new Promise((resolve) => setTimeout(resolve, 100));
for (let i = 0; i < 3; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `pre-local-${i}` } },
index: i,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
expect(receivedOnB.length).toBe(3);
const receivedOnA: unknown[] = [];
const subA = await replicaA.subscribe(streamId, (event: unknown) => receivedOnA.push(event));
await new Promise((resolve) => setTimeout(resolve, 100));
expect(receivedOnA.length).toBe(3);
for (let i = 0; i < 3; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `post-local-${i}` } },
index: i + 3,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
expect(receivedOnB.length).toBe(6);
expect(receivedOnA.length).toBe(6);
for (let i = 0; i < 3; i++) {
const data = (receivedOnB[i] as Record<string, unknown>).data as Record<string, unknown>;
const delta = data.delta as Record<string, unknown>;
const content = delta.content as Record<string, unknown>;
expect(content.text).toBe(`pre-local-${i}`);
}
for (let i = 0; i < 3; i++) {
const data = (receivedOnB[i + 3] as Record<string, unknown>).data as Record<
string,
unknown
>;
const delta = data.delta as Record<string, unknown>;
const content = delta.content as Record<string, unknown>;
expect(content.text).toBe(`post-local-${i}`);
}
subA?.unsubscribe();
subB?.unsubscribe();
await replicaA.destroy();
await replicaB.destroy();
});
test('should deliver buffered events locally AND publish live events cross-replica', async () => {
const replicaA = new GenerationJobManagerClass();
const servicesA = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
replicaA.configure(servicesA);
replicaA.initialize();
const streamId = `cross-buf-live-${Date.now()}`;
await replicaA.createJob(streamId, 'user-1');
await replicaA.emitChunk(streamId, {
created: true,
message: { text: 'hello' },
streamId,
} as CreatedEvent);
const receivedOnA: unknown[] = [];
const subA = await replicaA.subscribe(streamId, (event: unknown) => receivedOnA.push(event));
await new Promise((resolve) => setTimeout(resolve, 100));
expect(receivedOnA.length).toBe(1);
expect((receivedOnA[0] as Record<string, unknown>).created).toBe(true);
const replicaB = new GenerationJobManagerClass();
const servicesB = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
replicaB.configure(servicesB);
replicaB.initialize();
const receivedOnB: unknown[] = [];
const subB = await replicaB.subscribe(streamId, (event: unknown) => receivedOnB.push(event));
await new Promise((resolve) => setTimeout(resolve, 100));
for (let i = 0; i < 3; i++) {
await replicaA.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `word${i} ` } },
index: i,
},
});
}
/** B joined after A published seq 0, so B's reorder buffer force-flushes after REORDER_TIMEOUT_MS (500ms) */
await new Promise((resolve) => setTimeout(resolve, 700));
expect(receivedOnA.length).toBe(4);
expect(receivedOnB.length).toBe(4);
expect((receivedOnB[0] as CreatedEvent).created).toBe(true);
subA?.unsubscribe();
subB?.unsubscribe();
await replicaA.destroy();
await replicaB.destroy();
});
});
describeRedis('Concurrent Subscriber Readiness (Redis)', () => {
test('should return ready promise to all concurrent subscribers for same stream', async () => {
const subscriber = (
ioredisClient as unknown as { duplicate: () => typeof ioredisClient }
).duplicate()!;
const transport = new RedisEventTransport(ioredisClient as never, subscriber as never);
const streamId = `concurrent-sub-${Date.now()}`;
const sub1 = transport.subscribe(streamId, {
onChunk: () => {},
onDone: () => {},
});
const sub2 = transport.subscribe(streamId, {
onChunk: () => {},
onDone: () => {},
});
expect(sub1.ready).toBeDefined();
expect(sub2.ready).toBeDefined();
await Promise.all([sub1.ready, sub2.ready]);
sub1.unsubscribe();
sub2.unsubscribe();
transport.destroy();
subscriber.disconnect();
});
});
describeRedis('Sequence Reset Safety (Redis)', () => {
test('should not receive stale pre-subscribe events via Redis after sequence reset', async () => {
const manager = new GenerationJobManagerClass();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
manager.configure(services);
manager.initialize();
const streamId = `seq-stale-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: 'pre-sub-0' } },
index: 0,
},
});
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: 'pre-sub-1' } },
index: 1,
},
});
const receivedEvents: unknown[] = [];
const sub = await manager.subscribe(streamId, (event: unknown) => receivedEvents.push(event));
await new Promise((resolve) => setTimeout(resolve, 100));
expect(receivedEvents.length).toBe(2);
expect(
((receivedEvents[0] as Record<string, unknown>).data as Record<string, unknown>).delta,
).toBeDefined();
for (let i = 0; i < 5; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `post-sub-${i}` } },
index: i + 2,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
expect(receivedEvents.length).toBe(7);
const texts = receivedEvents.map(
(e) =>
(
((e as Record<string, unknown>).data as Record<string, unknown>).delta as Record<
string,
unknown
>
).content as Record<string, unknown>,
);
expect((texts[0] as Record<string, unknown>).text).toBe('pre-sub-0');
expect((texts[1] as Record<string, unknown>).text).toBe('pre-sub-1');
for (let i = 0; i < 5; i++) {
expect((texts[i + 2] as Record<string, unknown>).text).toBe(`post-sub-${i}`);
}
sub?.unsubscribe();
await manager.destroy();
});
test('should not reset sequence when second subscriber joins mid-stream', async () => {
const manager = new GenerationJobManagerClass();
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
manager.configure({ ...services, cleanupOnComplete: false });
manager.initialize();
const streamId = `seq-2nd-sub-${Date.now()}`;
await manager.createJob(streamId, 'user-1');
const eventsA: unknown[] = [];
const subA = await manager.subscribe(streamId, (event: unknown) => eventsA.push(event));
await new Promise((resolve) => setTimeout(resolve, 50));
for (let i = 0; i < 3; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `chunk-${i}` } },
index: i,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 100));
expect(eventsA.length).toBe(3);
const eventsB: unknown[] = [];
const subB = await manager.subscribe(streamId, (event: unknown) => eventsB.push(event));
for (let i = 3; i < 6; i++) {
await manager.emitChunk(streamId, {
event: 'on_message_delta',
data: {
delta: { content: { type: 'text', text: `chunk-${i}` } },
index: i,
},
});
}
await new Promise((resolve) => setTimeout(resolve, 300));
expect(eventsA.length).toBe(6);
expect(eventsB.length).toBe(3);
for (let i = 0; i < 6; i++) {
const text = (
(
((eventsA[i] as Record<string, unknown>).data as Record<string, unknown>)
.delta as Record<string, unknown>
).content as Record<string, unknown>
).text;
expect(text).toBe(`chunk-${i}`);
}
subA?.unsubscribe();
subB?.unsubscribe();
await manager.destroy();
});
});
describeRedis('Subscribe Error Recovery (Redis)', () => {
test('should allow resubscription after Redis subscribe failure', async () => {
const subscriber = (
ioredisClient as unknown as { duplicate: () => typeof ioredisClient }
).duplicate()!;
const realSubscribe = subscriber.subscribe.bind(subscriber);
let callCount = 0;
subscriber.subscribe = ((...args: Parameters<typeof subscriber.subscribe>) => {
callCount++;
if (callCount === 1) {
return Promise.reject(new Error('Simulated Redis SUBSCRIBE failure'));
}
return realSubscribe(...args);
}) as typeof subscriber.subscribe;
const transport = new RedisEventTransport(ioredisClient as never, subscriber as never);
const streamId = `err-retry-${Date.now()}`;
const sub1 = transport.subscribe(streamId, {
onChunk: () => {},
onDone: () => {},
});
await expect(sub1.ready).rejects.toThrow('Simulated Redis SUBSCRIBE failure');
const receivedEvents: unknown[] = [];
sub1.unsubscribe();
const sub2 = transport.subscribe(streamId, {
onChunk: (event: unknown) => receivedEvents.push(event),
onDone: () => {},
});
expect(sub2.ready).toBeDefined();
await sub2.ready;
expect(callCount).toBe(2);
await transport.emitChunk(streamId, {
event: 'test',
data: { value: 'hello' },
});
await new Promise((resolve) => setTimeout(resolve, 100));
expect(receivedEvents.length).toBe(1);
sub2.unsubscribe();
transport.destroy();
subscriber.disconnect();
});
});
describe('createStreamServices Auto-Detection', () => {
testRedis('should use Redis when useRedis is true and client is available', () => {
const services = createStreamServices({
useRedis: true,
redisClient: ioredisClient,
});
expect(services.isRedis).toBe(true);
services.eventTransport.destroy();
});
test('should fall back to in-memory when useRedis is false', () => {
const services = createStreamServices({ useRedis: false });
expect(services.isRedis).toBe(false);
});
test('should allow forcing in-memory via config override', async () => {
const services = createStreamServices({ useRedis: false });
expect(services.isRedis).toBe(false);
});
});
/**
* The production topology for interrupt & steer: the steer POST is routed to
* whichever replica the load balancer picks, which is usually NOT the replica
* running the generation. Everything else in the preempt suite runs against a
* single manager, so the hop that actually carries the request — non-owner
* publishes, owner arms, owner's SDK poll flips — has no coverage.
*
* Two `GenerationJobManagerClass` instances are a faithful pair of replicas
* here: `runtimeState` and `ownedJobs` are private instance fields, there is
* no module-level mutable state between them, and `createStreamServices`
* duplicates a dedicated subscriber connection per call. Separate OS
* processes would exercise the same objects over the same Redis.
*/
describeRedis('Cross-Replica Runtime Signals (Redis, multiple manager instances)', () => {
const replicas: GenerationJobManagerClass[] = [];
function createReplica(redisSubscriber?: Redis | Cluster): GenerationJobManagerClass {
const manager = new GenerationJobManagerClass();
manager.configure(
createStreamServices({
useRedis: true,
redisClient: ioredisClient!,
redisSubscriber,
}),
);
manager.initialize();
replicas.push(manager);
return manager;
}
afterEach(async () => {
/** `destroy()`, not `eventTransport.destroy()`: it also clears the
* manager's cleanup interval and disposes the job store and its timer.
* Dropping only the transport leaves each manager alive inside its own
* interval closure, still doing cleanup work against a dead transport. */
for (const manager of replicas.splice(0)) {
await manager.destroy().catch(() => {});
}
});
/**
* Redis pub/sub never replays and the owner's `SUBSCRIBE` is fired
* detached, so a one-shot publish can be dropped simply for arriving
* before anyone is listening — a fixed sleep only decides how often that
* happens on a loaded worker. Republishing until the owner's state
* converges is safe because both operations are idempotent set writes
* keyed by steerId, and it removes the timing assumption entirely.
*/
async function publishUntil(
publish: () => Promise<unknown>,
settled: () => boolean,
what: string,
timeoutMs = 15000,
): Promise<void> {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
await publish();
for (let attempt = 0; attempt < 10; attempt++) {
if (settled()) {
return;
}
await new Promise((resolve) => setTimeout(resolve, 25));
}
}
throw new Error(`Timed out waiting for ${what}`);
}
test('a replacement waits for the exact generation owner to acknowledge abort', async () => {
const owner = createReplica();
const router = createReplica();
const streamId = `${testPrefix}-replacement-owner-ack-${Date.now()}`;
const predecessor = await owner.createJob(streamId, 'user-1', streamId, {
initialMetadata: { generationProtocolVersion: 2 },
});
const replacement = await router.createJob(streamId, 'user-1', streamId, {
initialMetadata: { generationProtocolVersion: 2 },
});
expect(predecessor.abortController.signal.aborted).toBe(true);
expect(replacement.abortController.signal.aborted).toBe(false);
expect(await router.getJobStore().getJob(streamId)).toMatchObject({
createdAt: replacement.createdAt,
status: 'running',
});
}, 40000);
test('a bystander subscriber cannot acknowledge for a disconnected generation owner', async () => {
const ownerSubscriber = (ioredisClient as Redis).duplicate();
const owner = createReplica(ownerSubscriber);
const bystander = createReplica();
const router = createReplica();
const streamId = `${testPrefix}-replacement-bystander-ack-${Date.now()}`;
const predecessor = await owner.createJob(streamId, 'user-1', streamId, {
initialMetadata: { generationProtocolVersion: 2 },
});
const bystanderJob = await bystander.getJob(streamId);
const bystanderSubscription = await bystander.subscribe(streamId, () => undefined);
expect(bystanderJob?.createdAt).toBe(predecessor.createdAt);
expect(bystanderSubscription).not.toBeNull();
ownerSubscriber.disconnect();
await expect(
router.createJob(streamId, 'user-1', streamId, {
initialMetadata: { generationProtocolVersion: 2 },
}),
).rejects.toThrow('predecessor handoff could not be confirmed');
expect(predecessor.abortController.signal.aborted).toBe(false);
expect(bystanderJob?.abortController.signal.aborted).toBe(true);
expect(await router.getJobStore().getJob(streamId)).toMatchObject({
status: 'error',
error: 'Generation predecessor handoff could not be confirmed',
replacedJobs: [
expect.objectContaining({
createdAt: predecessor.createdAt,
status: 'running',
}),
],
});
bystanderSubscription?.unsubscribe();
}, 40000);
test('a steer routed to a non-owning replica arms the owner and flips its poll', async () => {
const owner = createReplica();
const router = createReplica();
const streamId = `${testPrefix}-preempt-xreplica-${Date.now()}`;
const job = await owner.createJob(streamId, 'user-1', undefined, {
initialMetadata: { preemptCapable: true },
});
/**
* The routing replica reads the job, which installs a FACADE runtime
* entry on it. That facade must not make it look like an owner.
*/
const seenByRouter = await router.getJob(streamId);
expect(seenByRouter?.createdAt).toBe(job.createdAt);
expect(router.isPreemptRequested(streamId)).toBe(false);
/**
* Guards the round-trip that shipped broken once: `preemptCapable` was
* serialized but never deserialized, so every Redis deployment read it
* back as undefined and the feature was a silent no-op. A same-replica
* read cannot catch that — this one crosses Redis.
*/
expect(seenByRouter?.metadata?.preemptCapable).toBe(true);
const enqueued = await router.steering.enqueueVersioned(
streamId,
{
steerId: 'steer-x',
text: 'interrupt me',
userId: 'user-1',
createdAt: Date.now(),
},
true,
job.createdAt,
);
if (typeof enqueued === 'number') {
throw new Error(`Unexpected enqueue rejection: ${enqueued}`);
}
/** The whole point: the owner's level-triggered poll flips from a
* request that was accepted on a different replica. */
await publishUntil(
() =>
router.requestPreempt(
streamId,
enqueued.item.steerId,
job.createdAt,
enqueued.item.preemptRevision ?? 0,
),
() => owner.isPreemptRequested(streamId),
'the owner to arm',
);
expect(owner.getArmedPreemptIds(streamId)).toContain('steer-x');
expect(await router.steering.cancel(streamId, 'steer-x', job.createdAt)).toBe(true);
await publishUntil(
() => router.noteSteersRemoved(streamId, ['steer-x'], job.createdAt),
() => !owner.isPreemptRequested(streamId),
'the owner to disarm',
);
expect(owner.getArmedPreemptIds(streamId)).not.toContain('steer-x');
}, 40000);
/**
* The in-memory store fences this in TypeScript; Redis fences it inside
* STEER_ENQUEUE_LUA, which only a real server can execute. That Lua path
* is where the `preemptCapable` P1 hid, so it gets its own coverage.
*/
test('the enqueue Lua refuses an item fenced to a replaced generation', async () => {
const owner = createReplica();
const streamId = `${testPrefix}-enqueue-fenced-${Date.now()}`;
const first = await owner.createJob(streamId, 'user-1');
const replacement = await owner.createJob(streamId, 'user-1');
expect(replacement.createdAt).not.toBe(first.createdAt);
const stale = await owner.steering.enqueue(
streamId,
{
steerId: 'steer-stale-epoch',
text: 'belongs to the previous run',
userId: 'user-1',
createdAt: Date.now(),
},
first.createdAt,
);
expect(stale).toBe(STEER_ENQUEUE_NOT_RUNNING);
expect(await owner.steering.peek(streamId)).toEqual([]);
const live = await owner.steering.enqueue(
streamId,
{
steerId: 'steer-live-epoch',
text: 'belongs to the live run',
userId: 'user-1',
createdAt: Date.now(),
},
replacement.createdAt,
);
expect(live).toBe(1);
}, 30000);
/**
* Redis parks leftover steers inside its terminal-transition Lua, which
* projects each item field by field — so a field added to SteerQueueItem
* is silently dropped there unless the projection is updated too. A steer
* recovered from `/chat/status` would come back without its interrupting
* label even though every non-Redis path preserves it.
*/
test('the terminal-transition Lua keeps preempt on parked steers', async () => {
const owner = createReplica();
const streamId = `${testPrefix}-park-preempt-${Date.now()}`;
const job = await owner.createJob(streamId, 'user-1');
await owner.steering.enqueue(
streamId,
{
steerId: 'steer-parked-preempt',
text: 'interrupt me',
userId: 'user-1',
createdAt: Date.now(),
preempt: true,
},
job.createdAt,
);
const store = new RedisJobStore(ioredisClient!);
const moved = await store.transitionStatus(streamId, {
from: 'running',
to: 'complete',
});
expect(moved).toBe(true);
const claimed = await store.claimParkedSteers(streamId, 'user-1');
expect(claimed).toBeDefined();
const parked = JSON.parse(claimed as string) as {
steers: Array<{ steerId: string; preempt?: boolean }>;
};
expect(parked.steers).toHaveLength(1);
expect(parked.steers[0].steerId).toBe('steer-parked-preempt');
expect(parked.steers[0].preempt).toBe(true);
}, 30000);
test('a stale generation id from another replica cannot arm the live job', async () => {
const owner = createReplica();
const router = createReplica();
const streamId = `${testPrefix}-preempt-xreplica-stale-${Date.now()}`;
const job = await owner.createJob(streamId, 'user-1', undefined, {
initialMetadata: { preemptCapable: true },
});
const enqueuePreempt = async (steerId: string) => {
const enqueued = await router.steering.enqueueVersioned(
streamId,
{
steerId,
text: steerId,
userId: 'user-1',
createdAt: Date.now(),
},
true,
job.createdAt,
);
if (typeof enqueued === 'number') {
throw new Error(`Unexpected enqueue rejection for ${steerId}: ${enqueued}`);
}
return enqueued.item;
};
const controlBefore = await enqueuePreempt('control-before');
const stale = await enqueuePreempt('steer-stale');
const controlAfter = await enqueuePreempt('control-after');
/**
* A negative assertion cannot be established by waiting: an undelivered
* stale arm and a fenced one look identical. So bracket it between two
* control arms on the same channel. The first proves the owner is
* listening BEFORE the stale one is published, and the second — sent
* after it, over the same publisher connection, so Redis orders it
* behind — proves the stale arm has already had its chance to land.
*/
await publishUntil(
() =>
router.requestPreempt(
streamId,
controlBefore.steerId,
job.createdAt,
controlBefore.preemptRevision ?? 0,
),
() => owner.getArmedPreemptIds(streamId).includes('control-before'),
'the first control arm',
);
await router.requestPreempt(
streamId,
stale.steerId,
job.createdAt - 1,
stale.preemptRevision ?? 0,
);
await publishUntil(
() =>
router.requestPreempt(
streamId,
controlAfter.steerId,
job.createdAt,
controlAfter.preemptRevision ?? 0,
),
() => owner.getArmedPreemptIds(streamId).includes('control-after'),
'the second control arm',
);
expect(owner.getArmedPreemptIds(streamId)).not.toContain('steer-stale');
}, 40000);
});
});