Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
⚡ perf: Halve Per-Delta Redis Round Trips in Resumable Streams (#14313)
Danny Avila8e5ef1fb31e9
diff --git a/packages/api/src/stream/GenerationJobManager.ts b/packages/api/src/stream/GenerationJobManager.tsindex f6e304d989..46f819effd 100644--- a/packages/api/src/stream/GenerationJobManager.ts+++ b/packages/api/src/stream/GenerationJobManager.ts@@ -1691,9 +1691,15 @@ class GenerationJobManagerClass {return null;}- const result = await this.jobStore.getContentParts(streamId);+ /** Independent reads (streamId-only): parallel to collapse 3 Redis round trips into 1.+ * Safe despite readCachedGraph's cache-drop side effect — each call catches its own+ * unusable-graph throw and falls back to reconstruction, so ordering cannot change the result. */+ const [result, runSteps, queuedSteers] = await Promise.all([+ this.jobStore.getContentParts(streamId),+ this.jobStore.getRunSteps(streamId),+ this.jobStore.peekSteers(streamId),+ ]);const aggregatedContent = result?.content ?? [];- const runSteps = await this.jobStore.getRunSteps(streamId);let titleEvent: t.ResumeState['titleEvent'];if (jobData.titleEvent) {try {@@ -1733,7 +1739,7 @@ class GenerationJobManagerClass {}/** Steers still queued (not yet injected); injected ones are already in aggregatedContent. */- const pendingSteers = (await this.jobStore.peekSteers(streamId)).map(toPendingSteer);+ const pendingSteers = queuedSteers.map(toPendingSteer);logger.debug(`[GenerationJobManager] getResumeState:`, {streamId,diff --git a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.tsindex e254cf66ba..f20da80e91 100644--- a/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts+++ b/packages/api/src/stream/__tests__/RedisEventTransport.stream_integration.spec.ts@@ -1,5 +1,5 @@-import type { Redis, Cluster } from 'ioredis';import { logger } from '@librechat/data-schemas';+import type { Redis, Cluster } from 'ioredis';import { createMockPublisher } from './helpers/publisher';logger.silent = true;@@ -182,6 +182,88 @@ describe('RedisEventTransport Integration Tests', () => {});});+ describe('Payload fidelity through server-side seq splicing', () => {+ /** The seq is spliced into the payload by Lua rather than encoded with it, so the+ * fragments either side must reassemble to exactly what JSON.stringify would emit.+ * Shapes here are the ones a naive cjson round-trip would corrupt. */+ test('should round-trip payload shapes that cjson would coerce', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `payload-fidelity-${Date.now()}`;+ const received: unknown[] = [];++ transport.subscribe(streamId, {+ onChunk: (event) => received.push(event),+ });++ await new Promise((resolve) => setTimeout(resolve, 100));++ const payloads: unknown[] = [+ { empty: [], nestedEmpty: { inner: [] } },+ { float: 0.1234567890123, negative: -273.15, zero: 0 },+ { unicode: 'héllo 🌍 "quoted" \\ backslash\n newline' },+ { nullish: null, emptyString: '', emptyObject: {} },+ { text: 'ordinary delta' },+ ];++ for (const payload of payloads) {+ await transport.emitChunk(streamId, payload);+ }++ await new Promise((resolve) => setTimeout(resolve, 300));++ expect(received).toEqual(payloads);++ transport.destroy();+ subscriber.disconnect();+ });++ test('should assign 0-indexed sequences and set a TTL on the counter only once', async () => {+ if (!ioredisClient) {+ console.warn('Redis not available, skipping test');+ return;+ }++ const { RedisEventTransport } = await import('../implementations/RedisEventTransport');++ const subscriber = (ioredisClient as Redis).duplicate();+ const transport = new RedisEventTransport(ioredisClient, subscriber);++ const streamId = `seq-alloc-${Date.now()}`;+ /** Bare key: the client applies REDIS_KEY_PREFIX itself, and ioredis prefixes EVAL keys+ * the same way, so both sides land on the same prefixed key. */+ const seqKey = `stream:{${streamId}}:seq`;++ transport.subscribe(streamId, { onChunk: () => {} });+ await new Promise((resolve) => setTimeout(resolve, 100));++ await transport.emitChunk(streamId, { index: 0 });+ /** First INCR arms the TTL; it must never be refreshed, or a long stream could+ * have its counter reset mid-generation. */+ const ttlAfterFirst = await (ioredisClient as Redis).ttl(seqKey);+ expect(ttlAfterFirst).toBeGreaterThan(0);++ for (let i = 1; i < 5; i++) {+ await transport.emitChunk(streamId, { index: i });+ }++ /** Counter is 1-based in Redis; seq is 0-based, so 5 emits => counter 5, last seq 4. */+ expect(await (ioredisClient as Redis).get(seqKey)).toBe('5');+ expect(await (ioredisClient as Redis).ttl(seqKey)).toBeLessThanOrEqual(ttlAfterFirst);++ transport.destroy();+ subscriber.disconnect();+ });+ });+describe('Sequential Event Ordering', () => {test('should maintain strict order when emitChunk is awaited', async () => {if (!ioredisClient) {diff --git a/packages/api/src/stream/__tests__/helpers/publisher.ts b/packages/api/src/stream/__tests__/helpers/publisher.tsindex 3e1799d373..be15c67535 100644--- a/packages/api/src/stream/__tests__/helpers/publisher.ts+++ b/packages/api/src/stream/__tests__/helpers/publisher.ts@@ -4,12 +4,13 @@ export interface MockPublisher {expire: jest.Mock;get: jest.Mock;del: jest.Mock;+ eval: jest.Mock;}/** Mock publisher with Redis command simulation for atomic sequence counters */export function createMockPublisher(): MockPublisher {const counters = new Map<string, number>();- return {+ const publisher: MockPublisher = {publish: jest.fn().mockResolvedValue(1),incr: jest.fn().mockImplementation((key: string) => {const current = (counters.get(key) ?? 0) + 1;@@ -27,5 +28,33 @@ export function createMockPublisher(): MockPublisher {}return Promise.resolve(keys.length);}),+ eval: jest.fn(),};++ /**+ * Stands in for PUBLISH_SEQ_LUA, which allocates the sequence and publishes in one server-side+ * round trip. Delegates to the incr/publish mocks rather than reimplementing them, so a test+ * can still fail either half independently and observe the ordering between them.+ */+ publisher.eval.mockImplementation(+ async (+ _script: string,+ _numKeys: number,+ seqKey: string,+ channel: string,+ prefix: string,+ suffix: string,+ ttlSeconds: string,+ ) => {+ const val = (await publisher.incr(seqKey)) as number;+ if (val === 1) {+ await publisher.expire(seqKey, Number(ttlSeconds));+ }+ const seq = val - 1;+ await publisher.publish(channel, `${prefix}${seq}${suffix}`);+ return seq;+ },+ );++ return publisher;}diff --git a/packages/api/src/stream/implementations/RedisEventTransport.ts b/packages/api/src/stream/implementations/RedisEventTransport.tsindex d0eaebad47..eb332ece04 100644--- a/packages/api/src/stream/implementations/RedisEventTransport.ts+++ b/packages/api/src/stream/implementations/RedisEventTransport.ts@@ -50,6 +50,32 @@ interface ReorderBuffer {flushTimeout: ReturnType<typeof setTimeout> | null;}+/**+ * Allocate a sequence number and publish the event in a single round trip.+ *+ * The payload is spliced server-side rather than round-tripped through `cjson`: decoding and+ * re-encoding arbitrary event data would coerce empty arrays to objects and alter float+ * precision. The caller pre-serializes everything around the seq, so this only concatenates.+ *+ * The TTL is set once on the first INCR and never refreshed, so an active stream cannot have+ * its counter reset mid-generation.+ *+ * The channel is passed as ARGV, not KEYS: ioredis applies `keyPrefix` to EVAL keys but never+ * to a pub/sub channel, so keying it here would publish to a prefixed channel that no+ * subscriber listens on. PUBLISH is broadcast cluster-wide rather than slot-routed, so it does+ * not need to be a key for Cluster correctness.+ *+ * KEYS: [sequence]+ * ARGV: [channel, payloadPrefix, payloadSuffix, sequenceTtlSeconds]+ * RETURNS: the 0-indexed seq assigned to this event+ */+const PUBLISH_SEQ_LUA =+ 'local val = redis.call("INCR", KEYS[1]) ' ++ 'if val == 1 then redis.call("EXPIRE", KEYS[1], tonumber(ARGV[4])) end ' ++ 'local seq = val - 1 ' ++ 'redis.call("PUBLISH", ARGV[1], ARGV[2] .. string.format("%d", seq) .. ARGV[3]) ' ++ 'return seq';+/** Max time (ms) to wait for out-of-order messages before force-flushing */const REORDER_TIMEOUT_MS = 500;/** Max messages to buffer before force-flushing (prevents memory issues) */@@ -126,21 +152,40 @@ export class RedisEventTransport implements IEventTransport {private static readonly SEQUENCE_TTL_SECONDS = 86400;/**- * Get next sequence number for a stream (0-indexed, backed by Redis INCR).- * A 24-hour TTL is set on the first INCR only (val === 1) as a safety net- * for orphaned keys from crashed processes. It is never refreshed, so an- * active stream cannot have its counter reset mid-generation.- * Keys are also deleted explicitly by cleanup() on normal stream teardown.+ * Split a seq-less message into the JSON fragments surrounding its `seq`, so the sequence+ * can be spliced in by {@link PUBLISH_SEQ_LUA} without re-encoding the payload.+ *+ * Omitting a field (e.g. `data: undefined`) yields an empty tail, matching what+ * `JSON.stringify` would have dropped from the whole-object encoding.*/- private async getNextSequence(streamId: string): Promise<number> {- const key = KEYS.sequence(streamId);- const val = await this.publisher.incr(key);- if (val === 1) {- this.publisher.expire(key, RedisEventTransport.SEQUENCE_TTL_SECONDS).catch((err) => {- logger.warn(`[RedisEventTransport] Failed to set TTL on sequence key ${key}:`, err);- });- }- return val - 1;+ private static buildPayloadParts(message: Omit<PubSubMessage, 'seq'>): [string, string] {+ const { type, ...rest } = message;+ const encodedRest = JSON.stringify(rest);+ const inner = encodedRest.slice(1, -1);+ return [`{"type":${JSON.stringify(type)},"seq":`, inner.length > 0 ? `,${inner}}` : '}'];+ }++ /**+ * Allocate a sequence number and publish, in one Redis round trip.+ *+ * Keys are deleted explicitly by cleanup() on normal stream teardown; the TTL is a safety+ * net for orphaned keys from crashed processes.+ */+ private async publishWithSequence(+ streamId: string,+ message: Omit<PubSubMessage, 'seq'>,+ ): Promise<number> {+ const [prefix, suffix] = RedisEventTransport.buildPayloadParts(message);+ const seq = await this.publisher.eval(+ PUBLISH_SEQ_LUA,+ 1,+ KEYS.sequence(streamId),+ CHANNELS.events(streamId),+ prefix,+ suffix,+ String(RedisEventTransport.SEQUENCE_TTL_SECONDS),+ );+ return seq as number;}/** Reset subscriber reorder buffer state to initial values */@@ -516,15 +561,12 @@ export class RedisEventTransport implements IEventTransport {* Publish a chunk event to all subscribers across all instances.* Includes sequence number for ordered delivery in Redis Cluster mode.*- * Performance: each emit requires two sequential Redis round-trips (INCR + PUBLISH).- * This is the unavoidable cost of cross-replica sequence coordination.+ * Performance: sequence allocation and publish share one round trip. This runs per streamed+ * delta, so the saved round trip is multiplied by the token count of every response.*/async emitChunk(streamId: string, event: unknown): Promise<void> {try {- const channel = CHANNELS.events(streamId);- const seq = await this.getNextSequence(streamId);- const message: PubSubMessage = { type: EventTypes.CHUNK, seq, data: event };- await this.publisher.publish(channel, JSON.stringify(message));+ await this.publishWithSequence(streamId, { type: EventTypes.CHUNK, data: event });} catch (err) {logger.error(`[RedisEventTransport] Failed to publish chunk:`, err);}@@ -535,12 +577,8 @@ export class RedisEventTransport implements IEventTransport {* Includes sequence number to ensure delivery after all chunks.*/async emitDone(streamId: string, event: unknown): Promise<void> {- const channel = CHANNELS.events(streamId);- const seq = await this.getNextSequence(streamId);- const message: PubSubMessage = { type: EventTypes.DONE, seq, data: event };-try {- await this.publisher.publish(channel, JSON.stringify(message));+ await this.publishWithSequence(streamId, { type: EventTypes.DONE, data: event });} catch (err) {logger.error(`[RedisEventTransport] Failed to publish done:`, err);throw err;@@ -552,12 +590,8 @@ export class RedisEventTransport implements IEventTransport {* Includes sequence number to ensure delivery after all chunks.*/async emitError(streamId: string, error: string): Promise<void> {- const channel = CHANNELS.events(streamId);- const seq = await this.getNextSequence(streamId);- const message: PubSubMessage = { type: EventTypes.ERROR, seq, error };-try {- await this.publisher.publish(channel, JSON.stringify(message));+ await this.publishWithSequence(streamId, { type: EventTypes.ERROR, error });} catch (err) {logger.error(`[RedisEventTransport] Failed to publish error:`, err);throw err;