Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🌊 feat: Wire Adaptive Stream Smoothing Across Google, Bedrock, and Zero-Disable Semantics (#14660)

Danny Avila33c8801ba6e5
diff --git a/.env.example b/.env.example
index 2f2eb94067..a38ee4d660 100644
--- a/.env.example
+++ b/.env.example
@@ -1001,6 +1001,12 @@ HELP_AND_FAQ_URL=https://librechat.ai
# Redis CPU at high token rates) at the cost of up to one window of added delivery
# latency. Enable only after EVERY replica runs a build with batch-frame support;
# older subscribers drop coalesced frames. Values are capped at 1000.
+# Keep the window <= the stream-smoothing cadence (`streamRate`, default 25ms):
+# each smoothing tick emits its pieces in one burst, so a tick-sized window
+# captures exactly one batch per tick; a larger window re-batches the paced
+# deltas and quantizes the smoothed cadence at delivery. With smoothing
+# disabled (`streamRate: 0`) there is no cadence to preserve — the window is
+# then purely the Redis-cost vs delivery-latency tradeoff described above.
# STREAM_DELTA_COALESCE_MS=25
# Single Redis instance
diff --git a/librechat.example.yaml b/librechat.example.yaml
index b8f6ce4bbb..423894017c 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -494,7 +494,9 @@ endpoints:
# Anthropic endpoint configuration with Vertex AI support
# Use this to run Anthropic Claude models through Google Cloud Vertex AI
# anthropic:
- # # (optional) Stream rate limiting in milliseconds
+ # # (optional) Override the adaptive stream-smoothing cadence in milliseconds.
+ # # Agents SDK-backed providers smooth at 25ms by default; set 0 to disable
+ # # smoothing. (Legacy Assistants/Ollama paths sleep this long per chunk instead.)
# streamRate: 20
# # (optional) Title model for conversation titles
# titleModel: claude-3.5-haiku # Use the visible model name (key from models config)
diff --git a/packages/api/src/endpoints/anthropic/initialize.ts b/packages/api/src/endpoints/anthropic/initialize.ts
index 94a86294a2..232ff80405 100644
--- a/packages/api/src/endpoints/anthropic/initialize.ts
+++ b/packages/api/src/endpoints/anthropic/initialize.ts
@@ -85,12 +85,12 @@ export async function initializeAnthropic({
const result = getLLMConfig(credentials, clientOptions);
- if (anthropicConfig?.streamRate) {
- (result.llmConfig as Record<string, unknown>)._lc_stream_delay = anthropicConfig.streamRate;
+ if (anthropicConfig?.streamRate != null) {
+ result.llmConfig._lc_stream_delay = anthropicConfig.streamRate;
}
- if (allConfig?.streamRate) {
- (result.llmConfig as Record<string, unknown>)._lc_stream_delay = allConfig.streamRate;
+ if (allConfig?.streamRate != null) {
+ result.llmConfig._lc_stream_delay = allConfig.streamRate;
}
return result;
diff --git a/packages/api/src/endpoints/bedrock/initialize.spec.ts b/packages/api/src/endpoints/bedrock/initialize.spec.ts
index 3e7c4eaf63..8bbd3d5f64 100644
--- a/packages/api/src/endpoints/bedrock/initialize.spec.ts
+++ b/packages/api/src/endpoints/bedrock/initialize.spec.ts
@@ -1106,3 +1106,44 @@ describe('initializeBedrock', () => {
});
});
});
+
+describe('initializeBedrock streamRate resolution', () => {
+ beforeEach(() => {
+ process.env.BEDROCK_AWS_ACCESS_KEY_ID = 'test-access-key';
+ process.env.BEDROCK_AWS_SECRET_ACCESS_KEY = 'test-secret-key';
+ process.env.BEDROCK_AWS_DEFAULT_REGION = 'us-east-1';
+ });
+
+ async function delayFor(config: Record<string, unknown>): Promise<unknown> {
+ const result = await initializeBedrock(createMockParams({ config }));
+ return (result.llmConfig as Record<string, unknown>)._lc_stream_delay;
+ }
+
+ it('wires `endpoints.bedrock.streamRate` into llmConfig._lc_stream_delay', async () => {
+ await expect(
+ delayFor({ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 } } }),
+ ).resolves.toBe(25);
+ });
+
+ it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => {
+ await expect(
+ delayFor({
+ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { activityLabel: true } },
+ }),
+ ).resolves.toBe(25);
+ });
+
+ it('lets `endpoints.all.streamRate` (including 0) override the endpoint value', async () => {
+ await expect(
+ delayFor({
+ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { streamRate: 0 } },
+ }),
+ ).resolves.toBe(0);
+ });
+
+ it('leaves the delay unset when neither level configures a streamRate', async () => {
+ await expect(
+ delayFor({ endpoints: { all: { activityLabel: true } } }),
+ ).resolves.toBeUndefined();
+ });
+});
diff --git a/packages/api/src/endpoints/bedrock/initialize.ts b/packages/api/src/endpoints/bedrock/initialize.ts
index 09307dfba0..7686da24dd 100644
--- a/packages/api/src/endpoints/bedrock/initialize.ts
+++ b/packages/api/src/endpoints/bedrock/initialize.ts
@@ -325,6 +325,14 @@ export async function initializeBedrock({
}
}
+ const streamRate =
+ appConfig?.endpoints?.all?.streamRate != null
+ ? appConfig.endpoints.all.streamRate
+ : (bedrockConfig?.streamRate as number | undefined);
+ if (streamRate != null) {
+ llmConfig._lc_stream_delay = streamRate;
+ }
+
return {
llmConfig,
configOptions,
diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.ts
index 0d334878ec..a896ec6966 100644
--- a/packages/api/src/endpoints/custom/initialize.ts
+++ b/packages/api/src/endpoints/custom/initialize.ts
@@ -346,7 +346,7 @@ export async function initializeCustom({
const streamRate = clientOptions.streamRate as number | undefined;
if (streamRate != null) {
- (options.llmConfig as Record<string, unknown>)._lc_stream_delay = streamRate;
+ options.llmConfig._lc_stream_delay = streamRate;
}
return options;
diff --git a/packages/api/src/endpoints/google/initialize.spec.ts b/packages/api/src/endpoints/google/initialize.spec.ts
index a6239805ef..44cf9994b5 100644
--- a/packages/api/src/endpoints/google/initialize.spec.ts
+++ b/packages/api/src/endpoints/google/initialize.spec.ts
@@ -165,3 +165,60 @@ describe('initializeGoogle', () => {
});
});
});
+
+describe('initializeGoogle streamRate resolution', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ process.env.GOOGLE_KEY = 'test-api-key';
+ });
+
+ async function initWithConfig(
+ endpointsConfig: Record<string, unknown>,
+ ): Promise<Record<string, unknown>> {
+ const req = createReq();
+ (req as unknown as { config: Record<string, unknown> }).config = {
+ endpoints: endpointsConfig,
+ };
+ const result = await initializeGoogle({
+ req,
+ endpoint: EModelEndpoint.google,
+ model_parameters: { model: 'gemini-2.5-flash' },
+ db: createDb(),
+ });
+ return result.llmConfig as Record<string, unknown>;
+ }
+
+ it('wires the endpoint streamRate into llmConfig._lc_stream_delay', async () => {
+ const llmConfig = await initWithConfig({ [EModelEndpoint.google]: { streamRate: 25 } });
+ expect(llmConfig._lc_stream_delay).toBe(25);
+ });
+
+ it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => {
+ const llmConfig = await initWithConfig({
+ [EModelEndpoint.google]: { streamRate: 25 },
+ all: { activityLabel: true },
+ });
+ expect(llmConfig._lc_stream_delay).toBe(25);
+ });
+
+ it('lets `endpoints.all.streamRate` override the endpoint value', async () => {
+ const llmConfig = await initWithConfig({
+ [EModelEndpoint.google]: { streamRate: 25 },
+ all: { streamRate: 10 },
+ });
+ expect(llmConfig._lc_stream_delay).toBe(10);
+ });
+
+ it('lets `endpoints.all.streamRate: 0` disable smoothing explicitly', async () => {
+ const llmConfig = await initWithConfig({
+ [EModelEndpoint.google]: { streamRate: 25 },
+ all: { streamRate: 0 },
+ });
+ expect(llmConfig._lc_stream_delay).toBe(0);
+ });
+
+ it('leaves the delay unset when neither level configures a streamRate', async () => {
+ const llmConfig = await initWithConfig({ all: { activityLabel: true } });
+ expect(llmConfig._lc_stream_delay).toBeUndefined();
+ });
+});
diff --git a/packages/api/src/endpoints/google/initialize.ts b/packages/api/src/endpoints/google/initialize.ts
index 833956ad1a..4543171875 100644
--- a/packages/api/src/endpoints/google/initialize.ts
+++ b/packages/api/src/endpoints/google/initialize.ts
@@ -84,7 +84,7 @@ export async function initializeGoogle({
clientOptions.titleModel = googleConfig.titleModel;
}
- if (allConfig) {
+ if (allConfig?.streamRate != null) {
clientOptions.streamRate = allConfig.streamRate;
}
@@ -124,5 +124,11 @@ export async function initializeGoogle({
...clientOptions,
};
- return getGoogleConfig(credentials, clientOptions);
+ const result = getGoogleConfig(credentials, clientOptions);
+
+ if (clientOptions.streamRate != null) {
+ result.llmConfig._lc_stream_delay = clientOptions.streamRate;
+ }
+
+ return result;
}
diff --git a/packages/api/src/endpoints/openai/initialize.ts b/packages/api/src/endpoints/openai/initialize.ts
index 9e9418e037..3423a87584 100644
--- a/packages/api/src/endpoints/openai/initialize.ts
+++ b/packages/api/src/endpoints/openai/initialize.ts
@@ -194,21 +194,19 @@ export async function initializeOpenAI({
(options as InitializeResultBase).useLegacyContent = true;
}
- const azureRate = modelName?.includes('gpt-4') ? 30 : 17;
-
let streamRate: number | undefined;
if (isAzureOpenAI && azureConfig) {
- streamRate = azureConfig.streamRate ?? azureRate;
+ streamRate = azureConfig.streamRate;
} else if (!isAzureOpenAI && openAIConfig) {
streamRate = openAIConfig.streamRate;
}
- if (allConfig?.streamRate) {
+ if (allConfig?.streamRate != null) {
streamRate = allConfig.streamRate;
}
- if (streamRate) {
+ if (streamRate != null) {
options.llmConfig._lc_stream_delay = streamRate;
}
diff --git a/packages/api/src/utils/generators.ts b/packages/api/src/utils/generators.ts
index 2e0416cb89..2466e4d3b3 100644
--- a/packages/api/src/utils/generators.ts
+++ b/packages/api/src/utils/generators.ts
@@ -92,6 +92,14 @@ export function createStreamEventHandlers(res: ServerResponse): {
};
}
+/**
+ * @deprecated No longer used internally: stream pacing lives in
+ * `@librechat/agents` (adaptive smoothing via `_lc_stream_delay` /
+ * `streamRate`), and LangChain runs callback handlers in the background, so
+ * a sleep here never paced token delivery. Kept as a compatibility shim for
+ * external `@librechat/api` consumers; removal is reserved for a major
+ * release.
+ */
export function createHandleLLMNewToken(streamRate: number) {
return async function (): Promise<void> {
if (streamRate) {
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 33328ab0f6..73861605f3 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -591,7 +591,14 @@ export const defaultAssistantsVersion = {
};
export const baseEndpointSchema = z.object({
- streamRate: z.number().optional(),
+ /**
+ * Milliseconds between visible streamed chunks. Agents SDK-backed
+ * providers (openAI, custom, anthropic, google, bedrock, agents) smooth
+ * adaptively at 25ms by default; set to override the cadence, 0 to
+ * disable smoothing. Legacy Assistants and Ollama paths instead sleep
+ * this long per provider chunk (default 1ms), with no adaptive smoothing.
+ */
+ streamRate: z.number().min(0).optional(),
baseURL: z.string().optional(),
/**
* Custom request headers forwarded to the provider on every request. Values