Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🌊 feat: Wire Adaptive Stream Smoothing Across Google, Bedrock, and Zero-Disable Semantics (#14660)
Danny Avila33c8801ba6e5
diff --git a/.env.example b/.env.exampleindex 2f2eb94067..a38ee4d660 100644--- a/.env.example+++ b/.env.example@@ -1001,6 +1001,12 @@ HELP_AND_FAQ_URL=https://librechat.ai# Redis CPU at high token rates) at the cost of up to one window of added delivery# latency. Enable only after EVERY replica runs a build with batch-frame support;# older subscribers drop coalesced frames. Values are capped at 1000.+# Keep the window <= the stream-smoothing cadence (`streamRate`, default 25ms):+# each smoothing tick emits its pieces in one burst, so a tick-sized window+# captures exactly one batch per tick; a larger window re-batches the paced+# deltas and quantizes the smoothed cadence at delivery. With smoothing+# disabled (`streamRate: 0`) there is no cadence to preserve — the window is+# then purely the Redis-cost vs delivery-latency tradeoff described above.# STREAM_DELTA_COALESCE_MS=25# Single Redis instancediff --git a/librechat.example.yaml b/librechat.example.yamlindex b8f6ce4bbb..423894017c 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -494,7 +494,9 @@ endpoints:# Anthropic endpoint configuration with Vertex AI support# Use this to run Anthropic Claude models through Google Cloud Vertex AI# anthropic:- # # (optional) Stream rate limiting in milliseconds+ # # (optional) Override the adaptive stream-smoothing cadence in milliseconds.+ # # Agents SDK-backed providers smooth at 25ms by default; set 0 to disable+ # # smoothing. (Legacy Assistants/Ollama paths sleep this long per chunk instead.)# streamRate: 20# # (optional) Title model for conversation titles# titleModel: claude-3.5-haiku # Use the visible model name (key from models config)diff --git a/packages/api/src/endpoints/anthropic/initialize.ts b/packages/api/src/endpoints/anthropic/initialize.tsindex 94a86294a2..232ff80405 100644--- a/packages/api/src/endpoints/anthropic/initialize.ts+++ b/packages/api/src/endpoints/anthropic/initialize.ts@@ -85,12 +85,12 @@ export async function initializeAnthropic({const result = getLLMConfig(credentials, clientOptions);- if (anthropicConfig?.streamRate) {- (result.llmConfig as Record<string, unknown>)._lc_stream_delay = anthropicConfig.streamRate;+ if (anthropicConfig?.streamRate != null) {+ result.llmConfig._lc_stream_delay = anthropicConfig.streamRate;}- if (allConfig?.streamRate) {- (result.llmConfig as Record<string, unknown>)._lc_stream_delay = allConfig.streamRate;+ if (allConfig?.streamRate != null) {+ result.llmConfig._lc_stream_delay = allConfig.streamRate;}return result;diff --git a/packages/api/src/endpoints/bedrock/initialize.spec.ts b/packages/api/src/endpoints/bedrock/initialize.spec.tsindex 3e7c4eaf63..8bbd3d5f64 100644--- a/packages/api/src/endpoints/bedrock/initialize.spec.ts+++ b/packages/api/src/endpoints/bedrock/initialize.spec.ts@@ -1106,3 +1106,44 @@ describe('initializeBedrock', () => {});});});++describe('initializeBedrock streamRate resolution', () => {+ beforeEach(() => {+ process.env.BEDROCK_AWS_ACCESS_KEY_ID = 'test-access-key';+ process.env.BEDROCK_AWS_SECRET_ACCESS_KEY = 'test-secret-key';+ process.env.BEDROCK_AWS_DEFAULT_REGION = 'us-east-1';+ });++ async function delayFor(config: Record<string, unknown>): Promise<unknown> {+ const result = await initializeBedrock(createMockParams({ config }));+ return (result.llmConfig as Record<string, unknown>)._lc_stream_delay;+ }++ it('wires `endpoints.bedrock.streamRate` into llmConfig._lc_stream_delay', async () => {+ await expect(+ delayFor({ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 } } }),+ ).resolves.toBe(25);+ });++ it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => {+ await expect(+ delayFor({+ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { activityLabel: true } },+ }),+ ).resolves.toBe(25);+ });++ it('lets `endpoints.all.streamRate` (including 0) override the endpoint value', async () => {+ await expect(+ delayFor({+ endpoints: { [EModelEndpoint.bedrock]: { streamRate: 25 }, all: { streamRate: 0 } },+ }),+ ).resolves.toBe(0);+ });++ it('leaves the delay unset when neither level configures a streamRate', async () => {+ await expect(+ delayFor({ endpoints: { all: { activityLabel: true } } }),+ ).resolves.toBeUndefined();+ });+});diff --git a/packages/api/src/endpoints/bedrock/initialize.ts b/packages/api/src/endpoints/bedrock/initialize.tsindex 09307dfba0..7686da24dd 100644--- a/packages/api/src/endpoints/bedrock/initialize.ts+++ b/packages/api/src/endpoints/bedrock/initialize.ts@@ -325,6 +325,14 @@ export async function initializeBedrock({}}+ const streamRate =+ appConfig?.endpoints?.all?.streamRate != null+ ? appConfig.endpoints.all.streamRate+ : (bedrockConfig?.streamRate as number | undefined);+ if (streamRate != null) {+ llmConfig._lc_stream_delay = streamRate;+ }+return {llmConfig,configOptions,diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.tsindex 0d334878ec..a896ec6966 100644--- a/packages/api/src/endpoints/custom/initialize.ts+++ b/packages/api/src/endpoints/custom/initialize.ts@@ -346,7 +346,7 @@ export async function initializeCustom({const streamRate = clientOptions.streamRate as number | undefined;if (streamRate != null) {- (options.llmConfig as Record<string, unknown>)._lc_stream_delay = streamRate;+ options.llmConfig._lc_stream_delay = streamRate;}return options;diff --git a/packages/api/src/endpoints/google/initialize.spec.ts b/packages/api/src/endpoints/google/initialize.spec.tsindex a6239805ef..44cf9994b5 100644--- a/packages/api/src/endpoints/google/initialize.spec.ts+++ b/packages/api/src/endpoints/google/initialize.spec.ts@@ -165,3 +165,60 @@ describe('initializeGoogle', () => {});});});++describe('initializeGoogle streamRate resolution', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ process.env.GOOGLE_KEY = 'test-api-key';+ });++ async function initWithConfig(+ endpointsConfig: Record<string, unknown>,+ ): Promise<Record<string, unknown>> {+ const req = createReq();+ (req as unknown as { config: Record<string, unknown> }).config = {+ endpoints: endpointsConfig,+ };+ const result = await initializeGoogle({+ req,+ endpoint: EModelEndpoint.google,+ model_parameters: { model: 'gemini-2.5-flash' },+ db: createDb(),+ });+ return result.llmConfig as Record<string, unknown>;+ }++ it('wires the endpoint streamRate into llmConfig._lc_stream_delay', async () => {+ const llmConfig = await initWithConfig({ [EModelEndpoint.google]: { streamRate: 25 } });+ expect(llmConfig._lc_stream_delay).toBe(25);+ });++ it('preserves the endpoint streamRate when `endpoints.all` exists without one', async () => {+ const llmConfig = await initWithConfig({+ [EModelEndpoint.google]: { streamRate: 25 },+ all: { activityLabel: true },+ });+ expect(llmConfig._lc_stream_delay).toBe(25);+ });++ it('lets `endpoints.all.streamRate` override the endpoint value', async () => {+ const llmConfig = await initWithConfig({+ [EModelEndpoint.google]: { streamRate: 25 },+ all: { streamRate: 10 },+ });+ expect(llmConfig._lc_stream_delay).toBe(10);+ });++ it('lets `endpoints.all.streamRate: 0` disable smoothing explicitly', async () => {+ const llmConfig = await initWithConfig({+ [EModelEndpoint.google]: { streamRate: 25 },+ all: { streamRate: 0 },+ });+ expect(llmConfig._lc_stream_delay).toBe(0);+ });++ it('leaves the delay unset when neither level configures a streamRate', async () => {+ const llmConfig = await initWithConfig({ all: { activityLabel: true } });+ expect(llmConfig._lc_stream_delay).toBeUndefined();+ });+});diff --git a/packages/api/src/endpoints/google/initialize.ts b/packages/api/src/endpoints/google/initialize.tsindex 833956ad1a..4543171875 100644--- a/packages/api/src/endpoints/google/initialize.ts+++ b/packages/api/src/endpoints/google/initialize.ts@@ -84,7 +84,7 @@ export async function initializeGoogle({clientOptions.titleModel = googleConfig.titleModel;}- if (allConfig) {+ if (allConfig?.streamRate != null) {clientOptions.streamRate = allConfig.streamRate;}@@ -124,5 +124,11 @@ export async function initializeGoogle({...clientOptions,};- return getGoogleConfig(credentials, clientOptions);+ const result = getGoogleConfig(credentials, clientOptions);++ if (clientOptions.streamRate != null) {+ result.llmConfig._lc_stream_delay = clientOptions.streamRate;+ }++ return result;}diff --git a/packages/api/src/endpoints/openai/initialize.ts b/packages/api/src/endpoints/openai/initialize.tsindex 9e9418e037..3423a87584 100644--- a/packages/api/src/endpoints/openai/initialize.ts+++ b/packages/api/src/endpoints/openai/initialize.ts@@ -194,21 +194,19 @@ export async function initializeOpenAI({(options as InitializeResultBase).useLegacyContent = true;}- const azureRate = modelName?.includes('gpt-4') ? 30 : 17;-let streamRate: number | undefined;if (isAzureOpenAI && azureConfig) {- streamRate = azureConfig.streamRate ?? azureRate;+ streamRate = azureConfig.streamRate;} else if (!isAzureOpenAI && openAIConfig) {streamRate = openAIConfig.streamRate;}- if (allConfig?.streamRate) {+ if (allConfig?.streamRate != null) {streamRate = allConfig.streamRate;}- if (streamRate) {+ if (streamRate != null) {options.llmConfig._lc_stream_delay = streamRate;}diff --git a/packages/api/src/utils/generators.ts b/packages/api/src/utils/generators.tsindex 2e0416cb89..2466e4d3b3 100644--- a/packages/api/src/utils/generators.ts+++ b/packages/api/src/utils/generators.ts@@ -92,6 +92,14 @@ export function createStreamEventHandlers(res: ServerResponse): {};}+/**+ * @deprecated No longer used internally: stream pacing lives in+ * `@librechat/agents` (adaptive smoothing via `_lc_stream_delay` /+ * `streamRate`), and LangChain runs callback handlers in the background, so+ * a sleep here never paced token delivery. Kept as a compatibility shim for+ * external `@librechat/api` consumers; removal is reserved for a major+ * release.+ */export function createHandleLLMNewToken(streamRate: number) {return async function (): Promise<void> {if (streamRate) {diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 33328ab0f6..73861605f3 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -591,7 +591,14 @@ export const defaultAssistantsVersion = {};export const baseEndpointSchema = z.object({- streamRate: z.number().optional(),+ /**+ * Milliseconds between visible streamed chunks. Agents SDK-backed+ * providers (openAI, custom, anthropic, google, bedrock, agents) smooth+ * adaptively at 25ms by default; set to override the cadence, 0 to+ * disable smoothing. Legacy Assistants and Ollama paths instead sleep+ * this long per provider chunk (default 1ms), with no adaptive smoothing.+ */+ streamRate: z.number().min(0).optional(),baseURL: z.string().optional(),/*** Custom request headers forwarded to the provider on every request. Values