Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧊 fix: In-Memory Endpoint Token Config Cache Isolation (#12673)
Danny Avilaa613caced3c4
diff --git a/packages/api/src/agents/__tests__/initialize.test.ts b/packages/api/src/agents/__tests__/initialize.test.tsindex fe1493628a..be7377954a 100644--- a/packages/api/src/agents/__tests__/initialize.test.ts+++ b/packages/api/src/agents/__tests__/initialize.test.ts@@ -3,6 +3,7 @@ import { EModelEndpoint } from 'librechat-data-provider';import type { Agent } from 'librechat-data-provider';import type { ServerRequest, InitializeResultBase, EndpointTokenConfig } from '~/types';import type { InitializeAgentDbMethods } from '../initialize';+import { DEFAULT_MAX_CONTEXT_TOKENS } from '../initialize';// Mock loggerjest.mock('winston', () => ({@@ -308,11 +309,8 @@ describe('initializeAgent — maxContextTokens', () => {db,);- // 0 is not used as-is; the formula kicks in.- // optionalChainWithEmptyCheck(0, 200000, 18000) returns 0 (not null/undefined),- // then Number(0) || 18000 = 18000 (the fallback default).expect(result.maxContextTokens).not.toBe(0);- const expected = Math.round((18000 - maxOutputTokens) * 0.95);+ const expected = Math.round((DEFAULT_MAX_CONTEXT_TOKENS - maxOutputTokens) * 0.95);expect(result.maxContextTokens).toBe(expected);});diff --git a/packages/api/src/agents/initialize.ts b/packages/api/src/agents/initialize.tsindex 1ebd20bb01..5105b21305 100644--- a/packages/api/src/agents/initialize.ts+++ b/packages/api/src/agents/initialize.ts@@ -68,6 +68,8 @@ export type InitializedAgent = Agent & {maxToolResultChars?: number;};+export const DEFAULT_MAX_CONTEXT_TOKENS = 32000;+/*** Parameters for initializing an agent* Matches the CJS signature from api/server/services/Endpoints/agents/agent.js@@ -357,7 +359,7 @@ export async function initializeAgent(providerEndpointMap[overrideProvider as keyof typeof providerEndpointMap],options.endpointTokenConfig,),- 18000,+ DEFAULT_MAX_CONTEXT_TOKENS,);if (@@ -414,7 +416,7 @@ export async function initializeAgent(agent.additional_instructions = artifactsPromptResult ?? undefined;}- const agentMaxContextNum = Number(agentMaxContextTokens) || 18000;+ const agentMaxContextNum = Number(agentMaxContextTokens) || DEFAULT_MAX_CONTEXT_TOKENS;const maxOutputTokensNum = Number(maxOutputTokens) || 0;const baseContextTokens = Math.max(0, agentMaxContextNum - maxOutputTokensNum);diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.tsnew file mode 100644index 0000000000..16b016f9bd--- /dev/null+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.ts@@ -0,0 +1,113 @@+import { CacheKeys, Time } from 'librechat-data-provider';++jest.mock('@keyv/redis', () => ({+ default: jest.fn(),+}));++jest.mock('../../redisClients', () => ({+ keyvRedisClient: null,+ ioredisClient: null,+}));++jest.mock('../../redisUtils', () => ({+ batchDeleteKeys: jest.fn(),+ scanKeys: jest.fn(),+}));++jest.mock('@librechat/data-schemas', () => ({+ logger: {+ error: jest.fn(),+ warn: jest.fn(),+ debug: jest.fn(),+ },+}));++describe('standardCache - in-memory memoization', () => {+ afterEach(() => {+ jest.resetModules();+ });++ async function loadFactory() {+ jest.doMock('../../cacheConfig', () => ({+ cacheConfig: {+ FORCED_IN_MEMORY_CACHE_NAMESPACES: [],+ REDIS_KEY_PREFIX: '',+ GLOBAL_PREFIX_SEPARATOR: '>>',+ },+ }));+ return import('../../cacheFactory');+ }++ it('returns the same instance for repeated calls with the same namespace', async () => {+ const { standardCache } = await loadFactory();+ const a = standardCache('test-ns');+ const b = standardCache('test-ns');+ expect(a).toBe(b);+ });++ it('returns different instances for different namespaces', async () => {+ const { standardCache } = await loadFactory();+ const a = standardCache('ns-one');+ const b = standardCache('ns-two');+ expect(a).not.toBe(b);+ });++ it('shares data across separate standardCache calls for the same namespace', async () => {+ const { standardCache } = await loadFactory();+ const writer = standardCache(CacheKeys.TOKEN_CONFIG);+ await writer.set('model-a', { context: 128000 });++ const reader = standardCache(CacheKeys.TOKEN_CONFIG);+ expect(await reader.get('model-a')).toEqual({ context: 128000 });+ });++ it('does not leak data between different namespaces', async () => {+ const { standardCache } = await loadFactory();+ const cacheA = standardCache('ns-a');+ const cacheB = standardCache('ns-b');++ await cacheA.set('key', 'value-a');+ await cacheB.set('key', 'value-b');++ expect(await cacheA.get('key')).toBe('value-a');+ expect(await cacheB.get('key')).toBe('value-b');+ });++ it('first caller TTL wins for a given namespace', async () => {+ const { standardCache } = await loadFactory();+ const first = standardCache('ttl-ns', 500);+ const second = standardCache('ttl-ns', 99999);+ expect(first).toBe(second);++ type KeyvWithOpts = typeof first & { opts: { ttl?: number } };+ expect((first as KeyvWithOpts).opts.ttl).toBe(500);+ });++ it('does not memoize when a custom fallbackStore is provided', async () => {+ const { standardCache } = await loadFactory();+ const storeA = new Map();+ const storeB = new Map();+ const a = standardCache('fb-ns', undefined, storeA);+ const b = standardCache('fb-ns', undefined, storeB);+ expect(a).not.toBe(b);+ });++ it('tokenConfigCache shares data with standardCache(TOKEN_CONFIG)', async () => {+ const { standardCache, tokenConfigCache } = await loadFactory();+ const direct = standardCache(CacheKeys.TOKEN_CONFIG, Time.THIRTY_MINUTES);+ await direct.set('openrouter', { 'gpt-4': { context: 128000, prompt: 5, completion: 15 } });++ const convenience = tokenConfigCache();+ expect(await convenience.get('openrouter')).toEqual({+ 'gpt-4': { context: 128000, prompt: 5, completion: 15 },+ });+ expect(convenience).toBe(direct);+ });++ it('tokenConfigCache creates the instance with THIRTY_MINUTES TTL', async () => {+ const { tokenConfigCache } = await loadFactory();+ const cache = tokenConfigCache();+ type KeyvWithOpts = typeof cache & { opts: { ttl?: number } };+ expect((cache as KeyvWithOpts).opts.ttl).toBe(Time.THIRTY_MINUTES);+ });+});diff --git a/packages/api/src/cache/cacheFactory.ts b/packages/api/src/cache/cacheFactory.tsindex 2d7817c2ad..5c5db865db 100644--- a/packages/api/src/cache/cacheFactory.ts+++ b/packages/api/src/cache/cacheFactory.ts@@ -9,18 +9,33 @@ const KeyvRedis = require('@keyv/redis').default as typeof import('@keyv/redis')import { Keyv } from 'keyv';import createMemoryStore from 'memorystore';import { RedisStore } from 'rate-limit-redis';-import { Time } from 'librechat-data-provider';import { logger } from '@librechat/data-schemas';import session, { MemoryStore } from 'express-session';+import { Time, CacheKeys } from 'librechat-data-provider';import { RedisStore as ConnectRedis } from 'connect-redis';import type { SendCommandFn } from 'rate-limit-redis';import { keyvRedisClient, ioredisClient } from './redisClients';+import { batchDeleteKeys, scanKeys } from './redisUtils';import { cacheConfig } from './cacheConfig';import { violationFile } from './keyvFiles';-import { batchDeleteKeys, scanKeys } from './redisUtils';++/**+ * Memoized in-memory Keyv instances keyed by namespace.+ * Without Redis, each `new Keyv()` gets its own internal Map, so callers that+ * write in one call-site and read in another would see an empty store.+ * Memoizing ensures a single shared Map per namespace across the entire bundle.+ *+ * Only applies to the plain in-memory path (no Redis, no custom fallbackStore).+ */+const inMemoryCacheMap = new Map<string, Keyv>();/*** Creates a cache instance using Redis or a fallback store. Suitable for general caching needs.+ *+ * **In-memory mode** (no Redis, no custom fallbackStore): instances are memoized by+ * namespace so that every call-site shares the same underlying `Map`. The first+ * caller's TTL wins for a given namespace.+ ** @param namespace - The cache namespace.* @param ttl - Time to live for cache entries.* @param fallbackStore - Optional fallback store if Redis is not used.@@ -73,9 +88,19 @@ export const standardCache = (namespace: string, ttl?: number, fallbackStore?: oif (fallbackStore) {return new Keyv({ store: fallbackStore, namespace, ttl });}- return new Keyv({ namespace, ttl });+ const existing = inMemoryCacheMap.get(namespace);+ if (existing) {+ return existing;+ }+ const cache = new Keyv({ namespace, ttl });+ inMemoryCacheMap.set(namespace, cache);+ return cache;};+/** Convenience accessor for the TOKEN_CONFIG cache namespace. */+export const tokenConfigCache = (): Keyv =>+ standardCache(CacheKeys.TOKEN_CONFIG, Time.THIRTY_MINUTES);+/*** Creates a cache instance for storing violation data.* Uses a file-based fallback store if Redis is not enabled.diff --git a/packages/api/src/endpoints/custom/initialize.spec.ts b/packages/api/src/endpoints/custom/initialize.spec.tsindex eddd7cb515..0accf3171a 100644--- a/packages/api/src/endpoints/custom/initialize.spec.ts+++ b/packages/api/src/endpoints/custom/initialize.spec.ts@@ -20,6 +20,7 @@ jest.mock('~/endpoints/models', () => ({jest.mock('~/cache', () => ({standardCache: jest.fn(() => ({ get: jest.fn().mockResolvedValue(null) })),+ tokenConfigCache: jest.fn(() => ({ get: jest.fn().mockResolvedValue(null) })),}));jest.mock('~/utils', () => ({diff --git a/packages/api/src/endpoints/custom/initialize.ts b/packages/api/src/endpoints/custom/initialize.tsindex ea0d2dbf5d..18719c4f02 100644--- a/packages/api/src/endpoints/custom/initialize.ts+++ b/packages/api/src/endpoints/custom/initialize.ts@@ -1,5 +1,4 @@import {- CacheKeys,ErrorTypes,envVarRegex,FetchTokenConfig,@@ -13,7 +12,7 @@ import { isUserProvided, checkUserKeyExpiry } from '~/utils';import { getCustomEndpointConfig } from '~/app/config';import { fetchModels } from '~/endpoints/models';import { validateEndpointURL } from '~/auth';-import { standardCache } from '~/cache';+import { tokenConfigCache } from '~/cache';const { PROXY } = process.env;@@ -136,7 +135,7 @@ export async function initializeCustom({const userId = req.user?.id ?? '';- const cache = standardCache(CacheKeys.TOKEN_CONFIG);+ const cache = tokenConfigCache();/** tokenConfig is an optional extended property on custom endpoints */const hasTokenConfig = (endpointConfig as Record<string, unknown>).tokenConfig != null;const tokenKey =diff --git a/packages/api/src/endpoints/models.ts b/packages/api/src/endpoints/models.tsindex cc91c1a4fa..fd296e59ec 100644--- a/packages/api/src/endpoints/models.ts+++ b/packages/api/src/endpoints/models.ts@@ -19,7 +19,7 @@ import {logAxiosError,inputSchema,} from '~/utils';-import { standardCache } from '~/cache';+import { standardCache, tokenConfigCache } from '~/cache';export interface FetchModelsParams {/** User ID for API requests */@@ -195,8 +195,7 @@ export async function fetchModels({const validationResult = inputSchema.safeParse(input);if (validationResult.success && createTokenConfig) {const endpointTokenConfig = processModelData(input);- const cache = standardCache(CacheKeys.TOKEN_CONFIG);- await cache.set(tokenKey ?? name, endpointTokenConfig);+ await tokenConfigCache().set(tokenKey ?? name, endpointTokenConfig);}models = input.data.map((item: { id: string }) => item.id);} catch (error) {