Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧯 fix: Outlast Failed Auth Cache Eviction on Credential Changes (#16435)
Marco Beretta26b69883556e
diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.jsindex 1edae18156..53d73125df 100644--- a/api/cache/getLogStores.js+++ b/api/cache/getLogStores.js@@ -66,7 +66,9 @@ const namespaces = {CacheKeys.OPENID_EXCHANGED_TOKENS,Time.TEN_MINUTES,),- [CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC),+ [CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC, undefined, undefined, {+ throwOnErrors: true,+ }),[CacheKeys.ADMIN_OAUTH_EXCHANGE]: standardCache(CacheKeys.ADMIN_OAUTH_EXCHANGE,Time.THIRTY_SECONDS,diff --git a/api/server/controllers/AuthController.js b/api/server/controllers/AuthController.jsindex 107f0dbdff..f5e3a230f0 100644--- a/api/server/controllers/AuthController.js+++ b/api/server/controllers/AuthController.js@@ -24,6 +24,7 @@ const {const {deleteAllUserSessions,deletePasskeysByUser,+ awaitAuthUserDocEviction,getUserById,findSession,updateUser,@@ -326,6 +327,7 @@ const resetPasswordController = createResetPasswordController({resetPassword,deleteAllUserSessions,deletePasskeysByUser,+ awaitAuthUserDocEviction,});const refreshController = async (req, res) => {diff --git a/api/server/controllers/AuthController.spec.js b/api/server/controllers/AuthController.spec.jsindex 50090636ea..6a1866c1f9 100644--- a/api/server/controllers/AuthController.spec.js+++ b/api/server/controllers/AuthController.spec.js@@ -42,6 +42,7 @@ jest.mock('~/models', () => ({deleteSession: jest.fn(),deleteAllUserSessions: jest.fn(),deletePasskeysByUser: jest.fn(),+ awaitAuthUserDocEviction: jest.fn(),getUserById: jest.fn(),findSession: jest.fn(),updateUser: jest.fn(),@@ -147,6 +148,7 @@ const {deleteTokens,deleteAllUserSessions,deletePasskeysByUser,+ awaitAuthUserDocEviction,} = require('~/models');const {createRefreshTokenBridgeFlightKey,@@ -2483,6 +2485,7 @@ describe('resetPasswordController', () => {resetPassword.mockReset().mockResolvedValue(serviceResult);deleteAllUserSessions.mockReset().mockResolvedValue(undefined);deletePasskeysByUser.mockReset().mockResolvedValue(undefined);+ awaitAuthUserDocEviction.mockReset().mockResolvedValue(undefined);req = {body: {userId: 'user-123',@@ -2511,6 +2514,31 @@ describe('resetPasswordController', () => {expect(res.json).toHaveBeenCalledWith(serviceResult);});+ it('confirms the reset only after revoking and settling the auth cache', async () => {+ let finishEviction;+ awaitAuthUserDocEviction.mockReturnValue(+ new Promise((resolve) => {+ finishEviction = resolve;+ }),+ );++ const reset = resetPasswordController(req, res);+ await new Promise((resolve) => setImmediate(resolve));+ const responseCalls = res.status.mock.calls.length;+ finishEviction();+ await reset;++ expect(awaitAuthUserDocEviction).toHaveBeenCalledWith('user-123');+ expect(deleteAllUserSessions.mock.invocationCallOrder[0]).toBeLessThan(+ awaitAuthUserDocEviction.mock.invocationCallOrder[0],+ );+ expect(deletePasskeysByUser.mock.invocationCallOrder[0]).toBeLessThan(+ awaitAuthUserDocEviction.mock.invocationCallOrder[0],+ );+ expect(responseCalls).toBe(0);+ expect(res.status).toHaveBeenCalledWith(200);+ });+it('does not revoke sessions or passkeys when reset fails', async () => {const resetError = new Error('Invalid token');resetPassword.mockResolvedValue(resetError);@@ -2519,6 +2547,7 @@ describe('resetPasswordController', () => {expect(deleteAllUserSessions).not.toHaveBeenCalled();expect(deletePasskeysByUser).not.toHaveBeenCalled();+ expect(awaitAuthUserDocEviction).not.toHaveBeenCalled();expect(res.status).toHaveBeenCalledWith(400);expect(res.json).toHaveBeenCalledWith(resetError);});diff --git a/config/reset-password.js b/config/reset-password.jsindex 1d239f58b1..3144ade839 100644--- a/config/reset-password.js+++ b/config/reset-password.js@@ -85,9 +85,15 @@ const resetPassword = async () => {* A passkey signs in on its own, so leaving one in place would keep an attacker* logged in after an administrator believes the account has been recovered.*/- const { deletedCount } = await Passkey.deleteMany({ user: user._id });- /** A refresh session outlives the stamp: refreshing mints a token issued after it. */- await Session.deleteMany({ user: user._id });+ let deletedCount;+ try {+ ({ deletedCount } = await Passkey.deleteMany({ user: user._id }));+ /** A refresh session outlives the stamp: refreshing mints a token issued after it. */+ await Session.deleteMany({ user: user._id });+ } finally {+ /** The password is already changed, so the cache settles even when a revocation fails. */+ await methods.awaitAuthUserDocEviction(user._id.toString());+ }console.log('Password successfully reset!');if (deletedCount > 0) {diff --git a/e2e/specs/mock/scenarios/reset-password.spec.ts b/e2e/specs/mock/scenarios/reset-password.spec.tsindex fc14e00759..a25739132a 100644--- a/e2e/specs/mock/scenarios/reset-password.spec.ts+++ b/e2e/specs/mock/scenarios/reset-password.spec.ts@@ -4,7 +4,11 @@ import os from 'os';import { Keyv } from 'keyv';import KeyvRedis from '@keyv/redis';import { expect, test } from '@playwright/test';-import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';+import {+ CacheKeys,+ AUTH_USER_DOC_BY_ID_PREFIX,+ AUTH_USER_DOC_CACHE_TTL_MS,+} from 'librechat-data-provider';import { spawn } from 'node:child_process';import { randomInt, randomUUID } from 'node:crypto';import { withMongo, seedPasskey, deleteUserByEmail } from '../db';@@ -40,7 +44,13 @@ function runResetCli(email: string,cwd: string,env: NodeJS.ProcessEnv,-): Promise<{ code: number | null; stdout: string; stderr: string }> {+): Promise<{+ code: number | null;+ stdout: string;+ stderr: string;+ /** Milliseconds between confirming the new password and the CLI reporting success. */+ reportDelayMs?: number;+}> {const child = spawn(process.execPath, [path.join(REPO_ROOT, 'config/reset-password.js')], {cwd,env: {@@ -59,14 +69,20 @@ function runResetCli(['Confirm new password: ', NEW_PASSWORD],];let stdout = '';+ let confirmedAt: number | undefined;+ let reportedAt: number | undefined;child.stdout?.on('data', (chunk: Buffer) => {stdout += chunk.toString();answers.forEach((answer, index) => {if (answer !== null && stdout.includes(answer[0])) {child.stdin?.write(`${answer[1]}\n`);answers[index] = null;+ confirmedAt = Date.now();}});+ if (reportedAt === undefined && stdout.includes('Password successfully reset!')) {+ reportedAt = Date.now();+ }});let stderr = '';child.stderr?.on('data', (chunk: Buffer) => (stderr += chunk.toString()));@@ -82,7 +98,11 @@ function runResetCli(child.on('close', (code) => {clearTimeout(timer);child.stdin?.end();- resolve({ code, stdout, stderr });+ const reportDelayMs =+ confirmedAt !== undefined && reportedAt !== undefined+ ? reportedAt - confirmedAt+ : undefined;+ resolve({ code, stdout, stderr, reportDelayMs });});});}@@ -230,4 +250,85 @@ test.describe('CLI password reset', () => {}});}++ test('reports the reset only after unevictable cached credentials expire @scenario:cli-reset-outlasts-failed-cache-eviction', async ({+ playwright,+ baseURL,+ }) => {+ test.setTimeout(90_000);+ const request = await playwright.request.newContext({+ baseURL,+ extraHTTPHeaders: {+ 'X-Forwarded-For': `127.${randomInt(1, 255)}.${randomInt(1, 255)}.${randomInt(1, 255)}`,+ },+ });+ const email = `cli-evict-${randomUUID().slice(0, 8)}@example.com`;+ const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-reset-'));+ const redisUri = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379';+ const prefix = `cli-evict-${randomUUID()}`;+ const adapter = new KeyvRedis(redisUri, { throwOnErrors: true, connectionTimeout: 5000 });+ const cache = new Keyv(adapter, { namespace: CacheKeys.AUTH_USER_DOC });+ adapter.namespace = prefix;+ adapter.keyPrefixSeparator = '::';+ const keys: string[] = [];+ try {+ const register = await request.post('/api/auth/register', {+ data: { email, name: 'CLI Evict', password: PASSWORD, confirm_password: PASSWORD },+ });+ expect(register.ok()).toBeTruthy();+ const login = await request.post('/api/auth/login', { data: { email, password: PASSWORD } });+ expect(login.ok()).toBeTruthy();+ const { token, user } = (await login.json()) as { token: string; user: { _id: string } };+ await seedPasskey(email, randomUUID(), 'Eviction regression');++ const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${user._id}`;+ const controlKey = 'auth-user-doc:another-user';+ keys.push(indexKey, controlKey);+ await cache.set(controlKey, { credentialsChangedAt: null }, 120_000);+ /** A hash where the index string belongs makes the CLI's GET fail with WRONGTYPE. */+ const client = await adapter.getClient();+ await client.hSet(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 'unreadable', '1');+ await client.pExpire(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 120_000);++ const env = { ...process.env };+ Object.assign(env, {+ USE_REDIS: 'true',+ USE_REDIS_CLUSTER: 'false',+ REDIS_URI: redisUri,+ REDIS_KEY_PREFIX: prefix,+ REDIS_KEY_PREFIX_VAR: '',+ AUTH_USER_CACHE_MODE: 'on',+ FORCED_IN_MEMORY_CACHE_NAMESPACES: '',+ });++ const cli = await runResetCli(email, cwd, env);+ expect(cli.code, cli.stderr.slice(-400)).toBe(0);+ expect(cli.stdout).toContain('Password successfully reset!');+ expect(cli.reportDelayMs).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS);+ expect(await cache.get(controlKey)).toEqual({ credentialsChangedAt: null });+ await withMongo(async (db) => {+ const resetUser = await db.collection('users').findOne({ email });+ expect(resetUser?.credentialsChangedAt).toBeTruthy();+ expect(await db.collection('passkeys').countDocuments({ user: resetUser!._id })).toBe(0);+ expect(await db.collection('sessions').countDocuments({ user: resetUser!._id })).toBe(0);+ });+ const rejected = await request.get('/api/user', {+ headers: { Authorization: `Bearer ${token}` },+ });+ expect(rejected.status()).toBe(401);+ const newLogin = await request.post('/api/auth/login', {+ data: { email, password: NEW_PASSWORD },+ });+ expect(newLogin.ok()).toBeTruthy();+ } finally {+ try {+ await Promise.all(keys.map((key) => cache.delete(key)));+ } finally {+ await cache.disconnect();+ await request.dispose();+ fs.rmSync(cwd, { recursive: true, force: true });+ await deleteUserByEmail(email);+ }+ }+ });});diff --git a/packages/api/src/auth/credentials.eviction.spec.ts b/packages/api/src/auth/credentials.eviction.spec.tsnew file mode 100644index 0000000000..db254d9422--- /dev/null+++ b/packages/api/src/auth/credentials.eviction.spec.ts@@ -0,0 +1,85 @@+import { Keyv } from 'keyv';+import mongoose from 'mongoose';+import { MongoMemoryServer } from 'mongodb-memory-server';+import { createMethods, createModels } from '@librechat/data-schemas';+import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';+import type { IUser } from '@librechat/data-schemas';+import {+ getCachedAuthUserDoc,+ setCachedAuthUserDoc,+ buildAuthUserDocCacheKey,+ AUTH_USER_DOC_CACHE_TTL_MS,+} from './userDocCache';+import { isTokenIssuedBeforeCredentialChange } from './credentials';++let mongoServer: MongoMemoryServer;+const originalCacheMode = process.env.AUTH_USER_CACHE_MODE;++beforeAll(async () => {+ mongoServer = await MongoMemoryServer.create();+ await mongoose.connect(mongoServer.getUri());+ createModels(mongoose);+});++afterAll(async () => {+ await mongoose.disconnect();+ await mongoServer.stop();+});++afterEach(() => {+ if (originalCacheMode === undefined) {+ delete process.env.AUTH_USER_CACHE_MODE;+ } else {+ process.env.AUTH_USER_CACHE_MODE = originalCacheMode;+ }+});++describe('credential change with a failing auth cache', () => {+ it('rejects an old OpenID access token once the credential barrier passes, even when the reverse index cannot be read', async () => {+ process.env.AUTH_USER_CACHE_MODE = 'on';+ const store = new Keyv();+ /** The index read fails the way a throwing Redis store rejects a WRONGTYPE reply. */+ const failingStore = {+ get: async (key: string) => {+ if (key.startsWith(AUTH_USER_DOC_BY_ID_PREFIX)) {+ throw new Error('WRONGTYPE Operation against a key holding the wrong kind of value');+ }+ return store.get(key);+ },+ set: (key: string, value: unknown, ttl?: number) => store.set(key, value, ttl),+ delete: (key: string) => store.delete(key),+ };+ const methods = createMethods(mongoose, {+ getCache: (key: string) => (key === CacheKeys.AUTH_USER_DOC ? failingStore : undefined),+ });+ const user = (await methods.createUser(+ { email: '[email protected]', provider: 'openid', openidId: 'subject-1' },+ undefined,+ true,+ true,+ )) as Partial<IUser>;+ const userId = String(user._id);+ const cacheKey = buildAuthUserDocCacheKey({ strategy: 'openid-jwt', subject: 'subject-1' });+ if (!cacheKey) {+ throw new Error('expected a cache key');+ }+ await setCachedAuthUserDoc(store, cacheKey, user);+ const oldToken = { iat: Math.floor(Date.now() / 1000) - 60 };+ expect(+ isTokenIssuedBeforeCredentialChange(oldToken, await getCachedAuthUserDoc(store, cacheKey)),+ ).toBe(false);++ const updated = await methods.updateUser(userId, {+ password: 'new-password-hash',+ credentialsChangedAt: new Date(),+ });+ const startedAt = Date.now();+ await methods.awaitAuthUserDocEviction(userId);++ expect(Date.now() - startedAt).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS);+ expect(updated?.credentialsChangedAt).toBeInstanceOf(Date);+ await expect(getCachedAuthUserDoc(store, cacheKey)).resolves.toBeUndefined();+ const current = await methods.findUser({ openidId: 'subject-1' });+ expect(isTokenIssuedBeforeCredentialChange(oldToken, current)).toBe(true);+ }, 20_000);+});diff --git a/packages/api/src/auth/passwordReset.ts b/packages/api/src/auth/passwordReset.tsindex 3c3a2bf75b..50fc044b99 100644--- a/packages/api/src/auth/passwordReset.ts+++ b/packages/api/src/auth/passwordReset.ts@@ -45,6 +45,8 @@ export interface PasswordResetControllerDeps {) => Promise<{ message: string } | Error>;deleteAllUserSessions: (query: { userId: string }) => Promise<{ deletedCount?: number }>;deletePasskeysByUser: (userId: string) => Promise<{ deletedCount?: number }>;+ /** Resolves once no cached auth document from before the reset can be served. */+ awaitAuthUserDocEviction: (userId: string) => Promise<void>;}export type PasswordResetOutcome =@@ -81,6 +83,8 @@ export function createResetPasswordController(deps: PasswordResetControllerDeps)passkeys.reason,);}+ /** After the revocations, so nothing can mint a post-stamp token while it waits. */+ await deps.awaitAuthUserDocEviction(userId);return res.status(200).json(result);} catch (error) {diff --git a/packages/api/src/auth/userDocCache.spec.ts b/packages/api/src/auth/userDocCache.spec.tsindex 4c70c438ab..766e33a4d9 100644--- a/packages/api/src/auth/userDocCache.spec.ts+++ b/packages/api/src/auth/userDocCache.spec.ts@@ -13,6 +13,7 @@ import {import { cacheConfig } from '~/cache/cacheConfig';jest.mock('@librechat/data-schemas', () => ({+ ...jest.requireActual('@librechat/data-schemas'),logger: {warn: jest.fn(),},@@ -188,10 +189,33 @@ describe('auth user document cache helpers', () => {expect(store.set).toHaveBeenCalledWith(buildAuthUserDocReverseIndexKey(userId.toString()),[cacheKey],- AUTH_USER_DOC_CACHE_TTL_MS,+ AUTH_USER_DOC_CACHE_TTL_MS * 2,);});+ it('does not cache a document the reverse index could not record', async () => {+ const store = makeStore();+ const userId = new Types.ObjectId();+ const indexKey = buildAuthUserDocReverseIndexKey(userId.toString());+ store.set.mockImplementation(async (key: string, value: unknown) => {+ if (key === indexKey) {+ throw new Error('redis unavailable');+ }+ store.values.set(key, value);+ return true;+ });++ await setCachedAuthUserDoc(store, 'auth-user-doc:v2:orphan', {+ _id: userId,+ email: '[email protected]',+ });++ expect(store.values.has('auth-user-doc:v2:orphan')).toBe(false);+ expect(logger.warn).toHaveBeenCalledWith('[authUserDocCache] Cache write failed', {+ error: 'redis unavailable',+ });+ });+it('deduplicates reverse-index keys and caps the remembered set', async () => {const store = makeStore();const objectId = new Types.ObjectId();diff --git a/packages/api/src/auth/userDocCache.ts b/packages/api/src/auth/userDocCache.tsindex 7a308e3823..4fa91a4772 100644--- a/packages/api/src/auth/userDocCache.ts+++ b/packages/api/src/auth/userDocCache.ts@@ -1,11 +1,15 @@import { createHash } from 'crypto';-import { logger } from '@librechat/data-schemas';-import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';+import { logger, evictAuthUserDocs } from '@librechat/data-schemas';+import {+ CacheKeys,+ AUTH_USER_DOC_BY_ID_PREFIX,+ AUTH_USER_DOC_CACHE_TTL_MS,+} from 'librechat-data-provider';import type { IUser } from '@librechat/data-schemas';import { cacheConfig } from '~/cache/cacheConfig';const AUTH_USER_DOC_CACHE_VERSION = 2;-export const AUTH_USER_DOC_CACHE_TTL_MS = 5000;+export { AUTH_USER_DOC_CACHE_TTL_MS };export type AuthUserDocCacheMode = 'off' | 'on';@@ -164,6 +168,10 @@ export async function getCachedAuthUserDoc(}}+/**+ * The reverse index is written first and outlives every document it names, so eviction can+ * always find a cached document: when the index write fails, the document is not cached.+ */export async function setCachedAuthUserDoc(store: AuthUserDocCacheStore,cacheKey: string,@@ -171,6 +179,10 @@ export async function setCachedAuthUserDoc(): Promise<void> {try {const sanitized = sanitizeUserForCache(user);+ const userId = getUserId(sanitized);+ if (userId) {+ await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS * 2);+ }await store.set(cacheKey,{@@ -180,10 +192,6 @@ export async function setCachedAuthUserDoc(} satisfies CachedAuthUserDoc,AUTH_USER_DOC_CACHE_TTL_MS,);- const userId = getUserId(sanitized);- if (userId) {- await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS);- }} catch (error) {logger.warn('[authUserDocCache] Cache write failed', {error: error instanceof Error ? error.message : String(error),@@ -198,25 +206,5 @@ export async function invalidateCachedAuthUserDoc(if (!store) {return;}- try {- const keys = new Set<string>();- if (input.cacheKey) {- keys.add(input.cacheKey);- }- if (input.userId) {- const indexKey = buildAuthUserDocReverseIndexKey(input.userId);- const indexed = await store.get<string[]>(indexKey);- if (Array.isArray(indexed)) {- for (const key of indexed) {- keys.add(key);- }- }- await store.delete(indexKey);- }- await Promise.all([...keys].map((key) => store.delete(key)));- } catch (error) {- logger.warn('[authUserDocCache] Cache invalidation failed', {- error: error instanceof Error ? error.message : String(error),- });- }+ await evictAuthUserDocs(store, input);}diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.tsindex 325bcd3e0f..dc2e77e005 100644--- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts@@ -189,6 +189,21 @@ describe('standardCache', () => {expect(cacheWithOpts.opts.ttl).toBe(5000);});+ test('rejects failed operations only for a cache created with throwOnErrors', async () => {+ const cacheFactory = await import('../../cacheFactory');+ const { ioredisClient } = await import('../../redisClients');+ if (!ioredisClient) throw new Error('ioredisClient is null');+ const strict = cacheFactory.standardCache('strict-namespace', undefined, undefined, {+ throwOnErrors: true,+ });+ const lenient = cacheFactory.standardCache('lenient-namespace');+ await ioredisClient.hset('strict-namespace:wrong-type', 'f', 'v');+ await ioredisClient.hset('lenient-namespace:wrong-type', 'f', 'v');++ await expect(strict.get('wrong-type')).rejects.toThrow(/WRONGTYPE/);+ await expect(lenient.get('wrong-type')).resolves.toBeUndefined();+ });+test('should handle TTL correctly', async () => {const cacheFactory = await import('../../cacheFactory');testCache = cacheFactory.standardCache('ttl-test', 1000); // 1 second TTLdiff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.tsindex 32a1492555..0ab85a87aa 100644--- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts@@ -67,7 +67,7 @@ describe('standardCache - CONFIG_STORE vs TOOL_CACHE namespace isolation', () =>const toolCache = standardCache(CacheKeys.TOOL_CACHE);expect(MockKeyvRedis).toHaveBeenCalledTimes(1);- expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient);+ expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient, { throwOnErrors: false });expect(toolCache).toBeDefined();});diff --git a/packages/api/src/cache/cacheFactory.ts b/packages/api/src/cache/cacheFactory.tsindex 07e443cac8..61e0d6d47a 100644--- a/packages/api/src/cache/cacheFactory.ts+++ b/packages/api/src/cache/cacheFactory.ts@@ -85,9 +85,17 @@ async function clearRedisNamespace(namespace: string): Promise<void> {* @param namespace - The cache namespace.* @param ttl - Time to live for cache entries.* @param fallbackStore - Optional fallback store if Redis is not used.+ * @param options.throwOnErrors - Rejects failed Redis operations instead of resolving them as a+ * miss or a no-op delete, for callers that must know an eviction did not happen. The first+ * caller's value wins for a shared namespace and TTL.* @returns Cache instance.*/-export const standardCache = (namespace: string, ttl?: number, fallbackStore?: object): Keyv => {+export const standardCache = (+ namespace: string,+ ttl?: number,+ fallbackStore?: object,+ options: { throwOnErrors?: boolean } = {},+): Keyv => {if (keyvRedisClient && !cacheConfig.FORCED_IN_MEMORY_CACHE_NAMESPACES?.includes(namespace)) {const byTtl = redisCacheMap.get(namespace);const existing = byTtl?.get(ttl);@@ -95,8 +103,9 @@ export const standardCache = (namespace: string, ttl?: number, fallbackStore?: oreturn existing;}try {- const keyvRedis = new KeyvRedis(keyvRedisClient);- const cache = new Keyv(keyvRedis, { namespace, ttl });+ const { throwOnErrors = false } = options;+ const keyvRedis = new KeyvRedis(keyvRedisClient, { throwOnErrors });+ const cache = new Keyv(keyvRedis, { namespace, ttl, throwOnErrors });keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX;keyvRedis.keyPrefixSeparator = cacheConfig.GLOBAL_PREFIX_SEPARATOR;diff --git a/packages/api/src/favorites/pinned.spec.ts b/packages/api/src/favorites/pinned.spec.tsindex 26e298db6e..c3a57e27f2 100644--- a/packages/api/src/favorites/pinned.spec.ts+++ b/packages/api/src/favorites/pinned.spec.ts@@ -11,6 +11,7 @@ import {import { createPinnedOrderHandlers } from './pinned';jest.mock('@librechat/data-schemas', () => ({+ ...jest.requireActual('@librechat/data-schemas'),logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },}));diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 6b43805364..0a20b6351d 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -4192,6 +4192,9 @@ export enum CacheKeys {export const AUTH_USER_DOC_BY_ID_PREFIX = 'auth-user-doc-byid';+/** Lifetime of a cached auth user document, and so the longest a stale one can be served. */+export const AUTH_USER_DOC_CACHE_TTL_MS = 5000;+/*** Enum for violation types, used to identify, log, and cache violations.*/diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.tsindex 44df9b4cd4..1ba6b2812d 100644--- a/packages/data-schemas/src/methods/user.methods.spec.ts+++ b/packages/data-schemas/src/methods/user.methods.spec.ts@@ -1,6 +1,10 @@import mongoose from 'mongoose';import { MongoMemoryServer } from 'mongodb-memory-server';-import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';+import {+ CacheKeys,+ AUTH_USER_DOC_BY_ID_PREFIX,+ AUTH_USER_DOC_CACHE_TTL_MS,+} from 'librechat-data-provider';import type * as t from '~/types';import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';import balanceSchema from '~/schema/balance';@@ -480,6 +484,119 @@ describe('User Methods - Database Tests', () => {expect(cache.delete).toHaveBeenCalledWith(indexKey);});+ describe('when auth cache eviction fails', () => {+ const cachedKeys = ['auth-cache-key-a', 'auth-cache-key-b'];++ /** Rejects the named operations the way the throwing Redis-backed auth cache does. */+ function makeFailingCache(failing: { indexRead?: boolean; deleteKey?: string }) {+ return {+ get: jest.fn(async () => {+ if (failing.indexRead) {+ throw new Error('WRONGTYPE');+ }+ return cachedKeys;+ }),+ set: jest.fn().mockResolvedValue(true),+ delete: jest.fn(async (key: string) => {+ if (key === failing.deleteKey) {+ throw new Error('redis unavailable');+ }+ return true;+ }),+ };+ }++ async function createCachedUser() {+ enableAuthUserDocCache();+ const user = await User.create({+ name: 'Credential User',+ email: '[email protected]',+ provider: 'openid',+ });+ return user._id?.toString() ?? '';+ }++ test.each<[string, { indexRead?: boolean; deleteKey?: string }]>([+ ['the reverse index read fails', { indexRead: true }],+ ['an indexed document delete fails', { deleteKey: 'auth-cache-key-a' }],+ ])('waits out the cache TTL at the credential barrier when %s', async (_label, failing) => {+ const userId = await createCachedUser();+ const cache = makeFailingCache(failing);+ const delay = jest.fn().mockResolvedValue(undefined);+ const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });++ const updated = await methodsWithCache.updateUser(userId, {+ password: 'new-password-hash',+ credentialsChangedAt: new Date(),+ });+ expect(updated?.credentialsChangedAt).toBeInstanceOf(Date);+ expect(delay).not.toHaveBeenCalled();++ await methodsWithCache.awaitAuthUserDocEviction(userId);++ expect(delay).toHaveBeenCalledTimes(1);+ expect(delay.mock.calls[0][0]).toBeGreaterThan(AUTH_USER_DOC_CACHE_TTL_MS);+ expect(cache.delete).not.toHaveBeenCalledWith(`${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`);+ if (!failing.indexRead) {+ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b');+ }+ });++ test('passes the barrier at once when only the index delete fails', async () => {+ const userId = await createCachedUser();+ const cache = makeFailingCache({ deleteKey: `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}` });+ const delay = jest.fn().mockResolvedValue(undefined);+ const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });++ await methodsWithCache.awaitAuthUserDocEviction(userId);++ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a');+ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b');+ expect(delay).not.toHaveBeenCalled();+ });++ test('passes the barrier at once when a retried eviction succeeds', async () => {+ const userId = await createCachedUser();+ const cache = makeFailingCache({ indexRead: true });+ const delay = jest.fn().mockResolvedValue(undefined);+ const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });++ await methodsWithCache.updateUser(userId, { credentialsChangedAt: new Date() });+ cache.get.mockResolvedValue(cachedKeys);+ await methodsWithCache.awaitAuthUserDocEviction(userId);++ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a');+ expect(delay).not.toHaveBeenCalled();+ });++ test('waits at the barrier when the cache can read but not delete', async () => {+ const userId = await createCachedUser();+ const delay = jest.fn().mockResolvedValue(undefined);+ const methodsWithCache = createUserMethods(mongoose, {+ getCache: () => ({ get: jest.fn(), set: jest.fn() }),+ delay,+ });++ await methodsWithCache.awaitAuthUserDocEviction(userId);++ expect(delay).toHaveBeenCalledTimes(1);+ });++ test('keeps updates themselves best effort', async () => {+ const userId = await createCachedUser();+ const delay = jest.fn().mockResolvedValue(undefined);+ const methodsWithCache = createUserMethods(mongoose, {+ getCache: () => makeFailingCache({ indexRead: true }),+ delay,+ });++ const updated = await methodsWithCache.updateUser(userId, { name: 'Renamed' });++ expect(updated?.name).toBe('Renamed');+ expect(delay).not.toHaveBeenCalled();+ });+ });+test('should invalidate cached auth user documents on delete', async () => {enableAuthUserDocCache();const user = await User.create({diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.tsindex 07de2ded81..cfa9057a10 100644--- a/packages/data-schemas/src/methods/user.ts+++ b/packages/data-schemas/src/methods/user.ts@@ -1,14 +1,16 @@import mongoose, { FilterQuery } from 'mongoose';import {- AUTH_USER_DOC_BY_ID_PREFIX,+ AUTH_USER_DOC_CACHE_TTL_MS,CacheKeys,type RefillIntervalUnit,type StatefulCodeEnvironment,} from 'librechat-data-provider';import type { IUser, BalanceConfig, CreateUserRequest, UserDeleteResult } from '~/types';import type { CacheStore } from '~/types';+import { evictAuthUserDocs } from '~/utils/eviction';import { escapeRegExp } from '~/utils/string';import { signPayload } from '~/crypto';+import logger from '~/config/winston';/** Default JWT session expiry: 15 minutes in milliseconds */export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15;@@ -19,8 +21,15 @@ const MAX_SUBAGENT_ADMISSION_FENCES = 32;interface UserMethodDeps {getCache?: (key: string) => CacheStore | undefined;+ /** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */+ delay?: (ms: number) => Promise<void>;}+const wait = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));++/** Timers can fire a millisecond early, so the wait outlasts the cache TTL by this much. */+const AUTH_USER_DOC_EXPIRY_MARGIN_MS = 100;+function isAuthUserDocCacheEnabled(): boolean {return process.env.AUTH_USER_CACHE_MODE === 'on';}@@ -55,6 +64,7 @@ export function createUserMethods(expectedState?: FilterQuery<IUser>,options?: { preserveExpiresAt?: boolean },) => Promise<IUser | null>;+ awaitAuthUserDocEviction: (userId: string) => Promise<void>;claimSamlIdentity: (userId: string,samlId: string,@@ -333,6 +343,24 @@ export function createUserMethods(return updated;}+ /**+ * The barrier a credential change passes before it is confirmed. A cached document without+ * the new credentialsChangedAt keeps pre-change access tokens verifying until it expires, so+ * eviction is retried and, when it still cannot remove every document, this resolves only+ * after the cache TTL. Callers revoke sessions and passkeys first: during the wait those+ * could otherwise mint access tokens issued after the stamp.+ */+ async function awaitAuthUserDocEviction(userId: string): Promise<void> {+ if (await invalidateAuthUserDocCache(userId)) {+ return;+ }+ logger.warn(+ '[awaitAuthUserDocEviction] Cached auth documents were not evicted after a credential change; waiting for them to expire',+ { userId, waitMs: AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS },+ );+ await (deps.delay ?? wait)(AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS);+ }+/** Atomically updates a SAML user only when the incoming identity can claim the document. */async function claimSamlIdentity(userId: string,@@ -358,26 +386,20 @@ export function createUserMethods(return updated;}- async function invalidateAuthUserDocCache(userId: string): Promise<void> {+ /** Resolves false only when a cached document for the user may still be served. */+ async function invalidateAuthUserDocCache(userId: string): Promise<boolean> {if (!isAuthUserDocCacheEnabled()) {- return;+ return true;}const cache = deps.getCache?.(CacheKeys.AUTH_USER_DOC);- if (!cache?.get || !cache?.delete) {- return;+ if (!cache?.get) {+ return true;}- try {- const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`;- const cachedKeys = await cache.get(indexKey);- if (Array.isArray(cachedKeys)) {- await Promise.all(- cachedKeys.map((key) => (typeof key === 'string' ? cache.delete?.(key) : undefined)),- );- }- await cache.delete(indexKey);- } catch {- // Cache invalidation must not make a user update fail.+ const remove = cache.delete?.bind(cache);+ if (!remove) {+ return false;}+ return evictAuthUserDocs({ get: (key) => cache.get(key), delete: remove }, { userId });}/**@@ -863,6 +885,7 @@ export function createUserMethods(countUsers,createUser,updateUser,+ awaitAuthUserDocEviction,claimSamlIdentity,acceptTerms,searchUsers,diff --git a/packages/data-schemas/src/utils/eviction.spec.ts b/packages/data-schemas/src/utils/eviction.spec.tsnew file mode 100644index 0000000000..4114788ab5--- /dev/null+++ b/packages/data-schemas/src/utils/eviction.spec.ts@@ -0,0 +1,111 @@+import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider';+import { evictAuthUserDocs } from './eviction';++const INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-1`;+const OTHER_INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-2`;++/** A Map-backed store that rejects the operations named for a key, as a throwing Redis store does. */+function makeStore(failures: { get?: string[]; delete?: string[] } = {}) {+ const values = new Map<string, unknown>([+ [INDEX_KEY, ['doc-a', 'doc-b']],+ ['doc-a', { id: 'user-1' }],+ ['doc-b', { id: 'user-1' }],+ ['doc-c', { id: 'user-1' }],+ [OTHER_INDEX_KEY, ['doc-other']],+ ['doc-other', { id: 'user-2' }],+ ]);+ return {+ values,+ get: jest.fn(async (key: string) => {+ if (failures.get?.includes(key)) {+ throw new Error(`GET ${key} failed`);+ }+ return values.get(key);+ }),+ delete: jest.fn(async (key: string) => {+ if (failures.delete?.includes(key)) {+ throw new Error(`DEL ${key} failed`);+ }+ return values.delete(key);+ }),+ };+}++function expectOtherUserCached(store: ReturnType<typeof makeStore>) {+ expect(store.values.get(OTHER_INDEX_KEY)).toEqual(['doc-other']);+ expect(store.values.get('doc-other')).toEqual({ id: 'user-2' });+}++describe('evictAuthUserDocs', () => {+ it('lets a retry find a document an earlier failed delete left behind', async () => {+ const store = makeStore({ delete: ['doc-a'] });+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);++ store.delete.mockImplementation(async (key: string) => store.values.delete(key));+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true);++ expect(store.values.has('doc-a')).toBe(false);+ expect(store.values.has(INDEX_KEY)).toBe(false);+ });++ it('deletes the indexed and explicit documents and the index', async () => {+ const store = makeStore();++ await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe(+ true,+ );++ for (const key of [INDEX_KEY, 'doc-a', 'doc-b', 'doc-c']) {+ expect(store.values.has(key)).toBe(false);+ }+ expectOtherUserCached(store);+ });++ it('reports a failed index read and keeps the index for a retry', async () => {+ const store = makeStore({ get: [INDEX_KEY] });++ await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe(+ false,+ );++ expect(store.values.has('doc-c')).toBe(false);+ expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);+ expect(store.values.has('doc-a')).toBe(true);+ expectOtherUserCached(store);+ });++ it.each([+ ['a string', 'doc-a'],+ ['a list with a non-string entry', ['doc-a', 42]],+ ])('treats a reverse index holding %s as unreadable', async (_label, indexed) => {+ const store = makeStore();+ store.values.set(INDEX_KEY, indexed);++ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);++ expect(store.values.get(INDEX_KEY)).toEqual(indexed);+ expect(store.values.has('doc-a')).toBe(true);+ });++ it('reports a failed document delete, deletes the others and keeps the index for a retry', async () => {+ const store = makeStore({ delete: ['doc-a'] });++ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);++ expect(store.values.has('doc-a')).toBe(true);+ expect(store.values.has('doc-b')).toBe(false);+ expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);+ expectOtherUserCached(store);+ });++ it('counts documents as evicted when only the index delete fails', async () => {+ const store = makeStore({ delete: [INDEX_KEY] });++ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true);++ expect(store.values.has('doc-a')).toBe(false);+ expect(store.values.has('doc-b')).toBe(false);+ expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);+ expectOtherUserCached(store);+ });+});diff --git a/packages/data-schemas/src/utils/eviction.ts b/packages/data-schemas/src/utils/eviction.tsnew file mode 100644index 0000000000..f238b38a32--- /dev/null+++ b/packages/data-schemas/src/utils/eviction.ts@@ -0,0 +1,77 @@+import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider';+import logger from '~/config/winston';++/** Store operations eviction needs; a failure is only seen when the store throws it. */+export interface AuthUserDocEvictionStore {+ get: (key: string) => Promise<unknown>;+ delete: (key: string) => Promise<unknown>;+}++function isKeyList(value: unknown): value is string[] {+ return Array.isArray(value) && value.every((key) => typeof key === 'string');+}++function describeError(error: unknown): string {+ return error instanceof Error ? error.message : String(error);+}++/**+ * Deletes the cached auth user documents for a user, attempting every delete even when+ * another fails. Resolves false when a document may still be served: the reverse index could+ * not be read, so its keys are unknown, or a document delete failed. The index is deleted only+ * after every document it names is gone, so a retry can still find what an earlier attempt+ * left behind. A failed delete of the index alone leaves only the names of deleted keys, so it+ * is logged and still counts as evicted. Never throws, so callers keep their own write results.+ */+export async function evictAuthUserDocs(+ store: AuthUserDocEvictionStore,+ input: { userId?: string; cacheKey?: string },+): Promise<boolean> {+ const keys = new Set<string>(input.cacheKey ? [input.cacheKey] : []);+ const indexKey = input.userId ? `${AUTH_USER_DOC_BY_ID_PREFIX}:${input.userId}` : undefined;+ let indexRead = true;+ if (indexKey) {+ try {+ const indexed = await store.get(indexKey);+ if (indexed != null && !isKeyList(indexed)) {+ throw new Error('Reverse index is not a list of cache keys');+ }+ for (const key of indexed ?? []) {+ keys.add(key);+ }+ } catch (error) {+ indexRead = false;+ logger.warn('[authUserDocCache] Reverse index read failed during eviction', {+ userId: input.userId,+ error: describeError(error),+ });+ }+ }++ const results = await Promise.allSettled([...keys].map((key) => store.delete(key)));+ let documentsDeleted = true;+ for (const result of results) {+ if (result.status === 'rejected') {+ documentsDeleted = false;+ logger.warn('[authUserDocCache] Cached document delete failed during eviction', {+ userId: input.userId,+ error: describeError(result.reason),+ });+ }+ }+ if (!indexRead || !documentsDeleted) {+ return false;+ }++ if (indexKey) {+ try {+ await store.delete(indexKey);+ } catch (error) {+ logger.warn('[authUserDocCache] Reverse index delete failed during eviction', {+ userId: input.userId,+ error: describeError(error),+ });+ }+ }+ return true;+}diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.tsindex fecdfcc7ed..f5a4c6fd48 100644--- a/packages/data-schemas/src/utils/index.ts+++ b/packages/data-schemas/src/utils/index.ts@@ -8,5 +8,6 @@ export * from './objectId';export * from './yaml';export * from './stripUIResourceMarkers';export * from './fading';+export * from './eviction';export { buildIndexWithRetry, createIndexesWithRetry, isIndexBuildInProgress } from './retry';export type { IndexBuildOptions } from './retry';