Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧯 fix: Outlast Failed Auth Cache Eviction on Credential Changes (#16435)

Marco Beretta26b69883556e
diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.js
index 1edae18156..53d73125df 100644
--- a/api/cache/getLogStores.js
+++ b/api/cache/getLogStores.js
@@ -66,7 +66,9 @@ const namespaces = {
CacheKeys.OPENID_EXCHANGED_TOKENS,
Time.TEN_MINUTES,
),
- [CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC),
+ [CacheKeys.AUTH_USER_DOC]: standardCache(CacheKeys.AUTH_USER_DOC, undefined, undefined, {
+ throwOnErrors: true,
+ }),
[CacheKeys.ADMIN_OAUTH_EXCHANGE]: standardCache(
CacheKeys.ADMIN_OAUTH_EXCHANGE,
Time.THIRTY_SECONDS,
diff --git a/api/server/controllers/AuthController.js b/api/server/controllers/AuthController.js
index 107f0dbdff..f5e3a230f0 100644
--- a/api/server/controllers/AuthController.js
+++ b/api/server/controllers/AuthController.js
@@ -24,6 +24,7 @@ const {
const {
deleteAllUserSessions,
deletePasskeysByUser,
+ awaitAuthUserDocEviction,
getUserById,
findSession,
updateUser,
@@ -326,6 +327,7 @@ const resetPasswordController = createResetPasswordController({
resetPassword,
deleteAllUserSessions,
deletePasskeysByUser,
+ awaitAuthUserDocEviction,
});
const refreshController = async (req, res) => {
diff --git a/api/server/controllers/AuthController.spec.js b/api/server/controllers/AuthController.spec.js
index 50090636ea..6a1866c1f9 100644
--- a/api/server/controllers/AuthController.spec.js
+++ b/api/server/controllers/AuthController.spec.js
@@ -42,6 +42,7 @@ jest.mock('~/models', () => ({
deleteSession: jest.fn(),
deleteAllUserSessions: jest.fn(),
deletePasskeysByUser: jest.fn(),
+ awaitAuthUserDocEviction: jest.fn(),
getUserById: jest.fn(),
findSession: jest.fn(),
updateUser: jest.fn(),
@@ -147,6 +148,7 @@ const {
deleteTokens,
deleteAllUserSessions,
deletePasskeysByUser,
+ awaitAuthUserDocEviction,
} = require('~/models');
const {
createRefreshTokenBridgeFlightKey,
@@ -2483,6 +2485,7 @@ describe('resetPasswordController', () => {
resetPassword.mockReset().mockResolvedValue(serviceResult);
deleteAllUserSessions.mockReset().mockResolvedValue(undefined);
deletePasskeysByUser.mockReset().mockResolvedValue(undefined);
+ awaitAuthUserDocEviction.mockReset().mockResolvedValue(undefined);
req = {
body: {
userId: 'user-123',
@@ -2511,6 +2514,31 @@ describe('resetPasswordController', () => {
expect(res.json).toHaveBeenCalledWith(serviceResult);
});
+ it('confirms the reset only after revoking and settling the auth cache', async () => {
+ let finishEviction;
+ awaitAuthUserDocEviction.mockReturnValue(
+ new Promise((resolve) => {
+ finishEviction = resolve;
+ }),
+ );
+
+ const reset = resetPasswordController(req, res);
+ await new Promise((resolve) => setImmediate(resolve));
+ const responseCalls = res.status.mock.calls.length;
+ finishEviction();
+ await reset;
+
+ expect(awaitAuthUserDocEviction).toHaveBeenCalledWith('user-123');
+ expect(deleteAllUserSessions.mock.invocationCallOrder[0]).toBeLessThan(
+ awaitAuthUserDocEviction.mock.invocationCallOrder[0],
+ );
+ expect(deletePasskeysByUser.mock.invocationCallOrder[0]).toBeLessThan(
+ awaitAuthUserDocEviction.mock.invocationCallOrder[0],
+ );
+ expect(responseCalls).toBe(0);
+ expect(res.status).toHaveBeenCalledWith(200);
+ });
+
it('does not revoke sessions or passkeys when reset fails', async () => {
const resetError = new Error('Invalid token');
resetPassword.mockResolvedValue(resetError);
@@ -2519,6 +2547,7 @@ describe('resetPasswordController', () => {
expect(deleteAllUserSessions).not.toHaveBeenCalled();
expect(deletePasskeysByUser).not.toHaveBeenCalled();
+ expect(awaitAuthUserDocEviction).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(400);
expect(res.json).toHaveBeenCalledWith(resetError);
});
diff --git a/config/reset-password.js b/config/reset-password.js
index 1d239f58b1..3144ade839 100644
--- a/config/reset-password.js
+++ b/config/reset-password.js
@@ -85,9 +85,15 @@ const resetPassword = async () => {
* A passkey signs in on its own, so leaving one in place would keep an attacker
* logged in after an administrator believes the account has been recovered.
*/
- const { deletedCount } = await Passkey.deleteMany({ user: user._id });
- /** A refresh session outlives the stamp: refreshing mints a token issued after it. */
- await Session.deleteMany({ user: user._id });
+ let deletedCount;
+ try {
+ ({ deletedCount } = await Passkey.deleteMany({ user: user._id }));
+ /** A refresh session outlives the stamp: refreshing mints a token issued after it. */
+ await Session.deleteMany({ user: user._id });
+ } finally {
+ /** The password is already changed, so the cache settles even when a revocation fails. */
+ await methods.awaitAuthUserDocEviction(user._id.toString());
+ }
console.log('Password successfully reset!');
if (deletedCount > 0) {
diff --git a/e2e/specs/mock/scenarios/reset-password.spec.ts b/e2e/specs/mock/scenarios/reset-password.spec.ts
index fc14e00759..a25739132a 100644
--- a/e2e/specs/mock/scenarios/reset-password.spec.ts
+++ b/e2e/specs/mock/scenarios/reset-password.spec.ts
@@ -4,7 +4,11 @@ import os from 'os';
import { Keyv } from 'keyv';
import KeyvRedis from '@keyv/redis';
import { expect, test } from '@playwright/test';
-import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
+import {
+ CacheKeys,
+ AUTH_USER_DOC_BY_ID_PREFIX,
+ AUTH_USER_DOC_CACHE_TTL_MS,
+} from 'librechat-data-provider';
import { spawn } from 'node:child_process';
import { randomInt, randomUUID } from 'node:crypto';
import { withMongo, seedPasskey, deleteUserByEmail } from '../db';
@@ -40,7 +44,13 @@ function runResetCli(
email: string,
cwd: string,
env: NodeJS.ProcessEnv,
-): Promise<{ code: number | null; stdout: string; stderr: string }> {
+): Promise<{
+ code: number | null;
+ stdout: string;
+ stderr: string;
+ /** Milliseconds between confirming the new password and the CLI reporting success. */
+ reportDelayMs?: number;
+}> {
const child = spawn(process.execPath, [path.join(REPO_ROOT, 'config/reset-password.js')], {
cwd,
env: {
@@ -59,14 +69,20 @@ function runResetCli(
['Confirm new password: ', NEW_PASSWORD],
];
let stdout = '';
+ let confirmedAt: number | undefined;
+ let reportedAt: number | undefined;
child.stdout?.on('data', (chunk: Buffer) => {
stdout += chunk.toString();
answers.forEach((answer, index) => {
if (answer !== null && stdout.includes(answer[0])) {
child.stdin?.write(`${answer[1]}\n`);
answers[index] = null;
+ confirmedAt = Date.now();
}
});
+ if (reportedAt === undefined && stdout.includes('Password successfully reset!')) {
+ reportedAt = Date.now();
+ }
});
let stderr = '';
child.stderr?.on('data', (chunk: Buffer) => (stderr += chunk.toString()));
@@ -82,7 +98,11 @@ function runResetCli(
child.on('close', (code) => {
clearTimeout(timer);
child.stdin?.end();
- resolve({ code, stdout, stderr });
+ const reportDelayMs =
+ confirmedAt !== undefined && reportedAt !== undefined
+ ? reportedAt - confirmedAt
+ : undefined;
+ resolve({ code, stdout, stderr, reportDelayMs });
});
});
}
@@ -230,4 +250,85 @@ test.describe('CLI password reset', () => {
}
});
}
+
+ test('reports the reset only after unevictable cached credentials expire @scenario:cli-reset-outlasts-failed-cache-eviction', async ({
+ playwright,
+ baseURL,
+ }) => {
+ test.setTimeout(90_000);
+ const request = await playwright.request.newContext({
+ baseURL,
+ extraHTTPHeaders: {
+ 'X-Forwarded-For': `127.${randomInt(1, 255)}.${randomInt(1, 255)}.${randomInt(1, 255)}`,
+ },
+ });
+ const email = `cli-evict-${randomUUID().slice(0, 8)}@example.com`;
+ const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-reset-'));
+ const redisUri = process.env.REDIS_URI ?? 'redis://127.0.0.1:6379';
+ const prefix = `cli-evict-${randomUUID()}`;
+ const adapter = new KeyvRedis(redisUri, { throwOnErrors: true, connectionTimeout: 5000 });
+ const cache = new Keyv(adapter, { namespace: CacheKeys.AUTH_USER_DOC });
+ adapter.namespace = prefix;
+ adapter.keyPrefixSeparator = '::';
+ const keys: string[] = [];
+ try {
+ const register = await request.post('/api/auth/register', {
+ data: { email, name: 'CLI Evict', password: PASSWORD, confirm_password: PASSWORD },
+ });
+ expect(register.ok()).toBeTruthy();
+ const login = await request.post('/api/auth/login', { data: { email, password: PASSWORD } });
+ expect(login.ok()).toBeTruthy();
+ const { token, user } = (await login.json()) as { token: string; user: { _id: string } };
+ await seedPasskey(email, randomUUID(), 'Eviction regression');
+
+ const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${user._id}`;
+ const controlKey = 'auth-user-doc:another-user';
+ keys.push(indexKey, controlKey);
+ await cache.set(controlKey, { credentialsChangedAt: null }, 120_000);
+ /** A hash where the index string belongs makes the CLI's GET fail with WRONGTYPE. */
+ const client = await adapter.getClient();
+ await client.hSet(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 'unreadable', '1');
+ await client.pExpire(`${prefix}::${CacheKeys.AUTH_USER_DOC}:${indexKey}`, 120_000);
+
+ const env = { ...process.env };
+ Object.assign(env, {
+ USE_REDIS: 'true',
+ USE_REDIS_CLUSTER: 'false',
+ REDIS_URI: redisUri,
+ REDIS_KEY_PREFIX: prefix,
+ REDIS_KEY_PREFIX_VAR: '',
+ AUTH_USER_CACHE_MODE: 'on',
+ FORCED_IN_MEMORY_CACHE_NAMESPACES: '',
+ });
+
+ const cli = await runResetCli(email, cwd, env);
+ expect(cli.code, cli.stderr.slice(-400)).toBe(0);
+ expect(cli.stdout).toContain('Password successfully reset!');
+ expect(cli.reportDelayMs).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS);
+ expect(await cache.get(controlKey)).toEqual({ credentialsChangedAt: null });
+ await withMongo(async (db) => {
+ const resetUser = await db.collection('users').findOne({ email });
+ expect(resetUser?.credentialsChangedAt).toBeTruthy();
+ expect(await db.collection('passkeys').countDocuments({ user: resetUser!._id })).toBe(0);
+ expect(await db.collection('sessions').countDocuments({ user: resetUser!._id })).toBe(0);
+ });
+ const rejected = await request.get('/api/user', {
+ headers: { Authorization: `Bearer ${token}` },
+ });
+ expect(rejected.status()).toBe(401);
+ const newLogin = await request.post('/api/auth/login', {
+ data: { email, password: NEW_PASSWORD },
+ });
+ expect(newLogin.ok()).toBeTruthy();
+ } finally {
+ try {
+ await Promise.all(keys.map((key) => cache.delete(key)));
+ } finally {
+ await cache.disconnect();
+ await request.dispose();
+ fs.rmSync(cwd, { recursive: true, force: true });
+ await deleteUserByEmail(email);
+ }
+ }
+ });
});
diff --git a/packages/api/src/auth/credentials.eviction.spec.ts b/packages/api/src/auth/credentials.eviction.spec.ts
new file mode 100644
index 0000000000..db254d9422
--- /dev/null
+++ b/packages/api/src/auth/credentials.eviction.spec.ts
@@ -0,0 +1,85 @@
+import { Keyv } from 'keyv';
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import { createMethods, createModels } from '@librechat/data-schemas';
+import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
+import type { IUser } from '@librechat/data-schemas';
+import {
+ getCachedAuthUserDoc,
+ setCachedAuthUserDoc,
+ buildAuthUserDocCacheKey,
+ AUTH_USER_DOC_CACHE_TTL_MS,
+} from './userDocCache';
+import { isTokenIssuedBeforeCredentialChange } from './credentials';
+
+let mongoServer: MongoMemoryServer;
+const originalCacheMode = process.env.AUTH_USER_CACHE_MODE;
+
+beforeAll(async () => {
+ mongoServer = await MongoMemoryServer.create();
+ await mongoose.connect(mongoServer.getUri());
+ createModels(mongoose);
+});
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer.stop();
+});
+
+afterEach(() => {
+ if (originalCacheMode === undefined) {
+ delete process.env.AUTH_USER_CACHE_MODE;
+ } else {
+ process.env.AUTH_USER_CACHE_MODE = originalCacheMode;
+ }
+});
+
+describe('credential change with a failing auth cache', () => {
+ it('rejects an old OpenID access token once the credential barrier passes, even when the reverse index cannot be read', async () => {
+ process.env.AUTH_USER_CACHE_MODE = 'on';
+ const store = new Keyv();
+ /** The index read fails the way a throwing Redis store rejects a WRONGTYPE reply. */
+ const failingStore = {
+ get: async (key: string) => {
+ if (key.startsWith(AUTH_USER_DOC_BY_ID_PREFIX)) {
+ throw new Error('WRONGTYPE Operation against a key holding the wrong kind of value');
+ }
+ return store.get(key);
+ },
+ set: (key: string, value: unknown, ttl?: number) => store.set(key, value, ttl),
+ delete: (key: string) => store.delete(key),
+ };
+ const methods = createMethods(mongoose, {
+ getCache: (key: string) => (key === CacheKeys.AUTH_USER_DOC ? failingStore : undefined),
+ });
+ const user = (await methods.createUser(
+ { email: '[email protected]', provider: 'openid', openidId: 'subject-1' },
+ undefined,
+ true,
+ true,
+ )) as Partial<IUser>;
+ const userId = String(user._id);
+ const cacheKey = buildAuthUserDocCacheKey({ strategy: 'openid-jwt', subject: 'subject-1' });
+ if (!cacheKey) {
+ throw new Error('expected a cache key');
+ }
+ await setCachedAuthUserDoc(store, cacheKey, user);
+ const oldToken = { iat: Math.floor(Date.now() / 1000) - 60 };
+ expect(
+ isTokenIssuedBeforeCredentialChange(oldToken, await getCachedAuthUserDoc(store, cacheKey)),
+ ).toBe(false);
+
+ const updated = await methods.updateUser(userId, {
+ password: 'new-password-hash',
+ credentialsChangedAt: new Date(),
+ });
+ const startedAt = Date.now();
+ await methods.awaitAuthUserDocEviction(userId);
+
+ expect(Date.now() - startedAt).toBeGreaterThanOrEqual(AUTH_USER_DOC_CACHE_TTL_MS);
+ expect(updated?.credentialsChangedAt).toBeInstanceOf(Date);
+ await expect(getCachedAuthUserDoc(store, cacheKey)).resolves.toBeUndefined();
+ const current = await methods.findUser({ openidId: 'subject-1' });
+ expect(isTokenIssuedBeforeCredentialChange(oldToken, current)).toBe(true);
+ }, 20_000);
+});
diff --git a/packages/api/src/auth/passwordReset.ts b/packages/api/src/auth/passwordReset.ts
index 3c3a2bf75b..50fc044b99 100644
--- a/packages/api/src/auth/passwordReset.ts
+++ b/packages/api/src/auth/passwordReset.ts
@@ -45,6 +45,8 @@ export interface PasswordResetControllerDeps {
) => Promise<{ message: string } | Error>;
deleteAllUserSessions: (query: { userId: string }) => Promise<{ deletedCount?: number }>;
deletePasskeysByUser: (userId: string) => Promise<{ deletedCount?: number }>;
+ /** Resolves once no cached auth document from before the reset can be served. */
+ awaitAuthUserDocEviction: (userId: string) => Promise<void>;
}
export type PasswordResetOutcome =
@@ -81,6 +83,8 @@ export function createResetPasswordController(deps: PasswordResetControllerDeps)
passkeys.reason,
);
}
+ /** After the revocations, so nothing can mint a post-stamp token while it waits. */
+ await deps.awaitAuthUserDocEviction(userId);
return res.status(200).json(result);
} catch (error) {
diff --git a/packages/api/src/auth/userDocCache.spec.ts b/packages/api/src/auth/userDocCache.spec.ts
index 4c70c438ab..766e33a4d9 100644
--- a/packages/api/src/auth/userDocCache.spec.ts
+++ b/packages/api/src/auth/userDocCache.spec.ts
@@ -13,6 +13,7 @@ import {
import { cacheConfig } from '~/cache/cacheConfig';
jest.mock('@librechat/data-schemas', () => ({
+ ...jest.requireActual('@librechat/data-schemas'),
logger: {
warn: jest.fn(),
},
@@ -188,10 +189,33 @@ describe('auth user document cache helpers', () => {
expect(store.set).toHaveBeenCalledWith(
buildAuthUserDocReverseIndexKey(userId.toString()),
[cacheKey],
- AUTH_USER_DOC_CACHE_TTL_MS,
+ AUTH_USER_DOC_CACHE_TTL_MS * 2,
);
});
+ it('does not cache a document the reverse index could not record', async () => {
+ const store = makeStore();
+ const userId = new Types.ObjectId();
+ const indexKey = buildAuthUserDocReverseIndexKey(userId.toString());
+ store.set.mockImplementation(async (key: string, value: unknown) => {
+ if (key === indexKey) {
+ throw new Error('redis unavailable');
+ }
+ store.values.set(key, value);
+ return true;
+ });
+
+ await setCachedAuthUserDoc(store, 'auth-user-doc:v2:orphan', {
+ _id: userId,
+ email: '[email protected]',
+ });
+
+ expect(store.values.has('auth-user-doc:v2:orphan')).toBe(false);
+ expect(logger.warn).toHaveBeenCalledWith('[authUserDocCache] Cache write failed', {
+ error: 'redis unavailable',
+ });
+ });
+
it('deduplicates reverse-index keys and caps the remembered set', async () => {
const store = makeStore();
const objectId = new Types.ObjectId();
diff --git a/packages/api/src/auth/userDocCache.ts b/packages/api/src/auth/userDocCache.ts
index 7a308e3823..4fa91a4772 100644
--- a/packages/api/src/auth/userDocCache.ts
+++ b/packages/api/src/auth/userDocCache.ts
@@ -1,11 +1,15 @@
import { createHash } from 'crypto';
-import { logger } from '@librechat/data-schemas';
-import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
+import { logger, evictAuthUserDocs } from '@librechat/data-schemas';
+import {
+ CacheKeys,
+ AUTH_USER_DOC_BY_ID_PREFIX,
+ AUTH_USER_DOC_CACHE_TTL_MS,
+} from 'librechat-data-provider';
import type { IUser } from '@librechat/data-schemas';
import { cacheConfig } from '~/cache/cacheConfig';
const AUTH_USER_DOC_CACHE_VERSION = 2;
-export const AUTH_USER_DOC_CACHE_TTL_MS = 5000;
+export { AUTH_USER_DOC_CACHE_TTL_MS };
export type AuthUserDocCacheMode = 'off' | 'on';
@@ -164,6 +168,10 @@ export async function getCachedAuthUserDoc(
}
}
+/**
+ * The reverse index is written first and outlives every document it names, so eviction can
+ * always find a cached document: when the index write fails, the document is not cached.
+ */
export async function setCachedAuthUserDoc(
store: AuthUserDocCacheStore,
cacheKey: string,
@@ -171,6 +179,10 @@ export async function setCachedAuthUserDoc(
): Promise<void> {
try {
const sanitized = sanitizeUserForCache(user);
+ const userId = getUserId(sanitized);
+ if (userId) {
+ await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS * 2);
+ }
await store.set(
cacheKey,
{
@@ -180,10 +192,6 @@ export async function setCachedAuthUserDoc(
} satisfies CachedAuthUserDoc,
AUTH_USER_DOC_CACHE_TTL_MS,
);
- const userId = getUserId(sanitized);
- if (userId) {
- await rememberUserCacheKey(store, userId, cacheKey, AUTH_USER_DOC_CACHE_TTL_MS);
- }
} catch (error) {
logger.warn('[authUserDocCache] Cache write failed', {
error: error instanceof Error ? error.message : String(error),
@@ -198,25 +206,5 @@ export async function invalidateCachedAuthUserDoc(
if (!store) {
return;
}
- try {
- const keys = new Set<string>();
- if (input.cacheKey) {
- keys.add(input.cacheKey);
- }
- if (input.userId) {
- const indexKey = buildAuthUserDocReverseIndexKey(input.userId);
- const indexed = await store.get<string[]>(indexKey);
- if (Array.isArray(indexed)) {
- for (const key of indexed) {
- keys.add(key);
- }
- }
- await store.delete(indexKey);
- }
- await Promise.all([...keys].map((key) => store.delete(key)));
- } catch (error) {
- logger.warn('[authUserDocCache] Cache invalidation failed', {
- error: error instanceof Error ? error.message : String(error),
- });
- }
+ await evictAuthUserDocs(store, input);
}
diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts
index 325bcd3e0f..dc2e77e005 100644
--- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts
+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.cache_integration.spec.ts
@@ -189,6 +189,21 @@ describe('standardCache', () => {
expect(cacheWithOpts.opts.ttl).toBe(5000);
});
+ test('rejects failed operations only for a cache created with throwOnErrors', async () => {
+ const cacheFactory = await import('../../cacheFactory');
+ const { ioredisClient } = await import('../../redisClients');
+ if (!ioredisClient) throw new Error('ioredisClient is null');
+ const strict = cacheFactory.standardCache('strict-namespace', undefined, undefined, {
+ throwOnErrors: true,
+ });
+ const lenient = cacheFactory.standardCache('lenient-namespace');
+ await ioredisClient.hset('strict-namespace:wrong-type', 'f', 'v');
+ await ioredisClient.hset('lenient-namespace:wrong-type', 'f', 'v');
+
+ await expect(strict.get('wrong-type')).rejects.toThrow(/WRONGTYPE/);
+ await expect(lenient.get('wrong-type')).resolves.toBeUndefined();
+ });
+
test('should handle TTL correctly', async () => {
const cacheFactory = await import('../../cacheFactory');
testCache = cacheFactory.standardCache('ttl-test', 1000); // 1 second TTL
diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts
index 32a1492555..0ab85a87aa 100644
--- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts
+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts
@@ -67,7 +67,7 @@ describe('standardCache - CONFIG_STORE vs TOOL_CACHE namespace isolation', () =>
const toolCache = standardCache(CacheKeys.TOOL_CACHE);
expect(MockKeyvRedis).toHaveBeenCalledTimes(1);
- expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient);
+ expect(MockKeyvRedis).toHaveBeenCalledWith(mockKeyvRedisClient, { throwOnErrors: false });
expect(toolCache).toBeDefined();
});
diff --git a/packages/api/src/cache/cacheFactory.ts b/packages/api/src/cache/cacheFactory.ts
index 07e443cac8..61e0d6d47a 100644
--- a/packages/api/src/cache/cacheFactory.ts
+++ b/packages/api/src/cache/cacheFactory.ts
@@ -85,9 +85,17 @@ async function clearRedisNamespace(namespace: string): Promise<void> {
* @param namespace - The cache namespace.
* @param ttl - Time to live for cache entries.
* @param fallbackStore - Optional fallback store if Redis is not used.
+ * @param options.throwOnErrors - Rejects failed Redis operations instead of resolving them as a
+ * miss or a no-op delete, for callers that must know an eviction did not happen. The first
+ * caller's value wins for a shared namespace and TTL.
* @returns Cache instance.
*/
-export const standardCache = (namespace: string, ttl?: number, fallbackStore?: object): Keyv => {
+export const standardCache = (
+ namespace: string,
+ ttl?: number,
+ fallbackStore?: object,
+ options: { throwOnErrors?: boolean } = {},
+): Keyv => {
if (keyvRedisClient && !cacheConfig.FORCED_IN_MEMORY_CACHE_NAMESPACES?.includes(namespace)) {
const byTtl = redisCacheMap.get(namespace);
const existing = byTtl?.get(ttl);
@@ -95,8 +103,9 @@ export const standardCache = (namespace: string, ttl?: number, fallbackStore?: o
return existing;
}
try {
- const keyvRedis = new KeyvRedis(keyvRedisClient);
- const cache = new Keyv(keyvRedis, { namespace, ttl });
+ const { throwOnErrors = false } = options;
+ const keyvRedis = new KeyvRedis(keyvRedisClient, { throwOnErrors });
+ const cache = new Keyv(keyvRedis, { namespace, ttl, throwOnErrors });
keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX;
keyvRedis.keyPrefixSeparator = cacheConfig.GLOBAL_PREFIX_SEPARATOR;
diff --git a/packages/api/src/favorites/pinned.spec.ts b/packages/api/src/favorites/pinned.spec.ts
index 26e298db6e..c3a57e27f2 100644
--- a/packages/api/src/favorites/pinned.spec.ts
+++ b/packages/api/src/favorites/pinned.spec.ts
@@ -11,6 +11,7 @@ import {
import { createPinnedOrderHandlers } from './pinned';
jest.mock('@librechat/data-schemas', () => ({
+ ...jest.requireActual('@librechat/data-schemas'),
logger: { error: jest.fn(), warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
}));
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index 6b43805364..0a20b6351d 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -4192,6 +4192,9 @@ export enum CacheKeys {
export const AUTH_USER_DOC_BY_ID_PREFIX = 'auth-user-doc-byid';
+/** Lifetime of a cached auth user document, and so the longest a stale one can be served. */
+export const AUTH_USER_DOC_CACHE_TTL_MS = 5000;
+
/**
* Enum for violation types, used to identify, log, and cache violations.
*/
diff --git a/packages/data-schemas/src/methods/user.methods.spec.ts b/packages/data-schemas/src/methods/user.methods.spec.ts
index 44df9b4cd4..1ba6b2812d 100644
--- a/packages/data-schemas/src/methods/user.methods.spec.ts
+++ b/packages/data-schemas/src/methods/user.methods.spec.ts
@@ -1,6 +1,10 @@
import mongoose from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
-import { AUTH_USER_DOC_BY_ID_PREFIX, CacheKeys } from 'librechat-data-provider';
+import {
+ CacheKeys,
+ AUTH_USER_DOC_BY_ID_PREFIX,
+ AUTH_USER_DOC_CACHE_TTL_MS,
+} from 'librechat-data-provider';
import type * as t from '~/types';
import { createUserMethods, USER_DELETION_FENCE_STALE_MS } from './user';
import balanceSchema from '~/schema/balance';
@@ -480,6 +484,119 @@ describe('User Methods - Database Tests', () => {
expect(cache.delete).toHaveBeenCalledWith(indexKey);
});
+ describe('when auth cache eviction fails', () => {
+ const cachedKeys = ['auth-cache-key-a', 'auth-cache-key-b'];
+
+ /** Rejects the named operations the way the throwing Redis-backed auth cache does. */
+ function makeFailingCache(failing: { indexRead?: boolean; deleteKey?: string }) {
+ return {
+ get: jest.fn(async () => {
+ if (failing.indexRead) {
+ throw new Error('WRONGTYPE');
+ }
+ return cachedKeys;
+ }),
+ set: jest.fn().mockResolvedValue(true),
+ delete: jest.fn(async (key: string) => {
+ if (key === failing.deleteKey) {
+ throw new Error('redis unavailable');
+ }
+ return true;
+ }),
+ };
+ }
+
+ async function createCachedUser() {
+ enableAuthUserDocCache();
+ const user = await User.create({
+ name: 'Credential User',
+ email: '[email protected]',
+ provider: 'openid',
+ });
+ return user._id?.toString() ?? '';
+ }
+
+ test.each<[string, { indexRead?: boolean; deleteKey?: string }]>([
+ ['the reverse index read fails', { indexRead: true }],
+ ['an indexed document delete fails', { deleteKey: 'auth-cache-key-a' }],
+ ])('waits out the cache TTL at the credential barrier when %s', async (_label, failing) => {
+ const userId = await createCachedUser();
+ const cache = makeFailingCache(failing);
+ const delay = jest.fn().mockResolvedValue(undefined);
+ const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });
+
+ const updated = await methodsWithCache.updateUser(userId, {
+ password: 'new-password-hash',
+ credentialsChangedAt: new Date(),
+ });
+ expect(updated?.credentialsChangedAt).toBeInstanceOf(Date);
+ expect(delay).not.toHaveBeenCalled();
+
+ await methodsWithCache.awaitAuthUserDocEviction(userId);
+
+ expect(delay).toHaveBeenCalledTimes(1);
+ expect(delay.mock.calls[0][0]).toBeGreaterThan(AUTH_USER_DOC_CACHE_TTL_MS);
+ expect(cache.delete).not.toHaveBeenCalledWith(`${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`);
+ if (!failing.indexRead) {
+ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b');
+ }
+ });
+
+ test('passes the barrier at once when only the index delete fails', async () => {
+ const userId = await createCachedUser();
+ const cache = makeFailingCache({ deleteKey: `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}` });
+ const delay = jest.fn().mockResolvedValue(undefined);
+ const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });
+
+ await methodsWithCache.awaitAuthUserDocEviction(userId);
+
+ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a');
+ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-b');
+ expect(delay).not.toHaveBeenCalled();
+ });
+
+ test('passes the barrier at once when a retried eviction succeeds', async () => {
+ const userId = await createCachedUser();
+ const cache = makeFailingCache({ indexRead: true });
+ const delay = jest.fn().mockResolvedValue(undefined);
+ const methodsWithCache = createUserMethods(mongoose, { getCache: () => cache, delay });
+
+ await methodsWithCache.updateUser(userId, { credentialsChangedAt: new Date() });
+ cache.get.mockResolvedValue(cachedKeys);
+ await methodsWithCache.awaitAuthUserDocEviction(userId);
+
+ expect(cache.delete).toHaveBeenCalledWith('auth-cache-key-a');
+ expect(delay).not.toHaveBeenCalled();
+ });
+
+ test('waits at the barrier when the cache can read but not delete', async () => {
+ const userId = await createCachedUser();
+ const delay = jest.fn().mockResolvedValue(undefined);
+ const methodsWithCache = createUserMethods(mongoose, {
+ getCache: () => ({ get: jest.fn(), set: jest.fn() }),
+ delay,
+ });
+
+ await methodsWithCache.awaitAuthUserDocEviction(userId);
+
+ expect(delay).toHaveBeenCalledTimes(1);
+ });
+
+ test('keeps updates themselves best effort', async () => {
+ const userId = await createCachedUser();
+ const delay = jest.fn().mockResolvedValue(undefined);
+ const methodsWithCache = createUserMethods(mongoose, {
+ getCache: () => makeFailingCache({ indexRead: true }),
+ delay,
+ });
+
+ const updated = await methodsWithCache.updateUser(userId, { name: 'Renamed' });
+
+ expect(updated?.name).toBe('Renamed');
+ expect(delay).not.toHaveBeenCalled();
+ });
+ });
+
test('should invalidate cached auth user documents on delete', async () => {
enableAuthUserDocCache();
const user = await User.create({
diff --git a/packages/data-schemas/src/methods/user.ts b/packages/data-schemas/src/methods/user.ts
index 07de2ded81..cfa9057a10 100644
--- a/packages/data-schemas/src/methods/user.ts
+++ b/packages/data-schemas/src/methods/user.ts
@@ -1,14 +1,16 @@
import mongoose, { FilterQuery } from 'mongoose';
import {
- AUTH_USER_DOC_BY_ID_PREFIX,
+ AUTH_USER_DOC_CACHE_TTL_MS,
CacheKeys,
type RefillIntervalUnit,
type StatefulCodeEnvironment,
} from 'librechat-data-provider';
import type { IUser, BalanceConfig, CreateUserRequest, UserDeleteResult } from '~/types';
import type { CacheStore } from '~/types';
+import { evictAuthUserDocs } from '~/utils/eviction';
import { escapeRegExp } from '~/utils/string';
import { signPayload } from '~/crypto';
+import logger from '~/config/winston';
/** Default JWT session expiry: 15 minutes in milliseconds */
export const DEFAULT_SESSION_EXPIRY: number = 1000 * 60 * 15;
@@ -19,8 +21,15 @@ const MAX_SUBAGENT_ADMISSION_FENCES = 32;
interface UserMethodDeps {
getCache?: (key: string) => CacheStore | undefined;
+ /** Resolves after the given milliseconds; tests pass one that need not wait out the cache TTL. */
+ delay?: (ms: number) => Promise<void>;
}
+const wait = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
+
+/** Timers can fire a millisecond early, so the wait outlasts the cache TTL by this much. */
+const AUTH_USER_DOC_EXPIRY_MARGIN_MS = 100;
+
function isAuthUserDocCacheEnabled(): boolean {
return process.env.AUTH_USER_CACHE_MODE === 'on';
}
@@ -55,6 +64,7 @@ export function createUserMethods(
expectedState?: FilterQuery<IUser>,
options?: { preserveExpiresAt?: boolean },
) => Promise<IUser | null>;
+ awaitAuthUserDocEviction: (userId: string) => Promise<void>;
claimSamlIdentity: (
userId: string,
samlId: string,
@@ -333,6 +343,24 @@ export function createUserMethods(
return updated;
}
+ /**
+ * The barrier a credential change passes before it is confirmed. A cached document without
+ * the new credentialsChangedAt keeps pre-change access tokens verifying until it expires, so
+ * eviction is retried and, when it still cannot remove every document, this resolves only
+ * after the cache TTL. Callers revoke sessions and passkeys first: during the wait those
+ * could otherwise mint access tokens issued after the stamp.
+ */
+ async function awaitAuthUserDocEviction(userId: string): Promise<void> {
+ if (await invalidateAuthUserDocCache(userId)) {
+ return;
+ }
+ logger.warn(
+ '[awaitAuthUserDocEviction] Cached auth documents were not evicted after a credential change; waiting for them to expire',
+ { userId, waitMs: AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS },
+ );
+ await (deps.delay ?? wait)(AUTH_USER_DOC_CACHE_TTL_MS + AUTH_USER_DOC_EXPIRY_MARGIN_MS);
+ }
+
/** Atomically updates a SAML user only when the incoming identity can claim the document. */
async function claimSamlIdentity(
userId: string,
@@ -358,26 +386,20 @@ export function createUserMethods(
return updated;
}
- async function invalidateAuthUserDocCache(userId: string): Promise<void> {
+ /** Resolves false only when a cached document for the user may still be served. */
+ async function invalidateAuthUserDocCache(userId: string): Promise<boolean> {
if (!isAuthUserDocCacheEnabled()) {
- return;
+ return true;
}
const cache = deps.getCache?.(CacheKeys.AUTH_USER_DOC);
- if (!cache?.get || !cache?.delete) {
- return;
+ if (!cache?.get) {
+ return true;
}
- try {
- const indexKey = `${AUTH_USER_DOC_BY_ID_PREFIX}:${userId}`;
- const cachedKeys = await cache.get(indexKey);
- if (Array.isArray(cachedKeys)) {
- await Promise.all(
- cachedKeys.map((key) => (typeof key === 'string' ? cache.delete?.(key) : undefined)),
- );
- }
- await cache.delete(indexKey);
- } catch {
- // Cache invalidation must not make a user update fail.
+ const remove = cache.delete?.bind(cache);
+ if (!remove) {
+ return false;
}
+ return evictAuthUserDocs({ get: (key) => cache.get(key), delete: remove }, { userId });
}
/**
@@ -863,6 +885,7 @@ export function createUserMethods(
countUsers,
createUser,
updateUser,
+ awaitAuthUserDocEviction,
claimSamlIdentity,
acceptTerms,
searchUsers,
diff --git a/packages/data-schemas/src/utils/eviction.spec.ts b/packages/data-schemas/src/utils/eviction.spec.ts
new file mode 100644
index 0000000000..4114788ab5
--- /dev/null
+++ b/packages/data-schemas/src/utils/eviction.spec.ts
@@ -0,0 +1,111 @@
+import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider';
+import { evictAuthUserDocs } from './eviction';
+
+const INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-1`;
+const OTHER_INDEX_KEY = `${AUTH_USER_DOC_BY_ID_PREFIX}:user-2`;
+
+/** A Map-backed store that rejects the operations named for a key, as a throwing Redis store does. */
+function makeStore(failures: { get?: string[]; delete?: string[] } = {}) {
+ const values = new Map<string, unknown>([
+ [INDEX_KEY, ['doc-a', 'doc-b']],
+ ['doc-a', { id: 'user-1' }],
+ ['doc-b', { id: 'user-1' }],
+ ['doc-c', { id: 'user-1' }],
+ [OTHER_INDEX_KEY, ['doc-other']],
+ ['doc-other', { id: 'user-2' }],
+ ]);
+ return {
+ values,
+ get: jest.fn(async (key: string) => {
+ if (failures.get?.includes(key)) {
+ throw new Error(`GET ${key} failed`);
+ }
+ return values.get(key);
+ }),
+ delete: jest.fn(async (key: string) => {
+ if (failures.delete?.includes(key)) {
+ throw new Error(`DEL ${key} failed`);
+ }
+ return values.delete(key);
+ }),
+ };
+}
+
+function expectOtherUserCached(store: ReturnType<typeof makeStore>) {
+ expect(store.values.get(OTHER_INDEX_KEY)).toEqual(['doc-other']);
+ expect(store.values.get('doc-other')).toEqual({ id: 'user-2' });
+}
+
+describe('evictAuthUserDocs', () => {
+ it('lets a retry find a document an earlier failed delete left behind', async () => {
+ const store = makeStore({ delete: ['doc-a'] });
+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);
+
+ store.delete.mockImplementation(async (key: string) => store.values.delete(key));
+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true);
+
+ expect(store.values.has('doc-a')).toBe(false);
+ expect(store.values.has(INDEX_KEY)).toBe(false);
+ });
+
+ it('deletes the indexed and explicit documents and the index', async () => {
+ const store = makeStore();
+
+ await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe(
+ true,
+ );
+
+ for (const key of [INDEX_KEY, 'doc-a', 'doc-b', 'doc-c']) {
+ expect(store.values.has(key)).toBe(false);
+ }
+ expectOtherUserCached(store);
+ });
+
+ it('reports a failed index read and keeps the index for a retry', async () => {
+ const store = makeStore({ get: [INDEX_KEY] });
+
+ await expect(evictAuthUserDocs(store, { userId: 'user-1', cacheKey: 'doc-c' })).resolves.toBe(
+ false,
+ );
+
+ expect(store.values.has('doc-c')).toBe(false);
+ expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);
+ expect(store.values.has('doc-a')).toBe(true);
+ expectOtherUserCached(store);
+ });
+
+ it.each([
+ ['a string', 'doc-a'],
+ ['a list with a non-string entry', ['doc-a', 42]],
+ ])('treats a reverse index holding %s as unreadable', async (_label, indexed) => {
+ const store = makeStore();
+ store.values.set(INDEX_KEY, indexed);
+
+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);
+
+ expect(store.values.get(INDEX_KEY)).toEqual(indexed);
+ expect(store.values.has('doc-a')).toBe(true);
+ });
+
+ it('reports a failed document delete, deletes the others and keeps the index for a retry', async () => {
+ const store = makeStore({ delete: ['doc-a'] });
+
+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(false);
+
+ expect(store.values.has('doc-a')).toBe(true);
+ expect(store.values.has('doc-b')).toBe(false);
+ expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);
+ expectOtherUserCached(store);
+ });
+
+ it('counts documents as evicted when only the index delete fails', async () => {
+ const store = makeStore({ delete: [INDEX_KEY] });
+
+ await expect(evictAuthUserDocs(store, { userId: 'user-1' })).resolves.toBe(true);
+
+ expect(store.values.has('doc-a')).toBe(false);
+ expect(store.values.has('doc-b')).toBe(false);
+ expect(store.values.get(INDEX_KEY)).toEqual(['doc-a', 'doc-b']);
+ expectOtherUserCached(store);
+ });
+});
diff --git a/packages/data-schemas/src/utils/eviction.ts b/packages/data-schemas/src/utils/eviction.ts
new file mode 100644
index 0000000000..f238b38a32
--- /dev/null
+++ b/packages/data-schemas/src/utils/eviction.ts
@@ -0,0 +1,77 @@
+import { AUTH_USER_DOC_BY_ID_PREFIX } from 'librechat-data-provider';
+import logger from '~/config/winston';
+
+/** Store operations eviction needs; a failure is only seen when the store throws it. */
+export interface AuthUserDocEvictionStore {
+ get: (key: string) => Promise<unknown>;
+ delete: (key: string) => Promise<unknown>;
+}
+
+function isKeyList(value: unknown): value is string[] {
+ return Array.isArray(value) && value.every((key) => typeof key === 'string');
+}
+
+function describeError(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
+
+/**
+ * Deletes the cached auth user documents for a user, attempting every delete even when
+ * another fails. Resolves false when a document may still be served: the reverse index could
+ * not be read, so its keys are unknown, or a document delete failed. The index is deleted only
+ * after every document it names is gone, so a retry can still find what an earlier attempt
+ * left behind. A failed delete of the index alone leaves only the names of deleted keys, so it
+ * is logged and still counts as evicted. Never throws, so callers keep their own write results.
+ */
+export async function evictAuthUserDocs(
+ store: AuthUserDocEvictionStore,
+ input: { userId?: string; cacheKey?: string },
+): Promise<boolean> {
+ const keys = new Set<string>(input.cacheKey ? [input.cacheKey] : []);
+ const indexKey = input.userId ? `${AUTH_USER_DOC_BY_ID_PREFIX}:${input.userId}` : undefined;
+ let indexRead = true;
+ if (indexKey) {
+ try {
+ const indexed = await store.get(indexKey);
+ if (indexed != null && !isKeyList(indexed)) {
+ throw new Error('Reverse index is not a list of cache keys');
+ }
+ for (const key of indexed ?? []) {
+ keys.add(key);
+ }
+ } catch (error) {
+ indexRead = false;
+ logger.warn('[authUserDocCache] Reverse index read failed during eviction', {
+ userId: input.userId,
+ error: describeError(error),
+ });
+ }
+ }
+
+ const results = await Promise.allSettled([...keys].map((key) => store.delete(key)));
+ let documentsDeleted = true;
+ for (const result of results) {
+ if (result.status === 'rejected') {
+ documentsDeleted = false;
+ logger.warn('[authUserDocCache] Cached document delete failed during eviction', {
+ userId: input.userId,
+ error: describeError(result.reason),
+ });
+ }
+ }
+ if (!indexRead || !documentsDeleted) {
+ return false;
+ }
+
+ if (indexKey) {
+ try {
+ await store.delete(indexKey);
+ } catch (error) {
+ logger.warn('[authUserDocCache] Reverse index delete failed during eviction', {
+ userId: input.userId,
+ error: describeError(error),
+ });
+ }
+ }
+ return true;
+}
diff --git a/packages/data-schemas/src/utils/index.ts b/packages/data-schemas/src/utils/index.ts
index fecdfcc7ed..f5a4c6fd48 100644
--- a/packages/data-schemas/src/utils/index.ts
+++ b/packages/data-schemas/src/utils/index.ts
@@ -8,5 +8,6 @@ export * from './objectId';
export * from './yaml';
export * from './stripUIResourceMarkers';
export * from './fading';
+export * from './eviction';
export { buildIndexWithRetry, createIndexesWithRetry, isIndexBuildInProgress } from './retry';
export type { IndexBuildOptions } from './retry';