Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🔄 fix: Judge Tenant Profiles on Their Own Base, Clear Config Caches on Principal Deletion, and Add a Config Reload Route (#17397)

Danny Avila4d061fdd4662
diff --git a/api/server/routes/admin/config.js b/api/server/routes/admin/config.js
index 2962476556..44a33d29c9 100644
--- a/api/server/routes/admin/config.js
+++ b/api/server/routes/admin/config.js
@@ -11,6 +11,7 @@ const {
const {
getAppConfig,
overrideCacheTtl,
+ reloadConfigCaches,
invalidateConfigCaches,
} = require('~/server/services/Config');
const { requireJwtAuth } = require('~/server/middleware');
@@ -35,6 +36,7 @@ const handlers = createAdminConfigHandlers({
hasCapability,
getAppConfig,
invalidateConfigCaches,
+ reloadConfigCaches,
recordAuditEntry: db.recordAuditEntry,
auditFailClosed: process.env.AUDIT_LOG_FAIL_CLOSED === 'true',
overrideCacheTtlMs: overrideCacheTtl,
@@ -46,6 +48,7 @@ router.use(requireJwtAuth, requireAdminAccess, rejectUnsafeConfigKeys);
router.get('/', handlers.listConfigs);
router.get('/meta', handlers.getConfigMeta);
router.get('/base', handlers.getBaseConfig);
+router.post('/reload', handlers.reloadConfig);
router.get('/:principalType/:principalId', handlers.getConfig);
router.put('/:principalType/:principalId', handlers.upsertConfigOverrides);
router.patch('/:principalType/:principalId/fields', handlers.patchConfigField);
diff --git a/api/server/routes/admin/config.test.js b/api/server/routes/admin/config.test.js
index 951ea44b9c..8ce98a4c66 100644
--- a/api/server/routes/admin/config.test.js
+++ b/api/server/routes/admin/config.test.js
@@ -41,6 +41,7 @@ jest.mock('~/server/services/Config', () => ({
getAppConfig: jest.fn(async () => ({ config: {} })),
overrideCacheTtl: 15_000,
invalidateConfigCaches: jest.fn(async () => undefined),
+ reloadConfigCaches: jest.fn(async () => ({ cleared: true, broadcast: 'sent' })),
}));
jest.mock('~/models', () => mockDb);
@@ -164,4 +165,44 @@ describe('admin config routes', () => {
overrideCacheTtlMs: 15_000,
});
});
+
+ it('reloads the config behind the same auth, through the Config service reload', async () => {
+ const { reloadConfigCaches, invalidateConfigCaches } = require('~/server/services/Config');
+ await request(app).post('/api/admin/config/reload').expect(401);
+ hasAdminAccess = false;
+ await request(app)
+ .post('/api/admin/config/reload')
+ .set('authorization', 'Bearer admin')
+ .expect(403);
+ expect(reloadConfigCaches).not.toHaveBeenCalled();
+
+ hasAdminAccess = true;
+ const response = await request(app)
+ .post('/api/admin/config/reload')
+ .set('authorization', 'Bearer admin')
+ .expect(200);
+
+ expect(response.body).toEqual({ success: true, broadcast: true });
+ expect(reloadConfigCaches).toHaveBeenCalledTimes(1);
+ expect(invalidateConfigCaches).not.toHaveBeenCalled();
+ expect(mockDb.writeConfigDocument).not.toHaveBeenCalled();
+ expect(mockDb.recordAuditEntry).toHaveBeenCalledWith(
+ expect.objectContaining({ action: 'config.reloaded' }),
+ );
+ });
+
+ it('answers 503 when the Config service could not tell the other instances', async () => {
+ const { reloadConfigCaches } = require('~/server/services/Config');
+ reloadConfigCaches.mockResolvedValueOnce({ cleared: true, broadcast: 'failed' });
+
+ const response = await request(app)
+ .post('/api/admin/config/reload')
+ .set('authorization', 'Bearer admin')
+ .expect(503);
+
+ expect(response.body).toEqual({
+ error: expect.any(String),
+ code: 'CONFIG_RELOAD_BROADCAST_FAILED',
+ });
+ });
});
diff --git a/api/server/routes/admin/groups.js b/api/server/routes/admin/groups.js
index 339abf75d8..a7a11b9970 100644
--- a/api/server/routes/admin/groups.js
+++ b/api/server/routes/admin/groups.js
@@ -2,6 +2,7 @@ const express = require('express');
const { createAdminGroupsHandlers } = require('@librechat/api');
const { SystemCapabilities } = require('@librechat/data-schemas');
const { requireCapability } = require('~/server/middleware/roles/capabilities');
+const { invalidateConfigCaches } = require('~/server/services/Config');
const { requireJwtAuth } = require('~/server/middleware');
const db = require('~/models');
@@ -26,6 +27,7 @@ const handlers = createAdminGroupsHandlers({
deleteAclEntries: db.deleteAclEntries,
restoreConfig: db.restoreConfig,
findConfigByPrincipal: db.findConfigByPrincipal,
+ invalidateConfigCaches,
recordAuditEntry: db.recordAuditEntry,
auditFailClosed: process.env.AUDIT_LOG_FAIL_CLOSED === 'true',
});
diff --git a/api/server/routes/admin/groups.test.js b/api/server/routes/admin/groups.test.js
new file mode 100644
index 0000000000..1cddcb0cf7
--- /dev/null
+++ b/api/server/routes/admin/groups.test.js
@@ -0,0 +1,88 @@
+const express = require('express');
+const request = require('supertest');
+
+const GROUP_ID = '507f1f77bcf86cd799439011';
+
+const mockRequireJwtAuth = jest.fn((req, res, next) => {
+ if (req.headers.authorization !== 'Bearer admin') {
+ return res.status(401).json({ message: 'Unauthorized' });
+ }
+ req.user = { id: 'user-1', _id: 'user-1', role: 'ADMIN', tenantId: 'tenant-a' };
+ next();
+});
+const mockDb = {
+ findGroupById: jest.fn(),
+ deleteGroup: jest.fn(),
+ deleteConfig: jest.fn(),
+ restoreConfig: jest.fn(),
+ findConfigByPrincipal: jest.fn(),
+ deleteAclEntries: jest.fn(),
+ recordAuditEntry: jest.fn(),
+};
+
+jest.mock('~/server/middleware', () => ({ requireJwtAuth: mockRequireJwtAuth }));
+
+jest.mock('~/server/middleware/roles/capabilities', () => ({
+ requireCapability: jest.fn(() => (_req, _res, next) => next()),
+}));
+
+jest.mock('~/server/services/Config', () => ({
+ invalidateConfigCaches: jest.fn(async () => undefined),
+}));
+
+jest.mock('~/models', () => mockDb);
+
+function createApp() {
+ const app = express();
+ app.use(express.json());
+ app.use('/api/admin/groups', require('./groups'));
+ return app;
+}
+
+describe('admin group routes', () => {
+ let app;
+
+ beforeAll(() => {
+ app = createApp();
+ });
+
+ beforeEach(() => {
+ const group = { _id: GROUP_ID, name: 'Support', memberIds: ['member-1'] };
+ mockDb.findGroupById.mockResolvedValue(group);
+ mockDb.deleteGroup.mockResolvedValue(group);
+ mockDb.deleteAclEntries.mockResolvedValue({ deletedCount: 0 });
+ mockDb.findConfigByPrincipal.mockResolvedValue(null);
+ });
+
+ it("clears the config caches through the Config service when a deletion removes the group's profile", async () => {
+ const { invalidateConfigCaches } = require('~/server/services/Config');
+ mockDb.deleteConfig.mockResolvedValue({
+ principalType: 'group',
+ principalId: GROUP_ID,
+ overrides: { interface: { customWelcome: 'group hello' } },
+ });
+
+ await request(app)
+ .delete(`/api/admin/groups/${GROUP_ID}`)
+ .set('authorization', 'Bearer admin')
+ .expect(200);
+
+ expect(mockDb.deleteConfig).toHaveBeenCalledWith('group', GROUP_ID);
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(1);
+ expect(invalidateConfigCaches).toHaveBeenCalledWith('tenant-a');
+ expect(mockDb.deleteGroup).toHaveBeenCalledWith(GROUP_ID);
+ });
+
+ it('clears nothing when the group had no profile', async () => {
+ const { invalidateConfigCaches } = require('~/server/services/Config');
+ mockDb.deleteConfig.mockResolvedValue(null);
+
+ await request(app)
+ .delete(`/api/admin/groups/${GROUP_ID}`)
+ .set('authorization', 'Bearer admin')
+ .expect(200);
+
+ expect(mockDb.deleteConfig).toHaveBeenCalledTimes(1);
+ expect(invalidateConfigCaches).not.toHaveBeenCalled();
+ });
+});
diff --git a/api/server/routes/admin/users.js b/api/server/routes/admin/users.js
index efe861248f..74b1bcbf11 100644
--- a/api/server/routes/admin/users.js
+++ b/api/server/routes/admin/users.js
@@ -27,7 +27,11 @@ const {
restoreUserSchedulesFromDeletion,
} = require('~/server/services/Schedules');
const db = require('~/models');
-const { getAppConfig, invalidateCodeEnvironmentConfigCache } = require('~/server/services/Config');
+const {
+ getAppConfig,
+ invalidateConfigCaches,
+ invalidateCodeEnvironmentConfigCache,
+} = require('~/server/services/Config');
const router = express.Router();
@@ -71,6 +75,7 @@ const handlers = createAdminUsersHandlers({
deleteUserCodeEnvironments: db.deleteUserCodeEnvironments,
invalidateCodeEnvironmentConfigCache,
deleteConfig: db.deleteConfig,
+ invalidateConfigCaches,
deleteAclEntries: db.deleteAclEntries,
deletePasskeysByUser: db.deletePasskeysByUser,
resolveErasure,
diff --git a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js
index 1207273c5d..bdcc1dda7b 100644
--- a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js
+++ b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js
@@ -36,7 +36,7 @@ function writeYaml(config) {
writeYaml({});
-const { getAppConfig, invalidateConfigCaches } = require('../app');
+const { getAppConfig, reloadConfigCaches, invalidateConfigCaches } = require('../app');
afterAll(() => {
if (previousConfigPath === undefined) {
@@ -76,4 +76,49 @@ describe('admin config writes and librechat.yaml', () => {
expect((await getAppConfig({ baseOnly: true })).filters).toBeUndefined();
expect((await getAppConfig({ role: 'USER' })).filters).toBeUndefined();
});
+
+ it('reload the file for the reload route, reporting a local clear and no broadcast without Redis', async () => {
+ writeYaml({});
+ await invalidateConfigCaches();
+ expect((await getAppConfig({ baseOnly: true })).filters).toBeUndefined();
+
+ writeYaml({ filters });
+ const report = await reloadConfigCaches();
+
+ expect(report).toEqual({ cleared: true, loaded: { ok: true }, broadcast: 'none' });
+ expect((await getAppConfig({ baseOnly: true })).filters?.prompts?.pii).toEqual(
+ expect.objectContaining(filters.prompts.pii),
+ );
+ });
+
+ it('report a librechat.yaml the loader refuses on the reload route, keeping the last good config', async () => {
+ writeYaml({ filters });
+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: true } });
+ const lastGood = await getAppConfig({ baseOnly: true });
+ expect(lastGood.filters).toBeDefined();
+
+ writeYaml({ bogusTopLevelKey: true, interface: { customWelcome: 42 } });
+ const report = await reloadConfigCaches();
+
+ expect(report).toEqual({
+ cleared: true,
+ loaded: {
+ ok: false,
+ error: {
+ code: 'CONFIG_INVALID',
+ issues: expect.arrayContaining([
+ { path: 'bogusTopLevelKey', segments: ['bogusTopLevelKey'], code: 'unrecognized_keys' },
+ {
+ path: 'interface.customWelcome',
+ segments: ['interface', 'customWelcome'],
+ code: 'invalid_type',
+ },
+ ]),
+ },
+ },
+ broadcast: 'skipped',
+ });
+ expect(JSON.stringify(report)).not.toContain('42');
+ expect((await getAppConfig({ baseOnly: true })).filters).toEqual(lastGood.filters);
+ });
});
diff --git a/api/server/services/Config/app.js b/api/server/services/Config/app.js
index 839373266d..d769134b72 100644
--- a/api/server/services/Config/app.js
+++ b/api/server/services/Config/app.js
@@ -71,8 +71,13 @@ const appConfigService = createAppConfigService({
});
},
});
-const { getAppConfig, clearAppConfigCache, clearOverrideCache, overrideCacheTtl } =
- appConfigService;
+const {
+ getAppConfig,
+ clearAppConfigCache,
+ clearOverrideCache,
+ reloadBaseConfig,
+ overrideCacheTtl,
+} = appConfigService;
// Config owns the reader; models never import this module to obtain it.
db.initializeMessageBudget(getAppConfig);
@@ -91,6 +96,7 @@ const configCacheInvalidation = createConfigCacheInvalidation({
{ label: 'clearOverrideCache', clear: clearOverrideCache },
{ label: 'clearMcpConfigCache', clear: () => clearMcpConfigCache() },
],
+ load: reloadBaseConfig,
pubsub: createRedisConfigInvalidationPubSub({
useRedis: cacheConfig.USE_REDIS,
client: ioredisClient,
@@ -106,10 +112,21 @@ function invalidateConfigCaches(tenantId) {
return configCacheInvalidation.invalidate(tenantId);
}
+/**
+ * The same clear for the admin reload route, then librechat.yaml loaded again here and, when it
+ * loaded, the broadcast; resolves with what each step reached.
+ * @param {string} [tenantId] - Optional tenant ID to scope override cache clearing.
+ * @returns {Promise<import('@librechat/api').ConfigReloadReport>}
+ */
+function reloadConfigCaches(tenantId) {
+ return configCacheInvalidation.reload(tenantId);
+}
+
module.exports = {
getAppConfig,
overrideCacheTtl,
invalidateConfigCaches,
+ reloadConfigCaches,
getCodeEnvironmentRegistry,
invalidateCodeEnvironmentConfigCache,
};
diff --git a/docs/mcp-overrides.md b/docs/mcp-overrides.md
index 10325c7ea4..ef332ef9de 100644
--- a/docs/mcp-overrides.md
+++ b/docs/mcp-overrides.md
@@ -66,19 +66,52 @@ withheld instead of falling back to `librechat.yaml`.
### When a change takes effect
-Each process keeps a principal's decision for `MCP_REGISTRY_CACHE_TTL` (5 seconds by default),
-including a decision made while the configuration could not be read. The decision is derived from
-the merged configuration, which each process also caches, per user and role, for 60 seconds.
-Hiding or unhiding a server therefore takes effect:
-
-- immediately in the process that handled the admin config write, which clears both caches;
-- within 60 seconds plus `MCP_REGISTRY_CACHE_TTL` (65 seconds by default) in every other process,
- which learns of the write only when its caches expire. The same bound applies to the writing
- process when the `APP_CONFIG` cache is Redis-backed, because merged configurations are then left
- to expire rather than cleared.
-
-Group membership is not part of either cache key, so adding a user to a group or removing one from
-it also takes up to that long to change what the user can see.
+The decision is derived from the principal's merged configuration. Each process caches the merged
+configuration per user and role for 60 seconds, and the decision for `MCP_REGISTRY_CACHE_TTL`
+(5 seconds by default), including a decision made while the configuration could not be read.
+
+Every admin config write that changes a profile clears these caches before it answers: the merged
+configurations (of the admin's tenant, or of every tenant for an admin without one), the hiding
+decisions and the cached `librechat.yaml`. Deleting a group or role through the admin API clears
+them the same way when it removes (or, if the deletion cannot finish, restores) that principal's
+profile, and so does `POST /api/admin/config/reload`, which changes no profile. A merge that
+read the configuration before the clear is never cached after it, so a request racing the write
+cannot put the old decision back. Hiding or unhiding a server therefore takes effect:
+
+- in the process that handled the write, for every request after the response, the admin's next
+ read included;
+- with Redis (`USE_REDIS`), in every other process as soon as it receives the clear the write
+ publishes before it answers. A process whose subscription is down, for example while Redis is
+ unreachable, catches up when its caches expire, within 60 seconds plus `MCP_REGISTRY_CACHE_TTL`
+ (65 seconds by default);
+- without Redis, in every other process only when its caches expire, within the same 65 seconds.
+
+The cached `librechat.yaml` does not expire. A process the clear does not reach (every other process
+without Redis; with Redis, one whose subscription is down, or all of them when the publish fails)
+keeps its old `librechat.yaml` until a later clear reaches it or it restarts, so a server added to
+or removed from the file does not change there in the meantime.
+
+`POST /api/admin/config/reload` is for an admin without a tenant, since it reaches every tenant.
+The answering process loads `librechat.yaml` again before it answers, and publishes the clear only
+once the file loaded. It answers 422 `CONFIG_RELOAD_INVALID`, with the location of each schema
+issue, when the file fails to load: that process keeps its last good configuration and the others
+are not told, so nothing changes until the file is fixed and reloaded. It answers 503
+`CONFIG_RELOAD_BROADCAST_FAILED` when the answering process reloaded but the clear could not be
+published (for example while Redis is unreachable), so the reload can be retried, and
+`{ "success": true, "broadcast": false }` without Redis, when only the answering process reloaded.
+It cannot tell whether a subscribed process missed a clear that was published, or whether another
+process, which loads the file on its next read, failed to load it.
+
+When `APP_CONFIG` is removed from `FORCED_IN_MEMORY_CACHE_NAMESPACES`, merged configurations live in
+Redis, where they are not listed and cleared. They then expire after 60 seconds in every process,
+the writing one included; the hiding decisions are still cleared.
+
+Hiding is enforced on every use, not only in what the catalog lists, so a server stays unusable
+once the principal's decision hides it: a stale catalog entry or a persisted agent that still names
+it fails at checkout, as described above.
+
+Group membership is not part of either cache key, and adding a user to a group or removing one from
+it clears nothing, so a membership change takes up to 65 seconds to change what the user can see.
Hiding does not stop the shared connections started at boot for other principals, and it keeps the
principal's stored OAuth tokens. Revoking those tokens (uninstalling the server or deleting the
diff --git a/e2e/specs/mock/content-filters.helpers.ts b/e2e/specs/mock/content-filters.helpers.ts
index 6da40585bb..18926c1171 100644
--- a/e2e/specs/mock/content-filters.helpers.ts
+++ b/e2e/specs/mock/content-filters.helpers.ts
@@ -14,10 +14,7 @@ import { getPrimaryE2EUser } from '../../setup/users.mock';
const PROJECT_ROOT = path.resolve(__dirname, '../../..');
const GENERATED_CONFIG_ROOT = path.join(PROJECT_ROOT, 'e2e/.generated');
-const RELOAD_SENTINEL = `e2e-content-filter-reload-${process.pid}-${randomUUID()}`;
-const RELOAD_SENTINEL_ROLE_PATH = `/api/admin/roles/${encodeURIComponent(RELOAD_SENTINEL)}`;
-const RELOAD_SENTINEL_PATH = `/api/admin/config/role/${encodeURIComponent(RELOAD_SENTINEL)}`;
-const RELOAD_PRIORITY = 10;
+const RELOAD_PATH = '/api/admin/config/reload';
type RequestFetchOptions = NonNullable<Parameters<APIRequestContext['fetch']>[1]>;
@@ -181,41 +178,21 @@ function getConfigFromResult(result: RequestResult): RuntimeConfig {
}
/**
- * The sentinel's profile belongs to a role of its own: config writes need an existing principal,
- * and no user is a member, so its empty profile changes no one's config.
- */
-async function ensureReloadSentinelRole(request: APIRequestContext, token: string): Promise<void> {
- const result = await requestResult(request, {
- path: '/api/admin/roles',
- token,
- method: 'POST',
- data: { name: RELOAD_SENTINEL },
- });
- expect([201, 409], `Expected reload sentinel role to exist: ${result.text}`).toContain(
- result.status,
- );
-}
-
-/**
- * A committed admin config write is what makes the server reload librechat.yaml; a request with
- * nothing to write reloads nothing. A PUT that carries `priority` always writes the sentinel,
- * and the returned `config` confirms it did (a no-op answers with only a message).
+ * Makes the server load librechat.yaml again: the admin config reload route clears the cached
+ * file and merged configs (on every instance, with Redis) without writing a config profile, and
+ * loads the file before it answers. It answers 200 only once the file loaded and every instance
+ * it could reach has been told (`broadcast` is false without Redis), 422 when the file fails to
+ * load, and 503 when the other instances could not be told.
*/
async function triggerConfigReload(request: APIRequestContext, token: string): Promise<void> {
- await ensureReloadSentinelRole(request, token);
const result = await requestResult(request, {
- path: RELOAD_SENTINEL_PATH,
+ path: RELOAD_PATH,
token,
- method: 'PUT',
- data: { overrides: {}, priority: RELOAD_PRIORITY },
+ method: 'POST',
});
- expect(result.ok, `Expected config reload trigger to succeed: ${result.text}`).toBe(true);
- expect(result.body, result.text).toEqual(
- expect.objectContaining({
- config: expect.objectContaining({ principalId: RELOAD_SENTINEL }),
- }),
- );
+ expect(result.ok, `Expected config reload to succeed: ${result.text}`).toBe(true);
+ expect(result.body, result.text).toEqual({ success: true, broadcast: expect.any(Boolean) });
}
async function getLoadedConfig(
@@ -250,25 +227,6 @@ async function getLoadedProtectionState(
};
}
-async function deleteReloadSentinel(request: APIRequestContext, token: string): Promise<void> {
- const result = await requestResult(request, {
- path: RELOAD_SENTINEL_PATH,
- token,
- method: 'DELETE',
- });
- expect([200, 404], `Expected reload sentinel cleanup to succeed: ${result.text}`).toContain(
- result.status,
- );
- const role = await requestResult(request, {
- path: RELOAD_SENTINEL_ROLE_PATH,
- token,
- method: 'DELETE',
- });
- expect([200, 404], `Expected reload sentinel role cleanup to succeed: ${role.text}`).toContain(
- role.status,
- );
-}
-
export async function loginAdmin(request: APIRequestContext): Promise<string> {
const { email, password } = getPrimaryE2EUser();
const response = await request.post('/api/auth/login', {
@@ -378,26 +336,22 @@ export async function restoreRuntimeFilters(
const baselineConfig = parseRuntimeConfig(baseline.contents);
atomicWrite(baseline, baseline.contents);
- try {
- await triggerConfigReload(request, token);
- await expect
- .poll(async () => getLoadedProtectionState(request, token), {
- timeout: 30000,
- intervals: [100, 250, 500, 1000],
- })
- .toEqual({
- base: {
- filters: baselineConfig.filters,
- messageFilter: baselineConfig.messageFilter,
- },
- effective: {
- filters: baselineConfig.filters,
- messageFilter: baselineConfig.messageFilter,
- },
- });
- } finally {
- await deleteReloadSentinel(request, token);
- }
+ await triggerConfigReload(request, token);
+ await expect
+ .poll(async () => getLoadedProtectionState(request, token), {
+ timeout: 30000,
+ intervals: [100, 250, 500, 1000],
+ })
+ .toEqual({
+ base: {
+ filters: baselineConfig.filters,
+ messageFilter: baselineConfig.messageFilter,
+ },
+ effective: {
+ filters: baselineConfig.filters,
+ messageFilter: baselineConfig.messageFilter,
+ },
+ });
baselineState = undefined;
}
diff --git a/packages/api/src/admin/cascade.integration.spec.ts b/packages/api/src/admin/cascade.integration.spec.ts
new file mode 100644
index 0000000000..1612c5df13
--- /dev/null
+++ b/packages/api/src/admin/cascade.integration.spec.ts
@@ -0,0 +1,351 @@
+import { Keyv } from 'keyv';
+import mongoose from 'mongoose';
+import { MongoMemoryServer } from 'mongodb-memory-server';
+import { PrincipalType, PrincipalModel } from 'librechat-data-provider';
+import type { AppConfig } from '@librechat/data-schemas';
+import type { Response } from 'express';
+import type { ServerRequest } from '~/types/http';
+
+process.env.CREDS_KEY =
+ process.env.CREDS_KEY ?? '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef';
+process.env.CREDS_IV = process.env.CREDS_IV ?? '0123456789abcdef0123456789abcdef';
+
+type DataSchemas = typeof import('@librechat/data-schemas');
+type Methods = ReturnType<DataSchemas['createMethods']>;
+type AdminGroupsDeps = Parameters<typeof import('./groups').createAdminGroupsHandlers>[0];
+type AdminRolesDeps = Parameters<typeof import('./roles').createAdminRolesHandlers>[0];
+
+interface MockRes {
+ statusCode: number;
+ body: undefined | Record<string, unknown>;
+ status: jest.Mock;
+ json: jest.Mock;
+}
+
+/**
+ * Deleting a group or role removes its config profile, and putting the profile back when the
+ * deletion cannot finish restores it. Both are config changes, so merged configs cached before
+ * them must not outlive them: the real config service, its in-memory cache and the cache
+ * invalidation `api/server/services/Config/app.js` wires, over real collections.
+ */
+
+let mongoServer: MongoMemoryServer;
+let methods: Methods;
+let dataSchemas: DataSchemas;
+let createAdminGroupsHandlers: typeof import('./groups').createAdminGroupsHandlers;
+let createAdminRolesHandlers: typeof import('./roles').createAdminRolesHandlers;
+let createAppConfigService: typeof import('~/app/service').createAppConfigService;
+let createConfigCacheInvalidation: typeof import('~/app/invalidation').createConfigCacheInvalidation;
+
+const LIFECYCLE_TIMEOUT_MS = 180_000;
+
+beforeAll(async () => {
+ const [schemas, groupsModule, rolesModule, serviceModule, invalidationModule, server] =
+ await Promise.all([
+ import('@librechat/data-schemas'),
+ import('./groups'),
+ import('./roles'),
+ import('~/app/service'),
+ import('~/app/invalidation'),
+ MongoMemoryServer.create(),
+ ]);
+ dataSchemas = schemas;
+ mongoServer = server;
+ ({ createAdminGroupsHandlers } = groupsModule);
+ ({ createAdminRolesHandlers } = rolesModule);
+ ({ createAppConfigService } = serviceModule);
+ ({ createConfigCacheInvalidation } = invalidationModule);
+ jest.spyOn(schemas.logger, 'error').mockReturnValue(schemas.logger);
+ jest.spyOn(schemas.logger, 'warn').mockReturnValue(schemas.logger);
+ jest.spyOn(schemas.logger, 'info').mockReturnValue(schemas.logger);
+
+ await mongoose.connect(mongoServer.getUri());
+ schemas.createModels(mongoose);
+ await mongoose.models.Config.init();
+ methods = schemas.createMethods(mongoose);
+}, LIFECYCLE_TIMEOUT_MS);
+
+afterAll(async () => {
+ await mongoose.disconnect();
+ await mongoServer?.stop();
+}, LIFECYCLE_TIMEOUT_MS);
+
+beforeEach(async () => {
+ await Promise.all(
+ ['Config', 'Group', 'Role', 'User', 'AclEntry', 'SystemGrant'].map((name) =>
+ mongoose.models[name].deleteMany({}),
+ ),
+ );
+});
+
+function mockRes(): Response & MockRes {
+ const res: MockRes = {
+ statusCode: 200,
+ body: undefined,
+ status: jest.fn((code: number) => {
+ res.statusCode = code;
+ return res;
+ }),
+ json: jest.fn((data: MockRes['body']) => {
+ res.body = data;
+ return res;
+ }),
+ };
+ return res as Partial<Response> as Response & MockRes;
+}
+
+function adminRequest(params: Record<string, string>): ServerRequest {
+ return {
+ params,
+ body: {},
+ query: {},
+ user: { id: 'admin', role: 'ADMIN' },
+ } as Partial<ServerRequest> as ServerRequest;
+}
+
+/**
+ * The config service with an in-memory cache, and the invalidation the Config service wires over
+ * it (spied on, so a test can wait for or hold one).
+ */
+function createConfigCaches() {
+ const keyv = new Keyv({ namespace: 'APP_CONFIG' });
+ const entries = keyv.opts.store as Map<string, unknown>;
+ /** Keyv's in-memory store, with the key listing the override cache clears by. */
+ const cache = {
+ get: (key: string) => keyv.get(key),
+ set: (key: string, value: unknown, ttl?: number) => keyv.set(key, value, ttl),
+ delete: (key: string) => keyv.delete(key),
+ opts: { store: { keys: () => entries.keys() } },
+ };
+ const service = createAppConfigService({
+ loadBaseConfig: () => dataSchemas.AppService({ config: {} }),
+ setCachedTools: async () => undefined,
+ getCache: () => cache,
+ cacheKeys: { APP_CONFIG: 'APP_CONFIG' },
+ getApplicableConfigs: methods.getApplicableConfigs,
+ getUserPrincipals: methods.getUserPrincipals,
+ });
+ const invalidation = createConfigCacheInvalidation({
+ clears: [
+ { label: 'clearAppConfigCache', clear: () => service.clearAppConfigCache() },
+ { label: 'clearOverrideCache', clear: service.clearOverrideCache },
+ ],
+ });
+ return {
+ getAppConfig: service.getAppConfig,
+ invalidateConfigCaches: jest.fn((tenantId?: string) => invalidation.invalidate(tenantId)),
+ };
+}
+
+const welcomeOf = (config: AppConfig): string | undefined =>
+ config.interfaceConfig?.customWelcome as string | undefined;
+
+function groupHandlers(overrides: Partial<AdminGroupsDeps>) {
+ return createAdminGroupsHandlers({
+ listGroups: jest.fn(),
+ countGroups: jest.fn(),
+ findGroupById: methods.findGroupById,
+ createGroup: methods.createGroup,
+ updateGroupFieldsTracked: methods.updateGroupFieldsTracked,
+ deleteGroup: methods.deleteGroup,
+ addUserToGroup: methods.addUserToGroup,
+ removeUserFromGroup: methods.removeUserFromGroup,
+ removeMemberByIdTracked: methods.removeMemberByIdTracked,
+ findUsers: methods.findUsers,
+ deleteConfig: methods.deleteConfig,
+ deleteAclEntries: methods.deleteAclEntries,
+ restoreConfig: methods.restoreConfig,
+ findConfigByPrincipal: methods.findConfigByPrincipal,
+ ...overrides,
+ });
+}
+
+function roleHandlers(overrides: Partial<AdminRolesDeps>) {
+ return createAdminRolesHandlers({
+ listRoles: methods.listRoles,
+ countRoles: methods.countRoles,
+ getRoleByName: methods.getRoleByName,
+ createRoleByName: methods.createRoleByName,
+ updateRoleByName: methods.updateRoleByName,
+ updateAccessPermissions: methods.updateAccessPermissions,
+ deleteRoleByName: methods.deleteRoleByName,
+ findUser: methods.findUser,
+ updateUser: methods.updateUser,
+ updateUsersByRole: methods.updateUsersByRole,
+ findUserIdsByRole: methods.findUserIdsByRole,
+ updateUsersRoleByIds: methods.updateUsersRoleByIds,
+ listUsersByRole: methods.listUsersByRole,
+ countUsersByRole: methods.countUsersByRole,
+ deleteConfig: methods.deleteConfig,
+ deleteAclEntries: async (filter) => {
+ await methods.deleteAclEntries(filter);
+ },
+ deleteGrantsForPrincipal: methods.deleteGrantsForPrincipal,
+ restoreConfig: methods.restoreConfig,
+ findConfigByPrincipal: methods.findConfigByPrincipal,
+ findGrantsForPrincipal: methods.findGrantsForPrincipal,
+ ...overrides,
+ });
+}
+
+/** A member of a group whose profile sets the welcome text, and that group's id. */
+async function memberOfProfiledGroup(): Promise<{ userId: string; groupId: string }> {
+ const userId = (
+ await methods.createUser(
+ { email: '[email protected]', name: 'Member', provider: 'local', role: 'USER' },
+ undefined,
+ true,
+ )
+ ).toString();
+ const group = await methods.createGroup({
+ name: 'Engineering',
+ source: 'local',
+ memberIds: [userId],
+ });
+ const groupId = group._id.toString();
+ await methods.upsertConfig(
+ PrincipalType.GROUP,
+ groupId,
+ PrincipalModel.GROUP,
+ { interface: { customWelcome: 'group welcome' } },
+ 20,
+ );
+ return { userId, groupId };
+}
+
+describe('deleting a principal clears the merged configs its profile reached', () => {
+ it('stops serving a deleted group’s profile to its members, before the response', async () => {
+ const caches = createConfigCaches();
+ const { userId, groupId } = await memberOfProfiledGroup();
+ const read = () => caches.getAppConfig({ userId, role: 'USER' });
+ expect(welcomeOf(await read())).toBe('group welcome');
+
+ const res = mockRes();
+ await groupHandlers({ invalidateConfigCaches: caches.invalidateConfigCaches }).deleteGroup(
+ adminRequest({ id: groupId }),
+ res,
+ );
+
+ expect(res.statusCode).toBe(200);
+ expect(caches.invalidateConfigCaches).toHaveBeenCalledWith(undefined);
+ expect(welcomeOf(await read())).toBeUndefined();
+ });
+
+ it('serves a group’s profile again once a deletion that cannot finish puts it back', async () => {
+ const caches = createConfigCaches();
+ const { userId, groupId } = await memberOfProfiledGroup();
+ const read = () => caches.getAppConfig({ userId, role: 'USER' });
+ expect(welcomeOf(await read())).toBe('group welcome');
+ let profileRemoved: () => void = () => undefined;
+ const removed = new Promise<void>((resolve) => (profileRemoved = resolve));
+ const invalidateConfigCaches = jest.fn(async (tenantId?: string) => {
+ await caches.invalidateConfigCaches(tenantId);
+ profileRemoved();
+ });
+
+ const res = mockRes();
+ await groupHandlers({
+ invalidateConfigCaches,
+ /** A member's request lands while the profile is gone, then the ACL removal fails. */
+ deleteAclEntries: async () => {
+ await Promise.race([removed, new Promise((resolve) => setTimeout(resolve, 2_000))]);
+ expect(welcomeOf(await read())).toBeUndefined();
+ throw new Error('acl store down');
+ },
+ }).deleteGroup(adminRequest({ id: groupId }), res);
+
+ expect(res.statusCode).toBe(500);
+ expect(res.body?.code).toBe('GROUP_CLEANUP_FAILED');
+ expect(await methods.findConfigByPrincipal(PrincipalType.GROUP, groupId)).not.toBeNull();
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(2);
+ expect(welcomeOf(await read())).toBe('group welcome');
+ });
+
+ it('leaves the caches alone when the group had no profile', async () => {
+ const caches = createConfigCaches();
+ const group = await methods.createGroup({ name: 'Plain', source: 'local' });
+
+ const res = mockRes();
+ await groupHandlers({ invalidateConfigCaches: caches.invalidateConfigCaches }).deleteGroup(
+ adminRequest({ id: group._id.toString() }),
+ res,
+ );
+
+ expect(res.statusCode).toBe(200);
+ expect(caches.invalidateConfigCaches).not.toHaveBeenCalled();
+ });
+
+ it('stops serving a deleted role’s profile, before the response', async () => {
+ const caches = createConfigCaches();
+ await methods.createRoleByName({ name: 'editor', permissions: {} });
+ await methods.upsertConfig(
+ PrincipalType.ROLE,
+ 'editor',
+ PrincipalModel.ROLE,
+ { interface: { customWelcome: 'editor welcome' } },
+ 10,
+ );
+ /** A request whose cached user still names the role, as one can until its user is reloaded. */
+ const read = () => caches.getAppConfig({ role: 'editor' });
+ expect(welcomeOf(await read())).toBe('editor welcome');
+
+ let release: () => void = () => undefined;
+ const invalidateConfigCaches = jest.fn(async (tenantId?: string) => {
+ await new Promise<void>((resolve) => (release = resolve));
+ await caches.invalidateConfigCaches(tenantId);
+ });
+ const res = mockRes();
+ const deleting = roleHandlers({ invalidateConfigCaches }).deleteRole(
+ adminRequest({ name: 'editor' }),
+ res,
+ );
+ while (invalidateConfigCaches.mock.calls.length === 0) {
+ await new Promise((resolve) => setImmediate(resolve));
+ }
+ /** The response waits for the invalidation. */
+ expect(res.status).not.toHaveBeenCalled();
+ release();
+ await deleting;
+
+ expect(res.statusCode).toBe(200);
+ expect(await methods.findConfigByPrincipal(PrincipalType.ROLE, 'editor')).toBeNull();
+ expect(welcomeOf(await read())).toBeUndefined();
+ });
+
+ it('serves a role’s profile again once a deletion that cannot finish puts it back', async () => {
+ const caches = createConfigCaches();
+ await methods.createRoleByName({ name: 'editor', permissions: {} });
+ await methods.upsertConfig(
+ PrincipalType.ROLE,
+ 'editor',
+ PrincipalModel.ROLE,
+ { interface: { customWelcome: 'editor welcome' } },
+ 10,
+ );
+ const read = () => caches.getAppConfig({ role: 'editor' });
+ expect(welcomeOf(await read())).toBe('editor welcome');
+ let profileRemoved: () => void = () => undefined;
+ const removed = new Promise<void>((resolve) => (profileRemoved = resolve));
+ const invalidateConfigCaches = jest.fn(async (tenantId?: string) => {
+ await caches.invalidateConfigCaches(tenantId);
+ profileRemoved();
+ });
+
+ const res = mockRes();
+ await roleHandlers({
+ invalidateConfigCaches,
+ /** A request lands while the profile is gone, then the grant removal fails. */
+ deleteGrantsForPrincipal: async () => {
+ await Promise.race([removed, new Promise((resolve) => setTimeout(resolve, 2_000))]);
+ expect(welcomeOf(await read())).toBeUndefined();
+ throw new Error('grant store down');
+ },
+ }).deleteRole(adminRequest({ name: 'editor' }), res);
+
+ expect(res.statusCode).toBe(500);
+ expect(res.body?.code).toBe('ROLE_CLEANUP_FAILED');
+ expect(await methods.findConfigByPrincipal(PrincipalType.ROLE, 'editor')).not.toBeNull();
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(2);
+ expect(welcomeOf(await read())).toBe('editor welcome');
+ });
+});
diff --git a/packages/api/src/admin/cascade.spec.ts b/packages/api/src/admin/cascade.spec.ts
index 933b56133a..f8d6c7cada 100644
--- a/packages/api/src/admin/cascade.spec.ts
+++ b/packages/api/src/admin/cascade.spec.ts
@@ -1,4 +1,6 @@
-import { cascadeStep, runCascade } from './cascade';
+import { PrincipalType } from 'librechat-data-provider';
+import type { ConfigRecord } from '@librechat/data-schemas';
+import { cascadeStep, runCascade, invalidatingConfigStore } from './cascade';
describe('runCascade', () => {
it('reports failed steps and restores only the steps that were removed', async () => {
@@ -103,3 +105,59 @@ describe('runCascade', () => {
expect(restoreVoid).toHaveBeenCalledWith('snapshot');
});
});
+
+describe('invalidatingConfigStore', () => {
+ const profile = { principalId: 'g1' } as ConfigRecord;
+
+ function wrap(store: Parameters<typeof invalidatingConfigStore>[0]) {
+ const invalidateConfigCaches = jest.fn().mockResolvedValue(undefined);
+ const wrapped = invalidatingConfigStore(store, {
+ invalidateConfigCaches,
+ tenantId: 'tenant-a',
+ logPrefix: '[test]',
+ });
+ return { wrapped, invalidateConfigCaches };
+ }
+
+ it('invalidates after a removal or restore that changed the store, and only then', async () => {
+ const { wrapped, invalidateConfigCaches } = wrap({
+ deleteConfig: jest.fn().mockResolvedValueOnce(profile).mockResolvedValueOnce(null),
+ restoreConfig: jest.fn().mockResolvedValueOnce(true).mockResolvedValueOnce(false),
+ });
+
+ expect(await wrapped.deleteConfig(PrincipalType.GROUP, 'g1')).toBe(profile);
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(1);
+ expect(await wrapped.deleteConfig(PrincipalType.GROUP, 'g1')).toBeNull();
+ expect(await wrapped.restoreConfig?.(profile)).toBe(true);
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(2);
+ expect(await wrapped.restoreConfig?.(profile)).toBe(false);
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(2);
+ expect(invalidateConfigCaches).toHaveBeenCalledWith('tenant-a');
+ });
+
+ it('invalidates after a call that rejects, which may have committed, and rethrows', async () => {
+ const failure = new Error('ack lost');
+ const { wrapped, invalidateConfigCaches } = wrap({
+ deleteConfig: jest.fn().mockRejectedValue(failure),
+ restoreConfig: jest.fn().mockRejectedValue(failure),
+ });
+
+ await expect(wrapped.deleteConfig(PrincipalType.ROLE, 'editor')).rejects.toBe(failure);
+ await expect(wrapped.restoreConfig?.(profile)).rejects.toBe(failure);
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(2);
+ });
+
+ it('resolves with the change when the invalidation fails, since the change committed', async () => {
+ const wrapped = invalidatingConfigStore(
+ { deleteConfig: jest.fn().mockResolvedValue(profile) },
+ {
+ invalidateConfigCaches: jest.fn().mockRejectedValue(new Error('redis down')),
+ tenantId: undefined,
+ logPrefix: '[test]',
+ },
+ );
+
+ await expect(wrapped.deleteConfig(PrincipalType.USER, 'u1')).resolves.toBe(profile);
+ expect(wrapped.restoreConfig).toBeUndefined();
+ });
+});
diff --git a/packages/api/src/admin/cascade.ts b/packages/api/src/admin/cascade.ts
index 00155c3843..50a44e1544 100644
--- a/packages/api/src/admin/cascade.ts
+++ b/packages/api/src/admin/cascade.ts
@@ -1,4 +1,6 @@
import { logger } from '@librechat/data-schemas';
+import type { PrincipalType } from 'librechat-data-provider';
+import type { ConfigRecord } from '@librechat/data-schemas';
import { getSafeErrorMetadata } from '~/utils/errors';
/** One removal in a delete cascade, with the way to put back what it removed. */
@@ -113,3 +115,70 @@ export async function runCascade(steps: CascadeStep[], logPrefix: string): Promi
return { failed, removed, compensate };
}
+
+/** The config store calls a principal's deletion makes: remove its profile, or put it back. */
+export interface ConfigProfileStore {
+ deleteConfig: (principalType: PrincipalType, principalId: string) => Promise<ConfigRecord | null>;
+ /** Re-creates a removed profile only when none exists; resolves to whether it inserted. */
+ restoreConfig?: (snapshot: ConfigRecord) => Promise<boolean>;
+}
+
+/**
+ * `store` with each change followed by the cache invalidation a config write runs, awaited before
+ * the deletion goes on: this instance's merged configs (and, with Redis, every other instance's)
+ * stop applying a removed profile, or apply a restored one again, before the response. A call
+ * that rejects may still have committed (its acknowledgement lost), so it invalidates too; one
+ * that changed nothing (no profile to remove, or one already back in place) does not. A failed
+ * invalidation is logged, not thrown: the change it follows has committed.
+ */
+export function invalidatingConfigStore(
+ store: ConfigProfileStore,
+ invalidation: {
+ invalidateConfigCaches?: (tenantId?: string) => Promise<void>;
+ tenantId: string | undefined;
+ logPrefix: string;
+ },
+): ConfigProfileStore {
+ const { invalidateConfigCaches, tenantId, logPrefix } = invalidation;
+ if (!invalidateConfigCaches) {
+ return store;
+ }
+ async function invalidate(): Promise<void> {
+ try {
+ await invalidateConfigCaches?.(tenantId);
+ } catch (error) {
+ logger.error(`${logPrefix} config cache invalidation failed:`, getSafeErrorMetadata(error));
+ }
+ }
+ async function changing<T>(
+ change: () => Promise<T>,
+ changed: (result: T) => boolean,
+ ): Promise<T> {
+ let result: T;
+ try {
+ result = await change();
+ } catch (error) {
+ await invalidate();
+ throw error;
+ }
+ if (changed(result)) {
+ await invalidate();
+ }
+ return result;
+ }
+ const { deleteConfig, restoreConfig } = store;
+ return {
+ deleteConfig: (principalType, principalId) =>
+ changing(
+ () => deleteConfig(principalType, principalId),
+ (removed) => removed != null,
+ ),
+ restoreConfig:
+ restoreConfig &&
+ ((snapshot) =>
+ changing(
+ () => restoreConfig(snapshot),
+ (inserted) => inserted,
+ )),
+ };
+}
diff --git a/packages/api/src/admin/config.changes.integration.spec.ts b/packages/api/src/admin/config.changes.integration.spec.ts
index 5cce76f60d..23b2dbfc28 100644
--- a/packages/api/src/admin/config.changes.integration.spec.ts
+++ b/packages/api/src/admin/config.changes.integration.spec.ts
@@ -2207,6 +2207,66 @@ describe('POST …/changes: request checks', () => {
expect(stored!.tombstones).toEqual([]);
});
+ it('lets unset remove part of what an older server stored outside configSchema, and still refuses writes there', async () => {
+ const params = nextParams();
+ await seedDocument(
+ params.principalId,
+ {
+ paths: { uploads: '/tmp/x', imageOutput: '/tmp/y' },
+ interfaceConfig: { agents: false, prompts: false },
+ interface: { customWelcome: 'hi' },
+ },
+ ['config.version', 'config.cache', 'interfaceConfig.bookmarks'],
+ );
+
+ const { status, body } = await send(
+ {
+ ops: [
+ { op: 'unset', fieldPath: 'paths.uploads' },
+ { op: 'unset', fieldPath: 'config.version' },
+ { op: 'unset', fieldPath: 'interfaceConfig.agents' },
+ { op: 'set', fieldPath: 'interface.customWelcome', value: 'hello' },
+ ],
+ },
+ params,
+ );
+
+ expect({ status, code: body.code, issues: body.issues }).toEqual({
+ status: 200,
+ code: undefined,
+ issues: undefined,
+ });
+ expect(body.applied.map((ref) => ref.opIndex)).toEqual([0, 1, 2, 3]);
+ const stored = await storedDocument(params.principalId);
+ expect(stored!.overrides).toEqual({
+ paths: { imageOutput: '/tmp/y' },
+ interfaceConfig: { prompts: false },
+ interface: { customWelcome: 'hello' },
+ });
+ expect(stored!.tombstones).toEqual(['config.cache', 'interfaceConfig.bookmarks']);
+
+ for (const op of [
+ { op: 'set', fieldPath: 'paths.imageOutput', value: '/tmp/z' },
+ { op: 'tombstone', fieldPath: 'interfaceConfig.prompts' },
+ { op: 'rename', fieldPath: 'paths', from: 'imageOutput', to: 'uploads' },
+ ]) {
+ const refused = await send(
+ { ops: [{ op: 'unset', fieldPath: 'paths.uploads' }, op] },
+ params,
+ );
+ expect({
+ status: refused.status,
+ code: refused.body.code,
+ opIndex: refused.body.opIndex,
+ }).toEqual({
+ status: 400,
+ code: 'UNKNOWN_SECTION',
+ opIndex: 1,
+ });
+ }
+ expect((await storedDocument(params.principalId))!.overrides).toEqual(stored!.overrides);
+ });
+
it('hides a whole inherited array but refuses a tombstone inside one', async () => {
const params = nextParams();
const whole = await send({ ops: [{ op: 'tombstone', fieldPath: 'endpoints.custom' }] }, params);
diff --git a/packages/api/src/admin/config.handler.spec.ts b/packages/api/src/admin/config.handler.spec.ts
index 8a10a9c465..d441c959c1 100644
--- a/packages/api/src/admin/config.handler.spec.ts
+++ b/packages/api/src/admin/config.handler.spec.ts
@@ -1,5 +1,6 @@
import type { ConfigDraft, ConfigRecord } from '@librechat/data-schemas';
import type { Response } from 'express';
+import type { ConfigReloadReport } from '~/app/invalidation';
import type { ServerRequest } from '~/types/http';
process.env.CREDS_KEY =
@@ -1988,6 +1989,7 @@ describe('createAdminConfigHandlers', () => {
body: { ops: [{ op: 'set', fieldPath: 'interface.modelSelect', value: false }] },
},
},
+ { name: 'reloadConfig', reqOverrides: {} },
];
describe('upsertConfigOverrides: scope-lifecycle auth (ASSIGN_CONFIGS)', () => {
@@ -4508,6 +4510,207 @@ describe('createAdminConfigHandlers', () => {
});
});
+ describe('reloadConfig', () => {
+ const admin = { id: 'u1', role: 'ADMIN', _id: { toString: () => 'u1' } };
+ const target = { type: 'config', id: 'librechat.yaml', name: 'librechat.yaml' };
+
+ it('clears the config caches before answering, writes nothing and audits the reload', async () => {
+ let release: () => void = () => undefined;
+ const reloadConfigCaches = jest.fn(
+ () =>
+ new Promise<ConfigReloadReport>(
+ (resolve) => (release = () => resolve({ cleared: true, broadcast: 'sent' })),
+ ),
+ );
+ const invalidateConfigCaches = jest.fn().mockResolvedValue(undefined);
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const { handlers, deps } = createHandlers({
+ reloadConfigCaches,
+ invalidateConfigCaches,
+ recordAuditEntry,
+ });
+ const res = mockRes();
+
+ const reloading = handlers.reloadConfig(mockReq({ user: admin }), res);
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(reloadConfigCaches).toHaveBeenCalledWith();
+ expect(res.status).not.toHaveBeenCalled();
+ release();
+ await reloading;
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body).toEqual({ success: true, broadcast: true });
+ expect(invalidateConfigCaches).not.toHaveBeenCalled();
+ expect(deps.writeConfigDocument).not.toHaveBeenCalled();
+ expect(deps.hasConfigCapability).toHaveBeenCalledWith(
+ expect.objectContaining({ id: 'u1', tenantId: undefined }),
+ null,
+ 'manage',
+ );
+ expect(recordAuditEntry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ action: 'config.reloaded',
+ target,
+ metadata: { broadcast: 'sent' },
+ }),
+ );
+ expect(recordAuditEntry.mock.calls[0][0].outcome).toBeUndefined();
+ });
+
+ it('refuses a tenant admin with broad manage, since the reload reaches every tenant', async () => {
+ const reloadConfigCaches = jest.fn().mockResolvedValue({ cleared: true, broadcast: 'sent' });
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const { handlers } = createHandlers({ reloadConfigCaches, recordAuditEntry });
+ const res = mockRes();
+
+ await handlers.reloadConfig(mockReq({ user: { ...admin, tenantId: 'tenant-a' } }), res);
+
+ expect(res.statusCode).toBe(403);
+ expect(res.body).toEqual({ error: expect.any(String), code: 'FORBIDDEN' });
+ expect(reloadConfigCaches).not.toHaveBeenCalled();
+ expect(recordAuditEntry).not.toHaveBeenCalled();
+ });
+
+ it('answers 422 CONFIG_RELOAD_INVALID with the issue locations when librechat.yaml is refused', async () => {
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const issues = [
+ { path: 'bogusTopLevelKey', segments: ['bogusTopLevelKey'], code: 'unrecognized_keys' },
+ ];
+ const report: ConfigReloadReport = {
+ cleared: true,
+ loaded: { ok: false, error: { code: 'CONFIG_INVALID', issues } },
+ broadcast: 'skipped',
+ };
+ const { handlers } = createHandlers({
+ reloadConfigCaches: jest.fn().mockResolvedValue(report),
+ recordAuditEntry,
+ });
+ const res = mockRes();
+
+ await handlers.reloadConfig(mockReq(), res);
+
+ expect(res.statusCode).toBe(422);
+ expect(res.body).toEqual({
+ error: expect.any(String),
+ code: 'CONFIG_RELOAD_INVALID',
+ issues,
+ });
+ expect(recordAuditEntry).toHaveBeenCalledTimes(1);
+ expect(recordAuditEntry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ action: 'config.reloaded',
+ outcome: 'failure',
+ target,
+ metadata: { code: 'CONFIG_RELOAD_INVALID', broadcast: 'skipped' },
+ }),
+ );
+ });
+
+ it('answers broadcast: false when there is no pub/sub to tell other instances', async () => {
+ const { handlers } = createHandlers({
+ reloadConfigCaches: jest.fn().mockResolvedValue({ cleared: true, broadcast: 'none' }),
+ });
+ const res = mockRes();
+
+ await handlers.reloadConfig(mockReq(), res);
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body).toEqual({ success: true, broadcast: false });
+ });
+
+ it.each([
+ {
+ name: 'the publish to the other instances failed',
+ report: { cleared: true, broadcast: 'failed' },
+ status: 503,
+ code: 'CONFIG_RELOAD_BROADCAST_FAILED',
+ },
+ {
+ name: 'a clear on this instance failed',
+ report: { cleared: false, broadcast: 'sent' },
+ status: 500,
+ code: 'CONFIG_RELOAD_FAILED',
+ },
+ {
+ name: 'both failed',
+ report: { cleared: false, broadcast: 'failed' },
+ status: 500,
+ code: 'CONFIG_RELOAD_FAILED',
+ },
+ {
+ name: 'building the config from the loaded file failed',
+ report: {
+ cleared: true,
+ loaded: { ok: false, error: { code: 'LOAD_FAILED' } },
+ broadcast: 'skipped',
+ },
+ status: 500,
+ code: 'CONFIG_RELOAD_FAILED',
+ },
+ ])(
+ 'answers $status $code and audits a failure when $name',
+ async ({ report, status, code }) => {
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const { handlers } = createHandlers({
+ reloadConfigCaches: jest.fn().mockResolvedValue(report),
+ recordAuditEntry,
+ });
+ const res = mockRes();
+
+ await handlers.reloadConfig(mockReq(), res);
+
+ expect(res.statusCode).toBe(status);
+ expect(res.body).toEqual({ error: expect.any(String), code });
+ expect(res.body).not.toHaveProperty('success');
+ expect(recordAuditEntry).toHaveBeenCalledTimes(1);
+ expect(recordAuditEntry).toHaveBeenCalledWith(
+ expect.objectContaining({
+ action: 'config.reloaded',
+ outcome: 'failure',
+ target,
+ metadata: { code, broadcast: report.broadcast },
+ }),
+ );
+ },
+ );
+
+ it('refuses a section-scoped manager, and answers 501 without the reload dependency', async () => {
+ const reloadConfigCaches = jest.fn().mockResolvedValue({ cleared: true, broadcast: 'sent' });
+ const scoped = createHandlers({
+ reloadConfigCaches,
+ hasConfigCapability: jest.fn(
+ async (_user: unknown, section: string | null) => section != null,
+ ),
+ });
+ const forbidden = mockRes();
+ await scoped.handlers.reloadConfig(mockReq(), forbidden);
+ const unconfigured = mockRes();
+ await createHandlers({
+ invalidateConfigCaches: jest.fn().mockResolvedValue(undefined),
+ }).handlers.reloadConfig(mockReq(), unconfigured);
+
+ expect(forbidden.statusCode).toBe(403);
+ expect(forbidden.body).toEqual({ error: 'Insufficient permissions', code: 'FORBIDDEN' });
+ expect(reloadConfigCaches).not.toHaveBeenCalled();
+ expect(unconfigured.statusCode).toBe(501);
+ });
+
+ it('answers 500 without echoing the failure, and audits nothing, when the reload throws', async () => {
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const { handlers } = createHandlers({
+ reloadConfigCaches: jest.fn().mockRejectedValue(new Error('redis://secret@host down')),
+ recordAuditEntry,
+ });
+ const res = mockRes();
+
+ await handlers.reloadConfig(mockReq(), res);
+
+ expect(res.statusCode).toBe(500);
+ expect(res.body).toEqual({ error: 'Failed to reload config' });
+ expect(recordAuditEntry).not.toHaveBeenCalled();
+ });
+ });
+
describe('updateConfigPriority', () => {
const params = { principalType: 'role', principalId: 'editor' };
const document = {
diff --git a/packages/api/src/admin/config.precedence.integration.spec.ts b/packages/api/src/admin/config.precedence.integration.spec.ts
index 861c516ae4..f866a57513 100644
--- a/packages/api/src/admin/config.precedence.integration.spec.ts
+++ b/packages/api/src/admin/config.precedence.integration.spec.ts
@@ -1,5 +1,6 @@
import mongoose, { Types } from 'mongoose';
import { MongoMemoryServer } from 'mongodb-memory-server';
+import { PrincipalType, PrincipalModel } from 'librechat-data-provider';
import type { TCustomConfig } from 'librechat-data-provider';
import type { Response } from 'express';
import type { ServerRequest } from '~/types/http';
@@ -298,6 +299,281 @@ describe('validation on the Base floor (W2-3)', () => {
});
});
+ describe("a tenant's group or user profile written by a caller without a tenant", () => {
+ const inTenant = <T>(run: () => Promise<T>): Promise<T> =>
+ dataSchemas.tenantStorage.run({ tenantId: 'tenant-a' }, run);
+ const setModel = { entries: [{ fieldPath: 'memory.agent.model', value: 'gpt-4.1' }] };
+ const welcome = { entries: [{ fieldPath: 'interface.customWelcome', value: 'hi' }] };
+
+ /** A tenant-a group or user with a profile its tenant's admin created. */
+ async function tenantProfile(
+ handlers: AdminConfigHandlers,
+ principalType: 'group' | 'user',
+ ): Promise<{ principalType: string; principalId: string }> {
+ const principalId = await inTenant(() =>
+ principalType === 'group'
+ ? createGroup('Tenant Engineering')
+ : createUser('[email protected]'),
+ );
+ const params = { principalType, principalId };
+ const created = await inTenant(() =>
+ call(handlers.patchConfigField, { params, body: welcome }),
+ );
+ expect(created.statusCode).toBe(200);
+ expect((await stored(principalType, principalId))?.tenantId).toBe('tenant-a');
+ return params;
+ }
+
+ it("refuses a write that only fails on that tenant's Base, on every write route", async () => {
+ const handlers = createHandlers(providerYaml);
+ await inTenant(() => switchBaseToAgentId(handlers));
+ const group = await tenantProfile(handlers, 'group');
+ const platformGroup = { principalType: 'group', principalId: await createGroup('Platform') };
+
+ const patched = await call(handlers.patchConfigField, { params: group, body: setModel });
+ const replaced = await call(handlers.upsertConfigOverrides, {
+ params: group,
+ body: { overrides: { memory: { agent: { model: 'gpt-4.1' } } } },
+ });
+ const changed = await call(handlers.applyConfigChanges, {
+ params: group,
+ body: { ops: [{ op: 'set', fieldPath: 'memory.agent.model', value: 'gpt-4.1' }] },
+ });
+ const platform = await call(handlers.patchConfigField, {
+ params: platformGroup,
+ body: setModel,
+ });
+
+ for (const res of [patched, replaced, changed]) {
+ expect(res.statusCode).toBe(400);
+ expect(res.body).toMatchObject({
+ code: 'CONFIG_OVERRIDE_INVALID',
+ issues: [expect.objectContaining({ code: 'union_dropped_key' })],
+ });
+ }
+ expect((await stored(group.principalType, group.principalId))?.overrides).toEqual({
+ interface: { customWelcome: 'hi' },
+ });
+ /** A platform group's profile still merges over the platform Base, which allows it. */
+ expect(platform.statusCode).toBe(200);
+ });
+
+ it("accepts a write that is valid only on that tenant's Base, judged on librechat.yaml as the tenant resolves it", async () => {
+ const getAppConfig = jest.fn(async () => ({ config: agentIdYaml }) as never);
+ const handlers = createHandlers(agentIdYaml, { getAppConfig });
+ await inTenant(() => switchBaseToProvider(handlers));
+ const user = await tenantProfile(handlers, 'user');
+ getAppConfig.mockClear();
+
+ const patched = await call(handlers.patchConfigField, { params: user, body: setModel });
+
+ expect(patched.statusCode).toBe(200);
+ expect((await stored(user.principalType, user.principalId))?.overrides).toEqual({
+ interface: { customWelcome: 'hi' },
+ memory: { agent: { model: 'gpt-4.1' } },
+ });
+ expect(getAppConfig).toHaveBeenCalledWith({ tenantId: 'tenant-a', baseOnly: true });
+ expect(getAppConfig).not.toHaveBeenCalledWith({ tenantId: undefined, baseOnly: true });
+ });
+
+ /** librechat.yaml scoped per tenant: `tenantYaml` is what only tenant-a resolves. */
+ const yamlByTenant = (tenantYaml: Partial<TCustomConfig>) =>
+ jest.fn(
+ async (options?: { tenantId?: string }) =>
+ ({
+ config: options?.tenantId === 'tenant-a' ? tenantYaml : {},
+ mcpConfig: options?.tenantId === 'tenant-a' ? tenantYaml.mcpServers : undefined,
+ }) as never,
+ );
+
+ it('allows the env references librechat.yaml uses as that tenant resolves it, on PUT, PATCH and POST …/changes', async () => {
+ /** A custom endpoint librechat.yaml defines for tenant-a only, keyed from the environment. */
+ const regional = {
+ name: 'Regional',
+ tenantId: 'tenant-a',
+ apiKey: '${REGIONAL_KEY}',
+ baseURL: 'https://regional.example.com/v1',
+ models: { default: ['m'] },
+ };
+ const getAppConfig = yamlByTenant({ endpoints: { custom: [regional] } } as never);
+ const handlers = createHandlers({}, { getAppConfig });
+ const group = await tenantProfile(handlers, 'group');
+ const platformGroup = { principalType: 'group', principalId: await createGroup('Platform') };
+ const entry = {
+ name: 'Second Region',
+ apiKey: '${REGIONAL_KEY}',
+ baseURL: 'https://second.example.com/v1',
+ models: { default: ['m'] },
+ };
+ const replace = (params: Record<string, string>) =>
+ call(handlers.upsertConfigOverrides, {
+ params,
+ body: { overrides: { endpoints: { custom: [entry] } } },
+ });
+ getAppConfig.mockClear();
+
+ const replaced = await replace(group);
+ const patched = await call(handlers.patchConfigField, {
+ params: group,
+ body: { entries: [{ fieldPath: 'endpoints.custom', value: [entry] }] },
+ });
+ const changed = await call(handlers.applyConfigChanges, {
+ params: group,
+ body: { ops: [{ op: 'upsertEntry', fieldPath: 'endpoints.custom', entry }] },
+ });
+
+ for (const res of [replaced, patched, changed]) {
+ expect(res.statusCode).toBe(200);
+ }
+ expect(getAppConfig).not.toHaveBeenCalledWith({ tenantId: undefined, baseOnly: true });
+ expect(getAppConfig).toHaveBeenCalledWith({ tenantId: 'tenant-a', baseOnly: true });
+
+ /** The platform group merges over the platform's librechat.yaml, which references no such name. */
+ const platform = await replace(platformGroup);
+ expect(platform.statusCode).toBe(400);
+ expect(platform.body).toMatchObject({
+ code: 'CONFIG_OVERRIDE_INVALID',
+ issues: [expect.objectContaining({ code: 'env_reference_not_allowed' })],
+ });
+ });
+
+ it("judges a field delete's credential re-entry on librechat.yaml as that tenant resolves it", async () => {
+ const url = 'https://tools.tenant.example/mcp';
+ const getAppConfig = yamlByTenant({
+ mcpServers: { tools: { type: 'streamable-http', url } },
+ } as never);
+ const handlers = createHandlers({}, { getAppConfig });
+ const group = await tenantProfile(handlers, 'group');
+ const keyed = await inTenant(() =>
+ call(handlers.patchConfigField, {
+ params: group,
+ body: {
+ entries: [
+ { fieldPath: 'mcpServers.tools.url', value: url },
+ {
+ fieldPath: 'mcpServers.tools.apiKey',
+ value: { source: 'admin', authorization_type: 'bearer', key: 'sk-tenant-key' },
+ },
+ ],
+ },
+ }),
+ );
+ expect(keyed.statusCode).toBe(200);
+
+ /** tenant-a's librechat.yaml still sends the key to the same URL once the profile's goes. */
+ const removed = await call(handlers.deleteConfigField, {
+ params: group,
+ query: { fieldPath: 'mcpServers.tools.url' },
+ });
+
+ expect(removed.statusCode).toBe(200);
+ expect((await stored(group.principalType, group.principalId))?.overrides?.mcpServers).toEqual(
+ { tools: { apiKey: expect.objectContaining({ source: 'admin' }) } },
+ );
+ });
+
+ it('answers 409 when the document moved to another partition after the write resolved it', async () => {
+ const handlers = createHandlers(providerYaml);
+ const group = await tenantProfile(handlers, 'group');
+ let reads = 0;
+ const moving = createHandlers(providerYaml, {
+ findConfigByPrincipal: async (principalType, principalId, options) => {
+ if (principalId === group.principalId && ++reads === 2) {
+ await methods.deleteConfig(PrincipalType.GROUP, group.principalId);
+ await methods.writeConfigDocument(
+ PrincipalType.GROUP,
+ group.principalId,
+ {
+ principalModel: PrincipalModel.GROUP,
+ priority: 20,
+ overrides: {},
+ tombstones: [],
+ isActive: true,
+ },
+ dataSchemas.ABSENT_CONFIG_REVISION,
+ );
+ }
+ return methods.findConfigByPrincipal(principalType, principalId, options);
+ },
+ });
+
+ const patched = await call(moving.patchConfigField, { params: group, body: welcome });
+
+ expect(patched.statusCode).toBe(409);
+ expect(patched.body?.code).toBe('CONFIG_VERSION_CONFLICT');
+ const moved = await stored(group.principalType, group.principalId);
+ expect(moved?.tenantId).toBeUndefined();
+ expect(moved?.overrides).toEqual({});
+ });
+
+ it("reports the credentials the profile drops over that tenant's Base, on every write route", async () => {
+ const handlers = createHandlers();
+ const based = await inTenant(() =>
+ call(handlers.patchConfigField, {
+ params: BASE,
+ body: {
+ entries: [
+ {
+ fieldPath: 'endpoints.custom',
+ value: [
+ {
+ name: 'Gateway',
+ apiKey: 'sk-tenant-gateway-key',
+ baseURL: 'https://gateway.example.com/v1',
+ models: { default: ['gpt-4o'] },
+ },
+ ],
+ },
+ ],
+ },
+ }),
+ );
+ expect(based.statusCode).toBe(200);
+ const group = await tenantProfile(handlers, 'group');
+ const moved = await inTenant(() =>
+ call(handlers.patchConfigField, {
+ params: group,
+ body: {
+ entries: [
+ {
+ fieldPath: 'endpoints.custom',
+ value: [{ name: 'Gateway', baseURL: 'https://elsewhere.example/v1' }],
+ },
+ { fieldPath: 'endpoints.agents.disableBuilder', value: true },
+ { fieldPath: 'endpoints.agents.recursionLimit', value: 30 },
+ ],
+ },
+ }),
+ );
+ const dropped = [
+ {
+ path: 'endpoints.custom["Gateway"].apiKey',
+ segments: ['endpoints', 'custom', 'Gateway', 'apiKey'],
+ reason: 'secret_rebinding',
+ },
+ ];
+ expect(moved.body?.ignored).toEqual(dropped);
+
+ const patched = await call(handlers.patchConfigField, {
+ params: group,
+ body: { entries: [{ fieldPath: 'endpoints.agents.recursionLimit', value: 40 }] },
+ });
+ const tombstoned = await call(handlers.tombstoneConfigField, {
+ params: group,
+ body: { fieldPath: 'endpoints.agents.recursionLimit' },
+ });
+ const deleted = await call(handlers.deleteConfigField, {
+ params: group,
+ query: { fieldPath: 'endpoints.agents.disableBuilder' },
+ });
+
+ for (const res of [patched, tombstoned, deleted]) {
+ expect(res.statusCode).toBe(200);
+ expect(res.body?.ignored).toEqual(dropped);
+ }
+ });
+ });
+
it('ignores an inactive Base, which the merge skips too', async () => {
const handlers = createHandlers(providerYaml);
await createRole('editor');
diff --git a/packages/api/src/admin/config.ts b/packages/api/src/admin/config.ts
index a8857689ff..1dcad3b809 100644
--- a/packages/api/src/admin/config.ts
+++ b/packages/api/src/admin/config.ts
@@ -16,7 +16,9 @@ import {
import {
logger,
getAtPath,
+ getTenantId,
unsetAtPath,
+ tenantStorage,
isPlainRecord,
isDotFreePath,
isSegmentPrefix,
@@ -58,7 +60,9 @@ import {
MAX_CONFIG_PATH_DEPTH,
getDefaultConfigPriority,
BASE_CONFIG_PRINCIPAL_ID,
+ SYSTEM_TENANT_ID,
isConfigRevisionConflict,
+ ConfigRevisionConflictError,
isConfigOverrideStateEmpty,
toConfigDraft,
toConfigIdentityPath,
@@ -89,6 +93,8 @@ import type {
AdminConfigIgnoredSecret,
AdminConfigChangesResponse,
AdminConfigChangeErrorResponse,
+ AdminConfigReloadResponse,
+ AdminConfigReloadErrorResponse,
ConfigMutationOutcome,
ConfigMutationPlan,
ConfigIdentityStep,
@@ -96,6 +102,7 @@ import type {
import type { TCustomConfig } from 'librechat-data-provider';
import type { Response } from 'express';
import type { CapabilityUser } from '~/middleware/capabilities';
+import type { ConfigReloadReport } from '~/app/invalidation';
import type { EnvReferencePolicy } from '~/app/allowlist';
import type { ConfigSecretRename } from './secrets';
import type { SecretReentryIssue } from './binding';
@@ -153,6 +160,9 @@ import { buildAdminConfigMeta } from './meta';
const UNSAFE_SEGMENTS = /(?:^|\.)(__[\w]*|constructor|prototype)(?:\.|$)/;
const MAX_PATCH_ENTRIES = 100;
+/** The audit target of a reload: the file every instance loads again. */
+const CONFIG_FILE_TARGET = 'librechat.yaml';
+const RELOAD_AUDIT_TARGET = { type: 'config', id: CONFIG_FILE_TARGET, name: CONFIG_FILE_TARGET };
const BASE_ONLY_OVERRIDE_SECTIONS = new Set<string>(BASE_ONLY_CONFIG_SECTIONS);
const BASE_PRINCIPAL_OVERRIDE_SECTIONS = new Set<string>(BASE_PRINCIPAL_CONFIG_SECTIONS);
const PROCESS_MCP_CONFIG_ERROR =
@@ -662,6 +672,12 @@ export interface AdminConfigDeps {
}) => Promise<AppConfig>;
/** Invalidate the config caches a mutation affects; awaited before the response is sent. */
invalidateConfigCaches?: (tenantId?: string) => Promise<void>;
+ /**
+ * The same clear for `POST /reload`, then librechat.yaml loaded again on this instance and,
+ * once it loaded, the broadcast, resolving with what each step reached instead of only logging
+ * a failure, so the route can answer for it.
+ */
+ reloadConfigCaches?: (tenantId?: string) => Promise<ConfigReloadReport>;
/** Optional audit emission for override mutations; fail-open unless `auditFailClosed`. */
recordAuditEntry?: RecordAuditEntry;
/** When true, a failed audit write surfaces as a 5xx after the mutation has
@@ -970,6 +986,43 @@ function unstorableKeyResponse(res: Response, operation: string): Response {
});
}
+/**
+ * The answer to a reload that did not load librechat.yaml on this instance or did not reach
+ * every instance, or `null` when it did. A refused file is reported first, since retrying cannot
+ * fix it; its issues carry locations and zod codes only.
+ */
+function getReloadFailure(
+ report: ConfigReloadReport,
+): { status: number; body: AdminConfigReloadErrorResponse } | null {
+ const loadFailure = report.loaded?.ok === false ? report.loaded.error : undefined;
+ if (loadFailure?.code === 'CONFIG_INVALID') {
+ return {
+ status: 422,
+ body: {
+ error: 'librechat.yaml failed to load; the last good config is still served',
+ code: 'CONFIG_RELOAD_INVALID',
+ issues: loadFailure.issues.map(({ path, segments, code }) => ({ path, segments, code })),
+ },
+ };
+ }
+ if (!report.cleared || loadFailure) {
+ return {
+ status: 500,
+ body: { error: 'Failed to reload config', code: 'CONFIG_RELOAD_FAILED' },
+ };
+ }
+ if (report.broadcast !== 'failed') {
+ return null;
+ }
+ return {
+ status: 503,
+ body: {
+ error: 'Config reloaded on this instance only; the other instances could not be notified',
+ code: 'CONFIG_RELOAD_BROADCAST_FAILED',
+ },
+ };
+}
+
/**
* Rejects a write that would send a stored secret to a destination it was not
* entered for. Reports only locations and stable codes, never a value; `segments`
@@ -994,6 +1047,33 @@ function secretReentryResponse(res: Response, issues: SecretReentryIssue[]): Res
*/
type FloorOutcome<T> = ConfigMutationOutcome<T> & { base?: ConfigRecord | null };
+/**
+ * The tenant partitions a write to one principal's document involves. `tenantId` is the one the
+ * document merges in (`undefined`: the deployment's own, tenantless), so the write is judged on
+ * librechat.yaml as that partition resolves it and on that partition's Base. `activeTenantId` is
+ * the request's own (the caller's), where the store addresses role documents and creates a
+ * document the write makes.
+ */
+interface WriteScope {
+ tenantId: string | undefined;
+ activeTenantId: string | undefined;
+}
+
+/** The tenant partition a stored document belongs to; `null` and `''` are tenantless. */
+function getConfigPartition(record: Pick<ConfigRecord, 'tenantId'>): string | undefined {
+ return record.tenantId || undefined;
+}
+
+/**
+ * The partition the config store reads and writes role documents in for this request
+ * (`tenantPartition`): the active tenant's, or the deployment's own outside a tenant and under
+ * the system context.
+ */
+function getActivePartition(): string | undefined {
+ const tenantId = getTenantId();
+ return tenantId && tenantId !== SYSTEM_TENANT_ID ? tenantId : undefined;
+}
+
/**
* Why a planned unset or tombstone wrote nothing: no document to unset, no principal to create
* one for, or a stored credential it would send elsewhere without entering it again.
@@ -1865,6 +1945,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
deleteConfigOverrides: (req: ServerRequest, res: Response) => Promise<Response>;
toggleConfig: (req: ServerRequest, res: Response) => Promise<Response>;
updateConfigPriority: (req: ServerRequest, res: Response) => Promise<Response>;
+ reloadConfig: (req: ServerRequest, res: Response) => Promise<Response>;
} {
const {
listAllConfigs,
@@ -1889,6 +1970,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
hasCapability = async () => false,
getAppConfig,
invalidateConfigCaches,
+ reloadConfigCaches,
recordAuditEntry,
auditFailClosed,
overrideCacheTtlMs = DEFAULT_OVERRIDE_CACHE_TTL,
@@ -1916,6 +1998,53 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
return findConfigByPrincipal(principalType, principalId, { includeInactive: true });
}
+ /**
+ * The partitions a write to the principal's document involves (`WriteScope`). A caller inside
+ * a tenant reaches only that tenant's documents, and a role's document is always the one in
+ * the caller's partition. A caller without a tenant addresses group and user documents by id,
+ * whichever tenant holds them, so the stored document names the partition the write is
+ * judged in: a profile merges over the Base of its own partition, not the caller's.
+ */
+ async function resolveWriteScope(
+ principalType: PrincipalType,
+ principalId: string,
+ ): Promise<WriteScope> {
+ const activeTenantId = getActivePartition();
+ if (activeTenantId || principalType === PrincipalType.ROLE) {
+ return { tenantId: activeTenantId, activeTenantId };
+ }
+ const stored = await readStoredConfig(principalType, principalId);
+ return { tenantId: stored ? getConfigPartition(stored) : undefined, activeTenantId };
+ }
+
+ /** The write's scope and librechat.yaml as the partition it is judged in resolves it. */
+ async function loadWriteScope(
+ principalType: PrincipalType,
+ principalId: string,
+ ): Promise<{ scope: WriteScope; baseConfig: AppConfig | undefined }> {
+ const scope = await resolveWriteScope(principalType, principalId);
+ return { scope, baseConfig: await getBaseAppConfig(scope.tenantId) };
+ }
+
+ /**
+ * Reads the document a write plans from, in the partition the write is judged in: the stored
+ * document's own, or the caller's for a document the write would create. A document that no
+ * longer lands there (removed and created again in another partition since the scope was
+ * resolved) answers a revision conflict rather than being judged on another partition's Base.
+ */
+ async function readScopedConfig(
+ principalType: PrincipalType,
+ principalId: string,
+ scope: WriteScope,
+ ): Promise<ConfigRecord | null> {
+ const stored = await readStoredConfig(principalType, principalId);
+ const partition = stored ? getConfigPartition(stored) : scope.activeTenantId;
+ if (partition !== scope.tenantId) {
+ throw new ConfigRevisionConflictError(stored ? getConfigRevision(stored) : null);
+ }
+ return stored;
+ }
+
/**
* Clears the caches a write affects before the response goes out, so the caller's next read
* sees the write. A failure is logged, not surfaced: the write itself has committed.
@@ -1995,7 +2124,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
* (`canCreateFor`), and a removal that changes where a stored credential is sent needs it
* entered again, as a write would: a web search URL beside a stored key, an Azure group's
* destination beside its stored key, or a field of an MCP server that keeps a stored credential
- * (judged over librechat.yaml, as on every write). A
+ * (judged over librechat.yaml as the document's partition resolves it, as on every write). A
* write that lands in between is re-read and re-checked, or answers 409 under
* `expectedRevision`. An unset stops when there is no document.
*/
@@ -2005,19 +2134,19 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
target: FieldTarget,
removal: { tombstone: true; priority?: number } | { tombstone: false },
expectedRevision: ConfigRevision | undefined,
- tenantId: string | undefined,
+ scope: WriteScope,
): Promise<ConfigMutationOutcome<RemovalStop>> {
const paths = getRemovalPaths(target);
const baseConfig =
MCP_SECTIONS.has(target.segments[0]) ||
isAzureGroupPath(target.segments) ||
isBoundKeyPath(target.segments)
- ? await getBaseAppConfig(tenantId)
+ ? await getBaseAppConfig(scope.tenantId)
: undefined;
const base: Partial<TCustomConfig> = baseConfig?.config ?? {};
return runConfigMutation<RemovalStop>(
{
- read: () => readStoredConfig(principalType, principalId),
+ read: () => readScopedConfig(principalType, principalId, scope),
write: (draft, revision) =>
writeConfigDocument(principalType, principalId, draft, revision),
},
@@ -2089,9 +2218,10 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
* always merges first), then this document. A user or group can have other profiles in
* between, so this is what the document does over the floor. PUT, PATCH and change batches
* report it (a dry run for the document it plans) on the Base their write was last judged on
- * (`base`), and field removals and tombstones in a section with bound secrets read it; the
- * base config is loaded only when the document overrides or tombstones such a section. A
- * failure is logged and reports nothing: the write itself has committed.
+ * (`base`), and field removals and tombstones in a section with bound secrets read it, in the
+ * partition the document merges in (`tenantId`); the base config is loaded only when the
+ * document overrides or tombstones such a section. A failure is logged and reports nothing:
+ * the write itself has committed.
*/
async function reportIgnored(
written: ConfigOverrideLayer,
@@ -2115,7 +2245,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
const isBase = isBaseConfigPrincipal(written);
let floor: ConfigRecord | null = null;
if (!isBase) {
- floor = loaded?.base !== undefined ? loaded.base : await readActiveBase();
+ floor = loaded?.base !== undefined ? loaded.base : await readActiveBase(tenantId);
}
const ignored: AdminConfigIgnoredSecret[] = [];
mergeOverrideLayers(baseConfig, floor ? [floor, written] : [written], {
@@ -2195,11 +2325,16 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
}
/**
- * The active Base document, from the caller's tenant partition (platform-level without one),
- * as the config store addresses every role document.
+ * The active Base document of one tenant partition (`undefined`: the deployment's own). The
+ * config store addresses every role document in the active partition, so another tenant's
+ * Base is read in that tenant's context; the partition rules themselves stay the store's.
*/
- function readActiveBase(): Promise<ConfigRecord | null> {
- return findConfigByPrincipal(PrincipalType.ROLE, BASE_CONFIG_PRINCIPAL_ID);
+ function readActiveBase(tenantId: string | undefined): Promise<ConfigRecord | null> {
+ const read = () => findConfigByPrincipal(PrincipalType.ROLE, BASE_CONFIG_PRINCIPAL_ID);
+ if (tenantId === getActivePartition()) {
+ return read();
+ }
+ return tenantStorage.run({ ...tenantStorage.getStore(), tenantId }, read);
}
/**
@@ -2228,11 +2363,12 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
* it, and is never turned into a failure.
*/
async function judgeOnLandedBase<T>(
+ tenantId: string | undefined,
read: ConfigRecord | null,
judge: (landed: ConfigRecord | null) => Promise<ConfigMutationPlan<T>>,
): Promise<ConfigMutationPlan<T> | null> {
try {
- const landed = await readActiveBase();
+ const landed = await readActiveBase(tenantId);
if ((landed && getConfigRevision(landed)) === (read && getConfigRevision(read))) {
return null;
}
@@ -2251,8 +2387,11 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
* tombstones (the floor): `librechat.yaml` with the active Base document resolved over it,
* exactly as the merge layers Base under every profile (Base always merges first), or
* `librechat.yaml` alone for Base itself. The principal's other profiles vary per user and
- * are left to the merge. Base is read from the caller's tenant partition, so a caller without
- * a tenant validates on the platform-level Base and never on a tenant's. With `yaml` `null`
+ * are left to the merge. Base is read from the partition the document merges in
+ * (`scope.tenantId`): the caller's for a role, and for a group or user document a caller
+ * without a tenant addresses by id, the tenant that holds it, whose Base that profile merges
+ * over; each attempt reads the document in that partition only (`readScopedConfig`). `yaml` is
+ * librechat.yaml as that partition resolves it. With `yaml` `null`
* the write does not validate on the floor, and `plan` gets an empty one. `plan` also gets the
* Base document itself (`null` for Base, or when none is active), which binds the keys the
* write enters to the destinations they are sent to over it (`bindPlannedSecrets`).
@@ -2271,7 +2410,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
async function runOnValidationFloor<T>(
principalType: PrincipalType,
principalId: string,
- tenantId: string | undefined,
+ scope: WriteScope,
yaml: Partial<TCustomConfig> | null,
plan: (
stored: ConfigRecord | null,
@@ -2292,8 +2431,8 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
{
read: async () => {
const [stored, base] = await Promise.all([
- readStoredConfig(principalType, principalId),
- readsBase ? readActiveBase() : null,
+ readScopedConfig(principalType, principalId, scope),
+ readsBase ? readActiveBase(scope.tenantId) : null,
]);
attempt.base = base;
return stored;
@@ -2310,7 +2449,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
if (!outcome.written) {
return { ...outcome, base: attempt.base };
}
- const judged = await judgeOnLandedBase(attempt.base, (landed) => {
+ const judged = await judgeOnLandedBase(scope.tenantId, attempt.base, (landed) => {
attempt.base = landed;
return plan(outcome.previous, floorOver(landed), landed);
});
@@ -2323,7 +2462,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
outcome.record,
outcome.previous,
);
- await invalidateCaches(tenantId, 'withdrawn write');
+ await invalidateCaches(scope.activeTenantId, 'withdrawn write');
return { written: false, current, result: judged.result, base: attempt.base };
}
@@ -2636,8 +2775,9 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
}
}
+ const scope = await resolveWriteScope(principalType, principalId);
const baseConfig =
- overrideSections.length > 0 ? await getBaseAppConfig(user.tenantId) : undefined;
+ overrideSections.length > 0 ? await getBaseAppConfig(scope.tenantId) : undefined;
const baseYaml: Partial<TCustomConfig> = baseConfig?.config ?? {};
const envPolicy = getEnvPolicy(baseYaml);
const envIssues = getEnvReferenceIssues(filteredOverrides, envPolicy);
@@ -2664,7 +2804,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
outcome = await runOnValidationFloor<PutStop>(
principalType,
principalId,
- user.tenantId,
+ scope,
yaml,
async (stored, floor, base) => {
if (!stored && !(await canCreateFor(principalType, principalId))) {
@@ -2786,7 +2926,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
created ? 201 : 200,
config,
hasBroadManage ? FULL_READ_ACCESS : NO_READ_ACCESS,
- await reportIgnored(config, user.tenantId, { baseConfig, envPolicy, base: outcome.base }),
+ await reportIgnored(config, scope.tenantId, { baseConfig, envPolicy, base: outcome.base }),
);
} catch (error) {
if (isFieldNameRejection(error)) {
@@ -2941,8 +3081,8 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
}
const requestedPriority = hasBroadManage ? priority : undefined;
- const [baseConfig, access] = await Promise.all([
- getBaseAppConfig(user.tenantId),
+ const [{ scope, baseConfig }, access] = await Promise.all([
+ loadWriteScope(principalType, principalId),
resolveWriteReadAccess(user, hasBroadManage, writtenSections),
]);
const baseYaml: Partial<TCustomConfig> = baseConfig?.config ?? {};
@@ -2970,7 +3110,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
outcome = await runOnValidationFloor<PatchStop>(
principalType,
principalId,
- user.tenantId,
+ scope,
baseYaml,
async (stored, floor, base) => {
if (!stored && !(await canCreateFor(principalType, principalId))) {
@@ -3072,7 +3212,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
200,
config,
access,
- await reportIgnored(config, user.tenantId, { baseConfig, envPolicy, base: outcome.base }),
+ await reportIgnored(config, scope.tenantId, { baseConfig, envPolicy, base: outcome.base }),
);
} catch (error) {
if (isFieldNameRejection(error)) {
@@ -3191,7 +3331,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
read.target,
{ tombstone: true, priority: requestedPriority },
expected.value,
- user.tenantId,
+ await resolveWriteScope(principalType, principalId),
);
} catch (error) {
return await writeFailureResponse(res, error, principalType, principalId);
@@ -3208,7 +3348,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
metadata: { operation: 'tombstone', fieldPath },
});
const ignored = isBoundSecretSection(parseFieldPath(fieldPath)[0])
- ? await reportIgnored(config, user.tenantId)
+ ? await reportIgnored(config, getConfigPartition(config))
: [];
return sendWrittenConfig(res, 200, config, access, ignored);
} catch (error) {
@@ -3304,7 +3444,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
target,
{ tombstone: false },
expected.value,
- user.tenantId,
+ await resolveWriteScope(principalType, principalId),
);
if (!outcome.written) {
return removalStopResponse(res, outcome.result);
@@ -3339,7 +3479,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
metadata: { operation: 'unset', fieldPath: describeFieldTarget(target) },
});
const ignored = isBoundSecretSection(target.segments[0])
- ? await reportIgnored(config, user.tenantId)
+ ? await reportIgnored(config, getConfigPartition(config))
: [];
return sendWrittenConfig(res, 200, config, access, ignored);
} catch (error) {
@@ -3466,8 +3606,8 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
const requestedPriority = hasBroadManage ? (priority as number | undefined) : undefined;
/** A dry run returns no document, so it resolves no read access: it shows only what it writes. */
- const [baseConfig, access] = await Promise.all([
- getBaseAppConfig(user.tenantId),
+ const [{ scope, baseConfig }, access] = await Promise.all([
+ loadWriteScope(principalType, principalId),
dryRun
? toWrittenSectionsAccess(hasBroadManage, managedSections)
: resolveWriteReadAccess(user, hasBroadManage, managedSections),
@@ -3610,7 +3750,14 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
}
let planned = state;
if (documentChanged) {
- const issues = getConfigChangeIssues(state, floor, stored, written);
+ /**
+ * Only an unset reaches a section configSchema does not define (`prepareChange` refuses
+ * every other op there), and it only removes what an older server stored. What it leaves
+ * of the section is the document's own `unknown_section` issue, which the merge ignores
+ * and which must not be blamed on the removal.
+ */
+ const attributable = written.filter((segments) => isAcceptedOverrideSection(segments[0]));
+ const issues = getConfigChangeIssues(state, floor, stored, attributable);
if (issues.length > 0) {
return { draft: null, result: { stop: { kind: 'issues', issues } } };
}
@@ -3664,7 +3811,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
outcome = await runOnValidationFloor<ChangesResult>(
principalType,
principalId,
- user.tenantId,
+ scope,
baseYaml,
plan,
expected.value,
@@ -3705,7 +3852,7 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
tombstones: report.planned.tombstones,
};
const ignoredSecrets = rebound
- ? await reportIgnored(rebound, user.tenantId, {
+ ? await reportIgnored(rebound, scope.tenantId, {
baseConfig,
envPolicy,
base: outcome.base,
@@ -3897,6 +4044,71 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
}
}
+ /**
+ * POST /reload — Reload librechat.yaml on every instance without a config write.
+ *
+ * Runs the clear every committed config write runs, for every tenant: this instance drops its
+ * cached librechat.yaml, its merged configs and its MCP config-source servers. It then loads
+ * the file again here before answering, so a file the loader refuses is reported (422, with
+ * the issues' locations) instead of only logged; that instance keeps its last good config, as
+ * any reload does, and the other instances are not told. Once the file loaded, the clear is
+ * published to every other instance with Redis, which load the file on their next read. A
+ * write only logs a failed clear or publish, having committed already; here the reload is the
+ * whole operation, so the response reports it. An instance that misses the publish keeps its
+ * cached librechat.yaml, which never expires, so a 503 tells the operator to retry rather than
+ * wait. Only an admin without a tenant with broad `manage:configs` may reload: the file, the
+ * MCP servers it defines and the caches dropped belong to every tenant, and a tenant admin
+ * cannot edit the file.
+ */
+ async function reloadConfig(req: ServerRequest, res: Response): Promise<Response> {
+ try {
+ const user = getCapabilityUser(req);
+ if (!user) {
+ return res.status(401).json({ error: 'Authentication required', code: 'AUTH_REQUIRED' });
+ }
+
+ if (user.tenantId) {
+ return res.status(403).json({
+ error: 'Only an admin without a tenant can reload librechat.yaml',
+ code: 'FORBIDDEN',
+ });
+ }
+
+ if (!(await hasConfigCapability(user, null, 'manage'))) {
+ return res.status(403).json({ error: 'Insufficient permissions', code: 'FORBIDDEN' });
+ }
+
+ if (!reloadConfigCaches) {
+ return res.status(501).json({ error: 'Config reload is not configured' });
+ }
+
+ const report = await reloadConfigCaches();
+ const failure = getReloadFailure(report);
+ if (failure) {
+ await emitAudit(req, {
+ action: 'config.reloaded',
+ outcome: 'failure',
+ target: RELOAD_AUDIT_TARGET,
+ metadata: { code: failure.body.code, broadcast: report.broadcast },
+ });
+ return res.status(failure.status).json(failure.body);
+ }
+ await emitAudit(req, {
+ action: 'config.reloaded',
+ target: RELOAD_AUDIT_TARGET,
+ metadata: { broadcast: report.broadcast },
+ });
+ const body: AdminConfigReloadResponse = {
+ success: true,
+ broadcast: report.broadcast === 'sent',
+ };
+ return res.status(200).json(body);
+ } catch (error) {
+ logger.error('[adminConfig] reloadConfig error:', getSafeErrorMetadata(error));
+ return res.status(500).json({ error: 'Failed to reload config' });
+ }
+ }
+
/**
* PATCH /:principalType/:principalId/active — Toggle isActive.
*/
@@ -3998,5 +4210,6 @@ export function createAdminConfigHandlers(deps: AdminConfigDeps): {
deleteConfigOverrides,
toggleConfig,
updateConfigPriority,
+ reloadConfig,
};
}
diff --git a/packages/api/src/admin/groups.ts b/packages/api/src/admin/groups.ts
index 474dc2a2d1..6fd567dc05 100644
--- a/packages/api/src/admin/groups.ts
+++ b/packages/api/src/admin/groups.ts
@@ -14,8 +14,8 @@ import type { Response } from 'express';
import type { ValidationError } from '~/types/error';
import type { ServerRequest } from '~/types/http';
import type { RecordAuditEntry } from './audit';
+import { cascadeStep, runCascade, invalidatingConfigStore } from './cascade';
import { getSafeErrorMetadata } from '~/utils/errors';
-import { cascadeStep, runCascade } from './cascade';
import { createAdminAuditEmitter } from './audit';
import { parsePagination } from './pagination';
@@ -99,6 +99,9 @@ export interface AdminGroupsDeps {
principalId: string,
options?: { includeInactive?: boolean },
) => Promise<ConfigRecord | null>;
+ /** Clears the merged-config caches (on every instance, with Redis) after the deletion removes
+ * or restores the group's config override, before the response, as config writes do. */
+ invalidateConfigCaches?: (tenantId?: string) => Promise<void>;
/** Optional audit emission for group mutations; fail-open unless `auditFailClosed`. */
recordAuditEntry?: RecordAuditEntry;
/** When true, a failed audit write surfaces as a 5xx after the mutation has
@@ -135,6 +138,7 @@ export function createAdminGroupsHandlers(deps: AdminGroupsDeps): {
deleteAclEntries,
restoreConfig,
findConfigByPrincipal,
+ invalidateConfigCaches,
recordAuditEntry,
auditFailClosed,
} = deps;
@@ -377,6 +381,11 @@ export function createAdminGroupsHandlers(deps: AdminGroupsDeps): {
return res.status(404).json({ error: 'Group not found' });
}
const target = groupTarget(existing, id);
+ const { deleteConfig: removeProfile, restoreConfig: restoreProfile } =
+ invalidatingConfigStore(
+ { deleteConfig, restoreConfig },
+ { invalidateConfigCaches, tenantId: req.user?.tenantId, logPrefix: '[adminGroups]' },
+ );
/**
* The attempt is recorded as `pending` before any removal so a crash mid-cascade
* still leaves a trail, but the write is started without being awaited: deleting a
@@ -418,15 +427,15 @@ export function createAdminGroupsHandlers(deps: AdminGroupsDeps): {
[
cascadeStep(
'config',
- () => deleteConfig(PrincipalType.GROUP, id),
- restoreConfig
+ () => removeProfile(PrincipalType.GROUP, id),
+ restoreProfile
? async (config) => {
if (!config) {
return;
}
let restored: boolean;
try {
- restored = await restoreConfig(config);
+ restored = await restoreProfile(config);
restoreFailed = false;
} catch (error) {
restoreFailed = true;
diff --git a/packages/api/src/admin/meta.spec.ts b/packages/api/src/admin/meta.spec.ts
index caa9f07ee9..40e465e113 100644
--- a/packages/api/src/admin/meta.spec.ts
+++ b/packages/api/src/admin/meta.spec.ts
@@ -1,8 +1,10 @@
import { join } from 'path';
import mongoose from 'mongoose';
import { readFileSync } from 'fs';
+import { EventEmitter } from 'events';
import mongoSanitize from 'express-mongo-sanitize';
import {
+ logger,
AppService,
getConfigRevision,
loadWebSearchConfig,
@@ -56,6 +58,7 @@ import {
import { createRequestSanitizer, rejectUnsafeConfigKeys } from '~/middleware/sanitize';
import { resolvePreAuthAppConfig, resolvePreLoginAppConfig } from '~/app/resolve';
import { MCPServersRegistry } from '~/mcp/registry/MCPServersRegistry';
+import { createConfigCacheInvalidation } from '~/app/invalidation';
import { createMCPPolicyResolver } from '~/mcp/registry/policy';
import { createEnvReferencePolicy } from '~/app/allowlist';
import { resolveTitleConfig } from '~/endpoints/config';
@@ -63,6 +66,7 @@ import { createAppConfigService } from '~/app/service';
import { mergeOverrideLayers } from '~/app/overrides';
import { createAdminConfigHandlers } from './config';
import { createAdminRolesHandlers } from './roles';
+import { ConfigReloadError } from '~/app/loader';
import { ADMIN_CONFIG_FEATURES } from './meta';
import { loadWebSearchAuth } from '~/web';
@@ -293,6 +297,7 @@ describe('GET /api/admin/config/meta', () => {
'prunesEmptyParents',
'readFilteredWriteResponses',
'recordSecrets',
+ 'reloadRoute',
'roleRenameMovesProfile',
'secretRebinding',
'sectionAllowlist',
@@ -345,6 +350,7 @@ describe('GET /api/admin/config/meta', () => {
yamlSecretPreview: true,
envAllowlist: true,
secretRebinding: true,
+ reloadRoute: true,
});
});
@@ -1510,6 +1516,127 @@ const FEATURE_PROBES: Record<AdminConfigFeatureFlag, () => Promise<boolean>> = {
restated.body?.ignored === undefined
);
},
+ async reloadRoute() {
+ /** Two instances, each loading librechat.yaml into its own cache, on one pub/sub channel. */
+ const subscribers = new Set<EventEmitter>();
+ const state = { reachable: true, fileValid: true };
+ const createInstance = () => {
+ let loads = 0;
+ const entries = new Map<string, unknown>();
+ const service = createAppConfigService({
+ loadBaseConfig: async (mode) => {
+ loads += 1;
+ if (!state.fileValid && mode === 'reload') {
+ throw new ConfigReloadError('Invalid custom config file', undefined, [
+ { code: 'unrecognized_keys', keys: ['bogusTopLevelKey'], path: [], message: '' },
+ ]);
+ }
+ return { config: {} } as AppConfig;
+ },
+ setCachedTools: async () => undefined,
+ getCache: () => ({
+ get: async (key: string) => entries.get(key),
+ set: async (key: string, value: unknown) => entries.set(key, value),
+ delete: async (key: string) => entries.delete(key),
+ }),
+ cacheKeys: { APP_CONFIG: 'app_config' },
+ getApplicableConfigs: async () => [],
+ getUserPrincipals: async () => [],
+ });
+ const subscriber = Object.assign(new EventEmitter(), {
+ subscribe: async () => subscribers.add(subscriber),
+ unsubscribe: async () => subscribers.delete(subscriber),
+ });
+ const invalidation = createConfigCacheInvalidation({
+ clears: [
+ { label: 'clearAppConfigCache', clear: () => service.clearAppConfigCache() },
+ { label: 'clearOverrideCache', clear: service.clearOverrideCache },
+ ],
+ load: service.reloadBaseConfig,
+ pubsub: {
+ publisher: {
+ publish: async (channel: string, message: string) => {
+ if (!state.reachable) {
+ throw new Error('Connection is closed.');
+ }
+ subscribers.forEach((other) => other.emit('message', channel, message));
+ return subscribers.size;
+ },
+ },
+ subscriber,
+ },
+ });
+ const read = async () => {
+ await service.getAppConfig({ baseOnly: true });
+ return loads;
+ };
+ return { invalidation, read };
+ };
+ const [first, second] = [createInstance(), createInstance()];
+ await settle();
+ await Promise.all([first.read(), second.read()]);
+
+ const writeConfigDocument = jest.fn();
+ const recordAuditEntry = jest.fn().mockResolvedValue(undefined);
+ const reload = async (deps: Partial<AdminConfigDeps>, init: MockReqInit = {}) => {
+ const res = mockRes();
+ await createHandlers({
+ writeConfigDocument,
+ recordAuditEntry,
+ reloadConfigCaches: first.invalidation.reload,
+ ...deps,
+ }).reloadConfig(mockReq(init), res);
+ await settle();
+ return { res, loads: [await first.read(), await second.read()] };
+ };
+ const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const sectionScoped = await reload({
+ hasConfigCapability: jest.fn(async (_user, section) => section != null),
+ });
+ const tenantAdmin = await reload(
+ {},
+ { user: { id: 'u2', role: 'ADMIN', tenantId: 'tenant-a' } as ServerRequest['user'] },
+ );
+ const reloaded = await reload({});
+ const unconfigured = await reload({ reloadConfigCaches: undefined });
+ state.fileValid = false;
+ const refused = await reload({});
+ state.fileValid = true;
+ state.reachable = false;
+ const unbroadcast = await reload({});
+ error.mockRestore();
+ const audited = recordAuditEntry.mock.calls.map(([entry]) => entry);
+ return (
+ sectionScoped.res.statusCode === 403 &&
+ sectionScoped.res.body?.code === 'FORBIDDEN' &&
+ sectionScoped.loads.join() === '1,1' &&
+ tenantAdmin.res.statusCode === 403 &&
+ tenantAdmin.res.body?.code === 'FORBIDDEN' &&
+ tenantAdmin.loads.join() === '1,1' &&
+ reloaded.res.statusCode === 200 &&
+ JSON.stringify(reloaded.res.body) === JSON.stringify({ success: true, broadcast: true }) &&
+ reloaded.loads.join() === '2,2' &&
+ unconfigured.res.statusCode === 501 &&
+ unconfigured.loads.join() === '2,2' &&
+ refused.res.statusCode === 422 &&
+ refused.res.body?.code === 'CONFIG_RELOAD_INVALID' &&
+ JSON.stringify(refused.res.body?.issues) ===
+ JSON.stringify([
+ { path: 'bogusTopLevelKey', segments: ['bogusTopLevelKey'], code: 'unrecognized_keys' },
+ ]) &&
+ refused.loads.join() === '3,2' &&
+ unbroadcast.res.statusCode === 503 &&
+ unbroadcast.res.body?.code === 'CONFIG_RELOAD_BROADCAST_FAILED' &&
+ unbroadcast.loads.join() === '4,2' &&
+ writeConfigDocument.mock.calls.length === 0 &&
+ audited.length === 3 &&
+ audited.every((entry) => entry.action === 'config.reloaded') &&
+ audited[0].outcome === undefined &&
+ audited[1].outcome === 'failure' &&
+ audited[1].metadata?.code === 'CONFIG_RELOAD_INVALID' &&
+ audited[2].outcome === 'failure'
+ );
+ },
};
describe('meta feature flags', () => {
diff --git a/packages/api/src/admin/meta.ts b/packages/api/src/admin/meta.ts
index 9313724f4c..a92a8a14f5 100644
--- a/packages/api/src/admin/meta.ts
+++ b/packages/api/src/admin/meta.ts
@@ -39,6 +39,7 @@ export const ADMIN_CONFIG_FEATURES: Readonly<AdminConfigFeatures> = {
yamlSecretPreview: true,
envAllowlist: true,
secretRebinding: true,
+ reloadRoute: true,
};
/**
diff --git a/packages/api/src/admin/roles.ts b/packages/api/src/admin/roles.ts
index 7606ca20d2..f49f506207 100644
--- a/packages/api/src/admin/roles.ts
+++ b/packages/api/src/admin/roles.ts
@@ -22,8 +22,8 @@ import type { ServerRequest } from '~/types/http';
import type { RecordAuditEntry } from './audit';
import type { CascadeStep } from './cascade';
import { createAdminAuditEmitter, diffRolePermissions, configTarget } from './audit';
+import { cascadeStep, runCascade, invalidatingConfigStore } from './cascade';
import { getSafeErrorMetadata } from '~/utils/errors';
-import { cascadeStep, runCascade } from './cascade';
import { parsePagination } from './pagination';
const systemRoleValues = new Set<string>(Object.values(SystemRoles));
@@ -235,7 +235,8 @@ export interface AdminRolesDeps {
toPrincipalId: string,
options?: { ids?: readonly string[] },
) => Promise<string[]>;
- /** Clears merged-config caches keyed by role name after a rename moved a profile. */
+ /** Clears the merged-config caches (on every instance, with Redis) once a rename moved a
+ * profile, or a deletion removed or restored one, before the response, as config writes do. */
invalidateConfigCaches?: (tenantId?: string) => Promise<void>;
/** Optional audit emission for role mutations and the grants removed by the
* role-deletion cascade. Failure is logged unless `auditFailClosed` is set. */
@@ -887,16 +888,21 @@ export function createAdminRolesHandlers(deps: AdminRolesDeps): {
const grantSnapshot = findGrantsForPrincipal
? await findGrantsForPrincipal(PrincipalType.ROLE, name, { tenantId })
: null;
+ const { deleteConfig: removeProfile, restoreConfig: restoreProfile } =
+ invalidatingConfigStore(
+ { deleteConfig, restoreConfig },
+ { invalidateConfigCaches, tenantId, logPrefix: '[adminRoles]' },
+ );
const configStep = cascadeStep(
'config',
- () => deleteConfig(PrincipalType.ROLE, name),
- restoreConfig
+ () => removeProfile(PrincipalType.ROLE, name),
+ restoreProfile
? async (config) => {
if (!config) {
return;
}
/** Insert-only: an override written after the snapshot is kept. */
- const restored = await restoreConfig(config);
+ const restored = await restoreProfile(config);
if (!restored) {
logger.warn('[adminRoles] config restore skipped, override exists:', name);
}
diff --git a/packages/api/src/admin/secrets.integration.spec.ts b/packages/api/src/admin/secrets.integration.spec.ts
index d4b5ef0c83..7db0338e08 100644
--- a/packages/api/src/admin/secrets.integration.spec.ts
+++ b/packages/api/src/admin/secrets.integration.spec.ts
@@ -3842,6 +3842,7 @@ describe('write guards on every handler that changes a document', () => {
toggleConfig: 'switches the document on or off; the merge judges the layers it brings in',
updateConfigPriority:
'sets only the priority, keeping overrides and tombstones; the merge judges the order it brings in',
+ reloadConfig: 'clears the config caches; no document is read or written',
};
it('lists every handler', () => {
diff --git a/packages/api/src/admin/users.spec.ts b/packages/api/src/admin/users.spec.ts
index b3582c33f1..d8add91c08 100644
--- a/packages/api/src/admin/users.spec.ts
+++ b/packages/api/src/admin/users.spec.ts
@@ -500,6 +500,42 @@ describe('createAdminUsersHandlers', () => {
expect(deps.deletePasskeysByUser).toHaveBeenCalledWith(validUserId);
});
+ it("clears the config caches after removing the user's config override, before answering", async () => {
+ let release: () => void = () => undefined;
+ const invalidateConfigCaches = jest.fn(
+ () => new Promise<void>((resolve) => (release = resolve)),
+ );
+ const deps = createDeps({
+ deleteConfig: jest.fn().mockResolvedValue({ principalId: validUserId }),
+ invalidateConfigCaches,
+ });
+ const { req, res, status } = createReqRes({
+ params: { id: validUserId },
+ user: { _id: new Types.ObjectId(), role: 'admin', tenantId: 'tenant-a' },
+ });
+
+ const deleting = createAdminUsersHandlers(deps).deleteUser(req, res);
+ await new Promise((resolve) => setImmediate(resolve));
+ expect(invalidateConfigCaches).toHaveBeenCalledWith('tenant-a');
+ expect(status).not.toHaveBeenCalled();
+ release();
+ await deleting;
+
+ expect(status).toHaveBeenCalledWith(200);
+ expect(invalidateConfigCaches).toHaveBeenCalledTimes(1);
+ });
+
+ it('leaves the config caches alone when the user had no config override', async () => {
+ const invalidateConfigCaches = jest.fn().mockResolvedValue(undefined);
+ const deps = createDeps({ invalidateConfigCaches });
+ const { req, res, status } = createReqRes({ params: { id: validUserId } });
+
+ await createAdminUsersHandlers(deps).deleteUser(req, res);
+
+ expect(status).toHaveBeenCalledWith(200);
+ expect(invalidateConfigCaches).not.toHaveBeenCalled();
+ });
+
it('runs the shared erasure cascade for the target before removing the user', async () => {
const checkpointer = { ttl: 60 };
const erasureDeps = { marker: true };
diff --git a/packages/api/src/admin/users.ts b/packages/api/src/admin/users.ts
index 20fbd519c4..2e3ce18b29 100644
--- a/packages/api/src/admin/users.ts
+++ b/packages/api/src/admin/users.ts
@@ -16,6 +16,7 @@ import type { UserGenerationControl } from '~/user/generations';
import type { UserErasureDeps } from '~/user/erasure';
import type { ServerRequest } from '~/types/http';
import { stopUserGenerations } from '~/user/generations';
+import { invalidatingConfigStore } from './cascade';
import { parsePagination } from './pagination';
import { eraseUserData } from '~/user/erasure';
@@ -108,6 +109,9 @@ export interface AdminUsersDeps {
principalType: PrincipalType,
principalId: string | Types.ObjectId,
) => Promise<ConfigRecord | null>;
+ /** Clears the merged-config caches (on every instance, with Redis) after the deletion removes
+ * the user's config override, before the response, as config writes do. */
+ invalidateConfigCaches?: (tenantId?: string) => Promise<void>;
deleteAclEntries: (filter: {
principalType: PrincipalType;
principalId: string | Types.ObjectId;
@@ -138,6 +142,7 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
deleteUserById,
invalidateCodeEnvironmentConfigCache,
deleteConfig,
+ invalidateConfigCaches,
deleteAclEntries,
deletePasskeysByUser,
resolveErasure,
@@ -309,8 +314,12 @@ export function createAdminUsersHandlers(deps: AdminUsersDeps): {
}
const objectId = new Types.ObjectId(id);
+ const profiles = invalidatingConfigStore(
+ { deleteConfig },
+ { invalidateConfigCaches, tenantId: req.user?.tenantId, logPrefix: '[adminUsers]' },
+ );
const cleanupResults = await Promise.allSettled([
- deleteConfig(PrincipalType.USER, id),
+ profiles.deleteConfig(PrincipalType.USER, id),
...(codeEnvironmentCleanupSafe ? [deleteUserCodeEnvironments(objectId)] : []),
deleteAclEntries({ principalType: PrincipalType.USER, principalId: objectId }),
deletePasskeysByUser(id),
diff --git a/packages/api/src/app/invalidation.redis.spec.ts b/packages/api/src/app/invalidation.redis.spec.ts
index b05f739bee..170df2e796 100644
--- a/packages/api/src/app/invalidation.redis.spec.ts
+++ b/packages/api/src/app/invalidation.redis.spec.ts
@@ -173,6 +173,29 @@ describe('createRedisConfigInvalidationPubSub', () => {
server = await startRespServer();
});
+ it('reports a reload whose broadcast Redis did not take, and one it did', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const instance = await startInstance('deploy-a');
+ await waitFor(() => (instance.pubsub.publisher as Redis).status === 'ready');
+
+ await expect(instance.invalidation.reload('tenant-a')).resolves.toEqual({
+ cleared: true,
+ broadcast: 'sent',
+ });
+
+ await server.close();
+ await waitFor(() => (instance.pubsub.publisher as Redis).status !== 'ready');
+ const started = Date.now();
+ await expect(instance.invalidation.reload('tenant-a')).resolves.toEqual({
+ cleared: true,
+ broadcast: 'failed',
+ });
+
+ expect(Date.now() - started).toBeLessThan(1000);
+ expect(instance.clearOverrides).toHaveBeenCalledTimes(2);
+ server = await startRespServer();
+ });
+
it('subscribes once Redis comes back when the first SUBSCRIBE failed during an outage', async () => {
const error = jest.spyOn(logger, 'error').mockReturnValue(logger);
const channel = 'librechat:config:invalidate:deploy-a';
diff --git a/packages/api/src/app/invalidation.spec.ts b/packages/api/src/app/invalidation.spec.ts
index f50d631814..6f671d3ed1 100644
--- a/packages/api/src/app/invalidation.spec.ts
+++ b/packages/api/src/app/invalidation.spec.ts
@@ -242,4 +242,141 @@ describe('createConfigCacheInvalidation', () => {
expect.anything(),
);
});
+
+ describe('reload', () => {
+ it('reports a clear here and a broadcast the other instance acts on', async () => {
+ const broker = createBroker();
+ const first = createInstance(broker, 'first');
+ const second = createInstance(broker, 'second');
+ await flush();
+
+ await expect(first.invalidation.reload('tenant-a')).resolves.toEqual({
+ cleared: true,
+ broadcast: 'sent',
+ });
+ await flush();
+
+ expect(first.clearOverrides).toHaveBeenCalledWith('tenant-a');
+ expect(second.clearOverrides).toHaveBeenCalledWith('tenant-a');
+ });
+
+ it('reports a failed publish, keeping the local clear, instead of only logging it', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const clear = jest.fn().mockResolvedValue(undefined);
+ const invalidation = createConfigCacheInvalidation({
+ clears: [{ label: 'clearAppConfigCache', clear }],
+ pubsub: {
+ publisher: { publish: jest.fn().mockRejectedValue(new Error('redis down')) },
+ subscriber: createBroker().subscriber(),
+ },
+ });
+
+ await expect(invalidation.reload()).resolves.toEqual({ cleared: true, broadcast: 'failed' });
+ expect(clear).toHaveBeenCalledTimes(1);
+ });
+
+ it('reports a failed local clear, still publishing it to the other instances', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const broker = createBroker();
+ const second = createInstance(broker, 'second');
+ const invalidation = createConfigCacheInvalidation({
+ clears: [
+ { label: 'clearAppConfigCache', clear: jest.fn().mockRejectedValue(new Error('x')) },
+ { label: 'clearOverrideCache', clear: jest.fn().mockResolvedValue(undefined) },
+ ],
+ pubsub: { publisher: broker.publisher(), subscriber: broker.subscriber() },
+ instanceId: 'first',
+ });
+ await flush();
+
+ await expect(invalidation.reload()).resolves.toEqual({ cleared: false, broadcast: 'sent' });
+ await flush();
+ expect(second.clearOverrides).toHaveBeenCalledTimes(1);
+ });
+
+ it('loads the config here after the clear and before the broadcast, reporting the load', async () => {
+ const broker = createBroker();
+ const second = createInstance(broker, 'second');
+ const publisher = broker.publisher();
+ const clear = jest.fn().mockResolvedValue(undefined);
+ const load = jest.fn(async () => ({ ok: true as const }));
+ const invalidation = createConfigCacheInvalidation({
+ clears: [{ label: 'clearAppConfigCache', clear }],
+ load,
+ pubsub: { publisher, subscriber: broker.subscriber() },
+ instanceId: 'first',
+ });
+ await flush();
+
+ await expect(invalidation.reload()).resolves.toEqual({
+ cleared: true,
+ loaded: { ok: true },
+ broadcast: 'sent',
+ });
+ await flush();
+
+ const [cleared] = clear.mock.invocationCallOrder;
+ const [loaded] = load.mock.invocationCallOrder;
+ const [published] = publisher.publish.mock.invocationCallOrder;
+ expect(cleared).toBeLessThan(loaded);
+ expect(loaded).toBeLessThan(published);
+ expect(second.clearOverrides).toHaveBeenCalledTimes(1);
+ });
+
+ it('reports a failed load and does not tell the other instances, which keep their config', async () => {
+ const broker = createBroker();
+ const second = createInstance(broker, 'second');
+ const publisher = broker.publisher();
+ const clear = jest.fn().mockResolvedValue(undefined);
+ const failure = {
+ code: 'CONFIG_INVALID' as const,
+ issues: [{ path: 'bogus', segments: ['bogus'], code: 'unrecognized_keys' }],
+ };
+ const invalidation = createConfigCacheInvalidation({
+ clears: [{ label: 'clearAppConfigCache', clear }],
+ load: async () => ({ ok: false, error: failure }),
+ pubsub: { publisher, subscriber: broker.subscriber() },
+ instanceId: 'first',
+ });
+ await flush();
+
+ await expect(invalidation.reload()).resolves.toEqual({
+ cleared: true,
+ loaded: { ok: false, error: failure },
+ broadcast: 'skipped',
+ });
+ await flush();
+
+ expect(clear).toHaveBeenCalledTimes(1);
+ expect(publisher.publish).not.toHaveBeenCalled();
+ expect(second.clearOverrides).not.toHaveBeenCalled();
+ });
+
+ it('leaves the load to the next read for a write invalidation', async () => {
+ const load = jest.fn(async () => ({ ok: true as const }));
+ const clear = jest.fn().mockResolvedValue(undefined);
+ const invalidation = createConfigCacheInvalidation({
+ clears: [{ label: 'clearAppConfigCache', clear }],
+ load,
+ });
+
+ await invalidation.invalidate();
+
+ expect(clear).toHaveBeenCalledTimes(1);
+ expect(load).not.toHaveBeenCalled();
+ });
+
+ it('reports no broadcast without a pub/sub', async () => {
+ const clear = jest.fn().mockResolvedValue(undefined);
+ const invalidation = createConfigCacheInvalidation({
+ clears: [{ label: 'clearAppConfigCache', clear }],
+ });
+
+ await expect(invalidation.reload('tenant-a')).resolves.toEqual({
+ cleared: true,
+ broadcast: 'none',
+ });
+ expect(clear).toHaveBeenCalledWith('tenant-a');
+ });
+ });
});
diff --git a/packages/api/src/app/invalidation.ts b/packages/api/src/app/invalidation.ts
index f6804a6eee..21ed04bfe1 100644
--- a/packages/api/src/app/invalidation.ts
+++ b/packages/api/src/app/invalidation.ts
@@ -1,6 +1,7 @@
import { randomUUID } from 'crypto';
import { logger } from '@librechat/data-schemas';
import type { Cluster, Redis } from 'ioredis';
+import type { ConfigLoadResult } from './loader';
import { createIoRedisSubscriber, duplicateIoRedisClient } from '~/cache/redisUtils';
import { getSafeErrorMetadata } from '~/utils/errors';
@@ -38,6 +39,21 @@ export interface ConfigCacheClear {
clear: (tenantId?: string) => Promise<void>;
}
+/** What one reload reached; failures are logged and reported, never thrown. */
+export interface ConfigReloadReport {
+ /** Whether every clear on this instance succeeded. */
+ cleared: boolean;
+ /** Loading the config again on this instance after the clear; absent without a `load` step. */
+ loaded?: ConfigLoadResult;
+ /**
+ * `sent` once published to the other instances, `failed` when publishing failed, `none`
+ * without a pub/sub (no other instance is told), `skipped` when the load here failed, so the
+ * others were not told either. An instance that misses the broadcast keeps its cached
+ * librechat.yaml, which never expires, until it receives a later one or restarts.
+ */
+ broadcast: 'sent' | 'failed' | 'none' | 'skipped';
+}
+
export interface ConfigCacheInvalidation {
/**
* Clears this instance's config caches and tells every other instance to do the same.
@@ -45,6 +61,13 @@ export interface ConfigCacheInvalidation {
* write that triggered them has already committed.
*/
invalidate(tenantId?: string): Promise<void>;
+ /**
+ * The same clear, then the `load` step, then the broadcast, resolving with what each reached,
+ * for a reload no write triggered: its caller answers for the outcome, so it must learn of a
+ * failed clear, load or broadcast instead of only having it logged. A reload whose load fails
+ * here is not broadcast, so the other instances keep serving what they have.
+ */
+ reload(tenantId?: string): Promise<ConfigReloadReport>;
/** Stops listening for other instances' invalidations. */
close(): Promise<void>;
}
@@ -77,24 +100,29 @@ function parseMessage(message: string): InvalidationMessage | null {
*/
export function createConfigCacheInvalidation(params: {
clears: readonly ConfigCacheClear[];
+ /** Loads the cleared config again on this instance, for `reload` only; never rejects. */
+ load?: () => Promise<ConfigLoadResult>;
pubsub?: ConfigInvalidationPubSub;
/** Identifies this instance, so it ignores its own broadcasts. */
instanceId?: string;
}): ConfigCacheInvalidation {
- const { clears, pubsub } = params;
+ const { clears, load, pubsub } = params;
const instanceId = params.instanceId ?? randomUUID();
const channel = pubsub?.channel ?? CONFIG_INVALIDATION_CHANNEL;
- async function clearLocal(tenantId?: string): Promise<void> {
+ /** Runs every clear; resolves whether all of them succeeded, logging each that failed. */
+ async function clearLocal(tenantId?: string): Promise<boolean> {
const results = await Promise.allSettled(clears.map(({ clear }) => clear(tenantId)));
- results.forEach((result, index) => {
- if (result.status === 'rejected') {
- logger.error(
- `[invalidateConfigCaches] ${clears[index].label} failed:`,
- getSafeErrorMetadata(result.reason),
- );
+ return results.reduce((cleared, result, index) => {
+ if (result.status === 'fulfilled') {
+ return cleared;
}
- });
+ logger.error(
+ `[invalidateConfigCaches] ${clears[index].label} failed:`,
+ getSafeErrorMetadata(result.reason),
+ );
+ return false;
+ }, true);
}
const onMessage = (received: string, message: string): void => {
@@ -144,22 +172,40 @@ export function createConfigCacheInvalidation(params: {
subscribe();
}
- async function invalidate(tenantId?: string): Promise<void> {
- await clearLocal(tenantId);
+ async function broadcast(tenantId?: string): Promise<ConfigReloadReport['broadcast']> {
if (!pubsub) {
- return;
+ return 'none';
}
const message: InvalidationMessage = tenantId
? { origin: instanceId, tenantId }
: { origin: instanceId };
try {
await pubsub.publisher.publish(channel, JSON.stringify(message));
+ return 'sent';
} catch (error) {
logger.error(
- '[invalidateConfigCaches] Publishing to other instances failed; they may serve cached config until it expires:',
+ '[invalidateConfigCaches] Publishing to other instances failed; they keep their cached librechat.yaml and serve cached merged configs until those expire:',
getSafeErrorMetadata(error),
);
+ return 'failed';
+ }
+ }
+
+ async function reload(tenantId?: string): Promise<ConfigReloadReport> {
+ const cleared = await clearLocal(tenantId);
+ if (!load) {
+ return { cleared, broadcast: await broadcast(tenantId) };
}
+ const loaded = await load();
+ if (!loaded.ok) {
+ return { cleared, loaded, broadcast: 'skipped' };
+ }
+ return { cleared, loaded, broadcast: await broadcast(tenantId) };
+ }
+
+ async function invalidate(tenantId?: string): Promise<void> {
+ await clearLocal(tenantId);
+ await broadcast(tenantId);
}
async function close(): Promise<void> {
@@ -172,7 +218,7 @@ export function createConfigCacheInvalidation(params: {
await pubsub.subscriber.unsubscribe(channel);
}
- return { invalidate, close };
+ return { invalidate, reload, close };
}
/**
diff --git a/packages/api/src/app/loader.ts b/packages/api/src/app/loader.ts
index 23de375031..47e03574d0 100644
--- a/packages/api/src/app/loader.ts
+++ b/packages/api/src/app/loader.ts
@@ -13,6 +13,7 @@ import {
validateSettingDefinitions,
} from 'librechat-data-provider';
import type { TCustomConfig, TEndpoint } from 'librechat-data-provider';
+import type { ConfigOverrideIssue } from '@librechat/data-schemas';
import type { ZodIssue } from 'zod';
import { isAgentInferenceExtension } from '../agents/inference';
import { checkConfigPolicyLinks } from './interface';
@@ -46,6 +47,40 @@ export class ConfigReloadError extends Error {
}
}
+/**
+ * Why librechat.yaml did not load again: `CONFIG_INVALID` when the loader refused the file
+ * (missing, unreadable or not fetched, unparseable, or failing `configSchema`, whose issues it
+ * lists by location), `LOAD_FAILED` when anything else failed, such as building the config from
+ * a file that loaded. The instance keeps its last good config.
+ */
+export type ConfigLoadFailure =
+ | { code: 'CONFIG_INVALID'; issues: ConfigOverrideIssue[] }
+ | { code: 'LOAD_FAILED' };
+
+export type ConfigLoadResult = { ok: true } | { ok: false; error: ConfigLoadFailure };
+
+function toLoadIssue(segments: string[], code: string): ConfigOverrideIssue {
+ return { path: segments.join('.'), segments, code };
+}
+
+/**
+ * Locations and zod codes only, since schema messages can echo the file's values; an
+ * unrecognized key is reported at its own location.
+ */
+export function getConfigLoadFailure(error: unknown): ConfigLoadFailure {
+ if (!(error instanceof ConfigReloadError)) {
+ return { code: 'LOAD_FAILED' };
+ }
+ const issues = (error.validationErrors ?? []).flatMap((issue) => {
+ const segments = issue.path.map(String);
+ if (issue.code !== 'unrecognized_keys') {
+ return [toLoadIssue(segments, issue.code)];
+ }
+ return issue.keys.map((key) => toLoadIssue([...segments, key], issue.code));
+ });
+ return { code: 'CONFIG_INVALID', issues };
+}
+
const OPENROUTER_PROMPT_CACHE_DEFAULT = {
key: 'promptCache',
default: true,
diff --git a/packages/api/src/app/service.spec.ts b/packages/api/src/app/service.spec.ts
index 1b8728a50a..a325f52ec2 100644
--- a/packages/api/src/app/service.spec.ts
+++ b/packages/api/src/app/service.spec.ts
@@ -16,6 +16,7 @@ import {
} from './service';
import { loadCustomEndpointsConfig } from '~/endpoints/custom/config';
import { getProviderConfig } from '~/endpoints/config/providers';
+import { ConfigReloadError } from './loader';
/** Extends AppConfig with mock fields used by merge behavior tests. */
/** `version` is a plain string section, so a test can mark which override layer it read. */
@@ -1766,6 +1767,105 @@ describe('createAppConfigService', () => {
});
});
+ describe('reloadBaseConfig', () => {
+ const invalidFile = () =>
+ new ConfigReloadError('Invalid custom config file', undefined, [
+ { code: 'unrecognized_keys', keys: ['bogusTopLevelKey'], path: [], message: 'bogus' },
+ {
+ code: 'invalid_type',
+ expected: 'string',
+ received: 'number',
+ path: ['interface', 'customWelcome'],
+ message: 'Expected string, received number',
+ },
+ ]);
+
+ it('loads librechat.yaml again after a clear and reports it loaded', async () => {
+ const deps = createDeps();
+ const { getAppConfig, clearAppConfigCache, reloadBaseConfig } = createAppConfigService(deps);
+ const initial = await getAppConfig({ baseOnly: true });
+ const next = { ...initial, interfaceConfig: { modelSelect: false } };
+ deps.loadBaseConfig.mockResolvedValueOnce(next);
+
+ await clearAppConfigCache();
+ await expect(reloadBaseConfig()).resolves.toEqual({ ok: true });
+
+ expect(deps.loadBaseConfig).toHaveBeenLastCalledWith('reload');
+ expect(await getAppConfig({ baseOnly: true })).toEqual(next);
+ expect(deps.loadBaseConfig).toHaveBeenCalledTimes(2);
+ });
+
+ it('reports a refused file by location and zod code, keeping the last good config', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const deps = createDeps();
+ const { getAppConfig, clearAppConfigCache, reloadBaseConfig } = createAppConfigService(deps);
+ const initial = await getAppConfig({ baseOnly: true });
+ deps.loadBaseConfig.mockRejectedValueOnce(invalidFile());
+
+ await clearAppConfigCache();
+ const result = await reloadBaseConfig();
+
+ expect(result).toEqual({
+ ok: false,
+ error: {
+ code: 'CONFIG_INVALID',
+ issues: [
+ { path: 'bogusTopLevelKey', segments: ['bogusTopLevelKey'], code: 'unrecognized_keys' },
+ {
+ path: 'interface.customWelcome',
+ segments: ['interface', 'customWelcome'],
+ code: 'invalid_type',
+ },
+ ],
+ },
+ });
+ expect(JSON.stringify(result)).not.toContain('Expected string');
+ expect(deps._cache._store.get('app_config:_BASE_')).toBe(initial);
+ expect(await getAppConfig({ baseOnly: true })).toBe(initial);
+ });
+
+ it('loads again and reports the failure when a read already restored the last good config', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const deps = createDeps();
+ const { getAppConfig, clearAppConfigCache, reloadBaseConfig } = createAppConfigService(deps);
+ const initial = await getAppConfig({ baseOnly: true });
+ deps.loadBaseConfig.mockRejectedValue(invalidFile());
+
+ await clearAppConfigCache();
+ expect(await getAppConfig({ baseOnly: true })).toBe(initial);
+ const result = await reloadBaseConfig();
+
+ expect(result).toMatchObject({ ok: false, error: { code: 'CONFIG_INVALID' } });
+ expect(deps.loadBaseConfig).toHaveBeenCalledTimes(3);
+ });
+
+ it('reports a failure to build the config from a file that loaded as LOAD_FAILED', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const deps = createDeps();
+ const { getAppConfig, clearAppConfigCache, reloadBaseConfig } = createAppConfigService(deps);
+ const initial = await getAppConfig({ baseOnly: true });
+ deps.loadBaseConfig.mockRejectedValueOnce(new Error('AppService failed: secret detail'));
+
+ await clearAppConfigCache();
+ const result = await reloadBaseConfig();
+
+ expect(result).toEqual({ ok: false, error: { code: 'LOAD_FAILED' } });
+ expect(await getAppConfig({ baseOnly: true })).toBe(initial);
+ });
+
+ it('resolves with the failure instead of rejecting when there is no last good config', async () => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const deps = createDeps();
+ deps.loadBaseConfig.mockRejectedValueOnce(invalidFile());
+ const { reloadBaseConfig } = createAppConfigService(deps);
+
+ await expect(reloadBaseConfig()).resolves.toMatchObject({
+ ok: false,
+ error: { code: 'CONFIG_INVALID' },
+ });
+ });
+ });
+
describe('clearOverrideCache', () => {
it('clears all override caches when no tenantId is provided', async () => {
const deps = createDeps({
diff --git a/packages/api/src/app/service.ts b/packages/api/src/app/service.ts
index f845cf5068..281ff99d8e 100644
--- a/packages/api/src/app/service.ts
+++ b/packages/api/src/app/service.ts
@@ -16,17 +16,24 @@ import {
} from 'librechat-data-provider';
import type { AppConfig, ConfigOverrideLayer } from '@librechat/data-schemas';
import type { Types } from 'mongoose';
+import type { CustomConfigLoadMode, ConfigLoadFailure, ConfigLoadResult } from './loader';
import type { EnvReferencePolicy } from './allowlist';
-import type { CustomConfigLoadMode } from './loader';
import type { SecretRebinding } from './rebinding';
import { createEnvReferencePolicy, parseEnvAllowlist } from './allowlist';
import { resolveRuntimeConfigSecrets } from '~/admin/secrets';
import { normalizeEndpointReferences } from './references';
import { mergeOverrideLayers } from './overrides';
+import { getConfigLoadFailure } from './loader';
import { checkAppConfigTheme } from './theme';
const BASE_CONFIG_KEY = '_BASE_';
+/** A base-config load: the config it serves, and why it failed when that is the last good one. */
+interface BaseConfigLoad {
+ config: AppConfig;
+ failure?: ConfigLoadFailure;
+}
+
function hiddenCustomEndpoints(
config: AppConfig,
tenantId?: string,
@@ -407,6 +414,8 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
getAppConfig: (options?: GetAppConfigOptions) => Promise<AppConfig>;
clearAppConfigCache: () => Promise<void>;
clearOverrideCache: (tenantId?: string) => Promise<void>;
+ /** Loads librechat.yaml again on this instance and resolves with the outcome; never rejects. */
+ reloadBaseConfig: () => Promise<ConfigLoadResult>;
/** The TTL in ms this service caches per-principal merged configs for. */
overrideCacheTtl: number;
} {
@@ -469,7 +478,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
return policy;
}
let lastGoodBaseConfig: AppConfig | undefined;
- let baseConfigFlight: { promise: Promise<AppConfig>; generation: number } | undefined;
+ let baseConfigFlight: { promise: Promise<BaseConfigLoad>; generation: number } | undefined;
/**
* Base-config generation: `clearAppConfigCache` bumps it before deleting `_BASE_`. A reload
@@ -564,56 +573,62 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
* An async store (Redis) can land the write after a clear's delete, so a write a clear
* overtook in flight is removed again.
*/
- async function loadAndCacheBaseConfig(
+ async function publishBaseConfig(
mode: CustomConfigLoadMode,
generation: number,
): Promise<AppConfig> {
- try {
- logger.info('[ensureBaseConfig] Loading base configuration...');
- const loaded = await loadBaseConfig(mode);
- if (!loaded) {
- throw new Error('Failed to initialize app configuration through AppService.');
- }
+ logger.info('[ensureBaseConfig] Loading base configuration...');
+ const loaded = await loadBaseConfig(mode);
+ if (!loaded) {
+ throw new Error('Failed to initialize app configuration through AppService.');
+ }
- const baseConfig = completeEffectiveConfig(loaded);
- if (generation !== baseGeneration) {
- return baseConfig;
- }
- if (baseConfig.availableTools) {
- await setCachedTools(baseConfig.availableTools);
- }
- if (generation !== baseGeneration) {
- return baseConfig;
- }
- await cache.set(BASE_CONFIG_KEY, baseConfig);
- if (generation !== baseGeneration) {
- await cache.delete(BASE_CONFIG_KEY);
- return baseConfig;
- }
- lastGoodBaseConfig = baseConfig;
+ const baseConfig = completeEffectiveConfig(loaded);
+ if (generation !== baseGeneration) {
return baseConfig;
+ }
+ if (baseConfig.availableTools) {
+ await setCachedTools(baseConfig.availableTools);
+ }
+ if (generation !== baseGeneration) {
+ return baseConfig;
+ }
+ await cache.set(BASE_CONFIG_KEY, baseConfig);
+ if (generation !== baseGeneration) {
+ await cache.delete(BASE_CONFIG_KEY);
+ return baseConfig;
+ }
+ lastGoodBaseConfig = baseConfig;
+ return baseConfig;
+ }
+
+ /** `publishBaseConfig`, keeping the last good config, and saying why, when a reload fails. */
+ async function loadAndCacheBaseConfig(
+ mode: CustomConfigLoadMode,
+ generation: number,
+ ): Promise<BaseConfigLoad> {
+ try {
+ return { config: await publishBaseConfig(mode, generation) };
} catch (error) {
if (mode === 'startup') {
throw error;
}
- return restoreLastGoodBaseConfig(error, generation);
+ const config = await restoreLastGoodBaseConfig(error, generation);
+ return { config, failure: getConfigLoadFailure(error) };
}
}
/**
- * Ensure the YAML-derived base config is loaded and cached.
- * Returns the `_BASE_` config (YAML + AppService). No DB queries.
- *
* Reads join the load in flight for the current generation. A load for a newer generation
* starts once the stale one settles, so the loader's process-wide side effects (such as the
* subagent cap) end with the newest librechat.yaml rather than whichever read finished last.
*/
- async function ensureBaseConfig(refresh?: boolean): Promise<AppConfig> {
+ async function loadBase(refresh?: boolean): Promise<BaseConfigLoad> {
const cached = (await cache.get(BASE_CONFIG_KEY)) as AppConfig | undefined;
if (cached) {
lastGoodBaseConfig ??= cached;
if (!refresh) {
- return cached;
+ return { config: cached };
}
}
@@ -637,6 +652,30 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
}
}
+ /**
+ * Ensure the YAML-derived base config is loaded and cached.
+ * Returns the `_BASE_` config (YAML + AppService). No DB queries.
+ */
+ async function ensureBaseConfig(refresh?: boolean): Promise<AppConfig> {
+ return (await loadBase(refresh)).config;
+ }
+
+ /**
+ * Loads librechat.yaml again on this instance (or joins the load of it in flight), as the next
+ * read after `clearAppConfigCache` would, and resolves with the outcome instead of only logging
+ * a failure, so the admin reload route can answer for it. A failed load keeps the last good
+ * config, as every reload does. Never rejects.
+ */
+ async function reloadBaseConfig(): Promise<ConfigLoadResult> {
+ try {
+ const { failure } = await loadBase(true);
+ return failure ? { ok: false, error: failure } : { ok: true };
+ } catch (error) {
+ logger.error('[reloadBaseConfig] Failed to load the base configuration:', error);
+ return { ok: false, error: getConfigLoadFailure(error) };
+ }
+ }
+
/**
* Get the app configuration, optionally merged with DB overrides for the given principal.
*
@@ -845,6 +884,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
getAppConfig,
clearAppConfigCache,
clearOverrideCache,
+ reloadBaseConfig,
overrideCacheTtl,
};
}
diff --git a/packages/data-schemas/src/types/admin.ts b/packages/data-schemas/src/types/admin.ts
index 9cae7e8175..47d5e92039 100644
--- a/packages/data-schemas/src/types/admin.ts
+++ b/packages/data-schemas/src/types/admin.ts
@@ -60,6 +60,35 @@ export type AdminConfigDeleteResponse = {
success: boolean;
};
+/**
+ * 200 from POST `/reload`. `broadcast` is false when the deployment has no Redis: only the
+ * instance that answered reloaded, and any other keeps its cached librechat.yaml.
+ */
+export type AdminConfigReloadResponse = {
+ success: true;
+ broadcast: boolean;
+};
+
+/**
+ * A POST `/reload` that did not load librechat.yaml or did not reach every instance:
+ * - 422 `CONFIG_RELOAD_INVALID`: the loader refused the file (missing or, with a remote
+ * `CONFIG_PATH`, not fetched; unparseable; or failing `configSchema`). `issues` lists the
+ * schema failures by location and zod code, with no values, and is empty when the file could
+ * not be read or parsed. The answering instance keeps its last good config, and the other
+ * instances are not told to reload.
+ * - 500 `CONFIG_RELOAD_FAILED`: a clear on the answering instance failed, or the file loaded but
+ * building the config from it failed (the last good config is kept, and nothing is broadcast).
+ * - 503 `CONFIG_RELOAD_BROADCAST_FAILED`: the answering instance reloaded, but the other
+ * instances could not be told (Redis unreachable).
+ *
+ * The 500 and 503 are safe to retry; a 422 with `issues` needs the file fixed first.
+ */
+export type AdminConfigReloadErrorResponse = {
+ error: string;
+ code: 'CONFIG_RELOAD_INVALID' | 'CONFIG_RELOAD_FAILED' | 'CONFIG_RELOAD_BROADCAST_FAILED';
+ issues?: Array<{ path: string; segments: string[]; code: string }>;
+};
+
/* ── Admin API request types ────────────────────────────────────────── */
/**
@@ -265,7 +294,8 @@ export type AdminConfigChangeErrorResponse = {
* unambiguously. A database engine that cannot store a dotted key answers 422
* `CONFIG_KEY_NOT_STORABLE` and stores nothing.
* - `sectionAllowlist`: writes to a top-level key `configSchema` does not define are rejected,
- * and stored ones are ignored at merge.
+ * and stored ones are ignored at merge. Removing what an older server stored under one, whole
+ * or in part (DELETE `/fields`, or an `unset` in `POST …/changes`), is still allowed.
* - `readFilteredWriteResponses`: write responses are filtered by the caller's read access.
* - `preAuthBaseOverrides`: Base (`__base__`) overrides apply before sign-in: the signed-out
* `GET /api/config` serves Base `registration.socialLogins` and the pre-login interface fields,
@@ -316,7 +346,11 @@ export type AdminConfigChangeErrorResponse = {
* user 100; Base 0), and a write that omits `priority` keeps the stored one, so a PUT no
* longer resets it. A profile write validates on `librechat.yaml` with the active Base
* document resolved over it, as the merge applies it; when Base changes before the write
- * commits, the write is judged on that Base, and taken back (400) when it fails there.
+ * commits, the write is judged on that Base, and taken back (400) when it fails there. Both are
+ * the ones of the tenant partition the profile merges in: a group or user profile an admin
+ * without a tenant writes for a tenant's principal is judged on that tenant's librechat.yaml
+ * scope and Base, and answers 409 `CONFIG_VERSION_CONFLICT` if it moves to another partition
+ * while the write runs.
* - `priorityRoute`: PATCH `/:principalType/:principalId/priority` `{ priority,
* expectedRevision? }` sets only the priority (overrides, tombstones and `isActive` stay),
* for broad `manage:configs` only. No document answers 404 `CONFIG_NOT_FOUND`, an invalid
@@ -421,6 +455,23 @@ export type AdminConfigChangeErrorResponse = {
* reference librechat.yaml uses beside a new destination keeps it, so for allowlisted names
* (`envAllowlist`) the rule stops a destination change that leaves the reference out, not one
* that restates it. Header maps are not covered.
+ * - `reloadRoute`: `POST /reload` reloads librechat.yaml on every instance without a config
+ * write. It is for an admin without a tenant who holds broad `manage:configs` (otherwise 403
+ * `FORBIDDEN`), since it reaches every tenant. The answering instance drops its cached
+ * librechat.yaml, every merged config and its MCP config-source servers, then loads the file
+ * again before it answers. When the file loaded, it tells every other instance (with Redis) to
+ * do the same; each of them loads the file on its next read, keeping its last good config if
+ * that fails. It answers 200 `{ success: true, broadcast }` (`AdminConfigReloadResponse`), with
+ * `broadcast: false` when there is no Redis to publish on. When the loader refuses the file it
+ * answers 422 `CONFIG_RELOAD_INVALID` with `issues` by location, keeps the last good config and
+ * broadcasts nothing; a failed clear, or a failure to build the config from a file that loaded,
+ * answers 500 `CONFIG_RELOAD_FAILED`; a failed publish after this instance reloaded answers 503
+ * `CONFIG_RELOAD_BROADCAST_FAILED` (`AdminConfigReloadErrorResponse`). An instance that misses
+ * the broadcast (the publish failed, or its subscription was down) keeps its old
+ * librechat.yaml, which never expires, until a later reload or config write reaches it or it
+ * restarts. Every reload that runs is audited as `config.reloaded` with `metadata.broadcast`, a
+ * failed one with outcome `failure` and its `code`. A committed config write still reloads the
+ * file too, on the next read.
*/
export type AdminConfigFeatureFlag =
| 'dottedKeys'
@@ -441,7 +492,8 @@ export type AdminConfigFeatureFlag =
| 'azureGroupSecrets'
| 'yamlSecretPreview'
| 'envAllowlist'
- | 'secretRebinding';
+ | 'secretRebinding'
+ | 'reloadRoute';
export type AdminConfigFeatures = Record<AdminConfigFeatureFlag, boolean>;
@@ -598,6 +650,8 @@ export const AUDIT_ACTIONS = [
'config.updated',
'config.deleted',
'config.toggled',
+ /** librechat.yaml reloaded without a config write; target `{ type: 'config', id: 'librechat.yaml' }`. */
+ 'config.reloaded',
/** Account lifecycle; `role_changed` covers IdP-driven role assignment. */
'user.deleted',
'user.role_changed',
@@ -644,6 +698,7 @@ export const AUDIT_ACTION_CATEGORY: Record<AuditAction, AuditCategory> = {
'config.updated': 'config',
'config.deleted': 'config',
'config.toggled': 'config',
+ 'config.reloaded': 'config',
'user.deleted': 'user',
'user.role_changed': 'user',
'acl.granted': 'acl',