Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🪪 feat: Name the Build and Config in Telemetry, Configure Stack Depth, Keep Scoped Frames (#17515)
Danny Avila696a1b6b56f9
diff --git a/.env.example b/.env.exampleindex 6def91d11b..d7c0a8a77c 100644--- a/.env.example+++ b/.env.example@@ -381,6 +381,7 @@ NODE_MAX_OLD_SPACE_SIZE=6144# OTEL_LOGS_ENABLED=false# OTEL_LOGS_LEVEL=info# OTEL_SERVICE_NAME=librechat+# Defaults to the image's BUILD_COMMIT, then the npm package version.# OTEL_SERVICE_VERSION=# Exporter protocol for every signal: http/protobuf (default, port 4318) or grpc (port 4317).# Per-signal overrides: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, OTEL_EXPORTER_OTLP_LOGS_PROTOCOL.diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.jsindex 8cc6dc5260..c9e5cdd6de 100644--- a/api/cache/getLogStores.js+++ b/api/cache/getLogStores.js@@ -53,7 +53,9 @@ const namespaces = {[CacheKeys.USER_PRINCIPALS]: userPrincipalsCache() ?? disabledCache,/** Authorization IDs stay uncached because a failed shared invalidation cannot fail closed. */[CacheKeys.PROMPT_GROUPS_ACCESS]: disabledCache,- [CacheKeys.APP_CONFIG]: standardCache(CacheKeys.APP_CONFIG),+ [CacheKeys.APP_CONFIG]: standardCache(CacheKeys.APP_CONFIG, undefined, undefined, {+ throwOnErrors: true,+ }),[CacheKeys.CONFIG_STORE]: standardCache(CacheKeys.CONFIG_STORE),[CacheKeys.TOOL_CACHE]: standardCache(CacheKeys.TOOL_CACHE),[CacheKeys.PENDING_REQ]: standardCache(CacheKeys.PENDING_REQ),diff --git a/api/cache/getLogStores.spec.js b/api/cache/getLogStores.spec.jsindex a6bd4f63a8..29ed2931b2 100644--- a/api/cache/getLogStores.spec.js+++ b/api/cache/getLogStores.spec.js@@ -1,8 +1,24 @@+jest.mock('@librechat/api', () => {+ const api = jest.requireActual('@librechat/api');+ return { ...api, standardCache: jest.fn(api.standardCache) };+});++const { standardCache } = require('@librechat/api');const { logger } = require('@librechat/data-schemas');const { CacheKeys } = require('librechat-data-provider');const getLogStores = require('./getLogStores');+const appConfigCacheArgs = standardCache.mock.calls.find(([key]) => key === CacheKeys.APP_CONFIG);describe('auditCache', () => {+ it('requires APP_CONFIG publication errors to propagate', () => {+ expect(appConfigCacheArgs).toEqual([+ CacheKeys.APP_CONFIG,+ undefined,+ undefined,+ { throwOnErrors: true },+ ]);+ });+it('logs counts and TTL info without cache keys or values', async () => {const secretKey = 'user-id-1234567890';const secretValue = 'super-secret-access-token';diff --git a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.jsindex bdcc1dda7b..3e24e018ac 100644--- a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js+++ b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js@@ -9,6 +9,7 @@ const os = require('os');const path = require('path');const yaml = require('js-yaml');const { Constants } = require('librechat-data-provider');+const { loadAndFormatTools } = require('~/server/services/start/tools');jest.mock('~/server/services/start/tools', () => ({loadAndFormatTools: jest.fn(() => ({})),@@ -122,3 +123,30 @@ describe('admin config writes and librechat.yaml', () => {expect((await getAppConfig({ baseOnly: true })).filters).toEqual(lastGood.filters);});});++describe('published telemetry settings', () => {+ const processDefault = Error.stackTraceLimit;++ afterEach(async () => {+ writeYaml({});+ await reloadConfigCaches();+ });++ it('keeps the last good stack limit if tool loading rejects a reload', async () => {+ writeYaml({ telemetry: { stackTraceLimit: 25 } });+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: true } });+ expect(Error.stackTraceLimit).toBe(25);++ writeYaml({ telemetry: { stackTraceLimit: 60 } });+ loadAndFormatTools.mockImplementationOnce(() => {+ throw new Error('tool scan failed');+ });+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: false } });+ expect(Error.stackTraceLimit).toBe(25);+ expect((await getAppConfig({ baseOnly: true })).config.telemetry.stackTraceLimit).toBe(25);++ writeYaml({});+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: true } });+ expect(Error.stackTraceLimit).toBe(processDefault);+ });+});diff --git a/api/server/services/Config/app.js b/api/server/services/Config/app.jsindex cd6a67aa36..665d4f1d95 100644--- a/api/server/services/Config/app.js+++ b/api/server/services/Config/app.js@@ -5,6 +5,7 @@ const {isEnabled,expandCustomEndpoint,createAppConfigService,+ applyLoadedConfig,clearMcpConfigCache,createCodeEnvironmentRegistry,createConfigCacheInvalidation,@@ -52,6 +53,7 @@ const loadBaseConfig = async (mode) => {const appConfigService = createAppConfigService({loadBaseConfig,+ onBaseConfigPublished: applyLoadedConfig,setCachedTools,getCache: getLogStores,cacheKeys: CacheKeys,diff --git a/api/server/services/Config/loadCustomConfig.js b/api/server/services/Config/loadCustomConfig.jsindex c253c4f14b..555f3d0873 100644--- a/api/server/services/Config/loadCustomConfig.js+++ b/api/server/services/Config/loadCustomConfig.js@@ -1,5 +1,10 @@const path = require('path');-const { loadYaml, redactConfigSecretMaps, createCustomConfigLoader } = require('@librechat/api');+const {+ loadYaml,+ recordLoadedConfig,+ redactConfigSecretMaps,+ createCustomConfigLoader,+} = require('@librechat/api');const projectRoot = path.resolve(__dirname, '..', '..', '..', '..');const defaultConfigPath = path.resolve(projectRoot, 'librechat.yaml');@@ -8,4 +13,5 @@ module.exports = createCustomConfigLoader({loadLocal: loadYaml,defaultConfigPath,redactConfig: redactConfigSecretMaps,+ onLoad: recordLoadedConfig,});diff --git a/librechat.example.yaml b/librechat.example.yamlindex d77eea84f9..832dedec57 100644--- a/librechat.example.yaml+++ b/librechat.example.yaml@@ -10,7 +10,10 @@ cache: true# Browser telemetry and session recording are independent opt-ins. Collector URL,# authentication and service settings remain in .env. YAML flags override .env;# omitting them preserves the corresponding environment settings.+# stackTraceLimit (1 to 1000) sets how many frames each error stack records; unset keeps+# Node's default of 10, or --stack-trace-limit from NODE_OPTIONS.# telemetry:+# stackTraceLimit: 50# rum:# enabled: true# sessionReplay:diff --git a/packages/api/src/admin/config.report.spec.ts b/packages/api/src/admin/config.report.spec.tsindex 2df759c5a8..5b49302115 100644--- a/packages/api/src/admin/config.report.spec.ts+++ b/packages/api/src/admin/config.report.spec.ts@@ -323,6 +323,43 @@ describe('write report', () => {await methods.deleteConfig('role' as never, '__base__');});+ it.each([+ ['role', '__base__'],+ ['role', 'telemetry-role'],+ ['group', 'telemetry-group'],+ ['user', 'telemetry-user'],+ ])(+ 'skips process-wide stack limits in %s %s overrides and tombstones',+ async (principalType, principalId) => {+ const params = { principalType, principalId };+ const handlers = createHandlers();+ const skipped = ['telemetry.stackTraceLimit:deployment_only_path'];+ const put = await call(handlers.upsertConfigOverrides, {+ params,+ body: { overrides: { telemetry: { stackTraceLimit: 60 }, memory: { disabled: true } } },+ });+ expect(put.statusCode).toBe(201);+ expect(describeAll(put.body?.skipped)).toEqual(skipped);+ expect((await stored(params))?.overrides).toEqual({ memory: { disabled: true } });++ const patch = await call(handlers.patchConfigField, {+ params,+ body: { entries: [{ fieldPath: 'telemetry.stackTraceLimit', value: 60 }] },+ });+ expect(patch.statusCode).toBe(200);+ expect(describeAll(patch.body?.skipped)).toEqual(skipped);++ const tombstone = await call(handlers.tombstoneConfigField, {+ params,+ body: { fieldPath: 'telemetry.stackTraceLimit' },+ });+ expect(tombstone.statusCode).toBe(200);+ expect(describeAll(tombstone.body?.skipped)).toEqual(skipped);+ expect((await stored(params))?.tombstones).toEqual([]);+ await methods.deleteConfig(principalType as never, principalId);+ },+ );+it('a tombstone of a field only librechat.yaml sets is skipped; deleting a stored one is cleanup', async () => {const params = nextParams();await seed(params, { rateLimits: { fileUploads: { ipMax: 1 } }, memory: { disabled: true } });diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.tsindex dc5ace62c5..5db9cab775 100644--- a/packages/api/src/app/index.ts+++ b/packages/api/src/app/index.ts@@ -17,6 +17,7 @@ export * from './server';export * from './readiness';export * from './origin';export * from './agents';+export { applyLoadedConfig, recordLoadedConfig } from './runtime';export { resolveBuildInfo } from './build';export type { BuildInfo } from './build';export { buildPreLoginInterface, withSafePolicyLinks } from './interface';diff --git a/packages/api/src/app/loader.spec.ts b/packages/api/src/app/loader.spec.tsindex b6dff512dc..a0530423b8 100644--- a/packages/api/src/app/loader.spec.ts+++ b/packages/api/src/app/loader.spec.ts@@ -1,7 +1,7 @@import path from 'path';import { logger } from '@librechat/data-schemas';import type { TCustomConfig } from 'librechat-data-provider';-import { createCustomConfigLoader, ConfigReloadError } from './loader';+import { createCustomConfigLoader, ConfigReloadError, hashCustomConfig } from './loader';import { loadYaml } from '~/utils/yaml';const fixture = (name: string) => path.join(__dirname, '__fixtures__', 'theme', `${name}.yaml`);@@ -278,3 +278,105 @@ describe('createCustomConfigLoader remote fetch failures', () => {await expect(loaderFor(refused)()).resolves.toBeNull();});});++describe('createCustomConfigLoader onLoad', () => {+ const originalConfigPath = process.env.CONFIG_PATH;++ beforeEach(() => {+ jest.spyOn(logger, 'error').mockImplementation(() => logger);+ jest.spyOn(logger, 'info').mockImplementation(() => logger);+ });++ afterEach(() => {+ jest.restoreAllMocks();+ if (originalConfigPath === undefined) {+ delete process.env.CONFIG_PATH;+ } else {+ process.env.CONFIG_PATH = originalConfigPath;+ }+ });++ /** The loader parses a string source as YAML, so comments and layout reach the digest input. */+ const createTextLoader = (text: string, onLoad: jest.Mock) => {+ process.env.CONFIG_PATH = '/virtual/librechat.yaml';+ return createCustomConfigLoader({+ loadLocal: () => text,+ defaultConfigPath: '/virtual/librechat.yaml',+ redactConfig: (value: TCustomConfig) => value,+ onLoad,+ });+ };++ const loadText = async (text: string) => {+ const onLoad = jest.fn();+ const config = await createTextLoader(text, onLoad)(false);+ return { config, onLoad };+ };++ const digestOf = async (text: string): Promise<string> => {+ const { onLoad } = await loadText(text);+ expect(onLoad).toHaveBeenCalledTimes(1);+ return onLoad.mock.calls[0][0].digest;+ };++ it('hands the loaded config and a short digest to onLoad', async () => {+ const { config, onLoad } = await loadText(+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n',+ );++ expect(config?.telemetry?.stackTraceLimit).toBe(40);+ expect(onLoad).toHaveBeenCalledWith({+ config,+ digest: expect.stringMatching(/^[0-9a-f]{12}$/),+ });+ });++ it('keeps the digest across comments, layout and key order, and moves it on a value change', async () => {+ const base = await digestOf('version: 1.3.18\ncache: true\ninterface:\n modelSelect: true\n');+ const reordered = await digestOf(+ '# comment\ninterface: { modelSelect: true }\n\ncache: true\nversion: 1.3.18\n',+ );+ const changed = await digestOf(+ 'version: 1.3.18\ncache: false\ninterface:\n modelSelect: true\n',+ );++ expect(reordered).toBe(base);+ expect(changed).not.toBe(base);+ });++ it('hashes the file as written, not the modelSpecs the schema fills in', async () => {+ const text =+ 'version: 1.3.18\nmodelSpecs:\n list:\n - name: spec\n label: Spec\n' ++ ' preset:\n endpoint: openAI\n model: gpt-4o\n';+ const { config, onLoad } = await loadText(text);++ expect(config?.modelSpecs).toBeDefined();+ expect(onLoad.mock.calls[0][0].digest).toBe(+ hashCustomConfig({+ version: '1.3.18',+ modelSpecs: {+ list: [{ name: 'spec', label: 'Spec', preset: { endpoint: 'openAI', model: 'gpt-4o' } }],+ },+ }),+ );+ });++ it('does not call onLoad when a reload is rejected', async () => {+ const onLoad = jest.fn();+ const load = createTextLoader('version: 1.3.18\nnotAKey: true\n', onLoad);++ await expect(load(false, { mode: 'reload' })).rejects.toBeInstanceOf(ConfigReloadError);+ expect(onLoad).not.toHaveBeenCalled();+ });+});++describe('hashCustomConfig', () => {+ it('ignores key order and keeps array order', () => {+ expect(hashCustomConfig({ a: 1, b: { c: 2, d: [1, 2] } })).toBe(+ hashCustomConfig({ b: { d: [1, 2], c: 2 }, a: 1 }),+ );+ expect(hashCustomConfig({ b: { d: [2, 1], c: 2 }, a: 1 })).not.toBe(+ hashCustomConfig({ a: 1, b: { c: 2, d: [1, 2] } }),+ );+ });+});diff --git a/packages/api/src/app/loader.ts b/packages/api/src/app/loader.tsindex 58b6ca0779..63b2c888ee 100644--- a/packages/api/src/app/loader.ts+++ b/packages/api/src/app/loader.ts@@ -1,5 +1,6 @@import axios from 'axios';import yaml from 'js-yaml';+import { createHash } from 'node:crypto';import { logger } from '@librechat/data-schemas';import {configSchema,@@ -22,11 +23,40 @@ export interface CustomConfigLoadOptions {mode?: CustomConfigLoadMode;}+export interface LoadedCustomConfig {+ config: TCustomConfig;+ /** {@link hashCustomConfig} of the file as parsed, before any check rewrote part of it. */+ digest: string;+}+export interface CustomConfigLoaderOptions {defaultConfigPath: string;loadLocal: (configPath: string) => unknown;fetchRemote?: (configPath: string) => Promise<unknown>;redactConfig: (config: TCustomConfig) => TCustomConfig;+ /** Runs after each successful load, at startup and on reload, before the config is returned. */+ onLoad?: (loaded: LoadedCustomConfig) => void;+}++const CONFIG_DIGEST_LENGTH = 12;++/** Keys are unique within an object, and code-unit order does not vary by locale. */+function sortKeys(_key: string, value: unknown): unknown {+ if (value === null || typeof value !== 'object' || Array.isArray(value)) {+ return value;+ }+ return Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : 1)));+}++/**+ * A short, stable hash of a parsed librechat.yaml: comments, formatting and key order do not+ * change it, so it moves only when the configuration does. Values are hashed, never logged.+ */+export function hashCustomConfig(config: unknown): string {+ return createHash('sha256')+ .update(JSON.stringify(config, sortKeys))+ .digest('hex')+ .slice(0, CONFIG_DIGEST_LENGTH);}export class ConfigReloadError extends Error {@@ -139,6 +169,7 @@ export function createCustomConfigLoader({defaultConfigPath,loadLocal,redactConfig,+ onLoad,fetchRemote = async (configPath: string): Promise<unknown> =>(await axios.get(configPath, serviceRequestBudget())).data,}: CustomConfigLoaderOptions): (@@ -229,6 +260,8 @@ export function createCustomConfigLoader({}}+ /** The checks below return copies, but `modelSpecs` is assigned onto this object later. */+ const parsedConfig = loadedConfig;const themeCheck = checkConfigTheme(loadedConfig);loadedConfig = themeCheck.config;reportThemeCheck(configPath, themeCheck.errors, themeCheck.warnings);@@ -288,6 +321,7 @@ export function createCustomConfigLoader({process.exit(1);}+ const digest = hashCustomConfig(parsedConfig);const customConfig = loadedConfig as TCustomConfig;/** Expanded copies are discarded: AppService serves them, and the config stays as written. */for (const endpoint of (customConfig.endpoints?.custom ?? []) as TEndpoint[]) {@@ -304,6 +338,7 @@ export function createCustomConfigLoader({logger.debug('Custom config:', loggableConfig);}+ onLoad?.({ config: customConfig, digest });loadedSuccessfully = true;return customConfig;} catch (error) {diff --git a/packages/api/src/app/runtime.spec.ts b/packages/api/src/app/runtime.spec.tsnew file mode 100644index 0000000000..0789c8b336--- /dev/null+++ b/packages/api/src/app/runtime.spec.ts@@ -0,0 +1,380 @@+import { Keyv } from 'keyv';+import KeyvRedis from '@keyv/redis';+import { createClient } from '@redis/client';+import { PrincipalType } from 'librechat-data-provider';+import { logger, AppService } from '@librechat/data-schemas';+import type { TCustomConfig } from 'librechat-data-provider';+import type { AppConfigServiceDeps } from './service';+import { applyLoadedConfig, recordLoadedConfig, applyStackTraceLimit } from './runtime';+import { getConfigDigest, resetConfigDigestForTests } from '~/telemetry/digest';+import { startRespServer } from '~/cache/__tests__/resp.helper';+import { createCustomConfigLoader } from './loader';+import { createAppConfigService } from './service';++function frameCount(depth: number): number {+ const recurse = (remaining: number): Error =>+ remaining === 0 ? new Error('deep') : recurse(remaining - 1);+ return (recurse(depth).stack ?? '').split('\n').filter((line) => line.includes(' at ')).length;+}++describe('applyStackTraceLimit', () => {+ const processDefault = Error.stackTraceLimit;++ afterEach(() => {+ applyStackTraceLimit(undefined);+ });++ it('leaves the process default alone when no limit is configured', () => {+ applyStackTraceLimit(undefined);++ expect(Error.stackTraceLimit).toBe(processDefault);+ });++ it('records deeper stacks once a limit is set and restores the default when it is removed', () => {+ applyStackTraceLimit(60);++ expect(Error.stackTraceLimit).toBe(60);+ expect(frameCount(80)).toBe(60);++ applyStackTraceLimit(undefined);++ expect(Error.stackTraceLimit).toBe(processDefault);+ });+});++describe('applyLoadedConfig', () => {+ const originalConfigPath = process.env.CONFIG_PATH;+ const processDefault = Error.stackTraceLimit;++ afterEach(() => {+ applyStackTraceLimit(undefined);+ resetConfigDigestForTests();+ jest.restoreAllMocks();+ if (originalConfigPath === undefined) {+ delete process.env.CONFIG_PATH;+ } else {+ process.env.CONFIG_PATH = originalConfigPath;+ }+ });++ function createRuntime(+ baseCache?: ReturnType<AppConfigServiceDeps['getCache']>,+ getApplicableConfigs: AppConfigServiceDeps['getApplicableConfigs'] = async () => [],+ ) {+ process.env.CONFIG_PATH = '/virtual/librechat.yaml';+ let text = 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 25\n';+ const load = createCustomConfigLoader({+ loadLocal: () => text,+ defaultConfigPath: '/virtual/librechat.yaml',+ redactConfig: (value: TCustomConfig) => value,+ onLoad: recordLoadedConfig,+ });+ const stored = new Map<string, unknown>();+ const cache = {+ get: jest.fn(async (key: string) => stored.get(key)),+ set: jest.fn(async (key: string, value: unknown): Promise<boolean | void> => {+ stored.set(key, value);+ }),+ delete: async (key: string) => stored.delete(key),+ };+ const formatTools = jest.fn(() => ({}));+ const buildConfig = jest.fn(AppService);+ const setCachedTools = jest.fn(async (): Promise<boolean | void> => {});+ const service = createAppConfigService({+ loadBaseConfig: async (mode) => {+ const config = (await load(false, { mode })) ?? {};+ const systemTools = formatTools();+ return buildConfig({ config, systemTools });+ },+ onBaseConfigPublished: applyLoadedConfig,+ setCachedTools,+ getCache: () => baseCache ?? cache,+ cacheKeys: { APP_CONFIG: 'app_config' },+ getApplicableConfigs,+ getUserPrincipals: async () => [],+ });+ return {+ ...service,+ load,+ cache,+ formatTools,+ buildConfig,+ setCachedTools,+ setText: (value: string) => {+ text = value;+ },+ };+ }++ it('does not change runtime settings when the file is only parsed', async () => {+ const runtime = createRuntime();++ await runtime.load(false);++ expect(Error.stackTraceLimit).toBe(processDefault);+ expect(getConfigDigest()).toBeUndefined();+ });++ it('applies telemetry settings on publication and restores defaults when the key is removed', async () => {+ jest.spyOn(logger, 'info').mockImplementation(() => logger);+ const runtime = createRuntime();++ await runtime.getAppConfig({ baseOnly: true });+ const first = getConfigDigest();++ expect(Error.stackTraceLimit).toBe(25);+ expect(first).toMatch(/^[0-9a-f]{12}$/);++ runtime.setText('version: 1.3.18\n');+ await expect(runtime.reloadBaseConfig()).resolves.toEqual({ ok: true });++ expect(Error.stackTraceLimit).toBe(processDefault);+ expect(getConfigDigest()).toMatch(/^[0-9a-f]{12}$/);+ expect(getConfigDigest()).not.toBe(first);+ });++ it('initializes local runtime settings before serving a serialized shared base cache', async () => {+ const publisher = createRuntime();+ const published = await publisher.getAppConfig({ baseOnly: true });+ const digest = getConfigDigest();+ const cached = JSON.parse(JSON.stringify(published));+ applyStackTraceLimit(undefined);+ resetConfigDigestForTests();++ const restarted = createRuntime();+ restarted.cache.get.mockResolvedValue(cached);+ const local = await restarted.getAppConfig({ baseOnly: true });++ expect(restarted.buildConfig).toHaveBeenCalledTimes(1);+ expect(local).not.toBe(cached);+ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);++ restarted.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');+ restarted.buildConfig.mockRejectedValueOnce(new Error('reload failed'));+ await expect(restarted.reloadBaseConfig()).resolves.toMatchObject({ ok: false });+ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);+ });++ it('keeps local YAML and runtime identity when another process replaces the shared base', async () => {+ const runtime = createRuntime();+ const local = await runtime.getAppConfig({ baseOnly: true });+ const digest = getConfigDigest();+ runtime.cache.get.mockResolvedValue({ config: { telemetry: { stackTraceLimit: 60 } } });++ expect(await runtime.getAppConfig({ baseOnly: true })).toBe(local);+ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);+ expect(runtime.buildConfig).toHaveBeenCalledTimes(1);++ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n');+ await runtime.clearAppConfigCache();+ const reloaded = await runtime.getAppConfig({ baseOnly: true });+ expect(reloaded.config.telemetry?.stackTraceLimit).toBe(40);+ expect(Error.stackTraceLimit).toBe(40);+ expect(getConfigDigest()).not.toBe(digest);+ });++ it('partitions serialized merged caches by local base and reuses identical bases', async () => {+ const values = new Map<string, string>();+ const cache = {+ get: async (key: string) => {+ const value = values.get(key);+ return value ? JSON.parse(value) : undefined;+ },+ set: async (key: string, value: unknown) => {+ values.set(key, JSON.stringify(value));+ },+ delete: async (key: string) => values.delete(key),+ };+ const first = createRuntime(cache);+ first.setText(+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 25\nmemory:\n disabled: true\n',+ );+ await first.getAppConfig({ baseOnly: true });+ const firstDigest = getConfigDigest();+ const second = createRuntime(cache);+ second.setText(+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\nmemory:\n disabled: false\n',+ );+ await second.getAppConfig({ role: 'USER' });++ const local = await first.getAppConfig({ role: 'USER' });+ expect(local.config.telemetry?.stackTraceLimit).toBe(25);+ expect(local.memory?.disabled).toBe(true);+ expect([...values.keys()].filter((key) => key.startsWith('_OVERRIDE_'))).toHaveLength(2);++ const identical = createRuntime(cache);+ identical.setText(+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 25\nmemory:\n disabled: true\n',+ );+ expect((await identical.getAppConfig({ role: 'USER' })).memory?.disabled).toBe(true);+ expect(getConfigDigest()).toBe(firstDigest);+ expect([...values.keys()].filter((key) => key.startsWith('_OVERRIDE_'))).toHaveLength(2);++ identical.setText(+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\nmemory:\n disabled: false\n',+ );+ await identical.clearAppConfigCache();+ expect((await identical.getAppConfig({ role: 'USER' })).memory?.disabled).toBe(false);+ expect(Error.stackTraceLimit).toBe(40);+ expect(getConfigDigest()).not.toBe(firstDigest);+ expect([...values.keys()].filter((key) => key.startsWith('_OVERRIDE_'))).toHaveLength(3);+ });++ it('serves database restrictions uncached through Redis write and connection failures', async () => {+ jest.spyOn(logger, 'warn').mockReturnValue(logger);+ const server = await startRespServer();+ const client = createClient({+ url: server.url,+ socket: { reconnectStrategy: false, connectTimeout: 100 },+ });+ const adapter = new KeyvRedis(client, { throwOnErrors: true });+ const cache = new Keyv({ store: adapter, namespace: 'APP_CONFIG', throwOnErrors: true });+ cache.on('error', () => undefined);+ const getApplicableConfigs = jest.fn(+ async (): ReturnType<AppConfigServiceDeps['getApplicableConfigs']> => [+ {+ principalType: PrincipalType.ROLE,+ principalId: 'USER',+ priority: 10,+ overrides: { interface: { modelSelect: false } },+ },+ ],+ );+ try {+ await client.connect();+ const runtime = createRuntime(+ {+ get: (key) => cache.get(key),+ set: (key, value) => cache.set(key, value),+ delete: (key) => cache.delete(key),+ },+ getApplicableConfigs,+ );+ await runtime.getAppConfig({ baseOnly: true });+ const digest = getConfigDigest();+ server.readonly = true;++ expect(+ (await runtime.getAppConfig({ role: 'USER', failClosed: true })).interfaceConfig+ ?.modelSelect,+ ).toBe(false);+ await server.close();+ client.destroy();+ expect(+ (await runtime.getAppConfig({ role: 'USER', failClosed: true })).interfaceConfig+ ?.modelSelect,+ ).toBe(false);++ expect(getApplicableConfigs).toHaveBeenCalledTimes(2);+ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);+ } finally {+ if (client.isOpen) client.destroy();+ await server.close();+ }+ });++ it('keeps last-good telemetry when the real Redis adapter rejects SET, then retries', async () => {+ jest.spyOn(logger, 'error').mockImplementation(() => logger);+ const server = await startRespServer();+ const client = createClient({ url: server.url, socket: { reconnectStrategy: false } });+ const adapter = new KeyvRedis(client, { throwOnErrors: true });+ const cache = new Keyv({ store: adapter, namespace: 'APP_CONFIG', throwOnErrors: true });+ cache.on('error', () => undefined);+ try {+ await client.connect();+ const runtime = createRuntime({+ get: (key) => cache.get(key),+ set: (key, value) => cache.set(key, value),+ delete: (key) => cache.delete(key),+ });+ const lastGood = await runtime.getAppConfig({ baseOnly: true });+ const digest = getConfigDigest();+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');+ server.readonly = true;++ await expect(runtime.reloadBaseConfig()).resolves.toMatchObject({ ok: false });+ expect(await runtime.getAppConfig({ baseOnly: true })).toBe(lastGood);+ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);++ server.readonly = false;+ await expect(runtime.reloadBaseConfig()).resolves.toEqual({ ok: true });+ expect(Error.stackTraceLimit).toBe(60);+ expect(getConfigDigest()).not.toBe(digest);+ } finally {+ client.destroy();+ await server.close();+ }+ });++ it.each([+ 'validation',+ 'tools scan',+ 'AppService',+ 'tool cache',+ 'base cache',+ 'tool cache refusal',+ 'base cache refusal',+ ])('keeps the last good digest and limit when %s fails', async (stage) => {+ jest.spyOn(logger, 'error').mockImplementation(() => logger);+ const runtime = createRuntime();+ const lastGood = await runtime.getAppConfig({ baseOnly: true });+ const digest = getConfigDigest();+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');+ const failure = new Error('publication failed');+ if (stage === 'validation') {+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 0\n');+ } else if (stage === 'tools scan') {+ runtime.formatTools.mockImplementationOnce(() => {+ throw failure;+ });+ } else if (stage === 'AppService') {+ runtime.buildConfig.mockRejectedValueOnce(failure);+ } else if (stage === 'tool cache') {+ runtime.setCachedTools.mockRejectedValueOnce(failure);+ } else if (stage === 'base cache') {+ runtime.cache.set.mockRejectedValueOnce(failure);+ } else if (stage === 'tool cache refusal') {+ runtime.setCachedTools.mockResolvedValueOnce(false);+ } else {+ runtime.cache.set.mockResolvedValueOnce(false);+ }++ await expect(runtime.reloadBaseConfig()).resolves.toMatchObject({ ok: false });++ expect(await runtime.getAppConfig({ baseOnly: true })).toBe(lastGood);+ expect(getConfigDigest()).toBe(digest);+ expect(Error.stackTraceLimit).toBe(25);++ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n');+ await expect(runtime.reloadBaseConfig()).resolves.toEqual({ ok: true });+ expect(Error.stackTraceLimit).toBe(40);+ expect(getConfigDigest()).not.toBe(digest);+ });++ it('does not apply a config overtaken by a clear during publication', async () => {+ const runtime = createRuntime();+ await runtime.getAppConfig({ baseOnly: true });+ const digest = getConfigDigest();+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');+ runtime.cache.set.mockImplementationOnce(async () => {+ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);+ await runtime.clearAppConfigCache();+ });++ await runtime.reloadBaseConfig();++ expect(Error.stackTraceLimit).toBe(25);+ expect(getConfigDigest()).toBe(digest);++ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n');+ await runtime.reloadBaseConfig();+ expect(Error.stackTraceLimit).toBe(40);+ expect(getConfigDigest()).not.toBe(digest);+ });+});diff --git a/packages/api/src/app/runtime.ts b/packages/api/src/app/runtime.tsnew file mode 100644index 0000000000..d15ef1b6a1--- /dev/null+++ b/packages/api/src/app/runtime.ts@@ -0,0 +1,31 @@+import type { AppConfig } from '@librechat/data-schemas';+import type { LoadedCustomConfig } from './loader';+import { setConfigDigest } from '~/telemetry/digest';++/** The limit the process had before config first set one: 10, or `--stack-trace-limit`. */+let processStackTraceLimit: number | undefined;++const loadedDigests = new WeakMap<object, string>();++/** Stages file identity without changing the running configuration. */+export function recordLoadedConfig({ config, digest }: LoadedCustomConfig): void {+ loadedDigests.set(config, digest);+}++/**+ * Sets `Error.stackTraceLimit` for errors created from now on. With no limit it restores the+ * process default, so removing the key and reloading undoes an earlier value.+ */+export function applyStackTraceLimit(limit: number | undefined): void {+ processStackTraceLimit ??= Error.stackTraceLimit;+ Error.stackTraceLimit = limit ?? processStackTraceLimit;+}++/** Applies process-wide settings only once the YAML-derived base config is published. */+export function applyLoadedConfig({ config }: AppConfig): void {+ applyStackTraceLimit(config?.telemetry?.stackTraceLimit);+ const digest = config && loadedDigests.get(config);+ if (digest) {+ setConfigDigest(digest);+ }+}diff --git a/packages/api/src/app/service.spec.ts b/packages/api/src/app/service.spec.tsindex 0c868b767f..74a67e6a0e 100644--- a/packages/api/src/app/service.spec.ts+++ b/packages/api/src/app/service.spec.ts@@ -1,4 +1,9 @@-import { PrincipalType, getMaxSubagents, setMaxSubagents } from 'librechat-data-provider';+import {+ FileSources,+ PrincipalType,+ getMaxSubagents,+ setMaxSubagents,+} from 'librechat-data-provider';import {logger,AppService,@@ -35,9 +40,8 @@ function createMockCache(namespace = 'app_config') {const store = new Map();return {get: jest.fn((key) => Promise.resolve(store.get(`${namespace}:${key}`))),- set: jest.fn((key, value) => {+ set: jest.fn(async (key, value): Promise<boolean | void> => {store.set(`${namespace}:${key}`, value);- return Promise.resolve(undefined);}),delete: jest.fn((key) => {store.delete(`${namespace}:${key}`);@@ -1175,6 +1179,72 @@ describe('createAppConfigService', () => {expect(deps.getApplicableConfigs).toHaveBeenCalledTimes(1);});+ describe.each([false, true])('optional override cache with failClosed=%s', (failClosed) => {+ it.each(['read', 'write', 'cleanup'])(+ 'serves current restrictions when cache %s fails',+ async (stage) => {+ const warn = jest.spyOn(logger, 'warn').mockReturnValue(logger);+ const deps = createDeps({+ getApplicableConfigs: jest+ .fn()+ .mockResolvedValue([+ { isActive: true, priority: 10, overrides: { interface: { modelSelect: false } } },+ ]),+ });+ const service = createAppConfigService(deps);+ await service.getAppConfig({ baseOnly: true });+ const failure = new Error('cache failure with private content');+ if (stage === 'read') {+ deps._cache.get.mockRejectedValueOnce(failure);+ } else if (stage === 'write') {+ deps._cache.set.mockRejectedValueOnce(failure);+ } else {+ deps._cache.set.mockImplementationOnce(async () => {+ await service.clearAppConfigCache();+ });+ deps._cache.delete.mockImplementation(async (key) => {+ if (key.startsWith('_OVERRIDE_')) throw failure;+ return deps._cache._store.delete(`app_config:${key}`);+ });+ }++ const resolved = await service.getAppConfig({ role: 'USER', failClosed });++ expect(resolved.interfaceConfig?.modelSelect).toBe(false);+ expect(deps.getApplicableConfigs).toHaveBeenCalledTimes(1);+ if (stage === 'read') expect(deps._cache.set).toHaveBeenCalledTimes(1);+ expect(warn).toHaveBeenCalled();+ expect(JSON.stringify(warn.mock.calls)).not.toContain('private content');+ },+ );+ });++ it('separates shared merges when identical YAML derives different runtime settings', async () => {+ const cache = createMockCache();+ const firstDeps = createDeps({+ getCache: () => cache,+ loadBaseConfig: jest.fn().mockResolvedValue({+ config: {},+ interfaceConfig: { modelSelect: false },+ }),+ });+ const secondDeps = createDeps({+ getCache: () => cache,+ loadBaseConfig: jest.fn().mockResolvedValue({+ config: {},+ interfaceConfig: { modelSelect: true },+ }),+ });+ const first = createAppConfigService(firstDeps);+ const second = createAppConfigService(secondDeps);++ expect((await first.getAppConfig({ role: 'USER' })).interfaceConfig?.modelSelect).toBe(false);+ expect((await second.getAppConfig({ role: 'USER' })).interfaceConfig?.modelSelect).toBe(true);+ expect((await first.getAppConfig({ role: 'USER' })).interfaceConfig?.modelSelect).toBe(false);+ expect(secondDeps.getApplicableConfigs).toHaveBeenCalledTimes(1);+ expect([...cache._store.keys()].filter((key) => key.includes('_OVERRIDE_'))).toHaveLength(2);+ });+it('uses separate cache keys per userId (no cross-user contamination)', async () => {const deps = createDeps({getApplicableConfigs: jest@@ -1204,7 +1274,7 @@ describe('createAppConfigService', () => {const cachedKeys = [...deps._cache._store.keys()];const overrideKey = cachedKeys.find((k) => k.includes('_OVERRIDE_:'));expect(overrideKey).toMatch(- /^app_config:_OVERRIDE_:__default__:global:uid1:tenant-v1:[a-f0-9]{64}$/,+ /^app_config:_OVERRIDE_:__default__:global:uid1:base-v2:[a-f0-9]{64}$/,);});@@ -1373,7 +1443,7 @@ describe('createAppConfigService', () => {k.includes('_OVERRIDE_:'),);expect(overrideKey).toMatch(- /^app_config:_OVERRIDE_:tenant-a:tenant:USER:tenant-v1:[a-f0-9]{64}$/,+ /^app_config:_OVERRIDE_:tenant-a:tenant:USER:base-v2:[a-f0-9]{64}$/,);expect(overrideKey).not.toContain('__default__');});@@ -1578,7 +1648,7 @@ describe('createAppConfigService', () => {expect([...deps._cache._store.keys()]).toEqual(expect.arrayContaining([expect.stringMatching(- /^app_config:_OVERRIDE_:__default__:global:USER:uid1:tenant-v1:[a-f0-9]{64}$/,+ /^app_config:_OVERRIDE_:__default__:global:USER:uid1:base-v2:[a-f0-9]{64}$/,),]),);@@ -1871,6 +1941,80 @@ describe('createAppConfigService', () => {expect(await getAppConfig({ baseOnly: true })).toBe(initial);});+ describe.each([false, true])('restoration after a clear=%s', (clear) => {+ it.each(['tool rejection', 'tool refusal', 'base rejection', 'base refusal'])(+ 'repairs a %s on the next ordinary read',+ async (stage) => {+ jest.spyOn(logger, 'error').mockReturnValue(logger);+ const good: AppConfig = {+ config: {},+ fileStrategy: FileSources.local,+ imageOutputType: 'png',+ availableTools: {+ good: {+ type: 'function',+ function: { name: 'good', description: 'good', parameters: { type: 'object' } },+ },+ },+ };+ const rejected: AppConfig = {+ ...good,+ availableTools: {+ rejected: {+ type: 'function',+ function: {+ name: 'rejected',+ description: 'rejected',+ parameters: { type: 'object' },+ },+ },+ },+ };+ let tools = good.availableTools;+ let refuseTools = false;+ const deps = createDeps({+ loadBaseConfig: jest.fn().mockResolvedValue(good),+ setCachedTools: jest.fn(async (next: AppConfig['availableTools']) => {+ if (next === good.availableTools && refuseTools) {+ refuseTools = false;+ if (stage.endsWith('rejection')) throw new Error('tool restore unavailable');+ return false;+ }+ tools = next;+ return true;+ }),+ });+ const service = createAppConfigService(deps);+ await service.getAppConfig({ baseOnly: true });+ if (clear) await service.clearAppConfigCache();+ refuseTools = stage.startsWith('tool');+ deps.loadBaseConfig.mockResolvedValueOnce(rejected).mockRejectedValue(invalidFile());+ let writes = 0;+ deps._cache.set.mockImplementation(async (key, value) => {+ writes++;+ if (writes === 1) throw new Error('new base unavailable');+ if (writes === 2 && stage.startsWith('base')) {+ if (stage.endsWith('rejection')) throw new Error('base restore unavailable');+ return false;+ }+ deps._cache._store.set(`app_config:${key}`, value);+ });++ await expect(service.reloadBaseConfig()).resolves.toMatchObject({ ok: false });+ if (stage.startsWith('tool')) expect(tools).toBe(rejected.availableTools);+ const loads = deps.loadBaseConfig.mock.calls.length;++ expect(await service.getAppConfig({ baseOnly: true })).toBe(good);+ expect(deps.loadBaseConfig).toHaveBeenCalledTimes(loads + 1);+ expect(tools).toBe(good.availableTools);+ expect(deps._cache._store.get('app_config:_BASE_')).toBe(good);++ expect(await service.getAppConfig({ baseOnly: true })).toBe(good);+ expect(deps.loadBaseConfig).toHaveBeenCalledTimes(loads + 1);+ },+ );+ });+it('resolves with the failure instead of rejecting when there is no last good config', async () => {jest.spyOn(logger, 'error').mockReturnValue(logger);const deps = createDeps();diff --git a/packages/api/src/app/service.ts b/packages/api/src/app/service.tsindex db2cdcd4bb..62bc22343c 100644--- a/packages/api/src/app/service.ts+++ b/packages/api/src/app/service.ts@@ -30,6 +30,7 @@ import { resolveOverrideLayers, mergeOverrideLayersWithSource } from './overrideimport { createEnvReferencePolicy, parseEnvAllowlist } from './allowlist';import { resolveRuntimeConfigSecrets } from '~/admin/secrets';import { normalizeEndpointReferences } from './references';+import { getSafeErrorMetadata } from '~/utils/errors';import { getConfigLoadFailure } from './loader';import { checkAppConfigTheme } from './theme';@@ -478,8 +479,10 @@ interface CacheStore {export interface AppConfigServiceDeps {/** Load the base AppConfig from YAML + AppService processing. */loadBaseConfig: (mode?: CustomConfigLoadMode) => Promise<AppConfig | undefined>;+ /** Synchronous process-wide effects, after tools and the current base config publish. */+ onBaseConfigPublished?: (config: AppConfig) => void;/** Cache tools after base config is loaded. */- setCachedTools: (tools: Record<string, unknown>) => Promise<void>;+ setCachedTools: (tools: Record<string, unknown>) => Promise<boolean | void>;/** Get a cache store by key. */getCache: (key: string) => CacheStore;/** The CacheKeys constants from librechat-data-provider. */@@ -613,7 +616,7 @@ export function _resetOverrideStrictCache(): void {_warnedNoTenantInStrictMode = false;}-/** Versioned so older heads cannot supply override entries built under a stale tenant ID. */+/** Versioned so older heads cannot supply entries built without the local base identity. */function overrideCacheKey(role: string | undefined,userId: string | undefined,@@ -623,7 +626,7 @@ function overrideCacheKey(const tenant = tenantId ? `${tenantId}:tenant` : '__default__:global';const principal =userId && role ? `${role}:${userId}` : userId || role || BASE_CONFIG_PRINCIPAL_ID;- return `_OVERRIDE_:${tenant}:${principal}:tenant-v1:${scopeVersion}`;+ return `_OVERRIDE_:${tenant}:${principal}:base-v2:${scopeVersion}`;}/**@@ -696,6 +699,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {} {const {loadBaseConfig,+ onBaseConfigPublished,setCachedTools,getCache,cacheKeys,@@ -710,6 +714,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {const cache = getCache(cacheKeys.APP_CONFIG);const envAllowlist = parseEnvAllowlist(overrideEnvAllowlist);const envPolicies = new WeakMap<AppConfig, Map<string, EnvReferencePolicy>>();+ const scopeVersions = new WeakMap<AppConfig, string>();const warnRebinding = createRebindingWarner();const runtimeConfigs = new WeakMap<AppConfig, AppConfig>();@@ -754,6 +759,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {return policy;}let lastGoodBaseConfig: AppConfig | undefined;+ let publishedBaseGeneration = -1;let baseConfigFlight: { promise: Promise<BaseConfigLoad>; generation: number } | undefined;/**@@ -825,6 +831,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {if (generation !== baseGeneration) {return lastGood;}+ publishedBaseGeneration = -1;const restorations = [cache.set(BASE_CONFIG_KEY, lastGood)];if (lastGood.availableTools) {restorations.push(setCachedTools(lastGood.availableTools));@@ -832,13 +839,20 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {const results = await Promise.allSettled(restorations);for (const result of results) {if (result.status === 'rejected') {- logger.error('[ensureBaseConfig] Failed to restore last-good config state:', result.reason);+ logger.error(+ '[ensureBaseConfig] Failed to restore last-good config state:',+ getSafeErrorMetadata(result.reason),+ );+ } else if (result.value === false) {+ logger.error('[ensureBaseConfig] Cache refused last-good config restoration.');}}if (generation !== baseGeneration) {await cache.delete(BASE_CONFIG_KEY).catch((deleteError: unknown) => {logger.error('[ensureBaseConfig] Failed to drop an overtaken restoration:', deleteError);});+ } else if (results.every((result) => result.status === 'fulfilled' && result.value !== false)) {+ publishedBaseGeneration = generation;}return lastGood;}@@ -863,18 +877,22 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {if (generation !== baseGeneration) {return baseConfig;}- if (baseConfig.availableTools) {- await setCachedTools(baseConfig.availableTools);+ if (baseConfig.availableTools && (await setCachedTools(baseConfig.availableTools)) === false) {+ throw new Error('Tool cache refused base configuration publication.');}if (generation !== baseGeneration) {return baseConfig;}- await cache.set(BASE_CONFIG_KEY, baseConfig);+ if ((await cache.set(BASE_CONFIG_KEY, baseConfig)) === false) {+ throw new Error('Base cache refused configuration publication.');+ }if (generation !== baseGeneration) {await cache.delete(BASE_CONFIG_KEY);return baseConfig;}+ onBaseConfigPublished?.(loaded);lastGoodBaseConfig = baseConfig;+ publishedBaseGeneration = generation;return baseConfig;}@@ -900,12 +918,9 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {* subagent cap) end with the newest librechat.yaml rather than whichever read finished last.*/async function loadBase(refresh?: boolean): Promise<BaseConfigLoad> {- const cached = (await cache.get(BASE_CONFIG_KEY)) as AppConfig | undefined;- if (cached) {- lastGoodBaseConfig ??= cached;- if (!refresh) {- return { config: cached };- }+ // Another process's cached YAML cannot replace this process's runtime configuration.+ if (!refresh && lastGoodBaseConfig && publishedBaseGeneration === baseGeneration) {+ return { config: lastGoodBaseConfig };}const generation = baseGeneration;@@ -1048,22 +1063,31 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {}};- const scopeVersion = createHash('sha256')- .update(- JSON.stringify([- baseConfig.endpoints?.custom?.map(({ name, tenantId }) => [name, tenantId]),- baseConfig.config?.endpoints?.custom?.map(({ name, tenantId }) => [name, tenantId]),- ]),- )- .digest('hex');+ let scopeVersion = scopeVersions.get(baseConfig);+ if (!scopeVersion) {+ // Hash once per local base, including derived settings and override policy, never values in keys.+ scopeVersion = createHash('sha256')+ .update(JSON.stringify([baseConfig, rederiveEndpoints, overrideEnvAllowlist]))+ .digest('hex');+ scopeVersions.set(baseConfig, scopeVersion);+ }const cacheKey = overrideCacheKey(role, userId, cacheTenantId, scopeVersion);/*** The key names a role and user, not a principal set: principals resolved elsewhere are* cached only when a user id ties them to the key.*/- const cacheable = bypassOverrideCache !== true && (resolvedPrincipals == null || !!userId);+ let cacheable = bypassOverrideCache !== true && (resolvedPrincipals == null || !!userId);if (cacheable && !refresh) {- const cachedMerged = (await cache.get(cacheKey)) as AppConfig | undefined;+ let cachedMerged: AppConfig | undefined;+ try {+ cachedMerged = (await cache.get(cacheKey)) as AppConfig | undefined;+ } catch (error) {+ cacheable = false;+ logger.warn(+ '[getAppConfig] Failed to read merged config cache; resolving uncached.',+ getSafeErrorMetadata(error),+ );+ }if (cachedMerged) {return await augment(withRuntimeSecrets(cachedMerged));}@@ -1108,9 +1132,16 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {}if (cacheable && getOverrideGeneration(cacheTenantId) === generation) {- await cache.set(cacheKey, merged, overrideCacheTtl);- if (getOverrideGeneration(cacheTenantId) !== generation) {- await cache.delete(cacheKey);+ try {+ await cache.set(cacheKey, merged, overrideCacheTtl);+ if (getOverrideGeneration(cacheTenantId) !== generation) {+ await cache.delete(cacheKey);+ }+ } catch (error) {+ logger.warn(+ '[getAppConfig] Failed to cache merged config; serving the resolved configuration.',+ getSafeErrorMetadata(error),+ );}}return await augment(withRuntimeSecrets(merged));diff --git a/packages/api/src/telemetry/config.spec.ts b/packages/api/src/telemetry/config.spec.tsindex 79753466df..ded1988c34 100644--- a/packages/api/src/telemetry/config.spec.ts+++ b/packages/api/src/telemetry/config.spec.ts@@ -65,6 +65,22 @@ describe('getTelemetryConfig', () => {expect(config.serviceVersion).toBe('0.8.5');});+ it('reports the baked BUILD_COMMIT ahead of the package version', () => {+ const commit = 'f177bed1757c1b7a4e2bd2a7d1c7a3d0e9a6b5c4';++ expect(+ getTelemetryConfig({ BUILD_COMMIT: ` ${commit} `, npm_package_version: '0.8.5' })+ .serviceVersion,+ ).toBe(commit);+ expect(+ getTelemetryConfig({ BUILD_COMMIT: commit, OTEL_SERVICE_VERSION: '1.2.3' }).serviceVersion,+ ).toBe('1.2.3');+ expect(+ getTelemetryConfig({ BUILD_COMMIT: ' ', npm_package_version: '0.8.5' }).serviceVersion,+ ).toBe('0.8.5');+ expect(getTelemetryConfig({}).serviceVersion).toBeUndefined();+ });+it('enables ioredis instrumentation only when explicitly configured', () => {const config = getTelemetryConfig({OTEL_IOREDIS_TRACING_ENABLED: 'true',diff --git a/packages/api/src/telemetry/config.ts b/packages/api/src/telemetry/config.tsindex cb87846199..0ae2567cac 100644--- a/packages/api/src/telemetry/config.ts+++ b/packages/api/src/telemetry/config.ts@@ -36,8 +36,11 @@ export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): Telemeconst tracingEnabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled;const logsEnabled = isTruthy(env.OTEL_LOGS_ENABLED) && !sdkDisabled;const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;+ /** Images bake BUILD_COMMIT, which names the exact code a pod runs; the package version does not. */const serviceVersion =- normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);+ normalizeEnvValue(env.OTEL_SERVICE_VERSION) ??+ normalizeEnvValue(env.BUILD_COMMIT) ??+ normalizeEnvValue(env.npm_package_version);const ioredisTracingEnabled = isTruthy(env.OTEL_IOREDIS_TRACING_ENABLED);const logsLevel = normalizeEnvValue(env.OTEL_LOGS_LEVEL)?.toLowerCase() ?? DEFAULT_LOGS_LEVEL;diff --git a/packages/api/src/telemetry/digest.ts b/packages/api/src/telemetry/digest.tsnew file mode 100644index 0000000000..bb17352241--- /dev/null+++ b/packages/api/src/telemetry/digest.ts@@ -0,0 +1,22 @@+/**+ * Names the librechat.yaml a process runs with on its log records and request spans. It is an+ * attribute rather than a resource attribute because the resource is fixed when the SDK starts,+ * before the config loads, and an admin reload can change the config without a restart.+ */+export const CONFIG_DIGEST_ATTRIBUTE = 'librechat.config.digest';++let configDigest: string | undefined;++/** Records the digest of the config that just loaded; the last successful load wins. */+export function setConfigDigest(digest: string): void {+ configDigest = digest;+}++export function getConfigDigest(): string | undefined {+ return configDigest;+}++/** Test hook — not exported from the package barrel. */+export function resetConfigDigestForTests(): void {+ configDigest = undefined;+}diff --git a/packages/api/src/telemetry/grpc.spec.ts b/packages/api/src/telemetry/grpc.spec.tsindex 27b283faae..b6529e7c38 100644--- a/packages/api/src/telemetry/grpc.spec.ts+++ b/packages/api/src/telemetry/grpc.spec.ts@@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas';import { Server, ServerCredentials } from '@grpc/grpc-js';import type { sendUnaryData, ServerUnaryCall, ServiceDefinition } from '@grpc/grpc-js';import { initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } from './sdk';+import { resetConfigDigestForTests, setConfigDigest } from './digest';const LOGS_EXPORT_PATH = '/opentelemetry.proto.collector.logs.v1.LogsService/Export';const TRACES_EXPORT_PATH = '/opentelemetry.proto.collector.trace.v1.TraceService/Export';@@ -11,6 +12,8 @@ const SPAN_NAME = 'grpc-spec-span';const INFO_MESSAGE = 'grpc-spec-info-message';const DEBUG_MESSAGE = 'grpc-spec-debug-message';const SECRET_SUFFIX = 'GRPCSPECSECRET1234567890';+const BUILD_COMMIT = 'c0ffee0123456789abcdef0123456789abcdef01';+const CONFIG_DIGEST = 'feedc0de4321';const EXPORTER_ENV_KEYS = ['OTEL_EXPORTER_OTLP_ENDPOINT','OTEL_EXPORTER_OTLP_PROTOCOL',@@ -89,7 +92,9 @@ describe('OTLP/gRPC export', () => {process.env.OTEL_METRICS_EXPORTER = 'none';process.env.OTEL_NODE_RESOURCE_DETECTORS = 'none';+ setConfigDigest(CONFIG_DIGEST);const controller = initializeTelemetry({+ BUILD_COMMIT,OTEL_LOGS_ENABLED: 'true',OTEL_SERVICE_NAME: SERVICE_NAME,OTEL_TRACING_ENABLED: 'true',@@ -108,6 +113,7 @@ describe('OTLP/gRPC export', () => {afterAll(async () => {await resetTelemetryForTests();+ resetConfigDigestForTests();for (const key of EXPORTER_ENV_KEYS) {const value = previousEnv[key];if (value === undefined) {@@ -134,6 +140,16 @@ describe('OTLP/gRPC export', () => {expect(payload.includes(DEBUG_MESSAGE)).toBe(false);});+ it('names the build in the resource and the loaded config on each record', () => {+ const logs = Buffer.concat(receivedLogs);+ const traces = Buffer.concat(receivedTraces);++ expect(logs.includes(BUILD_COMMIT)).toBe(true);+ expect(traces.includes(BUILD_COMMIT)).toBe(true);+ expect(logs.includes('librechat.config.digest')).toBe(true);+ expect(logs.includes(CONFIG_DIGEST)).toBe(true);+ });+it('exports spans over gRPC', () => {const payload = Buffer.concat(receivedTraces);diff --git a/packages/api/src/telemetry/logs.ts b/packages/api/src/telemetry/logs.tsindex a7b75839c0..1b12aaafc1 100644--- a/packages/api/src/telemetry/logs.ts+++ b/packages/api/src/telemetry/logs.ts@@ -2,11 +2,24 @@ import winston from 'winston';import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';import { logger, baseLogFormat, jsonTruncateFormat } from '@librechat/data-schemas';import type { TelemetryConfig } from './config';+import { CONFIG_DIGEST_ATTRIBUTE, getConfigDigest } from './digest';import { emitTelemetryWarning } from './warnings';import { DEFAULT_LOGS_LEVEL } from './config';let activeTransport: OpenTelemetryTransportV3 | undefined;+/**+ * Names the loaded librechat.yaml on exported records. Winston hands each transport its own copy+ * of the record, so file and console output keep their shape.+ */+const configDigestFormat = winston.format((info) => {+ const digest = getConfigDigest();+ if (digest) {+ info[CONFIG_DIGEST_ATTRIBUTE] = digest;+ }+ return info;+});+/*** An unknown level name would resolve to `undefined` inside winston and silently* drop every record, so fall back and say so instead.@@ -37,7 +50,7 @@ export function attachLogsTransport(config: TelemetryConfig): void {const transport = new OpenTelemetryTransportV3({level: resolveLogsLevel(config.logsLevel),- format: winston.format.combine(baseLogFormat, jsonTruncateFormat()),+ format: winston.format.combine(baseLogFormat, jsonTruncateFormat(), configDigestFormat()),});logger.add(transport);activeTransport = transport;diff --git a/packages/api/src/telemetry/middleware.spec.ts b/packages/api/src/telemetry/middleware.spec.tsindex 8be4bea321..d80d53d545 100644--- a/packages/api/src/telemetry/middleware.spec.ts+++ b/packages/api/src/telemetry/middleware.spec.ts@@ -5,6 +5,7 @@ import type { NextFunction, Response } from 'express';import type { Span } from '@opentelemetry/api';import type { ServerRequest } from '~/types';import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';+import { resetConfigDigestForTests, setConfigDigest } from './digest';import { observeRedisOperation } from '~/cache/redisTelemetry';import { getTelemetryRequestSpan } from './sdk';@@ -118,6 +119,23 @@ describe('telemetryMiddleware', () => {);});+ it('names the loaded config on the request span once a digest is recorded', () => {+ const span = createSpan();+ mockGetTelemetryRequestSpan.mockReturnValue(span);++ setConfigDigest('0123456789ab');+ try {+ telemetryMiddleware(createRequest(), createResponse() as Response, jest.fn());+ } finally {+ resetConfigDigestForTests();+ }++ expect(span.setAttributes).toHaveBeenCalledWith({+ 'http.request.method': 'POST',+ 'librechat.config.digest': '0123456789ab',+ });+ });+it('adds aggregated Redis activity to the stored request span', async () => {const requestSpan = createSpan();const res = createResponse(200);diff --git a/packages/api/src/telemetry/middleware.ts b/packages/api/src/telemetry/middleware.tsindex cd912f2806..bbbd538868 100644--- a/packages/api/src/telemetry/middleware.ts+++ b/packages/api/src/telemetry/middleware.ts@@ -8,6 +8,7 @@ import {runWithRedisRequestTelemetry,} from '~/cache/redisTelemetry';import { getErrorType, getSafeSpanException } from './safeException';+import { CONFIG_DIGEST_ATTRIBUTE, getConfigDigest } from './digest';import { getTelemetryRequestSpan } from './sdk';import { DEFAULT_HEALTH_PATH } from './config';@@ -115,9 +116,12 @@ export function telemetryMiddleware(req: ServerRequest, res: Response, next: Nexreturn;}- span.setAttributes({- 'http.request.method': req.method,- });+ const attributes: Attributes = { 'http.request.method': req.method };+ const configDigest = getConfigDigest();+ if (configDigest) {+ attributes[CONFIG_DIGEST_ATTRIBUTE] = configDigest;+ }+ span.setAttributes(attributes);const redisTelemetry = createRedisRequestTelemetry(span);let completed = false;diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.tsindex 677fbc7b9f..be96bd60ed 100644--- a/packages/data-provider/src/config.spec.ts+++ b/packages/data-provider/src/config.spec.ts@@ -57,6 +57,17 @@ describe('independent browser replay opt-in', () => {expect(telemetrySchema.safeParse(telemetry).success).toBe(false);});});++describe('telemetry.stackTraceLimit', () => {+ it('stays absent unless configured, so the process default applies', () => {+ expect(telemetrySchema.parse({})).toEqual({});+ expect(telemetrySchema.parse({ stackTraceLimit: 50 })).toEqual({ stackTraceLimit: 50 });+ });++ it.each([0, -1, 2.5, 1001, '50'])('rejects %p', (stackTraceLimit) => {+ expect(telemetrySchema.safeParse({ stackTraceLimit }).success).toBe(false);+ });+});import { EModelEndpoint, isDocumentSupportedProvider } from './schemas';import { getEndpointFileConfig, mergeFileConfig } from './file-config';import { DEFAULT_MCP_APP_CSP_LIMITS } from './mcp/csp';@@ -3281,6 +3292,10 @@ describe('DEPLOYMENT_ONLY_CONFIG_PATHS', () => {});});+ it('keeps the process-wide stack limit deployment-only', () => {+ expect(DEPLOYMENT_ONLY_CONFIG_PATHS['telemetry.stackTraceLimit']).toBe('operator');+ });+it('overlaps neither the base-only sections nor the base-only paths', () => {const baseOnly = new Set<string>([...BASE_ONLY_CONFIG_SECTIONS, ...BASE_ONLY_CONFIG_PATHS]);for (const path of Object.keys(DEPLOYMENT_ONLY_CONFIG_PATHS)) {diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex fa55aca890..df875c20fc 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -168,6 +168,7 @@ export type DeploymentOnlyConfigReason = 'startup' | 'operator' | 'unused';* a stored override had no effect rather than a dangerous one.*/export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly<Record<string, DeploymentOnlyConfigReason>> = {+ 'telemetry.stackTraceLimit': 'operator',rateLimits: 'operator',cloudfront: 'operator',permissions: 'operator',@@ -3521,6 +3522,12 @@ export const RUM_COLLECTOR_ACK_HEADER = 'x-librechat-rum-accepted';export const telemetrySchema = z.object({+ /**+ * Frames V8 records per error stack (`Error.stackTraceLimit`), applied when librechat.yaml+ * loads. Unset keeps the process default: 10, or `--stack-trace-limit` from NODE_OPTIONS.+ * Deeper stacks cost more on every thrown error.+ */+ stackTraceLimit: z.number().int().min(1).max(1000).optional(),rum: z.object({enabled: z.boolean().optional(),diff --git a/packages/data-schemas/src/config/parsers.spec.ts b/packages/data-schemas/src/config/parsers.spec.tsindex 66c8dbd438..a43b0b229b 100644--- a/packages/data-schemas/src/config/parsers.spec.ts+++ b/packages/data-schemas/src/config/parsers.spec.ts@@ -80,6 +80,29 @@ describe('redactMessage', () => {expect(redactMessage('mention @handle and v1.2.3')).toBe('mention @handle and v1.2.3');});+ it('keeps scoped package paths in stack frames and still masks addresses beside them', () => {+ const frames = [+ ' at Run.processStream (/app/node_modules/@librechat/agents/dist/cjs/run.cjs:120:5)',+ ' at async KeyvMongo.get (/app/node_modules/@keyv/mongo/dist/index.cjs:10:3)',+ ' at /app/node_modules/@langchain/core/dist/runnables/base.cjs:1:2',+ ' at file:///app/node_modules/@scope/pkg/index.mjs:1:1',+ ' at load (C:\\app\\node_modules\\@scope\\pkg\\index.js:1:1)',+ '{"stack":"at f (C:\\\\app\\\\node_modules\\\\@scope\\\\pkg\\\\x.js:1:2)"}',+ ];+ for (const frame of frames) {+ expect(redactMessage(frame)).toBe(frame);+ }++ expect(+ redactMessage('Error: no user [email protected]\n at f (/app/node_modules/@a/b/c.js:1:2)'),+ ).toBe('Error: no user [REDACTED_EMAIL]\n at f (/app/node_modules/@a/b/c.js:1:2)');+ expect(redactMessage('user jane/@example.com ok')).toBe('user [REDACTED_EMAIL] ok');+ expect(redactMessage('/home/[email protected]/file.js')).toBe('[REDACTED_EMAIL]');+ expect(redactMessage('see https://social.example/@jane.doe ok')).toBe(+ 'see [REDACTED_EMAIL] ok',+ );+ });+it('masks the whole token for apostrophes, long local parts and combining-mark domains', () => {expect(redactMessage("mary.jane'[email protected]")).toBe('[REDACTED_EMAIL]');expect(redactMessage(`${'a'.repeat(100)}@example.com ok`)).toBe('[REDACTED_EMAIL] ok');@@ -814,6 +837,20 @@ describe('redactFormat', () => {expect(info.nested).toEqual({ '[REDACTED_EMAIL]': 1, '[REDACTED_EMAIL]#2': 2, other: 3 });});+ it('keeps scoped package frame locations in a redacted Error stack', () => {+ const error = new Error('lookup failed for [email protected]');+ error.stack = [+ 'Error: lookup failed for [email protected]',+ ' at KeyvMongo.get (/app/node_modules/@keyv/mongo/dist/index.cjs:10:3)',+ ' at retryWithBackoff (/app/packages/api/dist/index.cjs:5:7)',+ ].join('\n');+ const info = runRedactFormat({ level: 'error', message: 'cache read', error });+ const stack = String((info.error as Error).stack);+ expect(stack).not.toContain('jane');+ expect(stack).toContain('(/app/node_modules/@keyv/mongo/dist/index.cjs:10:3)');+ expect(stack).toContain('(/app/packages/api/dist/index.cjs:5:7)');+ });+it.each(['error', 'warn', 'info', 'debug'])('redacts info.message for %s level', (level) => {const info = runRedactFormat({ level, message: 'Bearer secretvalue' });expect(info.message).toBe('Bearer [REDACTED]');diff --git a/packages/data-schemas/src/config/parsers.ts b/packages/data-schemas/src/config/parsers.tsindex 79c0db762f..e65ba82385 100644--- a/packages/data-schemas/src/config/parsers.ts+++ b/packages/data-schemas/src/config/parsers.ts@@ -61,9 +61,11 @@ const sensitiveKeys: RegExp[] = [/*** Matches a whole delimiter-bounded token that holds an address. The lookbehind lets the scan start* only at a token boundary, so long non-email text is scanned once instead of once per character.+ * An `@` between path separators (`node_modules/@scope/pkg/...`) names a scoped package in a file+ * path or stack frame, not an address, so frame locations survive.*/const EMAIL_PATTERN =- /(?:(?<!\\)\\"(?:[^"\\\r\n]|\\{3}"|\\{4}|\\{2}(?!\\)|\\[bfnrt/]|\\u[0-9a-fA-F]{4})*\\"|(?<!\\)"(?:[^"\\\r\n]|\\.)*"|(?<![^\s<>[\],;"])[^\s<>[\],;"]+)@(?:\[[^\]\s@]+\]|[^\s<>[\],;"@]+\.[^\s<>[\],;"@]+)/gu;+ /(?:(?<!\\)\\"(?:[^"\\\r\n]|\\{3}"|\\{4}|\\{2}(?!\\)|\\[bfnrt/]|\\u[0-9a-fA-F]{4})*\\"|(?<!\\)"(?:[^"\\\r\n]|\\.)*"|(?<![^\s<>[\],;"])[^\s<>[\],;"]+)(?!(?<=[/\\])@[^\s<>[\],;"@/\\]+[/\\])@(?:\[[^\]\s@]+\]|[^\s<>[\],;"@]+\.[^\s<>[\],;"@]+)/gu;/** Set while redacting splat arguments at levels whose later format strips heavy error fields. */let skipHeavyErrorFields = false;/** URL-encoded addresses (`user%40host.tld`), anchored at a token start so scans stay linear. */