Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🪪 feat: Name the Build and Config in Telemetry, Configure Stack Depth, Keep Scoped Frames (#17515)

Danny Avila696a1b6b56f9
diff --git a/.env.example b/.env.example
index 6def91d11b..d7c0a8a77c 100644
--- a/.env.example
+++ b/.env.example
@@ -381,6 +381,7 @@ NODE_MAX_OLD_SPACE_SIZE=6144
# OTEL_LOGS_ENABLED=false
# OTEL_LOGS_LEVEL=info
# OTEL_SERVICE_NAME=librechat
+# Defaults to the image's BUILD_COMMIT, then the npm package version.
# OTEL_SERVICE_VERSION=
# Exporter protocol for every signal: http/protobuf (default, port 4318) or grpc (port 4317).
# Per-signal overrides: OTEL_EXPORTER_OTLP_TRACES_PROTOCOL, OTEL_EXPORTER_OTLP_LOGS_PROTOCOL.
diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.js
index 8cc6dc5260..c9e5cdd6de 100644
--- a/api/cache/getLogStores.js
+++ b/api/cache/getLogStores.js
@@ -53,7 +53,9 @@ const namespaces = {
[CacheKeys.USER_PRINCIPALS]: userPrincipalsCache() ?? disabledCache,
/** Authorization IDs stay uncached because a failed shared invalidation cannot fail closed. */
[CacheKeys.PROMPT_GROUPS_ACCESS]: disabledCache,
- [CacheKeys.APP_CONFIG]: standardCache(CacheKeys.APP_CONFIG),
+ [CacheKeys.APP_CONFIG]: standardCache(CacheKeys.APP_CONFIG, undefined, undefined, {
+ throwOnErrors: true,
+ }),
[CacheKeys.CONFIG_STORE]: standardCache(CacheKeys.CONFIG_STORE),
[CacheKeys.TOOL_CACHE]: standardCache(CacheKeys.TOOL_CACHE),
[CacheKeys.PENDING_REQ]: standardCache(CacheKeys.PENDING_REQ),
diff --git a/api/cache/getLogStores.spec.js b/api/cache/getLogStores.spec.js
index a6bd4f63a8..29ed2931b2 100644
--- a/api/cache/getLogStores.spec.js
+++ b/api/cache/getLogStores.spec.js
@@ -1,8 +1,24 @@
+jest.mock('@librechat/api', () => {
+ const api = jest.requireActual('@librechat/api');
+ return { ...api, standardCache: jest.fn(api.standardCache) };
+});
+
+const { standardCache } = require('@librechat/api');
const { logger } = require('@librechat/data-schemas');
const { CacheKeys } = require('librechat-data-provider');
const getLogStores = require('./getLogStores');
+const appConfigCacheArgs = standardCache.mock.calls.find(([key]) => key === CacheKeys.APP_CONFIG);
describe('auditCache', () => {
+ it('requires APP_CONFIG publication errors to propagate', () => {
+ expect(appConfigCacheArgs).toEqual([
+ CacheKeys.APP_CONFIG,
+ undefined,
+ undefined,
+ { throwOnErrors: true },
+ ]);
+ });
+
it('logs counts and TTL info without cache keys or values', async () => {
const secretKey = 'user-id-1234567890';
const secretValue = 'super-secret-access-token';
diff --git a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js
index bdcc1dda7b..3e24e018ac 100644
--- a/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js
+++ b/api/server/services/Config/__tests__/invalidateConfigCaches.reload.spec.js
@@ -9,6 +9,7 @@ const os = require('os');
const path = require('path');
const yaml = require('js-yaml');
const { Constants } = require('librechat-data-provider');
+const { loadAndFormatTools } = require('~/server/services/start/tools');
jest.mock('~/server/services/start/tools', () => ({
loadAndFormatTools: jest.fn(() => ({})),
@@ -122,3 +123,30 @@ describe('admin config writes and librechat.yaml', () => {
expect((await getAppConfig({ baseOnly: true })).filters).toEqual(lastGood.filters);
});
});
+
+describe('published telemetry settings', () => {
+ const processDefault = Error.stackTraceLimit;
+
+ afterEach(async () => {
+ writeYaml({});
+ await reloadConfigCaches();
+ });
+
+ it('keeps the last good stack limit if tool loading rejects a reload', async () => {
+ writeYaml({ telemetry: { stackTraceLimit: 25 } });
+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: true } });
+ expect(Error.stackTraceLimit).toBe(25);
+
+ writeYaml({ telemetry: { stackTraceLimit: 60 } });
+ loadAndFormatTools.mockImplementationOnce(() => {
+ throw new Error('tool scan failed');
+ });
+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: false } });
+ expect(Error.stackTraceLimit).toBe(25);
+ expect((await getAppConfig({ baseOnly: true })).config.telemetry.stackTraceLimit).toBe(25);
+
+ writeYaml({});
+ await expect(reloadConfigCaches()).resolves.toMatchObject({ loaded: { ok: true } });
+ expect(Error.stackTraceLimit).toBe(processDefault);
+ });
+});
diff --git a/api/server/services/Config/app.js b/api/server/services/Config/app.js
index cd6a67aa36..665d4f1d95 100644
--- a/api/server/services/Config/app.js
+++ b/api/server/services/Config/app.js
@@ -5,6 +5,7 @@ const {
isEnabled,
expandCustomEndpoint,
createAppConfigService,
+ applyLoadedConfig,
clearMcpConfigCache,
createCodeEnvironmentRegistry,
createConfigCacheInvalidation,
@@ -52,6 +53,7 @@ const loadBaseConfig = async (mode) => {
const appConfigService = createAppConfigService({
loadBaseConfig,
+ onBaseConfigPublished: applyLoadedConfig,
setCachedTools,
getCache: getLogStores,
cacheKeys: CacheKeys,
diff --git a/api/server/services/Config/loadCustomConfig.js b/api/server/services/Config/loadCustomConfig.js
index c253c4f14b..555f3d0873 100644
--- a/api/server/services/Config/loadCustomConfig.js
+++ b/api/server/services/Config/loadCustomConfig.js
@@ -1,5 +1,10 @@
const path = require('path');
-const { loadYaml, redactConfigSecretMaps, createCustomConfigLoader } = require('@librechat/api');
+const {
+ loadYaml,
+ recordLoadedConfig,
+ redactConfigSecretMaps,
+ createCustomConfigLoader,
+} = require('@librechat/api');
const projectRoot = path.resolve(__dirname, '..', '..', '..', '..');
const defaultConfigPath = path.resolve(projectRoot, 'librechat.yaml');
@@ -8,4 +13,5 @@ module.exports = createCustomConfigLoader({
loadLocal: loadYaml,
defaultConfigPath,
redactConfig: redactConfigSecretMaps,
+ onLoad: recordLoadedConfig,
});
diff --git a/librechat.example.yaml b/librechat.example.yaml
index d77eea84f9..832dedec57 100644
--- a/librechat.example.yaml
+++ b/librechat.example.yaml
@@ -10,7 +10,10 @@ cache: true
# Browser telemetry and session recording are independent opt-ins. Collector URL,
# authentication and service settings remain in .env. YAML flags override .env;
# omitting them preserves the corresponding environment settings.
+# stackTraceLimit (1 to 1000) sets how many frames each error stack records; unset keeps
+# Node's default of 10, or --stack-trace-limit from NODE_OPTIONS.
# telemetry:
+# stackTraceLimit: 50
# rum:
# enabled: true
# sessionReplay:
diff --git a/packages/api/src/admin/config.report.spec.ts b/packages/api/src/admin/config.report.spec.ts
index 2df759c5a8..5b49302115 100644
--- a/packages/api/src/admin/config.report.spec.ts
+++ b/packages/api/src/admin/config.report.spec.ts
@@ -323,6 +323,43 @@ describe('write report', () => {
await methods.deleteConfig('role' as never, '__base__');
});
+ it.each([
+ ['role', '__base__'],
+ ['role', 'telemetry-role'],
+ ['group', 'telemetry-group'],
+ ['user', 'telemetry-user'],
+ ])(
+ 'skips process-wide stack limits in %s %s overrides and tombstones',
+ async (principalType, principalId) => {
+ const params = { principalType, principalId };
+ const handlers = createHandlers();
+ const skipped = ['telemetry.stackTraceLimit:deployment_only_path'];
+ const put = await call(handlers.upsertConfigOverrides, {
+ params,
+ body: { overrides: { telemetry: { stackTraceLimit: 60 }, memory: { disabled: true } } },
+ });
+ expect(put.statusCode).toBe(201);
+ expect(describeAll(put.body?.skipped)).toEqual(skipped);
+ expect((await stored(params))?.overrides).toEqual({ memory: { disabled: true } });
+
+ const patch = await call(handlers.patchConfigField, {
+ params,
+ body: { entries: [{ fieldPath: 'telemetry.stackTraceLimit', value: 60 }] },
+ });
+ expect(patch.statusCode).toBe(200);
+ expect(describeAll(patch.body?.skipped)).toEqual(skipped);
+
+ const tombstone = await call(handlers.tombstoneConfigField, {
+ params,
+ body: { fieldPath: 'telemetry.stackTraceLimit' },
+ });
+ expect(tombstone.statusCode).toBe(200);
+ expect(describeAll(tombstone.body?.skipped)).toEqual(skipped);
+ expect((await stored(params))?.tombstones).toEqual([]);
+ await methods.deleteConfig(principalType as never, principalId);
+ },
+ );
+
it('a tombstone of a field only librechat.yaml sets is skipped; deleting a stored one is cleanup', async () => {
const params = nextParams();
await seed(params, { rateLimits: { fileUploads: { ipMax: 1 } }, memory: { disabled: true } });
diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.ts
index dc5ace62c5..5db9cab775 100644
--- a/packages/api/src/app/index.ts
+++ b/packages/api/src/app/index.ts
@@ -17,6 +17,7 @@ export * from './server';
export * from './readiness';
export * from './origin';
export * from './agents';
+export { applyLoadedConfig, recordLoadedConfig } from './runtime';
export { resolveBuildInfo } from './build';
export type { BuildInfo } from './build';
export { buildPreLoginInterface, withSafePolicyLinks } from './interface';
diff --git a/packages/api/src/app/loader.spec.ts b/packages/api/src/app/loader.spec.ts
index b6dff512dc..a0530423b8 100644
--- a/packages/api/src/app/loader.spec.ts
+++ b/packages/api/src/app/loader.spec.ts
@@ -1,7 +1,7 @@
import path from 'path';
import { logger } from '@librechat/data-schemas';
import type { TCustomConfig } from 'librechat-data-provider';
-import { createCustomConfigLoader, ConfigReloadError } from './loader';
+import { createCustomConfigLoader, ConfigReloadError, hashCustomConfig } from './loader';
import { loadYaml } from '~/utils/yaml';
const fixture = (name: string) => path.join(__dirname, '__fixtures__', 'theme', `${name}.yaml`);
@@ -278,3 +278,105 @@ describe('createCustomConfigLoader remote fetch failures', () => {
await expect(loaderFor(refused)()).resolves.toBeNull();
});
});
+
+describe('createCustomConfigLoader onLoad', () => {
+ const originalConfigPath = process.env.CONFIG_PATH;
+
+ beforeEach(() => {
+ jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ jest.spyOn(logger, 'info').mockImplementation(() => logger);
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ if (originalConfigPath === undefined) {
+ delete process.env.CONFIG_PATH;
+ } else {
+ process.env.CONFIG_PATH = originalConfigPath;
+ }
+ });
+
+ /** The loader parses a string source as YAML, so comments and layout reach the digest input. */
+ const createTextLoader = (text: string, onLoad: jest.Mock) => {
+ process.env.CONFIG_PATH = '/virtual/librechat.yaml';
+ return createCustomConfigLoader({
+ loadLocal: () => text,
+ defaultConfigPath: '/virtual/librechat.yaml',
+ redactConfig: (value: TCustomConfig) => value,
+ onLoad,
+ });
+ };
+
+ const loadText = async (text: string) => {
+ const onLoad = jest.fn();
+ const config = await createTextLoader(text, onLoad)(false);
+ return { config, onLoad };
+ };
+
+ const digestOf = async (text: string): Promise<string> => {
+ const { onLoad } = await loadText(text);
+ expect(onLoad).toHaveBeenCalledTimes(1);
+ return onLoad.mock.calls[0][0].digest;
+ };
+
+ it('hands the loaded config and a short digest to onLoad', async () => {
+ const { config, onLoad } = await loadText(
+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n',
+ );
+
+ expect(config?.telemetry?.stackTraceLimit).toBe(40);
+ expect(onLoad).toHaveBeenCalledWith({
+ config,
+ digest: expect.stringMatching(/^[0-9a-f]{12}$/),
+ });
+ });
+
+ it('keeps the digest across comments, layout and key order, and moves it on a value change', async () => {
+ const base = await digestOf('version: 1.3.18\ncache: true\ninterface:\n modelSelect: true\n');
+ const reordered = await digestOf(
+ '# comment\ninterface: { modelSelect: true }\n\ncache: true\nversion: 1.3.18\n',
+ );
+ const changed = await digestOf(
+ 'version: 1.3.18\ncache: false\ninterface:\n modelSelect: true\n',
+ );
+
+ expect(reordered).toBe(base);
+ expect(changed).not.toBe(base);
+ });
+
+ it('hashes the file as written, not the modelSpecs the schema fills in', async () => {
+ const text =
+ 'version: 1.3.18\nmodelSpecs:\n list:\n - name: spec\n label: Spec\n' +
+ ' preset:\n endpoint: openAI\n model: gpt-4o\n';
+ const { config, onLoad } = await loadText(text);
+
+ expect(config?.modelSpecs).toBeDefined();
+ expect(onLoad.mock.calls[0][0].digest).toBe(
+ hashCustomConfig({
+ version: '1.3.18',
+ modelSpecs: {
+ list: [{ name: 'spec', label: 'Spec', preset: { endpoint: 'openAI', model: 'gpt-4o' } }],
+ },
+ }),
+ );
+ });
+
+ it('does not call onLoad when a reload is rejected', async () => {
+ const onLoad = jest.fn();
+ const load = createTextLoader('version: 1.3.18\nnotAKey: true\n', onLoad);
+
+ await expect(load(false, { mode: 'reload' })).rejects.toBeInstanceOf(ConfigReloadError);
+ expect(onLoad).not.toHaveBeenCalled();
+ });
+});
+
+describe('hashCustomConfig', () => {
+ it('ignores key order and keeps array order', () => {
+ expect(hashCustomConfig({ a: 1, b: { c: 2, d: [1, 2] } })).toBe(
+ hashCustomConfig({ b: { d: [1, 2], c: 2 }, a: 1 }),
+ );
+ expect(hashCustomConfig({ b: { d: [2, 1], c: 2 }, a: 1 })).not.toBe(
+ hashCustomConfig({ a: 1, b: { c: 2, d: [1, 2] } }),
+ );
+ });
+});
diff --git a/packages/api/src/app/loader.ts b/packages/api/src/app/loader.ts
index 58b6ca0779..63b2c888ee 100644
--- a/packages/api/src/app/loader.ts
+++ b/packages/api/src/app/loader.ts
@@ -1,5 +1,6 @@
import axios from 'axios';
import yaml from 'js-yaml';
+import { createHash } from 'node:crypto';
import { logger } from '@librechat/data-schemas';
import {
configSchema,
@@ -22,11 +23,40 @@ export interface CustomConfigLoadOptions {
mode?: CustomConfigLoadMode;
}
+export interface LoadedCustomConfig {
+ config: TCustomConfig;
+ /** {@link hashCustomConfig} of the file as parsed, before any check rewrote part of it. */
+ digest: string;
+}
+
export interface CustomConfigLoaderOptions {
defaultConfigPath: string;
loadLocal: (configPath: string) => unknown;
fetchRemote?: (configPath: string) => Promise<unknown>;
redactConfig: (config: TCustomConfig) => TCustomConfig;
+ /** Runs after each successful load, at startup and on reload, before the config is returned. */
+ onLoad?: (loaded: LoadedCustomConfig) => void;
+}
+
+const CONFIG_DIGEST_LENGTH = 12;
+
+/** Keys are unique within an object, and code-unit order does not vary by locale. */
+function sortKeys(_key: string, value: unknown): unknown {
+ if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+ return value;
+ }
+ return Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : 1)));
+}
+
+/**
+ * A short, stable hash of a parsed librechat.yaml: comments, formatting and key order do not
+ * change it, so it moves only when the configuration does. Values are hashed, never logged.
+ */
+export function hashCustomConfig(config: unknown): string {
+ return createHash('sha256')
+ .update(JSON.stringify(config, sortKeys))
+ .digest('hex')
+ .slice(0, CONFIG_DIGEST_LENGTH);
}
export class ConfigReloadError extends Error {
@@ -139,6 +169,7 @@ export function createCustomConfigLoader({
defaultConfigPath,
loadLocal,
redactConfig,
+ onLoad,
fetchRemote = async (configPath: string): Promise<unknown> =>
(await axios.get(configPath, serviceRequestBudget())).data,
}: CustomConfigLoaderOptions): (
@@ -229,6 +260,8 @@ export function createCustomConfigLoader({
}
}
+ /** The checks below return copies, but `modelSpecs` is assigned onto this object later. */
+ const parsedConfig = loadedConfig;
const themeCheck = checkConfigTheme(loadedConfig);
loadedConfig = themeCheck.config;
reportThemeCheck(configPath, themeCheck.errors, themeCheck.warnings);
@@ -288,6 +321,7 @@ export function createCustomConfigLoader({
process.exit(1);
}
+ const digest = hashCustomConfig(parsedConfig);
const customConfig = loadedConfig as TCustomConfig;
/** Expanded copies are discarded: AppService serves them, and the config stays as written. */
for (const endpoint of (customConfig.endpoints?.custom ?? []) as TEndpoint[]) {
@@ -304,6 +338,7 @@ export function createCustomConfigLoader({
logger.debug('Custom config:', loggableConfig);
}
+ onLoad?.({ config: customConfig, digest });
loadedSuccessfully = true;
return customConfig;
} catch (error) {
diff --git a/packages/api/src/app/runtime.spec.ts b/packages/api/src/app/runtime.spec.ts
new file mode 100644
index 0000000000..0789c8b336
--- /dev/null
+++ b/packages/api/src/app/runtime.spec.ts
@@ -0,0 +1,380 @@
+import { Keyv } from 'keyv';
+import KeyvRedis from '@keyv/redis';
+import { createClient } from '@redis/client';
+import { PrincipalType } from 'librechat-data-provider';
+import { logger, AppService } from '@librechat/data-schemas';
+import type { TCustomConfig } from 'librechat-data-provider';
+import type { AppConfigServiceDeps } from './service';
+import { applyLoadedConfig, recordLoadedConfig, applyStackTraceLimit } from './runtime';
+import { getConfigDigest, resetConfigDigestForTests } from '~/telemetry/digest';
+import { startRespServer } from '~/cache/__tests__/resp.helper';
+import { createCustomConfigLoader } from './loader';
+import { createAppConfigService } from './service';
+
+function frameCount(depth: number): number {
+ const recurse = (remaining: number): Error =>
+ remaining === 0 ? new Error('deep') : recurse(remaining - 1);
+ return (recurse(depth).stack ?? '').split('\n').filter((line) => line.includes(' at ')).length;
+}
+
+describe('applyStackTraceLimit', () => {
+ const processDefault = Error.stackTraceLimit;
+
+ afterEach(() => {
+ applyStackTraceLimit(undefined);
+ });
+
+ it('leaves the process default alone when no limit is configured', () => {
+ applyStackTraceLimit(undefined);
+
+ expect(Error.stackTraceLimit).toBe(processDefault);
+ });
+
+ it('records deeper stacks once a limit is set and restores the default when it is removed', () => {
+ applyStackTraceLimit(60);
+
+ expect(Error.stackTraceLimit).toBe(60);
+ expect(frameCount(80)).toBe(60);
+
+ applyStackTraceLimit(undefined);
+
+ expect(Error.stackTraceLimit).toBe(processDefault);
+ });
+});
+
+describe('applyLoadedConfig', () => {
+ const originalConfigPath = process.env.CONFIG_PATH;
+ const processDefault = Error.stackTraceLimit;
+
+ afterEach(() => {
+ applyStackTraceLimit(undefined);
+ resetConfigDigestForTests();
+ jest.restoreAllMocks();
+ if (originalConfigPath === undefined) {
+ delete process.env.CONFIG_PATH;
+ } else {
+ process.env.CONFIG_PATH = originalConfigPath;
+ }
+ });
+
+ function createRuntime(
+ baseCache?: ReturnType<AppConfigServiceDeps['getCache']>,
+ getApplicableConfigs: AppConfigServiceDeps['getApplicableConfigs'] = async () => [],
+ ) {
+ process.env.CONFIG_PATH = '/virtual/librechat.yaml';
+ let text = 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 25\n';
+ const load = createCustomConfigLoader({
+ loadLocal: () => text,
+ defaultConfigPath: '/virtual/librechat.yaml',
+ redactConfig: (value: TCustomConfig) => value,
+ onLoad: recordLoadedConfig,
+ });
+ const stored = new Map<string, unknown>();
+ const cache = {
+ get: jest.fn(async (key: string) => stored.get(key)),
+ set: jest.fn(async (key: string, value: unknown): Promise<boolean | void> => {
+ stored.set(key, value);
+ }),
+ delete: async (key: string) => stored.delete(key),
+ };
+ const formatTools = jest.fn(() => ({}));
+ const buildConfig = jest.fn(AppService);
+ const setCachedTools = jest.fn(async (): Promise<boolean | void> => {});
+ const service = createAppConfigService({
+ loadBaseConfig: async (mode) => {
+ const config = (await load(false, { mode })) ?? {};
+ const systemTools = formatTools();
+ return buildConfig({ config, systemTools });
+ },
+ onBaseConfigPublished: applyLoadedConfig,
+ setCachedTools,
+ getCache: () => baseCache ?? cache,
+ cacheKeys: { APP_CONFIG: 'app_config' },
+ getApplicableConfigs,
+ getUserPrincipals: async () => [],
+ });
+ return {
+ ...service,
+ load,
+ cache,
+ formatTools,
+ buildConfig,
+ setCachedTools,
+ setText: (value: string) => {
+ text = value;
+ },
+ };
+ }
+
+ it('does not change runtime settings when the file is only parsed', async () => {
+ const runtime = createRuntime();
+
+ await runtime.load(false);
+
+ expect(Error.stackTraceLimit).toBe(processDefault);
+ expect(getConfigDigest()).toBeUndefined();
+ });
+
+ it('applies telemetry settings on publication and restores defaults when the key is removed', async () => {
+ jest.spyOn(logger, 'info').mockImplementation(() => logger);
+ const runtime = createRuntime();
+
+ await runtime.getAppConfig({ baseOnly: true });
+ const first = getConfigDigest();
+
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(first).toMatch(/^[0-9a-f]{12}$/);
+
+ runtime.setText('version: 1.3.18\n');
+ await expect(runtime.reloadBaseConfig()).resolves.toEqual({ ok: true });
+
+ expect(Error.stackTraceLimit).toBe(processDefault);
+ expect(getConfigDigest()).toMatch(/^[0-9a-f]{12}$/);
+ expect(getConfigDigest()).not.toBe(first);
+ });
+
+ it('initializes local runtime settings before serving a serialized shared base cache', async () => {
+ const publisher = createRuntime();
+ const published = await publisher.getAppConfig({ baseOnly: true });
+ const digest = getConfigDigest();
+ const cached = JSON.parse(JSON.stringify(published));
+ applyStackTraceLimit(undefined);
+ resetConfigDigestForTests();
+
+ const restarted = createRuntime();
+ restarted.cache.get.mockResolvedValue(cached);
+ const local = await restarted.getAppConfig({ baseOnly: true });
+
+ expect(restarted.buildConfig).toHaveBeenCalledTimes(1);
+ expect(local).not.toBe(cached);
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+
+ restarted.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');
+ restarted.buildConfig.mockRejectedValueOnce(new Error('reload failed'));
+ await expect(restarted.reloadBaseConfig()).resolves.toMatchObject({ ok: false });
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+ });
+
+ it('keeps local YAML and runtime identity when another process replaces the shared base', async () => {
+ const runtime = createRuntime();
+ const local = await runtime.getAppConfig({ baseOnly: true });
+ const digest = getConfigDigest();
+ runtime.cache.get.mockResolvedValue({ config: { telemetry: { stackTraceLimit: 60 } } });
+
+ expect(await runtime.getAppConfig({ baseOnly: true })).toBe(local);
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+ expect(runtime.buildConfig).toHaveBeenCalledTimes(1);
+
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n');
+ await runtime.clearAppConfigCache();
+ const reloaded = await runtime.getAppConfig({ baseOnly: true });
+ expect(reloaded.config.telemetry?.stackTraceLimit).toBe(40);
+ expect(Error.stackTraceLimit).toBe(40);
+ expect(getConfigDigest()).not.toBe(digest);
+ });
+
+ it('partitions serialized merged caches by local base and reuses identical bases', async () => {
+ const values = new Map<string, string>();
+ const cache = {
+ get: async (key: string) => {
+ const value = values.get(key);
+ return value ? JSON.parse(value) : undefined;
+ },
+ set: async (key: string, value: unknown) => {
+ values.set(key, JSON.stringify(value));
+ },
+ delete: async (key: string) => values.delete(key),
+ };
+ const first = createRuntime(cache);
+ first.setText(
+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 25\nmemory:\n disabled: true\n',
+ );
+ await first.getAppConfig({ baseOnly: true });
+ const firstDigest = getConfigDigest();
+ const second = createRuntime(cache);
+ second.setText(
+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\nmemory:\n disabled: false\n',
+ );
+ await second.getAppConfig({ role: 'USER' });
+
+ const local = await first.getAppConfig({ role: 'USER' });
+ expect(local.config.telemetry?.stackTraceLimit).toBe(25);
+ expect(local.memory?.disabled).toBe(true);
+ expect([...values.keys()].filter((key) => key.startsWith('_OVERRIDE_'))).toHaveLength(2);
+
+ const identical = createRuntime(cache);
+ identical.setText(
+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 25\nmemory:\n disabled: true\n',
+ );
+ expect((await identical.getAppConfig({ role: 'USER' })).memory?.disabled).toBe(true);
+ expect(getConfigDigest()).toBe(firstDigest);
+ expect([...values.keys()].filter((key) => key.startsWith('_OVERRIDE_'))).toHaveLength(2);
+
+ identical.setText(
+ 'version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\nmemory:\n disabled: false\n',
+ );
+ await identical.clearAppConfigCache();
+ expect((await identical.getAppConfig({ role: 'USER' })).memory?.disabled).toBe(false);
+ expect(Error.stackTraceLimit).toBe(40);
+ expect(getConfigDigest()).not.toBe(firstDigest);
+ expect([...values.keys()].filter((key) => key.startsWith('_OVERRIDE_'))).toHaveLength(3);
+ });
+
+ it('serves database restrictions uncached through Redis write and connection failures', async () => {
+ jest.spyOn(logger, 'warn').mockReturnValue(logger);
+ const server = await startRespServer();
+ const client = createClient({
+ url: server.url,
+ socket: { reconnectStrategy: false, connectTimeout: 100 },
+ });
+ const adapter = new KeyvRedis(client, { throwOnErrors: true });
+ const cache = new Keyv({ store: adapter, namespace: 'APP_CONFIG', throwOnErrors: true });
+ cache.on('error', () => undefined);
+ const getApplicableConfigs = jest.fn(
+ async (): ReturnType<AppConfigServiceDeps['getApplicableConfigs']> => [
+ {
+ principalType: PrincipalType.ROLE,
+ principalId: 'USER',
+ priority: 10,
+ overrides: { interface: { modelSelect: false } },
+ },
+ ],
+ );
+ try {
+ await client.connect();
+ const runtime = createRuntime(
+ {
+ get: (key) => cache.get(key),
+ set: (key, value) => cache.set(key, value),
+ delete: (key) => cache.delete(key),
+ },
+ getApplicableConfigs,
+ );
+ await runtime.getAppConfig({ baseOnly: true });
+ const digest = getConfigDigest();
+ server.readonly = true;
+
+ expect(
+ (await runtime.getAppConfig({ role: 'USER', failClosed: true })).interfaceConfig
+ ?.modelSelect,
+ ).toBe(false);
+ await server.close();
+ client.destroy();
+ expect(
+ (await runtime.getAppConfig({ role: 'USER', failClosed: true })).interfaceConfig
+ ?.modelSelect,
+ ).toBe(false);
+
+ expect(getApplicableConfigs).toHaveBeenCalledTimes(2);
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+ } finally {
+ if (client.isOpen) client.destroy();
+ await server.close();
+ }
+ });
+
+ it('keeps last-good telemetry when the real Redis adapter rejects SET, then retries', async () => {
+ jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ const server = await startRespServer();
+ const client = createClient({ url: server.url, socket: { reconnectStrategy: false } });
+ const adapter = new KeyvRedis(client, { throwOnErrors: true });
+ const cache = new Keyv({ store: adapter, namespace: 'APP_CONFIG', throwOnErrors: true });
+ cache.on('error', () => undefined);
+ try {
+ await client.connect();
+ const runtime = createRuntime({
+ get: (key) => cache.get(key),
+ set: (key, value) => cache.set(key, value),
+ delete: (key) => cache.delete(key),
+ });
+ const lastGood = await runtime.getAppConfig({ baseOnly: true });
+ const digest = getConfigDigest();
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');
+ server.readonly = true;
+
+ await expect(runtime.reloadBaseConfig()).resolves.toMatchObject({ ok: false });
+ expect(await runtime.getAppConfig({ baseOnly: true })).toBe(lastGood);
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+
+ server.readonly = false;
+ await expect(runtime.reloadBaseConfig()).resolves.toEqual({ ok: true });
+ expect(Error.stackTraceLimit).toBe(60);
+ expect(getConfigDigest()).not.toBe(digest);
+ } finally {
+ client.destroy();
+ await server.close();
+ }
+ });
+
+ it.each([
+ 'validation',
+ 'tools scan',
+ 'AppService',
+ 'tool cache',
+ 'base cache',
+ 'tool cache refusal',
+ 'base cache refusal',
+ ])('keeps the last good digest and limit when %s fails', async (stage) => {
+ jest.spyOn(logger, 'error').mockImplementation(() => logger);
+ const runtime = createRuntime();
+ const lastGood = await runtime.getAppConfig({ baseOnly: true });
+ const digest = getConfigDigest();
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');
+ const failure = new Error('publication failed');
+ if (stage === 'validation') {
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 0\n');
+ } else if (stage === 'tools scan') {
+ runtime.formatTools.mockImplementationOnce(() => {
+ throw failure;
+ });
+ } else if (stage === 'AppService') {
+ runtime.buildConfig.mockRejectedValueOnce(failure);
+ } else if (stage === 'tool cache') {
+ runtime.setCachedTools.mockRejectedValueOnce(failure);
+ } else if (stage === 'base cache') {
+ runtime.cache.set.mockRejectedValueOnce(failure);
+ } else if (stage === 'tool cache refusal') {
+ runtime.setCachedTools.mockResolvedValueOnce(false);
+ } else {
+ runtime.cache.set.mockResolvedValueOnce(false);
+ }
+
+ await expect(runtime.reloadBaseConfig()).resolves.toMatchObject({ ok: false });
+
+ expect(await runtime.getAppConfig({ baseOnly: true })).toBe(lastGood);
+ expect(getConfigDigest()).toBe(digest);
+ expect(Error.stackTraceLimit).toBe(25);
+
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n');
+ await expect(runtime.reloadBaseConfig()).resolves.toEqual({ ok: true });
+ expect(Error.stackTraceLimit).toBe(40);
+ expect(getConfigDigest()).not.toBe(digest);
+ });
+
+ it('does not apply a config overtaken by a clear during publication', async () => {
+ const runtime = createRuntime();
+ await runtime.getAppConfig({ baseOnly: true });
+ const digest = getConfigDigest();
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 60\n');
+ runtime.cache.set.mockImplementationOnce(async () => {
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+ await runtime.clearAppConfigCache();
+ });
+
+ await runtime.reloadBaseConfig();
+
+ expect(Error.stackTraceLimit).toBe(25);
+ expect(getConfigDigest()).toBe(digest);
+
+ runtime.setText('version: 1.3.18\ntelemetry:\n stackTraceLimit: 40\n');
+ await runtime.reloadBaseConfig();
+ expect(Error.stackTraceLimit).toBe(40);
+ expect(getConfigDigest()).not.toBe(digest);
+ });
+});
diff --git a/packages/api/src/app/runtime.ts b/packages/api/src/app/runtime.ts
new file mode 100644
index 0000000000..d15ef1b6a1
--- /dev/null
+++ b/packages/api/src/app/runtime.ts
@@ -0,0 +1,31 @@
+import type { AppConfig } from '@librechat/data-schemas';
+import type { LoadedCustomConfig } from './loader';
+import { setConfigDigest } from '~/telemetry/digest';
+
+/** The limit the process had before config first set one: 10, or `--stack-trace-limit`. */
+let processStackTraceLimit: number | undefined;
+
+const loadedDigests = new WeakMap<object, string>();
+
+/** Stages file identity without changing the running configuration. */
+export function recordLoadedConfig({ config, digest }: LoadedCustomConfig): void {
+ loadedDigests.set(config, digest);
+}
+
+/**
+ * Sets `Error.stackTraceLimit` for errors created from now on. With no limit it restores the
+ * process default, so removing the key and reloading undoes an earlier value.
+ */
+export function applyStackTraceLimit(limit: number | undefined): void {
+ processStackTraceLimit ??= Error.stackTraceLimit;
+ Error.stackTraceLimit = limit ?? processStackTraceLimit;
+}
+
+/** Applies process-wide settings only once the YAML-derived base config is published. */
+export function applyLoadedConfig({ config }: AppConfig): void {
+ applyStackTraceLimit(config?.telemetry?.stackTraceLimit);
+ const digest = config && loadedDigests.get(config);
+ if (digest) {
+ setConfigDigest(digest);
+ }
+}
diff --git a/packages/api/src/app/service.spec.ts b/packages/api/src/app/service.spec.ts
index 0c868b767f..74a67e6a0e 100644
--- a/packages/api/src/app/service.spec.ts
+++ b/packages/api/src/app/service.spec.ts
@@ -1,4 +1,9 @@
-import { PrincipalType, getMaxSubagents, setMaxSubagents } from 'librechat-data-provider';
+import {
+ FileSources,
+ PrincipalType,
+ getMaxSubagents,
+ setMaxSubagents,
+} from 'librechat-data-provider';
import {
logger,
AppService,
@@ -35,9 +40,8 @@ function createMockCache(namespace = 'app_config') {
const store = new Map();
return {
get: jest.fn((key) => Promise.resolve(store.get(`${namespace}:${key}`))),
- set: jest.fn((key, value) => {
+ set: jest.fn(async (key, value): Promise<boolean | void> => {
store.set(`${namespace}:${key}`, value);
- return Promise.resolve(undefined);
}),
delete: jest.fn((key) => {
store.delete(`${namespace}:${key}`);
@@ -1175,6 +1179,72 @@ describe('createAppConfigService', () => {
expect(deps.getApplicableConfigs).toHaveBeenCalledTimes(1);
});
+ describe.each([false, true])('optional override cache with failClosed=%s', (failClosed) => {
+ it.each(['read', 'write', 'cleanup'])(
+ 'serves current restrictions when cache %s fails',
+ async (stage) => {
+ const warn = jest.spyOn(logger, 'warn').mockReturnValue(logger);
+ const deps = createDeps({
+ getApplicableConfigs: jest
+ .fn()
+ .mockResolvedValue([
+ { isActive: true, priority: 10, overrides: { interface: { modelSelect: false } } },
+ ]),
+ });
+ const service = createAppConfigService(deps);
+ await service.getAppConfig({ baseOnly: true });
+ const failure = new Error('cache failure with private content');
+ if (stage === 'read') {
+ deps._cache.get.mockRejectedValueOnce(failure);
+ } else if (stage === 'write') {
+ deps._cache.set.mockRejectedValueOnce(failure);
+ } else {
+ deps._cache.set.mockImplementationOnce(async () => {
+ await service.clearAppConfigCache();
+ });
+ deps._cache.delete.mockImplementation(async (key) => {
+ if (key.startsWith('_OVERRIDE_')) throw failure;
+ return deps._cache._store.delete(`app_config:${key}`);
+ });
+ }
+
+ const resolved = await service.getAppConfig({ role: 'USER', failClosed });
+
+ expect(resolved.interfaceConfig?.modelSelect).toBe(false);
+ expect(deps.getApplicableConfigs).toHaveBeenCalledTimes(1);
+ if (stage === 'read') expect(deps._cache.set).toHaveBeenCalledTimes(1);
+ expect(warn).toHaveBeenCalled();
+ expect(JSON.stringify(warn.mock.calls)).not.toContain('private content');
+ },
+ );
+ });
+
+ it('separates shared merges when identical YAML derives different runtime settings', async () => {
+ const cache = createMockCache();
+ const firstDeps = createDeps({
+ getCache: () => cache,
+ loadBaseConfig: jest.fn().mockResolvedValue({
+ config: {},
+ interfaceConfig: { modelSelect: false },
+ }),
+ });
+ const secondDeps = createDeps({
+ getCache: () => cache,
+ loadBaseConfig: jest.fn().mockResolvedValue({
+ config: {},
+ interfaceConfig: { modelSelect: true },
+ }),
+ });
+ const first = createAppConfigService(firstDeps);
+ const second = createAppConfigService(secondDeps);
+
+ expect((await first.getAppConfig({ role: 'USER' })).interfaceConfig?.modelSelect).toBe(false);
+ expect((await second.getAppConfig({ role: 'USER' })).interfaceConfig?.modelSelect).toBe(true);
+ expect((await first.getAppConfig({ role: 'USER' })).interfaceConfig?.modelSelect).toBe(false);
+ expect(secondDeps.getApplicableConfigs).toHaveBeenCalledTimes(1);
+ expect([...cache._store.keys()].filter((key) => key.includes('_OVERRIDE_'))).toHaveLength(2);
+ });
+
it('uses separate cache keys per userId (no cross-user contamination)', async () => {
const deps = createDeps({
getApplicableConfigs: jest
@@ -1204,7 +1274,7 @@ describe('createAppConfigService', () => {
const cachedKeys = [...deps._cache._store.keys()];
const overrideKey = cachedKeys.find((k) => k.includes('_OVERRIDE_:'));
expect(overrideKey).toMatch(
- /^app_config:_OVERRIDE_:__default__:global:uid1:tenant-v1:[a-f0-9]{64}$/,
+ /^app_config:_OVERRIDE_:__default__:global:uid1:base-v2:[a-f0-9]{64}$/,
);
});
@@ -1373,7 +1443,7 @@ describe('createAppConfigService', () => {
k.includes('_OVERRIDE_:'),
);
expect(overrideKey).toMatch(
- /^app_config:_OVERRIDE_:tenant-a:tenant:USER:tenant-v1:[a-f0-9]{64}$/,
+ /^app_config:_OVERRIDE_:tenant-a:tenant:USER:base-v2:[a-f0-9]{64}$/,
);
expect(overrideKey).not.toContain('__default__');
});
@@ -1578,7 +1648,7 @@ describe('createAppConfigService', () => {
expect([...deps._cache._store.keys()]).toEqual(
expect.arrayContaining([
expect.stringMatching(
- /^app_config:_OVERRIDE_:__default__:global:USER:uid1:tenant-v1:[a-f0-9]{64}$/,
+ /^app_config:_OVERRIDE_:__default__:global:USER:uid1:base-v2:[a-f0-9]{64}$/,
),
]),
);
@@ -1871,6 +1941,80 @@ describe('createAppConfigService', () => {
expect(await getAppConfig({ baseOnly: true })).toBe(initial);
});
+ describe.each([false, true])('restoration after a clear=%s', (clear) => {
+ it.each(['tool rejection', 'tool refusal', 'base rejection', 'base refusal'])(
+ 'repairs a %s on the next ordinary read',
+ async (stage) => {
+ jest.spyOn(logger, 'error').mockReturnValue(logger);
+ const good: AppConfig = {
+ config: {},
+ fileStrategy: FileSources.local,
+ imageOutputType: 'png',
+ availableTools: {
+ good: {
+ type: 'function',
+ function: { name: 'good', description: 'good', parameters: { type: 'object' } },
+ },
+ },
+ };
+ const rejected: AppConfig = {
+ ...good,
+ availableTools: {
+ rejected: {
+ type: 'function',
+ function: {
+ name: 'rejected',
+ description: 'rejected',
+ parameters: { type: 'object' },
+ },
+ },
+ },
+ };
+ let tools = good.availableTools;
+ let refuseTools = false;
+ const deps = createDeps({
+ loadBaseConfig: jest.fn().mockResolvedValue(good),
+ setCachedTools: jest.fn(async (next: AppConfig['availableTools']) => {
+ if (next === good.availableTools && refuseTools) {
+ refuseTools = false;
+ if (stage.endsWith('rejection')) throw new Error('tool restore unavailable');
+ return false;
+ }
+ tools = next;
+ return true;
+ }),
+ });
+ const service = createAppConfigService(deps);
+ await service.getAppConfig({ baseOnly: true });
+ if (clear) await service.clearAppConfigCache();
+ refuseTools = stage.startsWith('tool');
+ deps.loadBaseConfig.mockResolvedValueOnce(rejected).mockRejectedValue(invalidFile());
+ let writes = 0;
+ deps._cache.set.mockImplementation(async (key, value) => {
+ writes++;
+ if (writes === 1) throw new Error('new base unavailable');
+ if (writes === 2 && stage.startsWith('base')) {
+ if (stage.endsWith('rejection')) throw new Error('base restore unavailable');
+ return false;
+ }
+ deps._cache._store.set(`app_config:${key}`, value);
+ });
+
+ await expect(service.reloadBaseConfig()).resolves.toMatchObject({ ok: false });
+ if (stage.startsWith('tool')) expect(tools).toBe(rejected.availableTools);
+ const loads = deps.loadBaseConfig.mock.calls.length;
+
+ expect(await service.getAppConfig({ baseOnly: true })).toBe(good);
+ expect(deps.loadBaseConfig).toHaveBeenCalledTimes(loads + 1);
+ expect(tools).toBe(good.availableTools);
+ expect(deps._cache._store.get('app_config:_BASE_')).toBe(good);
+
+ expect(await service.getAppConfig({ baseOnly: true })).toBe(good);
+ expect(deps.loadBaseConfig).toHaveBeenCalledTimes(loads + 1);
+ },
+ );
+ });
+
it('resolves with the failure instead of rejecting when there is no last good config', async () => {
jest.spyOn(logger, 'error').mockReturnValue(logger);
const deps = createDeps();
diff --git a/packages/api/src/app/service.ts b/packages/api/src/app/service.ts
index db2cdcd4bb..62bc22343c 100644
--- a/packages/api/src/app/service.ts
+++ b/packages/api/src/app/service.ts
@@ -30,6 +30,7 @@ import { resolveOverrideLayers, mergeOverrideLayersWithSource } from './override
import { createEnvReferencePolicy, parseEnvAllowlist } from './allowlist';
import { resolveRuntimeConfigSecrets } from '~/admin/secrets';
import { normalizeEndpointReferences } from './references';
+import { getSafeErrorMetadata } from '~/utils/errors';
import { getConfigLoadFailure } from './loader';
import { checkAppConfigTheme } from './theme';
@@ -478,8 +479,10 @@ interface CacheStore {
export interface AppConfigServiceDeps {
/** Load the base AppConfig from YAML + AppService processing. */
loadBaseConfig: (mode?: CustomConfigLoadMode) => Promise<AppConfig | undefined>;
+ /** Synchronous process-wide effects, after tools and the current base config publish. */
+ onBaseConfigPublished?: (config: AppConfig) => void;
/** Cache tools after base config is loaded. */
- setCachedTools: (tools: Record<string, unknown>) => Promise<void>;
+ setCachedTools: (tools: Record<string, unknown>) => Promise<boolean | void>;
/** Get a cache store by key. */
getCache: (key: string) => CacheStore;
/** The CacheKeys constants from librechat-data-provider. */
@@ -613,7 +616,7 @@ export function _resetOverrideStrictCache(): void {
_warnedNoTenantInStrictMode = false;
}
-/** Versioned so older heads cannot supply override entries built under a stale tenant ID. */
+/** Versioned so older heads cannot supply entries built without the local base identity. */
function overrideCacheKey(
role: string | undefined,
userId: string | undefined,
@@ -623,7 +626,7 @@ function overrideCacheKey(
const tenant = tenantId ? `${tenantId}:tenant` : '__default__:global';
const principal =
userId && role ? `${role}:${userId}` : userId || role || BASE_CONFIG_PRINCIPAL_ID;
- return `_OVERRIDE_:${tenant}:${principal}:tenant-v1:${scopeVersion}`;
+ return `_OVERRIDE_:${tenant}:${principal}:base-v2:${scopeVersion}`;
}
/**
@@ -696,6 +699,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
} {
const {
loadBaseConfig,
+ onBaseConfigPublished,
setCachedTools,
getCache,
cacheKeys,
@@ -710,6 +714,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
const cache = getCache(cacheKeys.APP_CONFIG);
const envAllowlist = parseEnvAllowlist(overrideEnvAllowlist);
const envPolicies = new WeakMap<AppConfig, Map<string, EnvReferencePolicy>>();
+ const scopeVersions = new WeakMap<AppConfig, string>();
const warnRebinding = createRebindingWarner();
const runtimeConfigs = new WeakMap<AppConfig, AppConfig>();
@@ -754,6 +759,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
return policy;
}
let lastGoodBaseConfig: AppConfig | undefined;
+ let publishedBaseGeneration = -1;
let baseConfigFlight: { promise: Promise<BaseConfigLoad>; generation: number } | undefined;
/**
@@ -825,6 +831,7 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
if (generation !== baseGeneration) {
return lastGood;
}
+ publishedBaseGeneration = -1;
const restorations = [cache.set(BASE_CONFIG_KEY, lastGood)];
if (lastGood.availableTools) {
restorations.push(setCachedTools(lastGood.availableTools));
@@ -832,13 +839,20 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
const results = await Promise.allSettled(restorations);
for (const result of results) {
if (result.status === 'rejected') {
- logger.error('[ensureBaseConfig] Failed to restore last-good config state:', result.reason);
+ logger.error(
+ '[ensureBaseConfig] Failed to restore last-good config state:',
+ getSafeErrorMetadata(result.reason),
+ );
+ } else if (result.value === false) {
+ logger.error('[ensureBaseConfig] Cache refused last-good config restoration.');
}
}
if (generation !== baseGeneration) {
await cache.delete(BASE_CONFIG_KEY).catch((deleteError: unknown) => {
logger.error('[ensureBaseConfig] Failed to drop an overtaken restoration:', deleteError);
});
+ } else if (results.every((result) => result.status === 'fulfilled' && result.value !== false)) {
+ publishedBaseGeneration = generation;
}
return lastGood;
}
@@ -863,18 +877,22 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
if (generation !== baseGeneration) {
return baseConfig;
}
- if (baseConfig.availableTools) {
- await setCachedTools(baseConfig.availableTools);
+ if (baseConfig.availableTools && (await setCachedTools(baseConfig.availableTools)) === false) {
+ throw new Error('Tool cache refused base configuration publication.');
}
if (generation !== baseGeneration) {
return baseConfig;
}
- await cache.set(BASE_CONFIG_KEY, baseConfig);
+ if ((await cache.set(BASE_CONFIG_KEY, baseConfig)) === false) {
+ throw new Error('Base cache refused configuration publication.');
+ }
if (generation !== baseGeneration) {
await cache.delete(BASE_CONFIG_KEY);
return baseConfig;
}
+ onBaseConfigPublished?.(loaded);
lastGoodBaseConfig = baseConfig;
+ publishedBaseGeneration = generation;
return baseConfig;
}
@@ -900,12 +918,9 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
* subagent cap) end with the newest librechat.yaml rather than whichever read finished last.
*/
async function loadBase(refresh?: boolean): Promise<BaseConfigLoad> {
- const cached = (await cache.get(BASE_CONFIG_KEY)) as AppConfig | undefined;
- if (cached) {
- lastGoodBaseConfig ??= cached;
- if (!refresh) {
- return { config: cached };
- }
+ // Another process's cached YAML cannot replace this process's runtime configuration.
+ if (!refresh && lastGoodBaseConfig && publishedBaseGeneration === baseGeneration) {
+ return { config: lastGoodBaseConfig };
}
const generation = baseGeneration;
@@ -1048,22 +1063,31 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
}
};
- const scopeVersion = createHash('sha256')
- .update(
- JSON.stringify([
- baseConfig.endpoints?.custom?.map(({ name, tenantId }) => [name, tenantId]),
- baseConfig.config?.endpoints?.custom?.map(({ name, tenantId }) => [name, tenantId]),
- ]),
- )
- .digest('hex');
+ let scopeVersion = scopeVersions.get(baseConfig);
+ if (!scopeVersion) {
+ // Hash once per local base, including derived settings and override policy, never values in keys.
+ scopeVersion = createHash('sha256')
+ .update(JSON.stringify([baseConfig, rederiveEndpoints, overrideEnvAllowlist]))
+ .digest('hex');
+ scopeVersions.set(baseConfig, scopeVersion);
+ }
const cacheKey = overrideCacheKey(role, userId, cacheTenantId, scopeVersion);
/**
* The key names a role and user, not a principal set: principals resolved elsewhere are
* cached only when a user id ties them to the key.
*/
- const cacheable = bypassOverrideCache !== true && (resolvedPrincipals == null || !!userId);
+ let cacheable = bypassOverrideCache !== true && (resolvedPrincipals == null || !!userId);
if (cacheable && !refresh) {
- const cachedMerged = (await cache.get(cacheKey)) as AppConfig | undefined;
+ let cachedMerged: AppConfig | undefined;
+ try {
+ cachedMerged = (await cache.get(cacheKey)) as AppConfig | undefined;
+ } catch (error) {
+ cacheable = false;
+ logger.warn(
+ '[getAppConfig] Failed to read merged config cache; resolving uncached.',
+ getSafeErrorMetadata(error),
+ );
+ }
if (cachedMerged) {
return await augment(withRuntimeSecrets(cachedMerged));
}
@@ -1108,9 +1132,16 @@ export function createAppConfigService(deps: AppConfigServiceDeps): {
}
if (cacheable && getOverrideGeneration(cacheTenantId) === generation) {
- await cache.set(cacheKey, merged, overrideCacheTtl);
- if (getOverrideGeneration(cacheTenantId) !== generation) {
- await cache.delete(cacheKey);
+ try {
+ await cache.set(cacheKey, merged, overrideCacheTtl);
+ if (getOverrideGeneration(cacheTenantId) !== generation) {
+ await cache.delete(cacheKey);
+ }
+ } catch (error) {
+ logger.warn(
+ '[getAppConfig] Failed to cache merged config; serving the resolved configuration.',
+ getSafeErrorMetadata(error),
+ );
}
}
return await augment(withRuntimeSecrets(merged));
diff --git a/packages/api/src/telemetry/config.spec.ts b/packages/api/src/telemetry/config.spec.ts
index 79753466df..ded1988c34 100644
--- a/packages/api/src/telemetry/config.spec.ts
+++ b/packages/api/src/telemetry/config.spec.ts
@@ -65,6 +65,22 @@ describe('getTelemetryConfig', () => {
expect(config.serviceVersion).toBe('0.8.5');
});
+ it('reports the baked BUILD_COMMIT ahead of the package version', () => {
+ const commit = 'f177bed1757c1b7a4e2bd2a7d1c7a3d0e9a6b5c4';
+
+ expect(
+ getTelemetryConfig({ BUILD_COMMIT: ` ${commit} `, npm_package_version: '0.8.5' })
+ .serviceVersion,
+ ).toBe(commit);
+ expect(
+ getTelemetryConfig({ BUILD_COMMIT: commit, OTEL_SERVICE_VERSION: '1.2.3' }).serviceVersion,
+ ).toBe('1.2.3');
+ expect(
+ getTelemetryConfig({ BUILD_COMMIT: ' ', npm_package_version: '0.8.5' }).serviceVersion,
+ ).toBe('0.8.5');
+ expect(getTelemetryConfig({}).serviceVersion).toBeUndefined();
+ });
+
it('enables ioredis instrumentation only when explicitly configured', () => {
const config = getTelemetryConfig({
OTEL_IOREDIS_TRACING_ENABLED: 'true',
diff --git a/packages/api/src/telemetry/config.ts b/packages/api/src/telemetry/config.ts
index cb87846199..0ae2567cac 100644
--- a/packages/api/src/telemetry/config.ts
+++ b/packages/api/src/telemetry/config.ts
@@ -36,8 +36,11 @@ export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): Teleme
const tracingEnabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled;
const logsEnabled = isTruthy(env.OTEL_LOGS_ENABLED) && !sdkDisabled;
const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;
+ /** Images bake BUILD_COMMIT, which names the exact code a pod runs; the package version does not. */
const serviceVersion =
- normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);
+ normalizeEnvValue(env.OTEL_SERVICE_VERSION) ??
+ normalizeEnvValue(env.BUILD_COMMIT) ??
+ normalizeEnvValue(env.npm_package_version);
const ioredisTracingEnabled = isTruthy(env.OTEL_IOREDIS_TRACING_ENABLED);
const logsLevel = normalizeEnvValue(env.OTEL_LOGS_LEVEL)?.toLowerCase() ?? DEFAULT_LOGS_LEVEL;
diff --git a/packages/api/src/telemetry/digest.ts b/packages/api/src/telemetry/digest.ts
new file mode 100644
index 0000000000..bb17352241
--- /dev/null
+++ b/packages/api/src/telemetry/digest.ts
@@ -0,0 +1,22 @@
+/**
+ * Names the librechat.yaml a process runs with on its log records and request spans. It is an
+ * attribute rather than a resource attribute because the resource is fixed when the SDK starts,
+ * before the config loads, and an admin reload can change the config without a restart.
+ */
+export const CONFIG_DIGEST_ATTRIBUTE = 'librechat.config.digest';
+
+let configDigest: string | undefined;
+
+/** Records the digest of the config that just loaded; the last successful load wins. */
+export function setConfigDigest(digest: string): void {
+ configDigest = digest;
+}
+
+export function getConfigDigest(): string | undefined {
+ return configDigest;
+}
+
+/** Test hook — not exported from the package barrel. */
+export function resetConfigDigestForTests(): void {
+ configDigest = undefined;
+}
diff --git a/packages/api/src/telemetry/grpc.spec.ts b/packages/api/src/telemetry/grpc.spec.ts
index 27b283faae..b6529e7c38 100644
--- a/packages/api/src/telemetry/grpc.spec.ts
+++ b/packages/api/src/telemetry/grpc.spec.ts
@@ -3,6 +3,7 @@ import { logger } from '@librechat/data-schemas';
import { Server, ServerCredentials } from '@grpc/grpc-js';
import type { sendUnaryData, ServerUnaryCall, ServiceDefinition } from '@grpc/grpc-js';
import { initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } from './sdk';
+import { resetConfigDigestForTests, setConfigDigest } from './digest';
const LOGS_EXPORT_PATH = '/opentelemetry.proto.collector.logs.v1.LogsService/Export';
const TRACES_EXPORT_PATH = '/opentelemetry.proto.collector.trace.v1.TraceService/Export';
@@ -11,6 +12,8 @@ const SPAN_NAME = 'grpc-spec-span';
const INFO_MESSAGE = 'grpc-spec-info-message';
const DEBUG_MESSAGE = 'grpc-spec-debug-message';
const SECRET_SUFFIX = 'GRPCSPECSECRET1234567890';
+const BUILD_COMMIT = 'c0ffee0123456789abcdef0123456789abcdef01';
+const CONFIG_DIGEST = 'feedc0de4321';
const EXPORTER_ENV_KEYS = [
'OTEL_EXPORTER_OTLP_ENDPOINT',
'OTEL_EXPORTER_OTLP_PROTOCOL',
@@ -89,7 +92,9 @@ describe('OTLP/gRPC export', () => {
process.env.OTEL_METRICS_EXPORTER = 'none';
process.env.OTEL_NODE_RESOURCE_DETECTORS = 'none';
+ setConfigDigest(CONFIG_DIGEST);
const controller = initializeTelemetry({
+ BUILD_COMMIT,
OTEL_LOGS_ENABLED: 'true',
OTEL_SERVICE_NAME: SERVICE_NAME,
OTEL_TRACING_ENABLED: 'true',
@@ -108,6 +113,7 @@ describe('OTLP/gRPC export', () => {
afterAll(async () => {
await resetTelemetryForTests();
+ resetConfigDigestForTests();
for (const key of EXPORTER_ENV_KEYS) {
const value = previousEnv[key];
if (value === undefined) {
@@ -134,6 +140,16 @@ describe('OTLP/gRPC export', () => {
expect(payload.includes(DEBUG_MESSAGE)).toBe(false);
});
+ it('names the build in the resource and the loaded config on each record', () => {
+ const logs = Buffer.concat(receivedLogs);
+ const traces = Buffer.concat(receivedTraces);
+
+ expect(logs.includes(BUILD_COMMIT)).toBe(true);
+ expect(traces.includes(BUILD_COMMIT)).toBe(true);
+ expect(logs.includes('librechat.config.digest')).toBe(true);
+ expect(logs.includes(CONFIG_DIGEST)).toBe(true);
+ });
+
it('exports spans over gRPC', () => {
const payload = Buffer.concat(receivedTraces);
diff --git a/packages/api/src/telemetry/logs.ts b/packages/api/src/telemetry/logs.ts
index a7b75839c0..1b12aaafc1 100644
--- a/packages/api/src/telemetry/logs.ts
+++ b/packages/api/src/telemetry/logs.ts
@@ -2,11 +2,24 @@ import winston from 'winston';
import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';
import { logger, baseLogFormat, jsonTruncateFormat } from '@librechat/data-schemas';
import type { TelemetryConfig } from './config';
+import { CONFIG_DIGEST_ATTRIBUTE, getConfigDigest } from './digest';
import { emitTelemetryWarning } from './warnings';
import { DEFAULT_LOGS_LEVEL } from './config';
let activeTransport: OpenTelemetryTransportV3 | undefined;
+/**
+ * Names the loaded librechat.yaml on exported records. Winston hands each transport its own copy
+ * of the record, so file and console output keep their shape.
+ */
+const configDigestFormat = winston.format((info) => {
+ const digest = getConfigDigest();
+ if (digest) {
+ info[CONFIG_DIGEST_ATTRIBUTE] = digest;
+ }
+ return info;
+});
+
/**
* An unknown level name would resolve to `undefined` inside winston and silently
* drop every record, so fall back and say so instead.
@@ -37,7 +50,7 @@ export function attachLogsTransport(config: TelemetryConfig): void {
const transport = new OpenTelemetryTransportV3({
level: resolveLogsLevel(config.logsLevel),
- format: winston.format.combine(baseLogFormat, jsonTruncateFormat()),
+ format: winston.format.combine(baseLogFormat, jsonTruncateFormat(), configDigestFormat()),
});
logger.add(transport);
activeTransport = transport;
diff --git a/packages/api/src/telemetry/middleware.spec.ts b/packages/api/src/telemetry/middleware.spec.ts
index 8be4bea321..d80d53d545 100644
--- a/packages/api/src/telemetry/middleware.spec.ts
+++ b/packages/api/src/telemetry/middleware.spec.ts
@@ -5,6 +5,7 @@ import type { NextFunction, Response } from 'express';
import type { Span } from '@opentelemetry/api';
import type { ServerRequest } from '~/types';
import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
+import { resetConfigDigestForTests, setConfigDigest } from './digest';
import { observeRedisOperation } from '~/cache/redisTelemetry';
import { getTelemetryRequestSpan } from './sdk';
@@ -118,6 +119,23 @@ describe('telemetryMiddleware', () => {
);
});
+ it('names the loaded config on the request span once a digest is recorded', () => {
+ const span = createSpan();
+ mockGetTelemetryRequestSpan.mockReturnValue(span);
+
+ setConfigDigest('0123456789ab');
+ try {
+ telemetryMiddleware(createRequest(), createResponse() as Response, jest.fn());
+ } finally {
+ resetConfigDigestForTests();
+ }
+
+ expect(span.setAttributes).toHaveBeenCalledWith({
+ 'http.request.method': 'POST',
+ 'librechat.config.digest': '0123456789ab',
+ });
+ });
+
it('adds aggregated Redis activity to the stored request span', async () => {
const requestSpan = createSpan();
const res = createResponse(200);
diff --git a/packages/api/src/telemetry/middleware.ts b/packages/api/src/telemetry/middleware.ts
index cd912f2806..bbbd538868 100644
--- a/packages/api/src/telemetry/middleware.ts
+++ b/packages/api/src/telemetry/middleware.ts
@@ -8,6 +8,7 @@ import {
runWithRedisRequestTelemetry,
} from '~/cache/redisTelemetry';
import { getErrorType, getSafeSpanException } from './safeException';
+import { CONFIG_DIGEST_ATTRIBUTE, getConfigDigest } from './digest';
import { getTelemetryRequestSpan } from './sdk';
import { DEFAULT_HEALTH_PATH } from './config';
@@ -115,9 +116,12 @@ export function telemetryMiddleware(req: ServerRequest, res: Response, next: Nex
return;
}
- span.setAttributes({
- 'http.request.method': req.method,
- });
+ const attributes: Attributes = { 'http.request.method': req.method };
+ const configDigest = getConfigDigest();
+ if (configDigest) {
+ attributes[CONFIG_DIGEST_ATTRIBUTE] = configDigest;
+ }
+ span.setAttributes(attributes);
const redisTelemetry = createRedisRequestTelemetry(span);
let completed = false;
diff --git a/packages/data-provider/src/config.spec.ts b/packages/data-provider/src/config.spec.ts
index 677fbc7b9f..be96bd60ed 100644
--- a/packages/data-provider/src/config.spec.ts
+++ b/packages/data-provider/src/config.spec.ts
@@ -57,6 +57,17 @@ describe('independent browser replay opt-in', () => {
expect(telemetrySchema.safeParse(telemetry).success).toBe(false);
});
});
+
+describe('telemetry.stackTraceLimit', () => {
+ it('stays absent unless configured, so the process default applies', () => {
+ expect(telemetrySchema.parse({})).toEqual({});
+ expect(telemetrySchema.parse({ stackTraceLimit: 50 })).toEqual({ stackTraceLimit: 50 });
+ });
+
+ it.each([0, -1, 2.5, 1001, '50'])('rejects %p', (stackTraceLimit) => {
+ expect(telemetrySchema.safeParse({ stackTraceLimit }).success).toBe(false);
+ });
+});
import { EModelEndpoint, isDocumentSupportedProvider } from './schemas';
import { getEndpointFileConfig, mergeFileConfig } from './file-config';
import { DEFAULT_MCP_APP_CSP_LIMITS } from './mcp/csp';
@@ -3281,6 +3292,10 @@ describe('DEPLOYMENT_ONLY_CONFIG_PATHS', () => {
});
});
+ it('keeps the process-wide stack limit deployment-only', () => {
+ expect(DEPLOYMENT_ONLY_CONFIG_PATHS['telemetry.stackTraceLimit']).toBe('operator');
+ });
+
it('overlaps neither the base-only sections nor the base-only paths', () => {
const baseOnly = new Set<string>([...BASE_ONLY_CONFIG_SECTIONS, ...BASE_ONLY_CONFIG_PATHS]);
for (const path of Object.keys(DEPLOYMENT_ONLY_CONFIG_PATHS)) {
diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts
index fa55aca890..df875c20fc 100644
--- a/packages/data-provider/src/config.ts
+++ b/packages/data-provider/src/config.ts
@@ -168,6 +168,7 @@ export type DeploymentOnlyConfigReason = 'startup' | 'operator' | 'unused';
* a stored override had no effect rather than a dangerous one.
*/
export const DEPLOYMENT_ONLY_CONFIG_PATHS: Readonly<Record<string, DeploymentOnlyConfigReason>> = {
+ 'telemetry.stackTraceLimit': 'operator',
rateLimits: 'operator',
cloudfront: 'operator',
permissions: 'operator',
@@ -3521,6 +3522,12 @@ export const RUM_COLLECTOR_ACK_HEADER = 'x-librechat-rum-accepted';
export const telemetrySchema = z
.object({
+ /**
+ * Frames V8 records per error stack (`Error.stackTraceLimit`), applied when librechat.yaml
+ * loads. Unset keeps the process default: 10, or `--stack-trace-limit` from NODE_OPTIONS.
+ * Deeper stacks cost more on every thrown error.
+ */
+ stackTraceLimit: z.number().int().min(1).max(1000).optional(),
rum: z
.object({
enabled: z.boolean().optional(),
diff --git a/packages/data-schemas/src/config/parsers.spec.ts b/packages/data-schemas/src/config/parsers.spec.ts
index 66c8dbd438..a43b0b229b 100644
--- a/packages/data-schemas/src/config/parsers.spec.ts
+++ b/packages/data-schemas/src/config/parsers.spec.ts
@@ -80,6 +80,29 @@ describe('redactMessage', () => {
expect(redactMessage('mention @handle and v1.2.3')).toBe('mention @handle and v1.2.3');
});
+ it('keeps scoped package paths in stack frames and still masks addresses beside them', () => {
+ const frames = [
+ ' at Run.processStream (/app/node_modules/@librechat/agents/dist/cjs/run.cjs:120:5)',
+ ' at async KeyvMongo.get (/app/node_modules/@keyv/mongo/dist/index.cjs:10:3)',
+ ' at /app/node_modules/@langchain/core/dist/runnables/base.cjs:1:2',
+ ' at file:///app/node_modules/@scope/pkg/index.mjs:1:1',
+ ' at load (C:\\app\\node_modules\\@scope\\pkg\\index.js:1:1)',
+ '{"stack":"at f (C:\\\\app\\\\node_modules\\\\@scope\\\\pkg\\\\x.js:1:2)"}',
+ ];
+ for (const frame of frames) {
+ expect(redactMessage(frame)).toBe(frame);
+ }
+
+ expect(
+ redactMessage('Error: no user [email protected]\n at f (/app/node_modules/@a/b/c.js:1:2)'),
+ ).toBe('Error: no user [REDACTED_EMAIL]\n at f (/app/node_modules/@a/b/c.js:1:2)');
+ expect(redactMessage('user jane/@example.com ok')).toBe('user [REDACTED_EMAIL] ok');
+ expect(redactMessage('/home/[email protected]/file.js')).toBe('[REDACTED_EMAIL]');
+ expect(redactMessage('see https://social.example/@jane.doe ok')).toBe(
+ 'see [REDACTED_EMAIL] ok',
+ );
+ });
+
it('masks the whole token for apostrophes, long local parts and combining-mark domains', () => {
expect(redactMessage("mary.jane'[email protected]")).toBe('[REDACTED_EMAIL]');
expect(redactMessage(`${'a'.repeat(100)}@example.com ok`)).toBe('[REDACTED_EMAIL] ok');
@@ -814,6 +837,20 @@ describe('redactFormat', () => {
expect(info.nested).toEqual({ '[REDACTED_EMAIL]': 1, '[REDACTED_EMAIL]#2': 2, other: 3 });
});
+ it('keeps scoped package frame locations in a redacted Error stack', () => {
+ const error = new Error('lookup failed for [email protected]');
+ error.stack = [
+ 'Error: lookup failed for [email protected]',
+ ' at KeyvMongo.get (/app/node_modules/@keyv/mongo/dist/index.cjs:10:3)',
+ ' at retryWithBackoff (/app/packages/api/dist/index.cjs:5:7)',
+ ].join('\n');
+ const info = runRedactFormat({ level: 'error', message: 'cache read', error });
+ const stack = String((info.error as Error).stack);
+ expect(stack).not.toContain('jane');
+ expect(stack).toContain('(/app/node_modules/@keyv/mongo/dist/index.cjs:10:3)');
+ expect(stack).toContain('(/app/packages/api/dist/index.cjs:5:7)');
+ });
+
it.each(['error', 'warn', 'info', 'debug'])('redacts info.message for %s level', (level) => {
const info = runRedactFormat({ level, message: 'Bearer secretvalue' });
expect(info.message).toBe('Bearer [REDACTED]');
diff --git a/packages/data-schemas/src/config/parsers.ts b/packages/data-schemas/src/config/parsers.ts
index 79c0db762f..e65ba82385 100644
--- a/packages/data-schemas/src/config/parsers.ts
+++ b/packages/data-schemas/src/config/parsers.ts
@@ -61,9 +61,11 @@ const sensitiveKeys: RegExp[] = [
/**
* Matches a whole delimiter-bounded token that holds an address. The lookbehind lets the scan start
* only at a token boundary, so long non-email text is scanned once instead of once per character.
+ * An `@` between path separators (`node_modules/@scope/pkg/...`) names a scoped package in a file
+ * path or stack frame, not an address, so frame locations survive.
*/
const EMAIL_PATTERN =
- /(?:(?<!\\)\\"(?:[^"\\\r\n]|\\{3}"|\\{4}|\\{2}(?!\\)|\\[bfnrt/]|\\u[0-9a-fA-F]{4})*\\"|(?<!\\)"(?:[^"\\\r\n]|\\.)*"|(?<![^\s<>[\],;"])[^\s<>[\],;"]+)@(?:\[[^\]\s@]+\]|[^\s<>[\],;"@]+\.[^\s<>[\],;"@]+)/gu;
+ /(?:(?<!\\)\\"(?:[^"\\\r\n]|\\{3}"|\\{4}|\\{2}(?!\\)|\\[bfnrt/]|\\u[0-9a-fA-F]{4})*\\"|(?<!\\)"(?:[^"\\\r\n]|\\.)*"|(?<![^\s<>[\],;"])[^\s<>[\],;"]+)(?!(?<=[/\\])@[^\s<>[\],;"@/\\]+[/\\])@(?:\[[^\]\s@]+\]|[^\s<>[\],;"@]+\.[^\s<>[\],;"@]+)/gu;
/** Set while redacting splat arguments at levels whose later format strips heavy error fields. */
let skipHeavyErrorFields = false;
/** URL-encoded addresses (`user%40host.tld`), anchored at a token start so scans stay linear. */