Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

📡 feat: Add Backend OpenTelemetry Tracing (#12909)

Danny Avila050b7fd43ae0
diff --git a/.env.example b/.env.example
index 816b4cd14d..e1608ec907 100644
--- a/.env.example
+++ b/.env.example
@@ -110,6 +110,23 @@ NODE_MAX_OLD_SPACE_SIZE=6144
# LANGFUSE_SECRET_KEY=
# LANGFUSE_BASE_URL=
+#=======================#
+# OpenTelemetry Tracing #
+#=======================#
+
+# Enables backend OpenTelemetry tracing. General backend visibility only;
+# use Langfuse for GenAI-specific prompt/model observability.
+# OTEL_TRACING_ENABLED=false
+# OTEL_SERVICE_NAME=librechat
+# OTEL_SERVICE_VERSION=
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
+# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
+# OTEL_EXPORTER_OTLP_HEADERS=
+# OTEL_TRACES_EXPORTER=otlp
+# OTEL_TRACES_SAMPLER=parentbased_always_on
+# OTEL_LOG_LEVEL=INFO
+# OTEL_SDK_DISABLED=false
+
#===================================================#
# Endpoints #
#===================================================#
diff --git a/api/package.json b/api/package.json
index 80c3e5fd9b..8b3ff32a30 100644
--- a/api/package.json
+++ b/api/package.json
@@ -52,6 +52,16 @@
"@microsoft/microsoft-graph-client": "^3.0.7",
"@modelcontextprotocol/sdk": "^1.29.0",
"@node-saml/passport-saml": "^5.1.0",
+ "@opentelemetry/api": "^1.9.0",
+ "@opentelemetry/instrumentation-express": "^0.56.0",
+ "@opentelemetry/instrumentation-http": "^0.207.0",
+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",
+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",
+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",
+ "@opentelemetry/instrumentation-undici": "^0.18.0",
+ "@opentelemetry/resources": "^2.6.1",
+ "@opentelemetry/sdk-node": "^0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.39.0",
"@smithy/node-http-handler": "^4.4.5",
"ai-tokenizer": "^1.0.6",
"axios": "^1.16.0",
diff --git a/api/server/index.js b/api/server/index.js
index a8146fbc1c..9e094724f7 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -1,4 +1,4 @@
-require('dotenv').config();
+const telemetry = require('./telemetry');
const fs = require('fs');
const path = require('path');
require('module-alias')({ base: path.resolve(__dirname, '..') });
@@ -25,22 +25,22 @@ const {
updateInterfacePermissions,
} = require('@librechat/api');
const { connectDb, indexSync } = require('~/db');
-const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
const {
- getRoleByName,
updateAccessPermissions,
- seedDatabase,
sweepOrphanedPreviews,
+ getRoleByName,
+ seedDatabase,
} = require('~/models');
+const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');
const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');
const createValidateImageRequest = require('./middleware/validateImageRequest');
const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');
const { checkMigrations } = require('./services/start/migration');
+const optionalJwtAuth = require('./middleware/optionalJwtAuth');
const initializeMCPs = require('./services/initializeMCPs');
const configureSocialLogins = require('./socialLogins');
const { getAppConfig } = require('./services/Config');
const staticCache = require('./utils/staticCache');
-const optionalJwtAuth = require('./middleware/optionalJwtAuth');
const noIndex = require('./middleware/noIndex');
const routes = require('./routes');
@@ -146,6 +146,10 @@ const startServer = async () => {
app.use(staticCache(appConfig.paths.fonts));
app.use(staticCache(appConfig.paths.assets));
+ if (telemetry.enabled) {
+ app.use(telemetry.telemetryMiddleware);
+ }
+
if (!ALLOW_SOCIAL_LOGIN) {
console.warn('Social logins are disabled. Set ALLOW_SOCIAL_LOGIN=true to enable them.');
}
@@ -227,6 +231,10 @@ const startServer = async () => {
res.send(updatedIndexHtml);
});
+ /** Record trace errors before the final error controller. */
+ if (telemetry.enabled) {
+ app.use(telemetry.telemetryErrorMiddleware);
+ }
/** Error handler (must be last - Express identifies error middleware by its 4-arg signature) */
app.use(ErrorController);
diff --git a/api/server/index.spec.js b/api/server/index.spec.js
index 7b3d062fce..573770e282 100644
--- a/api/server/index.spec.js
+++ b/api/server/index.spec.js
@@ -1,4 +1,5 @@
const fs = require('fs');
+const path = require('path');
const request = require('supertest');
const { MongoMemoryServer } = require('mongodb-memory-server');
const mongoose = require('mongoose');
@@ -32,6 +33,56 @@ jest.mock('~/config', () => ({
}),
}));
+jest.mock(
+ '@librechat/api/telemetry',
+ () => ({
+ initializeTelemetry: jest.fn(() => ({
+ enabled: false,
+ status: 'disabled',
+ shutdown: jest.fn(),
+ })),
+ telemetryMiddleware: jest.fn((_req, _res, next) => next()),
+ telemetryErrorMiddleware: jest.fn((err, _req, _res, next) => next(err)),
+ }),
+ { virtual: true },
+);
+
+describe('Telemetry wiring', () => {
+ const source = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8');
+
+ it('loads telemetry before other server imports', () => {
+ const firstStatement = source
+ .split('\n')
+ .map((line) => line.trim())
+ .find(Boolean);
+
+ expect(firstStatement).toBe("const telemetry = require('./telemetry');");
+ });
+
+ it('mounts telemetry middleware after static assets and before routes', () => {
+ const telemetryMiddlewareIndex = source.indexOf('app.use(telemetry.telemetryMiddleware);');
+ const staticAssetsIndex = source.indexOf('app.use(staticCache(appConfig.paths.assets));');
+ const apiRoutesIndex = source.indexOf("app.use('/api/auth'");
+
+ expect(telemetryMiddlewareIndex).toBeGreaterThan(-1);
+ expect(staticAssetsIndex).toBeGreaterThan(-1);
+ expect(apiRoutesIndex).toBeGreaterThan(-1);
+ expect(staticAssetsIndex).toBeLessThan(telemetryMiddlewareIndex);
+ expect(telemetryMiddlewareIndex).toBeLessThan(apiRoutesIndex);
+ });
+
+ it('mounts telemetry error middleware before ErrorController', () => {
+ const telemetryErrorMiddlewareIndex = source.indexOf(
+ 'app.use(telemetry.telemetryErrorMiddleware);',
+ );
+ const errorControllerIndex = source.indexOf('app.use(ErrorController);');
+
+ expect(telemetryErrorMiddlewareIndex).toBeGreaterThan(-1);
+ expect(errorControllerIndex).toBeGreaterThan(-1);
+ expect(telemetryErrorMiddlewareIndex).toBeLessThan(errorControllerIndex);
+ });
+});
+
describe('Server Configuration', () => {
// Increase the default timeout to allow for Mongo cleanup
jest.setTimeout(30_000);
diff --git a/api/server/telemetry.js b/api/server/telemetry.js
new file mode 100644
index 0000000000..cb0058355b
--- /dev/null
+++ b/api/server/telemetry.js
@@ -0,0 +1,40 @@
+require('dotenv').config();
+
+function isTruthy(value) {
+ return value?.trim().toLowerCase() === 'true';
+}
+
+function isTelemetryEnabled() {
+ return isTruthy(process.env.OTEL_TRACING_ENABLED) && !isTruthy(process.env.OTEL_SDK_DISABLED);
+}
+
+if (isTelemetryEnabled()) {
+ const {
+ initializeTelemetry,
+ telemetryMiddleware,
+ telemetryErrorMiddleware,
+ } = require('@librechat/api/telemetry');
+ const controller = initializeTelemetry();
+
+ module.exports = {
+ get enabled() {
+ return controller.enabled;
+ },
+ get status() {
+ return controller.status;
+ },
+ shutdown: controller.shutdown,
+ telemetryMiddleware,
+ telemetryErrorMiddleware,
+ };
+} else {
+ module.exports = {
+ enabled: false,
+ get status() {
+ return 'disabled';
+ },
+ shutdown: async () => {},
+ telemetryMiddleware: (_req, _res, next) => next(),
+ telemetryErrorMiddleware: (err, _req, _res, next) => next(err),
+ };
+}
diff --git a/api/server/telemetry.spec.js b/api/server/telemetry.spec.js
new file mode 100644
index 0000000000..8f848da8e7
--- /dev/null
+++ b/api/server/telemetry.spec.js
@@ -0,0 +1,91 @@
+describe('telemetry bootstrap', () => {
+ const originalEnv = process.env;
+
+ beforeEach(() => {
+ jest.resetModules();
+ process.env = { ...originalEnv };
+ delete process.env.OTEL_SDK_DISABLED;
+ delete process.env.OTEL_TRACING_ENABLED;
+ jest.doMock('dotenv', () => ({
+ config: jest.fn(),
+ }));
+ });
+
+ afterEach(() => {
+ process.env = originalEnv;
+ jest.dontMock('dotenv');
+ jest.dontMock('@librechat/api/telemetry');
+ jest.resetModules();
+ });
+
+ it('does not load OpenTelemetry packages by default', () => {
+ jest.doMock(
+ '@librechat/api/telemetry',
+ () => {
+ throw new Error('telemetry package should not load when tracing is disabled');
+ },
+ { virtual: true },
+ );
+
+ const telemetry = require('./telemetry');
+
+ expect(telemetry.enabled).toBe(false);
+ expect(telemetry.status).toBe('disabled');
+ });
+
+ it('does not load OpenTelemetry packages when the SDK is disabled', () => {
+ process.env.OTEL_SDK_DISABLED = 'true';
+ process.env.OTEL_TRACING_ENABLED = 'true';
+ jest.doMock(
+ '@librechat/api/telemetry',
+ () => {
+ throw new Error('telemetry package should not load when the SDK is disabled');
+ },
+ { virtual: true },
+ );
+
+ const telemetry = require('./telemetry');
+
+ expect(telemetry.enabled).toBe(false);
+ expect(telemetry.status).toBe('disabled');
+ });
+
+ it('loads and exposes telemetry middleware when tracing is enabled', () => {
+ process.env.OTEL_TRACING_ENABLED = 'true';
+ let enabled = true;
+ let status = 'starting';
+ const telemetryMiddleware = jest.fn();
+ const telemetryErrorMiddleware = jest.fn();
+ const initializeTelemetry = jest.fn(() => ({
+ get enabled() {
+ return enabled;
+ },
+ get status() {
+ return status;
+ },
+ shutdown: jest.fn(),
+ }));
+ jest.doMock(
+ '@librechat/api/telemetry',
+ () => ({
+ initializeTelemetry,
+ telemetryMiddleware,
+ telemetryErrorMiddleware,
+ }),
+ { virtual: true },
+ );
+
+ const telemetry = require('./telemetry');
+
+ expect(initializeTelemetry).toHaveBeenCalledTimes(1);
+ expect(telemetry.enabled).toBe(true);
+ expect(telemetry.status).toBe('starting');
+ expect(telemetry.telemetryMiddleware).toBe(telemetryMiddleware);
+ expect(telemetry.telemetryErrorMiddleware).toBe(telemetryErrorMiddleware);
+
+ enabled = false;
+ status = 'failed';
+ expect(telemetry.enabled).toBe(false);
+ expect(telemetry.status).toBe('failed');
+ });
+});
diff --git a/package-lock.json b/package-lock.json
index dbc9fb994c..c25619187e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -67,6 +67,16 @@
"@microsoft/microsoft-graph-client": "^3.0.7",
"@modelcontextprotocol/sdk": "^1.29.0",
"@node-saml/passport-saml": "^5.1.0",
+ "@opentelemetry/api": "^1.9.0",
+ "@opentelemetry/instrumentation-express": "^0.56.0",
+ "@opentelemetry/instrumentation-http": "^0.207.0",
+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",
+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",
+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",
+ "@opentelemetry/instrumentation-undici": "^0.18.0",
+ "@opentelemetry/resources": "^2.6.1",
+ "@opentelemetry/sdk-node": "^0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.39.0",
"@smithy/node-http-handler": "^4.4.5",
"ai-tokenizer": "^1.0.6",
"axios": "^1.16.0",
@@ -12866,7 +12876,6 @@
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.1.tgz",
"integrity": "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g==",
"license": "Apache-2.0",
- "peer": true,
"dependencies": {
"@opentelemetry/semantic-conventions": "^1.29.0"
},
@@ -14245,6 +14254,120 @@
"@opentelemetry/api": "^1.3.0"
}
},
+ "node_modules/@opentelemetry/instrumentation-express": {
+ "version": "0.56.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.56.0.tgz",
+ "integrity": "sha512-rMV0WUTtAGEhHrHl3uDRIO97EkNUp4ewrW2iRVuP7kaV5qRT2b1pPV5PE75oR3GyLLSTooSAzGWl6CTm8eftKQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/core": "^2.0.0",
+ "@opentelemetry/instrumentation": "^0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.27.0"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.3.0"
+ }
+ },
+ "node_modules/@opentelemetry/instrumentation-http": {
+ "version": "0.207.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.207.0.tgz",
+ "integrity": "sha512-FC4i5hVixTzuhg4SV2ycTEAYx+0E2hm+GwbdoVPSA6kna0pPVI4etzaA9UkpJ9ussumQheFXP6rkGIaFJjMxsw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/core": "2.2.0",
+ "@opentelemetry/instrumentation": "0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.29.0",
+ "forwarded-parse": "2.1.2"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.3.0"
+ }
+ },
+ "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz",
+ "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/semantic-conventions": "^1.29.0"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": ">=1.0.0 <1.10.0"
+ }
+ },
+ "node_modules/@opentelemetry/instrumentation-ioredis": {
+ "version": "0.55.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.55.0.tgz",
+ "integrity": "sha512-ASuBMzh0ImmfOnWj9vCPtBMqSjr54/r/HluUIylwZB7xzTU6gL2SfybxySJMzEL9+386gJJVApwQktVznAtrWA==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/instrumentation": "^0.207.0",
+ "@opentelemetry/redis-common": "^0.38.2"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.3.0"
+ }
+ },
+ "node_modules/@opentelemetry/instrumentation-mongodb": {
+ "version": "0.60.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.60.0.tgz",
+ "integrity": "sha512-8mKW2oyyWdYOKYpu70AGGAvLnExGlOoFT+Ylk9hJvWYHR5f6IrmVqwUMlzPM2WC4ihQ9crvqrtldMPmlVezTqg==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/instrumentation": "^0.207.0"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.3.0"
+ }
+ },
+ "node_modules/@opentelemetry/instrumentation-mongoose": {
+ "version": "0.54.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.54.0.tgz",
+ "integrity": "sha512-R2JzMrxiz3R9m+cO6iVC85bma3E3vdBr06F+D4zGcEkJS9FaCRw6+Hdb+aQ1AaDRwEMTaw+F28+6VdMpniBOXQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/core": "^2.0.0",
+ "@opentelemetry/instrumentation": "^0.207.0"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.3.0"
+ }
+ },
+ "node_modules/@opentelemetry/instrumentation-undici": {
+ "version": "0.18.0",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.18.0.tgz",
+ "integrity": "sha512-NalxLuZV621Xq4IhQkC+OXoZjAT8Xf6vYRdTjHitOXMU+4l/peRY05V7wGr4d7huf+vjyQry0XKlyhsEr4ouNw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@opentelemetry/core": "^2.0.0",
+ "@opentelemetry/instrumentation": "^0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.24.0"
+ },
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ },
+ "peerDependencies": {
+ "@opentelemetry/api": "^1.7.0"
+ }
+ },
"node_modules/@opentelemetry/instrumentation/node_modules/@opentelemetry/api-logs": {
"version": "0.207.0",
"resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.207.0.tgz",
@@ -14504,12 +14627,20 @@
"@opentelemetry/api": ">=1.0.0 <1.10.0"
}
},
+ "node_modules/@opentelemetry/redis-common": {
+ "version": "0.38.3",
+ "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz",
+ "integrity": "sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": "^18.19.0 || >=20.6.0"
+ }
+ },
"node_modules/@opentelemetry/resources": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.6.1.tgz",
"integrity": "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA==",
"license": "Apache-2.0",
- "peer": true,
"dependencies": {
"@opentelemetry/core": "2.6.1",
"@opentelemetry/semantic-conventions": "^1.29.0"
@@ -28134,6 +28265,12 @@
"node": ">= 0.6"
}
},
+ "node_modules/forwarded-parse": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz",
+ "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==",
+ "license": "MIT"
+ },
"node_modules/fraction.js": {
"version": "5.3.4",
"resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz",
@@ -28286,6 +28423,67 @@
"node": ">=14"
}
},
+ "node_modules/gcp-metadata": {
+ "version": "5.3.0",
+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz",
+ "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==",
+ "license": "Apache-2.0",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "gaxios": "^5.0.0",
+ "json-bigint": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/gcp-metadata/node_modules/agent-base": {
+ "version": "6.0.2",
+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+ "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+ "license": "MIT",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6.0.0"
+ }
+ },
+ "node_modules/gcp-metadata/node_modules/gaxios": {
+ "version": "5.1.3",
+ "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz",
+ "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==",
+ "license": "Apache-2.0",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "extend": "^3.0.2",
+ "https-proxy-agent": "^5.0.0",
+ "is-stream": "^2.0.0",
+ "node-fetch": "^2.6.9"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/gcp-metadata/node_modules/https-proxy-agent": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+ "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+ "license": "MIT",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "agent-base": "6",
+ "debug": "4"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/generic-names": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/generic-names/-/generic-names-4.0.0.tgz",
@@ -28585,6 +28783,7 @@
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz",
"integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==",
+ "license": "Apache-2.0",
"dependencies": {
"gaxios": "^6.1.1",
"google-logging-utils": "^0.0.2",
@@ -28598,6 +28797,7 @@
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz",
"integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==",
+ "license": "Apache-2.0",
"engines": {
"node": ">=14"
}
@@ -34726,6 +34926,52 @@
"node": ">=20.19.0"
}
},
+ "node_modules/mongodb-memory-server-core/node_modules/gaxios": {
+ "version": "7.1.4",
+ "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz",
+ "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "extend": "^3.0.2",
+ "https-proxy-agent": "^7.0.1",
+ "node-fetch": "^3.3.2"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/mongodb-memory-server-core/node_modules/gcp-metadata": {
+ "version": "7.0.1",
+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz",
+ "integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "gaxios": "^7.0.0",
+ "google-logging-utils": "^1.0.0",
+ "json-bigint": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/mongodb-memory-server-core/node_modules/google-logging-utils": {
+ "version": "1.1.3",
+ "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz",
+ "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "optional": true,
+ "peer": true,
+ "engines": {
+ "node": ">=14"
+ }
+ },
"node_modules/mongodb-memory-server-core/node_modules/mongodb": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.1.1.tgz",
@@ -34787,6 +35033,27 @@
"node": ">=20.19.0"
}
},
+ "node_modules/mongodb-memory-server-core/node_modules/node-fetch": {
+ "version": "3.3.2",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
+ "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "peer": true,
+ "dependencies": {
+ "data-uri-to-buffer": "^4.0.0",
+ "fetch-blob": "^3.1.4",
+ "formdata-polyfill": "^4.0.10"
+ },
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/node-fetch"
+ }
+ },
"node_modules/mongoose": {
"version": "8.23.1",
"resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.23.1.tgz",
@@ -44692,6 +44959,16 @@
"@librechat/agents": "^3.1.86",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.29.0",
+ "@opentelemetry/api": "^1.9.0",
+ "@opentelemetry/instrumentation-express": "^0.56.0",
+ "@opentelemetry/instrumentation-http": "^0.207.0",
+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",
+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",
+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",
+ "@opentelemetry/instrumentation-undici": "^0.18.0",
+ "@opentelemetry/resources": "^2.6.1",
+ "@opentelemetry/sdk-node": "^0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.39.0",
"@smithy/node-http-handler": "^4.4.5",
"ai-tokenizer": "^1.0.6",
"axios": "^1.16.0",
diff --git a/packages/api/package.json b/packages/api/package.json
index 1168c21bc6..048df30ba1 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -10,6 +10,10 @@
".": {
"require": "./dist/index.js",
"types": "./dist/types/index.d.ts"
+ },
+ "./telemetry": {
+ "require": "./dist/telemetry.js",
+ "types": "./dist/types/telemetry/index.d.ts"
}
},
"scripts": {
@@ -103,6 +107,16 @@
"@librechat/agents": "^3.1.86",
"@librechat/data-schemas": "*",
"@modelcontextprotocol/sdk": "^1.29.0",
+ "@opentelemetry/api": "^1.9.0",
+ "@opentelemetry/instrumentation-express": "^0.56.0",
+ "@opentelemetry/instrumentation-http": "^0.207.0",
+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",
+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",
+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",
+ "@opentelemetry/instrumentation-undici": "^0.18.0",
+ "@opentelemetry/resources": "^2.6.1",
+ "@opentelemetry/sdk-node": "^0.207.0",
+ "@opentelemetry/semantic-conventions": "^1.39.0",
"@smithy/node-http-handler": "^4.4.5",
"ai-tokenizer": "^1.0.6",
"axios": "^1.16.0",
diff --git a/packages/api/rollup.config.js b/packages/api/rollup.config.js
index 9a9de35e8f..becc482291 100644
--- a/packages/api/rollup.config.js
+++ b/packages/api/rollup.config.js
@@ -45,7 +45,10 @@ const plugins = [
];
const cjsBuild = {
- input: 'src/index.ts',
+ input: {
+ index: 'src/index.ts',
+ telemetry: 'src/telemetry/index.ts',
+ },
output: {
dir: 'dist',
format: 'cjs',
diff --git a/packages/api/src/telemetry/config.spec.ts b/packages/api/src/telemetry/config.spec.ts
new file mode 100644
index 0000000000..128f07a5a3
--- /dev/null
+++ b/packages/api/src/telemetry/config.spec.ts
@@ -0,0 +1,46 @@
+import { getTelemetryConfig } from './config';
+
+describe('getTelemetryConfig', () => {
+ it('defaults tracing off', () => {
+ const config = getTelemetryConfig({});
+
+ expect(config.enabled).toBe(false);
+ expect(config.sdkDisabled).toBe(false);
+ expect(config.serviceName).toBe('librechat');
+ expect(config.healthPath).toBe('/health');
+ });
+
+ it('enables tracing only when OTEL_TRACING_ENABLED is true', () => {
+ expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'true' }).enabled).toBe(true);
+ expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'TRUE' }).enabled).toBe(true);
+ expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'false' }).enabled).toBe(false);
+ });
+
+ it('lets OTEL_SDK_DISABLED override tracing enablement', () => {
+ const config = getTelemetryConfig({
+ OTEL_SDK_DISABLED: 'true',
+ OTEL_TRACING_ENABLED: 'true',
+ });
+
+ expect(config.enabled).toBe(false);
+ expect(config.sdkDisabled).toBe(true);
+ });
+
+ it('uses standard service env vars when provided', () => {
+ const config = getTelemetryConfig({
+ OTEL_SERVICE_NAME: ' librechat-api ',
+ OTEL_SERVICE_VERSION: ' 1.2.3 ',
+ });
+
+ expect(config.serviceName).toBe('librechat-api');
+ expect(config.serviceVersion).toBe('1.2.3');
+ });
+
+ it('falls back to npm package version when service version is absent', () => {
+ const config = getTelemetryConfig({
+ npm_package_version: '0.8.5',
+ });
+
+ expect(config.serviceVersion).toBe('0.8.5');
+ });
+});
diff --git a/packages/api/src/telemetry/config.ts b/packages/api/src/telemetry/config.ts
new file mode 100644
index 0000000000..340197aa65
--- /dev/null
+++ b/packages/api/src/telemetry/config.ts
@@ -0,0 +1,43 @@
+const DEFAULT_SERVICE_NAME = 'librechat';
+export const DEFAULT_HEALTH_PATH = '/health';
+
+export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | 'stopped';
+
+export interface TelemetryConfig {
+ enabled: boolean;
+ healthPath: string;
+ sdkDisabled: boolean;
+ serviceName: string;
+ serviceVersion?: string;
+}
+
+function isTruthy(value?: string | boolean | null): boolean {
+ if (typeof value === 'boolean') {
+ return value;
+ }
+ if (typeof value === 'string') {
+ return value.trim().toLowerCase() === 'true';
+ }
+ return false;
+}
+
+function normalizeEnvValue(value?: string): string | undefined {
+ const trimmed = value?.trim();
+ return trimmed ? trimmed : undefined;
+}
+
+export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): TelemetryConfig {
+ const sdkDisabled = isTruthy(env.OTEL_SDK_DISABLED);
+ const enabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled;
+ const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;
+ const serviceVersion =
+ normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);
+
+ return {
+ enabled,
+ serviceName,
+ sdkDisabled,
+ serviceVersion,
+ healthPath: DEFAULT_HEALTH_PATH,
+ };
+}
diff --git a/packages/api/src/telemetry/index.ts b/packages/api/src/telemetry/index.ts
new file mode 100644
index 0000000000..2eb188bf84
--- /dev/null
+++ b/packages/api/src/telemetry/index.ts
@@ -0,0 +1,5 @@
+export { getTelemetryConfig } from './config';
+export { initializeTelemetry, shutdownTelemetry } from './sdk';
+export { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
+export type { TelemetryConfig, TelemetryStatus } from './config';
+export type { TelemetryController } from './sdk';
diff --git a/packages/api/src/telemetry/middleware.spec.ts b/packages/api/src/telemetry/middleware.spec.ts
new file mode 100644
index 0000000000..271d8c0a11
--- /dev/null
+++ b/packages/api/src/telemetry/middleware.spec.ts
@@ -0,0 +1,395 @@
+import { EventEmitter } from 'node:events';
+import { SpanStatusCode, trace } from '@opentelemetry/api';
+import type { NextFunction, Response } from 'express';
+import type { Span } from '@opentelemetry/api';
+import type { ServerRequest } from '~/types';
+import { getTelemetryRequestSpan } from './sdk';
+import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';
+
+jest.mock('./sdk', () => ({
+ getTelemetryRequestSpan: jest.fn(),
+}));
+
+const mockGetTelemetryRequestSpan = getTelemetryRequestSpan as jest.MockedFunction<
+ typeof getTelemetryRequestSpan
+>;
+
+interface MockResponse extends EventEmitter {
+ statusCode: number;
+ writableEnded: boolean;
+}
+
+function createSpan(): jest.Mocked<Span> {
+ const span = {} as jest.Mocked<Span>;
+ span.addEvent = jest.fn<jest.Mocked<Span>, Parameters<Span['addEvent']>>(() => span);
+ span.addLink = jest.fn<jest.Mocked<Span>, Parameters<Span['addLink']>>(() => span);
+ span.addLinks = jest.fn<jest.Mocked<Span>, Parameters<Span['addLinks']>>(() => span);
+ span.end = jest.fn<void, Parameters<Span['end']>>();
+ span.isRecording = jest.fn<boolean, Parameters<Span['isRecording']>>(() => true);
+ span.recordException = jest.fn<void, Parameters<Span['recordException']>>();
+ span.setAttribute = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttribute']>>(() => span);
+ span.setAttributes = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttributes']>>(() => span);
+ span.setStatus = jest.fn<jest.Mocked<Span>, Parameters<Span['setStatus']>>(() => span);
+ span.spanContext = jest.fn<ReturnType<Span['spanContext']>, Parameters<Span['spanContext']>>(
+ () => ({
+ spanId: '0000000000000000',
+ traceFlags: 0,
+ traceId: '00000000000000000000000000000000',
+ }),
+ );
+ span.updateName = jest.fn<jest.Mocked<Span>, Parameters<Span['updateName']>>(() => span);
+ return span;
+}
+
+function createResponse(statusCode = 200): MockResponse {
+ const res = new EventEmitter() as MockResponse;
+ res.statusCode = statusCode;
+ res.writableEnded = false;
+ return res;
+}
+
+function createRequest(overrides: Partial<ServerRequest> = {}): ServerRequest {
+ return {
+ baseUrl: '/api/messages',
+ body: {
+ prompt: 'do not capture this prompt',
+ text: 'do not capture this body',
+ },
+ headers: {
+ authorization: 'Bearer do-not-capture-this-auth-header',
+ cookie: 'session=do-not-capture-this-cookie',
+ 'x-api-key': 'do-not-capture-this-api-key',
+ },
+ method: 'POST',
+ path: '/api/messages/conversation-1',
+ route: { path: '/:conversationId' },
+ user: {
+ email: '[email protected]',
+ id: 'user-1',
+ tenantId: 'tenant-1',
+ } as ServerRequest['user'],
+ ...overrides,
+ } as ServerRequest;
+}
+
+afterEach(() => {
+ mockGetTelemetryRequestSpan.mockReset();
+});
+
+describe('telemetryMiddleware', () => {
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it('passes through without an active span', () => {
+ const next = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);
+
+ telemetryMiddleware(createRequest(), createResponse() as Response, next);
+
+ expect(next).toHaveBeenCalledTimes(1);
+ });
+
+ it('uses the stored request span for deferred completion attributes', () => {
+ const activeSpan = createSpan();
+ const requestSpan = createSpan();
+ const req = createRequest();
+ const res = createResponse(202);
+ const next: NextFunction = jest.fn();
+ mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);
+
+ telemetryMiddleware(req, res as Response, next);
+ res.emit('finish');
+
+ expect(next).toHaveBeenCalledTimes(1);
+ expect(trace.getActiveSpan).not.toHaveBeenCalled();
+ expect(activeSpan.setAttributes).not.toHaveBeenCalled();
+ expect(requestSpan.setAttributes).toHaveBeenCalledWith({
+ 'http.request.method': 'POST',
+ });
+ expect(requestSpan.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'http.response.status_code': 202,
+ 'http.route': '/api/messages/:conversationId',
+ }),
+ );
+ });
+
+ it('records safe route and identity attributes without body content', () => {
+ const span = createSpan();
+ const req = createRequest();
+ const res = createResponse(201);
+ const next: NextFunction = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(req, res as Response, next);
+ res.emit('finish');
+
+ expect(next).toHaveBeenCalledTimes(1);
+ expect(span.setAttributes).toHaveBeenCalledWith({
+ 'http.request.method': 'POST',
+ });
+ expect(span.setAttributes).toHaveBeenCalledWith({
+ 'enduser.id': 'user-1',
+ 'librechat.tenant.id': 'tenant-1',
+ });
+ expect(span.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'http.response.status_code': 201,
+ 'http.route': '/api/messages/:conversationId',
+ }),
+ );
+
+ const capturedAttributes = JSON.stringify(span.setAttributes.mock.calls);
+ expect(capturedAttributes).not.toContain('do not capture this prompt');
+ expect(capturedAttributes).not.toContain('do not capture this body');
+ expect(capturedAttributes).not.toContain('[email protected]');
+ expect(capturedAttributes).not.toContain('conversation-1');
+ expect(capturedAttributes).not.toContain('do-not-capture-this-auth-header');
+ expect(capturedAttributes).not.toContain('do-not-capture-this-cookie');
+ expect(capturedAttributes).not.toContain('do-not-capture-this-api-key');
+ });
+
+ it('records identity attributes populated by downstream middleware', () => {
+ const span = createSpan();
+ const req = createRequest({
+ headers: {},
+ user: undefined,
+ });
+ const res = createResponse(200);
+ const next: NextFunction = jest.fn(() => {
+ req.user = {
+ id: 'late-user',
+ tenantId: 'late-tenant',
+ } as ServerRequest['user'];
+ });
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(req, res as Response, next);
+
+ expect(next).toHaveBeenCalledTimes(1);
+ expect(span.setAttributes).toHaveBeenCalledTimes(1);
+
+ res.emit('finish');
+
+ expect(span.setAttributes).toHaveBeenCalledWith({
+ 'enduser.id': 'late-user',
+ 'librechat.tenant.id': 'late-tenant',
+ });
+ });
+
+ it('does not derive tenant identity from request headers', () => {
+ const span = createSpan();
+ const req = createRequest({
+ headers: {
+ 'x-tenant-id': 'spoofed-tenant',
+ },
+ user: undefined,
+ });
+ const res = createResponse(200);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(req, res as Response, jest.fn());
+ res.emit('finish');
+
+ expect(span.setAttributes).not.toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'librechat.tenant.id': 'spoofed-tenant',
+ }),
+ );
+ expect(JSON.stringify(span.setAttributes.mock.calls)).not.toContain('spoofed-tenant');
+ });
+
+ it('ignores health checks', () => {
+ const span = createSpan();
+ const next = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(
+ createRequest({
+ baseUrl: '',
+ path: '/health',
+ route: undefined,
+ }),
+ createResponse() as Response,
+ next,
+ );
+
+ expect(next).toHaveBeenCalledTimes(1);
+ expect(span.setAttributes).not.toHaveBeenCalled();
+ });
+
+ it('uses a low-cardinality fallback for unmatched API routes', () => {
+ const span = createSpan();
+ const req = createRequest({
+ baseUrl: '',
+ path: '/api/nonexistent/123',
+ route: undefined,
+ });
+ const res = createResponse(404);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(req, res as Response, jest.fn());
+ res.emit('finish');
+
+ expect(span.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'http.route': '/api/*',
+ 'http.response.status_code': 404,
+ }),
+ );
+ });
+
+ it('uses a low-cardinality fallback for unmatched SPA routes', () => {
+ const span = createSpan();
+ const req = createRequest({
+ baseUrl: '',
+ path: '/chat/conversation-id',
+ route: undefined,
+ });
+ const res = createResponse(200);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(req, res as Response, jest.fn());
+ res.emit('finish');
+
+ expect(span.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'http.route': 'spa_fallback',
+ 'http.response.status_code': 200,
+ }),
+ );
+ });
+
+ it('marks server responses as errored', () => {
+ const span = createSpan();
+ const req = createRequest();
+ const res = createResponse(500);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(req, res as Response, jest.fn());
+ res.emit('finish');
+
+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+ });
+
+ it('records completion attributes only once when finish and close both fire', () => {
+ const span = createSpan();
+ const res = createResponse(200);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(createRequest(), res as Response, jest.fn());
+ res.emit('finish');
+ res.emit('close');
+
+ expect(span.setAttributes).toHaveBeenCalledTimes(3);
+ });
+
+ it('marks client disconnects before finish as aborted errors', () => {
+ const span = createSpan();
+ const res = createResponse(200);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryMiddleware(createRequest(), res as Response, jest.fn());
+ res.emit('close');
+
+ expect(span.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'http.response.status_code': 499,
+ 'librechat.request.aborted': true,
+ }),
+ );
+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+ });
+});
+
+describe('telemetryErrorMiddleware', () => {
+ afterEach(() => {
+ jest.restoreAllMocks();
+ });
+
+ it('records exceptions and forwards the error', () => {
+ const span = createSpan();
+ const error = new TypeError('boom');
+ const next = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
+
+ expect(span.recordException).toHaveBeenCalledWith(error);
+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+ expect(span.setAttributes).toHaveBeenCalledWith({
+ 'enduser.id': 'user-1',
+ 'librechat.tenant.id': 'tenant-1',
+ });
+ expect(span.setAttributes).toHaveBeenCalledWith({
+ 'error.type': 'TypeError',
+ 'http.route': '/api/messages/:conversationId',
+ });
+ expect(next).toHaveBeenCalledWith(error);
+ });
+
+ it('records exceptions on the stored request span when available', () => {
+ const activeSpan = createSpan();
+ const requestSpan = createSpan();
+ const error = new TypeError('boom');
+ const next = jest.fn();
+ mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);
+
+ telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
+
+ expect(trace.getActiveSpan).not.toHaveBeenCalled();
+ expect(activeSpan.recordException).not.toHaveBeenCalled();
+ expect(requestSpan.recordException).toHaveBeenCalledWith(error);
+ expect(requestSpan.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+ expect(next).toHaveBeenCalledWith(error);
+ });
+
+ it('handles non-Error values without throwing', () => {
+ const span = createSpan();
+ const next = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryErrorMiddleware('boom', createRequest(), createResponse() as Response, next);
+
+ expect(span.recordException).toHaveBeenCalledWith('boom');
+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+ expect(span.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'error.type': 'string',
+ 'http.route': '/api/messages/:conversationId',
+ }),
+ );
+ expect(next).toHaveBeenCalledWith('boom');
+ });
+
+ it('handles null error values without throwing', () => {
+ const span = createSpan();
+ const next = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);
+
+ telemetryErrorMiddleware(null, createRequest(), createResponse() as Response, next);
+
+ expect(span.recordException).not.toHaveBeenCalled();
+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
+ expect(span.setAttributes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ 'error.type': 'null',
+ 'http.route': '/api/messages/:conversationId',
+ }),
+ );
+ expect(next).toHaveBeenCalledWith(null);
+ });
+
+ it('forwards the error without an active span', () => {
+ const error = new Error('boom');
+ const next = jest.fn();
+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);
+
+ telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);
+
+ expect(next).toHaveBeenCalledWith(error);
+ });
+});
diff --git a/packages/api/src/telemetry/middleware.ts b/packages/api/src/telemetry/middleware.ts
new file mode 100644
index 0000000000..d9c110c1b9
--- /dev/null
+++ b/packages/api/src/telemetry/middleware.ts
@@ -0,0 +1,171 @@
+import { SpanStatusCode, trace } from '@opentelemetry/api';
+import type { Span, Attributes } from '@opentelemetry/api';
+import type { NextFunction, Response } from 'express';
+import type { ServerRequest } from '~/types';
+import { getTelemetryRequestSpan } from './sdk';
+import { DEFAULT_HEALTH_PATH } from './config';
+
+const CLIENT_CLOSED_REQUEST_STATUS_CODE = 499;
+
+type ExpressErrorValue =
+ | Error
+ | string
+ | number
+ | boolean
+ | bigint
+ | symbol
+ | object
+ | null
+ | undefined;
+
+function getUserId(req: ServerRequest): string | undefined {
+ return req.user?.id;
+}
+
+function getTenantId(req: ServerRequest): string | undefined {
+ return req.user?.tenantId;
+}
+
+function isHealthPath(req: ServerRequest): boolean {
+ return req.path === DEFAULT_HEALTH_PATH;
+}
+
+function isApiPath(req: ServerRequest): boolean {
+ return req.path === '/api' || req.path.startsWith('/api/');
+}
+
+function getRoutePath(req: ServerRequest): string {
+ const routePath = req.route?.path;
+ if (typeof routePath === 'string') {
+ return `${req.baseUrl}${routePath}`;
+ }
+
+ if (isHealthPath(req)) {
+ return '/health';
+ }
+
+ if (isApiPath(req)) {
+ return '/api/*';
+ }
+
+ return 'spa_fallback';
+}
+
+function setIdentityAttributes(span: Span, req: ServerRequest): void {
+ const userId = getUserId(req);
+ const tenantId = getTenantId(req);
+
+ if (!userId && !tenantId) {
+ return;
+ }
+
+ const attributes: Attributes = {};
+
+ if (userId) {
+ attributes['enduser.id'] = userId;
+ }
+
+ if (tenantId) {
+ attributes['librechat.tenant.id'] = tenantId;
+ }
+
+ span.setAttributes(attributes);
+}
+
+function setCompletionAttributes(
+ span: Span,
+ req: ServerRequest,
+ res: Response,
+ aborted = false,
+): void {
+ const statusCode = aborted ? CLIENT_CLOSED_REQUEST_STATUS_CODE : res.statusCode;
+ const routePath = getRoutePath(req);
+ const attributes: Attributes = {
+ 'http.route': routePath,
+ 'http.response.status_code': statusCode,
+ };
+
+ if (aborted) {
+ attributes['librechat.request.aborted'] = true;
+ }
+
+ setIdentityAttributes(span, req);
+ span.setAttributes(attributes);
+
+ if (aborted || statusCode >= 500) {
+ span.setStatus({ code: SpanStatusCode.ERROR });
+ }
+}
+
+export function telemetryMiddleware(req: ServerRequest, res: Response, next: NextFunction): void {
+ if (isHealthPath(req)) {
+ next();
+ return;
+ }
+
+ const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();
+ if (!span) {
+ next();
+ return;
+ }
+
+ span.setAttributes({
+ 'http.request.method': req.method,
+ });
+
+ let completed = false;
+ const complete = () => {
+ if (completed) {
+ return;
+ }
+ completed = true;
+ setCompletionAttributes(span, req, res);
+ };
+
+ const close = () => {
+ if (completed) {
+ return;
+ }
+ completed = true;
+ setCompletionAttributes(span, req, res, !res.writableEnded);
+ };
+
+ res.once('finish', complete);
+ res.once('close', close);
+ next();
+}
+
+export function telemetryErrorMiddleware(
+ err: ExpressErrorValue,
+ req: ServerRequest,
+ _res: Response,
+ next: NextFunction,
+): void {
+ const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();
+ if (span) {
+ const routePath = getRoutePath(req);
+ if (err) {
+ span.recordException(err instanceof Error ? err : String(err));
+ }
+ span.setStatus({ code: SpanStatusCode.ERROR });
+ setIdentityAttributes(span, req);
+ span.setAttributes({
+ 'error.type': getErrorType(err),
+ 'http.route': routePath,
+ });
+ }
+
+ next(err);
+}
+
+function getErrorType(err: ExpressErrorValue): string {
+ if (err instanceof Error) {
+ return err.name || err.constructor.name;
+ }
+
+ if (err === null) {
+ return 'null';
+ }
+
+ return typeof err;
+}
diff --git a/packages/api/src/telemetry/sdk.spec.ts b/packages/api/src/telemetry/sdk.spec.ts
new file mode 100644
index 0000000000..548ecc6f2a
--- /dev/null
+++ b/packages/api/src/telemetry/sdk.spec.ts
@@ -0,0 +1,357 @@
+import { Socket } from 'node:net';
+import { IncomingMessage } from 'node:http';
+import type { Span } from '@opentelemetry/api';
+
+interface HttpInstrumentationOptions {
+ requestHook?: (span: Span, request: object) => void;
+ startIncomingSpanHook?: (request: IncomingMessage) => Record<string, string>;
+}
+
+const mockStart = jest.fn();
+const mockShutdown = jest.fn();
+const mockNodeSDK = jest.fn(() => ({
+ start: mockStart,
+ shutdown: mockShutdown,
+}));
+const mockExpressInstrumentation = jest.fn(() => ({ name: 'express' }));
+const mockHttpInstrumentation = jest.fn((options?: HttpInstrumentationOptions) => ({
+ name: 'http',
+ options,
+}));
+const mockIORedisInstrumentation = jest.fn(() => ({ name: 'ioredis' }));
+const mockMongoDBInstrumentation = jest.fn(() => ({ name: 'mongodb' }));
+const mockMongooseInstrumentation = jest.fn(() => ({ name: 'mongoose' }));
+const mockUndiciInstrumentation = jest.fn(() => ({ name: 'undici' }));
+const mockResourceFromAttributes = jest.fn((attributes: object) => ({ attributes }));
+
+jest.mock(
+ '@opentelemetry/sdk-node',
+ () => ({
+ NodeSDK: mockNodeSDK,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/instrumentation-express',
+ () => ({
+ ExpressInstrumentation: mockExpressInstrumentation,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/instrumentation-http',
+ () => ({
+ HttpInstrumentation: mockHttpInstrumentation,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/instrumentation-ioredis',
+ () => ({
+ IORedisInstrumentation: mockIORedisInstrumentation,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/instrumentation-mongodb',
+ () => ({
+ MongoDBInstrumentation: mockMongoDBInstrumentation,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/instrumentation-mongoose',
+ () => ({
+ MongooseInstrumentation: mockMongooseInstrumentation,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/instrumentation-undici',
+ () => ({
+ UndiciInstrumentation: mockUndiciInstrumentation,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/resources',
+ () => ({
+ resourceFromAttributes: mockResourceFromAttributes,
+ }),
+ { virtual: true },
+);
+
+jest.mock(
+ '@opentelemetry/semantic-conventions',
+ () => ({
+ ATTR_SERVICE_NAME: 'service.name',
+ ATTR_SERVICE_VERSION: 'service.version',
+ }),
+ { virtual: true },
+);
+
+async function flushSignalShutdown(): Promise<void> {
+ await new Promise((resolve) => setImmediate(resolve));
+}
+
+describe('telemetry SDK lifecycle', () => {
+ let emitWarningSpy: jest.SpyInstance;
+ let getTelemetryRequestSpan: (typeof import('./sdk'))['getTelemetryRequestSpan'];
+ let initializeTelemetry: (typeof import('./sdk'))['initializeTelemetry'];
+ let resetTelemetryForTests: (typeof import('./sdk'))['resetTelemetryForTests'];
+ let shutdownTelemetry: (typeof import('./sdk'))['shutdownTelemetry'];
+
+ beforeEach(async () => {
+ jest.clearAllMocks();
+ ({ getTelemetryRequestSpan, initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } =
+ await import('./sdk'));
+ await resetTelemetryForTests();
+ Reflect.deleteProperty(globalThis, 'Bun');
+ emitWarningSpy = jest.spyOn(process, 'emitWarning').mockImplementation(() => true);
+ });
+
+ afterEach(async () => {
+ await resetTelemetryForTests();
+ emitWarningSpy.mockRestore();
+ Reflect.deleteProperty(globalThis, 'Bun');
+ });
+
+ it('does not initialize when tracing is disabled by default', () => {
+ const controller = initializeTelemetry({});
+
+ expect(controller.enabled).toBe(false);
+ expect(controller.status).toBe('disabled');
+ expect(mockNodeSDK).not.toHaveBeenCalled();
+ });
+
+ it('does not initialize when OTEL_SDK_DISABLED is true', () => {
+ const controller = initializeTelemetry({
+ OTEL_SDK_DISABLED: 'true',
+ OTEL_TRACING_ENABLED: 'true',
+ });
+
+ expect(controller.enabled).toBe(false);
+ expect(controller.status).toBe('disabled');
+ expect(mockNodeSDK).not.toHaveBeenCalled();
+ });
+
+ it('does not initialize under Bun runtime', () => {
+ Object.defineProperty(globalThis, 'Bun', {
+ configurable: true,
+ value: {},
+ });
+
+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ expect(controller.enabled).toBe(false);
+ expect(controller.status).toBe('disabled');
+ expect(mockNodeSDK).not.toHaveBeenCalled();
+ });
+
+ it('starts the Node SDK once when enabled', () => {
+ const first = initializeTelemetry({
+ OTEL_SERVICE_NAME: 'librechat-test',
+ OTEL_TRACING_ENABLED: 'true',
+ });
+ const second = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ expect(first.enabled).toBe(true);
+ expect(first.status).toBe('started');
+ expect(second.enabled).toBe(true);
+ expect(mockNodeSDK).toHaveBeenCalledTimes(1);
+ expect(mockStart).toHaveBeenCalledTimes(1);
+ expect(mockResourceFromAttributes).toHaveBeenCalledWith({
+ 'service.name': 'librechat-test',
+ });
+ expect(mockHttpInstrumentation).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headersToSpanAttributes: {
+ client: { requestHeaders: [], responseHeaders: [] },
+ server: { requestHeaders: [], responseHeaders: [] },
+ },
+ }),
+ );
+ expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1);
+ expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1);
+ expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1);
+ expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1);
+ expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1);
+ });
+
+ it('tracks HTTP server request spans for completion updates', () => {
+ const span = {} as Span;
+ const request = new IncomingMessage(new Socket());
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+ const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
+ const requestHook = instrumentationOptions?.requestHook;
+
+ if (!requestHook) {
+ throw new Error('HTTP instrumentation requestHook was not configured');
+ }
+
+ requestHook(span, request);
+
+ expect(getTelemetryRequestSpan(request)).toBe(span);
+ });
+
+ it('redacts incoming URL attributes before HTTP spans are exported', () => {
+ const request = new IncomingMessage(new Socket());
+ request.url = '/oauth/callback?code=secret-code&state=secret-state';
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+ const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];
+ const startIncomingSpanHook = instrumentationOptions?.startIncomingSpanHook;
+
+ if (!startIncomingSpanHook) {
+ throw new Error('HTTP instrumentation startIncomingSpanHook was not configured');
+ }
+
+ const attributes = startIncomingSpanHook(request);
+
+ expect(attributes).toEqual({
+ 'http.target': 'spa_fallback?[REDACTED]',
+ 'http.url': 'spa_fallback?[REDACTED]',
+ 'url.full': 'spa_fallback?[REDACTED]',
+ 'url.path': 'spa_fallback',
+ 'url.query': '[REDACTED]',
+ });
+ expect(JSON.stringify(attributes)).not.toContain('secret-code');
+ expect(JSON.stringify(attributes)).not.toContain('secret-state');
+ });
+
+ it('reflects lifecycle status from the controller getter', async () => {
+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ expect(controller.status).toBe('started');
+ await controller.shutdown();
+ expect(controller.status).toBe('stopped');
+ expect(controller.enabled).toBe(false);
+ });
+
+ it('handles async SDK start failures without throwing', async () => {
+ mockStart.mockRejectedValueOnce(new Error('async start failed'));
+
+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ expect(controller.enabled).toBe(true);
+ expect(controller.status).toBe('starting');
+ await controller.shutdown();
+ expect(controller.enabled).toBe(false);
+ expect(controller.status).toBe('failed');
+ expect(emitWarningSpy).toHaveBeenCalledWith(
+ 'OpenTelemetry initialization failed: async start failed',
+ { code: 'LIBRECHAT_OTEL' },
+ );
+ });
+
+ it('returns failed status without throwing when SDK start fails', () => {
+ mockStart.mockImplementationOnce(() => {
+ throw new Error('start failed');
+ });
+
+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ expect(controller.enabled).toBe(false);
+ expect(controller.status).toBe('failed');
+ expect(emitWarningSpy).toHaveBeenCalledWith(
+ 'OpenTelemetry initialization failed: start failed',
+ { code: 'LIBRECHAT_OTEL' },
+ );
+ });
+
+ it('shuts down the active SDK idempotently', async () => {
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+ await shutdownTelemetry();
+ await shutdownTelemetry();
+
+ expect(mockShutdown).toHaveBeenCalledTimes(1);
+ });
+
+ it('coalesces concurrent shutdown calls', async () => {
+ let resolveShutdown: () => void = () => undefined;
+ mockShutdown.mockReturnValueOnce(
+ new Promise<void>((resolve) => {
+ resolveShutdown = resolve;
+ }),
+ );
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ const firstShutdown = shutdownTelemetry();
+ const secondShutdown = shutdownTelemetry();
+ expect(mockShutdown).toHaveBeenCalledTimes(1);
+
+ resolveShutdown();
+ await Promise.all([firstShutdown, secondShutdown]);
+ });
+
+ it('keeps the active SDK available when shutdown fails', async () => {
+ mockShutdown.mockRejectedValueOnce(new Error('shutdown failed'));
+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ await expect(shutdownTelemetry()).rejects.toThrow('shutdown failed');
+ expect(controller.status).toBe('started');
+
+ await shutdownTelemetry();
+ expect(mockShutdown).toHaveBeenCalledTimes(2);
+ expect(controller.status).toBe('stopped');
+ });
+
+ it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(
+ 'does not force process exit when another %s handler is registered',
+ async (signal) => {
+ const otherHandler = jest.fn();
+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+ process.once(signal, otherHandler);
+
+ process.emit(signal, signal);
+ await flushSignalShutdown();
+
+ expect(mockShutdown).toHaveBeenCalledTimes(1);
+ expect(otherHandler).toHaveBeenCalledTimes(1);
+ expect(killSpy).not.toHaveBeenCalled();
+
+ killSpy.mockRestore();
+ },
+ );
+
+ it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(
+ 'reraises the shutdown %s signal when telemetry is the only signal handler',
+ async (signal) => {
+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ process.emit(signal, signal);
+ await flushSignalShutdown();
+
+ expect(mockShutdown).toHaveBeenCalledTimes(1);
+ expect(killSpy).toHaveBeenCalledWith(process.pid, signal);
+
+ killSpy.mockRestore();
+ },
+ );
+
+ it('warns and reraises the signal when shutdown rejects', async () => {
+ mockShutdown.mockRejectedValueOnce(new Error('signal shutdown failed'));
+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);
+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });
+
+ process.emit('SIGTERM', 'SIGTERM');
+ await flushSignalShutdown();
+
+ expect(mockShutdown).toHaveBeenCalledTimes(1);
+ expect(emitWarningSpy).toHaveBeenCalledWith(
+ 'OpenTelemetry shutdown failed: signal shutdown failed',
+ { code: 'LIBRECHAT_OTEL' },
+ );
+ expect(killSpy).toHaveBeenCalledWith(process.pid, 'SIGTERM');
+
+ killSpy.mockRestore();
+ });
+});
diff --git a/packages/api/src/telemetry/sdk.ts b/packages/api/src/telemetry/sdk.ts
new file mode 100644
index 0000000000..49bd8a7ffc
--- /dev/null
+++ b/packages/api/src/telemetry/sdk.ts
@@ -0,0 +1,328 @@
+import { IncomingMessage } from 'node:http';
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { resourceFromAttributes } from '@opentelemetry/resources';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici';
+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';
+import { MongoDBInstrumentation } from '@opentelemetry/instrumentation-mongodb';
+import { MongooseInstrumentation } from '@opentelemetry/instrumentation-mongoose';
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node';
+import type { Span, Attributes } from '@opentelemetry/api';
+import type { TelemetryConfig, TelemetryStatus } from './config';
+import { getTelemetryConfig } from './config';
+
+export interface TelemetryController {
+ readonly enabled: boolean;
+ readonly status: TelemetryStatus;
+ shutdown: () => Promise<void>;
+}
+
+const WARNING_CODE = 'LIBRECHAT_OTEL';
+const REDACTED_QUERY_VALUE = '[REDACTED]';
+const SIGNAL_SHUTDOWN_TIMEOUT_MS = 5_000;
+
+interface RegisteredSignal {
+ signal: NodeJS.Signals;
+ listener: NodeJS.SignalsListener;
+}
+
+let activeSdk: NodeSDK | undefined;
+let pendingSdk: NodeSDK | undefined;
+let startPromise: Promise<void> | undefined;
+let shutdownPromise: Promise<void> | undefined;
+let status: TelemetryStatus = 'stopped';
+let registeredSignals: RegisteredSignal[] = [];
+let requestSpans = new WeakMap<IncomingMessage, Span>();
+
+function isBunRuntime(): boolean {
+ return Reflect.get(globalThis, 'Bun') != null;
+}
+
+function shouldIgnoreIncomingRequest(request: IncomingMessage, healthPath: string): boolean {
+ return request.url === healthPath || request.url?.startsWith(`${healthPath}?`) === true;
+}
+
+function getIncomingUrlInfo(request: IncomingMessage): { hasQuery: boolean; pathname: string } {
+ const rawUrl = request.url ?? '/';
+
+ try {
+ const parsedUrl = new URL(rawUrl, 'http://localhost');
+ return {
+ hasQuery: parsedUrl.search.length > 1,
+ pathname: parsedUrl.pathname || '/',
+ };
+ } catch {
+ const queryIndex = rawUrl.indexOf('?');
+ return {
+ hasQuery: queryIndex >= 0 && queryIndex < rawUrl.length - 1,
+ pathname: queryIndex >= 0 ? rawUrl.slice(0, queryIndex) || '/' : rawUrl || '/',
+ };
+ }
+}
+
+function getLowCardinalityUrlPath(pathname: string, healthPath: string): string {
+ if (pathname === healthPath) {
+ return healthPath;
+ }
+
+ if (pathname === '/api' || pathname.startsWith('/api/')) {
+ return '/api/*';
+ }
+
+ return 'spa_fallback';
+}
+
+function getSanitizedIncomingUrlAttributes(
+ request: IncomingMessage,
+ healthPath: string,
+): Attributes {
+ const { hasQuery, pathname } = getIncomingUrlInfo(request);
+ const safePath = getLowCardinalityUrlPath(pathname, healthPath);
+ const safeTarget = hasQuery ? `${safePath}?${REDACTED_QUERY_VALUE}` : safePath;
+ const attributes: Attributes = {
+ 'http.target': safeTarget,
+ 'http.url': safeTarget,
+ 'url.full': safeTarget,
+ 'url.path': safePath,
+ };
+
+ if (hasQuery) {
+ attributes['url.query'] = REDACTED_QUERY_VALUE;
+ }
+
+ return attributes;
+}
+
+function getResourceAttributes(config: TelemetryConfig): Attributes {
+ const attributes: Attributes = {
+ [ATTR_SERVICE_NAME]: config.serviceName,
+ };
+
+ if (config.serviceVersion) {
+ attributes[ATTR_SERVICE_VERSION] = config.serviceVersion;
+ }
+
+ return attributes;
+}
+
+function createSdk(config: TelemetryConfig): NodeSDK {
+ const sdkConfig: Partial<NodeSDKConfiguration> = {
+ resource: resourceFromAttributes(getResourceAttributes(config)),
+ instrumentations: [
+ new HttpInstrumentation({
+ headersToSpanAttributes: {
+ client: { requestHeaders: [], responseHeaders: [] },
+ server: { requestHeaders: [], responseHeaders: [] },
+ },
+ requestHook: (span: Span, request: object) => {
+ if (request instanceof IncomingMessage) {
+ requestSpans.set(request, span);
+ }
+ },
+ startIncomingSpanHook: (request: IncomingMessage) =>
+ getSanitizedIncomingUrlAttributes(request, config.healthPath),
+ ignoreIncomingRequestHook: (request: IncomingMessage) =>
+ shouldIgnoreIncomingRequest(request, config.healthPath),
+ }),
+ new ExpressInstrumentation(),
+ new MongoDBInstrumentation(),
+ new MongooseInstrumentation(),
+ new IORedisInstrumentation(),
+ new UndiciInstrumentation(),
+ ],
+ };
+
+ return new NodeSDK(sdkConfig);
+}
+
+export function getTelemetryRequestSpan(request: IncomingMessage): Span | undefined {
+ return requestSpans.get(request);
+}
+
+/**
+ * NodeSDK.start has been synchronous in some supported OpenTelemetry versions
+ * and promise-returning in others, so the lifecycle wrapper accepts either form.
+ */
+function startSdk(sdk: NodeSDK): void | Promise<void> {
+ return (sdk as NodeSDK & { start: () => void | Promise<void> }).start();
+}
+
+function emitWarning(message: string): void {
+ process.emitWarning(message, { code: WARNING_CODE });
+}
+
+function getErrorMessage(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
+
+function isControllerEnabled(): boolean {
+ return status === 'starting' || status === 'started';
+}
+
+function makeController(): TelemetryController {
+ return {
+ get enabled() {
+ return isControllerEnabled();
+ },
+ get status() {
+ return status;
+ },
+ shutdown: shutdownTelemetry,
+ };
+}
+
+function unregisterShutdownHandlers(): void {
+ for (const { signal, listener } of registeredSignals) {
+ process.removeListener(signal, listener);
+ }
+ registeredSignals = [];
+}
+
+function registerShutdownHandlers(): void {
+ if (registeredSignals.length > 0) {
+ return;
+ }
+
+ const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT'];
+ registeredSignals = signals.map((signal) => {
+ const listener: NodeJS.SignalsListener = () => {
+ const shouldReraiseSignal = process.listenerCount(signal) === 0;
+ withTimeout(shutdownTelemetry(), SIGNAL_SHUTDOWN_TIMEOUT_MS)
+ .catch((error) => {
+ emitWarning(`OpenTelemetry shutdown failed: ${getErrorMessage(error)}`);
+ })
+ .finally(() => {
+ if (shouldReraiseSignal) {
+ process.kill(process.pid, signal);
+ }
+ });
+ };
+ process.once(signal, listener);
+ return { signal, listener };
+ });
+}
+
+function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
+ let timeout: NodeJS.Timeout | undefined;
+ const timeoutPromise = new Promise<never>((_, reject) => {
+ timeout = setTimeout(() => {
+ reject(new Error(`timed out after ${timeoutMs}ms`));
+ }, timeoutMs);
+ timeout.unref?.();
+ });
+
+ return Promise.race([promise, timeoutPromise]).finally(() => {
+ if (timeout) {
+ clearTimeout(timeout);
+ }
+ });
+}
+
+export function initializeTelemetry(env: NodeJS.ProcessEnv = process.env): TelemetryController {
+ if (activeSdk || pendingSdk) {
+ return makeController();
+ }
+
+ const config = getTelemetryConfig(env);
+ if (!config.enabled || isBunRuntime()) {
+ status = 'disabled';
+ return makeController();
+ }
+
+ try {
+ const sdk = createSdk(config);
+ const result = startSdk(sdk);
+ if (result) {
+ pendingSdk = sdk;
+ status = 'starting';
+ const pendingStart = result
+ .then(() => {
+ if (pendingSdk === sdk) {
+ pendingSdk = undefined;
+ activeSdk = sdk;
+ status = 'started';
+ registerShutdownHandlers();
+ }
+ })
+ .catch((error) => {
+ if (pendingSdk === sdk) {
+ pendingSdk = undefined;
+ status = 'failed';
+ emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);
+ }
+ });
+ startPromise = pendingStart;
+ void pendingStart.finally(() => {
+ if (startPromise === pendingStart) {
+ startPromise = undefined;
+ }
+ });
+ return makeController();
+ }
+
+ activeSdk = sdk;
+ status = 'started';
+ registerShutdownHandlers();
+ return makeController();
+ } catch (error) {
+ status = 'failed';
+ emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);
+ return makeController();
+ }
+}
+
+async function performShutdownTelemetry(): Promise<void> {
+ if (startPromise) {
+ await startPromise;
+ }
+
+ if (!activeSdk) {
+ status = status === 'started' ? 'stopped' : status;
+ return;
+ }
+
+ const sdk = activeSdk;
+ try {
+ await sdk.shutdown();
+ activeSdk = undefined;
+ status = 'stopped';
+ unregisterShutdownHandlers();
+ } catch (error) {
+ status = 'started';
+ throw error;
+ }
+}
+
+export function shutdownTelemetry(): Promise<void> {
+ if (!shutdownPromise) {
+ shutdownPromise = performShutdownTelemetry().finally(() => {
+ shutdownPromise = undefined;
+ });
+ }
+
+ return shutdownPromise;
+}
+
+export async function resetTelemetryForTests(): Promise<void> {
+ try {
+ if (startPromise) {
+ await startPromise.catch(() => undefined);
+ }
+
+ if (shutdownPromise) {
+ await shutdownPromise.catch(() => undefined);
+ } else if (activeSdk) {
+ await Promise.resolve(activeSdk.shutdown()).catch(() => undefined);
+ }
+ } finally {
+ activeSdk = undefined;
+ pendingSdk = undefined;
+ startPromise = undefined;
+ shutdownPromise = undefined;
+ status = 'stopped';
+ requestSpans = new WeakMap<IncomingMessage, Span>();
+ unregisterShutdownHandlers();
+ }
+}