Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
📡 feat: Add Backend OpenTelemetry Tracing (#12909)
Danny Avila050b7fd43ae0
diff --git a/.env.example b/.env.exampleindex 816b4cd14d..e1608ec907 100644--- a/.env.example+++ b/.env.example@@ -110,6 +110,23 @@ NODE_MAX_OLD_SPACE_SIZE=6144# LANGFUSE_SECRET_KEY=# LANGFUSE_BASE_URL=+#=======================#+# OpenTelemetry Tracing #+#=======================#++# Enables backend OpenTelemetry tracing. General backend visibility only;+# use Langfuse for GenAI-specific prompt/model observability.+# OTEL_TRACING_ENABLED=false+# OTEL_SERVICE_NAME=librechat+# OTEL_SERVICE_VERSION=+# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318+# OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=+# OTEL_EXPORTER_OTLP_HEADERS=+# OTEL_TRACES_EXPORTER=otlp+# OTEL_TRACES_SAMPLER=parentbased_always_on+# OTEL_LOG_LEVEL=INFO+# OTEL_SDK_DISABLED=false+#===================================================## Endpoints ##===================================================#diff --git a/api/package.json b/api/package.jsonindex 80c3e5fd9b..8b3ff32a30 100644--- a/api/package.json+++ b/api/package.json@@ -52,6 +52,16 @@"@microsoft/microsoft-graph-client": "^3.0.7","@modelcontextprotocol/sdk": "^1.29.0","@node-saml/passport-saml": "^5.1.0",+ "@opentelemetry/api": "^1.9.0",+ "@opentelemetry/instrumentation-express": "^0.56.0",+ "@opentelemetry/instrumentation-http": "^0.207.0",+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",+ "@opentelemetry/instrumentation-undici": "^0.18.0",+ "@opentelemetry/resources": "^2.6.1",+ "@opentelemetry/sdk-node": "^0.207.0",+ "@opentelemetry/semantic-conventions": "^1.39.0","@smithy/node-http-handler": "^4.4.5","ai-tokenizer": "^1.0.6","axios": "^1.16.0",diff --git a/api/server/index.js b/api/server/index.jsindex a8146fbc1c..9e094724f7 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -1,4 +1,4 @@-require('dotenv').config();+const telemetry = require('./telemetry');const fs = require('fs');const path = require('path');require('module-alias')({ base: path.resolve(__dirname, '..') });@@ -25,22 +25,22 @@ const {updateInterfacePermissions,} = require('@librechat/api');const { connectDb, indexSync } = require('~/db');-const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');const {- getRoleByName,updateAccessPermissions,- seedDatabase,sweepOrphanedPreviews,+ getRoleByName,+ seedDatabase,} = require('~/models');+const initializeOAuthReconnectManager = require('./services/initializeOAuthReconnectManager');const { capabilityContextMiddleware } = require('./middleware/roles/capabilities');const createValidateImageRequest = require('./middleware/validateImageRequest');const { jwtLogin, ldapLogin, passportLogin } = require('~/strategies');const { checkMigrations } = require('./services/start/migration');+const optionalJwtAuth = require('./middleware/optionalJwtAuth');const initializeMCPs = require('./services/initializeMCPs');const configureSocialLogins = require('./socialLogins');const { getAppConfig } = require('./services/Config');const staticCache = require('./utils/staticCache');-const optionalJwtAuth = require('./middleware/optionalJwtAuth');const noIndex = require('./middleware/noIndex');const routes = require('./routes');@@ -146,6 +146,10 @@ const startServer = async () => {app.use(staticCache(appConfig.paths.fonts));app.use(staticCache(appConfig.paths.assets));+ if (telemetry.enabled) {+ app.use(telemetry.telemetryMiddleware);+ }+if (!ALLOW_SOCIAL_LOGIN) {console.warn('Social logins are disabled. Set ALLOW_SOCIAL_LOGIN=true to enable them.');}@@ -227,6 +231,10 @@ const startServer = async () => {res.send(updatedIndexHtml);});+ /** Record trace errors before the final error controller. */+ if (telemetry.enabled) {+ app.use(telemetry.telemetryErrorMiddleware);+ }/** Error handler (must be last - Express identifies error middleware by its 4-arg signature) */app.use(ErrorController);diff --git a/api/server/index.spec.js b/api/server/index.spec.jsindex 7b3d062fce..573770e282 100644--- a/api/server/index.spec.js+++ b/api/server/index.spec.js@@ -1,4 +1,5 @@const fs = require('fs');+const path = require('path');const request = require('supertest');const { MongoMemoryServer } = require('mongodb-memory-server');const mongoose = require('mongoose');@@ -32,6 +33,56 @@ jest.mock('~/config', () => ({}),}));+jest.mock(+ '@librechat/api/telemetry',+ () => ({+ initializeTelemetry: jest.fn(() => ({+ enabled: false,+ status: 'disabled',+ shutdown: jest.fn(),+ })),+ telemetryMiddleware: jest.fn((_req, _res, next) => next()),+ telemetryErrorMiddleware: jest.fn((err, _req, _res, next) => next(err)),+ }),+ { virtual: true },+);++describe('Telemetry wiring', () => {+ const source = fs.readFileSync(path.join(__dirname, 'index.js'), 'utf8');++ it('loads telemetry before other server imports', () => {+ const firstStatement = source+ .split('\n')+ .map((line) => line.trim())+ .find(Boolean);++ expect(firstStatement).toBe("const telemetry = require('./telemetry');");+ });++ it('mounts telemetry middleware after static assets and before routes', () => {+ const telemetryMiddlewareIndex = source.indexOf('app.use(telemetry.telemetryMiddleware);');+ const staticAssetsIndex = source.indexOf('app.use(staticCache(appConfig.paths.assets));');+ const apiRoutesIndex = source.indexOf("app.use('/api/auth'");++ expect(telemetryMiddlewareIndex).toBeGreaterThan(-1);+ expect(staticAssetsIndex).toBeGreaterThan(-1);+ expect(apiRoutesIndex).toBeGreaterThan(-1);+ expect(staticAssetsIndex).toBeLessThan(telemetryMiddlewareIndex);+ expect(telemetryMiddlewareIndex).toBeLessThan(apiRoutesIndex);+ });++ it('mounts telemetry error middleware before ErrorController', () => {+ const telemetryErrorMiddlewareIndex = source.indexOf(+ 'app.use(telemetry.telemetryErrorMiddleware);',+ );+ const errorControllerIndex = source.indexOf('app.use(ErrorController);');++ expect(telemetryErrorMiddlewareIndex).toBeGreaterThan(-1);+ expect(errorControllerIndex).toBeGreaterThan(-1);+ expect(telemetryErrorMiddlewareIndex).toBeLessThan(errorControllerIndex);+ });+});+describe('Server Configuration', () => {// Increase the default timeout to allow for Mongo cleanupjest.setTimeout(30_000);diff --git a/api/server/telemetry.js b/api/server/telemetry.jsnew file mode 100644index 0000000000..cb0058355b--- /dev/null+++ b/api/server/telemetry.js@@ -0,0 +1,40 @@+require('dotenv').config();++function isTruthy(value) {+ return value?.trim().toLowerCase() === 'true';+}++function isTelemetryEnabled() {+ return isTruthy(process.env.OTEL_TRACING_ENABLED) && !isTruthy(process.env.OTEL_SDK_DISABLED);+}++if (isTelemetryEnabled()) {+ const {+ initializeTelemetry,+ telemetryMiddleware,+ telemetryErrorMiddleware,+ } = require('@librechat/api/telemetry');+ const controller = initializeTelemetry();++ module.exports = {+ get enabled() {+ return controller.enabled;+ },+ get status() {+ return controller.status;+ },+ shutdown: controller.shutdown,+ telemetryMiddleware,+ telemetryErrorMiddleware,+ };+} else {+ module.exports = {+ enabled: false,+ get status() {+ return 'disabled';+ },+ shutdown: async () => {},+ telemetryMiddleware: (_req, _res, next) => next(),+ telemetryErrorMiddleware: (err, _req, _res, next) => next(err),+ };+}diff --git a/api/server/telemetry.spec.js b/api/server/telemetry.spec.jsnew file mode 100644index 0000000000..8f848da8e7--- /dev/null+++ b/api/server/telemetry.spec.js@@ -0,0 +1,91 @@+describe('telemetry bootstrap', () => {+ const originalEnv = process.env;++ beforeEach(() => {+ jest.resetModules();+ process.env = { ...originalEnv };+ delete process.env.OTEL_SDK_DISABLED;+ delete process.env.OTEL_TRACING_ENABLED;+ jest.doMock('dotenv', () => ({+ config: jest.fn(),+ }));+ });++ afterEach(() => {+ process.env = originalEnv;+ jest.dontMock('dotenv');+ jest.dontMock('@librechat/api/telemetry');+ jest.resetModules();+ });++ it('does not load OpenTelemetry packages by default', () => {+ jest.doMock(+ '@librechat/api/telemetry',+ () => {+ throw new Error('telemetry package should not load when tracing is disabled');+ },+ { virtual: true },+ );++ const telemetry = require('./telemetry');++ expect(telemetry.enabled).toBe(false);+ expect(telemetry.status).toBe('disabled');+ });++ it('does not load OpenTelemetry packages when the SDK is disabled', () => {+ process.env.OTEL_SDK_DISABLED = 'true';+ process.env.OTEL_TRACING_ENABLED = 'true';+ jest.doMock(+ '@librechat/api/telemetry',+ () => {+ throw new Error('telemetry package should not load when the SDK is disabled');+ },+ { virtual: true },+ );++ const telemetry = require('./telemetry');++ expect(telemetry.enabled).toBe(false);+ expect(telemetry.status).toBe('disabled');+ });++ it('loads and exposes telemetry middleware when tracing is enabled', () => {+ process.env.OTEL_TRACING_ENABLED = 'true';+ let enabled = true;+ let status = 'starting';+ const telemetryMiddleware = jest.fn();+ const telemetryErrorMiddleware = jest.fn();+ const initializeTelemetry = jest.fn(() => ({+ get enabled() {+ return enabled;+ },+ get status() {+ return status;+ },+ shutdown: jest.fn(),+ }));+ jest.doMock(+ '@librechat/api/telemetry',+ () => ({+ initializeTelemetry,+ telemetryMiddleware,+ telemetryErrorMiddleware,+ }),+ { virtual: true },+ );++ const telemetry = require('./telemetry');++ expect(initializeTelemetry).toHaveBeenCalledTimes(1);+ expect(telemetry.enabled).toBe(true);+ expect(telemetry.status).toBe('starting');+ expect(telemetry.telemetryMiddleware).toBe(telemetryMiddleware);+ expect(telemetry.telemetryErrorMiddleware).toBe(telemetryErrorMiddleware);++ enabled = false;+ status = 'failed';+ expect(telemetry.enabled).toBe(false);+ expect(telemetry.status).toBe('failed');+ });+});diff --git a/package-lock.json b/package-lock.jsonindex dbc9fb994c..c25619187e 100644--- a/package-lock.json+++ b/package-lock.json@@ -67,6 +67,16 @@"@microsoft/microsoft-graph-client": "^3.0.7","@modelcontextprotocol/sdk": "^1.29.0","@node-saml/passport-saml": "^5.1.0",+ "@opentelemetry/api": "^1.9.0",+ "@opentelemetry/instrumentation-express": "^0.56.0",+ "@opentelemetry/instrumentation-http": "^0.207.0",+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",+ "@opentelemetry/instrumentation-undici": "^0.18.0",+ "@opentelemetry/resources": "^2.6.1",+ "@opentelemetry/sdk-node": "^0.207.0",+ "@opentelemetry/semantic-conventions": "^1.39.0","@smithy/node-http-handler": "^4.4.5","ai-tokenizer": "^1.0.6","axios": "^1.16.0",@@ -12866,7 +12876,6 @@"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.6.1.tgz","integrity": "sha512-8xHSGWpJP9wBxgBpnqGL0R3PbdWQndL1Qp50qrg71+B28zK5OQmUgcDKLJgzyAAV38t4tOyLMGDD60LneR5W8g==","license": "Apache-2.0",- "peer": true,"dependencies": {"@opentelemetry/semantic-conventions": "^1.29.0"},@@ -14245,6 +14254,120 @@"@opentelemetry/api": "^1.3.0"}},+ "node_modules/@opentelemetry/instrumentation-express": {+ "version": "0.56.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.56.0.tgz",+ "integrity": "sha512-rMV0WUTtAGEhHrHl3uDRIO97EkNUp4ewrW2iRVuP7kaV5qRT2b1pPV5PE75oR3GyLLSTooSAzGWl6CTm8eftKQ==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/core": "^2.0.0",+ "@opentelemetry/instrumentation": "^0.207.0",+ "@opentelemetry/semantic-conventions": "^1.27.0"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": "^1.3.0"+ }+ },+ "node_modules/@opentelemetry/instrumentation-http": {+ "version": "0.207.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.207.0.tgz",+ "integrity": "sha512-FC4i5hVixTzuhg4SV2ycTEAYx+0E2hm+GwbdoVPSA6kna0pPVI4etzaA9UkpJ9ussumQheFXP6rkGIaFJjMxsw==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/core": "2.2.0",+ "@opentelemetry/instrumentation": "0.207.0",+ "@opentelemetry/semantic-conventions": "^1.29.0",+ "forwarded-parse": "2.1.2"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": "^1.3.0"+ }+ },+ "node_modules/@opentelemetry/instrumentation-http/node_modules/@opentelemetry/core": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.2.0.tgz",+ "integrity": "sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/semantic-conventions": "^1.29.0"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": ">=1.0.0 <1.10.0"+ }+ },+ "node_modules/@opentelemetry/instrumentation-ioredis": {+ "version": "0.55.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.55.0.tgz",+ "integrity": "sha512-ASuBMzh0ImmfOnWj9vCPtBMqSjr54/r/HluUIylwZB7xzTU6gL2SfybxySJMzEL9+386gJJVApwQktVznAtrWA==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/instrumentation": "^0.207.0",+ "@opentelemetry/redis-common": "^0.38.2"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": "^1.3.0"+ }+ },+ "node_modules/@opentelemetry/instrumentation-mongodb": {+ "version": "0.60.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.60.0.tgz",+ "integrity": "sha512-8mKW2oyyWdYOKYpu70AGGAvLnExGlOoFT+Ylk9hJvWYHR5f6IrmVqwUMlzPM2WC4ihQ9crvqrtldMPmlVezTqg==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/instrumentation": "^0.207.0"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": "^1.3.0"+ }+ },+ "node_modules/@opentelemetry/instrumentation-mongoose": {+ "version": "0.54.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.54.0.tgz",+ "integrity": "sha512-R2JzMrxiz3R9m+cO6iVC85bma3E3vdBr06F+D4zGcEkJS9FaCRw6+Hdb+aQ1AaDRwEMTaw+F28+6VdMpniBOXQ==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/core": "^2.0.0",+ "@opentelemetry/instrumentation": "^0.207.0"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": "^1.3.0"+ }+ },+ "node_modules/@opentelemetry/instrumentation-undici": {+ "version": "0.18.0",+ "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.18.0.tgz",+ "integrity": "sha512-NalxLuZV621Xq4IhQkC+OXoZjAT8Xf6vYRdTjHitOXMU+4l/peRY05V7wGr4d7huf+vjyQry0XKlyhsEr4ouNw==",+ "license": "Apache-2.0",+ "dependencies": {+ "@opentelemetry/core": "^2.0.0",+ "@opentelemetry/instrumentation": "^0.207.0",+ "@opentelemetry/semantic-conventions": "^1.24.0"+ },+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ },+ "peerDependencies": {+ "@opentelemetry/api": "^1.7.0"+ }+ },"node_modules/@opentelemetry/instrumentation/node_modules/@opentelemetry/api-logs": {"version": "0.207.0","resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.207.0.tgz",@@ -14504,12 +14627,20 @@"@opentelemetry/api": ">=1.0.0 <1.10.0"}},+ "node_modules/@opentelemetry/redis-common": {+ "version": "0.38.3",+ "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz",+ "integrity": "sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==",+ "license": "Apache-2.0",+ "engines": {+ "node": "^18.19.0 || >=20.6.0"+ }+ },"node_modules/@opentelemetry/resources": {"version": "2.6.1","resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.6.1.tgz","integrity": "sha512-lID/vxSuKWXM55XhAKNoYXu9Cutoq5hFdkbTdI/zDKQktXzcWBVhNsOkiZFTMU9UtEWuGRNe0HUgmsFldIdxVA==","license": "Apache-2.0",- "peer": true,"dependencies": {"@opentelemetry/core": "2.6.1","@opentelemetry/semantic-conventions": "^1.29.0"@@ -28134,6 +28265,12 @@"node": ">= 0.6"}},+ "node_modules/forwarded-parse": {+ "version": "2.1.2",+ "resolved": "https://registry.npmjs.org/forwarded-parse/-/forwarded-parse-2.1.2.tgz",+ "integrity": "sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==",+ "license": "MIT"+ },"node_modules/fraction.js": {"version": "5.3.4","resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz",@@ -28286,6 +28423,67 @@"node": ">=14"}},+ "node_modules/gcp-metadata": {+ "version": "5.3.0",+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz",+ "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==",+ "license": "Apache-2.0",+ "optional": true,+ "peer": true,+ "dependencies": {+ "gaxios": "^5.0.0",+ "json-bigint": "^1.0.0"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/gcp-metadata/node_modules/agent-base": {+ "version": "6.0.2",+ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",+ "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",+ "license": "MIT",+ "optional": true,+ "peer": true,+ "dependencies": {+ "debug": "4"+ },+ "engines": {+ "node": ">= 6.0.0"+ }+ },+ "node_modules/gcp-metadata/node_modules/gaxios": {+ "version": "5.1.3",+ "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz",+ "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==",+ "license": "Apache-2.0",+ "optional": true,+ "peer": true,+ "dependencies": {+ "extend": "^3.0.2",+ "https-proxy-agent": "^5.0.0",+ "is-stream": "^2.0.0",+ "node-fetch": "^2.6.9"+ },+ "engines": {+ "node": ">=12"+ }+ },+ "node_modules/gcp-metadata/node_modules/https-proxy-agent": {+ "version": "5.0.1",+ "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",+ "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",+ "license": "MIT",+ "optional": true,+ "peer": true,+ "dependencies": {+ "agent-base": "6",+ "debug": "4"+ },+ "engines": {+ "node": ">= 6"+ }+ },"node_modules/generic-names": {"version": "4.0.0","resolved": "https://registry.npmjs.org/generic-names/-/generic-names-4.0.0.tgz",@@ -28585,6 +28783,7 @@"version": "6.1.1","resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz","integrity": "sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==",+ "license": "Apache-2.0","dependencies": {"gaxios": "^6.1.1","google-logging-utils": "^0.0.2",@@ -28598,6 +28797,7 @@"version": "0.0.2","resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-0.0.2.tgz","integrity": "sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==",+ "license": "Apache-2.0","engines": {"node": ">=14"}@@ -34726,6 +34926,52 @@"node": ">=20.19.0"}},+ "node_modules/mongodb-memory-server-core/node_modules/gaxios": {+ "version": "7.1.4",+ "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz",+ "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==",+ "dev": true,+ "license": "Apache-2.0",+ "optional": true,+ "peer": true,+ "dependencies": {+ "extend": "^3.0.2",+ "https-proxy-agent": "^7.0.1",+ "node-fetch": "^3.3.2"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "node_modules/mongodb-memory-server-core/node_modules/gcp-metadata": {+ "version": "7.0.1",+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-7.0.1.tgz",+ "integrity": "sha512-UcO3kefx6dCcZkgcTGgVOTFb7b1LlQ02hY1omMjjrrBzkajRMCFgYOjs7J71WqnuG1k2b+9ppGL7FsOfhZMQKQ==",+ "dev": true,+ "license": "Apache-2.0",+ "optional": true,+ "peer": true,+ "dependencies": {+ "gaxios": "^7.0.0",+ "google-logging-utils": "^1.0.0",+ "json-bigint": "^1.0.0"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "node_modules/mongodb-memory-server-core/node_modules/google-logging-utils": {+ "version": "1.1.3",+ "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz",+ "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==",+ "dev": true,+ "license": "Apache-2.0",+ "optional": true,+ "peer": true,+ "engines": {+ "node": ">=14"+ }+ },"node_modules/mongodb-memory-server-core/node_modules/mongodb": {"version": "7.1.1","resolved": "https://registry.npmjs.org/mongodb/-/mongodb-7.1.1.tgz",@@ -34787,6 +35033,27 @@"node": ">=20.19.0"}},+ "node_modules/mongodb-memory-server-core/node_modules/node-fetch": {+ "version": "3.3.2",+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",+ "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",+ "dev": true,+ "license": "MIT",+ "optional": true,+ "peer": true,+ "dependencies": {+ "data-uri-to-buffer": "^4.0.0",+ "fetch-blob": "^3.1.4",+ "formdata-polyfill": "^4.0.10"+ },+ "engines": {+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/node-fetch"+ }+ },"node_modules/mongoose": {"version": "8.23.1","resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.23.1.tgz",@@ -44692,6 +44959,16 @@"@librechat/agents": "^3.1.86","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.29.0",+ "@opentelemetry/api": "^1.9.0",+ "@opentelemetry/instrumentation-express": "^0.56.0",+ "@opentelemetry/instrumentation-http": "^0.207.0",+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",+ "@opentelemetry/instrumentation-undici": "^0.18.0",+ "@opentelemetry/resources": "^2.6.1",+ "@opentelemetry/sdk-node": "^0.207.0",+ "@opentelemetry/semantic-conventions": "^1.39.0","@smithy/node-http-handler": "^4.4.5","ai-tokenizer": "^1.0.6","axios": "^1.16.0",diff --git a/packages/api/package.json b/packages/api/package.jsonindex 1168c21bc6..048df30ba1 100644--- a/packages/api/package.json+++ b/packages/api/package.json@@ -10,6 +10,10 @@".": {"require": "./dist/index.js","types": "./dist/types/index.d.ts"+ },+ "./telemetry": {+ "require": "./dist/telemetry.js",+ "types": "./dist/types/telemetry/index.d.ts"}},"scripts": {@@ -103,6 +107,16 @@"@librechat/agents": "^3.1.86","@librechat/data-schemas": "*","@modelcontextprotocol/sdk": "^1.29.0",+ "@opentelemetry/api": "^1.9.0",+ "@opentelemetry/instrumentation-express": "^0.56.0",+ "@opentelemetry/instrumentation-http": "^0.207.0",+ "@opentelemetry/instrumentation-ioredis": "^0.55.0",+ "@opentelemetry/instrumentation-mongodb": "^0.60.0",+ "@opentelemetry/instrumentation-mongoose": "^0.54.0",+ "@opentelemetry/instrumentation-undici": "^0.18.0",+ "@opentelemetry/resources": "^2.6.1",+ "@opentelemetry/sdk-node": "^0.207.0",+ "@opentelemetry/semantic-conventions": "^1.39.0","@smithy/node-http-handler": "^4.4.5","ai-tokenizer": "^1.0.6","axios": "^1.16.0",diff --git a/packages/api/rollup.config.js b/packages/api/rollup.config.jsindex 9a9de35e8f..becc482291 100644--- a/packages/api/rollup.config.js+++ b/packages/api/rollup.config.js@@ -45,7 +45,10 @@ const plugins = [];const cjsBuild = {- input: 'src/index.ts',+ input: {+ index: 'src/index.ts',+ telemetry: 'src/telemetry/index.ts',+ },output: {dir: 'dist',format: 'cjs',diff --git a/packages/api/src/telemetry/config.spec.ts b/packages/api/src/telemetry/config.spec.tsnew file mode 100644index 0000000000..128f07a5a3--- /dev/null+++ b/packages/api/src/telemetry/config.spec.ts@@ -0,0 +1,46 @@+import { getTelemetryConfig } from './config';++describe('getTelemetryConfig', () => {+ it('defaults tracing off', () => {+ const config = getTelemetryConfig({});++ expect(config.enabled).toBe(false);+ expect(config.sdkDisabled).toBe(false);+ expect(config.serviceName).toBe('librechat');+ expect(config.healthPath).toBe('/health');+ });++ it('enables tracing only when OTEL_TRACING_ENABLED is true', () => {+ expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'true' }).enabled).toBe(true);+ expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'TRUE' }).enabled).toBe(true);+ expect(getTelemetryConfig({ OTEL_TRACING_ENABLED: 'false' }).enabled).toBe(false);+ });++ it('lets OTEL_SDK_DISABLED override tracing enablement', () => {+ const config = getTelemetryConfig({+ OTEL_SDK_DISABLED: 'true',+ OTEL_TRACING_ENABLED: 'true',+ });++ expect(config.enabled).toBe(false);+ expect(config.sdkDisabled).toBe(true);+ });++ it('uses standard service env vars when provided', () => {+ const config = getTelemetryConfig({+ OTEL_SERVICE_NAME: ' librechat-api ',+ OTEL_SERVICE_VERSION: ' 1.2.3 ',+ });++ expect(config.serviceName).toBe('librechat-api');+ expect(config.serviceVersion).toBe('1.2.3');+ });++ it('falls back to npm package version when service version is absent', () => {+ const config = getTelemetryConfig({+ npm_package_version: '0.8.5',+ });++ expect(config.serviceVersion).toBe('0.8.5');+ });+});diff --git a/packages/api/src/telemetry/config.ts b/packages/api/src/telemetry/config.tsnew file mode 100644index 0000000000..340197aa65--- /dev/null+++ b/packages/api/src/telemetry/config.ts@@ -0,0 +1,43 @@+const DEFAULT_SERVICE_NAME = 'librechat';+export const DEFAULT_HEALTH_PATH = '/health';++export type TelemetryStatus = 'disabled' | 'failed' | 'started' | 'starting' | 'stopped';++export interface TelemetryConfig {+ enabled: boolean;+ healthPath: string;+ sdkDisabled: boolean;+ serviceName: string;+ serviceVersion?: string;+}++function isTruthy(value?: string | boolean | null): boolean {+ if (typeof value === 'boolean') {+ return value;+ }+ if (typeof value === 'string') {+ return value.trim().toLowerCase() === 'true';+ }+ return false;+}++function normalizeEnvValue(value?: string): string | undefined {+ const trimmed = value?.trim();+ return trimmed ? trimmed : undefined;+}++export function getTelemetryConfig(env: NodeJS.ProcessEnv = process.env): TelemetryConfig {+ const sdkDisabled = isTruthy(env.OTEL_SDK_DISABLED);+ const enabled = isTruthy(env.OTEL_TRACING_ENABLED) && !sdkDisabled;+ const serviceName = normalizeEnvValue(env.OTEL_SERVICE_NAME) ?? DEFAULT_SERVICE_NAME;+ const serviceVersion =+ normalizeEnvValue(env.OTEL_SERVICE_VERSION) ?? normalizeEnvValue(env.npm_package_version);++ return {+ enabled,+ serviceName,+ sdkDisabled,+ serviceVersion,+ healthPath: DEFAULT_HEALTH_PATH,+ };+}diff --git a/packages/api/src/telemetry/index.ts b/packages/api/src/telemetry/index.tsnew file mode 100644index 0000000000..2eb188bf84--- /dev/null+++ b/packages/api/src/telemetry/index.ts@@ -0,0 +1,5 @@+export { getTelemetryConfig } from './config';+export { initializeTelemetry, shutdownTelemetry } from './sdk';+export { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';+export type { TelemetryConfig, TelemetryStatus } from './config';+export type { TelemetryController } from './sdk';diff --git a/packages/api/src/telemetry/middleware.spec.ts b/packages/api/src/telemetry/middleware.spec.tsnew file mode 100644index 0000000000..271d8c0a11--- /dev/null+++ b/packages/api/src/telemetry/middleware.spec.ts@@ -0,0 +1,395 @@+import { EventEmitter } from 'node:events';+import { SpanStatusCode, trace } from '@opentelemetry/api';+import type { NextFunction, Response } from 'express';+import type { Span } from '@opentelemetry/api';+import type { ServerRequest } from '~/types';+import { getTelemetryRequestSpan } from './sdk';+import { telemetryErrorMiddleware, telemetryMiddleware } from './middleware';++jest.mock('./sdk', () => ({+ getTelemetryRequestSpan: jest.fn(),+}));++const mockGetTelemetryRequestSpan = getTelemetryRequestSpan as jest.MockedFunction<+ typeof getTelemetryRequestSpan+>;++interface MockResponse extends EventEmitter {+ statusCode: number;+ writableEnded: boolean;+}++function createSpan(): jest.Mocked<Span> {+ const span = {} as jest.Mocked<Span>;+ span.addEvent = jest.fn<jest.Mocked<Span>, Parameters<Span['addEvent']>>(() => span);+ span.addLink = jest.fn<jest.Mocked<Span>, Parameters<Span['addLink']>>(() => span);+ span.addLinks = jest.fn<jest.Mocked<Span>, Parameters<Span['addLinks']>>(() => span);+ span.end = jest.fn<void, Parameters<Span['end']>>();+ span.isRecording = jest.fn<boolean, Parameters<Span['isRecording']>>(() => true);+ span.recordException = jest.fn<void, Parameters<Span['recordException']>>();+ span.setAttribute = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttribute']>>(() => span);+ span.setAttributes = jest.fn<jest.Mocked<Span>, Parameters<Span['setAttributes']>>(() => span);+ span.setStatus = jest.fn<jest.Mocked<Span>, Parameters<Span['setStatus']>>(() => span);+ span.spanContext = jest.fn<ReturnType<Span['spanContext']>, Parameters<Span['spanContext']>>(+ () => ({+ spanId: '0000000000000000',+ traceFlags: 0,+ traceId: '00000000000000000000000000000000',+ }),+ );+ span.updateName = jest.fn<jest.Mocked<Span>, Parameters<Span['updateName']>>(() => span);+ return span;+}++function createResponse(statusCode = 200): MockResponse {+ const res = new EventEmitter() as MockResponse;+ res.statusCode = statusCode;+ res.writableEnded = false;+ return res;+}++function createRequest(overrides: Partial<ServerRequest> = {}): ServerRequest {+ return {+ baseUrl: '/api/messages',+ body: {+ prompt: 'do not capture this prompt',+ text: 'do not capture this body',+ },+ headers: {+ authorization: 'Bearer do-not-capture-this-auth-header',+ cookie: 'session=do-not-capture-this-cookie',+ 'x-api-key': 'do-not-capture-this-api-key',+ },+ method: 'POST',+ path: '/api/messages/conversation-1',+ route: { path: '/:conversationId' },+ user: {+ email: '[email protected]',+ id: 'user-1',+ tenantId: 'tenant-1',+ } as ServerRequest['user'],+ ...overrides,+ } as ServerRequest;+}++afterEach(() => {+ mockGetTelemetryRequestSpan.mockReset();+});++describe('telemetryMiddleware', () => {+ afterEach(() => {+ jest.restoreAllMocks();+ });++ it('passes through without an active span', () => {+ const next = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);++ telemetryMiddleware(createRequest(), createResponse() as Response, next);++ expect(next).toHaveBeenCalledTimes(1);+ });++ it('uses the stored request span for deferred completion attributes', () => {+ const activeSpan = createSpan();+ const requestSpan = createSpan();+ const req = createRequest();+ const res = createResponse(202);+ const next: NextFunction = jest.fn();+ mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);++ telemetryMiddleware(req, res as Response, next);+ res.emit('finish');++ expect(next).toHaveBeenCalledTimes(1);+ expect(trace.getActiveSpan).not.toHaveBeenCalled();+ expect(activeSpan.setAttributes).not.toHaveBeenCalled();+ expect(requestSpan.setAttributes).toHaveBeenCalledWith({+ 'http.request.method': 'POST',+ });+ expect(requestSpan.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'http.response.status_code': 202,+ 'http.route': '/api/messages/:conversationId',+ }),+ );+ });++ it('records safe route and identity attributes without body content', () => {+ const span = createSpan();+ const req = createRequest();+ const res = createResponse(201);+ const next: NextFunction = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(req, res as Response, next);+ res.emit('finish');++ expect(next).toHaveBeenCalledTimes(1);+ expect(span.setAttributes).toHaveBeenCalledWith({+ 'http.request.method': 'POST',+ });+ expect(span.setAttributes).toHaveBeenCalledWith({+ 'enduser.id': 'user-1',+ 'librechat.tenant.id': 'tenant-1',+ });+ expect(span.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'http.response.status_code': 201,+ 'http.route': '/api/messages/:conversationId',+ }),+ );++ const capturedAttributes = JSON.stringify(span.setAttributes.mock.calls);+ expect(capturedAttributes).not.toContain('do not capture this prompt');+ expect(capturedAttributes).not.toContain('do not capture this body');+ expect(capturedAttributes).not.toContain('[email protected]');+ expect(capturedAttributes).not.toContain('conversation-1');+ expect(capturedAttributes).not.toContain('do-not-capture-this-auth-header');+ expect(capturedAttributes).not.toContain('do-not-capture-this-cookie');+ expect(capturedAttributes).not.toContain('do-not-capture-this-api-key');+ });++ it('records identity attributes populated by downstream middleware', () => {+ const span = createSpan();+ const req = createRequest({+ headers: {},+ user: undefined,+ });+ const res = createResponse(200);+ const next: NextFunction = jest.fn(() => {+ req.user = {+ id: 'late-user',+ tenantId: 'late-tenant',+ } as ServerRequest['user'];+ });+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(req, res as Response, next);++ expect(next).toHaveBeenCalledTimes(1);+ expect(span.setAttributes).toHaveBeenCalledTimes(1);++ res.emit('finish');++ expect(span.setAttributes).toHaveBeenCalledWith({+ 'enduser.id': 'late-user',+ 'librechat.tenant.id': 'late-tenant',+ });+ });++ it('does not derive tenant identity from request headers', () => {+ const span = createSpan();+ const req = createRequest({+ headers: {+ 'x-tenant-id': 'spoofed-tenant',+ },+ user: undefined,+ });+ const res = createResponse(200);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(req, res as Response, jest.fn());+ res.emit('finish');++ expect(span.setAttributes).not.toHaveBeenCalledWith(+ expect.objectContaining({+ 'librechat.tenant.id': 'spoofed-tenant',+ }),+ );+ expect(JSON.stringify(span.setAttributes.mock.calls)).not.toContain('spoofed-tenant');+ });++ it('ignores health checks', () => {+ const span = createSpan();+ const next = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(+ createRequest({+ baseUrl: '',+ path: '/health',+ route: undefined,+ }),+ createResponse() as Response,+ next,+ );++ expect(next).toHaveBeenCalledTimes(1);+ expect(span.setAttributes).not.toHaveBeenCalled();+ });++ it('uses a low-cardinality fallback for unmatched API routes', () => {+ const span = createSpan();+ const req = createRequest({+ baseUrl: '',+ path: '/api/nonexistent/123',+ route: undefined,+ });+ const res = createResponse(404);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(req, res as Response, jest.fn());+ res.emit('finish');++ expect(span.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'http.route': '/api/*',+ 'http.response.status_code': 404,+ }),+ );+ });++ it('uses a low-cardinality fallback for unmatched SPA routes', () => {+ const span = createSpan();+ const req = createRequest({+ baseUrl: '',+ path: '/chat/conversation-id',+ route: undefined,+ });+ const res = createResponse(200);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(req, res as Response, jest.fn());+ res.emit('finish');++ expect(span.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'http.route': 'spa_fallback',+ 'http.response.status_code': 200,+ }),+ );+ });++ it('marks server responses as errored', () => {+ const span = createSpan();+ const req = createRequest();+ const res = createResponse(500);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(req, res as Response, jest.fn());+ res.emit('finish');++ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });+ });++ it('records completion attributes only once when finish and close both fire', () => {+ const span = createSpan();+ const res = createResponse(200);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(createRequest(), res as Response, jest.fn());+ res.emit('finish');+ res.emit('close');++ expect(span.setAttributes).toHaveBeenCalledTimes(3);+ });++ it('marks client disconnects before finish as aborted errors', () => {+ const span = createSpan();+ const res = createResponse(200);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryMiddleware(createRequest(), res as Response, jest.fn());+ res.emit('close');++ expect(span.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'http.response.status_code': 499,+ 'librechat.request.aborted': true,+ }),+ );+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });+ });+});++describe('telemetryErrorMiddleware', () => {+ afterEach(() => {+ jest.restoreAllMocks();+ });++ it('records exceptions and forwards the error', () => {+ const span = createSpan();+ const error = new TypeError('boom');+ const next = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);++ expect(span.recordException).toHaveBeenCalledWith(error);+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });+ expect(span.setAttributes).toHaveBeenCalledWith({+ 'enduser.id': 'user-1',+ 'librechat.tenant.id': 'tenant-1',+ });+ expect(span.setAttributes).toHaveBeenCalledWith({+ 'error.type': 'TypeError',+ 'http.route': '/api/messages/:conversationId',+ });+ expect(next).toHaveBeenCalledWith(error);+ });++ it('records exceptions on the stored request span when available', () => {+ const activeSpan = createSpan();+ const requestSpan = createSpan();+ const error = new TypeError('boom');+ const next = jest.fn();+ mockGetTelemetryRequestSpan.mockReturnValue(requestSpan);+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(activeSpan);++ telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);++ expect(trace.getActiveSpan).not.toHaveBeenCalled();+ expect(activeSpan.recordException).not.toHaveBeenCalled();+ expect(requestSpan.recordException).toHaveBeenCalledWith(error);+ expect(requestSpan.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });+ expect(next).toHaveBeenCalledWith(error);+ });++ it('handles non-Error values without throwing', () => {+ const span = createSpan();+ const next = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryErrorMiddleware('boom', createRequest(), createResponse() as Response, next);++ expect(span.recordException).toHaveBeenCalledWith('boom');+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });+ expect(span.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'error.type': 'string',+ 'http.route': '/api/messages/:conversationId',+ }),+ );+ expect(next).toHaveBeenCalledWith('boom');+ });++ it('handles null error values without throwing', () => {+ const span = createSpan();+ const next = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(span);++ telemetryErrorMiddleware(null, createRequest(), createResponse() as Response, next);++ expect(span.recordException).not.toHaveBeenCalled();+ expect(span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });+ expect(span.setAttributes).toHaveBeenCalledWith(+ expect.objectContaining({+ 'error.type': 'null',+ 'http.route': '/api/messages/:conversationId',+ }),+ );+ expect(next).toHaveBeenCalledWith(null);+ });++ it('forwards the error without an active span', () => {+ const error = new Error('boom');+ const next = jest.fn();+ jest.spyOn(trace, 'getActiveSpan').mockReturnValue(undefined);++ telemetryErrorMiddleware(error, createRequest(), createResponse() as Response, next);++ expect(next).toHaveBeenCalledWith(error);+ });+});diff --git a/packages/api/src/telemetry/middleware.ts b/packages/api/src/telemetry/middleware.tsnew file mode 100644index 0000000000..d9c110c1b9--- /dev/null+++ b/packages/api/src/telemetry/middleware.ts@@ -0,0 +1,171 @@+import { SpanStatusCode, trace } from '@opentelemetry/api';+import type { Span, Attributes } from '@opentelemetry/api';+import type { NextFunction, Response } from 'express';+import type { ServerRequest } from '~/types';+import { getTelemetryRequestSpan } from './sdk';+import { DEFAULT_HEALTH_PATH } from './config';++const CLIENT_CLOSED_REQUEST_STATUS_CODE = 499;++type ExpressErrorValue =+ | Error+ | string+ | number+ | boolean+ | bigint+ | symbol+ | object+ | null+ | undefined;++function getUserId(req: ServerRequest): string | undefined {+ return req.user?.id;+}++function getTenantId(req: ServerRequest): string | undefined {+ return req.user?.tenantId;+}++function isHealthPath(req: ServerRequest): boolean {+ return req.path === DEFAULT_HEALTH_PATH;+}++function isApiPath(req: ServerRequest): boolean {+ return req.path === '/api' || req.path.startsWith('/api/');+}++function getRoutePath(req: ServerRequest): string {+ const routePath = req.route?.path;+ if (typeof routePath === 'string') {+ return `${req.baseUrl}${routePath}`;+ }++ if (isHealthPath(req)) {+ return '/health';+ }++ if (isApiPath(req)) {+ return '/api/*';+ }++ return 'spa_fallback';+}++function setIdentityAttributes(span: Span, req: ServerRequest): void {+ const userId = getUserId(req);+ const tenantId = getTenantId(req);++ if (!userId && !tenantId) {+ return;+ }++ const attributes: Attributes = {};++ if (userId) {+ attributes['enduser.id'] = userId;+ }++ if (tenantId) {+ attributes['librechat.tenant.id'] = tenantId;+ }++ span.setAttributes(attributes);+}++function setCompletionAttributes(+ span: Span,+ req: ServerRequest,+ res: Response,+ aborted = false,+): void {+ const statusCode = aborted ? CLIENT_CLOSED_REQUEST_STATUS_CODE : res.statusCode;+ const routePath = getRoutePath(req);+ const attributes: Attributes = {+ 'http.route': routePath,+ 'http.response.status_code': statusCode,+ };++ if (aborted) {+ attributes['librechat.request.aborted'] = true;+ }++ setIdentityAttributes(span, req);+ span.setAttributes(attributes);++ if (aborted || statusCode >= 500) {+ span.setStatus({ code: SpanStatusCode.ERROR });+ }+}++export function telemetryMiddleware(req: ServerRequest, res: Response, next: NextFunction): void {+ if (isHealthPath(req)) {+ next();+ return;+ }++ const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();+ if (!span) {+ next();+ return;+ }++ span.setAttributes({+ 'http.request.method': req.method,+ });++ let completed = false;+ const complete = () => {+ if (completed) {+ return;+ }+ completed = true;+ setCompletionAttributes(span, req, res);+ };++ const close = () => {+ if (completed) {+ return;+ }+ completed = true;+ setCompletionAttributes(span, req, res, !res.writableEnded);+ };++ res.once('finish', complete);+ res.once('close', close);+ next();+}++export function telemetryErrorMiddleware(+ err: ExpressErrorValue,+ req: ServerRequest,+ _res: Response,+ next: NextFunction,+): void {+ const span = getTelemetryRequestSpan(req) ?? trace.getActiveSpan();+ if (span) {+ const routePath = getRoutePath(req);+ if (err) {+ span.recordException(err instanceof Error ? err : String(err));+ }+ span.setStatus({ code: SpanStatusCode.ERROR });+ setIdentityAttributes(span, req);+ span.setAttributes({+ 'error.type': getErrorType(err),+ 'http.route': routePath,+ });+ }++ next(err);+}++function getErrorType(err: ExpressErrorValue): string {+ if (err instanceof Error) {+ return err.name || err.constructor.name;+ }++ if (err === null) {+ return 'null';+ }++ return typeof err;+}diff --git a/packages/api/src/telemetry/sdk.spec.ts b/packages/api/src/telemetry/sdk.spec.tsnew file mode 100644index 0000000000..548ecc6f2a--- /dev/null+++ b/packages/api/src/telemetry/sdk.spec.ts@@ -0,0 +1,357 @@+import { Socket } from 'node:net';+import { IncomingMessage } from 'node:http';+import type { Span } from '@opentelemetry/api';++interface HttpInstrumentationOptions {+ requestHook?: (span: Span, request: object) => void;+ startIncomingSpanHook?: (request: IncomingMessage) => Record<string, string>;+}++const mockStart = jest.fn();+const mockShutdown = jest.fn();+const mockNodeSDK = jest.fn(() => ({+ start: mockStart,+ shutdown: mockShutdown,+}));+const mockExpressInstrumentation = jest.fn(() => ({ name: 'express' }));+const mockHttpInstrumentation = jest.fn((options?: HttpInstrumentationOptions) => ({+ name: 'http',+ options,+}));+const mockIORedisInstrumentation = jest.fn(() => ({ name: 'ioredis' }));+const mockMongoDBInstrumentation = jest.fn(() => ({ name: 'mongodb' }));+const mockMongooseInstrumentation = jest.fn(() => ({ name: 'mongoose' }));+const mockUndiciInstrumentation = jest.fn(() => ({ name: 'undici' }));+const mockResourceFromAttributes = jest.fn((attributes: object) => ({ attributes }));++jest.mock(+ '@opentelemetry/sdk-node',+ () => ({+ NodeSDK: mockNodeSDK,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/instrumentation-express',+ () => ({+ ExpressInstrumentation: mockExpressInstrumentation,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/instrumentation-http',+ () => ({+ HttpInstrumentation: mockHttpInstrumentation,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/instrumentation-ioredis',+ () => ({+ IORedisInstrumentation: mockIORedisInstrumentation,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/instrumentation-mongodb',+ () => ({+ MongoDBInstrumentation: mockMongoDBInstrumentation,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/instrumentation-mongoose',+ () => ({+ MongooseInstrumentation: mockMongooseInstrumentation,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/instrumentation-undici',+ () => ({+ UndiciInstrumentation: mockUndiciInstrumentation,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/resources',+ () => ({+ resourceFromAttributes: mockResourceFromAttributes,+ }),+ { virtual: true },+);++jest.mock(+ '@opentelemetry/semantic-conventions',+ () => ({+ ATTR_SERVICE_NAME: 'service.name',+ ATTR_SERVICE_VERSION: 'service.version',+ }),+ { virtual: true },+);++async function flushSignalShutdown(): Promise<void> {+ await new Promise((resolve) => setImmediate(resolve));+}++describe('telemetry SDK lifecycle', () => {+ let emitWarningSpy: jest.SpyInstance;+ let getTelemetryRequestSpan: (typeof import('./sdk'))['getTelemetryRequestSpan'];+ let initializeTelemetry: (typeof import('./sdk'))['initializeTelemetry'];+ let resetTelemetryForTests: (typeof import('./sdk'))['resetTelemetryForTests'];+ let shutdownTelemetry: (typeof import('./sdk'))['shutdownTelemetry'];++ beforeEach(async () => {+ jest.clearAllMocks();+ ({ getTelemetryRequestSpan, initializeTelemetry, resetTelemetryForTests, shutdownTelemetry } =+ await import('./sdk'));+ await resetTelemetryForTests();+ Reflect.deleteProperty(globalThis, 'Bun');+ emitWarningSpy = jest.spyOn(process, 'emitWarning').mockImplementation(() => true);+ });++ afterEach(async () => {+ await resetTelemetryForTests();+ emitWarningSpy.mockRestore();+ Reflect.deleteProperty(globalThis, 'Bun');+ });++ it('does not initialize when tracing is disabled by default', () => {+ const controller = initializeTelemetry({});++ expect(controller.enabled).toBe(false);+ expect(controller.status).toBe('disabled');+ expect(mockNodeSDK).not.toHaveBeenCalled();+ });++ it('does not initialize when OTEL_SDK_DISABLED is true', () => {+ const controller = initializeTelemetry({+ OTEL_SDK_DISABLED: 'true',+ OTEL_TRACING_ENABLED: 'true',+ });++ expect(controller.enabled).toBe(false);+ expect(controller.status).toBe('disabled');+ expect(mockNodeSDK).not.toHaveBeenCalled();+ });++ it('does not initialize under Bun runtime', () => {+ Object.defineProperty(globalThis, 'Bun', {+ configurable: true,+ value: {},+ });++ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ expect(controller.enabled).toBe(false);+ expect(controller.status).toBe('disabled');+ expect(mockNodeSDK).not.toHaveBeenCalled();+ });++ it('starts the Node SDK once when enabled', () => {+ const first = initializeTelemetry({+ OTEL_SERVICE_NAME: 'librechat-test',+ OTEL_TRACING_ENABLED: 'true',+ });+ const second = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ expect(first.enabled).toBe(true);+ expect(first.status).toBe('started');+ expect(second.enabled).toBe(true);+ expect(mockNodeSDK).toHaveBeenCalledTimes(1);+ expect(mockStart).toHaveBeenCalledTimes(1);+ expect(mockResourceFromAttributes).toHaveBeenCalledWith({+ 'service.name': 'librechat-test',+ });+ expect(mockHttpInstrumentation).toHaveBeenCalledWith(+ expect.objectContaining({+ headersToSpanAttributes: {+ client: { requestHeaders: [], responseHeaders: [] },+ server: { requestHeaders: [], responseHeaders: [] },+ },+ }),+ );+ expect(mockExpressInstrumentation).toHaveBeenCalledTimes(1);+ expect(mockMongoDBInstrumentation).toHaveBeenCalledTimes(1);+ expect(mockMongooseInstrumentation).toHaveBeenCalledTimes(1);+ expect(mockIORedisInstrumentation).toHaveBeenCalledTimes(1);+ expect(mockUndiciInstrumentation).toHaveBeenCalledTimes(1);+ });++ it('tracks HTTP server request spans for completion updates', () => {+ const span = {} as Span;+ const request = new IncomingMessage(new Socket());+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });+ const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];+ const requestHook = instrumentationOptions?.requestHook;++ if (!requestHook) {+ throw new Error('HTTP instrumentation requestHook was not configured');+ }++ requestHook(span, request);++ expect(getTelemetryRequestSpan(request)).toBe(span);+ });++ it('redacts incoming URL attributes before HTTP spans are exported', () => {+ const request = new IncomingMessage(new Socket());+ request.url = '/oauth/callback?code=secret-code&state=secret-state';+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });+ const instrumentationOptions = mockHttpInstrumentation.mock.calls[0]?.[0];+ const startIncomingSpanHook = instrumentationOptions?.startIncomingSpanHook;++ if (!startIncomingSpanHook) {+ throw new Error('HTTP instrumentation startIncomingSpanHook was not configured');+ }++ const attributes = startIncomingSpanHook(request);++ expect(attributes).toEqual({+ 'http.target': 'spa_fallback?[REDACTED]',+ 'http.url': 'spa_fallback?[REDACTED]',+ 'url.full': 'spa_fallback?[REDACTED]',+ 'url.path': 'spa_fallback',+ 'url.query': '[REDACTED]',+ });+ expect(JSON.stringify(attributes)).not.toContain('secret-code');+ expect(JSON.stringify(attributes)).not.toContain('secret-state');+ });++ it('reflects lifecycle status from the controller getter', async () => {+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ expect(controller.status).toBe('started');+ await controller.shutdown();+ expect(controller.status).toBe('stopped');+ expect(controller.enabled).toBe(false);+ });++ it('handles async SDK start failures without throwing', async () => {+ mockStart.mockRejectedValueOnce(new Error('async start failed'));++ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ expect(controller.enabled).toBe(true);+ expect(controller.status).toBe('starting');+ await controller.shutdown();+ expect(controller.enabled).toBe(false);+ expect(controller.status).toBe('failed');+ expect(emitWarningSpy).toHaveBeenCalledWith(+ 'OpenTelemetry initialization failed: async start failed',+ { code: 'LIBRECHAT_OTEL' },+ );+ });++ it('returns failed status without throwing when SDK start fails', () => {+ mockStart.mockImplementationOnce(() => {+ throw new Error('start failed');+ });++ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ expect(controller.enabled).toBe(false);+ expect(controller.status).toBe('failed');+ expect(emitWarningSpy).toHaveBeenCalledWith(+ 'OpenTelemetry initialization failed: start failed',+ { code: 'LIBRECHAT_OTEL' },+ );+ });++ it('shuts down the active SDK idempotently', async () => {+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });+ await shutdownTelemetry();+ await shutdownTelemetry();++ expect(mockShutdown).toHaveBeenCalledTimes(1);+ });++ it('coalesces concurrent shutdown calls', async () => {+ let resolveShutdown: () => void = () => undefined;+ mockShutdown.mockReturnValueOnce(+ new Promise<void>((resolve) => {+ resolveShutdown = resolve;+ }),+ );+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ const firstShutdown = shutdownTelemetry();+ const secondShutdown = shutdownTelemetry();+ expect(mockShutdown).toHaveBeenCalledTimes(1);++ resolveShutdown();+ await Promise.all([firstShutdown, secondShutdown]);+ });++ it('keeps the active SDK available when shutdown fails', async () => {+ mockShutdown.mockRejectedValueOnce(new Error('shutdown failed'));+ const controller = initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ await expect(shutdownTelemetry()).rejects.toThrow('shutdown failed');+ expect(controller.status).toBe('started');++ await shutdownTelemetry();+ expect(mockShutdown).toHaveBeenCalledTimes(2);+ expect(controller.status).toBe('stopped');+ });++ it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(+ 'does not force process exit when another %s handler is registered',+ async (signal) => {+ const otherHandler = jest.fn();+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });+ process.once(signal, otherHandler);++ process.emit(signal, signal);+ await flushSignalShutdown();++ expect(mockShutdown).toHaveBeenCalledTimes(1);+ expect(otherHandler).toHaveBeenCalledTimes(1);+ expect(killSpy).not.toHaveBeenCalled();++ killSpy.mockRestore();+ },+ );++ it.each<NodeJS.Signals>(['SIGTERM', 'SIGINT'])(+ 'reraises the shutdown %s signal when telemetry is the only signal handler',+ async (signal) => {+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ process.emit(signal, signal);+ await flushSignalShutdown();++ expect(mockShutdown).toHaveBeenCalledTimes(1);+ expect(killSpy).toHaveBeenCalledWith(process.pid, signal);++ killSpy.mockRestore();+ },+ );++ it('warns and reraises the signal when shutdown rejects', async () => {+ mockShutdown.mockRejectedValueOnce(new Error('signal shutdown failed'));+ const killSpy = jest.spyOn(process, 'kill').mockImplementation(() => true);+ initializeTelemetry({ OTEL_TRACING_ENABLED: 'true' });++ process.emit('SIGTERM', 'SIGTERM');+ await flushSignalShutdown();++ expect(mockShutdown).toHaveBeenCalledTimes(1);+ expect(emitWarningSpy).toHaveBeenCalledWith(+ 'OpenTelemetry shutdown failed: signal shutdown failed',+ { code: 'LIBRECHAT_OTEL' },+ );+ expect(killSpy).toHaveBeenCalledWith(process.pid, 'SIGTERM');++ killSpy.mockRestore();+ });+});diff --git a/packages/api/src/telemetry/sdk.ts b/packages/api/src/telemetry/sdk.tsnew file mode 100644index 0000000000..49bd8a7ffc--- /dev/null+++ b/packages/api/src/telemetry/sdk.ts@@ -0,0 +1,328 @@+import { IncomingMessage } from 'node:http';+import { NodeSDK } from '@opentelemetry/sdk-node';+import { resourceFromAttributes } from '@opentelemetry/resources';+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';+import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici';+import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';+import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';+import { MongoDBInstrumentation } from '@opentelemetry/instrumentation-mongodb';+import { MongooseInstrumentation } from '@opentelemetry/instrumentation-mongoose';+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';+import type { NodeSDKConfiguration } from '@opentelemetry/sdk-node';+import type { Span, Attributes } from '@opentelemetry/api';+import type { TelemetryConfig, TelemetryStatus } from './config';+import { getTelemetryConfig } from './config';++export interface TelemetryController {+ readonly enabled: boolean;+ readonly status: TelemetryStatus;+ shutdown: () => Promise<void>;+}++const WARNING_CODE = 'LIBRECHAT_OTEL';+const REDACTED_QUERY_VALUE = '[REDACTED]';+const SIGNAL_SHUTDOWN_TIMEOUT_MS = 5_000;++interface RegisteredSignal {+ signal: NodeJS.Signals;+ listener: NodeJS.SignalsListener;+}++let activeSdk: NodeSDK | undefined;+let pendingSdk: NodeSDK | undefined;+let startPromise: Promise<void> | undefined;+let shutdownPromise: Promise<void> | undefined;+let status: TelemetryStatus = 'stopped';+let registeredSignals: RegisteredSignal[] = [];+let requestSpans = new WeakMap<IncomingMessage, Span>();++function isBunRuntime(): boolean {+ return Reflect.get(globalThis, 'Bun') != null;+}++function shouldIgnoreIncomingRequest(request: IncomingMessage, healthPath: string): boolean {+ return request.url === healthPath || request.url?.startsWith(`${healthPath}?`) === true;+}++function getIncomingUrlInfo(request: IncomingMessage): { hasQuery: boolean; pathname: string } {+ const rawUrl = request.url ?? '/';++ try {+ const parsedUrl = new URL(rawUrl, 'http://localhost');+ return {+ hasQuery: parsedUrl.search.length > 1,+ pathname: parsedUrl.pathname || '/',+ };+ } catch {+ const queryIndex = rawUrl.indexOf('?');+ return {+ hasQuery: queryIndex >= 0 && queryIndex < rawUrl.length - 1,+ pathname: queryIndex >= 0 ? rawUrl.slice(0, queryIndex) || '/' : rawUrl || '/',+ };+ }+}++function getLowCardinalityUrlPath(pathname: string, healthPath: string): string {+ if (pathname === healthPath) {+ return healthPath;+ }++ if (pathname === '/api' || pathname.startsWith('/api/')) {+ return '/api/*';+ }++ return 'spa_fallback';+}++function getSanitizedIncomingUrlAttributes(+ request: IncomingMessage,+ healthPath: string,+): Attributes {+ const { hasQuery, pathname } = getIncomingUrlInfo(request);+ const safePath = getLowCardinalityUrlPath(pathname, healthPath);+ const safeTarget = hasQuery ? `${safePath}?${REDACTED_QUERY_VALUE}` : safePath;+ const attributes: Attributes = {+ 'http.target': safeTarget,+ 'http.url': safeTarget,+ 'url.full': safeTarget,+ 'url.path': safePath,+ };++ if (hasQuery) {+ attributes['url.query'] = REDACTED_QUERY_VALUE;+ }++ return attributes;+}++function getResourceAttributes(config: TelemetryConfig): Attributes {+ const attributes: Attributes = {+ [ATTR_SERVICE_NAME]: config.serviceName,+ };++ if (config.serviceVersion) {+ attributes[ATTR_SERVICE_VERSION] = config.serviceVersion;+ }++ return attributes;+}++function createSdk(config: TelemetryConfig): NodeSDK {+ const sdkConfig: Partial<NodeSDKConfiguration> = {+ resource: resourceFromAttributes(getResourceAttributes(config)),+ instrumentations: [+ new HttpInstrumentation({+ headersToSpanAttributes: {+ client: { requestHeaders: [], responseHeaders: [] },+ server: { requestHeaders: [], responseHeaders: [] },+ },+ requestHook: (span: Span, request: object) => {+ if (request instanceof IncomingMessage) {+ requestSpans.set(request, span);+ }+ },+ startIncomingSpanHook: (request: IncomingMessage) =>+ getSanitizedIncomingUrlAttributes(request, config.healthPath),+ ignoreIncomingRequestHook: (request: IncomingMessage) =>+ shouldIgnoreIncomingRequest(request, config.healthPath),+ }),+ new ExpressInstrumentation(),+ new MongoDBInstrumentation(),+ new MongooseInstrumentation(),+ new IORedisInstrumentation(),+ new UndiciInstrumentation(),+ ],+ };++ return new NodeSDK(sdkConfig);+}++export function getTelemetryRequestSpan(request: IncomingMessage): Span | undefined {+ return requestSpans.get(request);+}++/**+ * NodeSDK.start has been synchronous in some supported OpenTelemetry versions+ * and promise-returning in others, so the lifecycle wrapper accepts either form.+ */+function startSdk(sdk: NodeSDK): void | Promise<void> {+ return (sdk as NodeSDK & { start: () => void | Promise<void> }).start();+}++function emitWarning(message: string): void {+ process.emitWarning(message, { code: WARNING_CODE });+}++function getErrorMessage(error: unknown): string {+ return error instanceof Error ? error.message : String(error);+}++function isControllerEnabled(): boolean {+ return status === 'starting' || status === 'started';+}++function makeController(): TelemetryController {+ return {+ get enabled() {+ return isControllerEnabled();+ },+ get status() {+ return status;+ },+ shutdown: shutdownTelemetry,+ };+}++function unregisterShutdownHandlers(): void {+ for (const { signal, listener } of registeredSignals) {+ process.removeListener(signal, listener);+ }+ registeredSignals = [];+}++function registerShutdownHandlers(): void {+ if (registeredSignals.length > 0) {+ return;+ }++ const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT'];+ registeredSignals = signals.map((signal) => {+ const listener: NodeJS.SignalsListener = () => {+ const shouldReraiseSignal = process.listenerCount(signal) === 0;+ withTimeout(shutdownTelemetry(), SIGNAL_SHUTDOWN_TIMEOUT_MS)+ .catch((error) => {+ emitWarning(`OpenTelemetry shutdown failed: ${getErrorMessage(error)}`);+ })+ .finally(() => {+ if (shouldReraiseSignal) {+ process.kill(process.pid, signal);+ }+ });+ };+ process.once(signal, listener);+ return { signal, listener };+ });+}++function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {+ let timeout: NodeJS.Timeout | undefined;+ const timeoutPromise = new Promise<never>((_, reject) => {+ timeout = setTimeout(() => {+ reject(new Error(`timed out after ${timeoutMs}ms`));+ }, timeoutMs);+ timeout.unref?.();+ });++ return Promise.race([promise, timeoutPromise]).finally(() => {+ if (timeout) {+ clearTimeout(timeout);+ }+ });+}++export function initializeTelemetry(env: NodeJS.ProcessEnv = process.env): TelemetryController {+ if (activeSdk || pendingSdk) {+ return makeController();+ }++ const config = getTelemetryConfig(env);+ if (!config.enabled || isBunRuntime()) {+ status = 'disabled';+ return makeController();+ }++ try {+ const sdk = createSdk(config);+ const result = startSdk(sdk);+ if (result) {+ pendingSdk = sdk;+ status = 'starting';+ const pendingStart = result+ .then(() => {+ if (pendingSdk === sdk) {+ pendingSdk = undefined;+ activeSdk = sdk;+ status = 'started';+ registerShutdownHandlers();+ }+ })+ .catch((error) => {+ if (pendingSdk === sdk) {+ pendingSdk = undefined;+ status = 'failed';+ emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);+ }+ });+ startPromise = pendingStart;+ void pendingStart.finally(() => {+ if (startPromise === pendingStart) {+ startPromise = undefined;+ }+ });+ return makeController();+ }++ activeSdk = sdk;+ status = 'started';+ registerShutdownHandlers();+ return makeController();+ } catch (error) {+ status = 'failed';+ emitWarning(`OpenTelemetry initialization failed: ${getErrorMessage(error)}`);+ return makeController();+ }+}++async function performShutdownTelemetry(): Promise<void> {+ if (startPromise) {+ await startPromise;+ }++ if (!activeSdk) {+ status = status === 'started' ? 'stopped' : status;+ return;+ }++ const sdk = activeSdk;+ try {+ await sdk.shutdown();+ activeSdk = undefined;+ status = 'stopped';+ unregisterShutdownHandlers();+ } catch (error) {+ status = 'started';+ throw error;+ }+}++export function shutdownTelemetry(): Promise<void> {+ if (!shutdownPromise) {+ shutdownPromise = performShutdownTelemetry().finally(() => {+ shutdownPromise = undefined;+ });+ }++ return shutdownPromise;+}++export async function resetTelemetryForTests(): Promise<void> {+ try {+ if (startPromise) {+ await startPromise.catch(() => undefined);+ }++ if (shutdownPromise) {+ await shutdownPromise.catch(() => undefined);+ } else if (activeSdk) {+ await Promise.resolve(activeSdk.shutdown()).catch(() => undefined);+ }+ } finally {+ activeSdk = undefined;+ pendingSdk = undefined;+ startPromise = undefined;+ shutdownPromise = undefined;+ status = 'stopped';+ requestSpans = new WeakMap<IncomingMessage, Span>();+ unregisterShutdownHandlers();+ }+}