Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🪭 feat: Add opt-in Langfuse fanout gateway + collector (#13872)
Ravi Kumar La0529c9af7fb
diff --git a/.env.example b/.env.exampleindex 09749f7e05..75b4c490aa 100644--- a/.env.example+++ b/.env.example@@ -132,6 +132,60 @@ NODE_MAX_OLD_SPACE_SIZE=6144# LANGFUSE_SECRET_KEY=# LANGFUSE_BASE_URL=+# Optional Langfuse fanout for tenant-scoped Langfuse projects.+# The fanout gateway is opt-in: add docker-compose.langfuse-fanout.yml,+# deploy-compose.langfuse-fanout.yml, or enable helm langfuseFanout.+# Tenant public/secret keys are read from LibreChat tenant app configuration.+# Tenant Langfuse base URLs must be set in tenant app configuration and match+# one of the known startup destinations. Tenant API keys can be added or changed+# at runtime through tenant app configuration.+# See otel/langfuse-fanout/README.md.+# LANGFUSE_FANOUT_ENABLED=false+# LANGFUSE_FANOUT_COLLECTOR_URL=http://langfuse-fanout-collector:4318+# Emergency switch: unset/false defaults enabled; set true to keep central fanout export but skip tenant trace/score export.+# LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=false+# Langfuse Cloud base URL options: https://cloud.langfuse.com (EU),+# https://us.cloud.langfuse.com (US), https://jp.cloud.langfuse.com (JP).+# Gateway-only central trace/media export URL. LibreChat feedback scores use+# LANGFUSE_BASE_URL, so set both URLs to the same non-EU region when applicable.+# LANGFUSE_FANOUT_CENTRAL_BASE_URL=https://cloud.langfuse.com+# Gateway-only Basic auth header for central trace/media export. LibreChat feedback+# scores use LANGFUSE_PUBLIC_KEY/LANGFUSE_SECRET_KEY instead.+# LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER=Basic <base64-public-colon-secret>+# Compose's included gateway config supports the three listed destination keys.+# Add custom keys only when the gateway is started with matching destination URLs.+# LANGFUSE_FANOUT_TENANT_DESTINATIONS=eu=https://cloud.langfuse.com,us=https://us.cloud.langfuse.com,jp=https://jp.cloud.langfuse.com+# Compose's collector config routes only these destination keys. The gateway+# fails startup when LANGFUSE_FANOUT_TENANT_DESTINATIONS contains another key.+# LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS=eu,us,jp+# Gateway base URL used to build one-time media upload URLs. Compose sets this+# to its private service URL; Helm derives an internal service URL unless set.+# LANGFUSE_FANOUT_PUBLIC_URL=http://langfuse-fanout-collector:4318+# Internal gateway-to-collector trace endpoint. Compose sets this automatically.+# LANGFUSE_FANOUT_TRACE_COLLECTOR_URL=http://langfuse-fanout-otel:4319+# Redis-backed one-time upload plans let multiple gateway pods handle Langfuse+# media create/upload requests. Compose sets this to its private Redis service.+# LANGFUSE_FANOUT_REDIS_URI=redis://langfuse-fanout-redis:6379+# LANGFUSE_FANOUT_REDIS_USERNAME=+# LANGFUSE_FANOUT_REDIS_PASSWORD=+# LANGFUSE_FANOUT_REDIS_KEY_PREFIX=langfuse-fanout+# Internal collector receiver bind address. Helm uses 127.0.0.1 because the+# collector is a sidecar; Compose uses 0.0.0.0 on the private fanout network.+# LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT=0.0.0.0:4319+# Static Compose collector destination URLs. Helm derives these from values.+# LANGFUSE_FANOUT_TENANT_EU_BASE_URL=https://cloud.langfuse.com+# LANGFUSE_FANOUT_TENANT_US_BASE_URL=https://us.cloud.langfuse.com+# LANGFUSE_FANOUT_TENANT_JP_BASE_URL=https://jp.cloud.langfuse.com+# LANGFUSE_FANOUT_UPSTREAM_TIMEOUT=30s+# Optional bearer token for scraping the fanout gateway /metrics endpoint.+# If unset, /metrics returns 401. The gateway also accepts METRICS_SECRET when present.+# LANGFUSE_FANOUT_METRICS_SECRET=+# LANGFUSE_FANOUT_MEMORY_LIMIT_MIB=256+# LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB=64+# LANGFUSE_FANOUT_BATCH_TIMEOUT=1s+# LANGFUSE_FANOUT_BATCH_SEND_SIZE=128+# LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT=1000+#=======================## OpenTelemetry Tracing ##=======================#diff --git a/.github/workflows/langfuse-fanout.yml b/.github/workflows/langfuse-fanout.ymlnew file mode 100644index 0000000000..b9aeaf4616--- /dev/null+++ b/.github/workflows/langfuse-fanout.yml@@ -0,0 +1,43 @@+name: Langfuse Fanout++on:+ workflow_dispatch:+ pull_request:+ paths:+ - '.github/workflows/langfuse-fanout.yml'+ - 'otel/langfuse-fanout/**'++permissions:+ contents: read++concurrency:+ group: langfuse-fanout-${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: true++jobs:+ go-tests:+ name: Go tests+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@v4++ - name: Set up Go+ uses: actions/setup-go@v5+ with:+ go-version-file: otel/langfuse-fanout/go.mod+ cache-dependency-path: otel/langfuse-fanout/go.sum++ - name: Check Go formatting+ working-directory: otel/langfuse-fanout+ run: |+ unformatted="$(gofmt -l .)"+ if [ -n "$unformatted" ]; then+ echo "::error::Go files are not gofmt-formatted:"+ echo "$unformatted"+ exit 1+ fi++ - name: Run Go tests+ working-directory: otel/langfuse-fanout+ run: go test ./...diff --git a/api/server/routes/__tests__/messages-delete.spec.js b/api/server/routes/__tests__/messages-delete.spec.jsindex 36c4e8e9e6..cf3a7fb775 100644--- a/api/server/routes/__tests__/messages-delete.spec.js+++ b/api/server/routes/__tests__/messages-delete.spec.js@@ -11,6 +11,8 @@ jest.mock('@librechat/agents', () => ({jest.mock('@librechat/api', () => ({unescapeLaTeX: jest.fn((x) => x),countTokens: jest.fn().mockResolvedValue(10),+ sendFeedbackScore: jest.fn().mockResolvedValue(undefined),+ traceIdForMessage: jest.fn((messageId) => `trace-${messageId}`),}));jest.mock('@librechat/data-schemas', () => ({@@ -49,6 +51,7 @@ jest.mock('~/server/middleware/requireJwtAuth', () => (req, res, next) => next()jest.mock('~/server/middleware', () => ({requireJwtAuth: (req, res, next) => next(),validateMessageReq: (req, res, next) => next(),+ configMiddleware: (req, res, next) => next(),}));jest.mock('~/db/models', () => ({diff --git a/api/server/routes/messages.js b/api/server/routes/messages.jsindex 2aaed7cd5f..1f6188b379 100644--- a/api/server/routes/messages.js+++ b/api/server/routes/messages.js@@ -10,7 +10,7 @@ const {mergeQuotedTextForCount,} = require('@librechat/api');const { findAllArtifacts, replaceArtifactContent } = require('~/server/services/Artifacts/update');-const { requireJwtAuth, validateMessageReq } = require('~/server/middleware');+const { requireJwtAuth, validateMessageReq, configMiddleware } = require('~/server/middleware');const db = require('~/models');const router = express.Router();@@ -398,49 +398,56 @@ router.put('/:conversationId/:messageId', validateMessageReq, async (req, res) =}});-router.put('/:conversationId/:messageId/feedback', validateMessageReq, async (req, res) => {- try {- const { conversationId, messageId } = req.params;- const { feedback } = req.body;+router.put(+ '/:conversationId/:messageId/feedback',+ validateMessageReq,+ configMiddleware,+ async (req, res) => {+ try {+ const { conversationId, messageId } = req.params;+ const { feedback } = req.body;++ const updatedMessage = await db.updateMessage(+ req?.user?.id,+ {+ messageId,+ feedback: feedback || null,+ },+ { context: 'updateFeedback' },+ );- const updatedMessage = await db.updateMessage(- req?.user?.id,- {- messageId,- feedback: feedback || null,- },- { context: 'updateFeedback' },- );+ // Best-effort: Assistants messages do not have deterministic AgentRun traces.+ if (!isAssistantsEndpoint(updatedMessage.endpoint)) {+ sendFeedbackScore({+ traceId: traceIdForMessage(messageId),+ feedback: updatedMessage.feedback,+ appConfig: req.config,+ metadata: {+ messageId: updatedMessage.messageId ?? messageId,+ parentMessageId: updatedMessage.parentMessageId,+ conversationId: updatedMessage.conversationId ?? conversationId,+ sessionId: updatedMessage.conversationId ?? conversationId,+ userId: req?.user?.id,+ tenantId: req?.user?.tenantId,+ endpoint: updatedMessage.endpoint,+ sender: updatedMessage.sender,+ isCreatedByUser: updatedMessage.isCreatedByUser,+ tokenCount: updatedMessage.tokenCount,+ },+ }).catch((err) => logger.error('[langfuse] feedback score failed:', err));+ }- // Best-effort: Assistants messages do not have deterministic AgentRun traces.- if (!isAssistantsEndpoint(updatedMessage.endpoint)) {- sendFeedbackScore({- traceId: traceIdForMessage(messageId),+ res.json({+ messageId,+ conversationId,feedback: updatedMessage.feedback,- metadata: {- messageId: updatedMessage.messageId ?? messageId,- parentMessageId: updatedMessage.parentMessageId,- conversationId: updatedMessage.conversationId ?? conversationId,- sessionId: updatedMessage.conversationId ?? conversationId,- userId: req?.user?.id,- endpoint: updatedMessage.endpoint,- sender: updatedMessage.sender,- isCreatedByUser: updatedMessage.isCreatedByUser,- tokenCount: updatedMessage.tokenCount,- },- }).catch((err) => logger.error('[langfuse] feedback score failed:', err));+ });+ } catch (error) {+ logger.error('Error updating message feedback:', error);+ res.status(500).json({ error: 'Failed to update feedback' });}-- res.json({- messageId,- conversationId,- feedback: updatedMessage.feedback,- });- } catch (error) {- logger.error('Error updating message feedback:', error);- res.status(500).json({ error: 'Failed to update feedback' });- }-});+ },+);router.delete('/:conversationId/:messageId', validateMessageReq, async (req, res) => {try {diff --git a/deploy-compose.langfuse-fanout.yml b/deploy-compose.langfuse-fanout.ymlnew file mode 100644index 0000000000..15f57c1d88--- /dev/null+++ b/deploy-compose.langfuse-fanout.yml@@ -0,0 +1,76 @@+services:+ api:+ depends_on:+ - langfuse-fanout-collector+ environment:+ - LANGFUSE_FANOUT_ENABLED=true+ - LANGFUSE_FANOUT_COLLECTOR_URL=http://langfuse-fanout-collector:4318+ - LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=${LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED-false}+ - LANGFUSE_FANOUT_TENANT_DESTINATIONS=${LANGFUSE_FANOUT_TENANT_DESTINATIONS:-eu=https://cloud.langfuse.com,us=https://us.cloud.langfuse.com,jp=https://jp.cloud.langfuse.com}+ networks:+ - default+ - langfuse-fanout++ langfuse-fanout-collector:+ build:+ context: .+ dockerfile: otel/langfuse-fanout/Dockerfile+ image: librechat-langfuse-fanout:local+ restart: always+ depends_on:+ - langfuse-fanout-otel+ - langfuse-fanout-redis+ env_file:+ - .env+ environment:+ - LANGFUSE_FANOUT_CENTRAL_BASE_URL=${LANGFUSE_FANOUT_CENTRAL_BASE_URL:-https://cloud.langfuse.com}+ - LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER=${LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER:?Set LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER to central Langfuse Basic auth header}+ - LANGFUSE_FANOUT_TENANT_DESTINATIONS=${LANGFUSE_FANOUT_TENANT_DESTINATIONS:-eu=https://cloud.langfuse.com,us=https://us.cloud.langfuse.com,jp=https://jp.cloud.langfuse.com}+ - LANGFUSE_FANOUT_UPSTREAM_TIMEOUT=${LANGFUSE_FANOUT_UPSTREAM_TIMEOUT:-30s}+ - LANGFUSE_FANOUT_PUBLIC_URL=${LANGFUSE_FANOUT_PUBLIC_URL:-http://langfuse-fanout-collector:4318}+ - LANGFUSE_FANOUT_TRACE_COLLECTOR_URL=http://langfuse-fanout-otel:4319+ - LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS=eu,us,jp+ - LANGFUSE_FANOUT_REDIS_URI=${LANGFUSE_FANOUT_REDIS_URI:-redis://langfuse-fanout-redis:6379}+ - LANGFUSE_FANOUT_REDIS_USERNAME=${LANGFUSE_FANOUT_REDIS_USERNAME:-}+ - LANGFUSE_FANOUT_REDIS_PASSWORD=${LANGFUSE_FANOUT_REDIS_PASSWORD:-}+ - LANGFUSE_FANOUT_REDIS_KEY_PREFIX=${LANGFUSE_FANOUT_REDIS_KEY_PREFIX:-langfuse-fanout}+ expose:+ - '4318'+ networks:+ - langfuse-fanout++ langfuse-fanout-redis:+ image: redis:7.4-alpine+ restart: always+ expose:+ - '6379'+ networks:+ - langfuse-fanout++ langfuse-fanout-otel:+ image: otel/opentelemetry-collector-contrib:0.143.0+ restart: always+ command: ['--config=/etc/otelcol/otelcol.yaml']+ env_file:+ - .env+ environment:+ - LANGFUSE_FANOUT_CENTRAL_BASE_URL=${LANGFUSE_FANOUT_CENTRAL_BASE_URL:-https://cloud.langfuse.com}+ - LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER=${LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER:?Set LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER to central Langfuse Basic auth header}+ - LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT=0.0.0.0:4319+ - LANGFUSE_FANOUT_TENANT_EU_BASE_URL=${LANGFUSE_FANOUT_TENANT_EU_BASE_URL:-https://cloud.langfuse.com}+ - LANGFUSE_FANOUT_TENANT_US_BASE_URL=${LANGFUSE_FANOUT_TENANT_US_BASE_URL:-https://us.cloud.langfuse.com}+ - LANGFUSE_FANOUT_TENANT_JP_BASE_URL=${LANGFUSE_FANOUT_TENANT_JP_BASE_URL:-https://jp.cloud.langfuse.com}+ - LANGFUSE_FANOUT_MEMORY_LIMIT_MIB=${LANGFUSE_FANOUT_MEMORY_LIMIT_MIB:-256}+ - LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB=${LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB:-64}+ - LANGFUSE_FANOUT_BATCH_TIMEOUT=${LANGFUSE_FANOUT_BATCH_TIMEOUT:-1s}+ - LANGFUSE_FANOUT_BATCH_SEND_SIZE=${LANGFUSE_FANOUT_BATCH_SEND_SIZE:-128}+ - LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT=${LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT:-1000}+ volumes:+ - ./otel/langfuse-fanout/otelcol.yaml:/etc/otelcol/otelcol.yaml:ro+ expose:+ - '4319'+ networks:+ - langfuse-fanout++networks:+ langfuse-fanout:diff --git a/docker-compose.langfuse-fanout.yml b/docker-compose.langfuse-fanout.ymlnew file mode 100644index 0000000000..15f57c1d88--- /dev/null+++ b/docker-compose.langfuse-fanout.yml@@ -0,0 +1,76 @@+services:+ api:+ depends_on:+ - langfuse-fanout-collector+ environment:+ - LANGFUSE_FANOUT_ENABLED=true+ - LANGFUSE_FANOUT_COLLECTOR_URL=http://langfuse-fanout-collector:4318+ - LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=${LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED-false}+ - LANGFUSE_FANOUT_TENANT_DESTINATIONS=${LANGFUSE_FANOUT_TENANT_DESTINATIONS:-eu=https://cloud.langfuse.com,us=https://us.cloud.langfuse.com,jp=https://jp.cloud.langfuse.com}+ networks:+ - default+ - langfuse-fanout++ langfuse-fanout-collector:+ build:+ context: .+ dockerfile: otel/langfuse-fanout/Dockerfile+ image: librechat-langfuse-fanout:local+ restart: always+ depends_on:+ - langfuse-fanout-otel+ - langfuse-fanout-redis+ env_file:+ - .env+ environment:+ - LANGFUSE_FANOUT_CENTRAL_BASE_URL=${LANGFUSE_FANOUT_CENTRAL_BASE_URL:-https://cloud.langfuse.com}+ - LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER=${LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER:?Set LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER to central Langfuse Basic auth header}+ - LANGFUSE_FANOUT_TENANT_DESTINATIONS=${LANGFUSE_FANOUT_TENANT_DESTINATIONS:-eu=https://cloud.langfuse.com,us=https://us.cloud.langfuse.com,jp=https://jp.cloud.langfuse.com}+ - LANGFUSE_FANOUT_UPSTREAM_TIMEOUT=${LANGFUSE_FANOUT_UPSTREAM_TIMEOUT:-30s}+ - LANGFUSE_FANOUT_PUBLIC_URL=${LANGFUSE_FANOUT_PUBLIC_URL:-http://langfuse-fanout-collector:4318}+ - LANGFUSE_FANOUT_TRACE_COLLECTOR_URL=http://langfuse-fanout-otel:4319+ - LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS=eu,us,jp+ - LANGFUSE_FANOUT_REDIS_URI=${LANGFUSE_FANOUT_REDIS_URI:-redis://langfuse-fanout-redis:6379}+ - LANGFUSE_FANOUT_REDIS_USERNAME=${LANGFUSE_FANOUT_REDIS_USERNAME:-}+ - LANGFUSE_FANOUT_REDIS_PASSWORD=${LANGFUSE_FANOUT_REDIS_PASSWORD:-}+ - LANGFUSE_FANOUT_REDIS_KEY_PREFIX=${LANGFUSE_FANOUT_REDIS_KEY_PREFIX:-langfuse-fanout}+ expose:+ - '4318'+ networks:+ - langfuse-fanout++ langfuse-fanout-redis:+ image: redis:7.4-alpine+ restart: always+ expose:+ - '6379'+ networks:+ - langfuse-fanout++ langfuse-fanout-otel:+ image: otel/opentelemetry-collector-contrib:0.143.0+ restart: always+ command: ['--config=/etc/otelcol/otelcol.yaml']+ env_file:+ - .env+ environment:+ - LANGFUSE_FANOUT_CENTRAL_BASE_URL=${LANGFUSE_FANOUT_CENTRAL_BASE_URL:-https://cloud.langfuse.com}+ - LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER=${LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER:?Set LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER to central Langfuse Basic auth header}+ - LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT=0.0.0.0:4319+ - LANGFUSE_FANOUT_TENANT_EU_BASE_URL=${LANGFUSE_FANOUT_TENANT_EU_BASE_URL:-https://cloud.langfuse.com}+ - LANGFUSE_FANOUT_TENANT_US_BASE_URL=${LANGFUSE_FANOUT_TENANT_US_BASE_URL:-https://us.cloud.langfuse.com}+ - LANGFUSE_FANOUT_TENANT_JP_BASE_URL=${LANGFUSE_FANOUT_TENANT_JP_BASE_URL:-https://jp.cloud.langfuse.com}+ - LANGFUSE_FANOUT_MEMORY_LIMIT_MIB=${LANGFUSE_FANOUT_MEMORY_LIMIT_MIB:-256}+ - LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB=${LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB:-64}+ - LANGFUSE_FANOUT_BATCH_TIMEOUT=${LANGFUSE_FANOUT_BATCH_TIMEOUT:-1s}+ - LANGFUSE_FANOUT_BATCH_SEND_SIZE=${LANGFUSE_FANOUT_BATCH_SEND_SIZE:-128}+ - LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT=${LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT:-1000}+ volumes:+ - ./otel/langfuse-fanout/otelcol.yaml:/etc/otelcol/otelcol.yaml:ro+ expose:+ - '4319'+ networks:+ - langfuse-fanout++networks:+ langfuse-fanout:diff --git a/helm/librechat/readme.md b/helm/librechat/readme.mdindex 685c41e1e7..e30addf8c5 100755--- a/helm/librechat/readme.md+++ b/helm/librechat/readme.md@@ -53,3 +53,45 @@ also register this LibreChat callback URL with your identity provider:```texthttps://<librechat-domain>/api/admin/oauth/openid/callback```++## Langfuse Fanout++The chart can optionally deploy a Langfuse fanout gateway with an internal+OpenTelemetry Collector sidecar. The gateway handles Langfuse media fanout and+proxies traces to the collector; the collector forwards tenant-scoped Langfuse+traces to both a central Langfuse project and the tenant Langfuse project. It is+disabled by default.++When enabled, the chart also sets `LANGFUSE_FANOUT_ENABLED` and+`LANGFUSE_FANOUT_COLLECTOR_URL` for the LibreChat app unless those values are+already provided in `librechat.configEnv`.++Set `librechat.configEnv.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=true` to keep+central trace export flowing through the fanout gateway while disabling tenant trace+and score export. When omitted, false, or blank, tenant export remains available+if tenant keys and a known destination are configured.++Langfuse tenant base URLs are selected from the startup-configured destination+map rendered into LibreChat and the fanout gateway. Tenant API keys can still be added+through tenant app configuration at runtime without restarting either component.+The internal collector provides trace memory limiting, batching, tenant routing,+and removal of LibreChat-only routing attributes before export.++The fanout gateway stores one-time media upload plans in Redis so media create+and byte-upload requests can land on different gateway replicas. Set+`langfuseFanout.redis.uri` for an external Redis service, or enable the bundled+Redis chart with `redis.enabled=true` and let the chart derive the internal URI.+Scale the gateway manually with `langfuseFanout.replicaCount`; the chart does+not create a fanout HPA.+The internal collector receiver is bound to `127.0.0.1:4319` by default because+only the gateway sidecar should send traces to it.++The gateway exposes Prometheus metrics at `/metrics`. Configure+`langfuseFanout.metrics.secret.name` and `.key` to pass a bearer token secret to+the gateway; if omitted, `/metrics` returns 401. Use+`langfuseFanout.service.annotations` for scrape annotations when your cluster+uses annotation-based discovery. The gateway container also has configurable+`/healthz` liveness and readiness probes under `langfuseFanout`.++See [`otel/langfuse-fanout/README.md`](../../otel/langfuse-fanout/README.md)+for the central Langfuse secret and values example.diff --git a/helm/librechat/templates/_helpers.tpl b/helm/librechat/templates/_helpers.tplindex 4c242d9582..fc05ea974c 100755--- a/helm/librechat/templates/_helpers.tpl+++ b/helm/librechat/templates/_helpers.tpl@@ -28,13 +28,21 @@ If release name contains chart name it will be used as a full name.Common labels*/}}{{- define "librechat.labels" -}}-helm.sh/chart: {{ include "librechat.chart" . }}-{{ include "librechat.selectorLabels" . }}-{{- if .Chart.AppVersion }}-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}+{{- include "librechat.standardLabels" (dict "root" . "selectorLabels" (include "librechat.selectorLabels" .)) }}{{- end }}-app.kubernetes.io/managed-by: {{ .Release.Service }}++{{/*+Standard labels for chart-managed workloads.+*/}}+{{- define "librechat.standardLabels" -}}+{{- $root := .root -}}+helm.sh/chart: {{ include "librechat.chart" $root }}+{{ .selectorLabels }}+{{- if $root.Chart.AppVersion }}+app.kubernetes.io/version: {{ $root.Chart.AppVersion | quote }}{{- end }}+app.kubernetes.io/managed-by: {{ $root.Release.Service }}+{{- end -}}{{/*Selector labels@@ -44,6 +52,96 @@ app.kubernetes.io/name: {{ include "librechat.fullname" . }}app.kubernetes.io/instance: {{ .Release.Name }}{{- end }}+{{/*+Langfuse fanout collector service name.+*/}}+{{- define "librechat.langfuseFanout.fullname" -}}+{{- printf "%s-langfuse-fanout" (include "librechat.fullname" .) | trunc 63 | trimSuffix "-" }}+{{- end }}++{{/*+Langfuse fanout collector selector labels.+*/}}+{{- define "librechat.langfuseFanout.selectorLabels" -}}+app.kubernetes.io/name: {{ include "librechat.langfuseFanout.fullname" . }}+app.kubernetes.io/instance: {{ .Release.Name }}+{{- end }}++{{/*+Langfuse fanout collector labels.+*/}}+{{- define "librechat.langfuseFanout.labels" -}}+{{- include "librechat.standardLabels" (dict "root" . "selectorLabels" (include "librechat.langfuseFanout.selectorLabels" .)) }}+{{- end }}++{{/*+Validate Langfuse fanout destination keys. LibreChat normalizes destination+keys to lowercase before putting them on trace attributes, so Helm values must+already use the same lowercase key shape for collector routing to match.+*/}}+{{- define "librechat.langfuseFanout.validateDestinationKey" -}}+{{- $name := printf "%v" . -}}+{{- if not (regexMatch "^[a-z][a-z0-9_-]*$" $name) -}}+{{- fail (printf "langfuseFanout.tenant.destinations key %q is invalid; use lowercase keys matching ^[a-z][a-z0-9_-]*$ so LibreChat trace attributes match collector routes" $name) -}}+{{- end -}}+{{- end }}++{{/*+Render the environment variable name used by the collector for a destination.+*/}}+{{- define "librechat.langfuseFanout.destinationBaseUrlEnvName" -}}+{{- printf "LANGFUSE_FANOUT_TENANT_%s_BASE_URL" (. | printf "%v" | upper | replace "-" "_") -}}+{{- end }}++{{/*+Validate the full destination key set. Destination keys can contain hyphens and+underscores, but the collector base URL env vars replace hyphens with+underscores. Reject keys such as foo-bar and foo_bar because they would render+the same LANGFUSE_FANOUT_TENANT_FOO_BAR_BASE_URL env var.+*/}}+{{- define "librechat.langfuseFanout.validateDestinationKeys" -}}+{{- $seenEnvNames := dict -}}+{{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations -}}+{{- include "librechat.langfuseFanout.validateDestinationKey" $name -}}+{{- $envName := include "librechat.langfuseFanout.destinationBaseUrlEnvName" $name -}}+{{- if hasKey $seenEnvNames $envName -}}+{{- fail (printf "langfuseFanout.tenant.destinations keys %q and %q both render %s; use destination keys that remain unique after uppercasing and replacing '-' with '_' for env vars" (get $seenEnvNames $envName) $name $envName) -}}+{{- end -}}+{{- $_ := set $seenEnvNames $envName $name -}}+{{- end -}}+{{- end }}++{{/*+Render the fanout destination list consumed by LibreChat and the fanout gateway.+*/}}+{{- define "librechat.langfuseFanout.tenantDestinationsEnv" -}}+{{- include "librechat.langfuseFanout.validateDestinationKeys" . -}}+{{- $tenantDestinations := list -}}+{{- range $name, $destination := .Values.langfuseFanout.tenant.destinations -}}+{{- $tenantDestinations = append $tenantDestinations (printf "%s=%s" $name $destination.baseUrl) -}}+{{- end -}}+{{- join "," $tenantDestinations -}}+{{- end }}++{{/*+Render the fanout destination key list consumed by the gateway as a startup+guard against media destinations the collector cannot route traces to.+*/}}+{{- define "librechat.langfuseFanout.tenantDestinationKeysEnv" -}}+{{- include "librechat.langfuseFanout.validateDestinationKeys" . -}}+{{- $tenantDestinationKeys := list -}}+{{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations -}}+{{- $tenantDestinationKeys = append $tenantDestinationKeys $name -}}+{{- end -}}+{{- join "," $tenantDestinationKeys -}}+{{- end }}++{{/*+Bundled Redis URI used when the Redis subchart is enabled.+*/}}+{{- define "librechat.bundledRedisURI" -}}+{{- printf "redis://%s-master.%s.svc.cluster.local:6379" (include "common.names.fullname" .Subcharts.redis) (.Release.Namespace | lower) -}}+{{- end }}{{/*RAG Selector labelsdiff --git a/helm/librechat/templates/configmap-env.yaml b/helm/librechat/templates/configmap-env.yamlindex 5fd43940d2..e6dc7a5855 100755--- a/helm/librechat/templates/configmap-env.yaml+++ b/helm/librechat/templates/configmap-env.yaml@@ -19,11 +19,25 @@ data:USE_REDIS: "true"{{- end }}{{- if and (not (dig "configEnv" "REDIS_URI" "" .Values.librechat)) .Values.redis.enabled }}- REDIS_URI: redis://{{ include "common.names.fullname" .Subcharts.redis }}-master.{{ .Release.Namespace | lower }}.svc.cluster.local:6379+ REDIS_URI: {{ include "librechat.bundledRedisURI" . }}{{- end }}{{- if and $adminPanelUrl (not $configAdminPanelUrl) }}ADMIN_PANEL_URL: {{ $adminPanelUrl | quote }}{{- end }}+ {{- if and .Values.langfuseFanout.enabled (not (hasKey $configEnv "LANGFUSE_FANOUT_ENABLED")) }}+ LANGFUSE_FANOUT_ENABLED: "true"+ {{- end }}+ {{- if and .Values.langfuseFanout.enabled (not (hasKey $configEnv "LANGFUSE_FANOUT_COLLECTOR_URL")) }}+ LANGFUSE_FANOUT_COLLECTOR_URL: http://{{ include "librechat.langfuseFanout.fullname" . }}.{{ .Release.Namespace | lower }}.svc.cluster.local:{{ .Values.langfuseFanout.service.port }}+ {{- end }}+ {{- if and .Values.langfuseFanout.enabled (not (hasKey $configEnv "LANGFUSE_FANOUT_TENANT_DESTINATIONS")) }}+ {{- include "librechat.langfuseFanout.validateDestinationKeys" . }}+ {{- $tenantDestinations := list }}+ {{- range $name, $destination := .Values.langfuseFanout.tenant.destinations }}+ {{- $tenantDestinations = append $tenantDestinations (printf "%s=%s" $name $destination.baseUrl) }}+ {{- end }}+ LANGFUSE_FANOUT_TENANT_DESTINATIONS: {{ join "," $tenantDestinations | quote }}+ {{- end }}{{- if $configEnv }}{{- $renderedConfigEnv := $configEnv }}{{- if and $adminPanelUrl (hasKey $configEnv "ADMIN_PANEL_URL") (not $configAdminPanelUrl) }}diff --git a/helm/librechat/templates/langfuse-fanout-configmap.yaml b/helm/librechat/templates/langfuse-fanout-configmap.yamlnew file mode 100644index 0000000000..2ae8a22ea5--- /dev/null+++ b/helm/librechat/templates/langfuse-fanout-configmap.yaml@@ -0,0 +1,101 @@+{{- if .Values.langfuseFanout.enabled }}+{{- include "librechat.langfuseFanout.validateDestinationKeys" . }}+apiVersion: v1+kind: ConfigMap+metadata:+ name: {{ include "librechat.langfuseFanout.fullname" . }}-config+ labels:+ {{- include "librechat.langfuseFanout.labels" . | nindent 4 }}+data:+ otelcol.yaml: |+ extensions:+ headers_setter/tenant_passthrough:+ headers:+ - action: upsert+ key: Authorization+ from_context: authorization++ receivers:+ otlp:+ protocols:+ http:+ endpoint: ${env:LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT}+ include_metadata: true+ traces_url_path: /api/public/otel/v1/traces++ connectors:+ routing/langfuse_tenant_destination:+ error_mode: ignore+ table:+ {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}+ {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}+ - context: span+ condition: attributes["librechat.langfuse.destination"] == {{ $name | quote }}+ pipelines: [traces/tenant_{{ $name }}]+ {{- end }}++ processors:+ memory_limiter:+ check_interval: 1s+ limit_mib: ${env:LANGFUSE_FANOUT_MEMORY_LIMIT_MIB}+ spike_limit_mib: ${env:LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB}+ filter/tenant_export:+ error_mode: ignore+ traces:+ span:+ - attributes["librechat.langfuse.tenant_export.enabled"] != "true"+ attributes/drop_librechat_routing:+ actions:+ - key: librechat.langfuse.tenant_export.enabled+ action: delete+ - key: librechat.langfuse.destination+ action: delete+ batch/central:+ timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}+ send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}+ {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}+ {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}+ batch/by_auth_{{ $name }}:+ timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}+ send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}+ metadata_keys: [authorization]+ metadata_cardinality_limit: ${env:LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT}+ {{- end }}++ exporters:+ otlphttp/central:+ # Langfuse Cloud base URL options: https://cloud.langfuse.com (EU),+ # https://us.cloud.langfuse.com (US), https://jp.cloud.langfuse.com (JP).+ endpoint: "${env:LANGFUSE_FANOUT_CENTRAL_BASE_URL}/api/public/otel"+ headers:+ Authorization: "${env:LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER}"+ x-langfuse-ingestion-version: "4"+ {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}+ {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}+ otlphttp/tenant_{{ $name }}:+ endpoint: "${env:{{ include "librechat.langfuseFanout.destinationBaseUrlEnvName" $name }}}/api/public/otel"+ auth:+ authenticator: headers_setter/tenant_passthrough+ headers:+ x-langfuse-ingestion-version: "4"+ {{- end }}++ service:+ extensions: [headers_setter/tenant_passthrough]+ pipelines:+ traces/central:+ receivers: [otlp]+ processors: [memory_limiter, attributes/drop_librechat_routing, batch/central]+ exporters: [otlphttp/central]+ traces/tenant:+ receivers: [otlp]+ processors: [memory_limiter, filter/tenant_export]+ exporters: [routing/langfuse_tenant_destination]+ {{- range $name, $_destination := .Values.langfuseFanout.tenant.destinations }}+ {{- include "librechat.langfuseFanout.validateDestinationKey" $name }}+ traces/tenant_{{ $name }}:+ receivers: [routing/langfuse_tenant_destination]+ processors: [attributes/drop_librechat_routing, batch/by_auth_{{ $name }}]+ exporters: [otlphttp/tenant_{{ $name }}]+ {{- end }}+{{- end }}diff --git a/helm/librechat/templates/langfuse-fanout-deployment.yaml b/helm/librechat/templates/langfuse-fanout-deployment.yamlnew file mode 100644index 0000000000..f17775b43a--- /dev/null+++ b/helm/librechat/templates/langfuse-fanout-deployment.yaml@@ -0,0 +1,135 @@+{{- if .Values.langfuseFanout.enabled }}+{{- include "librechat.langfuseFanout.validateDestinationKeys" . }}+{{- $redisURI := .Values.langfuseFanout.redis.uri }}+{{- if and (not $redisURI) .Values.redis.enabled }}+{{- $redisURI = include "librechat.bundledRedisURI" . }}+{{- end }}+{{- if not $redisURI }}+{{- fail "langfuseFanout.redis.uri is required when langfuseFanout.enabled=true unless redis.enabled=true" }}+{{- end }}+{{- $publicURL := .Values.langfuseFanout.publicUrl }}+{{- if not $publicURL }}+{{- $publicURL = printf "http://%s.%s.svc.cluster.local:%v" (include "librechat.langfuseFanout.fullname" .) (.Release.Namespace | lower) .Values.langfuseFanout.service.port }}+{{- end }}+apiVersion: apps/v1+kind: Deployment+metadata:+ name: {{ include "librechat.langfuseFanout.fullname" . }}+ labels:+ {{- include "librechat.langfuseFanout.labels" . | nindent 4 }}+spec:+ replicas: {{ .Values.langfuseFanout.replicaCount }}+ selector:+ matchLabels:+ {{- include "librechat.langfuseFanout.selectorLabels" . | nindent 6 }}+ template:+ metadata:+ annotations:+ checksum/langfuse-fanout-config: {{ toYaml .Values.langfuseFanout | sha256sum }}+ {{- with .Values.langfuseFanout.podAnnotations }}+ {{- toYaml . | nindent 8 }}+ {{- end }}+ labels:+ {{- include "librechat.langfuseFanout.labels" . | nindent 8 }}+ {{- with .Values.langfuseFanout.podLabels }}+ {{- toYaml . | nindent 8 }}+ {{- end }}+ spec:+ containers:+ - name: langfuse-fanout+ image: "{{ .Values.langfuseFanout.image.repository }}:{{ .Values.langfuseFanout.image.tag }}"+ imagePullPolicy: {{ .Values.langfuseFanout.image.pullPolicy }}+ ports:+ - name: otlp-http+ containerPort: 4318+ protocol: TCP+ env:+ - name: LANGFUSE_FANOUT_CENTRAL_BASE_URL+ value: {{ .Values.langfuseFanout.central.baseUrl | quote }}+ - name: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER+ valueFrom:+ secretKeyRef:+ name: {{ required "langfuseFanout.central.authHeaderSecret.name is required when langfuseFanout.enabled=true" .Values.langfuseFanout.central.authHeaderSecret.name | quote }}+ key: {{ .Values.langfuseFanout.central.authHeaderSecret.key | quote }}+ - name: LANGFUSE_FANOUT_TENANT_DESTINATIONS+ value: {{ include "librechat.langfuseFanout.tenantDestinationsEnv" . | quote }}+ - name: LANGFUSE_FANOUT_UPSTREAM_TIMEOUT+ value: {{ .Values.langfuseFanout.upstreamTimeout | quote }}+ - name: LANGFUSE_FANOUT_TRACE_COLLECTOR_URL+ value: {{ .Values.langfuseFanout.traceCollectorUrl | quote }}+ - name: LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS+ value: {{ include "librechat.langfuseFanout.tenantDestinationKeysEnv" . | quote }}+ - name: LANGFUSE_FANOUT_REDIS_URI+ value: {{ $redisURI | quote }}+ {{- with .Values.langfuseFanout.redis.username }}+ - name: LANGFUSE_FANOUT_REDIS_USERNAME+ value: {{ . | quote }}+ {{- end }}+ {{- with .Values.langfuseFanout.redis.passwordSecret.name }}+ - name: LANGFUSE_FANOUT_REDIS_PASSWORD+ valueFrom:+ secretKeyRef:+ name: {{ . | quote }}+ key: {{ $.Values.langfuseFanout.redis.passwordSecret.key | quote }}+ {{- end }}+ - name: LANGFUSE_FANOUT_REDIS_KEY_PREFIX+ value: {{ .Values.langfuseFanout.redis.keyPrefix | quote }}+ - name: LANGFUSE_FANOUT_PUBLIC_URL+ value: {{ $publicURL | quote }}+ {{- with .Values.langfuseFanout.metrics.secret.name }}+ - name: LANGFUSE_FANOUT_METRICS_SECRET+ valueFrom:+ secretKeyRef:+ name: {{ . | quote }}+ key: {{ $.Values.langfuseFanout.metrics.secret.key | quote }}+ {{- end }}+ livenessProbe:+ {{- toYaml .Values.langfuseFanout.livenessProbe | nindent 12 }}+ readinessProbe:+ {{- toYaml .Values.langfuseFanout.readinessProbe | nindent 12 }}+ resources:+ {{- toYaml .Values.langfuseFanout.resources | nindent 12 }}+ - name: otelcol+ image: "{{ .Values.langfuseFanout.otelCollector.image.repository }}:{{ .Values.langfuseFanout.otelCollector.image.tag }}"+ imagePullPolicy: {{ .Values.langfuseFanout.otelCollector.image.pullPolicy }}+ args: ["--config=/etc/otelcol/otelcol.yaml"]+ ports:+ - name: otlp-internal+ containerPort: 4319+ protocol: TCP+ env:+ - name: LANGFUSE_FANOUT_CENTRAL_BASE_URL+ value: {{ .Values.langfuseFanout.central.baseUrl | quote }}+ - name: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER+ valueFrom:+ secretKeyRef:+ name: {{ required "langfuseFanout.central.authHeaderSecret.name is required when langfuseFanout.enabled=true" .Values.langfuseFanout.central.authHeaderSecret.name | quote }}+ key: {{ .Values.langfuseFanout.central.authHeaderSecret.key | quote }}+ - name: LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT+ value: {{ .Values.langfuseFanout.otelCollector.receiverEndpoint | quote }}+ {{- range $name, $destination := .Values.langfuseFanout.tenant.destinations }}+ - name: {{ include "librechat.langfuseFanout.destinationBaseUrlEnvName" $name }}+ value: {{ $destination.baseUrl | quote }}+ {{- end }}+ - name: LANGFUSE_FANOUT_MEMORY_LIMIT_MIB+ value: {{ .Values.langfuseFanout.memoryLimitMiB | quote }}+ - name: LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB+ value: {{ .Values.langfuseFanout.memorySpikeLimitMiB | quote }}+ - name: LANGFUSE_FANOUT_BATCH_TIMEOUT+ value: {{ .Values.langfuseFanout.batchTimeout | quote }}+ - name: LANGFUSE_FANOUT_BATCH_SEND_SIZE+ value: {{ .Values.langfuseFanout.batchSendSize | quote }}+ - name: LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT+ value: {{ .Values.langfuseFanout.metadataCardinalityLimit | quote }}+ volumeMounts:+ - name: config+ mountPath: /etc/otelcol/otelcol.yaml+ subPath: otelcol.yaml+ readOnly: true+ resources:+ {{- toYaml .Values.langfuseFanout.otelCollector.resources | nindent 12 }}+ volumes:+ - name: config+ configMap:+ name: {{ include "librechat.langfuseFanout.fullname" . }}-config+{{- end }}diff --git a/helm/librechat/templates/langfuse-fanout-service.yaml b/helm/librechat/templates/langfuse-fanout-service.yamlnew file mode 100644index 0000000000..ea625f653b--- /dev/null+++ b/helm/librechat/templates/langfuse-fanout-service.yaml@@ -0,0 +1,23 @@+{{- if .Values.langfuseFanout.enabled }}+apiVersion: v1+kind: Service+metadata:+ name: {{ include "librechat.langfuseFanout.fullname" . }}+ labels:+ {{- include "librechat.langfuseFanout.labels" . | nindent 4 }}+ {{- with .Values.langfuseFanout.service.annotations }}+ annotations:+ {{- range $key, $value := . }}+ {{ $key }}: {{ $value | quote }}+ {{- end }}+ {{- end }}+spec:+ type: {{ .Values.langfuseFanout.service.type }}+ ports:+ - name: otlp-http+ port: {{ .Values.langfuseFanout.service.port }}+ targetPort: otlp-http+ protocol: TCP+ selector:+ {{- include "librechat.langfuseFanout.selectorLabels" . | nindent 4 }}+{{- end }}diff --git a/helm/librechat/tests/langfuse_fanout_selector_test.sh b/helm/librechat/tests/langfuse_fanout_selector_test.shnew file mode 100755index 0000000000..52da4579fb--- /dev/null+++ b/helm/librechat/tests/langfuse_fanout_selector_test.sh@@ -0,0 +1,160 @@+#!/usr/bin/env bash+# Regression test for Langfuse fanout Helm selectors.+#+# The fanout collector must not share the main LibreChat app selector labels.+# Otherwise the main Service can route HTTP traffic to the OTEL collector pod.++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+CHART_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"+REPO_ROOT="$(cd "${CHART_DIR}/../.." && pwd)"+RENDER_CHART_DIR="$(mktemp -d -t librechat-fanout-chart.XXXXXX)"+RENDERED_FILE="$(mktemp -t librechat-fanout-render.XXXXXX)"+INVALID_RENDER_ERROR="$(mktemp -t librechat-fanout-invalid-key.XXXXXX)"+COLLISION_RENDER_ERROR="$(mktemp -t librechat-fanout-colliding-key.XXXXXX)"+trap 'rm -rf "${RENDER_CHART_DIR}"; rm -f "${RENDERED_FILE}" "${INVALID_RENDER_ERROR}" "${COLLISION_RENDER_ERROR}"' EXIT++if ! command -v helm >/dev/null 2>&1; then+ echo "FAIL: helm not on PATH" >&2+ exit 1+fi++mkdir -p "${RENDER_CHART_DIR}/templates"+awk '/^dependencies:/{ exit } { print }' "${CHART_DIR}/Chart.yaml" > "${RENDER_CHART_DIR}/Chart.yaml"+cp "${CHART_DIR}/values.yaml" "${RENDER_CHART_DIR}/values.yaml"+cp "${CHART_DIR}/templates/_helpers.tpl" "${RENDER_CHART_DIR}/templates/_helpers.tpl"+cp "${CHART_DIR}/templates/service.yaml" "${RENDER_CHART_DIR}/templates/service.yaml"+cp "${CHART_DIR}/templates/langfuse-fanout-service.yaml" \+ "${RENDER_CHART_DIR}/templates/langfuse-fanout-service.yaml"+cp "${CHART_DIR}/templates/langfuse-fanout-deployment.yaml" \+ "${RENDER_CHART_DIR}/templates/langfuse-fanout-deployment.yaml"++helm template librechat "${RENDER_CHART_DIR}" \+ --set langfuseFanout.enabled=true \+ --set langfuseFanout.central.authHeaderSecret.name=langfuse-central \+ --set langfuseFanout.redis.uri=redis://langfuse-fanout-redis:6379 \+ --show-only templates/service.yaml \+ --show-only templates/langfuse-fanout-service.yaml \+ --show-only templates/langfuse-fanout-deployment.yaml \+ > "${RENDERED_FILE}"++if helm template librechat "${RENDER_CHART_DIR}" \+ --set langfuseFanout.enabled=true \+ --set langfuseFanout.central.authHeaderSecret.name=langfuse-central \+ --set langfuseFanout.redis.uri=redis://langfuse-fanout-redis:6379 \+ --set langfuseFanout.tenant.destinations.EU.baseUrl=https://cloud.langfuse.com \+ --show-only templates/langfuse-fanout-deployment.yaml \+ > /dev/null 2> "${INVALID_RENDER_ERROR}"; then+ echo "FAIL: Helm accepted invalid uppercase Langfuse fanout destination key" >&2+ exit 1+fi++if ! grep -q 'langfuseFanout.tenant.destinations key "EU" is invalid' "${INVALID_RENDER_ERROR}"; then+ echo "FAIL: invalid destination key render did not explain the key contract" >&2+ cat "${INVALID_RENDER_ERROR}" >&2+ exit 1+fi++if helm template librechat "${RENDER_CHART_DIR}" \+ --set langfuseFanout.enabled=true \+ --set langfuseFanout.central.authHeaderSecret.name=langfuse-central \+ --set langfuseFanout.redis.uri=redis://langfuse-fanout-redis:6379 \+ --set langfuseFanout.tenant.destinations.foo-bar.baseUrl=https://foo-bar.example.com \+ --set langfuseFanout.tenant.destinations.foo_bar.baseUrl=https://foo-bar.example.com \+ --show-only templates/langfuse-fanout-deployment.yaml \+ > /dev/null 2> "${COLLISION_RENDER_ERROR}"; then+ echo "FAIL: Helm accepted colliding Langfuse fanout destination env var keys" >&2+ exit 1+fi++if ! grep -q 'both render LANGFUSE_FANOUT_TENANT_FOO_BAR_BASE_URL' "${COLLISION_RENDER_ERROR}"; then+ echo "FAIL: colliding destination key render did not explain the env var collision" >&2+ cat "${COLLISION_RENDER_ERROR}" >&2+ exit 1+fi++if ! command -v node >/dev/null 2>&1; then+ echo "FAIL: node not on PATH" >&2+ exit 1+fi++NODE_PATH="${REPO_ROOT}/node_modules${NODE_PATH:+:${NODE_PATH}}" \+RENDERED_FILE="${RENDERED_FILE}" node <<'NODE'+const fs = require('fs');+const yaml = require('js-yaml');++const docs = yaml+ .loadAll(fs.readFileSync(process.env.RENDERED_FILE, 'utf8'))+ .filter(Boolean);++function fail(message) {+ console.error(`FAIL: ${message}`);+ process.exit(1);+}++function find(kind, name) {+ const doc = docs.find((candidate) => candidate.kind === kind && candidate.metadata?.name === name);+ if (!doc) {+ fail(`missing ${kind}/${name}`);+ }+ return doc;+}++function isSubset(subset, labels) {+ return Object.entries(subset ?? {}).every(([key, value]) => labels?.[key] === value);+}++function envValue(env, name) {+ return (env ?? []).find((entry) => entry.name === name)?.value;+}++const mainService = find('Service', 'librechat-librechat');+const fanoutService = find('Service', 'librechat-librechat-langfuse-fanout');+const fanoutDeployment = find('Deployment', 'librechat-librechat-langfuse-fanout');+const fanoutContainer = fanoutDeployment.spec?.template?.spec?.containers?.find(+ (container) => container.name === 'langfuse-fanout',+);+if (!fanoutContainer) {+ fail('missing langfuse-fanout container');+}++const mainSelector = mainService.spec?.selector ?? {};+const fanoutSelector = fanoutService.spec?.selector ?? {};+const fanoutMatchLabels = fanoutDeployment.spec?.selector?.matchLabels ?? {};+const fanoutPodLabels = fanoutDeployment.spec?.template?.metadata?.labels ?? {};+const fanoutMetadataLabels = fanoutDeployment.metadata?.labels ?? {};++if (isSubset(mainSelector, fanoutPodLabels)) {+ fail('main Service selector matches fanout pod labels');+}+if (!isSubset(fanoutSelector, fanoutPodLabels)) {+ fail('fanout Service selector does not match fanout pod labels');+}+if (!isSubset(fanoutMatchLabels, fanoutPodLabels)) {+ fail('fanout Deployment selector is not a subset of pod labels');+}+if (mainSelector['app.kubernetes.io/name'] === fanoutSelector['app.kubernetes.io/name']) {+ fail('main and fanout Services share app.kubernetes.io/name selectors');+}+if (fanoutMetadataLabels['app.kubernetes.io/name'] !== fanoutSelector['app.kubernetes.io/name']) {+ fail('fanout Deployment metadata labels do not use fanout app name');+}+if (envValue(fanoutContainer.env, 'LANGFUSE_FANOUT_REDIS_URI') !== 'redis://langfuse-fanout-redis:6379') {+ fail('fanout Deployment did not render configured Redis URI');+}+if (+ envValue(fanoutContainer.env, 'LANGFUSE_FANOUT_PUBLIC_URL') !==+ 'http://librechat-librechat-langfuse-fanout.default.svc.cluster.local:4318'+) {+ fail('fanout Deployment did not render derived public URL');+}+if (fanoutContainer.livenessProbe?.httpGet?.path !== '/healthz') {+ fail('fanout Deployment missing /healthz liveness probe');+}+if (fanoutContainer.readinessProbe?.httpGet?.path !== '/healthz') {+ fail('fanout Deployment missing /healthz readiness probe');+}++console.log('PASS: Langfuse fanout selectors are isolated from the main LibreChat Service');+NODEdiff --git a/helm/librechat/values.yaml b/helm/librechat/values.yamlindex e5321c2a93..bcefc74342 100755--- a/helm/librechat/values.yaml+++ b/helm/librechat/values.yaml@@ -284,6 +284,77 @@ dnsConfig: {}updateStrategy:type: RollingUpdate+langfuseFanout:+ enabled: false+ replicaCount: 1+ image:+ repository: librechat-langfuse-fanout+ tag: "latest"+ pullPolicy: IfNotPresent+ otelCollector:+ receiverEndpoint: 127.0.0.1:4319+ image:+ repository: otel/opentelemetry-collector-contrib+ tag: "0.143.0"+ pullPolicy: IfNotPresent+ resources: {}+ service:+ type: ClusterIP+ port: 4318+ annotations: {}+ central:+ baseUrl: https://cloud.langfuse.com+ authHeaderSecret:+ name: ""+ key: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER+ metrics:+ secret:+ # Optional bearer token secret for scraping the gateway's /metrics endpoint.+ # When omitted, /metrics returns 401.+ name: ""+ key: METRICS_SECRET+ redis:+ # Redis stores short-lived Langfuse media upload plans so multiple gateway+ # replicas can handle create/upload requests. If empty and redis.enabled is+ # true, the chart derives the bundled Redis service URI.+ uri: ""+ username: ""+ passwordSecret:+ name: ""+ key: REDIS_PASSWORD+ keyPrefix: langfuse-fanout+ tenant:+ # Destination map keys are emitted by LibreChat as trace attributes and+ # matched by the gateway. Use lowercase keys matching ^[a-z][a-z0-9_-]*$.+ destinations:+ eu:+ baseUrl: https://cloud.langfuse.com+ us:+ baseUrl: https://us.cloud.langfuse.com+ jp:+ baseUrl: https://jp.cloud.langfuse.com+ upstreamTimeout: 30s+ # Optional override for one-time media upload URLs returned by the gateway.+ # When empty, the chart derives the internal fanout Service URL.+ publicUrl: ""+ traceCollectorUrl: http://127.0.0.1:4319+ memoryLimitMiB: 256+ memorySpikeLimitMiB: 64+ batchTimeout: 1s+ batchSendSize: 128+ metadataCardinalityLimit: 1000+ livenessProbe:+ httpGet:+ path: /healthz+ port: 4318+ readinessProbe:+ httpGet:+ path: /healthz+ port: 4318+ resources: {}+ podAnnotations: {}+ podLabels: {}+# Extra ConfigMaps to be created alongside the main onesadditionalConfigMaps: {}# custom: # suffix of the ConfigMap namediff --git a/otel/langfuse-fanout/Dockerfile b/otel/langfuse-fanout/Dockerfilenew file mode 100644index 0000000000..22be19c5c7--- /dev/null+++ b/otel/langfuse-fanout/Dockerfile@@ -0,0 +1,13 @@+FROM golang:1.23-alpine AS builder+WORKDIR /src+COPY otel/langfuse-fanout/go.mod otel/langfuse-fanout/go.sum* ./+RUN go mod download+COPY otel/langfuse-fanout/ ./+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/langfuse-fanout ./cmd/langfuse-fanout++FROM alpine:3.22+RUN addgroup -S app && adduser -S app -G app+COPY --from=builder /out/langfuse-fanout /usr/local/bin/langfuse-fanout+USER app+EXPOSE 4318+ENTRYPOINT ["/usr/local/bin/langfuse-fanout"]diff --git a/otel/langfuse-fanout/README.md b/otel/langfuse-fanout/README.mdnew file mode 100644index 0000000000..f78fa16bab--- /dev/null+++ b/otel/langfuse-fanout/README.md@@ -0,0 +1,285 @@+# Langfuse Fanout Gateway++LibreChat can send tenant-scoped agent traces to a tenant Langfuse project and+also copy those traces to a central Langfuse project. When trace payloads+contain Langfuse media references, the gateway also copies the media upload to+central and tenant Langfuse storage. This is optional and is disabled unless you+explicitly deploy the fanout gateway.++The deployment is a hybrid:++- the Go gateway is the only endpoint LibreChat talks to;+- trace requests are proxied to an internal OpenTelemetry collector;+- the collector owns trace memory limiting, batching, routing, and export;+- the Go gateway owns Langfuse media create/upload/patch fanout.++## How It Works++- Agent traces use Langfuse OTLP ingestion.+- LibreChat sends tenant traces to the local fanout gateway when+ `LANGFUSE_FANOUT_ENABLED=true` and `LANGFUSE_FANOUT_COLLECTOR_URL` points at+ the fanout gateway.+- The gateway forwards trace requests to the internal OpenTelemetry collector+ at `LANGFUSE_FANOUT_TRACE_COLLECTOR_URL`.+- The collector exports every trace to the central Langfuse project using+ `LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER`. This prebuilt header is collector-only;+ the LibreChat app derives central score auth from `LANGFUSE_PUBLIC_KEY` and+ `LANGFUSE_SECRET_KEY`.+- The collector also exports tenant-enabled traces to the tenant Langfuse+ project by routing on `librechat.langfuse.destination`, then forwarding the+ tenant `Authorization` header that LibreChat attaches to the OTLP request.+- For tenant-exportable runs, LibreChat uses a destination-scoped gateway URL+ like `http://langfuse-fanout-collector:4318/tenant/us`. Langfuse media upload+ requests do not carry span attributes, so this path gives the gateway the+ destination needed to copy media into the tenant's Langfuse region. For+ traces on this path, the gateway restores the internal tenant routing+ attributes before handing the request to the collector.+- Before export, the collector deletes the internal `librechat.langfuse.*`+ routing attributes from central and tenant traces.+- Langfuse media upload is fanned out by calling `POST /api/public/media` on+ central and tenant Langfuse, returning a one-time gateway upload URL, then+ uploading the received bytes to each upstream presigned upload URL. The SDK's+ `PATCH /api/public/media/{mediaId}` status call is also fanned out.+- Tenant export is conditional. LibreChat uses a destination-scoped gateway URL+ only when tenant keys are configured, the tenant base URL matches a configured+ startup destination, and `LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED` is not true.+ Other traces are still exported to central through the gateway without tenant+ auth.+- User feedback scores use Langfuse's direct REST API from the LibreChat API+ process. Central scores use LibreChat's normal central Langfuse env config;+ tenant scores use tenant app configuration when tenant fanout is enabled.++Tenant Langfuse keys are expected to come from LibreChat app configuration, for+example from an admin panel or another configuration data source. They are not+defined in this gateway config.++## Limitations++- Langfuse base URLs are startup configuration. `LANGFUSE_FANOUT_CENTRAL_BASE_URL`+ and `LANGFUSE_FANOUT_TENANT_DESTINATIONS` must be known when LibreChat and the+ gateway start. Tenant app configuration may choose any configured tenant+ destination.+- Tenant Langfuse API keys can be added, changed, or disabled in tenant app+ configuration at runtime without restarting LibreChat or the gateway.+- Tenant app configuration must set a Langfuse base URL matching one of the+ startup destinations before tenant trace/score export is enabled; keys alone+ are treated as central-only.+- `LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=true` can be set on LibreChat as an+ emergency switch to stop tenant trace and score export while keeping central+ gateway export active. When omitted, false, or blank, tenant export remains+ available if tenant keys and a known destination are configured.+- This supports Langfuse Cloud and self-hosted Langfuse as long as each allowed+ tenant base URL is configured at LibreChat/gateway startup. Runtime tenant+ config selects from those known destinations; it does not inject arbitrary+ export URLs into the gateway.+- The provided Compose gateway config is a three-region Langfuse Cloud preset+ (`eu`, `us`, `jp`). Compose's static collector config routes only those keys;+ the gateway fails startup when `LANGFUSE_FANOUT_TENANT_DESTINATIONS` contains+ a key outside `LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS`. For self-hosted or+ additional destination keys, update the collector config too or use Helm.+- Helm binds the internal collector receiver to `127.0.0.1:4319` because the+ collector is a sidecar. Compose binds it to `0.0.0.0:4319` on the private+ `langfuse-fanout` network. Do not publish the internal collector receiver+ outside the fanout deployment; tenant routing validation happens in the+ gateway before traces reach the collector.+- The gateway stores short-lived one-time media upload plans in Redis. This lets+ media create and byte-upload requests land on different gateway replicas.+ Compose includes a private Redis container; Helm can derive the URI from the+ bundled Redis chart or use an explicit `langfuseFanout.redis.uri`.+- The gateway requires an explicit public/internal base URL for one-time upload+ URLs. Compose sets `LANGFUSE_FANOUT_PUBLIC_URL` to its private gateway+ service URL. Helm derives the fanout Service DNS name unless `publicUrl` is+ set.+- Media fanout is not transactional across central and tenant projects. If one+ destination accepts `POST /api/public/media` and another fails, LibreChat sees+ a gateway error and will not upload bytes, but the successful destination may+ retain a short-lived, unused media record.+- Trace batching is handled by the collector. By default it flushes after 128+ items or 1 second, and tenant batches are separated by the request+ `Authorization` metadata.+- The gateway exposes Prometheus metrics at `/metrics` using the same bearer+ token shape as LibreChat. Set `LANGFUSE_FANOUT_METRICS_SECRET`, or provide+ `METRICS_SECRET` in the gateway environment. When neither is set, `/metrics`+ returns 401.++## Docker Compose++Set the central Langfuse destination in `.env`:++```dotenv+# Used by LibreChat for central feedback scores. Set this to the same non-EU+# region as LANGFUSE_FANOUT_CENTRAL_BASE_URL when applicable.+LANGFUSE_BASE_URL=https://cloud.langfuse.com++# Used by the gateway for central trace and media export.+LANGFUSE_FANOUT_CENTRAL_BASE_URL=https://cloud.langfuse.com+LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER=Basic <base64-public-key-colon-secret-key>+# Compose's included gateway config supports these three destination keys.+LANGFUSE_FANOUT_TENANT_DESTINATIONS=eu=https://cloud.langfuse.com,us=https://us.cloud.langfuse.com,jp=https://jp.cloud.langfuse.com+LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS=eu,us,jp+LANGFUSE_FANOUT_TENANT_EU_BASE_URL=https://cloud.langfuse.com+LANGFUSE_FANOUT_TENANT_US_BASE_URL=https://us.cloud.langfuse.com+LANGFUSE_FANOUT_TENANT_JP_BASE_URL=https://jp.cloud.langfuse.com+LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED=false+LANGFUSE_FANOUT_UPSTREAM_TIMEOUT=30s+LANGFUSE_FANOUT_PUBLIC_URL=http://langfuse-fanout-collector:4318+LANGFUSE_FANOUT_REDIS_URI=redis://langfuse-fanout-redis:6379+LANGFUSE_FANOUT_REDIS_USERNAME=+LANGFUSE_FANOUT_REDIS_PASSWORD=+LANGFUSE_FANOUT_REDIS_KEY_PREFIX=langfuse-fanout+LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT=0.0.0.0:4319+LANGFUSE_FANOUT_METRICS_SECRET=<metrics-bearer-token>+LANGFUSE_FANOUT_MEMORY_LIMIT_MIB=256+LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB=64+LANGFUSE_FANOUT_BATCH_TIMEOUT=1s+LANGFUSE_FANOUT_BATCH_SEND_SIZE=128+LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT=1000+```++Langfuse Cloud base URL options:++| Region | Base URL |+| ------ | ------------------------------- |+| EU | `https://cloud.langfuse.com` |+| US | `https://us.cloud.langfuse.com` |+| JP | `https://jp.cloud.langfuse.com` |++Then start LibreChat with the fanout override:++```sh+docker compose -f docker-compose.yml -f docker-compose.langfuse-fanout.yml up -d+```++For the deployed compose stack:++```sh+docker compose -f deploy-compose.yml -f deploy-compose.langfuse-fanout.yml up -d+```++The override builds the fanout gateway image, sets `LANGFUSE_FANOUT_ENABLED=true`, and points LibreChat at+`http://langfuse-fanout-collector:4318`. It also starts an internal+`langfuse-fanout-otel` service on the private fanout network for trace export.++## Helm++Create a secret containing the central Langfuse Basic auth header:++```sh+kubectl create secret generic langfuse-central \+ --from-literal=LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER='Basic <base64-public-key-colon-secret-key>'+```++Enable the gateway in values. Use either the bundled Redis chart as shown here+or set `langfuseFanout.redis.uri` to an external Redis service.++```yaml+redis:+ enabled: true++langfuseFanout:+ enabled: true+ central:+ baseUrl: https://cloud.langfuse.com+ authHeaderSecret:+ name: langfuse-central+ key: LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER+ metrics:+ secret:+ name: librechat-metrics+ key: METRICS_SECRET+ tenant:+ destinations:+ eu:+ baseUrl: https://cloud.langfuse.com+ us:+ baseUrl: https://us.cloud.langfuse.com+ jp:+ baseUrl: https://jp.cloud.langfuse.com+ upstreamTimeout: 30s+ publicUrl: ""+ otelCollector:+ receiverEndpoint: 127.0.0.1:4319+ redis:+ uri: ""+ username: ""+ passwordSecret:+ name: ""+ key: REDIS_PASSWORD+ keyPrefix: langfuse-fanout+ memoryLimitMiB: 256+ memorySpikeLimitMiB: 64+ batchTimeout: 1s+ batchSendSize: 128+ metadataCardinalityLimit: 1000+```++The chart renders one fanout Deployment with two containers: the gateway on+`4318` and an internal OpenTelemetry collector on `4319`. The Service exposes+only the gateway. The chart also injects `LANGFUSE_FANOUT_ENABLED` plus+`LANGFUSE_FANOUT_COLLECTOR_URL` into the LibreChat app ConfigMap when they are+not already supplied in `librechat.configEnv`.++Set `langfuseFanout.redis.uri` when using an external Redis service. If Redis+requires auth, set `langfuseFanout.redis.username` and point+`langfuseFanout.redis.passwordSecret.name`/`.key` at an existing Kubernetes+Secret. When using the bundled Redis chart with auth enabled, create a password+Secret for the gateway or provide an explicit authenticated URI.+Prefer `passwordSecret` over embedding credentials in `redis.uri`, because the+URI is rendered directly into the Deployment environment.+Scale the gateway manually with `langfuseFanout.replicaCount`; the chart does+not create a fanout HPA. The gateway container has configurable `/healthz`+liveness and readiness probes under `langfuseFanout`.++Useful gateway metrics include:++- `langfuse_fanout_http_requests_total`+- `langfuse_fanout_upstream_requests_total`+- `langfuse_fanout_trace_exports_total`+- `langfuse_fanout_media_upload_plans_created_total`+- `langfuse_fanout_media_upload_plans_completed_total`+- `langfuse_fanout_media_upload_plan_misses_total`+- `langfuse_fanout_media_upload_plan_store_errors_total`+- `langfuse_fanout_media_upload_bytes`+- `langfuse_fanout_media_divergence_total`++`langfuse_fanout_media_divergence_total{kind="media_id"}` is the correctness+signal for trace/media token fanout. `kind="upload_url_presence"` records that+some destinations returned an upload URL while others treated the media as+already uploaded.++## Notes++- The gateway handles Langfuse media uploads and proxies traces to the internal+ collector. Feedback scores go directly to Langfuse's REST API from the+ LibreChat API process.+- `LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER` must be a full Basic auth header and is+ consumed by the fanout deployment only. The app does not use it for scores.+- `LANGFUSE_FANOUT_CENTRAL_BASE_URL` is also consumed by the fanout deployment only.+ For non-EU central feedback scores, set LibreChat's normal `LANGFUSE_BASE_URL`+ to the same central Langfuse region.+- Tenant destinations default to the three configured Langfuse Cloud regions. Add or+ override `langfuseFanout.tenant.destinations` in Helm for self-hosted or+ custom destinations.+- `LANGFUSE_FANOUT_UPSTREAM_TIMEOUT` tunes the timeout for gateway calls to+ Langfuse APIs and presigned media upload URLs.+- `LANGFUSE_FANOUT_PUBLIC_URL` pins the base URL returned for the SDK's+ one-time media upload. The gateway fails startup when it is unset or invalid;+ this avoids trusting request `Host` or `X-Forwarded-Host` headers.+- `LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS` is a startup guard that must contain+ every key in `LANGFUSE_FANOUT_TENANT_DESTINATIONS`; this prevents media+ fanout from accepting a destination the collector cannot route traces to.+- `LANGFUSE_FANOUT_REDIS_URI`, optional `LANGFUSE_FANOUT_REDIS_USERNAME`,+ optional `LANGFUSE_FANOUT_REDIS_PASSWORD`, and+ `LANGFUSE_FANOUT_REDIS_KEY_PREFIX` configure the shared one-time media upload+ plan store. The gateway fails startup without a Redis URI.+- `LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT` controls the internal collector+ receiver bind address.+- `LANGFUSE_FANOUT_METRICS_SECRET` protects the gateway `/metrics` endpoint.+ If unset, the gateway falls back to `METRICS_SECRET` when present.+- `LANGFUSE_FANOUT_MEMORY_LIMIT_MIB`,+ `LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB`, `LANGFUSE_FANOUT_BATCH_TIMEOUT`,+ `LANGFUSE_FANOUT_BATCH_SEND_SIZE`, and+ `LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT` tune the internal collector.+- `LANGFUSE_FANOUT_COLLECTOR_URL` is the local gateway URL used by LibreChat.+ The env name is kept for compatibility with the original collector shape; it+ is not a Langfuse Cloud base URL.diff --git a/otel/langfuse-fanout/cmd/langfuse-fanout/main.go b/otel/langfuse-fanout/cmd/langfuse-fanout/main.gonew file mode 100644index 0000000000..147dd3e92a--- /dev/null+++ b/otel/langfuse-fanout/cmd/langfuse-fanout/main.go@@ -0,0 +1,1183 @@+package main++import (+ "bytes"+ "compress/gzip"+ "context"+ "crypto/rand"+ "crypto/subtle"+ "encoding/hex"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "log"+ "net/http"+ "net/url"+ "os"+ "os/signal"+ "strconv"+ "strings"+ "sync"+ "syscall"+ "time"++ "github.com/prometheus/client_golang/prometheus/promhttp"+ "github.com/redis/go-redis/v9"+ tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ commonv1 "go.opentelemetry.io/proto/otlp/common/v1"+ tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"+ "google.golang.org/protobuf/proto"+)++const (+ defaultListenAddr = ":4318"+ defaultTraceCollector = "http://127.0.0.1:4319"+ centralName = "central"+ tenantPrefix = "/tenant/"+ mediaUploadProxyPath = "/__langfuse-fanout/media-upload/"+ otelTracePath = "/api/public/otel/v1/traces"+ mediaPath = "/api/public/media"+ metricsPath = "/metrics"+ tenantExportAttribute = "librechat.langfuse.tenant_export.enabled"+ tenantDestAttribute = "librechat.langfuse.destination"+)++type config struct {+ listenAddr string+ traceCollectorURL string+ publicURL string+ metricsSecret string+ traceDestinationKeys map[string]bool+ central destination+ tenants map[string]string+ redis redisConfig+ uploadStore uploadPlanStore+ client *http.Client+}++type redisConfig struct {+ uri string+ username string+ password string+ keyPrefix string+}++type destination struct {+ name string+ baseURL string+ authorization string+}++type route struct {+ destination string+ path string+}++type uploadDestination struct {+ Name string `json:"name"`+ UploadURL string `json:"uploadUrl"`+}++type uploadPlan struct {+ ExpiresAt time.Time `json:"expiresAt"`+ Destinations []uploadDestination `json:"destinations"`+ ContentLength int64 `json:"contentLength"`+}++type uploadPlanStore interface {+ Put(ctx context.Context, uploadID string, plan uploadPlan) error+ Take(ctx context.Context, uploadID string) (uploadPlan, bool, error)+ Ping(ctx context.Context) error+ Close() error+}++type gateway struct {+ cfg config+ metrics *gatewayMetrics+ metricsHTTP http.Handler+}++type mediaUploadResponse struct {+ UploadURL *string `json:"uploadUrl"`+ MediaID string `json:"mediaId"`+}++func main() {+ cfg, err := loadConfig()+ if err != nil {+ log.Fatalf("failed to load config: %v", err)+ }+ uploadStore, err := newRedisUploadPlanStore(cfg.redis)+ if err != nil {+ log.Fatalf("failed to initialize Redis upload plan store: %v", err)+ }+ defer uploadStore.Close()+ cfg.uploadStore = uploadStore++ gw := newGateway(cfg)+ ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)+ defer stop()++ mux := http.NewServeMux()+ mux.HandleFunc("/", gw.handle)++ server := &http.Server{+ Addr: cfg.listenAddr,+ Handler: mux,+ ReadHeaderTimeout: 10 * time.Second,+ }+ log.Printf("langfuse fanout gateway listening on %s", cfg.listenAddr)+ errCh := make(chan error, 1)+ go func() {+ errCh <- server.ListenAndServe()+ }()++ select {+ case <-ctx.Done():+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ if err := server.Shutdown(shutdownCtx); err != nil {+ log.Printf("server shutdown failed: %v", err)+ }+ case err := <-errCh:+ if err != nil && !errors.Is(err, http.ErrServerClosed) {+ log.Fatal(err)+ }+ }+}++func newGateway(cfg config) *gateway {+ if cfg.uploadStore == nil {+ panic("langfuse fanout gateway requires an upload plan store")+ }+ metrics := newGatewayMetrics()+ return &gateway{+ cfg: cfg,+ metrics: metrics,+ metricsHTTP: promhttp.HandlerFor(metrics.registry, promhttp.HandlerOpts{}),+ }+}++func loadConfig() (config, error) {+ centralBaseURL := normalizeBaseURL(os.Getenv("LANGFUSE_FANOUT_CENTRAL_BASE_URL"))+ centralAuth := strings.TrimSpace(os.Getenv("LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER"))+ if centralBaseURL == "" {+ return config{}, errors.New("LANGFUSE_FANOUT_CENTRAL_BASE_URL is required")+ }+ if centralAuth == "" {+ return config{}, errors.New("LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER is required")+ }++ tenants := map[string]string{}+ for _, item := range strings.Split(os.Getenv("LANGFUSE_FANOUT_TENANT_DESTINATIONS"), ",") {+ item = strings.TrimSpace(item)+ if item == "" {+ continue+ }+ key, value, ok := strings.Cut(item, "=")+ if !ok {+ continue+ }+ key = normalizeDestinationKey(key)+ if key == "" {+ continue+ }+ if baseURL := normalizeBaseURL(value); baseURL != "" {+ tenants[key] = baseURL+ }+ }+ traceDestinationKeys := parseDestinationKeys(os.Getenv("LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS"))+ if len(tenants) > 0 && len(traceDestinationKeys) == 0 {+ return config{}, errors.New("LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS is required when LANGFUSE_FANOUT_TENANT_DESTINATIONS is set")+ }+ for key := range tenants {+ if len(traceDestinationKeys) > 0 && !traceDestinationKeys[key] {+ return config{}, fmt.Errorf("tenant destination %q is not present in LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS", key)+ }+ }+ rawPublicURL := strings.TrimSpace(os.Getenv("LANGFUSE_FANOUT_PUBLIC_URL"))+ publicURL := normalizeBaseURL(rawPublicURL)+ if publicURL == "" {+ return config{}, errors.New("LANGFUSE_FANOUT_PUBLIC_URL must be an absolute HTTP(S) URL")+ }+ redisURI := strings.TrimSpace(os.Getenv("LANGFUSE_FANOUT_REDIS_URI"))+ if redisURI == "" {+ return config{}, errors.New("LANGFUSE_FANOUT_REDIS_URI is required for media upload plan storage")+ }++ return config{+ listenAddr: envOrDefault("LANGFUSE_FANOUT_LISTEN_ADDR", defaultListenAddr),+ traceCollectorURL: normalizeCollectorURL(envOrDefault("LANGFUSE_FANOUT_TRACE_COLLECTOR_URL", defaultTraceCollector)),+ publicURL: publicURL,+ metricsSecret: firstNonEmptyEnv("LANGFUSE_FANOUT_METRICS_SECRET", "METRICS_SECRET"),+ traceDestinationKeys: traceDestinationKeys,+ central: destination{+ name: centralName,+ baseURL: centralBaseURL,+ authorization: centralAuth,+ },+ tenants: tenants,+ redis: redisConfig{+ uri: redisURI,+ username: strings.TrimSpace(os.Getenv("LANGFUSE_FANOUT_REDIS_USERNAME")),+ password: strings.TrimSpace(os.Getenv("LANGFUSE_FANOUT_REDIS_PASSWORD")),+ keyPrefix: envOrDefault("LANGFUSE_FANOUT_REDIS_KEY_PREFIX", "langfuse-fanout"),+ },+ client: &http.Client{+ Timeout: parseDurationEnv("LANGFUSE_FANOUT_UPSTREAM_TIMEOUT", 30*time.Second),+ },+ }, nil+}++func (g *gateway) handle(w http.ResponseWriter, r *http.Request) {+ startedAt := time.Now()+ recorder := &statusRecorder{ResponseWriter: w, status: http.StatusOK}+ w = recorder+ defer func() {+ if r.URL.Path != metricsPath && g.metrics != nil {+ g.metrics.recordHTTP(r.Method, normalizeMetricPath(r.URL.Path), recorder.status, time.Since(startedAt))+ }+ }()++ route := parseRoute(r.URL.Path)+ switch {+ case route.path == otelTracePath && r.Method == http.MethodPost:+ g.handleTraces(w, r, route)+ case route.path == mediaPath && r.Method == http.MethodPost:+ g.handleMediaCreate(w, r, route)+ case strings.HasPrefix(route.path, mediaPath+"/") && r.Method == http.MethodGet:+ g.handleMediaGet(w, r, route)+ case strings.HasPrefix(route.path, mediaPath+"/") && r.Method == http.MethodPatch:+ g.handleMediaPatch(w, r, route)+ case strings.HasPrefix(r.URL.Path, mediaUploadProxyPath) && r.Method == http.MethodPut:+ g.handleMediaUpload(w, r)+ case r.URL.Path == "/healthz":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("ok"))+ case r.URL.Path == metricsPath && r.Method == http.MethodGet:+ g.handleMetrics(w, r)+ default:+ http.Error(w, "langfuse fanout gateway only supports OTLP traces and media upload APIs", http.StatusNotImplemented)+ }+}++func (g *gateway) handleTraces(w http.ResponseWriter, r *http.Request, route route) {+ body, err := readMaybeGzip(r)+ if err != nil {+ http.Error(w, "failed to read request body", http.StatusBadRequest)+ return+ }++ contentType := r.Header.Get("Content-Type")+ if route.destination != "" &&+ g.cfg.tenants[route.destination] != "" &&+ strings.TrimSpace(r.Header.Get("Authorization")) != "" {+ body, err = addTenantRouteAttributes(body, contentType, route.destination)+ if err != nil {+ http.Error(w, "failed to add OTLP tenant routing attributes", http.StatusBadRequest)+ return+ }+ }++ contentEncoding := ""+ if strings.EqualFold(r.Header.Get("Content-Encoding"), "gzip") {+ contentEncoding = "gzip"+ body, err = gzipBytes(body)+ if err != nil {+ http.Error(w, "failed to encode request body", http.StatusInternalServerError)+ return+ }+ }++ resp, err := g.forwardTraceToCollector(r.Context(), r.Header, body, contentType, contentEncoding)+ if err != nil {+ g.recordTraceExport(route, "error")+ http.Error(w, fmt.Sprintf("trace collector export failed: %v", err), http.StatusBadGateway)+ return+ }+ defer resp.Body.Close()++ g.recordTraceExport(route, "success")+ copyResponseHeaders(w.Header(), resp.Header)+ w.WriteHeader(resp.StatusCode)+ _, _ = io.Copy(w, resp.Body)+}++func (g *gateway) handleMediaCreate(w http.ResponseWriter, r *http.Request, route route) {+ body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 2<<20))+ if err != nil {+ http.Error(w, "failed to read media create request", http.StatusBadRequest)+ return+ }+ if err := g.cfg.uploadStore.Ping(r.Context()); err != nil {+ g.recordUploadPlanStoreError("ping")+ http.Error(w, fmt.Sprintf("media upload plan store unavailable: %v", err), http.StatusBadGateway)+ return+ }++ destinations := g.mediaDestinations(route, r.Header.Get("Authorization"))+ if len(destinations) == 0 {+ http.Error(w, "no media destinations configured", http.StatusBadGateway)+ return+ }++ type mediaCreateResult struct {+ destination destination+ response mediaUploadResponse+ err error+ }+ responses := make([]mediaCreateResult, len(destinations))+ var wg sync.WaitGroup+ for index, dest := range destinations {+ index, dest := index, dest+ wg.Add(1)+ go func() {+ defer wg.Done()+ response, err := g.postMediaCreate(r.Context(), dest, body, r.Header.Get("Content-Type"))+ responses[index] = mediaCreateResult{destination: dest, response: response, err: err}+ }()+ }+ wg.Wait()+ for _, result := range responses {+ if result.err != nil {+ http.Error(w, fmt.Sprintf("%s media create failed: %v", result.destination.name, result.err), http.StatusBadGateway)+ return+ }+ }++ mediaID := responses[0].response.MediaID+ if mediaID == "" {+ http.Error(w, "upstream media create returned empty mediaId", http.StatusBadGateway)+ return+ }+ // Langfuse derives mediaId from the content hash today, so all fanout+ // destinations should converge on the same id for the same POST body.+ for _, response := range responses[1:] {+ if response.response.MediaID != mediaID {+ log.Printf(+ "upstream media IDs differ: %s=%s %s=%s",+ responses[0].destination.name,+ mediaID,+ response.destination.name,+ response.response.MediaID,+ )+ g.recordMediaDivergence("media_id", response.destination.name)+ http.Error(w, "upstream media IDs differ across destinations", http.StatusBadGateway)+ return+ }+ }++ uploadPlan := uploadPlan{+ ExpiresAt: time.Now().Add(time.Hour),+ Destinations: []uploadDestination{},+ }+ var requestBody struct {+ ContentLength int64 `json:"contentLength"`+ }+ _ = json.Unmarshal(body, &requestBody)+ uploadPlan.ContentLength = requestBody.ContentLength++ hadUploadURL := false+ missingUploadURLDestinations := []string{}+ for _, response := range responses {+ if response.response.UploadURL == nil || *response.response.UploadURL == "" {+ missingUploadURLDestinations = append(missingUploadURLDestinations, response.destination.name)+ continue+ }+ hadUploadURL = true+ uploadPlan.Destinations = append(uploadPlan.Destinations, uploadDestination{+ Name: response.destination.name,+ UploadURL: *response.response.UploadURL,+ })+ }+ if hadUploadURL {+ for _, destination := range missingUploadURLDestinations {+ g.recordMediaDivergence("upload_url_presence", destination)+ }+ }++ result := mediaUploadResponse{MediaID: mediaID}+ if len(uploadPlan.Destinations) > 0 {+ uploadID, err := randomID()+ if err != nil {+ http.Error(w, "failed to create media upload id", http.StatusInternalServerError)+ return+ }+ if err := g.storeUpload(r.Context(), uploadID, uploadPlan); err != nil {+ g.recordUploadPlanStoreError("put")+ http.Error(w, fmt.Sprintf("failed to store media upload plan: %v", err), http.StatusBadGateway)+ return+ }+ g.recordUploadPlanCreated(uploadPlan)+ uploadURL := g.absoluteURL(mediaUploadProxyPath + uploadID)+ result.UploadURL = &uploadURL+ }++ writeJSON(w, http.StatusCreated, result)+}++func (g *gateway) handleMediaPatch(w http.ResponseWriter, r *http.Request, route route) {+ body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 1<<20))+ if err != nil {+ http.Error(w, "failed to read media patch request", http.StatusBadRequest)+ return+ }++ destinations := g.mediaDestinations(route, r.Header.Get("Authorization"))+ if len(destinations) == 0 {+ http.Error(w, "no media destinations configured", http.StatusBadGateway)+ return+ }++ type patchResult struct {+ destination string+ err error+ }+ results := make([]patchResult, len(destinations))+ var wg sync.WaitGroup+ for index, dest := range destinations {+ index, dest := index, dest+ wg.Add(1)+ go func() {+ defer wg.Done()+ results[index] = patchResult{+ destination: dest.name,+ err: g.patchMedia(r.Context(), dest, route.path, body, r.Header.Get("Content-Type")),+ }+ }()+ }+ wg.Wait()+ for _, result := range results {+ if result.err != nil {+ http.Error(w, fmt.Sprintf("%s media patch failed: %v", result.destination, result.err), http.StatusBadGateway)+ return+ }+ }+ w.WriteHeader(http.StatusNoContent)+}++func (g *gateway) handleMediaGet(w http.ResponseWriter, r *http.Request, route route) {+ destinations := g.mediaDestinations(route, r.Header.Get("Authorization"))+ if len(destinations) == 0 {+ http.Error(w, "no media destinations configured", http.StatusBadGateway)+ return+ }+ target := destinations[0]+ if route.destination != "" {+ target = destinations[len(destinations)-1]+ }+ resp := g.getMedia(r.Context(), target, route.path, r.URL.RawQuery)+ if resp == nil {+ http.Error(w, "media get failed", http.StatusBadGateway)+ return+ }+ defer resp.Body.Close()+ copyResponseHeaders(w.Header(), resp.Header)+ w.WriteHeader(resp.StatusCode)+ _, _ = io.Copy(w, resp.Body)+}++func (g *gateway) getMedia(ctx context.Context, target destination, path string, rawQuery string) *http.Response {+ upstreamURL := target.baseURL + path+ if rawQuery != "" {+ upstreamURL += "?" + rawQuery+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil)+ if err != nil {+ return nil+ }+ req.Header.Set("Authorization", target.authorization)+ resp, err := g.doUpstream(req, "media_get", target.name)+ if err != nil {+ return nil+ }+ return resp+}++func (g *gateway) handleMetrics(w http.ResponseWriter, r *http.Request) {+ if g.cfg.metricsSecret == "" {+ w.WriteHeader(http.StatusUnauthorized)+ return+ }+ const prefix = "Bearer "+ auth := r.Header.Get("Authorization")+ if len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {+ w.WriteHeader(http.StatusUnauthorized)+ return+ }+ token := strings.TrimSpace(auth[len(prefix):])+ if subtle.ConstantTimeCompare([]byte(token), []byte(g.cfg.metricsSecret)) != 1 {+ w.WriteHeader(http.StatusUnauthorized)+ return+ }+ g.metricsHTTP.ServeHTTP(w, r)+}++func (g *gateway) handleMediaUpload(w http.ResponseWriter, r *http.Request) {+ uploadID := strings.TrimPrefix(r.URL.Path, mediaUploadProxyPath)++ plan, ok, err := g.takeUpload(r.Context(), uploadID)+ if err != nil {+ g.recordUploadPlanStoreError("take")+ http.Error(w, "failed to load media upload plan", http.StatusBadGateway)+ return+ }+ if !ok {+ g.recordUploadPlanMiss()+ http.Error(w, "unknown or expired upload", http.StatusNotFound)+ return+ }++ body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxUploadBytes(plan.ContentLength)))+ if err != nil {+ if err := g.restoreUpload(r.Context(), uploadID, plan); err != nil {+ g.recordUploadPlanStoreError("restore")+ }+ http.Error(w, "failed to read upload body", http.StatusBadRequest)+ return+ }++ type uploadResult struct {+ destination string+ status int+ err error+ }+ results := make([]uploadResult, len(plan.Destinations))+ var wg sync.WaitGroup+ for index, dest := range plan.Destinations {+ index, dest := index, dest+ wg.Add(1)+ go func() {+ defer wg.Done()+ code, err := g.putMedia(r.Context(), dest, body, r.Header)+ results[index] = uploadResult{destination: dest.Name, status: code, err: err}+ }()+ }+ wg.Wait()+ status := http.StatusOK+ for _, result := range results {+ if result.err != nil {+ if err := g.restoreUpload(r.Context(), uploadID, plan); err != nil {+ g.recordUploadPlanStoreError("restore")+ }+ http.Error(w, fmt.Sprintf("%s upload failed: %v", result.destination, result.err), http.StatusBadGateway)+ return+ }+ if result.status > status {+ status = result.status+ }+ }+ g.recordUploadPlanCompleted(plan)+ w.WriteHeader(status)+}++func (g *gateway) mediaDestinations(route route, tenantAuth string) []destination {+ destinations := []destination{g.cfg.central}+ if route.destination == "" {+ return destinations+ }+ baseURL := g.cfg.tenants[route.destination]+ if baseURL == "" || strings.TrimSpace(tenantAuth) == "" {+ return destinations+ }+ return append(destinations, destination{+ name: "tenant_" + route.destination,+ baseURL: baseURL,+ authorization: strings.TrimSpace(tenantAuth),+ })+}++func (g *gateway) forwardTraceToCollector(ctx context.Context, headers http.Header, body []byte, contentType string, contentEncoding string) (*http.Response, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, g.cfg.traceCollectorURL+otelTracePath, bytes.NewReader(body))+ if err != nil {+ return nil, err+ }+ req.Header.Set("Content-Type", contentTypeOrDefault(contentType, "application/x-protobuf"))+ if value := strings.TrimSpace(headers.Get("Authorization")); value != "" {+ req.Header.Set("Authorization", value)+ }+ if contentEncoding != "" {+ req.Header.Set("Content-Encoding", contentEncoding)+ }+ resp, err := g.doUpstream(req, "trace_collector", "collector")+ if err != nil {+ return nil, err+ }+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ defer resp.Body.Close()+ text, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return nil, fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(text)))+ }+ return resp, nil+}++func (g *gateway) postMediaCreate(ctx context.Context, dest destination, body []byte, contentType string) (mediaUploadResponse, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, dest.baseURL+mediaPath, bytes.NewReader(body))+ if err != nil {+ return mediaUploadResponse{}, err+ }+ req.Header.Set("Authorization", dest.authorization)+ req.Header.Set("Content-Type", contentTypeOrDefault(contentType, "application/json"))+ resp, err := g.doUpstream(req, "media_create", dest.name)+ if err != nil {+ return mediaUploadResponse{}, err+ }+ defer resp.Body.Close()+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ text, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return mediaUploadResponse{}, fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(text)))+ }+ var result mediaUploadResponse+ if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {+ return mediaUploadResponse{}, err+ }+ return result, nil+}++func (g *gateway) patchMedia(ctx context.Context, dest destination, path string, body []byte, contentType string) error {+ req, err := http.NewRequestWithContext(ctx, http.MethodPatch, dest.baseURL+path, bytes.NewReader(body))+ if err != nil {+ return err+ }+ req.Header.Set("Authorization", dest.authorization)+ req.Header.Set("Content-Type", contentTypeOrDefault(contentType, "application/json"))+ return g.doExpect2xx("media_patch", dest.name, req)+}++func (g *gateway) putMedia(ctx context.Context, dest uploadDestination, body []byte, originalHeaders http.Header) (int, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodPut, dest.UploadURL, bytes.NewReader(body))+ if err != nil {+ return 0, err+ }+ if value := originalHeaders.Get("Content-Type"); value != "" {+ if !allowedUploadContentType(value) {+ return 0, fmt.Errorf("unsupported upload content type %q", value)+ }+ req.Header.Set("Content-Type", value)+ }+ if value := originalHeaders.Get("Content-Encoding"); value != "" {+ req.Header.Set("Content-Encoding", value)+ }+ if isAzureUploadURL(dest.UploadURL) {+ if value := originalHeaders.Get("x-ms-blob-type"); value != "" {+ req.Header.Set("x-ms-blob-type", value)+ }+ } else if !isGCSUploadURL(dest.UploadURL) {+ if value := originalHeaders.Get("x-amz-checksum-sha256"); value != "" {+ req.Header.Set("x-amz-checksum-sha256", value)+ }+ }+ resp, err := g.doUpstream(req, "media_upload", dest.Name)+ if err != nil {+ return 0, err+ }+ defer resp.Body.Close()+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ text, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return resp.StatusCode, fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(text)))+ }+ return resp.StatusCode, nil+}++func (g *gateway) doExpect2xx(operation string, destination string, req *http.Request) error {+ resp, err := g.doUpstream(req, operation, destination)+ if err != nil {+ return err+ }+ defer resp.Body.Close()+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ text, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(text)))+ }+ return nil+}++func (g *gateway) doUpstream(req *http.Request, operation string, destination string) (*http.Response, error) {+ startedAt := time.Now()+ resp, err := g.cfg.client.Do(req)+ if err != nil {+ if g.metrics != nil {+ g.metrics.recordUpstream(operation, destination, "error", time.Since(startedAt))+ }+ return nil, err+ }+ if g.metrics != nil {+ g.metrics.recordUpstream(operation, destination, statusClass(resp.StatusCode), time.Since(startedAt))+ }+ return resp, nil+}++func (g *gateway) recordTraceExport(route route, result string) {+ if g.metrics == nil {+ return+ }+ destination := centralName+ if route.destination != "" {+ destination = "tenant_" + route.destination+ }+ g.metrics.recordTraceExport(destination, result)+}++func (g *gateway) recordMediaDivergence(kind string, destination string) {+ if g.metrics != nil {+ g.metrics.recordMediaDivergence(kind, destination)+ }+}++func (g *gateway) recordUploadPlanCreated(plan uploadPlan) {+ if g.metrics != nil {+ g.metrics.recordUploadPlanCreated(plan.ContentLength)+ }+}++func (g *gateway) recordUploadPlanCompleted(plan uploadPlan) {+ if len(plan.Destinations) == 0 {+ return+ }+ if g.metrics != nil {+ g.metrics.recordUploadPlanCompleted()+ }+}++func (g *gateway) recordUploadPlanMiss() {+ if g.metrics != nil {+ g.metrics.recordUploadPlanMiss()+ }+}++func (g *gateway) recordUploadPlanStoreError(operation string) {+ if g.metrics != nil {+ g.metrics.recordUploadPlanStoreError(operation)+ }+}++func addTenantRouteAttributes(body []byte, contentType string, destination string) ([]byte, error) {+ if isJSONContentType(contentType) {+ return addJSONTenantRouteAttributes(body, destination)+ }+ return addProtobufTenantRouteAttributes(body, destination)+}++func addProtobufTenantRouteAttributes(body []byte, destination string) ([]byte, error) {+ var request tracepb.ExportTraceServiceRequest+ if err := proto.Unmarshal(body, &request); err != nil {+ return nil, err+ }++ for _, resourceSpan := range request.ResourceSpans {+ for _, scopeSpan := range resourceSpan.ScopeSpans {+ for _, span := range scopeSpan.Spans {+ upsertSpanStringAttribute(span, tenantDestAttribute, destination)+ upsertSpanStringAttribute(span, tenantExportAttribute, "true")+ }+ }+ }++ return proto.Marshal(&request)+}++func upsertSpanStringAttribute(span *tracev1.Span, key string, value string) {+ for _, attribute := range span.Attributes {+ if attribute.Key == key {+ attribute.Value = stringAnyValue(value)+ return+ }+ }+ span.Attributes = append(span.Attributes, &commonv1.KeyValue{+ Key: key,+ Value: stringAnyValue(value),+ })+}++func stringAnyValue(value string) *commonv1.AnyValue {+ return &commonv1.AnyValue{+ Value: &commonv1.AnyValue_StringValue{StringValue: value},+ }+}++func addJSONTenantRouteAttributes(body []byte, destination string) ([]byte, error) {+ var request map[string]any+ if err := json.Unmarshal(body, &request); err != nil {+ return nil, err+ }+ resourceSpans, _ := request["resourceSpans"].([]any)+ for _, resourceSpan := range resourceSpans {+ resourceSpanMap, _ := resourceSpan.(map[string]any)+ scopeSpans, _ := resourceSpanMap["scopeSpans"].([]any)+ for _, scopeSpan := range scopeSpans {+ scopeSpanMap, _ := scopeSpan.(map[string]any)+ spans, _ := scopeSpanMap["spans"].([]any)+ for _, span := range spans {+ spanMap, _ := span.(map[string]any)+ upsertJSONSpanStringAttribute(spanMap, tenantDestAttribute, destination)+ upsertJSONSpanStringAttribute(spanMap, tenantExportAttribute, "true")+ }+ }+ }+ return json.Marshal(request)+}++func upsertJSONSpanStringAttribute(span map[string]any, key string, value string) {+ attrs, _ := span["attributes"].([]any)+ for _, attr := range attrs {+ attrMap, _ := attr.(map[string]any)+ if attrMap["key"] == key {+ attrMap["value"] = map[string]any{"stringValue": value}+ return+ }+ }+ span["attributes"] = append(attrs, map[string]any{+ "key": key,+ "value": map[string]any{"stringValue": value},+ })+}++func stringValue(value *commonv1.AnyValue) string {+ if value == nil {+ return ""+ }+ if stringValue := value.GetStringValue(); stringValue != "" {+ return stringValue+ }+ if value.GetBoolValue() {+ return "true"+ }+ return ""+}++func (g *gateway) storeUpload(ctx context.Context, uploadID string, plan uploadPlan) error {+ return g.cfg.uploadStore.Put(ctx, uploadID, plan)+}++func (g *gateway) takeUpload(ctx context.Context, uploadID string) (uploadPlan, bool, error) {+ if !validUploadID(uploadID) {+ return uploadPlan{}, false, nil+ }+ plan, ok, err := g.cfg.uploadStore.Take(ctx, uploadID)+ if err != nil || !ok {+ return uploadPlan{}, ok, err+ }+ return plan, true, nil+}++func (g *gateway) restoreUpload(ctx context.Context, uploadID string, plan uploadPlan) error {+ if time.Now().After(plan.ExpiresAt) {+ return nil+ }+ return g.cfg.uploadStore.Put(ctx, uploadID, plan)+}++type redisUploadPlanStore struct {+ client *redis.Client+ prefix string+}++func newRedisUploadPlanStore(cfg redisConfig) (*redisUploadPlanStore, error) {+ options, err := redis.ParseURL(cfg.uri)+ if err != nil {+ return nil, fmt.Errorf("parse LANGFUSE_FANOUT_REDIS_URI: %w", err)+ }+ if cfg.username != "" {+ options.Username = cfg.username+ }+ if cfg.password != "" {+ options.Password = cfg.password+ }+ client := redis.NewClient(options)+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ if err := client.Ping(ctx).Err(); err != nil {+ _ = client.Close()+ return nil, fmt.Errorf("ping Redis: %w", err)+ }+ return &redisUploadPlanStore{+ client: client,+ prefix: normalizeRedisKeyPrefix(cfg.keyPrefix),+ }, nil+}++func (s *redisUploadPlanStore) Put(ctx context.Context, uploadID string, plan uploadPlan) error {+ if !validUploadID(uploadID) {+ return errors.New("invalid upload id")+ }+ ttl := time.Until(plan.ExpiresAt)+ if ttl <= 0 {+ return errors.New("upload plan already expired")+ }+ body, err := json.Marshal(plan)+ if err != nil {+ return err+ }+ return s.client.Set(ctx, s.key(uploadID), body, ttl).Err()+}++func (s *redisUploadPlanStore) Take(ctx context.Context, uploadID string) (uploadPlan, bool, error) {+ if !validUploadID(uploadID) {+ return uploadPlan{}, false, nil+ }+ value, err := redisTakeScript.Run(ctx, s.client, []string{s.key(uploadID)}).Text()+ if errors.Is(err, redis.Nil) {+ return uploadPlan{}, false, nil+ }+ if err != nil {+ return uploadPlan{}, false, err+ }+ var plan uploadPlan+ if err := json.Unmarshal([]byte(value), &plan); err != nil {+ return uploadPlan{}, false, err+ }+ return plan, true, nil+}++func (s *redisUploadPlanStore) Ping(ctx context.Context) error {+ return s.client.Ping(ctx).Err()+}++func (s *redisUploadPlanStore) Close() error {+ return s.client.Close()+}++func (s *redisUploadPlanStore) key(uploadID string) string {+ return s.prefix + ":media-upload:" + uploadID+}++var redisTakeScript = redis.NewScript(`+local value = redis.call("GET", KEYS[1])+if value then+ redis.call("DEL", KEYS[1])+end+return value+`)++func normalizeRedisKeyPrefix(value string) string {+ value = strings.TrimSpace(value)+ if value == "" {+ return "langfuse-fanout"+ }+ return strings.TrimRight(value, ":")+}++func validUploadID(value string) bool {+ if value == "" || len(value) > 128 {+ return false+ }+ for _, r := range value {+ switch {+ case r >= 'a' && r <= 'f':+ case r >= '0' && r <= '9':+ default:+ return false+ }+ }+ return true+}++func parseRoute(path string) route {+ if !strings.HasPrefix(path, tenantPrefix) {+ return route{path: path}+ }+ rest := strings.TrimPrefix(path, tenantPrefix)+ destination, suffix, ok := strings.Cut(rest, "/")+ if !ok {+ return route{path: path}+ }+ normalizedDestination := normalizeDestinationKey(destination)+ if normalizedDestination == "" {+ return route{path: path}+ }+ return route{destination: normalizedDestination, path: "/" + suffix}+}++func readMaybeGzip(r *http.Request) ([]byte, error) {+ if !strings.EqualFold(r.Header.Get("Content-Encoding"), "gzip") {+ return io.ReadAll(io.LimitReader(r.Body, 20<<20))+ }+ reader, err := gzip.NewReader(r.Body)+ if err != nil {+ return nil, err+ }+ defer reader.Close()+ return io.ReadAll(io.LimitReader(reader, 20<<20))+}++func gzipBytes(body []byte) ([]byte, error) {+ var buffer bytes.Buffer+ writer := gzip.NewWriter(&buffer)+ if _, err := writer.Write(body); err != nil {+ return nil, err+ }+ if err := writer.Close(); err != nil {+ return nil, err+ }+ return buffer.Bytes(), nil+}++func (g *gateway) absoluteURL(path string) string {+ return g.cfg.publicURL + path+}++func writeJSON(w http.ResponseWriter, status int, value any) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(status)+ _ = json.NewEncoder(w).Encode(value)+}++func copyResponseHeaders(target http.Header, source http.Header) {+ for key, values := range source {+ if strings.EqualFold(key, "Content-Length") {+ continue+ }+ for _, value := range values {+ target.Add(key, value)+ }+ }+}++func normalizeBaseURL(value string) string {+ value = strings.TrimSpace(value)+ if value == "" {+ return ""+ }+ parsed, err := url.Parse(value)+ if err != nil || parsed.Host == "" {+ return ""+ }+ if parsed.Scheme != "http" && parsed.Scheme != "https" {+ return ""+ }+ parsed.Path = strings.TrimRight(parsed.Path, "/")+ parsed.RawQuery = ""+ parsed.Fragment = ""+ return strings.TrimRight(parsed.String(), "/")+}++func normalizeCollectorURL(value string) string {+ value = strings.TrimSpace(value)+ if value == "" {+ return defaultTraceCollector+ }+ return strings.TrimRight(value, "/")+}++func normalizeDestinationKey(value string) string {+ value = strings.TrimSpace(strings.ToLower(value))+ if value == "" {+ return ""+ }+ var builder strings.Builder+ for _, r := range value {+ switch {+ case r >= 'a' && r <= 'z':+ builder.WriteRune(r)+ case r >= '0' && r <= '9':+ builder.WriteRune(r)+ case r == '_' || r == '-':+ builder.WriteRune(r)+ default:+ builder.WriteRune('_')+ }+ }+ normalized := builder.String()+ if normalized == "" || normalized[0] < 'a' || normalized[0] > 'z' {+ return ""+ }+ return normalized+}++func parseDestinationKeys(value string) map[string]bool {+ result := map[string]bool{}+ for _, item := range strings.Split(value, ",") {+ key := normalizeDestinationKey(item)+ if key != "" {+ result[key] = true+ }+ }+ return result+}++func contentTypeOrDefault(value string, fallback string) string {+ if strings.TrimSpace(value) == "" {+ return fallback+ }+ return value+}++func isJSONContentType(value string) bool {+ mediaType := strings.ToLower(strings.TrimSpace(strings.Split(value, ";")[0]))+ return mediaType == "application/json" || strings.HasSuffix(mediaType, "+json")+}++func isGCSUploadURL(value string) bool {+ parsed, err := url.Parse(value)+ if err != nil {+ return false+ }+ host := parsed.Hostname()+ return host == "storage.googleapis.com" || strings.HasSuffix(host, ".storage.googleapis.com")+}++func isAzureUploadURL(value string) bool {+ parsed, err := url.Parse(value)+ if err != nil {+ return false+ }+ host := parsed.Hostname()+ return strings.Contains(host, ".blob.core.") || strings.Contains(host, ".blob.storage.")+}++func allowedUploadContentType(value string) bool {+ mediaType := strings.ToLower(strings.TrimSpace(strings.Split(value, ";")[0]))+ return mediaType == "application/octet-stream" ||+ mediaType == "application/pdf" ||+ strings.HasPrefix(mediaType, "image/") ||+ strings.HasPrefix(mediaType, "audio/") ||+ strings.HasPrefix(mediaType, "video/")+}++func envOrDefault(key string, fallback string) string {+ if value := strings.TrimSpace(os.Getenv(key)); value != "" {+ return value+ }+ return fallback+}++func firstNonEmptyEnv(keys ...string) string {+ for _, key := range keys {+ if value := strings.TrimSpace(os.Getenv(key)); value != "" {+ return value+ }+ }+ return ""+}++func parseDurationEnv(key string, fallback time.Duration) time.Duration {+ value := strings.TrimSpace(os.Getenv(key))+ if value == "" {+ return fallback+ }+ duration, err := time.ParseDuration(value)+ if err == nil {+ return duration+ }+ seconds, err := strconv.Atoi(value)+ if err != nil {+ return fallback+ }+ return time.Duration(seconds) * time.Second+}++func randomID() (string, error) {+ var bytes [16]byte+ if _, err := rand.Read(bytes[:]); err != nil {+ return "", err+ }+ return hex.EncodeToString(bytes[:]), nil+}++func maxUploadBytes(contentLength int64) int64 {+ if contentLength <= 0 {+ return 256 << 20+ }+ return contentLength + (1 << 20)+}diff --git a/otel/langfuse-fanout/cmd/langfuse-fanout/main_test.go b/otel/langfuse-fanout/cmd/langfuse-fanout/main_test.gonew file mode 100644index 0000000000..aaf8b593c2--- /dev/null+++ b/otel/langfuse-fanout/cmd/langfuse-fanout/main_test.go@@ -0,0 +1,929 @@+package main++import (+ "bytes"+ "compress/gzip"+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "sync"+ "testing"+ "time"++ tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ commonv1 "go.opentelemetry.io/proto/otlp/common/v1"+ resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1"+ tracev1 "go.opentelemetry.io/proto/otlp/trace/v1"+ "google.golang.org/protobuf/proto"+)++func TestLoadConfigRejectsDestinationsMissingTraceRoutes(t *testing.T) {+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_BASE_URL", "https://cloud.langfuse.com")+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER", "Basic central")+ t.Setenv("LANGFUSE_FANOUT_PUBLIC_URL", "http://fanout.local:4318")+ t.Setenv("LANGFUSE_FANOUT_TENANT_DESTINATIONS", "eu=https://cloud.langfuse.com,ca=https://example.com")+ t.Setenv("LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS", "eu,us,jp")++ _, err := loadConfig()+ if err == nil || !strings.Contains(err.Error(), `tenant destination "ca"`) {+ t.Fatalf("expected missing trace route error, got %v", err)+ }+}++func TestLoadConfigRejectsTenantDestinationsWithoutTraceKeys(t *testing.T) {+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_BASE_URL", "https://cloud.langfuse.com")+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER", "Basic central")+ t.Setenv("LANGFUSE_FANOUT_PUBLIC_URL", "http://fanout.local:4318")+ t.Setenv("LANGFUSE_FANOUT_TENANT_DESTINATIONS", "eu=https://cloud.langfuse.com")+ t.Setenv("LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS", "")++ _, err := loadConfig()+ if err == nil || !strings.Contains(err.Error(), "LANGFUSE_FANOUT_TRACE_DESTINATION_KEYS is required") {+ t.Fatalf("expected missing trace keys error, got %v", err)+ }+}++func TestLoadConfigRejectsInvalidPublicURL(t *testing.T) {+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_BASE_URL", "https://cloud.langfuse.com")+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER", "Basic central")+ t.Setenv("LANGFUSE_FANOUT_PUBLIC_URL", "ftp://example.com")++ _, err := loadConfig()+ if err == nil || !strings.Contains(err.Error(), "LANGFUSE_FANOUT_PUBLIC_URL") {+ t.Fatalf("expected invalid public URL error, got %v", err)+ }+}++func TestLoadConfigRequiresRedisURI(t *testing.T) {+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_BASE_URL", "https://cloud.langfuse.com")+ t.Setenv("LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER", "Basic central")+ t.Setenv("LANGFUSE_FANOUT_PUBLIC_URL", "http://fanout.local:4318")++ _, err := loadConfig()+ if err == nil || !strings.Contains(err.Error(), "LANGFUSE_FANOUT_REDIS_URI") {+ t.Fatalf("expected missing Redis URI error, got %v", err)+ }+}++func TestNormalizeBaseURLAllowsOnlyHTTPAndHTTPS(t *testing.T) {+ if got := normalizeBaseURL("http://localhost:3000/path/"); got != "http://localhost:3000/path" {+ t.Fatalf("http URL normalized to %q", got)+ }+ if got := normalizeBaseURL("https://cloud.langfuse.com/"); got != "https://cloud.langfuse.com" {+ t.Fatalf("https URL normalized to %q", got)+ }+ if got := normalizeBaseURL("file:///tmp/langfuse"); got != "" {+ t.Fatalf("file URL should be rejected, got %q", got)+ }+}++func TestTraceProxyForwardsExistingRoutingAttributesToCollector(t *testing.T) {+ t.Parallel()++ var collectorTrace []byte+ collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != otelTracePath {+ t.Fatalf("unexpected collector path %s", r.URL.Path)+ }+ if got := r.Header.Get("Authorization"); got != "Basic tenant" {+ t.Fatalf("collector auth = %q", got)+ }+ collectorTrace, _ = io.ReadAll(r.Body)+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte("{}"))+ }))+ defer collector.Close()++ gw := newTestGatewayWithCollector(collector.URL)+ body := buildTraceRequest(t, map[string]string{+ tenantExportAttribute: "true",+ tenantDestAttribute: "eu",+ "kept": "value",+ })++ req := httptest.NewRequest(http.MethodPost, otelTracePath, bytes.NewReader(body))+ req.Header.Set("Content-Type", "application/x-protobuf")+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if resp.Body.String() != "{}" {+ t.Fatalf("expected collector response body, got %s", resp.Body.String())+ }+ if len(collectorTrace) == 0 {+ t.Fatal("expected collector export")+ }++ attrs := parseTraceAttributes(t, collectorTrace)+ if attrs[tenantExportAttribute] != "true" || attrs[tenantDestAttribute] != "eu" {+ t.Fatalf("collector trace missing routing attrs: %#v", attrs)+ }+ if attrs["kept"] != "value" {+ t.Fatalf("collector trace lost kept attr: %#v", attrs)+ }+}++func TestGzipTraceProxyAddsRoutingAttributesFromPath(t *testing.T) {+ t.Parallel()++ var collectorTrace []byte+ var collectorEncoding string+ collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ collectorEncoding = r.Header.Get("Content-Encoding")+ reader, err := gzip.NewReader(r.Body)+ if err != nil {+ t.Fatalf("collector gzip reader: %v", err)+ }+ defer reader.Close()+ collectorTrace, _ = io.ReadAll(reader)+ _, _ = w.Write([]byte(`{"partialSuccess":{}}`))+ }))+ defer collector.Close()++ var zipped bytes.Buffer+ zipper := gzip.NewWriter(&zipped)+ if _, err := zipper.Write(buildTraceRequest(t, map[string]string{"kept": "value"})); err != nil {+ t.Fatal(err)+ }+ if err := zipper.Close(); err != nil {+ t.Fatal(err)+ }++ gw := newTestGatewayWithCollector(collector.URL)+ req := httptest.NewRequest(http.MethodPost, tenantPrefix+"eu"+otelTracePath, bytes.NewReader(zipped.Bytes()))+ req.Header.Set("Content-Type", "application/x-protobuf")+ req.Header.Set("Content-Encoding", "gzip")+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if collectorEncoding != "gzip" {+ t.Fatalf("collector encoding = %q", collectorEncoding)+ }+ if resp.Body.String() != `{"partialSuccess":{}}` {+ t.Fatalf("expected collector response body, got %s", resp.Body.String())+ }+ attrs := parseTraceAttributes(t, collectorTrace)+ if attrs[tenantExportAttribute] != "true" || attrs[tenantDestAttribute] != "eu" || attrs["kept"] != "value" {+ t.Fatalf("collector trace attrs = %#v", attrs)+ }+}++func TestJSONTraceProxyAddsRoutingAttributesFromPath(t *testing.T) {+ t.Parallel()++ var collectorTrace []byte+ var collectorContentType string+ collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != otelTracePath {+ t.Fatalf("unexpected collector path %s", r.URL.Path)+ }+ if got := r.Header.Get("Authorization"); got != "Basic tenant" {+ t.Fatalf("collector auth = %q", got)+ }+ collectorContentType = r.Header.Get("Content-Type")+ collectorTrace, _ = io.ReadAll(r.Body)+ _, _ = w.Write([]byte("{}"))+ }))+ defer collector.Close()++ gw := newTestGatewayWithCollector(collector.URL)+ body := buildJSONTraceRequest(t, map[string]any{+ "kept": "value",+ })++ req := httptest.NewRequest(http.MethodPost, tenantPrefix+"eu"+otelTracePath, bytes.NewReader(body))+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if len(collectorTrace) == 0 {+ t.Fatal("expected collector export")+ }+ if collectorContentType != "application/json" {+ t.Fatalf("content type = %q", collectorContentType)+ }+ attrs := parseJSONTraceAttributes(t, collectorTrace)+ if attrs[tenantExportAttribute] != "true" || attrs[tenantDestAttribute] != "eu" {+ t.Fatalf("collector trace missing routing attrs: %#v", attrs)+ }+ if attrs["kept"] != "value" {+ t.Fatalf("collector trace lost kept attr: %#v", attrs)+ }+}++func TestMediaUploadFansOutToCentralAndTenant(t *testing.T) {+ t.Parallel()++ var mu sync.Mutex+ uploads := map[string]string{}+ upstream := func(name string) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == mediaPath:+ uploadURL := "http://" + r.Host + "/upload/" + name+ writeJSON(w, http.StatusCreated, mediaUploadResponse{+ MediaID: "same-media-id",+ UploadURL: &uploadURL,+ })+ case r.Method == http.MethodPut && r.URL.Path == "/upload/"+name:+ body, _ := io.ReadAll(r.Body)+ mu.Lock()+ uploads[name] = string(body)+ mu.Unlock()+ w.WriteHeader(http.StatusOK)+ case r.Method == http.MethodPatch && r.URL.Path == mediaPath+"/same-media-id":+ w.WriteHeader(http.StatusNoContent)+ default:+ http.NotFound(w, r)+ }+ }))+ }+ central := upstream("central")+ defer central.Close()+ tenant := upstream("tenant")+ defer tenant.Close()++ store := newFakeUploadPlanStore()+ createGateway := newTestGatewayWithStore(central.URL, map[string]string{"eu": tenant.URL}, store)+ uploadGateway := newTestGatewayWithStore(central.URL, map[string]string{"eu": tenant.URL}, store)+ createBody := `{"traceId":"trace","contentType":"image/png","contentLength":5,"sha256Hash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","field":"input"}`+ req := httptest.NewRequest(http.MethodPost, tenantPrefix+"eu"+mediaPath, strings.NewReader(createBody))+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ createGateway.handle(resp, req)+ if resp.Code != http.StatusCreated {+ t.Fatalf("create status = %d, body = %s", resp.Code, resp.Body.String())+ }+ var create mediaUploadResponse+ if err := json.NewDecoder(resp.Body).Decode(&create); err != nil {+ t.Fatal(err)+ }+ if create.MediaID != "same-media-id" || create.UploadURL == nil || !strings.Contains(*create.UploadURL, mediaUploadProxyPath) {+ t.Fatalf("unexpected create response: %#v", create)+ }+ uploadID := strings.TrimPrefix(newUploadURLPath(t, *create.UploadURL), mediaUploadProxyPath)+ store.mu.Lock()+ storedPlan := store.plans[uploadID]+ store.mu.Unlock()+ storedPlanJSON, err := json.Marshal(storedPlan)+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(string(storedPlanJSON), "Basic ") {+ t.Fatalf("stored upload plan leaked authorization: %s", storedPlanJSON)+ }++ uploadReq := httptest.NewRequest(http.MethodPut, *create.UploadURL, strings.NewReader("hello"))+ uploadReq.Header.Set("Content-Type", "image/png")+ uploadResp := httptest.NewRecorder()+ uploadGateway.handle(uploadResp, uploadReq)+ if uploadResp.Code != http.StatusOK {+ t.Fatalf("upload status = %d, body = %s", uploadResp.Code, uploadResp.Body.String())+ }++ patchReq := httptest.NewRequest(http.MethodPatch, tenantPrefix+"eu"+mediaPath+"/same-media-id", strings.NewReader(`{"uploadHttpStatus":200}`))+ patchReq.Header.Set("Authorization", "Basic tenant")+ patchResp := httptest.NewRecorder()+ uploadGateway.handle(patchResp, patchReq)+ if patchResp.Code != http.StatusNoContent {+ t.Fatalf("patch status = %d, body = %s", patchResp.Code, patchResp.Body.String())+ }++ mu.Lock()+ defer mu.Unlock()+ if uploads["central"] != "hello" || uploads["tenant"] != "hello" {+ t.Fatalf("uploads = %#v", uploads)+ }+}++func TestMediaUploadRejectsInvalidIDBeforeReadingBody(t *testing.T) {+ t.Parallel()++ gw := newTestGateway("http://central.invalid", nil)+ reader := &failingReader{}+ req := httptest.NewRequest(http.MethodPut, mediaUploadProxyPath+"not-valid", reader)+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusNotFound {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if reader.read {+ t.Fatal("invalid upload id should not read request body")+ }+}++func TestMediaUploadIsOneTime(t *testing.T) {+ t.Parallel()++ var uploads int+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPut || r.URL.Path != "/upload" {+ http.NotFound(w, r)+ return+ }+ uploads+++ w.WriteHeader(http.StatusOK)+ }))+ defer upstream.Close()++ store := newFakeUploadPlanStore()+ uploadID := "abcdef1234"+ store.Put(context.Background(), uploadID, uploadPlan{+ ExpiresAt: time.Now().Add(time.Hour),+ ContentLength: 5,+ Destinations: []uploadDestination{{+ Name: "central",+ UploadURL: upstream.URL + "/upload",+ }},+ })+ gw := newTestGatewayWithStore(upstream.URL, nil, store)++ for index, expectedStatus := range []int{http.StatusOK, http.StatusNotFound} {+ req := httptest.NewRequest(http.MethodPut, mediaUploadProxyPath+uploadID, strings.NewReader("hello"))+ req.Header.Set("Content-Type", "image/png")+ resp := httptest.NewRecorder()+ gw.handle(resp, req)+ if resp.Code != expectedStatus {+ t.Fatalf("attempt %d status = %d, body = %s", index+1, resp.Code, resp.Body.String())+ }+ }+ if uploads != 1 {+ t.Fatalf("uploads = %d", uploads)+ }+}++func TestMediaUploadOversizeRestoresPlanForRetry(t *testing.T) {+ t.Parallel()++ var uploads int+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPut || r.URL.Path != "/upload" {+ http.NotFound(w, r)+ return+ }+ uploads+++ w.WriteHeader(http.StatusOK)+ }))+ defer upstream.Close()++ store := newFakeUploadPlanStore()+ uploadID := "abcdef1234"+ store.Put(context.Background(), uploadID, uploadPlan{+ ExpiresAt: time.Now().Add(time.Hour),+ ContentLength: 1,+ Destinations: []uploadDestination{{+ Name: "central",+ UploadURL: upstream.URL + "/upload",+ }},+ })+ gw := newTestGatewayWithStore(upstream.URL, nil, store)++ oversizeReq := httptest.NewRequest(+ http.MethodPut,+ mediaUploadProxyPath+uploadID,+ strings.NewReader(strings.Repeat("x", int(maxUploadBytes(1))+1)),+ )+ oversizeReq.Header.Set("Content-Type", "image/png")+ oversizeResp := httptest.NewRecorder()+ gw.handle(oversizeResp, oversizeReq)+ if oversizeResp.Code != http.StatusBadRequest {+ t.Fatalf("oversize status = %d, body = %s", oversizeResp.Code, oversizeResp.Body.String())+ }++ retryReq := httptest.NewRequest(http.MethodPut, mediaUploadProxyPath+uploadID, strings.NewReader("ok"))+ retryReq.Header.Set("Content-Type", "image/png")+ retryResp := httptest.NewRecorder()+ gw.handle(retryResp, retryReq)+ if retryResp.Code != http.StatusOK {+ t.Fatalf("retry status = %d, body = %s", retryResp.Code, retryResp.Body.String())+ }+ if uploads != 1 {+ t.Fatalf("uploads = %d", uploads)+ }+}++func TestMediaUploadUnsupportedContentTypeRestoresPlanForRetry(t *testing.T) {+ t.Parallel()++ var uploads int+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPut || r.URL.Path != "/upload" {+ http.NotFound(w, r)+ return+ }+ uploads+++ w.WriteHeader(http.StatusOK)+ }))+ defer upstream.Close()++ store := newFakeUploadPlanStore()+ uploadID := "abcdef1234"+ store.Put(context.Background(), uploadID, uploadPlan{+ ExpiresAt: time.Now().Add(time.Hour),+ ContentLength: 5,+ Destinations: []uploadDestination{{+ Name: "central",+ UploadURL: upstream.URL + "/upload",+ }},+ })+ gw := newTestGatewayWithStore(upstream.URL, nil, store)++ badReq := httptest.NewRequest(http.MethodPut, mediaUploadProxyPath+uploadID, strings.NewReader("hello"))+ badReq.Header.Set("Content-Type", "text/html")+ badResp := httptest.NewRecorder()+ gw.handle(badResp, badReq)+ if badResp.Code != http.StatusBadGateway {+ t.Fatalf("bad content-type status = %d, body = %s", badResp.Code, badResp.Body.String())+ }++ retryReq := httptest.NewRequest(http.MethodPut, mediaUploadProxyPath+uploadID, strings.NewReader("hello"))+ retryReq.Header.Set("Content-Type", "image/png")+ retryResp := httptest.NewRecorder()+ gw.handle(retryResp, retryReq)+ if retryResp.Code != http.StatusOK {+ t.Fatalf("retry status = %d, body = %s", retryResp.Code, retryResp.Body.String())+ }+ if uploads != 1 {+ t.Fatalf("uploads = %d", uploads)+ }+}++func TestMediaCreateUsesConfiguredPublicUploadURL(t *testing.T) {+ t.Parallel()++ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost || r.URL.Path != mediaPath {+ http.NotFound(w, r)+ return+ }+ uploadURL := "http://" + r.Host + "/upload"+ writeJSON(w, http.StatusCreated, mediaUploadResponse{+ MediaID: "same-media-id",+ UploadURL: &uploadURL,+ })+ }))+ defer upstream.Close()++ gw := newTestGateway(upstream.URL, nil)+ gw.cfg.publicURL = "https://fanout.example.com/base"+ req := httptest.NewRequest(http.MethodPost, mediaPath, strings.NewReader(`{"contentLength":5}`))+ req.Host = "attacker.example.com"+ req.Header.Set("X-Forwarded-Host", "attacker.example.com")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusCreated {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ var create mediaUploadResponse+ if err := json.NewDecoder(resp.Body).Decode(&create); err != nil {+ t.Fatal(err)+ }+ if create.UploadURL == nil || !strings.HasPrefix(*create.UploadURL, "https://fanout.example.com/base/") {+ t.Fatalf("unexpected upload URL: %#v", create.UploadURL)+ }+ if strings.Contains(*create.UploadURL, "attacker.example.com") {+ t.Fatalf("upload URL trusted request host: %s", *create.UploadURL)+ }+}++func TestMediaGetUsesTenantDestinationForTenantRoute(t *testing.T) {+ t.Parallel()++ var centralGets int+ central := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ centralGets+++ writeJSON(w, http.StatusOK, map[string]string{"url": "central"})+ }))+ defer central.Close()+ var tenantGets int+ tenant := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ tenantGets+++ if r.URL.Path != mediaPath+"/media-id" {+ t.Fatalf("unexpected tenant path %s", r.URL.Path)+ }+ if got := r.Header.Get("Authorization"); got != "Basic tenant" {+ t.Fatalf("tenant auth = %q", got)+ }+ writeJSON(w, http.StatusOK, map[string]string{"url": "tenant"})+ }))+ defer tenant.Close()++ gw := newTestGateway(central.URL, map[string]string{"eu": tenant.URL})+ req := httptest.NewRequest(http.MethodGet, tenantPrefix+"eu"+mediaPath+"/media-id", nil)+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if centralGets != 0 || tenantGets != 1 {+ t.Fatalf("centralGets=%d tenantGets=%d", centralGets, tenantGets)+ }+ if !strings.Contains(resp.Body.String(), "tenant") {+ t.Fatalf("unexpected body: %s", resp.Body.String())+ }+}++func TestMediaGetUsesCentralDestinationForCentralRoute(t *testing.T) {+ t.Parallel()++ var centralGets int+ central := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ centralGets+++ if got := r.Header.Get("Authorization"); got != "Basic central" {+ t.Fatalf("central auth = %q", got)+ }+ writeJSON(w, http.StatusOK, map[string]string{"url": "central"})+ }))+ defer central.Close()+ var tenantGets int+ tenant := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ tenantGets+++ w.WriteHeader(http.StatusOK)+ }))+ defer tenant.Close()++ gw := newTestGateway(central.URL, map[string]string{"eu": tenant.URL})+ req := httptest.NewRequest(http.MethodGet, mediaPath+"/media-id", nil)+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if centralGets != 1 || tenantGets != 0 {+ t.Fatalf("centralGets=%d tenantGets=%d", centralGets, tenantGets)+ }+ if !strings.Contains(resp.Body.String(), "central") {+ t.Fatalf("unexpected body: %s", resp.Body.String())+ }+}++func TestMediaCreateRejectsDifferentMediaIDs(t *testing.T) {+ t.Parallel()++ upstream := func(mediaID string) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost || r.URL.Path != mediaPath {+ http.NotFound(w, r)+ return+ }+ writeJSON(w, http.StatusCreated, mediaUploadResponse{MediaID: mediaID})+ }))+ }+ central := upstream("central-id")+ defer central.Close()+ tenant := upstream("tenant-id")+ defer tenant.Close()++ gw := newTestGateway(central.URL, map[string]string{"eu": tenant.URL})+ req := httptest.NewRequest(http.MethodPost, tenantPrefix+"eu"+mediaPath, strings.NewReader(`{"contentLength":0}`))+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusBadGateway {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }+ if !strings.Contains(resp.Body.String(), "media IDs differ") {+ t.Fatalf("unexpected body: %s", resp.Body.String())+ }++ metrics := scrapeMetrics(t, gw)+ if !strings.Contains(metrics, `langfuse_fanout_media_divergence_total{destination="tenant_eu",kind="media_id"} 1`) {+ t.Fatalf("missing media_id divergence metric:\n%s", metrics)+ }+}++func TestMediaCreateRecordsUploadURLPresenceDivergenceIndependentOfOrder(t *testing.T) {+ t.Parallel()++ central := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost || r.URL.Path != mediaPath {+ http.NotFound(w, r)+ return+ }+ writeJSON(w, http.StatusCreated, mediaUploadResponse{MediaID: "same-media-id"})+ }))+ defer central.Close()++ tenant := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost || r.URL.Path != mediaPath {+ http.NotFound(w, r)+ return+ }+ uploadURL := "http://" + r.Host + "/upload"+ writeJSON(w, http.StatusCreated, mediaUploadResponse{+ MediaID: "same-media-id",+ UploadURL: &uploadURL,+ })+ }))+ defer tenant.Close()++ gw := newTestGateway(central.URL, map[string]string{"eu": tenant.URL})+ req := httptest.NewRequest(http.MethodPost, tenantPrefix+"eu"+mediaPath, strings.NewReader(`{"contentLength":5}`))+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusCreated {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }++ metrics := scrapeMetrics(t, gw)+ if !strings.Contains(metrics, `langfuse_fanout_media_divergence_total{destination="central",kind="upload_url_presence"} 1`) {+ t.Fatalf("missing upload_url_presence divergence metric:\n%s", metrics)+ }+}++func TestMetricsEndpointRequiresBearerToken(t *testing.T) {+ t.Parallel()++ gw := newTestGateway("http://central.invalid", nil)++ unauthorized := httptest.NewRecorder()+ gw.handle(unauthorized, httptest.NewRequest(http.MethodGet, metricsPath, nil))+ if unauthorized.Code != http.StatusUnauthorized {+ t.Fatalf("unauthorized status = %d", unauthorized.Code)+ }++ wrong := httptest.NewRecorder()+ wrongReq := httptest.NewRequest(http.MethodGet, metricsPath, nil)+ wrongReq.Header.Set("Authorization", "Bearer wrong")+ gw.handle(wrong, wrongReq)+ if wrong.Code != http.StatusUnauthorized {+ t.Fatalf("wrong token status = %d", wrong.Code)+ }++ authorized := httptest.NewRecorder()+ authorizedReq := httptest.NewRequest(http.MethodGet, metricsPath, nil)+ authorizedReq.Header.Set("Authorization", "Bearer test-secret")+ gw.handle(authorized, authorizedReq)+ if authorized.Code != http.StatusOK {+ t.Fatalf("authorized status = %d, body = %s", authorized.Code, authorized.Body.String())+ }+ if !strings.Contains(authorized.Body.String(), "go_goroutines") {+ t.Fatalf("missing gateway metrics:\n%s", authorized.Body.String())+ }+}++func TestTraceProxyRecordsPrometheusMetrics(t *testing.T) {+ t.Parallel()++ collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte("{}"))+ }))+ defer collector.Close()++ gw := newTestGatewayWithCollector(collector.URL)+ body := buildTraceRequest(t, nil)+ req := httptest.NewRequest(http.MethodPost, tenantPrefix+"eu"+otelTracePath, bytes.NewReader(body))+ req.Header.Set("Content-Type", "application/x-protobuf")+ req.Header.Set("Authorization", "Basic tenant")+ resp := httptest.NewRecorder()++ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", resp.Code, resp.Body.String())+ }++ metrics := scrapeMetrics(t, gw)+ if !strings.Contains(metrics, `langfuse_fanout_trace_exports_total{destination="tenant_eu",result="success"} 1`) {+ t.Fatalf("missing trace export metric:\n%s", metrics)+ }+ if !strings.Contains(metrics, `langfuse_fanout_upstream_requests_total{destination="collector",operation="trace_collector",status_class="2xx"} 1`) {+ t.Fatalf("missing upstream collector metric:\n%s", metrics)+ }+}++func newTestGateway(centralURL string, tenants map[string]string) *gateway {+ return newTestGatewayWithStore(centralURL, tenants, newFakeUploadPlanStore())+}++func newTestGatewayWithStore(centralURL string, tenants map[string]string, store uploadPlanStore) *gateway {+ return newGateway(config{+ traceCollectorURL: "http://collector.invalid",+ publicURL: "http://fanout.local:4318",+ metricsSecret: "test-secret",+ central: destination{+ name: centralName,+ baseURL: centralURL,+ authorization: "Basic central",+ },+ tenants: tenants,+ uploadStore: store,+ client: &http.Client{Timeout: 5 * time.Second},+ })+}++func newTestGatewayWithCollector(collectorURL string) *gateway {+ gateway := newTestGateway("http://central.invalid", map[string]string{"eu": "http://tenant.invalid"})+ gateway.cfg.traceCollectorURL = collectorURL+ return gateway+}++func buildTraceRequest(t *testing.T, attrs map[string]string) []byte {+ t.Helper()+ spanAttrs := make([]*commonv1.KeyValue, 0, len(attrs))+ for key, value := range attrs {+ spanAttrs = append(spanAttrs, &commonv1.KeyValue{+ Key: key,+ Value: &commonv1.AnyValue{+ Value: &commonv1.AnyValue_StringValue{StringValue: value},+ },+ })+ }+ request := &tracepb.ExportTraceServiceRequest{+ ResourceSpans: []*tracev1.ResourceSpans{{+ Resource: &resourcev1.Resource{},+ ScopeSpans: []*tracev1.ScopeSpans{{+ Spans: []*tracev1.Span{{+ TraceId: []byte("1234567890123456"),+ SpanId: []byte("12345678"),+ Name: "test-span",+ Attributes: spanAttrs,+ }},+ }},+ }},+ }+ body, err := proto.Marshal(request)+ if err != nil {+ t.Fatal(err)+ }+ return body+}++func buildJSONTraceRequest(t *testing.T, attrs map[string]any) []byte {+ t.Helper()+ spanAttrs := make([]map[string]any, 0, len(attrs))+ for key, value := range attrs {+ anyValue := map[string]any{}+ switch typed := value.(type) {+ case bool:+ anyValue["boolValue"] = typed+ default:+ anyValue["stringValue"] = typed+ }+ spanAttrs = append(spanAttrs, map[string]any{+ "key": key,+ "value": anyValue,+ })+ }+ request := map[string]any{+ "resourceSpans": []any{+ map[string]any{+ "resource": map[string]any{},+ "scopeSpans": []any{+ map[string]any{+ "spans": []any{+ map[string]any{+ "traceId": "31323334353637383930313233343536",+ "spanId": "3132333435363738",+ "name": "test-span",+ "attributes": spanAttrs,+ },+ },+ },+ },+ },+ },+ }+ body, err := json.Marshal(request)+ if err != nil {+ t.Fatal(err)+ }+ return body+}++func parseTraceAttributes(t *testing.T, body []byte) map[string]string {+ t.Helper()+ var request tracepb.ExportTraceServiceRequest+ if err := proto.Unmarshal(body, &request); err != nil {+ t.Fatal(err)+ }+ result := map[string]string{}+ for _, resourceSpan := range request.ResourceSpans {+ for _, scopeSpan := range resourceSpan.ScopeSpans {+ for _, span := range scopeSpan.Spans {+ for _, attr := range span.Attributes {+ result[attr.Key] = stringValue(attr.Value)+ }+ }+ }+ }+ return result+}++func parseJSONTraceAttributes(t *testing.T, body []byte) map[string]string {+ t.Helper()+ var request map[string]any+ if err := json.Unmarshal(body, &request); err != nil {+ t.Fatal(err)+ }+ result := map[string]string{}+ resourceSpans, _ := request["resourceSpans"].([]any)+ for _, resourceSpan := range resourceSpans {+ resourceSpanMap, _ := resourceSpan.(map[string]any)+ scopeSpans, _ := resourceSpanMap["scopeSpans"].([]any)+ for _, scopeSpan := range scopeSpans {+ scopeSpanMap, _ := scopeSpan.(map[string]any)+ spans, _ := scopeSpanMap["spans"].([]any)+ for _, span := range spans {+ spanMap, _ := span.(map[string]any)+ attrs, _ := spanMap["attributes"].([]any)+ for _, attr := range attrs {+ attrMap, _ := attr.(map[string]any)+ valueMap, _ := attrMap["value"].(map[string]any)+ if key, ok := attrMap["key"].(string); ok {+ if value, ok := valueMap["stringValue"].(string); ok {+ result[key] = value+ }+ }+ }+ }+ }+ }+ return result+}++func scrapeMetrics(t *testing.T, gw *gateway) string {+ t.Helper()+ req := httptest.NewRequest(http.MethodGet, metricsPath, nil)+ req.Header.Set("Authorization", "Bearer test-secret")+ resp := httptest.NewRecorder()+ gw.handle(resp, req)+ if resp.Code != http.StatusOK {+ t.Fatalf("metrics status = %d, body = %s", resp.Code, resp.Body.String())+ }+ return resp.Body.String()+}++func newUploadURLPath(t *testing.T, value string) string {+ t.Helper()+ parsed, err := url.Parse(value)+ if err != nil {+ t.Fatal(err)+ }+ return parsed.Path+}++type fakeUploadPlanStore struct {+ mu sync.Mutex+ plans map[string]uploadPlan+}++func newFakeUploadPlanStore() *fakeUploadPlanStore {+ return &fakeUploadPlanStore{plans: map[string]uploadPlan{}}+}++func (s *fakeUploadPlanStore) Put(_ context.Context, uploadID string, plan uploadPlan) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.plans[uploadID] = plan+ return nil+}++func (s *fakeUploadPlanStore) Take(_ context.Context, uploadID string) (uploadPlan, bool, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ plan, ok := s.plans[uploadID]+ delete(s.plans, uploadID)+ return plan, ok, nil+}++func (s *fakeUploadPlanStore) Ping(_ context.Context) error {+ return nil+}++func (s *fakeUploadPlanStore) Close() error {+ return nil+}++type failingReader struct {+ read bool+}++func (r *failingReader) Read(_ []byte) (int, error) {+ r.read = true+ return 0, errors.New("read should not be called")+}diff --git a/otel/langfuse-fanout/cmd/langfuse-fanout/metrics.go b/otel/langfuse-fanout/cmd/langfuse-fanout/metrics.gonew file mode 100644index 0000000000..22259732e8--- /dev/null+++ b/otel/langfuse-fanout/cmd/langfuse-fanout/metrics.go@@ -0,0 +1,242 @@+package main++import (+ "fmt"+ "net/http"+ "strings"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/collectors"+)++type gatewayMetrics struct {+ registry *prometheus.Registry+ httpRequests *prometheus.CounterVec+ httpDuration *prometheus.HistogramVec+ upstreamRequests *prometheus.CounterVec+ upstreamDuration *prometheus.HistogramVec+ traceExports *prometheus.CounterVec+ mediaDivergence *prometheus.CounterVec+ uploadPlansCreated prometheus.Counter+ uploadPlansCompleted prometheus.Counter+ uploadPlanMisses prometheus.Counter+ uploadPlanStoreErrors *prometheus.CounterVec+ uploadBytes prometheus.Histogram+}++func newGatewayMetrics() *gatewayMetrics {+ registry := prometheus.NewRegistry()+ registry.MustRegister(+ collectors.NewGoCollector(),+ collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),+ )++ metrics := &gatewayMetrics{+ registry: registry,+ httpRequests: prometheus.NewCounterVec(prometheus.CounterOpts{+ Name: "langfuse_fanout_http_requests_total",+ Help: "Total HTTP requests handled by the Langfuse fanout gateway.",+ }, []string{"method", "path", "status"}),+ httpDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{+ Name: "langfuse_fanout_http_request_duration_seconds",+ Help: "HTTP request duration for the Langfuse fanout gateway.",+ Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},+ }, []string{"method", "path", "status"}),+ upstreamRequests: prometheus.NewCounterVec(prometheus.CounterOpts{+ Name: "langfuse_fanout_upstream_requests_total",+ Help: "Total upstream requests made by the Langfuse fanout gateway.",+ }, []string{"operation", "destination", "status_class"}),+ upstreamDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{+ Name: "langfuse_fanout_upstream_request_duration_seconds",+ Help: "Upstream request duration for Langfuse and collector calls.",+ Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},+ }, []string{"operation", "destination", "status_class"}),+ traceExports: prometheus.NewCounterVec(prometheus.CounterOpts{+ Name: "langfuse_fanout_trace_exports_total",+ Help: "Total trace export attempts through the Langfuse fanout gateway.",+ }, []string{"destination", "result"}),+ mediaDivergence: prometheus.NewCounterVec(prometheus.CounterOpts{+ Name: "langfuse_fanout_media_divergence_total",+ Help: "Media fanout upstream response divergence by kind. Values are counts only; no media IDs or URLs are exposed.",+ }, []string{"kind", "destination"}),+ uploadPlansCreated: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "langfuse_fanout_media_upload_plans_created_total",+ Help: "Total media upload fanout plans stored for SDK byte upload.",+ }),+ uploadPlansCompleted: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "langfuse_fanout_media_upload_plans_completed_total",+ Help: "Total media upload fanout plans consumed by SDK byte upload.",+ }),+ uploadPlanMisses: prometheus.NewCounter(prometheus.CounterOpts{+ Name: "langfuse_fanout_media_upload_plan_misses_total",+ Help: "Total media upload attempts that did not find a stored fanout plan.",+ }),+ uploadPlanStoreErrors: prometheus.NewCounterVec(prometheus.CounterOpts{+ Name: "langfuse_fanout_media_upload_plan_store_errors_total",+ Help: "Total Redis upload plan store errors by operation.",+ }, []string{"operation"}),+ uploadBytes: prometheus.NewHistogram(prometheus.HistogramOpts{+ Name: "langfuse_fanout_media_upload_bytes",+ Help: "Configured media upload content length for fanout upload plans.",+ Buckets: []float64{1_000, 10_000, 100_000, 1_000_000, 5_000_000, 10_000_000, 25_000_000, 50_000_000, 100_000_000, 250_000_000},+ }),+ }++ registry.MustRegister(+ metrics.httpRequests,+ metrics.httpDuration,+ metrics.upstreamRequests,+ metrics.upstreamDuration,+ metrics.traceExports,+ metrics.mediaDivergence,+ metrics.uploadPlansCreated,+ metrics.uploadPlansCompleted,+ metrics.uploadPlanMisses,+ metrics.uploadPlanStoreErrors,+ metrics.uploadBytes,+ )+ return metrics+}++func (m *gatewayMetrics) recordHTTP(method string, path string, status int, duration time.Duration) {+ if m == nil {+ return+ }+ labels := prometheus.Labels{+ "method": method,+ "path": path,+ "status": fmt.Sprintf("%d", status),+ }+ m.httpRequests.With(labels).Inc()+ m.httpDuration.With(labels).Observe(duration.Seconds())+}++func (m *gatewayMetrics) recordUpstream(operation string, destination string, statusClass string, duration time.Duration) {+ if m == nil {+ return+ }+ labels := prometheus.Labels{+ "operation": operation,+ "destination": normalizeMetricLabel(destination),+ "status_class": statusClass,+ }+ m.upstreamRequests.With(labels).Inc()+ m.upstreamDuration.With(labels).Observe(duration.Seconds())+}++func (m *gatewayMetrics) recordTraceExport(destination string, result string) {+ if m == nil {+ return+ }+ m.traceExports.WithLabelValues(normalizeMetricLabel(destination), result).Inc()+}++func (m *gatewayMetrics) recordMediaDivergence(kind string, destination string) {+ if m == nil {+ return+ }+ m.mediaDivergence.WithLabelValues(normalizeMetricLabel(kind), normalizeMetricLabel(destination)).Inc()+}++func (m *gatewayMetrics) recordUploadPlanCreated(contentLength int64) {+ if m == nil {+ return+ }+ m.uploadPlansCreated.Inc()+ if contentLength > 0 {+ m.uploadBytes.Observe(float64(contentLength))+ }+}++func (m *gatewayMetrics) recordUploadPlanCompleted() {+ if m == nil {+ return+ }+ m.uploadPlansCompleted.Inc()+}++func (m *gatewayMetrics) recordUploadPlanMiss() {+ if m == nil {+ return+ }+ m.uploadPlanMisses.Inc()+}++func (m *gatewayMetrics) recordUploadPlanStoreError(operation string) {+ if m == nil {+ return+ }+ m.uploadPlanStoreErrors.WithLabelValues(normalizeMetricLabel(operation)).Inc()+}++type statusRecorder struct {+ http.ResponseWriter+ status int+}++func (w *statusRecorder) WriteHeader(status int) {+ w.status = status+ w.ResponseWriter.WriteHeader(status)+}++func normalizeMetricPath(path string) string {+ route := parseRoute(path)+ switch {+ case route.path == otelTracePath:+ if route.destination == "" {+ return otelTracePath+ }+ return tenantPrefix + "#destination" + otelTracePath+ case route.path == mediaPath:+ if route.destination == "" {+ return mediaPath+ }+ return tenantPrefix + "#destination" + mediaPath+ case strings.HasPrefix(route.path, mediaPath+"/"):+ if route.destination == "" {+ return mediaPath + "/#mediaId"+ }+ return tenantPrefix + "#destination" + mediaPath + "/#mediaId"+ case strings.HasPrefix(path, mediaUploadProxyPath):+ return mediaUploadProxyPath + "#uploadId"+ case path == "/healthz":+ return "/healthz"+ default:+ return "/#path"+ }+}++func normalizeMetricLabel(value string) string {+ value = strings.TrimSpace(strings.ToLower(value))+ if value == "" {+ return "unknown"+ }+ var builder strings.Builder+ for _, r := range value {+ switch {+ case r >= 'a' && r <= 'z':+ builder.WriteRune(r)+ case r >= '0' && r <= '9':+ builder.WriteRune(r)+ case r == '_' || r == '-' || r == ':':+ builder.WriteRune(r)+ default:+ builder.WriteRune('_')+ }+ if builder.Len() >= 80 {+ break+ }+ }+ if builder.Len() == 0 {+ return "unknown"+ }+ return builder.String()+}++func statusClass(status int) string {+ if status <= 0 {+ return "error"+ }+ return fmt.Sprintf("%dxx", status/100)+}diff --git a/otel/langfuse-fanout/go.mod b/otel/langfuse-fanout/go.modnew file mode 100644index 0000000000..eed620d0e8--- /dev/null+++ b/otel/langfuse-fanout/go.mod@@ -0,0 +1,29 @@+module github.com/danny-avila/librechat/otel/langfuse-fanout++go 1.23.0++require (+ github.com/prometheus/client_golang v1.23.2+ github.com/redis/go-redis/v9 v9.17.2+ go.opentelemetry.io/proto/otlp v1.9.0+ google.golang.org/protobuf v1.36.10+)++require (+ github.com/beorn7/perks v1.0.1 // indirect+ github.com/cespare/xxhash/v2 v2.3.0 // indirect+ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect+ github.com/kr/text v0.2.0 // indirect+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect+ github.com/prometheus/client_model v0.6.2 // indirect+ github.com/prometheus/common v0.66.1 // indirect+ github.com/prometheus/procfs v0.16.1 // indirect+ go.yaml.in/yaml/v2 v2.4.2 // indirect+ golang.org/x/net v0.43.0 // indirect+ golang.org/x/sys v0.35.0 // indirect+ golang.org/x/text v0.28.0 // indirect+ google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect+ google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect+ google.golang.org/grpc v1.75.1 // indirect+)diff --git a/otel/langfuse-fanout/go.sum b/otel/langfuse-fanout/go.sumnew file mode 100644index 0000000000..a662c24fa9--- /dev/null+++ b/otel/langfuse-fanout/go.sum@@ -0,0 +1,90 @@+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=+github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=+github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=+go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=+go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=+go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=+go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=+go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=+gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=+google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY=+google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE=+google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE=+google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc=+google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=+google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=diff --git a/otel/langfuse-fanout/otelcol.yaml b/otel/langfuse-fanout/otelcol.yamlnew file mode 100644index 0000000000..21b1567d28--- /dev/null+++ b/otel/langfuse-fanout/otelcol.yaml@@ -0,0 +1,114 @@+extensions:+ headers_setter/tenant_passthrough:+ headers:+ - action: upsert+ key: Authorization+ from_context: authorization++receivers:+ otlp:+ protocols:+ http:+ endpoint: ${env:LANGFUSE_FANOUT_OTEL_RECEIVER_ENDPOINT}+ include_metadata: true+ traces_url_path: /api/public/otel/v1/traces++connectors:+ routing/langfuse_tenant_destination:+ error_mode: ignore+ table:+ - context: span+ condition: attributes["librechat.langfuse.destination"] == "eu"+ pipelines: [traces/tenant_eu]+ - context: span+ condition: attributes["librechat.langfuse.destination"] == "jp"+ pipelines: [traces/tenant_jp]+ - context: span+ condition: attributes["librechat.langfuse.destination"] == "us"+ pipelines: [traces/tenant_us]++processors:+ memory_limiter:+ check_interval: 1s+ limit_mib: ${env:LANGFUSE_FANOUT_MEMORY_LIMIT_MIB}+ spike_limit_mib: ${env:LANGFUSE_FANOUT_MEMORY_SPIKE_LIMIT_MIB}+ filter/tenant_export:+ error_mode: ignore+ traces:+ span:+ - attributes["librechat.langfuse.tenant_export.enabled"] != "true"+ attributes/drop_librechat_routing:+ actions:+ - key: librechat.langfuse.tenant_export.enabled+ action: delete+ - key: librechat.langfuse.destination+ action: delete+ batch/central:+ timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}+ send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}+ batch/by_auth_eu:+ timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}+ send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}+ metadata_keys: [authorization]+ metadata_cardinality_limit: ${env:LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT}+ batch/by_auth_jp:+ timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}+ send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}+ metadata_keys: [authorization]+ metadata_cardinality_limit: ${env:LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT}+ batch/by_auth_us:+ timeout: ${env:LANGFUSE_FANOUT_BATCH_TIMEOUT}+ send_batch_size: ${env:LANGFUSE_FANOUT_BATCH_SEND_SIZE}+ metadata_keys: [authorization]+ metadata_cardinality_limit: ${env:LANGFUSE_FANOUT_METADATA_CARDINALITY_LIMIT}++exporters:+ otlphttp/central:+ # Langfuse Cloud base URL options: https://cloud.langfuse.com (EU),+ # https://us.cloud.langfuse.com (US), https://jp.cloud.langfuse.com (JP).+ endpoint: '${env:LANGFUSE_FANOUT_CENTRAL_BASE_URL}/api/public/otel'+ headers:+ Authorization: '${env:LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER}'+ x-langfuse-ingestion-version: '4'+ otlphttp/tenant_eu:+ endpoint: '${env:LANGFUSE_FANOUT_TENANT_EU_BASE_URL}/api/public/otel'+ auth:+ authenticator: headers_setter/tenant_passthrough+ headers:+ x-langfuse-ingestion-version: '4'+ otlphttp/tenant_jp:+ endpoint: '${env:LANGFUSE_FANOUT_TENANT_JP_BASE_URL}/api/public/otel'+ auth:+ authenticator: headers_setter/tenant_passthrough+ headers:+ x-langfuse-ingestion-version: '4'+ otlphttp/tenant_us:+ endpoint: '${env:LANGFUSE_FANOUT_TENANT_US_BASE_URL}/api/public/otel'+ auth:+ authenticator: headers_setter/tenant_passthrough+ headers:+ x-langfuse-ingestion-version: '4'++service:+ extensions: [headers_setter/tenant_passthrough]+ pipelines:+ traces/central:+ receivers: [otlp]+ processors: [memory_limiter, attributes/drop_librechat_routing, batch/central]+ exporters: [otlphttp/central]+ traces/tenant:+ receivers: [otlp]+ processors: [memory_limiter, filter/tenant_export]+ exporters: [routing/langfuse_tenant_destination]+ traces/tenant_eu:+ receivers: [routing/langfuse_tenant_destination]+ processors: [attributes/drop_librechat_routing, batch/by_auth_eu]+ exporters: [otlphttp/tenant_eu]+ traces/tenant_jp:+ receivers: [routing/langfuse_tenant_destination]+ processors: [attributes/drop_librechat_routing, batch/by_auth_jp]+ exporters: [otlphttp/tenant_jp]+ traces/tenant_us:+ receivers: [routing/langfuse_tenant_destination]+ processors: [attributes/drop_librechat_routing, batch/by_auth_us]+ exporters: [otlphttp/tenant_us]diff --git a/packages/api/src/agents/__tests__/run-summarization.test.ts b/packages/api/src/agents/__tests__/run-summarization.test.tsindex 868010b1e8..e4b3a85dbb 100644--- a/packages/api/src/agents/__tests__/run-summarization.test.ts+++ b/packages/api/src/agents/__tests__/run-summarization.test.ts@@ -203,6 +203,16 @@ function makeAppConfig(customEndpoints: TestCustomEndpoint[]): AppConfig {beforeEach(() => {jest.clearAllMocks();+ delete process.env.LANGFUSE_PUBLIC_KEY;+ delete process.env.LANGFUSE_SECRET_KEY;+ delete process.env.LANGFUSE_BASE_URL;+ delete process.env.LANGFUSE_BASEURL;+ delete process.env.LANGFUSE_HOST;+ delete process.env.LANGFUSE_FANOUT_ENABLED;+ delete process.env.LANGFUSE_FANOUT_COLLECTOR_URL;+ delete process.env.LANGFUSE_FANOUT_TENANT_BASE_URL;+ delete process.env.LANGFUSE_FANOUT_TENANT_DESTINATIONS;+ delete process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED;});// ---------------------------------------------------------------------------@@ -1111,10 +1121,12 @@ async function callAndCaptureRunConfig({overrides,user,tenantId,+ appConfig,}: {overrides?: Record<string, unknown>;user?: Record<string, unknown>;tenantId?: string;+ appConfig?: AppConfig;} = {}): Promise<Record<string, unknown>> {const agents = [makeAgent(overrides)];const signal = new AbortController().signal;@@ -1126,6 +1138,7 @@ async function callAndCaptureRunConfig({streamUsage: true,user: user as never,tenantId,+ appConfig,});const createMock = Run.create as jest.Mock;@@ -1168,6 +1181,543 @@ describe('Langfuse run config', () => {tags: ['tenant:tenant-2'],});});++ it('adds tenant Langfuse credentials from tenant-scoped app config', async () => {+ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ fanout: {+ enabled: true,+ collectorUrl: 'http://langfuse-fanout-collector:4318',+ },+ },+ } as unknown as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'http://langfuse-fanout-collector:4318/tenant/eu',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ librechatTraceAttributes: {+ 'librechat.langfuse.tenant_export.enabled': 'true',+ 'librechat.langfuse.destination': 'eu',+ },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses central env Langfuse config when deployment fanout is not enabled', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ publicKey: 'pk-central',+ secretKey: 'sk-central',+ baseUrl: 'https://central.langfuse.example',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses deployment fanout collector URL without auth when only tenant keys are configured', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'https://cloud.langfuse.com';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('routes tenant fanout traces to the configured destination for the tenant base URL', async () => {+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://us.cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toMatchObject({+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'http://collector-from-env:4318/tenant/us',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ librechatTraceAttributes: {+ 'librechat.langfuse.tenant_export.enabled': 'true',+ 'librechat.langfuse.destination': 'us',+ },+ });+ });++ it('normalizes trailing slashes when building the tenant-scoped fanout URL', async () => {+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318/';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect((callArgs.langfuse as { baseUrl?: string } | undefined)?.baseUrl).toBe(+ 'http://collector-from-env:4318/tenant/eu',+ );+ });++ it.each(['1', 'yes', 'on'])(+ 'routes tenant fanout traces when global fanout is %s',+ async (value) => {+ process.env.LANGFUSE_FANOUT_ENABLED = value;+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://us.cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toMatchObject({+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'http://collector-from-env:4318/tenant/us',+ librechatTraceAttributes: {+ 'librechat.langfuse.tenant_export.enabled': 'true',+ 'librechat.langfuse.destination': 'us',+ },+ });+ },+ );++ it.each(['false', '0', 'no', 'off'])(+ 'uses central env Langfuse config when global fanout is %s',+ async (value) => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = value;+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ publicKey: 'pk-central',+ secretKey: 'sk-central',+ baseUrl: 'https://central.langfuse.example',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ },+ );++ it('does not append a tenant route to baseUrl when fanout is disabled', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'false';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toMatchObject({+ publicKey: 'pk-central',+ secretKey: 'sk-central',+ baseUrl: 'https://central.langfuse.example',+ });+ expect(callArgs.langfuse).not.toMatchObject({+ baseUrl: 'http://collector-from-env:4318/tenant/eu',+ librechatTraceAttributes: expect.any(Object),+ });+ });++ it('uses central env Langfuse config when fanout has no collector URL', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ publicKey: 'pk-central',+ secretKey: 'sk-central',+ baseUrl: 'https://central.langfuse.example',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses deployment fanout collector URL without auth when the tenant base URL is not a configured destination', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';+ process.env.LANGFUSE_FANOUT_TENANT_DESTINATIONS = 'eu=https://cloud.langfuse.com';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://unconfigured-langfuse.example.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses deployment fanout collector URL without auth when tenant Langfuse config has no keys', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {},+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses deployment fanout collector URL without auth when app config is missing under fanout env', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses deployment fanout collector URL without auth when tenant fanout export is disabled', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = 'true';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('does not disable tenant fanout export for a blank emergency toggle', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = ' ';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318/tenant/eu',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ tags: ['tenant:tenant-1'],+ librechatTraceAttributes: {+ 'librechat.langfuse.tenant_export.enabled': 'true',+ 'librechat.langfuse.destination': 'eu',+ },+ });+ });++ it.each(['true', '1', 'yes', 'on'])(+ 'uses deployment fanout collector URL without auth when the emergency toggle is %s',+ async (value) => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = value;++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ },+ );++ it.each(['false', '0', 'no', 'off'])(+ 'routes tenant fanout traces when the emergency toggle is %s',+ async (value) => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = value;++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ baseUrl: 'http://collector-from-env:4318/tenant/eu',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ tags: ['tenant:tenant-1'],+ librechatTraceAttributes: {+ 'librechat.langfuse.tenant_export.enabled': 'true',+ 'librechat.langfuse.destination': 'eu',+ },+ });+ },+ );++ it('uses central env Langfuse config when tenant fanout.enabled=false overrides deployment fanout env', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ fanout: {+ enabled: false,+ },+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ publicKey: 'pk-central',+ secretKey: 'sk-central',+ baseUrl: 'https://central.langfuse.example',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('uses central env Langfuse config when tenant fanout.enabled is the string false', async () => {+ process.env.LANGFUSE_PUBLIC_KEY = 'pk-central';+ process.env.LANGFUSE_SECRET_KEY = 'sk-central';+ process.env.LANGFUSE_BASE_URL = 'https://central.langfuse.example';+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector-from-env:4318';++ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ baseUrl: 'https://cloud.langfuse.com',+ fanout: {+ enabled: 'false',+ },+ },+ } as unknown as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ publicKey: 'pk-central',+ secretKey: 'sk-central',+ baseUrl: 'https://central.langfuse.example',+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('honors tenant Langfuse enabled=false as a tracing opt-out', async () => {+ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ enabled: false,+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ },+ } as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ enabled: false,+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });++ it('honors tenant Langfuse enabled as the string false', async () => {+ const callArgs = await callAndCaptureRunConfig({+ tenantId: 'tenant-1',+ appConfig: {+ langfuse: {+ enabled: 'false',+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ },+ } as unknown as AppConfig,+ });++ expect(callArgs.langfuse).toEqual({+ deterministicTraceId: true,+ enabled: false,+ metadata: { 'librechat.tenant.id': 'tenant-1' },+ tags: ['tenant:tenant-1'],+ });+ });});// ---------------------------------------------------------------------------diff --git a/packages/api/src/agents/openai/service.spec.ts b/packages/api/src/agents/openai/service.spec.tsindex 542ea54b6a..48892e627f 100644--- a/packages/api/src/agents/openai/service.spec.ts+++ b/packages/api/src/agents/openai/service.spec.ts@@ -1,5 +1,5 @@-import { createAgentChatCompletion } from './service';import type { ChatCompletionDependencies } from './service';+import { createAgentChatCompletion } from './service';jest.mock('@librechat/data-schemas', () => ({logger: {@@ -10,7 +10,11 @@ jest.mock('@librechat/data-schemas', () => ({},}));-type CreateRunArgs = { user?: Record<string, unknown> };+type CreateRunArgs = {+ user?: Record<string, unknown>;+ tenantId?: string;+ appConfig?: Record<string, unknown>;+};type ProcessStreamConfig = { configurable?: Record<string, unknown> };function createMockReq(user?: Record<string, unknown>) {@@ -104,4 +108,36 @@ describe('createAgentChatCompletion - MCP permission user propagation', () => {expect(streamConfig.configurable?.user).toEqual({ id: 'api-user' });expect(streamConfig.configurable?.user).not.toHaveProperty('role');});++ it('forwards appConfig and tenantId to createRun', async () => {+ const appConfig = {+ endpoints: {+ agents: { capabilities: ['execute_code'] },+ },+ langfuse: {+ publicKey: 'pk-tenant-1',+ secretKey: 'sk-tenant-1',+ },+ interfaceConfig: {+ modelSelect: true,+ },+ };+ deps.appConfig = appConfig as never;+ const req = createMockReq({+ id: 'user-123',+ tenantId: 'tenant-1',+ role: 'USER',+ });++ await createAgentChatCompletion(req, createMockRes(), deps);++ expect(createRun).toHaveBeenCalledTimes(1);+ const runArgs = createRun.mock.calls[0][0] as CreateRunArgs;+ expect(runArgs.tenantId).toBe('tenant-1');+ expect(runArgs.appConfig).toEqual({+ endpoints: appConfig.endpoints,+ langfuse: appConfig.langfuse,+ });+ expect(runArgs.appConfig).not.toHaveProperty('interfaceConfig');+ });});diff --git a/packages/api/src/agents/openai/service.ts b/packages/api/src/agents/openai/service.tsindex 22b06226dd..65493e8375 100644--- a/packages/api/src/agents/openai/service.ts+++ b/packages/api/src/agents/openai/service.ts@@ -69,15 +69,17 @@ export interface ChatCompletionDependencies {/** Create agent run */createRun?: CreateRunFn;/**- * App config. Optional, but required for agents with `execute_code` in- * their tools: the helper derives `codeEnvAvailable` from+ * App config. Optional for basic chat, but required for tenant-scoped+ * Langfuse fanout and for agents with `execute_code` in their tools:+ * tenant Langfuse keys are forwarded to `createRun`, and the helper derives+ * `codeEnvAvailable` from* `appConfig?.endpoints?.agents?.capabilities` and forwards it into* `deps.initializeAgent`. When `appConfig` is omitted, the resolved* `codeEnvAvailable` is `undefined`, so `initializeAgent` skips the* `execute_code` → `bash_tool` + `read_file` expansion entirely and* code-requesting agents silently lose sandbox tools. Pass `appConfig`* (even a minimal shape with just `endpoints.agents.capabilities`) to- * keep code execution working.+ * keep tenant tracing and code execution working.*/appConfig?: AppConfig;/** Tool execute options for event-driven tool execution */@@ -176,6 +178,7 @@ type CreateRunFn = (params: {requestBody: Record<string, unknown>;user: Record<string, unknown>;tenantId?: string;+ appConfig?: Pick<AppConfig, 'endpoints' | 'langfuse'>;tokenCounter?: (message: unknown) => number;}) => Promise<{Graph?: unknown;@@ -526,6 +529,12 @@ export async function createAgentChatCompletion(},user: safeUser,tenantId: typeof reqUser?.tenantId === 'string' ? reqUser.tenantId : undefined,+ appConfig: deps.appConfig+ ? {+ endpoints: deps.appConfig.endpoints,+ langfuse: deps.appConfig.langfuse,+ }+ : undefined,});if (run) {diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.tsindex a0d0f548e8..e23f8270e9 100644--- a/packages/api/src/agents/run.ts+++ b/packages/api/src/agents/run.ts@@ -39,6 +39,7 @@ import { getProviderConfig } from '~/endpoints/config/providers';import { extractDefaultParams } from '~/endpoints/openai/llm';import { resolveHeaders, createSafeUser } from '~/utils/env';import { getOpenAIConfig } from '~/endpoints/openai/config';+import { buildLangfuseConfig } from '~/langfuse/config';import { resolveConfigHeaders } from '~/utils/headers';import { applyTestRunHook } from '~/agents/testHook';import { isUserProvided } from '~/utils/common';@@ -849,17 +850,6 @@ function buildSubagentConfigs(return configs;}-function buildLangfuseConfig(tenantIdInput?: unknown) {- const tenantId = typeof tenantIdInput === 'string' ? tenantIdInput.trim() : '';- return {- deterministicTraceId: true,- ...(tenantId !== '' && {- metadata: { 'librechat.tenant.id': tenantId },- tags: [`tenant:${tenantId}`],- }),- };-}-/*** Creates a new Run instance with custom handlers and configuration.*@@ -1165,7 +1155,7 @@ export async function createRun({// feedback can be scored against the trace without a lookup (see the// feedback route in api/server/routes/messages.js). No-op unless Langfuse// tracing is enabled. Requires @librechat/agents >= 3.2.21.- langfuse: buildLangfuseConfig(tenantId ?? user?.tenantId),+ langfuse: buildLangfuseConfig({ appConfig, tenantId: tenantId ?? user?.tenantId }),...(enableToolOutputReferences && {toolOutputReferences: { enabled: true },}),diff --git a/packages/api/src/langfuse/config.ts b/packages/api/src/langfuse/config.tsnew file mode 100644index 0000000000..c45a77f0dd--- /dev/null+++ b/packages/api/src/langfuse/config.ts@@ -0,0 +1,135 @@+import type { AppConfig } from '@librechat/data-schemas';+import type { RunConfig } from '@librechat/agents';+import { resolveLangfuseTenantDestination } from './tenantDestinations';+import { isTrueEnv, normalizeBoolean } from './utils';+import { normalizeString } from '~/utils/text';++type LangfuseRunConfig = NonNullable<RunConfig['langfuse']>;+type LangfuseAppConfig = NonNullable<AppConfig['langfuse']>;+export type LangfuseFanoutConfig = LangfuseAppConfig['fanout'] & {+ collectorUrl?: string;+};+type LangfuseRunConfigWithTraceAttributes = LangfuseRunConfig & {+ librechatTraceAttributes?: Record<string, string | number | boolean | null | undefined>;+};+const TENANT_EXPORT_ATTRIBUTE = 'librechat.langfuse.tenant_export.enabled';+const TENANT_DESTINATION_ATTRIBUTE = 'librechat.langfuse.destination';+const DEFAULT_BASE_URL = 'https://cloud.langfuse.com';++function appendPath(baseUrl: string, path: string): string {+ return `${baseUrl.replace(/\/+$/, '')}${path}`;+}++export function isLangfuseTenantExportEnabled(): boolean {+ return !isTrueEnv(process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED);+}++export function isLangfuseFanoutEnabled(fanout?: LangfuseFanoutConfig): boolean {+ const enabled = normalizeBoolean(fanout?.enabled);+ return enabled !== false && (enabled === true || isTrueEnv(process.env.LANGFUSE_FANOUT_ENABLED));+}++function mergeTraceMetadata(+ base: LangfuseRunConfig['metadata'],+ tenantId?: string,+): LangfuseRunConfig['metadata'] | undefined {+ if (!tenantId) {+ return base;+ }+ return {+ ...(base ?? {}),+ 'librechat.tenant.id': tenantId,+ };+}++function mergeTags(tags: string[] | undefined, tenantId?: string): string[] | undefined {+ if (!tenantId) {+ return tags;+ }+ return [...new Set([...(tags ?? []), `tenant:${tenantId}`])];+}++function applyCentralEnvConfig(langfuse: LangfuseRunConfigWithTraceAttributes): void {+ const publicKey = normalizeString(process.env.LANGFUSE_PUBLIC_KEY);+ const secretKey = normalizeString(process.env.LANGFUSE_SECRET_KEY);+ if (publicKey && secretKey) {+ langfuse.publicKey = publicKey;+ langfuse.secretKey = secretKey;+ langfuse.baseUrl =+ normalizeString(process.env.LANGFUSE_BASE_URL) ??+ normalizeString(process.env.LANGFUSE_HOST) ??+ normalizeString(process.env.LANGFUSE_BASEURL) ??+ DEFAULT_BASE_URL;+ }+}++export function buildLangfuseConfig({+ appConfig,+ tenantId,+}: {+ appConfig?: AppConfig;+ tenantId?: string;+} = {}): LangfuseRunConfig {+ const normalizedTenantId = normalizeString(tenantId);+ const config = appConfig?.langfuse;++ const langfuse: LangfuseRunConfigWithTraceAttributes = {+ deterministicTraceId: true,+ };+ const metadata = mergeTraceMetadata(undefined, normalizedTenantId);+ const tags = mergeTags(undefined, normalizedTenantId);+ if (metadata) {+ langfuse.metadata = metadata;+ }+ if (tags) {+ langfuse.tags = tags;+ }++ if (normalizeBoolean(config?.enabled) === false) {+ return {+ ...langfuse,+ enabled: false,+ };+ }++ const publicKey = normalizeString(config?.publicKey);+ const secretKey = normalizeString(config?.secretKey);+ const hasTenantCredentials = Boolean(publicKey && secretKey);+ const fanout = config?.fanout as LangfuseFanoutConfig | undefined;+ const fanoutEnabled = isLangfuseFanoutEnabled(fanout);+ const fanoutCollectorUrl =+ normalizeString(fanout?.collectorUrl) ??+ normalizeString(process.env.LANGFUSE_FANOUT_COLLECTOR_URL);+ const tenantDestination = resolveLangfuseTenantDestination(config?.baseUrl);+ const tenantExportDestination = hasTenantCredentials ? tenantDestination : undefined;+ const tenantExportCollectorUrl = fanoutCollectorUrl;+ const tenantExportEnabled =+ hasTenantCredentials &&+ fanoutEnabled &&+ isLangfuseTenantExportEnabled() &&+ tenantExportDestination != null &&+ tenantExportCollectorUrl != null;++ if (tenantExportEnabled && tenantExportDestination && tenantExportCollectorUrl) {+ langfuse.publicKey = publicKey;+ langfuse.secretKey = secretKey;+ langfuse.baseUrl = appendPath(+ tenantExportCollectorUrl,+ `/tenant/${tenantExportDestination.key}`,+ );+ // TODO: Add support in @librechat/agents for Langfuse additionalHeaders and+ // route by headers if we need multiple tenant Langfuse exports for one run.+ // The destination-scoped URL is the current app-to-gateway routing contract.+ langfuse.librechatTraceAttributes = {+ ...(langfuse.librechatTraceAttributes ?? {}),+ [TENANT_EXPORT_ATTRIBUTE]: 'true',+ [TENANT_DESTINATION_ATTRIBUTE]: tenantExportDestination.key,+ };+ } else if (fanoutEnabled && fanoutCollectorUrl) {+ langfuse.baseUrl = fanoutCollectorUrl;+ } else {+ applyCentralEnvConfig(langfuse);+ }++ return langfuse;+}diff --git a/packages/api/src/langfuse/destinations.ts b/packages/api/src/langfuse/destinations.tsnew file mode 100644index 0000000000..0b0bc4dd45--- /dev/null+++ b/packages/api/src/langfuse/destinations.ts@@ -0,0 +1,118 @@+import type { AppConfig } from '@librechat/data-schemas';+import type { LangfuseFanoutConfig } from './config';+import { isLangfuseFanoutEnabled, isLangfuseTenantExportEnabled } from './config';+import { isFalseEnv, normalizeBoolean, toBasicAuthorization } from './utils';+import { resolveLangfuseTenantDestination } from './tenantDestinations';+import { normalizeString } from '~/utils/text';++const DEFAULT_BASE_URL = 'https://cloud.langfuse.com';++export type LangfuseScoreDestination = {+ name: 'central' | 'tenant';+ baseUrl: string;+ authorization: string;+};++function isSampleRateEnabled(value?: string): boolean {+ if (value == null || value.trim() === '') {+ return true;+ }+ const parsed = Number(value);+ return !Number.isFinite(parsed) || parsed !== 0;+}++function isTracingEnabled(): boolean {+ return (+ !isFalseEnv(process.env.LANGFUSE_TRACING_ENABLED) &&+ isSampleRateEnabled(process.env.LANGFUSE_SAMPLE_RATE)+ );+}++function getCentralEnvBaseUrl(): string {+ return (+ normalizeString(process.env.LANGFUSE_BASE_URL) ??+ normalizeString(process.env.LANGFUSE_HOST) ??+ normalizeString(process.env.LANGFUSE_BASEURL) ??+ DEFAULT_BASE_URL+ );+}++function getCentralScoreDestination(): LangfuseScoreDestination | undefined {+ if (!isTracingEnabled()) {+ return undefined;+ }++ // Central feedback scores are sent directly by the app, not through the+ // collector, so they use LibreChat's normal central Langfuse credentials.+ // LANGFUSE_FANOUT_CENTRAL_AUTH_HEADER is intentionally collector-only.+ const publicKey = normalizeString(process.env.LANGFUSE_PUBLIC_KEY);+ const secretKey = normalizeString(process.env.LANGFUSE_SECRET_KEY);+ if (!publicKey || !secretKey) {+ return undefined;+ }++ return {+ name: 'central',+ baseUrl: getCentralEnvBaseUrl(),+ authorization: toBasicAuthorization(publicKey, secretKey),+ };+}++function getTenantScoreDestination(appConfig?: AppConfig): LangfuseScoreDestination | undefined {+ if (!isTracingEnabled()) {+ return undefined;+ }+ if (!isLangfuseTenantExportEnabled()) {+ return undefined;+ }++ const config = appConfig?.langfuse;+ if (normalizeBoolean(config?.enabled) === false) {+ return undefined;+ }+ const fanout = config?.fanout as LangfuseFanoutConfig | undefined;+ if (!isLangfuseFanoutEnabled(fanout)) {+ return undefined;+ }+ const fanoutCollectorUrl =+ normalizeString(fanout?.collectorUrl) ??+ normalizeString(process.env.LANGFUSE_FANOUT_COLLECTOR_URL);+ if (!fanoutCollectorUrl) {+ return undefined;+ }++ const publicKey = normalizeString(config?.publicKey);+ const secretKey = normalizeString(config?.secretKey);+ if (!publicKey || !secretKey) {+ return undefined;+ }+ const destination = resolveLangfuseTenantDestination(config?.baseUrl);+ if (!destination) {+ return undefined;+ }++ return {+ name: 'tenant',+ baseUrl: destination.baseUrl,+ authorization: toBasicAuthorization(publicKey, secretKey),+ };+}++/**+ * Score fanout uses Langfuse's direct REST API. Trace fanout may use the OTLP+ * collector via appConfig.langfuse.fanout.collectorUrl/LANGFUSE_FANOUT_COLLECTOR_URL.+ */+export function getScoreDestinations(appConfig?: AppConfig): LangfuseScoreDestination[] {+ const destinations = [getCentralScoreDestination(), getTenantScoreDestination(appConfig)].filter(+ (destination): destination is LangfuseScoreDestination => Boolean(destination),+ );+ const seen = new Set<string>();+ return destinations.filter((destination) => {+ const key = `${destination.baseUrl}\n${destination.authorization}`;+ if (seen.has(key)) {+ return false;+ }+ seen.add(key);+ return true;+ });+}diff --git a/packages/api/src/langfuse/feedback.spec.ts b/packages/api/src/langfuse/feedback.spec.tsindex 3de2f727a4..75278d38df 100644--- a/packages/api/src/langfuse/feedback.spec.ts+++ b/packages/api/src/langfuse/feedback.spec.ts@@ -1,8 +1,11 @@+import type { AppConfig } from '@librechat/data-schemas';+jest.mock('@librechat/data-schemas',() => ({logger: {debug: jest.fn(),+ error: jest.fn(),},}),{ virtual: true },@@ -17,6 +20,14 @@ const langfuseEnvKeys = ['LANGFUSE_TRACING_ENABLED','LANGFUSE_SAMPLE_RATE','LANGFUSE_TRACING_ENVIRONMENT',+ 'LANGFUSE_FANOUT_ENABLED',+ 'LANGFUSE_FANOUT_COLLECTOR_URL',+ 'LANGFUSE_FANOUT_TENANT_BASE_URL',+ 'LANGFUSE_FANOUT_TENANT_DESTINATIONS',+ 'LANGFUSE_FANOUT_TENANT_EU_BASE_URL',+ 'LANGFUSE_FANOUT_TENANT_US_BASE_URL',+ 'LANGFUSE_FANOUT_TENANT_JP_BASE_URL',+ 'LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED',];let fetchMock: jest.SpiedFunction<typeof fetch>;@@ -31,6 +42,11 @@ function setLangfuseCredentials() {process.env.LANGFUSE_SECRET_KEY = 'secret-key';}+function enableTenantFanout() {+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector:4318';+}+async function loadFeedback(): Promise<typeof import('./feedback')> {jest.resetModules();return import('./feedback');@@ -40,6 +56,21 @@ function getFetchMock(): jest.SpiedFunction<typeof fetch> {return fetchMock;}+function getTenantAuthorization(+ publicKey = 'tenant-public-key',+ secretKey = 'tenant-secret-key',+): string {+ return `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString('base64')}`;+}++function getCentralAuthorization(): string {+ return getTenantAuthorization('public-key', 'secret-key');+}++function appConfigWithLangfuse(langfuse: AppConfig['langfuse']): AppConfig {+ return { langfuse } as AppConfig;+}+describe('Langfuse feedback scores', () => {beforeEach(() => {clearLangfuseEnv();@@ -120,18 +151,656 @@ describe('Langfuse feedback scores', () => {);});- it('skips scores when Langfuse tracing is disabled', async () => {- process.env.LANGFUSE_TRACING_ENABLED = 'false';+ it('posts feedback scores to central fanout and tenant Langfuse projects', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';const { sendFeedbackScore } = await loadFeedback();await sendFeedbackScore({traceId: 'trace-id',- feedback: { rating: 'thumbsDown' },+ feedback: { rating: 'thumbsDown', tag: 'wrong' },+ metadata: { tenantId: 'tenant-a' },+ appConfig: {+ langfuse: {+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'http://tenant-langfuse:3000',+ },+ } as AppConfig,});- expect(getFetchMock()).not.toHaveBeenCalled();+ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(getFetchMock()).toHaveBeenNthCalledWith(+ 1,+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({+ Authorization: getCentralAuthorization(),+ }),+ }),+ );+ expect(getFetchMock()).toHaveBeenNthCalledWith(+ 2,+ 'http://tenant-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({+ Authorization: getTenantAuthorization(),+ }),+ }),+ );+ const [, tenantInit] = getFetchMock().mock.calls[1];+ expect(JSON.parse(tenantInit?.body as string)).toMatchObject({+ id: 'feedback-trace-id',+ traceId: 'trace-id',+ name: 'user-feedback',+ value: 0,+ metadata: {+ rating: 'thumbsDown',+ tag: 'wrong',+ tenantId: 'tenant-a',+ },+ });+ });++ it('skips tenant feedback scores when tenant keys are configured without a tenant base URL', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('posts tenant feedback scores to the configured destination for the tenant base URL', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://us.cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenNthCalledWith(+ 2,+ 'https://us.cloud.langfuse.com/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({+ Authorization: getTenantAuthorization(),+ }),+ }),+ );+ });++ it('skips tenant feedback scores when the tenant base URL is not a configured destination', async () => {+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_DESTINATIONS = 'eu=https://cloud.langfuse.com';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://unconfigured-langfuse.example.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('deletes feedback scores from central and tenant Langfuse projects', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: null,+ appConfig: {+ langfuse: {+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'http://tenant-langfuse:3000',+ },+ } as AppConfig,+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(getFetchMock()).toHaveBeenNthCalledWith(+ 1,+ 'http://central-langfuse:3000/api/public/scores/feedback-trace-id',+ expect.objectContaining({+ method: 'DELETE',+ headers: { Authorization: getCentralAuthorization() },+ }),+ );+ expect(getFetchMock()).toHaveBeenNthCalledWith(+ 2,+ 'http://tenant-langfuse:3000/api/public/scores/feedback-trace-id',+ expect.objectContaining({+ method: 'DELETE',+ headers: {+ Authorization: getTenantAuthorization(),+ },+ }),+ );+ });++ it('posts feedback scores to tenant Langfuse when no central destination is configured', async () => {+ enableTenantFanout();+ delete process.env.LANGFUSE_PUBLIC_KEY;+ delete process.env.LANGFUSE_SECRET_KEY;+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'http://tenant-langfuse:3000',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://tenant-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getTenantAuthorization() }),+ }),+ );+ });++ it('skips tenant scores when tenant Langfuse is disabled but keeps central scores', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ enabled: false,+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('skips tenant scores when tenant Langfuse enabled is the string false', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ enabled: 'false',+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ } as unknown as AppConfig['langfuse']),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('skips tenant scores when tenant fanout export is disabled but keeps central scores', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = 'true';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );});+ it('does not disable tenant scores for a blank emergency toggle', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = ' ';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'https://cloud.langfuse.com/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getTenantAuthorization() }),+ }),+ );+ });++ it.each(['true', '1', 'yes', 'on'])(+ 'disables tenant scores when the emergency toggle is %s',+ async (value) => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = value;+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ },+ );++ it.each(['false', '0', 'no', 'off'])(+ 'does not disable tenant scores when the emergency toggle is %s',+ async (value) => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_EXPORT_DISABLED = value;+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'https://cloud.langfuse.com/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getTenantAuthorization() }),+ }),+ );+ },+ );++ it('skips tenant scores when global fanout is disabled', async () => {+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('skips tenant scores when tenant fanout is disabled in app config', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ fanout: { enabled: false },+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('skips tenant scores when tenant fanout enabled is the string false', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ fanout: { enabled: 'false' },+ } as unknown as AppConfig['langfuse']),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('skips tenant scores when fanout has no collector URL', async () => {+ process.env.LANGFUSE_FANOUT_ENABLED = 'true';+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ });++ it('deduplicates matching central and tenant score destinations', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_PUBLIC_KEY = 'tenant-public-key';+ process.env.LANGFUSE_SECRET_KEY = 'tenant-secret-key';+ process.env.LANGFUSE_BASE_URL = 'https://cloud.langfuse.com';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'https://cloud.langfuse.com/api/public/scores',+ expect.objectContaining({ method: 'POST' }),+ );+ });++ it('attempts every destination and reports partial feedback score failures', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ fetchMock+ .mockResolvedValueOnce(new Response('central down', { status: 500 }))+ .mockResolvedValueOnce(new Response(null, { status: 200 }));+ const { sendFeedbackScore } = await loadFeedback();+ const { logger } = await import('@librechat/data-schemas');++ await expect(+ sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'http://tenant-langfuse:3000',+ }),+ }),+ ).rejects.toThrow('langfuse central score create failed: score create 500: central down');++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(logger.error).toHaveBeenCalledWith(+ expect.stringContaining('[langfuse] central feedback score send failed'),+ expect.any(Error),+ );+ });++ it('reports tenant feedback score failures after central succeeds', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ fetchMock+ .mockResolvedValueOnce(new Response(null, { status: 200 }))+ .mockResolvedValueOnce(new Response('tenant down', { status: 503 }));+ const { sendFeedbackScore } = await loadFeedback();+ const { logger } = await import('@librechat/data-schemas');++ await expect(+ sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'http://tenant-langfuse:3000',+ }),+ }),+ ).rejects.toThrow('langfuse tenant score create failed: score create 503: tenant down');++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(logger.debug).toHaveBeenCalledWith(+ expect.stringContaining('[langfuse] central feedback score sent'),+ );+ expect(logger.error).toHaveBeenCalledWith(+ expect.stringContaining('[langfuse] tenant feedback score send failed'),+ expect.any(Error),+ );+ });++ it('aggregates feedback score failures when every destination fails', async () => {+ enableTenantFanout();+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ process.env.LANGFUSE_FANOUT_TENANT_BASE_URL = 'http://tenant-langfuse:3000';+ fetchMock+ .mockResolvedValueOnce(new Response('central down', { status: 500 }))+ .mockResolvedValueOnce(new Response('tenant down', { status: 503 }));+ const { sendFeedbackScore } = await loadFeedback();++ await expect(+ sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'http://tenant-langfuse:3000',+ }),+ }),+ ).rejects.toThrow(+ 'langfuse central score create failed: score create 500: central down; langfuse tenant score create failed: score create 503: tenant down',+ );++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ });++ it.each(['false', '0', 'no', 'off'])(+ 'skips scores when Langfuse tracing is disabled with %s',+ async (value) => {+ process.env.LANGFUSE_TRACING_ENABLED = value;+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsDown' },+ });++ expect(getFetchMock()).not.toHaveBeenCalled();+ },+ );++ it.each(['true', '1', 'yes', 'on'])(+ 'enables tenant scores when global fanout is %s',+ async (value) => {+ process.env.LANGFUSE_FANOUT_ENABLED = value;+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector:4318';+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(2);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'https://cloud.langfuse.com/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getTenantAuthorization() }),+ }),+ );+ },+ );++ it.each(['false', '0', 'no', 'off'])(+ 'keeps tenant scores disabled when global fanout is %s',+ async (value) => {+ process.env.LANGFUSE_FANOUT_ENABLED = value;+ process.env.LANGFUSE_FANOUT_COLLECTOR_URL = 'http://collector:4318';+ process.env.LANGFUSE_BASE_URL = 'http://central-langfuse:3000';+ const { sendFeedbackScore } = await loadFeedback();++ await sendFeedbackScore({+ traceId: 'trace-id',+ feedback: { rating: 'thumbsUp' },+ appConfig: appConfigWithLangfuse({+ publicKey: 'tenant-public-key',+ secretKey: 'tenant-secret-key',+ baseUrl: 'https://cloud.langfuse.com',+ }),+ });++ expect(getFetchMock()).toHaveBeenCalledTimes(1);+ expect(getFetchMock()).toHaveBeenCalledWith(+ 'http://central-langfuse:3000/api/public/scores',+ expect.objectContaining({+ method: 'POST',+ headers: expect.objectContaining({ Authorization: getCentralAuthorization() }),+ }),+ );+ },+ );+it('skips scores when Langfuse sampling is set to zero', async () => {process.env.LANGFUSE_SAMPLE_RATE = '0';const { sendFeedbackScore } = await loadFeedback();diff --git a/packages/api/src/langfuse/feedback.ts b/packages/api/src/langfuse/feedback.tsindex 14d80df048..0e58ade588 100644--- a/packages/api/src/langfuse/feedback.ts+++ b/packages/api/src/langfuse/feedback.ts@@ -1,4 +1,6 @@import { logger } from '@librechat/data-schemas';+import type { AppConfig } from '@librechat/data-schemas';+import { getScoreDestinations, type LangfuseScoreDestination } from './destinations';export type LangfuseFeedback = {rating?: 'thumbsUp' | 'thumbsDown';@@ -13,39 +15,23 @@ export type SendFeedbackScoreParams = {feedback?: LangfuseFeedback | null;metadata?: LangfuseFeedbackMetadata;observationId?: string;+ appConfig?: AppConfig;};-const DEFAULT_BASE_URL = 'https://cloud.langfuse.com';-const BASE =- process.env.LANGFUSE_BASE_URL ??- process.env.LANGFUSE_HOST ??- process.env.LANGFUSE_BASEURL ??- DEFAULT_BASE_URL;--function isFalseEnv(value?: string): boolean {- return value != null && ['0', 'false', 'no', 'off'].includes(value.trim().toLowerCase());-}--function isSampleRateEnabled(value?: string): boolean {- if (value == null || value.trim() === '') {- return true;- }- const parsed = Number(value);- return !Number.isFinite(parsed) || parsed !== 0;-}--const ENABLED =- Boolean(process.env.LANGFUSE_PUBLIC_KEY && process.env.LANGFUSE_SECRET_KEY) &&- !isFalseEnv(process.env.LANGFUSE_TRACING_ENABLED) &&- isSampleRateEnabled(process.env.LANGFUSE_SAMPLE_RATE);-const AUTHORIZATION = ENABLED- ? 'Basic ' +- Buffer.from(`${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`).toString(- 'base64',- )- : undefined;const ENVIRONMENT = process.env.LANGFUSE_TRACING_ENVIRONMENT;+type LangfuseScorePayload = {+ id: string;+ traceId: string;+ name: 'user-feedback';+ value: number;+ dataType: 'BOOLEAN';+ comment?: string;+ metadata: Record<string, string | number | boolean>;+ observationId?: string;+ environment?: string;+};+function cleanMetadata(metadata: LangfuseFeedbackMetadata,): Record<string, string | number | boolean> {@@ -61,30 +47,47 @@ function cleanMetadata();}-export async function sendFeedbackScore({- traceId,- feedback,- metadata = {},- observationId,-}: SendFeedbackScoreParams): Promise<void> {- if (!ENABLED || !AUTHORIZATION || !traceId) {- return;+async function deleteScore(destination: LangfuseScoreDestination, scoreId: string): Promise<void> {+ const res = await fetch(+ `${destination.baseUrl}/api/public/scores/${encodeURIComponent(scoreId)}`,+ {+ method: 'DELETE',+ headers: { Authorization: destination.authorization },+ },+ );+ if (!res.ok && res.status !== 404) {+ throw new Error(`score delete ${res.status}: ${await res.text()}`);}+}- const scoreId = `feedback-${traceId}`;-- if (!feedback?.rating) {- const res = await fetch(`${BASE}/api/public/scores/${encodeURIComponent(scoreId)}`, {- method: 'DELETE',- headers: { Authorization: AUTHORIZATION },- });- if (!res.ok && res.status !== 404) {- throw new Error(`langfuse score delete ${res.status}: ${await res.text()}`);- }- return;+async function createScore(+ destination: LangfuseScoreDestination,+ payload: LangfuseScorePayload,+): Promise<void> {+ const res = await fetch(`${destination.baseUrl}/api/public/scores`, {+ method: 'POST',+ headers: { Authorization: destination.authorization, 'Content-Type': 'application/json' },+ body: JSON.stringify(payload),+ });+ if (!res.ok) {+ throw new Error(`score create ${res.status}: ${await res.text()}`);}+}- const body = {+function buildScorePayload({+ scoreId,+ traceId,+ feedback,+ metadata,+ observationId,+}: {+ scoreId: string;+ traceId: string;+ feedback: LangfuseFeedback;+ metadata: LangfuseFeedbackMetadata;+ observationId?: string;+}): LangfuseScorePayload {+ return {id: scoreId,traceId,name: 'user-feedback',@@ -95,14 +98,64 @@ export async function sendFeedbackScore({...(observationId ? { observationId } : {}),...(ENVIRONMENT ? { environment: ENVIRONMENT } : {}),};+}- const res = await fetch(`${BASE}/api/public/scores`, {- method: 'POST',- headers: { Authorization: AUTHORIZATION, 'Content-Type': 'application/json' },- body: JSON.stringify(body),+export async function sendFeedbackScore({+ traceId,+ feedback,+ metadata = {},+ observationId,+ appConfig,+}: SendFeedbackScoreParams): Promise<void> {+ if (!traceId) {+ return;+ }++ const destinations = getScoreDestinations(appConfig);+ if (destinations.length === 0) {+ return;+ }++ const scoreId = `feedback-${traceId}`;+ const payload = feedback?.rating+ ? buildScorePayload({ scoreId, traceId, feedback, metadata, observationId })+ : undefined;++ const results = await Promise.allSettled(+ destinations.map((destination) =>+ payload ? createScore(destination, payload) : deleteScore(destination, scoreId),+ ),+ );+ const failures: string[] = [];++ results.forEach((result, index) => {+ const destination = destinations[index];+ if (!destination) {+ return;+ }+ if (result.status === 'fulfilled') {+ logger.debug(+ `[langfuse] ${destination.name} feedback score ${+ payload ? 'sent' : 'deleted'+ } for trace ${traceId} (${feedback?.rating ?? 'none'})`,+ );+ return;+ }++ logger.error(+ `[langfuse] ${destination.name} feedback score ${+ payload ? 'send' : 'delete'+ } failed for trace ${traceId}:`,+ result.reason,+ );+ failures.push(+ `langfuse ${destination.name} score ${payload ? 'create' : 'delete'} failed: ${+ result.reason instanceof Error ? result.reason.message : String(result.reason)+ }`,+ );});- if (!res.ok) {- throw new Error(`langfuse score create ${res.status}: ${await res.text()}`);++ if (failures.length > 0) {+ throw new Error(failures.join('; '));}- logger.debug(`[langfuse] feedback score sent for trace ${traceId} (${feedback.rating})`);}diff --git a/packages/api/src/langfuse/tenantDestinations.ts b/packages/api/src/langfuse/tenantDestinations.tsnew file mode 100644index 0000000000..85a733aede--- /dev/null+++ b/packages/api/src/langfuse/tenantDestinations.ts@@ -0,0 +1,111 @@+import { normalizeString } from '~/utils/text';++const DEFAULT_TENANT_DESTINATIONS: Array<[string, string]> = [+ ['eu', 'https://cloud.langfuse.com'],+ ['us', 'https://us.cloud.langfuse.com'],+ ['jp', 'https://jp.cloud.langfuse.com'],+];++const DESTINATIONS_ENV = 'LANGFUSE_FANOUT_TENANT_DESTINATIONS';+const LEGACY_TENANT_BASE_URL_ENV = 'LANGFUSE_FANOUT_TENANT_BASE_URL';++export type LangfuseTenantDestination = {+ key: string;+ baseUrl: string;+};++function normalizeDestinationKey(value: string): string | undefined {+ const normalized = value+ .trim()+ .toLowerCase()+ .replace(/[^a-z0-9_-]+/g, '_');+ return /^[a-z][a-z0-9_-]*$/.test(normalized) ? normalized : undefined;+}++function normalizeBaseUrl(value: unknown): string | undefined {+ const normalized = normalizeString(value);+ if (!normalized) {+ return undefined;+ }++ try {+ const url = new URL(normalized);+ if (url.protocol !== 'https:' && url.protocol !== 'http:') {+ return undefined;+ }+ url.pathname = url.pathname.replace(/\/+$/, '');+ url.search = '';+ url.hash = '';+ return url.toString().replace(/\/+$/, '');+ } catch {+ return undefined;+ }+}++function destinationEnvName(key: string): string {+ return `LANGFUSE_FANOUT_TENANT_${key.toUpperCase().replace(/[^A-Z0-9]+/g, '_')}_BASE_URL`;+}++function parseDestinationList(value: string | undefined): LangfuseTenantDestination[] {+ if (!value) {+ return [];+ }++ return value+ .split(',')+ .map((item) => item.trim())+ .filter(Boolean)+ .map((item) => {+ const index = item.indexOf('=');+ if (index < 0) {+ return undefined;+ }+ const key = normalizeDestinationKey(item.slice(0, index));+ const baseUrl = normalizeBaseUrl(item.slice(index + 1));+ return key && baseUrl ? { key, baseUrl } : undefined;+ })+ .filter((destination): destination is LangfuseTenantDestination => Boolean(destination));+}++function uniqueDestinations(+ destinations: LangfuseTenantDestination[],+): LangfuseTenantDestination[] {+ const byKey = new Map<string, LangfuseTenantDestination>();+ for (const destination of destinations) {+ byKey.set(destination.key, destination);+ }+ return [...byKey.values()];+}++export function getLangfuseTenantDestinations(): LangfuseTenantDestination[] {+ const configuredValue = normalizeString(process.env[DESTINATIONS_ENV]);+ const configured = parseDestinationList(configuredValue);+ if (configuredValue) {+ return uniqueDestinations(configured);+ }++ const legacyBaseUrl = normalizeBaseUrl(process.env[LEGACY_TENANT_BASE_URL_ENV]);+ const defaults = DEFAULT_TENANT_DESTINATIONS.map(([key, defaultBaseUrl]) => ({+ key,+ baseUrl:+ normalizeBaseUrl(process.env[destinationEnvName(key)]) ??+ (key === 'eu' ? legacyBaseUrl : undefined) ??+ defaultBaseUrl,+ }));++ return uniqueDestinations(defaults);+}++export function resolveLangfuseTenantDestination(+ baseUrl: unknown,+): LangfuseTenantDestination | undefined {+ const normalizedBaseUrl = normalizeBaseUrl(baseUrl);++ if (!normalizedBaseUrl) {+ return undefined;+ }++ return getLangfuseTenantDestinations().find(+ (destination) => destination.baseUrl === normalizedBaseUrl,+ );+}diff --git a/packages/api/src/langfuse/utils.ts b/packages/api/src/langfuse/utils.tsnew file mode 100644index 0000000000..064b1e9d80--- /dev/null+++ b/packages/api/src/langfuse/utils.ts@@ -0,0 +1,32 @@+export function toBasicAuthorization(publicKey: string, secretKey: string): string {+ return `Basic ${Buffer.from(`${publicKey}:${secretKey}`).toString('base64')}`;+}++const TRUE_ENV_VALUES = new Set(['1', 'true', 'yes', 'on']);+const FALSE_ENV_VALUES = new Set(['0', 'false', 'no', 'off']);++export function normalizeBoolean(value: unknown): boolean | undefined {+ if (typeof value === 'boolean') {+ return value;+ }+ if (typeof value !== 'string') {+ return undefined;+ }++ const normalized = value.trim().toLowerCase();+ if (TRUE_ENV_VALUES.has(normalized)) {+ return true;+ }+ if (FALSE_ENV_VALUES.has(normalized)) {+ return false;+ }+ return undefined;+}++export function isTrueEnv(value: unknown): boolean {+ return normalizeBoolean(value) === true;+}++export function isFalseEnv(value: unknown): boolean {+ return normalizeBoolean(value) === false;+}diff --git a/packages/api/src/utils/text.spec.ts b/packages/api/src/utils/text.spec.tsindex cbafb25af7..5a0c21a018 100644--- a/packages/api/src/utils/text.spec.ts+++ b/packages/api/src/utils/text.spec.ts@@ -1,4 +1,4 @@-import { processTextWithTokenLimit, TokenCountFn } from './text';+import { normalizeString, processTextWithTokenLimit, TokenCountFn } from './text';import Tokenizer, { countTokens } from './tokenizer';jest.mock('@librechat/data-schemas', () => ({@@ -9,6 +9,15 @@ jest.mock('@librechat/data-schemas', () => ({},}));+describe('normalizeString', () => {+ it('trims non-empty strings and treats blank or non-string values as undefined', () => {+ expect(normalizeString(' value ')).toBe('value');+ expect(normalizeString(' ')).toBeUndefined();+ expect(normalizeString(null)).toBeUndefined();+ expect(normalizeString(123)).toBeUndefined();+ });+});+/*** OLD IMPLEMENTATION (Binary Search) - kept for comparison testing* This is the original algorithm that caused CPU spikesdiff --git a/packages/api/src/utils/text.ts b/packages/api/src/utils/text.tsindex 5273670e10..00e9e81d86 100644--- a/packages/api/src/utils/text.ts+++ b/packages/api/src/utils/text.ts@@ -3,6 +3,10 @@ import { logger } from '@librechat/data-schemas';/** Token count function that can be sync or async */export type TokenCountFn = (text: string) => number | Promise<number>;+export function normalizeString(value: unknown): string | undefined {+ return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;+}+/*** Safety buffer multiplier applied to character position estimates during truncation.*diff --git a/packages/data-provider/specs/config-schemas.spec.ts b/packages/data-provider/specs/config-schemas.spec.tsindex 236295483b..96915b604d 100644--- a/packages/data-provider/specs/config-schemas.spec.ts+++ b/packages/data-provider/specs/config-schemas.spec.ts@@ -1178,3 +1178,32 @@ describe('specsConfigSchema', () => {expect(result.success).toBe(false);});});++describe('configSchema langfuse', () => {+ it('accepts tenant Langfuse fanout config', () => {+ const result = configSchema.safeParse({+ version: '1.3.7',+ langfuse: {+ publicKey: 'pk-lf-tenant',+ secretKey: 'sk-lf-tenant',+ fanout: {+ enabled: true,+ collectorUrl: 'http://langfuse-fanout-collector:4318',+ },+ },+ });++ expect(result.success).toBe(true);+ });++ it('accepts an explicit tenant Langfuse opt-out', () => {+ const result = configSchema.safeParse({+ version: '1.3.7',+ langfuse: {+ enabled: false,+ },+ });++ expect(result.success).toBe(true);+ });+});diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.tsindex 0c54da1658..01db2feeed 100644--- a/packages/data-provider/src/config.ts+++ b/packages/data-provider/src/config.ts@@ -1754,11 +1754,27 @@ export const messageFilterSchema = z.object({export type MessageFilterConfig = z.infer<typeof messageFilterSchema>;+export const langfuseConfigSchema = z.object({+ enabled: z.boolean().optional(),+ publicKey: z.string().optional(),+ secretKey: z.string().optional(),+ baseUrl: z.string().optional(),+ fanout: z+ .object({+ enabled: z.boolean().optional(),+ collectorUrl: z.string().optional(),+ })+ .optional(),+});++export type LangfuseConfig = z.infer<typeof langfuseConfigSchema>;+export const configSchema = z.object({version: z.string(),cache: z.boolean().default(true),ocr: ocrSchema.optional(),webSearch: webSearchSchema.optional(),+ langfuse: langfuseConfigSchema.optional(),memory: memorySchema.optional(),summarization: summarizationConfigSchema.optional(),skillSync: skillSyncConfigSchema,diff --git a/packages/data-schemas/src/app/service.ts b/packages/data-schemas/src/app/service.tsindex 6a2b4ac16e..c999c842b7 100644--- a/packages/data-schemas/src/app/service.ts+++ b/packages/data-schemas/src/app/service.ts@@ -1,7 +1,8 @@import {+ AgentCapabilities,EModelEndpoint,getConfigDefaults,- AgentCapabilities,+ langfuseConfigSchema,skillSyncConfigSchema,summarizationConfigSchema,} from 'librechat-data-provider';@@ -73,6 +74,21 @@ export function loadSkillSyncConfig(config: DeepPartial<TCustomConfig>): AppConfreturn parsed.data;}+export function loadLangfuseConfig(config: DeepPartial<TCustomConfig>): AppConfig['langfuse'] {+ const raw = config.langfuse;+ if (!raw || typeof raw !== 'object') {+ return undefined;+ }++ const parsed = langfuseConfigSchema.safeParse(raw);+ if (!parsed.success) {+ logger.warn('[AppService] Invalid Langfuse config', parsed.error.flatten());+ return undefined;+ }++ return parsed.data;+}+export type Paths = {root: string;uploads: string;@@ -134,6 +150,7 @@ export const AppService = async (params?: {const turnstileConfig = loadTurnstileConfig(config, configDefaults);const speech = config.speech;const messageFilter = config.messageFilter;+ const langfuse = loadLangfuseConfig(config);const defaultConfig = {ocr,@@ -151,6 +168,7 @@ export const AppService = async (params?: {transactions,filteredTools,includedTools,+ langfuse,messageFilter,summarization,availableTools,diff --git a/packages/data-schemas/src/types/app.ts b/packages/data-schemas/src/types/app.tsindex 0eb0f4d60b..4940261f5f 100644--- a/packages/data-schemas/src/types/app.ts+++ b/packages/data-schemas/src/types/app.ts@@ -65,6 +65,8 @@ export interface AppConfig {webSearch?: TCustomConfig['webSearch'];/** Message filter configuration (PII and future filter types) */messageFilter?: TCustomConfig['messageFilter'];+ /** Langfuse tracing configuration */+ langfuse?: TCustomConfig['langfuse'];/** Skill sync configuration */skillSync?: SkillSyncConfig;/** File storage strategy ('local', 's3', 'firebase', 'azure_blob', 'cloudfront') */