Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: Add Langfuse tracing (#2510)

Mike Fortmanbfb98b0b666d
diff --git a/.env.example b/.env.example
index 8b22fee8d..1d05f965f 100644
--- a/.env.example
+++ b/.env.example
@@ -431,11 +431,27 @@ LANGFLOW_COMPONENTS_INDEX_PATH=/app/flows/component_index.json
# Langfuse tracing (optional)
# Get keys from https://cloud.langfuse.com or your self-hosted instance
+# Exception to the note above: the backend also reads these when
+# OPENRAG_TRACING_ENABLED=true (below), so both trace to the same project.
LANGFUSE_SECRET_KEY=
LANGFUSE_PUBLIC_KEY=
-# Leave empty for Langfuse Cloud, or set for self-hosted (e.g., http://localhost:3002)
-# Example of self hosting outside of the docker network: LANGFUSE_HOST=http://host.docker.internal:3000
-LANGFUSE_HOST=
+# Langfuse URL, as shown in Langfuse's setup steps: https://cloud.langfuse.com (EU),
+# https://us.cloud.langfuse.com (US), or your self-hosted instance (e.g., http://localhost:3002).
+# Self hosting outside of the docker network: LANGFUSE_BASE_URL=http://host.docker.internal:3000
+# Langflow needs it set to trace. LANGFUSE_HOST is still accepted as an older name.
+LANGFUSE_BASE_URL=
+
+# Backend tracing to Langfuse (optional, default off). Traces every model call
+# the backend makes through its LLM gateway — embeddings and chat, all
+# providers — with model, provider, latency, token usage, cost and errors.
+# Requires the LANGFUSE_* keys above and the `tracing` extra (`uv sync --extra
+# tracing`; included in the published backend image).
+OPENRAG_TRACING_ENABLED=false
+# Traces carry metadata only by default. Set to true to also record prompts,
+# completions and embedding inputs (document text).
+OPENRAG_TRACING_CAPTURE_CONTENT=false
+# In OPENRAG_RUN_MODE=saas, content capture additionally requires this.
+OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS=false
# OPENRAG_LLM_TOKEN is seeded to the non-empty string "None" in Langflow
# (docker-compose OPENRAG_LLM_TOKEN=None). Langflow treats an empty Credential
diff --git a/Dockerfile.backend b/Dockerfile.backend
index d1ac25f75..98180d1aa 100644
--- a/Dockerfile.backend
+++ b/Dockerfile.backend
@@ -48,11 +48,12 @@ WORKDIR /app
# Dependency layer: reuse as long as lockfile and pyproject don't change
# --extra apm bakes in the Instana tracer so the published image can turn APM on
-# with INSTANA_ENABLED alone. The extra exists so source installs that will never
-# enable it don't pay for the tracer; images ship with the capability present.
+# with INSTANA_ENABLED alone, and --extra tracing does the same for Langfuse
+# tracing (OPENRAG_TRACING_ENABLED). The extras exist so source installs that will never
+# enable them don't pay for the tracers; images ship with the capability present.
COPY pyproject.toml uv.lock ./
RUN --mount=type=cache,target=/root/.cache/uv,uid=0,gid=0 \
- uv sync --extra apm
+ uv sync --extra apm --extra tracing
# Application (invalidates after deps only when src/flows change)
COPY src/ ./src/
diff --git a/docker-compose.yml b/docker-compose.yml
index 5eed863ba..c3a7692ff 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -142,6 +142,16 @@ services:
- DOCLING_SERVE_URL=${DOCLING_SERVE_URL:-http://host.docker.internal:5001}
- DOCLING_SERVE_VERIFY_SSL=${DOCLING_SERVE_VERIFY_SSL:-true}
- CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS-http://localhost:3000}
+ # Backend LLM tracing to Langfuse (opt-in, default off). Shares the
+ # LANGFUSE_* keys with the langflow service below so both land in one
+ # project. Metadata only unless content capture is opted into.
+ - OPENRAG_TRACING_ENABLED=${OPENRAG_TRACING_ENABLED:-false}
+ - OPENRAG_TRACING_CAPTURE_CONTENT=${OPENRAG_TRACING_CAPTURE_CONTENT:-false}
+ - OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS=${OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS:-false}
+ - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-}
+ - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-}
+ - LANGFUSE_BASE_URL=${LANGFUSE_BASE_URL:-}
+ - LANGFUSE_HOST=${LANGFUSE_HOST:-}
# Instana APM (opt-in, default off — see the INSTANA_* block in .env.example).
# The agent runs on the HOST (`make instana-agent-up`, host networking), so a
# containerized backend reaches it via the host.docker.internal alias mapped
@@ -230,6 +240,7 @@ services:
- LANGFLOW_COMPONENTS_INDEX_PATH=/app/flows/component_index.json
- LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY:-}
- LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-}
+ - LANGFUSE_BASE_URL=${LANGFUSE_BASE_URL:-}
- LANGFUSE_HOST=${LANGFUSE_HOST:-}
- LANGFLOW_DEACTIVATE_TRACING
# Hop token is injected per Langflow run. Seed a non-empty placeholder
diff --git a/docs/docs/get-started/docker.mdx b/docs/docs/get-started/docker.mdx
index 4bfcd730a..85e797199 100644
--- a/docs/docs/get-started/docker.mdx
+++ b/docs/docs/get-started/docker.mdx
@@ -118,7 +118,7 @@ The following variables are required or recommended:
* **`LANGFUSE_SECRET_KEY`**: A secret key for your Langfuse project.
* **`LANGFUSE_PUBLIC_KEY`**: A public key for your Langfuse project.
- * **`LANGFUSE_HOST`**: Required for self-hosted Langfuse deployments. Leave empty for Langfuse Cloud.
+ * **`LANGFUSE_BASE_URL`**: The URL of your Langfuse instance, as shown in your Langfuse project's setup steps. For example, `https://cloud.langfuse.com` (EU), `https://us.cloud.langfuse.com` (US), or your self-hosted address. `LANGFUSE_HOST` is accepted as an older name.
8. Save your `.env` file.
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index cb9dabe65..4d60344f9 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -284,9 +284,9 @@ For better security, it is recommended to set `LANGFLOW_SUPERUSER_PASSWORD` so t
| `LANGFLOW_SUPERUSER` | `admin` | Username for the Langflow administrator user. |
| `LANGFLOW_SUPERUSER_PASSWORD` | Not set | Langflow administrator password. If this variable isn't set, then the Langflow server starts _without_ authentication enabled. It is recommended to set `LANGFLOW_SUPERUSER_PASSWORD` so the [Langflow server starts with authentication enabled](https://docs.langflow.org/api-keys-and-authentication#start-a-langflow-server-with-authentication-enabled). |
| `LANGFLOW_CHAT_FLOW_ID`, `LANGFLOW_INGEST_FLOW_ID`, `NUDGES_FLOW_ID`, `LANGFLOW_URL_INGEST_FLOW_ID` | Built-in flow IDs | These variables are set automatically to the IDs of the chat, Docling ingestion, URL ingestion, and nudges [flows](/agents). The default values are found in [`.env.example`](https://github.com/langflow-ai/openrag/blob/main/.env.example). Only change these values if you want to replace a built-in flow with your own custom flow. The flow JSON must be present in your version of the OpenRAG codebase. For example, if you [deploy self-managed services](/docker), you can add the flow JSON to your local clone of the OpenRAG repository before deploying OpenRAG. |
-| `LANGFUSE_SECRET_KEY` | Not set | Optional Langfuse secret key to enable the [Langflow integration with Langfuse](https://docs.langflow.org/integrations-langfuse). |
-| `LANGFUSE_PUBLIC_KEY` | Not set | Optional Langfuse public key to enable the [Langflow integration with Langfuse](https://docs.langflow.org/integrations-langfuse). |
-| `LANGFUSE_HOST` | Not set | Leave empty for Langfuse Cloud. Required for self-hosted Langfuse deployments if `LANGFUSE_SECRET_KEY` and `LANGFUSE_PUBLIC_KEY` are set. The address must be relative to the OpenRAG container deployment. For example, `http://localhost:3002` or `http://host.docker.internal:3000`. |
+| `LANGFUSE_SECRET_KEY` | Not set | Optional Langfuse secret key to enable the [Langflow integration with Langfuse](https://docs.langflow.org/integrations-langfuse). Also used by [backend tracing](#backend-langfuse-tracing) when it's enabled. |
+| `LANGFUSE_PUBLIC_KEY` | Not set | Optional Langfuse public key to enable the [Langflow integration with Langfuse](https://docs.langflow.org/integrations-langfuse). Also used by [backend tracing](#backend-langfuse-tracing) when it's enabled. |
+| `LANGFUSE_BASE_URL` | Not set | The URL of your Langfuse instance, as shown in your Langfuse project's setup steps. For example, `https://cloud.langfuse.com` (EU), `https://us.cloud.langfuse.com` (US), or a self-hosted address. Self-hosted addresses must be relative to the OpenRAG container deployment, such as `http://localhost:3002` or `http://host.docker.internal:3000`. Langflow doesn't trace unless this is set. `LANGFUSE_HOST` is accepted as an older name for this variable. |
| `LANGFLOW_TIMEOUT` | `2400` | Total Langflow HTTP timeout limit in seconds. Increase this value if you experience timeouts when ingesting very large PDFs. See also [`INGESTION_TIMEOUT`](#document-processing-settings).|
| `LANGFLOW_CONNECT_TIMEOUT` | `30` | Langflow HTTP connection timeout limit in seconds. Increase this value if you experience timeouts when ingesting very large PDFs. |
| `LANGFLOW_KEY` | Automatically generated | A Langflow API key to run flows with Langflow API calls. Because Langflow API keys are server-specific, allow OpenRAG to generate this key initially. You can create additional Langflow API keys after deploying OpenRAG. |
@@ -416,3 +416,30 @@ These variables don't affect the backend, and they aren't used by Kubernetes dep
`make instana-agent-up` starts a privileged container in the host PID namespace with access to the Docker socket, which is effectively root access to the host.
It's intended for local development machines only. Don't run it on a shared or production host.
:::
+
+### Backend tracing with Langfuse {#backend-langfuse-tracing}
+
+OpenRAG Backend can send a trace to [Langfuse](https://langfuse.com) for every model call it makes, both embeddings and chat completions, for every provider.
+This includes calls that Langflow flows make through the backend's model gateway.
+Each trace records the model, provider, latency, token usage, cost where the provider reports it, and the error for failed calls.
+
+Backend tracing is off by default. It uses the same `LANGFUSE_*` keys as the [Langflow integration with Langfuse](#langflow-settings), so backend and Langflow traces land in the same Langfuse project.
+
+To enable it, set the Langfuse keys and `OPENRAG_TRACING_ENABLED=true` in your `.env` file, then [restart the OpenRAG services](/manage-services):
+
+```bash
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+LANGFUSE_BASE_URL=https://cloud.langfuse.com # or https://us.cloud.langfuse.com, or your self-hosted URL
+OPENRAG_TRACING_ENABLED=true
+```
+
+In Kubernetes, set `backend.tracing.enabled: true` in the Helm chart, which reuses the keys under `langflow.langfuse`. With the operator, set the same variables in the backend's `spec.env`.
+
+By default, traces carry metadata only. Prompts, completions, and the document text sent for embedding are replaced with `redacted-by-litellm` before the trace leaves the backend.
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `OPENRAG_TRACING_ENABLED` | `false` | Whether OpenRAG Backend sends traces to Langfuse. Requires `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY`. If either key is missing, the backend logs a warning and runs untraced. The tracing dependencies are an optional extra: the backend container image includes them, and a backend run from source needs `uv sync --extra tracing`. Read at startup, so changes require a backend restart. |
+| `OPENRAG_TRACING_CAPTURE_CONTENT` | `false` | Whether traces include prompts, completions, and embedding inputs. Embedding inputs are your document text, so enable this only when that content may leave your deployment. |
+| `OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS` | `false` | In `OPENRAG_RUN_MODE=saas`, content capture also requires this variable. Has no effect in other run modes. |
diff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
index 3f77c641b..a11d9596d 100644
--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
@@ -251,6 +251,23 @@ stringData:
{{- end }}
SESSION_SECRET={{ $sessionSecret | quote }}
+ # Backend LLM tracing to Langfuse. Reuses the Langflow Langfuse keys so
+ # backend and Langflow traces land in the same project.
+ OPENRAG_TRACING_ENABLED={{ ternary "true" "false" .Values.backend.tracing.enabled }}
+ {{- if .Values.backend.tracing.enabled }}
+ OPENRAG_TRACING_CAPTURE_CONTENT={{ ternary "true" "false" .Values.backend.tracing.captureContent }}
+ OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS={{ ternary "true" "false" .Values.backend.tracing.allowContentInSaas }}
+ {{- if .Values.langflow.langfuse.secretKey }}
+ LANGFUSE_SECRET_KEY={{ .Values.langflow.langfuse.secretKey | quote }}
+ {{- end }}
+ {{- if .Values.langflow.langfuse.publicKey }}
+ LANGFUSE_PUBLIC_KEY={{ .Values.langflow.langfuse.publicKey | quote }}
+ {{- end }}
+ {{- if .Values.langflow.langfuse.host }}
+ LANGFUSE_BASE_URL={{ .Values.langflow.langfuse.host | quote }}
+ {{- end }}
+ {{- end }}
+
# Instana APM
# INSTANA_AGENT_HOST is deliberately absent: it is the pod's own node IP,
# which this file cannot know at render time. The backend Deployment injects
diff --git a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
index 5673808b1..6f8fde768 100644
--- a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
@@ -142,7 +142,7 @@ stringData:
LANGFUSE_PUBLIC_KEY={{ .Values.langflow.langfuse.publicKey | quote }}
{{- end }}
{{- if .Values.langflow.langfuse.host }}
- LANGFUSE_HOST={{ .Values.langflow.langfuse.host | quote }}
+ LANGFUSE_BASE_URL={{ .Values.langflow.langfuse.host | quote }}
{{- end }}
# Variables to expose to flows (NOTE: must use valid env var names; no hyphens)
diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yaml
index f5dc12a3f..b36e53095 100644
--- a/kubernetes/helm/openrag/values.yaml
+++ b/kubernetes/helm/openrag/values.yaml
@@ -382,6 +382,15 @@ backend:
enabled: false
port: 8100
+ # Backend LLM tracing to Langfuse (opt-in, default off). Uses the keys and
+ # host under langflow.langfuse. Traces carry metadata only (model, provider,
+ # latency, tokens, errors) unless captureContent is set; in SaaS run mode
+ # allowContentInSaas must also be set.
+ tracing:
+ enabled: false
+ captureContent: false
+ allowContentInSaas: false
+
# Instana APM (opt-in, default off).
#
# Requires an Instana agent already running in the cluster — install it
diff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.go
index 2893a04c1..4e36dd3ab 100644
--- a/kubernetes/operator/internal/controller/env.go
+++ b/kubernetes/operator/internal/controller/env.go
@@ -199,6 +199,15 @@ func NewEnvVarManager() *EnvVarManager {
"INSTANA_TRACING_DISABLE": "logging",
"INSTANA_STACK_TRACE": "error",
"INSTANA_SECRETS": `regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z`,
+
+ // Backend tracing to Langfuse (opt-in, default off). Enabling it
+ // also needs LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY (and
+ // LANGFUSE_HOST for self-hosted) on the backend via spec.env.
+ // Traces carry metadata only unless content capture is opted into,
+ // which in SaaS run mode takes the second flag as well.
+ "OPENRAG_TRACING_ENABLED": "false",
+ "OPENRAG_TRACING_CAPTURE_CONTENT": "false",
+ "OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS": "false",
},
DefaultOpenRagFEEnvVars: map[string]string{
// Frontend environment variables will be added here
diff --git a/kubernetes/operator/internal/controller/tracing_test.go b/kubernetes/operator/internal/controller/tracing_test.go
new file mode 100644
index 000000000..daad7cb9e
--- /dev/null
+++ b/kubernetes/operator/internal/controller/tracing_test.go
@@ -0,0 +1,32 @@
+package controller
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ corev1 "k8s.io/api/core/v1"
+)
+
+// Backend tracing to Langfuse is opt-in, and content capture is a separate
+// opt-in on top of it. These pin both defaults off and confirm a CR can turn
+// tracing on and supply the Langfuse keys to the backend.
+
+func TestBackendTracing_OffByDefault(t *testing.T) {
+ env := resolvedBackendEnv(t, nil)
+
+ assert.Equal(t, "false", env["OPENRAG_TRACING_ENABLED"])
+ assert.Equal(t, "false", env["OPENRAG_TRACING_CAPTURE_CONTENT"])
+ assert.Equal(t, "false", env["OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS"])
+}
+
+func TestBackendTracing_EnabledFromCRSpec(t *testing.T) {
+ env := resolvedBackendEnv(t, []corev1.EnvVar{
+ {Name: "OPENRAG_TRACING_ENABLED", Value: "true"},
+ {Name: "LANGFUSE_PUBLIC_KEY", Value: "pk-lf-test"},
+ })
+
+ assert.Equal(t, "true", env["OPENRAG_TRACING_ENABLED"])
+ assert.Equal(t, "pk-lf-test", env["LANGFUSE_PUBLIC_KEY"])
+ assert.Equal(t, "false", env["OPENRAG_TRACING_CAPTURE_CONTENT"],
+ "enabling tracing must not enable content capture")
+}
diff --git a/pyproject.toml b/pyproject.toml
index 0b3672692..cb86871a7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -84,6 +84,9 @@ dependencies = [
# be adopted explicitly, not resolved into an install.
# src/main.py imports it defensively, so its absence downgrades to a warning.
apm = ["instana>=3.17.0,<4"]
+tracing = [
+ "langfuse>=4.16,<5",
+]
[dependency-groups]
dev = [
@@ -93,6 +96,8 @@ dev = [
"pytest-cov>=4.0.0",
"ruff>=0.15.12",
"mypy>=2.0.0",
+ # The tracing extra, so its unit tests (tests/unit/observability) run.
+ "langfuse>=4.16,<5",
]
[project.scripts]
diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py
index 026d379e7..b05bd664e 100644
--- a/src/api/provider_validation.py
+++ b/src/api/provider_validation.py
@@ -1001,6 +1001,7 @@ async def _test_litellm_provider(
"""Validate arbitrary providers through the same LiteLLM adapter used at runtime."""
import litellm
+ from observability.tracing import gateway_trace_kwargs
from services.llm_gateway import LlmGatewayError, with_stale_connection_retry
from services.model_catalog import litellm_provider_key
@@ -1029,6 +1030,7 @@ async def _test_litellm_provider(
input=["OpenRAG provider validation"],
**credentials,
**(runtime_kwargs or {}),
+ **gateway_trace_kwargs("embedding", "provider_check", provider=provider),
),
provider=provider,
model=litellm_model,
@@ -1042,6 +1044,7 @@ async def _test_litellm_provider(
max_tokens=4,
**credentials,
**(runtime_kwargs or {}),
+ **gateway_trace_kwargs("chat", "provider_check", provider=provider),
),
provider=provider,
model=litellm_model,
diff --git a/src/api/v1/llm.py b/src/api/v1/llm.py
index 7e9d2aeda..4d008087b 100644
--- a/src/api/v1/llm.py
+++ b/src/api/v1/llm.py
@@ -10,6 +10,7 @@ from fastapi import Depends, Request
from fastapi.responses import JSONResponse, StreamingResponse
from dependencies import require_llm_proxy_any_permission, require_llm_proxy_permission
+from observability.tracing import trace_context
from services.langflow_llm_token_service import HOP_PURPOSE_CHAT
from services.llm_gateway import LlmGatewayError, chat_completions, embeddings
from services.model_catalog import (
@@ -113,6 +114,24 @@ async def _read_json_body(request: Request) -> dict[str, Any] | JSONResponse:
return body
+def _trace_caller(request: Request, user: User, *, api_purpose: str | None = None):
+ """Label the gateway calls this request makes: Langflow run or API caller.
+
+ ``api_purpose`` overrides the gateway's purpose for API callers only;
+ Langflow runs keep the gateway's (their hop token says chat or ingest).
+ """
+ run_id = getattr(request.state, "llm_hop_run_id", None)
+ if not run_id:
+ return trace_context(source="api", user_id=user.user_id, purpose=api_purpose)
+ hop_purpose = getattr(request.state, "llm_hop_purpose", None)
+ return trace_context(
+ source="langflow",
+ user_id=user.user_id,
+ session_id=run_id,
+ attributes={"openrag.langflow_run_purpose": hop_purpose} if hop_purpose else None,
+ )
+
+
async def chat_completions_endpoint(
request: Request,
user: User = Depends(_chat_use),
@@ -122,7 +141,10 @@ async def chat_completions_endpoint(
if isinstance(body, JSONResponse):
return body
try:
- result = await chat_completions(body)
+ # The provider call starts here even when streaming, so the label
+ # covers it; the stream is only read after the block exits.
+ with _trace_caller(request, user):
+ result = await chat_completions(body)
except LlmGatewayError as exc:
return _gateway_error(exc)
if body.get("stream"):
@@ -150,7 +172,10 @@ async def embeddings_endpoint(
# behind a bulk ingest at a concurrency-limited provider.
interactive = getattr(request.state, "llm_hop_purpose", None) == HOP_PURPOSE_CHAT
try:
- result = await embeddings(body, interactive=interactive)
+ # A non-interactive call here isn't necessarily ingestion: the gateway
+ # would label an API caller's embeddings `ingest`.
+ with _trace_caller(request, user, api_purpose="api"):
+ result = await embeddings(body, interactive=interactive)
except LlmGatewayError as exc:
return _gateway_error(exc)
return JSONResponse(result)
diff --git a/src/api/v1/search.py b/src/api/v1/search.py
index a8ac58624..98bffff4d 100644
--- a/src/api/v1/search.py
+++ b/src/api/v1/search.py
@@ -18,6 +18,7 @@ from dependencies import (
get_search_service,
require_api_key_permission,
)
+from observability.tracing import trace_context
from session_manager import User
from utils.logging_config import get_logger
from utils.opensearch_utils import DISK_SPACE_ERROR_MESSAGE, OpenSearchDiskSpaceError
@@ -73,14 +74,18 @@ async def search_endpoint(
)
try:
- result = await search_service.search(
- query,
- user_id=user.user_id,
- jwt_token=user.jwt_token,
- filters=resolved_filters or {},
- limit=resolved_limit,
- score_threshold=resolved_score_threshold,
- )
+ # The search trace records which saved filter was applied.
+ with trace_context(
+ attributes={"openrag.search.filter_id": body.filter_id} if body.filter_id else None
+ ):
+ result = await search_service.search(
+ query,
+ user_id=user.user_id,
+ jwt_token=user.jwt_token,
+ filters=resolved_filters or {},
+ limit=resolved_limit,
+ score_threshold=resolved_score_threshold,
+ )
results = [
{
diff --git a/src/app/lifespan.py b/src/app/lifespan.py
index 5eefb6408..1e5b87cb1 100644
--- a/src/app/lifespan.py
+++ b/src/app/lifespan.py
@@ -201,6 +201,13 @@ async def run_startup(app: FastAPI):
run_mode=get_run_mode(),
)
+ # Backend LLM tracing to Langfuse (OPENRAG_TRACING_ENABLED, default off).
+ # Registered before any gateway call can run; misconfiguration warns and
+ # leaves the backend untraced rather than failing startup.
+ from observability.tracing import configure_tracing
+
+ configure_tracing()
+
# Open the SQL engine on uvicorn's live loop (NOT the one used by
# asyncio.run(create_app()) which closed already). All RBAC code
# uses RBACService's lazy factory that reads db.engine.SessionLocal
diff --git a/src/config/settings.py b/src/config/settings.py
index 831b4911b..032eb90e3 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -395,6 +395,58 @@ def is_ingest_preview_flag_enabled() -> bool:
return raw in ("true", "1", "yes", "on")
+def is_tracing_flag_enabled() -> bool:
+ """Opt-in flag for backend LLM/ingest/search tracing to Langfuse.
+
+ Default off. Gates registration of the LiteLLM ``langfuse_otel`` callback in
+ ``observability.tracing``; see that module for the content-capture policy.
+ """
+ raw = os.getenv("OPENRAG_TRACING_ENABLED", "false").strip().lower()
+ return raw in ("true", "1", "yes", "on")
+
+
+def is_tracing_capture_content_flag_enabled() -> bool:
+ """Opt-in to put prompts, completions, chunk text, queries and filenames in traces.
+
+ Default off (metadata only). In SaaS run mode this alone is not enough;
+ ``OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS`` must also be set.
+ """
+ raw = os.getenv("OPENRAG_TRACING_CAPTURE_CONTENT", "false").strip().lower()
+ return raw in ("true", "1", "yes", "on")
+
+
+def is_tracing_content_allowed_in_saas() -> bool:
+ """Second key required for trace content capture in SaaS run mode. Default off."""
+ raw = os.getenv("OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS", "false").strip().lower()
+ return raw in ("true", "1", "yes", "on")
+
+
+def langfuse_keys_configured() -> bool:
+ """Whether both Langfuse API keys are set (shared with the Langflow container)."""
+ return bool(
+ os.getenv("LANGFUSE_PUBLIC_KEY", "").strip()
+ and os.getenv("LANGFUSE_SECRET_KEY", "").strip()
+ )
+
+
+# With no host set, the Langfuse SDK defaults to Langfuse Cloud EU while
+# LiteLLM's langfuse_otel defaults to Cloud US; follow the SDK.
+LANGFUSE_DEFAULT_HOST = "https://cloud.langfuse.com"
+
+
+def get_langfuse_host() -> str:
+ """Langfuse URL for backend tracing, shared by LiteLLM's exporter and our spans.
+
+ ``LANGFUSE_BASE_URL`` (the name Langfuse's setup steps use) first, then the
+ older ``LANGFUSE_HOST`` — Langflow's precedence — else Langfuse Cloud EU.
+ """
+ return (
+ os.getenv("LANGFUSE_BASE_URL", "").strip()
+ or os.getenv("LANGFUSE_HOST", "").strip()
+ or LANGFUSE_DEFAULT_HOST
+ )
+
+
def is_workspace_oauth_overrides_enabled() -> bool:
"""Feature flag for workspace-level OAuth connector credential overrides.
diff --git a/src/dependencies.py b/src/dependencies.py
index 0be0ff44f..e36b65b4a 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -206,6 +206,9 @@ async def get_llm_proxy_user(
# Which kind of Langflow run minted it; the gateway gives chat
# (query) embeddings their own lane beside bulk ingestion.
request.state.llm_hop_purpose = claims.get("purpose")
+ # Unique per Langflow run (each run mints its own token); traces use
+ # it as the session that groups the run's model calls.
+ request.state.llm_hop_run_id = claims.get("jti")
return user
return await resolve_api_key_user(request, api_key_service, session_manager)
diff --git a/src/models/processors.py b/src/models/processors.py
index b14aa7f94..cf1968d32 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -5,6 +5,7 @@ import time
from typing import TYPE_CHECKING, Any, Literal
from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
+from observability.spans import trace_root, trace_span
from session_manager import AnonymousUser
from utils.document_processing import (
extract_relevant,
@@ -33,6 +34,32 @@ TEXT_PARSER_LABEL = "Text Parser"
DUPLICATE_FILENAME_WARNING = "A file with this name already exists."
+
+def _docling_counts(doc: Any) -> dict[str, int]:
+ """What Docling produced, for the parse span: element counts, never text."""
+ if not isinstance(doc, dict):
+ return {}
+ return {key: len(doc.get(key) or ()) for key in ("pages", "texts", "tables", "pictures")}
+
+
+def _chunk_type_counts(chunks: list[dict]) -> dict[str, int]:
+ """Chunk count, and per type (text/table/picture) where non-zero."""
+ counts: dict[str, int] = {"chunks": len(chunks)}
+ for chunk in chunks:
+ key = f"chunks_{chunk.get('type') or 'unknown'}"
+ counts[key] = counts.get(key, 0) + 1
+ return counts
+
+
+def _prompt_tokens(response: Any) -> int:
+ """Input tokens an embedding response reports, or 0."""
+ usage = response.get("usage") if isinstance(response, dict) else None
+ if not isinstance(usage, dict):
+ return 0
+ tokens = usage.get("prompt_tokens") or usage.get("total_tokens") or 0
+ return tokens if isinstance(tokens, int) else 0
+
+
# Both outcomes finish the file the same way — SKIPPED, counted successful,
# duplicate warning — but only "skip" means a document with this name is
# actually indexed. "skip_in_flight" means another file in this batch is still
@@ -560,227 +587,300 @@ class TaskProcessor:
if chunk_overlap is None:
chunk_overlap = config.knowledge.chunk_overlap
- # Get user's OpenSearch client with JWT for OIDC auth
- opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
- owner_user_id, jwt_token
- )
-
- # Check if already exists
- if await self.check_document_exists(file_hash, opensearch_client):
- return {"status": "unchanged", "id": file_hash}
+ file_ext = os.path.splitext(file_path)[1].lower()
- logger.info(
- "Processing document with embedding model",
- embedding_model=embedding_model,
- file_hash=file_hash,
- )
+ # One trace per file, with a span per stage; no-ops while tracing is off.
+ with trace_root(
+ "ingest.file",
+ user_id=owner_user_id,
+ tags=["ingest_path:openrag", f"connector:{connector_type}"],
+ attributes={
+ "document_id": file_hash,
+ "ingest_path": "openrag",
+ "connector_type": connector_type,
+ "embedding_model": embedding_model,
+ "embedding_provider": embedding_provider,
+ "file.extension": file_ext or None,
+ "file.size": file_size,
+ "sample_data": is_sample_data or None,
+ },
+ ) as ingest_trace:
+ ingest_trace.set_content(filename=original_filename or os.path.basename(file_path))
+
+ # Get user's OpenSearch client with JWT for OIDC auth
+ opensearch_client = self.document_service.session_manager.get_user_opensearch_client(
+ owner_user_id, jwt_token
+ )
- # Check if this is a .txt or .md file - use simple processing instead of docling
- file_ext = os.path.splitext(file_path)[1].lower()
+ # Check if already exists
+ if await self.check_document_exists(file_hash, opensearch_client):
+ ingest_trace.set(status="unchanged")
+ return {"status": "unchanged", "id": file_hash}
- if file_ext in (".txt", ".md"):
- # Simple text file processing without docling
logger.info(
- "Processing as plain text file (bypassing docling)",
- file_path=file_path,
+ "Processing document with embedding model",
+ embedding_model=embedding_model,
file_hash=file_hash,
)
- slim_doc = process_text_file(file_path)
- slim_doc["parser"] = TEXT_PARSER_LABEL
- else:
- full_doc = await self.docling_service.convert_file(
- file_path,
- user_id=owner_user_id,
- auth_header=jwt_token,
- ocr=ocr,
- picture_descriptions=picture_descriptions,
- )
- slim_doc = extract_relevant(full_doc)
- slim_doc["parser"] = DOCLING_PARSER_LABEL
-
- # Override filename with original_filename if provided
- if original_filename:
- slim_doc["filename"] = original_filename
- if chunk_size is not None:
- try:
- cs = int(chunk_size)
- except (TypeError, ValueError):
- cs = 0
- if cs > 0:
+ # Check if this is a .txt or .md file - use simple processing instead of docling
+ if file_ext in (".txt", ".md"):
+ # Simple text file processing without docling
+ logger.info(
+ "Processing as plain text file (bypassing docling)",
+ file_path=file_path,
+ file_hash=file_hash,
+ )
+ with trace_span("ingest.parse", parser=TEXT_PARSER_LABEL) as parse:
+ slim_doc = process_text_file(file_path)
+ parse.set(chunks=len(slim_doc["chunks"]))
+ slim_doc["parser"] = TEXT_PARSER_LABEL
+ else:
+ with trace_span(
+ "ingest.parse",
+ parser=DOCLING_PARSER_LABEL,
+ # Unset per-request options fall back to config, as in Docling.
+ ocr=ocr if ocr is not None else getattr(config.knowledge, "ocr", None),
+ picture_descriptions=picture_descriptions
+ if picture_descriptions is not None
+ else getattr(config.knowledge, "picture_descriptions", None),
+ table_structure=getattr(config.knowledge, "table_structure", None),
+ ) as parse:
+ full_doc = await self.docling_service.convert_file(
+ file_path,
+ user_id=owner_user_id,
+ auth_header=jwt_token,
+ ocr=ocr,
+ picture_descriptions=picture_descriptions,
+ )
+ parse.set(**_docling_counts(full_doc))
+ with trace_span("ingest.extract") as extract:
+ slim_doc = extract_relevant(full_doc)
+ extract.set(**_chunk_type_counts(slim_doc["chunks"]))
+ slim_doc["parser"] = DOCLING_PARSER_LABEL
+ ingest_trace.set(mimetype=slim_doc.get("mimetype"))
+
+ # Override filename with original_filename if provided
+ if original_filename:
+ slim_doc["filename"] = original_filename
+
+ if chunk_size is not None:
try:
- co = int(chunk_overlap) if chunk_overlap is not None else 0
+ cs = int(chunk_size)
except (TypeError, ValueError):
- co = 0
- if co < cs:
- slim_doc["chunks"] = resplit_chunks_character_windows(
- slim_doc["chunks"], cs, max(0, co)
+ cs = 0
+ if cs > 0:
+ try:
+ co = int(chunk_overlap) if chunk_overlap is not None else 0
+ except (TypeError, ValueError):
+ co = 0
+ if co < cs:
+ with trace_span(
+ "ingest.resplit",
+ chunk_size=cs,
+ chunk_overlap=max(0, co),
+ chunks_in=len(slim_doc["chunks"]),
+ ) as resplit:
+ slim_doc["chunks"] = resplit_chunks_character_windows(
+ slim_doc["chunks"], cs, max(0, co)
+ )
+ resplit.set(chunks_out=len(slim_doc["chunks"]))
+
+ litellm_embedding_model = (
+ await self.models_service.get_litellm_model_name(
+ embedding_model,
+ provider=embedding_provider,
+ )
+ if self.models_service is not None
+ else embedding_model
+ )
+
+ litellm_model_lower = litellm_embedding_model.lower() if litellm_embedding_model else ""
+ if "watsonx" in litellm_model_lower:
+ max_tokens = 500
+ elif "ollama" in litellm_model_lower:
+ max_tokens = 2000
+ else:
+ max_tokens = 8000
+
+ with trace_span(
+ "ingest.token_cap", max_tokens=max_tokens, chunks_in=len(slim_doc["chunks"])
+ ) as token_cap:
+ # Filter out chunks with empty or whitespace-only text before generating embeddings.
+ # This ensures the length of chunks matches the length of the embeddings array,
+ # since chunk_texts_for_embeddings also drops empty texts.
+ slim_doc["chunks"] = [
+ c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()
+ ]
+ non_empty = len(slim_doc["chunks"])
+
+ # Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.
+ slim_doc["chunks"] = split_chunks_by_max_tokens(
+ slim_doc["chunks"], max_tokens, litellm_embedding_model
+ )
+ # Re-filter out chunks with empty or whitespace-only text that may have resulted from splitting
+ slim_doc["chunks"] = [
+ c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()
+ ]
+ token_cap.set(non_empty=non_empty, chunks_out=len(slim_doc["chunks"]))
+ texts = [c["text"] for c in slim_doc["chunks"]]
+
+ text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)
+ embeddings = []
+ # Every provider goes through the gateway, not just the aliased ones.
+ # The gateway resolves credentials per call from config; the direct
+ # client resolved them from process-global environment written once per
+ # process, which is never unset when a provider is deconfigured and is
+ # shared by every provider in the process. Gating on "is this provider
+ # aliased?" also meant the correctness of ingestion depended on an
+ # unrelated routing detail.
+ from services.llm_gateway import embeddings as gateway_embeddings
+ from services.llm_gateway import qualified_model_id
+
+ gateway_model = qualified_model_id(embedding_provider, embedding_model)
+
+ with trace_span("ingest.embed", batches=len(text_batches), inputs=len(texts)) as embed:
+ prompt_tokens = 0
+ for batch in text_batches:
+ response = await gateway_embeddings({"model": gateway_model, "input": batch})
+ data = response.get("data", [])
+ embeddings.extend(
+ [
+ item["embedding"] if isinstance(item, dict) else item.embedding
+ for item in data
+ ]
)
+ prompt_tokens += _prompt_tokens(response)
+ embed.set(
+ vectors=len(embeddings),
+ dimensions=len(embeddings[0]) if embeddings else None,
+ prompt_tokens=prompt_tokens or None,
+ )
- # Filter out chunks with empty or whitespace-only text before generating embeddings.
- # This ensures the length of chunks matches the length of the embeddings array,
- # since chunk_texts_for_embeddings also drops empty texts.
- slim_doc["chunks"] = [c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()]
+ if not embeddings or len(embeddings) == 0:
+ logger.error(
+ "No embeddings generated — document may be empty or unreadable",
+ file_hash=file_hash,
+ embedding_model=embedding_model,
+ )
+ ingest_trace.fail("NoEmbeddings")
+ return {
+ "status": "error",
+ "error": "No text content could be extracted from document",
+ }
- litellm_embedding_model = (
- await self.models_service.get_litellm_model_name(
- embedding_model,
- provider=embedding_provider,
+ from services.document_index_writer import (
+ DocumentIndexChunk,
+ DocumentIndexContext,
+ DocumentIndexWriter,
)
- if self.models_service is not None
- else embedding_model
- )
- litellm_model_lower = litellm_embedding_model.lower() if litellm_embedding_model else ""
- if "watsonx" in litellm_model_lower:
- max_tokens = 500
- elif "ollama" in litellm_model_lower:
- max_tokens = 2000
- else:
- max_tokens = 8000
+ document_index_writer = getattr(self.document_service, "document_index_writer", None)
+ if document_index_writer is None:
+ document_index_writer = DocumentIndexWriter()
- # Split any chunks that exceed max_tokens before embedding, ensuring chunks and embeddings align 1-to-1.
- slim_doc["chunks"] = split_chunks_by_max_tokens(
- slim_doc["chunks"], max_tokens, litellm_embedding_model
- )
- # Re-filter out chunks with empty or whitespace-only text that may have resulted from splitting
- slim_doc["chunks"] = [c for c in slim_doc["chunks"] if c.get("text") and c["text"].strip()]
- texts = [c["text"] for c in slim_doc["chunks"]]
-
- text_batches = chunk_texts_for_embeddings(texts, max_tokens=max_tokens)
- embeddings = []
- # Every provider goes through the gateway, not just the aliased ones.
- # The gateway resolves credentials per call from config; the direct
- # client resolved them from process-global environment written once per
- # process, which is never unset when a provider is deconfigured and is
- # shared by every provider in the process. Gating on "is this provider
- # aliased?" also meant the correctness of ingestion depended on an
- # unrelated routing detail.
- from services.llm_gateway import embeddings as gateway_embeddings
- from services.llm_gateway import qualified_model_id
-
- gateway_model = qualified_model_id(embedding_provider, embedding_model)
-
- for batch in text_batches:
- response = await gateway_embeddings({"model": gateway_model, "input": batch})
- data = response.get("data", [])
- embeddings.extend(
- [item["embedding"] if isinstance(item, dict) else item.embedding for item in data]
+ # Owner is always the authenticated uploading/syncing user unless shared=True,
+ # in which case owner fields are omitted so DLS makes the doc visible to all users.
+ owner, owner_name, owner_email = resolve_shared_owner_fields(
+ owner_user_id, owner_name, owner_email, shared
)
+ if acl:
+ allowed_users = acl.allowed_users or []
+ allowed_groups = acl.allowed_groups or []
+ allowed_principals = acl.allowed_principals or []
+ allowed_principal_labels = acl.allowed_principal_labels or []
+ else:
+ allowed_users = []
+ allowed_groups = []
+ allowed_principals = []
+ allowed_principal_labels = []
- if not embeddings or len(embeddings) == 0:
- logger.error(
- "No embeddings generated — document may be empty or unreadable",
- file_hash=file_hash,
+ filename = original_filename if original_filename else slim_doc["filename"]
+ index_context = DocumentIndexContext(
+ document_id=file_hash,
+ filename=filename,
+ mimetype=slim_doc["mimetype"],
embedding_model=embedding_model,
+ embedding_provider=embedding_provider,
+ owner=owner,
+ owner_name=owner_name,
+ owner_email=owner_email,
+ file_size=file_size,
+ connector_type=connector_type,
+ allowed_users=allowed_users,
+ allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=allowed_principal_labels,
+ is_sample_data=is_sample_data,
)
- return {"status": "error", "error": "No text content could be extracted from document"}
-
- from services.document_index_writer import (
- DocumentIndexChunk,
- DocumentIndexContext,
- DocumentIndexWriter,
- )
-
- document_index_writer = getattr(self.document_service, "document_index_writer", None)
- if document_index_writer is None:
- document_index_writer = DocumentIndexWriter()
+ parser_name = slim_doc.get("parser")
+ if not parser_name:
+ if file_ext in (".txt", ".md"):
+ parser_name = TEXT_PARSER_LABEL
+ else:
+ parser_name = DOCLING_PARSER_LABEL
+
+ chunk_metadata = {"parser": parser_name}
+ if chunk_size is not None:
+ chunk_metadata["chunk_size"] = chunk_size
+ if chunk_overlap is not None:
+ chunk_metadata["chunk_overlap"] = chunk_overlap
+ if connector_file_id:
+ chunk_metadata["connector_file_id"] = connector_file_id
+
+ index_chunks = [
+ DocumentIndexChunk(
+ chunk_id=f"{file_hash}_{i}",
+ text=chunk["text"],
+ vector=vect,
+ page=chunk["page"],
+ metadata=chunk_metadata,
+ )
+ for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True))
+ ]
+
+ with trace_span("ingest.index") as index:
+ # Clear stale chunks from a prior indexing of this document. Chunks are
+ # stored under ids {file_hash}_{i}; if the new chunk count is lower
+ # than the prior one, trailing chunks would otherwise survive the
+ # writer's idempotent upsert.
+ # DLS-safe: enumerate visible chunk ids with the scoped user client,
+ # then delete concrete ids with the trusted backend client.
+ try:
+ from utils.opensearch_delete import (
+ collect_visible_document_ids,
+ delete_document_ids,
+ )
- # Clear stale chunks from a prior indexing of this document. Chunks are
- # stored under ids {file_hash}_{i}; if the new chunk count is lower
- # than the prior one, trailing chunks would otherwise survive the
- # writer's idempotent upsert.
- # DLS-safe: enumerate visible chunk ids with the scoped user client,
- # then delete concrete ids with the trusted backend client.
- try:
- from utils.opensearch_delete import (
- collect_visible_document_ids,
- delete_document_ids,
- )
+ write_client = clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
- write_client = clients.opensearch
- if write_client is None:
- raise RuntimeError("Backend OpenSearch write client is unavailable")
+ stale_chunk_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ query={"term": {"document_id": file_hash}},
+ )
+ await delete_document_ids(
+ write_client,
+ index=get_index_name(),
+ document_ids=stale_chunk_ids,
+ refresh=True,
+ )
+ index.set(stale_chunks_deleted=len(stale_chunk_ids))
+ except Exception as e:
+ logger.warning(
+ "Failed to clear stale chunks before re-index; proceeding",
+ file_hash=file_hash,
+ error=str(e),
+ )
+ index.set(stale_cleanup_failed=True)
- stale_chunk_ids = await collect_visible_document_ids(
- opensearch_client,
- index=get_index_name(),
- query={"term": {"document_id": file_hash}},
- )
- await delete_document_ids(
- write_client,
- index=get_index_name(),
- document_ids=stale_chunk_ids,
- refresh=True,
- )
- except Exception as e:
- logger.warning(
- "Failed to clear stale chunks before re-index; proceeding",
- file_hash=file_hash,
- error=str(e),
- )
+ await document_index_writer.index_chunks(index_context, index_chunks, final=True)
+ index.set(chunks_indexed=len(index_chunks))
- # Owner is always the authenticated uploading/syncing user unless shared=True,
- # in which case owner fields are omitted so DLS makes the doc visible to all users.
- owner, owner_name, owner_email = resolve_shared_owner_fields(
- owner_user_id, owner_name, owner_email, shared
- )
- if acl:
- allowed_users = acl.allowed_users or []
- allowed_groups = acl.allowed_groups or []
- allowed_principals = acl.allowed_principals or []
- allowed_principal_labels = acl.allowed_principal_labels or []
- else:
- allowed_users = []
- allowed_groups = []
- allowed_principals = []
- allowed_principal_labels = []
-
- filename = original_filename if original_filename else slim_doc["filename"]
- index_context = DocumentIndexContext(
- document_id=file_hash,
- filename=filename,
- mimetype=slim_doc["mimetype"],
- embedding_model=embedding_model,
- embedding_provider=embedding_provider,
- owner=owner,
- owner_name=owner_name,
- owner_email=owner_email,
- file_size=file_size,
- connector_type=connector_type,
- allowed_users=allowed_users,
- allowed_groups=allowed_groups,
- allowed_principals=allowed_principals,
- allowed_principal_labels=allowed_principal_labels,
- is_sample_data=is_sample_data,
- )
- parser_name = slim_doc.get("parser")
- if not parser_name:
- if file_ext in (".txt", ".md"):
- parser_name = TEXT_PARSER_LABEL
- else:
- parser_name = DOCLING_PARSER_LABEL
-
- chunk_metadata = {"parser": parser_name}
- if chunk_size is not None:
- chunk_metadata["chunk_size"] = chunk_size
- if chunk_overlap is not None:
- chunk_metadata["chunk_overlap"] = chunk_overlap
- if connector_file_id:
- chunk_metadata["connector_file_id"] = connector_file_id
-
- index_chunks = [
- DocumentIndexChunk(
- chunk_id=f"{file_hash}_{i}",
- text=chunk["text"],
- vector=vect,
- page=chunk["page"],
- metadata=chunk_metadata,
- )
- for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True))
- ]
- await document_index_writer.index_chunks(index_context, index_chunks, final=True)
- return {"status": "indexed", "id": file_hash}
+ ingest_trace.set(status="indexed", chunks=len(index_chunks))
+ return {"status": "indexed", "id": file_hash}
async def process_item(self, upload_task: UploadTask, item: Any, file_task: FileTask) -> None:
"""
diff --git a/src/observability/litellm_langfuse.py b/src/observability/litellm_langfuse.py
new file mode 100644
index 000000000..eb8735761
--- /dev/null
+++ b/src/observability/litellm_langfuse.py
@@ -0,0 +1,89 @@
+"""LiteLLM's Langfuse OTel logger, without provider error text unless content capture is on.
+
+``litellm.turn_off_message_logging`` redacts prompts and completions but not
+failures: on a failed call LiteLLM records the exception (an ``exception``
+event with its message and stack trace) and sets ``error.message`` and
+``error.stack_trace`` from the provider's error text, which can echo the
+prompt or the chunk being embedded. With capture off, those are dropped before
+export; ``error.type`` and ``error.code`` stay. Our own spans already do the
+same (``observability.spans``).
+
+Imported only once tracing is enabled: it imports LiteLLM's OTel integration
+and the OTel SDK.
+"""
+
+from typing import Any
+
+from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+from litellm.integrations.opentelemetry import OpenTelemetryConfig
+from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor
+from opentelemetry.trace import Status
+
+from observability.tracing import TRACING_CALLBACK
+
+_ERROR_TEXT_ATTRIBUTES = frozenset({"error.message", "error.stack_trace"})
+_EXCEPTION_EVENT = "exception"
+
+
+def _without_error_text(span: ReadableSpan) -> ReadableSpan:
+ """``span`` minus exception events, error message / stack trace and status text."""
+ attributes = span.attributes or {}
+ if (
+ not _ERROR_TEXT_ATTRIBUTES.intersection(attributes)
+ and not any(event.name == _EXCEPTION_EVENT for event in span.events)
+ and not span.status.description
+ ):
+ return span
+ return ReadableSpan(
+ name=span.name,
+ context=span.context,
+ parent=span.parent,
+ resource=span.resource,
+ attributes={k: v for k, v in attributes.items() if k not in _ERROR_TEXT_ATTRIBUTES},
+ events=tuple(event for event in span.events if event.name != _EXCEPTION_EVENT),
+ links=span.links,
+ kind=span.kind,
+ status=Status(span.status.status_code),
+ start_time=span.start_time,
+ end_time=span.end_time,
+ instrumentation_scope=span.instrumentation_scope,
+ )
+
+
+class _ErrorTextFilter(SpanProcessor):
+ """Wraps LiteLLM's span processor; hands it ended spans without error text."""
+
+ def __init__(self, inner: SpanProcessor):
+ self._inner = inner
+
+ def on_start(self, span: Any, parent_context: Any = None) -> None:
+ self._inner.on_start(span, parent_context=parent_context)
+
+ def on_end(self, span: ReadableSpan) -> None:
+ self._inner.on_end(_without_error_text(span))
+
+ def shutdown(self) -> None:
+ self._inner.shutdown()
+
+ def force_flush(self, timeout_millis: int = 30000) -> bool:
+ return self._inner.force_flush(timeout_millis)
+
+
+class _RedactingLangfuseOtelLogger(LangfuseOtelLogger):
+ """``LangfuseOtelLogger`` whose every span processor drops error text."""
+
+ def _get_span_processor(self, *args: Any, **kwargs: Any) -> SpanProcessor:
+ return _ErrorTextFilter(super()._get_span_processor(*args, **kwargs))
+
+
+def build_langfuse_logger(*, capture_content: bool, exporter: Any = None) -> LangfuseOtelLogger:
+ """The LiteLLM callback that exports gateway calls to Langfuse.
+
+ ``exporter`` replaces the OTLP export to Langfuse (tests); otherwise the
+ logger reads ``LANGFUSE_*`` from the environment like LiteLLM's own.
+ """
+ logger_class = LangfuseOtelLogger if capture_content else _RedactingLangfuseOtelLogger
+ config = OpenTelemetryConfig(exporter=exporter) if exporter is not None else None
+ # Registered under LiteLLM's own name for this integration, which keeps it
+ # off the global tracer provider (``OpenTelemetry._skip_set_global``).
+ return logger_class(config=config, callback_name=TRACING_CALLBACK)
diff --git a/src/observability/spans.py b/src/observability/spans.py
new file mode 100644
index 000000000..0fa7530da
--- /dev/null
+++ b/src/observability/spans.py
@@ -0,0 +1,185 @@
+"""OpenRAG's own spans in Langfuse: one trace per ingested file or search.
+
+The gateway's model calls are traced by LiteLLM (see ``observability.tracing``);
+these are the spans around them. LiteLLM parents each call under the active
+OTel span, so an embedding call made inside an ``ingest.embed`` span nests
+there in Langfuse.
+
+Built on the Langfuse SDK with a private TracerProvider. Left to itself the SDK
+installs its provider as the global one — or, with Instana's already global,
+attaches its exporter to Instana's and ships every APM span to Langfuse.
+
+Values recorded with ``Span.set`` are ids, counts and settings, sent whatever
+the content setting; anything that can carry user content (filenames, query
+text, exception messages) goes through ``Span.set_content`` or is dropped
+unless content capture is on. When tracing is off every helper is a no-op.
+"""
+
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from typing import Any
+
+from config.settings import get_langfuse_host
+from observability.tracing import content_capture_enabled, current_trace_context, trace_context
+from utils.run_mode_utils import get_run_mode
+
+# Langfuse client our spans export through; None while tracing is off.
+_client: Any = None
+
+_PREFIX = "openrag."
+# Attribute on an exception naming the innermost span it failed in, so the
+# root can say where the trace failed.
+_FAILED_SPAN_ATTR = "_openrag_failed_span"
+
+
+def init_span_client(*, span_exporter: Any = None) -> None:
+ """Create the Langfuse client for our spans. Called by ``configure_tracing``.
+
+ ``span_exporter`` replaces the OTLP export to Langfuse (tests).
+ """
+ from langfuse import Langfuse
+ from opentelemetry.sdk.resources import Resource
+ from opentelemetry.sdk.trace import TracerProvider
+ from opentelemetry.sdk.trace.sampling import ALWAYS_ON
+
+ global _client
+ _client = Langfuse(
+ base_url=get_langfuse_host(),
+ tracer_provider=TracerProvider(
+ # Not parent-based: a background ingest runs inside the request's
+ # (possibly unsampled Instana) context. Roots detach from it anyway.
+ sampler=ALWAYS_ON,
+ resource=Resource.create({"service.name": "openrag-backend"}),
+ ),
+ span_exporter=span_exporter,
+ )
+
+
+def _prefixed(attributes: Mapping[str, Any]) -> dict[str, Any]:
+ return {
+ key if key.startswith(_PREFIX) else f"{_PREFIX}{key}": value
+ for key, value in attributes.items()
+ if value is not None
+ }
+
+
+class Span:
+ """Handle on one of our spans; every method is a no-op while tracing is off."""
+
+ def __init__(self, observation: Any = None):
+ self._observation = observation
+
+ def set(self, **attributes: Any) -> None:
+ """Record ids, counts and settings as ``openrag.*`` metadata. Never content."""
+ if self._observation is None:
+ return
+ values = _prefixed(attributes)
+ if values:
+ self._observation.update(metadata=values)
+
+ def set_content(self, **attributes: Any) -> None:
+ """Record values that can carry user content, only when capture is on."""
+ if content_capture_enabled():
+ self.set(**attributes)
+
+ def fail(self, error_type: str, message: str | None = None) -> None:
+ """Mark the span failed. ``message`` is content and kept only when capture is on."""
+ if self._observation is None:
+ return
+ status = f"{error_type}: {message}" if message and content_capture_enabled() else error_type
+ self._observation.update(level="ERROR", status_message=status)
+ self.set(error_type=error_type)
+
+
+_NOOP = Span()
+
+
+@contextmanager
+def _observe(name: str, attributes: Mapping[str, Any]) -> Iterator[Span]:
+ """One Langfuse span, current for the block; failures recorded without their text.
+
+ The SDK's span records an escaping exception with its message and stack
+ trace, which can carry file paths or document text. So the exception is
+ caught inside the span, recorded by type, and re-raised once it has ended.
+ """
+ error: BaseException | None = None
+ metadata = _prefixed(attributes) or None
+ with _client.start_as_current_observation(name=name, as_type="span", metadata=metadata) as obs:
+ span = Span(obs)
+ try:
+ yield span
+ except BaseException as exc:
+ error = exc
+ failed_in = getattr(exc, _FAILED_SPAN_ATTR, None)
+ error_type = type(exc).__name__
+ if failed_in is None:
+ span.fail(error_type, str(exc))
+ try:
+ setattr(exc, _FAILED_SPAN_ATTR, name)
+ except AttributeError:
+ pass
+ else:
+ span.fail(f"{error_type} in {failed_in}", str(exc))
+ if error is not None:
+ raise error
+
+
+@contextmanager
+def trace_root(
+ name: str,
+ *,
+ attributes: Mapping[str, Any],
+ user_id: str | None = None,
+ session_id: str | None = None,
+ tags: list[str] | None = None,
+) -> Iterator[Span]:
+ """Start a new Langfuse trace named ``name`` for the block.
+
+ Always a new trace, never a child of whatever span is current: background
+ work inherits the request's context, whose span (Instana's) never reaches
+ Langfuse. ``attributes`` become trace metadata on every span in the trace,
+ together with the current ``trace_context``'s; user and session default
+ to the context's too. Gateway calls made inside are labelled the same way.
+ """
+ if _client is None:
+ yield _NOOP
+ return
+
+ from langfuse import propagate_attributes
+ from opentelemetry import context as otel_context
+
+ context = current_trace_context()
+ user_id = user_id or context.get("user_id")
+ session_id = session_id or context.get("session_id")
+ trace_attributes = {
+ **context.get("attributes", {}),
+ "openrag.run_mode": get_run_mode(),
+ **_prefixed(attributes),
+ }
+
+ token = otel_context.attach(otel_context.Context())
+ try:
+ with (
+ _observe(name, {}) as root,
+ propagate_attributes(
+ trace_name=name,
+ user_id=user_id or None,
+ session_id=session_id or None,
+ tags=tags or None,
+ metadata={key: str(value) for key, value in trace_attributes.items()},
+ ),
+ trace_context(user_id=user_id, session_id=session_id, attributes=trace_attributes),
+ ):
+ yield root
+ finally:
+ otel_context.detach(token)
+
+
+@contextmanager
+def trace_span(name: str, **attributes: Any) -> Iterator[Span]:
+ """A stage span under the current one; ``attributes`` as for ``Span.set``."""
+ if _client is None:
+ yield _NOOP
+ return
+ with _observe(name, attributes) as span:
+ yield span
diff --git a/src/observability/tracing.py b/src/observability/tracing.py
new file mode 100644
index 000000000..2782ec008
--- /dev/null
+++ b/src/observability/tracing.py
@@ -0,0 +1,221 @@
+"""Backend tracing to Langfuse.
+
+Every model call the backend makes goes through LiteLLM in
+``services/llm_gateway.py`` — including the embedding and chat calls Langflow
+proxies to ``/v1/*`` — so tracing hooks in once, as LiteLLM's ``langfuse_otel``
+callback, rather than at each call site. That callback builds its own private
+OpenTelemetry provider exporting OTLP to ``$LANGFUSE_BASE_URL/api/public/otel`` and
+does not replace the global provider, and it parents each call under whatever
+OTel span is active, so ingest and search spans opened around gateway calls
+nest them automatically.
+
+Off by default (``OPENRAG_TRACING_ENABLED``). When off, nothing is registered
+and neither LiteLLM nor the OTel SDK is imported here.
+
+Content policy: traces carry metadata only (model, provider, latency, token
+usage, cost, errors) unless content capture is opted into — and in SaaS run
+mode that takes a second flag. Decision record:
+``local/plans/backend-tracing-decision.md``.
+"""
+
+import os
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from contextvars import ContextVar
+from types import MappingProxyType
+from typing import Any
+
+from config.settings import (
+ get_langfuse_host,
+ is_tracing_capture_content_flag_enabled,
+ is_tracing_content_allowed_in_saas,
+ is_tracing_flag_enabled,
+ langfuse_keys_configured,
+)
+from utils.logging_config import get_logger
+from utils.run_mode_utils import get_run_mode, is_run_mode_saas
+
+logger = get_logger(__name__)
+
+TRACING_CALLBACK = "langfuse_otel"
+
+# LiteLLM setting, read from the environment per request (``get_secret_bool``).
+_LITELLM_REQUEST_SPAN_ENV_VAR = "USE_OTEL_LITELLM_REQUEST_SPAN"
+# OpenTelemetry SDK setting, read when a TracerProvider is constructed.
+_OTEL_SAMPLER_ENV_VAR = "OTEL_TRACES_SAMPLER"
+
+# Set by configure_tracing(); gates the per-call metadata below so a backend
+# running untraced passes LiteLLM exactly what it did before tracing existed.
+_enabled = False
+
+# Who the work in progress is for (see trace_context): read by gateway calls
+# and by the ingest/search traces in observability.spans.
+_call_context: ContextVar[Mapping[str, Any]] = ContextVar(
+ "openrag_trace_context", default=MappingProxyType({})
+)
+
+
+def content_capture_enabled() -> bool:
+ """Whether traces may carry prompts, completions, chunk text, queries and filenames."""
+ if not is_tracing_capture_content_flag_enabled():
+ return False
+ if is_run_mode_saas():
+ return is_tracing_content_allowed_in_saas()
+ return True
+
+
+def configure_tracing() -> bool:
+ """Register the Langfuse callback on LiteLLM when tracing is enabled.
+
+ Returns True when tracing is active. Misconfiguration (missing keys, missing
+ ``tracing`` extra) warns and leaves the backend running untraced rather than
+ failing startup — the same stance as the Instana gate.
+ """
+ if not is_tracing_flag_enabled():
+ return False
+
+ if not langfuse_keys_configured():
+ logger.warning(
+ "OPENRAG_TRACING_ENABLED is set but LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY "
+ "are missing; backend tracing is disabled."
+ )
+ return False
+
+ try:
+ import opentelemetry.exporter.otlp.proto.http.trace_exporter # noqa: F401
+ import opentelemetry.sdk.trace # noqa: F401
+ except ImportError:
+ logger.warning(
+ "OPENRAG_TRACING_ENABLED is set but the tracing dependencies are not "
+ "installed; backend tracing is disabled. Install the optional extra with "
+ "`uv sync --extra tracing`."
+ )
+ return False
+
+ import litellm
+
+ # Without this, a call made while another OTel span is active (an ingest or
+ # search span, or Instana's HTTP span) gets no span of its own: LiteLLM
+ # writes its attributes onto that caller span instead — usually already
+ # ended by the time the async callback runs, or owned by another exporter —
+ # so nothing reaches Langfuse. With it, every call is its own child span.
+ os.environ.setdefault(_LITELLM_REQUEST_SPAN_ENV_VAR, "true")
+ # LiteLLM builds its tracer providers with the SDK default sampler, which
+ # follows the parent's sampling decision. Instana's HTTP spans are active
+ # during every request and are never sampled, so with Instana on that would
+ # silently drop every LLM span. Record unconditionally; the spans still join
+ # the request's trace id. Read when each provider is built, i.e. after this.
+ os.environ.setdefault(_OTEL_SAMPLER_ENV_VAR, "always_on")
+ # LiteLLM reads its host only from LANGFUSE_OTEL_HOST / LANGFUSE_HOST, so
+ # hand it the one our spans use too (LANGFUSE_OTEL_HOST, LiteLLM's own
+ # override, still wins for LiteLLM).
+ if not os.environ.get("LANGFUSE_OTEL_HOST"):
+ os.environ["LANGFUSE_HOST"] = get_langfuse_host()
+
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+
+ from observability import litellm_langfuse
+
+ capture = content_capture_enabled()
+ # Read by LiteLLM on every request: forces its OTel logger to NO_CONTENT,
+ # so prompts, completions and embedding inputs never leave the process.
+ litellm.turn_off_message_logging = not capture
+ # A logger instance rather than the "langfuse_otel" string: that flag
+ # doesn't cover failures, whose provider error text the logger built here
+ # drops when capture is off.
+ if not any(isinstance(cb, LangfuseOtelLogger) for cb in litellm.callbacks):
+ litellm.callbacks.append(litellm_langfuse.build_langfuse_logger(capture_content=capture))
+
+ from observability.spans import init_span_client
+
+ init_span_client()
+
+ global _enabled
+ _enabled = True
+ logger.info("Backend tracing enabled", backend="langfuse", capture_content=capture)
+ return True
+
+
+@contextmanager
+def trace_context(
+ *,
+ source: str | None = None,
+ user_id: str | None = None,
+ session_id: str | None = None,
+ purpose: str | None = None,
+ attributes: Mapping[str, Any] | None = None,
+) -> Iterator[None]:
+ """Describe the work inside this block to its traces: gateway calls and roots.
+
+ ``source`` is who asked (``langflow`` or the default ``openrag``),
+ ``user_id`` and ``session_id`` become Langfuse's user and session, and
+ ``purpose`` overrides the call site's own (e.g. ``dimension_probe``).
+ ``attributes`` are extra ``openrag.*`` trace metadata. Blocks nest: inner
+ values win, and leaving a block restores the outer ones.
+ """
+ fields = dict(_call_context.get())
+ for key, value in (
+ ("source", source),
+ ("user_id", user_id),
+ ("session_id", session_id),
+ ("purpose", purpose),
+ ):
+ if value is not None:
+ fields[key] = value
+ if attributes:
+ fields["attributes"] = {**fields.get("attributes", {}), **attributes}
+ token = _call_context.set(MappingProxyType(fields))
+ try:
+ yield
+ finally:
+ _call_context.reset(token)
+
+
+def current_trace_context() -> Mapping[str, Any]:
+ """The fields set by the enclosing ``trace_context`` blocks (read-only)."""
+ return _call_context.get()
+
+
+def gateway_trace_kwargs(
+ kind: str, purpose: str, *, provider: str | None = None, **sizes: Any
+) -> dict[str, Any]:
+ """LiteLLM kwargs that label one gateway call in Langfuse; ``{}`` when untraced.
+
+ ``kind`` is ``embedding`` or ``chat``; ``purpose`` is what the call is for
+ (``ingest``, ``query``, ``completion``, ...) unless the call context
+ overrides it. ``provider`` is OpenRAG's provider id, which can differ from
+ LiteLLM's route (``watsonx_onprem``, custom providers). ``sizes`` are counts recorded as ``openrag.<kind>.<name>``;
+ None values are left out. Never pass content here: this metadata is sent
+ whatever the content-capture setting.
+
+ LiteLLM's ``langfuse_otel`` maps ``generation_name`` to the observation
+ name, ``trace_user_id`` / ``session_id`` to Langfuse's user and session,
+ and ``trace_metadata`` keys to metadata; it drops other keys. LiteLLM
+ consumes ``metadata`` itself and never sends it to the provider.
+ """
+ if not _enabled:
+ return {}
+ context = _call_context.get()
+ purpose = context.get("purpose") or purpose
+ source = context.get("source") or "openrag"
+ trace_metadata: dict[str, Any] = {
+ "openrag.source": source,
+ "openrag.purpose": purpose,
+ "openrag.run_mode": get_run_mode(),
+ **context.get("attributes", {}),
+ }
+ if provider:
+ trace_metadata["openrag.provider"] = provider
+ trace_metadata.update(
+ {f"openrag.{kind}.{name}": value for name, value in sizes.items() if value is not None}
+ )
+ metadata: dict[str, Any] = {
+ "generation_name": f"{kind}.{purpose}",
+ "tags": [f"source:{source}", f"purpose:{purpose}"],
+ "trace_metadata": trace_metadata,
+ }
+ if context.get("user_id"):
+ metadata["trace_user_id"] = context["user_id"]
+ if context.get("session_id"):
+ metadata["session_id"] = context["session_id"]
+ return {"metadata": metadata}
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index 47af817d0..c2119e835 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -141,15 +141,17 @@ class LangflowFileService:
# mapping's vector width, so it has to reach the same endpoint the real
# embedding call will. The direct client resolved credentials from
# process-global environment instead of from config.
+ from observability.tracing import trace_context
from services.llm_gateway import embeddings as gateway_embeddings
from services.llm_gateway import qualified_model_id
- response = await gateway_embeddings(
- {
- "model": qualified_model_id(embedding_provider, embedding_model),
- "input": ["dimension probe"],
- }
- )
+ with trace_context(purpose="dimension_probe"):
+ response = await gateway_embeddings(
+ {
+ "model": qualified_model_id(embedding_provider, embedding_model),
+ "input": ["dimension probe"],
+ }
+ )
data = response.get("data") or []
if not data:
raise RuntimeError("Embedding provider returned no data for dimension probe")
diff --git a/src/services/llm_gateway.py b/src/services/llm_gateway.py
index 5140a439e..a73f7e2ac 100644
--- a/src/services/llm_gateway.py
+++ b/src/services/llm_gateway.py
@@ -18,6 +18,7 @@ from itertools import pairwise
from typing import Any, Literal, NamedTuple
from uuid import uuid4
+from observability.tracing import gateway_trace_kwargs, trace_context
from services import provider_error_log
from services.model_catalog import is_known_provider, litellm_provider_key
from utils.logging_config import get_logger
@@ -1402,6 +1403,14 @@ async def chat_completions(
**credentials,
**runtime_kwargs,
**kwargs,
+ **gateway_trace_kwargs(
+ "chat",
+ "completion",
+ provider=provider,
+ messages=len(messages) if isinstance(messages, list) else None,
+ tools=len(body["tools"]) if isinstance(body.get("tools"), list) else None,
+ stream=stream,
+ ),
),
provider=provider,
model=litellm_model,
@@ -1833,6 +1842,19 @@ def _embedding_input(value: Any) -> Any:
return [value] if isinstance(value, str) else value
+def _embedding_trace_sizes(embedding_input: Any) -> dict[str, int | None]:
+ """How many inputs and how much text, for the trace; never the text itself."""
+ if not isinstance(embedding_input, list) or not embedding_input:
+ return {}
+ if all(isinstance(item, int) for item in embedding_input):
+ return {"inputs": 1} # one pre-tokenised input
+ all_text = all(isinstance(item, str) for item in embedding_input)
+ return {
+ "inputs": len(embedding_input),
+ "input_chars": sum(map(len, embedding_input)) if all_text else None,
+ }
+
+
#: How many times one input is cut smaller before the gateway gives up on it.
_MAX_OVERFLOW_SPLIT_ROUNDS = 3
#: Pieces are sized under the proportion the provider reported: tokens are not
@@ -2003,6 +2025,7 @@ async def embeddings(
runtime_kwargs = _provider_runtime_kwargs(provider, cfg)
limiter = _embedding_limiter(provider, cfg)
lane = _INTERACTIVE_LANE if interactive else _BULK_LANE
+ purpose = "query" if interactive else "ingest"
embedding_input = _embedding_input(body.get("input"))
batch_size = _WATSONX_ONPREM_EMBEDDING_BATCH_SIZE if provider == "watsonx_onprem" else None
@@ -2017,6 +2040,12 @@ async def embeddings(
input=inputs,
**credentials,
**runtime_kwargs,
+ **gateway_trace_kwargs(
+ "embedding",
+ purpose,
+ provider=provider,
+ **_embedding_trace_sizes(inputs),
+ ),
),
provider=provider,
model=litellm_model,
@@ -2069,8 +2098,16 @@ async def embeddings(
response = {}
data: list[dict[str, Any]] = []
usage: dict[str, int | float] = {}
+ sub_batches = -(-len(embedding_input) // batch_size)
for offset in range(0, len(embedding_input), batch_size):
- payload = await embed(embedding_input[offset : offset + batch_size])
+ # Numbered in the trace; an over-length input's pieces keep it.
+ with trace_context(
+ attributes={
+ "openrag.embedding.sub_batch": offset // batch_size + 1,
+ "openrag.embedding.sub_batches": sub_batches,
+ }
+ ):
+ payload = await embed(embedding_input[offset : offset + batch_size])
if not response:
response = {
key: value for key, value in payload.items() if key not in {"data", "usage"}
diff --git a/src/services/search_service.py b/src/services/search_service.py
index b124d6f01..56b64789b 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -1,5 +1,6 @@
import asyncio
import copy
+import json
import os
import re
from collections import Counter
@@ -10,6 +11,7 @@ from agentd.tool_decorator import tool
from auth_context import get_auth_context
from config.embedding_constants import get_declared_default_embedding_model
from config.settings import get_embedding_model, get_index_name, get_openrag_config
+from observability.spans import Span, trace_root, trace_span
from services.llm_gateway import LlmGatewayError
from services.llm_gateway import embeddings as gateway_embeddings
from utils.container_utils import transform_localhost_url
@@ -216,6 +218,22 @@ def _apply_exact_match_file_filter(
return chunks, aggregations
+def _record_search_response(span: Span, response: dict[str, Any]) -> None:
+ """What a search returned, for its trace: counts, scores and ids, never text."""
+ results = response.get("results") or []
+ span.set(
+ **{
+ "search.hit_count": len(results),
+ "search.top_scores": [
+ round(r["score"], 4) for r in results[:5] if isinstance(r.get("score"), int | float)
+ ],
+ "search.chunk_ids": [r["chunk_id"] for r in results[:10] if r.get("chunk_id")],
+ "search.degraded": True if response.get("warnings") else None,
+ "search.error": response.get("error"),
+ }
+ )
+
+
def register_search_service(service: "SearchService") -> None:
"""
Explicitly register the active search service for the @tool wrapper.
@@ -264,7 +282,9 @@ class SearchService:
except Exception as e:
logger.warning("[SEARCH] Could not configure Ollama endpoint from config", error=str(e))
- async def search_tool(self, query: str, embedding_model: str = None) -> dict[str, Any]:
+ async def search_tool(
+ self, query: str, embedding_model: str = None, *, source: str = "agent_tool"
+ ) -> dict[str, Any]:
"""
Use this tool to search for documents relevant to the query.
@@ -273,10 +293,14 @@ class SearchService:
embedding_model (str): Optional override for embedding model.
If not provided, uses the current embedding
model from configuration.
+ source (str): Who is searching, for the trace: ``agent_tool`` or
+ ``api`` (see ``search``).
Returns:
dict (str, Any): {"results": [chunks]} on success
"""
+ from auth_context import get_score_threshold, get_search_filters, get_search_limit
+
# Strategy: Use provided model, or default to the configured embedding
# model. This assumes documents are embedded with that model by default.
# Future enhancement: Could auto-detect available models in corpus.
@@ -287,6 +311,38 @@ class SearchService:
get_openrag_config().knowledge.embedding_provider
)
)
+ filters = get_search_filters() or {}
+ # One trace per search; a no-op while tracing is off. Filter values are
+ # filenames and owners, so only how many of each is recorded.
+ with trace_root(
+ "search",
+ user_id=get_auth_context()[0],
+ tags=[f"search_source:{source}"],
+ attributes={
+ "search.source": source,
+ "search.k": get_search_limit(),
+ "search.score_threshold": get_score_threshold(),
+ "search.query_chars": len(query) if isinstance(query, str) else None,
+ "search.wildcard": isinstance(query, str) and query.strip() == "*",
+ "search.filters": json.dumps(
+ {
+ key: len(values)
+ for key, values in filters.items()
+ if isinstance(values, list)
+ }
+ )
+ if filters
+ else None,
+ "embedding_model": embedding_model,
+ },
+ ) as search_trace:
+ search_trace.set_content(**{"search.query": query})
+ response = await self._search(query, embedding_model)
+ _record_search_response(search_trace, response)
+ return response
+
+ async def _search(self, query: str, embedding_model: str) -> dict[str, Any]:
+ """The search itself; ``search_tool`` wraps it in its trace."""
embedding_field_name = get_embedding_field_name(embedding_model)
logger.info(
@@ -476,23 +532,29 @@ class SearchService:
# one broken model (e.g. provider credentials removed after ingest)
# doesn't take down the entire search. If all models fail we fall
# back to keyword-only search below.
- embedding_results = await asyncio.gather(
- *[embed_with_space(space) for space in available_spaces],
- return_exceptions=True,
- )
+ with trace_span(
+ "search.embed",
+ spaces=len(available_spaces),
+ space_ids=[space.space_id for space in available_spaces],
+ ) as embed_span:
+ embedding_results = await asyncio.gather(
+ *[embed_with_space(space) for space in available_spaces],
+ return_exceptions=True,
+ )
- for space, result in zip(available_spaces, embedding_results, strict=False):
- if isinstance(result, BaseException):
- failed_models.append(space.space_id)
- logger.warning(
- "Skipping model with failed embedding; continuing with others",
- model=space.space_id,
- error=str(result),
- )
- continue
- if isinstance(result, tuple) and result[1] is not None:
- successful_space, embedding = result
- query_embeddings[successful_space.field_identity] = embedding
+ for space, result in zip(available_spaces, embedding_results, strict=False):
+ if isinstance(result, BaseException):
+ failed_models.append(space.space_id)
+ logger.warning(
+ "Skipping model with failed embedding; continuing with others",
+ model=space.space_id,
+ error=str(result),
+ )
+ continue
+ if isinstance(result, tuple) and result[1] is not None:
+ successful_space, embedding = result
+ query_embeddings[successful_space.field_identity] = embedding
+ embed_span.set(succeeded=len(query_embeddings), failed=len(failed_models))
logger.info(
"Generated query embeddings",
@@ -712,63 +774,68 @@ class SearchService:
search_params = {"terminate_after": 0}
- try:
- index_name = get_index_name()
- logger.info(f"Sending query to index '{index_name}'..")
- results = await opensearch_client.search(
- index=index_name, body=search_body, params=search_params
- )
- except RequestError as e:
- error_message = str(e)
- if is_disk_space_error(e):
- logger.error(
- "OpenSearch query blocked by disk space constraint",
- error=error_message,
- )
- raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
- if (
- fallback_search_body is not None
- and "unknown field [num_candidates]" in error_message.lower()
- ):
- logger.warning(
- "OpenSearch cluster does not support num_candidates; retrying without it"
+ with trace_span("search.query", knn_fields=len(query_embeddings), size=limit) as query_span:
+ try:
+ index_name = get_index_name()
+ logger.info(f"Sending query to index '{index_name}'..")
+ results = await opensearch_client.search(
+ index=index_name, body=search_body, params=search_params
)
- try:
- results = await opensearch_client.search(
- index=get_index_name(),
- body=fallback_search_body,
- params=search_params,
+ except RequestError as e:
+ error_message = str(e)
+ if is_disk_space_error(e):
+ logger.error(
+ "OpenSearch query blocked by disk space constraint",
+ error=error_message,
+ )
+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
+ if (
+ fallback_search_body is not None
+ and "unknown field [num_candidates]" in error_message.lower()
+ ):
+ logger.warning(
+ "OpenSearch cluster does not support num_candidates; retrying without it"
)
- except RequestError as retry_error:
- if is_disk_space_error(retry_error):
+ query_span.set(retried_without_num_candidates=True)
+ try:
+ results = await opensearch_client.search(
+ index=get_index_name(),
+ body=fallback_search_body,
+ params=search_params,
+ )
+ except RequestError as retry_error:
+ if is_disk_space_error(retry_error):
+ logger.error(
+ "OpenSearch retry blocked by disk space constraint",
+ error=str(retry_error),
+ )
+ raise OpenSearchDiskSpaceError(
+ DISK_SPACE_ERROR_MESSAGE
+ ) from retry_error
logger.error(
- "OpenSearch retry blocked by disk space constraint",
+ "OpenSearch retry without num_candidates failed",
error=str(retry_error),
+ search_body=fallback_search_body,
)
- raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from retry_error
+ raise
+ else:
logger.error(
- "OpenSearch retry without num_candidates failed",
- error=str(retry_error),
- search_body=fallback_search_body,
+ "OpenSearch query failed", error=error_message, search_body=search_body
)
raise
- else:
- logger.error(
- "OpenSearch query failed", error=error_message, search_body=search_body
- )
+ except OpenSearchDiskSpaceError:
raise
- except OpenSearchDiskSpaceError:
- raise
- except Exception as e:
- if is_disk_space_error(e):
- logger.error(
- "OpenSearch query blocked by disk space constraint",
- error=str(e),
- )
- raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
- logger.error("OpenSearch query failed", error=str(e), search_body=search_body)
- # Re-raise the exception so the API returns the error to frontend
- raise
+ except Exception as e:
+ if is_disk_space_error(e):
+ logger.error(
+ "OpenSearch query blocked by disk space constraint",
+ error=str(e),
+ )
+ raise OpenSearchDiskSpaceError(DISK_SPACE_ERROR_MESSAGE) from e
+ logger.error("OpenSearch query failed", error=str(e), search_body=search_body)
+ # Re-raise the exception so the API returns the error to frontend
+ raise
+ query_span.set(hits=len(results["hits"]["hits"]), took_ms=results.get("took"))
# Transform results (keep for backward compatibility)
chunks = []
@@ -877,4 +944,4 @@ class SearchService:
set_search_limit(limit)
set_score_threshold(score_threshold)
- return await self.search_tool(query, embedding_model=embedding_model)
+ return await self.search_tool(query, embedding_model=embedding_model, source="api")
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 11b361039..2741bbe8c 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -16,6 +16,7 @@ from models.tasks import (
TaskStatus,
UploadTask,
)
+from observability.tracing import trace_context
from session_manager import AnonymousUser
from utils.filename_claims import (
INFLIGHT_CLAIM_WINNER_FAILED_ERROR,
@@ -850,10 +851,15 @@ class TaskService:
try:
# Add timeout protection to prevent indefinite hangs
- await self._process_with_timeout(
- processor.process_item(upload_task, item, file_task),
- timeout_seconds=self.ingestion_timeout,
- )
+ # Traces opened for this file (and its model calls)
+ # carry the task; an upload's files share a session.
+ with trace_context(
+ session_id=task_id, attributes={"openrag.task_id": task_id}
+ ):
+ await self._process_with_timeout(
+ processor.process_item(upload_task, item, file_task),
+ timeout_seconds=self.ingestion_timeout,
+ )
logger.info(
"File processing task succeeded",
diff --git a/src/tui/config_fields.py b/src/tui/config_fields.py
index fc9139e39..b77958746 100644
--- a/src/tui/config_fields.py
+++ b/src/tui/config_fields.py
@@ -383,14 +383,29 @@ CONFIG_SECTIONS: list[ConfigSection] = [
secret=True,
),
ConfigField(
- "langfuse_host",
- "LANGFUSE_HOST",
+ "langfuse_base_url",
+ "LANGFUSE_BASE_URL",
"Host",
placeholder="https://cloud.langfuse.com",
helper_text="Leave empty for Langfuse Cloud, or set for self-hosted",
),
+ ConfigField(
+ "openrag_tracing_enabled",
+ "OPENRAG_TRACING_ENABLED",
+ "Trace Backend",
+ placeholder="false",
+ helper_text="If true, the OpenRAG backend also traces its model calls to Langfuse",
+ ),
+ ConfigField(
+ "openrag_tracing_capture_content",
+ "OPENRAG_TRACING_CAPTURE_CONTENT",
+ "Capture Content",
+ placeholder="false",
+ advanced=True,
+ helper_text="If true, backend traces include prompts, completions and document text",
+ ),
],
- gate_prompt="Configure Langfuse tracing?",
+ gate_prompt="Configure Langfuse tracing (Langflow and backend)?",
),
# ── Storage ─────────────────────────────────────────────────
ConfigSection(
diff --git a/src/tui/managers/env_manager.py b/src/tui/managers/env_manager.py
index c5f4924bb..c33608a7b 100644
--- a/src/tui/managers/env_manager.py
+++ b/src/tui/managers/env_manager.py
@@ -82,7 +82,10 @@ class EnvConfig:
# Langfuse settings (optional)
langfuse_secret_key: str = ""
langfuse_public_key: str = ""
- langfuse_host: str = ""
+ langfuse_base_url: str = ""
+ # Backend tracing to Langfuse (optional; empty = unset, backend default off)
+ openrag_tracing_enabled: str = ""
+ openrag_tracing_capture_content: str = ""
# Langflow auth settings
langflow_auto_login: str = "False"
@@ -244,7 +247,12 @@ class EnvManager:
"OPENRAG_VERSION": "openrag_version",
"LANGFUSE_SECRET_KEY": "langfuse_secret_key", # pragma: allowlist secret
"LANGFUSE_PUBLIC_KEY": "langfuse_public_key", # pragma: allowlist secret
- "LANGFUSE_HOST": "langfuse_host",
+ # Older name for LANGFUSE_BASE_URL: loaded so it's migrated on save.
+ # Listed first so LANGFUSE_BASE_URL wins when both are set.
+ "LANGFUSE_HOST": "langfuse_base_url",
+ "LANGFUSE_BASE_URL": "langfuse_base_url",
+ "OPENRAG_TRACING_ENABLED": "openrag_tracing_enabled",
+ "OPENRAG_TRACING_CAPTURE_CONTENT": "openrag_tracing_capture_content",
}
def _collect_preserved_env_lines(self) -> list[str]:
@@ -694,7 +702,12 @@ class EnvManager:
langfuse_vars = [
("LANGFUSE_SECRET_KEY", self.config.langfuse_secret_key),
("LANGFUSE_PUBLIC_KEY", self.config.langfuse_public_key),
- ("LANGFUSE_HOST", self.config.langfuse_host),
+ ("LANGFUSE_BASE_URL", self.config.langfuse_base_url),
+ ("OPENRAG_TRACING_ENABLED", self.config.openrag_tracing_enabled),
+ (
+ "OPENRAG_TRACING_CAPTURE_CONTENT",
+ self.config.openrag_tracing_capture_content,
+ ),
]
langfuse_written = False
diff --git a/tests/unit/api/test_provider_check_tracing.py b/tests/unit/api/test_provider_check_tracing.py
new file mode 100644
index 000000000..f96e7974c
--- /dev/null
+++ b/tests/unit/api/test_provider_check_tracing.py
@@ -0,0 +1,43 @@
+"""Provider checks call LiteLLM directly; their traces say so instead of looking like real traffic."""
+
+import pytest
+
+from api import provider_validation
+
+
+def tracing_on(monkeypatch):
+ from observability import tracing
+
+ monkeypatch.setattr(tracing, "_enabled", True)
+
+
+ ("litellm_call", "embedding_model", "llm_model", "kind"),
+ [
+ ("litellm.aembedding", "text-embedding-3-small", None, "embedding"),
+ ("litellm.acompletion", None, "gpt-4o-mini", "chat"),
+ ],
+)
+async def test_provider_checks_are_traced_as_provider_checks(
+ monkeypatch, tracing_on, litellm_call, embedding_model, llm_model, kind
+):
+ captured: dict = {}
+
+ async def _capture(**kwargs):
+ captured.update(kwargs)
+
+ monkeypatch.setattr(litellm_call, _capture)
+
+ await provider_validation._test_litellm_provider(
+ provider="openai",
+ credentials={"api_key": "k"},
+ runtime_kwargs={},
+ embedding_model=embedding_model,
+ llm_model=llm_model,
+ )
+
+ metadata = captured["metadata"]
+ assert metadata["generation_name"] == f"{kind}.provider_check"
+ assert metadata["trace_metadata"]["openrag.provider"] == "openai"
diff --git a/tests/unit/api/test_v1_llm.py b/tests/unit/api/test_v1_llm.py
index b2c2b4052..4e0565e7d 100644
--- a/tests/unit/api/test_v1_llm.py
+++ b/tests/unit/api/test_v1_llm.py
@@ -253,3 +253,105 @@ async def test_gateway_error_detail_is_not_returned_to_the_caller(monkeypatch):
assert "10.0.0.5" not in body
assert "RuntimeError" not in body
assert json.loads(body)["error"]["type"] == "api_error"
+
+
+# ---------------------------------------------------------------------------
+# Tracing: who a /v1 call is for
+# ---------------------------------------------------------------------------
+
+
+def tracing_on(monkeypatch):
+ from observability import tracing
+
+ monkeypatch.setattr(tracing, "_enabled", True)
+
+
+def _metadata_seen_by_gateway(kind: str):
+ """A fake gateway call that records the trace metadata it would send."""
+ from observability.tracing import gateway_trace_kwargs
+
+ seen = {}
+
+ async def fake(*args, **kwargs):
+ seen.update(gateway_trace_kwargs(kind, "ingest")["metadata"])
+ if kind == "chat":
+ return {"choices": [{"message": {"role": "assistant", "content": "ok"}}]}
+ return {"object": "list", "data": [{"embedding": [0.2]}]}
+
+ return fake, seen
+
+
+async def test_langflow_embedding_calls_are_traced_to_the_user_and_run(monkeypatch, tracing_on):
+ fake, seen = _metadata_seen_by_gateway("embedding")
+ monkeypatch.setattr(v1_llm, "embeddings", fake)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
+ request.state = SimpleNamespace(llm_hop_purpose="ingest", llm_hop_run_id="run-1")
+
+ await v1_llm.embeddings_endpoint(request, user=_user())
+
+ assert seen["trace_metadata"]["openrag.source"] == "langflow"
+ assert seen["trace_metadata"]["openrag.langflow_run_purpose"] == "ingest"
+ assert seen["trace_user_id"] == "u1"
+ assert seen["session_id"] == "run-1"
+
+
+async def test_langflow_chat_calls_are_traced_to_the_user_and_run(monkeypatch, tracing_on):
+ fake, seen = _metadata_seen_by_gateway("chat")
+ monkeypatch.setattr(v1_llm, "chat_completions", fake)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "gpt-4o-mini", "messages": []})
+ request.state = SimpleNamespace(llm_hop_purpose="chat", llm_hop_run_id="run-2")
+
+ await v1_llm.chat_completions_endpoint(request, user=_user())
+
+ assert seen["trace_metadata"]["openrag.source"] == "langflow"
+ assert seen["trace_user_id"] == "u1"
+ assert seen["session_id"] == "run-2"
+
+
+async def test_api_key_calls_are_traced_as_api_without_a_session(monkeypatch, tracing_on):
+ fake, seen = _metadata_seen_by_gateway("embedding")
+ monkeypatch.setattr(v1_llm, "embeddings", fake)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
+ request.state = SimpleNamespace() # API key / JWT: not a Langflow run
+
+ await v1_llm.embeddings_endpoint(request, user=_user())
+
+ assert seen["trace_metadata"]["openrag.source"] == "api"
+ assert seen["trace_user_id"] == "u1"
+ assert "session_id" not in seen
+ assert "openrag.langflow_run_purpose" not in seen["trace_metadata"]
+
+
+async def test_api_key_embedding_calls_get_their_own_purpose(monkeypatch, tracing_on):
+ """Not every non-interactive embedding is ingestion: an API caller's is labelled as such."""
+ fake, seen = _metadata_seen_by_gateway("embedding")
+ monkeypatch.setattr(v1_llm, "embeddings", fake)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
+ request.state = SimpleNamespace()
+
+ await v1_llm.embeddings_endpoint(request, user=_user())
+
+ assert seen["generation_name"] == "embedding.api"
+ assert seen["trace_metadata"]["openrag.purpose"] == "api"
+
+
+async def test_langflow_ingest_embedding_calls_stay_ingest(monkeypatch, tracing_on):
+ fake, seen = _metadata_seen_by_gateway("embedding")
+ monkeypatch.setattr(v1_llm, "embeddings", fake)
+ request = MagicMock()
+ request.json = AsyncMock(return_value={"model": "text-embedding-3-small", "input": ["x"]})
+ request.state = SimpleNamespace(llm_hop_purpose="ingest", llm_hop_run_id="run-1")
+
+ await v1_llm.embeddings_endpoint(request, user=_user())
+
+ assert seen["generation_name"] == "embedding.ingest"
diff --git a/tests/unit/api/test_v1_search_trace_context.py b/tests/unit/api/test_v1_search_trace_context.py
new file mode 100644
index 000000000..058ff9e80
--- /dev/null
+++ b/tests/unit/api/test_v1_search_trace_context.py
@@ -0,0 +1,52 @@
+"""/v1/search tells the search trace which saved knowledge filter it applied."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from api.v1 import search as v1_search
+from observability.tracing import current_trace_context
+
+
+def _service(seen: list):
+ async def search(query, **_kwargs):
+ seen.append(dict(current_trace_context()))
+ return {"results": []}
+
+ return SimpleNamespace(search=search)
+
+
+async def test_the_applied_filter_id_is_in_the_search_trace_context(monkeypatch):
+ seen: list = []
+ monkeypatch.setattr(
+ v1_search,
+ "resolve_filter_id",
+ AsyncMock(return_value={"filters": {}, "limit": 10, "score_threshold": 0}),
+ )
+ user = SimpleNamespace(user_id="u1", jwt_token=None)
+
+ await v1_search.search_endpoint(
+ v1_search.SearchV1Body(query="q", filter_id="filter-1"),
+ search_service=_service(seen),
+ user=user,
+ knowledge_filter_service=None,
+ )
+
+ assert seen[0]["attributes"] == {"openrag.search.filter_id": "filter-1"}
+
+
+async def test_no_filter_id_adds_nothing():
+ seen: list = []
+ user = SimpleNamespace(user_id="u1", jwt_token=None)
+
+ await v1_search.search_endpoint(
+ v1_search.SearchV1Body(query="q"),
+ search_service=_service(seen),
+ user=user,
+ knowledge_filter_service=None,
+ )
+
+ assert "attributes" not in seen[0]
diff --git a/tests/unit/dependencies/test_llm_proxy_user_purpose.py b/tests/unit/dependencies/test_llm_proxy_user_purpose.py
index dda507a04..7246989b1 100644
--- a/tests/unit/dependencies/test_llm_proxy_user_purpose.py
+++ b/tests/unit/dependencies/test_llm_proxy_user_purpose.py
@@ -49,3 +49,19 @@ async def test_a_non_hop_token_records_no_purpose(monkeypatch):
assert user is api_user
assert not hasattr(request.state, "llm_hop_purpose")
+
+
+async def test_a_hop_token_records_its_run_id():
+ """Each Langflow run mints its own hop token; its id groups the run's calls in traces."""
+ service = LangflowLlmTokenService(secret=SECRET, ttl_seconds=60)
+ first = _request(service.create_token(user_id="alice", purpose=HOP_PURPOSE_CHAT))
+ second = _request(service.create_token(user_id="alice", purpose=HOP_PURPOSE_CHAT))
+
+ for request in (first, second):
+ await dependencies.get_llm_proxy_user(
+ request, token_service=service, api_key_service=None, session_manager=None
+ )
+
+ assert first.state.llm_hop_run_id
+ assert first.state.llm_hop_run_id != second.state.llm_hop_run_id
diff --git a/tests/unit/observability/conftest.py b/tests/unit/observability/conftest.py
new file mode 100644
index 000000000..99916f36b
--- /dev/null
+++ b/tests/unit/observability/conftest.py
@@ -0,0 +1,44 @@
+"""A real Langfuse SDK client for our spans, exporting to memory."""
+
+import pytest
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from observability import spans, tracing
+
+
[email protected](scope="session")
+def _span_client_and_exporter():
+ exporter = InMemorySpanExporter()
+ patch = pytest.MonkeyPatch()
+ # The SDK caches one client per public key; this key is only used here.
+ patch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-openrag-spans-test")
+ patch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-openrag-spans-test")
+ patch.setenv("LANGFUSE_HOST", "http://127.0.0.1:9")
+ try:
+ spans.init_span_client(span_exporter=exporter)
+ client = spans._client
+ finally:
+ patch.undo()
+ spans._client = None
+ yield client, exporter
+ client.shutdown()
+
+
+def finished(_span_client_and_exporter, monkeypatch):
+ """Turn tracing on; returns a function giving the exported spans by name.
+
+ Content capture is off unless a test patches ``spans.content_capture_enabled``.
+ """
+ client, exporter = _span_client_and_exporter
+ exporter.clear()
+ monkeypatch.setattr(spans, "_client", client)
+ monkeypatch.setattr(tracing, "_enabled", True)
+ monkeypatch.delenv("OPENRAG_RUN_MODE", raising=False)
+ monkeypatch.setattr(spans, "content_capture_enabled", lambda: False)
+
+ def _finished():
+ client.flush()
+ return {span.name: span for span in exporter.get_finished_spans()}
+
+ return _finished
diff --git a/tests/unit/observability/test_gateway_trace_metadata.py b/tests/unit/observability/test_gateway_trace_metadata.py
new file mode 100644
index 000000000..1ef0ee6de
--- /dev/null
+++ b/tests/unit/observability/test_gateway_trace_metadata.py
@@ -0,0 +1,100 @@
+"""What each gateway call tells Langfuse about itself, beyond LiteLLM's defaults.
+
+LiteLLM's `langfuse_otel` callback reads the call's `metadata` kwarg:
+`generation_name` names the observation, `trace_user_id` / `session_id` set
+Langfuse's user and session, and `trace_metadata` keys become filterable
+metadata (other keys are dropped). Verified against Langfuse Cloud.
+"""
+
+import pytest
+
+from observability import tracing
+from observability.tracing import gateway_trace_kwargs, trace_context
+
+
+def enabled(monkeypatch):
+ monkeypatch.setattr(tracing, "_enabled", True)
+ monkeypatch.delenv("OPENRAG_RUN_MODE", raising=False)
+
+
+def test_nothing_is_added_when_tracing_is_off(monkeypatch):
+ monkeypatch.setattr(tracing, "_enabled", False)
+
+ assert gateway_trace_kwargs("embedding", "ingest", inputs=3) == {}
+
+
+def test_names_the_observation_after_kind_and_purpose(enabled):
+ metadata = gateway_trace_kwargs("embedding", "query")["metadata"]
+
+ assert metadata["generation_name"] == "embedding.query"
+ assert metadata["trace_metadata"]["openrag.purpose"] == "query"
+
+
+def test_backend_calls_are_attributed_to_openrag_by_default(enabled):
+ metadata = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+
+ assert metadata["trace_metadata"]["openrag.source"] == "openrag"
+ assert metadata["trace_metadata"]["openrag.run_mode"] == "oss"
+ assert set(metadata["tags"]) == {"source:openrag", "purpose:ingest"}
+ assert "trace_user_id" not in metadata
+ assert "session_id" not in metadata
+
+
+def test_sizes_are_recorded_as_openrag_attributes(enabled):
+ metadata = gateway_trace_kwargs("embedding", "ingest", inputs=3, input_chars=120)["metadata"]
+
+ assert metadata["trace_metadata"]["openrag.embedding.inputs"] == 3
+ assert metadata["trace_metadata"]["openrag.embedding.input_chars"] == 120
+
+
+def test_none_sizes_are_left_out(enabled):
+ metadata = gateway_trace_kwargs("chat", "completion", tools=None)["metadata"]
+
+ assert "openrag.chat.tools" not in metadata["trace_metadata"]
+
+
+def test_call_context_sets_source_user_and_session(enabled):
+ with trace_context(source="langflow", user_id="user-1", session_id="run-1"):
+ metadata = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+
+ assert metadata["trace_metadata"]["openrag.source"] == "langflow"
+ assert metadata["trace_user_id"] == "user-1"
+ assert metadata["session_id"] == "run-1"
+ assert "source:langflow" in metadata["tags"]
+
+
+def test_call_context_purpose_overrides_the_callers_default(enabled):
+ with trace_context(purpose="dimension_probe"):
+ metadata = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+
+ assert metadata["generation_name"] == "embedding.dimension_probe"
+
+
+def test_call_context_attributes_are_added(enabled):
+ with trace_context(attributes={"openrag.langflow_run_purpose": "chat"}):
+ metadata = gateway_trace_kwargs("chat", "completion")["metadata"]
+
+ assert metadata["trace_metadata"]["openrag.langflow_run_purpose"] == "chat"
+
+
+def test_call_context_nests_and_restores(enabled):
+ with trace_context(source="langflow", user_id="user-1"):
+ with trace_context(purpose="query"):
+ inner = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+ outer = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+ after = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+
+ assert inner["generation_name"] == "embedding.query"
+ assert inner["trace_user_id"] == "user-1"
+ assert outer["generation_name"] == "embedding.ingest"
+ assert after["trace_metadata"]["openrag.source"] == "openrag"
+ assert "trace_user_id" not in after
+
+
+def test_empty_user_and_session_are_left_out(enabled):
+ with trace_context(user_id="", session_id=None):
+ metadata = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+
+ assert "trace_user_id" not in metadata
+ assert "session_id" not in metadata
diff --git a/tests/unit/observability/test_ingest_trace.py b/tests/unit/observability/test_ingest_trace.py
new file mode 100644
index 000000000..97b183cab
--- /dev/null
+++ b/tests/unit/observability/test_ingest_trace.py
@@ -0,0 +1,354 @@
+"""One Langfuse trace per file on the non-Langflow ingest path (issue 03).
+
+`process_document_standard` runs against fakes for Docling, the gateway and
+OpenSearch; the spans go through the real Langfuse SDK into memory.
+"""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from langfuse._client.attributes import LangfuseOtelSpanAttributes as LF
+from opentelemetry import trace
+
+from observability import spans
+from observability.tracing import trace_context
+
+STAGES = {
+ "ingest.parse",
+ "ingest.extract",
+ "ingest.resplit",
+ "ingest.token_cap",
+ "ingest.embed",
+ "ingest.index",
+}
+
+
+def _metadata(span, key):
+ return span.attributes.get(f"{LF.OBSERVATION_METADATA}.openrag.{key}")
+
+
+def _trace_metadata(span, key):
+ return span.attributes.get(f"{LF.TRACE_METADATA}.openrag.{key}")
+
+
+DOCLING_DOC = {
+ "origin": {"filename": "report.pdf", "mimetype": "application/pdf"},
+ "pages": {"1": {}, "2": {}},
+ "texts": [
+ {"text": "Intro paragraph", "prov": [{"page_no": 1}]},
+ {"text": "Page two text", "prov": [{"page_no": 2}]},
+ {"text": "More on two", "prov": [{"page_no": 2}]},
+ ],
+ "tables": [
+ {
+ "prov": [{"page_no": 2}],
+ "data": {
+ "table_cells": [
+ {"start_row_offset_idx": 0, "start_col_offset_idx": 0, "text": "a"},
+ ]
+ },
+ }
+ ],
+ "pictures": [{"prov": [{"page_no": 1}], "annotations": []}],
+}
+
+
+class _Pipeline:
+ """`process_document_standard` with every external dependency faked."""
+
+ def __init__(self, monkeypatch, *, exists=False, stale_ids=("doc-1_0", "doc-1_1")):
+ from models import processors as processors_mod
+ from models.processors import TaskProcessor
+ from services import llm_gateway as gateway_mod
+ from utils import opensearch_delete
+
+ self.opensearch = AsyncMock()
+ self.opensearch.exists = AsyncMock(return_value=exists)
+ self.check_exists = exists
+
+ document_service = MagicMock()
+ document_service.session_manager.get_user_opensearch_client = MagicMock(
+ return_value=self.opensearch
+ )
+ self.indexed: list = []
+
+ class _Writer:
+ async def index_chunks(writer_self, context, chunks, *, final=False):
+ self.indexed.extend(chunks)
+
+ document_service.document_index_writer = _Writer()
+
+ models_service = MagicMock()
+ models_service.get_litellm_model_name = AsyncMock(return_value="text-embedding-3-small")
+
+ self.docling = MagicMock()
+ self.docling.convert_file = AsyncMock(return_value=DOCLING_DOC)
+
+ self.processor = TaskProcessor(
+ document_service=document_service,
+ models_service=models_service,
+ docling_service=self.docling,
+ )
+ self.processor.check_document_exists = AsyncMock(return_value=exists)
+
+ config = MagicMock()
+ config.knowledge.embedding_model = "text-embedding-3-small"
+ config.knowledge.embedding_provider = "openai"
+ config.knowledge.chunk_size = None
+ config.knowledge.chunk_overlap = None
+ config.knowledge.ocr = True
+ config.knowledge.picture_descriptions = False
+ config.knowledge.table_structure = True
+ monkeypatch.setattr(processors_mod, "get_openrag_config", lambda: config)
+ monkeypatch.setattr(processors_mod, "get_index_name", lambda: "test-index")
+ fake_clients = MagicMock()
+ fake_clients.opensearch = AsyncMock()
+ monkeypatch.setattr(processors_mod, "clients", fake_clients)
+
+ async def _collect(*_args, **_kwargs):
+ return list(stale_ids)
+
+ async def _delete(*_args, **_kwargs):
+ return None
+
+ monkeypatch.setattr(opensearch_delete, "collect_visible_document_ids", _collect)
+ monkeypatch.setattr(opensearch_delete, "delete_document_ids", _delete)
+
+ self.embed_parents: list[str] = []
+ self.embed_error: Exception | None = None
+
+ async def _embeddings(body, **_kwargs):
+ self.embed_parents.append(getattr(trace.get_current_span(), "name", None))
+ if self.embed_error is not None:
+ raise self.embed_error
+ return {
+ "data": [{"embedding": [0.1, 0.2, 0.3]} for _ in body["input"]],
+ "usage": {"prompt_tokens": 5 * len(body["input"]), "total_tokens": 0},
+ }
+
+ monkeypatch.setattr(gateway_mod, "embeddings", _embeddings)
+
+ async def run(self, path: str, **kwargs):
+ defaults = {
+ "file_hash": "doc-1",
+ "owner_user_id": "user-1",
+ "connector_type": "local",
+ "file_size": 1234,
+ }
+ return await self.processor.process_document_standard(
+ file_path=path, **{**defaults, **kwargs}
+ )
+
+
+def text_file(tmp_path):
+ path = tmp_path / "notes.txt"
+ path.write_text("First paragraph.\n\nSecond paragraph.")
+ return str(path)
+
+
+def pdf_file(tmp_path):
+ path = tmp_path / "report.pdf"
+ path.write_bytes(b"%PDF-fake")
+ return str(path)
+
+
+# --- one trace per file ----------------------------------------------------
+
+
+async def test_an_indexed_text_file_is_one_trace_with_its_stages(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ with trace_context(session_id="task-1", attributes={"openrag.task_id": "task-1"}):
+ result = await pipeline.run(text_file)
+
+ assert result["status"] == "indexed"
+ exported = finished()
+ root = exported["ingest.file"]
+ assert root.parent is None
+ assert set(exported) == {
+ "ingest.file",
+ "ingest.parse",
+ "ingest.token_cap",
+ "ingest.embed",
+ "ingest.index",
+ }
+ for name in set(exported) - {"ingest.file"}:
+ assert exported[name].context.trace_id == root.context.trace_id
+
+ assert root.attributes[LF.TRACE_USER_ID] == "user-1"
+ assert root.attributes[LF.TRACE_SESSION_ID] == "task-1"
+ assert "ingest_path:openrag" in root.attributes[LF.TRACE_TAGS]
+ assert _trace_metadata(root, "task_id") == "task-1"
+ assert _trace_metadata(root, "document_id") == "doc-1"
+ assert _trace_metadata(root, "ingest_path") == "openrag"
+ assert _trace_metadata(root, "connector_type") == "local"
+ assert _trace_metadata(root, "embedding_model") == "text-embedding-3-small"
+ assert _trace_metadata(root, "embedding_provider") == "openai"
+ assert _trace_metadata(root, "run_mode") == "oss"
+ assert _trace_metadata(root, "file.extension") == ".txt"
+ assert _metadata(root, "status") == "indexed"
+ assert _metadata(root, "chunks") == 1
+
+
+async def test_stage_counts_for_a_text_file(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ await pipeline.run(text_file)
+
+ exported = finished()
+ parse = exported["ingest.parse"]
+ assert _metadata(parse, "parser") == "Text Parser"
+ assert _metadata(parse, "chunks") == 1
+
+ cap = exported["ingest.token_cap"]
+ assert _metadata(cap, "max_tokens") == 8000
+ assert _metadata(cap, "chunks_in") == 1
+ assert _metadata(cap, "chunks_out") == 1
+
+ embed = exported["ingest.embed"]
+ assert _metadata(embed, "batches") == 1
+ assert _metadata(embed, "inputs") == 1
+ assert _metadata(embed, "vectors") == 1
+ assert _metadata(embed, "dimensions") == 3
+ assert _metadata(embed, "prompt_tokens") == 5
+
+ index = exported["ingest.index"]
+ assert _metadata(index, "stale_chunks_deleted") == 2
+ assert _metadata(index, "chunks_indexed") == 1
+
+
+async def test_embedding_calls_nest_under_the_embed_stage(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ await pipeline.run(text_file)
+
+ assert pipeline.embed_parents == ["ingest.embed"]
+
+
+async def test_docling_parse_and_extract_counts(monkeypatch, finished, pdf_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ await pipeline.run(pdf_file, ocr=None, picture_descriptions=True)
+
+ exported = finished()
+ parse = exported["ingest.parse"]
+ assert _metadata(parse, "parser").startswith("Docling")
+ assert _metadata(parse, "pages") == 2
+ assert _metadata(parse, "texts") == 3
+ assert _metadata(parse, "tables") == 1
+ assert _metadata(parse, "pictures") == 1
+ # Unset per-request options fall back to the knowledge config, as Docling does.
+ assert _metadata(parse, "ocr") is True
+ assert _metadata(parse, "picture_descriptions") is True
+ assert _metadata(parse, "table_structure") is True
+
+ extract = exported["ingest.extract"]
+ assert _metadata(extract, "chunks") == 3
+ assert _metadata(extract, "chunks_text") == 2
+ assert _metadata(extract, "chunks_table") == 1
+ assert _metadata(extract, "chunks_picture") is None
+
+
+async def test_resplit_records_its_window_and_counts(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ await pipeline.run(text_file, chunk_size=10, chunk_overlap=2)
+
+ resplit = finished()["ingest.resplit"]
+ assert _metadata(resplit, "chunk_size") == 10
+ assert _metadata(resplit, "chunk_overlap") == 2
+ assert _metadata(resplit, "chunks_in") == 1
+ assert _metadata(resplit, "chunks_out") == 5
+
+
+# --- short and failed traces ------------------------------------------------
+
+
+async def test_an_unchanged_file_is_a_short_trace_without_error(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch, exists=True)
+
+ result = await pipeline.run(text_file)
+
+ assert result == {"status": "unchanged", "id": "doc-1"}
+ exported = finished()
+ assert set(exported) == {"ingest.file"}
+ root = exported["ingest.file"]
+ assert _metadata(root, "status") == "unchanged"
+ assert root.attributes.get(LF.OBSERVATION_LEVEL) != "ERROR"
+
+
+async def test_an_embedding_failure_marks_the_stage_and_the_trace(monkeypatch, finished, text_file):
+ from services.llm_gateway import LlmGatewayError
+
+ pipeline = _Pipeline(monkeypatch)
+ pipeline.embed_error = LlmGatewayError("upstream said no", 502)
+
+ with pytest.raises(LlmGatewayError):
+ await pipeline.run(text_file)
+
+ exported = finished()
+ assert exported["ingest.embed"].attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ root = exported["ingest.file"]
+ assert root.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "LlmGatewayError in ingest.embed"
+ assert "ingest.index" not in exported
+
+
+async def test_a_document_without_text_marks_the_trace_failed(monkeypatch, finished, tmp_path):
+ pipeline = _Pipeline(monkeypatch)
+ empty = tmp_path / "empty.txt"
+ empty.write_text(" ")
+
+ result = await pipeline.run(str(empty))
+
+ assert result["status"] == "error"
+ root = finished()["ingest.file"]
+ assert root.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "NoEmbeddings"
+
+
+# --- content policy ---------------------------------------------------------
+
+
+async def test_filename_is_left_out_unless_capture_is_on(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ await pipeline.run(text_file, original_filename="q3-salaries.txt")
+ for span in finished().values():
+ assert "q3-salaries" not in str(dict(span.attributes))
+
+ monkeypatch.setattr(spans, "content_capture_enabled", lambda: True)
+ await pipeline.run(text_file, original_filename="q3-salaries.txt")
+ assert _metadata(finished()["ingest.file"], "filename") == "q3-salaries.txt"
+
+
+async def test_chunk_text_never_reaches_a_span(monkeypatch, finished, text_file):
+ pipeline = _Pipeline(monkeypatch)
+
+ await pipeline.run(text_file)
+
+ for span in finished().values():
+ assert "First paragraph" not in str(dict(span.attributes))
+
+
+async def test_untraced_ingest_behaves_the_same(monkeypatch, text_file):
+ monkeypatch.setattr(spans, "_client", None)
+ pipeline = _Pipeline(monkeypatch)
+
+ result = await pipeline.run(text_file)
+
+ assert result == {"status": "indexed", "id": "doc-1"}
+ assert len(pipeline.indexed) == 1
diff --git a/tests/unit/observability/test_search_trace.py b/tests/unit/observability/test_search_trace.py
new file mode 100644
index 000000000..5d6ab8583
--- /dev/null
+++ b/tests/unit/observability/test_search_trace.py
@@ -0,0 +1,230 @@
+"""One Langfuse trace per search the backend serves (issue 05).
+
+Covers the search API (UI and /v1) and the agent `search_tool`. Retrieval in
+Langflow's agent flow runs its own OpenSearch query; the backend only sees its
+query embedding, traced by the gateway as `embedding.query`.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+from langfuse._client.attributes import LangfuseOtelSpanAttributes as LF
+from opentelemetry import trace
+
+from auth_context import set_auth_context, set_score_threshold, set_search_filters, set_search_limit
+from observability import spans
+from observability.tracing import trace_context
+from services.search_service import SearchService
+
+
+def _metadata(span, key):
+ return span.attributes.get(f"{LF.OBSERVATION_METADATA}.openrag.{key}")
+
+
+def _trace_metadata(span, key):
+ return span.attributes.get(f"{LF.TRACE_METADATA}.openrag.{key}")
+
+
+HITS = [
+ {
+ "_id": "doc-a_0",
+ "_score": 0.91,
+ "_source": {"filename": "a.pdf", "text": "alpha secret text"},
+ },
+ {"_id": "doc-a_3", "_score": 0.72, "_source": {"filename": "a.pdf", "text": "more alpha"}},
+ {"_id": "doc-b_1", "_score": 0.5, "_source": {"filename": "b.pdf", "text": "beta"}},
+]
+
+
+class _OpenSearch:
+ def __init__(self, error: Exception | None = None):
+ self.error = error
+
+ async def search(self, *, index, body, params):
+ if body.get("size") == 0 and "embedding_spaces" in body.get("aggs", {}):
+ return {
+ "aggregations": {
+ "embedding_spaces": {
+ "buckets": [{"key": {"space_id": "openai:text-embedding-3-small"}}]
+ }
+ }
+ }
+ if body.get("size") == 0:
+ return {"aggregations": {}}
+ if self.error is not None:
+ raise self.error
+ return {"took": 7, "hits": {"hits": HITS}, "aggregations": {}}
+
+
+def service(monkeypatch):
+ opensearch = _OpenSearch()
+ svc = object.__new__(SearchService)
+ svc.session_manager = SimpleNamespace(get_user_opensearch_client=lambda *_: opensearch)
+ svc.models_service = None
+ svc.opensearch = opensearch
+ svc.embed_parents = []
+ svc.embed_error = None
+
+ async def create_embedding(body, *, interactive=False):
+ svc.embed_parents.append(getattr(trace.get_current_span(), "name", None))
+ if svc.embed_error is not None:
+ raise svc.embed_error
+ return {"data": [{"embedding": [0.1, 0.2]}]}
+
+ monkeypatch.setattr("services.search_service.gateway_embeddings", create_embedding)
+ monkeypatch.setattr("services.search_service.get_index_name", lambda: "documents")
+ monkeypatch.setattr("services.search_service.get_embedding_model", lambda: None)
+ monkeypatch.setattr(
+ "services.search_service.get_openrag_config",
+ lambda: SimpleNamespace(
+ knowledge=SimpleNamespace(
+ embedding_provider="openai", legacy_embedding_provider_map={}
+ ),
+ providers=SimpleNamespace(ollama=SimpleNamespace(endpoint=None)),
+ ),
+ )
+ set_auth_context("user-1", "jwt")
+ set_search_filters({})
+ set_search_limit(10)
+ set_score_threshold(0)
+ return svc
+
+
+async def _api_search(service, query="gamma", **kwargs):
+ return await service.search(
+ query,
+ user_id="user-1",
+ jwt_token="jwt",
+ filters=kwargs.pop("filters", {}),
+ limit=kwargs.pop("limit", 5),
+ score_threshold=kwargs.pop("score_threshold", 0.2),
+ embedding_model="text-embedding-3-small",
+ )
+
+
+async def test_an_api_search_is_one_trace_with_embed_and_query_stages(service, finished):
+ response = await _api_search(
+ service, filters={"data_sources": ["a.pdf", "b.pdf"], "owners": ["u"]}
+ )
+
+ assert response["total"] == 3
+ exported = finished()
+ root = exported["search"]
+ assert root.parent is None
+ assert exported["search.embed"].parent.span_id == root.context.span_id
+ assert exported["search.query"].parent.span_id == root.context.span_id
+
+ assert root.attributes[LF.TRACE_USER_ID] == "user-1"
+ assert "search_source:api" in root.attributes[LF.TRACE_TAGS]
+ assert _trace_metadata(root, "search.source") == "api"
+ assert _trace_metadata(root, "search.k") == "5"
+ assert _trace_metadata(root, "search.score_threshold") == "0.2"
+ assert _trace_metadata(root, "search.query_chars") == "5"
+ assert _trace_metadata(root, "search.filters") == '{"data_sources": 2, "owners": 1}'
+ assert _trace_metadata(root, "embedding_model") == "text-embedding-3-small"
+
+ assert _metadata(root, "search.hit_count") == 3
+ assert _metadata(root, "search.top_scores") == "[0.91, 0.72, 0.5]"
+ assert _metadata(root, "search.chunk_ids") == '["doc-a_0", "doc-a_3", "doc-b_1"]'
+
+
+async def test_stage_spans_record_spaces_and_opensearch_results(service, finished):
+ await _api_search(service)
+
+ exported = finished()
+ embed = exported["search.embed"]
+ assert _metadata(embed, "spaces") == 1
+ assert _metadata(embed, "succeeded") == 1
+ assert _metadata(embed, "failed") == 0
+ assert _metadata(embed, "space_ids") == '["openai:text-embedding-3-small"]'
+
+ query = exported["search.query"]
+ assert _metadata(query, "knn_fields") == 1
+ assert _metadata(query, "hits") == 3
+ assert _metadata(query, "took_ms") == 7
+
+
+async def test_query_embeddings_nest_under_the_embed_stage(service, finished):
+ await _api_search(service)
+
+ assert service.embed_parents == ["search.embed"]
+
+
+async def test_the_agent_tool_is_traced_as_agent_tool(service, finished):
+ await service.search_tool("gamma", embedding_model="text-embedding-3-small")
+
+ root = finished()["search"]
+ assert _trace_metadata(root, "search.source") == "agent_tool"
+
+
+async def test_a_knowledge_filter_from_the_context_is_recorded(service, finished):
+ with trace_context(attributes={"openrag.search.filter_id": "filter-1"}):
+ await _api_search(service)
+
+ assert _trace_metadata(finished()["search"], "search.filter_id") == "filter-1"
+
+
+async def test_query_and_chunk_text_stay_out_unless_capture_is_on(service, finished, monkeypatch):
+ await _api_search(service, query="alpha secret")
+ for span in finished().values():
+ attributes = str(dict(span.attributes))
+ assert "alpha secret" not in attributes
+ assert "a.pdf" not in attributes
+
+ monkeypatch.setattr(spans, "content_capture_enabled", lambda: True)
+ await _api_search(service, query="alpha secret")
+ assert _metadata(finished()["search"], "search.query") == "alpha secret"
+
+
+async def test_a_failed_query_embedding_degrades_without_failing(service, finished):
+ from services.llm_gateway import LlmGatewayError
+
+ service.embed_error = LlmGatewayError("provider removed", 400)
+
+ response = await _api_search(service)
+
+ assert response["warnings"][0]["code"] == "embedding_unavailable"
+ exported = finished()
+ assert _metadata(exported["search.embed"], "failed") == 1
+ root = exported["search"]
+ assert _metadata(root, "search.degraded") is True
+ assert root.attributes.get(LF.OBSERVATION_LEVEL) != "ERROR"
+
+
+async def test_an_opensearch_failure_marks_the_query_stage_and_the_trace(service, finished):
+ service.opensearch.error = RuntimeError("cluster unavailable")
+
+ with pytest.raises(RuntimeError):
+ await _api_search(service)
+
+ exported = finished()
+ assert exported["search.query"].attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ root = exported["search"]
+ assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "RuntimeError in search.query"
+
+
+async def test_a_wildcard_listing_has_no_embed_stage(service, finished):
+ await _api_search(service, query="*")
+
+ exported = finished()
+ assert "search.embed" not in exported
+ assert _trace_metadata(exported["search"], "search.wildcard") == "True"
+
+
+async def test_untraced_search_returns_the_same(service, monkeypatch):
+ monkeypatch.setattr(spans, "_client", None)
+
+ response = await _api_search(service)
+
+ assert [chunk["chunk_id"] for chunk in response["results"]] == ["doc-a_0", "doc-a_3", "doc-b_1"]
diff --git a/tests/unit/observability/test_spans.py b/tests/unit/observability/test_spans.py
new file mode 100644
index 000000000..2588e896e
--- /dev/null
+++ b/tests/unit/observability/test_spans.py
@@ -0,0 +1,214 @@
+"""OpenRAG's own spans (ingest, search) as Langfuse observations.
+
+Spans go through the real Langfuse SDK client into an in-memory exporter, so
+these assert the OTel attributes Langfuse ingests.
+"""
+
+import pytest
+from langfuse._client.attributes import LangfuseOtelSpanAttributes as LF
+from opentelemetry import context as otel_context
+from opentelemetry import trace
+
+from observability import spans
+from observability.spans import trace_root, trace_span
+from observability.tracing import gateway_trace_kwargs, trace_context
+
+
+def _metadata(span, key):
+ return span.attributes.get(f"{LF.OBSERVATION_METADATA}.{key}")
+
+
+def _trace_metadata(span, key):
+ return span.attributes.get(f"{LF.TRACE_METADATA}.{key}")
+
+
+# --- off -------------------------------------------------------------------
+
+
+def test_off_by_default_everything_is_a_no_op(monkeypatch):
+ monkeypatch.setattr(spans, "_client", None)
+
+ with trace_root("ingest.file", attributes={"openrag.document_id": "d"}) as root:
+ with trace_span("ingest.parse") as parse:
+ parse.set(chunks=3)
+ parse.set_content(filename="secret.pdf")
+ parse.fail("ValueError")
+ root.set(status="indexed")
+
+
+def test_off_exceptions_still_propagate(monkeypatch):
+ monkeypatch.setattr(spans, "_client", None)
+
+ with pytest.raises(ValueError), trace_root("ingest.file", attributes={}):
+ raise ValueError("boom")
+
+
+# --- shape -----------------------------------------------------------------
+
+
+def test_root_and_stage_spans_nest(finished):
+ with trace_root("ingest.file", attributes={}), trace_span("ingest.parse"):
+ pass
+
+ exported = finished()
+ root, parse = exported["ingest.file"], exported["ingest.parse"]
+ assert root.parent is None
+ assert parse.parent.span_id == root.context.span_id
+ assert parse.context.trace_id == root.context.trace_id
+
+
+def test_a_root_starts_its_own_trace_inside_another_span(finished):
+ """Background ingest inherits the request's (e.g. Instana's) span; the file gets its own trace."""
+ foreign = trace.NonRecordingSpan(
+ trace.SpanContext(
+ trace_id=0x1234,
+ span_id=0x5678,
+ is_remote=False,
+ trace_flags=trace.TraceFlags(trace.TraceFlags.SAMPLED),
+ )
+ )
+ token = otel_context.attach(trace.set_span_in_context(foreign))
+ try:
+ with trace_root("ingest.file", attributes={}):
+ pass
+ assert trace.get_current_span() is foreign
+ finally:
+ otel_context.detach(token)
+
+ root = finished()["ingest.file"]
+ assert root.parent is None
+ assert root.context.trace_id != 0x1234
+
+
+def test_root_sets_trace_name_user_session_tags_and_metadata(finished):
+ with trace_root(
+ "ingest.file",
+ user_id="user-1",
+ session_id="task-1",
+ tags=["ingest_path:openrag"],
+ attributes={"openrag.document_id": "doc-1", "openrag.file.size": 42, "skip": None},
+ ):
+ pass
+
+ root = finished()["ingest.file"]
+ assert root.attributes[LF.TRACE_NAME] == "ingest.file"
+ assert root.attributes[LF.TRACE_USER_ID] == "user-1"
+ assert root.attributes[LF.TRACE_SESSION_ID] == "task-1"
+ assert list(root.attributes[LF.TRACE_TAGS]) == ["ingest_path:openrag"]
+ assert _trace_metadata(root, "openrag.document_id") == "doc-1"
+ assert _trace_metadata(root, "openrag.file.size") == "42"
+ assert _trace_metadata(root, "openrag.run_mode") == "oss"
+ assert _trace_metadata(root, "skip") is None
+
+
+def test_root_takes_user_session_and_attributes_from_the_trace_context(finished):
+ with (
+ trace_context(
+ user_id="user-2", session_id="task-2", attributes={"openrag.task_id": "task-2"}
+ ),
+ trace_root("ingest.file", attributes={}),
+ ):
+ pass
+
+ root = finished()["ingest.file"]
+ assert root.attributes[LF.TRACE_USER_ID] == "user-2"
+ assert root.attributes[LF.TRACE_SESSION_ID] == "task-2"
+ assert _trace_metadata(root, "openrag.task_id") == "task-2"
+
+
+def test_stage_spans_carry_the_trace_metadata(finished):
+ with trace_root("ingest.file", attributes={"openrag.document_id": "doc-1"}):
+ with trace_span("ingest.embed"):
+ pass
+
+ assert _trace_metadata(finished()["ingest.embed"], "openrag.document_id") == "doc-1"
+
+
+def test_gateway_calls_inside_a_root_carry_its_user_session_and_attributes(finished):
+ with trace_root(
+ "ingest.file",
+ user_id="user-1",
+ session_id="task-1",
+ attributes={"openrag.document_id": "doc-1"},
+ ):
+ metadata = gateway_trace_kwargs("embedding", "ingest")["metadata"]
+
+ assert metadata["trace_user_id"] == "user-1"
+ assert metadata["session_id"] == "task-1"
+ assert metadata["trace_metadata"]["openrag.document_id"] == "doc-1"
+
+
+# --- values ----------------------------------------------------------------
+
+
+def test_set_records_openrag_metadata_and_merges(finished):
+ with (
+ trace_root("ingest.file", attributes={}),
+ trace_span("ingest.resplit", chunk_size=1000) as span,
+ ):
+ span.set(chunks_in=4)
+ span.set(chunks_out=9, skipped=None)
+
+ resplit = finished()["ingest.resplit"]
+ assert _metadata(resplit, "openrag.chunk_size") == 1000
+ assert _metadata(resplit, "openrag.chunks_in") == 4
+ assert _metadata(resplit, "openrag.chunks_out") == 9
+ assert _metadata(resplit, "openrag.skipped") is None
+
+
+def test_content_is_dropped_unless_capture_is_on(finished, monkeypatch):
+ with trace_root("ingest.file", attributes={}) as root:
+ root.set_content(filename="q3-salaries.pdf")
+ with_capture_off = finished()["ingest.file"]
+ assert "q3-salaries.pdf" not in str(dict(with_capture_off.attributes))
+
+ monkeypatch.setattr(spans, "content_capture_enabled", lambda: True)
+ with trace_root("ingest.file", attributes={}) as root:
+ root.set_content(filename="q3-salaries.pdf")
+ assert _metadata(finished()["ingest.file"], "openrag.filename") == "q3-salaries.pdf"
+
+
+# --- failures --------------------------------------------------------------
+
+
+def test_a_failing_stage_marks_itself_and_the_trace_and_reraises(finished):
+ with pytest.raises(ValueError):
+ with trace_root("ingest.file", attributes={}), trace_span("ingest.embed"):
+ raise ValueError("chunk text: the quarterly salaries are")
+
+ exported = finished()
+ embed, root = exported["ingest.embed"], exported["ingest.file"]
+ assert embed.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ assert embed.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "ValueError"
+ assert root.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "ValueError in ingest.embed"
+
+
+def test_exception_text_stays_out_of_spans_unless_capture_is_on(finished):
+ with pytest.raises(ValueError):
+ with trace_root("ingest.file", attributes={}), trace_span("ingest.embed"):
+ raise ValueError("chunk text: the quarterly salaries are")
+
+ for span in finished().values():
+ assert "quarterly salaries" not in str(dict(span.attributes))
+ assert not any(event.name == "exception" for event in span.events)
+
+
+def test_exception_text_is_kept_when_capture_is_on(finished, monkeypatch):
+ monkeypatch.setattr(spans, "content_capture_enabled", lambda: True)
+
+ with pytest.raises(ValueError), trace_root("ingest.file", attributes={}):
+ raise ValueError("bad page 3")
+
+ root = finished()["ingest.file"]
+ assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "ValueError: bad page 3"
+
+
+def test_fail_marks_a_span_without_an_exception(finished):
+ with trace_root("ingest.file", attributes={}) as root:
+ root.fail("NoEmbeddings")
+
+ root_span = finished()["ingest.file"]
+ assert root_span.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
+ assert root_span.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "NoEmbeddings"
+ assert _metadata(root_span, "openrag.error_type") == "NoEmbeddings"
diff --git a/tests/unit/observability/test_tracing.py b/tests/unit/observability/test_tracing.py
new file mode 100644
index 000000000..cf7dc233a
--- /dev/null
+++ b/tests/unit/observability/test_tracing.py
@@ -0,0 +1,474 @@
+"""Tests for backend LLM tracing (`observability/tracing.py`).
+
+Tracing is off by default, and when it is on, traces carry metadata only unless
+content capture is explicitly opted into (twice, in SaaS). Both failure modes
+are silent — tracing that never starts, or prompts and chunk text leaking into
+Langfuse — so the gate and the redaction switch are pinned here.
+"""
+
+import pytest
+
+from observability import spans, tracing
+from observability.tracing import TRACING_CALLBACK, configure_tracing, content_capture_enabled
+
+_TRACING_ENV = (
+ "OPENRAG_TRACING_ENABLED",
+ "OPENRAG_TRACING_CAPTURE_CONTENT",
+ "OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS",
+ "OPENRAG_RUN_MODE",
+ "LANGFUSE_PUBLIC_KEY",
+ "LANGFUSE_SECRET_KEY",
+ "LANGFUSE_HOST",
+ "LANGFUSE_BASE_URL",
+ "LANGFUSE_OTEL_HOST",
+ "USE_OTEL_LITELLM_REQUEST_SPAN",
+ "OTEL_TRACES_SAMPLER",
+)
+
+
[email protected](autouse=True)
+def _isolated_tracing(monkeypatch):
+ """Clear tracing env and restore LiteLLM's process-global callback state."""
+ import litellm
+
+ for name in _TRACING_ENV:
+ # setenv first so teardown restores the original value (or absence)
+ # even when configure_tracing() writes the variable itself.
+ monkeypatch.setenv(name, "")
+ monkeypatch.delenv(name)
+ # LiteLLM copies `callbacks` into these lists on first use and dedupes by
+ # class, so a logger from one test would otherwise swallow the next test's.
+ for name in (
+ "callbacks",
+ "input_callback",
+ "success_callback",
+ "failure_callback",
+ "_async_success_callback",
+ "_async_failure_callback",
+ ):
+ monkeypatch.setattr(litellm, name, [])
+ monkeypatch.setattr(litellm, "turn_off_message_logging", False)
+ monkeypatch.setattr(tracing, "_enabled", False)
+ # configure_tracing() builds a real Langfuse client for our spans; never
+ # leave one behind for later tests to export through.
+ monkeypatch.setattr(spans, "_client", None)
+
+
+def _langfuse_loggers() -> list:
+ """The Langfuse OTel loggers registered on LiteLLM."""
+ import litellm
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+
+ return [cb for cb in litellm.callbacks if isinstance(cb, LangfuseOtelLogger)]
+
+
+def keys(monkeypatch):
+ monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test")
+ monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test")
+
+
+def test_disabled_by_default_registers_no_callback(keys):
+ import litellm
+
+ assert configure_tracing() is False
+ assert TRACING_CALLBACK not in litellm.callbacks
+ assert _langfuse_loggers() == []
+
+
[email protected]("value", ["false", "0", "", "no"])
+def test_falsy_flag_registers_no_callback(monkeypatch, keys, value):
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", value)
+
+ assert configure_tracing() is False
+ assert TRACING_CALLBACK not in litellm.callbacks
+ assert _langfuse_loggers() == []
+
+
+def test_enabled_without_keys_warns_and_stays_off(monkeypatch, caplog):
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ warnings = []
+ monkeypatch.setattr(tracing.logger, "warning", lambda msg, **kw: warnings.append(msg))
+
+ assert configure_tracing() is False
+ assert TRACING_CALLBACK not in litellm.callbacks
+ assert _langfuse_loggers() == []
+ assert any("LANGFUSE_PUBLIC_KEY" in w for w in warnings)
+
+
+def test_enabled_with_keys_registers_langfuse_otel_callback(monkeypatch, keys):
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+
+ assert configure_tracing() is True
+ (logger,) = _langfuse_loggers()
+ # Registered as LiteLLM's langfuse_otel, which keeps it off the global provider.
+ assert logger.callback_name == TRACING_CALLBACK
+ assert logger._skip_set_global() is True
+
+
+def test_registration_is_idempotent(monkeypatch, keys):
+ """Startup can run more than once in a process (tests, reloads)."""
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+
+ configure_tracing()
+ configure_tracing()
+
+ assert len(_langfuse_loggers()) == 1
+
+
+def test_enabled_redacts_message_content_by_default(monkeypatch, keys):
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+
+ configure_tracing()
+
+ assert litellm.turn_off_message_logging is True
+
+
+def test_capture_opt_in_keeps_content_in_oss(monkeypatch, keys):
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")
+
+ configure_tracing()
+
+ assert litellm.turn_off_message_logging is False
+
+
+ ("run_mode", "capture", "allow_saas", "expected"),
+ [
+ ("oss", "false", "false", False),
+ ("oss", "true", "false", True),
+ ("on_prem", "true", "false", True),
+ ("saas", "true", "false", False),
+ ("saas", "false", "true", False),
+ ("saas", "true", "true", True),
+ ],
+)
+def test_content_capture_policy(monkeypatch, run_mode, capture, allow_saas, expected):
+ monkeypatch.setenv("OPENRAG_RUN_MODE", run_mode)
+ monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", capture)
+ monkeypatch.setenv("OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS", allow_saas)
+
+ assert content_capture_enabled() is expected
+
+
+def test_saas_without_second_flag_redacts(monkeypatch, keys):
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_RUN_MODE", "saas")
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")
+
+ configure_tracing()
+
+ assert litellm.turn_off_message_logging is True
+
+
+def test_empty_host_targets_the_same_langfuse_cloud_as_langflow(monkeypatch, keys):
+ """LiteLLM defaults to the US cloud; Langflow and the Langfuse SDK default to EU."""
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+
+ configure_tracing()
+
+ config = LangfuseOtelLogger.get_langfuse_otel_config()
+ assert config.endpoint == "https://cloud.langfuse.com/api/public/otel"
+
+
+def test_explicit_host_is_respected(monkeypatch, keys):
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.internal:3000")
+
+ configure_tracing()
+
+ config = LangfuseOtelLogger.get_langfuse_otel_config()
+ assert config.endpoint == "http://langfuse.internal:3000/api/public/otel"
+
+
+def test_base_url_is_used_as_the_host(monkeypatch, keys):
+ """Langfuse's setup snippets name the host LANGFUSE_BASE_URL; LiteLLM reads LANGFUSE_HOST."""
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ monkeypatch.setenv("LANGFUSE_BASE_URL", "https://us.cloud.langfuse.com")
+
+ configure_tracing()
+
+ config = LangfuseOtelLogger.get_langfuse_otel_config()
+ assert config.endpoint == "https://us.cloud.langfuse.com/api/public/otel"
+
+
+def test_base_url_wins_over_host_like_langflow(monkeypatch, keys):
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ monkeypatch.setenv("LANGFUSE_BASE_URL", "https://us.cloud.langfuse.com")
+ monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com")
+
+ configure_tracing()
+
+ config = LangfuseOtelLogger.get_langfuse_otel_config()
+ assert config.endpoint == "https://us.cloud.langfuse.com/api/public/otel"
+
+
+def test_missing_tracing_extra_warns_and_stays_off(monkeypatch, keys):
+ import builtins
+
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+ real_import = builtins.__import__
+
+ def _no_otel_sdk(name, *args, **kwargs):
+ if name.startswith("opentelemetry.sdk") or name.startswith("opentelemetry.exporter"):
+ raise ImportError(name)
+ return real_import(name, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "__import__", _no_otel_sdk)
+ warnings = []
+ monkeypatch.setattr(tracing.logger, "warning", lambda msg, **kw: warnings.append(msg))
+
+ assert configure_tracing() is False
+ assert TRACING_CALLBACK not in litellm.callbacks
+ assert _langfuse_loggers() == []
+ assert any("--extra tracing" in w for w in warnings)
+
+
+# ── LiteLLM contract ───────────────────────────────────────────────────────
+# Run real LiteLLM calls (mock_response, no network) through the Langfuse OTel
+# logger with an in-memory exporter in place of OTLP. These pin the pinned
+# LiteLLM's behaviour that tracing relies on: redaction, nesting under the
+# caller's active span, and failures recorded on the span.
+
+_SECRETS = ("SECRET-PROMPT", "SECRET-ANSWER", "SECRET-CHUNK")
+
+
+def traced(monkeypatch, keys):
+ """Configure tracing, then swap the OTLP-exporting callback for an in-memory one."""
+ import litellm
+ from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
+ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+ from observability import litellm_langfuse
+
+ monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
+
+ def _configure():
+ exporter = InMemorySpanExporter()
+ # configure_tracing() builds its LiteLLM logger through this; hand it
+ # an in-memory exporter instead of OTLP to Langfuse.
+ real_build = litellm_langfuse.build_langfuse_logger
+ monkeypatch.setattr(
+ litellm_langfuse,
+ "build_langfuse_logger",
+ lambda **kwargs: real_build(**kwargs, exporter=exporter),
+ )
+ assert configure_tracing() is True
+ assert any(isinstance(cb, LangfuseOtelLogger) for cb in litellm.callbacks)
+ return exporter
+
+ return _configure
+
+
+def caller_tracer():
+ from opentelemetry.sdk.trace import TracerProvider
+
+ return TracerProvider().get_tracer("openrag-test")
+
+
+async def _drain(exporter, expected: int):
+ """LiteLLM runs logging callbacks in the background; wait for the spans."""
+ import asyncio
+
+ for _ in range(100):
+ spans = [s for s in exporter.get_finished_spans() if s.name == "litellm_request"]
+ if len(spans) >= expected:
+ return spans
+ await asyncio.sleep(0.02)
+ raise AssertionError(f"expected {expected} litellm_request spans, got {len(spans)}")
+
+
+def _span_text(span) -> str:
+ return (
+ repr(dict(span.attributes))
+ + repr([dict(e.attributes or {}) for e in span.events])
+ + repr(span.status.description)
+ )
+
+
+async def _completion(content="SECRET-PROMPT", response="SECRET-ANSWER"):
+ import litellm
+
+ return await litellm.acompletion(
+ model="openai/gpt-4o-mini",
+ messages=[{"role": "user", "content": content}],
+ mock_response=response,
+ )
+
+
+async def _embedding():
+ import litellm
+
+ return await litellm.aembedding(
+ model="openai/text-embedding-3-small", input=["SECRET-CHUNK"], mock_response=[0.1, 0.2]
+ )
+
+
+async def test_gateway_calls_are_traced_without_content_by_default(traced):
+ exporter = traced()
+
+ await _completion()
+ await _embedding()
+
+ spans = await _drain(exporter, 2)
+ assert {span.attributes["llm.model_name"] for span in spans} == {
+ "gpt-4o-mini",
+ "text-embedding-3-small",
+ }
+ for span in spans:
+ assert not any(secret in _span_text(span) for secret in _SECRETS)
+
+
+async def test_content_is_traced_when_capture_is_opted_in(monkeypatch, traced):
+ monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")
+ exporter = traced()
+
+ await _completion()
+
+ (span,) = await _drain(exporter, 1)
+ assert "SECRET-PROMPT" in _span_text(span)
+
+
+async def test_gateway_spans_nest_under_the_callers_active_span(traced, caller_tracer):
+ """Ingest/search spans (03, 05) rely on this: no ids are threaded through call sites."""
+ exporter = traced()
+
+ with caller_tracer.start_as_current_span("ingest.embed") as parent:
+ await _embedding()
+ parent_ctx = parent.get_span_context()
+
+ (span,) = await _drain(exporter, 1)
+ assert span.context.trace_id == parent_ctx.trace_id
+ assert span.parent.span_id == parent_ctx.span_id
+
+
+async def test_gateway_spans_survive_an_unsampled_foreign_parent(traced):
+ """Instana's HTTP spans are active during requests and are never sampled.
+
+ The SDK's default parent-based sampler would drop every LLM span beneath
+ them, so Langfuse would see nothing whenever Instana is also enabled.
+ """
+ from opentelemetry import trace
+ from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags
+
+ exporter = traced()
+ instana_like = NonRecordingSpan(
+ SpanContext(
+ trace_id=0xD20FD29B103E15D2,
+ span_id=0xD20FD29B103E15D2,
+ is_remote=False,
+ trace_flags=TraceFlags(TraceFlags.DEFAULT),
+ )
+ )
+
+ with trace.use_span(instana_like):
+ await _embedding()
+
+ (span,) = await _drain(exporter, 1)
+ assert span.context.trace_id == 0xD20FD29B103E15D2
+
+
+async def test_failed_call_is_traced_with_the_error(traced):
+ import litellm
+ from opentelemetry.trace import StatusCode
+
+ exporter = traced()
+
+ with pytest.raises(litellm.InternalServerError):
+ await _completion(response=Exception("upstream exploded"))
+
+ (span,) = await _drain(exporter, 1)
+ assert span.status.status_code is StatusCode.ERROR
+ assert "error.type" in span.attributes
+ # Provider error text can echo the prompt or chunk: content, so dropped
+ # unless capture is on (turn_off_message_logging doesn't cover failures).
+ assert "upstream exploded" not in _span_text(span)
+ assert "error.message" not in span.attributes
+ assert "error.stack_trace" not in span.attributes
+ assert not any(event.name == "exception" for event in span.events)
+
+
+async def test_failed_call_keeps_the_error_text_when_capture_is_on(monkeypatch, traced):
+ import litellm
+
+ monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")
+ exporter = traced()
+
+ with pytest.raises(litellm.InternalServerError):
+ await _completion(response=Exception("upstream exploded"))
+
+ (span,) = await _drain(exporter, 1)
+ assert "upstream exploded" in _span_text(span)
+
+
+ ("env", "expected"),
+ [
+ ({}, "https://cloud.langfuse.com"),
+ ({"LANGFUSE_HOST": "http://old:3000"}, "http://old:3000"),
+ ({"LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com"}, "https://us.cloud.langfuse.com"),
+ (
+ {
+ "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com",
+ "LANGFUSE_HOST": "http://old:3000",
+ },
+ "https://us.cloud.langfuse.com",
+ ),
+ ],
+)
+def test_langfuse_host_precedence(monkeypatch, env, expected):
+ from config.settings import get_langfuse_host
+
+ for name, value in env.items():
+ monkeypatch.setenv(name, value)
+
+ assert get_langfuse_host() == expected
+
+
+def test_span_client_resolves_the_host_itself(monkeypatch, keys):
+ """Our spans and LiteLLM's must export to the same Langfuse, whatever ran first."""
+ import langfuse
+
+ captured = {}
+
+ class _FakeLangfuse:
+ def __init__(self, **kwargs):
+ captured.update(kwargs)
+
+ monkeypatch.setattr(langfuse, "Langfuse", _FakeLangfuse)
+ monkeypatch.setenv("LANGFUSE_BASE_URL", "https://us.cloud.langfuse.com")
+
+ spans.init_span_client()
+
+ assert captured["base_url"] == "https://us.cloud.langfuse.com"
diff --git a/tests/unit/services/test_embedding_calls_use_the_gateway.py b/tests/unit/services/test_embedding_calls_use_the_gateway.py
index 3ebd8b30a..55fee5df2 100644
--- a/tests/unit/services/test_embedding_calls_use_the_gateway.py
+++ b/tests/unit/services/test_embedding_calls_use_the_gateway.py
@@ -270,3 +270,26 @@ async def test_the_dimension_probe_goes_through_the_gateway(monkeypatch) -> None
assert await service._detect_embedding_dimensions("my-deployment", "azure_ai") == 1024
assert calls == [{"model": "azure_ai:my-deployment", "input": ["dimension probe"]}]
+
+
+async def test_the_dimension_probe_is_traced_as_a_probe(monkeypatch) -> None:
+ """Otherwise it shows in traces as a one-input ingest embedding."""
+ from observability import tracing
+ from observability.tracing import gateway_trace_kwargs
+ from services import llm_gateway as gateway_mod
+ from services.langflow_file_service import LangflowFileService
+
+ monkeypatch.setattr(tracing, "_enabled", True)
+ names: list[str] = []
+
+ async def fake_embeddings(body, **_kwargs):
+ names.append(gateway_trace_kwargs("embedding", "ingest")["metadata"]["generation_name"])
+ return {"data": [{"embedding": [0.0] * 1024}]}
+
+ monkeypatch.setattr(gateway_mod, "embeddings", fake_embeddings)
+
+ service = LangflowFileService(docling_service=AsyncMock())
+
+ await service._detect_embedding_dimensions("my-deployment", "azure_ai")
+ assert names == ["embedding.dimension_probe"]
diff --git a/tests/unit/services/test_llm_gateway.py b/tests/unit/services/test_llm_gateway.py
index bcad5e5cc..c25d28012 100644
--- a/tests/unit/services/test_llm_gateway.py
+++ b/tests/unit/services/test_llm_gateway.py
@@ -2698,3 +2698,152 @@ class TestToolsBesideReasoningEffort:
)
assert len(calls) == 1
+
+
+# ---------------------------------------------------------------------------
+# Tracing metadata: what each call tells Langfuse (only when tracing is on)
+# ---------------------------------------------------------------------------
+
+
+def tracing_on(monkeypatch):
+ from observability import tracing
+
+ monkeypatch.setattr(tracing, "_enabled", True)
+
+
+def _capture_embedding(monkeypatch) -> list[dict]:
+ calls: list[dict] = []
+
+ async def fake_aembedding(**kwargs):
+ calls.append(kwargs)
+ return {
+ "object": "list",
+ "data": [{"embedding": [0.1], "index": i} for i in range(len(kwargs["input"]))],
+ }
+
+ monkeypatch.setattr("litellm.aembedding", fake_aembedding)
+ return calls
+
+
+async def test_embeddings_send_no_tracing_metadata_when_tracing_is_off(monkeypatch):
+ from observability import tracing
+
+ monkeypatch.setattr(tracing, "_enabled", False)
+ calls = _capture_embedding(monkeypatch)
+
+ await embeddings({"model": "text-embedding-3-small", "input": ["hello"]}, config=_config())
+
+ assert "metadata" not in calls[0]
+
+
+async def test_bulk_embeddings_are_traced_as_ingest_with_sizes(monkeypatch, tracing_on):
+ calls = _capture_embedding(monkeypatch)
+
+ await embeddings(
+ {"model": "text-embedding-3-small", "input": ["hello", "world!"]}, config=_config()
+ )
+
+ metadata = calls[0]["metadata"]
+ assert metadata["generation_name"] == "embedding.ingest"
+ assert metadata["trace_metadata"]["openrag.provider"] == "openai"
+ assert metadata["trace_metadata"]["openrag.embedding.inputs"] == 2
+ assert metadata["trace_metadata"]["openrag.embedding.input_chars"] == 11
+
+
+async def test_interactive_embeddings_are_traced_as_query(monkeypatch, tracing_on):
+ calls = _capture_embedding(monkeypatch)
+
+ await embeddings(
+ {"model": "text-embedding-3-small", "input": "what is openrag"},
+ config=_config(),
+ interactive=True,
+ )
+
+ metadata = calls[0]["metadata"]
+ assert metadata["generation_name"] == "embedding.query"
+ assert metadata["trace_metadata"]["openrag.embedding.inputs"] == 1
+ assert metadata["trace_metadata"]["openrag.embedding.input_chars"] == 15
+
+
+async def test_embedding_trace_metadata_never_carries_the_input_text(monkeypatch, tracing_on):
+ calls = _capture_embedding(monkeypatch)
+
+ await embeddings(
+ {"model": "text-embedding-3-small", "input": ["secret chunk text"]}, config=_config()
+ )
+
+ assert "secret chunk text" not in json.dumps(calls[0]["metadata"])
+
+
+async def test_onprem_sub_batches_are_numbered_in_trace_metadata(monkeypatch, tracing_on):
+ providers = ProvidersConfig(
+ openai=OpenAIConfig(),
+ anthropic=AnthropicConfig(),
+ watsonx=WatsonXConfig(),
+ ollama=OllamaConfig(),
+ custom={
+ "watsonx_onprem": GenericProviderConfig(
+ credentials={
+ "api_base": "https://cpd.example.com",
+ "username": "cpduser",
+ "api_key": "secret",
+ },
+ configured=True,
+ )
+ },
+ )
+ calls = _capture_embedding(monkeypatch)
+ monkeypatch.setattr(
+ "services.llm_gateway._provider_runtime_kwargs",
+ lambda provider, config: {},
+ )
+
+ await embeddings(
+ {
+ "model": "watsonx_onprem:ibm/slate-30m-english-rtrvr",
+ "input": [f"chunk-{index}" for index in range(70)],
+ },
+ config=SimpleNamespace(providers=providers),
+ )
+
+ traced = [call["metadata"]["trace_metadata"] for call in calls]
+ assert [t["openrag.embedding.inputs"] for t in traced] == [32, 32, 6]
+ assert [t["openrag.embedding.sub_batch"] for t in traced] == [1, 2, 3]
+ assert {t["openrag.embedding.sub_batches"] for t in traced} == {3}
+
+
+async def test_chat_completions_are_traced_with_request_shape(monkeypatch, tracing_on):
+ captured = {}
+
+ async def fake_acompletion(**kwargs):
+ captured.update(kwargs)
+ return {"choices": [{"message": {"role": "assistant", "content": "hi"}}]}
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+
+ await chat_completions(
+ {
+ "model": "gpt-4o-mini",
+ "messages": [
+ {"role": "system", "content": "be brief"},
+ {"role": "user", "content": "hello"},
+ ],
+ "tools": [{"type": "function", "function": {"name": "search", "parameters": {}}}],
+ },
+ config=_config(),
+ )
+
+ metadata = captured["metadata"]
+ assert metadata["generation_name"] == "chat.completion"
+ assert metadata["trace_metadata"]["openrag.provider"] == "openai"
+ assert metadata["trace_metadata"]["openrag.chat.messages"] == 2
+ assert metadata["trace_metadata"]["openrag.chat.tools"] == 1
+ assert metadata["trace_metadata"]["openrag.chat.stream"] is False
+ assert "hello" not in json.dumps(metadata)
diff --git a/tests/unit/test_env_manager.py b/tests/unit/test_env_manager.py
index 721883b88..5488fcb87 100644
--- a/tests/unit/test_env_manager.py
+++ b/tests/unit/test_env_manager.py
@@ -248,3 +248,112 @@ class TestLangflowPasswordAndAutoLogin:
assert env_manager.validate_config() is False
assert "langflow_superuser_password" in env_manager.config.validation_errors
+
+
+# ---------------------------------------------------------------------------
+# Backend tracing settings
+# ---------------------------------------------------------------------------
+
+
+class TestBackendTracingSettings:
+ """The TUI manages backend tracing alongside the Langfuse keys it reuses."""
+
+ def test_tracing_flags_round_trip(self, env_manager, tmp_path):
+ from tui.managers.env_manager import EnvManager
+
+ env_manager.config.opensearch_password = "OpenSearchPass123!"
+ env_manager.config.langflow_superuser_password = "LangflowPass123!"
+ env_manager.config.langfuse_public_key = "pk-lf-test"
+ env_manager.config.openrag_tracing_enabled = "true"
+ env_manager.config.openrag_tracing_capture_content = "true"
+
+ with patch("tui.utils.version_check.get_current_version", return_value="1.0.0"):
+ assert env_manager.save_env_file() is True
+
+ content = (tmp_path / ".env").read_text()
+ assert content.count("OPENRAG_TRACING_ENABLED=") == 1
+ assert content.count("OPENRAG_TRACING_CAPTURE_CONTENT=") == 1
+
+ reloaded = EnvManager(env_file=tmp_path / ".env")
+ reloaded.load_existing_env()
+ assert reloaded.config.openrag_tracing_enabled == "true"
+ assert reloaded.config.openrag_tracing_capture_content == "true"
+
+ def test_unset_tracing_flags_are_not_written(self, env_manager, tmp_path):
+ env_manager.config.opensearch_password = "OpenSearchPass123!"
+ env_manager.config.langflow_superuser_password = "LangflowPass123!"
+
+ with patch("tui.utils.version_check.get_current_version", return_value="1.0.0"):
+ assert env_manager.save_env_file() is True
+
+ assert "OPENRAG_TRACING_" not in (tmp_path / ".env").read_text()
+
+ def test_tracing_fields_are_in_the_langfuse_section(self):
+ from tui.config_fields import CONFIG_SECTIONS
+
+ (langfuse,) = [s for s in CONFIG_SECTIONS if s.name == "Langfuse"]
+ env_vars = {f.env_var for f in langfuse.fields}
+ assert {"OPENRAG_TRACING_ENABLED", "OPENRAG_TRACING_CAPTURE_CONTENT"} <= env_vars
+
+
+class TestLangfuseBaseUrl:
+ """LANGFUSE_BASE_URL is the host name Langfuse's setup steps use; LANGFUSE_HOST is the older alias."""
+
+ @pytest.fixture(autouse=True)
+ def _clear_langfuse_host_env(self, monkeypatch):
+ # load_dotenv writes into os.environ, so clear what earlier loads left.
+ for var in ("LANGFUSE_HOST", "LANGFUSE_BASE_URL"):
+ monkeypatch.setenv(var, "")
+ monkeypatch.delenv(var)
+
+ def _save(self, env_manager):
+ env_manager.config.opensearch_password = "OpenSearchPass123!"
+ env_manager.config.langflow_superuser_password = "LangflowPass123!"
+ with patch("tui.utils.version_check.get_current_version", return_value="1.0.0"):
+ assert env_manager.save_env_file() is True
+
+ def test_host_is_written_as_base_url(self, env_manager, tmp_path):
+ env_manager.config.langfuse_base_url = "https://us.cloud.langfuse.com"
+
+ self._save(env_manager)
+
+ from tui.managers.env_manager import EnvManager
+
+ assert "LANGFUSE_HOST" not in (tmp_path / ".env").read_text()
+ reloaded = EnvManager(env_file=tmp_path / ".env")
+ reloaded.load_existing_env()
+ assert reloaded.config.langfuse_base_url == "https://us.cloud.langfuse.com"
+
+ def test_legacy_langfuse_host_is_migrated_to_base_url(self, tmp_path):
+ from tui.managers.env_manager import EnvManager
+
+ (tmp_path / ".env").write_text("LANGFUSE_HOST=http://langfuse.internal:3000\n")
+ manager = EnvManager(env_file=tmp_path / ".env")
+ manager.load_existing_env()
+ assert manager.config.langfuse_base_url == "http://langfuse.internal:3000"
+
+ self._save(manager)
+
+ content = (tmp_path / ".env").read_text()
+ assert "LANGFUSE_BASE_URL=" in content
+ assert "LANGFUSE_HOST" not in content
+
+ def test_base_url_wins_over_legacy_host(self, tmp_path):
+ from tui.managers.env_manager import EnvManager
+
+ (tmp_path / ".env").write_text(
+ "LANGFUSE_BASE_URL=https://us.cloud.langfuse.com\n"
+ "LANGFUSE_HOST=https://cloud.langfuse.com\n"
+ )
+ manager = EnvManager(env_file=tmp_path / ".env")
+ manager.load_existing_env()
+
+ assert manager.config.langfuse_base_url == "https://us.cloud.langfuse.com"
+
+ def test_langfuse_section_offers_base_url(self):
+ from tui.config_fields import CONFIG_SECTIONS
+
+ (langfuse,) = [s for s in CONFIG_SECTIONS if s.name == "Langfuse"]
+ env_vars = {f.env_var for f in langfuse.fields}
+ assert "LANGFUSE_BASE_URL" in env_vars
+ assert "LANGFUSE_HOST" not in env_vars
diff --git a/tests/unit/test_task_service_trace_context.py b/tests/unit/test_task_service_trace_context.py
new file mode 100644
index 000000000..ff5f34c07
--- /dev/null
+++ b/tests/unit/test_task_service_trace_context.py
@@ -0,0 +1,33 @@
+"""The task runner tells each file's trace which task and upload session it belongs to."""
+
+from unittest.mock import Mock
+
+import pytest
+
+from models.tasks import FileTask, TaskStatus, UploadTask
+from observability.tracing import current_trace_context
+from services.task_service import TaskService
+
+
+async def test_each_file_is_processed_inside_its_tasks_trace_context():
+ seen: list[dict] = []
+
+ class _Processor:
+ async def process_item(self, upload_task, item, file_task):
+ seen.append(dict(current_trace_context()))
+ file_task.status = TaskStatus.COMPLETED
+
+ service = TaskService(document_service=Mock(), ingestion_timeout=2)
+ task = UploadTask(
+ task_id="task-42",
+ total_files=1,
+ file_tasks={"/data/a.pdf": FileTask(file_path="/data/a.pdf", filename="a.pdf")},
+ processor=_Processor(),
+ )
+ service.task_store.setdefault("user1", {})["task-42"] = task
+
+ await service.background_custom_processor("user1", "task-42", ["/data/a.pdf"])
+
+ assert seen == [{"session_id": "task-42", "attributes": {"openrag.task_id": "task-42"}}]
+ assert current_trace_context() == {}
diff --git a/uv.lock b/uv.lock
index 98106bddd..4d51e69df 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
version = 1
-revision = 3
+revision = 2
requires-python = ">=3.13"
resolution-markers = [
"python_full_version >= '3.15'",
@@ -339,6 +339,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/5e/0b/e106f0fd7fa785867d9ffcc47dc9e6237c0e58f51058473b777487a98edc/azure_storage_blob-12.30.0-py3-none-any.whl", hash = "sha256:d415ac50b67a8da6b3ae7e9f1014b1b55cd7aafa0b8d4ca9b380568dc7360423", size = 435610, upload-time = "2026-06-08T11:45:37.213Z" },
]
+[[package]]
+name = "backoff"
+version = "2.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" },
+]
+
[[package]]
name = "beartype"
version = "0.22.9"
@@ -1547,6 +1556,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" },
]
+[[package]]
+name = "langfuse"
+version = "4.18.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "backoff" },
+ { name = "httpx" },
+ { name = "opentelemetry-api" },
+ { name = "opentelemetry-exporter-otlp-proto-http" },
+ { name = "opentelemetry-sdk" },
+ { name = "packaging" },
+ { name = "pydantic" },
+ { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
+ { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
+ { name = "wrapt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/78/32/cfa4cfa87a73d5e9bc527e568d9eaad171ad8d3ff3ec8a796247b4a577c4/langfuse-4.18.0.tar.gz", hash = "sha256:fd529365207ecdefd778ad1a35056a2acee032ff0d75e18368c106fbc3cd865c", size = 395814, upload-time = "2026-10-14T08:40:45.485Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/49/49/2569267383ee0d0afeaab904aefaf77eb69fed023354e833fb49ebe7411e/langfuse-4.18.0-py3-none-any.whl", hash = "sha256:b0f14f30f9f147ca662eaa9998af374fc57b9e8a894f0ae86789cbbd96448289", size = 709950, upload-time = "2026-10-14T08:40:47.196Z" },
+]
+
[[package]]
name = "librt"
version = "0.10.0"
@@ -2053,9 +2083,13 @@ dependencies = [
apm = [
{ name = "instana" },
]
+tracing = [
+ { name = "langfuse" },
+]
[package.dev-dependencies]
dev = [
+ { name = "langfuse" },
{ name = "mypy" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
@@ -2090,6 +2124,7 @@ requires-dist = [
{ name = "idna", specifier = ">=3.18" },
{ name = "instana", marker = "extra == 'apm'", specifier = ">=3.17.0,<4" },
{ name = "joserfc", specifier = ">=1.7.3,<2.0.0" },
+ { name = "langfuse", marker = "extra == 'tracing'", specifier = ">=4.16,<5" },
{ name = "litellm", specifier = ">=1.105.0,<2.0.0" },
{ name = "mcp", specifier = ">=1.28.1" },
{ name = "msal", specifier = ">=1.29.0" },
@@ -2118,10 +2153,11 @@ requires-dist = [
{ name = "uvicorn", specifier = ">=0.35.0" },
{ name = "zxcvbn", specifier = ">=4.5.0" },
]
-provides-extras = ["apm"]
+provides-extras = ["apm", "tracing"]
[package.metadata.requires-dev]
dev = [
+ { name = "langfuse", specifier = ">=4.16,<5" },
{ name = "mypy", specifier = ">=2.0.0" },
{ name = "pytest", specifier = ">=9.0.3" },
{ name = "pytest-asyncio", specifier = ">=0.21.0" },
@@ -2166,29 +2202,118 @@ async = [
[[package]]
name = "opentelemetry-api"
-version = "1.44.0"
+version = "1.45.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
+ { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" },
+]
+
+[[package]]
+name = "opentelemetry-exporter-http-transport"
+version = "0.66b1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/62/0c/e3ebdb4b507f66afcc905e6885a4946969bd75b45988492643356fbbdc63/opentelemetry_exporter_http_transport-0.66b1.tar.gz", hash = "sha256:443080203bf52586ce0b2ad901e8951c61833eab1aa539ae6f1f16fe9e8e7952", size = 11693, upload-time = "2026-10-06T17:32:59.65Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/04/69/6af86ff66492b481c6a4c05dcfd68beb47ed8ba046440a26a2aac76b95c7/opentelemetry_exporter_http_transport-0.66b1-py3-none-any.whl", hash = "sha256:2f95404bdee7f9d2d529c7de56c7bd86d014d774d8fbf137810e0167f8a492bf", size = 12155, upload-time = "2026-10-06T17:32:35.454Z" },
+]
+
+[package.optional-dependencies]
+requests = [
+ { name = "requests" },
+]
+
+[[package]]
+name = "opentelemetry-exporter-otlp-common"
+version = "0.66b1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-sdk" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cb/19/41de712173f43057e4532d42ece7d0c6d4210d353e5752433cb14987643f/opentelemetry_exporter_otlp_common-0.66b1.tar.gz", hash = "sha256:6b1403487a2185ac1feb45fd5546fdf8630ce71c36bcefaadf51e2130e9e23f9", size = 14325, upload-time = "2026-10-06T17:33:01.725Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fc/39/8c23d67665c762aa51840fa06f86e902e8f6f1693bc8d7e3d98cd6e2f753/opentelemetry_exporter_otlp_common-0.66b1-py3-none-any.whl", hash = "sha256:00ff8592c3a7cb729ff3fdc7ffa12372c243bdf2163e80c180994d0c7bd83ee9", size = 12385, upload-time = "2026-10-06T17:32:38.177Z" },
+]
+
+[[package]]
+name = "opentelemetry-exporter-otlp-proto-common"
+version = "1.45.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-proto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c1/8e/65e85e5137991a3c493b11682151d198638a5bc1dd4b4c5f67e013c57d7c/opentelemetry_exporter_otlp_proto_common-1.45.1.tar.gz", hash = "sha256:2e4adcc3a67bcf57804fc49514f0ef64974ca7590aa3491da389852b4a0628f6", size = 18873, upload-time = "2026-10-06T17:33:04.471Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/84/aa/92f225d353904e7f70b8b3e3c1b02db0cf56f744c2e83c581dc372e78873/opentelemetry_exporter_otlp_proto_common-1.45.1-py3-none-any.whl", hash = "sha256:2f446183ae7047b036226f1d846c41a834b0e8755ad13b51a51dd38952eb466c", size = 15393, upload-time = "2026-10-06T17:32:41.911Z" },
+]
+
+[[package]]
+name = "opentelemetry-exporter-otlp-proto-http"
+version = "1.45.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
+ { name = "googleapis-common-protos" },
+ { name = "opentelemetry-api" },
+ { name = "opentelemetry-exporter-http-transport", extra = ["requests"] },
+ { name = "opentelemetry-exporter-otlp-common" },
+ { name = "opentelemetry-exporter-otlp-proto-common" },
+ { name = "opentelemetry-proto" },
+ { name = "opentelemetry-sdk" },
+ { name = "requests" },
{ name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
{ name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1b/17/26487707ea4caa97b17e6e4b5fa72133a53512ffa2f5cf7a49ef284b29cb/opentelemetry_exporter_otlp_proto_http-1.45.1.tar.gz", hash = "sha256:45c218405ce3fd879596924b1874bf9a8f6880206d61065c5a912c8e5c297fb7", size = 28839, upload-time = "2026-10-06T17:33:05.713Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/1f/517eaa0187ba106a9da97160ce2add3a371812681dc440930b267f714e42/opentelemetry_exporter_otlp_proto_http-1.45.1-py3-none-any.whl", hash = "sha256:24a97cf3753c7fb52fad44a696e452ff371686339e2acf3309e2eda3d0230700", size = 22180, upload-time = "2026-10-06T17:32:43.946Z" },
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "1.45.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4b/7f/15f014fb195da6c2dbb6c71399b8e76824878718e94de6454038488eed28/opentelemetry_proto-1.45.1.tar.gz", hash = "sha256:79e0fb95e4616691a469439238aa9224d75779b3e108e895d1aa125ab29ca77c", size = 46488, upload-time = "2026-10-06T17:33:11.49Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ab/9a/42ec8180a769516ae757e893b69736826efceac7332553915b4528a91c6d/opentelemetry_proto-1.45.1-py3-none-any.whl", hash = "sha256:f38e2a8413053c180cd3d2637fbb279673ec2f6a6e09c995aafa2f452c52b46e", size = 72488, upload-time = "2026-10-06T17:32:53.057Z" },
+]
+
+[[package]]
+name = "opentelemetry-sdk"
+version = "1.45.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "opentelemetry-api" },
+ { name = "opentelemetry-semantic-conventions" },
+ { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
+ { name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a1/79/7392e21a1c8f0c61d90b223e31c7e48cb9d452e91a6b820ad24cca5f23c4/opentelemetry_sdk-1.45.1.tar.gz", hash = "sha256:63d24a6ca645019a631e6a51999c73e93adcac1196ca640b8ae78a7cc4762bf3", size = 218324, upload-time = "2026-10-06T17:33:13.26Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/95/3c/87c42b4bd6dd297536f04cd9383d212ac557ecd49f2cbdcd46da1c9ef5c8/opentelemetry_sdk-1.45.1-py3-none-any.whl", hash = "sha256:c604c11dc429810812348989115fa44bd558772a3d7442afc43d024f2c250ca4", size = 140063, upload-time = "2026-10-06T17:32:55.04Z" },
]
[[package]]
name = "opentelemetry-semantic-conventions"
-version = "0.65b0"
+version = "0.66b1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.15'" },
{ name = "typing-extensions", version = "4.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/46/e4/dbbfb2a010c4db2224a5114638acede6fe563d33cc20fb1752cebcbe6298/opentelemetry_semantic_conventions-0.66b1.tar.gz", hash = "sha256:497ca63bf383723411e8eaf60c8779e9877633c936bb641080adab59d0eb6ec8", size = 150250, upload-time = "2026-10-06T17:33:14.073Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/14/67f8aa798857f8cf686f515bf93d9bb877ce952ddc8efae0fa25b45ce0d6/opentelemetry_semantic_conventions-0.66b1-py3-none-any.whl", hash = "sha256:d4cddeb4315490b35213f55e2bdc9ac54bb1e4d318927475bed62b35545e581b", size = 206279, upload-time = "2026-10-06T17:32:56.103Z" },
]
[[package]]
@@ -2957,8 +3082,8 @@ name = "secretstorage"
version = "3.5.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "cryptography" },
- { name = "jeepney" },
+ { name = "cryptography", marker = "python_full_version >= '3.15' or sys_platform != 'win32'" },
+ { name = "jeepney", marker = "python_full_version >= '3.15' or sys_platform != 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" }
wheels = [