OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: Instrument Backend app via IBM Instana for APM and observability (#2265)
Mike Pawlowski7eca07d4871a
diff --git a/.env.example b/.env.exampleindex c1edb52f7..412b416ec 100644--- a/.env.example+++ b/.env.example@@ -332,6 +332,10 @@ NEXT_ALLOWED_DEV_ORIGINS=# LANGFLOW_WORKERS=2 # Langflow workers (default: 1)# DOCLING_WORKERS=2 # Docling workers (default: 1)+# Logging and Service+# Format for log messages (default: json)+LOG_FORMAT=+# OPTIONAL: Set the logging level for the Backend service# - Supported values: DEBUG, INFO, WARNING, ERROR, CRITICAL# - Default: INFO@@ -416,16 +420,134 @@ LANGFLOW_KEY_RETRY_DELAY=# Version of Langflow (informational/future use)# LANGFLOW_VERSION=-# Logging and Service-# Format for log messages (default: json)-LOG_FORMAT=-# Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)-LOG_LEVEL=# Service name used in logs (default: openrag)SERVICE_NAME=openrag# Secret key for session management (auto-generated if not provided)SESSION_SECRET=+# --- Instana APM -----------------------------------------------------------+# Enable auto-instrumentation via Instana's Python tracer.+# Requires a local Instana host agent (see `make instana-agent-up`).+# The tracer ships as the optional `apm` extra, so a host-run backend+# (`make backend`) needs `uv sync --extra apm` once; the backend container+# image already includes it. Without the package the backend logs a warning+# and runs untraced rather than failing to start.+# - Default: false+# INSTANA_ENABLED=true+#+# OPTIONAL: Local Instana agent host.+# The host agent (`make instana-agent-up`) uses host networking and publishes+# 42699 on the host's loopback, so the correct value depends on where the+# backend process runs:+# - `make backend` (backend on the host) → 127.0.0.1+# - `make dev` / `make dev-cpu` (in Docker) → host.docker.internal+# Leave unset to accept the per-topology default: docker-compose defaults the+# backend container to host.docker.internal; the tracer's own built-in default+# is `localhost`, which is what a host-run backend gets.+# INSTANA_AGENT_HOST=127.0.0.1+# OPTIONAL: Local Instana agent port+# - Default: 42699+# INSTANA_AGENT_PORT=42699+#+# OPTIONAL: Service name shown in Instana's Analyze Traces / Applications view.+# INSTANA_SERVICE_NAME="OpenRAG Backend"+#+# OPTIONAL: Custom zone the host/process is grouped under in the+# Infrastructure view. The process entry itself is named by the host agent from+# the process command line, not by the tracer.+# INSTANA_ZONE="openrag-dev"+#+# OPTIONAL: Instana tracer's own log verbosity+# - Default: info+# INSTANA_LOG_LEVEL=info+#+# Span categories/types the tracer must not trace. Comma-separated; `true`+# disables every category.+# - Tracer default: <empty> (everything traced)+# - Set here to `logging`, deliberately, and left uncommented so a fresh .env+# carries it.+#+# WHY: with logging spans on, every WARNING/ERROR logged inside an active+# request trace appends an Event to a list shared by ALL spans (the mutable+# default of InstanaSpan.__init__ in instana 3.17.0), which is never freed, and+# each new log span re-walks the whole list. That is an unbounded memory leak+# and O(n^2) CPU on a long-running backend: measured here at 10.9us -> 1010us+# per in-trace warning, still climbing after 2000 of them. `logging` brings it+# back to 26us and holds flat. src/ has ~750 warning/error call sites, many on+# request paths.+#+INSTANA_TRACING_DISABLE=logging+#+# When the tracer captures a Python stack trace for EXIT spans.+# Accepted: all | error | none (anything else logs a warning and falls back to+# `all`, so never leave this set to an empty string).+# - Tracer default: all+# - Set here to `error`, and left uncommented for the same reason as above.+#+# WHY: EXIT_SPANS covers httpx, urllib3 (OpenSearch), sqlalchemy, aiohttp-client+# and boto3, so `all` means a full traceback.extract_stack() — with linecache+# source reads — on every outbound HTTP call, every SQL query and every+# retrieval. Measured 127us at stack depth 30, 221us at 60, 355us at 100; a+# uvicorn/starlette/async route stack is deep and a chat request makes 10-30+# exit spans. `error` keeps the stack where it is actually diagnostic (errored+# spans) and drops it from the happy path.+INSTANA_STACK_TRACE=error+#+# OPTIONAL: Maximum frames kept per captured stack trace (1-40).+# - Default: 30+# INSTANA_STACK_TRACE_LENGTH=30+#+# Query-string parameters the tracer redacts before spans leave the host.+# Format: <matcher>:<name>[,<name>], matcher one of equals, equals-ignore-case,+# contains, contains-ignore-case, regex.+# - Tracer default: contains-ignore-case:key,pass,secret+# - Set here to a regex that keeps that credential coverage and adds+# OpenRAG's free-text query parameters, deliberately, and left uncommented+# so a fresh .env carries it.+#+# WHY: the tracer default only catches credential-shaped names. It does NOT+# redact free-text values, so GET /v2/files/search?q=<user query> is exported+# to Instana verbatim, along with the `search` and `filename` parameters used+# by file listing. Spans never carry request or response bodies, so chat+# prompts and document text stay local regardless of this setting.+#+# Set this to an empty value (not commented out) to fall back to the tracer's+# own weaker default; it logs a "Couldn't parse INSTANA_SECRETS" warning each+# time you do.+#+# NOTE: uses `\Z` rather than a trailing `$` to anchor exact names. The+# tracer matches with Python's re.match(), which already anchors at the+# start of the string, so a leading `^` would be redundant and a trailing+# `$` is the only anchor doing real work here. This Makefile includes and+# exports every `.env` line as GNU Make syntax (see Makefile's ENV_FILE+# loading), which treats a bare `$` as a Make variable reference — `$,`+# silently resolves to an undefined variable and eats both the `$` and the+# following comma. Reintroducing `$` here will silently break this+# redaction under `make backend` / `make dev` without any warning.+INSTANA_SECRETS=regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z+#+# Instana host agent key.+# Both this and the endpoint below come from your own tenant's+# Infrastructure > Add a Host wizard.+# WARNING: setting these enables `make instana-agent-up`, which starts a+# privileged container in the host PID namespace with Docker socket access.+# Local development only - do not run it on a shared or production host.+# INSTANA_AGENT_KEY=+# OPTIONAL: Instana host agent endpoint.+# The ingress is region-specific — the default below is only one of several,+# so take the value your tenant's Add a Host wizard shows you.+# - Default: ingress-coral-saas.instana.io+# INSTANA_AGENT_ENDPOINT=ingress-coral-saas.instana.io+# OPTIONAL: Instana host agent port+# - Default: 443+# INSTANA_AGENT_ENDPOINT_PORT=443+# OPTIONAL: Instana host agent image tag.+# Pinned in docker-compose.yml because the agent runs privileged with host PID+# and Docker socket access. Set this to try a newer agent without editing+# compose; see https://icr.io/v2/instana/agent/tags/list for available tags.+# - Default: 1.323.1+# INSTANA_AGENT_IMAGE_TAG=1.323.1+# OPTIONAL: JWT signing key for token issuance and verification.# If not set, OpenRAG generates an RSA key pair (private_key.pem / public_key.pem)# under OPENRAG_KEYS_PATH at startup and uses RS256 signing.diff --git a/Dockerfile.backend b/Dockerfile.backendindex 46c898630..d1ac25f75 100644--- a/Dockerfile.backend+++ b/Dockerfile.backend@@ -47,9 +47,12 @@ RUN microdnf install -y \WORKDIR /app# Dependency layer: reuse as long as lockfile and pyproject don't change+# --extra apm bakes in the Instana tracer so the published image can turn APM on+# with INSTANA_ENABLED alone. The extra exists so source installs that will never+# enable it don't pay for the tracer; images ship with the capability present.COPY pyproject.toml uv.lock ./RUN --mount=type=cache,target=/root/.cache/uv,uid=0,gid=0 \- uv sync+ uv sync --extra apm# Application (invalidates after deps only when src/flows change)COPY src/ ./src/diff --git a/Makefile b/Makefileindex d773c6a07..20e2a4634 100644--- a/Makefile+++ b/Makefile@@ -104,6 +104,7 @@ endef.PHONY: help check_tools help_docker help_dev help_test help_local help_utils help_operator \dev dev-cpu dev-local dev-local-cpu dev-local-build-lf dev-local-build-lf-cpu stop clean build logs \azurite-up azurite-down \+ instana-agent-up instana-agent-down \shell-backend shell-frontend install \test test-unit test-integration test-ci test-ci-local test-ci-suite test-sdk test-os-jwt lint \ci-build-images ci-save-images \@@ -360,6 +361,8 @@ help_local: ## Show local development commands@echo " $(PURPLE)make docling-stop$(NC) - Stop docling-serve"@echo " $(PURPLE)make azurite-up$(NC) - Start Azurite (local Azure Blob emulator) for connector testing"@echo " $(PURPLE)make azurite-down$(NC) - Stop Azurite emulator"+ @echo " $(PURPLE)make instana-agent-up$(NC) - Start the local Instana host agent (forwards to INSTANA_AGENT_ENDPOINT)"+ @echo " $(PURPLE)make instana-agent-down$(NC) - Stop the local Instana host agent"@echo ''@echo "$(PURPLE)Installation:$(NC)"@echo " $(PURPLE)make install$(NC) - Install all dependencies"@@ -869,6 +872,18 @@ azurite-down: ## Stop Azurite emulator$(COMPOSE_CMD) --profile azurite stop azurite@echo "$(PURPLE)Azurite stopped.$(NC)"+instana-agent-up: ## Start the local Instana host agent (forwards to INSTANA_AGENT_ENDPOINT)+ @echo "$(YELLOW)Starting Instana host agent...$(NC)"+ @echo "$(RED)WARNING: Local development only on Linux. Runs privileged, in the host PID namespace, with Docker socket access and uses host networking. Do not run on a shared or production host.$(NC)"+ @if [ -z "$$INSTANA_AGENT_KEY" ]; then echo "$(RED)INSTANA_AGENT_KEY not set in $(ENV_FILE)$(NC)"; exit 1; fi+ $(COMPOSE_CMD) --profile instana up -d instana-agent+ @echo "$(PURPLE)Instana agent started (listening on 127.0.0.1:42699).$(NC)"++instana-agent-down: ## Stop the local Instana host agent+ @echo "$(YELLOW)Stopping Instana agent...$(NC)"+ $(COMPOSE_CMD) --profile instana stop instana-agent+ @echo "$(PURPLE)Instana agent stopped.$(NC)"+####################### INSTALLATION######################diff --git a/docker-compose.yml b/docker-compose.ymlindex 4413de1fa..cb902ad35 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -125,6 +125,47 @@ services:- DOCLING_SERVE_URL=${DOCLING_SERVE_URL:-http://host.docker.internal:5001}- DOCLING_SERVE_VERIFY_SSL=${DOCLING_SERVE_VERIFY_SSL:-true}- CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS-http://localhost:3000}+ # Instana APM (opt-in, default off — see the INSTANA_* block in .env.example).+ # The agent runs on the HOST (`make instana-agent-up`, host networking), so a+ # containerized backend reaches it via the host.docker.internal alias mapped+ # in extra_hosts below — not via 127.0.0.1, which is the container loopback.+ - INSTANA_ENABLED=${INSTANA_ENABLED:-false}+ - INSTANA_AGENT_HOST=${INSTANA_AGENT_HOST:-host.docker.internal}+ - INSTANA_AGENT_PORT=${INSTANA_AGENT_PORT:-42699}+ # Performance guardrails — see the INSTANA_* block in .env.example for the+ # measurements behind these two defaults. Forwarded with a default rather+ # than bare so the safe value applies even when .env predates them:+ # `logging` avoids an unbounded per-warning span/event leak in the tracer,+ # `error` avoids a full stack capture on every httpx/urllib3/sqlalchemy+ # exit span.+ #+ # The two substitution forms differ on purpose. `-` (unset only) lets an+ # explicit empty value through, which is how you opt back into tracing+ # logging spans. `:-` (unset or empty) is required for the stack-trace+ # level because the tracer rejects an empty string, logging "Invalid+ # stack-trace value" on every boot and silently reverting to `all` — the+ # expensive setting this exists to avoid. Opt out with `all`, not ``.+ - INSTANA_TRACING_DISABLE=${INSTANA_TRACING_DISABLE-logging}+ - INSTANA_STACK_TRACE=${INSTANA_STACK_TRACE:-error}+ # Redact free-text query parameters before spans leave the host — a+ # privacy default, not a performance one. The tracer's own default+ # (contains-ignore-case:key,pass,secret) only catches credential-shaped+ # names, so without this OpenRAG's search text+ # (GET /v2/files/search?q=..., and the `search`/`filename` params) is+ # exported to Instana verbatim. Forwarded with `:-` (unset or empty),+ # same reasoning as INSTANA_STACK_TRACE above: an empty value fails the+ # tracer's own parser and falls back to the weak built-in default, so an+ # explicit empty override must not silently defeat this. Uses `\Z`+ # rather than a `$`-anchor (see the NOTE in .env.example) so the value+ # survives the Makefile's `include`/`export` of `.env` unmangled.+ - INSTANA_SECRETS=${INSTANA_SECRETS:-regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z}+ # Listed bare rather than as `VAR=${VAR}` on purpose: the tracer tests these+ # for *presence*, not truthiness, so forwarding an empty string would set a+ # blank service name and log an "Unknown INSTANA_LOG_LEVEL" warning on every+ # boot. The bare form is omitted from the container entirely when unset.+ - INSTANA_SERVICE_NAME+ - INSTANA_LOG_LEVEL+ - INSTANA_ZONEvolumes:- ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:U,z- ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,z@@ -244,6 +285,52 @@ services:volumes:- azurite-data:/data+ # Instana host agent: forwards traces/metrics to the Instana SaaS ingress+ # named by INSTANA_AGENT_ENDPOINT. The default below is one regional ingress;+ # get the right one for your tenant from Infrastructure > Add a Host, which+ # also issues the INSTANA_AGENT_KEY the agent needs.+ # Opt-in only — gated behind the "instana" compose profile.+ #+ # WARNING: local development only. This container runs privileged, in the host+ # PID namespace, with a read-write Docker socket — any one of those is+ # root-equivalent on the host. The agent needs them for host/infrastructure+ # metrics, but do not run this on a shared or production host.+ #+ # Start it with `make instana-agent-up`. Uses host networking so the agent+ # publishes 42699 directly on localhost, matching the dev-local topology+ # (backend runs on the host via `make backend`, not in Docker).+ # NOTE: flags below are a starting point — re-verify against the exact+ # Docker install command from the tenant's own "Add a Host Agent" wizard+ # (Infrastructure > Add a Host) before relying on this, since IBM adjusts+ # recommended flags between agent releases.+ instana-agent:+ # Pinned rather than :latest — this container runs privileged, with host PID+ # and a Docker socket mount, so an upstream retag would silently change what+ # runs as root on a developer machine. Override with INSTANA_AGENT_IMAGE_TAG+ # to test a newer agent; bump the default here once it is verified.+ image: icr.io/instana/agent:${INSTANA_AGENT_IMAGE_TAG:-1.323.1}+ container_name: instana-agent+ profiles: ["instana"]+ network_mode: host+ pid: host+ privileged: true+ environment:+ - INSTANA_AGENT_ENDPOINT=${INSTANA_AGENT_ENDPOINT:-ingress-coral-saas.instana.io}+ - INSTANA_AGENT_ENDPOINT_PORT=${INSTANA_AGENT_ENDPOINT_PORT:-443}+ - INSTANA_AGENT_KEY=${INSTANA_AGENT_KEY}+ volumes:+ # docker.sock stays read-write: talking to the Docker API means writing to+ # the socket, so :ro here would break container discovery. The rest are+ # :ro because the agent only reads them. Note this narrows scope rather+ # than enforcing a boundary — `privileged: true` above lets the container+ # remount them writable. If a future agent release needs write access,+ # these flags are the first thing to revert.+ - /var/run/docker.sock:/var/run/docker.sock+ - /dev:/dev:ro+ - /sys:/sys:ro+ - /var/log:/var/log:ro+ - /var/lib/docker/containers:/var/lib/docker/containers:ro+volumes:opensearch-data:azurite-data:\ No newline at end of filediff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdxindex 4c88e5a44..b682b004d 100644--- a/docs/docs/reference/configuration.mdx+++ b/docs/docs/reference/configuration.mdx@@ -198,3 +198,40 @@ Configure general system components, session management, and logging.| `LOG_LEVEL` | `INFO` | Logging level. Can be one of `DEBUG`, `INFO`, `WARNING`, `ERROR`, or `CRITICAL`. `DEBUG` provides the most detailed logs but can impact performance. || `SERVICE_NAME` | `openrag` | Service name used in log entries. || `SESSION_SECRET` | Automatically generated | Secret key for session management. If not provided, a secure random key is generated automatically. |++## Observability and APM settings {#observability-and-apm-settings}++Instrument OpenRAG Backend with [IBM Instana](https://www.ibm.com/products/instana) for application performance monitoring and distributed tracing.+Instrumentation is off by default.++For a complete walkthrough, including how to obtain an agent key and how to configure Kubernetes deployments, see [Monitor OpenRAG with IBM Instana](/reference/observability).++The first group configures OpenRAG Backend, which sends traces to an Instana host agent.++| Variable | Default | Description |+|----------|---------|-------------|+| `INSTANA_ENABLED` | `false` | Whether to load the Instana Python tracer in OpenRAG Backend. Accepts `true`, `1`, or `yes` to enable. This variable is read once at process start, so the backend must be restarted for a change to take effect. The tracer is an optional dependency: the backend container image includes it, and a backend run from source needs `uv sync --extra apm`. If it isn't installed, the backend logs a warning and runs untraced. |+| `INSTANA_AGENT_HOST` | `host.docker.internal` for containerized deployments; `localhost` for a backend that runs directly on the host | Address of the Instana host agent that the tracer sends spans to. In Kubernetes, leave this unset. The Helm chart and the operator inject the pod's own node IP, which is the address of the per-node agent DaemonSet. |+| `INSTANA_AGENT_PORT` | `42699` | Port of the Instana host agent. |+| `INSTANA_SERVICE_NAME` | Not set | Service name that OpenRAG Backend reports to Instana, shown in the **Applications** and **Analyze Traces** views. Leave this unset rather than setting it to an empty value, because the tracer checks it for presence rather than for a value, and an empty value registers a service with a blank name. |+| `INSTANA_ZONE` | Not set | Custom zone that the host and process are grouped under in the Instana **Infrastructure** view. Presence-sensitive in the same way as `INSTANA_SERVICE_NAME`. |+| `INSTANA_LOG_LEVEL` | `info` | Verbosity of the tracer's own logging, as one of `debug`, `info`, `warn`, or `error`. This is unrelated to [`LOG_LEVEL`](#system-settings), which controls OpenRAG's application logs. Presence-sensitive in the same way as `INSTANA_SERVICE_NAME`: an empty value logs an `Unknown INSTANA_LOG_LEVEL` warning on every start. |+| `INSTANA_SECRETS` | `regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z` | Query-string parameters that the tracer replaces with `<redacted>` before a span is sent, as `<matcher>:<name>[,<name>]` where the matcher is one of `equals`, `equals-ignore-case`, `contains`, `contains-ignore-case`, or `regex`. OpenRAG sets this rather than leaving it at the tracer's own `contains-ignore-case:key,pass,secret`, which only covers credential-shaped parameter names and not free-text ones such as the `q` of `GET /v2/files/search`. See [Redact query parameters](/reference/observability#redact-query-parameters). |+| `INSTANA_TRACING_DISABLE` | `logging` | Span categories or types that the tracer skips, as a comma-separated list, or `true` to disable tracing entirely. OpenRAG sets `logging` rather than leaving it empty, because a logging span is created for every `WARNING` and `ERROR` logged inside a request trace, at a cost that grows for as long as the process runs. See [Performance guardrails](/reference/observability#performance-guardrails). |+| `INSTANA_STACK_TRACE` | `error` | When the tracer captures a Python stack trace for an exit span, as one of `all`, `error`, or `none`. OpenRAG sets `error` rather than the tracer's own `all`, which captures a full stack on every outbound HTTP call, OpenSearch query, and database query. Any other value logs a warning and falls back to `all`, so use `all` to opt out rather than an empty value. See [Performance guardrails](/reference/observability#performance-guardrails). |+| `INSTANA_STACK_TRACE_LENGTH` | `30` | Maximum number of frames kept in a captured stack trace, between 1 and 40. Only has an effect when `INSTANA_STACK_TRACE` captures a stack. |++The second group configures the optional local Instana host agent that `make instana-agent-up` starts.+These variables don't affect the backend, and they aren't used by Kubernetes deployments, where the agent is installed separately as a DaemonSet.++| Variable | Default | Description |+|----------|---------|-------------|+| `INSTANA_AGENT_KEY` | Not set | Agent key for your Instana tenant, from **Infrastructure > Add a Host** in the Instana UI. Required by `make instana-agent-up`, which fails if the key isn't set. |+| `INSTANA_AGENT_ENDPOINT` | `ingress-coral-saas.instana.io` | Ingress endpoint that the host agent forwards data to. This value is region-specific, so use the endpoint that your tenant's **Add a Host** wizard shows rather than the default. |+| `INSTANA_AGENT_ENDPOINT_PORT` | `443` | Port of the ingress endpoint. |+| `INSTANA_AGENT_IMAGE_TAG` | `1.323.1` | Tag of the `icr.io/instana/agent` image that the local host agent runs. The tag is pinned rather than tracked on `latest` because the agent container runs privileged. For available tags, see [the agent's tag list](https://icr.io/v2/instana/agent/tags/list). |++:::warning+`make instana-agent-up` starts a privileged container in the host PID namespace with access to the Docker socket, which is effectively root access to the host.+It's intended for local development machines only. Don't run it on a shared or production host.+:::diff --git a/docs/docs/reference/observability.mdx b/docs/docs/reference/observability.mdxnew file mode 100644index 000000000..f5d7b175b--- /dev/null+++ b/docs/docs/reference/observability.mdx@@ -0,0 +1,265 @@+---+title: Monitor OpenRAG with IBM Instana+slug: /reference/observability+---++OpenRAG Backend can be instrumented with [IBM Instana](https://www.ibm.com/products/instana) for application performance monitoring (APM) and distributed tracing.++Instrumentation is off by default. When you set `INSTANA_ENABLED=true`, the backend loads Instana's Python tracer at startup, before the libraries the tracer instruments, and traces are collected automatically for:++* Incoming API requests to the backend (FastAPI and Starlette).+* Outgoing HTTP calls to Langflow, Docling, and model providers.+* OpenSearch queries.+* Database queries issued through SQLAlchemy, for both the default SQLite database and PostgreSQL.+* Backend log records, when logging spans are enabled — off by default via `INSTANA_TRACING_DISABLE=logging` (see [Logging spans](#logging-spans)).++Only OpenRAG Backend is instrumented. Langflow, the frontend, OpenSearch, and Docling Serve aren't traced by this feature. Instana's host agent reports infrastructure metrics for those containers separately.++## How traces reach Instana {#how-traces-reach-instana}++The tracer doesn't send data to your Instana tenant directly.+It sends spans to an Instana _host agent_ on port `42699`, and the agent forwards them to your tenant's ingress endpoint:++```text+OpenRAG Backend ──▶ Instana host agent ──▶ ingress-REGION-saas.instana.io:443+ (tracer) (:42699) (your Instana tenant)+```++Enabling APM therefore has two parts: configure the backend, and make sure an agent is reachable at `INSTANA_AGENT_HOST:INSTANA_AGENT_PORT`.+The correct agent address depends on where the backend process runs:++| Deployment | Where the agent runs | `INSTANA_AGENT_HOST` |+|---|---|---|+| [Self-managed containers](/docker) | On the Docker host, started by `make instana-agent-up` | `host.docker.internal`, the Docker Compose default |+| Backend on the host, such as `make backend` for [contributors](/support/contribute) | On the same host | `127.0.0.1` |+| Kubernetes | An `instana-agent` DaemonSet, one agent per node behind host port `42699` | The pod's own node IP, injected automatically |++## What gets exported {#what-gets-exported}++Trace data leaves your deployment: the tracer sends it to the host agent, and the agent forwards it to your Instana tenant.+Before you enable APM against a real corpus, it's worth knowing what a span carries.++Spans **don't** carry request or response bodies.+Chat prompts, retrieved chunk text, and uploaded document content are sent to the backend in request bodies, so they stay inside your deployment.++Spans **do** carry:++* The request path and method, such as `POST /v1/chat`, and the query string.+* Outgoing request URLs for calls to Langflow, Docling, model providers, and OpenSearch.+* SQL statement text for database queries. Statements are parameterized, so bound values aren't included, and database credentials are stripped from the connection URL.+* Log messages at `WARNING` level and above, when logging spans are enabled — off by default (see [Logging spans](#logging-spans)). A warning or an error that interpolates a filename, a user query, or chunk text exports that text.++The query string is the surface to pay attention to, because a few OpenRAG endpoints take free text there.+`GET /v2/files/search?q=...` carries a user's search terms, and file listing takes `search` and `filename` parameters.++### Redact query parameters {#redact-query-parameters}++`INSTANA_SECRETS` controls which query parameters the tracer replaces with `<redacted>` before a span is sent. Its format is `<matcher>:<name>[,<name>]`, where the matcher is one of `equals`, `equals-ignore-case`, `contains`, `contains-ignore-case`, or `regex`.++The tracer's own default is `contains-ignore-case:key,pass,secret`, which covers credential-shaped parameter names such as `api_key` but leaves free-text parameters untouched — including the `q`, `search`, and `filename` parameters above. OpenRAG ships a wider value instead, so this redaction applies out of the box; it's set for you in `.env.example`, `docker-compose.yml`, the Helm chart, and the operator:++```bash+INSTANA_SECRETS=regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z+```++`q`, `search`, and `filename` are anchored with `\Z` (end-of-string) rather+than `$`: the tracer matches with Python's `re.match()`, which already+anchors at the start of the string, and `.env` is parsed as GNU Make syntax+when loaded via `make backend` / `make dev` — a literal `$` there is+interpreted as a Make variable reference and silently corrupted.++The matcher applies to the parameter *name*, not its value, and only one matcher is in effect at a time.+To fall back to the tracer's own weaker default, set this to an empty value; the tracer logs a `Couldn't parse INSTANA_SECRETS` warning each time you do.++:::warning+Don't set `INSTANA_EXTRA_HTTP_HEADERS`.+It tells the tracer to copy the named request headers onto spans, and OpenRAG's headers carry session cookies, API keys, and bearer tokens.+:::++If your deployment can't export this data at all, keep `INSTANA_ENABLED=false`. There is no partial mode in which the backend is traced but request metadata stays local.++## Before you begin {#before-you-begin}++You need an Instana tenant, an agent key, and the ingress endpoint for your tenant's region.++To get the key and endpoint, open your Instana tenant and go to **Infrastructure > Add a Host**, then select the Docker installation method.+The command that the wizard shows you contains both values as `INSTANA_AGENT_KEY` and `INSTANA_AGENT_ENDPOINT`.++The ingress endpoint is region-specific, such as `ingress-coral-saas.instana.io`.+Use the value from your own wizard rather than the OpenRAG default, or the agent connects to the wrong region and reports no data.++### Install the tracer {#install-the-tracer}++The Instana Python tracer is an optional dependency, declared as the `apm` extra, so that deployments that never enable APM don't install it.++The OpenRAG Backend container image includes the extra already, so [self-managed](/docker) and Kubernetes deployments need nothing extra.++If you run the backend directly from source, such as `make backend` for [contributors](/support/contribute), install the extra once:++```bash+uv sync --extra apm+```++If `INSTANA_ENABLED` is set and the package isn't installed, the backend logs a warning at startup and runs without tracing rather than failing to start.++## Enable tracing in a self-managed deployment {#enable-tracing}++1. Add the Instana variables to your `.env` file:++ ```bash+ INSTANA_ENABLED=true+ INSTANA_AGENT_KEY=YOUR_AGENT_KEY+ INSTANA_AGENT_ENDPOINT=ingress-coral-saas.instana.io+ INSTANA_AGENT_ENDPOINT_PORT=443+ INSTANA_SERVICE_NAME="OpenRAG Backend"+ INSTANA_ZONE="openrag-dev"+ ```++ `INSTANA_SERVICE_NAME` and `INSTANA_ZONE` are optional, but they make OpenRAG easier to find in the Instana UI.+ For the complete list of variables, see [Observability and APM settings](/reference/configuration#observability-and-apm-settings).++ :::warning+ Don't set the optional variables to an empty value.+ The tracer checks `INSTANA_SERVICE_NAME`, `INSTANA_ZONE`, and `INSTANA_LOG_LEVEL` for presence rather than for a value, so `INSTANA_SERVICE_NAME=` registers a service with a blank name, and `INSTANA_LOG_LEVEL=` logs an `Unknown INSTANA_LOG_LEVEL` warning on every start.+ To leave a variable at its default, comment it out or remove it.+ :::++2. Start the Instana host agent:++ ```bash+ make instana-agent-up+ ```++ The target fails if `INSTANA_AGENT_KEY` isn't set. For what this agent does to your machine, see [Run the local host agent](#local-host-agent).++ If your agent runs somewhere else already, skip this step and set `INSTANA_AGENT_HOST` to that agent's address instead.++3. [Restart the OpenRAG services](/manage-services) so the backend picks up the new configuration.++ `INSTANA_ENABLED` is read once, at process start. Changing it requires a restart of the backend, not just a page reload.++4. Confirm that the backend loaded the tracer:++ ```bash+ docker compose logs openrag-backend | grep -i instana+ ```++5. In your Instana tenant, open **Applications** and look for your `INSTANA_SERVICE_NAME` value, or open **Analyze > Traces** and send a request through OpenRAG.++ Traces can take a minute or two to appear after the agent first connects.++To turn instrumentation off again, set `INSTANA_ENABLED=false` or remove it, restart the services, and run `make instana-agent-down`.++## Run the local host agent {#local-host-agent}++OpenRAG ships the host agent as an opt-in Docker Compose service under the `instana` profile, so it never starts as part of `make dev` or `docker compose up`.++| Command | Description |+|---|---|+| `make instana-agent-up` | Starts the `instana-agent` container. Requires `INSTANA_AGENT_KEY`. The agent listens on `127.0.0.1:42699` and forwards to `INSTANA_AGENT_ENDPOINT`. |+| `make instana-agent-down` | Stops the `instana-agent` container. |++:::warning+The host agent container runs as `privileged`, in the host PID namespace, with host networking and a read-write mount of the Docker socket.+The agent needs this access to monitor host and container infrastructure, and the same access is effectively root on the machine that runs it.++Use these targets on a local development machine only.+Don't run them on a shared or production host. Deploy the agent there through whatever process your organization uses for host agents.+:::++:::info+The agent service uses `network_mode: host`, which is supported on Linux Docker hosts, and on Docker Desktop only when host networking is enabled.+On other platforms, the container can start without publishing port `42699` on the host, and the backend then reports no traces even though the agent appears to be running.+On those platforms, install the agent natively with the wizard's non-Docker instructions, or point `INSTANA_AGENT_HOST` at an agent running elsewhere.+:::++The agent image is pinned in `docker-compose.yml` rather than tracked on `latest`, because a privileged container shouldn't change underneath you on a re-pull.+To try a newer agent without editing Compose, set `INSTANA_AGENT_IMAGE_TAG` to a tag from [the agent's tag list](https://icr.io/v2/instana/agent/tags/list).++## Enable tracing in Kubernetes {#kubernetes}++The OpenRAG Helm chart configures the backend tracer under `backend.instana`:++```yaml+backend:+ instana:+ enabled: true+ agentPort: 42699+ serviceName: "OpenRAG Backend"+ zone: "openrag-cpd"+```++Leave `agentHost` empty. The chart then injects the pod's own node IP through the Kubernetes Downward API, which is the address that the standard per-node agent DaemonSet expects.+Set `agentHost` explicitly only if your agent isn't a DaemonSet and is reached at a fixed address, such as through a Service.++The same presence rule applies as in `.env`: `serviceName`, `zone`, and `logLevel` are omitted from the rendered configuration when they're empty, so leave them empty rather than setting them to `""` deliberately.++`tracingDisable`, `stackTrace`, and `secrets` don't follow that rule. All three ship with a real, non-empty chart default (`logging`, `error`, and the redaction regex in [Redact query parameters](#redact-query-parameters) — see also [Performance guardrails](#performance-guardrails)), so leaving them out of your values override doesn't disable the guardrail or weaken the redaction; the chart's default still gets written. To get the tracer's own default instead, override them explicitly: `tracingDisable: ""`, `stackTrace: "all"`, or `secrets: ""`.++:::important+The OpenRAG chart doesn't install an Instana agent.+Install one separately with IBM's `instana-agent` Helm chart or operator.+The agent is a privileged, host-PID DaemonSet, so installing it is a cluster administration task.+:::++### Enable tracing with the OpenRAG operator {#operator}++For deployments managed by the [OpenRAG operator](https://github.com/langflow-ai/openrag/tree/main/kubernetes/operator), set the variables in the `OpenRAG` custom resource:++```yaml+apiVersion: openr.ag/v1alpha1+kind: OpenRAG+metadata:+ name: my-openrag+spec:+ backend:+ env:+ - name: INSTANA_ENABLED+ value: "true"+ - name: INSTANA_SERVICE_NAME+ value: "OpenRAG Backend"+ - name: INSTANA_ZONE+ value: "openrag-cpd"+```++Don't set `INSTANA_AGENT_HOST`. When `INSTANA_ENABLED` is truthy, the operator injects it from the pod's node IP, and setting it in the custom resource suppresses that injection.++## Configure the tracer's own logging {#tracer-logging}++`INSTANA_LOG_LEVEL` controls how much the tracer itself logs, as one of `debug`, `info`, `warn`, or `error`.+It's unrelated to [`LOG_LEVEL`](/reference/configuration#system-settings), which controls OpenRAG's application logs.+Set it to `debug` when you're diagnosing why traces don't reach Instana, and leave it unset otherwise.++## Performance guardrails {#performance-guardrails}++Two of the tracer's own defaults are expensive in a long-running backend, so OpenRAG ships a different value for each. Both are set for you in `.env.example`, `docker-compose.yml`, the Helm chart, and the operator — you only need to change them to opt out.++### Logging spans {#logging-spans}++`INSTANA_TRACING_DISABLE=logging` stops the tracer from creating a span for every `WARNING` and `ERROR` that OpenRAG logs while handling a request.++Left enabled, each of those spans records an event in a list that the tracer shares across all spans and never frees, and every later logging span walks the whole list before it's reported. The cost per logged warning therefore climbs for as long as the process runs, and the memory is never returned. Measured on the OpenRAG backend, a single warning logged inside a request trace costs about 1 ms with logging spans on and about 26 μs with them off.++You lose no information by disabling them. OpenRAG's own structured logs, controlled by [`LOG_LEVEL`](/reference/configuration#system-settings), already record the same warnings and errors, with more context than the span carries.++To trace logging spans anyway, set the variable to an empty value rather than removing it:++```bash+INSTANA_TRACING_DISABLE=+```++### Stack traces on exit spans {#exit-span-stack-traces}++`INSTANA_STACK_TRACE=error` limits stack capture to spans that actually failed.++An exit span is created for every call OpenRAG makes out of the process — every OpenSearch query, every request to Langflow, and every database query. At the tracer's own default of `all`, each one captures a full Python stack trace and reads the matching source lines from disk. OpenRAG's async request stack is deep enough that this costs 200 to 350 μs per exit span, and a single chat request makes tens of them.++`error` keeps the stack trace where it's diagnostic and drops it from the successful path. To capture stacks on every exit span, set `all` — not an empty value, which the tracer rejects with a warning and then falls back to `all` anyway:++```bash+INSTANA_STACK_TRACE=all+```++## Environment variable reference {#variables}++For every variable in this feature, including defaults, see [Observability and APM settings](/reference/configuration#observability-and-apm-settings).diff --git a/docs/sidebars.js b/docs/sidebars.jsindex d04c76060..5a65e51a5 100644--- a/docs/sidebars.js+++ b/docs/sidebars.js@@ -74,6 +74,11 @@ const sidebars = {label: "Chat",},"reference/configuration",+ {+ type: "doc",+ id: "reference/observability",+ label: "Monitor with IBM Instana",+ },{type: "doc",id: "reference/api-sdk-overview",diff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yamlindex 236b1452a..77b7235ec 100644--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml@@ -247,4 +247,32 @@ stringData:SERVICE_NAME={{ .Values.backend.serviceName | quote }}{{- end }}SESSION_SECRET={{ $sessionSecret | quote }}++ # Instana APM+ # INSTANA_AGENT_HOST is deliberately absent: it is the pod's own node IP,+ # which this file cannot know at render time. The backend Deployment injects+ # it through the Downward API instead. bootstrap.py loads .env with+ # override=False, so that container env var wins over anything here.+ INSTANA_ENABLED={{ ternary "true" "false" .Values.backend.instana.enabled }}+ {{- if .Values.backend.instana.enabled }}+ INSTANA_AGENT_PORT={{ .Values.backend.instana.agentPort }}+ {{- if .Values.backend.instana.serviceName }}+ INSTANA_SERVICE_NAME={{ .Values.backend.instana.serviceName | quote }}+ {{- end }}+ {{- if .Values.backend.instana.logLevel }}+ INSTANA_LOG_LEVEL={{ .Values.backend.instana.logLevel | quote }}+ {{- end }}+ {{- if .Values.backend.instana.zone }}+ INSTANA_ZONE={{ .Values.backend.instana.zone | quote }}+ {{- end }}+ {{- if .Values.backend.instana.tracingDisable }}+ INSTANA_TRACING_DISABLE={{ .Values.backend.instana.tracingDisable | quote }}+ {{- end }}+ {{- if .Values.backend.instana.stackTrace }}+ INSTANA_STACK_TRACE={{ .Values.backend.instana.stackTrace | quote }}+ {{- end }}+ {{- if .Values.backend.instana.secrets }}+ INSTANA_SECRETS={{ .Values.backend.instana.secrets | quote }}+ {{- end }}+ {{- end }}{{- end }}diff --git a/kubernetes/helm/openrag/templates/backend/deployment.yaml b/kubernetes/helm/openrag/templates/backend/deployment.yamlindex 80c3a295a..27c93d288 100644--- a/kubernetes/helm/openrag/templates/backend/deployment.yaml+++ b/kubernetes/helm/openrag/templates/backend/deployment.yaml@@ -214,6 +214,23 @@ spec:containerPort: {{ .Values.backend.ingestRouter.port }}protocol: TCP{{- end }}+ {{- if .Values.backend.instana.enabled }}+ # The backend otherwise takes its whole configuration from the mounted+ # .env file. Instana is the one exception: its agent runs as a+ # per-node DaemonSet behind a hostPort, so the address is the pod's+ # own node IP — a value that does not exist when the .env Secret is+ # rendered. bootstrap.py loads .env with override=False, so this+ # container env var takes precedence.+ env:+ - name: INSTANA_AGENT_HOST+ {{- if .Values.backend.instana.agentHost }}+ value: {{ .Values.backend.instana.agentHost | quote }}+ {{- else }}+ valueFrom:+ fieldRef:+ fieldPath: status.hostIP+ {{- end }}+ {{- end }}volumeMounts:- name: openrag-dotenvmountPath: /app/.envdiff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yamlindex f1033b343..592cf6349 100644--- a/kubernetes/helm/openrag/values.yaml+++ b/kubernetes/helm/openrag/values.yaml@@ -381,6 +381,50 @@ backend:enabled: falseport: 8100+ # Instana APM (opt-in, default off).+ #+ # Requires an Instana agent already running in the cluster — install it+ # separately with IBM's own instana-agent chart or operator. This chart never+ # deploys an agent: the agent needs a privileged, host-PID DaemonSet, which is+ # a cluster-admin concern, not an application one.+ instana:+ enabled: false+ # Agent address. Leave empty to use the pod's own node IP via the Downward+ # API, which is what the standard agent DaemonSet (hostPort 42699) expects.+ # Set explicitly only when routing to a fixed address instead, e.g. a+ # Service in front of a non-DaemonSet agent.+ agentHost: ""+ agentPort: 42699+ # All optional. Left empty they are omitted from the rendered .env entirely+ # rather than written as an empty value: the tracer tests these for+ # presence, not truthiness, so an empty string would set a blank service+ # name and log an "Unknown INSTANA_LOG_LEVEL" warning on every boot.+ serviceName: "" # Shown in Analyze Traces / Applications+ logLevel: "" # Tracer's own verbosity: debug|info|warn|error+ zone: "" # Custom zone in the Infrastructure view+ # Performance guardrails, plus one privacy default (secrets, below). Unlike+ # the three vars above these default to a real value rather than empty,+ # because the tracer's own defaults are either costly or, for secrets,+ # under-redacting — see the INSTANA_* block in .env.example for the+ # measurements. Set any of them to "" to fall back to the tracer's own+ # default.+ #+ # tracingDisable: span categories/types not to trace. `logging` avoids an+ # unbounded event-list leak (and O(n^2) CPU) on every WARNING/ERROR logged+ # inside a request trace, at no loss of information — the same events are+ # already in OpenRAG's structured logs.+ tracingDisable: "logging"+ # stackTrace: when to capture a Python stack for EXIT spans (all|error|none).+ # `all` means a full traceback on every httpx / urllib3 / sqlalchemy call.+ # Anything other than all|error|none logs a warning and falls back to `all`.+ stackTrace: "error"+ # secrets: query-string parameter names the tracer redacts before a span+ # leaves the host. Not a performance concern like the two above — this is+ # privacy: the tracer's own default only catches credential-shaped names,+ # so without it OpenRAG's search text (`q`, `search`, `filename` query+ # parameters) is exported to your Instana tenant verbatim.+ secrets: "regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\\Z,search\\Z,filename\\Z"+# ============================================================================# OpenRAG Frontend Configuration# ============================================================================diff --git a/kubernetes/operator/internal/controller/ENV_OVERRIDE.md b/kubernetes/operator/internal/controller/ENV_OVERRIDE.mdindex d98db513d..359345b38 100644--- a/kubernetes/operator/internal/controller/ENV_OVERRIDE.md+++ b/kubernetes/operator/internal/controller/ENV_OVERRIDE.md@@ -218,7 +218,49 @@ When the operator environment contains `OPTLF_LANGFLOW_WORKERS=8`, the prefix `O### ValueFrom Support-CR env vars using `valueFrom` (e.g., secrets, configmaps) are NOT evaluated at this stage. They are passed directly to the pod spec and resolved by Kubernetes at runtime. Only env vars with direct `value` fields are merged into the `.env` file.+CR env vars using `valueFrom` are resolved by the operator **during reconcile** and merged into the `.env` file like any other value — they are not passed through to the pod spec:++| `valueFrom` source | Behaviour |+|---|---|+| `secretKeyRef` | Read from the cluster and written into `.env`. Missing keys error unless `optional: true`. |+| `configMapKeyRef` | Same. |+| `fieldRef` | **Rejected** with an error — there is no pod yet at reconcile time. |+| `resourceFieldRef` | **Rejected**, same reason. |++Resolving secrets into the file rather than the container's `Env` is deliberate: it keeps credentials from showing up when someone runs `env` inside the pod.++### Downward API Exception: Instana++The backend container's `Env` is empty by design. One variable breaks that rule.++Instana's Kubernetes topology is one agent per node behind a hostPort, so the tracer's agent address is the pod's **own node IP** — exactly the kind of value `fieldRef` exists for, and exactly what the table above cannot express. `InstanaAgentHostEnvVar` therefore injects `INSTANA_AGENT_HOST` onto the container via `fieldRef: status.hostIP`. This is safe to layer on top of `.env` because the backend's `bootstrap.py` calls `load_dotenv(override=False)`, so a real container env var wins over the file. A node IP is not a credential, so the rationale above is unaffected.++Enable it through the CR:++```yaml+spec:+ backend:+ env:+ - name: INSTANA_ENABLED+ value: "true"+ # Optional. Omit rather than setting empty — the tracer tests these for+ # presence, not truthiness, so "" means a blank service name and an+ # "Unknown INSTANA_LOG_LEVEL" warning on every boot.+ - name: INSTANA_SERVICE_NAME+ value: "OpenRAG Backend"+ - name: INSTANA_ZONE+ value: "openrag-cpd"+```++`INSTANA_AGENT_HOST` is injected automatically and should be left unset. Setting it explicitly suppresses the injection and pins the tracer to that address instead — use that only for a non-DaemonSet agent reached through a Service.++`INSTANA_TRACING_DISABLE=logging` and `INSTANA_STACK_TRACE=error` are defaulted for you, and unlike the presence-sensitive vars above they carry a real value rather than being omitted. They are performance guardrails against two costly tracer defaults: a span per in-trace `WARNING`/`ERROR` whose bookkeeping grows for the life of the process, and a full Python stack capture on every outbound HTTP, OpenSearch, and database call. Override them through `spec.env` like any other backend var — with `all` rather than `""` for the stack-trace level, which the tracer rejects. See [Performance guardrails](https://docs.openr.ag/reference/observability#performance-guardrails).++`INSTANA_SECRETS` gets the same real-default treatment, for privacy rather than performance: the tracer's own default (`contains-ignore-case:key,pass,secret`) only redacts credential-shaped query-parameter names, so without it OpenRAG's search text (`q`, `search`, `filename` on `GET /v2/files/search` and file listing) is exported to your Instana tenant verbatim. Override it through `spec.env` like the two above.++Both decisions — whether to inject, and whether an explicit host already exists — are made from the **resolved** backend environment (the map `GetBackendEnvVars` returns), not from the raw `spec.env`. So `INSTANA_ENABLED` and `INSTANA_AGENT_HOST` behave the same whether they are literals, `secretKeyRef`, or `configMapKeyRef`: the operator sees exactly the value the backend will read out of its `.env`. Reading the raw `spec.env` instead would let the two disagree — a Secret-backed `INSTANA_ENABLED=true` would boot the tracer with no agent host, and a Secret-backed explicit host would be silently overridden by the injected node IP.++The operator never deploys an agent. Install one separately with IBM's `instana-agent` chart or operator: it needs a privileged, host-PID DaemonSet, which is a cluster-admin concern.## Testing@@ -230,9 +272,16 @@ See `env_test.go` for comprehensive test coverage of:- ValueFrom handling- Real-world scenarios+See `instana_test.go` for the Downward API exception:+- Default-off, and the truthiness values that enable it+- `status.hostIP` injection, and explicit-host suppression+- Operator-prefix precedence and `valueFrom` fallback+- The guard against giving presence-sensitive vars empty defaults+Run tests:```bashgo test -v ./internal/controller -run TestEnvVarManager+go test -v ./internal/controller -run Instana```## Best Practicesdiff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.goindex eb9d02d56..7103c3d2e 100644--- a/kubernetes/operator/internal/controller/env.go+++ b/kubernetes/operator/internal/controller/env.go@@ -157,6 +157,45 @@ func NewEnvVarManager() *EnvVarManager {"AZURE_STORAGE_ACCOUNT_NAME": "","AZURE_STORAGE_ACCOUNT_KEY": "","AZURE_STORAGE_ENDPOINT": "",++ // Instana APM (opt-in, default off). Requires an Instana agent+ // DaemonSet already running in the cluster; the operator never+ // deploys one — that is the cluster admin's responsibility.+ //+ // INSTANA_AGENT_HOST is intentionally omitted: it is the pod's own+ // node IP, which cannot be known when the .env file is rendered. It+ // is injected through the Downward API instead — see+ // InstanaAgentHostEnvVar.+ //+ // INSTANA_SERVICE_NAME / INSTANA_LOG_LEVEL / INSTANA_ZONE are also+ // intentionally omitted: the tracer tests those for *presence*, not+ // truthiness, and BuildEnvFileContent writes an empty default as+ // `KEY=`. That would set a blank service name and log an "Unknown+ // INSTANA_LOG_LEVEL" warning on every boot. Set them via spec.env.+ //+ // INSTANA_TRACING_DISABLE / INSTANA_STACK_TRACE are performance+ // guardrails, so unlike the vars above they carry a real default+ // instead of being omitted. The tracer's own defaults are costly on+ // a long-running backend: logging spans leak an event per in-trace+ // WARNING/ERROR into a list shared by all spans (never freed, and+ // re-walked by each new log span), and stack_trace=all captures a+ // full Python traceback on every httpx / urllib3 / sqlalchemy exit+ // span. See the INSTANA_* block in .env.example for the numbers.+ // Override via spec.env to reproduce the upstream behaviour.+ //+ // INSTANA_SECRETS carries a real default for a different reason:+ // privacy, not performance. The tracer's own default+ // (contains-ignore-case:key,pass,secret) only redacts+ // credential-shaped query-parameter names, so OpenRAG's free-text+ // search parameters (`q`, `search`, `filename` on+ // GET /v2/files/search and file listing) would otherwise be+ // exported to the Instana tenant verbatim. Override via spec.env+ // to reproduce that exposure.+ "INSTANA_ENABLED": "false",+ "INSTANA_AGENT_PORT": "42699",+ "INSTANA_TRACING_DISABLE": "logging",+ "INSTANA_STACK_TRACE": "error",+ "INSTANA_SECRETS": `regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z`,},DefaultOpenRagFEEnvVars: map[string]string{// Frontend environment variables will be added here@@ -315,6 +354,58 @@ func resolveEnvVarValue(ctx context.Context, c client.Client, namespace string,return "", false, nil}+// instanaAgentHostVar is the one backend variable that cannot live in the .env file.+const instanaAgentHostVar = "INSTANA_AGENT_HOST"++// InstanaAgentHostEnvVar returns the container-level env var that points the+// Instana tracer at the agent serving the pod's own node, or nil when it should+// not be injected.+//+// Every other backend variable is resolved into the mounted .env file (see+// mergeEnvVars). Instana is the exception: its Kubernetes topology is one agent+// per node behind a hostPort, so the agent address is the *node IP* — a value+// that does not exist when the .env file is rendered at reconcile time. That is+// also why spec.env rejects fieldRef (resolveEnvVarValue): there is no pod yet.+// So this single variable is injected through the Downward API on the container.+//+// The backend's bootstrap loads .env with override=False, so a real container+// env var wins over the file — which is what makes this layering work.+//+// backendEnv must be the *resolved* backend environment — the map GetBackendEnvVars+// returns, in which secretKeyRef/configMapKeyRef entries have already been read+// from the cluster. That is what the backend will actually see in its .env, so+// deciding from anything else lets the two disagree: a Secret-backed+// INSTANA_ENABLED=true would boot the tracer with no agent host, and a+// Secret-backed explicit host would be silently overridden by the fieldRef below.+//+// Returns nil when Instana is disabled, or when an explicit host was configured+// through any of the three levels, in which case that value stands.+func (m *EnvVarManager) InstanaAgentHostEnvVar(backendEnv map[string]string) *corev1.EnvVar {+ if !isTruthyEnvValue(backendEnv["INSTANA_ENABLED"]) {+ return nil+ }+ if backendEnv[instanaAgentHostVar] != "" {+ return nil+ }+ return &corev1.EnvVar{+ Name: instanaAgentHostVar,+ ValueFrom: &corev1.EnvVarSource{+ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.hostIP"},+ },+ }+}++// isTruthyEnvValue mirrors the backend's own gate in src/main.py, so the+// operator and the application agree on what "enabled" means.+func isTruthyEnvValue(v string) bool {+ switch strings.ToLower(strings.TrimSpace(v)) {+ case "true", "1", "yes":+ return true+ default:+ return false+ }+}+// BuildEnvFileContent converts a map of env vars to .env file format.// Values are written raw unless they contain characters that would break dotenv// parsing without quoting: newlines, carriage returns, or '#' (comment marker).diff --git a/kubernetes/operator/internal/controller/instana_test.go b/kubernetes/operator/internal/controller/instana_test.gonew file mode 100644index 000000000..9ac27be4d--- /dev/null+++ b/kubernetes/operator/internal/controller/instana_test.go@@ -0,0 +1,236 @@+package controller++import (+ "context"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ corev1 "k8s.io/api/core/v1"+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"+ "sigs.k8s.io/controller-runtime/pkg/client"+)++// The Instana agent address is the pod's own node IP, so it is the one backend+// variable that cannot be rendered into the mounted .env file. These tests pin+// the injection rules in InstanaAgentHostEnvVar.+//+// InstanaAgentHostEnvVar reads the *resolved* backend env — the map+// GetBackendEnvVars returns — so the decision cannot drift from what the backend+// will actually read out of its .env. The three-level precedence itself is+// mergeEnvVars' contract; the tests below exercise it through GetBackendEnvVars+// wherever the origin of a value is the thing under test.++// resolvedBackendEnv runs the real merge so these tests decide from the same map+// the reconciler hands to InstanaAgentHostEnvVar.+func resolvedBackendEnv(t *testing.T, crEnvVars []corev1.EnvVar, objs ...client.Object) map[string]string {+ t.Helper()++ r, c := reconciler(newScheme(t), objs...)+ env, err := r.EnvVarManager.GetBackendEnvVars(context.Background(), c, "test-ns", crEnvVars)+ require.NoError(t, err)+ return env+}++func TestInstanaAgentHostEnvVar_DisabledByDefault(t *testing.T) {+ m := NewEnvVarManager()++ assert.Nil(t, m.InstanaAgentHostEnvVar(resolvedBackendEnv(t, nil)),+ "Instana is opt-in; nothing should be injected when the flag is unset")+ assert.Equal(t, "false", m.DefaultOpenRagBEEnvVars["INSTANA_ENABLED"],+ "the default must keep the feature off")+}++func TestInstanaAgentHostEnvVar_InjectsNodeIPWhenEnabled(t *testing.T) {+ m := NewEnvVarManager()++ got := m.InstanaAgentHostEnvVar(map[string]string{"INSTANA_ENABLED": "true"})++ require.NotNil(t, got)+ assert.Equal(t, "INSTANA_AGENT_HOST", got.Name)+ assert.Empty(t, got.Value, "the value must come from the Downward API, not a literal")+ require.NotNil(t, got.ValueFrom)+ require.NotNil(t, got.ValueFrom.FieldRef)+ assert.Equal(t, "status.hostIP", got.ValueFrom.FieldRef.FieldPath)+}++func TestInstanaAgentHostEnvVar_TruthinessMatchesBackendGate(t *testing.T) {+ m := NewEnvVarManager()++ // Mirrors the gate in src/observability/instana_boot.py: ("true", "1", "yes"),+ // case-insensitive.+ for _, v := range []string{"true", "TRUE", "True", "1", "yes", "YES", " true "} {+ assert.NotNil(t, m.InstanaAgentHostEnvVar(map[string]string{"INSTANA_ENABLED": v}),+ "expected %q to enable Instana", v)+ }+ for _, v := range []string{"false", "0", "no", "", "off", "enabled"} {+ assert.Nil(t, m.InstanaAgentHostEnvVar(map[string]string{"INSTANA_ENABLED": v}),+ "expected %q to leave Instana disabled", v)+ }+}++func TestInstanaAgentHostEnvVar_ExplicitHostWins(t *testing.T) {+ m := NewEnvVarManager()++ got := m.InstanaAgentHostEnvVar(map[string]string{+ "INSTANA_ENABLED": "true",+ "INSTANA_AGENT_HOST": "instana-agent.instana-agent.svc",+ })++ assert.Nil(t, got,+ "an operator-injected fieldRef would override the .env value, so an explicit host must suppress it")+}++func TestInstanaAgentHostEnvVar_HonoursOperatorEnvPrefix(t *testing.T) {+ m := NewEnvVarManager()++ t.Setenv(OPENRAGBE_ENV_PREFIX+"INSTANA_ENABLED", "true")+ assert.NotNil(t, m.InstanaAgentHostEnvVar(resolvedBackendEnv(t, nil)),+ "level 2 (operator env) must be able to enable Instana on its own")++ // Level 3 (CR spec) still outranks level 2.+ assert.Nil(t, m.InstanaAgentHostEnvVar(+ resolvedBackendEnv(t, []corev1.EnvVar{{Name: "INSTANA_ENABLED", Value: "false"}})),+ "the CR spec must be able to turn it back off")+}++// A Secret- or ConfigMap-backed INSTANA_ENABLED reaches the backend through the+// .env file, so it must reach the host decision too. Deciding from the raw+// spec.Env instead would boot the tracer with no agent to talk to.+func TestInstanaAgentHostEnvVar_EnabledFromSecretRef(t *testing.T) {+ m := NewEnvVarManager()++ secret := &corev1.Secret{+ ObjectMeta: metav1.ObjectMeta{Name: "apm-config", Namespace: "test-ns"},+ Data: map[string][]byte{"instana-enabled": []byte("true")},+ }+ env := resolvedBackendEnv(t, []corev1.EnvVar{{+ Name: "INSTANA_ENABLED",+ ValueFrom: &corev1.EnvVarSource{+ SecretKeyRef: &corev1.SecretKeySelector{+ LocalObjectReference: corev1.LocalObjectReference{Name: "apm-config"},+ Key: "instana-enabled",+ },+ },+ }}, secret)++ require.Equal(t, "true", env["INSTANA_ENABLED"], "precondition: the .env gets the resolved value")+ assert.NotNil(t, m.InstanaAgentHostEnvVar(env),+ "a Secret-backed enable flag must still inject the agent host")+}++func TestInstanaAgentHostEnvVar_EnabledFromConfigMapRef(t *testing.T) {+ m := NewEnvVarManager()++ cm := &corev1.ConfigMap{+ ObjectMeta: metav1.ObjectMeta{Name: "apm-config", Namespace: "test-ns"},+ Data: map[string]string{"instana-enabled": "true"},+ }+ env := resolvedBackendEnv(t, []corev1.EnvVar{{+ Name: "INSTANA_ENABLED",+ ValueFrom: &corev1.EnvVarSource{+ ConfigMapKeyRef: &corev1.ConfigMapKeySelector{+ LocalObjectReference: corev1.LocalObjectReference{Name: "apm-config"},+ Key: "instana-enabled",+ },+ },+ }}, cm)++ require.Equal(t, "true", env["INSTANA_ENABLED"], "precondition: the .env gets the resolved value")+ assert.NotNil(t, m.InstanaAgentHostEnvVar(env),+ "a ConfigMap-backed enable flag must still inject the agent host")+}++// The mirror case: an explicit host from a Secret or ConfigMap lands in the .env,+// and a fieldRef injected next to it would silently win (bootstrap loads .env+// with override=False), so it must suppress the injection.+func TestInstanaAgentHostEnvVar_ExplicitHostFromRefWins(t *testing.T) {+ m := NewEnvVarManager()++ cm := &corev1.ConfigMap{+ ObjectMeta: metav1.ObjectMeta{Name: "apm-config", Namespace: "test-ns"},+ Data: map[string]string{"agent-host": "instana-agent.instana-agent.svc"},+ }+ env := resolvedBackendEnv(t, []corev1.EnvVar{+ {Name: "INSTANA_ENABLED", Value: "true"},+ {Name: "INSTANA_AGENT_HOST", ValueFrom: &corev1.EnvVarSource{+ ConfigMapKeyRef: &corev1.ConfigMapKeySelector{+ LocalObjectReference: corev1.LocalObjectReference{Name: "apm-config"},+ Key: "agent-host",+ },+ }},+ }, cm)++ require.Equal(t, "instana-agent.instana-agent.svc", env["INSTANA_AGENT_HOST"],+ "precondition: the .env gets the resolved host")+ assert.Nil(t, m.InstanaAgentHostEnvVar(env),+ "a ConfigMap-backed explicit host must suppress the node-IP fieldRef that would override it")+}++// An optional reference that does not resolve leaves the default in place rather+// than half-enabling the feature.+func TestInstanaAgentHostEnvVar_UnresolvedOptionalRefLeavesDefault(t *testing.T) {+ m := NewEnvVarManager()++ optional := true+ env := resolvedBackendEnv(t, []corev1.EnvVar{{+ Name: "INSTANA_ENABLED",+ ValueFrom: &corev1.EnvVarSource{+ SecretKeyRef: &corev1.SecretKeySelector{+ LocalObjectReference: corev1.LocalObjectReference{Name: "missing"},+ Key: "k",+ Optional: &optional,+ },+ },+ }})++ assert.Nil(t, m.InstanaAgentHostEnvVar(env),+ "an unresolvable enable flag should leave the default (off) in place")+}++func TestInstanaPresenceSensitiveVarsAreNotDefaulted(t *testing.T) {+ m := NewEnvVarManager()++ // BuildEnvFileContent writes an empty default as `KEY=`, which dotenv turns+ // into an empty string. The tracer tests these for presence, not truthiness,+ // so an empty default would set a blank service name and warn on every boot.+ //+ // INSTANA_TRACING_DISABLE, INSTANA_STACK_TRACE, and INSTANA_SECRETS are+ // deliberately NOT in this list: the tracer treats an empty value for+ // those as a parseable (if weaker) setting rather than "unset", so they+ // carry a real default instead — see+ // TestInstanaSecretsDefaultRedactsOpenRAGQueryParams below.+ for _, k := range []string{"INSTANA_SERVICE_NAME", "INSTANA_LOG_LEVEL", "INSTANA_ZONE", "INSTANA_AGENT_HOST"} {+ _, ok := m.DefaultOpenRagBEEnvVars[k]+ assert.False(t, ok, "%s must not carry an empty default", k)+ }+}++func TestInstanaSecretsDefaultRedactsOpenRAGQueryParams(t *testing.T) {+ m := NewEnvVarManager()++ assert.Equal(t,+ `regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\Z,search\Z,filename\Z`,+ m.DefaultOpenRagBEEnvVars["INSTANA_SECRETS"],+ "must keep the tracer's own credential coverage and add OpenRAG's free-text search parameters")+}++func TestBackendDeploymentInjectsInstanaHost(t *testing.T) {+ r, _ := reconciler(newScheme(t))++ cr := minimalCR("my-openrag", "my-ns")+ backendEnv := func(instanaHost *corev1.EnvVar) []corev1.EnvVar {+ return r.backendDeployment(cr, "my-ns", "hash", instanaHost).Spec.Template.Spec.Containers[0].Env+ }++ assert.Empty(t, backendEnv(nil),+ "the backend container keeps an empty Env by design; everything else lives in .env")++ env := backendEnv(r.EnvVarManager.InstanaAgentHostEnvVar(map[string]string{"INSTANA_ENABLED": "true"}))++ require.Len(t, env, 1, "only the agent host may be injected")+ assert.Equal(t, "INSTANA_AGENT_HOST", env[0].Name)+ require.NotNil(t, env[0].ValueFrom)+ require.NotNil(t, env[0].ValueFrom.FieldRef)+ assert.Equal(t, "status.hostIP", env[0].ValueFrom.FieldRef.FieldPath)+}diff --git a/kubernetes/operator/internal/controller/openrag_controller.go b/kubernetes/operator/internal/controller/openrag_controller.goindex 82eedb047..9b42dec9b 100644--- a/kubernetes/operator/internal/controller/openrag_controller.go+++ b/kubernetes/operator/internal/controller/openrag_controller.go@@ -125,8 +125,9 @@ func (r *OpenRAGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctif err := r.reconcileServiceAccounts(ctx, instance, targetNS); err != nil {return r.updateStatusError(ctx, instance, "service accounts", err)}- // Reconcile .env secrets and get their hashes to trigger pod restarts when secrets change- backendEnvHash, langflowEnvHash, err := r.reconcileEnvSecrets(ctx, instance, targetNS)+ // Reconcile .env secrets. Their hashes trigger pod restarts when a secret+ // changes; the resolved backend env feeds the Instana decision below.+ envSecrets, err := r.reconcileEnvSecrets(ctx, instance, targetNS)if err != nil {return r.updateStatusError(ctx, instance, "env secrets", err)}@@ -136,7 +137,8 @@ func (r *OpenRAGReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctif err := r.reconcileServices(ctx, instance, targetNS); err != nil {return r.updateStatusError(ctx, instance, "services", err)}- if err := r.reconcileDeployments(ctx, instance, targetNS, backendEnvHash, langflowEnvHash); err != nil {+ instanaHostEnv := r.EnvVarManager.InstanaAgentHostEnvVar(envSecrets.backendEnv)+ if err := r.reconcileDeployments(ctx, instance, targetNS, envSecrets.backendHash, envSecrets.langflowHash, instanaHostEnv); err != nil {return r.updateStatusError(ctx, instance, "deployments", err)}if err := r.reconcileDoclingComponents(ctx, instance, targetNS); err != nil {@@ -232,27 +234,37 @@ func parseEnvValue(envContent, key string) string {return ""}+// envSecretsResult is what reconcileEnvSecrets computed that later reconcile+// steps still need.+type envSecretsResult struct {+ // backendHash and langflowHash are SHA256 hashes of the rendered .env+ // content, stamped onto the pods so a changed secret triggers a restart.+ backendHash string+ langflowHash string+ // backendEnv is the fully resolved backend environment — the map the .env+ // file was rendered from, with secretKeyRef and configMapKeyRef entries+ // already read from the cluster. It is the only view in which a value's+ // origin no longer matters, so anything deciding from a backend variable+ // (see InstanaAgentHostEnvVar) must read it here rather than from spec.env.+ backendEnv map[string]string+}+// reconcileEnvSecrets creates / updates the backend and Langflow .env Secrets// from CR fields and fixed runtime defaults.// All sensitive values (whether user-provided or generated) are consolidated into .env files.-// Returns SHA256 hashes of the backend and langflow .env content to use as pod annotations.-func (r *OpenRAGReconciler) reconcileEnvSecrets(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string) (backendHash, langflowHash string, err error) {+func (r *OpenRAGReconciler) reconcileEnvSecrets(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string) (envSecretsResult, error) {// Build backend .env content with all secrets consolidated- backendEnvContent, err := r.buildBackendEnv(ctx, o, targetNS)+ backendEnvContent, backendEnv, err := r.buildBackendEnv(ctx, o, targetNS)if err != nil {- return "", "", fmt.Errorf("failed to build backend env: %w", err)+ return envSecretsResult{}, fmt.Errorf("failed to build backend env: %w", err)}// Build langflow .env content with all secrets consolidatedlangflowEnvContent, err := r.buildLangflowEnv(ctx, o, targetNS)if err != nil {- return "", "", fmt.Errorf("failed to build langflow env: %w", err)+ return envSecretsResult{}, fmt.Errorf("failed to build langflow env: %w", err)}- // Calculate SHA256 hashes of .env content for pod restart triggering- backendHash = calculateHash(backendEnvContent)- langflowHash = calculateHash(langflowEnvContent)-type envDef struct {name stringcontent string@@ -282,35 +294,43 @@ func (r *OpenRAGReconciler) reconcileEnvSecrets(ctx context.Context, o *openragvStringData: map[string]string{".env": d.content},}if err := r.setOwnerOrLabel(o, secret, targetNS); err != nil {- return "", "", err+ return envSecretsResult{}, err}if err := r.createOrUpdate(ctx, secret); err != nil {- return "", "", err+ return envSecretsResult{}, err}}- return backendHash, langflowHash, nil+ // Hash the .env content for pod restart triggering+ return envSecretsResult{+ backendHash: calculateHash(backendEnvContent),+ langflowHash: calculateHash(langflowEnvContent),+ backendEnv: backendEnv,+ }, nil}-func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string) (string, error) {+// buildBackendEnv renders the backend .env content and returns it alongside the+// resolved env map it was rendered from, so callers can read a final value+// (secretKeyRef and configMapKeyRef already resolved) without re-resolving.+func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string) (string, map[string]string, error) {// Start with defaults, operator env, and CR env (three-level priority)// This now resolves ALL env vars (including secrets/configmaps) for inclusion in .env fileenvVars, err := r.EnvVarManager.GetBackendEnvVars(ctx, r.Client, targetNS, o.Spec.Backend.Env)if err != nil {- return "", fmt.Errorf("failed to merge backend env vars: %w", err)+ return "", nil, fmt.Errorf("failed to merge backend env vars: %w", err)}// Get or generate encryption key (AES-256)// Priority: 1) User-provided secret in CR, 2) Existing value in .env, 3) Generate newencryptionKey, err := r.getOrGenerateSecret(ctx, o, targetNS, o.Spec.Backend.EncryptionKeySecret, "OPENRAG_ENCRYPTION_KEY", instanceResourceName(o, "be-env"), GenerateAESKeyString32)if err != nil {- return "", fmt.Errorf("failed to get encryption key: %w", err)+ return "", nil, fmt.Errorf("failed to get encryption key: %w", err)}envVars["OPENRAG_ENCRYPTION_KEY"] = encryptionKey// Get or generate JWT signing key (base64 secret)jwtSigningKey, err := r.getOrGenerateSecret(ctx, o, targetNS, o.Spec.Backend.JWTSigningKeySecret, "JWT_SIGNING_KEY", instanceResourceName(o, "be-env"), generateBase64SecretKey)if err != nil {- return "", fmt.Errorf("failed to get JWT signing key: %w", err)+ return "", nil, fmt.Errorf("failed to get JWT signing key: %w", err)}envVars["JWT_SIGNING_KEY"] = jwtSigningKey@@ -349,7 +369,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alp}username, err := r.readSecretValue(ctx, targetNS, usernameSecret)if err != nil {- return "", fmt.Errorf("failed to read OpenSearch username: %w", err)+ return "", nil, fmt.Errorf("failed to read OpenSearch username: %w", err)}if username != "" {envVars["OPENSEARCH_USERNAME"] = username@@ -362,7 +382,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alp}password, err := r.readSecretValue(ctx, targetNS, passwordSecret)if err != nil {- return "", fmt.Errorf("failed to read OpenSearch password: %w", err)+ return "", nil, fmt.Errorf("failed to read OpenSearch password: %w", err)}if password != "" {envVars["OPENSEARCH_PASSWORD"] = password@@ -386,7 +406,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alpif wx.APIKeySecret != nil {apiKey, err := r.readSecretValue(ctx, targetNS, wx.APIKeySecret)if err != nil {- return "", fmt.Errorf("failed to read WatsonX API key: %w", err)+ return "", nil, fmt.Errorf("failed to read WatsonX API key: %w", err)}if apiKey != "" {envVars["WATSONX_API_KEY"] = apiKey@@ -430,7 +450,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alpif oa.Google.ClientSecret != nil {clientSecret, err := r.readSecretValue(ctx, targetNS, oa.Google.ClientSecret)if err != nil {- return "", fmt.Errorf("failed to read Google OAuth client secret: %w", err)+ return "", nil, fmt.Errorf("failed to read Google OAuth client secret: %w", err)}if clientSecret != "" {envVars["GOOGLE_OAUTH_CLIENT_SECRET"] = clientSecret@@ -446,7 +466,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alpif oa.Microsoft.ClientSecret != nil {clientSecret, err := r.readSecretValue(ctx, targetNS, oa.Microsoft.ClientSecret)if err != nil {- return "", fmt.Errorf("failed to read Microsoft OAuth client secret: %w", err)+ return "", nil, fmt.Errorf("failed to read Microsoft OAuth client secret: %w", err)}if clientSecret != "" {envVars["MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET"] = clientSecret@@ -481,7 +501,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alp}// Convert map to .env file format- return r.EnvVarManager.BuildEnvFileContent(envVars), nil+ return r.EnvVarManager.BuildEnvFileContent(envVars), envVars, nil}func (r *OpenRAGReconciler) buildLangflowEnv(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string) (string, error) {@@ -695,10 +715,10 @@ func (r *OpenRAGReconciler) reconcileServices(ctx context.Context, o *openragv1areturn nil}-func (r *OpenRAGReconciler) reconcileDeployments(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string, backendEnvHash, langflowEnvHash string) error {+func (r *OpenRAGReconciler) reconcileDeployments(ctx context.Context, o *openragv1alpha1.OpenRAG, targetNS string, backendEnvHash, langflowEnvHash string, instanaHostEnv *corev1.EnvVar) error {deploys := []client.Object{r.frontendDeployment(o, targetNS),- r.backendDeployment(o, targetNS, backendEnvHash),+ r.backendDeployment(o, targetNS, backendEnvHash, instanaHostEnv),r.langflowDeployment(o, targetNS, langflowEnvHash),}for _, d := range deploys {@@ -764,7 +784,10 @@ func (r *OpenRAGReconciler) frontendDeployment(o *openragv1alpha1.OpenRAG, targe}}-func (r *OpenRAGReconciler) backendDeployment(o *openragv1alpha1.OpenRAG, targetNS string, envHash string) *appsv1.Deployment {+// backendDeployment builds the backend Deployment. instanaHostEnv is the single+// container-level env var the reconciler may inject (nil when Instana is off or+// an explicit agent host was configured) — see InstanaAgentHostEnvVar.+func (r *OpenRAGReconciler) backendDeployment(o *openragv1alpha1.OpenRAG, targetNS string, envHash string, instanaHostEnv *corev1.EnvVar) *appsv1.Deployment {spec := o.Spec.Backendreplicas := replicasOrDefault(spec.Replicas)@@ -804,6 +827,15 @@ func (r *OpenRAGReconciler) backendDeployment(o *openragv1alpha1.OpenRAG, target// The .env file is mounted and sourced by the applicationvar envVars []corev1.EnvVar // Empty - all vars are in .env file+ // Sole exception: the Instana agent address is the pod's own node IP, which+ // a .env file rendered at reconcile time cannot carry. See+ // InstanaAgentHostEnvVar. Nil unless Instana is enabled, and it carries a+ // node IP rather than a credential, so the "keep secrets out of `env`"+ // rationale above is unaffected.+ if instanaHostEnv != nil {+ envVars = append(envVars, *instanaHostEnv)+ }+baseLabels := componentLabels(o.Name, "be")deploymentLabels := mergeDeploymentLabels(baseLabels, spec.Labels)deploymentAnnotations := mergeDeploymentAnnotations(spec.Annotations)diff --git a/kubernetes/operator/internal/controller/openrag_controller_env_behavior_test.go b/kubernetes/operator/internal/controller/openrag_controller_env_behavior_test.goindex 9d602eb0e..a69f9e3db 100644--- a/kubernetes/operator/internal/controller/openrag_controller_env_behavior_test.go+++ b/kubernetes/operator/internal/controller/openrag_controller_env_behavior_test.go@@ -23,7 +23,7 @@ func TestComponentSpec_Env_DirectValues_GoToEnvFileOnly(t *testing.T) {}// Check .env file content contains direct values- backendEnvContent, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)assert.Contains(t, backendEnvContent, `CUSTOM_VAR=custom_value`,"Direct value should be in .env file")@@ -31,7 +31,7 @@ func TestComponentSpec_Env_DirectValues_GoToEnvFileOnly(t *testing.T) {"Direct value should be in .env file")// Check container env is EMPTY (no spec.Env)- deploy := r.backendDeployment(cr, "test-ns", "hash123")+ deploy := r.backendDeployment(cr, "test-ns", "hash123", nil)containerEnv := deploy.Spec.Template.Spec.Containers[0].Env// Container Env should be empty or only contain operator-managed vars@@ -77,7 +77,7 @@ func TestComponentSpec_Env_SecretRefs_ResolvedToEnvFile(t *testing.T) {}// Check .env file content - should contain RESOLVED secret value- backendEnvContent, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)assert.Contains(t, backendEnvContent, `DATABASE_PASSWORD=super-secret-password`,".env file should contain RESOLVED secret value")@@ -85,7 +85,7 @@ func TestComponentSpec_Env_SecretRefs_ResolvedToEnvFile(t *testing.T) {".env file should contain direct values")// Check container env - should be EMPTY (no spec.Env)- deploy := r.backendDeployment(cr, "test-ns", "hash123")+ deploy := r.backendDeployment(cr, "test-ns", "hash123", nil)containerEnv := deploy.Spec.Template.Spec.Containers[0].Env// Container Env should NOT contain DATABASE_PASSWORD or REGULAR_VAR@@ -127,13 +127,13 @@ func TestComponentSpec_Env_ConfigMapRefs_ResolvedToEnvFile(t *testing.T) {}// Check .env file - should contain resolved value- backendEnvContent, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)assert.Contains(t, backendEnvContent, `API_ENDPOINT=https://api.example.com`,".env file should contain resolved configmap value")// Check container env - should be empty- deploy := r.backendDeployment(cr, "test-ns", "hash123")+ deploy := r.backendDeployment(cr, "test-ns", "hash123", nil)containerEnv := deploy.Spec.Template.Spec.Containers[0].Envfor _, env := range containerEnv {@@ -209,7 +209,7 @@ func TestComponentSpec_Env_FieldRef_RejectedWithError(t *testing.T) {}// Should fail with clear error message- _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ _, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.Error(t, err)assert.Contains(t, err.Error(), "fieldRef is not supported","Should reject fieldRef with helpful error message")@@ -268,7 +268,7 @@ func TestComponentSpec_Env_MixedTypes_AllResolvedToEnvFile(t *testing.T) {}// Check .env file - should contain ALL 5 resolved values- backendEnvContent, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)assert.Contains(t, backendEnvContent, `LITERAL_1=value1`)assert.Contains(t, backendEnvContent, `SECRET_REF=secret-value-1`)@@ -277,7 +277,7 @@ func TestComponentSpec_Env_MixedTypes_AllResolvedToEnvFile(t *testing.T) {assert.Contains(t, backendEnvContent, `LITERAL_3=value3`)// Check container env - should be empty (no spec.Env vars)- deploy := r.backendDeployment(cr, "test-ns", "hash123")+ deploy := r.backendDeployment(cr, "test-ns", "hash123", nil)containerEnv := deploy.Spec.Template.Spec.Containers[0].Env// Verify none of the 5 env vars are in container envdiff --git a/kubernetes/operator/internal/controller/openrag_controller_test.go b/kubernetes/operator/internal/controller/openrag_controller_test.goindex baee42f92..8025d015a 100644--- a/kubernetes/operator/internal/controller/openrag_controller_test.go+++ b/kubernetes/operator/internal/controller/openrag_controller_test.go@@ -1283,7 +1283,7 @@ func TestEnvHash_StableAcrossReconciles(t *testing.T) {// Reconcile multiple timesvar hashes []stringfor i := 0; i < 5; i++ {- backendEnvContent, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)hash := calculateHash(backendEnvContent)hashes = append(hashes, hash)@@ -1306,7 +1306,7 @@ func TestEnvHash_ChangesWhenEnvChanges(t *testing.T) {r, _ := reconciler(s, cr)// Get hash with original env- backendEnvContent1, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent1, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)hash1 := calculateHash(backendEnvContent1)@@ -1316,7 +1316,7 @@ func TestEnvHash_ChangesWhenEnvChanges(t *testing.T) {}// Get hash with changed env- backendEnvContent2, err := r.buildBackendEnv(context.Background(), cr, "test-ns")+ backendEnvContent2, _, err := r.buildBackendEnv(context.Background(), cr, "test-ns")require.NoError(t, err)hash2 := calculateHash(backendEnvContent2)diff --git a/pyproject.toml b/pyproject.tomlindex 20aca986c..8667bd790 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -70,6 +70,18 @@ dependencies = ["cachetools>=5.5.0",]+# Optional extras. Install with `uv sync --extra <name>`.+[project.optional-dependencies]+# IBM Instana APM tracer. Optional because the feature is opt-in at runtime+# (INSTANA_ENABLED, default false) and the tracer monkey-patches httpx,+# opensearch-py, sqlalchemy, starlette, fastapi, and logging at import time —+# not something to install on every OSS deployment that will never enable it.+# The upper bound is deliberate for the same reason: a major release of a+# library that rewrites the runtime behaviour of the whole HTTP/DB stack should+# be adopted explicitly, not resolved into an install.+# src/main.py imports it defensively, so its absence downgrades to a warning.+apm = ["instana>=3.17.0,<4"]+[dependency-groups]dev = ["pytest>=9.0.3",diff --git a/src/api/settings/langflow_sync.py b/src/api/settings/langflow_sync.pyindex d53437440..258f09aea 100644--- a/src/api/settings/langflow_sync.py+++ b/src/api/settings/langflow_sync.py@@ -158,7 +158,30 @@ async def ensure_required_langflow_global_variables(config=None):target_val = _string_value(required_values.get(name)) if is_generic else curr_val+ # Langflow rejects an empty value with 400 "Variable value cannot be+ # empty". Optional providers (Ollama, watsonx) resolve to "" when+ # unconfigured, so keep whatever Langflow already holds instead of+ # pushing a value it will refuse. This matters most in the type+ # migration below: the DELETE has already landed by then, so a+ # refused recreate would drop the variable entirely.+ sync_value = is_generic and bool(target_val)+ if not sync_value:+ target_val = curr_val+if curr_type != target_type:+ if not target_val:+ # The retained value (see sync_value above) is also empty: the+ # DELETE below would succeed but the recreate POST would send ""+ # and get rejected, dropping the variable entirely. Defer the+ # type migration until the variable actually has a value.+ logger.debug(+ "Deferring Langflow global variable type migration until configured",+ variable_name=name,+ old_type=curr_type,+ new_type=target_type,+ )+ continue+logger.info("Migrating Langflow global variable type",variable_name=name,@@ -183,7 +206,7 @@ async def ensure_required_langflow_global_variables(config=None):raise RuntimeError(f"Failed to recreate Langflow global variable {name!r}: status_code={recreate_resp.status_code}")- elif has_default_fields or (is_generic and curr_val != target_val):+ elif has_default_fields or (sync_value and curr_val != target_val):logger.info("Updating Langflow global variable", variable_name=name, variable_id=var_id)@@ -193,7 +216,7 @@ async def ensure_required_langflow_global_variables(config=None):"default_fields": [],"type": target_type,}- if is_generic:+ if sync_value:patch_payload["value"] = target_valpatch_resp = await clients.langflow_request("PATCH", f"/api/v1/variables/{var_id}", json=patch_payload@@ -212,6 +235,16 @@ async def ensure_required_langflow_global_variables(config=None):if name not in existing_by_name:try:target_val = _string_value(required_values.get(name, ""))+ if not target_val:+ # Langflow rejects an empty value with 400. Variables sourced+ # from an unconfigured optional provider (OLLAMA_BASE_URL,+ # WATSONX_PROJECT_ID, WATSONX_URL) have nothing to set yet;+ # they are created once that provider is configured.+ logger.debug(+ "Skipping Langflow global variable with no configured value",+ variable_name=name,+ )+ continueawait _upsert_langflow_global_variable(name, target_val)except Exception as e:logger.warning(diff --git a/src/bootstrap.py b/src/bootstrap.pyindex a0e4a7842..79c7275e9 100644--- a/src/bootstrap.py+++ b/src/bootstrap.py@@ -25,6 +25,14 @@ def load_env():configure_from_env()+ # Instana patches httpx / urllib3 / sqlalchemy / starlette / logging at+ # import time, so the tracer has to load before anything imports them.+ # This is the earliest point where the flag is readable and a warning can+ # be logged, and it covers every entry point that imports bootstrap.+ from observability.instana_boot import boot_instana++ boot_instana()+# Execute immediately on importload_env()diff --git a/src/main.py b/src/main.pyindex 8edd56f2f..f8203aa96 100644--- a/src/main.py+++ b/src/main.py@@ -11,6 +11,7 @@ Most of what used to live in this file moved into focused modules:- utils/opensearch_init.py — OpenSearch index/security setup- utils/jwt_keygen.py — RSA keypair generation- utils/url_content_fetcher.py — URL → text-file helper+- observability/instana_boot.py — Instana APM gate (run from bootstrap)- api/health.py — liveness + readiness probesThis module is intentionally thin: import bootstrap, run Alembic, build@@ -18,6 +19,9 @@ the app, run uvicorn. The re-exports below preserve the`from main import …` contract used by tests and api/settings.py."""+# Importing bootstrap also boots the Instana tracer when INSTANA_ENABLED is+# set — it has to patch httpx / sqlalchemy / starlette before the imports+# below pull them in.import bootstrap # noqa: F401 — must be first; loads .env + structured loggingimport asynciodiff --git a/src/observability/__init__.py b/src/observability/__init__.pynew file mode 100644index 000000000..1c6e8fb1a--- /dev/null+++ b/src/observability/__init__.py@@ -0,0 +1,5 @@+"""Observability bootstrap helpers (APM tracers and friends).++Kept separate from `utils/` because the modules here run *before* the rest of+the application is importable — see `instana_boot` for why that matters.+"""diff --git a/src/observability/instana_boot.py b/src/observability/instana_boot.pynew file mode 100644index 000000000..81f92c374--- /dev/null+++ b/src/observability/instana_boot.py@@ -0,0 +1,78 @@+"""Instana APM gate.++This lives in its own module for one reason: the Instana tracer monkey-patches+``httpx``, ``urllib3``, ``sqlalchemy``, ``starlette``, ``fastapi``, and+``logging`` at *import* time, so it must be imported before any of them. That+rules out reading the flag from ``config/settings.py`` — importing that module+pulls in the very libraries the tracer has to patch first — which makes this+the documented exception to "config/settings.py is the only place that reads+``os.environ`` directly".++``bootstrap.load_env()`` calls :func:`boot_instana` immediately after loading+``.env`` and configuring structlog, which is the earliest point at which the+flag is readable and a warning can be logged.+"""++import os++INSTANA_ENABLED_ENV_VAR = "INSTANA_ENABLED"++# Same spelling the Kubernetes path accepts (`isTruthyEnvValue` in the+# operator's env.go), so a value that enables tracing there enables it here.+_TRUTHY_VALUES = frozenset({"true", "1", "yes"})+++def is_instana_enabled(value: str | None = None) -> bool:+ """Return True when ``INSTANA_ENABLED`` asks for tracing.++ Reads the process environment when ``value`` is not supplied, which is what+ the boot path does; passing it explicitly keeps the parsing testable.+ """+ if value is None:+ value = os.environ.get(INSTANA_ENABLED_ENV_VAR, "")+ return value.strip().lower() in _TRUTHY_VALUES+++TEST_OPT_IN_ENV_VAR = "OPENRAG_TEST_INSTANA"+++def is_instana_test_opt_in(value: str | None = None) -> bool:+ """Whether a developer explicitly opted an instrumented *test* run in.++ ``INSTANA_ENABLED`` alone can't be used for this: ``make test`` /+ ``make test-unit`` export every key present in ``.env`` (including+ ``INSTANA_ENABLED``, if a developer has it set there for local app+ development) into the pytest process before it starts, making it+ indistinguishable from an explicit shell override. This is a separate,+ test-only switch that is never written to ``.env.example`` and never+ exported by a normal Make flow.+ """+ if value is None:+ value = os.environ.get(TEST_OPT_IN_ENV_VAR, "")+ return is_instana_enabled(value)+++def boot_instana() -> bool:+ """Load the Instana tracer when enabled. Returns True when it was loaded.++ Importing ``instana`` *is* the activation step — the package runs+ ``boot_agent()`` at import time. A missing package is not fatal: the tracer+ ships as the optional ``apm`` extra, so an install without it warns and+ runs untraced rather than failing to start.+ """+ if not is_instana_enabled():+ return False++ try:+ import instana # noqa: F401 — the import is the activation+ except ImportError:+ from utils.logging_config import get_logger++ get_logger(__name__).warning(+ "INSTANA_ENABLED is set but the 'instana' package is not installed; "+ "APM tracing is disabled. Install the optional extra with "+ "`uv sync --extra apm`."+ )+ return False++ return Truediff --git a/tests/conftest.py b/tests/conftest.pyindex 7e5ab7eb5..89cfd7f24 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -17,6 +17,25 @@ if str(SRC) not in sys.path:# Load environment variablesload_dotenv()+# Keep the Instana tracer out of the test process unless a developer opted in+# explicitly with OPENRAG_TEST_INSTANA=true. INSTANA_ENABLED alone can't be used+# for this: `make test` / `make test-unit` export every key in .env (including+# INSTANA_ENABLED, if a developer has it set there for local app development)+# into the pytest process before it starts, making it indistinguishable from an+# explicit shell override.+#+# `from main import ...` below pulls in `bootstrap`, which boots the tracer at+# import time, so leaving INSTANA_ENABLED alone here would run the entire suite+# instrumented. That costs ~1.1s of extra import time per process and+# monkey-patches logging, asyncio.create_task, httpx, urllib3 and sqlalchemy+# underneath every test — and in instana 3.17.0 each WARNING/ERROR logged+# inside a trace leaks an event into a process-global list that every later+# log span re-walks. None of that belongs in a test run.+from observability.instana_boot import is_instana_test_opt_in # noqa: E402++if not is_instana_test_opt_in():+ os.environ["INSTANA_ENABLED"] = "false"+# Force no-auth mode for testing by setting OAuth credentials to empty strings# This ensures anonymous JWT tokens are created automaticallyos.environ["GOOGLE_OAUTH_CLIENT_ID"] = ""diff --git a/tests/unit/observability/__init__.py b/tests/unit/observability/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/observability/test_instana_boot.py b/tests/unit/observability/test_instana_boot.pynew file mode 100644index 000000000..75434fed2--- /dev/null+++ b/tests/unit/observability/test_instana_boot.py@@ -0,0 +1,117 @@+"""Tests for the Instana enable gate (`observability/instana_boot.py`).++The gate decides whether an APM tracer that monkey-patches httpx, urllib3,+sqlalchemy, starlette and logging gets loaded into the process, so its parsing+is worth pinning: the failure mode is silent (tracing simply never starts).+"""++import sys+from types import SimpleNamespace++import pytest++from observability.instana_boot import (+ INSTANA_ENABLED_ENV_VAR,+ TEST_OPT_IN_ENV_VAR,+ boot_instana,+ is_instana_enabled,+ is_instana_test_opt_in,+)+from utils import logging_config++[email protected]("value", ["true", "1", "yes"])+def test_accepted_values_enable_tracing(value):+ assert is_instana_enabled(value) is True++[email protected]("value", ["TRUE", "True", "Yes", "YES"])+def test_accepted_values_are_case_insensitive(value):+ assert is_instana_enabled(value) is True++[email protected]("value", [" true ", "\ttrue\n"])+def test_surrounding_whitespace_is_ignored(value):+ """A trailing space in a .env line or a Helm value must not disable tracing."""+ assert is_instana_enabled(value) is True++[email protected]("value", ["false", "0", "no", "", " ", "on", "enabled", "y"])+def test_everything_else_disables_tracing(value):+ """Only the documented spellings count.++ `on`/`enabled`/`y` are deliberately *not* accepted — this asserts the+ documented set rather than a guess, and keeps the backend in step with the+ operator's isTruthyEnvValue (kubernetes/operator/internal/controller/env.go).+ """+ assert is_instana_enabled(value) is False+++def test_unset_variable_disables_tracing(monkeypatch):+ monkeypatch.delenv(INSTANA_ENABLED_ENV_VAR, raising=False)+ assert is_instana_enabled() is False+++def test_environment_is_read_when_no_value_is_passed(monkeypatch):+ monkeypatch.setenv(INSTANA_ENABLED_ENV_VAR, "true")+ assert is_instana_enabled() is True+++def test_boot_is_a_no_op_when_disabled(monkeypatch):+ """The tracer must not be imported when the flag is off."""+ monkeypatch.setenv(INSTANA_ENABLED_ENV_VAR, "false")+ monkeypatch.delitem(sys.modules, "instana", raising=False)++ assert boot_instana() is False+ assert "instana" not in sys.modules+++def test_boot_warns_and_continues_when_the_package_is_missing(monkeypatch):+ """`instana` ships as the optional `apm` extra, so a missing package is not fatal."""+ monkeypatch.setenv(INSTANA_ENABLED_ENV_VAR, "true")+ monkeypatch.delitem(sys.modules, "instana", raising=False)++ class _BlockInstana:+ def find_spec(self, name, path=None, target=None):+ if name == "instana":+ raise ImportError("blocked by test")+ return None++ monkeypatch.setattr(sys, "meta_path", [_BlockInstana(), *sys.meta_path])++ # Assert on the logger call rather than on captured output: structlog is+ # configured against the stderr stream bound at import time, which neither+ # caplog nor capsys/capfd sees from inside a test.+ warnings: list[str] = []+ monkeypatch.setattr(+ logging_config,+ "get_logger",+ lambda *a, **kw: SimpleNamespace(warning=lambda msg, **kw: warnings.append(msg)),+ )++ assert boot_instana() is False+ assert len(warnings) == 1+ assert "uv sync --extra apm" in warnings[0]++[email protected]("value", ["true", "1", "yes", "TRUE", " true "])+def test_test_opt_in_accepts_the_same_spellings_as_is_instana_enabled(value):+ assert is_instana_test_opt_in(value) is True++[email protected]("value", ["false", "0", "no", ""])+def test_test_opt_in_is_off_by_default(value):+ assert is_instana_test_opt_in(value) is False+++def test_test_opt_in_ignores_an_env_derived_instana_enabled_value(monkeypatch):+ """Regression for discussion_r3854845897: a value `make test` exported from+ .env for INSTANA_ENABLED must not be mistaken for an explicit test opt-in."""+ monkeypatch.setenv(INSTANA_ENABLED_ENV_VAR, "true")+ monkeypatch.delenv(TEST_OPT_IN_ENV_VAR, raising=False)+ assert is_instana_test_opt_in() is False+++def test_test_opt_in_reads_its_own_env_var(monkeypatch):+ monkeypatch.setenv(TEST_OPT_IN_ENV_VAR, "true")+ assert is_instana_test_opt_in() is Truediff --git a/tests/unit/observability/test_instana_secrets.py b/tests/unit/observability/test_instana_secrets.pynew file mode 100644index 000000000..ffb988d15--- /dev/null+++ b/tests/unit/observability/test_instana_secrets.py@@ -0,0 +1,61 @@+"""Tests for the `INSTANA_SECRETS` default value (`.env.example`).++Regression for discussion_r3864664254: the Makefile `include`s and `export`s+every `.env` line as GNU Make syntax (see Makefile's ENV_FILE loading), which+treats a bare `$` as a Make variable reference. A `^name$`-anchored regex+value silently loses its `$` (and the following comma) under `make backend` /+`make dev`, so `q`/`search`/`filename` were never actually redacted despite+the tracer reporting no error. The fixed default anchors with `\\Z` instead of+`$`, which is `re.match()`-equivalent (the tracer already anchors at the+start of the string) and contains no `$` for Make to mangle.+"""++import re++import pytest++instana_secrets = pytest.importorskip("instana.util.secrets")++# Mirrors the INSTANA_SECRETS default in .env.example / docker-compose.yml /+# kubernetes/helm/openrag/values.yaml / kubernetes/operator's env.go.+INSTANA_SECRETS_DEFAULT = "regex:.*key.*,.*pass.*,.*secret.*,.*token.*,q\\Z,search\\Z,filename\\Z"+++def _kwlist():+ matcher, names = INSTANA_SECRETS_DEFAULT.split(":", 1)+ assert matcher == "regex"+ return names.split(",")+++def test_default_contains_no_dollar_sign():+ """A `$` here is silently corrupted by the Makefile's `.env` include/export."""+ assert "$" not in INSTANA_SECRETS_DEFAULT++[email protected]("name", ["q", "search", "filename", "api_key", "auth_token"])+def test_default_redacts_the_documented_query_parameters(name):+ assert instana_secrets.contains_secret(name, "regex", _kwlist()) is True++[email protected]("name", ["quantity", "research", "filename2", "myfilename"])+def test_default_does_not_over_redact_similarly_named_parameters(name):+ """Exact-match names must not swallow unrelated parameters that merely+ contain the same substring — this is what \\Z (vs. a bare `contains`+ match) buys us, same as the original `^name$` intent."""+ assert not instana_secrets.contains_secret(name, "regex", _kwlist())+++def test_default_matches_the_makefile_include_export_round_trip():+ """Guards the actual bug: simulates GNU Make's variable-reference parsing+ of a `$`-anchored value to show why it breaks, and confirms the current+ `\\Z`-anchored default has nothing for that parsing to corrupt."""++ def make_mangled(value: str) -> str:+ # `$X` where X is a single character is a Make variable reference;+ # an undefined one expands to "". A trailing lone `$` is literal.+ return re.sub(r"\$(.)", "", value)++ old_style_default = "regex:.*key.*,.*pass.*,.*secret.*,.*token.*,^q$,^search$,^filename$"+ assert make_mangled(old_style_default) != old_style_default++ assert make_mangled(INSTANA_SECRETS_DEFAULT) == INSTANA_SECRETS_DEFAULTdiff --git a/tests/unit/test_langflow_global_variables.py b/tests/unit/test_langflow_global_variables.pyindex 60ef90bb9..9a238a216 100644--- a/tests/unit/test_langflow_global_variables.py+++ b/tests/unit/test_langflow_global_variables.py@@ -455,3 +455,232 @@ async def test_ensure_required_langflow_global_variables_handles_failed_delete_passert "DELETE" in methodsassert "POST" in methodsassert "PATCH" in methods+++async def test_ensure_required_globals_skips_creating_empty_valued_variables(monkeypatch):+ """Unconfigured optional providers must not be POSTed to Langflow.++ Langflow answers 400 "Variable value cannot be empty", which the create+ helper reports as a generic "Failed to create Langflow global variable"+ warning on every boot.+ """++ async def mock_langflow_request(method, endpoint, **kwargs):+ if method == "GET":+ return _Response(json_data=[])+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients, "langflow_request", mock_langflow_request, raising=True+ )++ create_calls = []++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ create_calls.append((name, value, modify, variable_type))++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ # No ollama / watsonx configured -> OLLAMA_BASE_URL, WATSONX_PROJECT_ID and+ # WATSONX_URL all resolve to "".+ config = SimpleNamespace(providers=SimpleNamespace(), knowledge=SimpleNamespace())++ await langflow_sync.ensure_required_langflow_global_variables(config)++ created = {name for name, *_ in create_calls}+ assert "OLLAMA_BASE_URL" not in created+ assert "WATSONX_PROJECT_ID" not in created+ assert "WATSONX_URL" not in created+ # Variables that do have a value are still created.+ assert "OPENSEARCH_INDEX_NAME" in created+ assert all(value for _, value, *_ in create_calls)+++async def test_ensure_required_globals_creates_optional_provider_var_once_configured(monkeypatch):+ async def mock_langflow_request(method, endpoint, **kwargs):+ if method == "GET":+ return _Response(json_data=[])+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients, "langflow_request", mock_langflow_request, raising=True+ )++ create_calls = []++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ create_calls.append((name, value, modify, variable_type))++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ config = SimpleNamespace(+ providers=SimpleNamespace(ollama=SimpleNamespace(endpoint="http://localhost:11434")),+ knowledge=SimpleNamespace(),+ )++ await langflow_sync.ensure_required_langflow_global_variables(config)++ assert ("OLLAMA_BASE_URL", "http://localhost:11434", True, "Generic") in create_calls+++async def test_ensure_required_globals_does_not_blank_existing_value(monkeypatch):+ """An existing variable must not be PATCHed to "" when its provider is gone.++ Langflow rejects the empty value, so the PATCH would fail and log a warning+ every boot while leaving the stored value untouched anyway.+ """+ langflow_calls = []++ async def mock_langflow_request(method, endpoint, **kwargs):+ langflow_calls.append((method, endpoint, kwargs))+ if method == "GET":+ return _Response(+ json_data=[+ {+ "id": "var-1",+ "name": "OLLAMA_BASE_URL",+ "value": "http://localhost:11434",+ "type": "Generic",+ "default_fields": [],+ }+ ]+ )+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients, "langflow_request", mock_langflow_request, raising=True+ )++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ pass++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ # ollama no longer configured -> target value is ""+ config = SimpleNamespace(providers=SimpleNamespace(), knowledge=SimpleNamespace())++ await langflow_sync.ensure_required_langflow_global_variables(config)++ assert [c for c in langflow_calls if c[0] == "PATCH"] == []+++async def test_ensure_required_globals_type_migration_keeps_value_when_target_empty(monkeypatch):+ """A Credential->Generic migration must not recreate the variable with "".++ The DELETE has already landed by then, so a 400 on the recreate would drop+ the variable entirely.+ """+ langflow_calls = []++ async def mock_langflow_request(method, endpoint, **kwargs):+ langflow_calls.append((method, endpoint, kwargs))+ if method == "GET":+ return _Response(+ json_data=[+ {+ "id": "var-1",+ "name": "WATSONX_URL",+ "value": "https://us-south.ml.cloud.ibm.com",+ "type": "Credential",+ "default_fields": [],+ }+ ]+ )+ if method == "DELETE":+ return _Response(status_code=204)+ return _Response(status_code=201)++ monkeypatch.setattr(+ langflow_sync.clients, "langflow_request", mock_langflow_request, raising=True+ )++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ pass++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ config = SimpleNamespace(providers=SimpleNamespace(), knowledge=SimpleNamespace())++ await langflow_sync.ensure_required_langflow_global_variables(config)++ posts = [c for c in langflow_calls if c[0] == "POST"]+ assert len(posts) == 1+ # Recreated as Generic, but carrying the value Langflow already held.+ assert posts[0][2]["json"]["value"] == "https://us-south.ml.cloud.ibm.com"+ assert posts[0][2]["json"]["type"] == "Generic"+++async def test_ensure_required_globals_defers_type_migration_when_value_is_empty(monkeypatch):+ """A Credential->Generic migration must not delete a variable it can't recreate.++ Unlike the "keeps_value_when_target_empty" case above, here the *retained*+ value is also empty (an unconfigured WATSONX_URL stored as Credential).+ Deleting and then recreating with "" would 400, dropping the variable+ entirely, so the migration must be deferred until it has a value.+ """+ langflow_calls = []++ async def mock_langflow_request(method, endpoint, **kwargs):+ langflow_calls.append((method, endpoint, kwargs))+ if method == "GET":+ return _Response(+ json_data=[+ {+ "id": "var-1",+ "name": "WATSONX_URL",+ "value": "",+ "type": "Credential",+ "default_fields": [],+ }+ ]+ )+ return _Response(status_code=200)++ monkeypatch.setattr(+ langflow_sync.clients, "langflow_request", mock_langflow_request, raising=True+ )++ async def create_variable(name, value, modify=False, variable_type="Credential"):+ pass++ monkeypatch.setattr(+ langflow_sync.clients,+ "_create_langflow_global_variable",+ create_variable,+ raising=True,+ )++ # watsonx is not configured -> WATSONX_URL resolves to "".+ config = SimpleNamespace(providers=SimpleNamespace(), knowledge=SimpleNamespace())++ await langflow_sync.ensure_required_langflow_global_variables(config)++ methods = [c[0] for c in langflow_calls]+ assert "DELETE" not in methods+ assert "POST" not in methods+ assert "PATCH" not in methodsdiff --git a/uv.lock b/uv.lockindex 36e7be36e..86be91184 100644--- a/uv.lock+++ b/uv.lock@@ -298,6 +298,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" },]+[[package]]+name = "autowrapt"+version = "1.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "wrapt" },+]+sdist = { url = "https://files.pythonhosted.org/packages/83/09/4eccf7c33822a70696611520ada5dd1916f742344b0a7c96b90dd301178d/autowrapt-1.0.tar.gz", hash = "sha256:a0dcd0092477bb22e391f136659cc7f716da06ebda66a8b51ae4955cab0f97a2", size = 2687, upload-time = "2015-04-09T20:09:11.041Z" }+[[package]]name = "azure-core"version = "1.41.0"@@ -891,6 +900,12 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" },]+[[package]]+name = "fysom"+version = "2.1.6"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/32/e3/2b5d622edca8dffcdec421b28b014567e8ffa861e6483d4e6af5b7add878/fysom-2.1.6.tar.gz", hash = "sha256:e3617b79f19220c72b1c7e69d4d6c2a36a80cdbe80f4df7bdebab5ab4c893bf5", size = 19717, upload-time = "2021-09-10T14:17:22.894Z" }+[[package]]name = "google-api-core"version = "2.30.2"@@ -1235,6 +1250,26 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },]+[[package]]+name = "instana"+version = "3.17.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "autowrapt" },+ { name = "fysom" },+ { name = "opentelemetry-api" },+ { name = "opentelemetry-semantic-conventions" },+ { name = "psutil", marker = "sys_platform == 'win32'" },+ { name = "pyyaml" },+ { name = "requests" },+ { name = "typing-extensions" },+ { name = "urllib3" },+]+sdist = { url = "https://files.pythonhosted.org/packages/ce/a0/3d8bfe72026f9cc0f90fd58c0e63d70dfe2bae19d9c774f8ab4846d5e10e/instana-3.17.0.tar.gz", hash = "sha256:8d53a6aa59ca91aae6701f4d05f137b9a541ce01e8263b3a271adf1623d832a8", size = 364175, upload-time = "2026-08-14T05:47:30.146Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/9a/10/875126ab8b9fcd4af94bdb82d01b8ecb495affcc10725d1b636fe057898a/instana-3.17.0-py3-none-any.whl", hash = "sha256:87119594142369355c1bb5fe2a70a9c325c7ed0869b8a329c19d4ac2fa738811", size = 197041, upload-time = "2026-08-14T05:47:28.424Z" },+]+[[package]]name = "isodate"version = "0.7.2"@@ -1920,6 +1955,11 @@ dependencies = [{ name = "zxcvbn" },]+[package.optional-dependencies]+apm = [+ { name = "instana" },+]+[package.dev-dependencies]dev = [{ name = "mypy" },@@ -1952,6 +1992,7 @@ requires-dist = [{ name = "httpx", specifier = ">=0.27.0" },{ name = "ibm-cos-sdk", specifier = ">=2.13.0" },{ name = "idna", specifier = ">=3.18" },+ { name = "instana", marker = "extra == 'apm'", specifier = ">=3.17.0,<4" },{ name = "litellm", specifier = "==1.84.0" },{ name = "mcp", specifier = ">=1.28.1" },{ name = "msal", specifier = ">=1.29.0" },@@ -1980,6 +2021,7 @@ requires-dist = [{ name = "uvicorn", specifier = ">=0.35.0" },{ name = "zxcvbn", specifier = ">=4.5.0" },]+provides-extras = ["apm"][package.metadata.requires-dev]dev = [@@ -2027,15 +2069,27 @@ async = [[[package]]name = "opentelemetry-api"-version = "1.41.0"+version = "1.44.0"source = { registry = "https://pypi.org/simple" }dependencies = [- { name = "importlib-metadata" },{ name = "typing-extensions" },]-sdist = { url = "https://files.pythonhosted.org/packages/47/8e/3778a7e87801d994869a9396b9fc2a289e5f9be91ff54a27d41eace494b0/opentelemetry_api-1.41.0.tar.gz", hash = "sha256:9421d911326ec12dee8bc933f7839090cad7a3f13fcfb0f9e82f8174dc003c09", size = 71416, upload-time = "2026-04-09T14:38:34.544Z" }+sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" }wheels = [- { url = "https://files.pythonhosted.org/packages/58/ee/99ab786653b3bda9c37ade7e24a7b607a1b1f696063172768417539d876d/opentelemetry_api-1.41.0-py3-none-any.whl", hash = "sha256:0e77c806e6a89c9e4f8d372034622f3e1418a11bdbe1c80a50b3d3397ad0fa4f", size = 69007, upload-time = "2026-04-09T14:38:11.833Z" },+ { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },+]++[[package]]+name = "opentelemetry-semantic-conventions"+version = "0.65b0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "opentelemetry-api" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" },][[package]]@@ -3234,6 +3288,59 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/6f/28/258ebab549c2bf3e64d2b0217b973467394a9cea8c42f70418ca2c5d0d2e/websockets-16.0-py3-none-any.whl", hash = "sha256:1637db62fad1dc833276dded54215f2c7fa46912301a24bd94d45d46a011ceec", size = 171598, upload-time = "2026-01-10T09:23:45.395Z" },]+[[package]]+name = "wrapt"+version = "2.3.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/2b/b0/c1f5a970721f06b85c0cd5142e0ff8fe067708abd779b0c4f4be7d61d09f/wrapt-2.3.0.tar.gz", hash = "sha256:681a2d0eefd721998f90642762b8e75c2159ec531b20ad5e437245ea7b06a107", size = 131509, upload-time = "2026-07-28T06:06:14.895Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/8e/6e/0f88a072483e76b881e3fdcd6b6ffb4a5791002514fe541e72b1b73c859a/wrapt-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d3fb71e65b001adfc42684522eeccd9c21d8ba679945abc993439567b66e59f", size = 81960, upload-time = "2026-07-28T06:04:49.622Z" },+ { url = "https://files.pythonhosted.org/packages/d7/ff/b7e2776e7c294075eb712cc9ef573d1b818f393006d09787262b8fc871c4/wrapt-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51a7a4181c1295774812271fbcd7c909df372bc25579d4ed9eb875caaf0ae86f", size = 82435, upload-time = "2026-07-28T06:04:50.9Z" },+ { url = "https://files.pythonhosted.org/packages/d8/90/343bb5d0f1f9669bc252a6073f085b4abf862511bd5c9c9eaec754341f1d/wrapt-2.3.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9045917809c63fdf7abe3a2ceaed3d670b8ee4500ddd9291192d30aeb34467c5", size = 170350, upload-time = "2026-07-28T06:04:52.187Z" },+ { url = "https://files.pythonhosted.org/packages/59/f8/13b79a392930bd0dd6b86cbfbfe1c40944110456e1dc6d809e5c46ece904/wrapt-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54ca1d5573f69b5fe1d74f1f65799c68015e82f685efec9fd8cfa40a094c44d0", size = 170022, upload-time = "2026-07-28T06:04:53.599Z" },+ { url = "https://files.pythonhosted.org/packages/b2/fc/4f1b6918f5290db959d6e0c07f77385d87cede29c39c9cf8f145e9c82954/wrapt-2.3.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:242b60c21e30866e6a2fa606c612b47c553fa60c0eaeeeb7797fb842ac0ce609", size = 161043, upload-time = "2026-07-28T06:04:54.936Z" },+ { url = "https://files.pythonhosted.org/packages/01/e1/45d3cf74414780bdff6d0380467e003f6eb0f028b6c9403db868dbc7209c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3f3d7ec0a51fbfe00d3aef047641ff2c58b25565b4717fc1f90e050be01cba8", size = 168576, upload-time = "2026-07-28T06:04:56.261Z" },+ { url = "https://files.pythonhosted.org/packages/f3/73/2fa58dd97f191c997755e2c6d569a68f0c433db4e4b36099bdd7227b6cac/wrapt-2.3.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:261f53870cd4fb2bf38f9f972c56c728fd224cb7c65721307de59d9e7e6741ae", size = 159140, upload-time = "2026-07-28T06:04:57.754Z" },+ { url = "https://files.pythonhosted.org/packages/29/a8/08a56e2000a8816d449dcbad8c8b081697acbbd490821ceca0f9d8e8d20c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8159ec0b0cb7608175eb150de94c19e34f4d47ac655f5ca9baf45df6b688ffd3", size = 169263, upload-time = "2026-07-28T06:04:59.161Z" },+ { url = "https://files.pythonhosted.org/packages/9e/d4/354e1725e35a73b2af4fa70a3e024c7a5d1bf1802dfb862dcb668aae0253/wrapt-2.3.0-cp313-cp313-win32.whl", hash = "sha256:10461884b3014fbfc8eb7d09a93c5f246363e6711d9d881f95eb8c27fdef049f", size = 78241, upload-time = "2026-07-28T06:05:00.507Z" },+ { url = "https://files.pythonhosted.org/packages/6c/7e/34c87fa2174848dfee820322aaa318bab08913998ccecc8d2f57b4ad4639/wrapt-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:ac870cc97b73bb00ac353329e9559a4bebc47c4c86792ed9b23b58c15b6ad838", size = 81113, upload-time = "2026-07-28T06:05:01.839Z" },+ { url = "https://files.pythonhosted.org/packages/11/86/fcc9a530579e008c9478bb565a6cdfbfd33536660f069c8b91a6607c5050/wrapt-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:a65e8db2b4e90c2e7ade931086351c98ef420bf7a94ee08c95ac8a3cbbc43579", size = 80182, upload-time = "2026-07-28T06:05:03.152Z" },+ { url = "https://files.pythonhosted.org/packages/96/50/3864848b95b28ef73e17551fc8dccbff2628a834f52cf26a57f9c419fb83/wrapt-2.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:fd1f2f557dd3491fe75905e578f4db967393d40d1a8f468edc4d40ac7f2d5944", size = 83921, upload-time = "2026-07-28T06:05:04.476Z" },+ { url = "https://files.pythonhosted.org/packages/3b/4c/3d1921a60c3e8c71c540ff136e6a47a1fbccf7f671e818394889f7871d9c/wrapt-2.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9f5d2aec29dfc76c37e23897dee92766a3fd4f3bff3ae7fc9c6b4bf37d8c1360", size = 84412, upload-time = "2026-07-28T06:05:05.921Z" },+ { url = "https://files.pythonhosted.org/packages/fa/1a/4a796ff7adb26ada6d4b758c94d47a38320b085e7099afc088efbbcdb006/wrapt-2.3.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:646d20d413ffcd1b0a2f700076e2d0252d872dcb7754860a73e45a59ea883614", size = 207168, upload-time = "2026-07-28T06:05:07.256Z" },+ { url = "https://files.pythonhosted.org/packages/1d/3e/d7777776806c579b761bac2f91721dda9f04c7a1b380213c5935cc750ae6/wrapt-2.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:379f670f45b7bb8993edd9f6fc36c6cc65edb81cffa0b504be34acb0303fff0a", size = 214351, upload-time = "2026-07-28T06:05:08.945Z" },+ { url = "https://files.pythonhosted.org/packages/63/27/2d64d394df7bf181955b3bb562bf33c4492fb4be113f53071106d43ad8b5/wrapt-2.3.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6208f302f110295d64b22a7ac96500c791bf492dce4366e622e4912b077c9687", size = 199020, upload-time = "2026-07-28T06:05:10.418Z" },+ { url = "https://files.pythonhosted.org/packages/3e/3d/fb31d3db7d9834d265fb1a27a2adf0ddf51557c67458c97b22439ad6ae3d/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ed635a9ca4f3a5a2b900c10c69e823373bc00ebc114b459383596d3487da3570", size = 209969, upload-time = "2026-07-28T06:05:11.983Z" },+ { url = "https://files.pythonhosted.org/packages/1f/d1/8724b5da582e62070dc9bf4d8bf1972f317297eefd7ba1f2b5c6393ccf6c/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:e3b9eaa742ae7a0aaaaad4ca4b69469d757af2d6e6663ef1dadc47adec0aeb41", size = 196324, upload-time = "2026-07-28T06:05:13.557Z" },+ { url = "https://files.pythonhosted.org/packages/0d/5c/3d9ef411149543016ee6bcf3af707f787cebd946527452b94bf122e9b7b4/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d0f7284f88f4833705132d06d3b425a43095c2cbd07c58166aac3ab646ba12a4", size = 202610, upload-time = "2026-07-28T06:05:15.048Z" },+ { url = "https://files.pythonhosted.org/packages/13/9b/4fc042ceb757866dd4a5fc057b3b736f2b360d3703ce9f830d83dc9226e0/wrapt-2.3.0-cp313-cp313t-win32.whl", hash = "sha256:7ebb274aba688b043429eb1500ff8a76ce0cb8ac0812ca3e301f06247b8722b3", size = 79178, upload-time = "2026-07-28T06:05:16.469Z" },+ { url = "https://files.pythonhosted.org/packages/6b/ff/b94878f8eed809ca042685276bcea9f24e8c2ca7c9653bb80bbb920a68a5/wrapt-2.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c4bded758ad6f03b965830944a2f0bc5b2eb3767fe5a7310134315d1a6610e98", size = 82634, upload-time = "2026-07-28T06:05:18.026Z" },+ { url = "https://files.pythonhosted.org/packages/80/fb/663e1de5332a71685a729754312d327d4cada767c36e1c5a2db4c8de49e6/wrapt-2.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:d2cc64539da63e39ffb9c7ede849b6e8ddaaf7b3876b5cfb04efd85a5f3f4eb6", size = 81387, upload-time = "2026-07-28T06:05:19.417Z" },+ { url = "https://files.pythonhosted.org/packages/58/10/b073beaea89bc0d3670a75ff51139430a54b6af7ba7796507730634536dd/wrapt-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea52a0d0f08c584943d5764be0e84efa912c8da23c23e1e285ff2f5641c18fcc", size = 81978, upload-time = "2026-07-28T06:05:21.133Z" },+ { url = "https://files.pythonhosted.org/packages/b3/31/0916d9cebf848ed3f1a0c1888faee421747df77331e4db2bc527a9a85988/wrapt-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd85b0aa88efdb189d6ae2f35f4526943a8f091c38599c9c31478241c819e6a1", size = 82518, upload-time = "2026-07-28T06:05:22.562Z" },+ { url = "https://files.pythonhosted.org/packages/f5/73/31c1bf0f3384062751c2094dadb314916d70aa9b6bfd26d994b4a7b393fa/wrapt-2.3.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:141ed6211286a9660d8d6702de598b43f0934b4f0eda16393f100a80f501d945", size = 170187, upload-time = "2026-07-28T06:05:23.904Z" },+ { url = "https://files.pythonhosted.org/packages/ed/25/fce087d54b79b8905f3c3c9dd5f454bbd8d8acb80b960c4a6aee5b4659b3/wrapt-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e49885a62ec4ee854d1b9e6371fda6afd219917225752abf729a3f36d4df9a5", size = 169288, upload-time = "2026-07-28T06:05:25.378Z" },+ { url = "https://files.pythonhosted.org/packages/c7/30/0d09e6dddc6b7a7230ac77f50254b5980ab4fcd22976f72f8cc8a0404458/wrapt-2.3.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d6159c9b2fefec02314e1332dbbbfaf960e369dfd26bcf7f8b258b5732065b3", size = 160932, upload-time = "2026-07-28T06:05:27.022Z" },+ { url = "https://files.pythonhosted.org/packages/2c/ca/0913af0d2ec0c43865d32d615f518fea66c13c5c930e489e9b0de248e9a8/wrapt-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24da48596326ef8e448cfa837b454f638713d3531262375f00e5a9681682fc07", size = 169017, upload-time = "2026-07-28T06:05:28.501Z" },+ { url = "https://files.pythonhosted.org/packages/c3/f2/3d1e47ea81b822210f5df1bf942fd90780a75c055243d569b664529dea88/wrapt-2.3.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cd3a2edf0427013736b8127955cec62608c56e53ea47e82812ea32059cda407f", size = 159065, upload-time = "2026-07-28T06:05:30.01Z" },+ { url = "https://files.pythonhosted.org/packages/43/a5/ef2066ced8e5fca204e2b361e9708e36555b40949c583d997ea3b590817d/wrapt-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fa0df3bff4e7ce45759f33fd39335fe2f60477bb9ecf7b8aa41e7d07ee36a23", size = 168821, upload-time = "2026-07-28T06:05:31.649Z" },+ { url = "https://files.pythonhosted.org/packages/d5/e1/016104650d4e572fa91506eb396b3dd8efbccc9284fdc1c9479c3d21db28/wrapt-2.3.0-cp314-cp314-win32.whl", hash = "sha256:2935d5454b3f179a29b12cf390ee47246740ba2c3a7545b1b46ba31a5f2a4a0b", size = 78700, upload-time = "2026-07-28T06:05:33.391Z" },+ { url = "https://files.pythonhosted.org/packages/3d/97/6fdc20a9f2ca304748b3f0819cbf377d55260562777bf0b615431bc3c181/wrapt-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:cc2cea812e5cb179a796b766747e7d3b21088760d8deb95676d482b8c8e6fa7d", size = 81422, upload-time = "2026-07-28T06:05:34.774Z" },+ { url = "https://files.pythonhosted.org/packages/5e/a4/9cbd53bf05746bea2c392af39cb052427a8ec95cbd494d930733d8f44681/wrapt-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:22cc5c0a717bd4da87018ae0bffd4c19c6fb679d3ff357216ba566ab26c76cab", size = 80639, upload-time = "2026-07-28T06:05:36.228Z" },+ { url = "https://files.pythonhosted.org/packages/43/bb/6c5e4a0f66ea0d2b2dd267e8dd05a0014eea56840b3c8595d40b0a5d1f91/wrapt-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a6b5984cd65dd639546f0eb4b8eacf1c31cb2fe9fb5c27bffe240987cdb2cf84", size = 84030, upload-time = "2026-07-28T06:05:37.714Z" },+ { url = "https://files.pythonhosted.org/packages/6a/eb/a1aedf03283bc9cbf8a1783995ddc54e3c5a86878f19002d2c428494f4c5/wrapt-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c88abcf53daef80e01a75c7530e727fa6e2c1888fe83e3dcdba4c96216a1f5c7", size = 84419, upload-time = "2026-07-28T06:05:39.131Z" },+ { url = "https://files.pythonhosted.org/packages/63/61/50d511c0dc5105563849e86daa3e16ac7feef699f79fb05af45ea70107d5/wrapt-2.3.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85de890ff968196e92dd1ae73a9fb8970495e7650a457b1c9ef0ac3dd550bce2", size = 207171, upload-time = "2026-07-28T06:05:40.69Z" },+ { url = "https://files.pythonhosted.org/packages/3f/59/9b538cf7795217e810699d16bc88b96a830d9b5c403eb2ec2db6b5f2ae81/wrapt-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50f416b74d092bb9f41b424e90dd457f365f7ba4b11de62a23679769a21bd85c", size = 214329, upload-time = "2026-07-28T06:05:42.287Z" },+ { url = "https://files.pythonhosted.org/packages/b3/28/9935d62b1499e5c8b3d191e99ba4eb31ca237a0b699142011a837e9dc7ea/wrapt-2.3.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39febbee6d77301d31da6996b152ce52452da7c7ef72aba10c2fa976dff9c295", size = 199079, upload-time = "2026-07-28T06:05:43.958Z" },+ { url = "https://files.pythonhosted.org/packages/2b/01/4446b80fa2ffa47a3449b250d004ba1c1937f07f64a179608fec735df866/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:93513bec052c6cd987f9f580c3df068c8bc4ebae6543736be3ca7ec5959cafcd", size = 209992, upload-time = "2026-07-28T06:05:45.677Z" },+ { url = "https://files.pythonhosted.org/packages/d4/07/56f26c9f9979586a021e8148747004aba4498f49458c90b0502969b904e1/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:729126e667da34d251b8ebf8a45ef0c5ddadc21542b3d6e1abf4259ece6508df", size = 196334, upload-time = "2026-07-28T06:05:47.608Z" },+ { url = "https://files.pythonhosted.org/packages/8b/41/6d7bcc895b0f28b2250e10908f060687b9165429dcd7f22ddb3d4c031b74/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:626b69db2021aa01671ec7bbc9740e558522bd44c18cf2ce69bf3d666a014109", size = 202644, upload-time = "2026-07-28T06:05:49.183Z" },+ { url = "https://files.pythonhosted.org/packages/cd/25/7860927edba06b758b8852a6f02e832be715563c67a6795d94350bc81099/wrapt-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:629d73378082c00a8173031f9fb30a3ac6abbc894a5bfdfae71fabc60642d501", size = 79685, upload-time = "2026-07-28T06:05:50.976Z" },+ { url = "https://files.pythonhosted.org/packages/c4/0f/270bafe92fde3b069a39bc01e39ee79340895b335640df861d43d2a51885/wrapt-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:42869085687f0aefd57c0f636c3f9354f8ffb321a8ba9cb52d19beb796e561c5", size = 83104, upload-time = "2026-07-28T06:05:52.405Z" },+ { url = "https://files.pythonhosted.org/packages/55/b3/af176d79a8515a8a720eccdad9a96f6e31a30abf2865430c8c42adf2fd13/wrapt-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b1e5aa486e269b00ed35e64771c7d0ab8096cfd2643405ca8cd60ebedc099a51", size = 81774, upload-time = "2026-07-28T06:05:53.902Z" },+ { url = "https://files.pythonhosted.org/packages/00/39/3daf9f47be208606586de4568ba6713db53ebc8fd7a575aea1fe57983b69/wrapt-2.3.0-py3-none-any.whl", hash = "sha256:d8c7ed08477429752b8c44991f40ad7838b18332a160698740a6bfbc10d998a2", size = 61866, upload-time = "2026-07-28T06:06:12.9Z" },+]+[[package]]name = "yarl"version = "1.23.0"