Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix: k8s Attributes not automatically set in spans by Operator (#1091)

Ishan Jaincc1ee8c12676
diff --git a/docs/latest/operator/configuration/autoinstrumentation.mdx b/docs/latest/operator/configuration/autoinstrumentation.mdx
index 16eb8bea..dc8c6abd 100644
--- a/docs/latest/operator/configuration/autoinstrumentation.mdx
+++ b/docs/latest/operator/configuration/autoinstrumentation.mdx
@@ -382,15 +382,40 @@ Configure resource attributes that will be added to all telemetry data.
### Automatic Attributes
-The operator automatically sets these attributes (cannot be overridden):
+The operator automatically adds these Kubernetes resource attributes to `OTEL_RESOURCE_ATTRIBUTES`.
+If you also define `OTEL_RESOURCE_ATTRIBUTES` in your Deployment spec, your values take precedence on conflict.
| Attribute | Source | Example | Description |
|-----------|--------|---------|-------------|
-| `k8s.pod.name` | Pod metadata | `"chatbot-abc123"` | Kubernetes pod name |
+| `k8s.pod.name` | Downward API (`metadata.name`) | `"chatbot-abc123"` | Kubernetes pod name |
| `k8s.namespace.name` | Pod namespace | `"production"` | Kubernetes namespace |
| `k8s.deployment.name` | Owner reference | `"ai-chatbot"` | Deployment name (if applicable) |
| `k8s.replicaset.name` | Owner reference | `"ai-chatbot-xyz789"` | ReplicaSet name (if applicable) |
-| `k8s.node.name` | Pod spec | `"worker-node-1"` | Node where pod is running |
+| `k8s.node.name` | Downward API (`spec.nodeName`) | `"worker-node-1"` | Node where pod is running |
+
+### Overriding Automatic Attributes
+
+To override any automatic attribute, set `OTEL_RESOURCE_ATTRIBUTES` in your Deployment spec:
+
+```yaml
+spec:
+ containers:
+ - name: my-app
+ env:
+ - name: OTEL_RESOURCE_ATTRIBUTES
+ value: "k8s.deployment.name=my-custom-name,my.custom.attr=value"
+```
+
+The operator merges your attributes with its auto-detected ones. Your values win on conflict.
+
+### Pod Annotations
+
+| Annotation | Description | Example |
+|------------|-------------|---------|
+| `openlit.io/inject-python` | Force Python instrumentation on all containers (bypasses language detection) | `"true"` |
+| `openlit.io/container-languages` | Map containers to languages; `python` bypasses heuristics | `"app:python,sidecar:java"` |
+| `openlit.io/exclude-containers` | Exclude specific containers from instrumentation | `"sidecar,debug"` |
+| `openlit.io/include-containers` | Only instrument these containers (allowlist) | `"app,worker"` |
<AccordionGroup>
<Accordion title="Resource Attributes Examples">
@@ -436,4 +461,26 @@ The operator automatically sets these attributes (cannot be overridden):
serviceVersion: "v1.5.2"
```
</Accordion>
+
+ <Accordion title="Custom Image Language Detection">
+ If your Python application uses a custom Docker image that the operator cannot auto-detect
+ as Python (e.g. private registry images like `my-registry.com/litellm:v1.0`), use one of
+ these approaches:
+
+ ```yaml
+ # Option 1: Force all containers as Python
+ metadata:
+ annotations:
+ openlit.io/inject-python: "true"
+
+ # Option 2: Map specific containers to Python
+ metadata:
+ annotations:
+ openlit.io/container-languages: "litellm:python"
+ ```
+
+ The operator auto-detects common Python images (`python`, `django`, `flask`, `fastapi`,
+ `uvicorn`, `gunicorn`, `litellm`, `streamlit`, `gradio`, `airflow`, `mlflow`, `ray`, etc.)
+ and Python-related environment variables or commands.
+ </Accordion>
</AccordionGroup>
diff --git a/operator/README.md b/operator/README.md
index 01018f03..42eaa425 100644
--- a/operator/README.md
+++ b/operator/README.md
@@ -120,7 +120,6 @@ data:
OPENLIT_OTLP_ENDPOINT: "http://your-backend:4318"
OPENLIT_DEFAULT_ENVIRONMENT: "production"
OPENLIT_CAPTURE_MESSAGE_CONTENT: "true"
- OPENLIT_DETAILED_TRACING: "false"
```
## 📋 **Examples**
diff --git a/operator/internal/injector/injector.go b/operator/internal/injector/injector.go
index 3058b5c6..6f3d5fad 100644
--- a/operator/internal/injector/injector.go
+++ b/operator/internal/injector/injector.go
@@ -288,23 +288,59 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,
deploymentEnvironment = "kubernetes" // Simple fallback
}
- // Build comprehensive OTEL_RESOURCE_ATTRIBUTES following OpenTelemetry semantic conventions
- resourceAttrs := []string{
- fmt.Sprintf("service.name=%s", serviceName),
- fmt.Sprintf("service.namespace=%s", serviceNamespace),
- fmt.Sprintf("deployment.environment=%s", deploymentEnvironment),
+ // Build operator-generated resource attributes as a map (user can override any key)
+ operatorAttrs := map[string]string{
+ "service.name": serviceName,
+ "service.namespace": serviceNamespace,
+ "deployment.environment": deploymentEnvironment,
}
- // Add service.version only if available
if serviceVersion != "" {
- resourceAttrs = append(resourceAttrs, fmt.Sprintf("service.version=%s", serviceVersion))
+ operatorAttrs["service.version"] = serviceVersion
}
- // Only add service.instance.id if we have the actual pod name (not during admission webhook)
var serviceInstanceId string
if pod.Name != "" {
serviceInstanceId = i.generateServiceInstanceId(container, pod)
- resourceAttrs = append(resourceAttrs, fmt.Sprintf("service.instance.id=%s", serviceInstanceId))
+ operatorAttrs["service.instance.id"] = serviceInstanceId
+ }
+
+ // Add k8s.* resource attributes
+ if pod.Namespace != "" {
+ operatorAttrs["k8s.namespace.name"] = pod.Namespace
+ }
+
+ deploymentName, replicaSetName := i.extractK8sResourceNames(pod)
+ if deploymentName != "" {
+ operatorAttrs["k8s.deployment.name"] = deploymentName
+ }
+ if replicaSetName != "" {
+ operatorAttrs["k8s.replicaset.name"] = replicaSetName
+ }
+
+ // k8s.pod.name and k8s.node.name use downward API env var interpolation
+ // because they are not available at admission webhook time.
+ // Prefixed with OPENLIT_ to avoid colliding with user-defined env vars.
+ operatorAttrs["k8s.pod.name"] = "$(OPENLIT_K8S_POD_NAME)"
+ operatorAttrs["k8s.node.name"] = "$(OPENLIT_K8S_NODE_NAME)"
+
+ // Parse user's existing OTEL_RESOURCE_ATTRIBUTES and overlay on top (user wins on conflict)
+ for _, env := range container.Env {
+ if env.Name == "OTEL_RESOURCE_ATTRIBUTES" && env.Value != "" {
+ for _, pair := range strings.Split(env.Value, ",") {
+ parts := strings.SplitN(strings.TrimSpace(pair), "=", 2)
+ if len(parts) == 2 {
+ operatorAttrs[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+ }
+ }
+ break
+ }
+ }
+
+ // Serialize merged attributes
+ resourceAttrParts := make([]string, 0, len(operatorAttrs))
+ for k, v := range operatorAttrs {
+ resourceAttrParts = append(resourceAttrParts, fmt.Sprintf("%s=%s", k, v))
}
// Check if PYTHONPATH already exists in the container
@@ -324,6 +360,25 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,
}
envVars := []corev1.EnvVar{
+ // Downward API env vars for k8s attributes (must come before OTEL_RESOURCE_ATTRIBUTES
+ // so that $(OPENLIT_K8S_POD_NAME) and $(OPENLIT_K8S_NODE_NAME) interpolation works).
+ // Namespaced with OPENLIT_ prefix to avoid colliding with user-defined env vars.
+ {
+ Name: "OPENLIT_K8S_POD_NAME",
+ ValueFrom: &corev1.EnvVarSource{
+ FieldRef: &corev1.ObjectFieldSelector{
+ FieldPath: "metadata.name",
+ },
+ },
+ },
+ {
+ Name: "OPENLIT_K8S_NODE_NAME",
+ ValueFrom: &corev1.EnvVarSource{
+ FieldRef: &corev1.ObjectFieldSelector{
+ FieldPath: "spec.nodeName",
+ },
+ },
+ },
{
Name: "PYTHONPATH",
Value: pythonPathValue,
@@ -351,7 +406,7 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,
},
{
Name: "OTEL_RESOURCE_ATTRIBUTES",
- Value: strings.Join(resourceAttrs, ","),
+ Value: strings.Join(resourceAttrParts, ","),
},
// Optional service version
{
@@ -373,16 +428,6 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,
})
}
- // Add optional OpenLIT-specific environment variables (removed advanced options for now)
-
- if i.config.DetailedTracing {
- envVars = append(envVars, corev1.EnvVar{
- Name: "OPENLIT_DETAILED_TRACING",
- Value: "true",
- })
- }
-
- // Phoenix-specific variables removed - using pure OpenTelemetry approach for OpenInference
// Add custom instrumentation environment variables
if i.config.CustomPackages != "" {
envVars = append(envVars, corev1.EnvVar{
@@ -391,13 +436,16 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,
})
}
- log.Info("🌍 OpenTelemetry resource attributes configured",
+ log.Info("OpenTelemetry resource attributes configured",
"component", "injector",
"service.name", serviceName,
"service.namespace", serviceNamespace,
"service.instance.id", serviceInstanceId,
"service.version", serviceVersion,
"deployment.environment", deploymentEnvironment,
+ "k8s.namespace.name", pod.Namespace,
+ "k8s.deployment.name", deploymentName,
+ "k8s.replicaset.name", replicaSetName,
"otel.exporter.otlp.endpoint", i.config.OTLPEndpoint,
"instrumentation.provider", i.config.Provider,
"instrumentation.custom_packages", i.config.CustomPackages)
@@ -405,6 +453,34 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,
return envVars
}
+// extractK8sResourceNames returns (deploymentName, replicaSetName) from pod owner references.
+func (i *OpenLitInjector) extractK8sResourceNames(pod *corev1.Pod) (string, string) {
+ if len(pod.OwnerReferences) == 0 {
+ return "", ""
+ }
+
+ var deploymentName, replicaSetName string
+
+ for _, owner := range pod.OwnerReferences {
+ switch owner.Kind {
+ case "ReplicaSet":
+ replicaSetName = owner.Name
+ // Deployment name = ReplicaSet name minus the trailing hash suffix
+ if lastDash := strings.LastIndex(owner.Name, "-"); lastDash > 0 && lastDash < len(owner.Name)-5 {
+ deploymentName = owner.Name[:lastDash]
+ }
+ case "Deployment":
+ deploymentName = owner.Name
+ case "StatefulSet", "DaemonSet", "Job", "CronJob":
+ if deploymentName == "" {
+ deploymentName = owner.Name
+ }
+ }
+ }
+
+ return deploymentName, replicaSetName
+}
+
// generateServiceName calculates service.name following OpenTelemetry K8s conventions
// Priority: annotation -> app.instance -> app.name -> k8s.deployment.name -> k8s.replicaset.name ->
//
@@ -1003,8 +1079,17 @@ func (i *OpenLitInjector) validateSecurityContext(pod *corev1.Pod) error {
// shouldInstrumentContainer determines if a specific container should be instrumented
func (i *OpenLitInjector) shouldInstrumentContainer(container *corev1.Container, pod *corev1.Pod) bool {
- // Check for explicit container exclusion annotation
if pod.Annotations != nil {
+ // Force-inject annotation bypasses ALL heuristic checks:
+ // openlit.io/inject-python: "true"
+ if val, exists := pod.Annotations["openlit.io/inject-python"]; exists && strings.ToLower(val) == "true" {
+ log.Info("Force-injecting Python instrumentation via openlit.io/inject-python annotation",
+ "component", "injector",
+ "k8s.container.name", container.Name,
+ "k8s.pod.name", pod.Name)
+ return true
+ }
+
// Check for container-specific exclusion: openlit.io/exclude-containers: "container1,container2"
if excludeList, exists := pod.Annotations["openlit.io/exclude-containers"]; exists {
excludedContainers := strings.Split(excludeList, ",")
@@ -1055,8 +1140,17 @@ func (i *OpenLitInjector) shouldInstrumentContainer(container *corev1.Container,
}
}
- // If this container is mapped to a different language than we support, skip it
- if containerLanguage != "" && containerLanguage != "python" {
+ if containerLanguage == "python" {
+ // Explicitly marked as Python -- bypass all heuristic checks
+ log.Info("Container confirmed as Python via openlit.io/container-languages annotation",
+ "component", "injector",
+ "k8s.container.name", container.Name,
+ "k8s.pod.name", pod.Name)
+ return true
+ }
+
+ if containerLanguage != "" {
+ // Mapped to a non-Python language, skip it
log.Info("Container uses unsupported language - skipping instrumentation",
"component", "injector",
"k8s.container.name", container.Name,
@@ -1162,6 +1256,17 @@ func (i *OpenLitInjector) isPythonContainer(container *corev1.Container) bool {
"jupyter",
"anaconda",
"miniconda",
+ "litellm",
+ "streamlit",
+ "gradio",
+ "langserve",
+ "chainlit",
+ "prefect",
+ "airflow",
+ "dbt",
+ "mlflow",
+ "bentoml",
+ "ray",
}
for _, pattern := range pythonImagePatterns {
@@ -1200,7 +1305,13 @@ func (i *OpenLitInjector) isPythonContainer(container *corev1.Container) bool {
strings.Contains(argLower, "django") ||
strings.Contains(argLower, "flask") ||
strings.Contains(argLower, "gunicorn") ||
- strings.Contains(argLower, "uvicorn") {
+ strings.Contains(argLower, "uvicorn") ||
+ strings.Contains(argLower, "litellm") ||
+ strings.Contains(argLower, "streamlit") ||
+ strings.Contains(argLower, "celery") ||
+ strings.Contains(argLower, "airflow") ||
+ strings.Contains(argLower, "prefect") ||
+ strings.Contains(argLower, "mlflow") {
return true
}
}
@@ -1497,11 +1608,16 @@ func (i *OpenLitInjector) modifyContainerWithRecovery(container *corev1.Containe
container.Env = append(container.Env, envVars...)
+ // Apply CR-defined environment variables (spec.python.instrumentation.env)
+ if len(i.config.EnvVars) > 0 {
+ container.Env = append(container.Env, i.config.EnvVars...)
+ }
+
log.Info("Container modification completed",
"component", "injector",
"k8s.container.name", container.Name,
"instrumentation.provider", i.config.Provider,
- "env_vars.added", len(envVars),
+ "env_vars.added", len(envVars)+len(i.config.EnvVars),
"volume_mounts.added", 1)
return nil
diff --git a/operator/internal/injector/injector_test.go b/operator/internal/injector/injector_test.go
index ecc708c3..a5014e27 100644
--- a/operator/internal/injector/injector_test.go
+++ b/operator/internal/injector/injector_test.go
@@ -28,7 +28,6 @@ func (suite *InjectorTestSuite) SetupTest() {
ServiceName: "test-app",
ServiceNamespace: "default",
CaptureMessageContent: true,
- DetailedTracing: true,
SharedVolumeName: "instrumentation-packages",
SharedVolumePath: "/instrumentation-packages",
}
diff --git a/operator/internal/injector/types.go b/operator/internal/injector/types.go
index f765240c..42092937 100644
--- a/operator/internal/injector/types.go
+++ b/operator/internal/injector/types.go
@@ -24,7 +24,6 @@ type InjectorConfig struct {
ServiceName string `json:"serviceName"`
ServiceNamespace string `json:"serviceNamespace"`
CaptureMessageContent bool `json:"captureMessageContent"`
- DetailedTracing bool `json:"detailedTracing"`
// Volume and mount configuration
SharedVolumeName string `json:"sharedVolumeName"`
diff --git a/operator/internal/webhook/handler.go b/operator/internal/webhook/handler.go
index f4ab6d77..cefe10e0 100644
--- a/operator/internal/webhook/handler.go
+++ b/operator/internal/webhook/handler.go
@@ -434,7 +434,6 @@ func (h *Handler) createInjectorConfig(instrConfig *v1alpha1.AutoInstrumentation
// Instrumentation settings (set defaults)
CaptureMessageContent: true,
- DetailedTracing: true,
// Volume configuration (using configurable names instead of hardcoded)
SharedVolumeName: "instrumentation-packages",
diff --git a/operator/providers/openlit/sitecustomize.py b/operator/providers/openlit/sitecustomize.py
index faabae25..42efaa14 100644
--- a/operator/providers/openlit/sitecustomize.py
+++ b/operator/providers/openlit/sitecustomize.py
@@ -37,8 +37,6 @@ except Exception as e:
"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true"
).lower()
== "true",
- detailed_tracing=os.environ.get("OPENLIT_DETAILED_TRACING", "true").lower()
- == "true",
)
print("✅ OpenLIT fallback initialization successful!")