Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix: k8s Attributes not automatically set in spans by Operator (#1091)
Ishan Jaincc1ee8c12676
diff --git a/docs/latest/operator/configuration/autoinstrumentation.mdx b/docs/latest/operator/configuration/autoinstrumentation.mdxindex 16eb8bea..dc8c6abd 100644--- a/docs/latest/operator/configuration/autoinstrumentation.mdx+++ b/docs/latest/operator/configuration/autoinstrumentation.mdx@@ -382,15 +382,40 @@ Configure resource attributes that will be added to all telemetry data.### Automatic Attributes-The operator automatically sets these attributes (cannot be overridden):+The operator automatically adds these Kubernetes resource attributes to `OTEL_RESOURCE_ATTRIBUTES`.+If you also define `OTEL_RESOURCE_ATTRIBUTES` in your Deployment spec, your values take precedence on conflict.| Attribute | Source | Example | Description ||-----------|--------|---------|-------------|-| `k8s.pod.name` | Pod metadata | `"chatbot-abc123"` | Kubernetes pod name |+| `k8s.pod.name` | Downward API (`metadata.name`) | `"chatbot-abc123"` | Kubernetes pod name || `k8s.namespace.name` | Pod namespace | `"production"` | Kubernetes namespace || `k8s.deployment.name` | Owner reference | `"ai-chatbot"` | Deployment name (if applicable) || `k8s.replicaset.name` | Owner reference | `"ai-chatbot-xyz789"` | ReplicaSet name (if applicable) |-| `k8s.node.name` | Pod spec | `"worker-node-1"` | Node where pod is running |+| `k8s.node.name` | Downward API (`spec.nodeName`) | `"worker-node-1"` | Node where pod is running |++### Overriding Automatic Attributes++To override any automatic attribute, set `OTEL_RESOURCE_ATTRIBUTES` in your Deployment spec:++```yaml+spec:+ containers:+ - name: my-app+ env:+ - name: OTEL_RESOURCE_ATTRIBUTES+ value: "k8s.deployment.name=my-custom-name,my.custom.attr=value"+```++The operator merges your attributes with its auto-detected ones. Your values win on conflict.++### Pod Annotations++| Annotation | Description | Example |+|------------|-------------|---------|+| `openlit.io/inject-python` | Force Python instrumentation on all containers (bypasses language detection) | `"true"` |+| `openlit.io/container-languages` | Map containers to languages; `python` bypasses heuristics | `"app:python,sidecar:java"` |+| `openlit.io/exclude-containers` | Exclude specific containers from instrumentation | `"sidecar,debug"` |+| `openlit.io/include-containers` | Only instrument these containers (allowlist) | `"app,worker"` |<AccordionGroup><Accordion title="Resource Attributes Examples">@@ -436,4 +461,26 @@ The operator automatically sets these attributes (cannot be overridden):serviceVersion: "v1.5.2"```</Accordion>++ <Accordion title="Custom Image Language Detection">+ If your Python application uses a custom Docker image that the operator cannot auto-detect+ as Python (e.g. private registry images like `my-registry.com/litellm:v1.0`), use one of+ these approaches:++ ```yaml+ # Option 1: Force all containers as Python+ metadata:+ annotations:+ openlit.io/inject-python: "true"++ # Option 2: Map specific containers to Python+ metadata:+ annotations:+ openlit.io/container-languages: "litellm:python"+ ```++ The operator auto-detects common Python images (`python`, `django`, `flask`, `fastapi`,+ `uvicorn`, `gunicorn`, `litellm`, `streamlit`, `gradio`, `airflow`, `mlflow`, `ray`, etc.)+ and Python-related environment variables or commands.+ </Accordion></AccordionGroup>diff --git a/operator/README.md b/operator/README.mdindex 01018f03..42eaa425 100644--- a/operator/README.md+++ b/operator/README.md@@ -120,7 +120,6 @@ data:OPENLIT_OTLP_ENDPOINT: "http://your-backend:4318"OPENLIT_DEFAULT_ENVIRONMENT: "production"OPENLIT_CAPTURE_MESSAGE_CONTENT: "true"- OPENLIT_DETAILED_TRACING: "false"```## 📋 **Examples**diff --git a/operator/internal/injector/injector.go b/operator/internal/injector/injector.goindex 3058b5c6..6f3d5fad 100644--- a/operator/internal/injector/injector.go+++ b/operator/internal/injector/injector.go@@ -288,23 +288,59 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,deploymentEnvironment = "kubernetes" // Simple fallback}- // Build comprehensive OTEL_RESOURCE_ATTRIBUTES following OpenTelemetry semantic conventions- resourceAttrs := []string{- fmt.Sprintf("service.name=%s", serviceName),- fmt.Sprintf("service.namespace=%s", serviceNamespace),- fmt.Sprintf("deployment.environment=%s", deploymentEnvironment),+ // Build operator-generated resource attributes as a map (user can override any key)+ operatorAttrs := map[string]string{+ "service.name": serviceName,+ "service.namespace": serviceNamespace,+ "deployment.environment": deploymentEnvironment,}- // Add service.version only if availableif serviceVersion != "" {- resourceAttrs = append(resourceAttrs, fmt.Sprintf("service.version=%s", serviceVersion))+ operatorAttrs["service.version"] = serviceVersion}- // Only add service.instance.id if we have the actual pod name (not during admission webhook)var serviceInstanceId stringif pod.Name != "" {serviceInstanceId = i.generateServiceInstanceId(container, pod)- resourceAttrs = append(resourceAttrs, fmt.Sprintf("service.instance.id=%s", serviceInstanceId))+ operatorAttrs["service.instance.id"] = serviceInstanceId+ }++ // Add k8s.* resource attributes+ if pod.Namespace != "" {+ operatorAttrs["k8s.namespace.name"] = pod.Namespace+ }++ deploymentName, replicaSetName := i.extractK8sResourceNames(pod)+ if deploymentName != "" {+ operatorAttrs["k8s.deployment.name"] = deploymentName+ }+ if replicaSetName != "" {+ operatorAttrs["k8s.replicaset.name"] = replicaSetName+ }++ // k8s.pod.name and k8s.node.name use downward API env var interpolation+ // because they are not available at admission webhook time.+ // Prefixed with OPENLIT_ to avoid colliding with user-defined env vars.+ operatorAttrs["k8s.pod.name"] = "$(OPENLIT_K8S_POD_NAME)"+ operatorAttrs["k8s.node.name"] = "$(OPENLIT_K8S_NODE_NAME)"++ // Parse user's existing OTEL_RESOURCE_ATTRIBUTES and overlay on top (user wins on conflict)+ for _, env := range container.Env {+ if env.Name == "OTEL_RESOURCE_ATTRIBUTES" && env.Value != "" {+ for _, pair := range strings.Split(env.Value, ",") {+ parts := strings.SplitN(strings.TrimSpace(pair), "=", 2)+ if len(parts) == 2 {+ operatorAttrs[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])+ }+ }+ break+ }+ }++ // Serialize merged attributes+ resourceAttrParts := make([]string, 0, len(operatorAttrs))+ for k, v := range operatorAttrs {+ resourceAttrParts = append(resourceAttrParts, fmt.Sprintf("%s=%s", k, v))}// Check if PYTHONPATH already exists in the container@@ -324,6 +360,25 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,}envVars := []corev1.EnvVar{+ // Downward API env vars for k8s attributes (must come before OTEL_RESOURCE_ATTRIBUTES+ // so that $(OPENLIT_K8S_POD_NAME) and $(OPENLIT_K8S_NODE_NAME) interpolation works).+ // Namespaced with OPENLIT_ prefix to avoid colliding with user-defined env vars.+ {+ Name: "OPENLIT_K8S_POD_NAME",+ ValueFrom: &corev1.EnvVarSource{+ FieldRef: &corev1.ObjectFieldSelector{+ FieldPath: "metadata.name",+ },+ },+ },+ {+ Name: "OPENLIT_K8S_NODE_NAME",+ ValueFrom: &corev1.EnvVarSource{+ FieldRef: &corev1.ObjectFieldSelector{+ FieldPath: "spec.nodeName",+ },+ },+ },{Name: "PYTHONPATH",Value: pythonPathValue,@@ -351,7 +406,7 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,},{Name: "OTEL_RESOURCE_ATTRIBUTES",- Value: strings.Join(resourceAttrs, ","),+ Value: strings.Join(resourceAttrParts, ","),},// Optional service version{@@ -373,16 +428,6 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,})}- // Add optional OpenLIT-specific environment variables (removed advanced options for now)-- if i.config.DetailedTracing {- envVars = append(envVars, corev1.EnvVar{- Name: "OPENLIT_DETAILED_TRACING",- Value: "true",- })- }-- // Phoenix-specific variables removed - using pure OpenTelemetry approach for OpenInference// Add custom instrumentation environment variablesif i.config.CustomPackages != "" {envVars = append(envVars, corev1.EnvVar{@@ -391,13 +436,16 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,})}- log.Info("🌍 OpenTelemetry resource attributes configured",+ log.Info("OpenTelemetry resource attributes configured","component", "injector","service.name", serviceName,"service.namespace", serviceNamespace,"service.instance.id", serviceInstanceId,"service.version", serviceVersion,"deployment.environment", deploymentEnvironment,+ "k8s.namespace.name", pod.Namespace,+ "k8s.deployment.name", deploymentName,+ "k8s.replicaset.name", replicaSetName,"otel.exporter.otlp.endpoint", i.config.OTLPEndpoint,"instrumentation.provider", i.config.Provider,"instrumentation.custom_packages", i.config.CustomPackages)@@ -405,6 +453,34 @@ func (i *OpenLitInjector) buildEnvironmentVariables(container *corev1.Container,return envVars}+// extractK8sResourceNames returns (deploymentName, replicaSetName) from pod owner references.+func (i *OpenLitInjector) extractK8sResourceNames(pod *corev1.Pod) (string, string) {+ if len(pod.OwnerReferences) == 0 {+ return "", ""+ }++ var deploymentName, replicaSetName string++ for _, owner := range pod.OwnerReferences {+ switch owner.Kind {+ case "ReplicaSet":+ replicaSetName = owner.Name+ // Deployment name = ReplicaSet name minus the trailing hash suffix+ if lastDash := strings.LastIndex(owner.Name, "-"); lastDash > 0 && lastDash < len(owner.Name)-5 {+ deploymentName = owner.Name[:lastDash]+ }+ case "Deployment":+ deploymentName = owner.Name+ case "StatefulSet", "DaemonSet", "Job", "CronJob":+ if deploymentName == "" {+ deploymentName = owner.Name+ }+ }+ }++ return deploymentName, replicaSetName+}+// generateServiceName calculates service.name following OpenTelemetry K8s conventions// Priority: annotation -> app.instance -> app.name -> k8s.deployment.name -> k8s.replicaset.name ->//@@ -1003,8 +1079,17 @@ func (i *OpenLitInjector) validateSecurityContext(pod *corev1.Pod) error {// shouldInstrumentContainer determines if a specific container should be instrumentedfunc (i *OpenLitInjector) shouldInstrumentContainer(container *corev1.Container, pod *corev1.Pod) bool {- // Check for explicit container exclusion annotationif pod.Annotations != nil {+ // Force-inject annotation bypasses ALL heuristic checks:+ // openlit.io/inject-python: "true"+ if val, exists := pod.Annotations["openlit.io/inject-python"]; exists && strings.ToLower(val) == "true" {+ log.Info("Force-injecting Python instrumentation via openlit.io/inject-python annotation",+ "component", "injector",+ "k8s.container.name", container.Name,+ "k8s.pod.name", pod.Name)+ return true+ }+// Check for container-specific exclusion: openlit.io/exclude-containers: "container1,container2"if excludeList, exists := pod.Annotations["openlit.io/exclude-containers"]; exists {excludedContainers := strings.Split(excludeList, ",")@@ -1055,8 +1140,17 @@ func (i *OpenLitInjector) shouldInstrumentContainer(container *corev1.Container,}}- // If this container is mapped to a different language than we support, skip it- if containerLanguage != "" && containerLanguage != "python" {+ if containerLanguage == "python" {+ // Explicitly marked as Python -- bypass all heuristic checks+ log.Info("Container confirmed as Python via openlit.io/container-languages annotation",+ "component", "injector",+ "k8s.container.name", container.Name,+ "k8s.pod.name", pod.Name)+ return true+ }++ if containerLanguage != "" {+ // Mapped to a non-Python language, skip itlog.Info("Container uses unsupported language - skipping instrumentation","component", "injector","k8s.container.name", container.Name,@@ -1162,6 +1256,17 @@ func (i *OpenLitInjector) isPythonContainer(container *corev1.Container) bool {"jupyter","anaconda","miniconda",+ "litellm",+ "streamlit",+ "gradio",+ "langserve",+ "chainlit",+ "prefect",+ "airflow",+ "dbt",+ "mlflow",+ "bentoml",+ "ray",}for _, pattern := range pythonImagePatterns {@@ -1200,7 +1305,13 @@ func (i *OpenLitInjector) isPythonContainer(container *corev1.Container) bool {strings.Contains(argLower, "django") ||strings.Contains(argLower, "flask") ||strings.Contains(argLower, "gunicorn") ||- strings.Contains(argLower, "uvicorn") {+ strings.Contains(argLower, "uvicorn") ||+ strings.Contains(argLower, "litellm") ||+ strings.Contains(argLower, "streamlit") ||+ strings.Contains(argLower, "celery") ||+ strings.Contains(argLower, "airflow") ||+ strings.Contains(argLower, "prefect") ||+ strings.Contains(argLower, "mlflow") {return true}}@@ -1497,11 +1608,16 @@ func (i *OpenLitInjector) modifyContainerWithRecovery(container *corev1.Containecontainer.Env = append(container.Env, envVars...)+ // Apply CR-defined environment variables (spec.python.instrumentation.env)+ if len(i.config.EnvVars) > 0 {+ container.Env = append(container.Env, i.config.EnvVars...)+ }+log.Info("Container modification completed","component", "injector","k8s.container.name", container.Name,"instrumentation.provider", i.config.Provider,- "env_vars.added", len(envVars),+ "env_vars.added", len(envVars)+len(i.config.EnvVars),"volume_mounts.added", 1)return nildiff --git a/operator/internal/injector/injector_test.go b/operator/internal/injector/injector_test.goindex ecc708c3..a5014e27 100644--- a/operator/internal/injector/injector_test.go+++ b/operator/internal/injector/injector_test.go@@ -28,7 +28,6 @@ func (suite *InjectorTestSuite) SetupTest() {ServiceName: "test-app",ServiceNamespace: "default",CaptureMessageContent: true,- DetailedTracing: true,SharedVolumeName: "instrumentation-packages",SharedVolumePath: "/instrumentation-packages",}diff --git a/operator/internal/injector/types.go b/operator/internal/injector/types.goindex f765240c..42092937 100644--- a/operator/internal/injector/types.go+++ b/operator/internal/injector/types.go@@ -24,7 +24,6 @@ type InjectorConfig struct {ServiceName string `json:"serviceName"`ServiceNamespace string `json:"serviceNamespace"`CaptureMessageContent bool `json:"captureMessageContent"`- DetailedTracing bool `json:"detailedTracing"`// Volume and mount configurationSharedVolumeName string `json:"sharedVolumeName"`diff --git a/operator/internal/webhook/handler.go b/operator/internal/webhook/handler.goindex f4ab6d77..cefe10e0 100644--- a/operator/internal/webhook/handler.go+++ b/operator/internal/webhook/handler.go@@ -434,7 +434,6 @@ func (h *Handler) createInjectorConfig(instrConfig *v1alpha1.AutoInstrumentation// Instrumentation settings (set defaults)CaptureMessageContent: true,- DetailedTracing: true,// Volume configuration (using configurable names instead of hardcoded)SharedVolumeName: "instrumentation-packages",diff --git a/operator/providers/openlit/sitecustomize.py b/operator/providers/openlit/sitecustomize.pyindex faabae25..42efaa14 100644--- a/operator/providers/openlit/sitecustomize.py+++ b/operator/providers/openlit/sitecustomize.py@@ -37,8 +37,6 @@ except Exception as e:"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true").lower()== "true",- detailed_tracing=os.environ.get("OPENLIT_DETAILED_TRACING", "true").lower()- == "true",)print("✅ OpenLIT fallback initialization successful!")