Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

41.9 KB
Raw Blame
package engine

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"net/url"
	"sort"
	"strings"
	"time"

	"github.com/openlit/openlit/openlit-controller/internal/config"
	"github.com/openlit/openlit/openlit-controller/internal/openlit"
	"go.uber.org/zap"
)

const (
	openlitInstrumentationVolume    = "openlit-instrumentation-packages"
	openlitInstrumentationMountPath = "/instrumentation-packages"
	openlitInstrumentationInitName  = "openlit-auto-instrumentation"
	openlitManagedLabel             = "openlit.io/python-sdk-managed"
	openlitManagedVolumeLabel       = "openlit.io/python-sdk-volume"
	openlitManagedAnnotation        = "openlit.io/python-sdk-managed"
	openlitManagedConfigHash        = "openlit.io/python-sdk-config-hash"
	openlitManagedProfileAnnotation = "openlit.io/python-sdk-profile"
	openlitManagedSDKVersion        = "openlit.io/python-sdk-version"
	controllerManagedObservability  = "controller_managed"
	controllerManagedAgentMode      = "agent_observability"
	defaultDuplicatePolicy          = "block_if_existing_otel_detected"
)

type pythonSDKDecision string

const (
	pythonSDKDecisionApply pythonSDKDecision = "apply_controller_managed_sdk"
	pythonSDKDecisionAdopt pythonSDKDecision = "adopt_existing_openlit"
	pythonSDKDecisionBlock pythonSDKDecision = "block_existing_otel"
)

type pythonSDKPreflightResult struct {
	Decision pythonSDKDecision
	Source   string
	Conflict string
	Reason   string
}

var controllerManagedDisabledInstrumentors = []string{
	"openai",
	"anthropic",
	"cohere",
	"mistral",
	"groq",
	"google-ai-studio",
	"azure-ai-inference",
	"bedrock",
	"vertexai",
	"together",
	"fireworks",
	"deepseek",
	"ollama",
	"litellm",
	"httpx",
	"requests",
	"urllib",
	"urllib3",
	"aiohttp-client",
}

func (e *Engine) EnablePythonSDK(serviceID string, payload openlit.PythonSDKActionPayload) error {
	payload = e.normalizeSDKPayload(payload, "python")
	if isNodeRuntime(payload.TargetRuntime) {
		return e.EnableNodeJSSDK(serviceID, payload)
	}
	if strings.ToLower(payload.TargetRuntime) != "python" {
		return fmt.Errorf("unsupported SDK target_runtime %q", payload.TargetRuntime)
	}
	// sdk_version is attacker-controllable (it arrives in the poll action payload
	// and is interpolated into pip install commands, including shell strings for
	// the k8s init container / docker helper). Reject anything that isn't a plain
	// PEP 440-style version before it can reach any exec path.
	if !isValidSDKVersion(payload.SDKVersion) {
		return fmt.Errorf("invalid sdk_version %q: must match a PEP 440 version (letters, digits, . _ + -)", payload.SDKVersion)
	}

	svc, err := e.serviceSnapshot(serviceID)
	if err != nil {
		return err
	}

	if strings.ToLower(svc.LanguageRuntime) != "python" {
		e.setAgentObservabilityState(
			serviceID,
			"unsupported",
			"none",
			"",
			"Controller-managed Python SDK injection is only available for Python services",
		)
		return fmt.Errorf("service %q is not a Python runtime", svc.ServiceName)
	}
	if svc.AgentObservabilityStatus == "enabled" && svc.AgentObservabilitySource == "existing_openlit" {
		e.setAgentObservabilityState(
			serviceID,
			"enabled",
			"existing_openlit",
			"",
			"Existing OpenLIT instrumentation is not controller-managed and will not be removed automatically",
		)
		return fmt.Errorf("service %q uses existing OpenLIT instrumentation that is not controller-managed", svc.ServiceName)
	}

	sdkVersionLabel := payload.SDKVersion
	if sdkVersionLabel == "" {
		sdkVersionLabel = "latest"
	}

	switch e.deployMode {
	case config.DeployKubernetes:
		if err := e.enablePythonSDKKubernetes(svc, payload); err != nil {
			return err
		}
		e.setDesiredAgentObservabilityState(
			serviceID,
			"enabled",
			"controller_managed",
			"",
			"Controller-managed Python SDK rollout configured for Kubernetes workload",
		)
		e.setSDKVersionAttr(serviceID, sdkVersionLabel)
		return nil
	case config.DeployDocker:
		if err := e.enablePythonSDKDocker(svc, payload); err != nil {
			return err
		}
		e.setDesiredAgentObservabilityState(
			serviceID,
			"enabled",
			"controller_managed",
			"",
			"Controller-managed Python SDK rollout configured for Docker container",
		)
		e.setSDKVersionAttr(serviceID, sdkVersionLabel)
		return nil
	default:
		isContainerized := svc.ResourceAttributes["openlit.is_containerized"] == "true"
		containerID := svc.ResourceAttributes["container.id"]

		if isContainerized && containerID != "" {
			if e.container != nil && e.container.dockerClient != nil && e.container.dockerClient.canManage() {
				if err := e.enablePythonSDKDocker(svc, payload); err != nil {
					return err
				}
				e.setDesiredAgentObservabilityState(
					serviceID,
					"enabled",
					"controller_managed",
					"",
					"Controller-managed Python SDK rollout configured for Docker container (detected from Linux host)",
				)
				e.setSDKVersionAttr(serviceID, sdkVersionLabel)
				return nil
			}
			e.setDesiredAgentObservabilityState(
				serviceID,
				"unsupported",
				"none",
				"",
				"This process runs inside a container but the Docker socket is not available. Mount /var/run/docker.sock or use Docker-mode controller for Agent Observability.",
			)
			return fmt.Errorf("containerized process detected but Docker socket not available for Agent O11y")
		}

		if svc.ResourceAttributes["systemd.unit"] != "" {
			if err := e.enablePythonSDKLinux(svc, payload); err != nil {
				return err
			}
			e.setDesiredAgentObservabilityState(
				serviceID,
				"enabled",
				"controller_managed",
				"",
				"Controller-managed Python SDK rollout configured for systemd service",
			)
			e.setSDKVersionAttr(serviceID, sdkVersionLabel)
			return nil
		}

		if err := e.enablePythonSDKBareProcess(svc, payload); err != nil {
			return err
		}
		e.setSDKVersionAttr(serviceID, sdkVersionLabel)
		return nil
	}
}

func (e *Engine) DisablePythonSDK(serviceID string, payload openlit.PythonSDKActionPayload) error {
	if isNodeRuntime(payload.TargetRuntime) {
		return e.DisableNodeJSSDK(serviceID, payload)
	}
	if payload.TargetRuntime != "" && strings.ToLower(payload.TargetRuntime) != "python" {
		return fmt.Errorf("unsupported SDK target_runtime %q", payload.TargetRuntime)
	}
	svc, err := e.serviceSnapshot(serviceID)
	if err != nil {
		return err
	}

	if strings.ToLower(svc.LanguageRuntime) != "python" {
		e.setAgentObservabilityState(
			serviceID,
			"unsupported",
			"none",
			"",
			"Controller-managed Python SDK injection is only available for Python services",
		)
		return fmt.Errorf("service %q is not a Python runtime", svc.ServiceName)
	}

	switch e.deployMode {
	case config.DeployKubernetes:
		if err := e.disablePythonSDKKubernetes(svc); err != nil {
			return err
		}
		e.setDesiredAgentObservabilityState(
			serviceID,
			"disabled",
			"none",
			"",
			"Controller-managed Python SDK removed from Kubernetes workload",
		)
		return nil
	case config.DeployDocker:
		if err := e.disablePythonSDKDocker(svc); err != nil {
			return err
		}
		e.setDesiredAgentObservabilityState(
			serviceID,
			"disabled",
			"none",
			"",
			"Controller-managed Python SDK removed from Docker container",
		)
		return nil
	default:
		isContainerized := svc.ResourceAttributes["openlit.is_containerized"] == "true"
		containerID := svc.ResourceAttributes["container.id"]

		if isContainerized && containerID != "" {
			if e.container != nil && e.container.dockerClient != nil && e.container.dockerClient.canManage() {
				if err := e.disablePythonSDKDocker(svc); err != nil {
					return err
				}
				e.setDesiredAgentObservabilityState(
					serviceID,
					"disabled",
					"none",
					"",
					"Controller-managed Python SDK removed from Docker container (detected from Linux host)",
				)
				return nil
			}
			return fmt.Errorf("containerized process detected but Docker socket not available to disable Agent O11y")
		}

		if svc.ResourceAttributes["systemd.unit"] != "" {
			if err := e.disablePythonSDKLinux(svc); err != nil {
				return err
			}
			e.setDesiredAgentObservabilityState(
				serviceID,
				"disabled",
				"none",
				"",
				"Controller-managed Python SDK removed from systemd service",
			)
			return nil
		}

		if err := e.disablePythonSDKBareProcess(svc); err != nil {
			return err
		}
		return nil
	}
}

func (e *Engine) normalizeSDKPayload(payload openlit.SDKActionPayload, defaultRuntime string) openlit.SDKActionPayload {
	if payload.TargetRuntime == "" {
		payload.TargetRuntime = defaultRuntime
	}
	if payload.InstrumentationProfile == "" {
		payload.InstrumentationProfile = controllerManagedObservability
	}
	if payload.ObservabilityScope == "" {
		payload.ObservabilityScope = "agent"
	}
	if payload.DuplicatePolicy == "" {
		payload.DuplicatePolicy = defaultDuplicatePolicy
	}
	expCfg := e.GetExportConfig()
	if payload.OTLPEndpoint == "" {
		payload.OTLPEndpoint = expCfg.OTLPEndpoint
	}
	if payload.OTLPProtocol == "" {
		payload.OTLPProtocol = expCfg.OTLPProtocol
	}
	if payload.OTLPHeaders == nil && len(expCfg.OTLPHeaders) > 0 {
		payload.OTLPHeaders = expCfg.OTLPHeaders
	}
	if payload.OTLPTracesEndpoint == "" {
		payload.OTLPTracesEndpoint = expCfg.OTLPTracesEndpoint
	}
	if payload.OTLPMetricsEndpoint == "" {
		payload.OTLPMetricsEndpoint = expCfg.OTLPMetricsEndpoint
	}
	if payload.OTLPLogsEndpoint == "" {
		payload.OTLPLogsEndpoint = expCfg.OTLPLogsEndpoint
	}
	if payload.SDKVersion == "" {
		// PyPI (Python) and npm (Node) publish the `openlit` package under
		// independent version numbers, so each runtime gets its own default.
		// Falling the Node default back to e.sdkVersion is intentionally
		// avoided: a version valid on PyPI is unlikely to exist on npm and
		// would make `npm install openlit@<version>` fail.
		if isNodeRuntime(payload.TargetRuntime) {
			payload.SDKVersion = e.nodeSDKVersion
		} else {
			payload.SDKVersion = e.sdkVersion
		}
	}
	if payload.Environment == "" {
		payload.Environment = e.environment
	}
	return payload
}

func (e *Engine) serviceSnapshot(serviceID string) (*openlit.ServiceState, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()

	svc, ok := e.services[serviceID]
	if !ok {
		return nil, fmt.Errorf("service %q not found", serviceID)
	}

	copyAttrs := make(map[string]string, len(svc.ResourceAttributes))
	for key, value := range svc.ResourceAttributes {
		copyAttrs[key] = value
	}

	copied := *svc
	copied.ResourceAttributes = copyAttrs
	return &copied, nil
}

func (e *Engine) setAgentObservabilityState(
	serviceID, status, source, conflict, reason string,
) {
	e.mu.Lock()
	defer e.mu.Unlock()

	svc, ok := e.services[serviceID]
	if !ok {
		return
	}

	e.applyAgentStateLocked(svc, status, source, conflict, reason)
}

func (e *Engine) setDesiredAgentObservabilityState(
	serviceID, status, source, conflict, reason string,
) {
	e.mu.Lock()
	defer e.mu.Unlock()

	svc, ok := e.services[serviceID]
	if !ok {
		return
	}
	svc.DesiredAgentObservabilityStatus = status
	svc.DesiredAgentObservabilityReason = reason
	e.applyAgentStateLocked(svc, status, source, conflict, reason)
}

func (e *Engine) setSDKVersionAttr(serviceID, version string) {
	e.mu.Lock()
	defer e.mu.Unlock()
	svc, ok := e.services[serviceID]
	if !ok {
		return
	}
	if svc.ResourceAttributes == nil {
		svc.ResourceAttributes = make(map[string]string)
	}
	svc.ResourceAttributes["openlit.sdk.version"] = version
}

func (e *Engine) applyAgentStateLocked(
	svc *openlit.ServiceState,
	status, source, conflict, reason string,
) {
	svc.AgentObservabilityStatus = status
	svc.AgentObservabilitySource = source
	svc.ObservabilityConflict = conflict
	svc.ObservabilityReason = reason
	if svc.ResourceAttributes == nil {
		svc.ResourceAttributes = make(map[string]string)
	}

	if status != "" {
		svc.ResourceAttributes["openlit.agent_observability.status"] = status
	}
	if source != "" {
		svc.ResourceAttributes["openlit.agent_observability.source"] = source
	} else {
		delete(svc.ResourceAttributes, "openlit.agent_observability.source")
	}
	if conflict != "" {
		svc.ResourceAttributes["openlit.observability.conflict"] = conflict
	} else {
		delete(svc.ResourceAttributes, "openlit.observability.conflict")
	}
	if reason != "" {
		svc.ResourceAttributes["openlit.observability.reason"] = reason
	} else {
		delete(svc.ResourceAttributes, "openlit.observability.reason")
	}
}

func (e *Engine) enablePythonSDKKubernetes(
	svc *openlit.ServiceState,
	payload openlit.PythonSDKActionPayload,
) error {
	if e.container == nil || e.container.k8sClient == nil {
		return fmt.Errorf("kubernetes client unavailable for Python SDK injection")
	}

	namespace := svc.ResourceAttributes["k8s.namespace.name"]
	workloadKind := svc.ResourceAttributes["k8s.workload.kind"]
	workloadName := svc.DeploymentName
	containerName := svc.ResourceAttributes["container.name"]

	if workloadKind == "" || workloadKind == "Pod" {
		return e.enablePythonSDKNakedPod(svc, payload)
	}

	if namespace == "" || workloadName == "" {
		return fmt.Errorf("missing Kubernetes workload metadata for %s", svc.ServiceName)
	}

	workload, err := e.container.k8sClient.getWorkload(namespace, workloadKind, workloadName)
	if err != nil {
		e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to fetch workload %s/%s: %v", workloadKind, workloadName, err))
		return fmt.Errorf("fetching workload %s/%s: %w", workloadKind, workloadName, err)
	}

	container, err := findTargetContainer(workload, containerName)
	if err != nil {
		return err
	}

	preflight := evaluatePythonSDKPreflight(
		extractContainerEnv(container),
		flattenContainerCommand(container),
		payload.DuplicatePolicy,
	)
	if preflight.Decision == pythonSDKDecisionBlock {
		e.setAgentObservabilityState(svc.ID, "conflict", preflight.Source, preflight.Conflict, preflight.Reason)
		return fmt.Errorf("%s", preflight.Reason)
	}
	if preflight.Decision == pythonSDKDecisionAdopt {
		e.setAgentObservabilityState(svc.ID, "enabled", preflight.Source, "", preflight.Reason)
		return nil
	}

	patch, err := buildPythonSDKWorkloadPatch(workload, containerName, svc, payload, true)
	if err != nil {
		return err
	}
	if err := e.container.k8sClient.patchWorkload(namespace, workloadKind, workloadName, patch); err != nil {
		e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to patch workload %s/%s: %v", workloadKind, workloadName, err))
		return err
	}
	// Flip in-memory state so the next heartbeat carries 'enabled'. See
	// the matching comment in enablePythonSDKDocker for context.
	e.setAgentObservabilityState(
		svc.ID,
		"enabled",
		"controller_managed",
		"",
		fmt.Sprintf("OpenLIT SDK injected via %s/%s patch (workload %s)", workloadKind, workloadName, svc.WorkloadKey),
	)
	return nil
}

func (e *Engine) disablePythonSDKKubernetes(svc *openlit.ServiceState) error {
	if e.container == nil || e.container.k8sClient == nil {
		return fmt.Errorf("kubernetes client unavailable for Python SDK removal")
	}

	namespace := svc.ResourceAttributes["k8s.namespace.name"]
	workloadKind := svc.ResourceAttributes["k8s.workload.kind"]
	workloadName := svc.DeploymentName
	containerName := svc.ResourceAttributes["container.name"]

	if workloadKind == "" || workloadKind == "Pod" {
		return e.disablePythonSDKNakedPod(svc)
	}

	if namespace == "" || workloadName == "" {
		return fmt.Errorf("missing Kubernetes workload metadata for %s", svc.ServiceName)
	}

	workload, err := e.container.k8sClient.getWorkload(namespace, workloadKind, workloadName)
	if err != nil {
		e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to fetch workload %s/%s: %v", workloadKind, workloadName, err))
		return fmt.Errorf("fetching workload %s/%s: %w", workloadKind, workloadName, err)
	}

	patch, err := buildPythonSDKWorkloadPatch(workload, containerName, svc, openlit.PythonSDKActionPayload{}, false)
	if err != nil {
		return err
	}
	if err := e.container.k8sClient.patchWorkload(namespace, workloadKind, workloadName, patch); err != nil {
		e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to patch workload %s/%s: %v", workloadKind, workloadName, err))
		return err
	}
	e.setAgentObservabilityState(
		svc.ID,
		"disabled",
		"none",
		"",
		fmt.Sprintf("OpenLIT SDK removed via %s/%s patch (workload %s)", workloadKind, workloadName, svc.WorkloadKey),
	)
	return nil
}

func (e *Engine) enablePythonSDKNakedPod(svc *openlit.ServiceState, payload openlit.PythonSDKActionPayload) error {
	namespace := svc.ResourceAttributes["k8s.namespace.name"]
	podName := svc.ResourceAttributes["k8s.pod.name"]
	containerName := svc.ResourceAttributes["container.name"]
	if namespace == "" || podName == "" {
		return fmt.Errorf("missing pod metadata for naked pod %s", svc.ServiceName)
	}

	pod, err := e.container.k8sClient.getPod(namespace, podName)
	if err != nil {
		return fmt.Errorf("fetching naked pod %s/%s: %w", namespace, podName, err)
	}

	spec, _ := pod["spec"].(map[string]any)
	if spec == nil {
		return fmt.Errorf("pod %s/%s has no spec", namespace, podName)
	}

	containers, err := extractContainers(spec["containers"])
	if err != nil {
		return err
	}
	targetIndex, err := selectContainerIndex(containers, containerName)
	if err != nil {
		return err
	}

	preflight := evaluatePythonSDKPreflight(
		extractContainerEnv(containers[targetIndex]),
		flattenContainerCommand(containers[targetIndex]),
		payload.DuplicatePolicy,
	)
	if preflight.Decision == pythonSDKDecisionBlock {
		e.setAgentObservabilityState(svc.ID, "conflict", preflight.Source, preflight.Conflict, preflight.Reason)
		return fmt.Errorf("%s", preflight.Reason)
	}
	if preflight.Decision == pythonSDKDecisionAdopt {
		e.setAgentObservabilityState(svc.ID, "enabled", preflight.Source, "", preflight.Reason)
		return nil
	}

	appImage := resolveContainerImage(containers, containerName)
	if appImage == "" {
		return fmt.Errorf("cannot resolve image for naked pod container %q", containerName)
	}

	installCmd := buildPyPIInstallShellCmd(openlitInstrumentationMountPath, payload.SDKVersion)

	if err := applyPythonSDKContainerSettings(containers[targetIndex], svc, payload); err != nil {
		return err
	}

	nakedInitContainer := map[string]any{
		"name":    openlitInstrumentationInitName,
		"image":   appImage,
		"command": []any{"sh", "-c", installCmd},
		"volumeMounts": []any{
			map[string]any{
				"name":      openlitInstrumentationVolume,
				"mountPath": openlitInstrumentationMountPath,
			},
		},
	}
	if policy := resolveContainerImagePullPolicy(containers, containerName); policy != "" {
		nakedInitContainer["imagePullPolicy"] = policy
	}

	spec["initContainers"] = objectSliceToAny(
		upsertNamedObject(
			extractObjectSlice(spec["initContainers"]),
			openlitInstrumentationInitName,
			nakedInitContainer,
		),
	)
	spec["volumes"] = objectSliceToAny(
		upsertNamedObject(
			extractObjectSlice(spec["volumes"]),
			openlitInstrumentationVolume,
			map[string]any{
				"name":     openlitInstrumentationVolume,
				"emptyDir": map[string]any{},
			},
		),
	)
	spec["containers"] = objectSliceToAny(containers)

	metadata, _ := pod["metadata"].(map[string]any)
	annotations := copyStringMap(metadata["annotations"])
	annotations[openlitManagedAnnotation] = "true"
	annotations[openlitManagedProfileAnnotation] = controllerManagedObservability
	annotations[openlitManagedConfigHash] = pythonSDKConfigHash(svc, payload)
	versionLabel := payload.SDKVersion
	if versionLabel == "" {
		versionLabel = "latest"
	}
	annotations[openlitManagedSDKVersion] = versionLabel

	stripPodRuntimeFields(pod)

	if metaMap, ok := pod["metadata"].(map[string]any); ok {
		if annotations != nil {
			metaMap["annotations"] = stringMapToAny(annotations)
		}
	}

	// Capture the original spec for rollback BEFORE the destructive delete. A
	// naked pod has no controller to recreate it, so the delete→create window is
	// the danger zone: log the full original spec at Error level first so it is
	// recoverable from controller logs even if the process dies mid-recreate.
	originalPodJSON, marshalErr := json.Marshal(pod)
	if marshalErr != nil {
		return fmt.Errorf("marshaling naked pod %s/%s before SDK injection: %w", namespace, podName, marshalErr)
	}
	e.logger.Info("recreating naked pod for SDK injection; original spec captured for rollback",
		zap.String("namespace", namespace),
		zap.String("pod", podName),
		zap.ByteString("original_spec", originalPodJSON),
	)

	if err := e.container.k8sClient.deletePod(namespace, podName, 0); err != nil {
		return fmt.Errorf("deleting naked pod %s/%s for SDK injection: %w", namespace, podName, err)
	}

	if err := e.container.k8sClient.createPod(namespace, pod); err != nil {
		e.logger.Error("naked pod create failed after delete, attempting rollback with original spec",
			zap.String("namespace", namespace),
			zap.String("pod", podName),
			zap.Error(err),
		)
		var originalPod map[string]any
		if unmarshalErr := json.Unmarshal(originalPodJSON, &originalPod); unmarshalErr == nil {
			stripPodRuntimeFields(originalPod)
			// Retry the rollback a few times — a transient API error must not be
			// the difference between a recovered pod and a lost one.
			var restoreErr error
			for attempt := 1; attempt <= 3; attempt++ {
				if restoreErr = e.container.k8sClient.createPod(namespace, originalPod); restoreErr == nil {
					break
				}
				e.logger.Warn("naked pod rollback attempt failed, retrying",
					zap.String("namespace", namespace),
					zap.String("pod", podName),
					zap.Int("attempt", attempt),
					zap.Error(restoreErr),
				)
				time.Sleep(time.Duration(attempt) * time.Second)
			}
			if restoreErr != nil {
				e.logger.Error("CRITICAL: naked pod rollback failed — pod is deleted and could not be restored. Original spec is in the preceding 'original_spec' log line. Manual intervention required.",
					zap.String("namespace", namespace),
					zap.String("pod", podName),
					zap.Error(restoreErr),
				)
				e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "",
					fmt.Sprintf("CRITICAL: Pod %s/%s was deleted but could not be recreated. Manual intervention required.", namespace, podName))
			}
		}
		return fmt.Errorf("recreating naked pod %s/%s with SDK: %w", namespace, podName, err)
	}

	e.setAgentObservabilityState(
		svc.ID,
		"enabled",
		"controller_managed",
		"",
		fmt.Sprintf("OpenLIT SDK injected via naked pod %s/%s recreate (workload %s)", namespace, podName, svc.WorkloadKey),
	)
	return nil
}

func (e *Engine) disablePythonSDKNakedPod(svc *openlit.ServiceState) error {
	namespace := svc.ResourceAttributes["k8s.namespace.name"]
	podName := svc.ResourceAttributes["k8s.pod.name"]
	containerName := svc.ResourceAttributes["container.name"]
	if namespace == "" || podName == "" {
		return fmt.Errorf("missing pod metadata for naked pod %s", svc.ServiceName)
	}

	pod, err := e.container.k8sClient.getPod(namespace, podName)
	if err != nil {
		return fmt.Errorf("fetching naked pod %s/%s: %w", namespace, podName, err)
	}

	spec, _ := pod["spec"].(map[string]any)
	if spec == nil {
		return fmt.Errorf("pod %s/%s has no spec", namespace, podName)
	}

	containers, err := extractContainers(spec["containers"])
	if err != nil {
		return err
	}
	targetIndex, err := selectContainerIndex(containers, containerName)
	if err != nil {
		return err
	}

	removePythonSDKContainerSettings(containers[targetIndex])
	spec["initContainers"] = objectSliceToAny(
		removeNamedObject(
			extractObjectSlice(spec["initContainers"]),
			openlitInstrumentationInitName,
		),
	)
	spec["volumes"] = objectSliceToAny(
		removeNamedObject(
			extractObjectSlice(spec["volumes"]),
			openlitInstrumentationVolume,
		),
	)
	spec["containers"] = objectSliceToAny(containers)

	metadata, _ := pod["metadata"].(map[string]any)
	annotations := copyStringMap(metadata["annotations"])
	delete(annotations, openlitManagedAnnotation)
	delete(annotations, openlitManagedProfileAnnotation)
	delete(annotations, openlitManagedConfigHash)

	stripPodRuntimeFields(pod)

	if metaMap, ok := pod["metadata"].(map[string]any); ok {
		if annotations != nil {
			metaMap["annotations"] = stringMapToAny(annotations)
		}
	}

	originalPodJSON, _ := json.Marshal(pod)

	if err := e.container.k8sClient.deletePod(namespace, podName, 0); err != nil {
		return fmt.Errorf("deleting naked pod %s/%s for SDK removal: %w", namespace, podName, err)
	}

	if err := e.container.k8sClient.createPod(namespace, pod); err != nil {
		e.logger.Error("naked pod create failed after delete during disable, attempting rollback",
			zap.String("namespace", namespace),
			zap.String("pod", podName),
			zap.Error(err),
		)
		var originalPod map[string]any
		if unmarshalErr := json.Unmarshal(originalPodJSON, &originalPod); unmarshalErr == nil {
			stripPodRuntimeFields(originalPod)
			if restoreErr := e.container.k8sClient.createPod(namespace, originalPod); restoreErr != nil {
				e.logger.Error("CRITICAL: naked pod rollback failed — pod is deleted and could not be restored. Manual intervention required.",
					zap.String("namespace", namespace),
					zap.String("pod", podName),
					zap.Error(restoreErr),
				)
				e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "",
					fmt.Sprintf("CRITICAL: Pod %s/%s was deleted but could not be recreated. Manual intervention required.", namespace, podName))
			}
		}
		return fmt.Errorf("recreating naked pod %s/%s without SDK: %w", namespace, podName, err)
	}

	e.setAgentObservabilityState(
		svc.ID,
		"disabled",
		"none",
		"",
		fmt.Sprintf("OpenLIT SDK removed via naked pod %s/%s recreate (workload %s)", namespace, podName, svc.WorkloadKey),
	)
	return nil
}

func stripPodRuntimeFields(pod map[string]any) {
	delete(pod, "status")
	if metadata, ok := pod["metadata"].(map[string]any); ok {
		delete(metadata, "uid")
		delete(metadata, "resourceVersion")
		delete(metadata, "creationTimestamp")
		delete(metadata, "selfLink")
		delete(metadata, "managedFields")
	}
	if spec, ok := pod["spec"].(map[string]any); ok {
		delete(spec, "nodeName")
	}
}

func stringMapToAny(m map[string]string) map[string]any {
	result := make(map[string]any, len(m))
	for k, v := range m {
		result[k] = v
	}
	return result
}

func evaluatePythonSDKPreflight(
	env map[string]string,
	command []string,
	duplicatePolicy string,
) pythonSDKPreflightResult {
	lowerCommand := strings.ToLower(strings.Join(command, " "))

	isControllerManaged := strings.EqualFold(env["OPENLIT_CONTROLLER_MODE"], controllerManagedAgentMode)
	if strings.Contains(strings.ToLower(env["PYTHONPATH"]), openlitInstrumentationMountPath) {
		isControllerManaged = true
	}
	if isControllerManaged {
		return pythonSDKPreflightResult{
			Decision: pythonSDKDecisionAdopt,
			Source:   "controller_managed",
			Reason:   "Existing controller-managed OpenLIT Python SDK configuration detected",
		}
	}

	hasExistingOpenLIT :=
		env["OPENLIT_DISABLED_INSTRUMENTORS"] != "" ||
			env["OPENLIT_DISABLE_METRICS"] != "" ||
			env["OPENLIT_DISABLE_EVENTS"] != "" ||
			strings.Contains(lowerCommand, "openlit")
	if hasExistingOpenLIT {
		return pythonSDKPreflightResult{
			Decision: pythonSDKDecisionAdopt,
			Source:   "existing_openlit",
			Reason:   "Existing OpenLIT Python SDK configuration detected and adopted",
		}
	}

	hasExistingOTel :=
		env["OTEL_PYTHON_DISTRO"] != "" ||
			env["OTEL_PYTHON_CONFIGURATOR"] != "" ||
			hasAnyEnvWithPrefix(env, "OTEL_EXPORTER_") ||
			strings.Contains(lowerCommand, "opentelemetry-instrument")
	if hasExistingOTel && duplicatePolicy == "block_if_existing_otel_detected" {
		return pythonSDKPreflightResult{
			Decision: pythonSDKDecisionBlock,
			Source:   "existing_otel",
			Conflict: "existing_otel",
			Reason:   "Existing OpenTelemetry Python instrumentation detected on target workload",
		}
	}

	return pythonSDKPreflightResult{
		Decision: pythonSDKDecisionApply,
		Source:   "controller_managed",
		Reason:   "Controller-managed OpenLIT Python SDK will be applied",
	}
}

func flattenContainerCommand(container map[string]any) []string {
	command := make([]string, 0)
	command = append(command, extractStringSlice(container["command"])...)
	command = append(command, extractStringSlice(container["args"])...)
	return command
}

func resolveContainerImage(containers []map[string]any, containerName string) string {
	idx, err := selectContainerIndex(containers, containerName)
	if err != nil {
		return ""
	}
	image, _ := containers[idx]["image"].(string)
	return image
}

func resolveContainerImagePullPolicy(containers []map[string]any, containerName string) string {
	idx, err := selectContainerIndex(containers, containerName)
	if err != nil {
		return ""
	}
	policy, _ := containers[idx]["imagePullPolicy"].(string)
	return policy
}

func buildPythonSDKWorkloadPatch(
	workload map[string]any,
	containerName string,
	svc *openlit.ServiceState,
	payload openlit.PythonSDKActionPayload,
	enable bool,
) (map[string]any, error) {
	template, err := getTemplate(workload)
	if err != nil {
		return nil, err
	}
	templateMeta := ensureMap(template, "metadata")
	templateSpec := ensureMap(template, "spec")

	annotations := copyStringMap(templateMeta["annotations"])
	if enable {
		annotations[openlitManagedAnnotation] = "true"
		annotations[openlitManagedProfileAnnotation] = controllerManagedObservability
		annotations[openlitManagedConfigHash] = pythonSDKConfigHash(svc, payload)
		versionLabel := payload.SDKVersion
		if versionLabel == "" {
			versionLabel = "latest"
		}
		annotations[openlitManagedSDKVersion] = versionLabel
	} else {
		delete(annotations, openlitManagedAnnotation)
		delete(annotations, openlitManagedProfileAnnotation)
		delete(annotations, openlitManagedConfigHash)
		delete(annotations, openlitManagedSDKVersion)
	}

	containers, err := extractContainers(templateSpec["containers"])
	if err != nil {
		return nil, err
	}

	targetIndex, err := selectContainerIndex(containers, containerName)
	if err != nil {
		return nil, err
	}

	if enable {
		appImage := resolveContainerImage(containers, containerName)
		if appImage == "" {
			return nil, fmt.Errorf("cannot resolve image for container %q", containerName)
		}

		installCmd := buildPyPIInstallShellCmd(openlitInstrumentationMountPath, payload.SDKVersion)

		if err := applyPythonSDKContainerSettings(containers[targetIndex], svc, payload); err != nil {
			return nil, err
		}

		initContainer := map[string]any{
			"name":    openlitInstrumentationInitName,
			"image":   appImage,
			"command": []any{"sh", "-c", installCmd},
			"volumeMounts": []any{
				map[string]any{
					"name":      openlitInstrumentationVolume,
					"mountPath": openlitInstrumentationMountPath,
				},
			},
		}
		if policy := resolveContainerImagePullPolicy(containers, containerName); policy != "" {
			initContainer["imagePullPolicy"] = policy
		}

		templateSpec["initContainers"] = objectSliceToAny(
			upsertNamedObject(
				extractObjectSlice(templateSpec["initContainers"]),
				openlitInstrumentationInitName,
				initContainer,
			),
		)
		templateSpec["volumes"] = objectSliceToAny(
			upsertNamedObject(
				extractObjectSlice(templateSpec["volumes"]),
				openlitInstrumentationVolume,
				map[string]any{
					"name":     openlitInstrumentationVolume,
					"emptyDir": map[string]any{},
				},
			),
		)
	} else {
		removePythonSDKContainerSettings(containers[targetIndex])
		templateSpec["initContainers"] = objectSliceToAny(
			removeNamedObject(
				extractObjectSlice(templateSpec["initContainers"]),
				openlitInstrumentationInitName,
			),
		)
		templateSpec["volumes"] = objectSliceToAny(
			removeNamedObject(
				extractObjectSlice(templateSpec["volumes"]),
				openlitInstrumentationVolume,
			),
		)
	}

	templateSpec["containers"] = objectSliceToAny(containers)

	return map[string]any{
		"spec": map[string]any{
			"template": map[string]any{
				"metadata": map[string]any{
					"annotations": annotations,
				},
				"spec": templateSpec,
			},
		},
	}, nil
}

func k8sManagedPythonPath() string {
	return fmt.Sprintf("%s/%s:%s/%s",
		openlitInstrumentationMountPath, pythonSDKBootstrapDirName,
		openlitInstrumentationMountPath, pythonSDKPackagesDirName,
	)
}

func applyPythonSDKContainerSettings(
	container map[string]any,
	svc *openlit.ServiceState,
	payload openlit.PythonSDKActionPayload,
) error {
	env := extractObjectSlice(container["env"])
	pyPath, err := prependEnvValue(env, "PYTHONPATH", k8sManagedPythonPath())
	if err != nil {
		return err
	}
	env = pyPath
	env = upsertEnvValue(env, "OPENLIT_CONTROLLER_MODE", controllerManagedAgentMode)
	env = upsertEnvValue(env, "OTEL_SERVICE_NAME", svc.ServiceName)
	env = upsertEnvValue(env, "OTEL_EXPORTER_OTLP_ENDPOINT", payload.OTLPEndpoint)
	env = upsertEnvValue(env, "OTEL_DEPLOYMENT_ENVIRONMENT", payload.Environment)
	if svc.WorkloadKey != "" {
		env = mergeOtelResourceAttrInK8sEnv(env, "service.workload.key", svc.WorkloadKey)
	}
	if payload.OTLPProtocol != "" {
		env = upsertEnvValue(env, "OTEL_EXPORTER_OTLP_PROTOCOL", payload.OTLPProtocol)
	}
	if payload.OTLPTracesEndpoint != "" {
		env = upsertEnvValue(env, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", payload.OTLPTracesEndpoint)
	}
	if payload.OTLPMetricsEndpoint != "" {
		env = upsertEnvValue(env, "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", payload.OTLPMetricsEndpoint)
	}
	if payload.OTLPLogsEndpoint != "" {
		env = upsertEnvValue(env, "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", payload.OTLPLogsEndpoint)
	}
	if len(payload.OTLPHeaders) > 0 {
		env = upsertEnvValue(env, "OTEL_EXPORTER_OTLP_HEADERS", formatOTLPHeaders(payload.OTLPHeaders))
	}
	env = upsertEnvValue(
		env,
		"OPENLIT_DISABLED_INSTRUMENTORS",
		strings.Join(controllerManagedDisabledInstrumentors, ","),
	)
	container["env"] = objectSliceToAny(env)

	volumeMounts := extractObjectSlice(container["volumeMounts"])
	volumeMounts = upsertNamedObject(
		volumeMounts,
		openlitInstrumentationVolume,
		map[string]any{
			"name":      openlitInstrumentationVolume,
			"mountPath": openlitInstrumentationMountPath,
		},
	)
	container["volumeMounts"] = objectSliceToAny(volumeMounts)
	return nil
}

func removePythonSDKContainerSettings(container map[string]any) {
	env := extractObjectSlice(container["env"])
	for _, key := range []string{
		"OPENLIT_CONTROLLER_MODE",
		"OTEL_EXPORTER_OTLP_ENDPOINT",
		"OTEL_EXPORTER_OTLP_PROTOCOL",
		"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
		"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
		"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
		"OTEL_EXPORTER_OTLP_HEADERS",
		"OTEL_DEPLOYMENT_ENVIRONMENT",
		"OPENLIT_DISABLED_INSTRUMENTORS",
	} {
		env = removeEnvValue(env, key)
	}
	env = removeOtelResourceAttrFromK8sEnv(env, "service.workload.key")
	env = removePrefixedEnvValue(env, "PYTHONPATH", k8sManagedPythonPath())
	env = removePrefixedEnvValue(env, "PYTHONPATH", openlitInstrumentationMountPath)
	container["env"] = objectSliceToAny(env)

	volumeMounts := removeNamedObject(
		extractObjectSlice(container["volumeMounts"]),
		openlitInstrumentationVolume,
	)
	container["volumeMounts"] = objectSliceToAny(volumeMounts)
}

// formatOTLPHeaders converts a map of headers to the format specified by the
// OTel spec for OTEL_EXPORTER_OTLP_HEADERS: "key1=value1,key2=value2".
// Values containing ',' or '=' are percent-encoded per the spec. Keys are
// sorted for deterministic output.
func formatOTLPHeaders(headers map[string]string) string {
	keys := make([]string, 0, len(headers))
	for k := range headers {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	parts := make([]string, 0, len(keys))
	for _, k := range keys {
		v := headers[k]
		if strings.ContainsAny(v, ",=") {
			v = url.QueryEscape(v)
		}
		parts = append(parts, k+"="+v)
	}
	return strings.Join(parts, ",")
}

func pythonSDKConfigHash(svc *openlit.ServiceState, payload openlit.PythonSDKActionPayload) string {
	data, err := json.Marshal(map[string]any{
		"service": svc.ID,
		"payload": payload,
	})
	if err != nil {
		// Marshal of these plain structs should never fail, but if it does, fall
		// back to a value-derived representation so distinct configs don't all
		// collide onto the hash of "null".
		data = []byte(fmt.Sprintf("%s|%+v", svc.ID, payload))
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:8])
}

func getTemplate(workload map[string]any) (map[string]any, error) {
	spec, ok := workload["spec"].(map[string]any)
	if !ok {
		return nil, fmt.Errorf("workload spec missing")
	}
	template, ok := spec["template"].(map[string]any)
	if !ok {
		return nil, fmt.Errorf("workload template missing")
	}
	return template, nil
}

func ensureMap(parent map[string]any, key string) map[string]any {
	if existing, ok := parent[key].(map[string]any); ok {
		return existing
	}
	next := make(map[string]any)
	parent[key] = next
	return next
}

func extractContainers(value any) ([]map[string]any, error) {
	containers := extractObjectSlice(value)
	if len(containers) == 0 {
		return nil, fmt.Errorf("workload has no containers")
	}
	return containers, nil
}

func findTargetContainer(workload map[string]any, containerName string) (map[string]any, error) {
	template, err := getTemplate(workload)
	if err != nil {
		return nil, err
	}
	containers, err := extractContainers(ensureMap(template, "spec")["containers"])
	if err != nil {
		return nil, err
	}
	index, err := selectContainerIndex(containers, containerName)
	if err != nil {
		return nil, err
	}
	return containers[index], nil
}

func selectContainerIndex(containers []map[string]any, containerName string) (int, error) {
	// No target name specified: only unambiguous when there's exactly one container.
	if containerName == "" {
		if len(containers) == 1 {
			return 0, nil
		}
		return -1, fmt.Errorf("no container name specified and pod has %d containers", len(containers))
	}
	// A name was specified: require an exact match. Do NOT silently fall back to
	// containers[0] on mismatch — injecting into the wrong container is worse
	// than failing the action visibly.
	for index, container := range containers {
		if container["name"] == containerName {
			return index, nil
		}
	}
	return -1, fmt.Errorf("failed to resolve target container %q", containerName)
}

func extractObjectSlice(value any) []map[string]any {
	raw, ok := value.([]any)
	if !ok {
		return nil
	}
	result := make([]map[string]any, 0, len(raw))
	for _, item := range raw {
		if object, ok := item.(map[string]any); ok {
			result = append(result, object)
		}
	}
	return result
}

func objectSliceToAny(items []map[string]any) []any {
	result := make([]any, 0, len(items))
	for _, item := range items {
		result = append(result, item)
	}
	return result
}

func upsertNamedObject(items []map[string]any, name string, next map[string]any) []map[string]any {
	for index, item := range items {
		if item["name"] == name {
			items[index] = next
			return items
		}
	}
	return append(items, next)
}

func removeNamedObject(items []map[string]any, name string) []map[string]any {
	filtered := make([]map[string]any, 0, len(items))
	for _, item := range items {
		if item["name"] == name {
			continue
		}
		filtered = append(filtered, item)
	}
	return filtered
}

func extractContainerEnv(container map[string]any) map[string]string {
	env := make(map[string]string)
	for _, item := range extractObjectSlice(container["env"]) {
		name, _ := item["name"].(string)
		value, _ := item["value"].(string)
		if name != "" {
			env[name] = value
		}
	}
	return env
}

func upsertEnvValue(env []map[string]any, key, value string) []map[string]any {
	for index, item := range env {
		if item["name"] == key {
			env[index]["value"] = value
			delete(env[index], "valueFrom")
			return env
		}
	}
	return append(env, map[string]any{"name": key, "value": value})
}

// mergeOtelResourceAttrInK8sEnv merges a key=value into the
// OTEL_RESOURCE_ATTRIBUTES entry of a K8s container env list. If the env var
// is already set on the container we preserve its other pairs and just
// replace/append the requested key.
func mergeOtelResourceAttrInK8sEnv(env []map[string]any, key, value string) []map[string]any {
	for index, item := range env {
		if item["name"] != "OTEL_RESOURCE_ATTRIBUTES" {
			continue
		}
		existing, _ := item["value"].(string)
		env[index]["value"] = mergeResourceAttrValue(existing, key, value)
		delete(env[index], "valueFrom")
		return env
	}
	return append(env, map[string]any{
		"name":  "OTEL_RESOURCE_ATTRIBUTES",
		"value": mergeResourceAttrValue("", key, value),
	})
}

// removeOtelResourceAttrFromK8sEnv removes a single key from
// OTEL_RESOURCE_ATTRIBUTES. The env var itself is removed only when it would
// otherwise become empty (so we don't leave a stray controller-injected
// attribute behind when the user wasn't using OTEL_RESOURCE_ATTRIBUTES).
func removeOtelResourceAttrFromK8sEnv(env []map[string]any, key string) []map[string]any {
	for index, item := range env {
		if item["name"] != "OTEL_RESOURCE_ATTRIBUTES" {
			continue
		}
		existing, _ := item["value"].(string)
		filtered := removeResourceAttrKey(existing, key)
		if filtered == "" {
			return removeEnvValue(env, "OTEL_RESOURCE_ATTRIBUTES")
		}
		env[index]["value"] = filtered
		return env
	}
	return env
}

func removeEnvValue(env []map[string]any, key string) []map[string]any {
	filtered := make([]map[string]any, 0, len(env))
	for _, item := range env {
		if item["name"] == key {
			continue
		}
		filtered = append(filtered, item)
	}
	return filtered
}

func prependEnvValue(env []map[string]any, key, prefix string) ([]map[string]any, error) {
	for index, item := range env {
		if item["name"] != key {
			continue
		}
		if _, hasValueFrom := item["valueFrom"]; hasValueFrom {
			return nil, fmt.Errorf("%s uses valueFrom and cannot be safely rewritten", key)
		}
		existing, _ := item["value"].(string)
		if existing == "" {
			env[index]["value"] = prefix
			return env, nil
		}
		if strings.Contains(existing, prefix) {
			return env, nil
		}
		env[index]["value"] = fmt.Sprintf("%s:%s", prefix, existing)
		return env, nil
	}
	return append(env, map[string]any{"name": key, "value": prefix}), nil
}

func removePrefixedEnvValue(env []map[string]any, key, prefix string) []map[string]any {
	for index, item := range env {
		if item["name"] != key {
			continue
		}
		value, _ := item["value"].(string)
		value = strings.TrimPrefix(value, prefix+":")
		value = strings.TrimPrefix(value, prefix)
		if value == "" {
			return removeEnvValue(env, key)
		}
		env[index]["value"] = value
		return env
	}
	return env
}

func copyStringMap(value any) map[string]string {
	result := make(map[string]string)
	source, ok := value.(map[string]any)
	if !ok {
		return result
	}
	for key, item := range source {
		if str, ok := item.(string); ok {
			result[key] = str
		}
	}
	return result
}

func extractStringSlice(value any) []string {
	raw, ok := value.([]any)
	if !ok {
		return nil
	}
	result := make([]string, 0, len(raw))
	for _, item := range raw {
		if str, ok := item.(string); ok {
			result = append(result, str)
		}
	}
	return result
}