Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
package engine
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/openlit/openlit/openlit-controller/internal/openlit"
"go.uber.org/zap"
)
func (e *Engine) enablePythonSDKDocker(
svc *openlit.ServiceState,
payload openlit.PythonSDKActionPayload,
) error {
if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {
e.setAgentObservabilityState(
svc.ID,
"unsupported",
"none",
"",
"Docker Agent Observability requires a writable Docker socket and a Docker-capable controller",
)
return fmt.Errorf("docker controller does not have writable Docker API access")
}
containerID := svc.ResourceAttributes["container.id"]
if containerID == "" {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "Docker container metadata is missing for this workload")
return fmt.Errorf("missing docker container id for %s", svc.ServiceName)
}
inspect, err := e.container.dockerClient.inspectContainer(containerID)
if err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to inspect Docker container: %v", err))
return err
}
preflight := evaluatePythonSDKPreflight(
dockerInspectEnv(inspect),
dockerInspectCommand(inspect),
payload.DuplicatePolicy,
)
if preflight.Decision == pythonSDKDecisionBlock {
e.setAgentObservabilityState(svc.ID, "conflict", preflight.Source, preflight.Conflict, preflight.Reason)
return fmt.Errorf("%s", preflight.Reason)
}
if preflight.Decision == pythonSDKDecisionAdopt {
e.setAgentObservabilityState(svc.ID, "enabled", preflight.Source, "", preflight.Reason)
return nil
}
nextHash := pythonSDKConfigHash(svc, payload)
e.logger.Info("starting docker agent observability rollout",
zap.String("mode", "docker"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("previous_hash", dockerInspectManagedConfigHash(inspect)),
zap.String("next_hash", nextHash),
)
if err := ensureDockerContainerManageable(inspect); err != nil {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", err.Error())
return err
}
volumeName := dockerSDKVolumeName(svc)
if err := e.prepareDockerSDKVolume(inspect, svc, volumeName, payload.SDKVersion); err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", err.Error())
return err
}
payloadMap, err := buildDockerContainerCreatePayload(inspect, svc, payload, volumeName, true)
if err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to build container config: %v", err))
return err
}
if err := e.recreateDockerContainer(inspect, payloadMap); err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", err.Error())
return err
}
// Flip in-memory state so the next heartbeat carries
// `openlit.agent_observability.status='enabled'` into
// openlit_controller_services. Without this the read-path rollup keeps
// showing the old 'disabled' status until the scanner re-detects the
// container, which can leave the UI stuck in "Enabling..." for minutes.
e.setAgentObservabilityState(
svc.ID,
"enabled",
"controller_managed",
"",
fmt.Sprintf("OpenLIT SDK active via Docker container recreate (workload %s)", svc.WorkloadKey),
)
e.logger.Info("completed docker agent observability rollout",
zap.String("mode", "docker"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("previous_hash", dockerInspectManagedConfigHash(inspect)),
zap.String("next_hash", nextHash),
)
return nil
}
func (e *Engine) disablePythonSDKDocker(svc *openlit.ServiceState) error {
if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {
e.setAgentObservabilityState(
svc.ID,
"unsupported",
"none",
"",
"Docker Agent Observability requires a writable Docker socket and a Docker-capable controller",
)
return fmt.Errorf("docker controller does not have writable Docker API access")
}
containerID := svc.ResourceAttributes["container.id"]
if containerID == "" {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "Docker container metadata is missing for this workload")
return fmt.Errorf("missing docker container id for %s", svc.ServiceName)
}
inspect, err := e.container.dockerClient.inspectContainer(containerID)
if err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", fmt.Sprintf("Failed to inspect Docker container: %v", err))
return err
}
previousHash := dockerInspectManagedConfigHash(inspect)
e.logger.Info("starting docker agent observability removal",
zap.String("mode", "docker"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("previous_hash", previousHash),
zap.String("next_hash", ""),
)
volumeName := dockerSDKVolumeName(svc)
payloadMap, err := buildDockerContainerCreatePayload(inspect, svc, openlit.PythonSDKActionPayload{}, volumeName, false)
if err != nil {
return err
}
if err := e.recreateDockerContainer(inspect, payloadMap); err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", err.Error())
return err
}
_ = e.container.dockerClient.removeVolume(volumeName)
// Flip in-memory state so the next heartbeat reflects the disable. See
// the matching comment in enablePythonSDKDocker for why this is needed
// even though applyObservedAgentObservability eventually catches up.
e.setAgentObservabilityState(
svc.ID,
"disabled",
"none",
"",
fmt.Sprintf("OpenLIT SDK removed via Docker container recreate (workload %s)", svc.WorkloadKey),
)
e.logger.Info("completed docker agent observability removal",
zap.String("mode", "docker"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("previous_hash", previousHash),
zap.String("next_hash", ""),
)
return nil
}
func (e *Engine) prepareDockerSDKVolume(
inspect map[string]any,
svc *openlit.ServiceState,
volumeName string,
sdkVersion string,
) error {
if err := e.container.dockerClient.createVolume(volumeName, map[string]string{
openlitManagedLabel: "true",
openlitManagedVolumeLabel: svc.WorkloadKey,
}); err != nil {
return err
}
image := dockerInspectImage(inspect)
if image == "" {
return fmt.Errorf("docker sdk preparation requires image metadata")
}
installCmd := buildPyPIInstallShellCmd(openlitInstrumentationMountPath, sdkVersion)
helperPayload := map[string]any{
"Image": image,
"Entrypoint": []any{"sh"},
"Cmd": []any{"-c", installCmd},
"HostConfig": map[string]any{
"Binds": []any{fmt.Sprintf("%s:%s", volumeName, openlitInstrumentationMountPath)},
"NetworkMode": dockerInspectHostConfigValue(inspect, "NetworkMode"),
},
}
helperName := "openlit-python-sdk-installer-" + shortHash(svc.WorkloadKey+time.Now().String())
helperID, err := e.container.dockerClient.createContainer(helperName, helperPayload)
if err != nil {
return err
}
defer func() { _ = e.container.dockerClient.removeContainer(helperID, true) }()
if err := e.container.dockerClient.startContainer(helperID); err != nil {
return err
}
statusCode, err := e.container.dockerClient.waitContainer(helperID)
if err != nil {
return err
}
if statusCode != 0 {
return fmt.Errorf("docker sdk installer exited with status %d", statusCode)
}
return nil
}
func (e *Engine) recreateDockerContainer(inspect map[string]any, payload map[string]any) error {
containerID, _ := inspect["Id"].(string)
originalName := strings.TrimPrefix(dockerInspectName(inspect), "/")
if containerID == "" || originalName == "" {
return fmt.Errorf("docker container identity missing for recreate")
}
backupName := originalName + "-openlit-backup-" + shortHash(containerID+time.Now().String())
if err := e.container.dockerClient.stopContainer(containerID, 10); err != nil {
return err
}
if err := e.container.dockerClient.renameContainer(containerID, backupName); err != nil {
if startErr := e.container.dockerClient.startContainer(containerID); startErr != nil {
e.logger.Error("CRITICAL: container stopped and rename failed; could not restart original",
zap.String("container_id", containerID),
zap.String("workload", originalName),
zap.Error(startErr),
)
}
return err
}
newID, createErr := e.container.dockerClient.createContainer(originalName, payload)
if createErr != nil {
e.logger.Error("docker rollback: create failed, restoring original",
zap.String("workload", originalName),
zap.Error(createErr),
)
if renameErr := e.container.dockerClient.renameContainer(containerID, originalName); renameErr != nil {
e.logger.Error("docker rollback: rename-back failed",
zap.String("container_id", containerID),
zap.String("workload", originalName),
zap.Error(renameErr),
)
}
if startErr := e.container.dockerClient.startContainer(containerID); startErr != nil {
e.logger.Error("CRITICAL: docker rollback failed — container is stopped and could not be restored. Manual intervention required.",
zap.String("container_id", containerID),
zap.String("workload", originalName),
zap.Error(startErr),
)
}
return createErr
}
if err := e.container.dockerClient.startContainer(newID); err != nil {
e.logger.Error("docker rollback: start of new container failed, restoring original",
zap.String("workload", originalName),
zap.Error(err),
)
if removeErr := e.container.dockerClient.removeContainer(newID, true); removeErr != nil {
e.logger.Error("docker rollback: remove new container failed",
zap.String("new_container_id", newID),
zap.String("workload", originalName),
zap.Error(removeErr),
)
}
if renameErr := e.container.dockerClient.renameContainer(containerID, originalName); renameErr != nil {
e.logger.Error("docker rollback: rename-back failed",
zap.String("container_id", containerID),
zap.String("workload", originalName),
zap.Error(renameErr),
)
}
if startErr := e.container.dockerClient.startContainer(containerID); startErr != nil {
e.logger.Error("CRITICAL: docker rollback failed — container is stopped and could not be restored. Manual intervention required.",
zap.String("container_id", containerID),
zap.String("workload", originalName),
zap.Error(startErr),
)
}
return err
}
if err := e.container.dockerClient.removeContainer(containerID, true); err != nil {
e.logger.Warn("docker cleanup: failed to remove old container (new container is running)",
zap.String("old_container_id", containerID),
zap.String("workload", originalName),
zap.Error(err),
)
}
return nil
}
func buildDockerContainerCreatePayload(
inspect map[string]any,
svc *openlit.ServiceState,
payload openlit.PythonSDKActionPayload,
volumeName string,
enable bool,
) (map[string]any, error) {
configMap := dockerInspectConfig(inspect)
if configMap == nil {
return nil, fmt.Errorf("docker container config missing")
}
hostConfig := cloneMap(dockerInspectHostConfig(inspect))
networkingConfig := dockerInspectNetworkingConfig(inspect)
createPayload := map[string]any{}
copySelectedKeys(createPayload, configMap, []string{
"Hostname", "Domainname", "User", "AttachStdin", "AttachStdout", "AttachStderr",
"ExposedPorts", "Tty", "OpenStdin", "StdinOnce", "Env", "Cmd", "Healthcheck",
"ArgsEscaped", "Image", "Volumes", "WorkingDir", "Entrypoint", "NetworkDisabled",
"MacAddress", "Labels", "StopSignal", "StopTimeout", "Shell",
})
envValues := extractStringList(createPayload["Env"])
if enable {
envValues = upsertEnvString(envValues, "PYTHONPATH", dockerManagedPythonPath(envValues))
envValues = upsertEnvString(envValues, "OPENLIT_CONTROLLER_MODE", controllerManagedAgentMode)
envValues = upsertEnvString(envValues, "OTEL_SERVICE_NAME", svc.ServiceName)
envValues = upsertEnvString(envValues, "OTEL_EXPORTER_OTLP_ENDPOINT", payload.OTLPEndpoint)
envValues = upsertEnvString(envValues, "OTEL_DEPLOYMENT_ENVIRONMENT", payload.Environment)
if svc.WorkloadKey != "" {
envValues = mergeOtelResourceAttrInEnvString(envValues, "service.workload.key", svc.WorkloadKey)
}
if payload.OTLPProtocol != "" {
envValues = upsertEnvString(envValues, "OTEL_EXPORTER_OTLP_PROTOCOL", payload.OTLPProtocol)
}
if payload.OTLPTracesEndpoint != "" {
envValues = upsertEnvString(envValues, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", payload.OTLPTracesEndpoint)
}
if payload.OTLPMetricsEndpoint != "" {
envValues = upsertEnvString(envValues, "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", payload.OTLPMetricsEndpoint)
}
if payload.OTLPLogsEndpoint != "" {
envValues = upsertEnvString(envValues, "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", payload.OTLPLogsEndpoint)
}
if len(payload.OTLPHeaders) > 0 {
envValues = upsertEnvString(envValues, "OTEL_EXPORTER_OTLP_HEADERS", formatOTLPHeaders(payload.OTLPHeaders))
}
envValues = upsertEnvString(
envValues,
"OPENLIT_DISABLED_INSTRUMENTORS",
strings.Join(controllerManagedDisabledInstrumentors, ","),
)
labels := extractStringMap(createPayload["Labels"])
labels[openlitManagedLabel] = "true"
labels[openlitManagedVolumeLabel] = volumeName
labels[openlitManagedConfigHash] = pythonSDKConfigHash(svc, payload)
createPayload["Labels"] = labels
hostConfig["Binds"] = upsertBindMount(extractStringList(hostConfig["Binds"]), fmt.Sprintf("%s:%s", volumeName, openlitInstrumentationMountPath), openlitInstrumentationMountPath)
} else {
envValues = removeEnvString(envValues, "OPENLIT_CONTROLLER_MODE")
envValues = removeEnvString(envValues, "OTEL_EXPORTER_OTLP_ENDPOINT")
envValues = removeEnvString(envValues, "OTEL_EXPORTER_OTLP_PROTOCOL")
envValues = removeEnvString(envValues, "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
envValues = removeEnvString(envValues, "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT")
envValues = removeEnvString(envValues, "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT")
envValues = removeEnvString(envValues, "OTEL_EXPORTER_OTLP_HEADERS")
envValues = removeEnvString(envValues, "OTEL_DEPLOYMENT_ENVIRONMENT")
envValues = removeEnvString(envValues, "OPENLIT_DISABLED_INSTRUMENTORS")
envValues = removeOtelResourceAttrFromEnvString(envValues, "service.workload.key")
envValues = rewritePythonPathWithoutManagedSDK(envValues)
labels := extractStringMap(createPayload["Labels"])
delete(labels, openlitManagedLabel)
delete(labels, openlitManagedVolumeLabel)
delete(labels, openlitManagedConfigHash)
createPayload["Labels"] = labels
hostConfig["Binds"] = removeBindMount(extractStringList(hostConfig["Binds"]), openlitInstrumentationMountPath)
}
createPayload["Env"] = stringSliceToAny(envValues)
createPayload["HostConfig"] = hostConfig
if len(networkingConfig) > 0 {
createPayload["NetworkingConfig"] = networkingConfig
}
return createPayload, nil
}
func (e *Engine) enablePythonSDKLinux(
svc *openlit.ServiceState,
payload openlit.PythonSDKActionPayload,
) error {
if !linuxSystemdSDKSupported() {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "Linux Agent Observability requires systemd and root-level unit management access")
return fmt.Errorf("linux controller does not have systemd management support")
}
unit := svc.ResourceAttributes["systemd.unit"]
if unit == "" {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "This Linux workload is not managed by systemd")
return fmt.Errorf("linux workload %s is not systemd-managed", svc.ServiceName)
}
if svc.ResourceAttributes["systemd.scope"] == "user" {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "User-scoped systemd services are not supported yet")
return fmt.Errorf("user-scoped systemd unit %s is not supported", unit)
}
preflight := evaluatePythonSDKPreflight(
readEnviron(e.procRoot, svc.PID),
strings.Fields(svc.Cmdline),
payload.DuplicatePolicy,
)
if preflight.Decision == pythonSDKDecisionBlock {
e.setAgentObservabilityState(svc.ID, "conflict", preflight.Source, preflight.Conflict, preflight.Reason)
return fmt.Errorf("%s", preflight.Reason)
}
if preflight.Decision == pythonSDKDecisionAdopt {
e.setAgentObservabilityState(svc.ID, "enabled", preflight.Source, "", preflight.Reason)
return nil
}
sdkRoot := filepath.Join(systemdSDKStateDir, sanitizeSystemdUnit(unit))
nextHash := pythonSDKConfigHash(svc, payload)
previousHash := readSystemdDropInConfigHash(unit)
e.logger.Info("starting linux agent observability rollout",
zap.String("mode", "linux"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("unit", unit),
zap.String("previous_hash", previousHash),
zap.String("next_hash", nextHash),
)
if err := installPythonSDKFromPyPI(svc.ExePath, sdkRoot, payload.SDKVersion); err != nil {
e.setAgentObservabilityState(svc.ID, "error", "controller_managed", "", err.Error())
return err
}
env := readEnviron(e.procRoot, svc.PID)
sdPayload := systemdOTLPPayload{
OTLPEndpoint: payload.OTLPEndpoint,
OTLPProtocol: payload.OTLPProtocol,
OTLPTracesEndpoint: payload.OTLPTracesEndpoint,
OTLPMetricsEndpoint: payload.OTLPMetricsEndpoint,
OTLPLogsEndpoint: payload.OTLPLogsEndpoint,
Environment: payload.Environment,
}
if len(payload.OTLPHeaders) > 0 {
sdPayload.OTLPHeaders = formatOTLPHeaders(payload.OTLPHeaders)
}
dropIn := buildSystemdDropInContent(
unit,
sdkRoot,
svc.ServiceName,
svc.WorkloadKey,
sdPayload,
strings.Join(controllerManagedDisabledInstrumentors, ","),
env["PYTHONPATH"],
nextHash,
)
if err := writeSystemdDropIn(unit, dropIn); err != nil {
return err
}
if err := runSystemctl("daemon-reload"); err != nil {
_ = removeSystemdDropIn(unit)
e.logger.Warn("linux agent observability rollback triggered",
zap.String("unit", unit),
zap.String("rollback_result", "dropin_removed_after_daemon_reload_failure"),
zap.Error(err),
)
return err
}
if err := runSystemctl("try-restart", unit); err != nil {
_ = removeSystemdDropIn(unit)
_ = runSystemctl("daemon-reload")
e.logger.Warn("linux agent observability rollback triggered",
zap.String("unit", unit),
zap.String("rollback_result", "dropin_removed_after_restart_failure"),
zap.Error(err),
)
return err
}
// Flip in-memory state so the next heartbeat carries 'enabled'. See
// the matching comment in enablePythonSDKDocker for context.
e.setAgentObservabilityState(
svc.ID,
"enabled",
"controller_managed",
"",
fmt.Sprintf("OpenLIT SDK active via systemd drop-in (unit %s)", unit),
)
e.logger.Info("completed linux agent observability rollout",
zap.String("mode", "linux"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("unit", unit),
zap.String("previous_hash", previousHash),
zap.String("next_hash", nextHash),
)
return nil
}
func (e *Engine) disablePythonSDKLinux(svc *openlit.ServiceState) error {
if !linuxSystemdSDKSupported() {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "Linux Agent Observability requires systemd and root-level unit management access")
return fmt.Errorf("linux controller does not have systemd management support")
}
unit := svc.ResourceAttributes["systemd.unit"]
if unit == "" {
e.setAgentObservabilityState(svc.ID, "unsupported", "none", "", "This Linux workload is not managed by systemd")
return fmt.Errorf("linux workload %s is not systemd-managed", svc.ServiceName)
}
previousHash := readSystemdDropInConfigHash(unit)
e.logger.Info("starting linux agent observability removal",
zap.String("mode", "linux"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("unit", unit),
zap.String("previous_hash", previousHash),
zap.String("next_hash", ""),
)
if err := removeSystemdDropIn(unit); err != nil {
return err
}
if err := runSystemctl("daemon-reload"); err != nil {
return err
}
if err := runSystemctl("try-restart", unit); err != nil {
return err
}
e.setAgentObservabilityState(
svc.ID,
"disabled",
"none",
"",
fmt.Sprintf("OpenLIT SDK removed via systemd drop-in (unit %s)", unit),
)
e.logger.Info("completed linux agent observability removal",
zap.String("mode", "linux"),
zap.String("workload_key", svc.WorkloadKey),
zap.String("unit", unit),
zap.String("previous_hash", previousHash),
zap.String("next_hash", ""),
)
return nil
}
func ensureDockerContainerManageable(inspect map[string]any) error {
hostConfig := dockerInspectHostConfig(inspect)
restartPolicy, _ := hostConfig["RestartPolicy"].(map[string]any)
restartName, _ := restartPolicy["Name"].(string)
labels := extractStringMap(dockerInspectConfig(inspect)["Labels"])
if restartName != "" && restartName != "no" {
return nil
}
if labels["com.docker.compose.project"] != "" && labels["com.docker.compose.service"] != "" {
return nil
}
return fmt.Errorf("container is not controller-manageable: missing restart policy or compose metadata for safe recreate")
}
func dockerSDKVolumeName(svc *openlit.ServiceState) string {
return "openlit-python-sdk-" + shortHash(svc.WorkloadKey)
}
func dockerManagedPythonPath(existingEnv []string) string {
managedPath := fmt.Sprintf("%s/%s:%s/%s", openlitInstrumentationMountPath, pythonSDKBootstrapDirName, openlitInstrumentationMountPath, pythonSDKPackagesDirName)
for _, entry := range existingEnv {
key, value, ok := strings.Cut(entry, "=")
if ok && key == "PYTHONPATH" && value != "" {
if strings.Contains(value, managedPath) {
return value
}
return managedPath + ":" + value
}
}
return managedPath
}
func dockerInspectEnv(inspect map[string]any) map[string]string {
env := make(map[string]string)
for _, item := range extractStringList(dockerInspectConfig(inspect)["Env"]) {
key, value, ok := strings.Cut(item, "=")
if ok {
env[key] = value
}
}
return env
}
func dockerInspectCommand(inspect map[string]any) []string {
config := dockerInspectConfig(inspect)
command := make([]string, 0)
command = append(command, extractStringList(config["Entrypoint"])...)
command = append(command, extractStringList(config["Cmd"])...)
return command
}
func dockerInspectConfig(inspect map[string]any) map[string]any {
config, _ := inspect["Config"].(map[string]any)
return config
}
func dockerInspectHostConfig(inspect map[string]any) map[string]any {
hostConfig, _ := inspect["HostConfig"].(map[string]any)
if hostConfig == nil {
return map[string]any{}
}
return hostConfig
}
func dockerInspectHostConfigValue(inspect map[string]any, key string) any {
return dockerInspectHostConfig(inspect)[key]
}
func dockerInspectName(inspect map[string]any) string {
name, _ := inspect["Name"].(string)
return name
}
func dockerInspectManagedConfigHash(inspect map[string]any) string {
return extractStringMap(dockerInspectConfig(inspect)["Labels"])[openlitManagedConfigHash]
}
func dockerInspectImage(inspect map[string]any) string {
image, _ := dockerInspectConfig(inspect)["Image"].(string)
return image
}
func dockerInspectNetworkingConfig(inspect map[string]any) map[string]any {
networkSettings, _ := inspect["NetworkSettings"].(map[string]any)
networks, _ := networkSettings["Networks"].(map[string]any)
if len(networks) == 0 {
return nil
}
return map[string]any{
"EndpointsConfig": networks,
}
}
func upsertEnvString(env []string, key, value string) []string {
for i, entry := range env {
existingKey, _, ok := strings.Cut(entry, "=")
if ok && existingKey == key {
env[i] = key + "=" + value
return env
}
}
return append(env, key+"="+value)
}
// mergeResourceAttrValue merges a single key=value into a comma-separated
// OTEL_RESOURCE_ATTRIBUTES string (the OTel convention). If the key is already
// present, its value is replaced; otherwise the pair is appended. Used so that
// controller-injected attributes (e.g. service.workload.key) don't clobber
// user-defined OTEL_RESOURCE_ATTRIBUTES already set on the workload.
func mergeResourceAttrValue(existing, key, value string) string {
pair := key + "=" + value
if existing == "" {
return pair
}
parts := strings.Split(existing, ",")
out := make([]string, 0, len(parts)+1)
replaced := false
for _, p := range parts {
trimmed := strings.TrimSpace(p)
if trimmed == "" {
continue
}
if k, _, ok := strings.Cut(trimmed, "="); ok && k == key {
out = append(out, pair)
replaced = true
continue
}
out = append(out, trimmed)
}
if !replaced {
out = append(out, pair)
}
return strings.Join(out, ",")
}
// mergeOtelResourceAttrInEnvString merges a key=value into the
// OTEL_RESOURCE_ATTRIBUTES entry of a []string env list (the Docker /
// linux-bare shape).
func mergeOtelResourceAttrInEnvString(env []string, key, value string) []string {
for i, entry := range env {
k, v, ok := strings.Cut(entry, "=")
if ok && k == "OTEL_RESOURCE_ATTRIBUTES" {
env[i] = "OTEL_RESOURCE_ATTRIBUTES=" + mergeResourceAttrValue(v, key, value)
return env
}
}
return append(env, "OTEL_RESOURCE_ATTRIBUTES="+mergeResourceAttrValue("", key, value))
}
// removeResourceAttrKey strips a key from an OTEL_RESOURCE_ATTRIBUTES value,
// returning the remaining comma-separated pairs (empty string when nothing
// else is left).
func removeResourceAttrKey(existing, key string) string {
if existing == "" {
return ""
}
parts := strings.Split(existing, ",")
out := make([]string, 0, len(parts))
for _, p := range parts {
trimmed := strings.TrimSpace(p)
if trimmed == "" {
continue
}
if k, _, ok := strings.Cut(trimmed, "="); ok && k == key {
continue
}
out = append(out, trimmed)
}
return strings.Join(out, ",")
}
// removeOtelResourceAttrFromEnvString removes a single key from
// OTEL_RESOURCE_ATTRIBUTES on a []string env list (Docker / linux-bare).
// Drops the whole env var when the remaining list is empty.
func removeOtelResourceAttrFromEnvString(env []string, key string) []string {
for i, entry := range env {
k, v, ok := strings.Cut(entry, "=")
if !ok || k != "OTEL_RESOURCE_ATTRIBUTES" {
continue
}
filtered := removeResourceAttrKey(v, key)
if filtered == "" {
return append(env[:i], env[i+1:]...)
}
env[i] = "OTEL_RESOURCE_ATTRIBUTES=" + filtered
return env
}
return env
}
func removeEnvString(env []string, key string) []string {
filtered := make([]string, 0, len(env))
for _, entry := range env {
existingKey, _, ok := strings.Cut(entry, "=")
if ok && existingKey == key {
continue
}
filtered = append(filtered, entry)
}
return filtered
}
func rewritePythonPathWithoutManagedSDK(env []string) []string {
filtered := make([]string, 0, len(env))
managedPrefix := fmt.Sprintf("%s/%s:%s/%s", openlitInstrumentationMountPath, pythonSDKBootstrapDirName, openlitInstrumentationMountPath, pythonSDKPackagesDirName)
for _, entry := range env {
key, value, ok := strings.Cut(entry, "=")
if !ok || key != "PYTHONPATH" {
filtered = append(filtered, entry)
continue
}
value = strings.TrimPrefix(value, managedPrefix+":")
value = strings.TrimPrefix(value, managedPrefix)
if value != "" {
filtered = append(filtered, key+"="+value)
}
}
return filtered
}
func upsertBindMount(binds []string, mount string, target string) []string {
for _, existing := range binds {
if strings.HasSuffix(existing, ":"+target) {
return binds
}
}
return append(binds, mount)
}
func removeBindMount(binds []string, target string) []string {
filtered := make([]string, 0, len(binds))
for _, existing := range binds {
if strings.HasSuffix(existing, ":"+target) {
continue
}
filtered = append(filtered, existing)
}
return filtered
}
func extractStringList(value any) []string {
switch typed := value.(type) {
case []string:
return append([]string(nil), typed...)
case []any:
result := make([]string, 0, len(typed))
for _, item := range typed {
if str, ok := item.(string); ok {
result = append(result, str)
}
}
return result
default:
return nil
}
}
func stringSliceToAny(values []string) []any {
result := make([]any, 0, len(values))
for _, value := range values {
result = append(result, value)
}
return result
}
func extractStringMap(value any) map[string]string {
switch typed := value.(type) {
case map[string]string:
copied := make(map[string]string, len(typed))
for key, item := range typed {
copied[key] = item
}
return copied
case map[string]any:
copied := make(map[string]string, len(typed))
for key, item := range typed {
if str, ok := item.(string); ok {
copied[key] = str
}
}
return copied
default:
return map[string]string{}
}
}
func cloneMap(source map[string]any) map[string]any {
if source == nil {
return map[string]any{}
}
data, err := json.Marshal(source)
if err != nil {
result := make(map[string]any, len(source))
for k, v := range source {
result[k] = v
}
return result
}
var copied map[string]any
if err := json.Unmarshal(data, &copied); err != nil {
result := make(map[string]any, len(source))
for k, v := range source {
result[k] = v
}
return result
}
return copied
}
func copySelectedKeys(target map[string]any, source map[string]any, keys []string) {
for _, key := range keys {
if value, ok := source[key]; ok {
target[key] = value
}
}
}
func sanitizeSystemdUnit(unit string) string {
return strings.ReplaceAll(unit, "/", "_")
}
const bareProcessSDKStateDir = "/var/lib/openlit/python-sdk"
func (e *Engine) enablePythonSDKBareProcess(
svc *openlit.ServiceState,
payload openlit.PythonSDKActionPayload,
) error {
pythonBinary := svc.ExePath
if pythonBinary == "" {
e.setDesiredAgentObservabilityState(
svc.ID,
"unsupported",
"none",
"",
"Cannot determine the Python binary path for this process",
)
return fmt.Errorf("missing python executable path for bare process %s", svc.ServiceName)
}
preflight := evaluatePythonSDKPreflight(
readEnviron(e.procRoot, svc.PID),
strings.Fields(svc.Cmdline),
payload.DuplicatePolicy,
)
if preflight.Decision == pythonSDKDecisionBlock {
e.setDesiredAgentObservabilityState(svc.ID, "conflict", preflight.Source, preflight.Conflict, preflight.Reason)
return fmt.Errorf("%s", preflight.Reason)
}
if preflight.Decision == pythonSDKDecisionAdopt {
e.setDesiredAgentObservabilityState(svc.ID, "enabled", preflight.Source, "", preflight.Reason)
return nil
}
fingerprint := shortHash(svc.WorkloadKey)
sdkRoot := filepath.Join(bareProcessSDKStateDir, fingerprint)
e.logger.Info("installing Python SDK for bare process",
zap.String("service", svc.ServiceName),
zap.String("workload_key", svc.WorkloadKey),
zap.String("sdk_root", sdkRoot),
zap.String("python", pythonBinary),
)
if err := installPythonSDKFromPyPI(pythonBinary, sdkRoot, payload.SDKVersion); err != nil {
e.setDesiredAgentObservabilityState(
svc.ID,
"error",
"controller_managed",
"",
fmt.Sprintf("Failed to install OpenLIT SDK: %v", err),
)
return err
}
bootstrapDir := filepath.Join(sdkRoot, pythonSDKBootstrapDirName)
packagesDir := filepath.Join(sdkRoot, pythonSDKPackagesDirName)
pythonPath := bootstrapDir + ":" + packagesDir
existingPythonPath := readEnviron(e.procRoot, svc.PID)["PYTHONPATH"]
if existingPythonPath != "" {
pythonPath = pythonPath + ":" + existingPythonPath
}
envOverrides := map[string]string{
"PYTHONPATH": pythonPath,
"OPENLIT_CONTROLLER_MODE": "agent_observability",
"OTEL_SERVICE_NAME": svc.ServiceName,
"OTEL_EXPORTER_OTLP_ENDPOINT": payload.OTLPEndpoint,
"OTEL_DEPLOYMENT_ENVIRONMENT": payload.Environment,
"OPENLIT_DISABLED_INSTRUMENTORS": strings.Join(controllerManagedDisabledInstrumentors, ","),
}
if svc.WorkloadKey != "" {
// Merge into any existing OTEL_RESOURCE_ATTRIBUTES on the workload so we
// don't clobber user-defined attributes during process restart.
existingResourceAttrs := readEnviron(e.procRoot, svc.PID)["OTEL_RESOURCE_ATTRIBUTES"]
envOverrides["OTEL_RESOURCE_ATTRIBUTES"] = mergeResourceAttrValue(
existingResourceAttrs,
"service.workload.key",
svc.WorkloadKey,
)
}
if payload.OTLPProtocol != "" {
envOverrides["OTEL_EXPORTER_OTLP_PROTOCOL"] = payload.OTLPProtocol
}
if payload.OTLPTracesEndpoint != "" {
envOverrides["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = payload.OTLPTracesEndpoint
}
if payload.OTLPMetricsEndpoint != "" {
envOverrides["OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"] = payload.OTLPMetricsEndpoint
}
if payload.OTLPLogsEndpoint != "" {
envOverrides["OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"] = payload.OTLPLogsEndpoint
}
if len(payload.OTLPHeaders) > 0 {
envOverrides["OTEL_EXPORTER_OTLP_HEADERS"] = formatOTLPHeaders(payload.OTLPHeaders)
}
newPID, err := restartProcessWithEnv(e.procRoot, svc.PID, envOverrides)
if err != nil {
e.logger.Warn("bare process restart failed, falling back to manual mode",
zap.String("service", svc.ServiceName),
zap.Error(err),
)
manualReason := fmt.Sprintf(
"OpenLIT SDK installed at %s. To activate, set these environment variables and restart your process:\n"+
" PYTHONPATH=%s:$PYTHONPATH\n"+
" OPENLIT_CONTROLLER_MODE=agent_observability\n"+
" OTEL_SERVICE_NAME=%s\n"+
" OTEL_EXPORTER_OTLP_ENDPOINT=%s\n"+
" OTEL_DEPLOYMENT_ENVIRONMENT=%s\n"+
" OPENLIT_DISABLED_INSTRUMENTORS=%s",
sdkRoot,
bootstrapDir+":"+packagesDir,
svc.ServiceName,
payload.OTLPEndpoint,
payload.Environment,
strings.Join(controllerManagedDisabledInstrumentors, ","),
)
e.setDesiredAgentObservabilityState(svc.ID, "manual", "controller_managed", "", manualReason)
return nil
}
e.mu.Lock()
if s, ok := e.services[svc.ID]; ok {
s.PID = newPID
}
e.mu.Unlock()
e.setDesiredAgentObservabilityState(
svc.ID,
"enabled",
"controller_managed",
"",
fmt.Sprintf("OpenLIT SDK active via process restart (PID %d → %d)", svc.PID, newPID),
)
e.logger.Info("bare process Agent O11y: enabled via restart",
zap.String("service", svc.ServiceName),
zap.Int("old_pid", svc.PID),
zap.Int("new_pid", newPID),
zap.String("sdk_root", sdkRoot),
)
return nil
}
var openlitEnvKeysToStrip = []string{
"PYTHONPATH",
"OPENLIT_CONTROLLER_MODE",
"OTEL_SERVICE_NAME",
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_DEPLOYMENT_ENVIRONMENT",
"OPENLIT_DISABLED_INSTRUMENTORS",
}
func (e *Engine) disablePythonSDKBareProcess(
svc *openlit.ServiceState,
) error {
fingerprint := shortHash(svc.WorkloadKey)
sdkRoot := filepath.Join(bareProcessSDKStateDir, fingerprint)
e.logger.Info("disabling bare process Agent O11y",
zap.String("service", svc.ServiceName),
zap.Int("pid", svc.PID),
zap.String("sdk_root", sdkRoot),
)
// Surgically strip controller-injected env: drop the openlit keys entirely,
// but only remove our `service.workload.key` from OTEL_RESOURCE_ATTRIBUTES
// so any user-defined resource attributes survive.
existingEnv := readEnviron(e.procRoot, svc.PID)
if existingEnv == nil {
existingEnv = make(map[string]string)
}
for _, k := range openlitEnvKeysToStrip {
delete(existingEnv, k)
}
if existing := existingEnv["OTEL_RESOURCE_ATTRIBUTES"]; existing != "" {
if filtered := removeResourceAttrKey(existing, "service.workload.key"); filtered == "" {
delete(existingEnv, "OTEL_RESOURCE_ATTRIBUTES")
} else {
existingEnv["OTEL_RESOURCE_ATTRIBUTES"] = filtered
}
}
newPID, err := restartProcess(e.procRoot, svc.PID, existingEnv)
if err != nil {
e.logger.Warn("bare process restart for disable failed",
zap.String("service", svc.ServiceName),
zap.Error(err),
)
e.setDesiredAgentObservabilityState(
svc.ID,
"disabled",
"none",
"",
"Agent O11y disabled. Remove PYTHONPATH environment variable and restart the process.",
)
} else {
e.mu.Lock()
if s, ok := e.services[svc.ID]; ok {
s.PID = newPID
}
e.mu.Unlock()
e.setDesiredAgentObservabilityState(
svc.ID,
"disabled",
"none",
"",
fmt.Sprintf("Agent O11y disabled via process restart (PID %d → %d)", svc.PID, newPID),
)
e.logger.Info("bare process Agent O11y: disabled via restart",
zap.String("service", svc.ServiceName),
zap.Int("old_pid", svc.PID),
zap.Int("new_pid", newPID),
)
}
_ = os.RemoveAll(sdkRoot)
return nil
}