Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

15.4 KB
Raw Blame
package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"syscall"
	"time"

	"github.com/openlit/openlit/openlit-controller/internal/config"
	"github.com/openlit/openlit/openlit-controller/internal/engine"
	"github.com/openlit/openlit/openlit-controller/internal/openlit"
	"github.com/openlit/openlit/openlit-controller/internal/server"
	"go.uber.org/zap"
)

func main() {
	configPath := flag.String("config", "", "path to config file")
	openlitURL := flag.String("openlit-url", "", "OpenLIT dashboard URL")
	flag.Parse()

	logger, _ := zap.NewProduction()
	defer func() { _ = logger.Sync() }()

	if *openlitURL != "" {
		os.Setenv("OPENLIT_URL", *openlitURL)
	}

	cfg, err := config.Load(*configPath)
	if err != nil {
		logger.Fatal("failed to load config", zap.Error(err))
	}

	logger.Info("openlit-controller starting",
		zap.String("openlit_url", cfg.OpenlitURL),
		zap.String("api_listen", cfg.APIListen),
		zap.Duration("poll_interval", cfg.PollInterval),
		zap.String("obi_binary", cfg.OBIBinaryPath),
		zap.String("otlp_endpoint", cfg.OTLPEndpoint),
		zap.String("proc_root", cfg.ProcRoot),
	)

	eng := engine.New(logger, cfg.OBIBinaryPath, cfg.OTLPEndpoint, cfg.ProcRoot, cfg.Environment, cfg.SDKVersion, cfg.NodeSDKVersion, cfg.DeployMode)
	client := openlit.NewClient(cfg.OpenlitURL, cfg.APIKey, logger)

	var mode openlit.ControllerMode
	switch cfg.DeployMode {
	case config.DeployKubernetes:
		mode = openlit.ModeKubernetes
	case config.DeployDocker:
		mode = openlit.ModeDocker
	default:
		mode = openlit.ModeLinux
	}
	nodeName, _ := os.Hostname()

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	if err := eng.Start(ctx); err != nil {
		logger.Fatal("failed to start engine", zap.Error(err))
	}

	srv := server.New(cfg.APIListen, eng, logger)
	fatalCh := make(chan error, 1)
	go func() {
		if err := srv.Start(); err != nil && err != http.ErrServerClosed {
			logger.Error("REST API server failed", zap.Error(err))
			select {
			case fatalCh <- err:
			default:
			}
		}
	}()

	controllerAttrs := buildControllerResourceAttrs(mode, cfg.Environment, eng.ControllerCapabilities())
	go runPollLoop(ctx, client, eng, logger, cfg.PollInterval, nodeName, mode, cfg.ClusterID, controllerAttrs)

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)

	select {
	case <-sigCh:
		logger.Info("received shutdown signal")
	case err := <-fatalCh:
		logger.Error("initiating shutdown due to fatal component error", zap.Error(err))
	}

	cancel()
	logger.Info("shutting down")
	eng.Stop()

	shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer shutdownCancel()
	if err := srv.Shutdown(shutdownCtx); err != nil {
		logger.Warn("HTTP server shutdown error", zap.Error(err))
	}
}

func runPollLoop(
	ctx context.Context,
	client *openlit.Client,
	eng *engine.Engine,
	logger *zap.Logger,
	interval time.Duration,
	nodeName string,
	mode openlit.ControllerMode,
	clusterID string,
	controllerAttrs map[string]string,
) {
	iid := instanceID(mode)
	currentInterval := interval
	var currentConfigHash string

	resp := doPoll(ctx, client, eng, logger, iid, nodeName, mode, clusterID, nil, controllerAttrs, currentConfigHash)

	ticker := time.NewTicker(currentInterval)
	defer ticker.Stop()

	var pendingResults []openlit.ActionResult
	if resp != nil {
		pendingResults = resp.actionResults
		if resp.configHash != "" {
			currentConfigHash = resp.configHash
		}
		if next := resp.pollInterval; next > 0 && next != currentInterval {
			currentInterval = next
			ticker.Reset(currentInterval)
			logger.Info("poll interval updated from config", zap.Duration("interval", currentInterval))
		}
	}

	consecutiveFailures := 0
	const maxBackoffMultiplier = 4

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			resp := doPoll(ctx, client, eng, logger, iid, nodeName, mode, clusterID, pendingResults, controllerAttrs, currentConfigHash)
			if resp != nil {
				consecutiveFailures = 0
				pendingResults = resp.actionResults
				if resp.configHash != "" {
					currentConfigHash = resp.configHash
				}
				if next := resp.pollInterval; next > 0 && next != currentInterval {
					currentInterval = next
					ticker.Reset(currentInterval)
					logger.Info("poll interval updated from config", zap.Duration("interval", currentInterval))
				}
			} else {
				consecutiveFailures++
				if consecutiveFailures > 1 {
					backoff := consecutiveFailures
					if backoff > maxBackoffMultiplier {
						backoff = maxBackoffMultiplier
					}
					delay := time.Duration(backoff) * currentInterval / 2
					logger.Warn("poll failed, backing off",
						zap.Int("consecutive_failures", consecutiveFailures),
						zap.Duration("backoff", delay),
					)
					select {
					case <-ctx.Done():
						return
					case <-time.After(delay):
					}
				}
			}
		}
	}
}

type pollResult struct {
	actionResults []openlit.ActionResult
	pollInterval  time.Duration
	configHash    string
}

func doPoll(
	ctx context.Context,
	client *openlit.Client,
	eng *engine.Engine,
	logger *zap.Logger,
	instanceID string,
	nodeName string,
	mode openlit.ControllerMode,
	clusterID string,
	actionResults []openlit.ActionResult,
	controllerAttrs map[string]string,
	configHash string,
) *pollResult {
	discovered, instrumented := eng.ServiceCount()

	services := eng.GetServices()
	dsvcs := make([]openlit.DiscoveredService, 0, len(services))
	for _, svc := range services {
		dsvcs = append(dsvcs, openlit.DiscoveredService{
			ServiceName:              svc.ServiceName,
			WorkloadKey:              svc.WorkloadKey,
			Namespace:                svc.Namespace,
			LanguageRuntime:          svc.LanguageRuntime,
			LLMProviders:             svc.LLMProviders,
			OpenPorts:                svc.OpenPorts,
			DeploymentName:           svc.DeploymentName,
			PID:                      svc.PID,
			ExePath:                  svc.ExePath,
			InstrumentationStatus:    svc.InstrumentationStatus,
			AgentObservabilityStatus: svc.AgentObservabilityStatus,
			AgentObservabilitySource: svc.AgentObservabilitySource,
			ObservabilityConflict:    svc.ObservabilityConflict,
			ObservabilityReason:      svc.ObservabilityReason,
			LifecycleStatus:          svc.LifecycleStatus,
			FirstSeen:                svc.FirstSeen.UTC().Format("2006-01-02 15:04:05"),
			ResourceAttributes:       svc.ResourceAttributes,
		})
	}

	resp, err := client.Poll(ctx, &openlit.PollRequest{
		InstanceID:           instanceID,
		ClusterID:            clusterID,
		Version:              server.Version,
		Mode:                 mode,
		NodeName:             nodeName,
		ServicesDiscovered:   discovered,
		ServicesInstrumented: instrumented,
		Services:             dsvcs,
		ActionResults:        actionResults,
		ResourceAttributes:   controllerAttrs,
		ConfigHash:           configHash,
	})
	if err != nil {
		logger.Warn("poll failed", zap.Error(err))
		return nil
	}

	var results []openlit.ActionResult
	for _, action := range resp.Actions {
		result := executeAction(eng, action, logger)
		results = append(results, result)
	}

	pr := &pollResult{actionResults: results}

	if resp.ConfigHash != "" {
		pr.configHash = resp.ConfigHash
	}

	if resp.ConfigChanged && resp.Config != nil {
		if v, ok := resp.Config["poll_interval_seconds"]; ok {
			if seconds, ok := v.(float64); ok && seconds >= 5 && seconds <= 300 {
				pr.pollInterval = time.Duration(seconds) * time.Second
			}
		}

		if hostsRaw, ok := resp.Config["custom_llm_hosts"]; ok {
			if hostsList, ok := hostsRaw.([]interface{}); ok {
				specs := make([]string, 0, len(hostsList))
				for _, h := range hostsList {
					if s, ok := h.(string); ok {
						if s = strings.TrimSpace(s); s != "" {
							specs = append(specs, s)
						}
					}
				}
				eng.UpdateCustomHosts(specs)
			}
		}

		if exportRaw, ok := resp.Config["export"]; ok {
			if exportMap, ok := exportRaw.(map[string]interface{}); ok {
				// Only non-empty values override the controller's configured
				// (env-derived) endpoints. An empty/unset field in the dashboard
				// config means "use the controller's own OTEL_EXPORTER_OTLP_ENDPOINT",
				// so it must NOT clobber the working endpoint with "".
				newCfg := eng.GetExportConfig()
				if v, ok := exportMap["otlp_endpoint"].(string); ok && v != "" {
					newCfg.OTLPEndpoint = v
				}
				if v, ok := exportMap["otlp_protocol"].(string); ok && v != "" {
					newCfg.OTLPProtocol = v
				}
				if v, ok := exportMap["otlp_traces_endpoint"].(string); ok && v != "" {
					newCfg.OTLPTracesEndpoint = v
				}
				if v, ok := exportMap["otlp_metrics_endpoint"].(string); ok && v != "" {
					newCfg.OTLPMetricsEndpoint = v
				}
				if v, ok := exportMap["otlp_logs_endpoint"].(string); ok && v != "" {
					newCfg.OTLPLogsEndpoint = v
				}
				if headersRaw, ok := exportMap["otlp_headers"]; ok {
					if headersMap, ok := headersRaw.(map[string]interface{}); ok {
						headers := make(map[string]string, len(headersMap))
						for k, v := range headersMap {
							if sv, ok := v.(string); ok {
								headers[k] = sv
							}
						}
						newCfg.OTLPHeaders = headers
					}
				}
				eng.UpdateExportConfig(newCfg)
			}
		}
	}

	return pr
}

func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zap.Logger) openlit.ActionResult {
	var execErr error
	var snapshot string

	switch action.ActionType {
	case openlit.ActionInstrument:
		execErr = eng.InstrumentService(action.ServiceKey)
	case openlit.ActionUninstrument:
		execErr = eng.UninstrumentService(action.ServiceKey)
	case openlit.ActionEnableSDK, openlit.ActionEnablePythonSDK:
		payload, err := parsePythonSDKPayload(action.Payload)
		if err != nil {
			execErr = err
			break
		}
		execErr = eng.EnablePythonSDK(action.ServiceKey, payload)
	case openlit.ActionDisableSDK, openlit.ActionDisablePythonSDK:
		payload, err := parsePythonSDKPayload(action.Payload)
		if err != nil {
			execErr = err
			break
		}
		execErr = eng.DisablePythonSDK(action.ServiceKey, payload)
	case openlit.ActionStartWorkload:
		execErr = eng.StartWorkload(action.ServiceKey, action.Payload)
	case openlit.ActionStopWorkload:
		// StopWorkload returns a snapshot blob the dashboard persists
		// into desired_states_v2.config so a later Start can hand it
		// back. For K8s controlled workloads the blob carries just
		// (kind, ns, name, container_name); for K8s naked pods it
		// carries the full gzipped pod spec; for Linux bare processes
		// it carries (exe_path, argv, cwd, env-allowlist). Docker and
		// Linux systemd modes return "" because the durable identifier
		// (container name, unit name) is already encoded in
		// workload_key and the runtime preserves the object across Stop.
		snapshot, execErr = eng.StopWorkload(action.ServiceKey, action.Payload)
	case openlit.ActionRestartWorkload:
		execErr = eng.RestartWorkload(action.ServiceKey, action.Payload)
	default:
		logger.Warn("unknown action type", zap.String("type", action.ActionType))
		return openlit.ActionResult{
			ActionID: action.ID,
			Status:   "failed",
			Error:    fmt.Sprintf("unknown action type: %s", action.ActionType),
		}
	}

	if execErr != nil {
		logger.Error("action execution failed",
			zap.String("action_id", action.ID),
			zap.String("type", action.ActionType),
			zap.Error(execErr),
		)
		return openlit.ActionResult{
			ActionID: action.ID,
			Status:   "failed",
			Error:    execErr.Error(),
		}
	}

	logger.Info("action executed",
		zap.String("action_id", action.ID),
		zap.String("type", action.ActionType),
		zap.String("service", action.ServiceKey),
	)
	return openlit.ActionResult{
		ActionID: action.ID,
		Status:   "completed",
		Snapshot: snapshot,
	}
}

func parsePythonSDKPayload(raw string) (openlit.PythonSDKActionPayload, error) {
	payload := openlit.PythonSDKActionPayload{
		TargetRuntime:          "python",
		InstrumentationProfile: "controller_managed",
		DuplicatePolicy:        "block_if_existing_otel_detected",
		ObservabilityScope:     "agent",
	}
	if raw == "" || raw == "{}" {
		return payload, nil
	}
	if err := json.Unmarshal([]byte(raw), &payload); err != nil {
		return payload, fmt.Errorf("parse python SDK payload: %w", err)
	}
	if payload.TargetRuntime == "" {
		payload.TargetRuntime = "python"
	}
	if payload.InstrumentationProfile == "" {
		payload.InstrumentationProfile = "controller_managed"
	}
	if payload.DuplicatePolicy == "" {
		payload.DuplicatePolicy = "block_if_existing_otel_detected"
	}
	if payload.ObservabilityScope == "" {
		payload.ObservabilityScope = "agent"
	}
	return payload, nil
}

func buildControllerResourceAttrs(mode openlit.ControllerMode, environment string, capabilities []string) map[string]string {
	hostname, _ := os.Hostname()
	attrs := map[string]string{
		"host.name":               hostname,
		"os.type":                 runtime.GOOS,
		"host.arch":               runtime.GOARCH,
		"controller.version":      server.Version,
		"deployment.environment":  environment,
		"controller.capabilities": strings.Join(capabilities, ","),
	}

	switch mode {
	case openlit.ModeKubernetes:
		if v := os.Getenv("NODE_NAME"); v != "" {
			attrs["k8s.node.name"] = v
		}
		if v := os.Getenv("POD_NAME"); v != "" {
			attrs["k8s.pod.name"] = v
		}
		if v := os.Getenv("POD_NAMESPACE"); v != "" {
			attrs["k8s.namespace.name"] = v
		}
		if v := os.Getenv("POD_UID"); v != "" {
			attrs["k8s.pod.uid"] = v
		}
	case openlit.ModeDocker:
		attrs["container.runtime"] = "docker"
	}

	return attrs
}

func instanceID(mode openlit.ControllerMode) string {
	if env := os.Getenv("OPENLIT_INSTANCE_ID"); env != "" {
		return env
	}
	if mode == openlit.ModeKubernetes {
		if nodeName := os.Getenv("NODE_NAME"); nodeName != "" {
			return nodeName
		}
	}
	if mode == openlit.ModeDocker {
		if h, err := dockerStableID(); err == nil && h != "" {
			return h
		}
	}
	hostname, _ := os.Hostname()
	return hostname
}

func dockerStableID() (string, error) {
	hostName, err := dockerHostHostname()
	if err != nil {
		hostName, err = procHostHostname()
	}
	if err != nil || hostName == "" {
		return "", fmt.Errorf("could not determine host hostname: %w", err)
	}
	containerHostname, _ := os.Hostname()
	if containerHostname != "" && containerHostname != hostName {
		h := sha256.Sum256([]byte(containerHostname))
		hostName = hostName + "-" + hex.EncodeToString(h[:])[:6]
	}
	return hostName, nil
}

// dockerHostHostname queries the Docker Engine API for the real host hostname.
func dockerHostHostname() (string, error) {
	const sock = "/var/run/docker.sock"
	if _, err := os.Stat(sock); err != nil {
		return "", err
	}
	client := &http.Client{
		Transport: &http.Transport{
			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
				return net.DialTimeout("unix", sock, 2*time.Second)
			},
		},
		Timeout: 5 * time.Second,
	}
	resp, err := client.Get("http://docker/info")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return "", err
	}
	var info struct {
		Name string `json:"Name"`
	}
	if err := json.Unmarshal(body, &info); err != nil {
		return "", err
	}
	if info.Name == "" {
		return "", fmt.Errorf("empty Name in docker info")
	}
	return info.Name, nil
}

// procHostHostname reads the host hostname from the mounted /proc filesystem.
func procHostHostname() (string, error) {
	procRoot := os.Getenv("OPENLIT_PROC_ROOT")
	if procRoot == "" {
		procRoot = "/proc"
	}
	data, err := os.ReadFile(filepath.Join(procRoot, "sys/kernel/hostname"))
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(data)), nil
}