Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

Play pause stop (#1192)

Aman Agarwal7ca59852f631
diff --git a/openlit-controller/cmd/controller/main.go b/openlit-controller/cmd/controller/main.go
index 81c49c99..8171fcaa 100644
--- a/openlit-controller/cmd/controller/main.go
+++ b/openlit-controller/cmd/controller/main.go
@@ -223,6 +223,7 @@ func doPoll(
AgentObservabilitySource: svc.AgentObservabilitySource,
ObservabilityConflict: svc.ObservabilityConflict,
ObservabilityReason: svc.ObservabilityReason,
+ LifecycleStatus: svc.LifecycleStatus,
FirstSeen: svc.FirstSeen.UTC().Format("2006-01-02 15:04:05"),
ResourceAttributes: svc.ResourceAttributes,
})
@@ -304,6 +305,7 @@ func doPoll(
func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zap.Logger) openlit.ActionResult {
var execErr error
+ var snapshot string
switch action.ActionType {
case openlit.ActionInstrument:
@@ -324,6 +326,21 @@ func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zap
break
}
execErr = eng.DisablePythonSDK(action.ServiceKey, payload)
+ case openlit.ActionStartWorkload:
+ execErr = eng.StartWorkload(action.ServiceKey, action.Payload)
+ case openlit.ActionStopWorkload:
+ // StopWorkload returns a snapshot blob the dashboard persists
+ // into desired_states_v2.config so a later Start can hand it
+ // back. For K8s controlled workloads the blob carries just
+ // (kind, ns, name, container_name); for K8s naked pods it
+ // carries the full gzipped pod spec; for Linux bare processes
+ // it carries (exe_path, argv, cwd, env-allowlist). Docker and
+ // Linux systemd modes return "" because the durable identifier
+ // (container name, unit name) is already encoded in
+ // workload_key and the runtime preserves the object across Stop.
+ snapshot, execErr = eng.StopWorkload(action.ServiceKey, action.Payload)
+ case openlit.ActionRestartWorkload:
+ execErr = eng.RestartWorkload(action.ServiceKey, action.Payload)
default:
logger.Warn("unknown action type", zap.String("type", action.ActionType))
return openlit.ActionResult{
@@ -354,6 +371,7 @@ func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zap
return openlit.ActionResult{
ActionID: action.ID,
Status: "completed",
+ Snapshot: snapshot,
}
}
diff --git a/openlit-controller/internal/engine/container.go b/openlit-controller/internal/engine/container.go
index 0803ae9f..e96da17f 100644
--- a/openlit-controller/internal/engine/container.go
+++ b/openlit-controller/internal/engine/container.go
@@ -429,6 +429,186 @@ func (c *k8sAPIClient) patchWorkload(namespace, kind, name string, patch map[str
return nil
}
+// putWorkload performs an HTTP PUT against the workload resource with the
+// given full object. K8s uses the embedded `metadata.resourceVersion` for
+// optimistic concurrency: callers GET the object, mutate it, then PUT it
+// back; a 409 Conflict means someone else updated it first and the caller
+// should retry. Used by atomicScaleWithAnnotation and bumpRolloutAnnotation
+// to avoid blind merge-patch races when multiple DaemonSet controllers act
+// on the same workload at once.
+func (c *k8sAPIClient) putWorkload(namespace, kind, name string, workload map[string]any) (int, error) {
+ resource, err := k8sWorkloadResource(kind)
+ if err != nil {
+ return 0, err
+ }
+ body, err := json.Marshal(workload)
+ if err != nil {
+ return 0, fmt.Errorf("marshal workload: %w", err)
+ }
+ endpoint := fmt.Sprintf(
+ "%s/apis/apps/v1/namespaces/%s/%s/%s",
+ c.apiServer,
+ url.PathEscape(namespace),
+ resource,
+ url.PathEscape(name),
+ )
+ req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(body))
+ if err != nil {
+ return 0, err
+ }
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return 0, err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ responseBody, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, fmt.Errorf("K8s workload put returned %d: %s", resp.StatusCode, strings.TrimSpace(string(responseBody)))
+ }
+ return resp.StatusCode, nil
+}
+
+// atomicScaleWithAnnotation scales a workload to the target replica count,
+// recording the prior count in an annotation in the same atomic update.
+// It uses a read-modify-update loop with resourceVersion-based optimistic
+// concurrency: on 409 Conflict (someone else patched the workload first),
+// it re-reads and retries up to 3 times. The operation is idempotent: if
+// the workload is already at the target replicas AND the annotation
+// matches the recorded value, it returns nil without making an API call.
+//
+// recordAnnotationKey/Value semantics:
+// - Pass empty annotationKey to skip annotation handling entirely.
+// - When scaling down (replicas==0) and the workload was previously >0,
+// the prior replica count is recorded under recordAnnotationKey so
+// scale-up can restore it.
+// - When scaling up (replicas>0), the annotation is removed.
+func (c *k8sAPIClient) atomicScaleWithAnnotation(namespace, kind, name string, replicas int, recordAnnotationKey string) error {
+ const maxRetries = 3
+ for attempt := 0; attempt < maxRetries; attempt++ {
+ workload, err := c.getWorkload(namespace, kind, name)
+ if err != nil {
+ return err
+ }
+ spec, _ := workload["spec"].(map[string]any)
+ if spec == nil {
+ return fmt.Errorf("workload %s/%s has no spec", kind, name)
+ }
+ current := readReplicas(spec)
+ meta, _ := workload["metadata"].(map[string]any)
+ annotations := copyStringMap(meta["annotations"])
+
+ alreadyAtTarget := current == replicas
+ annotationMatches := true
+ if recordAnnotationKey != "" {
+ if replicas == 0 {
+ _, present := annotations[recordAnnotationKey]
+ annotationMatches = current == 0 && present
+ } else {
+ _, present := annotations[recordAnnotationKey]
+ annotationMatches = !present
+ }
+ }
+ if alreadyAtTarget && annotationMatches {
+ return nil
+ }
+
+ spec["replicas"] = replicas
+ if recordAnnotationKey != "" {
+ if replicas == 0 && current > 0 {
+ annotations[recordAnnotationKey] = fmt.Sprintf("%d", current)
+ } else if replicas > 0 {
+ delete(annotations, recordAnnotationKey)
+ }
+ }
+ if meta == nil {
+ meta = map[string]any{}
+ workload["metadata"] = meta
+ }
+ meta["annotations"] = stringMapToAny(annotations)
+
+ status, err := c.putWorkload(namespace, kind, name, workload)
+ if err == nil {
+ return nil
+ }
+ if status == http.StatusConflict {
+ continue
+ }
+ return err
+ }
+ return fmt.Errorf("atomic scale conflict after %d retries", maxRetries)
+}
+
+// readSavedReplicas returns the value of the saved-replicas annotation on a
+// workload, falling back to 1 when missing or unparsable. Used to decide
+// what to scale back to on Start.
+func (c *k8sAPIClient) readSavedReplicas(namespace, kind, name, annotationKey string) (int, error) {
+ workload, err := c.getWorkload(namespace, kind, name)
+ if err != nil {
+ return 0, err
+ }
+ meta, _ := workload["metadata"].(map[string]any)
+ annotations := copyStringMap(meta["annotations"])
+ raw, ok := annotations[annotationKey]
+ if !ok {
+ return 1, nil
+ }
+ var saved int
+ _, scanErr := fmt.Sscanf(raw, "%d", &saved)
+ if scanErr != nil || saved < 1 {
+ return 1, nil
+ }
+ return saved, nil
+}
+
+// bumpRolloutAnnotation patches the workload pod template annotations to
+// trigger a rolling restart (same trick `kubectl rollout restart` uses).
+// Uses an atomic read-modify-update so multiple controllers racing on the
+// same workload do not lose each other's writes.
+func (c *k8sAPIClient) bumpRolloutAnnotation(namespace, kind, name, annotationKey, annotationValue string) error {
+ const maxRetries = 3
+ for attempt := 0; attempt < maxRetries; attempt++ {
+ workload, err := c.getWorkload(namespace, kind, name)
+ if err != nil {
+ return err
+ }
+ template, err := getTemplate(workload)
+ if err != nil {
+ return err
+ }
+ tmplMeta := ensureMap(template, "metadata")
+ annotations := copyStringMap(tmplMeta["annotations"])
+ if existing, ok := annotations[annotationKey]; ok && existing == annotationValue {
+ return nil
+ }
+ annotations[annotationKey] = annotationValue
+ tmplMeta["annotations"] = stringMapToAny(annotations)
+
+ status, err := c.putWorkload(namespace, kind, name, workload)
+ if err == nil {
+ return nil
+ }
+ if status == http.StatusConflict {
+ continue
+ }
+ return err
+ }
+ return fmt.Errorf("rollout-restart conflict after %d retries", maxRetries)
+}
+
+func readReplicas(spec map[string]any) int {
+ if v, ok := spec["replicas"].(float64); ok {
+ return int(v)
+ }
+ if v, ok := spec["replicas"].(int); ok {
+ return v
+ }
+ return 0
+}
+
func k8sWorkloadResource(kind string) (string, error) {
switch strings.ToLower(kind) {
case "deployment":
@@ -790,6 +970,23 @@ func (d *dockerAPIClient) stopContainer(containerID string, timeoutSeconds int)
return nil
}
+// restartContainer issues a native Docker restart for the given container.
+// The container's name and ID are both preserved, so the controller's
+// workload_key (docker:<name>) remains stable across the restart.
+func (d *dockerAPIClient) restartContainer(containerID string, timeoutSeconds int) error {
+ endpoint := fmt.Sprintf("/containers/%s/restart?t=%d", containerID, timeoutSeconds)
+ resp, err := d.doRequest(http.MethodPost, endpoint, nil, "")
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusNoContent {
+ body, _ := io.ReadAll(resp.Body)
+ return fmt.Errorf("docker restart returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+ }
+ return nil
+}
+
func (d *dockerAPIClient) renameContainer(containerID, newName string) error {
resp, err := d.doRequest(http.MethodPost, fmt.Sprintf("/containers/%s/rename?name=%s", containerID, url.QueryEscape(newName)), nil, "")
if err != nil {
diff --git a/openlit-controller/internal/engine/engine.go b/openlit-controller/internal/engine/engine.go
index 8437db24..b9c6df07 100644
--- a/openlit-controller/internal/engine/engine.go
+++ b/openlit-controller/internal/engine/engine.go
@@ -66,6 +66,23 @@ const (
CapabilityPythonSDKDockerV1 = "python_sdk_injection_docker_v1"
CapabilityPythonSDKLinuxSystemdV1 = "python_sdk_injection_linux_systemd_v1"
CapabilityPythonSDKLinuxBareV1 = "python_sdk_injection_linux_bare_v1"
+ CapabilityLifecycleKubernetesV1 = "lifecycle_kubernetes_v1"
+ CapabilityLifecycleDockerV1 = "lifecycle_docker_v1"
+ CapabilityLifecycleLinuxSystemdV1 = "lifecycle_linux_systemd_v1"
+ CapabilityLifecycleLinuxBareV1 = "lifecycle_linux_bare_v1"
+)
+
+// Lifecycle resource-attribute / annotation keys shared between the
+// controller heartbeat and the dashboard rollup. Keep these constants in
+// sync with the SQL projections in
+// src/client/src/lib/platform/agents/index.ts.
+const (
+ LifecycleStatusAttr = "openlit.lifecycle.status"
+ LifecycleStatusRunning = "running"
+ LifecycleStatusStopped = "stopped"
+ LifecycleStatusRestarting = "restarting"
+ K8sSavedReplicasAnnotation = "openlit.io/saved-replicas"
+ K8sRolloutRestartAnnotation = "openlit.io/restartedAt"
)
func New(logger *zap.Logger, obiBinaryPath, otlpEndpoint, procRoot, environment, sdkVersion string, mode config.DeployMode) *Engine {
@@ -397,15 +414,19 @@ func (e *Engine) ControllerCapabilities() []string {
case config.DeployKubernetes:
if e.container != nil && e.container.k8sClient != nil {
capabilities = append(capabilities, CapabilityPythonSDKKubernetesV1)
+ capabilities = append(capabilities, CapabilityLifecycleKubernetesV1)
}
case config.DeployDocker:
if e.container != nil && e.container.dockerClient != nil && e.container.dockerClient.canManage() {
capabilities = append(capabilities, CapabilityPythonSDKDockerV1)
+ capabilities = append(capabilities, CapabilityLifecycleDockerV1)
}
default:
capabilities = append(capabilities, CapabilityPythonSDKLinuxBareV1)
+ capabilities = append(capabilities, CapabilityLifecycleLinuxBareV1)
if linuxSystemdSDKSupported() {
capabilities = append(capabilities, CapabilityPythonSDKLinuxSystemdV1)
+ capabilities = append(capabilities, CapabilityLifecycleLinuxSystemdV1)
}
}
@@ -494,3 +515,57 @@ func (e *Engine) ServiceCount() (discovered, instrumented int) {
}
return
}
+
+// setLifecycleState stamps the lifecycle status onto the matching in-memory
+// service so the next heartbeat reports it. Mirrors the
+// applyAgentStateLocked / setAgentObservabilityState pattern in python_sdk.go
+// and uses the same resource_attributes channel
+// (openlit.lifecycle.status) for the dashboard rollup.
+//
+// On Stop the scanner is likely to lose sight of the workload immediately
+// (the process exits / container exits / pod is gone), so this helper is the
+// last chance the controller has to surface the "stopped" state to the UI
+// before the row falls out of the heartbeat window. We also refresh
+// LastSeen so the row survives the next prune cycle (30-minute
+// not-instrumented cutoff in pruneStaleServices) while the user decides
+// whether to Play it back.
+func (e *Engine) setLifecycleState(workloadKey, status string) {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+
+ svc, ok := e.services[workloadKey]
+ if !ok {
+ return
+ }
+ svc.LifecycleStatus = status
+ svc.LastSeen = time.Now()
+ if svc.ResourceAttributes == nil {
+ svc.ResourceAttributes = make(map[string]string)
+ }
+ if status != "" {
+ svc.ResourceAttributes[LifecycleStatusAttr] = status
+ } else {
+ delete(svc.ResourceAttributes, LifecycleStatusAttr)
+ }
+}
+
+// snapshotService returns a defensive copy of a service entry so callers
+// can read its identity (workload_key, namespace, service_name,
+// container name) without holding the engine lock.
+func (e *Engine) snapshotService(workloadKey string) (*openlit.ServiceState, error) {
+ e.mu.RLock()
+ defer e.mu.RUnlock()
+ svc, ok := e.services[workloadKey]
+ if !ok {
+ return nil, fmt.Errorf("service %q not found", workloadKey)
+ }
+ copied := *svc
+ if svc.ResourceAttributes != nil {
+ clone := make(map[string]string, len(svc.ResourceAttributes))
+ for k, v := range svc.ResourceAttributes {
+ clone[k] = v
+ }
+ copied.ResourceAttributes = clone
+ }
+ return &copied, nil
+}
diff --git a/openlit-controller/internal/engine/lifecycle.go b/openlit-controller/internal/engine/lifecycle.go
new file mode 100644
index 00000000..82278e15
--- /dev/null
+++ b/openlit-controller/internal/engine/lifecycle.go
@@ -0,0 +1,715 @@
+// Package engine — lifecycle.go
+//
+// Implements the controller-side Play / Stop / Restart actions queued from
+// the dashboard. The critical invariant every code path here MUST preserve
+// is workload identity: an agent that gets stopped and started again must
+// keep the same workload_key (see events.go::buildWorkloadKey) and the same
+// service_name so the dashboard rollup re-points to the *same* agent_key
+// row instead of creating a duplicate.
+//
+// Per-mode identity preservation rules:
+// - K8s controlled workload: scale-to-0 (preserves Deployment/STS/DS
+// object, container_name, deployment_name); rollout-restart is an
+// annotation bump on the pod template.
+// - K8s naked pod: snapshot the pod spec before delete, recreate with the
+// *same* metadata.name on Start.
+// - Docker: stopContainer / startContainer / restartContainer all run
+// against the same container ID; name + ID are stable.
+// - Linux systemd: systemctl start/stop/restart of the same unit name.
+// - Linux bare process: capture (exe, args, cwd, env-allowlist) before
+// SIGTERM; re-exec with the same args produces an identical
+// sha1(exe|cmdline) fingerprint.
+
+package engine
+
+import (
+ "bytes"
+ "compress/gzip"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "os/exec"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/openlit/openlit/openlit-controller/internal/config"
+ "github.com/openlit/openlit/openlit-controller/internal/openlit"
+ "go.uber.org/zap"
+)
+
+// lifecycleBareProcessSnapshot is what we persist into the desired-state
+// config blob on Stop for a Linux bare process so Start can re-exec it
+// identically. The workload_key fingerprint that the dashboard rolls up
+// on is sha1(/proc/PID/exe-resolved | argv-cmdline), so the only fields
+// we strictly need to preserve identity are Args (argv) and the runtime
+// binary path that PATH resolution lands on for Args[0]. Cwd is needed
+// to give relative argv[0]s the same starting directory. The env
+// allowlist intentionally excludes arbitrary user env to avoid leaking
+// secrets through the dashboard.
+//
+// ExePath here is captured from argv[0] (i.e. the launch path the user
+// typed) rather than the resolved /proc/PID/exe symlink — it is purely
+// informational metadata for operators reading the desired-state row.
+// launchBareProcess relies on Args + Cwd to recreate identical argv +
+// PATH resolution on Start.
+type lifecycleBareProcessSnapshot struct {
+ ExePath string `json:"exe_path"`
+ Args []string `json:"args"`
+ Cwd string `json:"cwd"`
+ Env map[string]string `json:"env"`
+}
+
+// lifecycleK8sPodSnapshot wraps a gzip+base64-encoded pod spec. We
+// serialize it this way because real-world pod specs can be 5-15KB; the
+// desired-states config column is plain String and gets piped through
+// ClickHouse JSONEachRow, so keeping it compact + ASCII-safe matters.
+//
+// `Controlled` is set for Deployment/StatefulSet/DaemonSet workloads so
+// startK8s can re-derive the kind/namespace/name needed to scale back
+// up even when the controller's in-memory service cache has been
+// pruned (>30 min after Stop, or controller restart). Without it, a
+// long-stopped controlled workload becomes unrecoverable from the UI.
+//
+// Exactly one of `Controlled` or `GzippedPodB64` is populated per
+// snapshot: controlled workloads do not carry pod specs because Start
+// simply patches the existing Deployment back up, and naked pods do
+// not carry a Controlled struct because there is no parent object.
+type lifecycleK8sPodSnapshot struct {
+ GzippedPodB64 string `json:"gzipped_pod_b64,omitempty"`
+ Controlled *lifecycleK8sControlledSnapshot `json:"controlled,omitempty"`
+}
+
+type lifecycleK8sControlledSnapshot struct {
+ Kind string `json:"kind"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ ContainerName string `json:"container_name,omitempty"`
+}
+
+// envAllowlistPrefixes captures the env vars Stop/Start should preserve
+// for bare-process restart. Everything else (auth tokens, DB passwords,
+// arbitrary user env) is dropped on purpose -- the controller is not a
+// secret store. PATH/HOME keep the new process able to find its libs.
+var envAllowlistPrefixes = []string{
+ "PATH",
+ "HOME",
+ "LANG",
+ "LC_",
+ "PYTHONPATH",
+ "OPENLIT_",
+ "OTEL_",
+}
+
+// StartWorkload brings a previously-stopped workload back online. payload
+// is the raw JSON string queued by the dashboard; for naked pods and bare
+// processes it carries the snapshot captured at Stop time.
+func (e *Engine) StartWorkload(workloadKey, payload string) error {
+ svc, err := e.snapshotService(workloadKey)
+ if err != nil {
+ // For K8s naked pods + Linux bare processes the service is gone
+ // from e.services by the time Start arrives (Stop removed it).
+ // We still need to act, but with only the workload_key and the
+ // payload to go on. The payload carries the snapshot in those
+ // cases; for other modes we cannot proceed without an in-memory
+ // service entry.
+ if isModeRequiringInMemory(e.deployMode) {
+ return err
+ }
+ }
+
+ switch e.deployMode {
+ case config.DeployKubernetes:
+ return e.startK8s(workloadKey, svc, payload)
+ case config.DeployDocker:
+ if svc == nil {
+ return fmt.Errorf("docker workload %q not in scanner cache", workloadKey)
+ }
+ return e.startDocker(svc)
+ default:
+ return e.startLinux(workloadKey, svc, payload)
+ }
+}
+
+// StopWorkload takes a running workload offline. The returned snapshot is
+// included in the ActionResult so the dashboard can persist it into
+// desired_states_v2.config and feed it back on the next Start.
+func (e *Engine) StopWorkload(workloadKey, _ string) (string, error) {
+ svc, err := e.snapshotService(workloadKey)
+ if err != nil {
+ return "", err
+ }
+
+ switch e.deployMode {
+ case config.DeployKubernetes:
+ return e.stopK8s(svc)
+ case config.DeployDocker:
+ return "", e.stopDocker(svc)
+ default:
+ return e.stopLinux(svc)
+ }
+}
+
+// RestartWorkload is identity-preserving: pod names / container IDs /
+// systemd unit names are unchanged across the operation.
+func (e *Engine) RestartWorkload(workloadKey, _ string) error {
+ svc, err := e.snapshotService(workloadKey)
+ if err != nil {
+ return err
+ }
+
+ switch e.deployMode {
+ case config.DeployKubernetes:
+ return e.restartK8s(svc)
+ case config.DeployDocker:
+ return e.restartDocker(svc)
+ default:
+ return e.restartLinux(svc)
+ }
+}
+
+func isModeRequiringInMemory(mode config.DeployMode) bool {
+ // For K8s the dashboard-supplied payload + workload_key are enough to
+ // recreate naked pods; controlled workloads can be looked up directly
+ // from the API server. For Linux bare we likewise need only the
+ // payload + workload_key. Docker is the odd one out: Start runs
+ // against a stopped container, and we need its current container.id
+ // from the local scanner cache to issue the start.
+ return mode == config.DeployDocker
+}
+
+// ----- Kubernetes ---------------------------------------------------------
+
+func (e *Engine) stopK8s(svc *openlit.ServiceState) (string, error) {
+ if e.container == nil || e.container.k8sClient == nil {
+ return "", fmt.Errorf("kubernetes client unavailable for lifecycle stop")
+ }
+ namespace := svc.ResourceAttributes["k8s.namespace.name"]
+ workloadKind := svc.ResourceAttributes["k8s.workload.kind"]
+ workloadName := svc.DeploymentName
+ if workloadKind == "" || workloadKind == "Pod" {
+ return e.stopK8sNakedPod(svc)
+ }
+ if namespace == "" || workloadName == "" {
+ return "", fmt.Errorf("missing k8s workload metadata for %s", svc.ServiceName)
+ }
+ if err := e.container.k8sClient.atomicScaleWithAnnotation(
+ namespace, workloadKind, workloadName, 0, K8sSavedReplicasAnnotation,
+ ); err != nil {
+ return "", err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)
+ e.logger.Info("k8s workload stopped",
+ zap.String("workload_key", svc.WorkloadKey),
+ zap.String("kind", workloadKind),
+ zap.String("name", workloadName),
+ )
+ // Persist a small identifier blob so a later Start can re-derive
+ // (kind, ns, name) without depending on the controller's in-memory
+ // cache — which gets pruned after 30 min, or wiped entirely on
+ // controller restart.
+ snap := lifecycleK8sPodSnapshot{
+ Controlled: &lifecycleK8sControlledSnapshot{
+ Kind: workloadKind,
+ Namespace: namespace,
+ Name: workloadName,
+ ContainerName: svc.ResourceAttributes["container.name"],
+ },
+ }
+ out, err := json.Marshal(snap)
+ if err != nil {
+ // The scale already happened; losing the snapshot just degrades
+ // recovery, it doesn't undo Stop. Return success so the action
+ // completes and the UI reflects the stopped state.
+ e.logger.Warn("k8s controlled snapshot marshal failed",
+ zap.String("workload_key", svc.WorkloadKey),
+ zap.Error(err),
+ )
+ return "", nil
+ }
+ return string(out), nil
+}
+
+func (e *Engine) startK8s(workloadKey string, svc *openlit.ServiceState, payload string) error {
+ if e.container == nil || e.container.k8sClient == nil {
+ return fmt.Errorf("kubernetes client unavailable for lifecycle start")
+ }
+
+ // Dispatch order:
+ // 1. If the payload carries a parsed snapshot, trust it. This is
+ // the durable source of truth — it survives controller cache
+ // eviction and controller restarts.
+ // 2. Else fall back to the in-memory svc (fresh Stop→Play within
+ // the prune window).
+ // 3. Else (no payload, no svc) we're out of options — fail
+ // explicitly so the user sees a clear error.
+ if payload != "" && payload != "{}" {
+ var parsed lifecycleK8sPodSnapshot
+ if err := json.Unmarshal([]byte(payload), &parsed); err == nil {
+ if parsed.Controlled != nil {
+ return e.startK8sControlledFromSnapshot(workloadKey, parsed.Controlled)
+ }
+ if parsed.GzippedPodB64 != "" {
+ return e.startK8sNakedPod(workloadKey, svc, payload)
+ }
+ }
+ }
+
+ if svc != nil && (svc.ResourceAttributes["k8s.workload.kind"] != "" && svc.ResourceAttributes["k8s.workload.kind"] != "Pod") {
+ return e.startK8sControlledFromSnapshot(workloadKey, &lifecycleK8sControlledSnapshot{
+ Kind: svc.ResourceAttributes["k8s.workload.kind"],
+ Namespace: svc.ResourceAttributes["k8s.namespace.name"],
+ Name: svc.DeploymentName,
+ })
+ }
+
+ // Naked pod path. svc may be nil here (the pod was deleted on Stop).
+ return e.startK8sNakedPod(workloadKey, svc, payload)
+}
+
+// startK8sControlledFromSnapshot scales a Deployment/STS/DS workload
+// back up using the saved-replicas annotation, given just its
+// (kind, namespace, name). Works whether or not the controller has
+// the workload in its in-memory cache.
+func (e *Engine) startK8sControlledFromSnapshot(workloadKey string, snap *lifecycleK8sControlledSnapshot) error {
+ if snap == nil || snap.Kind == "" || snap.Namespace == "" || snap.Name == "" {
+ return fmt.Errorf("controlled snapshot missing kind/namespace/name (workload %s)", workloadKey)
+ }
+ saved, err := e.container.k8sClient.readSavedReplicas(
+ snap.Namespace, snap.Kind, snap.Name, K8sSavedReplicasAnnotation,
+ )
+ if err != nil {
+ return err
+ }
+ if err := e.container.k8sClient.atomicScaleWithAnnotation(
+ snap.Namespace, snap.Kind, snap.Name, saved, K8sSavedReplicasAnnotation,
+ ); err != nil {
+ return err
+ }
+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)
+ e.logger.Info("k8s workload scaled back up",
+ zap.String("workload_key", workloadKey),
+ zap.String("kind", snap.Kind),
+ zap.String("name", snap.Name),
+ zap.Int("replicas", saved),
+ )
+ return nil
+}
+
+func (e *Engine) restartK8s(svc *openlit.ServiceState) error {
+ if e.container == nil || e.container.k8sClient == nil {
+ return fmt.Errorf("kubernetes client unavailable for lifecycle restart")
+ }
+ namespace := svc.ResourceAttributes["k8s.namespace.name"]
+ workloadKind := svc.ResourceAttributes["k8s.workload.kind"]
+ workloadName := svc.DeploymentName
+ if workloadKind == "" || workloadKind == "Pod" {
+ return e.restartK8sNakedPod(svc)
+ }
+ stamp := time.Now().UTC().Format(time.RFC3339)
+ if err := e.container.k8sClient.bumpRolloutAnnotation(
+ namespace, workloadKind, workloadName, K8sRolloutRestartAnnotation, stamp,
+ ); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)
+ e.logger.Info("k8s workload restart triggered",
+ zap.String("workload_key", svc.WorkloadKey),
+ zap.String("kind", workloadKind),
+ zap.String("name", workloadName),
+ zap.String("stamp", stamp),
+ )
+ return nil
+}
+
+func (e *Engine) stopK8sNakedPod(svc *openlit.ServiceState) (string, error) {
+ namespace := svc.ResourceAttributes["k8s.namespace.name"]
+ podName := svc.ResourceAttributes["k8s.pod.name"]
+ if namespace == "" || podName == "" {
+ return "", fmt.Errorf("naked pod %s missing namespace or pod name", svc.ServiceName)
+ }
+ pod, err := e.container.k8sClient.getPod(namespace, podName)
+ if err != nil {
+ return "", err
+ }
+ stripPodRuntimeFields(pod)
+ blob, err := encodePodSnapshot(pod)
+ if err != nil {
+ return "", err
+ }
+ if err := e.container.k8sClient.deletePod(namespace, podName, 0); err != nil {
+ return "", err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)
+ e.logger.Info("k8s naked pod stopped",
+ zap.String("workload_key", svc.WorkloadKey),
+ zap.String("namespace", namespace),
+ zap.String("pod", podName),
+ )
+ return blob, nil
+}
+
+func (e *Engine) startK8sNakedPod(workloadKey string, svc *openlit.ServiceState, payload string) error {
+ if payload == "" || payload == "{}" {
+ return fmt.Errorf("naked pod start requires a snapshot payload (workload %s)", workloadKey)
+ }
+ pod, err := decodePodSnapshot(payload)
+ if err != nil {
+ return err
+ }
+ namespace := podMetaString(pod, "namespace")
+ podName := podMetaString(pod, "name")
+ if namespace == "" || podName == "" {
+ return fmt.Errorf("naked pod snapshot missing namespace or name (workload %s)", workloadKey)
+ }
+ stripPodRuntimeFields(pod)
+ if err := e.container.k8sClient.createPod(namespace, pod); err != nil {
+ return err
+ }
+ // At this point the scanner will rediscover the pod and re-populate
+ // e.services. We still seed the lifecycle status so the next poll
+ // reports "running" even if it lands before the rediscovery.
+ if svc != nil {
+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)
+ }
+ e.logger.Info("k8s naked pod started",
+ zap.String("workload_key", workloadKey),
+ zap.String("namespace", namespace),
+ zap.String("pod", podName),
+ )
+ return nil
+}
+
+func (e *Engine) restartK8sNakedPod(svc *openlit.ServiceState) error {
+ blob, err := e.stopK8sNakedPod(svc)
+ if err != nil {
+ return err
+ }
+ if err := e.startK8sNakedPod(svc.WorkloadKey, svc, blob); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRunning)
+ return nil
+}
+
+func encodePodSnapshot(pod map[string]any) (string, error) {
+ raw, err := json.Marshal(pod)
+ if err != nil {
+ return "", fmt.Errorf("marshal pod snapshot: %w", err)
+ }
+ var buf bytes.Buffer
+ gz := gzip.NewWriter(&buf)
+ if _, err := gz.Write(raw); err != nil {
+ _ = gz.Close()
+ return "", fmt.Errorf("gzip pod snapshot: %w", err)
+ }
+ if err := gz.Close(); err != nil {
+ return "", fmt.Errorf("gzip close: %w", err)
+ }
+ wrapped := lifecycleK8sPodSnapshot{
+ GzippedPodB64: base64.StdEncoding.EncodeToString(buf.Bytes()),
+ }
+ out, err := json.Marshal(wrapped)
+ if err != nil {
+ return "", err
+ }
+ return string(out), nil
+}
+
+func decodePodSnapshot(blob string) (map[string]any, error) {
+ var wrapped lifecycleK8sPodSnapshot
+ if err := json.Unmarshal([]byte(blob), &wrapped); err != nil {
+ return nil, fmt.Errorf("parse pod snapshot wrapper: %w", err)
+ }
+ if wrapped.GzippedPodB64 == "" {
+ return nil, fmt.Errorf("pod snapshot missing gzipped body")
+ }
+ gzBytes, err := base64.StdEncoding.DecodeString(wrapped.GzippedPodB64)
+ if err != nil {
+ return nil, fmt.Errorf("decode pod snapshot: %w", err)
+ }
+ gz, err := gzip.NewReader(bytes.NewReader(gzBytes))
+ if err != nil {
+ return nil, fmt.Errorf("gunzip pod snapshot: %w", err)
+ }
+ defer gz.Close()
+ var pod map[string]any
+ if err := json.NewDecoder(gz).Decode(&pod); err != nil {
+ return nil, fmt.Errorf("decode pod snapshot body: %w", err)
+ }
+ return pod, nil
+}
+
+func podMetaString(pod map[string]any, key string) string {
+ meta, _ := pod["metadata"].(map[string]any)
+ if meta == nil {
+ return ""
+ }
+ val, _ := meta[key].(string)
+ return val
+}
+
+// ----- Docker -------------------------------------------------------------
+
+func (e *Engine) stopDocker(svc *openlit.ServiceState) error {
+ if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {
+ return fmt.Errorf("docker client unavailable for lifecycle stop")
+ }
+ containerID := svc.ResourceAttributes["container.id"]
+ if containerID == "" {
+ return fmt.Errorf("docker workload %s missing container.id", svc.ServiceName)
+ }
+ if err := e.container.dockerClient.stopContainer(containerID, 10); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)
+ return nil
+}
+
+func (e *Engine) startDocker(svc *openlit.ServiceState) error {
+ if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {
+ return fmt.Errorf("docker client unavailable for lifecycle start")
+ }
+ containerID := svc.ResourceAttributes["container.id"]
+ if containerID == "" {
+ return fmt.Errorf("docker workload %s missing container.id", svc.ServiceName)
+ }
+ if err := e.container.dockerClient.startContainer(containerID); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRunning)
+ return nil
+}
+
+func (e *Engine) restartDocker(svc *openlit.ServiceState) error {
+ if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {
+ return fmt.Errorf("docker client unavailable for lifecycle restart")
+ }
+ containerID := svc.ResourceAttributes["container.id"]
+ if containerID == "" {
+ return fmt.Errorf("docker workload %s missing container.id", svc.ServiceName)
+ }
+ if err := e.container.dockerClient.restartContainer(containerID, 10); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)
+ return nil
+}
+
+// ----- Linux (systemd + bare process) -------------------------------------
+
+func (e *Engine) stopLinux(svc *openlit.ServiceState) (string, error) {
+ if unit := svc.ResourceAttributes["systemd.unit"]; unit != "" {
+ if !linuxSystemdSDKSupported() {
+ return "", fmt.Errorf("systemctl unavailable; cannot stop unit %s", unit)
+ }
+ if err := runSystemctl("stop", unit); err != nil {
+ return "", err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)
+ return "", nil
+ }
+ return e.stopBareProcess(svc)
+}
+
+func (e *Engine) startLinux(workloadKey string, svc *openlit.ServiceState, payload string) error {
+ if svc != nil {
+ if unit := svc.ResourceAttributes["systemd.unit"]; unit != "" {
+ if !linuxSystemdSDKSupported() {
+ return fmt.Errorf("systemctl unavailable; cannot start unit %s", unit)
+ }
+ if err := runSystemctl("start", unit); err != nil {
+ return err
+ }
+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)
+ return nil
+ }
+ }
+ // Bare process — workload may not be in e.services anymore.
+ return e.startBareProcess(workloadKey, payload)
+}
+
+func (e *Engine) restartLinux(svc *openlit.ServiceState) error {
+ if unit := svc.ResourceAttributes["systemd.unit"]; unit != "" {
+ if !linuxSystemdSDKSupported() {
+ return fmt.Errorf("systemctl unavailable; cannot restart unit %s", unit)
+ }
+ if err := runSystemctl("restart", unit); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)
+ return nil
+ }
+ return e.restartBareProcess(svc)
+}
+
+func (e *Engine) stopBareProcess(svc *openlit.ServiceState) (string, error) {
+ if svc.PID <= 0 {
+ return "", fmt.Errorf("bare process %s has no pid; cannot stop", svc.ServiceName)
+ }
+ snap := captureBareProcessSnapshot(e.procRoot, svc.PID)
+ blob, err := json.Marshal(snap)
+ if err != nil {
+ return "", fmt.Errorf("marshal bare-process snapshot: %w", err)
+ }
+ if err := syscall.Kill(svc.PID, syscall.SIGTERM); err != nil {
+ return "", fmt.Errorf("SIGTERM pid %d: %w", svc.PID, err)
+ }
+ if !waitForProcessExit(svc.PID, 5*time.Second) {
+ _ = syscall.Kill(svc.PID, syscall.SIGKILL)
+ waitForProcessExit(svc.PID, 2*time.Second)
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)
+ return string(blob), nil
+}
+
+func (e *Engine) startBareProcess(workloadKey, payload string) error {
+ if payload == "" || payload == "{}" {
+ return fmt.Errorf("bare process start requires a snapshot payload (workload %s)", workloadKey)
+ }
+ var snap lifecycleBareProcessSnapshot
+ if err := json.Unmarshal([]byte(payload), &snap); err != nil {
+ return fmt.Errorf("decode bare-process snapshot: %w", err)
+ }
+ if snap.ExePath == "" || len(snap.Args) == 0 {
+ return fmt.Errorf("bare process snapshot missing exe_path/args (workload %s)", workloadKey)
+ }
+ if err := launchBareProcess(snap); err != nil {
+ return err
+ }
+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)
+ e.logger.Info("bare process re-launched",
+ zap.String("workload_key", workloadKey),
+ zap.String("exe", snap.ExePath),
+ zap.Strings("args", snap.Args),
+ )
+ return nil
+}
+
+func (e *Engine) restartBareProcess(svc *openlit.ServiceState) error {
+ if svc.PID <= 0 {
+ return fmt.Errorf("bare process %s has no pid; cannot restart", svc.ServiceName)
+ }
+ envFull := readEnviron(e.procRoot, svc.PID)
+ if envFull == nil {
+ envFull = make(map[string]string)
+ }
+ if _, err := restartProcess(e.procRoot, svc.PID, envFull); err != nil {
+ return err
+ }
+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)
+ return nil
+}
+
+// launchBareProcess re-execs a previously-stopped bare process from a
+// captured snapshot. We deliberately mirror restartProcess's setsid setup so
+// the new process detaches from the controller (otherwise it would die when
+// the controller exits) but use os/exec rather than fork directly because the
+// controller runs single-threaded in this code path.
+//
+// Trust model for the dynamic argv: the snapshot is captured at Stop
+// time from /proc/<pid>/cmdline (trusted, locally read) and travels
+// through the dashboard's ClickHouse `desired_states_v2.config`
+// column. The argv is therefore only as trustworthy as ClickHouse
+// write access. Two mitigations:
+//
+// 1. The controller already runs as a privileged process that can
+// execute container ops, pip installs, kubectl apply, etc. — so a
+// ClickHouse-write attacker has equivalent paths to RCE via the
+// existing action queue. This call site does not widen the blast
+// radius beyond the controller's existing trust boundary.
+// 2. We still validate the argv defensively below to fail closed on
+// obviously-bogus payloads (empty argv, NUL bytes, empty argv[0])
+// rather than blindly forwarding to execve.
+//
+// Crucially, `exec.Command` does NOT spawn a shell — Go's os/exec
+// invokes execve(2) directly with the argv slice, so shell
+// metacharacters in any argv element are not interpreted.
+func launchBareProcess(snap lifecycleBareProcessSnapshot) error {
+ if err := validateBareProcessArgs(snap.Args); err != nil {
+ return err
+ }
+ // nolint:gosec // G204: argv is taken from a controller-captured
+ // snapshot of /proc/<pid>/cmdline; see the trust-model comment
+ // above for why a dynamic argv here is acceptable. execve is
+ // shell-free, and validateBareProcessArgs rejects malformed
+ // payloads.
+ cmd := exec.Command(snap.Args[0], snap.Args[1:]...)
+ cmd.Dir = snap.Cwd
+ envSlice := make([]string, 0, len(snap.Env))
+ for k, v := range snap.Env {
+ envSlice = append(envSlice, k+"="+v)
+ }
+ cmd.Env = envSlice
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
+ cmd.Stdout = nil
+ cmd.Stderr = nil
+ if err := cmd.Start(); err != nil {
+ return fmt.Errorf("start bare process: %w", err)
+ }
+ go func() { _ = cmd.Wait() }()
+ return nil
+}
+
+// validateBareProcessArgs guards the launchBareProcess execve call
+// against obviously-malformed payloads. The rules are deliberately
+// minimal — we are not trying to defend against an attacker who
+// already has ClickHouse write access (see the trust-model comment on
+// launchBareProcess), only against:
+//
+// - empty argv (would panic on Args[0])
+// - empty / NUL-containing argv[0] (execve rejects NULs and they
+// have caused surprising path-resolution behavior historically)
+// - NUL bytes anywhere in argv (same reasoning)
+//
+// We intentionally do NOT reject relative argv[0]: the original
+// process may legitimately have been launched as `./my-script` and
+// changing that on restart would break workload_key identity (which
+// folds argv into its fingerprint).
+func validateBareProcessArgs(args []string) error {
+ if len(args) == 0 {
+ return fmt.Errorf("bare process snapshot has empty argv")
+ }
+ if args[0] == "" {
+ return fmt.Errorf("bare process snapshot has empty argv[0]")
+ }
+ for i, a := range args {
+ if strings.ContainsRune(a, 0) {
+ return fmt.Errorf("bare process snapshot argv[%d] contains NUL byte", i)
+ }
+ }
+ return nil
+}
+
+// captureBareProcessSnapshot reads /proc/<pid>/* to build the minimum
+// payload Start needs to re-exec the workload identically. The env is
+// restricted to an allowlist of prefixes (PATH, OPENLIT_*, OTEL_*, etc.) so
+// the controller never exfiltrates arbitrary user env (which may contain
+// secrets) through the dashboard.
+func captureBareProcessSnapshot(procRoot string, pid int) lifecycleBareProcessSnapshot {
+ args := readCmdlineArgs(procRoot, pid)
+ cwd := readCwd(procRoot, pid)
+ if cwd == "" {
+ cwd = "/"
+ }
+ env := readEnviron(procRoot, pid)
+ filtered := make(map[string]string)
+ for k, v := range env {
+ for _, prefix := range envAllowlistPrefixes {
+ if k == prefix || strings.HasPrefix(k, prefix) {
+ filtered[k] = v
+ break
+ }
+ }
+ }
+ exePath := ""
+ if len(args) > 0 {
+ exePath = args[0]
+ }
+ return lifecycleBareProcessSnapshot{
+ ExePath: exePath,
+ Args: args,
+ Cwd: cwd,
+ Env: filtered,
+ }
+}
diff --git a/openlit-controller/internal/engine/lifecycle_test.go b/openlit-controller/internal/engine/lifecycle_test.go
new file mode 100644
index 00000000..bf726819
--- /dev/null
+++ b/openlit-controller/internal/engine/lifecycle_test.go
@@ -0,0 +1,173 @@
+// Tests the JSON wire format of lifecycleK8sPodSnapshot — the blob the
+// controller returns from Stop, the dashboard persists into
+// desired_states_v2.config, and the controller decodes on the next
+// Start. Both fields are `omitempty` so each snapshot kind serialises
+// to a different shape; if that invariant breaks, the dashboard's
+// requiresSnapshot check in lifecycle.ts and the controller's
+// startK8s dispatch in this file silently misbehave.
+
+package engine
+
+import (
+ "encoding/json"
+ "strings"
+ "testing"
+)
+
+func TestLifecycleK8sSnapshot_ControlledJSON(t *testing.T) {
+ snap := lifecycleK8sPodSnapshot{
+ Controlled: &lifecycleK8sControlledSnapshot{
+ Kind: "Deployment",
+ Namespace: "default",
+ Name: "demo-openai-app",
+ ContainerName: "demo-openai-app",
+ },
+ }
+ out, err := json.Marshal(snap)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ got := string(out)
+ // The dashboard's requiresSnapshot path treats "" and "{}" as
+ // empty. We want a non-empty body for controlled workloads so the
+ // poll route persists it.
+ if got == "{}" || got == "" {
+ t.Fatalf("controlled snapshot serialised as empty: %q", got)
+ }
+ // Naked-pod field must be absent for a controlled snapshot so the
+ // startK8s dispatcher does not race the wrong branch.
+ if strings.Contains(got, "gzipped_pod_b64") {
+ t.Fatalf("controlled snapshot leaked gzipped_pod_b64: %q", got)
+ }
+ if !strings.Contains(got, "\"controlled\"") {
+ t.Fatalf("expected controlled key in JSON, got %q", got)
+ }
+
+ // Round-trip must preserve every field.
+ var back lifecycleK8sPodSnapshot
+ if err := json.Unmarshal(out, &back); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if back.Controlled == nil {
+ t.Fatalf("round-trip lost controlled struct")
+ }
+ if back.Controlled.Kind != "Deployment" ||
+ back.Controlled.Namespace != "default" ||
+ back.Controlled.Name != "demo-openai-app" ||
+ back.Controlled.ContainerName != "demo-openai-app" {
+ t.Fatalf("round-trip mangled fields: %+v", back.Controlled)
+ }
+ if back.GzippedPodB64 != "" {
+ t.Fatalf("round-trip resurrected gzipped_pod_b64")
+ }
+}
+
+func TestLifecycleK8sSnapshot_NakedPodJSON(t *testing.T) {
+ snap := lifecycleK8sPodSnapshot{
+ GzippedPodB64: "H4sIAAAA",
+ }
+ out, err := json.Marshal(snap)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ got := string(out)
+ if strings.Contains(got, "\"controlled\"") {
+ t.Fatalf("naked-pod snapshot leaked controlled key: %q", got)
+ }
+ if !strings.Contains(got, "gzipped_pod_b64") {
+ t.Fatalf("missing gzipped_pod_b64: %q", got)
+ }
+}
+
+func TestValidateBareProcessArgs(t *testing.T) {
+ tests := []struct {
+ name string
+ args []string
+ wantErr bool
+ }{
+ {
+ name: "happy path: absolute path",
+ args: []string{"/usr/bin/python3", "myapp.py"},
+ },
+ {
+ name: "happy path: PATH-resolved binary",
+ args: []string{"python3", "-m", "myapp"},
+ },
+ {
+ name: "happy path: relative argv[0]",
+ args: []string{"./run.sh", "--flag"},
+ },
+ {
+ name: "empty argv",
+ args: nil,
+ wantErr: true,
+ },
+ {
+ name: "empty argv[0]",
+ args: []string{"", "myapp.py"},
+ wantErr: true,
+ },
+ {
+ name: "NUL in argv[0]",
+ args: []string{"python\x003", "myapp.py"},
+ wantErr: true,
+ },
+ {
+ name: "NUL in middle argv",
+ args: []string{"python3", "my\x00app.py"},
+ wantErr: true,
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ err := validateBareProcessArgs(tc.args)
+ if (err != nil) != tc.wantErr {
+ t.Fatalf("validateBareProcessArgs(%v): err=%v, wantErr=%v", tc.args, err, tc.wantErr)
+ }
+ })
+ }
+}
+
+func TestLifecycleK8sSnapshot_DispatchDiscriminator(t *testing.T) {
+ // Mirrors the dispatch in startK8s: a controlled blob is detected
+ // by parsed.Controlled != nil, a naked pod blob by
+ // parsed.GzippedPodB64 != "". Each input goes down exactly one
+ // branch.
+ tests := []struct {
+ name string
+ payload string
+ wantControlled bool
+ wantNakedPod bool
+ }{
+ {
+ name: "controlled",
+ payload: `{"controlled":{"kind":"Deployment","namespace":"x","name":"y"}}`,
+ wantControlled: true,
+ },
+ {
+ name: "naked_pod",
+ payload: `{"gzipped_pod_b64":"abc"}`,
+ wantNakedPod: true,
+ },
+ {
+ name: "empty",
+ payload: `{}`,
+ },
+ }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ var parsed lifecycleK8sPodSnapshot
+ if err := json.Unmarshal([]byte(tc.payload), &parsed); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ gotControlled := parsed.Controlled != nil
+ gotNakedPod := parsed.GzippedPodB64 != ""
+ if gotControlled != tc.wantControlled {
+ t.Errorf("controlled: got %v want %v", gotControlled, tc.wantControlled)
+ }
+ if gotNakedPod != tc.wantNakedPod {
+ t.Errorf("naked pod: got %v want %v", gotNakedPod, tc.wantNakedPod)
+ }
+ })
+ }
+}
diff --git a/openlit-controller/internal/openlit/types.go b/openlit-controller/internal/openlit/types.go
index 67e2b30d..3e32b072 100644
--- a/openlit-controller/internal/openlit/types.go
+++ b/openlit-controller/internal/openlit/types.go
@@ -15,6 +15,9 @@ const (
ActionUninstrument = "uninstrument"
ActionEnablePythonSDK = "enable_python_sdk"
ActionDisablePythonSDK = "disable_python_sdk"
+ ActionStartWorkload = "start_workload"
+ ActionStopWorkload = "stop_workload"
+ ActionRestartWorkload = "restart_workload"
)
// PollRequest is sent by the controller to OpenLIT on each poll cycle.
@@ -66,6 +69,13 @@ type ActionResult struct {
ActionID string `json:"action_id"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
+ // Snapshot carries action-specific state back to the dashboard. For
+ // stop_workload it holds a JSON blob describing how to recreate the
+ // workload later (pod spec for K8s naked pods, exe+args+cwd+env subset
+ // for bare Linux processes). The dashboard persists this into the
+ // matching openlit_controller_desired_states_v2.config row so a later
+ // start_workload action can hand it back to the controller.
+ Snapshot string `json:"snapshot,omitempty"`
}
type DiscoveredService struct {
@@ -83,8 +93,12 @@ type DiscoveredService struct {
AgentObservabilitySource string `json:"agent_observability_source,omitempty"`
ObservabilityConflict string `json:"observability_conflict,omitempty"`
ObservabilityReason string `json:"observability_reason,omitempty"`
- FirstSeen string `json:"first_seen,omitempty"`
- ResourceAttributes map[string]string `json:"resource_attributes,omitempty"`
+ // LifecycleStatus is the controller's last-known lifecycle state for the
+ // workload ("running"/"stopped"/"restarting"). Echoed via
+ // resource_attributes["openlit.lifecycle.status"] for the dashboard rollup.
+ LifecycleStatus string `json:"lifecycle_status,omitempty"`
+ FirstSeen string `json:"first_seen,omitempty"`
+ ResourceAttributes map[string]string `json:"resource_attributes,omitempty"`
}
type ServiceState struct {
@@ -101,6 +115,7 @@ type ServiceState struct {
AgentObservabilitySource string `json:"agent_observability_source,omitempty"`
ObservabilityConflict string `json:"observability_conflict,omitempty"`
ObservabilityReason string `json:"observability_reason,omitempty"`
+ LifecycleStatus string `json:"lifecycle_status,omitempty"`
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen"`
PID int `json:"pid,omitempty"`
diff --git a/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts b/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts
index e4fb6f1d..b6cec169 100644
--- a/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts
+++ b/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts
@@ -25,6 +25,7 @@ function makeAgent(overrides: Partial<UnifiedAgent> = {}): UnifiedAgent {
service_name: "svc",
environment: "prod",
cluster_id: "default",
+ workload_key: "docker:svc",
source: "controller",
controller_service_id: "ctrl-1",
controller_instance_id: "inst-1",
@@ -42,6 +43,8 @@ function makeAgent(overrides: Partial<UnifiedAgent> = {}): UnifiedAgent {
desired_instrumentation_status: "none",
agent_observability_status: "",
desired_agent_status: "none",
+ lifecycle_status: "running",
+ desired_lifecycle_status: "unknown",
pending_action: null,
pending_action_status: null,
first_seen: "2026-05-12 00:00:00",
diff --git a/src/client/src/app/(playground)/agents/[agentKey]/page.tsx b/src/client/src/app/(playground)/agents/[agentKey]/page.tsx
index d02cac52..ce0a003c 100644
--- a/src/client/src/app/(playground)/agents/[agentKey]/page.tsx
+++ b/src/client/src/app/(playground)/agents/[agentKey]/page.tsx
@@ -26,6 +26,7 @@ import { getObservabilityView } from "@/lib/platform/agents/observability-view";
import AgentHeader from "@/components/(playground)/agents/agent-header";
import AgentScopeProvider from "@/components/(playground)/agents/agent-scope-provider";
import AgentOverviewTab from "@/components/(playground)/agents/agent-overview-tab";
+import LifecycleActions from "@/components/(playground)/agents/lifecycle-actions";
import VersionTimeline, {
VersionChooser,
} from "@/components/(playground)/agents/version-timeline";
@@ -186,16 +187,18 @@ export default function AgentDetailPage() {
// fetch while the table advances within seconds.
const llmIntent = useAgentIntent(agentKey, "llm");
const agentIntent = useAgentIntent(agentKey, "agent");
+ const lifecycleIntent = useAgentIntent(agentKey, "lifecycle");
const detailTransitioning = useMemo(() => {
if (!agent) return false;
if (
getObservabilityView(agent, "llm", llmIntent).transitioning ||
- getObservabilityView(agent, "agent", agentIntent).transitioning
+ getObservabilityView(agent, "agent", agentIntent).transitioning ||
+ getObservabilityView(agent, "lifecycle", lifecycleIntent).transitioning
) {
return true;
}
return false;
- }, [agent, llmIntent, agentIntent]);
+ }, [agent, llmIntent, agentIntent, lifecycleIntent]);
const detailPollInFlightRef = useRef(false);
@@ -349,12 +352,19 @@ export default function AgentDetailPage() {
agent={agent}
onRefresh={fetchAgent}
rightSlot={
- <VersionChooser
- versions={versions}
- selectedHash={urlVersionHash}
- onSelectVersion={handleSelectVersion}
- onOpenAllVersions={() => setVersionsOpen(true)}
- />
+ <div className="flex items-center gap-2">
+ <LifecycleActions
+ agent={agent}
+ onRefresh={fetchAgent}
+ variant="header"
+ />
+ <VersionChooser
+ versions={versions}
+ selectedHash={urlVersionHash}
+ onSelectVersion={handleSelectVersion}
+ onOpenAllVersions={() => setVersionsOpen(true)}
+ />
+ </div>
}
/>
diff --git a/src/client/src/app/(playground)/agents/page.tsx b/src/client/src/app/(playground)/agents/page.tsx
index fe6d0dee..6d451c47 100644
--- a/src/client/src/app/(playground)/agents/page.tsx
+++ b/src/client/src/app/(playground)/agents/page.tsx
@@ -202,7 +202,17 @@ export default function AgentsPage() {
"agent",
null
).transitioning;
- if (podsPending > 0 || llmTransitioning || agentTransitioning) {
+ const lifecycleTransitioning = getObservabilityView(
+ service,
+ "lifecycle",
+ null
+ ).transitioning;
+ if (
+ podsPending > 0 ||
+ llmTransitioning ||
+ agentTransitioning ||
+ lifecycleTransitioning
+ ) {
keys.add(service.agent_key);
}
}
@@ -270,7 +280,14 @@ export default function AgentsPage() {
if (!intent) continue;
const serverView = getObservabilityView(agent, feature, null);
if (serverView.transitioning) continue;
- const intentTargetEnabled = intent.direction === "enabling";
+ // For lifecycle, `restarting` collapses back to `running`
+ // once the controller is done. We treat it like `starting`
+ // for convergence purposes -- the workload is up, so the
+ // intent has resolved.
+ const intentTargetEnabled =
+ intent.direction === "enabling" ||
+ intent.direction === "starting" ||
+ intent.direction === "restarting";
if (serverView.enabled === intentTargetEnabled) {
clearIntent(agentKey, feature);
}
diff --git a/src/client/src/app/(playground)/agents/service-table.tsx b/src/client/src/app/(playground)/agents/service-table.tsx
index 0305997a..efc7d8b4 100644
--- a/src/client/src/app/(playground)/agents/service-table.tsx
+++ b/src/client/src/app/(playground)/agents/service-table.tsx
@@ -28,6 +28,7 @@ import {
getSetAgentIntent,
useAgentIntent,
} from "@/selectors/agents-instrumentation";
+import LifecycleActions from "@/components/(playground)/agents/lifecycle-actions";
interface ServiceTableProps {
services: UnifiedAgent[];
@@ -51,6 +52,7 @@ type ServiceColumnKey =
| "service"
| "system"
| "providers"
+ | "lifecycle"
| "aiObservability"
| "agentObservability"
| "lastSeen";
@@ -59,10 +61,64 @@ function StaticDash() {
return <span className="text-xs text-stone-400 dark:text-stone-500">—</span>;
}
-function directionLabel(direction: "enabling" | "disabling" | null): string {
- return direction === "disabling"
- ? getMessage().AGENTS_SERVICE_ACTION_DISABLING
- : getMessage().AGENTS_SERVICE_ACTION_ENABLING;
+/**
+ * Returns a tooltip string describing *why* observability changes
+ * should be blocked for this agent, or `null` when the agent is in a
+ * state where toggling LLM / Agent observability is safe.
+ *
+ * We gate on the lifecycle observability-view so the same precedence
+ * (optimistic intent → pending action → desired mismatch → steady)
+ * decides what counts as "running" — that way clicking Stop in one
+ * row instantly disables its o11y toggles via the optimistic intent,
+ * without waiting for the controller round-trip.
+ *
+ * Controllers can mutate processes only while they exist: Docker /
+ * Linux modes literally have nothing to attach to once the workload
+ * is stopped, and a K8s instrument racing a Stop can leave a
+ * half-applied state. Gating closes that race entirely.
+ */
+function useObservabilityBlock(service: EnrichedAgent): string | null {
+ const lifecycleIntent = useAgentIntent(service.agent_key, "lifecycle");
+ if (service.source === "sdk" || !service.controller_service_id) {
+ return null;
+ }
+ const lifecycleView = getObservabilityView(
+ service,
+ "lifecycle",
+ lifecycleIntent
+ );
+ if (lifecycleView.transitioning) {
+ return getMessage().AGENTS_OBSERVABILITY_DISABLED_TRANSITIONING;
+ }
+ if (!lifecycleView.enabled) {
+ return getMessage().AGENTS_OBSERVABILITY_DISABLED_NOT_RUNNING;
+ }
+ return null;
+}
+
+function directionLabel(
+ direction:
+ | "enabling"
+ | "disabling"
+ | "starting"
+ | "stopping"
+ | "restarting"
+ | null
+): string {
+ switch (direction) {
+ case "disabling":
+ return getMessage().AGENTS_SERVICE_ACTION_DISABLING;
+ case "enabling":
+ return getMessage().AGENTS_SERVICE_ACTION_ENABLING;
+ case "starting":
+ return getMessage().AGENTS_LIFECYCLE_STARTING;
+ case "stopping":
+ return getMessage().AGENTS_LIFECYCLE_STOPPING;
+ case "restarting":
+ return getMessage().AGENTS_LIFECYCLE_RESTARTING;
+ default:
+ return getMessage().AGENTS_SERVICE_ACTION_WORKING;
+ }
}
function AIObservabilityCell({
@@ -76,6 +132,7 @@ function AIObservabilityCell({
const setIntent = useRootStore(getSetAgentIntent);
const clearIntent = useRootStore(getClearAgentIntent);
const intent = useAgentIntent(service.agent_key, "llm");
+ const blockReason = useObservabilityBlock(service);
const controllerServiceId = service.controller_service_id;
if (!controllerServiceId) {
@@ -84,6 +141,28 @@ function AIObservabilityCell({
const view = getObservabilityView(service, "llm", intent);
+ // Stopped or transitioning lifecycle → render a disabled button that
+ // still reflects current state, with a tooltip explaining why it is
+ // inert. We keep the Enable/Disable label honest so a user who comes
+ // back later understands what *will* happen once the agent is back.
+ if (blockReason && !view.transitioning) {
+ const label = view.enabled
+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE
+ : getMessage().AGENTS_SERVICE_ACTION_ENABLE;
+ return (
+ <button
+ type="button"
+ disabled
+ title={blockReason}
+ aria-label={blockReason}
+ onClick={(e) => e.stopPropagation()}
+ className="inline-flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-400 dark:text-stone-500 bg-stone-50 dark:bg-stone-900 cursor-not-allowed"
+ >
+ {label}
+ </button>
+ );
+ }
+
const handleAction = async (e: React.MouseEvent) => {
e.stopPropagation();
if (view.transitioning) return;
@@ -158,6 +237,7 @@ function AgentObservabilityCell({
const setIntent = useRootStore(getSetAgentIntent);
const clearIntent = useRootStore(getClearAgentIntent);
const intent = useAgentIntent(service.agent_key, "agent");
+ const blockReason = useObservabilityBlock(service);
const controllerServiceId = service.controller_service_id;
if (!controllerServiceId) {
@@ -170,6 +250,36 @@ function AgentObservabilityCell({
// flag the row visually. `view.isManual` is the canonical semantic read.
const isManual = view.isManual;
+ // Lifecycle gating: same logic as the LLM cell, but we still want to
+ // surface the `manual` badge so users can see the SDK status; we just
+ // disable the Disable button until the agent is running.
+ if (blockReason && !view.transitioning) {
+ const label = isManual
+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE
+ : view.enabled
+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE
+ : getMessage().AGENTS_SERVICE_ACTION_ENABLE;
+ return (
+ <div className="flex items-center gap-2">
+ {isManual && (
+ <span className="inline-flex items-center px-2 py-0.5 text-[10px] font-semibold rounded-full bg-amber-50 text-amber-700 dark:bg-amber-900/30 dark:text-amber-400 border border-amber-200 dark:border-amber-800">
+ {getMessage().AGENTS_SERVICE_MANUAL_BADGE}
+ </span>
+ )}
+ <button
+ type="button"
+ disabled
+ title={blockReason}
+ aria-label={blockReason}
+ onClick={(e) => e.stopPropagation()}
+ className="inline-flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-400 dark:text-stone-500 bg-stone-50 dark:bg-stone-900 cursor-not-allowed"
+ >
+ {label}
+ </button>
+ </div>
+ );
+ }
+
const handleAction = async (e: React.MouseEvent) => {
e.stopPropagation();
if (view.transitioning) return;
@@ -258,6 +368,35 @@ function AgentObservabilityCell({
);
}
+/**
+ * ActionsCell — Docker-Desktop-style action cluster pinned to the
+ * right of the row. SDK-only agents (no controller managing them)
+ * render an em-dash so the right edge stays visually clean instead of
+ * showing nothing at all; controller-managed rows render the full
+ * Play / Stop / Restart pill from LifecycleActions.
+ */
+function ActionsCell({
+ service,
+ onRefresh,
+}: {
+ service: EnrichedAgent;
+ onRefresh: () => void;
+}) {
+ return (
+ <div className="flex justify-end">
+ {service.source === "sdk" || !service.controller_service_id ? (
+ <StaticDash />
+ ) : (
+ <LifecycleActions
+ agent={service}
+ onRefresh={onRefresh}
+ variant="row"
+ />
+ )}
+ </div>
+ );
+}
+
const columns: Columns<ServiceColumnKey, EnrichedAgent> = {
service: {
header: () => getMessage().AGENTS_COLUMN_SERVICE,
@@ -336,6 +475,20 @@ const columns: Columns<ServiceColumnKey, EnrichedAgent> = {
</span>
),
},
+ lifecycle: {
+ header: () => (
+ <div className="flex justify-end">
+ {getMessage().AGENTS_COLUMN_ACTIONS}
+ </div>
+ ),
+ cell: ({ row, extraFunctions }) => (
+ <ActionsCell
+ service={row}
+ onRefresh={extraFunctions.onRefresh}
+ />
+ ),
+ enableHiding: false,
+ },
aiObservability: {
header: () => getMessage().AGENTS_COLUMN_LLM_OBSERVABILITY,
cell: ({ row, extraFunctions }) => (
@@ -356,6 +509,10 @@ const columns: Columns<ServiceColumnKey, EnrichedAgent> = {
},
};
+// Column display order is driven by this record's key order (see
+// DataTable). `lifecycle` is intentionally last so the action pill
+// pins to the right of the row, matching Docker Desktop's container
+// list layout.
const VISIBILITY_COLUMNS: Record<ServiceColumnKey, boolean> = {
service: true,
system: true,
@@ -363,6 +520,7 @@ const VISIBILITY_COLUMNS: Record<ServiceColumnKey, boolean> = {
lastSeen: true,
aiObservability: true,
agentObservability: true,
+ lifecycle: true,
};
export default function ServiceTable({
diff --git a/src/client/src/app/api/controller/catalog/[id]/restart/route.ts b/src/client/src/app/api/controller/catalog/[id]/restart/route.ts
new file mode 100644
index 00000000..539940cc
--- /dev/null
+++ b/src/client/src/app/api/controller/catalog/[id]/restart/route.ts
@@ -0,0 +1,10 @@
+import { getFeatureHandler } from "@/lib/platform/controller/features";
+
+export async function POST(
+ request: Request,
+ { params }: { params: Promise<{ id: string }> }
+) {
+ const { id } = await params;
+ const handler = getFeatureHandler("lifecycle")!;
+ return handler.applyOperation(id, "restart", {});
+}
diff --git a/src/client/src/app/api/controller/catalog/[id]/start/route.ts b/src/client/src/app/api/controller/catalog/[id]/start/route.ts
new file mode 100644
index 00000000..c57d6287
--- /dev/null
+++ b/src/client/src/app/api/controller/catalog/[id]/start/route.ts
@@ -0,0 +1,10 @@
+import { getFeatureHandler } from "@/lib/platform/controller/features";
+
+export async function POST(
+ request: Request,
+ { params }: { params: Promise<{ id: string }> }
+) {
+ const { id } = await params;
+ const handler = getFeatureHandler("lifecycle")!;
+ return handler.applyOperation(id, "start", {});
+}
diff --git a/src/client/src/app/api/controller/catalog/[id]/stop/route.ts b/src/client/src/app/api/controller/catalog/[id]/stop/route.ts
new file mode 100644
index 00000000..5eebe4f7
--- /dev/null
+++ b/src/client/src/app/api/controller/catalog/[id]/stop/route.ts
@@ -0,0 +1,10 @@
+import { getFeatureHandler } from "@/lib/platform/controller/features";
+
+export async function POST(
+ request: Request,
+ { params }: { params: Promise<{ id: string }> }
+) {
+ const { id } = await params;
+ const handler = getFeatureHandler("lifecycle")!;
+ return handler.applyOperation(id, "stop", {});
+}
diff --git a/src/client/src/app/api/controller/poll/route.ts b/src/client/src/app/api/controller/poll/route.ts
index 228be258..2201ce47 100644
--- a/src/client/src/app/api/controller/poll/route.ts
+++ b/src/client/src/app/api/controller/poll/route.ts
@@ -2,14 +2,18 @@ import {
upsertControllerInstance,
upsertServices,
getControllerConfig,
+ getControllerInstanceById,
getPendingActions,
markActionsAcknowledged,
completeAction,
getConfigHash,
queueAction,
+ getActionsByIds,
getFeatureDesiredStates,
getEnvironmentFeatureConfigs,
+ updateFeatureDesiredState,
} from "@/lib/platform/controller";
+import { KNOWN_ACTIONS } from "@/types/controller";
import { getAllFeatureHandlers } from "@/lib/platform/controller/features";
import type { ReportedService } from "@/lib/platform/controller/features";
import { getFirstDBConfig } from "@/lib/db-config";
@@ -193,14 +197,160 @@ async function phaseUpsertServices(
// --- Phase 4: Process action results ---
+/**
+ * Persist any lifecycle-Stop snapshots reported by the controller into
+ * the matching desired_states_v2.config row, keyed by
+ * (workload_key, cluster_id, feature='lifecycle'). Without this step
+ * the saved-state blob the controller computed at Stop time (pod spec /
+ * exe+cmdline+cwd+env for bare processes) would be lost, and a later
+ * Play would have nothing to recreate the workload with.
+ *
+ * Only completed stop_workload actions with a non-empty snapshot are
+ * persisted. Other action types are ignored here -- the standard
+ * completeAction path still runs for them.
+ *
+ * Returns the set of action_ids whose snapshot write failed, so the
+ * caller can defer their completeAction and let the controller retry
+ * Stop on the next poll. Controller-side stops are idempotent (the pod
+ * is already gone / process already dead) so a retry is safe; the
+ * value is that we get another chance to capture the snapshot. The
+ * snapshot itself was computed pre-destructively so the controller has
+ * it in memory only for the lifetime of this RPC, not on retry -- but
+ * the dashboard-side write being retried gives the most common
+ * failure (transient ClickHouse) a chance to recover without losing
+ * the blob.
+ */
+async function persistLifecycleStopSnapshots(
+ actionResults: any[],
+ instanceId: string,
+ dbId: string
+): Promise<Set<string>> {
+ const failed = new Set<string>();
+ const candidates = actionResults.filter(
+ (ar) =>
+ ar?.action_id &&
+ ar.status === "completed" &&
+ typeof ar.snapshot === "string" &&
+ ar.snapshot.length > 0 &&
+ ar.snapshot !== "{}"
+ );
+ if (candidates.length === 0) return failed;
+
+ const actionIds = candidates.map((ar) => ar.action_id as string);
+ const [actionsRes, instanceRes] = await Promise.all([
+ getActionsByIds(actionIds, instanceId, dbId),
+ getControllerInstanceById(instanceId, dbId),
+ ]);
+
+ if (actionsRes.err || !actionsRes.data) {
+ console.error(
+ "controller poll: getActionsByIds failed; deferring snapshot persistence:",
+ sanitizeLogValue(actionsRes.err || "no data")
+ );
+ // Defer ALL candidate completions so the controller will retry
+ // Stop on the next poll, giving us another shot at the lookup.
+ for (const ar of candidates) failed.add(ar.action_id as string);
+ return failed;
+ }
+ // Symmetric: an instance lookup failure must defer the same way the
+ // actions lookup does. Falling back to cluster_id="default" would
+ // silently write the snapshot into the wrong (workload_key,
+ // cluster_id) row in multi-cluster setups, so a later Start in the
+ // real cluster would see no snapshot and 409. Defer + retry instead.
+ if (instanceRes.err || !instanceRes.data?.[0]?.cluster_id) {
+ // instanceId rides in on the controller's poll request body, so
+ // it must be sanitized before being logged. We use
+ // `JSON.stringify` rather than the project's `sanitizeLogValue`
+ // helper or an inline regex replace because CodeQL's
+ // `js/log-injection` query has a built-in `JsonStringifySanitizer`
+ // barrier — using exactly the pattern the analyzer expects keeps
+ // future alerts from re-firing on this site. `JSON.stringify`
+ // escapes CR/LF (`\\n`, `\\r`) so log forgery via embedded
+ // newlines is impossible.
+ console.error(
+ "controller poll: getControllerInstanceById failed; deferring snapshot persistence:",
+ "instance",
+ JSON.stringify(instanceId),
+ ":",
+ sanitizeLogValue(instanceRes.err || "no data")
+ );
+ for (const ar of candidates) failed.add(ar.action_id as string);
+ return failed;
+ }
+ const clusterId = instanceRes.data[0].cluster_id;
+ const actionById = new Map(
+ actionsRes.data.map((a) => [a.id, a] as const)
+ );
+
+ // Run the writes in parallel — they target independent
+ // (workload_key, cluster_id) rows so ReplacingMergeTree merges are
+ // non-conflicting, and at typical scale we can have dozens of Stops
+ // in a single poll cycle (e.g. operator stopping a whole namespace).
+ await Promise.all(
+ candidates.map(async (ar) => {
+ const action = actionById.get(ar.action_id);
+ if (!action) return;
+ if (action.action_type !== KNOWN_ACTIONS.STOP_WORKLOAD) return;
+
+ const writeRes = await updateFeatureDesiredState(
+ action.service_key,
+ clusterId,
+ "lifecycle",
+ "stopped",
+ ar.snapshot as string,
+ dbId
+ );
+ if (writeRes?.err) {
+ // Same rationale as the instance-lookup branch above:
+ // ar.action_id and action.service_key both originate in
+ // the controller's poll request body, so we route them
+ // through `JSON.stringify` — CodeQL's
+ // `JsonStringifySanitizer` barrier. `writeRes.err` is a
+ // DB error string (not user input), so passing it
+ // through `sanitizeLogValue` is sufficient.
+ console.error(
+ "controller poll: persist lifecycle snapshot failed for action",
+ JSON.stringify(ar.action_id),
+ "workload",
+ JSON.stringify(action.service_key),
+ ":",
+ sanitizeLogValue(writeRes.err)
+ );
+ failed.add(ar.action_id);
+ }
+ })
+ );
+ return failed;
+}
+
async function phaseProcessActionResults(
actionResults: any[] | undefined,
instanceId: string,
dbId: string
) {
if (!Array.isArray(actionResults)) return;
- for (const ar of actionResults) {
- if (ar.action_id && ar.status) {
+
+ // Persist Stop snapshots BEFORE marking the action complete. If we
+ // reordered these, a controller crash between completeAction and the
+ // snapshot write would leave us with a successful stop_workload that
+ // has no recovery blob, breaking Play for naked pods / bare
+ // processes. Writing the snapshot first means completeAction is the
+ // commit point, and a write failure defers completeAction so the
+ // controller retries Stop on the next poll.
+ const deferred = await persistLifecycleStopSnapshots(
+ actionResults,
+ instanceId,
+ dbId
+ );
+
+ // completeAction calls run in parallel as well — each row in
+ // openlit_controller_actions is identified by a deterministic id so
+ // re-inserts deduplicate via ReplacingMergeTree. Parallelizing here
+ // keeps the poll round-trip O(slowest write) rather than O(N).
+ await Promise.all(
+ actionResults.map(async (ar) => {
+ if (!ar.action_id || !ar.status) return;
+ if (deferred.has(ar.action_id)) return;
const completeResult = await completeAction(
ar.action_id,
instanceId,
@@ -211,8 +361,8 @@ async function phaseProcessActionResults(
if (completeResult?.err) {
console.error("controller poll: completeAction failed");
}
- }
- }
+ })
+ );
}
// --- Phase 5: Generic feature reconciliation ---
diff --git a/src/client/src/clickhouse/migrations/create-controller-migration.ts b/src/client/src/clickhouse/migrations/create-controller-migration.ts
index 0039731c..83ed1cca 100644
--- a/src/client/src/clickhouse/migrations/create-controller-migration.ts
+++ b/src/client/src/clickhouse/migrations/create-controller-migration.ts
@@ -92,6 +92,7 @@ export default async function CreateControllerMigration(
updated_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(updated_at)
ORDER BY (workload_key, cluster_id, feature)
+ TTL updated_at + INTERVAL 90 DAY
SETTINGS index_granularity = 8192
`,
`
diff --git a/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx b/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx
index 426423bf..fe2f3df4 100644
--- a/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx
+++ b/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx
@@ -18,6 +18,14 @@ interface AgentConfigurationTabProps {
*
* Both blocks share `getObservabilityView` with the list-view cells so the
* Configuration tab can no longer drift from the list (the previous bug).
+ *
+ * Observability toggles are gated on lifecycle state: when the agent
+ * is stopped or transitioning, instrumenting (or removing
+ * instrumentation) is unsafe — there is no live process for the
+ * controller to attach to in Docker / Linux modes, and a K8s instrument
+ * action racing a Stop leaves a half-applied state. We block the
+ * toggle and surface the reason via the button tooltip rather than
+ * silently dropping the click.
*/
export default function AgentConfigurationTab({
agent,
@@ -25,8 +33,28 @@ export default function AgentConfigurationTab({
}: AgentConfigurationTabProps) {
const llmIntent = useAgentIntent(agent.agent_key, "llm");
const agentIntent = useAgentIntent(agent.agent_key, "agent");
+ const lifecycleIntent = useAgentIntent(agent.agent_key, "lifecycle");
const llmView = getObservabilityView(agent, "llm", llmIntent);
const agentView = getObservabilityView(agent, "agent", agentIntent);
+ const lifecycleView = getObservabilityView(
+ agent,
+ "lifecycle",
+ lifecycleIntent
+ );
+
+ // Only controller-managed agents have a real lifecycle to gate on.
+ // SDK-only rows fall through to ObservabilityBlock's static branch
+ // anyway, so passing null here is correct.
+ const isControllerManaged =
+ agent.source !== "sdk" && !!agent.controller_service_id;
+ let blockReason: string | null = null;
+ if (isControllerManaged) {
+ if (lifecycleView.transitioning) {
+ blockReason = getMessage().AGENTS_OBSERVABILITY_DISABLED_TRANSITIONING;
+ } else if (!lifecycleView.enabled) {
+ blockReason = getMessage().AGENTS_OBSERVABILITY_DISABLED_NOT_RUNNING;
+ }
+ }
return (
<div className="space-y-4">
@@ -44,6 +72,7 @@ export default function AgentConfigurationTab({
podSummary={llmView.podSummary}
onChange={onRefresh}
serviceName={agent.service_name}
+ blockReason={blockReason}
/>
<ObservabilityBlock
kind="agent"
@@ -55,6 +84,7 @@ export default function AgentConfigurationTab({
podSummary={agentView.podSummary}
onChange={onRefresh}
serviceName={agent.service_name}
+ blockReason={blockReason}
/>
</div>
</div>
diff --git a/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx b/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx
index d4ce5697..5f3662f8 100644
--- a/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx
+++ b/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx
@@ -1,6 +1,5 @@
"use client";
-import { ProviderIcon } from "@/components/svg/providers";
import { formatBrowserDateTime, parseDateString } from "@/utils/date";
import getMessage from "@/constants/messages";
import type { UnifiedAgent } from "@/types/agents";
@@ -59,23 +58,6 @@ export default function AgentMetadataGrid({ agent }: AgentMetadataGridProps) {
label={getMessage().AGENTS_METADATA_LAST_SEEN}
value={formatBrowserDateTime(agent.last_seen)}
/>
- {agent.providers.length > 0 && (
- <MetadataCard
- label={getMessage().AGENTS_MODEL_PROVIDERS}
- value={
- <div className="flex flex-wrap items-center gap-1.5">
- {agent.providers.map((p) => (
- <span key={p} className="flex items-center gap-1 text-xs">
- <ProviderIcon provider={p} className="w-3.5 h-3.5" />
- <span className="capitalize text-stone-700 dark:text-stone-200">
- {p}
- </span>
- </span>
- ))}
- </div>
- }
- />
- )}
</div>
);
}
diff --git a/src/client/src/components/(playground)/agents/lifecycle-actions.tsx b/src/client/src/components/(playground)/agents/lifecycle-actions.tsx
new file mode 100644
index 00000000..81e85a71
--- /dev/null
+++ b/src/client/src/components/(playground)/agents/lifecycle-actions.tsx
@@ -0,0 +1,301 @@
+"use client";
+
+/**
+ * LifecycleActions — Play / Stop / Restart for controller-managed
+ * workloads. Used by the agents list table (per-row, right-aligned
+ * "Actions" column) and the agent detail page (header `rightSlot`).
+ *
+ * Visual style mirrors Docker Desktop's container actions: the buttons
+ * sit inside a single bordered pill with internal dividers, so the
+ * affordance is always visible without hover. Two variants:
+ *
+ * - `row` : compact 28px icon buttons for the agents table.
+ * - `header` : 36px icon buttons with a touch more padding for the
+ * agent-detail header bar — large enough to be obvious
+ * next to the version chooser, small enough to not
+ * dominate the layout.
+ *
+ * Hidden entirely for SDK-only rows (no controller, no lifecycle to
+ * manage) and gracefully no-ops when the controller has not advertised
+ * `lifecycle_*_v1` capabilities yet (in that case the API returns 409
+ * and the toast surfaces the reason).
+ *
+ * State transitions flow through the shared observability-view so the
+ * spinner / direction / desired-mismatch precedence is the same as the
+ * LLM + Agent cells. Stop is the only destructive action and always
+ * opens a confirmation dialog.
+ */
+
+import { useState } from "react";
+import { Loader2, Play, RotateCcw, Square } from "lucide-react";
+import { toast } from "sonner";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import getMessage from "@/constants/messages";
+import {
+ getObservabilityView,
+ type Direction,
+} from "@/lib/platform/agents/observability-view";
+import useFetchWrapper from "@/utils/hooks/useFetchWrapper";
+import { useRootStore } from "@/store";
+import {
+ getClearAgentIntent,
+ getSetAgentIntent,
+ useAgentIntent,
+} from "@/selectors/agents-instrumentation";
+import type { UnifiedAgent } from "@/types/agents";
+
+export type LifecycleActionsVariant = "row" | "header";
+
+interface LifecycleActionsProps {
+ agent: UnifiedAgent;
+ onRefresh: () => void;
+ variant?: LifecycleActionsVariant;
+}
+
+function directionLabel(direction: Direction | null): string {
+ switch (direction) {
+ case "starting":
+ return getMessage().AGENTS_LIFECYCLE_STARTING;
+ case "stopping":
+ return getMessage().AGENTS_LIFECYCLE_STOPPING;
+ case "restarting":
+ return getMessage().AGENTS_LIFECYCLE_RESTARTING;
+ default:
+ return getMessage().AGENTS_SERVICE_ACTION_WORKING;
+ }
+}
+
+const SIZES: Record<
+ LifecycleActionsVariant,
+ { button: string; icon: string; pill: string }
+> = {
+ row: {
+ button: "w-8 h-8",
+ icon: "w-3.5 h-3.5",
+ pill: "h-8",
+ },
+ header: {
+ button: "w-9 h-9",
+ icon: "w-4 h-4",
+ pill: "h-9",
+ },
+};
+
+export default function LifecycleActions({
+ agent,
+ onRefresh,
+ variant = "row",
+}: LifecycleActionsProps) {
+ const { fireRequest, isLoading } = useFetchWrapper();
+ const setIntent = useRootStore(getSetAgentIntent);
+ const clearIntent = useRootStore(getClearAgentIntent);
+ const intent = useAgentIntent(agent.agent_key, "lifecycle");
+ // Symmetric to useObservabilityBlock in service-table.tsx: while
+ // LLM or Agent observability is mid-instrument, we don't want the
+ // user racing a Stop / Restart against that operation. The cheapest
+ // fix is to disable the lifecycle pill and surface the reason in a
+ // tooltip. Both reads go through getObservabilityView so the
+ // optimistic-intent / pending-action / desired-mismatch precedence
+ // is the same one used everywhere else.
+ const llmIntent = useAgentIntent(agent.agent_key, "llm");
+ const agentIntent = useAgentIntent(agent.agent_key, "agent");
+ const [confirmStopOpen, setConfirmStopOpen] = useState(false);
+
+ const controllerServiceId = agent.controller_service_id;
+ if (agent.source === "sdk" || !controllerServiceId) {
+ return null;
+ }
+
+ const view = getObservabilityView(agent, "lifecycle", intent);
+ const llmView = getObservabilityView(agent, "llm", llmIntent);
+ const agentView = getObservabilityView(agent, "agent", agentIntent);
+ const observabilityInFlight =
+ llmView.transitioning || agentView.transitioning;
+ const running = view.enabled;
+ const sz = SIZES[variant];
+
+ async function fire(
+ action: "start" | "stop" | "restart",
+ direction: "starting" | "stopping" | "restarting"
+ ) {
+ if (view.transitioning) return;
+ setIntent(agent.agent_key, "lifecycle", direction);
+ await fireRequest({
+ requestType: "POST",
+ url: `/api/controller/catalog/${controllerServiceId}/${action}`,
+ successCb: () => {
+ const queued =
+ action === "start"
+ ? getMessage().AGENTS_LIFECYCLE_QUEUED_PLAY(agent.service_name)
+ : action === "stop"
+ ? getMessage().AGENTS_LIFECYCLE_QUEUED_STOP(agent.service_name)
+ : getMessage().AGENTS_LIFECYCLE_QUEUED_RESTART(agent.service_name);
+ toast.success(queued);
+ onRefresh();
+ },
+ failureCb: (err: any) => {
+ clearIntent(agent.agent_key, "lifecycle");
+ toast.error(getMessage().AGENTS_LIFECYCLE_FAILED(err));
+ },
+ });
+ }
+
+ // Shared pill shell — one bordered container with internal dividers
+ // between buttons, so the action affordances are visible even when
+ // the cursor is nowhere near them. Matches Docker Desktop's
+ // container-row actions cluster.
+ const pillBase = `inline-flex ${sz.pill} items-center rounded-md border border-stone-200 dark:border-stone-700 bg-white dark:bg-stone-900 overflow-hidden divide-x divide-stone-200 dark:divide-stone-700 shadow-sm`;
+ const buttonBase = `inline-flex items-center justify-center ${sz.button} disabled:opacity-50 disabled:cursor-not-allowed transition-colors`;
+
+ if (view.transitioning) {
+ return (
+ <div className={pillBase} aria-busy="true" aria-label={directionLabel(view.direction)}>
+ <span
+ className={`${buttonBase} text-stone-500 dark:text-stone-400`}
+ title={directionLabel(view.direction)}
+ >
+ <Loader2 className={`${sz.icon} animate-spin`} />
+ </span>
+ </div>
+ );
+ }
+
+ // Observability-in-flight branch: keep the action affordance visible
+ // so users understand *what* would happen, but render it as a
+ // disabled pill with the explanatory tooltip. Stop/Restart vs. Play
+ // shape stays consistent with the live state so reading the row
+ // does not require a translation.
+ if (observabilityInFlight) {
+ const blockTooltip =
+ getMessage().AGENTS_LIFECYCLE_DISABLED_OBSERVABILITY_TRANSITIONING;
+ const disabledButton = (
+ node: React.ReactNode,
+ ariaLabel: string,
+ extraColor: string
+ ) => (
+ <span
+ className={`${buttonBase} ${extraColor} opacity-50 cursor-not-allowed`}
+ title={blockTooltip}
+ aria-label={ariaLabel}
+ aria-disabled="true"
+ >
+ {node}
+ </span>
+ );
+ return (
+ <div className={pillBase} onClick={(e) => e.stopPropagation()}>
+ {!running
+ ? disabledButton(
+ <Play className={sz.icon} />,
+ getMessage().AGENTS_LIFECYCLE_PLAY,
+ "text-green-600 dark:text-green-400"
+ )
+ : (
+ <>
+ {disabledButton(
+ <Square className={`${sz.icon} fill-current`} />,
+ getMessage().AGENTS_LIFECYCLE_STOP,
+ "text-red-600 dark:text-red-400"
+ )}
+ {disabledButton(
+ <RotateCcw className={sz.icon} />,
+ getMessage().AGENTS_LIFECYCLE_RESTART,
+ "text-stone-600 dark:text-stone-300"
+ )}
+ </>
+ )}
+ </div>
+ );
+ }
+
+ return (
+ <>
+ <div className={pillBase} onClick={(e) => e.stopPropagation()}>
+ {!running ? (
+ <button
+ onClick={(e) => {
+ e.stopPropagation();
+ fire("start", "starting");
+ }}
+ disabled={isLoading}
+ title={getMessage().AGENTS_LIFECYCLE_TOOLTIP_PLAY}
+ aria-label={getMessage().AGENTS_LIFECYCLE_PLAY}
+ className={`${buttonBase} text-green-600 dark:text-green-400 hover:bg-green-50 dark:hover:bg-green-950/40`}
+ >
+ <Play className={sz.icon} />
+ </button>
+ ) : (
+ <>
+ <button
+ onClick={(e) => {
+ e.stopPropagation();
+ setConfirmStopOpen(true);
+ }}
+ disabled={isLoading}
+ title={getMessage().AGENTS_LIFECYCLE_TOOLTIP_STOP}
+ aria-label={getMessage().AGENTS_LIFECYCLE_STOP}
+ className={`${buttonBase} text-red-600 dark:text-red-400 hover:bg-red-50 dark:hover:bg-red-950/40`}
+ >
+ <Square className={`${sz.icon} fill-current`} />
+ </button>
+ <button
+ onClick={(e) => {
+ e.stopPropagation();
+ fire("restart", "restarting");
+ }}
+ disabled={isLoading}
+ title={getMessage().AGENTS_LIFECYCLE_TOOLTIP_RESTART}
+ aria-label={getMessage().AGENTS_LIFECYCLE_RESTART}
+ className={`${buttonBase} text-stone-600 dark:text-stone-300 hover:bg-stone-100 dark:hover:bg-stone-800`}
+ >
+ <RotateCcw className={sz.icon} />
+ </button>
+ </>
+ )}
+ </div>
+ <Dialog open={confirmStopOpen} onOpenChange={setConfirmStopOpen}>
+ <DialogContent
+ onClick={(e) => e.stopPropagation()}
+ className="sm:max-w-md"
+ >
+ <DialogHeader>
+ <DialogTitle>
+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_TITLE}
+ </DialogTitle>
+ <DialogDescription>
+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_DESCRIPTION(
+ agent.service_name
+ )}
+ </DialogDescription>
+ </DialogHeader>
+ <DialogFooter>
+ <button
+ type="button"
+ onClick={() => setConfirmStopOpen(false)}
+ className="inline-flex items-center px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-600 dark:text-stone-300 hover:bg-stone-50 dark:hover:bg-stone-800"
+ >
+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_CANCEL}
+ </button>
+ <button
+ type="button"
+ onClick={() => {
+ setConfirmStopOpen(false);
+ fire("stop", "stopping");
+ }}
+ className="inline-flex items-center px-3 py-1.5 text-xs font-medium rounded-md bg-red-600 text-white hover:bg-red-700"
+ >
+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_CONFIRM}
+ </button>
+ </DialogFooter>
+ </DialogContent>
+ </Dialog>
+ </>
+ );
+}
diff --git a/src/client/src/components/(playground)/agents/observability-block.tsx b/src/client/src/components/(playground)/agents/observability-block.tsx
index b545f42c..fd49e422 100644
--- a/src/client/src/components/(playground)/agents/observability-block.tsx
+++ b/src/client/src/components/(playground)/agents/observability-block.tsx
@@ -26,13 +26,33 @@ interface ObservabilityBlockProps {
enabled: boolean;
/** True when an action is in flight (controller-side or optimistic). */
pending?: boolean;
- pendingDirection?: "enabling" | "disabling" | null;
+ // Direction is widened to the full Direction union (which includes
+ // lifecycle's starting/stopping/restarting) even though this block
+ // only ever renders for llm/agent, because the upstream
+ // observability-view now returns the widened type. The renderer
+ // below only branches on enabling/disabling so the lifecycle
+ // variants are inert here.
+ pendingDirection?:
+ | "enabling"
+ | "disabling"
+ | "starting"
+ | "stopping"
+ | "restarting"
+ | null;
/** Multi-pod rollup so we can show `Pods: ack/total` while a fan-out is
* propagating. `null` for SDK-only rows or single-pod workloads. */
podSummary?: PodSummary | null;
/** Triggered after a successful action so the parent can refresh. */
onChange?: () => void;
serviceName: string;
+ /**
+ * If set, the toggle renders as a disabled button with this text as
+ * the tooltip. Used to gate observability changes on lifecycle
+ * state (e.g. the agent is stopped or transitioning) — we don't
+ * want to queue an instrument action against a workload that has
+ * no live process for the controller to attach to.
+ */
+ blockReason?: string | null;
}
function kindLabel(kind: ObservabilityKind) {
@@ -57,6 +77,7 @@ export default function ObservabilityBlock({
podSummary,
onChange,
serviceName,
+ blockReason,
}: ObservabilityBlockProps) {
const { fireRequest, isLoading } = useFetchWrapper();
const setIntent = useRootStore(getSetAgentIntent);
@@ -142,6 +163,18 @@ export default function ObservabilityBlock({
? getMessage().AGENTS_SERVICE_ACTION_DISABLING
: getMessage().AGENTS_SERVICE_ACTION_ENABLING}
</button>
+ ) : blockReason ? (
+ <button
+ type="button"
+ disabled
+ title={blockReason}
+ aria-label={blockReason}
+ className="inline-flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-400 dark:text-stone-500 bg-stone-50 dark:bg-stone-900 cursor-not-allowed"
+ >
+ {enabled
+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE
+ : getMessage().AGENTS_SERVICE_ACTION_ENABLE}
+ </button>
) : (
<button
onClick={handleToggle}
diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.ts
index 70d2214e..693a0aa9 100644
--- a/src/client/src/constants/messages/en.ts
+++ b/src/client/src/constants/messages/en.ts
@@ -416,6 +416,43 @@ export const AGENTS_SDK_SOURCE_NOTE = "Source: SDK";
export const AGENTS_LOAD_MORE = "Load more";
export const AGENTS_LOAD_MORE_LOADING = "Loading...";
+// Agents - Lifecycle (Play / Stop / Restart for controller-managed workloads)
+export const AGENTS_COLUMN_ACTIONS = "Actions";
+export const AGENTS_LIFECYCLE_PLAY = "Play";
+export const AGENTS_LIFECYCLE_STOP = "Stop";
+export const AGENTS_LIFECYCLE_RESTART = "Restart";
+export const AGENTS_LIFECYCLE_STARTING = "Starting...";
+export const AGENTS_LIFECYCLE_STOPPING = "Stopping...";
+export const AGENTS_LIFECYCLE_RESTARTING = "Restarting...";
+export const AGENTS_LIFECYCLE_STATUS_RUNNING = "Running";
+export const AGENTS_LIFECYCLE_STATUS_STOPPED = "Stopped";
+export const AGENTS_LIFECYCLE_STATUS_RESTARTING = "Restarting";
+export const AGENTS_LIFECYCLE_STATUS_UNKNOWN = "Unknown";
+export const AGENTS_LIFECYCLE_TOOLTIP_PLAY = "Start the agent (scale up / start container / re-launch process)";
+export const AGENTS_LIFECYCLE_TOOLTIP_STOP = "Stop the agent (scale to 0 / stop container / SIGTERM)";
+export const AGENTS_LIFECYCLE_TOOLTIP_RESTART = "Restart the agent (rolling restart / restart container / SIGTERM + relaunch)";
+export const AGENTS_LIFECYCLE_TOOLTIP_DISABLED_SDK = "Lifecycle controls are available only for controller-managed agents.";
+export const AGENTS_LIFECYCLE_TOOLTIP_DISABLED_CAPABILITY = "The controller managing this agent does not advertise lifecycle support yet.";
+export const AGENTS_LIFECYCLE_TOOLTIP_PLAY_NO_SNAPSHOT = "No saved snapshot to restore. The controller cannot bring this agent back up without state captured at Stop time.";
+export const AGENTS_LIFECYCLE_CONFIRM_STOP_TITLE = "Stop agent?";
+export const AGENTS_LIFECYCLE_CONFIRM_STOP_DESCRIPTION = (serviceName: string) =>
+ `This will scale down or terminate the agent "${serviceName}". Telemetry will stop until you Play it again.`;
+export const AGENTS_LIFECYCLE_CONFIRM_STOP_CONFIRM = "Stop agent";
+export const AGENTS_LIFECYCLE_CONFIRM_STOP_CANCEL = "Cancel";
+export const AGENTS_LIFECYCLE_QUEUED_PLAY = (serviceName: string) =>
+ `Starting ${serviceName}...`;
+export const AGENTS_LIFECYCLE_QUEUED_STOP = (serviceName: string) =>
+ `Stopping ${serviceName}...`;
+export const AGENTS_LIFECYCLE_QUEUED_RESTART = (serviceName: string) =>
+ `Restarting ${serviceName}...`;
+export const AGENTS_LIFECYCLE_FAILED = (err: string) => `Lifecycle action failed: ${err}`;
+export const AGENTS_OBSERVABILITY_DISABLED_NOT_RUNNING =
+ "Start the agent to change observability — toggling LLM / Agent observability is only possible while the agent is running.";
+export const AGENTS_OBSERVABILITY_DISABLED_TRANSITIONING =
+ "Observability changes are blocked while the agent is starting, stopping, or restarting.";
+export const AGENTS_LIFECYCLE_DISABLED_OBSERVABILITY_TRANSITIONING =
+ "Lifecycle changes are blocked while LLM or Agent observability is being applied to this agent.";
+
// Agents - Detail Tabs
export const AGENTS_TAB_ANALYTICS = "Analytics";
export const AGENTS_TAB_CONVERSATIONS = "Conversations";
@@ -513,7 +550,6 @@ export const AGENTS_STAT_PID = "PID";
export const AGENTS_STAT_RUNTIME = "Runtime";
export const AGENTS_STAT_FIRST_SEEN = "First Seen";
export const AGENTS_STAT_RUNTIME_UNKNOWN = "Unknown";
-export const AGENTS_MODEL_PROVIDERS = "Model Providers";
export const AGENTS_RESOURCE_ATTRIBUTES = "Resource Attributes";
export const AGENTS_CONTROLS = "Controls";
export const AGENTS_LLM_OBSERVABILITY_LABEL = "LLM Observability";
@@ -535,8 +571,8 @@ export const AGENTS_MANUAL_SETUP_REQUIRED = "Manual setup required";
export const AGENTS_CONTAINERIZED_WARNING = "This process runs inside a container. Mount the Docker socket or use a Docker/Kubernetes-mode controller for Agent Observability.";
export const AGENTS_NAKED_POD_WARNING = "This is a naked pod (no Deployment, DaemonSet, or StatefulSet). Enabling or disabling Agent Observability will restart the pod.";
export const AGENTS_NAKED_POD_CONFIRM = "This pod has no Deployment or DaemonSet. Enabling Agent Observability will restart the pod. Continue?";
-export const AGENTS_AGENT_DEPLOY_QUEUED = "Agent observability is being deployed. The workload will be updated automatically.";
-export const AGENTS_AGENT_REMOVE_QUEUED = "Agent observability removal queued. The workload will be updated automatically.";
+export const AGENTS_AGENT_DEPLOY_QUEUED = "Agent observability is being deployed. The agent will be updated automatically.";
+export const AGENTS_AGENT_REMOVE_QUEUED = "Agent observability removal queued. The agent will be updated automatically.";
export const AGENTS_QUEUED_ACTION = (action: string) => `Queued ${action}`;
export const AGENTS_PENDING_INSTRUMENTING = "Instrumenting";
export const AGENTS_PENDING_UNINSTRUMENTING = "Uninstrumenting";
diff --git a/src/client/src/lib/platform/agents/index.ts b/src/client/src/lib/platform/agents/index.ts
index aedd83d5..196a8402 100644
--- a/src/client/src/lib/platform/agents/index.ts
+++ b/src/client/src/lib/platform/agents/index.ts
@@ -62,6 +62,8 @@ function rowToAgent(row: Record<string, unknown>): UnifiedAgent {
desired_instrumentation_status: (String(row.desired_instrumentation_status || "none") as UnifiedAgent["desired_instrumentation_status"]),
agent_observability_status: agentObsStatus,
desired_agent_status: (String(row.desired_agent_status || "none") as UnifiedAgent["desired_agent_status"]),
+ lifecycle_status: (String(row.lifecycle_status || "unknown") as UnifiedAgent["lifecycle_status"]),
+ desired_lifecycle_status: (String(row.desired_lifecycle_status || "unknown") as UnifiedAgent["desired_lifecycle_status"]),
pending_action: pendingAction ? pendingAction : null,
pending_action_status: pendingActionStatus
? (pendingActionStatus as UnifiedAgent["pending_action_status"])
@@ -93,7 +95,12 @@ WITH pod_set AS (
argMax(service_name, last_seen) AS service_name,
uniqExact(controller_instance_id) AS pods_total,
groupArrayDistinct(controller_instance_id) AS instance_ids,
- argMax(resource_attributes['openlit.agent_observability.status'], last_seen) AS agent_observability_status
+ argMax(resource_attributes['openlit.agent_observability.status'], last_seen) AS agent_observability_status,
+ -- Mirrors the agent_observability rollup. The controller stamps
+ -- resource_attributes['openlit.lifecycle.status'] on every
+ -- heartbeat after a Play / Stop / Restart so the UI sees the
+ -- actual state of the workload, not just the desired state.
+ argMax(resource_attributes['openlit.lifecycle.status'], last_seen) AS lifecycle_status
FROM ${CONTROLLER_SERVICES_TABLE}
FINAL
WHERE last_seen >= now() - INTERVAL 5 MINUTE
@@ -120,6 +127,21 @@ desired_agent AS (
WHERE feature = 'agent'
GROUP BY workload_key, cluster_id
),
+desired_lifecycle AS (
+ -- A row here means the user has explicitly Played or Stopped the
+ -- workload. It is also the source of truth for the snapshot blob
+ -- (config column) used to bring K8s naked pods and Linux bare
+ -- processes back up after Stop.
+ SELECT
+ workload_key,
+ cluster_id,
+ argMax(desired_status, updated_at) AS desired_status,
+ argMax(config, updated_at) AS config
+ FROM ${CONTROLLER_DESIRED_STATES_V2_TABLE}
+ FINAL
+ WHERE feature = 'lifecycle'
+ GROUP BY workload_key, cluster_id
+),
pod_action_latest AS (
-- Per (instance_id, service_key), keep the most recent action row.
-- We rename the max(updated_at) projection to last_updated_at so the
@@ -187,6 +209,24 @@ const SELECT_COLUMNS = `
-- not yet aged out of the materializer's 30-min lookback window).
coalesce(pod_set.agent_observability_status, if(s.source = 'sdk', 'enabled', '')) AS agent_observability_status,
if(desired_agent.desired_status != '', desired_agent.desired_status, 'none') AS desired_agent_status,
+ -- Lifecycle:
+ -- - For SDK-only rows there is nothing to lifecycle-manage; render
+ -- 'unknown' so the UI hides the buttons (the LifecycleCell also
+ -- guards on source != 'sdk').
+ -- - When pod_set has a value, that is the most authoritative
+ -- reading (it came from the latest heartbeat).
+ -- - When pod_set is empty AND the user has stopped the workload,
+ -- trust the desired_status='stopped' as the actual state -- the
+ -- heartbeat went silent because the workload is down. This is
+ -- what keeps the row visible as "Stopped" in the UI.
+ if(s.source = 'sdk' AND pod_set.lifecycle_status = '',
+ 'unknown',
+ if(pod_set.lifecycle_status != '',
+ pod_set.lifecycle_status,
+ if(desired_lifecycle.desired_status = 'stopped', 'stopped', 'running')
+ )
+ ) AS lifecycle_status,
+ if(desired_lifecycle.desired_status != '', desired_lifecycle.desired_status, 'unknown') AS desired_lifecycle_status,
if(pod_actions.pending_action != '', pod_actions.pending_action, '') AS pending_action,
if(pod_actions.pending_action_status != '', pod_actions.pending_action_status, '') AS pending_action_status,
s.first_seen AS first_seen,
@@ -220,6 +260,9 @@ const ROLLUP_JOINS = `
LEFT JOIN desired_agent
ON desired_agent.workload_key = s.workload_key
AND desired_agent.cluster_id = s.cluster_id
+ LEFT JOIN desired_lifecycle
+ ON desired_lifecycle.workload_key = s.workload_key
+ AND desired_lifecycle.cluster_id = s.cluster_id
LEFT JOIN pod_actions
ON pod_actions.service_key = s.workload_key
`;
@@ -269,12 +312,21 @@ async function loadAgents(params: ListAgentsParams): Promise<ListAgentsResult> {
// off so we always include the most recent materialized rows, even if the
// browser captured `end` slightly before the materializer wrote the row.
// Custom ranges still send both bounds.
+ //
+ // Stopped-workload exception: a workload the user has Stopped no longer
+ // emits heartbeats, so `s.last_seen` ages out fast. We special-case it
+ // by OR-ing in any row whose lifecycle desired-state is 'stopped' --
+ // those stay visible until the 90-day desired-states TTL collapses the
+ // row organically. Without this OR, clicking Stop would feel like the
+ // workload disappeared from the dashboard, which is exactly the bug
+ // the lifecycle UNION in the materializer is meant to prevent.
+ const stoppedRowClause = `desired_lifecycle.desired_status = 'stopped'`;
if (params.timeStart) {
where.push(
- `s.last_seen >= parseDateTimeBestEffort('${escape(params.timeStart)}')`
+ `(s.last_seen >= parseDateTimeBestEffort('${escape(params.timeStart)}') OR ${stoppedRowClause})`
);
} else {
- where.push(`s.last_seen >= now() - INTERVAL 30 DAY`);
+ where.push(`(s.last_seen >= now() - INTERVAL 30 DAY OR ${stoppedRowClause})`);
}
if (params.timeEnd) {
where.push(
diff --git a/src/client/src/lib/platform/agents/materialize.ts b/src/client/src/lib/platform/agents/materialize.ts
index d5bfafc2..d5644d88 100644
--- a/src/client/src/lib/platform/agents/materialize.ts
+++ b/src/client/src/lib/platform/agents/materialize.ts
@@ -9,7 +9,10 @@
*/
import { dataCollector, OTEL_TRACES_TABLE_NAME } from "@/lib/platform/common";
-import { CONTROLLER_SERVICES_TABLE } from "@/lib/platform/controller/table-details";
+import {
+ CONTROLLER_DESIRED_STATES_V2_TABLE,
+ CONTROLLER_SERVICES_TABLE,
+} from "@/lib/platform/controller/table-details";
import type { AgentSource } from "@/types/agents";
import { computeAgentKey, invalidateAgent } from "./index";
import { invalidatePrefix } from "./cache";
@@ -140,8 +143,16 @@ async function discoverAgents(
last_seen: string;
}>) || [];
+ // The materializer needs to keep stopped workloads in agents_summary
+ // even after their last heartbeat aged out of the 24h window. We do
+ // that by UNION-ing the active heartbeat rows with the
+ // lifetime-latest row for any workload whose lifecycle desired
+ // state is 'stopped'. Without this, a workload that has been
+ // stopped for >24h would silently disappear from the UI -- the
+ // agents-list query reads agents_summary, and agents_summary only
+ // keeps rows the materializer rewrites.
const ctrlQuery = `
- WITH latest AS (
+ WITH active AS (
SELECT
argMax(s.id, s.last_seen) AS id,
argMax(s.controller_instance_id, s.last_seen) AS controller_instance_id,
@@ -157,6 +168,46 @@ async function discoverAgents(
WHERE s.last_seen >= now() - INTERVAL 24 HOUR
${clusterPredicate}
GROUP BY concat(s.cluster_id, ':', s.service_name)
+ ),
+ stopped_keys AS (
+ SELECT workload_key, cluster_id
+ FROM ${CONTROLLER_DESIRED_STATES_V2_TABLE} FINAL
+ WHERE feature = 'lifecycle' AND desired_status = 'stopped'
+ AND workload_key != ''
+ ${clusterPredicate}
+ ),
+ stopped AS (
+ -- Lifetime lookup (no time bound) so stopped agents stay
+ -- materialized regardless of how long they have been down.
+ -- We bound the size by stopped_keys (small) and exclude
+ -- anything already covered by the active CTE to avoid
+ -- double-materializing the same workload. The dedup key is
+ -- the same (cluster_id, service_name) tuple the active CTE
+ -- groups by -- service_name alone would collide across
+ -- clusters (e.g. "checkout" running in cluster-prod while
+ -- stopped in cluster-staging).
+ SELECT
+ argMax(s.id, s.last_seen) AS id,
+ argMax(s.controller_instance_id, s.last_seen) AS controller_instance_id,
+ argMax(s.cluster_id, s.last_seen) AS cluster_id,
+ argMax(s.service_name, s.last_seen) AS service_name,
+ argMax(s.workload_key, s.last_seen) AS workload_key,
+ argMax(s.instrumentation_status, s.last_seen) AS instrumentation_status,
+ argMax(s.resource_attributes, s.last_seen) AS resource_attributes,
+ argMax(s.llm_providers, s.last_seen) AS llm_providers,
+ min(s.first_seen) AS first_seen,
+ max(s.last_seen) AS last_seen
+ FROM ${CONTROLLER_SERVICES_TABLE} s FINAL
+ WHERE (s.workload_key, s.cluster_id) IN (SELECT workload_key, cluster_id FROM stopped_keys)
+ AND s.workload_key != ''
+ ${clusterPredicate}
+ GROUP BY concat(s.cluster_id, ':', s.service_name)
+ HAVING (cluster_id, service_name) NOT IN (SELECT cluster_id, service_name FROM active)
+ ),
+ latest AS (
+ SELECT * FROM active
+ UNION ALL
+ SELECT * FROM stopped
)
SELECT
latest.id AS id,
diff --git a/src/client/src/lib/platform/agents/observability-view.ts b/src/client/src/lib/platform/agents/observability-view.ts
index de9cd5c3..b5f0c3de 100644
--- a/src/client/src/lib/platform/agents/observability-view.ts
+++ b/src/client/src/lib/platform/agents/observability-view.ts
@@ -29,8 +29,26 @@
import type { UnifiedAgent } from "@/types/agents";
-export type Feature = "llm" | "agent";
-export type Direction = "enabling" | "disabling";
+export type Feature = "llm" | "agent" | "lifecycle";
+/**
+ * Direction of the in-flight transition.
+ *
+ * For `llm` and `agent` features: only `enabling` / `disabling` apply.
+ * For `lifecycle`:
+ * - `starting` — Play queued while the workload is stopped.
+ * - `stopping` — Stop queued while the workload is running.
+ * - `restarting` — Restart queued; the workload will return to `running`.
+ * We expose `restarting` as a distinct direction (rather than reusing
+ * `enabling`) because the UI renders a different spinner glyph and a
+ * different toast for Restart, and the optimistic-intent store wants to
+ * match by exact action.
+ */
+export type Direction =
+ | "enabling"
+ | "disabling"
+ | "starting"
+ | "stopping"
+ | "restarting";
export interface OptimisticIntent {
feature: Feature;
@@ -82,11 +100,21 @@ const AGENT_ACTIONS = new Set<string>([
"enable_python_sdk",
"disable_python_sdk",
]);
+const LIFECYCLE_ACTIONS = new Set<string>([
+ "start_workload",
+ "stop_workload",
+ "restart_workload",
+]);
function isFeatureAction(feature: Feature, action: string): boolean {
- return feature === "llm"
- ? LLM_ACTIONS.has(action)
- : AGENT_ACTIONS.has(action);
+ switch (feature) {
+ case "llm":
+ return LLM_ACTIONS.has(action);
+ case "agent":
+ return AGENT_ACTIONS.has(action);
+ case "lifecycle":
+ return LIFECYCLE_ACTIONS.has(action);
+ }
}
function actionDirection(action: string): Direction | null {
@@ -97,6 +125,12 @@ function actionDirection(action: string): Direction | null {
case "uninstrument":
case "disable_python_sdk":
return "disabling";
+ case "start_workload":
+ return "starting";
+ case "stop_workload":
+ return "stopping";
+ case "restart_workload":
+ return "restarting";
default:
return null;
}
@@ -133,6 +167,24 @@ function agentDesiredEnabled(agent: UnifiedAgent): boolean {
return agent.desired_agent_status === "enabled";
}
+function lifecycleActualEnabled(agent: UnifiedAgent): boolean {
+ // "Enabled" for lifecycle means "the workload is up". `restarting`
+ // counts as enabled because the workload will be back in seconds and
+ // we don't want the Stop button to disappear during the bounce.
+ return (
+ agent.lifecycle_status === "running" ||
+ agent.lifecycle_status === "restarting"
+ );
+}
+
+function lifecycleDesiredEnabled(agent: UnifiedAgent): boolean {
+ // Default to running when the desired-state is unknown. Most rows
+ // start out without an explicit lifecycle desired state -- only Stop
+ // writes one. Treating unknown as `running` matches the user's
+ // expectation that an undiscovered workload is "on" by default.
+ return agent.desired_lifecycle_status !== "stopped";
+}
+
function buildPodSummary(agent: UnifiedAgent): PodSummary | null {
const total = agent.pods_total ?? 0;
if (total === 0) return null;
@@ -143,16 +195,36 @@ function buildPodSummary(agent: UnifiedAgent): PodSummary | null {
};
}
+function featureActualEnabled(feature: Feature, agent: UnifiedAgent): boolean {
+ switch (feature) {
+ case "llm":
+ return llmActualEnabled(agent);
+ case "agent":
+ return agentActualEnabled(agent);
+ case "lifecycle":
+ return lifecycleActualEnabled(agent);
+ }
+}
+
+function featureDesiredEnabled(feature: Feature, agent: UnifiedAgent): boolean {
+ switch (feature) {
+ case "llm":
+ return llmDesiredEnabled(agent);
+ case "agent":
+ return agentDesiredEnabled(agent);
+ case "lifecycle":
+ return lifecycleDesiredEnabled(agent);
+ }
+}
+
export function getObservabilityView(
agent: UnifiedAgent,
feature: Feature,
intent: OptimisticIntent | null,
now: number = Date.now()
): ObservabilityView {
- const actualEnabled =
- feature === "llm" ? llmActualEnabled(agent) : agentActualEnabled(agent);
- const desiredEnabled =
- feature === "llm" ? llmDesiredEnabled(agent) : agentDesiredEnabled(agent);
+ const actualEnabled = featureActualEnabled(feature, agent);
+ const desiredEnabled = featureDesiredEnabled(feature, agent);
const podSummary = buildPodSummary(agent);
const isManual = feature === "agent" && isManualAgentObservability(agent);
@@ -160,8 +232,15 @@ export function getObservabilityView(
// expires or the server's pending_action confirms it (in which case we
// fall through to the pending_action branch which is also a spinner).
if (intent && intent.feature === feature && intent.expiresAt > now) {
+ // For lifecycle: starting/restarting are conceptually "enabled"
+ // (workload will be up); stopping is "disabled" (workload will
+ // be down).
+ const optimisticEnabled =
+ intent.direction === "enabling" ||
+ intent.direction === "starting" ||
+ intent.direction === "restarting";
return {
- enabled: intent.direction === "enabling",
+ enabled: optimisticEnabled,
transitioning: true,
direction: intent.direction,
podSummary,
@@ -180,10 +259,14 @@ export function getObservabilityView(
if (hasServerPending) {
const direction = actionDirection(pendingActionRaw);
+ const willEnable =
+ direction === "enabling" ||
+ direction === "starting" ||
+ direction === "restarting";
return {
// Spinner button needs a stable "what it'll be next" so the label
// doesn't flicker between optimistic and pending_action phases.
- enabled: direction === "enabling",
+ enabled: willEnable,
transitioning: true,
direction,
podSummary,
@@ -194,16 +277,44 @@ export function getObservabilityView(
// (3) Desired state diverges from actual — usually the brief window
// between the API write and the first action row, or a controller retry
- // loop. Pure controller-source rows only. SDK / both rows always have
+ // loop. Pure controller-source rows only for `llm`/`agent`; lifecycle
+ // rows always go through controller-managed actions so the SDK-backed
+ // short-circuit does not apply. SDK / both rows always have
// `desired_*='none'` (the controller never wrote a desired state, the
// SDK self-enrolled by emitting traces) which would otherwise pin them
// permanently in "disabling..." — that's the regression bug.
const isSdkBacked = agent.source === "sdk" || agent.source === "both";
- if (!isSdkBacked && desiredEnabled !== actualEnabled) {
+ const skipForSdk = isSdkBacked && feature !== "lifecycle";
+ if (!skipForSdk && desiredEnabled !== actualEnabled) {
+ let direction: Direction;
+ if (feature === "lifecycle") {
+ direction = desiredEnabled ? "starting" : "stopping";
+ } else {
+ direction = desiredEnabled ? "enabling" : "disabling";
+ }
return {
enabled: actualEnabled,
transitioning: true,
- direction: desiredEnabled ? "enabling" : "disabling",
+ direction,
+ podSummary,
+ isManual,
+ source: "desired_mismatch",
+ };
+ }
+
+ // (3b) Lifecycle-specific in-flight signal. The controller stamps
+ // `openlit.lifecycle.status = "restarting"` on the in-memory service
+ // for the heartbeat immediately following a Restart. Both
+ // optimistic intent and pending_action will have already covered
+ // the user-initiated case, but a second browser tab that loads
+ // after the click would otherwise miss the spinner entirely.
+ // Treating actual=='restarting' as transitioning here is the
+ // failsafe that keeps the icon set consistent across tabs.
+ if (feature === "lifecycle" && agent.lifecycle_status === "restarting") {
+ return {
+ enabled: true,
+ transitioning: true,
+ direction: "restarting",
podSummary,
isManual,
source: "desired_mismatch",
diff --git a/src/client/src/lib/platform/controller/features/index.ts b/src/client/src/lib/platform/controller/features/index.ts
index 6e183a4a..94600850 100644
--- a/src/client/src/lib/platform/controller/features/index.ts
+++ b/src/client/src/lib/platform/controller/features/index.ts
@@ -11,5 +11,6 @@ export type {
import "./instrumentation";
import "./agent";
+import "./lifecycle";
import "./prompts";
import "./envs";
diff --git a/src/client/src/lib/platform/controller/features/lifecycle.ts b/src/client/src/lib/platform/controller/features/lifecycle.ts
new file mode 100644
index 00000000..f399eccd
--- /dev/null
+++ b/src/client/src/lib/platform/controller/features/lifecycle.ts
@@ -0,0 +1,365 @@
+/**
+ * Lifecycle feature handler — drives Play / Stop / Restart for
+ * controller-managed workloads.
+ *
+ * The handler is the dashboard-side mirror of openlit-controller's
+ * engine/lifecycle.go. It only writes desired state + queues the action;
+ * the controller does the actual work. The actual workload status comes
+ * back to the dashboard through two channels:
+ * 1. The controller's next heartbeat reports
+ * resource_attributes["openlit.lifecycle.status"]. That feeds the
+ * pod_set rollup in lib/platform/agents/index.ts.
+ * 2. For Stop, the controller's ActionResult carries a snapshot blob
+ * (pod spec for K8s naked pods, exe+cmdline+cwd+env-subset for bare
+ * Linux processes). The poll route persists the blob into
+ * desired_states_v2.config so a later Start can hand it back.
+ *
+ * The Restart path is *one-shot* on the reconcile loop: clicking Restart
+ * does NOT change desired_status (it stays 'running'), and we never emit
+ * a restart action from reconcile() — only directly from applyOperation.
+ * That prevents the reconciler from looping forever on Restart desired-
+ * vs-actual mismatches that have no closed-form fix.
+ */
+
+import {
+ getServiceById,
+ getControllerInstanceById,
+ getControllerIdsForWorkload,
+ getFeatureDesiredStates,
+ queueAction,
+ updateFeatureDesiredState,
+} from "@/lib/platform/controller";
+import { KNOWN_ACTIONS } from "@/types/controller";
+import type {
+ FeatureDesiredState,
+ EnvironmentFeatureConfig,
+} from "@/types/controller";
+import type {
+ FeatureHandler,
+ ReportedService,
+ ReconcileAction,
+} from "./registry";
+import { registerFeature } from "./registry";
+
+const FEATURE = "lifecycle";
+
+const DESIRED_RUNNING = "running";
+const DESIRED_STOPPED = "stopped";
+
+/**
+ * The controller advertises its per-mode lifecycle support via the
+ * `controller.capabilities` resource attribute (see
+ * engine.go::ControllerCapabilities). Modes that do not appear here mean
+ * the dashboard should not let the user click Play/Stop/Restart for
+ * workloads attached to that controller — the action would be queued
+ * but the Go engine would have nothing to do with it.
+ */
+function capabilityForMode(mode: string | undefined): { value: string; prefix: boolean } {
+ switch (mode) {
+ case "kubernetes":
+ return { value: "lifecycle_kubernetes_v1", prefix: false };
+ case "docker":
+ return { value: "lifecycle_docker_v1", prefix: false };
+ case "linux":
+ return { value: "lifecycle_linux_", prefix: true };
+ default:
+ return { value: "", prefix: false };
+ }
+}
+
+async function resolveLifecycleContext(
+ serviceId: string,
+ dbConfigId?: string
+): Promise<
+ | { error: Response }
+ | {
+ service: any;
+ supportsLifecycle: boolean;
+ mode: string;
+ targets: string[];
+ }
+> {
+ const serviceRes = await getServiceById(serviceId, dbConfigId);
+ if (!serviceRes.data || serviceRes.data.length === 0) {
+ return { error: Response.json({ error: "Service not found" }, { status: 404 }) };
+ }
+ const service = serviceRes.data[0];
+ if (!service.workload_key) {
+ return {
+ error: Response.json(
+ { error: "Service is missing workload_key" },
+ { status: 500 }
+ ),
+ };
+ }
+
+ const instanceRes = await getControllerInstanceById(
+ service.controller_instance_id,
+ dbConfigId
+ );
+ const instance = instanceRes.data?.[0];
+ const mode = instance?.mode || "linux";
+ const expected = capabilityForMode(mode);
+ const capabilities =
+ instance?.resource_attributes?.["controller.capabilities"] || "";
+ const supportsLifecycle = capabilities
+ .split(",")
+ .map((cap) => cap.trim())
+ .some((cap) =>
+ expected.prefix
+ ? cap.startsWith(expected.value)
+ : cap === expected.value
+ );
+
+ // Multi-controller fan-out: same as instrumentation.ts. For K8s
+ // workload-level operations (scale, rollout-restart) the action is
+ // idempotent across controllers — atomicScaleWithAnnotation/
+ // bumpRolloutAnnotation use resourceVersion-based optimistic
+ // concurrency so the second controller's update either no-ops or
+ // retries cleanly on conflict. For per-pod operations (naked pods,
+ // Linux modes) every owning controller acting against its own pod is
+ // the correct behaviour.
+ const controllerIds = await getControllerIdsForWorkload(
+ service.service_name,
+ service.namespace || "",
+ service.cluster_id || "default",
+ dbConfigId
+ );
+ const targets =
+ controllerIds.data?.map((r) => r.controller_instance_id) || [];
+ if (targets.length === 0) {
+ targets.push(service.controller_instance_id);
+ }
+
+ return { service, supportsLifecycle, mode, targets };
+}
+
+const lifecycleHandler: FeatureHandler = {
+ feature: FEATURE,
+
+ validatePayload(operation: string, _payload: Record<string, unknown>) {
+ if (
+ operation !== "start" &&
+ operation !== "stop" &&
+ operation !== "restart"
+ ) {
+ return `Unknown operation "${operation}" for feature "${FEATURE}". Expected "start", "stop", or "restart".`;
+ }
+ return null;
+ },
+
+ async applyOperation(
+ serviceId: string,
+ operation: string,
+ _payload: Record<string, unknown>,
+ dbConfigId?: string
+ ): Promise<Response> {
+ try {
+ const ctx = await resolveLifecycleContext(serviceId, dbConfigId);
+ if ("error" in ctx) return ctx.error;
+
+ if (!ctx.supportsLifecycle) {
+ return Response.json(
+ {
+ error:
+ "The selected controller does not advertise lifecycle support for this mode yet.",
+ },
+ { status: 409 }
+ );
+ }
+
+ const workloadKey = ctx.service.workload_key as string;
+ const clusterId = (ctx.service.cluster_id as string) || "default";
+
+ if (operation === "stop") {
+ // Write the desired state up-front so the UI sees the
+ // stopped intent immediately. The controller will return
+ // a snapshot blob on the action result, which the poll
+ // route persists into the same row's `config` column so
+ // a later Start can hand it back. We deliberately do not
+ // require the snapshot at write time -- naked pods and
+ // bare processes are the only modes that *need* one, and
+ // the controller computes it from local state.
+ await updateFeatureDesiredState(
+ workloadKey,
+ clusterId,
+ FEATURE,
+ DESIRED_STOPPED,
+ "{}",
+ dbConfigId
+ );
+ await Promise.all(
+ ctx.targets.map((cid) =>
+ queueAction(
+ cid,
+ KNOWN_ACTIONS.STOP_WORKLOAD,
+ workloadKey,
+ "{}",
+ dbConfigId
+ )
+ )
+ );
+ return Response.json({
+ status: "queued",
+ action: KNOWN_ACTIONS.STOP_WORKLOAD,
+ controllers: ctx.targets.length,
+ });
+ }
+
+ if (operation === "start") {
+ // Pull the snapshot stashed by the previous Stop. What
+ // the snapshot contains depends on the mode:
+ // - K8s naked pod: full gzipped pod spec (recreated
+ // by the controller).
+ // - Linux bare process: exe + argv + cwd + env
+ // allowlist (re-execed).
+ // - K8s controlled (Deployment/STS/DS): small
+ // {kind, ns, name, container_name} identifier that
+ // lets startK8s scale back up even when the
+ // controller's in-memory svc cache has been pruned.
+ // - Docker, systemd: "{}" — runtime preserves the
+ // object across Stop (container ID / unit file),
+ // no replay data needed.
+ const desiredRes = await getFeatureDesiredStates(
+ [workloadKey],
+ clusterId,
+ [FEATURE],
+ dbConfigId
+ );
+ const row = desiredRes.data?.find(
+ (r) => r.feature === FEATURE
+ );
+ const snapshot = row?.config || "{}";
+
+ // If the workload mode requires a snapshot but the
+ // desired-states row has none (e.g. controller was
+ // upgraded between Stop and Start, or the user clicks
+ // Play on a workload that the controller never stopped),
+ // fail closed -- the controller would otherwise error
+ // when trying to recreate the pod or re-exec the process.
+ //
+ // Systemd carve-out: Linux *systemd* units do NOT need a
+ // snapshot because the unit file still exists on disk
+ // and `systemctl start <unit>` is sufficient. Only Linux
+ // *bare* processes (no `systemd.unit` resource attr)
+ // need the captured exe+argv+cwd+env to re-exec.
+ const k8sKind = ctx.service.resource_attributes?.["k8s.workload.kind"];
+ const isLinuxBare =
+ ctx.mode === "linux" &&
+ !ctx.service.resource_attributes?.["systemd.unit"];
+ const isK8sNakedPod =
+ ctx.mode === "kubernetes" &&
+ (k8sKind === "" || k8sKind === "Pod" || k8sKind === undefined);
+ const requiresSnapshot = isLinuxBare || isK8sNakedPod;
+ if (requiresSnapshot && (!snapshot || snapshot === "{}")) {
+ return Response.json(
+ {
+ error:
+ "No saved snapshot is available to bring this workload back up. The controller cannot recreate naked pods or bare processes without a snapshot captured at Stop time.",
+ },
+ { status: 409 }
+ );
+ }
+
+ await updateFeatureDesiredState(
+ workloadKey,
+ clusterId,
+ FEATURE,
+ DESIRED_RUNNING,
+ snapshot,
+ dbConfigId
+ );
+ await Promise.all(
+ ctx.targets.map((cid) =>
+ queueAction(
+ cid,
+ KNOWN_ACTIONS.START_WORKLOAD,
+ workloadKey,
+ snapshot,
+ dbConfigId
+ )
+ )
+ );
+ return Response.json({
+ status: "queued",
+ action: KNOWN_ACTIONS.START_WORKLOAD,
+ controllers: ctx.targets.length,
+ });
+ }
+
+ // Restart: keep desired_status='running' (no flip). We do not
+ // touch desired_states here because the reconcile loop must
+ // not interpret Restart as a continuous desired-state
+ // mismatch.
+ await Promise.all(
+ ctx.targets.map((cid) =>
+ queueAction(
+ cid,
+ KNOWN_ACTIONS.RESTART_WORKLOAD,
+ workloadKey,
+ "{}",
+ dbConfigId
+ )
+ )
+ );
+ return Response.json({
+ status: "queued",
+ action: KNOWN_ACTIONS.RESTART_WORKLOAD,
+ controllers: ctx.targets.length,
+ });
+ } catch (error: any) {
+ return Response.json(
+ { error: error.message || "Lifecycle operation failed" },
+ { status: 500 }
+ );
+ }
+ },
+
+ reconcile(
+ reportedServices: ReportedService[],
+ desiredStates: Map<string, FeatureDesiredState>,
+ _envConfig?: EnvironmentFeatureConfig
+ ): ReconcileAction[] {
+ const actions: ReconcileAction[] = [];
+
+ for (const svc of reportedServices) {
+ const desired = desiredStates.get(svc.workload_key);
+ if (!desired) continue;
+
+ const actual =
+ svc.resource_attributes?.["openlit.lifecycle.status"] ||
+ "running";
+
+ if (
+ desired.desired_status === DESIRED_RUNNING &&
+ actual === "stopped"
+ ) {
+ actions.push({
+ actionType: KNOWN_ACTIONS.START_WORKLOAD,
+ serviceKey: svc.workload_key,
+ payload: desired.config || "{}",
+ });
+ }
+ if (
+ desired.desired_status === DESIRED_STOPPED &&
+ actual !== "stopped"
+ ) {
+ actions.push({
+ actionType: KNOWN_ACTIONS.STOP_WORKLOAD,
+ serviceKey: svc.workload_key,
+ payload: "{}",
+ });
+ }
+ // Restart is intentionally NOT reconciled. It's a one-shot
+ // action queued directly from applyOperation -- a reconcile
+ // emission would either loop forever or paper over a genuine
+ // stop with a restart.
+ }
+
+ return actions;
+ },
+};
+
+registerFeature(lifecycleHandler);
+
+export default lifecycleHandler;
diff --git a/src/client/src/lib/platform/controller/index.ts b/src/client/src/lib/platform/controller/index.ts
index e9679e01..39d6050a 100644
--- a/src/client/src/lib/platform/controller/index.ts
+++ b/src/client/src/lib/platform/controller/index.ts
@@ -685,6 +685,47 @@ export async function markActionsAcknowledged(
);
}
+/**
+ * Look up a batch of actions by ID for a given controller instance.
+ * Used by the poll route to inspect action_type / service_key for
+ * lifecycle Stop actions whose results carry a snapshot blob that
+ * needs persisting into desired_states_v2.config. Returning the
+ * created_at lets callers preserve ordering when sorting fanned-out
+ * fan-in actions.
+ */
+export async function getActionsByIds(
+ actionIds: string[],
+ instanceId: string,
+ dbConfigId?: string
+): Promise<{
+ err?: unknown;
+ data?: Array<{
+ id: string;
+ action_type: ActionType;
+ service_key: string;
+ }>;
+}> {
+ if (actionIds.length === 0) return { data: [] };
+ const escaped = actionIds
+ .map((id) => `'${escapeClickHouse(id)}'`)
+ .join(", ");
+ const query = `
+ SELECT id, action_type, service_key
+ FROM ${CONTROLLER_ACTIONS_TABLE}
+ FINAL
+ WHERE instance_id = '${escapeClickHouse(instanceId)}'
+ AND id IN (${escaped})
+ `;
+ return dataCollector({ query }, "query", dbConfigId) as Promise<{
+ err?: unknown;
+ data?: Array<{
+ id: string;
+ action_type: ActionType;
+ service_key: string;
+ }>;
+ }>;
+}
+
export async function completeAction(
actionId: string,
instanceId: string,
diff --git a/src/client/src/types/agents.ts b/src/client/src/types/agents.ts
index d92ec6e8..ab4138dd 100644
--- a/src/client/src/types/agents.ts
+++ b/src/client/src/types/agents.ts
@@ -58,6 +58,26 @@ export interface UnifiedAgent {
desired_instrumentation_status: "none" | "instrumented";
agent_observability_status: "enabled" | "disabled" | "manual" | "";
desired_agent_status: "none" | "enabled";
+ /**
+ * Current lifecycle state of the workload, rolled up from
+ * `resource_attributes['openlit.lifecycle.status']` on
+ * `openlit_controller_services` plus the matching
+ * `feature='lifecycle'` row in `openlit_controller_desired_states_v2`.
+ *
+ * - `running`: the controller's heartbeat reports the workload as up.
+ * - `stopped`: the user clicked Stop and the workload is scaled down.
+ * The row is still surfaced because the controller's desired-state
+ * table holds `desired_status='stopped'` indefinitely (up to the
+ * 90-day TTL) so the agent does not disappear from the list.
+ * - `restarting`: a Restart action is in flight.
+ * - `unknown`: SDK-only agents or rows the controller hasn't reported on.
+ */
+ lifecycle_status: "running" | "stopped" | "restarting" | "unknown";
+ /**
+ * Mirrors `agent_observability` desired-state shape so the UI can compare
+ * actual vs desired and render transition states identically.
+ */
+ desired_lifecycle_status: "running" | "stopped" | "unknown";
pending_action: string | null;
pending_action_status: "pending" | "acknowledged" | null;
diff --git a/src/client/src/types/controller.ts b/src/client/src/types/controller.ts
index e8eac3cb..e6b21b45 100644
--- a/src/client/src/types/controller.ts
+++ b/src/client/src/types/controller.ts
@@ -121,6 +121,9 @@ export const KNOWN_ACTIONS = {
REMOVE_PROMPTS: "remove_prompts",
PUSH_ENVS: "push_envs",
REMOVE_ENVS: "remove_envs",
+ START_WORKLOAD: "start_workload",
+ STOP_WORKLOAD: "stop_workload",
+ RESTART_WORKLOAD: "restart_workload",
} as const;
export interface FeatureDesiredState {