Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
Play pause stop (#1192)
Aman Agarwal7ca59852f631
diff --git a/openlit-controller/cmd/controller/main.go b/openlit-controller/cmd/controller/main.goindex 81c49c99..8171fcaa 100644--- a/openlit-controller/cmd/controller/main.go+++ b/openlit-controller/cmd/controller/main.go@@ -223,6 +223,7 @@ func doPoll(AgentObservabilitySource: svc.AgentObservabilitySource,ObservabilityConflict: svc.ObservabilityConflict,ObservabilityReason: svc.ObservabilityReason,+ LifecycleStatus: svc.LifecycleStatus,FirstSeen: svc.FirstSeen.UTC().Format("2006-01-02 15:04:05"),ResourceAttributes: svc.ResourceAttributes,})@@ -304,6 +305,7 @@ func doPoll(func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zap.Logger) openlit.ActionResult {var execErr error+ var snapshot stringswitch action.ActionType {case openlit.ActionInstrument:@@ -324,6 +326,21 @@ func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zapbreak}execErr = eng.DisablePythonSDK(action.ServiceKey, payload)+ case openlit.ActionStartWorkload:+ execErr = eng.StartWorkload(action.ServiceKey, action.Payload)+ case openlit.ActionStopWorkload:+ // StopWorkload returns a snapshot blob the dashboard persists+ // into desired_states_v2.config so a later Start can hand it+ // back. For K8s controlled workloads the blob carries just+ // (kind, ns, name, container_name); for K8s naked pods it+ // carries the full gzipped pod spec; for Linux bare processes+ // it carries (exe_path, argv, cwd, env-allowlist). Docker and+ // Linux systemd modes return "" because the durable identifier+ // (container name, unit name) is already encoded in+ // workload_key and the runtime preserves the object across Stop.+ snapshot, execErr = eng.StopWorkload(action.ServiceKey, action.Payload)+ case openlit.ActionRestartWorkload:+ execErr = eng.RestartWorkload(action.ServiceKey, action.Payload)default:logger.Warn("unknown action type", zap.String("type", action.ActionType))return openlit.ActionResult{@@ -354,6 +371,7 @@ func executeAction(eng *engine.Engine, action openlit.PendingAction, logger *zapreturn openlit.ActionResult{ActionID: action.ID,Status: "completed",+ Snapshot: snapshot,}}diff --git a/openlit-controller/internal/engine/container.go b/openlit-controller/internal/engine/container.goindex 0803ae9f..e96da17f 100644--- a/openlit-controller/internal/engine/container.go+++ b/openlit-controller/internal/engine/container.go@@ -429,6 +429,186 @@ func (c *k8sAPIClient) patchWorkload(namespace, kind, name string, patch map[strreturn nil}+// putWorkload performs an HTTP PUT against the workload resource with the+// given full object. K8s uses the embedded `metadata.resourceVersion` for+// optimistic concurrency: callers GET the object, mutate it, then PUT it+// back; a 409 Conflict means someone else updated it first and the caller+// should retry. Used by atomicScaleWithAnnotation and bumpRolloutAnnotation+// to avoid blind merge-patch races when multiple DaemonSet controllers act+// on the same workload at once.+func (c *k8sAPIClient) putWorkload(namespace, kind, name string, workload map[string]any) (int, error) {+ resource, err := k8sWorkloadResource(kind)+ if err != nil {+ return 0, err+ }+ body, err := json.Marshal(workload)+ if err != nil {+ return 0, fmt.Errorf("marshal workload: %w", err)+ }+ endpoint := fmt.Sprintf(+ "%s/apis/apps/v1/namespaces/%s/%s/%s",+ c.apiServer,+ url.PathEscape(namespace),+ resource,+ url.PathEscape(name),+ )+ req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(body))+ if err != nil {+ return 0, err+ }+ req.Header.Set("Authorization", "Bearer "+c.token)+ req.Header.Set("Content-Type", "application/json")++ resp, err := c.httpClient.Do(req)+ if err != nil {+ return 0, err+ }+ defer resp.Body.Close()++ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ responseBody, _ := io.ReadAll(resp.Body)+ return resp.StatusCode, fmt.Errorf("K8s workload put returned %d: %s", resp.StatusCode, strings.TrimSpace(string(responseBody)))+ }+ return resp.StatusCode, nil+}++// atomicScaleWithAnnotation scales a workload to the target replica count,+// recording the prior count in an annotation in the same atomic update.+// It uses a read-modify-update loop with resourceVersion-based optimistic+// concurrency: on 409 Conflict (someone else patched the workload first),+// it re-reads and retries up to 3 times. The operation is idempotent: if+// the workload is already at the target replicas AND the annotation+// matches the recorded value, it returns nil without making an API call.+//+// recordAnnotationKey/Value semantics:+// - Pass empty annotationKey to skip annotation handling entirely.+// - When scaling down (replicas==0) and the workload was previously >0,+// the prior replica count is recorded under recordAnnotationKey so+// scale-up can restore it.+// - When scaling up (replicas>0), the annotation is removed.+func (c *k8sAPIClient) atomicScaleWithAnnotation(namespace, kind, name string, replicas int, recordAnnotationKey string) error {+ const maxRetries = 3+ for attempt := 0; attempt < maxRetries; attempt++ {+ workload, err := c.getWorkload(namespace, kind, name)+ if err != nil {+ return err+ }+ spec, _ := workload["spec"].(map[string]any)+ if spec == nil {+ return fmt.Errorf("workload %s/%s has no spec", kind, name)+ }+ current := readReplicas(spec)+ meta, _ := workload["metadata"].(map[string]any)+ annotations := copyStringMap(meta["annotations"])++ alreadyAtTarget := current == replicas+ annotationMatches := true+ if recordAnnotationKey != "" {+ if replicas == 0 {+ _, present := annotations[recordAnnotationKey]+ annotationMatches = current == 0 && present+ } else {+ _, present := annotations[recordAnnotationKey]+ annotationMatches = !present+ }+ }+ if alreadyAtTarget && annotationMatches {+ return nil+ }++ spec["replicas"] = replicas+ if recordAnnotationKey != "" {+ if replicas == 0 && current > 0 {+ annotations[recordAnnotationKey] = fmt.Sprintf("%d", current)+ } else if replicas > 0 {+ delete(annotations, recordAnnotationKey)+ }+ }+ if meta == nil {+ meta = map[string]any{}+ workload["metadata"] = meta+ }+ meta["annotations"] = stringMapToAny(annotations)++ status, err := c.putWorkload(namespace, kind, name, workload)+ if err == nil {+ return nil+ }+ if status == http.StatusConflict {+ continue+ }+ return err+ }+ return fmt.Errorf("atomic scale conflict after %d retries", maxRetries)+}++// readSavedReplicas returns the value of the saved-replicas annotation on a+// workload, falling back to 1 when missing or unparsable. Used to decide+// what to scale back to on Start.+func (c *k8sAPIClient) readSavedReplicas(namespace, kind, name, annotationKey string) (int, error) {+ workload, err := c.getWorkload(namespace, kind, name)+ if err != nil {+ return 0, err+ }+ meta, _ := workload["metadata"].(map[string]any)+ annotations := copyStringMap(meta["annotations"])+ raw, ok := annotations[annotationKey]+ if !ok {+ return 1, nil+ }+ var saved int+ _, scanErr := fmt.Sscanf(raw, "%d", &saved)+ if scanErr != nil || saved < 1 {+ return 1, nil+ }+ return saved, nil+}++// bumpRolloutAnnotation patches the workload pod template annotations to+// trigger a rolling restart (same trick `kubectl rollout restart` uses).+// Uses an atomic read-modify-update so multiple controllers racing on the+// same workload do not lose each other's writes.+func (c *k8sAPIClient) bumpRolloutAnnotation(namespace, kind, name, annotationKey, annotationValue string) error {+ const maxRetries = 3+ for attempt := 0; attempt < maxRetries; attempt++ {+ workload, err := c.getWorkload(namespace, kind, name)+ if err != nil {+ return err+ }+ template, err := getTemplate(workload)+ if err != nil {+ return err+ }+ tmplMeta := ensureMap(template, "metadata")+ annotations := copyStringMap(tmplMeta["annotations"])+ if existing, ok := annotations[annotationKey]; ok && existing == annotationValue {+ return nil+ }+ annotations[annotationKey] = annotationValue+ tmplMeta["annotations"] = stringMapToAny(annotations)++ status, err := c.putWorkload(namespace, kind, name, workload)+ if err == nil {+ return nil+ }+ if status == http.StatusConflict {+ continue+ }+ return err+ }+ return fmt.Errorf("rollout-restart conflict after %d retries", maxRetries)+}++func readReplicas(spec map[string]any) int {+ if v, ok := spec["replicas"].(float64); ok {+ return int(v)+ }+ if v, ok := spec["replicas"].(int); ok {+ return v+ }+ return 0+}+func k8sWorkloadResource(kind string) (string, error) {switch strings.ToLower(kind) {case "deployment":@@ -790,6 +970,23 @@ func (d *dockerAPIClient) stopContainer(containerID string, timeoutSeconds int)return nil}+// restartContainer issues a native Docker restart for the given container.+// The container's name and ID are both preserved, so the controller's+// workload_key (docker:<name>) remains stable across the restart.+func (d *dockerAPIClient) restartContainer(containerID string, timeoutSeconds int) error {+ endpoint := fmt.Sprintf("/containers/%s/restart?t=%d", containerID, timeoutSeconds)+ resp, err := d.doRequest(http.MethodPost, endpoint, nil, "")+ if err != nil {+ return err+ }+ defer resp.Body.Close()+ if resp.StatusCode != http.StatusNoContent {+ body, _ := io.ReadAll(resp.Body)+ return fmt.Errorf("docker restart returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))+ }+ return nil+}+func (d *dockerAPIClient) renameContainer(containerID, newName string) error {resp, err := d.doRequest(http.MethodPost, fmt.Sprintf("/containers/%s/rename?name=%s", containerID, url.QueryEscape(newName)), nil, "")if err != nil {diff --git a/openlit-controller/internal/engine/engine.go b/openlit-controller/internal/engine/engine.goindex 8437db24..b9c6df07 100644--- a/openlit-controller/internal/engine/engine.go+++ b/openlit-controller/internal/engine/engine.go@@ -66,6 +66,23 @@ const (CapabilityPythonSDKDockerV1 = "python_sdk_injection_docker_v1"CapabilityPythonSDKLinuxSystemdV1 = "python_sdk_injection_linux_systemd_v1"CapabilityPythonSDKLinuxBareV1 = "python_sdk_injection_linux_bare_v1"+ CapabilityLifecycleKubernetesV1 = "lifecycle_kubernetes_v1"+ CapabilityLifecycleDockerV1 = "lifecycle_docker_v1"+ CapabilityLifecycleLinuxSystemdV1 = "lifecycle_linux_systemd_v1"+ CapabilityLifecycleLinuxBareV1 = "lifecycle_linux_bare_v1"+)++// Lifecycle resource-attribute / annotation keys shared between the+// controller heartbeat and the dashboard rollup. Keep these constants in+// sync with the SQL projections in+// src/client/src/lib/platform/agents/index.ts.+const (+ LifecycleStatusAttr = "openlit.lifecycle.status"+ LifecycleStatusRunning = "running"+ LifecycleStatusStopped = "stopped"+ LifecycleStatusRestarting = "restarting"+ K8sSavedReplicasAnnotation = "openlit.io/saved-replicas"+ K8sRolloutRestartAnnotation = "openlit.io/restartedAt")func New(logger *zap.Logger, obiBinaryPath, otlpEndpoint, procRoot, environment, sdkVersion string, mode config.DeployMode) *Engine {@@ -397,15 +414,19 @@ func (e *Engine) ControllerCapabilities() []string {case config.DeployKubernetes:if e.container != nil && e.container.k8sClient != nil {capabilities = append(capabilities, CapabilityPythonSDKKubernetesV1)+ capabilities = append(capabilities, CapabilityLifecycleKubernetesV1)}case config.DeployDocker:if e.container != nil && e.container.dockerClient != nil && e.container.dockerClient.canManage() {capabilities = append(capabilities, CapabilityPythonSDKDockerV1)+ capabilities = append(capabilities, CapabilityLifecycleDockerV1)}default:capabilities = append(capabilities, CapabilityPythonSDKLinuxBareV1)+ capabilities = append(capabilities, CapabilityLifecycleLinuxBareV1)if linuxSystemdSDKSupported() {capabilities = append(capabilities, CapabilityPythonSDKLinuxSystemdV1)+ capabilities = append(capabilities, CapabilityLifecycleLinuxSystemdV1)}}@@ -494,3 +515,57 @@ func (e *Engine) ServiceCount() (discovered, instrumented int) {}return}++// setLifecycleState stamps the lifecycle status onto the matching in-memory+// service so the next heartbeat reports it. Mirrors the+// applyAgentStateLocked / setAgentObservabilityState pattern in python_sdk.go+// and uses the same resource_attributes channel+// (openlit.lifecycle.status) for the dashboard rollup.+//+// On Stop the scanner is likely to lose sight of the workload immediately+// (the process exits / container exits / pod is gone), so this helper is the+// last chance the controller has to surface the "stopped" state to the UI+// before the row falls out of the heartbeat window. We also refresh+// LastSeen so the row survives the next prune cycle (30-minute+// not-instrumented cutoff in pruneStaleServices) while the user decides+// whether to Play it back.+func (e *Engine) setLifecycleState(workloadKey, status string) {+ e.mu.Lock()+ defer e.mu.Unlock()++ svc, ok := e.services[workloadKey]+ if !ok {+ return+ }+ svc.LifecycleStatus = status+ svc.LastSeen = time.Now()+ if svc.ResourceAttributes == nil {+ svc.ResourceAttributes = make(map[string]string)+ }+ if status != "" {+ svc.ResourceAttributes[LifecycleStatusAttr] = status+ } else {+ delete(svc.ResourceAttributes, LifecycleStatusAttr)+ }+}++// snapshotService returns a defensive copy of a service entry so callers+// can read its identity (workload_key, namespace, service_name,+// container name) without holding the engine lock.+func (e *Engine) snapshotService(workloadKey string) (*openlit.ServiceState, error) {+ e.mu.RLock()+ defer e.mu.RUnlock()+ svc, ok := e.services[workloadKey]+ if !ok {+ return nil, fmt.Errorf("service %q not found", workloadKey)+ }+ copied := *svc+ if svc.ResourceAttributes != nil {+ clone := make(map[string]string, len(svc.ResourceAttributes))+ for k, v := range svc.ResourceAttributes {+ clone[k] = v+ }+ copied.ResourceAttributes = clone+ }+ return &copied, nil+}diff --git a/openlit-controller/internal/engine/lifecycle.go b/openlit-controller/internal/engine/lifecycle.gonew file mode 100644index 00000000..82278e15--- /dev/null+++ b/openlit-controller/internal/engine/lifecycle.go@@ -0,0 +1,715 @@+// Package engine — lifecycle.go+//+// Implements the controller-side Play / Stop / Restart actions queued from+// the dashboard. The critical invariant every code path here MUST preserve+// is workload identity: an agent that gets stopped and started again must+// keep the same workload_key (see events.go::buildWorkloadKey) and the same+// service_name so the dashboard rollup re-points to the *same* agent_key+// row instead of creating a duplicate.+//+// Per-mode identity preservation rules:+// - K8s controlled workload: scale-to-0 (preserves Deployment/STS/DS+// object, container_name, deployment_name); rollout-restart is an+// annotation bump on the pod template.+// - K8s naked pod: snapshot the pod spec before delete, recreate with the+// *same* metadata.name on Start.+// - Docker: stopContainer / startContainer / restartContainer all run+// against the same container ID; name + ID are stable.+// - Linux systemd: systemctl start/stop/restart of the same unit name.+// - Linux bare process: capture (exe, args, cwd, env-allowlist) before+// SIGTERM; re-exec with the same args produces an identical+// sha1(exe|cmdline) fingerprint.++package engine++import (+ "bytes"+ "compress/gzip"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "os/exec"+ "strings"+ "syscall"+ "time"++ "github.com/openlit/openlit/openlit-controller/internal/config"+ "github.com/openlit/openlit/openlit-controller/internal/openlit"+ "go.uber.org/zap"+)++// lifecycleBareProcessSnapshot is what we persist into the desired-state+// config blob on Stop for a Linux bare process so Start can re-exec it+// identically. The workload_key fingerprint that the dashboard rolls up+// on is sha1(/proc/PID/exe-resolved | argv-cmdline), so the only fields+// we strictly need to preserve identity are Args (argv) and the runtime+// binary path that PATH resolution lands on for Args[0]. Cwd is needed+// to give relative argv[0]s the same starting directory. The env+// allowlist intentionally excludes arbitrary user env to avoid leaking+// secrets through the dashboard.+//+// ExePath here is captured from argv[0] (i.e. the launch path the user+// typed) rather than the resolved /proc/PID/exe symlink — it is purely+// informational metadata for operators reading the desired-state row.+// launchBareProcess relies on Args + Cwd to recreate identical argv ++// PATH resolution on Start.+type lifecycleBareProcessSnapshot struct {+ ExePath string `json:"exe_path"`+ Args []string `json:"args"`+ Cwd string `json:"cwd"`+ Env map[string]string `json:"env"`+}++// lifecycleK8sPodSnapshot wraps a gzip+base64-encoded pod spec. We+// serialize it this way because real-world pod specs can be 5-15KB; the+// desired-states config column is plain String and gets piped through+// ClickHouse JSONEachRow, so keeping it compact + ASCII-safe matters.+//+// `Controlled` is set for Deployment/StatefulSet/DaemonSet workloads so+// startK8s can re-derive the kind/namespace/name needed to scale back+// up even when the controller's in-memory service cache has been+// pruned (>30 min after Stop, or controller restart). Without it, a+// long-stopped controlled workload becomes unrecoverable from the UI.+//+// Exactly one of `Controlled` or `GzippedPodB64` is populated per+// snapshot: controlled workloads do not carry pod specs because Start+// simply patches the existing Deployment back up, and naked pods do+// not carry a Controlled struct because there is no parent object.+type lifecycleK8sPodSnapshot struct {+ GzippedPodB64 string `json:"gzipped_pod_b64,omitempty"`+ Controlled *lifecycleK8sControlledSnapshot `json:"controlled,omitempty"`+}++type lifecycleK8sControlledSnapshot struct {+ Kind string `json:"kind"`+ Namespace string `json:"namespace"`+ Name string `json:"name"`+ ContainerName string `json:"container_name,omitempty"`+}++// envAllowlistPrefixes captures the env vars Stop/Start should preserve+// for bare-process restart. Everything else (auth tokens, DB passwords,+// arbitrary user env) is dropped on purpose -- the controller is not a+// secret store. PATH/HOME keep the new process able to find its libs.+var envAllowlistPrefixes = []string{+ "PATH",+ "HOME",+ "LANG",+ "LC_",+ "PYTHONPATH",+ "OPENLIT_",+ "OTEL_",+}++// StartWorkload brings a previously-stopped workload back online. payload+// is the raw JSON string queued by the dashboard; for naked pods and bare+// processes it carries the snapshot captured at Stop time.+func (e *Engine) StartWorkload(workloadKey, payload string) error {+ svc, err := e.snapshotService(workloadKey)+ if err != nil {+ // For K8s naked pods + Linux bare processes the service is gone+ // from e.services by the time Start arrives (Stop removed it).+ // We still need to act, but with only the workload_key and the+ // payload to go on. The payload carries the snapshot in those+ // cases; for other modes we cannot proceed without an in-memory+ // service entry.+ if isModeRequiringInMemory(e.deployMode) {+ return err+ }+ }++ switch e.deployMode {+ case config.DeployKubernetes:+ return e.startK8s(workloadKey, svc, payload)+ case config.DeployDocker:+ if svc == nil {+ return fmt.Errorf("docker workload %q not in scanner cache", workloadKey)+ }+ return e.startDocker(svc)+ default:+ return e.startLinux(workloadKey, svc, payload)+ }+}++// StopWorkload takes a running workload offline. The returned snapshot is+// included in the ActionResult so the dashboard can persist it into+// desired_states_v2.config and feed it back on the next Start.+func (e *Engine) StopWorkload(workloadKey, _ string) (string, error) {+ svc, err := e.snapshotService(workloadKey)+ if err != nil {+ return "", err+ }++ switch e.deployMode {+ case config.DeployKubernetes:+ return e.stopK8s(svc)+ case config.DeployDocker:+ return "", e.stopDocker(svc)+ default:+ return e.stopLinux(svc)+ }+}++// RestartWorkload is identity-preserving: pod names / container IDs /+// systemd unit names are unchanged across the operation.+func (e *Engine) RestartWorkload(workloadKey, _ string) error {+ svc, err := e.snapshotService(workloadKey)+ if err != nil {+ return err+ }++ switch e.deployMode {+ case config.DeployKubernetes:+ return e.restartK8s(svc)+ case config.DeployDocker:+ return e.restartDocker(svc)+ default:+ return e.restartLinux(svc)+ }+}++func isModeRequiringInMemory(mode config.DeployMode) bool {+ // For K8s the dashboard-supplied payload + workload_key are enough to+ // recreate naked pods; controlled workloads can be looked up directly+ // from the API server. For Linux bare we likewise need only the+ // payload + workload_key. Docker is the odd one out: Start runs+ // against a stopped container, and we need its current container.id+ // from the local scanner cache to issue the start.+ return mode == config.DeployDocker+}++// ----- Kubernetes ---------------------------------------------------------++func (e *Engine) stopK8s(svc *openlit.ServiceState) (string, error) {+ if e.container == nil || e.container.k8sClient == nil {+ return "", fmt.Errorf("kubernetes client unavailable for lifecycle stop")+ }+ namespace := svc.ResourceAttributes["k8s.namespace.name"]+ workloadKind := svc.ResourceAttributes["k8s.workload.kind"]+ workloadName := svc.DeploymentName+ if workloadKind == "" || workloadKind == "Pod" {+ return e.stopK8sNakedPod(svc)+ }+ if namespace == "" || workloadName == "" {+ return "", fmt.Errorf("missing k8s workload metadata for %s", svc.ServiceName)+ }+ if err := e.container.k8sClient.atomicScaleWithAnnotation(+ namespace, workloadKind, workloadName, 0, K8sSavedReplicasAnnotation,+ ); err != nil {+ return "", err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)+ e.logger.Info("k8s workload stopped",+ zap.String("workload_key", svc.WorkloadKey),+ zap.String("kind", workloadKind),+ zap.String("name", workloadName),+ )+ // Persist a small identifier blob so a later Start can re-derive+ // (kind, ns, name) without depending on the controller's in-memory+ // cache — which gets pruned after 30 min, or wiped entirely on+ // controller restart.+ snap := lifecycleK8sPodSnapshot{+ Controlled: &lifecycleK8sControlledSnapshot{+ Kind: workloadKind,+ Namespace: namespace,+ Name: workloadName,+ ContainerName: svc.ResourceAttributes["container.name"],+ },+ }+ out, err := json.Marshal(snap)+ if err != nil {+ // The scale already happened; losing the snapshot just degrades+ // recovery, it doesn't undo Stop. Return success so the action+ // completes and the UI reflects the stopped state.+ e.logger.Warn("k8s controlled snapshot marshal failed",+ zap.String("workload_key", svc.WorkloadKey),+ zap.Error(err),+ )+ return "", nil+ }+ return string(out), nil+}++func (e *Engine) startK8s(workloadKey string, svc *openlit.ServiceState, payload string) error {+ if e.container == nil || e.container.k8sClient == nil {+ return fmt.Errorf("kubernetes client unavailable for lifecycle start")+ }++ // Dispatch order:+ // 1. If the payload carries a parsed snapshot, trust it. This is+ // the durable source of truth — it survives controller cache+ // eviction and controller restarts.+ // 2. Else fall back to the in-memory svc (fresh Stop→Play within+ // the prune window).+ // 3. Else (no payload, no svc) we're out of options — fail+ // explicitly so the user sees a clear error.+ if payload != "" && payload != "{}" {+ var parsed lifecycleK8sPodSnapshot+ if err := json.Unmarshal([]byte(payload), &parsed); err == nil {+ if parsed.Controlled != nil {+ return e.startK8sControlledFromSnapshot(workloadKey, parsed.Controlled)+ }+ if parsed.GzippedPodB64 != "" {+ return e.startK8sNakedPod(workloadKey, svc, payload)+ }+ }+ }++ if svc != nil && (svc.ResourceAttributes["k8s.workload.kind"] != "" && svc.ResourceAttributes["k8s.workload.kind"] != "Pod") {+ return e.startK8sControlledFromSnapshot(workloadKey, &lifecycleK8sControlledSnapshot{+ Kind: svc.ResourceAttributes["k8s.workload.kind"],+ Namespace: svc.ResourceAttributes["k8s.namespace.name"],+ Name: svc.DeploymentName,+ })+ }++ // Naked pod path. svc may be nil here (the pod was deleted on Stop).+ return e.startK8sNakedPod(workloadKey, svc, payload)+}++// startK8sControlledFromSnapshot scales a Deployment/STS/DS workload+// back up using the saved-replicas annotation, given just its+// (kind, namespace, name). Works whether or not the controller has+// the workload in its in-memory cache.+func (e *Engine) startK8sControlledFromSnapshot(workloadKey string, snap *lifecycleK8sControlledSnapshot) error {+ if snap == nil || snap.Kind == "" || snap.Namespace == "" || snap.Name == "" {+ return fmt.Errorf("controlled snapshot missing kind/namespace/name (workload %s)", workloadKey)+ }+ saved, err := e.container.k8sClient.readSavedReplicas(+ snap.Namespace, snap.Kind, snap.Name, K8sSavedReplicasAnnotation,+ )+ if err != nil {+ return err+ }+ if err := e.container.k8sClient.atomicScaleWithAnnotation(+ snap.Namespace, snap.Kind, snap.Name, saved, K8sSavedReplicasAnnotation,+ ); err != nil {+ return err+ }+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)+ e.logger.Info("k8s workload scaled back up",+ zap.String("workload_key", workloadKey),+ zap.String("kind", snap.Kind),+ zap.String("name", snap.Name),+ zap.Int("replicas", saved),+ )+ return nil+}++func (e *Engine) restartK8s(svc *openlit.ServiceState) error {+ if e.container == nil || e.container.k8sClient == nil {+ return fmt.Errorf("kubernetes client unavailable for lifecycle restart")+ }+ namespace := svc.ResourceAttributes["k8s.namespace.name"]+ workloadKind := svc.ResourceAttributes["k8s.workload.kind"]+ workloadName := svc.DeploymentName+ if workloadKind == "" || workloadKind == "Pod" {+ return e.restartK8sNakedPod(svc)+ }+ stamp := time.Now().UTC().Format(time.RFC3339)+ if err := e.container.k8sClient.bumpRolloutAnnotation(+ namespace, workloadKind, workloadName, K8sRolloutRestartAnnotation, stamp,+ ); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)+ e.logger.Info("k8s workload restart triggered",+ zap.String("workload_key", svc.WorkloadKey),+ zap.String("kind", workloadKind),+ zap.String("name", workloadName),+ zap.String("stamp", stamp),+ )+ return nil+}++func (e *Engine) stopK8sNakedPod(svc *openlit.ServiceState) (string, error) {+ namespace := svc.ResourceAttributes["k8s.namespace.name"]+ podName := svc.ResourceAttributes["k8s.pod.name"]+ if namespace == "" || podName == "" {+ return "", fmt.Errorf("naked pod %s missing namespace or pod name", svc.ServiceName)+ }+ pod, err := e.container.k8sClient.getPod(namespace, podName)+ if err != nil {+ return "", err+ }+ stripPodRuntimeFields(pod)+ blob, err := encodePodSnapshot(pod)+ if err != nil {+ return "", err+ }+ if err := e.container.k8sClient.deletePod(namespace, podName, 0); err != nil {+ return "", err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)+ e.logger.Info("k8s naked pod stopped",+ zap.String("workload_key", svc.WorkloadKey),+ zap.String("namespace", namespace),+ zap.String("pod", podName),+ )+ return blob, nil+}++func (e *Engine) startK8sNakedPod(workloadKey string, svc *openlit.ServiceState, payload string) error {+ if payload == "" || payload == "{}" {+ return fmt.Errorf("naked pod start requires a snapshot payload (workload %s)", workloadKey)+ }+ pod, err := decodePodSnapshot(payload)+ if err != nil {+ return err+ }+ namespace := podMetaString(pod, "namespace")+ podName := podMetaString(pod, "name")+ if namespace == "" || podName == "" {+ return fmt.Errorf("naked pod snapshot missing namespace or name (workload %s)", workloadKey)+ }+ stripPodRuntimeFields(pod)+ if err := e.container.k8sClient.createPod(namespace, pod); err != nil {+ return err+ }+ // At this point the scanner will rediscover the pod and re-populate+ // e.services. We still seed the lifecycle status so the next poll+ // reports "running" even if it lands before the rediscovery.+ if svc != nil {+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)+ }+ e.logger.Info("k8s naked pod started",+ zap.String("workload_key", workloadKey),+ zap.String("namespace", namespace),+ zap.String("pod", podName),+ )+ return nil+}++func (e *Engine) restartK8sNakedPod(svc *openlit.ServiceState) error {+ blob, err := e.stopK8sNakedPod(svc)+ if err != nil {+ return err+ }+ if err := e.startK8sNakedPod(svc.WorkloadKey, svc, blob); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRunning)+ return nil+}++func encodePodSnapshot(pod map[string]any) (string, error) {+ raw, err := json.Marshal(pod)+ if err != nil {+ return "", fmt.Errorf("marshal pod snapshot: %w", err)+ }+ var buf bytes.Buffer+ gz := gzip.NewWriter(&buf)+ if _, err := gz.Write(raw); err != nil {+ _ = gz.Close()+ return "", fmt.Errorf("gzip pod snapshot: %w", err)+ }+ if err := gz.Close(); err != nil {+ return "", fmt.Errorf("gzip close: %w", err)+ }+ wrapped := lifecycleK8sPodSnapshot{+ GzippedPodB64: base64.StdEncoding.EncodeToString(buf.Bytes()),+ }+ out, err := json.Marshal(wrapped)+ if err != nil {+ return "", err+ }+ return string(out), nil+}++func decodePodSnapshot(blob string) (map[string]any, error) {+ var wrapped lifecycleK8sPodSnapshot+ if err := json.Unmarshal([]byte(blob), &wrapped); err != nil {+ return nil, fmt.Errorf("parse pod snapshot wrapper: %w", err)+ }+ if wrapped.GzippedPodB64 == "" {+ return nil, fmt.Errorf("pod snapshot missing gzipped body")+ }+ gzBytes, err := base64.StdEncoding.DecodeString(wrapped.GzippedPodB64)+ if err != nil {+ return nil, fmt.Errorf("decode pod snapshot: %w", err)+ }+ gz, err := gzip.NewReader(bytes.NewReader(gzBytes))+ if err != nil {+ return nil, fmt.Errorf("gunzip pod snapshot: %w", err)+ }+ defer gz.Close()+ var pod map[string]any+ if err := json.NewDecoder(gz).Decode(&pod); err != nil {+ return nil, fmt.Errorf("decode pod snapshot body: %w", err)+ }+ return pod, nil+}++func podMetaString(pod map[string]any, key string) string {+ meta, _ := pod["metadata"].(map[string]any)+ if meta == nil {+ return ""+ }+ val, _ := meta[key].(string)+ return val+}++// ----- Docker -------------------------------------------------------------++func (e *Engine) stopDocker(svc *openlit.ServiceState) error {+ if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {+ return fmt.Errorf("docker client unavailable for lifecycle stop")+ }+ containerID := svc.ResourceAttributes["container.id"]+ if containerID == "" {+ return fmt.Errorf("docker workload %s missing container.id", svc.ServiceName)+ }+ if err := e.container.dockerClient.stopContainer(containerID, 10); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)+ return nil+}++func (e *Engine) startDocker(svc *openlit.ServiceState) error {+ if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {+ return fmt.Errorf("docker client unavailable for lifecycle start")+ }+ containerID := svc.ResourceAttributes["container.id"]+ if containerID == "" {+ return fmt.Errorf("docker workload %s missing container.id", svc.ServiceName)+ }+ if err := e.container.dockerClient.startContainer(containerID); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRunning)+ return nil+}++func (e *Engine) restartDocker(svc *openlit.ServiceState) error {+ if e.container == nil || e.container.dockerClient == nil || !e.container.dockerClient.canManage() {+ return fmt.Errorf("docker client unavailable for lifecycle restart")+ }+ containerID := svc.ResourceAttributes["container.id"]+ if containerID == "" {+ return fmt.Errorf("docker workload %s missing container.id", svc.ServiceName)+ }+ if err := e.container.dockerClient.restartContainer(containerID, 10); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)+ return nil+}++// ----- Linux (systemd + bare process) -------------------------------------++func (e *Engine) stopLinux(svc *openlit.ServiceState) (string, error) {+ if unit := svc.ResourceAttributes["systemd.unit"]; unit != "" {+ if !linuxSystemdSDKSupported() {+ return "", fmt.Errorf("systemctl unavailable; cannot stop unit %s", unit)+ }+ if err := runSystemctl("stop", unit); err != nil {+ return "", err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)+ return "", nil+ }+ return e.stopBareProcess(svc)+}++func (e *Engine) startLinux(workloadKey string, svc *openlit.ServiceState, payload string) error {+ if svc != nil {+ if unit := svc.ResourceAttributes["systemd.unit"]; unit != "" {+ if !linuxSystemdSDKSupported() {+ return fmt.Errorf("systemctl unavailable; cannot start unit %s", unit)+ }+ if err := runSystemctl("start", unit); err != nil {+ return err+ }+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)+ return nil+ }+ }+ // Bare process — workload may not be in e.services anymore.+ return e.startBareProcess(workloadKey, payload)+}++func (e *Engine) restartLinux(svc *openlit.ServiceState) error {+ if unit := svc.ResourceAttributes["systemd.unit"]; unit != "" {+ if !linuxSystemdSDKSupported() {+ return fmt.Errorf("systemctl unavailable; cannot restart unit %s", unit)+ }+ if err := runSystemctl("restart", unit); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)+ return nil+ }+ return e.restartBareProcess(svc)+}++func (e *Engine) stopBareProcess(svc *openlit.ServiceState) (string, error) {+ if svc.PID <= 0 {+ return "", fmt.Errorf("bare process %s has no pid; cannot stop", svc.ServiceName)+ }+ snap := captureBareProcessSnapshot(e.procRoot, svc.PID)+ blob, err := json.Marshal(snap)+ if err != nil {+ return "", fmt.Errorf("marshal bare-process snapshot: %w", err)+ }+ if err := syscall.Kill(svc.PID, syscall.SIGTERM); err != nil {+ return "", fmt.Errorf("SIGTERM pid %d: %w", svc.PID, err)+ }+ if !waitForProcessExit(svc.PID, 5*time.Second) {+ _ = syscall.Kill(svc.PID, syscall.SIGKILL)+ waitForProcessExit(svc.PID, 2*time.Second)+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusStopped)+ return string(blob), nil+}++func (e *Engine) startBareProcess(workloadKey, payload string) error {+ if payload == "" || payload == "{}" {+ return fmt.Errorf("bare process start requires a snapshot payload (workload %s)", workloadKey)+ }+ var snap lifecycleBareProcessSnapshot+ if err := json.Unmarshal([]byte(payload), &snap); err != nil {+ return fmt.Errorf("decode bare-process snapshot: %w", err)+ }+ if snap.ExePath == "" || len(snap.Args) == 0 {+ return fmt.Errorf("bare process snapshot missing exe_path/args (workload %s)", workloadKey)+ }+ if err := launchBareProcess(snap); err != nil {+ return err+ }+ e.setLifecycleState(workloadKey, LifecycleStatusRunning)+ e.logger.Info("bare process re-launched",+ zap.String("workload_key", workloadKey),+ zap.String("exe", snap.ExePath),+ zap.Strings("args", snap.Args),+ )+ return nil+}++func (e *Engine) restartBareProcess(svc *openlit.ServiceState) error {+ if svc.PID <= 0 {+ return fmt.Errorf("bare process %s has no pid; cannot restart", svc.ServiceName)+ }+ envFull := readEnviron(e.procRoot, svc.PID)+ if envFull == nil {+ envFull = make(map[string]string)+ }+ if _, err := restartProcess(e.procRoot, svc.PID, envFull); err != nil {+ return err+ }+ e.setLifecycleState(svc.WorkloadKey, LifecycleStatusRestarting)+ return nil+}++// launchBareProcess re-execs a previously-stopped bare process from a+// captured snapshot. We deliberately mirror restartProcess's setsid setup so+// the new process detaches from the controller (otherwise it would die when+// the controller exits) but use os/exec rather than fork directly because the+// controller runs single-threaded in this code path.+//+// Trust model for the dynamic argv: the snapshot is captured at Stop+// time from /proc/<pid>/cmdline (trusted, locally read) and travels+// through the dashboard's ClickHouse `desired_states_v2.config`+// column. The argv is therefore only as trustworthy as ClickHouse+// write access. Two mitigations:+//+// 1. The controller already runs as a privileged process that can+// execute container ops, pip installs, kubectl apply, etc. — so a+// ClickHouse-write attacker has equivalent paths to RCE via the+// existing action queue. This call site does not widen the blast+// radius beyond the controller's existing trust boundary.+// 2. We still validate the argv defensively below to fail closed on+// obviously-bogus payloads (empty argv, NUL bytes, empty argv[0])+// rather than blindly forwarding to execve.+//+// Crucially, `exec.Command` does NOT spawn a shell — Go's os/exec+// invokes execve(2) directly with the argv slice, so shell+// metacharacters in any argv element are not interpreted.+func launchBareProcess(snap lifecycleBareProcessSnapshot) error {+ if err := validateBareProcessArgs(snap.Args); err != nil {+ return err+ }+ // nolint:gosec // G204: argv is taken from a controller-captured+ // snapshot of /proc/<pid>/cmdline; see the trust-model comment+ // above for why a dynamic argv here is acceptable. execve is+ // shell-free, and validateBareProcessArgs rejects malformed+ // payloads.+ cmd := exec.Command(snap.Args[0], snap.Args[1:]...)+ cmd.Dir = snap.Cwd+ envSlice := make([]string, 0, len(snap.Env))+ for k, v := range snap.Env {+ envSlice = append(envSlice, k+"="+v)+ }+ cmd.Env = envSlice+ cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}+ cmd.Stdout = nil+ cmd.Stderr = nil+ if err := cmd.Start(); err != nil {+ return fmt.Errorf("start bare process: %w", err)+ }+ go func() { _ = cmd.Wait() }()+ return nil+}++// validateBareProcessArgs guards the launchBareProcess execve call+// against obviously-malformed payloads. The rules are deliberately+// minimal — we are not trying to defend against an attacker who+// already has ClickHouse write access (see the trust-model comment on+// launchBareProcess), only against:+//+// - empty argv (would panic on Args[0])+// - empty / NUL-containing argv[0] (execve rejects NULs and they+// have caused surprising path-resolution behavior historically)+// - NUL bytes anywhere in argv (same reasoning)+//+// We intentionally do NOT reject relative argv[0]: the original+// process may legitimately have been launched as `./my-script` and+// changing that on restart would break workload_key identity (which+// folds argv into its fingerprint).+func validateBareProcessArgs(args []string) error {+ if len(args) == 0 {+ return fmt.Errorf("bare process snapshot has empty argv")+ }+ if args[0] == "" {+ return fmt.Errorf("bare process snapshot has empty argv[0]")+ }+ for i, a := range args {+ if strings.ContainsRune(a, 0) {+ return fmt.Errorf("bare process snapshot argv[%d] contains NUL byte", i)+ }+ }+ return nil+}++// captureBareProcessSnapshot reads /proc/<pid>/* to build the minimum+// payload Start needs to re-exec the workload identically. The env is+// restricted to an allowlist of prefixes (PATH, OPENLIT_*, OTEL_*, etc.) so+// the controller never exfiltrates arbitrary user env (which may contain+// secrets) through the dashboard.+func captureBareProcessSnapshot(procRoot string, pid int) lifecycleBareProcessSnapshot {+ args := readCmdlineArgs(procRoot, pid)+ cwd := readCwd(procRoot, pid)+ if cwd == "" {+ cwd = "/"+ }+ env := readEnviron(procRoot, pid)+ filtered := make(map[string]string)+ for k, v := range env {+ for _, prefix := range envAllowlistPrefixes {+ if k == prefix || strings.HasPrefix(k, prefix) {+ filtered[k] = v+ break+ }+ }+ }+ exePath := ""+ if len(args) > 0 {+ exePath = args[0]+ }+ return lifecycleBareProcessSnapshot{+ ExePath: exePath,+ Args: args,+ Cwd: cwd,+ Env: filtered,+ }+}diff --git a/openlit-controller/internal/engine/lifecycle_test.go b/openlit-controller/internal/engine/lifecycle_test.gonew file mode 100644index 00000000..bf726819--- /dev/null+++ b/openlit-controller/internal/engine/lifecycle_test.go@@ -0,0 +1,173 @@+// Tests the JSON wire format of lifecycleK8sPodSnapshot — the blob the+// controller returns from Stop, the dashboard persists into+// desired_states_v2.config, and the controller decodes on the next+// Start. Both fields are `omitempty` so each snapshot kind serialises+// to a different shape; if that invariant breaks, the dashboard's+// requiresSnapshot check in lifecycle.ts and the controller's+// startK8s dispatch in this file silently misbehave.++package engine++import (+ "encoding/json"+ "strings"+ "testing"+)++func TestLifecycleK8sSnapshot_ControlledJSON(t *testing.T) {+ snap := lifecycleK8sPodSnapshot{+ Controlled: &lifecycleK8sControlledSnapshot{+ Kind: "Deployment",+ Namespace: "default",+ Name: "demo-openai-app",+ ContainerName: "demo-openai-app",+ },+ }+ out, err := json.Marshal(snap)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ got := string(out)+ // The dashboard's requiresSnapshot path treats "" and "{}" as+ // empty. We want a non-empty body for controlled workloads so the+ // poll route persists it.+ if got == "{}" || got == "" {+ t.Fatalf("controlled snapshot serialised as empty: %q", got)+ }+ // Naked-pod field must be absent for a controlled snapshot so the+ // startK8s dispatcher does not race the wrong branch.+ if strings.Contains(got, "gzipped_pod_b64") {+ t.Fatalf("controlled snapshot leaked gzipped_pod_b64: %q", got)+ }+ if !strings.Contains(got, "\"controlled\"") {+ t.Fatalf("expected controlled key in JSON, got %q", got)+ }++ // Round-trip must preserve every field.+ var back lifecycleK8sPodSnapshot+ if err := json.Unmarshal(out, &back); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ if back.Controlled == nil {+ t.Fatalf("round-trip lost controlled struct")+ }+ if back.Controlled.Kind != "Deployment" ||+ back.Controlled.Namespace != "default" ||+ back.Controlled.Name != "demo-openai-app" ||+ back.Controlled.ContainerName != "demo-openai-app" {+ t.Fatalf("round-trip mangled fields: %+v", back.Controlled)+ }+ if back.GzippedPodB64 != "" {+ t.Fatalf("round-trip resurrected gzipped_pod_b64")+ }+}++func TestLifecycleK8sSnapshot_NakedPodJSON(t *testing.T) {+ snap := lifecycleK8sPodSnapshot{+ GzippedPodB64: "H4sIAAAA",+ }+ out, err := json.Marshal(snap)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ got := string(out)+ if strings.Contains(got, "\"controlled\"") {+ t.Fatalf("naked-pod snapshot leaked controlled key: %q", got)+ }+ if !strings.Contains(got, "gzipped_pod_b64") {+ t.Fatalf("missing gzipped_pod_b64: %q", got)+ }+}++func TestValidateBareProcessArgs(t *testing.T) {+ tests := []struct {+ name string+ args []string+ wantErr bool+ }{+ {+ name: "happy path: absolute path",+ args: []string{"/usr/bin/python3", "myapp.py"},+ },+ {+ name: "happy path: PATH-resolved binary",+ args: []string{"python3", "-m", "myapp"},+ },+ {+ name: "happy path: relative argv[0]",+ args: []string{"./run.sh", "--flag"},+ },+ {+ name: "empty argv",+ args: nil,+ wantErr: true,+ },+ {+ name: "empty argv[0]",+ args: []string{"", "myapp.py"},+ wantErr: true,+ },+ {+ name: "NUL in argv[0]",+ args: []string{"python\x003", "myapp.py"},+ wantErr: true,+ },+ {+ name: "NUL in middle argv",+ args: []string{"python3", "my\x00app.py"},+ wantErr: true,+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ err := validateBareProcessArgs(tc.args)+ if (err != nil) != tc.wantErr {+ t.Fatalf("validateBareProcessArgs(%v): err=%v, wantErr=%v", tc.args, err, tc.wantErr)+ }+ })+ }+}++func TestLifecycleK8sSnapshot_DispatchDiscriminator(t *testing.T) {+ // Mirrors the dispatch in startK8s: a controlled blob is detected+ // by parsed.Controlled != nil, a naked pod blob by+ // parsed.GzippedPodB64 != "". Each input goes down exactly one+ // branch.+ tests := []struct {+ name string+ payload string+ wantControlled bool+ wantNakedPod bool+ }{+ {+ name: "controlled",+ payload: `{"controlled":{"kind":"Deployment","namespace":"x","name":"y"}}`,+ wantControlled: true,+ },+ {+ name: "naked_pod",+ payload: `{"gzipped_pod_b64":"abc"}`,+ wantNakedPod: true,+ },+ {+ name: "empty",+ payload: `{}`,+ },+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ var parsed lifecycleK8sPodSnapshot+ if err := json.Unmarshal([]byte(tc.payload), &parsed); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ gotControlled := parsed.Controlled != nil+ gotNakedPod := parsed.GzippedPodB64 != ""+ if gotControlled != tc.wantControlled {+ t.Errorf("controlled: got %v want %v", gotControlled, tc.wantControlled)+ }+ if gotNakedPod != tc.wantNakedPod {+ t.Errorf("naked pod: got %v want %v", gotNakedPod, tc.wantNakedPod)+ }+ })+ }+}diff --git a/openlit-controller/internal/openlit/types.go b/openlit-controller/internal/openlit/types.goindex 67e2b30d..3e32b072 100644--- a/openlit-controller/internal/openlit/types.go+++ b/openlit-controller/internal/openlit/types.go@@ -15,6 +15,9 @@ const (ActionUninstrument = "uninstrument"ActionEnablePythonSDK = "enable_python_sdk"ActionDisablePythonSDK = "disable_python_sdk"+ ActionStartWorkload = "start_workload"+ ActionStopWorkload = "stop_workload"+ ActionRestartWorkload = "restart_workload")// PollRequest is sent by the controller to OpenLIT on each poll cycle.@@ -66,6 +69,13 @@ type ActionResult struct {ActionID string `json:"action_id"`Status string `json:"status"`Error string `json:"error,omitempty"`+ // Snapshot carries action-specific state back to the dashboard. For+ // stop_workload it holds a JSON blob describing how to recreate the+ // workload later (pod spec for K8s naked pods, exe+args+cwd+env subset+ // for bare Linux processes). The dashboard persists this into the+ // matching openlit_controller_desired_states_v2.config row so a later+ // start_workload action can hand it back to the controller.+ Snapshot string `json:"snapshot,omitempty"`}type DiscoveredService struct {@@ -83,8 +93,12 @@ type DiscoveredService struct {AgentObservabilitySource string `json:"agent_observability_source,omitempty"`ObservabilityConflict string `json:"observability_conflict,omitempty"`ObservabilityReason string `json:"observability_reason,omitempty"`- FirstSeen string `json:"first_seen,omitempty"`- ResourceAttributes map[string]string `json:"resource_attributes,omitempty"`+ // LifecycleStatus is the controller's last-known lifecycle state for the+ // workload ("running"/"stopped"/"restarting"). Echoed via+ // resource_attributes["openlit.lifecycle.status"] for the dashboard rollup.+ LifecycleStatus string `json:"lifecycle_status,omitempty"`+ FirstSeen string `json:"first_seen,omitempty"`+ ResourceAttributes map[string]string `json:"resource_attributes,omitempty"`}type ServiceState struct {@@ -101,6 +115,7 @@ type ServiceState struct {AgentObservabilitySource string `json:"agent_observability_source,omitempty"`ObservabilityConflict string `json:"observability_conflict,omitempty"`ObservabilityReason string `json:"observability_reason,omitempty"`+ LifecycleStatus string `json:"lifecycle_status,omitempty"`FirstSeen time.Time `json:"first_seen"`LastSeen time.Time `json:"last_seen"`PID int `json:"pid,omitempty"`diff --git a/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts b/src/client/src/__tests__/lib/platform/agents/observability-view.test.tsindex e4fb6f1d..b6cec169 100644--- a/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts+++ b/src/client/src/__tests__/lib/platform/agents/observability-view.test.ts@@ -25,6 +25,7 @@ function makeAgent(overrides: Partial<UnifiedAgent> = {}): UnifiedAgent {service_name: "svc",environment: "prod",cluster_id: "default",+ workload_key: "docker:svc",source: "controller",controller_service_id: "ctrl-1",controller_instance_id: "inst-1",@@ -42,6 +43,8 @@ function makeAgent(overrides: Partial<UnifiedAgent> = {}): UnifiedAgent {desired_instrumentation_status: "none",agent_observability_status: "",desired_agent_status: "none",+ lifecycle_status: "running",+ desired_lifecycle_status: "unknown",pending_action: null,pending_action_status: null,first_seen: "2026-05-12 00:00:00",diff --git a/src/client/src/app/(playground)/agents/[agentKey]/page.tsx b/src/client/src/app/(playground)/agents/[agentKey]/page.tsxindex d02cac52..ce0a003c 100644--- a/src/client/src/app/(playground)/agents/[agentKey]/page.tsx+++ b/src/client/src/app/(playground)/agents/[agentKey]/page.tsx@@ -26,6 +26,7 @@ import { getObservabilityView } from "@/lib/platform/agents/observability-view";import AgentHeader from "@/components/(playground)/agents/agent-header";import AgentScopeProvider from "@/components/(playground)/agents/agent-scope-provider";import AgentOverviewTab from "@/components/(playground)/agents/agent-overview-tab";+import LifecycleActions from "@/components/(playground)/agents/lifecycle-actions";import VersionTimeline, {VersionChooser,} from "@/components/(playground)/agents/version-timeline";@@ -186,16 +187,18 @@ export default function AgentDetailPage() {// fetch while the table advances within seconds.const llmIntent = useAgentIntent(agentKey, "llm");const agentIntent = useAgentIntent(agentKey, "agent");+ const lifecycleIntent = useAgentIntent(agentKey, "lifecycle");const detailTransitioning = useMemo(() => {if (!agent) return false;if (getObservabilityView(agent, "llm", llmIntent).transitioning ||- getObservabilityView(agent, "agent", agentIntent).transitioning+ getObservabilityView(agent, "agent", agentIntent).transitioning ||+ getObservabilityView(agent, "lifecycle", lifecycleIntent).transitioning) {return true;}return false;- }, [agent, llmIntent, agentIntent]);+ }, [agent, llmIntent, agentIntent, lifecycleIntent]);const detailPollInFlightRef = useRef(false);@@ -349,12 +352,19 @@ export default function AgentDetailPage() {agent={agent}onRefresh={fetchAgent}rightSlot={- <VersionChooser- versions={versions}- selectedHash={urlVersionHash}- onSelectVersion={handleSelectVersion}- onOpenAllVersions={() => setVersionsOpen(true)}- />+ <div className="flex items-center gap-2">+ <LifecycleActions+ agent={agent}+ onRefresh={fetchAgent}+ variant="header"+ />+ <VersionChooser+ versions={versions}+ selectedHash={urlVersionHash}+ onSelectVersion={handleSelectVersion}+ onOpenAllVersions={() => setVersionsOpen(true)}+ />+ </div>}/>diff --git a/src/client/src/app/(playground)/agents/page.tsx b/src/client/src/app/(playground)/agents/page.tsxindex fe6d0dee..6d451c47 100644--- a/src/client/src/app/(playground)/agents/page.tsx+++ b/src/client/src/app/(playground)/agents/page.tsx@@ -202,7 +202,17 @@ export default function AgentsPage() {"agent",null).transitioning;- if (podsPending > 0 || llmTransitioning || agentTransitioning) {+ const lifecycleTransitioning = getObservabilityView(+ service,+ "lifecycle",+ null+ ).transitioning;+ if (+ podsPending > 0 ||+ llmTransitioning ||+ agentTransitioning ||+ lifecycleTransitioning+ ) {keys.add(service.agent_key);}}@@ -270,7 +280,14 @@ export default function AgentsPage() {if (!intent) continue;const serverView = getObservabilityView(agent, feature, null);if (serverView.transitioning) continue;- const intentTargetEnabled = intent.direction === "enabling";+ // For lifecycle, `restarting` collapses back to `running`+ // once the controller is done. We treat it like `starting`+ // for convergence purposes -- the workload is up, so the+ // intent has resolved.+ const intentTargetEnabled =+ intent.direction === "enabling" ||+ intent.direction === "starting" ||+ intent.direction === "restarting";if (serverView.enabled === intentTargetEnabled) {clearIntent(agentKey, feature);}diff --git a/src/client/src/app/(playground)/agents/service-table.tsx b/src/client/src/app/(playground)/agents/service-table.tsxindex 0305997a..efc7d8b4 100644--- a/src/client/src/app/(playground)/agents/service-table.tsx+++ b/src/client/src/app/(playground)/agents/service-table.tsx@@ -28,6 +28,7 @@ import {getSetAgentIntent,useAgentIntent,} from "@/selectors/agents-instrumentation";+import LifecycleActions from "@/components/(playground)/agents/lifecycle-actions";interface ServiceTableProps {services: UnifiedAgent[];@@ -51,6 +52,7 @@ type ServiceColumnKey =| "service"| "system"| "providers"+ | "lifecycle"| "aiObservability"| "agentObservability"| "lastSeen";@@ -59,10 +61,64 @@ function StaticDash() {return <span className="text-xs text-stone-400 dark:text-stone-500">—</span>;}-function directionLabel(direction: "enabling" | "disabling" | null): string {- return direction === "disabling"- ? getMessage().AGENTS_SERVICE_ACTION_DISABLING- : getMessage().AGENTS_SERVICE_ACTION_ENABLING;+/**+ * Returns a tooltip string describing *why* observability changes+ * should be blocked for this agent, or `null` when the agent is in a+ * state where toggling LLM / Agent observability is safe.+ *+ * We gate on the lifecycle observability-view so the same precedence+ * (optimistic intent → pending action → desired mismatch → steady)+ * decides what counts as "running" — that way clicking Stop in one+ * row instantly disables its o11y toggles via the optimistic intent,+ * without waiting for the controller round-trip.+ *+ * Controllers can mutate processes only while they exist: Docker /+ * Linux modes literally have nothing to attach to once the workload+ * is stopped, and a K8s instrument racing a Stop can leave a+ * half-applied state. Gating closes that race entirely.+ */+function useObservabilityBlock(service: EnrichedAgent): string | null {+ const lifecycleIntent = useAgentIntent(service.agent_key, "lifecycle");+ if (service.source === "sdk" || !service.controller_service_id) {+ return null;+ }+ const lifecycleView = getObservabilityView(+ service,+ "lifecycle",+ lifecycleIntent+ );+ if (lifecycleView.transitioning) {+ return getMessage().AGENTS_OBSERVABILITY_DISABLED_TRANSITIONING;+ }+ if (!lifecycleView.enabled) {+ return getMessage().AGENTS_OBSERVABILITY_DISABLED_NOT_RUNNING;+ }+ return null;+}++function directionLabel(+ direction:+ | "enabling"+ | "disabling"+ | "starting"+ | "stopping"+ | "restarting"+ | null+): string {+ switch (direction) {+ case "disabling":+ return getMessage().AGENTS_SERVICE_ACTION_DISABLING;+ case "enabling":+ return getMessage().AGENTS_SERVICE_ACTION_ENABLING;+ case "starting":+ return getMessage().AGENTS_LIFECYCLE_STARTING;+ case "stopping":+ return getMessage().AGENTS_LIFECYCLE_STOPPING;+ case "restarting":+ return getMessage().AGENTS_LIFECYCLE_RESTARTING;+ default:+ return getMessage().AGENTS_SERVICE_ACTION_WORKING;+ }}function AIObservabilityCell({@@ -76,6 +132,7 @@ function AIObservabilityCell({const setIntent = useRootStore(getSetAgentIntent);const clearIntent = useRootStore(getClearAgentIntent);const intent = useAgentIntent(service.agent_key, "llm");+ const blockReason = useObservabilityBlock(service);const controllerServiceId = service.controller_service_id;if (!controllerServiceId) {@@ -84,6 +141,28 @@ function AIObservabilityCell({const view = getObservabilityView(service, "llm", intent);+ // Stopped or transitioning lifecycle → render a disabled button that+ // still reflects current state, with a tooltip explaining why it is+ // inert. We keep the Enable/Disable label honest so a user who comes+ // back later understands what *will* happen once the agent is back.+ if (blockReason && !view.transitioning) {+ const label = view.enabled+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE+ : getMessage().AGENTS_SERVICE_ACTION_ENABLE;+ return (+ <button+ type="button"+ disabled+ title={blockReason}+ aria-label={blockReason}+ onClick={(e) => e.stopPropagation()}+ className="inline-flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-400 dark:text-stone-500 bg-stone-50 dark:bg-stone-900 cursor-not-allowed"+ >+ {label}+ </button>+ );+ }+const handleAction = async (e: React.MouseEvent) => {e.stopPropagation();if (view.transitioning) return;@@ -158,6 +237,7 @@ function AgentObservabilityCell({const setIntent = useRootStore(getSetAgentIntent);const clearIntent = useRootStore(getClearAgentIntent);const intent = useAgentIntent(service.agent_key, "agent");+ const blockReason = useObservabilityBlock(service);const controllerServiceId = service.controller_service_id;if (!controllerServiceId) {@@ -170,6 +250,36 @@ function AgentObservabilityCell({// flag the row visually. `view.isManual` is the canonical semantic read.const isManual = view.isManual;+ // Lifecycle gating: same logic as the LLM cell, but we still want to+ // surface the `manual` badge so users can see the SDK status; we just+ // disable the Disable button until the agent is running.+ if (blockReason && !view.transitioning) {+ const label = isManual+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE+ : view.enabled+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE+ : getMessage().AGENTS_SERVICE_ACTION_ENABLE;+ return (+ <div className="flex items-center gap-2">+ {isManual && (+ <span className="inline-flex items-center px-2 py-0.5 text-[10px] font-semibold rounded-full bg-amber-50 text-amber-700 dark:bg-amber-900/30 dark:text-amber-400 border border-amber-200 dark:border-amber-800">+ {getMessage().AGENTS_SERVICE_MANUAL_BADGE}+ </span>+ )}+ <button+ type="button"+ disabled+ title={blockReason}+ aria-label={blockReason}+ onClick={(e) => e.stopPropagation()}+ className="inline-flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-400 dark:text-stone-500 bg-stone-50 dark:bg-stone-900 cursor-not-allowed"+ >+ {label}+ </button>+ </div>+ );+ }+const handleAction = async (e: React.MouseEvent) => {e.stopPropagation();if (view.transitioning) return;@@ -258,6 +368,35 @@ function AgentObservabilityCell({);}+/**+ * ActionsCell — Docker-Desktop-style action cluster pinned to the+ * right of the row. SDK-only agents (no controller managing them)+ * render an em-dash so the right edge stays visually clean instead of+ * showing nothing at all; controller-managed rows render the full+ * Play / Stop / Restart pill from LifecycleActions.+ */+function ActionsCell({+ service,+ onRefresh,+}: {+ service: EnrichedAgent;+ onRefresh: () => void;+}) {+ return (+ <div className="flex justify-end">+ {service.source === "sdk" || !service.controller_service_id ? (+ <StaticDash />+ ) : (+ <LifecycleActions+ agent={service}+ onRefresh={onRefresh}+ variant="row"+ />+ )}+ </div>+ );+}+const columns: Columns<ServiceColumnKey, EnrichedAgent> = {service: {header: () => getMessage().AGENTS_COLUMN_SERVICE,@@ -336,6 +475,20 @@ const columns: Columns<ServiceColumnKey, EnrichedAgent> = {</span>),},+ lifecycle: {+ header: () => (+ <div className="flex justify-end">+ {getMessage().AGENTS_COLUMN_ACTIONS}+ </div>+ ),+ cell: ({ row, extraFunctions }) => (+ <ActionsCell+ service={row}+ onRefresh={extraFunctions.onRefresh}+ />+ ),+ enableHiding: false,+ },aiObservability: {header: () => getMessage().AGENTS_COLUMN_LLM_OBSERVABILITY,cell: ({ row, extraFunctions }) => (@@ -356,6 +509,10 @@ const columns: Columns<ServiceColumnKey, EnrichedAgent> = {},};+// Column display order is driven by this record's key order (see+// DataTable). `lifecycle` is intentionally last so the action pill+// pins to the right of the row, matching Docker Desktop's container+// list layout.const VISIBILITY_COLUMNS: Record<ServiceColumnKey, boolean> = {service: true,system: true,@@ -363,6 +520,7 @@ const VISIBILITY_COLUMNS: Record<ServiceColumnKey, boolean> = {lastSeen: true,aiObservability: true,agentObservability: true,+ lifecycle: true,};export default function ServiceTable({diff --git a/src/client/src/app/api/controller/catalog/[id]/restart/route.ts b/src/client/src/app/api/controller/catalog/[id]/restart/route.tsnew file mode 100644index 00000000..539940cc--- /dev/null+++ b/src/client/src/app/api/controller/catalog/[id]/restart/route.ts@@ -0,0 +1,10 @@+import { getFeatureHandler } from "@/lib/platform/controller/features";++export async function POST(+ request: Request,+ { params }: { params: Promise<{ id: string }> }+) {+ const { id } = await params;+ const handler = getFeatureHandler("lifecycle")!;+ return handler.applyOperation(id, "restart", {});+}diff --git a/src/client/src/app/api/controller/catalog/[id]/start/route.ts b/src/client/src/app/api/controller/catalog/[id]/start/route.tsnew file mode 100644index 00000000..c57d6287--- /dev/null+++ b/src/client/src/app/api/controller/catalog/[id]/start/route.ts@@ -0,0 +1,10 @@+import { getFeatureHandler } from "@/lib/platform/controller/features";++export async function POST(+ request: Request,+ { params }: { params: Promise<{ id: string }> }+) {+ const { id } = await params;+ const handler = getFeatureHandler("lifecycle")!;+ return handler.applyOperation(id, "start", {});+}diff --git a/src/client/src/app/api/controller/catalog/[id]/stop/route.ts b/src/client/src/app/api/controller/catalog/[id]/stop/route.tsnew file mode 100644index 00000000..5eebe4f7--- /dev/null+++ b/src/client/src/app/api/controller/catalog/[id]/stop/route.ts@@ -0,0 +1,10 @@+import { getFeatureHandler } from "@/lib/platform/controller/features";++export async function POST(+ request: Request,+ { params }: { params: Promise<{ id: string }> }+) {+ const { id } = await params;+ const handler = getFeatureHandler("lifecycle")!;+ return handler.applyOperation(id, "stop", {});+}diff --git a/src/client/src/app/api/controller/poll/route.ts b/src/client/src/app/api/controller/poll/route.tsindex 228be258..2201ce47 100644--- a/src/client/src/app/api/controller/poll/route.ts+++ b/src/client/src/app/api/controller/poll/route.ts@@ -2,14 +2,18 @@ import {upsertControllerInstance,upsertServices,getControllerConfig,+ getControllerInstanceById,getPendingActions,markActionsAcknowledged,completeAction,getConfigHash,queueAction,+ getActionsByIds,getFeatureDesiredStates,getEnvironmentFeatureConfigs,+ updateFeatureDesiredState,} from "@/lib/platform/controller";+import { KNOWN_ACTIONS } from "@/types/controller";import { getAllFeatureHandlers } from "@/lib/platform/controller/features";import type { ReportedService } from "@/lib/platform/controller/features";import { getFirstDBConfig } from "@/lib/db-config";@@ -193,14 +197,160 @@ async function phaseUpsertServices(// --- Phase 4: Process action results ---+/**+ * Persist any lifecycle-Stop snapshots reported by the controller into+ * the matching desired_states_v2.config row, keyed by+ * (workload_key, cluster_id, feature='lifecycle'). Without this step+ * the saved-state blob the controller computed at Stop time (pod spec /+ * exe+cmdline+cwd+env for bare processes) would be lost, and a later+ * Play would have nothing to recreate the workload with.+ *+ * Only completed stop_workload actions with a non-empty snapshot are+ * persisted. Other action types are ignored here -- the standard+ * completeAction path still runs for them.+ *+ * Returns the set of action_ids whose snapshot write failed, so the+ * caller can defer their completeAction and let the controller retry+ * Stop on the next poll. Controller-side stops are idempotent (the pod+ * is already gone / process already dead) so a retry is safe; the+ * value is that we get another chance to capture the snapshot. The+ * snapshot itself was computed pre-destructively so the controller has+ * it in memory only for the lifetime of this RPC, not on retry -- but+ * the dashboard-side write being retried gives the most common+ * failure (transient ClickHouse) a chance to recover without losing+ * the blob.+ */+async function persistLifecycleStopSnapshots(+ actionResults: any[],+ instanceId: string,+ dbId: string+): Promise<Set<string>> {+ const failed = new Set<string>();+ const candidates = actionResults.filter(+ (ar) =>+ ar?.action_id &&+ ar.status === "completed" &&+ typeof ar.snapshot === "string" &&+ ar.snapshot.length > 0 &&+ ar.snapshot !== "{}"+ );+ if (candidates.length === 0) return failed;++ const actionIds = candidates.map((ar) => ar.action_id as string);+ const [actionsRes, instanceRes] = await Promise.all([+ getActionsByIds(actionIds, instanceId, dbId),+ getControllerInstanceById(instanceId, dbId),+ ]);++ if (actionsRes.err || !actionsRes.data) {+ console.error(+ "controller poll: getActionsByIds failed; deferring snapshot persistence:",+ sanitizeLogValue(actionsRes.err || "no data")+ );+ // Defer ALL candidate completions so the controller will retry+ // Stop on the next poll, giving us another shot at the lookup.+ for (const ar of candidates) failed.add(ar.action_id as string);+ return failed;+ }+ // Symmetric: an instance lookup failure must defer the same way the+ // actions lookup does. Falling back to cluster_id="default" would+ // silently write the snapshot into the wrong (workload_key,+ // cluster_id) row in multi-cluster setups, so a later Start in the+ // real cluster would see no snapshot and 409. Defer + retry instead.+ if (instanceRes.err || !instanceRes.data?.[0]?.cluster_id) {+ // instanceId rides in on the controller's poll request body, so+ // it must be sanitized before being logged. We use+ // `JSON.stringify` rather than the project's `sanitizeLogValue`+ // helper or an inline regex replace because CodeQL's+ // `js/log-injection` query has a built-in `JsonStringifySanitizer`+ // barrier — using exactly the pattern the analyzer expects keeps+ // future alerts from re-firing on this site. `JSON.stringify`+ // escapes CR/LF (`\\n`, `\\r`) so log forgery via embedded+ // newlines is impossible.+ console.error(+ "controller poll: getControllerInstanceById failed; deferring snapshot persistence:",+ "instance",+ JSON.stringify(instanceId),+ ":",+ sanitizeLogValue(instanceRes.err || "no data")+ );+ for (const ar of candidates) failed.add(ar.action_id as string);+ return failed;+ }+ const clusterId = instanceRes.data[0].cluster_id;+ const actionById = new Map(+ actionsRes.data.map((a) => [a.id, a] as const)+ );++ // Run the writes in parallel — they target independent+ // (workload_key, cluster_id) rows so ReplacingMergeTree merges are+ // non-conflicting, and at typical scale we can have dozens of Stops+ // in a single poll cycle (e.g. operator stopping a whole namespace).+ await Promise.all(+ candidates.map(async (ar) => {+ const action = actionById.get(ar.action_id);+ if (!action) return;+ if (action.action_type !== KNOWN_ACTIONS.STOP_WORKLOAD) return;++ const writeRes = await updateFeatureDesiredState(+ action.service_key,+ clusterId,+ "lifecycle",+ "stopped",+ ar.snapshot as string,+ dbId+ );+ if (writeRes?.err) {+ // Same rationale as the instance-lookup branch above:+ // ar.action_id and action.service_key both originate in+ // the controller's poll request body, so we route them+ // through `JSON.stringify` — CodeQL's+ // `JsonStringifySanitizer` barrier. `writeRes.err` is a+ // DB error string (not user input), so passing it+ // through `sanitizeLogValue` is sufficient.+ console.error(+ "controller poll: persist lifecycle snapshot failed for action",+ JSON.stringify(ar.action_id),+ "workload",+ JSON.stringify(action.service_key),+ ":",+ sanitizeLogValue(writeRes.err)+ );+ failed.add(ar.action_id);+ }+ })+ );+ return failed;+}+async function phaseProcessActionResults(actionResults: any[] | undefined,instanceId: string,dbId: string) {if (!Array.isArray(actionResults)) return;- for (const ar of actionResults) {- if (ar.action_id && ar.status) {++ // Persist Stop snapshots BEFORE marking the action complete. If we+ // reordered these, a controller crash between completeAction and the+ // snapshot write would leave us with a successful stop_workload that+ // has no recovery blob, breaking Play for naked pods / bare+ // processes. Writing the snapshot first means completeAction is the+ // commit point, and a write failure defers completeAction so the+ // controller retries Stop on the next poll.+ const deferred = await persistLifecycleStopSnapshots(+ actionResults,+ instanceId,+ dbId+ );++ // completeAction calls run in parallel as well — each row in+ // openlit_controller_actions is identified by a deterministic id so+ // re-inserts deduplicate via ReplacingMergeTree. Parallelizing here+ // keeps the poll round-trip O(slowest write) rather than O(N).+ await Promise.all(+ actionResults.map(async (ar) => {+ if (!ar.action_id || !ar.status) return;+ if (deferred.has(ar.action_id)) return;const completeResult = await completeAction(ar.action_id,instanceId,@@ -211,8 +361,8 @@ async function phaseProcessActionResults(if (completeResult?.err) {console.error("controller poll: completeAction failed");}- }- }+ })+ );}// --- Phase 5: Generic feature reconciliation ---diff --git a/src/client/src/clickhouse/migrations/create-controller-migration.ts b/src/client/src/clickhouse/migrations/create-controller-migration.tsindex 0039731c..83ed1cca 100644--- a/src/client/src/clickhouse/migrations/create-controller-migration.ts+++ b/src/client/src/clickhouse/migrations/create-controller-migration.ts@@ -92,6 +92,7 @@ export default async function CreateControllerMigration(updated_at DateTime DEFAULT now()) ENGINE = ReplacingMergeTree(updated_at)ORDER BY (workload_key, cluster_id, feature)+ TTL updated_at + INTERVAL 90 DAYSETTINGS index_granularity = 8192`,`diff --git a/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx b/src/client/src/components/(playground)/agents/agent-configuration-tab.tsxindex 426423bf..fe2f3df4 100644--- a/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx+++ b/src/client/src/components/(playground)/agents/agent-configuration-tab.tsx@@ -18,6 +18,14 @@ interface AgentConfigurationTabProps {** Both blocks share `getObservabilityView` with the list-view cells so the* Configuration tab can no longer drift from the list (the previous bug).+ *+ * Observability toggles are gated on lifecycle state: when the agent+ * is stopped or transitioning, instrumenting (or removing+ * instrumentation) is unsafe — there is no live process for the+ * controller to attach to in Docker / Linux modes, and a K8s instrument+ * action racing a Stop leaves a half-applied state. We block the+ * toggle and surface the reason via the button tooltip rather than+ * silently dropping the click.*/export default function AgentConfigurationTab({agent,@@ -25,8 +33,28 @@ export default function AgentConfigurationTab({}: AgentConfigurationTabProps) {const llmIntent = useAgentIntent(agent.agent_key, "llm");const agentIntent = useAgentIntent(agent.agent_key, "agent");+ const lifecycleIntent = useAgentIntent(agent.agent_key, "lifecycle");const llmView = getObservabilityView(agent, "llm", llmIntent);const agentView = getObservabilityView(agent, "agent", agentIntent);+ const lifecycleView = getObservabilityView(+ agent,+ "lifecycle",+ lifecycleIntent+ );++ // Only controller-managed agents have a real lifecycle to gate on.+ // SDK-only rows fall through to ObservabilityBlock's static branch+ // anyway, so passing null here is correct.+ const isControllerManaged =+ agent.source !== "sdk" && !!agent.controller_service_id;+ let blockReason: string | null = null;+ if (isControllerManaged) {+ if (lifecycleView.transitioning) {+ blockReason = getMessage().AGENTS_OBSERVABILITY_DISABLED_TRANSITIONING;+ } else if (!lifecycleView.enabled) {+ blockReason = getMessage().AGENTS_OBSERVABILITY_DISABLED_NOT_RUNNING;+ }+ }return (<div className="space-y-4">@@ -44,6 +72,7 @@ export default function AgentConfigurationTab({podSummary={llmView.podSummary}onChange={onRefresh}serviceName={agent.service_name}+ blockReason={blockReason}/><ObservabilityBlockkind="agent"@@ -55,6 +84,7 @@ export default function AgentConfigurationTab({podSummary={agentView.podSummary}onChange={onRefresh}serviceName={agent.service_name}+ blockReason={blockReason}/></div></div>diff --git a/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx b/src/client/src/components/(playground)/agents/agent-metadata-grid.tsxindex d4ce5697..5f3662f8 100644--- a/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx+++ b/src/client/src/components/(playground)/agents/agent-metadata-grid.tsx@@ -1,6 +1,5 @@"use client";-import { ProviderIcon } from "@/components/svg/providers";import { formatBrowserDateTime, parseDateString } from "@/utils/date";import getMessage from "@/constants/messages";import type { UnifiedAgent } from "@/types/agents";@@ -59,23 +58,6 @@ export default function AgentMetadataGrid({ agent }: AgentMetadataGridProps) {label={getMessage().AGENTS_METADATA_LAST_SEEN}value={formatBrowserDateTime(agent.last_seen)}/>- {agent.providers.length > 0 && (- <MetadataCard- label={getMessage().AGENTS_MODEL_PROVIDERS}- value={- <div className="flex flex-wrap items-center gap-1.5">- {agent.providers.map((p) => (- <span key={p} className="flex items-center gap-1 text-xs">- <ProviderIcon provider={p} className="w-3.5 h-3.5" />- <span className="capitalize text-stone-700 dark:text-stone-200">- {p}- </span>- </span>- ))}- </div>- }- />- )}</div>);}diff --git a/src/client/src/components/(playground)/agents/lifecycle-actions.tsx b/src/client/src/components/(playground)/agents/lifecycle-actions.tsxnew file mode 100644index 00000000..81e85a71--- /dev/null+++ b/src/client/src/components/(playground)/agents/lifecycle-actions.tsx@@ -0,0 +1,301 @@+"use client";++/**+ * LifecycleActions — Play / Stop / Restart for controller-managed+ * workloads. Used by the agents list table (per-row, right-aligned+ * "Actions" column) and the agent detail page (header `rightSlot`).+ *+ * Visual style mirrors Docker Desktop's container actions: the buttons+ * sit inside a single bordered pill with internal dividers, so the+ * affordance is always visible without hover. Two variants:+ *+ * - `row` : compact 28px icon buttons for the agents table.+ * - `header` : 36px icon buttons with a touch more padding for the+ * agent-detail header bar — large enough to be obvious+ * next to the version chooser, small enough to not+ * dominate the layout.+ *+ * Hidden entirely for SDK-only rows (no controller, no lifecycle to+ * manage) and gracefully no-ops when the controller has not advertised+ * `lifecycle_*_v1` capabilities yet (in that case the API returns 409+ * and the toast surfaces the reason).+ *+ * State transitions flow through the shared observability-view so the+ * spinner / direction / desired-mismatch precedence is the same as the+ * LLM + Agent cells. Stop is the only destructive action and always+ * opens a confirmation dialog.+ */++import { useState } from "react";+import { Loader2, Play, RotateCcw, Square } from "lucide-react";+import { toast } from "sonner";+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";+import getMessage from "@/constants/messages";+import {+ getObservabilityView,+ type Direction,+} from "@/lib/platform/agents/observability-view";+import useFetchWrapper from "@/utils/hooks/useFetchWrapper";+import { useRootStore } from "@/store";+import {+ getClearAgentIntent,+ getSetAgentIntent,+ useAgentIntent,+} from "@/selectors/agents-instrumentation";+import type { UnifiedAgent } from "@/types/agents";++export type LifecycleActionsVariant = "row" | "header";++interface LifecycleActionsProps {+ agent: UnifiedAgent;+ onRefresh: () => void;+ variant?: LifecycleActionsVariant;+}++function directionLabel(direction: Direction | null): string {+ switch (direction) {+ case "starting":+ return getMessage().AGENTS_LIFECYCLE_STARTING;+ case "stopping":+ return getMessage().AGENTS_LIFECYCLE_STOPPING;+ case "restarting":+ return getMessage().AGENTS_LIFECYCLE_RESTARTING;+ default:+ return getMessage().AGENTS_SERVICE_ACTION_WORKING;+ }+}++const SIZES: Record<+ LifecycleActionsVariant,+ { button: string; icon: string; pill: string }+> = {+ row: {+ button: "w-8 h-8",+ icon: "w-3.5 h-3.5",+ pill: "h-8",+ },+ header: {+ button: "w-9 h-9",+ icon: "w-4 h-4",+ pill: "h-9",+ },+};++export default function LifecycleActions({+ agent,+ onRefresh,+ variant = "row",+}: LifecycleActionsProps) {+ const { fireRequest, isLoading } = useFetchWrapper();+ const setIntent = useRootStore(getSetAgentIntent);+ const clearIntent = useRootStore(getClearAgentIntent);+ const intent = useAgentIntent(agent.agent_key, "lifecycle");+ // Symmetric to useObservabilityBlock in service-table.tsx: while+ // LLM or Agent observability is mid-instrument, we don't want the+ // user racing a Stop / Restart against that operation. The cheapest+ // fix is to disable the lifecycle pill and surface the reason in a+ // tooltip. Both reads go through getObservabilityView so the+ // optimistic-intent / pending-action / desired-mismatch precedence+ // is the same one used everywhere else.+ const llmIntent = useAgentIntent(agent.agent_key, "llm");+ const agentIntent = useAgentIntent(agent.agent_key, "agent");+ const [confirmStopOpen, setConfirmStopOpen] = useState(false);++ const controllerServiceId = agent.controller_service_id;+ if (agent.source === "sdk" || !controllerServiceId) {+ return null;+ }++ const view = getObservabilityView(agent, "lifecycle", intent);+ const llmView = getObservabilityView(agent, "llm", llmIntent);+ const agentView = getObservabilityView(agent, "agent", agentIntent);+ const observabilityInFlight =+ llmView.transitioning || agentView.transitioning;+ const running = view.enabled;+ const sz = SIZES[variant];++ async function fire(+ action: "start" | "stop" | "restart",+ direction: "starting" | "stopping" | "restarting"+ ) {+ if (view.transitioning) return;+ setIntent(agent.agent_key, "lifecycle", direction);+ await fireRequest({+ requestType: "POST",+ url: `/api/controller/catalog/${controllerServiceId}/${action}`,+ successCb: () => {+ const queued =+ action === "start"+ ? getMessage().AGENTS_LIFECYCLE_QUEUED_PLAY(agent.service_name)+ : action === "stop"+ ? getMessage().AGENTS_LIFECYCLE_QUEUED_STOP(agent.service_name)+ : getMessage().AGENTS_LIFECYCLE_QUEUED_RESTART(agent.service_name);+ toast.success(queued);+ onRefresh();+ },+ failureCb: (err: any) => {+ clearIntent(agent.agent_key, "lifecycle");+ toast.error(getMessage().AGENTS_LIFECYCLE_FAILED(err));+ },+ });+ }++ // Shared pill shell — one bordered container with internal dividers+ // between buttons, so the action affordances are visible even when+ // the cursor is nowhere near them. Matches Docker Desktop's+ // container-row actions cluster.+ const pillBase = `inline-flex ${sz.pill} items-center rounded-md border border-stone-200 dark:border-stone-700 bg-white dark:bg-stone-900 overflow-hidden divide-x divide-stone-200 dark:divide-stone-700 shadow-sm`;+ const buttonBase = `inline-flex items-center justify-center ${sz.button} disabled:opacity-50 disabled:cursor-not-allowed transition-colors`;++ if (view.transitioning) {+ return (+ <div className={pillBase} aria-busy="true" aria-label={directionLabel(view.direction)}>+ <span+ className={`${buttonBase} text-stone-500 dark:text-stone-400`}+ title={directionLabel(view.direction)}+ >+ <Loader2 className={`${sz.icon} animate-spin`} />+ </span>+ </div>+ );+ }++ // Observability-in-flight branch: keep the action affordance visible+ // so users understand *what* would happen, but render it as a+ // disabled pill with the explanatory tooltip. Stop/Restart vs. Play+ // shape stays consistent with the live state so reading the row+ // does not require a translation.+ if (observabilityInFlight) {+ const blockTooltip =+ getMessage().AGENTS_LIFECYCLE_DISABLED_OBSERVABILITY_TRANSITIONING;+ const disabledButton = (+ node: React.ReactNode,+ ariaLabel: string,+ extraColor: string+ ) => (+ <span+ className={`${buttonBase} ${extraColor} opacity-50 cursor-not-allowed`}+ title={blockTooltip}+ aria-label={ariaLabel}+ aria-disabled="true"+ >+ {node}+ </span>+ );+ return (+ <div className={pillBase} onClick={(e) => e.stopPropagation()}>+ {!running+ ? disabledButton(+ <Play className={sz.icon} />,+ getMessage().AGENTS_LIFECYCLE_PLAY,+ "text-green-600 dark:text-green-400"+ )+ : (+ <>+ {disabledButton(+ <Square className={`${sz.icon} fill-current`} />,+ getMessage().AGENTS_LIFECYCLE_STOP,+ "text-red-600 dark:text-red-400"+ )}+ {disabledButton(+ <RotateCcw className={sz.icon} />,+ getMessage().AGENTS_LIFECYCLE_RESTART,+ "text-stone-600 dark:text-stone-300"+ )}+ </>+ )}+ </div>+ );+ }++ return (+ <>+ <div className={pillBase} onClick={(e) => e.stopPropagation()}>+ {!running ? (+ <button+ onClick={(e) => {+ e.stopPropagation();+ fire("start", "starting");+ }}+ disabled={isLoading}+ title={getMessage().AGENTS_LIFECYCLE_TOOLTIP_PLAY}+ aria-label={getMessage().AGENTS_LIFECYCLE_PLAY}+ className={`${buttonBase} text-green-600 dark:text-green-400 hover:bg-green-50 dark:hover:bg-green-950/40`}+ >+ <Play className={sz.icon} />+ </button>+ ) : (+ <>+ <button+ onClick={(e) => {+ e.stopPropagation();+ setConfirmStopOpen(true);+ }}+ disabled={isLoading}+ title={getMessage().AGENTS_LIFECYCLE_TOOLTIP_STOP}+ aria-label={getMessage().AGENTS_LIFECYCLE_STOP}+ className={`${buttonBase} text-red-600 dark:text-red-400 hover:bg-red-50 dark:hover:bg-red-950/40`}+ >+ <Square className={`${sz.icon} fill-current`} />+ </button>+ <button+ onClick={(e) => {+ e.stopPropagation();+ fire("restart", "restarting");+ }}+ disabled={isLoading}+ title={getMessage().AGENTS_LIFECYCLE_TOOLTIP_RESTART}+ aria-label={getMessage().AGENTS_LIFECYCLE_RESTART}+ className={`${buttonBase} text-stone-600 dark:text-stone-300 hover:bg-stone-100 dark:hover:bg-stone-800`}+ >+ <RotateCcw className={sz.icon} />+ </button>+ </>+ )}+ </div>+ <Dialog open={confirmStopOpen} onOpenChange={setConfirmStopOpen}>+ <DialogContent+ onClick={(e) => e.stopPropagation()}+ className="sm:max-w-md"+ >+ <DialogHeader>+ <DialogTitle>+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_TITLE}+ </DialogTitle>+ <DialogDescription>+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_DESCRIPTION(+ agent.service_name+ )}+ </DialogDescription>+ </DialogHeader>+ <DialogFooter>+ <button+ type="button"+ onClick={() => setConfirmStopOpen(false)}+ className="inline-flex items-center px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-600 dark:text-stone-300 hover:bg-stone-50 dark:hover:bg-stone-800"+ >+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_CANCEL}+ </button>+ <button+ type="button"+ onClick={() => {+ setConfirmStopOpen(false);+ fire("stop", "stopping");+ }}+ className="inline-flex items-center px-3 py-1.5 text-xs font-medium rounded-md bg-red-600 text-white hover:bg-red-700"+ >+ {getMessage().AGENTS_LIFECYCLE_CONFIRM_STOP_CONFIRM}+ </button>+ </DialogFooter>+ </DialogContent>+ </Dialog>+ </>+ );+}diff --git a/src/client/src/components/(playground)/agents/observability-block.tsx b/src/client/src/components/(playground)/agents/observability-block.tsxindex b545f42c..fd49e422 100644--- a/src/client/src/components/(playground)/agents/observability-block.tsx+++ b/src/client/src/components/(playground)/agents/observability-block.tsx@@ -26,13 +26,33 @@ interface ObservabilityBlockProps {enabled: boolean;/** True when an action is in flight (controller-side or optimistic). */pending?: boolean;- pendingDirection?: "enabling" | "disabling" | null;+ // Direction is widened to the full Direction union (which includes+ // lifecycle's starting/stopping/restarting) even though this block+ // only ever renders for llm/agent, because the upstream+ // observability-view now returns the widened type. The renderer+ // below only branches on enabling/disabling so the lifecycle+ // variants are inert here.+ pendingDirection?:+ | "enabling"+ | "disabling"+ | "starting"+ | "stopping"+ | "restarting"+ | null;/** Multi-pod rollup so we can show `Pods: ack/total` while a fan-out is* propagating. `null` for SDK-only rows or single-pod workloads. */podSummary?: PodSummary | null;/** Triggered after a successful action so the parent can refresh. */onChange?: () => void;serviceName: string;+ /**+ * If set, the toggle renders as a disabled button with this text as+ * the tooltip. Used to gate observability changes on lifecycle+ * state (e.g. the agent is stopped or transitioning) — we don't+ * want to queue an instrument action against a workload that has+ * no live process for the controller to attach to.+ */+ blockReason?: string | null;}function kindLabel(kind: ObservabilityKind) {@@ -57,6 +77,7 @@ export default function ObservabilityBlock({podSummary,onChange,serviceName,+ blockReason,}: ObservabilityBlockProps) {const { fireRequest, isLoading } = useFetchWrapper();const setIntent = useRootStore(getSetAgentIntent);@@ -142,6 +163,18 @@ export default function ObservabilityBlock({? getMessage().AGENTS_SERVICE_ACTION_DISABLING: getMessage().AGENTS_SERVICE_ACTION_ENABLING}</button>+ ) : blockReason ? (+ <button+ type="button"+ disabled+ title={blockReason}+ aria-label={blockReason}+ className="inline-flex items-center gap-1.5 px-3 py-1.5 text-xs font-medium rounded-md border border-stone-200 dark:border-stone-700 text-stone-400 dark:text-stone-500 bg-stone-50 dark:bg-stone-900 cursor-not-allowed"+ >+ {enabled+ ? getMessage().AGENTS_SERVICE_ACTION_DISABLE+ : getMessage().AGENTS_SERVICE_ACTION_ENABLE}+ </button>) : (<buttononClick={handleToggle}diff --git a/src/client/src/constants/messages/en.ts b/src/client/src/constants/messages/en.tsindex 70d2214e..693a0aa9 100644--- a/src/client/src/constants/messages/en.ts+++ b/src/client/src/constants/messages/en.ts@@ -416,6 +416,43 @@ export const AGENTS_SDK_SOURCE_NOTE = "Source: SDK";export const AGENTS_LOAD_MORE = "Load more";export const AGENTS_LOAD_MORE_LOADING = "Loading...";+// Agents - Lifecycle (Play / Stop / Restart for controller-managed workloads)+export const AGENTS_COLUMN_ACTIONS = "Actions";+export const AGENTS_LIFECYCLE_PLAY = "Play";+export const AGENTS_LIFECYCLE_STOP = "Stop";+export const AGENTS_LIFECYCLE_RESTART = "Restart";+export const AGENTS_LIFECYCLE_STARTING = "Starting...";+export const AGENTS_LIFECYCLE_STOPPING = "Stopping...";+export const AGENTS_LIFECYCLE_RESTARTING = "Restarting...";+export const AGENTS_LIFECYCLE_STATUS_RUNNING = "Running";+export const AGENTS_LIFECYCLE_STATUS_STOPPED = "Stopped";+export const AGENTS_LIFECYCLE_STATUS_RESTARTING = "Restarting";+export const AGENTS_LIFECYCLE_STATUS_UNKNOWN = "Unknown";+export const AGENTS_LIFECYCLE_TOOLTIP_PLAY = "Start the agent (scale up / start container / re-launch process)";+export const AGENTS_LIFECYCLE_TOOLTIP_STOP = "Stop the agent (scale to 0 / stop container / SIGTERM)";+export const AGENTS_LIFECYCLE_TOOLTIP_RESTART = "Restart the agent (rolling restart / restart container / SIGTERM + relaunch)";+export const AGENTS_LIFECYCLE_TOOLTIP_DISABLED_SDK = "Lifecycle controls are available only for controller-managed agents.";+export const AGENTS_LIFECYCLE_TOOLTIP_DISABLED_CAPABILITY = "The controller managing this agent does not advertise lifecycle support yet.";+export const AGENTS_LIFECYCLE_TOOLTIP_PLAY_NO_SNAPSHOT = "No saved snapshot to restore. The controller cannot bring this agent back up without state captured at Stop time.";+export const AGENTS_LIFECYCLE_CONFIRM_STOP_TITLE = "Stop agent?";+export const AGENTS_LIFECYCLE_CONFIRM_STOP_DESCRIPTION = (serviceName: string) =>+ `This will scale down or terminate the agent "${serviceName}". Telemetry will stop until you Play it again.`;+export const AGENTS_LIFECYCLE_CONFIRM_STOP_CONFIRM = "Stop agent";+export const AGENTS_LIFECYCLE_CONFIRM_STOP_CANCEL = "Cancel";+export const AGENTS_LIFECYCLE_QUEUED_PLAY = (serviceName: string) =>+ `Starting ${serviceName}...`;+export const AGENTS_LIFECYCLE_QUEUED_STOP = (serviceName: string) =>+ `Stopping ${serviceName}...`;+export const AGENTS_LIFECYCLE_QUEUED_RESTART = (serviceName: string) =>+ `Restarting ${serviceName}...`;+export const AGENTS_LIFECYCLE_FAILED = (err: string) => `Lifecycle action failed: ${err}`;+export const AGENTS_OBSERVABILITY_DISABLED_NOT_RUNNING =+ "Start the agent to change observability — toggling LLM / Agent observability is only possible while the agent is running.";+export const AGENTS_OBSERVABILITY_DISABLED_TRANSITIONING =+ "Observability changes are blocked while the agent is starting, stopping, or restarting.";+export const AGENTS_LIFECYCLE_DISABLED_OBSERVABILITY_TRANSITIONING =+ "Lifecycle changes are blocked while LLM or Agent observability is being applied to this agent.";+// Agents - Detail Tabsexport const AGENTS_TAB_ANALYTICS = "Analytics";export const AGENTS_TAB_CONVERSATIONS = "Conversations";@@ -513,7 +550,6 @@ export const AGENTS_STAT_PID = "PID";export const AGENTS_STAT_RUNTIME = "Runtime";export const AGENTS_STAT_FIRST_SEEN = "First Seen";export const AGENTS_STAT_RUNTIME_UNKNOWN = "Unknown";-export const AGENTS_MODEL_PROVIDERS = "Model Providers";export const AGENTS_RESOURCE_ATTRIBUTES = "Resource Attributes";export const AGENTS_CONTROLS = "Controls";export const AGENTS_LLM_OBSERVABILITY_LABEL = "LLM Observability";@@ -535,8 +571,8 @@ export const AGENTS_MANUAL_SETUP_REQUIRED = "Manual setup required";export const AGENTS_CONTAINERIZED_WARNING = "This process runs inside a container. Mount the Docker socket or use a Docker/Kubernetes-mode controller for Agent Observability.";export const AGENTS_NAKED_POD_WARNING = "This is a naked pod (no Deployment, DaemonSet, or StatefulSet). Enabling or disabling Agent Observability will restart the pod.";export const AGENTS_NAKED_POD_CONFIRM = "This pod has no Deployment or DaemonSet. Enabling Agent Observability will restart the pod. Continue?";-export const AGENTS_AGENT_DEPLOY_QUEUED = "Agent observability is being deployed. The workload will be updated automatically.";-export const AGENTS_AGENT_REMOVE_QUEUED = "Agent observability removal queued. The workload will be updated automatically.";+export const AGENTS_AGENT_DEPLOY_QUEUED = "Agent observability is being deployed. The agent will be updated automatically.";+export const AGENTS_AGENT_REMOVE_QUEUED = "Agent observability removal queued. The agent will be updated automatically.";export const AGENTS_QUEUED_ACTION = (action: string) => `Queued ${action}`;export const AGENTS_PENDING_INSTRUMENTING = "Instrumenting";export const AGENTS_PENDING_UNINSTRUMENTING = "Uninstrumenting";diff --git a/src/client/src/lib/platform/agents/index.ts b/src/client/src/lib/platform/agents/index.tsindex aedd83d5..196a8402 100644--- a/src/client/src/lib/platform/agents/index.ts+++ b/src/client/src/lib/platform/agents/index.ts@@ -62,6 +62,8 @@ function rowToAgent(row: Record<string, unknown>): UnifiedAgent {desired_instrumentation_status: (String(row.desired_instrumentation_status || "none") as UnifiedAgent["desired_instrumentation_status"]),agent_observability_status: agentObsStatus,desired_agent_status: (String(row.desired_agent_status || "none") as UnifiedAgent["desired_agent_status"]),+ lifecycle_status: (String(row.lifecycle_status || "unknown") as UnifiedAgent["lifecycle_status"]),+ desired_lifecycle_status: (String(row.desired_lifecycle_status || "unknown") as UnifiedAgent["desired_lifecycle_status"]),pending_action: pendingAction ? pendingAction : null,pending_action_status: pendingActionStatus? (pendingActionStatus as UnifiedAgent["pending_action_status"])@@ -93,7 +95,12 @@ WITH pod_set AS (argMax(service_name, last_seen) AS service_name,uniqExact(controller_instance_id) AS pods_total,groupArrayDistinct(controller_instance_id) AS instance_ids,- argMax(resource_attributes['openlit.agent_observability.status'], last_seen) AS agent_observability_status+ argMax(resource_attributes['openlit.agent_observability.status'], last_seen) AS agent_observability_status,+ -- Mirrors the agent_observability rollup. The controller stamps+ -- resource_attributes['openlit.lifecycle.status'] on every+ -- heartbeat after a Play / Stop / Restart so the UI sees the+ -- actual state of the workload, not just the desired state.+ argMax(resource_attributes['openlit.lifecycle.status'], last_seen) AS lifecycle_statusFROM ${CONTROLLER_SERVICES_TABLE}FINALWHERE last_seen >= now() - INTERVAL 5 MINUTE@@ -120,6 +127,21 @@ desired_agent AS (WHERE feature = 'agent'GROUP BY workload_key, cluster_id),+desired_lifecycle AS (+ -- A row here means the user has explicitly Played or Stopped the+ -- workload. It is also the source of truth for the snapshot blob+ -- (config column) used to bring K8s naked pods and Linux bare+ -- processes back up after Stop.+ SELECT+ workload_key,+ cluster_id,+ argMax(desired_status, updated_at) AS desired_status,+ argMax(config, updated_at) AS config+ FROM ${CONTROLLER_DESIRED_STATES_V2_TABLE}+ FINAL+ WHERE feature = 'lifecycle'+ GROUP BY workload_key, cluster_id+),pod_action_latest AS (-- Per (instance_id, service_key), keep the most recent action row.-- We rename the max(updated_at) projection to last_updated_at so the@@ -187,6 +209,24 @@ const SELECT_COLUMNS = `-- not yet aged out of the materializer's 30-min lookback window).coalesce(pod_set.agent_observability_status, if(s.source = 'sdk', 'enabled', '')) AS agent_observability_status,if(desired_agent.desired_status != '', desired_agent.desired_status, 'none') AS desired_agent_status,+ -- Lifecycle:+ -- - For SDK-only rows there is nothing to lifecycle-manage; render+ -- 'unknown' so the UI hides the buttons (the LifecycleCell also+ -- guards on source != 'sdk').+ -- - When pod_set has a value, that is the most authoritative+ -- reading (it came from the latest heartbeat).+ -- - When pod_set is empty AND the user has stopped the workload,+ -- trust the desired_status='stopped' as the actual state -- the+ -- heartbeat went silent because the workload is down. This is+ -- what keeps the row visible as "Stopped" in the UI.+ if(s.source = 'sdk' AND pod_set.lifecycle_status = '',+ 'unknown',+ if(pod_set.lifecycle_status != '',+ pod_set.lifecycle_status,+ if(desired_lifecycle.desired_status = 'stopped', 'stopped', 'running')+ )+ ) AS lifecycle_status,+ if(desired_lifecycle.desired_status != '', desired_lifecycle.desired_status, 'unknown') AS desired_lifecycle_status,if(pod_actions.pending_action != '', pod_actions.pending_action, '') AS pending_action,if(pod_actions.pending_action_status != '', pod_actions.pending_action_status, '') AS pending_action_status,s.first_seen AS first_seen,@@ -220,6 +260,9 @@ const ROLLUP_JOINS = `LEFT JOIN desired_agentON desired_agent.workload_key = s.workload_keyAND desired_agent.cluster_id = s.cluster_id+ LEFT JOIN desired_lifecycle+ ON desired_lifecycle.workload_key = s.workload_key+ AND desired_lifecycle.cluster_id = s.cluster_idLEFT JOIN pod_actionsON pod_actions.service_key = s.workload_key`;@@ -269,12 +312,21 @@ async function loadAgents(params: ListAgentsParams): Promise<ListAgentsResult> {// off so we always include the most recent materialized rows, even if the// browser captured `end` slightly before the materializer wrote the row.// Custom ranges still send both bounds.+ //+ // Stopped-workload exception: a workload the user has Stopped no longer+ // emits heartbeats, so `s.last_seen` ages out fast. We special-case it+ // by OR-ing in any row whose lifecycle desired-state is 'stopped' --+ // those stay visible until the 90-day desired-states TTL collapses the+ // row organically. Without this OR, clicking Stop would feel like the+ // workload disappeared from the dashboard, which is exactly the bug+ // the lifecycle UNION in the materializer is meant to prevent.+ const stoppedRowClause = `desired_lifecycle.desired_status = 'stopped'`;if (params.timeStart) {where.push(- `s.last_seen >= parseDateTimeBestEffort('${escape(params.timeStart)}')`+ `(s.last_seen >= parseDateTimeBestEffort('${escape(params.timeStart)}') OR ${stoppedRowClause})`);} else {- where.push(`s.last_seen >= now() - INTERVAL 30 DAY`);+ where.push(`(s.last_seen >= now() - INTERVAL 30 DAY OR ${stoppedRowClause})`);}if (params.timeEnd) {where.push(diff --git a/src/client/src/lib/platform/agents/materialize.ts b/src/client/src/lib/platform/agents/materialize.tsindex d5bfafc2..d5644d88 100644--- a/src/client/src/lib/platform/agents/materialize.ts+++ b/src/client/src/lib/platform/agents/materialize.ts@@ -9,7 +9,10 @@*/import { dataCollector, OTEL_TRACES_TABLE_NAME } from "@/lib/platform/common";-import { CONTROLLER_SERVICES_TABLE } from "@/lib/platform/controller/table-details";+import {+ CONTROLLER_DESIRED_STATES_V2_TABLE,+ CONTROLLER_SERVICES_TABLE,+} from "@/lib/platform/controller/table-details";import type { AgentSource } from "@/types/agents";import { computeAgentKey, invalidateAgent } from "./index";import { invalidatePrefix } from "./cache";@@ -140,8 +143,16 @@ async function discoverAgents(last_seen: string;}>) || [];+ // The materializer needs to keep stopped workloads in agents_summary+ // even after their last heartbeat aged out of the 24h window. We do+ // that by UNION-ing the active heartbeat rows with the+ // lifetime-latest row for any workload whose lifecycle desired+ // state is 'stopped'. Without this, a workload that has been+ // stopped for >24h would silently disappear from the UI -- the+ // agents-list query reads agents_summary, and agents_summary only+ // keeps rows the materializer rewrites.const ctrlQuery = `- WITH latest AS (+ WITH active AS (SELECTargMax(s.id, s.last_seen) AS id,argMax(s.controller_instance_id, s.last_seen) AS controller_instance_id,@@ -157,6 +168,46 @@ async function discoverAgents(WHERE s.last_seen >= now() - INTERVAL 24 HOUR${clusterPredicate}GROUP BY concat(s.cluster_id, ':', s.service_name)+ ),+ stopped_keys AS (+ SELECT workload_key, cluster_id+ FROM ${CONTROLLER_DESIRED_STATES_V2_TABLE} FINAL+ WHERE feature = 'lifecycle' AND desired_status = 'stopped'+ AND workload_key != ''+ ${clusterPredicate}+ ),+ stopped AS (+ -- Lifetime lookup (no time bound) so stopped agents stay+ -- materialized regardless of how long they have been down.+ -- We bound the size by stopped_keys (small) and exclude+ -- anything already covered by the active CTE to avoid+ -- double-materializing the same workload. The dedup key is+ -- the same (cluster_id, service_name) tuple the active CTE+ -- groups by -- service_name alone would collide across+ -- clusters (e.g. "checkout" running in cluster-prod while+ -- stopped in cluster-staging).+ SELECT+ argMax(s.id, s.last_seen) AS id,+ argMax(s.controller_instance_id, s.last_seen) AS controller_instance_id,+ argMax(s.cluster_id, s.last_seen) AS cluster_id,+ argMax(s.service_name, s.last_seen) AS service_name,+ argMax(s.workload_key, s.last_seen) AS workload_key,+ argMax(s.instrumentation_status, s.last_seen) AS instrumentation_status,+ argMax(s.resource_attributes, s.last_seen) AS resource_attributes,+ argMax(s.llm_providers, s.last_seen) AS llm_providers,+ min(s.first_seen) AS first_seen,+ max(s.last_seen) AS last_seen+ FROM ${CONTROLLER_SERVICES_TABLE} s FINAL+ WHERE (s.workload_key, s.cluster_id) IN (SELECT workload_key, cluster_id FROM stopped_keys)+ AND s.workload_key != ''+ ${clusterPredicate}+ GROUP BY concat(s.cluster_id, ':', s.service_name)+ HAVING (cluster_id, service_name) NOT IN (SELECT cluster_id, service_name FROM active)+ ),+ latest AS (+ SELECT * FROM active+ UNION ALL+ SELECT * FROM stopped)SELECTlatest.id AS id,diff --git a/src/client/src/lib/platform/agents/observability-view.ts b/src/client/src/lib/platform/agents/observability-view.tsindex de9cd5c3..b5f0c3de 100644--- a/src/client/src/lib/platform/agents/observability-view.ts+++ b/src/client/src/lib/platform/agents/observability-view.ts@@ -29,8 +29,26 @@import type { UnifiedAgent } from "@/types/agents";-export type Feature = "llm" | "agent";-export type Direction = "enabling" | "disabling";+export type Feature = "llm" | "agent" | "lifecycle";+/**+ * Direction of the in-flight transition.+ *+ * For `llm` and `agent` features: only `enabling` / `disabling` apply.+ * For `lifecycle`:+ * - `starting` — Play queued while the workload is stopped.+ * - `stopping` — Stop queued while the workload is running.+ * - `restarting` — Restart queued; the workload will return to `running`.+ * We expose `restarting` as a distinct direction (rather than reusing+ * `enabling`) because the UI renders a different spinner glyph and a+ * different toast for Restart, and the optimistic-intent store wants to+ * match by exact action.+ */+export type Direction =+ | "enabling"+ | "disabling"+ | "starting"+ | "stopping"+ | "restarting";export interface OptimisticIntent {feature: Feature;@@ -82,11 +100,21 @@ const AGENT_ACTIONS = new Set<string>(["enable_python_sdk","disable_python_sdk",]);+const LIFECYCLE_ACTIONS = new Set<string>([+ "start_workload",+ "stop_workload",+ "restart_workload",+]);function isFeatureAction(feature: Feature, action: string): boolean {- return feature === "llm"- ? LLM_ACTIONS.has(action)- : AGENT_ACTIONS.has(action);+ switch (feature) {+ case "llm":+ return LLM_ACTIONS.has(action);+ case "agent":+ return AGENT_ACTIONS.has(action);+ case "lifecycle":+ return LIFECYCLE_ACTIONS.has(action);+ }}function actionDirection(action: string): Direction | null {@@ -97,6 +125,12 @@ function actionDirection(action: string): Direction | null {case "uninstrument":case "disable_python_sdk":return "disabling";+ case "start_workload":+ return "starting";+ case "stop_workload":+ return "stopping";+ case "restart_workload":+ return "restarting";default:return null;}@@ -133,6 +167,24 @@ function agentDesiredEnabled(agent: UnifiedAgent): boolean {return agent.desired_agent_status === "enabled";}+function lifecycleActualEnabled(agent: UnifiedAgent): boolean {+ // "Enabled" for lifecycle means "the workload is up". `restarting`+ // counts as enabled because the workload will be back in seconds and+ // we don't want the Stop button to disappear during the bounce.+ return (+ agent.lifecycle_status === "running" ||+ agent.lifecycle_status === "restarting"+ );+}++function lifecycleDesiredEnabled(agent: UnifiedAgent): boolean {+ // Default to running when the desired-state is unknown. Most rows+ // start out without an explicit lifecycle desired state -- only Stop+ // writes one. Treating unknown as `running` matches the user's+ // expectation that an undiscovered workload is "on" by default.+ return agent.desired_lifecycle_status !== "stopped";+}+function buildPodSummary(agent: UnifiedAgent): PodSummary | null {const total = agent.pods_total ?? 0;if (total === 0) return null;@@ -143,16 +195,36 @@ function buildPodSummary(agent: UnifiedAgent): PodSummary | null {};}+function featureActualEnabled(feature: Feature, agent: UnifiedAgent): boolean {+ switch (feature) {+ case "llm":+ return llmActualEnabled(agent);+ case "agent":+ return agentActualEnabled(agent);+ case "lifecycle":+ return lifecycleActualEnabled(agent);+ }+}++function featureDesiredEnabled(feature: Feature, agent: UnifiedAgent): boolean {+ switch (feature) {+ case "llm":+ return llmDesiredEnabled(agent);+ case "agent":+ return agentDesiredEnabled(agent);+ case "lifecycle":+ return lifecycleDesiredEnabled(agent);+ }+}+export function getObservabilityView(agent: UnifiedAgent,feature: Feature,intent: OptimisticIntent | null,now: number = Date.now()): ObservabilityView {- const actualEnabled =- feature === "llm" ? llmActualEnabled(agent) : agentActualEnabled(agent);- const desiredEnabled =- feature === "llm" ? llmDesiredEnabled(agent) : agentDesiredEnabled(agent);+ const actualEnabled = featureActualEnabled(feature, agent);+ const desiredEnabled = featureDesiredEnabled(feature, agent);const podSummary = buildPodSummary(agent);const isManual = feature === "agent" && isManualAgentObservability(agent);@@ -160,8 +232,15 @@ export function getObservabilityView(// expires or the server's pending_action confirms it (in which case we// fall through to the pending_action branch which is also a spinner).if (intent && intent.feature === feature && intent.expiresAt > now) {+ // For lifecycle: starting/restarting are conceptually "enabled"+ // (workload will be up); stopping is "disabled" (workload will+ // be down).+ const optimisticEnabled =+ intent.direction === "enabling" ||+ intent.direction === "starting" ||+ intent.direction === "restarting";return {- enabled: intent.direction === "enabling",+ enabled: optimisticEnabled,transitioning: true,direction: intent.direction,podSummary,@@ -180,10 +259,14 @@ export function getObservabilityView(if (hasServerPending) {const direction = actionDirection(pendingActionRaw);+ const willEnable =+ direction === "enabling" ||+ direction === "starting" ||+ direction === "restarting";return {// Spinner button needs a stable "what it'll be next" so the label// doesn't flicker between optimistic and pending_action phases.- enabled: direction === "enabling",+ enabled: willEnable,transitioning: true,direction,podSummary,@@ -194,16 +277,44 @@ export function getObservabilityView(// (3) Desired state diverges from actual — usually the brief window// between the API write and the first action row, or a controller retry- // loop. Pure controller-source rows only. SDK / both rows always have+ // loop. Pure controller-source rows only for `llm`/`agent`; lifecycle+ // rows always go through controller-managed actions so the SDK-backed+ // short-circuit does not apply. SDK / both rows always have// `desired_*='none'` (the controller never wrote a desired state, the// SDK self-enrolled by emitting traces) which would otherwise pin them// permanently in "disabling..." — that's the regression bug.const isSdkBacked = agent.source === "sdk" || agent.source === "both";- if (!isSdkBacked && desiredEnabled !== actualEnabled) {+ const skipForSdk = isSdkBacked && feature !== "lifecycle";+ if (!skipForSdk && desiredEnabled !== actualEnabled) {+ let direction: Direction;+ if (feature === "lifecycle") {+ direction = desiredEnabled ? "starting" : "stopping";+ } else {+ direction = desiredEnabled ? "enabling" : "disabling";+ }return {enabled: actualEnabled,transitioning: true,- direction: desiredEnabled ? "enabling" : "disabling",+ direction,+ podSummary,+ isManual,+ source: "desired_mismatch",+ };+ }++ // (3b) Lifecycle-specific in-flight signal. The controller stamps+ // `openlit.lifecycle.status = "restarting"` on the in-memory service+ // for the heartbeat immediately following a Restart. Both+ // optimistic intent and pending_action will have already covered+ // the user-initiated case, but a second browser tab that loads+ // after the click would otherwise miss the spinner entirely.+ // Treating actual=='restarting' as transitioning here is the+ // failsafe that keeps the icon set consistent across tabs.+ if (feature === "lifecycle" && agent.lifecycle_status === "restarting") {+ return {+ enabled: true,+ transitioning: true,+ direction: "restarting",podSummary,isManual,source: "desired_mismatch",diff --git a/src/client/src/lib/platform/controller/features/index.ts b/src/client/src/lib/platform/controller/features/index.tsindex 6e183a4a..94600850 100644--- a/src/client/src/lib/platform/controller/features/index.ts+++ b/src/client/src/lib/platform/controller/features/index.ts@@ -11,5 +11,6 @@ export type {import "./instrumentation";import "./agent";+import "./lifecycle";import "./prompts";import "./envs";diff --git a/src/client/src/lib/platform/controller/features/lifecycle.ts b/src/client/src/lib/platform/controller/features/lifecycle.tsnew file mode 100644index 00000000..f399eccd--- /dev/null+++ b/src/client/src/lib/platform/controller/features/lifecycle.ts@@ -0,0 +1,365 @@+/**+ * Lifecycle feature handler — drives Play / Stop / Restart for+ * controller-managed workloads.+ *+ * The handler is the dashboard-side mirror of openlit-controller's+ * engine/lifecycle.go. It only writes desired state + queues the action;+ * the controller does the actual work. The actual workload status comes+ * back to the dashboard through two channels:+ * 1. The controller's next heartbeat reports+ * resource_attributes["openlit.lifecycle.status"]. That feeds the+ * pod_set rollup in lib/platform/agents/index.ts.+ * 2. For Stop, the controller's ActionResult carries a snapshot blob+ * (pod spec for K8s naked pods, exe+cmdline+cwd+env-subset for bare+ * Linux processes). The poll route persists the blob into+ * desired_states_v2.config so a later Start can hand it back.+ *+ * The Restart path is *one-shot* on the reconcile loop: clicking Restart+ * does NOT change desired_status (it stays 'running'), and we never emit+ * a restart action from reconcile() — only directly from applyOperation.+ * That prevents the reconciler from looping forever on Restart desired-+ * vs-actual mismatches that have no closed-form fix.+ */++import {+ getServiceById,+ getControllerInstanceById,+ getControllerIdsForWorkload,+ getFeatureDesiredStates,+ queueAction,+ updateFeatureDesiredState,+} from "@/lib/platform/controller";+import { KNOWN_ACTIONS } from "@/types/controller";+import type {+ FeatureDesiredState,+ EnvironmentFeatureConfig,+} from "@/types/controller";+import type {+ FeatureHandler,+ ReportedService,+ ReconcileAction,+} from "./registry";+import { registerFeature } from "./registry";++const FEATURE = "lifecycle";++const DESIRED_RUNNING = "running";+const DESIRED_STOPPED = "stopped";++/**+ * The controller advertises its per-mode lifecycle support via the+ * `controller.capabilities` resource attribute (see+ * engine.go::ControllerCapabilities). Modes that do not appear here mean+ * the dashboard should not let the user click Play/Stop/Restart for+ * workloads attached to that controller — the action would be queued+ * but the Go engine would have nothing to do with it.+ */+function capabilityForMode(mode: string | undefined): { value: string; prefix: boolean } {+ switch (mode) {+ case "kubernetes":+ return { value: "lifecycle_kubernetes_v1", prefix: false };+ case "docker":+ return { value: "lifecycle_docker_v1", prefix: false };+ case "linux":+ return { value: "lifecycle_linux_", prefix: true };+ default:+ return { value: "", prefix: false };+ }+}++async function resolveLifecycleContext(+ serviceId: string,+ dbConfigId?: string+): Promise<+ | { error: Response }+ | {+ service: any;+ supportsLifecycle: boolean;+ mode: string;+ targets: string[];+ }+> {+ const serviceRes = await getServiceById(serviceId, dbConfigId);+ if (!serviceRes.data || serviceRes.data.length === 0) {+ return { error: Response.json({ error: "Service not found" }, { status: 404 }) };+ }+ const service = serviceRes.data[0];+ if (!service.workload_key) {+ return {+ error: Response.json(+ { error: "Service is missing workload_key" },+ { status: 500 }+ ),+ };+ }++ const instanceRes = await getControllerInstanceById(+ service.controller_instance_id,+ dbConfigId+ );+ const instance = instanceRes.data?.[0];+ const mode = instance?.mode || "linux";+ const expected = capabilityForMode(mode);+ const capabilities =+ instance?.resource_attributes?.["controller.capabilities"] || "";+ const supportsLifecycle = capabilities+ .split(",")+ .map((cap) => cap.trim())+ .some((cap) =>+ expected.prefix+ ? cap.startsWith(expected.value)+ : cap === expected.value+ );++ // Multi-controller fan-out: same as instrumentation.ts. For K8s+ // workload-level operations (scale, rollout-restart) the action is+ // idempotent across controllers — atomicScaleWithAnnotation/+ // bumpRolloutAnnotation use resourceVersion-based optimistic+ // concurrency so the second controller's update either no-ops or+ // retries cleanly on conflict. For per-pod operations (naked pods,+ // Linux modes) every owning controller acting against its own pod is+ // the correct behaviour.+ const controllerIds = await getControllerIdsForWorkload(+ service.service_name,+ service.namespace || "",+ service.cluster_id || "default",+ dbConfigId+ );+ const targets =+ controllerIds.data?.map((r) => r.controller_instance_id) || [];+ if (targets.length === 0) {+ targets.push(service.controller_instance_id);+ }++ return { service, supportsLifecycle, mode, targets };+}++const lifecycleHandler: FeatureHandler = {+ feature: FEATURE,++ validatePayload(operation: string, _payload: Record<string, unknown>) {+ if (+ operation !== "start" &&+ operation !== "stop" &&+ operation !== "restart"+ ) {+ return `Unknown operation "${operation}" for feature "${FEATURE}". Expected "start", "stop", or "restart".`;+ }+ return null;+ },++ async applyOperation(+ serviceId: string,+ operation: string,+ _payload: Record<string, unknown>,+ dbConfigId?: string+ ): Promise<Response> {+ try {+ const ctx = await resolveLifecycleContext(serviceId, dbConfigId);+ if ("error" in ctx) return ctx.error;++ if (!ctx.supportsLifecycle) {+ return Response.json(+ {+ error:+ "The selected controller does not advertise lifecycle support for this mode yet.",+ },+ { status: 409 }+ );+ }++ const workloadKey = ctx.service.workload_key as string;+ const clusterId = (ctx.service.cluster_id as string) || "default";++ if (operation === "stop") {+ // Write the desired state up-front so the UI sees the+ // stopped intent immediately. The controller will return+ // a snapshot blob on the action result, which the poll+ // route persists into the same row's `config` column so+ // a later Start can hand it back. We deliberately do not+ // require the snapshot at write time -- naked pods and+ // bare processes are the only modes that *need* one, and+ // the controller computes it from local state.+ await updateFeatureDesiredState(+ workloadKey,+ clusterId,+ FEATURE,+ DESIRED_STOPPED,+ "{}",+ dbConfigId+ );+ await Promise.all(+ ctx.targets.map((cid) =>+ queueAction(+ cid,+ KNOWN_ACTIONS.STOP_WORKLOAD,+ workloadKey,+ "{}",+ dbConfigId+ )+ )+ );+ return Response.json({+ status: "queued",+ action: KNOWN_ACTIONS.STOP_WORKLOAD,+ controllers: ctx.targets.length,+ });+ }++ if (operation === "start") {+ // Pull the snapshot stashed by the previous Stop. What+ // the snapshot contains depends on the mode:+ // - K8s naked pod: full gzipped pod spec (recreated+ // by the controller).+ // - Linux bare process: exe + argv + cwd + env+ // allowlist (re-execed).+ // - K8s controlled (Deployment/STS/DS): small+ // {kind, ns, name, container_name} identifier that+ // lets startK8s scale back up even when the+ // controller's in-memory svc cache has been pruned.+ // - Docker, systemd: "{}" — runtime preserves the+ // object across Stop (container ID / unit file),+ // no replay data needed.+ const desiredRes = await getFeatureDesiredStates(+ [workloadKey],+ clusterId,+ [FEATURE],+ dbConfigId+ );+ const row = desiredRes.data?.find(+ (r) => r.feature === FEATURE+ );+ const snapshot = row?.config || "{}";++ // If the workload mode requires a snapshot but the+ // desired-states row has none (e.g. controller was+ // upgraded between Stop and Start, or the user clicks+ // Play on a workload that the controller never stopped),+ // fail closed -- the controller would otherwise error+ // when trying to recreate the pod or re-exec the process.+ //+ // Systemd carve-out: Linux *systemd* units do NOT need a+ // snapshot because the unit file still exists on disk+ // and `systemctl start <unit>` is sufficient. Only Linux+ // *bare* processes (no `systemd.unit` resource attr)+ // need the captured exe+argv+cwd+env to re-exec.+ const k8sKind = ctx.service.resource_attributes?.["k8s.workload.kind"];+ const isLinuxBare =+ ctx.mode === "linux" &&+ !ctx.service.resource_attributes?.["systemd.unit"];+ const isK8sNakedPod =+ ctx.mode === "kubernetes" &&+ (k8sKind === "" || k8sKind === "Pod" || k8sKind === undefined);+ const requiresSnapshot = isLinuxBare || isK8sNakedPod;+ if (requiresSnapshot && (!snapshot || snapshot === "{}")) {+ return Response.json(+ {+ error:+ "No saved snapshot is available to bring this workload back up. The controller cannot recreate naked pods or bare processes without a snapshot captured at Stop time.",+ },+ { status: 409 }+ );+ }++ await updateFeatureDesiredState(+ workloadKey,+ clusterId,+ FEATURE,+ DESIRED_RUNNING,+ snapshot,+ dbConfigId+ );+ await Promise.all(+ ctx.targets.map((cid) =>+ queueAction(+ cid,+ KNOWN_ACTIONS.START_WORKLOAD,+ workloadKey,+ snapshot,+ dbConfigId+ )+ )+ );+ return Response.json({+ status: "queued",+ action: KNOWN_ACTIONS.START_WORKLOAD,+ controllers: ctx.targets.length,+ });+ }++ // Restart: keep desired_status='running' (no flip). We do not+ // touch desired_states here because the reconcile loop must+ // not interpret Restart as a continuous desired-state+ // mismatch.+ await Promise.all(+ ctx.targets.map((cid) =>+ queueAction(+ cid,+ KNOWN_ACTIONS.RESTART_WORKLOAD,+ workloadKey,+ "{}",+ dbConfigId+ )+ )+ );+ return Response.json({+ status: "queued",+ action: KNOWN_ACTIONS.RESTART_WORKLOAD,+ controllers: ctx.targets.length,+ });+ } catch (error: any) {+ return Response.json(+ { error: error.message || "Lifecycle operation failed" },+ { status: 500 }+ );+ }+ },++ reconcile(+ reportedServices: ReportedService[],+ desiredStates: Map<string, FeatureDesiredState>,+ _envConfig?: EnvironmentFeatureConfig+ ): ReconcileAction[] {+ const actions: ReconcileAction[] = [];++ for (const svc of reportedServices) {+ const desired = desiredStates.get(svc.workload_key);+ if (!desired) continue;++ const actual =+ svc.resource_attributes?.["openlit.lifecycle.status"] ||+ "running";++ if (+ desired.desired_status === DESIRED_RUNNING &&+ actual === "stopped"+ ) {+ actions.push({+ actionType: KNOWN_ACTIONS.START_WORKLOAD,+ serviceKey: svc.workload_key,+ payload: desired.config || "{}",+ });+ }+ if (+ desired.desired_status === DESIRED_STOPPED &&+ actual !== "stopped"+ ) {+ actions.push({+ actionType: KNOWN_ACTIONS.STOP_WORKLOAD,+ serviceKey: svc.workload_key,+ payload: "{}",+ });+ }+ // Restart is intentionally NOT reconciled. It's a one-shot+ // action queued directly from applyOperation -- a reconcile+ // emission would either loop forever or paper over a genuine+ // stop with a restart.+ }++ return actions;+ },+};++registerFeature(lifecycleHandler);++export default lifecycleHandler;diff --git a/src/client/src/lib/platform/controller/index.ts b/src/client/src/lib/platform/controller/index.tsindex e9679e01..39d6050a 100644--- a/src/client/src/lib/platform/controller/index.ts+++ b/src/client/src/lib/platform/controller/index.ts@@ -685,6 +685,47 @@ export async function markActionsAcknowledged();}+/**+ * Look up a batch of actions by ID for a given controller instance.+ * Used by the poll route to inspect action_type / service_key for+ * lifecycle Stop actions whose results carry a snapshot blob that+ * needs persisting into desired_states_v2.config. Returning the+ * created_at lets callers preserve ordering when sorting fanned-out+ * fan-in actions.+ */+export async function getActionsByIds(+ actionIds: string[],+ instanceId: string,+ dbConfigId?: string+): Promise<{+ err?: unknown;+ data?: Array<{+ id: string;+ action_type: ActionType;+ service_key: string;+ }>;+}> {+ if (actionIds.length === 0) return { data: [] };+ const escaped = actionIds+ .map((id) => `'${escapeClickHouse(id)}'`)+ .join(", ");+ const query = `+ SELECT id, action_type, service_key+ FROM ${CONTROLLER_ACTIONS_TABLE}+ FINAL+ WHERE instance_id = '${escapeClickHouse(instanceId)}'+ AND id IN (${escaped})+ `;+ return dataCollector({ query }, "query", dbConfigId) as Promise<{+ err?: unknown;+ data?: Array<{+ id: string;+ action_type: ActionType;+ service_key: string;+ }>;+ }>;+}+export async function completeAction(actionId: string,instanceId: string,diff --git a/src/client/src/types/agents.ts b/src/client/src/types/agents.tsindex d92ec6e8..ab4138dd 100644--- a/src/client/src/types/agents.ts+++ b/src/client/src/types/agents.ts@@ -58,6 +58,26 @@ export interface UnifiedAgent {desired_instrumentation_status: "none" | "instrumented";agent_observability_status: "enabled" | "disabled" | "manual" | "";desired_agent_status: "none" | "enabled";+ /**+ * Current lifecycle state of the workload, rolled up from+ * `resource_attributes['openlit.lifecycle.status']` on+ * `openlit_controller_services` plus the matching+ * `feature='lifecycle'` row in `openlit_controller_desired_states_v2`.+ *+ * - `running`: the controller's heartbeat reports the workload as up.+ * - `stopped`: the user clicked Stop and the workload is scaled down.+ * The row is still surfaced because the controller's desired-state+ * table holds `desired_status='stopped'` indefinitely (up to the+ * 90-day TTL) so the agent does not disappear from the list.+ * - `restarting`: a Restart action is in flight.+ * - `unknown`: SDK-only agents or rows the controller hasn't reported on.+ */+ lifecycle_status: "running" | "stopped" | "restarting" | "unknown";+ /**+ * Mirrors `agent_observability` desired-state shape so the UI can compare+ * actual vs desired and render transition states identically.+ */+ desired_lifecycle_status: "running" | "stopped" | "unknown";pending_action: string | null;pending_action_status: "pending" | "acknowledged" | null;diff --git a/src/client/src/types/controller.ts b/src/client/src/types/controller.tsindex e8eac3cb..e6b21b45 100644--- a/src/client/src/types/controller.ts+++ b/src/client/src/types/controller.ts@@ -121,6 +121,9 @@ export const KNOWN_ACTIONS = {REMOVE_PROMPTS: "remove_prompts",PUSH_ENVS: "push_envs",REMOVE_ENVS: "remove_envs",+ START_WORKLOAD: "start_workload",+ STOP_WORKLOAD: "stop_workload",+ RESTART_WORKLOAD: "restart_workload",} as const;export interface FeatureDesiredState {