Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

36.8 KB
Raw Blame
package api

import (
	"bytes"
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/actionlifecycle"
	"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
	unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/auth"
)

type fakeDockerActionAgentCommander struct {
	results          []*agentexec.CommandResultPayload
	calls            []agentexec.ExecuteCommandPayload
	callAgents       []string
	typedCalls       []agentexec.DockerContainerLifecyclePayload
	typedUpdateCalls []agentexec.DockerContainerUpdatePayload
	connected        map[string]bool
	agentByHost      map[string]string
	queryResult      operationreceipt.QueryResult
	queries          []operationreceipt.Identity
}

type scopedFakeDockerActionAgentCommander struct {
	*fakeDockerActionAgentCommander
	tokenAgents map[string]string
	lastOrg     string
}

type observingDockerActionAgentCommander struct {
	*fakeDockerActionAgentCommander
	observations []agentexec.DockerContainerObservationPayload
}

func (f *observingDockerActionAgentCommander) ObserveDockerContainer(_ context.Context, _ string, req agentexec.DockerContainerObservationPayload) (*agentexec.DockerContainerObservationResultPayload, error) {
	if err := agentexec.BindDockerContainerObservationPayload(&req); err != nil {
		return nil, err
	}
	f.observations = append(f.observations, req)
	return &agentexec.DockerContainerObservationResultPayload{
		RequestID: req.RequestID, ActionID: req.ActionID, ProtocolVersion: req.ProtocolVersion, RequestDigest: req.RequestDigest,
		Observed: true,
		Snapshot: agentexec.DockerContainerObservationSnapshot{ContainerID: req.ContainerID, State: "running", Running: true, Health: agentexec.DockerContainerHealthHealthy, StartedAt: time.Now().UTC(), ObservedAt: time.Now().UTC()},
	}, nil
}

func (f *scopedFakeDockerActionAgentCommander) IsAgentConnectedForOrganization(organizationID, agentID string) bool {
	f.lastOrg = organizationID
	return f.IsAgentConnected(agentID)
}

func (f *scopedFakeDockerActionAgentCommander) GetAgentForHostForOrganization(organizationID, hostname string) (string, bool) {
	f.lastOrg = organizationID
	return f.GetAgentForHost(hostname)
}

func (f *scopedFakeDockerActionAgentCommander) GetAgentForTokenForOrganization(organizationID, tokenID string) (string, bool) {
	f.lastOrg = organizationID
	agentID, ok := f.tokenAgents[tokenID]
	return agentID, ok
}

func (f *scopedFakeDockerActionAgentCommander) GetAgentForIdentityForOrganization(organizationID, agentID, hostname string) (string, bool) {
	f.lastOrg = organizationID
	hostAgentID, ok := f.GetAgentForHost(hostname)
	return agentID, ok && hostAgentID == agentID && f.IsAgentConnected(agentID)
}

func dockerActionDispatchContext(t *testing.T, executor dockerContainerActionExecutor, record unified.ActionAuditRecord) context.Context {
	t.Helper()
	attempt, err := unified.NewActionDispatchAttempt(record.ID, time.Now())
	if err != nil {
		t.Fatal(err)
	}
	attempt, err = executor.BindActionDispatch(context.Background(), record, attempt)
	if err != nil {
		t.Fatal(err)
	}
	return actionlifecycle.ContextWithCommittedDispatchAttempt(context.Background(), attempt)
}

func (f *fakeDockerActionAgentCommander) AgentOperationReceiptVersion(string) int { return 1 }

func (f *fakeDockerActionAgentCommander) QueryAgentOperation(_ context.Context, _ string, identity operationreceipt.Identity) (operationreceipt.QueryResult, error) {
	f.queries = append(f.queries, identity)
	return f.queryResult, nil
}

func (f *fakeDockerActionAgentCommander) ExecuteDockerContainerLifecycle(_ context.Context, agentID string, req agentexec.DockerContainerLifecyclePayload) (*agentexec.DockerContainerLifecycleResultPayload, error) {
	f.callAgents = append(f.callAgents, agentID)
	f.typedCalls = append(f.typedCalls, req)
	now := time.Now().UTC()
	return &agentexec.DockerContainerLifecycleResultPayload{
		RequestID: req.RequestID, ActionID: req.ActionID, Operation: req.Operation, OperationVersion: req.OperationVersion, RequestDigest: req.RequestDigest, ContainerID: req.ContainerID,
		ExecutionPhase: agentexec.DockerContainerPhaseComplete, MutationStarted: true, MutationCompleted: true, ReadbackRan: true,
		Before: agentexec.DockerContainerLifecycleSnapshot{ContainerID: req.ContainerID, State: req.ExpectedState, Running: true, Health: agentexec.DockerContainerHealthNone, StartedAt: req.ExpectedStartedAt, ObservedAt: now.Add(-time.Second)},
		After:  agentexec.DockerContainerLifecycleSnapshot{ContainerID: req.ContainerID, State: "running", Running: true, Health: agentexec.DockerContainerHealthNone, StartedAt: now, ObservedAt: now},
	}, nil
}

func (f *fakeDockerActionAgentCommander) ExecuteDockerContainerUpdate(_ context.Context, agentID string, req agentexec.DockerContainerUpdatePayload) (*agentexec.DockerContainerUpdateResultPayload, error) {
	f.callAgents = append(f.callAgents, agentID)
	f.typedUpdateCalls = append(f.typedUpdateCalls, req)
	now := time.Now().UTC()
	newID := "fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210"
	return &agentexec.DockerContainerUpdateResultPayload{
		RequestID: req.RequestID, ActionID: req.ActionID, Operation: req.Operation, OperationVersion: req.OperationVersion, RequestDigest: req.RequestDigest, ContainerID: req.ContainerID,
		ExecutionPhase: agentexec.DockerContainerPhaseComplete, MutationStarted: true, MutationCompleted: true, ReadbackRan: true,
		NewContainerID: newID, ContainerName: "api",
		OldImageDigest: "sha256:1111111111111111111111111111111111111111111111111111111111111111",
		NewImageDigest: "sha256:2222222222222222222222222222222222222222222222222222222222222222",
		BackupCreated:  true, BackupContainer: "api_pulse_backup_20260714_000000",
		After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: newID, State: "running", Running: true, Health: agentexec.DockerContainerHealthNone, StartedAt: now, ObservedAt: now},
	}, nil
}

func (f *fakeDockerActionAgentCommander) ExecuteCommand(_ context.Context, agentID string, cmd agentexec.ExecuteCommandPayload) (*agentexec.CommandResultPayload, error) {
	f.callAgents = append(f.callAgents, agentID)
	f.calls = append(f.calls, cmd)
	if len(f.results) == 0 {
		return &agentexec.CommandResultPayload{RequestID: cmd.RequestID, Success: true, ExitCode: 0}, nil
	}
	result := f.results[0]
	f.results = f.results[1:]
	return result, nil
}

func (f *fakeDockerActionAgentCommander) GetAgentForHost(hostname string) (string, bool) {
	if f.agentByHost == nil {
		return "", false
	}
	agentID, ok := f.agentByHost[strings.TrimSpace(hostname)]
	return agentID, ok
}

func (f *fakeDockerActionAgentCommander) IsAgentConnected(agentID string) bool {
	if f.connected == nil {
		return true
	}
	return f.connected[agentID]
}

func dockerActionCapabilityNames(capabilities []unified.ResourceCapability) []string {
	names := make([]string, 0, len(capabilities))
	for _, capability := range capabilities {
		names = append(names, capability.Name)
	}
	return names
}

func dockerActionReadinessByName(readinesses []unified.ResourceActionReadiness, name string) (unified.ResourceActionReadiness, bool) {
	for _, readiness := range readinesses {
		if readiness.Name == name {
			return readiness, true
		}
	}
	return unified.ResourceActionReadiness{}, false
}

func TestDockerCommandSessionResolutionUsesTenantAndRecoversFromStaleReportToken(t *testing.T) {
	agents := &scopedFakeDockerActionAgentCommander{
		fakeDockerActionAgentCommander: &fakeDockerActionAgentCommander{
			connected:   map[string]bool{"canonical-agent": true, "stale-agent": true},
			agentByHost: map[string]string{"docker-host": "stale-agent"},
		},
		tokenAgents: map[string]string{"fresh-token": "canonical-agent"},
	}
	executor := dockerContainerActionExecutor{agents: agents}
	ctx := context.WithValue(context.Background(), OrgIDContextKey, "org-b")
	resource := unified.Resource{Docker: &unified.DockerData{
		TokenID:  "fresh-token",
		AgentID:  "stale-agent",
		Hostname: "docker-host",
	}}

	agentID, err := executor.connectedDockerCommandAgentID(ctx, resource)
	if err != nil {
		t.Fatalf("resolve command session: %v", err)
	}
	if agentID != "canonical-agent" || agents.lastOrg != "org-b" {
		t.Fatalf("resolved agent/org = %q/%q, want canonical-agent/org-b", agentID, agents.lastOrg)
	}

	delete(agents.tokenAgents, "fresh-token")
	agents.agentByHost["docker-host"] = "canonical-agent"
	resource.Docker.AgentID = "canonical-agent"
	if agentID, err := executor.connectedDockerCommandAgentID(ctx, resource); err != nil || agentID != "canonical-agent" {
		t.Fatalf("stale token exact-identity fallback = %q, %v; want canonical-agent", agentID, err)
	}

	resource.Docker.AgentID = "stale-agent"
	if agentID, err := executor.connectedDockerCommandAgentID(ctx, resource); err == nil {
		t.Fatalf("stale token unexpectedly crossed report identity to %q", agentID)
	}
}

func TestDockerContainerActionExecutorDispatchesPodmanRestartAndVerification(t *testing.T) {
	now := time.Now().UTC()
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot: models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{
			dockerContainerActionResource("app-container:api", "podman", "running", now),
		},
	})
	agents := &fakeDockerActionAgentCommander{}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)
	record := dockerContainerActionRecord("act_container", "app-container:api", "restart")

	result, err := executor.ExecuteAction(dockerActionDispatchContext(t, executor, record), record)
	if err != nil {
		t.Fatalf("ExecuteAction: %v", err)
	}
	if result == nil || !result.Success || result.Verification == nil || !result.Verification.Success {
		t.Fatalf("result = %#v, want successful execution and verification", result)
	}
	if len(agents.typedCalls) != 1 || len(agents.calls) != 0 {
		t.Fatalf("typed/raw agent calls = %d/%d, want one typed dispatch and zero raw commands", len(agents.typedCalls), len(agents.calls))
	}
	if got := agents.typedCalls[0].Operation; got != agentexec.DockerContainerOperationRestart {
		t.Fatalf("typed operation = %q", got)
	}
	if agents.typedCalls[0].Runtime != "podman" {
		t.Fatalf("runtime = %q", agents.typedCalls[0].Runtime)
	}
	if agents.typedCalls[0].RequestID != "act_container.dispatch.1" {
		t.Fatalf("dispatch request identity = %q", agents.typedCalls[0].RequestID)
	}
}

func TestDockerContainerActionExecutorUsesProductionDaemonObservationForIndependentTruth(t *testing.T) {
	now := time.Now().UTC()
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot:  models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{dockerContainerActionResource("app-container:api", "docker", "running", now)},
	})
	agents := &observingDockerActionAgentCommander{fakeDockerActionAgentCommander: &fakeDockerActionAgentCommander{}}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)
	if executor.observer == nil {
		t.Fatal("production constructor did not wire the typed daemon observer")
	}
	record := dockerContainerActionRecord("act_container", "app-container:api", "restart")
	result, err := executor.ExecuteAction(dockerActionDispatchContext(t, executor, record), record)
	if err != nil {
		t.Fatalf("ExecuteAction: %v", err)
	}
	if len(agents.observations) != 1 || agents.observations[0].ActionID != record.ID {
		t.Fatalf("observations=%#v", agents.observations)
	}
	truth := result.ActionResultV2
	if truth == nil || truth.Execution.Status != unified.ActionExecutionSucceeded || truth.Verification.Status != unified.ActionVerificationConfirmed || truth.Verification.EvidenceClass != unified.ActionEvidenceIndependent {
		t.Fatalf("canonical truth=%#v", truth)
	}
	if len(truth.Verification.Evidence) != 1 || truth.Verification.Evidence[0].ObserverTrustDomain == truth.Verification.Evidence[0].ExecutorTrustDomain {
		t.Fatalf("verification evidence=%#v", truth.Verification.Evidence)
	}
}

func TestDockerContainerActionExecutorResolvesCommandAgentByDockerHostname(t *testing.T) {
	now := time.Now().UTC()
	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	resource.Docker.AgentID = "docker-source-1"
	resource.Docker.Hostname = "tower"
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot:  models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{resource},
	})
	agents := &fakeDockerActionAgentCommander{
		connected: map[string]bool{
			"docker-source-1": false,
			"command-agent-1": true,
		},
		agentByHost: map[string]string{"tower": "command-agent-1"},
	}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)

	readiness := executor.CheckActionAvailable(context.Background(), unified.ActionRequest{
		RequestID:      "req-availability",
		ResourceID:     "app-container:api",
		CapabilityName: "restart",
		Reason:         "operator requested restart",
		RequestedBy:    "operator",
	}, resource)
	if !readiness.Available {
		t.Fatalf("CheckActionAvailable readiness = %#v, want available through hostname-resolved command agent", readiness)
	}

	record := dockerContainerActionRecord("act_container", "app-container:api", "restart")
	result, err := executor.ExecuteAction(dockerActionDispatchContext(t, executor, record), record)
	if err != nil {
		t.Fatalf("ExecuteAction: %v", err)
	}
	if result == nil || !result.Success {
		t.Fatalf("result = %#v, want successful execution", result)
	}
	if len(agents.callAgents) != 1 {
		t.Fatalf("call agents = %#v, want one typed dispatch through hostname-resolved agent", agents.callAgents)
	}
	for _, agentID := range agents.callAgents {
		if agentID != "command-agent-1" {
			t.Fatalf("called agent %q, want command-agent-1; all calls %#v", agentID, agents.callAgents)
		}
	}
}

func TestDockerContainerActionExecutorFailsWhenCapabilityNoLongerAdvertised(t *testing.T) {
	now := time.Now().UTC()
	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	resource.Capabilities = nil
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot:  models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{resource},
	})
	agents := &fakeDockerActionAgentCommander{}
	executor := newDockerContainerActionExecutor(h, agents)

	result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_container"), dockerContainerActionRecord("act_container", "app-container:api", "restart"))
	if err == nil || !strings.Contains(err.Error(), "does not currently advertise restart capability") {
		t.Fatalf("ExecuteAction err = %v, result = %#v", err, result)
	}
	if len(agents.calls) != 0 {
		t.Fatalf("agent calls = %#v, want none", agents.calls)
	}
}

func TestDockerContainerActionExecutorAvailabilityRequiresConnectedAgent(t *testing.T) {
	now := time.Now().UTC()
	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot:  models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{resource},
	})
	agents := &fakeDockerActionAgentCommander{connected: map[string]bool{"agent-1": false}}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)

	readiness := executor.CheckActionAvailable(context.Background(), unified.ActionRequest{
		RequestID:      "req-availability",
		ResourceID:     "app-container:api",
		CapabilityName: "restart",
		Reason:         "operator requested restart",
		RequestedBy:    "operator",
	}, resource)
	if readiness.Available || readiness.ReasonCode != "command_agent_disconnected" || readiness.Reason != "Docker / Podman command agent is not connected." {
		t.Fatalf("CheckActionAvailable readiness = %#v, want disconnected agent", readiness)
	}
	if len(agents.calls) != 0 {
		t.Fatalf("agent calls = %#v, want none", agents.calls)
	}
}

func TestDockerContainerActionExecutorStaleFixtureIsTripleZero(t *testing.T) {
	now := time.Now().UTC()
	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	resource.SourceStatus[unified.SourceDocker] = unified.SourceStatus{Status: "stale", LastSeen: now.Add(-time.Hour)}
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}})
	agents := &fakeDockerActionAgentCommander{}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)
	readiness := executor.CheckActionAvailable(context.Background(), unified.ActionRequest{ResourceID: resource.ID, CapabilityName: "restart"}, resource)
	if readiness.Available || readiness.ReasonCode != "stale_inventory" || len(agents.typedCalls) != 0 || len(agents.calls) != 0 {
		t.Fatalf("stale readiness=%#v typed=%d raw=%d", readiness, len(agents.typedCalls), len(agents.calls))
	}
}

func TestDockerContainerActionExecutorCallbackLossReconcilesReceiptWithoutRedispatch(t *testing.T) {
	now := time.Now().UTC()
	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	startedAt := now.Add(-time.Minute)
	resource.Docker.StartedAt = &startedAt
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}})
	agents := &fakeDockerActionAgentCommander{}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)
	record := dockerContainerActionRecord("act_container", resource.ID, "restart")
	attempt, err := unified.NewActionDispatchAttempt(record.ID, now.Add(-5*time.Second))
	if err != nil {
		t.Fatal(err)
	}
	attempt, err = executor.BindActionDispatch(context.Background(), record, attempt)
	if err != nil {
		t.Fatal(err)
	}
	facts := agentexec.DockerContainerLifecycleResultPayload{
		RequestID: attempt.ID, ActionID: record.ID, Operation: attempt.OperationKind, OperationVersion: attempt.OperationVersion, RequestDigest: attempt.RequestDigest, ContainerID: resource.Docker.ContainerID,
		ExecutionPhase: agentexec.DockerContainerPhaseComplete, MutationStarted: true, MutationCompleted: true, ReadbackRan: true,
		Before: agentexec.DockerContainerLifecycleSnapshot{ContainerID: resource.Docker.ContainerID, State: "running", Running: true, StartedAt: startedAt, ObservedAt: now.Add(-2 * time.Second)},
		After:  agentexec.DockerContainerLifecycleSnapshot{ContainerID: resource.Docker.ContainerID, State: "running", Running: true, StartedAt: now.Add(-time.Second), ObservedAt: now.Add(-time.Second)},
	}
	raw, _ := json.Marshal(facts)
	identity := operationreceipt.Identity{AttemptID: attempt.ID, ActionID: attempt.ActionID, OperationKind: attempt.OperationKind, OperationVersion: attempt.OperationVersion, RequestDigest: attempt.RequestDigest, AgentID: attempt.AgentID}
	agents.queryResult = operationreceipt.QueryResult{Version: operationreceipt.ProtocolVersion, Status: operationreceipt.QueryFoundTerminal, Record: &operationreceipt.Record{Identity: identity, State: operationreceipt.StateTerminal, AcceptedAt: now.Add(-4 * time.Second), StartedAt: now.Add(-3 * time.Second), TerminalAt: now, ResultKind: agentexec.DockerContainerLifecycleReceiptKind, ResultVersion: agentexec.DockerContainerLifecycleReceiptVersion, Result: raw}}
	result, _, found, err := executor.ReconcileActionDispatch(context.Background(), record, attempt)
	if err != nil || !found || result == nil || result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded || len(agents.typedCalls) != 0 || len(agents.queries) != 1 {
		t.Fatalf("reconcile result=%#v found=%v err=%v typed=%d queries=%d", result, found, err, len(agents.typedCalls), len(agents.queries))
	}
}

func TestHandlePlanActionRejectsDisconnectedDockerContainerAgent(t *testing.T) {
	now := time.Now().UTC()
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot: models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{
			dockerContainerActionResource("app-container:api", "docker", "running", now),
		},
	})
	h.SetActionExecutor(newDockerContainerActionExecutor(h, &fakeDockerActionAgentCommander{
		connected: map[string]bool{"agent-1": false},
	}))

	rec := httptest.NewRecorder()
	req := httptest.NewRequest(http.MethodPost, "/api/actions/plan", bytes.NewBufferString(`{
		"requestId":"req-disconnected-agent",
		"resourceId":"app-container:api",
		"capabilityName":"restart",
		"reason":"operator requested restart",
		"requestedBy":"operator"
	}`))
	h.HandlePlanAction(rec, actionHandlerTestRequest(req, ""))

	if rec.Code != http.StatusConflict {
		t.Fatalf("plan status = %d, want %d, body=%s", rec.Code, http.StatusConflict, rec.Body.String())
	}
	if !strings.Contains(rec.Body.String(), `"error":"action_execution_unavailable"`) ||
		!strings.Contains(rec.Body.String(), `"reason":"Docker / Podman command agent is not connected."`) ||
		!strings.Contains(rec.Body.String(), `"reasonCode":"command_agent_disconnected"`) {
		t.Fatalf("unexpected response body: %s", rec.Body.String())
	}
	store, err := h.getStore("default")
	if err != nil {
		t.Fatalf("get store: %v", err)
	}
	audits, err := store.GetActionAudits("app-container:api", time.Time{}, 10)
	if err != nil {
		t.Fatalf("GetActionAudits: %v", err)
	}
	if len(audits) != 0 {
		t.Fatalf("audits = %#v, want none for refused plan", audits)
	}
}

func TestResourceResponsesFilterDisconnectedDockerLifecycleCapabilities(t *testing.T) {
	now := time.Now().UTC()
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot: models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{
			dockerContainerActionResource("app-container:api", "docker", "running", now),
		},
	})
	h.SetActionExecutor(newDockerContainerActionExecutor(h, &fakeDockerActionAgentCommander{
		connected: map[string]bool{"agent-1": false},
	}))

	listRec := httptest.NewRecorder()
	listReq := httptest.NewRequest(http.MethodGet, "/api/resources?type=app-container", nil)
	h.HandleListResources(listRec, listReq)
	if listRec.Code != http.StatusOK {
		t.Fatalf("list status = %d, body=%s", listRec.Code, listRec.Body.String())
	}
	var list ResourcesResponse
	if err := json.Unmarshal(listRec.Body.Bytes(), &list); err != nil {
		t.Fatalf("decode list: %v", err)
	}
	if len(list.Data) != 1 {
		t.Fatalf("list data len = %d, want 1", len(list.Data))
	}
	if got := dockerActionCapabilityNames(list.Data[0].Capabilities); len(got) != 0 {
		t.Fatalf("list capabilities = %#v, want none", got)
	}
	readiness, ok := dockerActionReadinessByName(list.Data[0].ActionReadiness, "restart")
	if !ok || readiness.Available || readiness.ReasonCode != "command_agent_disconnected" {
		t.Fatalf("list action readiness = %#v, ok=%v; want disconnected restart", list.Data[0].ActionReadiness, ok)
	}

	detailRec := httptest.NewRecorder()
	detailReq := httptest.NewRequest(http.MethodGet, "/api/resources/app-container:api", nil)
	h.HandleGetResource(detailRec, detailReq)
	if detailRec.Code != http.StatusOK {
		t.Fatalf("detail status = %d, body=%s", detailRec.Code, detailRec.Body.String())
	}
	var detail unified.Resource
	if err := json.Unmarshal(detailRec.Body.Bytes(), &detail); err != nil {
		t.Fatalf("decode detail: %v", err)
	}
	if got := dockerActionCapabilityNames(detail.Capabilities); len(got) != 0 {
		t.Fatalf("detail capabilities = %#v, want none", got)
	}
	readiness, ok = dockerActionReadinessByName(detail.ActionReadiness, "restart")
	if !ok || readiness.Available || readiness.Reason != "Docker / Podman command agent is not connected." {
		t.Fatalf("detail action readiness = %#v, ok=%v; want disconnected restart", detail.ActionReadiness, ok)
	}
}

func TestHandleExecuteActionRejectsNeverAutoRemediateBeforeExecutor(t *testing.T) {
	now := time.Date(2026, 5, 4, 14, 0, 0, 0, time.UTC)
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot: models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{
			{
				ID:        "vm:42",
				Type:      unified.ResourceTypeVM,
				Name:      "web-42",
				Status:    unified.StatusWarning,
				LastSeen:  now,
				UpdatedAt: now,
				Sources:   []unified.DataSource{unified.SourceProxmox},
				Capabilities: []unified.ResourceCapability{
					{
						Name:                 "restart",
						Type:                 unified.CapabilityTypeCommon,
						Description:          "Restart the VM",
						MinimumApprovalLevel: unified.ApprovalAdmin,
						InternalHandler:      "proxmox.vm.restart",
					},
				},
			},
		},
	})
	executor := &stubActionExecutor{result: &unified.ExecutionResult{Success: true, Output: "should not run"}}
	h.SetActionExecutor(executor)

	planRec := httptest.NewRecorder()
	planReq := httptest.NewRequest(http.MethodPost, "/api/actions/plan", bytes.NewBufferString(`{
		"requestId":"agent-run-locked",
		"resourceId":"vm:42",
		"capabilityName":"restart",
		"reason":"Recover after confirmed outage",
		"requestedBy":"agent:oncall-helper"
	}`))
	h.HandlePlanAction(planRec, actionHandlerTestRequest(planReq, ""))
	if planRec.Code != http.StatusOK {
		t.Fatalf("plan status = %d, body=%s", planRec.Code, planRec.Body.String())
	}
	var plan unified.ActionPlan
	if err := json.Unmarshal(planRec.Body.Bytes(), &plan); err != nil {
		t.Fatalf("decode plan: %v", err)
	}

	decisionRec := httptest.NewRecorder()
	decisionReq := httptest.NewRequest(http.MethodPost, "/api/actions/"+plan.ActionID+"/decision", bytes.NewBufferString(`{"outcome":"approved"}`))
	decisionReq.SetPathValue("id", plan.ActionID)
	decisionReq = decisionReq.WithContext(auth.WithUser(decisionReq.Context(), "[email protected]"))
	h.HandleDecideAction(decisionRec, actionHandlerTestRequest(decisionReq, ""))
	if decisionRec.Code != http.StatusOK {
		t.Fatalf("decision status = %d, body=%s", decisionRec.Code, decisionRec.Body.String())
	}

	store, err := h.getStore("default")
	if err != nil {
		t.Fatalf("get store: %v", err)
	}
	if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
		CanonicalID:        "vm:42",
		NeverAutoRemediate: true,
		SetAt:              now,
		SetBy:              "[email protected]",
	}); err != nil {
		t.Fatalf("SetResourceOperatorState: %v", err)
	}

	executeRec := httptest.NewRecorder()
	executeReq := httptest.NewRequest(http.MethodPost, "/api/actions/"+plan.ActionID+"/execute", bytes.NewBufferString(`{}`))
	executeReq.SetPathValue("id", plan.ActionID)
	executeReq = executeReq.WithContext(auth.WithUser(executeReq.Context(), "[email protected]"))
	h.HandleExecuteAction(executeRec, actionHandlerTestRequest(executeReq, ""))
	if executeRec.Code != http.StatusConflict {
		t.Fatalf("execute status = %d, body=%s", executeRec.Code, executeRec.Body.String())
	}
	if executor.calls != 0 {
		t.Fatalf("executor calls = %d, want none", executor.calls)
	}
	audit, ok, err := store.GetActionAudit(plan.ActionID)
	if err != nil {
		t.Fatalf("GetActionAudit: %v", err)
	}
	if !ok || audit.State != unified.ActionStateFailed || audit.Result == nil || !strings.HasPrefix(audit.Result.ErrorMessage, "resource_remediation_locked:") {
		t.Fatalf("locked audit = %#v, ok=%v", audit, ok)
	}
}

func dockerContainerActionResource(id, runtime, state string, now time.Time) unified.Resource {
	return unified.Resource{
		ID:         id,
		Type:       unified.ResourceTypeAppContainer,
		Technology: runtime,
		Name:       "api",
		Status:     unified.StatusOnline,
		LastSeen:   now,
		UpdatedAt:  now,
		Sources:    []unified.DataSource{unified.SourceDocker},
		SourceStatus: map[unified.DataSource]unified.SourceStatus{
			unified.SourceDocker: {Status: "online", LastSeen: now},
		},
		Docker: &unified.DockerData{
			AgentID:        "agent-1",
			ContainerID:    "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
			ContainerState: state,
			Runtime:        runtime,
		},
		Capabilities: []unified.ResourceCapability{
			{
				Name:                 "restart",
				Type:                 unified.CapabilityTypeCommon,
				Description:          "Restart this container",
				MinimumApprovalLevel: unified.ApprovalAdmin,
				Platform:             runtime,
				InternalHandler:      dockerContainerLifecycleHandler,
			},
		},
	}
}

func dockerContainerActionRecord(actionID, resourceID, operation string) unified.ActionAuditRecord {
	return unified.ActionAuditRecord{
		ID:        actionID,
		CreatedAt: time.Now().UTC(),
		UpdatedAt: time.Now().UTC(),
		State:     unified.ActionStateExecuting,
		Request: unified.ActionRequest{
			RequestID:      "req-" + actionID,
			ResourceID:     resourceID,
			CapabilityName: operation,
			Reason:         "test execution",
			RequestedBy:    "agent:oncall-helper",
			Params:         map[string]any{},
		},
		Plan: unified.ActionPlan{
			ActionID:         actionID,
			RequestID:        "req-" + actionID,
			Allowed:          true,
			RequiresApproval: true,
			ApprovalPolicy:   unified.ApprovalAdmin,
			PlannedAt:        time.Now().UTC().Add(-time.Minute),
			ExpiresAt:        time.Now().UTC().Add(time.Minute),
			ResourceVersion:  "resource:sha256:test",
			PolicyVersion:    "policy:sha256:test",
			PlanHash:         "sha256:test",
		},
	}
}

func dockerContainerUpdateActionResource(id, runtime string, now time.Time) unified.Resource {
	resource := dockerContainerActionResource(id, runtime, "running", now)
	resource.Docker.Image = "ghcr.io/example/api:latest"
	resource.Docker.UpdateStatus = &unified.DockerUpdateStatusMeta{
		UpdateAvailable: true,
		CurrentDigest:   "sha256:" + strings.Repeat("9", 64),
		LatestDigest:    "sha256:" + strings.Repeat("a", 64),
	}
	resource.Capabilities = append(resource.Capabilities, unified.ResourceCapability{
		Name:                 "update",
		Type:                 unified.CapabilityTypeCommon,
		Description:          "Update this container",
		MinimumApprovalLevel: unified.ApprovalAdmin,
		Platform:             runtime,
		InternalHandler:      dockerContainerUpdateHandler,
	})
	return resource
}

func TestDockerContainerActionExecutorDispatchesTypedUpdate(t *testing.T) {
	now := time.Now().UTC()
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot: models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{
			dockerContainerUpdateActionResource("app-container:api", "docker", now),
		},
	})
	agents := &fakeDockerActionAgentCommander{}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)
	record := dockerContainerActionRecord("act_update", "app-container:api", "update")

	result, err := executor.ExecuteAction(dockerActionDispatchContext(t, executor, record), record)
	if err != nil {
		t.Fatalf("ExecuteAction: %v", err)
	}
	if result == nil || !result.Success || result.Verification == nil || !result.Verification.Success {
		t.Fatalf("result = %#v, want successful execution and verification", result)
	}
	if len(agents.typedUpdateCalls) != 1 || len(agents.typedCalls) != 0 || len(agents.calls) != 0 {
		t.Fatalf("update/lifecycle/raw agent calls = %d/%d/%d, want exactly one typed update dispatch", len(agents.typedUpdateCalls), len(agents.typedCalls), len(agents.calls))
	}
	call := agents.typedUpdateCalls[0]
	if call.Operation != agentexec.DockerContainerOperationUpdate || call.ExpectedImageDigest != "sha256:"+strings.Repeat("9", 64) || call.Runtime != "docker" {
		t.Fatalf("typed update payload = %+v", call)
	}
	if result.ActionResultV2 == nil || result.ActionResultV2.Compensation.Support != unified.ActionCompensationDeclared || result.ActionResultV2.Compensation.Status != unified.ActionCompensationNotNeeded {
		t.Fatalf("compensation truth = %+v, want declared/not_needed", result.ActionResultV2)
	}
}

func TestDockerContainerActionExecutorRefusesUpdateThroughLifecycleHandler(t *testing.T) {
	now := time.Now().UTC()
	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	resource.Docker.Image = "ghcr.io/example/api:latest"
	resource.Capabilities = append(resource.Capabilities, unified.ResourceCapability{
		Name:                 "update",
		Type:                 unified.CapabilityTypeCommon,
		MinimumApprovalLevel: unified.ApprovalAdmin,
		Platform:             "docker",
		InternalHandler:      dockerContainerLifecycleHandler,
	})
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot:  models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{resource},
	})
	agents := &fakeDockerActionAgentCommander{}
	executor := newDockerContainerActionExecutor(h, agents).(dockerContainerActionExecutor)
	record := dockerContainerActionRecord("act_update_wrong", "app-container:api", "update")

	if readiness := executor.CheckActionAvailable(context.Background(), record.Request, resource); readiness.Available || readiness.Name != "" {
		t.Fatalf("readiness = %+v, want fail-closed for wrong handler", readiness)
	}
}

func TestDockerContainerActionAvailabilityDetailNamesMissedLookup(t *testing.T) {
	now := time.Now().UTC()
	request := unified.ActionRequest{
		RequestID:      "req-detail",
		ResourceID:     "app-container:api",
		CapabilityName: "restart",
		Reason:         "operator requested restart",
		RequestedBy:    "operator",
	}

	resource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot:  models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{resource},
	})
	executor := newDockerContainerActionExecutor(h, &fakeDockerActionAgentCommander{
		connected: map[string]bool{"agent-1": false},
	}).(dockerContainerActionExecutor)

	readiness := executor.CheckActionAvailable(context.Background(), request, resource)
	if readiness.Available || readiness.ReasonCode != "command_agent_disconnected" ||
		readiness.Reason != "Docker / Podman command agent is not connected." {
		t.Fatalf("identity-fallback readiness = %#v, want disconnected agent", readiness)
	}
	if !strings.Contains(readiness.Detail, `agent "agent-1"`) {
		t.Fatalf("identity-fallback detail = %q, want the missed agent identity named", readiness.Detail)
	}

	tokenResource := dockerContainerActionResource("app-container:api", "docker", "running", now)
	tokenResource.Docker.TokenID = "rotated-token"
	tokenResource.Docker.Hostname = "docker-host"
	tokenExecutor := newDockerContainerActionExecutor(h, &scopedFakeDockerActionAgentCommander{
		fakeDockerActionAgentCommander: &fakeDockerActionAgentCommander{
			connected: map[string]bool{"agent-1": true},
		},
		tokenAgents: map[string]string{},
	}).(dockerContainerActionExecutor)

	readiness = tokenExecutor.CheckActionAvailable(context.Background(), request, tokenResource)
	if readiness.Available || readiness.ReasonCode != "command_agent_disconnected" ||
		readiness.Reason != "Docker / Podman command agent is not connected." {
		t.Fatalf("token-named readiness = %#v, want disconnected agent", readiness)
	}
	if !strings.Contains(readiness.Detail, "rotated-token") ||
		!strings.Contains(readiness.Detail, `exact agent "agent-1" and hostname "docker-host" identity`) {
		t.Fatalf("token-named detail = %q, want the stale token and failed exact identity proof named", readiness.Detail)
	}
}

func TestHandlePlanActionRefusalEnvelopeCarriesDetail(t *testing.T) {
	now := time.Now().UTC()
	h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
	h.SetStateProvider(resourceUnifiedSeedProvider{
		snapshot: models.StateSnapshot{LastUpdate: now},
		resources: []unified.Resource{
			dockerContainerActionResource("app-container:api", "docker", "running", now),
		},
	})
	h.SetActionExecutor(newDockerContainerActionExecutor(h, &fakeDockerActionAgentCommander{
		connected: map[string]bool{"agent-1": false},
	}))

	rec := httptest.NewRecorder()
	req := httptest.NewRequest(http.MethodPost, "/api/actions/plan", bytes.NewBufferString(`{
		"requestId":"req-refusal-detail",
		"resourceId":"app-container:api",
		"capabilityName":"restart",
		"reason":"operator requested restart",
		"requestedBy":"operator"
	}`))
	h.HandlePlanAction(rec, actionHandlerTestRequest(req, ""))

	if rec.Code != http.StatusConflict {
		t.Fatalf("plan status = %d, want %d, body=%s", rec.Code, http.StatusConflict, rec.Body.String())
	}
	var envelope struct {
		Details map[string]string `json:"details"`
	}
	if err := json.Unmarshal(rec.Body.Bytes(), &envelope); err != nil {
		t.Fatalf("decode refusal envelope: %v", err)
	}
	if !strings.Contains(envelope.Details["detail"], `agent "agent-1"`) {
		t.Fatalf("envelope detail = %q, want the missed agent identity named", envelope.Details["detail"])
	}
}