Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

32.8 KB
Raw Blame
package api

import (
	"encoding/json"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
	unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestDockerContainerExecutionResultDerivesTruthOnlyFromFacts(t *testing.T) {
	now := time.Now().UTC()
	req := agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}
	tests := []struct {
		name         string
		facts        agentexec.DockerContainerLifecycleResultPayload
		execution    unified.ActionExecutionStatus
		verification unified.ActionVerificationStatus
		class        unified.ActionEvidenceClass
	}{
		{name: "preflight no effect", facts: dockerResultFacts(now, false, false, false, true), execution: unified.ActionExecutionNotRun, verification: unified.ActionVerificationInconclusive, class: unified.ActionEvidenceNone},
		{name: "completed confirmed", facts: dockerResultFacts(now, true, true, true, true), execution: unified.ActionExecutionSucceeded, verification: unified.ActionVerificationConfirmed, class: unified.ActionEvidenceAgentAttested},
		{name: "completed contradicted", facts: dockerResultFacts(now, true, true, true, false), execution: unified.ActionExecutionSucceeded, verification: unified.ActionVerificationContradicted, class: unified.ActionEvidenceAgentAttested},
		{name: "partial unknown", facts: dockerResultFacts(now, true, false, false, false), execution: unified.ActionExecutionInconclusive, verification: unified.ActionVerificationInconclusive, class: unified.ActionEvidenceNone},
	}
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", req, tc.facts, nil, now)
			if err != nil {
				t.Fatal(err)
			}
			if result.ActionResultV2.Execution.Status != tc.execution || result.ActionResultV2.Verification.Status != tc.verification || result.ActionResultV2.Verification.EvidenceClass != tc.class {
				t.Fatalf("truth = %#v", result.ActionResultV2)
			}
			if result.ActionResultV2.Compensation.Support != unified.ActionCompensationUnavailable {
				t.Fatalf("restart acquired rollback: %#v", result.ActionResultV2.Compensation)
			}
		})
	}
}

func TestDockerContainerExecutionResultPreservesTypedPreflightRefusal(t *testing.T) {
	now := time.Now().UTC()
	facts := dockerResultFacts(now, false, false, false, true)
	facts.ReasonCode = agentexec.ActionRefusalTargetStateChanged
	result, err := dockerContainerExecutionResult(
		"app-container:fixture",
		"agent-1",
		agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart},
		facts,
		nil,
		now,
	)
	if err != nil {
		t.Fatal(err)
	}
	if got := result.ActionResultV2.Execution.ReasonCode; got != agentexec.ActionRefusalTargetStateChanged {
		t.Fatalf("execution reason = %q, want %q", got, agentexec.ActionRefusalTargetStateChanged)
	}
}

func TestDockerContainerExecutionResultStaleReadbackIsInconclusive(t *testing.T) {
	now := time.Now().UTC()
	facts := dockerResultFacts(now.Add(-time.Hour), true, true, true, true)
	result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}, facts, nil, now)
	if err != nil {
		t.Fatal(err)
	}
	if result.ActionResultV2.Verification.Status != unified.ActionVerificationInconclusive || result.ActionResultV2.Verification.ReasonCode != "stale_agent_readback" {
		t.Fatalf("verification = %#v", result.ActionResultV2.Verification)
	}
}

func TestDockerContainerExecutionResultPreservesBoundedPositiveAgentClockSkew(t *testing.T) {
	receivedAt := time.Now().UTC()
	facts := dockerResultFacts(receivedAt.Add(2*time.Second), true, true, true, true)
	result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}, facts, nil, receivedAt)
	if err != nil {
		t.Fatal(err)
	}
	verification := result.ActionResultV2.Verification
	if verification.Status != unified.ActionVerificationConfirmed || len(verification.Evidence) != 1 {
		t.Fatalf("verification = %#v", verification)
	}
	if !verification.Evidence[0].ObservedAt.Equal(facts.After.ObservedAt) || !verification.Evidence[0].ReceivedAt.Equal(receivedAt) {
		t.Fatalf("evidence timestamps = observed %s received %s, want original observation and receipt boundary %s", verification.Evidence[0].ObservedAt, verification.Evidence[0].ReceivedAt, receivedAt)
	}
}

func TestDockerContainerExecutionResultRejectsExcessivePositiveAgentClockSkew(t *testing.T) {
	receivedAt := time.Now().UTC()
	facts := dockerResultFacts(receivedAt.Add(6*time.Minute), true, true, true, true)
	result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}, facts, nil, receivedAt)
	if err != nil {
		t.Fatal(err)
	}
	verification := result.ActionResultV2.Verification
	if verification.Status != unified.ActionVerificationInconclusive || verification.ReasonCode != "stale_agent_readback" || len(verification.Evidence) != 0 {
		t.Fatalf("verification = %#v", verification)
	}
}

func TestDockerContainerUpdateExecutionResultPreservesBoundedPositiveAgentClockSkew(t *testing.T) {
	receivedAt := time.Now().UTC()
	observedAt := receivedAt.Add(2 * time.Second)
	facts := agentexec.DockerContainerUpdateResultPayload{
		Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
		MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID,
		After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, Health: agentexec.DockerContainerHealthHealthy, ObservedAt: observedAt},
	}
	result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, receivedAt)
	if err != nil {
		t.Fatal(err)
	}
	verification := result.ActionResultV2.Verification
	if verification.Status != unified.ActionVerificationConfirmed || len(verification.Evidence) != 1 {
		t.Fatalf("verification = %#v", verification)
	}
	if !verification.Evidence[0].ObservedAt.Equal(observedAt) || !verification.Evidence[0].ReceivedAt.Equal(receivedAt) {
		t.Fatalf("evidence timestamps = observed %s received %s, want original observation and receipt boundary %s", verification.Evidence[0].ObservedAt, verification.Evidence[0].ReceivedAt, receivedAt)
	}
}

func TestDockerContainerExecutionResultClassifiesDistinctDaemonObserverAsIndependent(t *testing.T) {
	now := time.Now().UTC()
	facts := dockerResultFacts(now, true, true, true, true)
	facts.ContainerID = dockerLifecycleTestID
	req := agentexec.DockerContainerLifecyclePayload{ActionID: "action-1", Operation: agentexec.DockerContainerOperationRestart}
	observation := &dockerContainerPostconditionObservation{
		ObserverID: "colima-direct-cli", TrustDomain: "daemon:colima-direct", Method: "docker_api_inspect",
		Snapshot: dockerObservationSnapshotFromLifecycle(facts.After), ReceivedAt: now,
	}
	result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", req, facts, observation, now)
	if err != nil {
		t.Fatal(err)
	}
	truth := result.ActionResultV2.Verification
	if truth.Status != unified.ActionVerificationConfirmed || truth.EvidenceClass != unified.ActionEvidenceIndependent || len(truth.Evidence) != 1 || truth.Evidence[0].Digest == "" || len(truth.Evidence[0].Refs) != 1 {
		t.Fatalf("independent verification = %#v", truth)
	}
}

func TestDockerContainerExecutionResultDoesNotVerifyUnrecoveredHealth(t *testing.T) {
	now := time.Now().UTC()
	facts := dockerResultFacts(now, true, true, true, true)
	facts.ContainerID = dockerLifecycleTestID
	req := agentexec.DockerContainerLifecyclePayload{ActionID: "action-1", Operation: agentexec.DockerContainerOperationRestart}
	for _, health := range []string{"", agentexec.DockerContainerHealthStarting, agentexec.DockerContainerHealthUnhealthy} {
		observation := &dockerContainerPostconditionObservation{
			ObserverID: "docker-observer", TrustDomain: "docker-daemon:agent-1", Method: "docker_api_inspect",
			Snapshot: dockerObservationSnapshotFromLifecycle(facts.After), ReceivedAt: now,
		}
		observation.Snapshot.Health = health
		result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", req, facts, observation, now)
		if err != nil {
			t.Fatal(err)
		}
		if result.ActionResultV2.Verification.Status == unified.ActionVerificationConfirmed {
			t.Fatalf("health %q produced confirmed verification: %#v", health, result.ActionResultV2.Verification)
		}
	}
}

func TestDockerContainerExecutionResultStartingHealthRemainsInconclusive(t *testing.T) {
	now := time.Now().UTC()
	facts := dockerResultFacts(now, true, false, false, true)
	facts.ContainerID = dockerLifecycleTestID
	observation := &dockerContainerPostconditionObservation{
		ObserverID: "daemon", TrustDomain: "docker-daemon:agent-1", Method: "docker_api_inspect",
		Snapshot: dockerObservationSnapshotFromLifecycle(facts.After), ReceivedAt: now,
	}
	observation.Snapshot.Health = agentexec.DockerContainerHealthStarting
	result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{ActionID: "action", Operation: agentexec.DockerContainerOperationRestart}, facts, observation, now)
	if err != nil {
		t.Fatal(err)
	}
	if result.ActionResultV2.Execution.Status != unified.ActionExecutionInconclusive || result.ActionResultV2.Verification.Status != unified.ActionVerificationInconclusive || result.ActionResultV2.Verification.ReasonCode != "container_health_starting" {
		t.Fatalf("transitional health was treated as a final contradiction: %#v", result.ActionResultV2)
	}
}

func dockerObservationSnapshotFromLifecycle(snapshot agentexec.DockerContainerLifecycleSnapshot) agentexec.DockerContainerObservationSnapshot {
	return agentexec.DockerContainerObservationSnapshot{
		ContainerID: snapshot.ContainerID, State: snapshot.State, Running: snapshot.Running, Health: snapshot.Health,
		StartedAt: snapshot.StartedAt, RestartCount: snapshot.RestartCount, ObservedAt: snapshot.ObservedAt,
	}
}

func TestDockerContainerExecutionResultTreatsLegacyMissingHealthAsInconclusive(t *testing.T) {
	now := time.Now().UTC()
	facts := dockerResultFacts(now, true, true, true, true)
	facts.After.Health = ""
	result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}, facts, nil, now)
	if err != nil {
		t.Fatal(err)
	}
	verification := result.ActionResultV2.Verification
	if verification.Status != unified.ActionVerificationInconclusive || verification.ReasonCode != "container_health_unknown" {
		t.Fatalf("legacy health verification = %#v", verification)
	}
}

func dockerResultFacts(now time.Time, started, completed, readback, matches bool) agentexec.DockerContainerLifecycleResultPayload {
	beforeStart := now.Add(-time.Minute)
	afterStart := now
	state, running := "running", true
	if !matches {
		afterStart = beforeStart
		state, running = "exited", false
	}
	return agentexec.DockerContainerLifecycleResultPayload{
		ExecutionPhase: agentexec.DockerContainerPhaseComplete, MutationStarted: started, MutationCompleted: completed, ReadbackRan: readback,
		Before: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, Health: agentexec.DockerContainerHealthHealthy, StartedAt: beforeStart, ObservedAt: now.Add(-time.Second)},
		After:  agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: state, Running: running, Health: agentexec.DockerContainerHealthHealthy, StartedAt: afterStart, ObservedAt: now},
	}
}

const dockerLifecycleTestID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"

// A replacement identity alone is not proof that its reported running state
// survived until the independent daemon readback.
func TestDockerContainerUpdateIndependentObservationMustMatchState(t *testing.T) {
	now := time.Now().UTC()
	for _, tc := range []struct {
		name, baseline, state, health string
		running                       bool
		want                          unified.ActionVerificationStatus
	}{
		{"running", "running", "running", "healthy", true, unified.ActionVerificationConfirmed},
		{"no healthcheck", "running", "running", "none", true, unified.ActionVerificationConfirmed},
		{"stopped original", "created", "created", "none", false, unified.ActionVerificationConfirmed},
		{"stopped after update", "running", "exited", "", false, unified.ActionVerificationContradicted},
		{"restarting", "running", "restarting", "", true, unified.ActionVerificationContradicted},
		{"unhealthy", "running", "running", "unhealthy", true, unified.ActionVerificationContradicted},
		{"starting healthcheck", "running", "running", "starting", true, unified.ActionVerificationInconclusive},
		{"unknown health", "running", "running", "", true, unified.ActionVerificationContradicted},
		{"missing agent readback", "", "running", "healthy", true, unified.ActionVerificationInconclusive},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := agentexec.DockerContainerUpdateResultPayload{
				Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
				MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID,
				After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: "running", Running: true, ObservedAt: now},
			}
			facts.After.State = tc.baseline
			facts.After.Running = tc.baseline == "running"
			facts.ReadbackRan = tc.baseline != ""
			observation := &dockerContainerPostconditionObservation{
				ObserverID: "daemon-1", TrustDomain: "daemon:1", Method: "daemon_inspect", ReceivedAt: now,
				Snapshot: agentexec.DockerContainerObservationSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now},
			}
			result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, observation, now)
			if err != nil {
				t.Fatal(err)
			}
			if got := result.ActionResultV2.Verification; got.Status != tc.want || got.EvidenceClass != unified.ActionEvidenceIndependent {
				t.Fatalf("verification = %+v, want %s / independent", got, tc.want)
			}
			if tc.health == "starting" && result.ActionResultV2.Verification.ReasonCode != "container_health_starting" {
				t.Fatalf("reason = %q, want container_health_starting", result.ActionResultV2.Verification.ReasonCode)
			}
			if result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded {
				t.Fatal("readback changed execution history")
			}
		})
	}
}

func TestDockerContainerUpdateAgentReadbackMustSupportRunningClaim(t *testing.T) {
	now := time.Now().UTC()
	for _, tc := range []struct {
		name, state, health string
		running             bool
		want                unified.ActionVerificationStatus
	}{
		{"healthy", "running", "healthy", true, unified.ActionVerificationConfirmed},
		{"no healthcheck", "running", "none", true, unified.ActionVerificationConfirmed},
		{"stopped preserved", "created", "none", false, unified.ActionVerificationConfirmed},
		{"unhealthy", "running", "unhealthy", true, unified.ActionVerificationContradicted},
		{"starting", "running", "starting", true, unified.ActionVerificationInconclusive},
		{"restarting", "restarting", "healthy", true, unified.ActionVerificationContradicted},
		{"unknown health", "running", "", true, unified.ActionVerificationInconclusive},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := agentexec.DockerContainerUpdateResultPayload{
				Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
				MutationStarted: true, MutationCompleted: true, ReadbackRan: true, NewContainerID: dockerLifecycleTestID,
				After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now},
			}
			result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, now)
			if err != nil {
				t.Fatal(err)
			}
			if got := result.ActionResultV2.Verification; got.Status != tc.want || got.EvidenceClass != unified.ActionEvidenceAgentAttested {
				t.Fatalf("verification = %+v, want %s / agent_attested", got, tc.want)
			}
			if tc.health == "starting" && result.ActionResultV2.Verification.ReasonCode != "container_health_starting" {
				t.Fatalf("reason = %q, want container_health_starting", result.ActionResultV2.Verification.ReasonCode)
			}
			if result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded {
				t.Fatal("readback changed execution history")
			}
		})
	}
}

// The execution and verification summaries are the sentences people read in
// the action result, so they must say what Pulse saw in plain words while the
// typed facts stay on the evidence.
func TestDockerContainerExecutionResultSummariesSayWhatPulseSaw(t *testing.T) {
	now := time.Now().UTC()
	beforeStart := now.Add(-time.Minute)
	for _, tc := range []struct {
		name, operation, state, health string
		running                        bool
		startedAt                      time.Time
		status                         unified.ActionVerificationStatus
		want                           string
	}{
		{name: "restart recovered", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthHealthy, startedAt: now, status: unified.ActionVerificationConfirmed, want: "The container restarted and is running and healthy."},
		{name: "restart without health check", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthNone, startedAt: now, status: unified.ActionVerificationConfirmed, want: "The container restarted and is running."},
		{name: "restart health failing", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthUnhealthy, startedAt: now, status: unified.ActionVerificationContradicted, want: "The container restarted, but its health check is failing. Check its logs."},
		{name: "restart crash loop", operation: agentexec.DockerContainerOperationRestart, state: "restarting", running: true, health: agentexec.DockerContainerHealthNone, startedAt: now, status: unified.ActionVerificationContradicted, want: "The container is not running. Docker reports it as restarting, which usually means it keeps crashing."},
		{name: "restart same start time", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthNone, startedAt: beforeStart, status: unified.ActionVerificationContradicted, want: "The container is running, but its start time did not change, so it did not restart."},
		{name: "restart earlier start time", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthNone, startedAt: beforeStart.Add(-time.Hour), status: unified.ActionVerificationContradicted, want: "The container is running, but its start time is not newer than before, so Pulse could not confirm a restart."},
		{name: "restart missing start time", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthNone, status: unified.ActionVerificationContradicted, want: "The container is running, but Pulse did not receive its start time, so it could not confirm a restart."},
		{name: "restart health pending", operation: agentexec.DockerContainerOperationRestart, state: "running", running: true, health: agentexec.DockerContainerHealthStarting, startedAt: now, status: unified.ActionVerificationInconclusive, want: "The container is running. Its health check was still starting when Pulse checked. Wait a moment, then refresh its status."},
		{name: "running state without running flag", operation: agentexec.DockerContainerOperationStart, state: "running", running: false, health: agentexec.DockerContainerHealthNone, startedAt: now, status: unified.ActionVerificationContradicted, want: "The container is not running. Docker reports it as running but not marked as running."},
		{name: "start exited again", operation: agentexec.DockerContainerOperationStart, state: "exited", running: false, health: agentexec.DockerContainerHealthNone, startedAt: now, status: unified.ActionVerificationContradicted, want: "The container is not running. Docker reports it as stopped."},
		{name: "start health failing", operation: agentexec.DockerContainerOperationStart, state: "running", running: true, health: agentexec.DockerContainerHealthUnhealthy, startedAt: now, status: unified.ActionVerificationContradicted, want: "The container started, but its health check is failing. Check its logs."},
		{name: "stop still running", operation: agentexec.DockerContainerOperationStop, state: "running", running: true, health: agentexec.DockerContainerHealthNone, startedAt: beforeStart, status: unified.ActionVerificationContradicted, want: "The container is still running."},
		{name: "stop paused", operation: agentexec.DockerContainerOperationStop, state: "paused", running: false, health: agentexec.DockerContainerHealthNone, startedAt: beforeStart, status: unified.ActionVerificationContradicted, want: "Pulse expected a stopped container. Docker reports it as paused."},
		{name: "stop paused but flagged running", operation: agentexec.DockerContainerOperationStop, state: "paused", running: true, health: agentexec.DockerContainerHealthNone, startedAt: beforeStart, status: unified.ActionVerificationContradicted, want: "Pulse expected a stopped container. Docker reports it as paused."},
		{name: "stop confirmed", operation: agentexec.DockerContainerOperationStop, state: "exited", running: false, health: agentexec.DockerContainerHealthNone, startedAt: beforeStart, status: unified.ActionVerificationConfirmed, want: "The container is stopped."},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := dockerResultFacts(now, true, true, true, true)
			facts.ContainerID = dockerLifecycleTestID
			req := agentexec.DockerContainerLifecyclePayload{ActionID: "action-1", Operation: tc.operation}
			observation := &dockerContainerPostconditionObservation{
				ObserverID: "docker-daemon:agent-1", TrustDomain: "docker-daemon:agent-1", Method: "typed_docker_daemon_observation",
				Snapshot:   agentexec.DockerContainerObservationSnapshot{ContainerID: dockerLifecycleTestID, State: tc.state, Running: tc.running, Health: tc.health, StartedAt: tc.startedAt, ObservedAt: now},
				ReceivedAt: now,
			}
			result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", req, facts, observation, now)
			if err != nil {
				t.Fatal(err)
			}
			truth := result.ActionResultV2
			if truth.Verification.Status != tc.status || truth.Verification.Summary != tc.want {
				t.Fatalf("verification = %q %q, want %q %q", truth.Verification.Status, truth.Verification.Summary, tc.status, tc.want)
			}
			_, past := dockerLifecycleVerbs(tc.operation)
			if want := "Docker " + past + " the container."; truth.Execution.Summary != want {
				t.Fatalf("execution summary = %q, want %q", truth.Execution.Summary, want)
			}
			if len(truth.Verification.Evidence) != 1 || !strings.HasPrefix(truth.Verification.Evidence[0].Summary, "Direct daemon observation:") {
				t.Fatalf("typed observation facts left the evidence: %#v", truth.Verification.Evidence)
			}
			if !strings.HasPrefix(result.Output, "Container lifecycle: phase=") {
				t.Fatalf("legacy output lost the typed facts: %q", result.Output)
			}
		})
	}
}

// The agent receipt does not carry health on the wire, so without a daemon
// observation the server cannot confirm a start or restart. The summary must
// say Pulse lacked the health information rather than blame Docker.
func TestDockerContainerAgentOnlySummaryAfterWireRoundTrip(t *testing.T) {
	now := time.Now().UTC()
	for _, tc := range []struct {
		name, state string
		running     bool
		want        string
	}{
		{name: "running", state: "running", running: true, want: "Docker reports the container as running. Pulse could not get its health information, so it could not confirm the result."},
		{name: "exited", state: "exited", running: false, want: "Docker reports the container as stopped. Pulse could not get its health information, so it could not confirm the result."},
		{name: "restarting", state: "restarting", running: true, want: "Docker reports the container as restarting, which usually means it keeps crashing. Pulse could not get its health information, so it could not confirm the result."},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := dockerResultFacts(now, true, true, true, true)
			facts.After.State, facts.After.Running = tc.state, tc.running
			encoded, err := json.Marshal(facts)
			if err != nil {
				t.Fatal(err)
			}
			var wire agentexec.DockerContainerLifecycleResultPayload
			if err := json.Unmarshal(encoded, &wire); err != nil {
				t.Fatal(err)
			}
			if wire.After.Health != "" {
				t.Fatalf("fixture expects health to stay off the wire, got %q", wire.After.Health)
			}
			result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}, wire, nil, now)
			if err != nil {
				t.Fatal(err)
			}
			verification := result.ActionResultV2.Verification
			if verification.Status != unified.ActionVerificationInconclusive || verification.ReasonCode != "container_health_unknown" || verification.Summary != tc.want {
				t.Fatalf("verification = %q %q %q, want %q", verification.Status, verification.ReasonCode, verification.Summary, tc.want)
			}
			if len(verification.Evidence) != 1 || !strings.HasPrefix(verification.Evidence[0].Summary, "Container lifecycle: phase=") {
				t.Fatalf("agent readback facts left the evidence: %#v", verification.Evidence)
			}
		})
	}
}

func TestDockerContainerExecutionSummariesCarryUncertaintyAndAgentError(t *testing.T) {
	now := time.Now().UTC()
	for _, tc := range []struct {
		name               string
		started, completed bool
		errText            string
		want               string
	}{
		{name: "refused", errText: "container preflight state no longer matches the request", want: "The agent did not restart the container. Agent said: container preflight state no longer matches the request."},
		{name: "partial", started: true, errText: "context deadline exceeded", want: "Docker was asked to restart the container, but did not confirm it, so the change may or may not have happened. Agent said: context deadline exceeded."},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := dockerResultFacts(now, tc.started, tc.completed, false, true)
			facts.ReasonCode = agentexec.ActionRefusalTargetStateChanged
			facts.Error = tc.errText
			result, err := dockerContainerExecutionResult("app-container:fixture", "agent-1", agentexec.DockerContainerLifecyclePayload{Operation: agentexec.DockerContainerOperationRestart}, facts, nil, now)
			if err != nil {
				t.Fatal(err)
			}
			if got := result.ActionResultV2.Execution.Summary; got != tc.want {
				t.Fatalf("execution summary = %q, want %q", got, tc.want)
			}
			if !strings.Contains(result.Output, "error="+tc.errText) {
				t.Fatalf("legacy output lost the agent error: %q", result.Output)
			}
		})
	}
}

func TestDockerContainerUpdateSummariesSayWhatPulseSaw(t *testing.T) {
	now := time.Now().UTC()
	for _, tc := range []struct {
		name, agentState, state, health string
		agentRunning, running           bool
		observedID                      string
		status                          unified.ActionVerificationStatus
		want                            string
	}{
		{name: "running healthy", agentState: "running", agentRunning: true, state: "running", running: true, health: agentexec.DockerContainerHealthHealthy, observedID: dockerLifecycleTestID, status: unified.ActionVerificationConfirmed, want: "The updated container is running and healthy."},
		{name: "kept stopped", agentState: "exited", state: "exited", health: agentexec.DockerContainerHealthNone, observedID: dockerLifecycleTestID, status: unified.ActionVerificationConfirmed, want: "The updated container is in place. Docker reports it as stopped."},
		{name: "kept created", agentState: "created", state: "created", health: agentexec.DockerContainerHealthNone, observedID: dockerLifecycleTestID, status: unified.ActionVerificationConfirmed, want: "The updated container is in place. Docker reports it as created but not started."},
		{name: "health failing", agentState: "running", agentRunning: true, state: "running", running: true, health: agentexec.DockerContainerHealthUnhealthy, observedID: dockerLifecycleTestID, status: unified.ActionVerificationContradicted, want: "The updated container is running, but its health check is failing. Check its logs."},
		{name: "exited after readback", agentState: "running", agentRunning: true, state: "exited", health: agentexec.DockerContainerHealthNone, observedID: dockerLifecycleTestID, status: unified.ActionVerificationContradicted, want: "Docker reports the updated container as stopped."},
		{name: "health pending", agentState: "running", agentRunning: true, state: "running", running: true, health: agentexec.DockerContainerHealthStarting, observedID: dockerLifecycleTestID, status: unified.ActionVerificationInconclusive, want: "The updated container is running. Its health check was still starting when Pulse checked. Wait a moment, then refresh its status."},
		{name: "different container", agentState: "running", agentRunning: true, state: "running", running: true, health: agentexec.DockerContainerHealthHealthy, observedID: strings.Repeat("f", 64), status: unified.ActionVerificationContradicted, want: "Pulse found a different container than the one the update created."},
		{name: "readback unavailable", state: "running", running: true, health: agentexec.DockerContainerHealthHealthy, observedID: dockerLifecycleTestID, status: unified.ActionVerificationInconclusive, want: "The agent did not report the updated container's state, so Pulse could not confirm the update."},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := agentexec.DockerContainerUpdateResultPayload{
				Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhaseComplete,
				MutationStarted: true, MutationCompleted: true, ReadbackRan: tc.agentState != "", NewContainerID: dockerLifecycleTestID,
				After: agentexec.DockerContainerLifecycleSnapshot{ContainerID: dockerLifecycleTestID, State: tc.agentState, Running: tc.agentRunning, ObservedAt: now},
			}
			observation := &dockerContainerPostconditionObservation{
				ObserverID: "docker-daemon:agent-1", TrustDomain: "docker-daemon:agent-1", Method: "typed_docker_daemon_observation",
				Snapshot:   agentexec.DockerContainerObservationSnapshot{ContainerID: tc.observedID, State: tc.state, Running: tc.running, Health: tc.health, ObservedAt: now},
				ReceivedAt: now,
			}
			result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, observation, now)
			if err != nil {
				t.Fatal(err)
			}
			truth := result.ActionResultV2
			if truth.Verification.Status != tc.status || truth.Verification.Summary != tc.want {
				t.Fatalf("verification = %q %q, want %q %q", truth.Verification.Status, truth.Verification.Summary, tc.status, tc.want)
			}
			if truth.Execution.Summary != "Docker replaced the container with one using the new image." {
				t.Fatalf("execution summary = %q", truth.Execution.Summary)
			}
			if !strings.HasPrefix(result.Output, "Container update: phase=") {
				t.Fatalf("legacy output lost the typed facts: %q", result.Output)
			}
		})
	}
}

func TestDockerContainerUpdateExecutionSummariesCarryUncertaintyAndAgentError(t *testing.T) {
	now := time.Now().UTC()
	for _, tc := range []struct {
		name    string
		started bool
		errText string
		want    string
	}{
		{name: "not started", errText: "Failed to pull image: manifest unknown: ", want: "The update stopped before changing the container. Agent said: Failed to pull image: manifest unknown."},
		{name: "partial", started: true, errText: "context deadline exceeded", want: "The update started, but Pulse could not confirm it completed. Agent said: context deadline exceeded."},
	} {
		t.Run(tc.name, func(t *testing.T) {
			facts := agentexec.DockerContainerUpdateResultPayload{
				Operation: agentexec.DockerContainerOperationUpdate, ActionID: "action-update", ExecutionPhase: agentexec.DockerContainerPhasePreflight,
				MutationStarted: tc.started, Error: tc.errText,
			}
			result, err := dockerContainerUpdateExecutionResult("app-container:fixture", "agent-1", facts, nil, now)
			if err != nil {
				t.Fatal(err)
			}
			if got := result.ActionResultV2.Execution.Summary; got != tc.want {
				t.Fatalf("execution summary = %q, want %q", got, tc.want)
			}
		})
	}
}