Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

26.4 KB
Raw Blame
package ai

import (
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func createInstallablePatrolObserver(t *testing.T, store *PatrolObjectiveStore, now time.Time, resourceIDs ...string) PatrolObjective {
	t.Helper()
	objective, err := store.Create(CreatePatrolObjectiveInput{
		Brief:       "Keep scoped resources online",
		ResourceIDs: resourceIDs,
	}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		EvidenceFit:      PatrolObserverEvidenceFitDirect,
		Interpretation:   "Every scoped canonical resource remains online.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:        `{"runtime":"pulse-resource-state/v1","path":"status","operator":"equals","value":"online","sample_interval_seconds":10,"wake_after_consecutive_failures":2}`,
		WakeEvidence:     "A scoped resource is not online for two consecutive samples.",
		RequirementsJSON: `{}`,
		Actor:            "patrol:model",
	}, now)
	if err != nil {
		t.Fatalf("propose observer: %v", err)
	}
	return objective
}

func TestPatrolObserverRuntimeInstallsEvaluatesAndLeasesWithoutObjectiveRevisionChurn(t *testing.T) {
	now := time.Date(2026, 8, 14, 1, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective := createInstallablePatrolObserver(t, store, now, "node-1")
	proposalRevision := objective.Revision
	patrol := NewPatrolService(nil, mockPatrolStateProvider{state: models.StateSnapshot{
		Nodes: []models.Node{{ID: "node-1", Name: "node-1", Status: "online"}},
	}})
	patrol.SetObjectiveStore(store)

	patrol.processObjectiveObservers(now.Add(time.Second))
	got, ok := store.Get(objective.ID, now.Add(time.Second))
	if !ok || got.Observer == nil {
		t.Fatalf("installed objective missing: %+v, found=%v", got, ok)
	}
	if got.Observer.State != PatrolObserverInstalled || got.Coverage.State != PatrolObjectiveCovered || got.Coverage.ReasonCode != "observer_healthy" {
		t.Fatalf("installed observer = %+v, coverage = %+v", got.Observer, got.Coverage)
	}
	if got.Observer.ValidUntil == nil || got.Observer.LastEvidenceAt == nil {
		t.Fatalf("observer lease was not persisted: %+v", got.Observer)
	}
	if got.Revision != proposalRevision+2 {
		t.Fatalf("objective revision = %d, want proposal revision + validation/install only (%d)", got.Revision, proposalRevision+2)
	}

	patrol.processObjectiveObservers(now.Add(2 * time.Minute))
	refreshed, _ := store.Get(objective.ID, now.Add(2*time.Minute))
	if refreshed.Revision != got.Revision {
		t.Fatalf("health heartbeat changed objective revision from %d to %d", got.Revision, refreshed.Revision)
	}
	if !refreshed.Observer.ValidUntil.After(*got.Observer.ValidUntil) {
		t.Fatalf("health lease did not advance: old=%v new=%v", got.Observer.ValidUntil, refreshed.Observer.ValidUntil)
	}
}

func TestPatrolObserverRuntimeRecordsExplicitUnsupportedValidationReason(t *testing.T) {
	now := time.Date(2026, 8, 14, 1, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep playback smooth", ResourceIDs: []string{"jellyfin"}}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "Observe playback buffering events.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerAPI},
		ProbeJSON:        `{"runtime":"jellyfin-api/v1","event":"playback"}`,
		WakeEvidence:     "Playback buffers.",
		RequirementsJSON: `{"network":["jellyfin"]}`,
	}, now)
	if err != nil {
		t.Fatalf("propose observer: %v", err)
	}
	patrol := NewPatrolService(nil, nil)
	patrol.SetObjectiveStore(store)
	patrol.processObjectiveObservers(now.Add(time.Second))

	got, _ := store.Get(objective.ID, now.Add(time.Second))
	if got.Observer == nil || got.Observer.State != PatrolObserverRejected {
		t.Fatalf("unsupported proposal state = %+v", got.Observer)
	}
	if got.Observer.FailureCode != "observer_trigger_unsupported" || got.Coverage.ReasonCode != "observer_trigger_unsupported" {
		t.Fatalf("validation failure was not projected explicitly: observer=%+v coverage=%+v", got.Observer, got.Coverage)
	}
	if got.Coverage.State != PatrolObjectiveUncovered || !strings.Contains(got.Coverage.Summary, "could not be validated") {
		t.Fatalf("unsupported proposal coverage = %+v", got.Coverage)
	}
	revised, err := store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: got.Revision,
		Interpretation:   "Keep the canonical Jellyfin resource online.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:        `{"runtime":"pulse-resource-state/v1","path":"status","operator":"equals","value":"online","sample_interval_seconds":30,"wake_after_consecutive_failures":2}`,
		WakeEvidence:     "The Jellyfin resource is not online twice.",
		RequirementsJSON: `{}`,
	}, now.Add(2*time.Second))
	if err != nil {
		t.Fatalf("replace rejected observer proposal: %v", err)
	}
	if revised.Observer == nil || revised.Observer.Version != got.Observer.Version+1 || revised.Observer.State != PatrolObserverProposed {
		t.Fatalf("replacement proposal = %+v", revised.Observer)
	}
}

func TestPatrolAvailabilityObserverUsesCanonicalScopedTargetAndWakesOnOutcomeBreach(t *testing.T) {
	now := time.Date(2026, 8, 14, 3, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective, err := store.Create(CreatePatrolObjectiveInput{
		Brief:       "Keep the front cameras reachable",
		ResourceIDs: []string{"frigate-1"},
	}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		EvidenceFit:      PatrolObserverEvidenceFitDirect,
		Interpretation:   "The existing camera availability check remains reachable.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:        `{"runtime":"pulse-availability-state/v1","target_id":"camera-front-http","path":"probe_outcome","operator":"equals","value":"reachable","sample_interval_seconds":10,"wake_after_consecutive_failures":2}`,
		WakeEvidence:     "The canonical camera endpoint is not reachable twice.",
		RequirementsJSON: `{}`,
	}, now)
	if err != nil {
		t.Fatalf("propose availability observer: %v", err)
	}

	checkedAt := now
	outcome := "reachable"
	provider := &mockUnifiedResourceProvider{getAllFunc: func() []unifiedresources.Resource {
		return []unifiedresources.Resource{{
			ID: "frigate-1", Type: unifiedresources.ResourceTypeAppContainer,
			AvailabilityChecks: []unifiedresources.AvailabilityData{{
				TargetID: "camera-front-http", LinkedResourceID: "frigate-1", Enabled: true,
				ProbeOutcome: outcome, LastChecked: &checkedAt,
			}},
		}}
	}}
	patrol := NewPatrolService(nil, nil)
	patrol.SetObjectiveStore(store)
	patrol.SetUnifiedResourceProvider(provider)
	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 10})
	patrol.SetTriggerManager(tm)

	patrol.processObjectiveObservers(now.Add(time.Second))
	installed, _ := store.Get(objective.ID, now.Add(time.Second))
	if installed.Observer == nil || installed.Observer.State != PatrolObserverInstalled || installed.Coverage.State != PatrolObjectiveCovered {
		t.Fatalf("availability observer was not installed: observer=%+v coverage=%+v", installed.Observer, installed.Coverage)
	}
	if tm.GetPendingCount() != 0 {
		t.Fatal("reachable availability target woke Patrol")
	}

	outcome = "unreachable"
	checkedAt = now.Add(11 * time.Second)
	patrol.processObjectiveObservers(now.Add(11 * time.Second))
	patrol.processObjectiveObservers(now.Add(21 * time.Second))
	if tm.GetPendingCount() != 1 {
		t.Fatalf("availability breach did not wake Patrol once; pending=%d", tm.GetPendingCount())
	}
	queued := tm.pendingTriggers[0]
	if queued.ObjectiveContext == nil || len(queued.ObjectiveContext.ObservedResourceIDs) != 1 || queued.ObjectiveContext.ObservedResourceIDs[0] != "frigate-1" {
		t.Fatalf("availability wake lost canonical owner context: %+v", queued.ObjectiveContext)
	}
	if !strings.Contains(queued.ObjectiveContext.Evidence, "camera-front-http") || !strings.Contains(queued.ObjectiveContext.Evidence, "equals reachable") {
		t.Fatalf("availability wake evidence = %q", queued.ObjectiveContext.Evidence)
	}
}

func TestPatrolAvailabilityObserverBindingFailsClosed(t *testing.T) {
	now := time.Date(2026, 8, 14, 3, 0, 0, 0, time.UTC)
	checkedAt := now
	for _, test := range []struct {
		name       string
		resourceID string
		targetID   string
		enabled    bool
		wantCode   string
	}{
		{name: "missing", resourceID: "camera-1", targetID: "different-target", enabled: true, wantCode: "observer_availability_target_missing"},
		{name: "disabled", resourceID: "camera-1", targetID: "camera-http", enabled: false, wantCode: "observer_availability_target_disabled"},
		{name: "out of scope", resourceID: "camera-2", targetID: "camera-http", enabled: true, wantCode: "observer_availability_target_out_of_scope"},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := NewInMemoryPatrolObjectiveStore()
			objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep camera online", ResourceIDs: []string{"camera-1"}}, now)
			if err != nil {
				t.Fatalf("create objective: %v", err)
			}
			objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
				ExpectedRevision: objective.Revision,
				Interpretation:   "Keep the endpoint reachable.",
				TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
				ProbeJSON:        `{"runtime":"pulse-availability-state/v1","target_id":"camera-http","path":"probe_outcome","operator":"equals","value":"reachable","sample_interval_seconds":30,"wake_after_consecutive_failures":2}`,
				WakeEvidence:     "Endpoint reachability breached.", RequirementsJSON: `{}`,
			}, now)
			if err != nil {
				t.Fatalf("propose observer: %v", err)
			}
			provider := &mockUnifiedResourceProvider{getAllFunc: func() []unifiedresources.Resource {
				return []unifiedresources.Resource{{
					ID: test.resourceID,
					AvailabilityChecks: []unifiedresources.AvailabilityData{{
						TargetID: test.targetID, Enabled: test.enabled, ProbeOutcome: "reachable", LastChecked: &checkedAt,
					}},
				}}
			}}
			patrol := NewPatrolService(nil, nil)
			patrol.SetObjectiveStore(store)
			patrol.SetUnifiedResourceProvider(provider)
			patrol.processObjectiveObservers(now.Add(time.Second))

			got, _ := store.Get(objective.ID, now.Add(time.Second))
			if got.Observer == nil || got.Observer.State != PatrolObserverRejected || got.Observer.FailureCode != test.wantCode {
				t.Fatalf("binding result = %+v, want rejected/%s", got.Observer, test.wantCode)
			}
		})
	}
}

func TestPatrolResourceMetricObserverWakesOnFreshThresholdBreach(t *testing.T) {
	now := time.Date(2026, 8, 14, 4, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective, err := store.Create(CreatePatrolObjectiveInput{
		Brief: "Keep this node cool", ResourceIDs: []string{"node-1"},
	}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "The canonical node temperature remains below 75 Celsius.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:        `{"runtime":"pulse-resource-metric/v1","metric":"temperature_celsius","operator":"less_than","threshold":75,"sample_interval_seconds":10,"wake_after_consecutive_failures":2,"max_evidence_age_seconds":60}`,
		WakeEvidence:     "Fresh temperature evidence is at or above 75 Celsius twice.",
		RequirementsJSON: `{}`,
	}, now)
	if err != nil {
		t.Fatalf("propose metric observer: %v", err)
	}
	temperature := 82.5
	provider := &mockUnifiedResourceProvider{getAllFunc: func() []unifiedresources.Resource {
		return []unifiedresources.Resource{{
			ID: "node-1", LastSeen: now.Add(20 * time.Second), Temperature: &temperature,
		}}
	}}
	patrol := NewPatrolService(nil, nil)
	patrol.SetObjectiveStore(store)
	patrol.SetUnifiedResourceProvider(provider)
	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 10})
	patrol.SetTriggerManager(tm)

	patrol.processObjectiveObservers(now.Add(time.Second))
	patrol.processObjectiveObservers(now.Add(11 * time.Second))
	if tm.GetPendingCount() != 1 {
		t.Fatalf("metric breach did not wake Patrol once; pending=%d", tm.GetPendingCount())
	}
	queued := tm.pendingTriggers[0]
	if queued.ObjectiveContext == nil || !strings.Contains(queued.ObjectiveContext.Evidence, "temperature_celsius") || !strings.Contains(queued.ObjectiveContext.Evidence, "node-1=82.50") {
		t.Fatalf("metric evidence = %+v", queued.ObjectiveContext)
	}
}

func TestPatrolEstateWideObjectiveUsesCurrentCanonicalResourceSet(t *testing.T) {
	now := time.Date(2026, 8, 14, 4, 30, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep disk use below 85 percent"}, now)
	if err != nil {
		t.Fatalf("create estate objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision, Interpretation: "Every current canonical resource with disk telemetry stays below 85 percent.",
		EvidenceFit:  PatrolObserverEvidenceFitDirect,
		TriggerKinds: []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:    `{"runtime":"pulse-resource-metric/v1","metric":"disk_percent","operator":"less_than","threshold":85,"sample_interval_seconds":10,"wake_after_consecutive_failures":1,"max_evidence_age_seconds":60}`,
		WakeEvidence: "A current resource breaches the disk objective.", RequirementsJSON: `{}`,
	}, now)
	if err != nil {
		t.Fatalf("propose observer: %v", err)
	}
	disk := &unifiedresources.MetricValue{Percent: 40}
	provider := &mockUnifiedResourceProvider{getAllFunc: func() []unifiedresources.Resource {
		return []unifiedresources.Resource{{ID: "node-1", LastSeen: now.Add(time.Second), Metrics: &unifiedresources.ResourceMetrics{Disk: disk}}}
	}}
	patrol := NewPatrolService(nil, nil)
	patrol.SetObjectiveStore(store)
	patrol.SetUnifiedResourceProvider(provider)
	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 10})
	patrol.SetTriggerManager(tm)
	patrol.processObjectiveObservers(now.Add(time.Second))
	installed, _ := store.Get(objective.ID, now.Add(time.Second))
	if installed.Coverage.State != PatrolObjectiveCovered {
		t.Fatalf("estate-wide objective coverage = %+v", installed.Coverage)
	}
	disk.Percent = 90
	patrol.processObjectiveObservers(now.Add(11 * time.Second))
	if tm.GetPendingCount() != 1 || tm.pendingTriggers[0].ObjectiveContext == nil || len(tm.pendingTriggers[0].ObjectiveContext.ObservedResourceIDs) != 1 || tm.pendingTriggers[0].ObjectiveContext.ObservedResourceIDs[0] != "node-1" {
		t.Fatalf("estate-wide breach did not bind the current canonical resource: %+v", tm.pendingTriggers)
	}
}

func TestPatrolResourceMetricObserverFailsClosedForStaleOrMissingEvidence(t *testing.T) {
	now := time.Date(2026, 8, 14, 4, 0, 0, 0, time.UTC)
	for _, test := range []struct {
		name       string
		resource   unifiedresources.Resource
		wantDetail string
	}{
		{name: "stale", resource: unifiedresources.Resource{ID: "node-1", LastSeen: now.Add(-10 * time.Minute), Metrics: &unifiedresources.ResourceMetrics{Disk: &unifiedresources.MetricValue{Percent: 20}}}, wantDetail: "metric_stale"},
		{name: "missing", resource: unifiedresources.Resource{ID: "node-1", LastSeen: now}, wantDetail: "metric_missing"},
	} {
		t.Run(test.name, func(t *testing.T) {
			store := NewInMemoryPatrolObjectiveStore()
			objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep disk below 85 percent", ResourceIDs: []string{"node-1"}}, now)
			if err != nil {
				t.Fatalf("create objective: %v", err)
			}
			objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
				ExpectedRevision: objective.Revision, Interpretation: "Disk remains below 85 percent.",
				TriggerKinds: []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
				ProbeJSON:    `{"runtime":"pulse-resource-metric/v1","metric":"disk_percent","operator":"less_than","threshold":85,"sample_interval_seconds":10,"wake_after_consecutive_failures":1,"max_evidence_age_seconds":60}`,
				WakeEvidence: "Disk metric is high or unavailable.", RequirementsJSON: `{}`,
			}, now)
			if err != nil {
				t.Fatalf("propose observer: %v", err)
			}
			patrol := NewPatrolService(nil, nil)
			patrol.SetObjectiveStore(store)
			patrol.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{getAllFunc: func() []unifiedresources.Resource { return []unifiedresources.Resource{test.resource} }})
			tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 10})
			patrol.SetTriggerManager(tm)
			patrol.processObjectiveObservers(now.Add(time.Second))
			if tm.GetPendingCount() != 1 || tm.pendingTriggers[0].ObjectiveContext == nil || !strings.Contains(tm.pendingTriggers[0].ObjectiveContext.Evidence, test.wantDetail) {
				t.Fatalf("fail-closed evidence = %+v", tm.pendingTriggers)
			}
		})
	}
}

func TestValidatePatrolResourceMetricObserverRejectsUnsafeBounds(t *testing.T) {
	now := time.Now().UTC()
	store := NewInMemoryPatrolObjectiveStore()
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep node cool", ResourceIDs: []string{"node-1"}}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision, Interpretation: "Temperature stays safe.",
		TriggerKinds: []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:    `{"runtime":"pulse-resource-metric/v1","metric":"temperature_celsius","operator":"less_than","threshold":1000,"sample_interval_seconds":10,"wake_after_consecutive_failures":2,"max_evidence_age_seconds":60}`,
		WakeEvidence: "Temperature is high.", RequirementsJSON: `{}`,
	}, now)
	if err != nil {
		t.Fatalf("propose observer: %v", err)
	}
	artifact, _ := store.GetObserverArtifact(objective.ID)
	_, validationErr := validatePatrolObserverArtifact(objective, artifact)
	if validationErr == nil || validationErr.code != "observer_threshold_out_of_bounds" {
		t.Fatalf("threshold validation error = %v", validationErr)
	}
}

func TestPatrolObserverRuntimeWakesModelOnlyAfterLocalFailureWindow(t *testing.T) {
	now := time.Date(2026, 8, 14, 1, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective := createInstallablePatrolObserver(t, store, now, "camera-1")
	patrol := NewPatrolService(nil, mockPatrolStateProvider{state: models.StateSnapshot{
		Hosts: []models.Host{{ID: "camera-1", Hostname: "camera-1", Status: "offline"}},
	}})
	patrol.SetObjectiveStore(store)
	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 10})
	patrol.SetTriggerManager(tm)

	patrol.processObjectiveObservers(now.Add(time.Second))
	if got := tm.GetPendingCount(); got != 0 {
		t.Fatalf("observer woke Patrol after first failed local sample; pending=%d", got)
	}
	patrol.processObjectiveObservers(now.Add(11 * time.Second))
	if got := tm.GetPendingCount(); got != 1 {
		t.Fatalf("observer did not queue one Patrol wake after failure window; pending=%d", got)
	}
	queued := tm.pendingTriggers[0]
	if queued.ObjectiveContext == nil {
		t.Fatal("observer wake discarded the triggering objective context")
	}
	currentObjective, found := store.Get(objective.ID, now.Add(11*time.Second))
	if !found {
		t.Fatal("triggering objective disappeared from store")
	}
	if queued.ObjectiveContext.ObjectiveID != currentObjective.ID || queued.ObjectiveContext.Revision != currentObjective.Revision || queued.ObjectiveContext.Brief != currentObjective.Brief {
		t.Fatalf("queued objective context = %+v, want exact objective %+v", queued.ObjectiveContext, currentObjective)
	}
	if queued.ObjectiveContext.ObserverID != currentObjective.Observer.ID || queued.ObjectiveContext.ObserverVersion != currentObjective.Observer.Version {
		t.Fatalf("queued observer binding = %+v, want %s/%d", queued.ObjectiveContext, currentObjective.Observer.ID, currentObjective.Observer.Version)
	}
	if len(queued.ObjectiveContext.ObservedResourceIDs) != 1 || queued.ObjectiveContext.ObservedResourceIDs[0] != "camera-1" || !strings.Contains(queued.ObjectiveContext.Evidence, "2 consecutive local samples") {
		t.Fatalf("queued objective evidence = %+v", queued.ObjectiveContext)
	}
	patrol.processObjectiveObservers(now.Add(21 * time.Second))
	if got := tm.GetPendingCount(); got != 1 {
		t.Fatalf("observer repeated wake while failure remained active; pending=%d", got)
	}
	got, _ := store.Get(objective.ID, now.Add(21*time.Second))
	if got.Coverage.State != PatrolObjectiveCovered {
		t.Fatalf("runtime health and objective outcome were conflated: %+v", got.Coverage)
	}
}

func TestPatrolObserverRuntimeRetriesOnlyUntilDurableObjectiveRunSucceeds(t *testing.T) {
	now := time.Date(2026, 8, 14, 2, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective := createInstallablePatrolObserver(t, store, now, "camera-1")
	patrol := NewPatrolService(nil, mockPatrolStateProvider{state: models.StateSnapshot{
		Hosts: []models.Host{{ID: "camera-1", Hostname: "camera-1", Status: "offline"}},
	}})
	patrol.SetObjectiveStore(store)
	tm := NewTriggerManager(TriggerManagerConfig{MaxPendingTriggers: 10})
	patrol.SetTriggerManager(tm)

	patrol.processObjectiveObservers(now.Add(time.Second))
	firstWakeAt := now.Add(11 * time.Second)
	patrol.processObjectiveObservers(firstWakeAt)
	if tm.GetPendingCount() != 1 {
		t.Fatal("initial objective wake was not queued")
	}
	clearPendingPatrolTriggers(tm)

	retryAt := firstWakeAt.Add(patrolObserverWakeRetryInterval + time.Second)
	patrol.processObjectiveObservers(retryAt)
	if tm.GetPendingCount() != 1 {
		t.Fatal("unacknowledged objective wake was not retried after bounded interval")
	}
	retried := tm.pendingTriggers[0]
	clearPendingPatrolTriggers(tm)

	patrol.runHistoryStore.Add(PatrolRunRecord{
		ID:               "objective-run-success",
		StartedAt:        retryAt,
		CompletedAt:      retryAt.Add(time.Second),
		Type:             "scoped",
		TriggerReason:    string(TriggerReasonObjectiveEvidence),
		Status:           "issues_found",
		FindingIDs:       []string{"camera-offline"},
		ObjectiveContext: clonePatrolObjectiveContext(retried.ObjectiveContext),
	})
	patrol.processObjectiveObservers(retryAt.Add(patrolObserverWakeRetryInterval + time.Second))
	if tm.GetPendingCount() != 0 {
		t.Fatal("observer retried after a successful durable objective run acknowledged delivery")
	}

	current, found := store.Get(objective.ID, retryAt)
	if !found || !patrol.objectiveWakeDelivered(current, retryAt) {
		t.Fatal("successful run was not recognized as exact objective delivery")
	}
}

func clearPendingPatrolTriggers(tm *TriggerManager) {
	tm.mu.Lock()
	tm.pendingTriggers = nil
	tm.mu.Unlock()
}

func TestValidatePatrolObserverArtifactRejectsUnknownExecutableFields(t *testing.T) {
	now := time.Now().UTC()
	store := NewInMemoryPatrolObjectiveStore()
	objective := createInstallablePatrolObserver(t, store, now, "node-1")
	artifact, _ := store.GetObserverArtifact(objective.ID)
	artifact.Probe = []byte(`{"runtime":"pulse-resource-state/v1","path":"status","operator":"equals","value":"online","sample_interval_seconds":30,"wake_after_consecutive_failures":2,"command":"rm -rf /"}`)
	_, err := validatePatrolObserverArtifact(objective, artifact)
	if err == nil || err.code != "observer_probe_invalid" {
		t.Fatalf("unknown executable field validation error = %v", err)
	}
}

func TestValidatePatrolAvailabilityObserverRejectsModelSuppliedNetworkAuthority(t *testing.T) {
	now := time.Now().UTC()
	store := NewInMemoryPatrolObjectiveStore()
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep camera reachable", ResourceIDs: []string{"camera-1"}}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "Keep the endpoint reachable.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:        `{"runtime":"pulse-availability-state/v1","target_id":"camera-http","path":"probe_outcome","operator":"equals","value":"reachable","sample_interval_seconds":30,"wake_after_consecutive_failures":2}`,
		WakeEvidence:     "Endpoint reachability breached.", RequirementsJSON: `{}`,
	}, now)
	if err != nil {
		t.Fatalf("propose observer: %v", err)
	}
	artifact, _ := store.GetObserverArtifact(objective.ID)
	artifact.Probe = []byte(`{"runtime":"pulse-availability-state/v1","target_id":"camera-http","path":"probe_outcome","operator":"equals","value":"reachable","sample_interval_seconds":30,"wake_after_consecutive_failures":2,"url":"http://169.254.169.254/latest/meta-data"}`)
	_, validationErr := validatePatrolObserverArtifact(objective, artifact)
	if validationErr == nil || validationErr.code != "observer_probe_invalid" {
		t.Fatalf("model-supplied URL validation error = %v", validationErr)
	}
}

func TestPatrolObjectiveIntentEditInvalidatesInstalledObserver(t *testing.T) {
	now := time.Date(2026, 8, 14, 1, 0, 0, 0, time.UTC)
	store := NewInMemoryPatrolObjectiveStore()
	objective := createInstallablePatrolObserver(t, store, now, "node-1")
	patrol := NewPatrolService(nil, mockPatrolStateProvider{state: models.StateSnapshot{
		Nodes: []models.Node{{ID: "node-1", Status: "online"}},
	}})
	patrol.SetObjectiveStore(store)
	patrol.processObjectiveObservers(now.Add(time.Second))
	installed, _ := store.Get(objective.ID, now.Add(time.Second))
	if installed.Coverage.State != PatrolObjectiveCovered {
		t.Fatalf("observer was not installed before edit: %+v", installed.Coverage)
	}

	revisedBrief := "Keep this node online and its workload responsive"
	updated, err := store.Update(objective.ID, UpdatePatrolObjectiveInput{
		ExpectedRevision: installed.Revision,
		Brief:            &revisedBrief,
		Actor:            "operator",
	}, now.Add(2*time.Second))
	if err != nil {
		t.Fatalf("update objective intent: %v", err)
	}
	if updated.Observer == nil || updated.Observer.State != PatrolObserverDisabled || updated.Observer.ValidUntil != nil {
		t.Fatalf("intent edit retained stale observer authority: %+v", updated.Observer)
	}
	if updated.Coverage.State != PatrolObjectiveUncovered || updated.Coverage.ReasonCode != "observer_disabled" {
		t.Fatalf("intent edit coverage = %+v", updated.Coverage)
	}
}