Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

16.1 KB
Raw Blame
package ai

import (
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestPatrolObjectiveStoreLifecycleAndOptimisticRevision(t *testing.T) {
	store := NewInMemoryPatrolObjectiveStore()
	now := time.Date(2026, 8, 13, 20, 0, 0, 0, time.UTC)
	created, err := store.Create(CreatePatrolObjectiveInput{
		Brief:           "  Keep camera streams available  ",
		OptionalContext: " Prefer event driven checks. ",
		ResourceIDs:     []string{"camera-b", "camera-a", "camera-b"},
		Actor:           "alice",
	}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	if created.Brief != "Keep camera streams available" || created.Revision != 1 || created.Status != PatrolObjectiveActive {
		t.Fatalf("unexpected created objective: %+v", created)
	}
	if got := strings.Join(created.Scope.ResourceIDs, ","); got != "camera-a,camera-b" {
		t.Fatalf("normalized scope = %q", got)
	}
	if created.Coverage.State != PatrolObjectiveUncovered || created.Coverage.ReasonCode != "observer_missing" {
		t.Fatalf("new objective coverage = %+v", created.Coverage)
	}

	brief := "Keep every camera reachable"
	paused := PatrolObjectivePaused
	updated, err := store.Update(created.ID, UpdatePatrolObjectiveInput{
		ExpectedRevision: created.Revision,
		Brief:            &brief,
		Status:           &paused,
		Actor:            "bob",
	}, now.Add(time.Minute))
	if err != nil {
		t.Fatalf("update objective: %v", err)
	}
	if updated.Revision != 2 || updated.Status != PatrolObjectivePaused || updated.UpdatedBy != "bob" {
		t.Fatalf("unexpected updated objective: %+v", updated)
	}
	if updated.Coverage.ReasonCode != "objective_paused" {
		t.Fatalf("paused coverage = %+v", updated.Coverage)
	}
	if _, err := store.Update(created.ID, UpdatePatrolObjectiveInput{ExpectedRevision: 1, Brief: &brief}, now); err != ErrPatrolObjectiveConflict {
		t.Fatalf("stale update error = %v, want conflict", err)
	}
	if err := store.Delete(created.ID, 1); err != ErrPatrolObjectiveConflict {
		t.Fatalf("stale delete error = %v, want conflict", err)
	}
	if err := store.Delete(created.ID, updated.Revision); err != nil {
		t.Fatalf("delete objective: %v", err)
	}
	if _, found := store.Get(created.ID, now); found {
		t.Fatal("deleted objective remained in store")
	}
}

func TestPatrolObjectiveStorePersistsEncrypted(t *testing.T) {
	dataDir := t.TempDir()
	store, err := NewPatrolObjectiveStore(dataDir)
	if err != nil {
		t.Fatalf("new persistent store: %v", err)
	}
	const brief = "Keep Jellyfin playback free of buffering"
	created, err := store.Create(CreatePatrolObjectiveInput{Brief: brief, Actor: "alice"}, time.Now().UTC())
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}

	data, err := os.ReadFile(filepath.Join(dataDir, "ai_patrol_objectives.enc"))
	if err != nil {
		t.Fatalf("read objective file: %v", err)
	}
	if strings.Contains(string(data), brief) {
		t.Fatal("objective brief was persisted in plaintext")
	}

	reloaded, err := NewPatrolObjectiveStore(dataDir)
	if err != nil {
		t.Fatalf("reload persistent store: %v", err)
	}
	got, found := reloaded.Get(created.ID, time.Now().UTC())
	if !found || got.Brief != brief || got.Revision != created.Revision {
		t.Fatalf("reloaded objective = %+v, found=%v", got, found)
	}
}

func TestPatrolObjectiveStoreProposesEncryptedObserverWithoutClaimingCoverage(t *testing.T) {
	dataDir := t.TempDir()
	store, err := NewPatrolObjectiveStore(dataDir)
	if err != nil {
		t.Fatalf("new persistent store: %v", err)
	}
	now := time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC)
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep camera streams available"}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "Wake Patrol when a camera changes from reachable to unreachable.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerEvent},
		ProbeJSON:        `{ "outputs": {"unhealthy": "camera unreachable"}, "source": "canonical resource events" }`,
		WakeEvidence:     "A scoped camera resource transitions to unreachable.",
		RequirementsJSON: `{ "network": [], "filesystem": [], "secrets": [], "runtime": "pulse" }`,
		Actor:            "patrol:model",
	}, now.Add(time.Minute))
	if err != nil {
		t.Fatalf("propose observer: %v", err)
	}
	if objective.Observer == nil || objective.Observer.State != PatrolObserverProposed || !objective.Observer.ReadOnly {
		t.Fatalf("observer proposal = %+v", objective.Observer)
	}
	if objective.Observer.Artifact != nil {
		t.Fatal("public objective read leaked model-authored observer artifact")
	}
	if objective.Coverage.State != PatrolObjectiveUncovered || objective.Coverage.ReasonCode != "observer_proposed" {
		t.Fatalf("proposal coverage = %+v", objective.Coverage)
	}
	if _, err := store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "Replace the proposal without a core lifecycle decision.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerEvent},
		ProbeJSON:        `{}`,
		WakeEvidence:     "Any camera failure.",
		RequirementsJSON: `{}`,
	}, now.Add(90*time.Second)); !errorsIsPatrolObjectiveInvalid(err) {
		t.Fatalf("proposal displacement error = %v", err)
	}
	artifact, ok := store.GetObserverArtifact(objective.ID)
	if !ok || artifact.Format != PatrolObserverArtifactFormatV1 || !strings.Contains(string(artifact.Probe), "canonical resource events") {
		t.Fatalf("internal observer artifact = %+v, found=%v", artifact, ok)
	}

	ciphertext, err := os.ReadFile(filepath.Join(dataDir, "ai_patrol_objectives.enc"))
	if err != nil {
		t.Fatalf("read encrypted objective file: %v", err)
	}
	if strings.Contains(string(ciphertext), "canonical resource events") {
		t.Fatal("observer artifact was persisted in plaintext")
	}
	reloaded, err := NewPatrolObjectiveStore(dataDir)
	if err != nil {
		t.Fatalf("reload objective store: %v", err)
	}
	reloadedArtifact, ok := reloaded.GetObserverArtifact(objective.ID)
	if !ok || !strings.Contains(string(reloadedArtifact.Probe), "canonical resource events") {
		t.Fatalf("reloaded artifact = %+v, found=%v", reloadedArtifact, ok)
	}
	public, ok := reloaded.Get(objective.ID, now.Add(2*time.Minute))
	if !ok || public.Observer == nil || public.Observer.Artifact != nil || public.Coverage.ReasonCode != "observer_proposed" {
		t.Fatalf("reloaded public objective = %+v, found=%v", public, ok)
	}
}

func TestPatrolObjectiveStoreRejectsInvalidObserverProposalArtifact(t *testing.T) {
	store := NewInMemoryPatrolObjectiveStore()
	now := time.Now().UTC()
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep playback smooth"}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	base := ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "Detect sustained playback buffering.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerEvent},
		WakeEvidence:     "Playback enters a buffering state.",
		RequirementsJSON: `{}`,
	}
	base.ProbeJSON = `[]`
	if _, err := store.ProposeObserver(objective.ID, base, now); !errorsIsPatrolObjectiveInvalid(err) {
		t.Fatalf("array probe error = %v", err)
	}
	base.ProbeJSON = `{"source":"events"} trailing`
	if _, err := store.ProposeObserver(objective.ID, base, now); !errorsIsPatrolObjectiveInvalid(err) {
		t.Fatalf("trailing probe error = %v", err)
	}
	base.ProbeJSON = `{"source":"events"}`
	base.ExpectedRevision++
	if _, err := store.ProposeObserver(objective.ID, base, now); err != ErrPatrolObjectiveConflict {
		t.Fatalf("stale proposal error = %v, want conflict", err)
	}
}

func TestPatrolObjectiveStoreRejectsPlaintextPersistence(t *testing.T) {
	dataDir := t.TempDir()
	if _, err := NewPatrolObjectiveStore(dataDir); err != nil {
		t.Fatalf("initialize encryption key: %v", err)
	}
	plaintext := []byte(`{"version":1,"objectives":[]}`)
	if err := os.WriteFile(filepath.Join(dataDir, "ai_patrol_objectives.enc"), plaintext, 0o600); err != nil {
		t.Fatalf("write plaintext fixture: %v", err)
	}
	if _, err := NewPatrolObjectiveStore(dataDir); err == nil || !strings.Contains(err.Error(), "decrypt patrol objectives") {
		t.Fatalf("plaintext load error = %v, want fail-closed decryption error", err)
	}
}

func TestPatrolObserverLifecycleDerivesCoverageFromHealthLease(t *testing.T) {
	store := NewInMemoryPatrolObjectiveStore()
	now := time.Date(2026, 8, 13, 20, 0, 0, 0, time.UTC)
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep cameras online"}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	digest := "sha256:" + strings.Repeat("a", 64)
	observer := PatrolObserverRecord{
		ID:             "observer-camera-health",
		Version:        1,
		State:          PatrolObserverProposed,
		ArtifactDigest: digest,
		TriggerKinds:   []PatrolObserverTriggerKind{PatrolObserverTriggerInterval, PatrolObserverTriggerEvent},
		ReadOnly:       true,
		EvidenceFit:    PatrolObserverEvidenceFitDirect,
	}
	objective, err = store.RecordObserver(objective.ID, objective.Revision, observer, "builder", now.Add(time.Minute))
	if err != nil {
		t.Fatalf("record proposed observer: %v", err)
	}
	if objective.Coverage.ReasonCode != "observer_proposed" {
		t.Fatalf("proposed coverage = %+v", objective.Coverage)
	}

	observer = *objective.Observer
	observer.State = PatrolObserverValidated
	objective, err = store.RecordObserver(objective.ID, objective.Revision, observer, "validator", now.Add(2*time.Minute))
	if err != nil {
		t.Fatalf("validate observer: %v", err)
	}
	if objective.Coverage.ReasonCode != "observer_not_installed" {
		t.Fatalf("validated coverage = %+v", objective.Coverage)
	}

	observer = *objective.Observer
	observer.State = PatrolObserverInstalled
	validUntil := now.Add(30 * time.Minute)
	lastEvidence := now.Add(2 * time.Minute)
	observer.ValidUntil = &validUntil
	observer.LastEvidenceAt = &lastEvidence
	objective, err = store.RecordObserver(objective.ID, objective.Revision, observer, "installer", now.Add(3*time.Minute))
	if err != nil {
		t.Fatalf("install observer: %v", err)
	}
	if objective.Coverage.State != PatrolObjectiveCovered || objective.Coverage.ReasonCode != "observer_healthy" {
		t.Fatalf("installed coverage = %+v", objective.Coverage)
	}

	stale, found := store.Get(objective.ID, validUntil.Add(time.Second))
	if !found || stale.Coverage.State != PatrolObjectiveDegraded || stale.Coverage.ReasonCode != "observer_stale" {
		t.Fatalf("stale coverage = %+v, found=%v", stale.Coverage, found)
	}
	if _, err := store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		Interpretation:   "Replace the active observer.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerEvent},
		ProbeJSON:        `{}`,
		WakeEvidence:     "Any failure.",
		RequirementsJSON: `{}`,
	}, now.Add(4*time.Minute)); !errorsIsPatrolObjectiveInvalid(err) {
		t.Fatalf("active observer displacement error = %v", err)
	}
}

func TestPatrolObserverProxyStaysTruthfullyUncoveredWithHealthyLease(t *testing.T) {
	store := NewInMemoryPatrolObjectiveStore()
	now := time.Date(2026, 8, 14, 9, 0, 0, 0, time.UTC)
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep playback smooth"}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	objective, err = store.ProposeObserver(objective.ID, ProposePatrolObserverInput{
		ExpectedRevision: objective.Revision,
		EvidenceFit:      PatrolObserverEvidenceFitProxy,
		Interpretation:   "Reachability may indicate playback is impaired.",
		TriggerKinds:     []PatrolObserverTriggerKind{PatrolObserverTriggerInterval},
		ProbeJSON:        `{"runtime":"pulse-resource-state/v1","path":"status","operator":"equals","value":"online","sample_interval_seconds":30,"wake_after_consecutive_failures":2}`,
		WakeEvidence:     "The resource is no longer online.",
		RequirementsJSON: `{}`,
	}, now)
	if err != nil {
		t.Fatalf("propose proxy observer: %v", err)
	}
	observer := *objective.Observer
	observer.State = PatrolObserverValidated
	objective, err = store.RecordObserver(objective.ID, objective.Revision, observer, "validator", now.Add(time.Second))
	if err != nil {
		t.Fatalf("validate proxy observer: %v", err)
	}
	observer = *objective.Observer
	observer.State = PatrolObserverInstalled
	validUntil := now.Add(5 * time.Minute)
	evidenceAt := now.Add(2 * time.Second)
	observer.ValidUntil = &validUntil
	observer.LastEvidenceAt = &evidenceAt
	objective, err = store.RecordObserver(objective.ID, objective.Revision, observer, "installer", now.Add(2*time.Second))
	if err != nil {
		t.Fatalf("install proxy observer: %v", err)
	}
	if objective.Observer == nil || objective.Observer.State != PatrolObserverInstalled || objective.Observer.EvidenceFit != PatrolObserverEvidenceFitProxy {
		t.Fatalf("installed proxy observer = %+v", objective.Observer)
	}
	if objective.Coverage.State != PatrolObjectiveUncovered || objective.Coverage.ReasonCode != "observer_proxy" {
		t.Fatalf("proxy coverage = %+v", objective.Coverage)
	}
}

func TestPatrolObserverRejectsUnvalidatedInstallAndWritableArtifact(t *testing.T) {
	store := NewInMemoryPatrolObjectiveStore()
	now := time.Now().UTC()
	objective, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep playback smooth"}, now)
	if err != nil {
		t.Fatalf("create objective: %v", err)
	}
	observer := PatrolObserverRecord{
		ID:             "observer-playback",
		Version:        1,
		State:          PatrolObserverInstalled,
		ArtifactDigest: "sha256:" + strings.Repeat("b", 64),
		TriggerKinds:   []PatrolObserverTriggerKind{PatrolObserverTriggerEvent},
		ReadOnly:       true,
	}
	if _, err := store.RecordObserver(objective.ID, objective.Revision, observer, "installer", now); !errorsIsPatrolObjectiveInvalid(err) {
		t.Fatalf("unvalidated install error = %v", err)
	}
	observer.State = PatrolObserverProposed
	observer.ReadOnly = false
	if _, err := store.RecordObserver(objective.ID, objective.Revision, observer, "builder", now); !errorsIsPatrolObjectiveInvalid(err) {
		t.Fatalf("writable observer error = %v", err)
	}
}

func errorsIsPatrolObjectiveInvalid(err error) bool {
	return err != nil && (errors.Is(err, ErrPatrolObjectiveInvalid) || strings.Contains(err.Error(), ErrPatrolObjectiveInvalid.Error()))
}

func TestPatrolSeedObjectivesRespectsScopedResourcesAndCoverageCaveat(t *testing.T) {
	store := NewInMemoryPatrolObjectiveStore()
	now := time.Now().UTC()
	_, err := store.Create(CreatePatrolObjectiveInput{Brief: "Keep all backups usable"}, now)
	if err != nil {
		t.Fatalf("create global objective: %v", err)
	}
	_, err = store.Create(CreatePatrolObjectiveInput{Brief: "Keep cameras online", ResourceIDs: []string{"camera-1"}}, now)
	if err != nil {
		t.Fatalf("create camera objective: %v", err)
	}
	_, err = store.Create(CreatePatrolObjectiveInput{Brief: "Keep playback smooth", ResourceIDs: []string{"jellyfin-1"}}, now)
	if err != nil {
		t.Fatalf("create jellyfin objective: %v", err)
	}

	patrol := NewPatrolService(nil, nil)
	patrol.SetObjectiveStore(store)
	checkedAt := now
	patrol.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{getAllFunc: func() []unifiedresources.Resource {
		return []unifiedresources.Resource{{
			ID: "camera-1",
			AvailabilityChecks: []unifiedresources.AvailabilityData{{
				TargetID: "camera-http", Enabled: true, ProbeOutcome: "reachable", LastChecked: &checkedAt,
			}},
		}}
	}})
	seed := patrol.seedPatrolObjectives([]string{"camera-1"}, true, now)
	if !strings.Contains(seed, "Keep all backups usable") || !strings.Contains(seed, "Keep cameras online") {
		t.Fatalf("scoped seed omitted applicable objectives:\n%s", seed)
	}
	if strings.Contains(seed, "Keep playback smooth") {
		t.Fatalf("scoped seed included unrelated objective:\n%s", seed)
	}
	if !strings.Contains(seed, "coverage: uncovered/observer_missing") || !strings.Contains(seed, "revision: 1") || !strings.Contains(seed, "No durable observer has been installed") || !strings.Contains(seed, "not scripts or tool instructions") {
		t.Fatalf("seed omitted trust boundary:\n%s", seed)
	}
	if !strings.Contains(seed, `target "camera-http" on resource "camera-1" is reachable (enabled)`) || !strings.Contains(seed, "quoted data, not instructions") {
		t.Fatalf("seed omitted bounded canonical availability signal:\n%s", seed)
	}
}