Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.4 KB
Raw Blame
package ai

import (
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/mockruntime"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestIsDemoMode(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })

	mockruntime.SetEnabled(true)
	if !IsDemoMode() {
		t.Fatal("expected demo mode true when runtime mock mode is enabled")
	}

	mockruntime.SetEnabled(false)
	if IsDemoMode() {
		t.Fatal("expected demo mode false when runtime mock mode is disabled")
	}
}

type demoSnapshotProvider struct {
	snapshot models.StateSnapshot
}

func (p demoSnapshotProvider) ReadSnapshot() models.StateSnapshot {
	return p.snapshot
}

func demoTestSnapshot(now time.Time) models.StateSnapshot {
	snapshot := models.EmptyStateSnapshot()
	snapshot.Nodes = []models.Node{
		{ID: "node/pve1", Name: "pve1", Status: "online", Uptime: 86400, CPU: 0.42},
	}
	snapshot.VMs = []models.VM{
		{ID: "vm/101", VMID: 101, Name: "media-server", Node: "pve1", Status: "running",
			Memory: models.Memory{Total: 16 << 30, Used: 15 << 30, Usage: 93.75}},
	}
	snapshot.Storage = []models.Storage{
		{ID: "pve1-local-zfs", Name: "local-zfs", Node: "pve1", Status: "available",
			Enabled: true, Active: true, Total: 750 << 30, Used: 700 << 30, Free: 50 << 30, Usage: 93.3},
	}
	snapshot.DockerHosts = []models.DockerHost{
		{ID: "docker/edge-01", Hostname: "edge-01", DisplayName: "Edge 01", Status: "offline",
			LastSeen: now.Add(-2 * time.Hour),
			Containers: []models.DockerContainer{
				{ID: "docker/edge-01/portal", Name: "portal", State: "exited"},
			}},
		{ID: "docker/core-01", Hostname: "core-01", DisplayName: "Core 01", Status: "online",
			LastSeen: now,
			Containers: []models.DockerContainer{
				{ID: "docker/core-01/uptime-kuma", Name: "uptime-kuma", State: "running",
					Health: "unhealthy", RestartCount: 4, Image: "louislam/uptime-kuma:1"},
			}},
	}
	snapshot.PBSInstances = []models.PBSInstance{
		{ID: "pbs/dr-vault", Name: "dr-vault", Status: "degraded", ConnectionHealth: "degraded"},
	}
	return snapshot
}

func TestPatrolService_RunDemoPatrolCycle_SynthesizesFromMockState(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(true)

	now := time.Now()
	service := NewPatrolService(nil, demoSnapshotProvider{snapshot: demoTestSnapshot(now)})
	if service.findings == nil || service.runHistoryStore == nil {
		t.Fatal("expected findings and run history to be initialized")
	}

	service.runDemoPatrolCycle(TriggerReasonStartup)

	findings := service.findings.GetActive(FindingSeverityInfo)
	if len(findings) < 4 {
		t.Fatalf("expected at least 4 demo findings (offline host, storage, pbs, container), got %d", len(findings))
	}
	byID := map[string]*Finding{}
	for _, f := range findings {
		if !isDemoFindingID(f.ID) {
			t.Fatalf("expected only demo finding IDs, got %q", f.ID)
		}
		byID[f.ID] = f
	}
	offline := byID["demo-docker-host-offline-edge-01"]
	if offline == nil {
		t.Fatalf("expected offline docker host finding, got IDs %v", keysOfDemoFindings(byID))
	}
	if offline.Severity != FindingSeverityCritical {
		t.Fatalf("expected offline host finding to be critical, got %s", offline.Severity)
	}
	storage := byID["demo-storage-capacity-pve1-local-zfs"]
	if storage == nil {
		t.Fatalf("expected storage capacity finding, got IDs %v", keysOfDemoFindings(byID))
	}
	if storage.Severity != FindingSeverityCritical {
		t.Fatalf("expected 93%% full storage finding to be critical, got %s", storage.Severity)
	}
	if !strings.Contains(storage.Title, "93%") {
		t.Fatalf("expected storage title to quote the observed usage, got %q", storage.Title)
	}

	// Run history: backfill plus the current run.
	if service.runHistoryStore.Count() != 11 {
		t.Fatalf("expected 10 backfill entries plus the current run, got %d", service.runHistoryStore.Count())
	}
	runs := service.GetRunHistory(1)
	if len(runs) != 1 || runs[0].Source != PatrolRunSourceDemo {
		t.Fatalf("expected newest run to be the demo run, got %+v", runs)
	}
	if runs[0].Status != "critical" {
		t.Fatalf("expected run status critical with a critical finding active, got %q", runs[0].Status)
	}
	if runs[0].ResourcesChecked == 0 {
		t.Fatalf("expected run to report resources checked, got %+v", runs[0])
	}

	// Second cycle refreshes instead of duplicating.
	service.runDemoPatrolCycle(TriggerReasonScheduled)
	refreshed := service.findings.GetActive(FindingSeverityInfo)
	if len(refreshed) != len(findings) {
		t.Fatalf("expected second cycle to refresh findings, got %d then %d", len(findings), len(refreshed))
	}
	if service.runHistoryStore.Count() != 12 {
		t.Fatalf("expected exactly one additional run record on second cycle, got %d", service.runHistoryStore.Count())
	}
}

func keysOfDemoFindings(m map[string]*Finding) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	return keys
}

// Demo findings name the unified resource, as real Patrol findings do. The
// Inbox names an alert's subject by the unified resource its reference
// resolves to, resource drawers ask for a resource's findings by that ID, and
// the alert-mirror reconcile folds a finding under its alert through it. A
// demo finding stamped with the snapshot's source ID joined none of them.
func TestPatrolService_RunDemoPatrolCycle_NamesUnifiedResources(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(true)

	now := time.Now()
	snapshot := demoTestSnapshot(now)
	// Mock-shaped Docker IDs: a host alert reference ("docker:<host ID>")
	// cannot carry a slash, and mock container IDs are short.
	snapshot.DockerHosts[0].ID = "edge-01-mock"
	snapshot.DockerHosts[1].ID = "core-01-mock"
	snapshot.DockerHosts[1].Containers[0].ID = "core-01-5666a65b2b52"
	// The busiest guest is an LXC container.
	snapshot.Containers = []models.Container{
		{ID: "pve1-ct-102", VMID: 102, Name: "dns-01", Node: "pve1", Status: "running", Type: "lxc",
			Memory: models.Memory{Total: 2 << 30, Used: 1946 << 20, Usage: 95.0}},
	}
	offlineAlert := models.Alert{ID: "docker:edge-01-mock-connectivity", Type: "docker-host-offline",
		ResourceID: alerts.DockerHostResourceID("edge-01-mock"), StartTime: now.Add(-time.Hour)}
	usageAlert := models.Alert{ID: "pve1-local-zfs-usage", Type: "usage", ResourceID: "pve1-local-zfs",
		StartTime: now.Add(-time.Hour)}
	snapshot.ActiveAlerts = []models.Alert{offlineAlert, usageAlert}

	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(snapshot)
	service := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
	service.SetReadState(registry)
	service.runDemoPatrolCycle(TriggerReasonStartup)

	containerResourceID := ""
	for _, resource := range registry.List() {
		if resource.Type == unifiedresources.ResourceTypeAppContainer && resource.Name == "uptime-kuma" {
			containerResourceID = resource.ID
		}
	}
	if containerResourceID == "" {
		t.Fatal("expected the registry to hold the uptime-kuma app container")
	}

	cases := []struct {
		findingID    string
		resourceID   string
		resourceType string
	}{
		// An alert on each resource carries these references.
		{"demo-docker-host-offline-edge-01", AttentionSubjectResourceID(registry, offlineAlert.ResourceID), "docker-host"},
		{"demo-storage-capacity-pve1-local-zfs", AttentionSubjectResourceID(registry, usageAlert.ResourceID), "storage"},
		{"demo-pbs-degraded-dr-vault", AttentionSubjectResourceID(registry, "pbs/dr-vault"), "pbs"},
		{"demo-guest-memory-pve1-ct-102", AttentionSubjectResourceID(registry, "pve1-ct-102"), "system-container"},
		// A short container ID binds no alert reference, so name the resource directly.
		{"demo-docker-container-core-01-uptime-kuma", containerResourceID, "app-container"},
	}
	for _, tc := range cases {
		finding := service.findings.Get(tc.findingID)
		if finding == nil {
			t.Fatalf("expected finding %s", tc.findingID)
		}
		if _, ok := registry.Get(tc.resourceID); !ok {
			t.Fatalf("%s: expected %q to be a unified resource", tc.findingID, tc.resourceID)
		}
		if finding.ResourceID != tc.resourceID {
			t.Fatalf("%s: resource ID = %q, want unified %q", tc.findingID, finding.ResourceID, tc.resourceID)
		}
		if finding.ResourceType != tc.resourceType {
			t.Fatalf("%s: resource type = %q, want %q", tc.findingID, finding.ResourceType, tc.resourceType)
		}
		onResource := service.GetFindingsForResource(tc.resourceID)
		if len(onResource) != 1 || onResource[0].ID != tc.findingID {
			t.Fatalf("%s: findings for resource %q = %v", tc.findingID, tc.resourceID, onResource)
		}
	}

	// Both findings restate an active alert, so they fold under it.
	for findingID, alertID := range map[string]string{
		"demo-docker-host-offline-edge-01":     offlineAlert.ID,
		"demo-storage-capacity-pve1-local-zfs": usageAlert.ID,
	} {
		if got := service.findings.Get(findingID).MirrorsAlertID; got != alertID {
			t.Fatalf("%s: mirrors alert %q, want %q", findingID, got, alertID)
		}
	}

	// Without a read state the views are derived from the same snapshot.
	for _, derived := range synthesizeDemoPatrolFindings(newPatrolRuntimeState(snapshot), now) {
		if want := service.findings.Get(derived.ID); want == nil || derived.ResourceID != want.ResourceID {
			t.Fatalf("%s: derived resource ID %q differs from the read state's", derived.ID, derived.ResourceID)
		}
	}

	// A registry that has not ingested the snapshot yet must not leave the
	// first cycle's findings on source IDs: the store keeps a finding's
	// resource for as long as the process runs.
	early := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
	early.SetReadState(unifiedresources.NewRegistry(nil))
	early.runDemoPatrolCycle(TriggerReasonStartup)
	for _, tc := range cases {
		finding := early.findings.Get(tc.findingID)
		if finding == nil {
			t.Fatalf("expected finding %s before the registry ingests", tc.findingID)
		}
		if finding.ResourceID != tc.resourceID {
			t.Fatalf("%s: resource ID before the registry ingests = %q, want %q", tc.findingID, finding.ResourceID, tc.resourceID)
		}
	}

	// Once the registry holds resources it is the only authority. A record it
	// lacks or refuses (an operator link can fold a Docker host into a guest)
	// keeps its source ID instead of an ID from a registry without those links.
	partial := snapshot
	partial.DockerHosts = nil
	partialRegistry := unifiedresources.NewRegistry(nil)
	partialRegistry.IngestSnapshot(partial)
	lagging := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
	lagging.SetReadState(partialRegistry)
	lagging.runDemoPatrolCycle(TriggerReasonStartup)
	for findingID, want := range map[string]string{
		"demo-docker-host-offline-edge-01":     "edge-01-mock",
		"demo-storage-capacity-pve1-local-zfs": AttentionSubjectResourceID(registry, usageAlert.ResourceID),
	} {
		finding := lagging.findings.Get(findingID)
		if finding == nil {
			t.Fatalf("expected finding %s with a registry lacking Docker hosts", findingID)
		}
		if finding.ResourceID != want {
			t.Fatalf("%s: resource ID with a registry lacking Docker hosts = %q, want %q", findingID, finding.ResourceID, want)
		}
	}
}

func TestPatrolService_RunDemoPatrolCycle_ResolvesStaleDemoFindings(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(true)

	now := time.Now()
	snapshot := demoTestSnapshot(now)
	service := NewPatrolService(nil, demoSnapshotProvider{snapshot: snapshot})
	service.runDemoPatrolCycle(TriggerReasonStartup)

	// Mock condition clears: the edge host comes back online.
	healed := demoTestSnapshot(now)
	healed.DockerHosts[0].Status = "online"
	service.SetStateProvider(demoSnapshotProvider{snapshot: healed})
	service.runDemoPatrolCycle(TriggerReasonScheduled)

	offline := service.findings.Get("demo-docker-host-offline-edge-01")
	if offline == nil {
		t.Fatal("expected offline host finding to still exist")
	}
	if !offline.IsResolved() {
		t.Fatal("expected offline host finding to auto-resolve once the mock condition cleared")
	}
}

func TestPatrolService_RunDemoPatrolCycle_ResolvesRuntimeFailureFinding(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(true)

	now := time.Now()
	service := NewPatrolService(nil, demoSnapshotProvider{snapshot: demoTestSnapshot(now)})

	// A real run failed before mock mode enabled and left the meta-finding.
	failure := patrolRuntimeFailure{
		Cause:       PatrolFailureCauseProviderConnection,
		Title:       "Pulse Patrol: Provider analysis error",
		Description: "Pulse Patrol reached the configured provider, but the provider did not complete the Patrol analysis request.",
	}
	stale := newPatrolRuntimeFailureFinding(failure, now.Add(-time.Hour))
	service.findings.Add(stale)

	service.runDemoPatrolCycle(TriggerReasonScheduled)

	refreshed := service.findings.Get(stale.ID)
	if refreshed == nil {
		t.Fatal("expected runtime failure finding to still exist")
	}
	if !refreshed.IsResolved() {
		t.Fatal("expected demo cycle to auto-resolve the stale provider error finding")
	}
}

func TestPatrolService_RunDemoPatrolCycle_NoStore(t *testing.T) {
	service := &PatrolService{}
	service.runDemoPatrolCycle(TriggerReasonStartup)
}

func TestPatrolService_RunDemoPatrolCycle_EmptyStateSkipsRecording(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(true)

	service := NewPatrolService(nil, demoSnapshotProvider{snapshot: models.EmptyStateSnapshot()})
	service.runDemoPatrolCycle(TriggerReasonStartup)

	if count := service.runHistoryStore.Count(); count != 0 {
		t.Fatalf("expected no run records for an empty mock state, got %d", count)
	}
}

func TestDemoFindingsExcludedFromPersistence(t *testing.T) {
	findings := map[string]*Finding{
		"demo-storage-capacity-pve1-local-zfs": {ID: "demo-storage-capacity-pve1-local-zfs"},
		"abc123def456":                         {ID: "abc123def456"},
	}
	records := findingsToRecords(findings)
	if _, ok := records["demo-storage-capacity-pve1-local-zfs"]; ok {
		t.Fatal("expected demo finding to be excluded from persistence records")
	}
	if _, ok := records["abc123def456"]; !ok {
		t.Fatal("expected real finding to be persisted")
	}
}

func TestPatrolRunHistoryFiltersDemoEvidenceOutsideDemoMode(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(false)

	service := NewPatrolService(nil, nil)
	now := time.Date(2026, 5, 8, 10, 0, 0, 0, time.UTC)
	service.runHistoryStore.Add(PatrolRunRecord{
		ID:               "live-run-1",
		StartedAt:        now.Add(-15 * time.Minute),
		CompletedAt:      now.Add(-14 * time.Minute),
		Type:             "patrol",
		ResourcesChecked: 12,
		FindingsSummary:  "No issues found",
		FindingIDs:       []string{},
		Status:           "healthy",
	})
	service.runHistoryStore.Add(PatrolRunRecord{
		ID:               "demo-run-legacy",
		StartedAt:        now.Add(-5 * time.Minute),
		CompletedAt:      now.Add(-4 * time.Minute),
		Type:             "patrol",
		ResourcesChecked: 47,
		ExistingFindings: 5,
		FindingsSummary:  "2 critical, 3 warnings",
		FindingIDs:       []string{"demo-storage-critical"},
		Status:           "issues_found",
	})

	runs := service.GetRunHistory(1)
	if len(runs) != 1 {
		t.Fatalf("expected one live run after filtering demo evidence, got %d", len(runs))
	}
	if runs[0].ID != "live-run-1" {
		t.Fatalf("expected live-run-1 after filtering demo evidence, got %q", runs[0].ID)
	}
	if _, ok := service.GetRunByID("demo-run-legacy"); ok {
		t.Fatal("expected legacy demo run lookup to be hidden outside demo mode")
	}
}

func TestPatrolRunHistoryKeepsDemoEvidenceInDemoMode(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(true)

	service := NewPatrolService(nil, nil)
	now := time.Date(2026, 5, 8, 10, 0, 0, 0, time.UTC)
	service.runHistoryStore.Add(PatrolRunRecord{
		ID:               "live-run-1",
		StartedAt:        now.Add(-15 * time.Minute),
		CompletedAt:      now.Add(-14 * time.Minute),
		Type:             "patrol",
		ResourcesChecked: 12,
		FindingsSummary:  "No issues found",
		FindingIDs:       []string{},
		Status:           "healthy",
	})
	service.runHistoryStore.Add(PatrolRunRecord{
		ID:               "demo-run-1",
		Source:           PatrolRunSourceDemo,
		StartedAt:        now.Add(-5 * time.Minute),
		CompletedAt:      now.Add(-4 * time.Minute),
		Type:             "patrol",
		ResourcesChecked: 47,
		ExistingFindings: 5,
		FindingsSummary:  "2 critical, 3 warnings",
		FindingIDs:       []string{"demo-storage-critical"},
		Status:           "issues_found",
	})

	runs := service.GetRunHistory(1)
	if len(runs) != 1 {
		t.Fatalf("expected one run, got %d", len(runs))
	}
	if runs[0].ID != "demo-run-1" || runs[0].Source != PatrolRunSourceDemo {
		t.Fatalf("expected current demo run in demo mode, got %+v", runs[0])
	}
	if _, ok := service.GetRunByID("demo-run-1"); !ok {
		t.Fatal("expected demo run lookup to be available in demo mode")
	}
}

func TestPatrolCoverageIgnoresPersistedDemoRunsOutsideDemoMode(t *testing.T) {
	original := mockruntime.IsEnabled()
	t.Cleanup(func() { mockruntime.SetEnabled(original) })
	mockruntime.SetEnabled(false)

	now := time.Date(2026, 5, 8, 10, 0, 0, 0, time.UTC)
	store := NewPatrolRunHistoryStore(10)
	store.Add(PatrolRunRecord{
		ID:               "demo-run-legacy",
		StartedAt:        now.Add(-5 * time.Minute),
		CompletedAt:      now.Add(-4 * time.Minute),
		Type:             "patrol",
		ResourcesChecked: 47,
		ErrorCount:       1,
		Status:           "error",
		FindingsSummary:  "2 critical, 3 warnings",
		FindingIDs:       []string{"demo-storage-critical"},
	})

	if factor, ok := summarizeRecentPatrolCoverage(store, now); ok {
		t.Fatalf("expected persisted demo run to be excluded from coverage scoring, got %+v", factor)
	}
}

func TestGenerateDemoAIResponse(t *testing.T) {
	tests := []struct {
		name     string
		prompt   string
		expected string
	}{
		{"patrol", "Analyze the infrastructure for issues", "ZFS pool 'local-zfs' is 94% full"},
		{"disk", "disk is full", "Disk Usage Analysis"},
		{"memory", "memory pressure", "Memory Analysis"},
		{"backup", "pbs backup status", "Backup Status Review"},
		{"cpu", "cpu load is high", "CPU/Performance Analysis"},
		{"hello", "hello there", "Pulse Assistant"},
		{"default", "status report", "This Demo Shows"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			resp := GenerateDemoAIResponse(tt.prompt)
			if resp == nil {
				t.Fatal("expected response")
			}
			if !strings.Contains(resp.Content, tt.expected) {
				t.Fatalf("expected response to contain %q, got %q", tt.expected, resp.Content)
			}
			if resp.Model == "" {
				t.Fatal("expected model to be set")
			}
		})
	}
}

func TestGenerateDemoAIStream(t *testing.T) {
	var content strings.Builder
	done := false

	resp, err := GenerateDemoAIStream("disk usage", func(event StreamEvent) {
		switch event.Type {
		case "content":
			chunk, ok := event.Data.(string)
			if !ok {
				t.Fatalf("expected string content chunk, got %T", event.Data)
			}
			content.WriteString(chunk)
		case "done":
			done = true
		}
	})
	if err != nil {
		t.Fatalf("GenerateDemoAIStream failed: %v", err)
	}
	if resp == nil {
		t.Fatal("expected response")
	}
	if !done {
		t.Fatal("expected done event")
	}
	if content.String() != resp.Content {
		t.Fatal("expected streamed content to match response content")
	}
}