Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

36.0 KB
Raw Blame
package ai

import (
	"context"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	ur "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

// mockReadState is a minimal ReadState implementation for tests that need
// ReadState-backed seed context functions.
type mockReadState struct {
	nodes       []*ur.NodeView
	vms         []*ur.VMView
	containers  []*ur.ContainerView
	hosts       []*ur.HostView
	dockerHosts []*ur.DockerHostView
	dockerCtrs  []*ur.DockerContainerView
	storage     []*ur.StoragePoolView
	physical    []*ur.PhysicalDiskView
	pbs         []*ur.PBSInstanceView
	pmg         []*ur.PMGInstanceView
	k8sClusters []*ur.K8sClusterView
	resources   map[string]*ur.Resource
}

func (m *mockReadState) Nodes() []*ur.NodeView             { return m.nodes }
func (m *mockReadState) VMs() []*ur.VMView                 { return m.vms }
func (m *mockReadState) Containers() []*ur.ContainerView   { return m.containers }
func (m *mockReadState) Hosts() []*ur.HostView             { return m.hosts }
func (m *mockReadState) DockerHosts() []*ur.DockerHostView { return m.dockerHosts }
func (m *mockReadState) DockerContainers() []*ur.DockerContainerView {
	return m.dockerCtrs
}
func (m *mockReadState) StoragePools() []*ur.StoragePoolView     { return m.storage }
func (m *mockReadState) PhysicalDisks() []*ur.PhysicalDiskView   { return m.physical }
func (m *mockReadState) PBSInstances() []*ur.PBSInstanceView     { return m.pbs }
func (m *mockReadState) PMGInstances() []*ur.PMGInstanceView     { return m.pmg }
func (m *mockReadState) K8sClusters() []*ur.K8sClusterView       { return m.k8sClusters }
func (m *mockReadState) K8sNodes() []*ur.K8sNodeView             { return nil }
func (m *mockReadState) Pods() []*ur.PodView                     { return nil }
func (m *mockReadState) K8sDeployments() []*ur.K8sDeploymentView { return nil }
func (m *mockReadState) Workloads() []*ur.WorkloadView           { return nil }
func (m *mockReadState) Infrastructure() []*ur.InfrastructureView {
	return nil
}

func (m *mockReadState) Get(id string) (*ur.Resource, bool) {
	if m.resources == nil {
		return nil, false
	}
	resource, ok := m.resources[id]
	if !ok || resource == nil {
		return nil, false
	}
	return resource, true
}

type precomputeMetricsHistoryProvider struct {
	metrics map[string][]models.MetricPoint
	storage map[string][]models.MetricPoint
}

func (p *precomputeMetricsHistoryProvider) GetNodeMetrics(nodeID string, metricType string, duration time.Duration) []models.MetricPoint {
	return p.metrics[nodeID+":"+metricType]
}

func (p *precomputeMetricsHistoryProvider) GetGuestMetrics(guestID string, metricType string, duration time.Duration) []models.MetricPoint {
	return p.metrics[guestID+":"+metricType]
}

func (p *precomputeMetricsHistoryProvider) GetAllGuestMetrics(guestID string, duration time.Duration) map[string][]models.MetricPoint {
	return nil
}

func (p *precomputeMetricsHistoryProvider) GetAllStorageMetrics(storageID string, duration time.Duration) map[string][]models.MetricPoint {
	if p.storage == nil {
		return nil
	}
	points := p.storage[storageID+":usage"]
	if len(points) == 0 {
		return nil
	}
	return map[string][]models.MetricPoint{"usage": points}
}

func TestSeedPrecomputeIntelligence_PopulatesSignals(t *testing.T) {
	now := time.Now()
	ps := NewPatrolService(nil, nil)

	bs := baseline.NewStore(baseline.StoreConfig{MinSamples: 1})
	baselinePoints := []baseline.MetricPoint{
		{Value: 10, Timestamp: now.Add(-5 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-4 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-3 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-2 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-1 * time.Hour)},
	}
	_ = bs.Learn("node-1", "node", "cpu", baselinePoints)
	_ = bs.Learn("node-1", "node", "memory", baselinePoints)
	_ = bs.Learn("vm-1", "vm", "memory", baselinePoints)
	_ = bs.Learn("vm-1", "vm", "disk", baselinePoints)
	_ = bs.Learn("ct-1", "container", "memory", baselinePoints)
	_ = bs.Learn("ct-1", "container", "disk", baselinePoints)
	_ = bs.Learn("storage-1", "storage", "usage", baselinePoints)
	ps.SetBaselineStore(bs)

	series := func(start float64) []models.MetricPoint {
		return []models.MetricPoint{
			{Value: start, Timestamp: now.Add(-5 * time.Hour)},
			{Value: start + 2, Timestamp: now.Add(-4 * time.Hour)},
			{Value: start + 4, Timestamp: now.Add(-3 * time.Hour)},
			{Value: start + 6, Timestamp: now.Add(-2 * time.Hour)},
			{Value: start + 8, Timestamp: now.Add(-1 * time.Hour)},
		}
	}
	mh := &precomputeMetricsHistoryProvider{
		metrics: map[string][]models.MetricPoint{
			"node-1:memory": series(70),
			"vm-1:memory":   series(60),
			"vm-1:disk":     series(50),
			"ct-1:memory":   series(55),
			"ct-1:disk":     series(45),
		},
		storage: map[string][]models.MetricPoint{
			"storage-1:usage": series(65),
		},
	}
	ps.SetMetricsHistoryProvider(mh)

	patternCfg := DefaultPatternConfig()
	patternCfg.MinOccurrences = 2
	patternCfg.PredictionLimit = 72 * time.Hour
	pd := NewPatternDetector(patternCfg)
	pd.RecordEvent(HistoricalEvent{ResourceID: "vm-1", EventType: EventHighCPU, Timestamp: now.Add(-36 * time.Hour)})
	pd.RecordEvent(HistoricalEvent{ResourceID: "vm-1", EventType: EventHighCPU, Timestamp: now.Add(-12 * time.Hour)})
	ps.SetPatternDetector(pd)

	cd := newLegacyChangeDetector(t,
		legacyCreatedChange("vm-1", "vm-1", "vm"),
		legacyStatusChange("vm-1", "vm-1", "vm", "running", "stopped"),
	)
	ps.SetChangeDetector(cd)

	corrCfg := DefaultCorrelationConfig()
	corrCfg.MinOccurrences = 1
	corrCfg.CorrelationWindow = time.Hour
	corr := NewCorrelationDetector(corrCfg)
	for i := 0; i < 3; i++ {
		base := now.Add(time.Duration(-30+i*5) * time.Minute)
		corr.RecordEvent(CorrelationEvent{ResourceID: "node-1", ResourceName: "node-1", ResourceType: "node", EventType: CorrelationEventHighCPU, Timestamp: base})
		corr.RecordEvent(CorrelationEvent{ResourceID: "vm-1", ResourceName: "vm-1", ResourceType: "vm", EventType: CorrelationEventRestart, Timestamp: base.Add(2 * time.Minute)})
	}
	ps.SetCorrelationDetector(corr)

	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "node-1", Name: "node-1", CPU: 80, Memory: models.Memory{Usage: 80}},
		},
		VMs: []models.VM{
			{ID: "vm-1", Name: "vm-1", Status: "running", CPU: 20, Memory: models.Memory{Usage: 70}, Disk: models.Disk{Usage: 60}},
		},
		Containers: []models.Container{
			{ID: "ct-1", Name: "ct-1", Status: "running", CPU: 10, Memory: models.Memory{Usage: 65}, Disk: models.Disk{Usage: 55}},
		},
		Storage: []models.Storage{
			{ID: "storage-1", Name: "local", Usage: 85},
		},
		ActiveAlerts: []models.Alert{{ID: "alert-1"}},
	}

	scoped := map[string]bool{"node-1": true, "vm-1": true, "ct-1": true, "storage-1": true}
	intel := ps.seedPrecomputeIntelligenceState(patrolRuntimeStateForTest(ps, state), scoped, now)

	if !intel.hasBaselineStore {
		t.Fatalf("expected baseline store flag to be true")
	}
	if len(intel.anomalies) == 0 {
		t.Fatalf("expected anomalies to be populated")
	}
	nameSet := false
	for _, a := range intel.anomalies {
		if a.ResourceName != "" {
			nameSet = true
			break
		}
	}
	if !nameSet {
		t.Fatalf("expected anomaly resource names to be set")
	}
	if len(intel.forecasts) == 0 {
		t.Fatalf("expected capacity forecasts")
	}
	if len(intel.predictions) == 0 {
		t.Fatalf("expected failure predictions")
	}
	if len(intel.recentChanges) == 0 {
		t.Fatalf("expected recent changes")
	}
	if len(intel.correlations) == 0 {
		t.Fatalf("expected correlations")
	}
	if intel.isQuiet {
		t.Fatalf("expected infrastructure to be non-quiet")
	}
}

func TestSeedBackupAnalysis_StaleAndRecent(t *testing.T) {
	now := time.Now()
	ps := NewPatrolService(nil, nil)

	// Wire ReadState — seedBackupAnalysis uses ReadState as sole path.
	vm1 := newTestVMView("qemu/101", "vm-1", 101, "pve1", "", ur.StatusOnline, false, nil)
	// Set LastBackup on vm-1's underlying resource via view construction.
	vm1Res := &ur.Resource{
		ID: "vm-1", Name: "vm-1", Type: ur.ResourceTypeVM, Status: ur.StatusOnline,
		Proxmox: &ur.ProxmoxData{VMID: 101, LastBackup: now.Add(-24 * time.Hour)},
	}
	vm1v := ur.NewVMView(vm1Res)
	vm1 = &vm1v

	ps.SetReadState(&mockReadState{
		vms: []*ur.VMView{
			vm1,
			newTestVMView("qemu/102", "vm-2", 102, "pve1", "", ur.StatusOnline, false, nil),
		},
		containers: []*ur.ContainerView{
			newTestContainerView("lxc/201", "ct-1", 201, "pve1", "", ur.StatusOnline, false, nil),
		},
	})

	state := models.StateSnapshot{
		PVEBackups: models.PVEBackups{
			BackupTasks:    []models.BackupTask{{VMID: 102, Status: "OK", EndTime: now.Add(-72 * time.Hour)}},
			StorageBackups: []models.StorageBackup{{VMID: 101, Time: now.Add(-36 * time.Hour)}},
		},
		PBSBackups: []models.PBSBackup{{
			VMID:       "102",
			Instance:   "pbs-main",
			Datastore:  "backup-vault",
			Namespace:  "tenant-a",
			BackupTime: now.Add(-60 * time.Hour),
		}},
	}

	output := ps.seedBackupAnalysisState(patrolRuntimeStateForTest(ps, state), nil, now)
	if output == "" {
		t.Fatalf("expected backup analysis output")
	}
	if !strings.Contains(output, "Guests with no backup in >48h") {
		t.Fatalf("expected stale backup section, got: %s", output)
	}
	if !strings.Contains(output, "ct-1 (never; no PVE/PBS backup seen)") {
		t.Fatalf("expected never-backed guest to include source scope, got: %s", output)
	}
	if !strings.Contains(output, "vm-2 (last:") ||
		!strings.Contains(output, "via PBS pbs-main/backup-vault namespace tenant-a") {
		t.Fatalf("expected stale vm entry to include PBS source context, got: %s", output)
	}
	if !strings.Contains(output, "Guests with recent backups: 1/3") {
		t.Fatalf("expected recent backup count, got: %s", output)
	}
}

func TestPatrolRunAIAnalysisForRecord_ReconcilesNoAcceptedActions(t *testing.T) {
	result := &AIAnalysisResult{
		Response: strings.Join([]string{
			"### Infrastructure Status",
			"All resources appear healthy.",
			"",
			"### Key Observations",
			"- Guest backup state should be checked.",
			"",
			"### Actions Taken",
			"- patrol_report_finding (warning): Orphaned backup for vm-101.",
		}, "\n"),
		RejectedFindings: 1,
	}

	output := patrolRunAIAnalysisForRecord(result, patrolRunAnalysisRecordContext{
		ResourcesChecked: 44,
	})

	if strings.Contains(output, "patrol_report_finding") {
		t.Fatalf("expected rejected tool call prose to be removed, got: %s", output)
	}
	if !strings.Contains(output, "Patrol scanned 44 resources; no accepted warning or critical Patrol findings recorded.") {
		t.Fatalf("expected reconciled status line, got: %s", output)
	}
	if !strings.Contains(output, "- No findings reported — all clear.") {
		t.Fatalf("expected no-action line, got: %s", output)
	}
	if !strings.Contains(output, "Guest backup state should be checked.") {
		t.Fatalf("expected key observations to be preserved, got: %s", output)
	}
}

func TestPatrolRunAIAnalysisForRecord_RendersAcceptedActions(t *testing.T) {
	result := &AIAnalysisResult{
		Response: strings.Join([]string{
			"Infrastructure Status:",
			"All clear.",
			"",
			"Key Observations:",
			"- Backup source context present.",
			"",
			"Actions Taken:",
			"- patrol_report_finding (warning): old wording.",
		}, "\n"),
		Findings: []*Finding{{
			Severity:     FindingSeverityWarning,
			Title:        "Orphaned backup requires cleanup",
			ResourceName: "pbs-main/backup-vault namespace tenant-a",
		}},
		ResolvedIDs: []string{"finding-123"},
	}

	output := patrolRunAIAnalysisForRecord(result, patrolRunAnalysisRecordContext{
		ResourcesChecked: 12,
		NewFindings:      1,
		ResolvedFindings: 1,
	})

	if strings.Contains(output, "old wording") || strings.Contains(output, "patrol_report_finding") {
		t.Fatalf("expected raw tool-call wording to be replaced, got: %s", output)
	}
	if !strings.Contains(output, "Patrol scanned 12 resources; accepted Patrol finding activity: 1 new, 1 resolved.") {
		t.Fatalf("expected accepted finding status, got: %s", output)
	}
	if !strings.Contains(output, "- warning: Orphaned backup requires cleanup (pbs-main/backup-vault namespace tenant-a)") {
		t.Fatalf("expected accepted finding action, got: %s", output)
	}
	if !strings.Contains(output, "- resolved: finding-123") {
		t.Fatalf("expected resolved action, got: %s", output)
	}
}

func TestPatrolRunAIAnalysisForRecordNeverClaimsAllClearForUncertainOrErroredRun(t *testing.T) {
	baseResponse := strings.Join([]string{
		"Infrastructure Status:", "All clear.", "", "Actions Taken:", "- none",
	}, "\n")
	uncertain := patrolRunAIAnalysisForRecord(&AIAnalysisResult{
		Response:    baseResponse,
		Assessments: []PatrolFindingAssessment{{FindingID: "finding-uncertain", Verdict: "uncertain"}},
	}, patrolRunAnalysisRecordContext{ResourcesChecked: 1, UncertainFindings: 1})
	if strings.Contains(uncertain, "No findings reported — all clear") || !strings.Contains(uncertain, "- uncertain: finding-uncertain") {
		t.Fatalf("uncertain run produced misleading actions: %s", uncertain)
	}

	errored := patrolRunAIAnalysisForRecord(&AIAnalysisResult{Response: baseResponse}, patrolRunAnalysisRecordContext{
		ResourcesChecked: 1, ErrorCount: 1,
	})
	if strings.Contains(errored, "No findings reported — all clear") || !strings.Contains(errored, "Analysis incomplete") {
		t.Fatalf("errored run produced misleading actions: %s", errored)
	}
}

func TestSeedBackupAnalysisState_ScopesGuestsToRuntime(t *testing.T) {
	now := time.Now()
	ps := NewPatrolService(nil, nil)

	vm1 := newTestVMView("qemu/101", "vm-1", 101, "pve1", "", ur.StatusOnline, false, nil)
	vm2 := newTestVMView("qemu/102", "vm-2", 102, "pve1", "", ur.StatusOnline, false, nil)

	runtimeState := newPatrolRuntimeState(models.StateSnapshot{
		PVEBackups: models.PVEBackups{
			BackupTasks: []models.BackupTask{
				{VMID: 101, Status: "OK", EndTime: now.Add(-24 * time.Hour)},
				{VMID: 102, Status: "OK", EndTime: now.Add(-72 * time.Hour)},
			},
		},
		PBSBackups: []models.PBSBackup{
			{VMID: "102", BackupTime: now.Add(-72 * time.Hour)},
		},
	})
	runtimeState.readState = &mockReadState{
		vms: []*ur.VMView{vm1, vm2},
	}

	output := ps.seedBackupAnalysisState(runtimeState, map[string]bool{vm1.ID(): true}, now)
	if output == "" {
		t.Fatalf("expected scoped backup analysis output")
	}
	if !strings.Contains(output, "Guests with recent backups: 1/1") {
		t.Fatalf("expected scoped recent backup count, got: %s", output)
	}
	if strings.Contains(output, "vm-2") {
		t.Fatalf("expected out-of-scope guest to be omitted, got: %s", output)
	}
}

func TestSeedPrecomputeIntelligenceState_UsesRuntimeReadState(t *testing.T) {
	now := time.Now()
	ps := NewPatrolService(nil, nil)

	bs := baseline.NewStore(baseline.StoreConfig{MinSamples: 1})
	baselinePoints := []baseline.MetricPoint{
		{Value: 10, Timestamp: now.Add(-5 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-4 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-3 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-2 * time.Hour)},
		{Value: 10, Timestamp: now.Add(-1 * time.Hour)},
	}
	_ = bs.Learn("node-1", "node", "cpu", baselinePoints)
	_ = bs.Learn("node-1", "node", "memory", baselinePoints)
	_ = bs.Learn("vm-1", "vm", "memory", baselinePoints)
	_ = bs.Learn("vm-1", "vm", "disk", baselinePoints)
	_ = bs.Learn("ct-1", "container", "memory", baselinePoints)
	_ = bs.Learn("ct-1", "container", "disk", baselinePoints)
	_ = bs.Learn("storage-1", "storage", "usage", baselinePoints)
	ps.SetBaselineStore(bs)

	series := func(start float64) []models.MetricPoint {
		return []models.MetricPoint{
			{Value: start, Timestamp: now.Add(-5 * time.Hour)},
			{Value: start + 2, Timestamp: now.Add(-4 * time.Hour)},
			{Value: start + 4, Timestamp: now.Add(-3 * time.Hour)},
			{Value: start + 6, Timestamp: now.Add(-2 * time.Hour)},
			{Value: start + 8, Timestamp: now.Add(-1 * time.Hour)},
		}
	}
	ps.SetMetricsHistoryProvider(&precomputeMetricsHistoryProvider{
		metrics: map[string][]models.MetricPoint{
			"node-1:memory": series(70),
			"vm-1:memory":   series(60),
			"vm-1:disk":     series(50),
			"ct-1:memory":   series(55),
			"ct-1:disk":     series(45),
		},
		storage: map[string][]models.MetricPoint{
			"storage-1:usage": series(65),
		},
	})

	runtimeState := newPatrolRuntimeState(models.StateSnapshot{ActiveAlerts: []models.Alert{{ID: "alert-1"}}})
	nodeView := ur.NewNodeView(&ur.Resource{
		ID:     "node-1",
		Name:   "node-1",
		Type:   ur.ResourceTypeAgent,
		Status: ur.StatusOnline,
		Metrics: &ur.ResourceMetrics{
			CPU:    &ur.MetricValue{Percent: 80},
			Memory: &ur.MetricValue{Percent: 80},
		},
	})
	vmView := ur.NewVMView(&ur.Resource{
		ID:     "vm-1",
		Name:   "vm-1",
		Type:   ur.ResourceTypeVM,
		Status: "running",
		Metrics: &ur.ResourceMetrics{
			CPU:    &ur.MetricValue{Percent: 20},
			Memory: &ur.MetricValue{Percent: 70},
			Disk:   &ur.MetricValue{Percent: 60},
		},
	})
	ctView := ur.NewContainerView(&ur.Resource{
		ID:     "ct-1",
		Name:   "ct-1",
		Type:   ur.ResourceTypeSystemContainer,
		Status: "running",
		Metrics: &ur.ResourceMetrics{
			CPU:    &ur.MetricValue{Percent: 10},
			Memory: &ur.MetricValue{Percent: 65},
			Disk:   &ur.MetricValue{Percent: 55},
		},
	})
	storageView := ur.NewStoragePoolView(&ur.Resource{
		ID:     "storage-1",
		Name:   "local",
		Type:   ur.ResourceTypeStorage,
		Status: ur.StatusOnline,
		Metrics: &ur.ResourceMetrics{
			Disk: &ur.MetricValue{Percent: 85},
		},
	})
	runtimeState.readState = &mockReadState{
		nodes:      []*ur.NodeView{&nodeView},
		vms:        []*ur.VMView{&vmView},
		containers: []*ur.ContainerView{&ctView},
		storage:    []*ur.StoragePoolView{&storageView},
	}

	ps.SetReadState(nil)

	scoped := map[string]bool{"node-1": true, "vm-1": true, "ct-1": true, "storage-1": true}
	intel := ps.seedPrecomputeIntelligenceState(runtimeState, scoped, now)
	if !intel.hasBaselineStore {
		t.Fatalf("expected baseline store flag to be true")
	}
	if len(intel.anomalies) == 0 {
		t.Fatalf("expected anomalies from runtime readState")
	}
	if len(intel.forecasts) == 0 {
		t.Fatalf("expected forecasts from runtime readState")
	}
}

func TestSeedPrecomputeIntelligenceState_UsesCanonicalResourceTimeline(t *testing.T) {
	now := time.Now()
	canonicalStore := ur.NewMemoryStore()
	if err := canonicalStore.RecordChange(ur.ResourceChange{
		ID:               "change-1",
		ObservedAt:       now.Add(-time.Hour),
		ResourceID:       "res-1",
		Kind:             ur.ChangeConfigUpdate,
		SourceType:       ur.SourcePulseDiff,
		SourceAdapter:    ur.AdapterOpsAgent,
		Actor:            "agent:ops-helper",
		RelatedResources: []string{"related-1"},
		Reason:           "Config refresh",
	}); err != nil {
		t.Fatalf("record canonical change: %v", err)
	}

	svc := &Service{
		orgID:                    "org-1",
		resourceExportStore:      canonicalStore,
		resourceExportStoreOrgID: "org-1",
	}
	ps := NewPatrolService(svc, nil)

	intel := ps.seedPrecomputeIntelligenceState(
		patrolRuntimeStateForTest(ps, models.StateSnapshot{}),
		map[string]bool{"res-1": true},
		now,
	)

	if len(intel.recentChanges) != 1 {
		t.Fatalf("expected 1 canonical recent change, got %d", len(intel.recentChanges))
	}
	change := intel.recentChanges[0]
	if change.ResourceID != "res-1" {
		t.Fatalf("expected canonical resource ID res-1, got %q", change.ResourceID)
	}
	for _, part := range []string{
		"**Config update**",
		"[pulse_diff/agent:ops-helper]",
		"; actor agent:ops-helper",
		"related: related-1",
	} {
		if !strings.Contains(change.Description, part) {
			t.Fatalf("expected canonical description to contain %q, got %q", part, change.Description)
		}
	}
}

func TestPatrolRecentChangeFromUnified_UsesCanonicalChangePresentation(t *testing.T) {
	change := ur.ResourceChange{
		ID:               "change-1",
		ObservedAt:       time.Date(2026, time.March, 19, 12, 30, 0, 0, time.UTC),
		ResourceID:       "res-1",
		Kind:             ur.ChangeAnomaly,
		From:             " old-state ",
		To:               " new-state ",
		SourceType:       ur.SourcePulseDiff,
		SourceAdapter:    ur.AdapterOpsAgent,
		Actor:            " agent:ops-helper ",
		Reason:           "  incident changed  ",
		RelatedResources: []string{"", " related-1 "},
	}

	memoryChange := memory.ChangeFromUnifiedResourceChange(change)
	if memoryChange.ID != "change-1" {
		t.Fatalf("unexpected memory change ID: %q", memoryChange.ID)
	}
	if memoryChange.ResourceName != "res-1" {
		t.Fatalf("unexpected memory change resource name: %q", memoryChange.ResourceName)
	}
	if !memoryChange.DetectedAt.Equal(change.ObservedAt) {
		t.Fatalf("unexpected memory change detectedAt: %v", memoryChange.DetectedAt)
	}
	if memoryChange.ChangeType != "Metric anomaly" {
		t.Fatalf("unexpected change type: %q", memoryChange.ChangeType)
	}
	if !strings.Contains(memoryChange.Description, "Metric anomaly") {
		t.Fatalf("expected canonical description to contain label, got %q", memoryChange.Description)
	}
	if !strings.Contains(memoryChange.Description, "[pulse_diff/agent:ops-helper]") {
		t.Fatalf("expected canonical description to contain provenance, got %q", memoryChange.Description)
	}
	if !strings.Contains(memoryChange.Description, "; actor agent:ops-helper") {
		t.Fatalf("expected canonical description to contain actor, got %q", memoryChange.Description)
	}
	if !strings.Contains(memoryChange.Description, "related: related-1") {
		t.Fatalf("expected canonical description to contain related resource, got %q", memoryChange.Description)
	}
}

func TestSeedFindingsAndContext_ResolvesMissingAndAddsNotes(t *testing.T) {
	now := time.Now()
	ps := NewPatrolService(nil, nil)

	missing := &Finding{
		ID:           "find-missing",
		Severity:     FindingSeverityInfo,
		Category:     FindingCategoryPerformance,
		ResourceID:   "vm-missing",
		ResourceName: "vm-missing",
		Title:        "Missing VM",
		Description:  "no longer exists",
		DetectedAt:   now.Add(-2 * time.Hour),
		LastSeenAt:   now.Add(-2 * time.Hour),
	}
	active := &Finding{
		ID:           "find-active",
		Severity:     FindingSeverityInfo,
		Category:     FindingCategoryPerformance,
		ResourceID:   "node-1",
		ResourceName: "node-1",
		Title:        "High CPU",
		Description:  "cpu high",
		UserNote:     "keep an eye",
		DetectedAt:   now.Add(-1 * time.Hour),
		LastSeenAt:   now.Add(-1 * time.Hour),
	}
	dismissed := &Finding{
		ID:           "find-dismissed",
		Severity:     FindingSeverityInfo,
		Category:     FindingCategoryPerformance,
		ResourceID:   "node-1",
		ResourceName: "node-1",
		Title:        "Noisy alerts",
		Description:  "expected",
		DetectedAt:   now.Add(-30 * time.Minute),
		LastSeenAt:   now.Add(-30 * time.Minute),
	}
	dismissedOutOfScope := &Finding{
		ID:           "find-dismissed-out-of-scope",
		Severity:     FindingSeverityInfo,
		Category:     FindingCategoryPerformance,
		ResourceID:   "node-2",
		ResourceName: "node-2",
		Title:        "Ignore node-2",
		Description:  "out of scope",
		DetectedAt:   now.Add(-20 * time.Minute),
		LastSeenAt:   now.Add(-20 * time.Minute),
	}

	ps.findings.Add(missing)
	ps.findings.Add(active)
	ps.findings.Add(dismissed)
	ps.findings.Add(dismissedOutOfScope)
	ps.findings.Dismiss(dismissed.ID, "expected_behavior", "known workload")
	ps.findings.Dismiss(dismissedOutOfScope.ID, "expected_behavior", "different resource")

	resolvedID := ""
	ps.unifiedFindingResolver = func(findingID string) {
		resolvedID = findingID
	}

	knowledgeStore, err := knowledge.NewStore(t.TempDir())
	if err != nil {
		t.Fatalf("failed to create knowledge store: %v", err)
	}
	if err := knowledgeStore.SaveNote("node-1", "node-1", "node", "config", "Pinned", "keep settings"); err != nil {
		t.Fatalf("failed to save knowledge note: %v", err)
	}
	if err := knowledgeStore.SaveNote("node-2", "node-2", "node", "config", "Ignore", "out of scope"); err != nil {
		t.Fatalf("failed to save out-of-scope knowledge note: %v", err)
	}
	ps.knowledgeStore = knowledgeStore

	// Set up readState so seedFindingsAndContext can build knownResources
	// and auto-resolve findings for resources that no longer exist.
	nodeView := ur.NewNodeView(&ur.Resource{ID: "node-1", Name: "node-1"})
	ps.readState = &mockReadState{nodes: []*ur.NodeView{&nodeView}}

	state := models.StateSnapshot{Nodes: []models.Node{{ID: "node-1", Name: "node-1"}}}
	output, seeded := ps.seedFindingsAndContextState(&PatrolScope{ResourceIDs: []string{"node-1"}}, patrolRuntimeStateForTest(ps, state))

	if resolvedID != missing.ID {
		t.Fatalf("expected unified resolver to be called for missing finding")
	}
	if missing.ResolvedAt == nil {
		t.Fatalf("expected missing finding to be resolved")
	}
	if len(seeded) != 1 || seeded[0] != active.ID {
		t.Fatalf("expected active finding to be seeded")
	}
	if !strings.Contains(output, "Active Findings to Re-check") {
		t.Fatalf("expected active findings section, got: %s", output)
	}
	if !strings.Contains(output, "User note: \"keep an eye\"") {
		t.Fatalf("expected user note in output, got: %s", output)
	}
	if !strings.Contains(output, "User Feedback on Previous Findings") {
		t.Fatalf("expected dismissed findings context, got: %s", output)
	}
	if strings.Contains(output, dismissedOutOfScope.Title) {
		t.Fatalf("expected out-of-scope dismissed feedback to be omitted, got: %s", output)
	}
	if strings.Contains(output, "out of scope") {
		t.Fatalf("expected out-of-scope knowledge note to be omitted, got: %s", output)
	}
	if !strings.Contains(output, "# User Notes") || !strings.Contains(output, "Saved Knowledge") {
		t.Fatalf("expected knowledge context, got: %s", output)
	}
}

func TestSeedFindingsAndContext_ScopedPatrolSkipsOutOfScopeFindingsWithoutResolving(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	inScope := &Finding{
		ID:           "finding-in-scope",
		ResourceID:   "node-1",
		ResourceName: "node-1",
		Title:        "Scoped finding",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryGeneral,
		DetectedAt:   time.Now().Add(-time.Hour),
		LastSeenAt:   time.Now().Add(-time.Hour),
	}
	outOfScope := &Finding{
		ID:           "finding-out-of-scope",
		ResourceID:   "node-2",
		ResourceName: "node-2",
		Title:        "Out of scope finding",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryGeneral,
		DetectedAt:   time.Now().Add(-time.Hour),
		LastSeenAt:   time.Now().Add(-time.Hour),
	}
	ps.findings.Add(inScope)
	ps.findings.Add(outOfScope)

	node1 := ur.NewNodeView(&ur.Resource{ID: "node-1", Name: "node-1"})
	node2 := ur.NewNodeView(&ur.Resource{ID: "node-2", Name: "node-2"})
	ps.SetReadState(&mockReadState{nodes: []*ur.NodeView{&node1, &node2}})

	scopedRuntime := newPatrolRuntimeState(models.StateSnapshot{
		Nodes: []models.Node{{ID: "node-1", Name: "node-1"}},
	})
	scopedRuntime.readState = &mockReadState{nodes: []*ur.NodeView{&node1}}

	output, seeded := ps.seedFindingsAndContextState(&PatrolScope{ResourceIDs: []string{"node-1"}}, scopedRuntime)

	if outOfScope.ResolvedAt != nil {
		t.Fatalf("expected out-of-scope finding to remain active, got resolved at %v", outOfScope.ResolvedAt)
	}
	if len(seeded) != 1 || seeded[0] != inScope.ID {
		t.Fatalf("expected only in-scope finding to be seeded, got %v", seeded)
	}
	if !strings.Contains(output, inScope.Title) {
		t.Fatalf("expected in-scope finding in output, got: %s", output)
	}
	if strings.Contains(output, outOfScope.Title) {
		t.Fatalf("expected out-of-scope finding to be omitted, got: %s", output)
	}
}

func TestSeedFindingsAndContext_DependencyEvidenceDoesNotJoinExactFindingLifecycle(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	now := time.Now().Add(-time.Hour)
	workload := &Finding{
		ID: "workload-finding", ResourceID: "workload-1", ResourceName: "workload-1",
		Title: "Workload unhealthy", Severity: FindingSeverityWarning, Category: FindingCategoryReliability,
		DetectedAt: now, LastSeenAt: now,
	}
	host := &Finding{
		ID: "host-finding", ResourceID: "host-1", ResourceName: "host-1",
		Title: "Host packages pending", Severity: FindingSeverityWarning, Category: FindingCategoryGeneral,
		DetectedAt: now, LastSeenAt: now,
	}
	dismissedHost := &Finding{
		ID: "dismissed-host-finding", ResourceID: "host-1", ResourceName: "host-1",
		Title: "Dismissed host issue", Severity: FindingSeverityWatch, Category: FindingCategoryGeneral,
		DetectedAt: now, LastSeenAt: now,
	}
	ps.findings.Add(workload)
	ps.findings.Add(host)
	ps.findings.Add(dismissedHost)
	ps.findings.Dismiss(dismissedHost.ID, "expected_behavior", "dependency-only feedback")

	workloadView := ur.NewNodeView(&ur.Resource{ID: "workload-1", Name: "workload-1"})
	hostView := ur.NewNodeView(&ur.Resource{ID: "host-1", Name: "host-1"})
	state := &mockReadState{nodes: []*ur.NodeView{&workloadView, &hostView}}
	ps.SetReadState(state)
	effective := newPatrolRuntimeState(models.StateSnapshot{Nodes: []models.Node{
		{ID: "workload-1", Name: "workload-1"},
		{ID: "host-1", Name: "host-1"},
	}})
	effective.readState = state

	output, seeded := ps.seedFindingsAndContextState(&PatrolScope{ResourceIDs: []string{"workload-1"}}, effective)
	if len(seeded) != 1 || seeded[0] != workload.ID {
		t.Fatalf("seeded findings = %v, want exact caller-scope finding only", seeded)
	}
	if !strings.Contains(output, workload.Title) || strings.Contains(output, host.Title) || strings.Contains(output, dismissedHost.Title) {
		t.Fatalf("dependency finding lifecycle leaked into exact scope: %s", output)
	}
	if host.ResolvedAt != nil {
		t.Fatalf("dependency finding was incorrectly resolved: %v", host.ResolvedAt)
	}
}

func TestSeedFindingsAndContext_KeepsSyntheticPatrolServiceFindings(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	serviceFinding := &Finding{
		ID:           generateFindingID("ai-service", "reliability", "ai-patrol-error"),
		Key:          "ai-patrol-error",
		ResourceID:   "ai-service",
		ResourceName: "Pulse Patrol Service",
		ResourceType: "service",
		Title:        "Pulse Patrol: Provider billing or quota issue",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryReliability,
		DetectedAt:   time.Now().Add(-2 * time.Hour),
		LastSeenAt:   time.Now().Add(-time.Hour),
	}
	ps.findings.Add(serviceFinding)

	node1 := ur.NewNodeView(&ur.Resource{ID: "node-1", Name: "node-1"})
	ps.SetReadState(&mockReadState{nodes: []*ur.NodeView{&node1}})

	runtime := newPatrolRuntimeState(models.StateSnapshot{
		Nodes: []models.Node{{ID: "node-1", Name: "node-1"}},
	})
	runtime.readState = &mockReadState{nodes: []*ur.NodeView{&node1}}

	output, seeded := ps.seedFindingsAndContextState(&PatrolScope{ResourceIDs: []string{"node-1"}}, runtime)

	if serviceFinding.ResolvedAt != nil {
		t.Fatalf("expected synthetic Patrol service finding to remain active, got resolved at %v", serviceFinding.ResolvedAt)
	}
	if len(seeded) != 1 || seeded[0] != serviceFinding.ID {
		t.Fatalf("expected synthetic Patrol service finding to stay seeded, got %v", seeded)
	}
	if !strings.Contains(output, serviceFinding.Title) {
		t.Fatalf("expected synthetic Patrol service finding in output, got: %s", output)
	}
}

func TestSeedFindingsAndContext_ScopedPatrolWithoutRuntimeResourcesOmitsGlobalKnowledge(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	knowledgeStore, err := knowledge.NewStore(t.TempDir())
	if err != nil {
		t.Fatalf("failed to create knowledge store: %v", err)
	}
	if err := knowledgeStore.SaveNote("node-1", "node-1", "node", "config", "Pinned", "keep settings"); err != nil {
		t.Fatalf("failed to save knowledge note: %v", err)
	}
	ps.knowledgeStore = knowledgeStore

	output, _ := ps.seedFindingsAndContextState(&PatrolScope{ResourceTypes: []string{"node"}}, newPatrolRuntimeState(models.StateSnapshot{}))
	if strings.Contains(output, "# User Notes") || strings.Contains(output, "keep settings") {
		t.Fatalf("expected scoped patrol without runtime resources to omit global knowledge, got: %s", output)
	}
}

func TestSeedFindingsAndContext_ScopedPatrolExplicitIDsStillUseKnowledgeFallback(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	knowledgeStore, err := knowledge.NewStore(t.TempDir())
	if err != nil {
		t.Fatalf("failed to create knowledge store: %v", err)
	}
	if err := knowledgeStore.SaveNote("node-1", "node-1", "node", "config", "Pinned", "keep settings"); err != nil {
		t.Fatalf("failed to save knowledge note: %v", err)
	}
	ps.knowledgeStore = knowledgeStore

	output, _ := ps.seedFindingsAndContextState(&PatrolScope{ResourceIDs: []string{"node-1"}}, newPatrolRuntimeState(models.StateSnapshot{}))
	if !strings.Contains(output, "# User Notes") || !strings.Contains(output, "keep settings") {
		t.Fatalf("expected explicit scoped patrol to retain knowledge fallback, got: %s", output)
	}
}

// newTestVMView creates a VMView with Proxmox fields needed for gatherGuestIntelligence tests.
// sourceID is the legacy guest ID (e.g. "qemu/100") stored in Proxmox.SourceID.
// status should be a normalized ResourceStatus (e.g. ur.StatusOnline, ur.StatusOffline).
func newTestVMView(sourceID, name string, vmid int, node, instance string, status ur.ResourceStatus, template bool, ips []string) *ur.VMView {
	r := &ur.Resource{
		ID:     "reg-" + sourceID, // unified registry ID (not used by gatherGuestIntelligence)
		Name:   name,
		Type:   ur.ResourceTypeVM,
		Status: status,
		Proxmox: &ur.ProxmoxData{
			SourceID: sourceID,
			VMID:     vmid,
			NodeName: node,
			Instance: instance,
			Template: template,
		},
		Identity: ur.ResourceIdentity{
			IPAddresses: ips,
		},
	}
	v := ur.NewVMView(r)
	return &v
}

// newTestContainerView creates a ContainerView with Proxmox fields needed for gatherGuestIntelligence tests.
// sourceID is the legacy guest ID (e.g. "lxc/101") stored in Proxmox.SourceID.
// status should be a normalized ResourceStatus (e.g. ur.StatusOnline, ur.StatusOffline).
func newTestContainerView(sourceID, name string, vmid int, node, instance string, status ur.ResourceStatus, template bool, ips []string) *ur.ContainerView {
	r := &ur.Resource{
		ID:     "reg-" + sourceID,
		Name:   name,
		Type:   ur.ResourceTypeSystemContainer,
		Status: status,
		Proxmox: &ur.ProxmoxData{
			SourceID: sourceID,
			VMID:     vmid,
			NodeName: node,
			Instance: instance,
			Template: template,
		},
		Identity: ur.ResourceIdentity{
			IPAddresses: ips,
		},
	}
	v := ur.NewContainerView(r)
	return &v
}

func TestGatherGuestIntelligence_ReadStatePath(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	// Wire ReadState instead of using state snapshot.
	// Status uses normalized StatusOnline/StatusOffline (registry normalizes "running" → "online").
	rs := &mockReadState{
		vms: []*ur.VMView{
			newTestVMView("qemu/100", "db-server", 100, "pve1", "", ur.StatusOnline, false, nil),
			newTestVMView("qemu/200", "unknown-vm", 200, "pve1", "", ur.StatusOnline, false, nil),
			newTestVMView("qemu/9000", "template-vm", 9000, "pve1", "", ur.StatusOffline, true, nil),
		},
		containers: []*ur.ContainerView{
			newTestContainerView("lxc/101", "web-proxy", 101, "pve1", "", ur.StatusOnline, false, nil),
		},
	}
	ps.SetReadState(rs)

	intel := ps.gatherGuestIntelligence(context.Background())

	// Expect 3 entries: 2 VMs (template skipped) + 1 container
	if len(intel) != 3 {
		t.Fatalf("expected 3 entries from ReadState path, got %d", len(intel))
	}
	if gi := intel["qemu/100"]; gi == nil || gi.Name != "db-server" {
		t.Fatalf("expected db-server entry, got: %+v", gi)
	}
	if gi := intel["qemu/200"]; gi == nil || gi.Name != "unknown-vm" {
		t.Fatalf("expected unknown-vm entry, got: %+v", gi)
	}
	if gi := intel["lxc/101"]; gi == nil || gi.GuestType != "system-container" {
		t.Fatalf("expected web-proxy container entry, got: %+v", gi)
	}
	// Template should be skipped
	if _, ok := intel["qemu/9000"]; ok {
		t.Fatal("template VM should be skipped")
	}
}

func TestGatherGuestIntelligence_ReadStateReachability(t *testing.T) {
	prober := &mockGuestProber{
		agents: map[string]string{"pve1": "agent-1"},
		results: map[string]map[string]PingResult{
			"agent-1": {
				"10.0.0.1": {Reachable: true},
				"10.0.0.2": {Reachable: false},
			},
		},
	}

	ps := NewPatrolService(nil, nil)
	ps.SetGuestProber(prober)

	rs := &mockReadState{
		vms: []*ur.VMView{
			newTestVMView("qemu/100", "vm-up", 100, "pve1", "", ur.StatusOnline, false, []string{"10.0.0.1"}),
			newTestVMView("qemu/101", "vm-down", 101, "pve1", "", ur.StatusOnline, false, []string{"10.0.0.2"}),
		},
	}
	ps.SetReadState(rs)

	intel := ps.gatherGuestIntelligence(context.Background())

	if len(intel) != 2 {
		t.Fatalf("expected 2 entries, got %d", len(intel))
	}
	if gi := intel["qemu/100"]; gi == nil || gi.Reachable == nil || !*gi.Reachable {
		t.Fatal("expected vm-up to be reachable")
	}
	if gi := intel["qemu/101"]; gi == nil || gi.Reachable == nil || *gi.Reachable {
		t.Fatal("expected vm-down to be unreachable")
	}
}