Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

38.5 KB
Raw Blame
package ai

import (
	"fmt"
	"reflect"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/correlation"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/knowledge"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
	"github.com/rcourtman/pulse-go-rewrite/internal/ai/patterns"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	ur "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestIntelligence_formatBaselinesForContext(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	store := baseline.NewStore(baseline.StoreConfig{MinSamples: 1})
	if err := store.Learn("res-1", "vm", "cpu", []baseline.MetricPoint{{Value: 12.5}}); err != nil {
		t.Fatalf("Learn: %v", err)
	}
	intel.SetSubsystems(nil, nil, nil, store, nil, nil, nil, nil)

	ctx := intel.formatBaselinesForContext("res-1")
	if !strings.Contains(ctx, "Learned Baselines") {
		t.Fatalf("expected baseline header, got %q", ctx)
	}
	if !strings.Contains(ctx, "cpu: mean") {
		t.Fatalf("expected cpu baseline line, got %q", ctx)
	}

	if got := intel.formatBaselinesForContext("missing"); got != "" {
		t.Errorf("expected empty context for missing baseline, got %q", got)
	}

	empty := NewIntelligence(IntelligenceConfig{})
	if got := empty.formatBaselinesForContext("res-1"); got != "" {
		t.Errorf("expected empty context with no baseline store, got %q", got)
	}
}

func TestIntelligence_formatAnomaliesForContext(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	if got := intel.formatAnomaliesForContext(nil); got != "" {
		t.Errorf("expected empty anomalies context, got %q", got)
	}

	anomalies := []AnomalyReport{{Metric: "cpu", Description: "CPU high"}}
	ctx := intel.formatAnomaliesForContext(anomalies)
	if !strings.Contains(ctx, "Current Anomalies") {
		t.Fatalf("expected anomalies header, got %q", ctx)
	}
	if !strings.Contains(ctx, "CPU high") {
		t.Fatalf("expected anomaly description, got %q", ctx)
	}
}

func TestIntelligence_formatAnomalyDescription(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	bl := &baseline.MetricBaseline{Mean: 10}

	above := intel.formatAnomalyDescription("cpu", 20, bl, 2.5)
	if !strings.Contains(above, "above baseline") {
		t.Errorf("expected above-baseline description, got %q", above)
	}

	below := intel.formatAnomalyDescription("cpu", 5, bl, -1.5)
	if !strings.Contains(below, "below baseline") {
		t.Errorf("expected below-baseline description, got %q", below)
	}
}

func TestIntelligence_FormatContext_Anomalies(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	intel.anomalyDetector = func(resourceID string) []AnomalyReport {
		return []AnomalyReport{{Metric: "cpu", Description: "CPU high"}}
	}

	ctx := intel.FormatContext("res-1")
	if !strings.Contains(ctx, "Current Anomalies") {
		t.Fatalf("expected anomalies context, got %q", ctx)
	}
	if !strings.Contains(ctx, "CPU high") {
		t.Fatalf("expected anomaly description, got %q", ctx)
	}
}

func TestIntelligence_getUpcomingRisks(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	predictions := []patterns.FailurePrediction{
		{ResourceID: "r1", EventType: patterns.EventHighCPU, DaysUntil: 2, Confidence: 0.6},
		{ResourceID: "r2", EventType: patterns.EventHighMemory, DaysUntil: 9, Confidence: 0.9},
		{ResourceID: "r3", EventType: patterns.EventDiskFull, DaysUntil: 4, Confidence: 0.4},
		{ResourceID: "r4", EventType: patterns.EventOOM, DaysUntil: 1, Confidence: 0.8},
		{ResourceID: "r5", EventType: patterns.EventRestart, DaysUntil: 5, Confidence: 0.7},
	}

	risk := intel.getUpcomingRisks(predictions, 2)
	if len(risk) != 2 {
		t.Fatalf("expected 2 risks, got %d", len(risk))
	}
	if risk[0].ResourceID != "r4" || risk[1].ResourceID != "r1" {
		t.Errorf("unexpected ordering: %v", []string{risk[0].ResourceID, risk[1].ResourceID})
	}
}

func TestIntelligence_countFindings(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	findings := []*Finding{
		nil,
		{Severity: FindingSeverityCritical},
		{Severity: FindingSeverityWarning},
		{Severity: FindingSeverityWatch},
		{Severity: FindingSeverityInfo},
	}
	counts := intel.countFindings(findings)
	if counts.Total != 4 {
		t.Errorf("expected total 4, got %d", counts.Total)
	}
	if counts.Critical != 1 || counts.Warning != 1 || counts.Watch != 1 || counts.Info != 1 {
		t.Errorf("unexpected counts: %+v", counts)
	}
}

func TestIntelligence_getTopFindings(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	base := time.Now()
	findings := []*Finding{
		{Severity: FindingSeverityWarning, DetectedAt: base.Add(-2 * time.Hour)},
		{Severity: FindingSeverityCritical, DetectedAt: base.Add(-3 * time.Hour), Title: "older critical"},
		{Severity: FindingSeverityCritical, DetectedAt: base.Add(-1 * time.Hour), Title: "newer critical"},
		{Severity: FindingSeverityWatch, DetectedAt: base.Add(-30 * time.Minute)},
	}

	top := intel.getTopFindings(findings, 3)
	if len(top) != 3 {
		t.Fatalf("expected 3 findings, got %d", len(top))
	}
	if top[0].Title != "newer critical" || top[1].Title != "older critical" {
		t.Errorf("unexpected ordering: %q, %q", top[0].Title, top[1].Title)
	}
	if top[2].Severity != FindingSeverityWarning {
		t.Errorf("expected warning in third position, got %s", top[2].Severity)
	}
}

func TestIntelligence_getTopFindings_Empty(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	if got := intel.getTopFindings(nil, 5); got != nil {
		t.Errorf("expected nil for empty findings, got %v", got)
	}
}

// When most recent runs error but at least one successful full run exists, the
// score model must drop out of grade A. The previous flat 10-point penalty
// produced grade A even with 8 of 9 recent runs failing — the user saw
// "Coverage incomplete, recent runs errored" alongside an "A 95/100" chip.
// Tiering by error ratio aligns the grade with the warning.
func TestIntelligence_GetSummary_DominantRecentErrorsDropOutOfGradeA(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(20)
	now := time.Now()
	// One successful full patrol — keeps hasSuccessfulFullRun=true so we land
	// on the tiered case-4 branch instead of the no-successful-run branches.
	runHistory.Add(PatrolRunRecord{
		ID:               "manual-ok",
		Type:             "patrol",
		TriggerReason:    "manual",
		CompletedAt:      now.Add(-2 * time.Hour),
		ErrorCount:       0,
		Status:           "healthy",
		ResourcesChecked: 50,
	})
	// Eight subsequent startup runs all errored — the dominant-error case
	// observed in the wild that previously still produced grade A.
	for i := 0; i < 8; i++ {
		runHistory.Add(PatrolRunRecord{
			ID:               fmt.Sprintf("startup-error-%d", i),
			Type:             "patrol",
			TriggerReason:    "startup",
			CompletedAt:      now.Add(-time.Duration(60-i*5) * time.Minute),
			ErrorCount:       1,
			Status:           "error",
			ResourcesChecked: 58,
		})
	}
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	if summary.OverallHealth.Grade == HealthGradeA {
		t.Fatalf("expected non-A grade when most recent runs errored, got %s (score %f)",
			summary.OverallHealth.Grade, summary.OverallHealth.Score)
	}
	var coverageFactor *HealthFactor
	for i := range summary.OverallHealth.Factors {
		if summary.OverallHealth.Factors[i].Category == "coverage" {
			coverageFactor = &summary.OverallHealth.Factors[i]
			break
		}
	}
	if coverageFactor == nil {
		t.Fatal("expected coverage factor in overall health")
	}
	if !strings.Contains(coverageFactor.Description, "Most recent Patrol runs encountered errors") {
		t.Fatalf("expected dominant-error description, got %q", coverageFactor.Description)
	}
}

// A single error among many successful runs should still warn but not punish
// the grade out of A — the "occasional error" case stays at the lighter 10-pt
// penalty so a transient single failure doesn't flip the headline.
func TestIntelligence_GetSummary_OccasionalRecentErrorStaysInGradeA(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(20)
	now := time.Now()
	for i := 0; i < 9; i++ {
		runHistory.Add(PatrolRunRecord{
			ID:               fmt.Sprintf("patrol-ok-%d", i),
			Type:             "patrol",
			TriggerReason:    "scheduled",
			CompletedAt:      now.Add(-time.Duration(60-i*5) * time.Minute),
			ErrorCount:       0,
			Status:           "healthy",
			ResourcesChecked: 50,
		})
	}
	runHistory.Add(PatrolRunRecord{
		ID:               "patrol-error-1",
		Type:             "patrol",
		TriggerReason:    "scheduled",
		CompletedAt:      now.Add(-2 * time.Minute),
		ErrorCount:       1,
		Status:           "error",
		ResourcesChecked: 58,
	})
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	// Light tier: -10 penalty against a 100 starting score = 90, still grade A.
	if summary.OverallHealth.Grade != HealthGradeA {
		t.Fatalf("expected grade A for 1-error-in-10, got %s (score %f)",
			summary.OverallHealth.Grade, summary.OverallHealth.Score)
	}
}

func TestIntelligence_GetSummary_DegradesWhenRecentPatrolCoverageIsScopedAndErroring(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(10)
	now := time.Now()
	runHistory.Add(PatrolRunRecord{
		ID:               "scoped-error-1",
		Type:             "scoped",
		TriggerReason:    "alert_fired",
		CompletedAt:      now.Add(-5 * time.Minute),
		ErrorCount:       1,
		Status:           "error",
		ResourcesChecked: 1,
	})
	runHistory.Add(PatrolRunRecord{
		ID:               "scoped-error-2",
		Type:             "scoped",
		TriggerReason:    "alert_fired",
		CompletedAt:      now.Add(-15 * time.Minute),
		ErrorCount:       1,
		Status:           "error",
		ResourcesChecked: 1,
	})
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	if summary.OverallHealth.Score >= 100 {
		t.Fatalf("expected reduced health score, got %f", summary.OverallHealth.Score)
	}
	if summary.OverallHealth.Grade == HealthGradeA {
		t.Fatalf("expected non-A grade, got %s", summary.OverallHealth.Grade)
	}
	if !strings.Contains(summary.OverallHealth.Prediction, "Run Patrol to check everything") {
		t.Fatalf("expected coverage warning prediction, got %q", summary.OverallHealth.Prediction)
	}
	foundCoverageFactor := false
	for _, factor := range summary.OverallHealth.Factors {
		if factor.Category == "coverage" {
			foundCoverageFactor = true
			break
		}
	}
	if !foundCoverageFactor {
		t.Fatal("expected coverage factor in overall health")
	}
}

func TestIntelligence_GetSummary_DegradesWhenRecentPatrolCoverageIsVerificationOnly(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(10)
	now := time.Now()
	runHistory.Add(PatrolRunRecord{
		ID:               "verification-1",
		Type:             "verification",
		TriggerReason:    "verification",
		CompletedAt:      now.Add(-2 * time.Minute),
		ErrorCount:       0,
		Status:           "healthy",
		ResourcesChecked: 1,
	})
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	if summary.OverallHealth.Score >= 100 {
		t.Fatalf("expected reduced health score, got %f", summary.OverallHealth.Score)
	}
	if summary.OverallHealth.Grade == HealthGradeA {
		t.Fatalf("expected non-A grade, got %s", summary.OverallHealth.Grade)
	}
	want := "Recent Patrol activity only covered follow-up checks. Run Patrol to check everything."
	if summary.OverallHealth.Prediction != want {
		t.Fatalf("expected verification-only coverage warning, got %q", summary.OverallHealth.Prediction)
	}
}

func TestIntelligence_GetSummary_DegradesWhenRecentFullPatrolErrored(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(10)
	now := time.Now()
	runHistory.Add(PatrolRunRecord{
		ID:               "patrol-error-1",
		Type:             "patrol",
		TriggerReason:    "startup",
		CompletedAt:      now.Add(-1 * time.Minute),
		ErrorCount:       1,
		Status:           "error",
		ResourcesChecked: 58,
	})
	runHistory.Add(PatrolRunRecord{
		ID:               "scoped-error-1",
		Type:             "scoped",
		TriggerReason:    "alert_fired",
		CompletedAt:      now.Add(-30 * time.Second),
		ErrorCount:       1,
		Status:           "error",
		ResourcesChecked: 1,
	})
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	if summary.OverallHealth.Score >= 100 {
		t.Fatalf("expected reduced health score, got %f", summary.OverallHealth.Score)
	}
	if summary.OverallHealth.Grade == HealthGradeA {
		t.Fatalf("expected non-A grade, got %s", summary.OverallHealth.Grade)
	}
	want := "Patrol needs a clean check: a recent check ended with errors, so current health may be incomplete."
	if summary.OverallHealth.Prediction != want {
		t.Fatalf("expected full-patrol coverage warning, got %q", summary.OverallHealth.Prediction)
	}
	foundCoverageFactor := false
	for _, factor := range summary.OverallHealth.Factors {
		if factor.Category != "coverage" {
			continue
		}
		foundCoverageFactor = true
		if factor.Description != want {
			t.Fatalf("expected matching coverage factor description, got %q", factor.Description)
		}
	}
	if !foundCoverageFactor {
		t.Fatal("expected coverage factor in overall health")
	}
}

func TestIntelligence_getLearningStats(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	knowledgeStore, err := knowledge.NewStore(t.TempDir())
	if err != nil {
		t.Fatalf("NewStore: %v", err)
	}
	_ = knowledgeStore.SaveNote("vm-1", "vm-1", "vm", "general", "Note", "Content")
	_ = knowledgeStore.SaveNote("vm-2", "vm-2", "vm", "general", "Note", "Content")

	baselineStore := baseline.NewStore(baseline.StoreConfig{MinSamples: 1})
	_ = baselineStore.Learn("vm-1", "vm", "cpu", []baseline.MetricPoint{{Value: 10}})

	patternDetector := patterns.NewDetector(patterns.DetectorConfig{
		MinOccurrences:  2,
		PatternWindow:   48 * time.Hour,
		PredictionLimit: 30 * 24 * time.Hour,
	})
	patternStart := time.Now().Add(-90 * time.Minute)
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-1", EventType: patterns.EventHighCPU, Timestamp: patternStart})
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-1", EventType: patterns.EventHighCPU, Timestamp: patternStart.Add(60 * time.Minute)})

	correlationDetector := correlation.NewDetector(correlation.Config{
		MinOccurrences:    1,
		CorrelationWindow: 2 * time.Hour,
		RetentionWindow:   24 * time.Hour,
	})
	corrStart := time.Now().Add(-30 * time.Minute)
	for i := 0; i < 2; i++ {
		base := corrStart.Add(time.Duration(i) * 10 * time.Minute)
		correlationDetector.RecordEvent(correlation.Event{ResourceID: "node-a", ResourceName: "node-a", ResourceType: "node", EventType: correlation.EventHighCPU, Timestamp: base})
		correlationDetector.RecordEvent(correlation.Event{ResourceID: "vm-b", ResourceName: "vm-b", ResourceType: "vm", EventType: correlation.EventRestart, Timestamp: base.Add(1 * time.Minute)})
	}

	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{MaxIncidents: 5})

	intel.SetSubsystems(nil, patternDetector, correlationDetector, baselineStore, incidentStore, knowledgeStore, nil, nil)

	stats := intel.getLearningStats()
	if stats.ResourcesWithKnowledge != 2 {
		t.Errorf("expected 2 resources with knowledge, got %d", stats.ResourcesWithKnowledge)
	}
	if stats.TotalNotes != 2 {
		t.Errorf("expected 2 total notes, got %d", stats.TotalNotes)
	}
	if stats.ResourcesWithBaselines != 1 {
		t.Errorf("expected 1 resource with baseline, got %d", stats.ResourcesWithBaselines)
	}
	if stats.PatternsDetected != 1 {
		t.Errorf("expected 1 pattern, got %d", stats.PatternsDetected)
	}
	if stats.CorrelationsLearned != 1 {
		t.Errorf("expected 1 correlation, got %d", stats.CorrelationsLearned)
	}
}

func TestIntelligence_SummarizePolicyPostureUsesSharedHelper(t *testing.T) {
	resources := []ur.Resource{
		{
			ID:   "public-1",
			Name: "public-vm",
			Type: ur.ResourceTypeVM,
			Tags: []string{"public"},
		},
		{
			ID:     "internal-1",
			Name:   "agent-1",
			Type:   ur.ResourceTypeAgent,
			Agent:  &ur.AgentData{Hostname: "agent-1"},
			Status: ur.StatusOnline,
		},
		{
			ID:     "sensitive-1",
			Name:   "db-1",
			Type:   ur.ResourceTypeVM,
			Tags:   []string{"database"},
			Status: ur.StatusOnline,
			Identity: ur.ResourceIdentity{
				Hostnames:   []string{"db.internal"},
				IPAddresses: []string{"10.0.0.10"},
			},
			Canonical: &ur.CanonicalIdentity{
				PlatformID: "db.internal",
				Aliases:    []string{"db-1"},
			},
		},
		{
			ID:     "restricted-1",
			Name:   "mail-gw",
			Type:   ur.ResourceTypePMG,
			Status: ur.StatusWarning,
			PMG:    &ur.PMGData{Hostname: "mail.internal"},
		},
	}

	summary := ur.SummarizePolicyPosture(ur.RefreshCanonicalMetadataSlice(resources))
	if summary == nil {
		t.Fatal("expected posture summary")
	}
	if summary.TotalResources != 4 {
		t.Fatalf("total resources = %d, want 4", summary.TotalResources)
	}
	if got := summary.SensitivityCounts[ur.ResourceSensitivityPublic]; got != 1 {
		t.Fatalf("public sensitivity count = %d, want 1", got)
	}
	if got := summary.SensitivityCounts[ur.ResourceSensitivityInternal]; got != 1 {
		t.Fatalf("internal sensitivity count = %d, want 1", got)
	}
	if got := summary.SensitivityCounts[ur.ResourceSensitivitySensitive]; got != 1 {
		t.Fatalf("sensitive sensitivity count = %d, want 1", got)
	}
	if got := summary.SensitivityCounts[ur.ResourceSensitivityRestricted]; got != 1 {
		t.Fatalf("restricted sensitivity count = %d, want 1", got)
	}
	if got := summary.RoutingCounts[ur.ResourceRoutingScopeCloudSummary]; got != 2 {
		t.Fatalf("cloud summary routing count = %d, want 2", got)
	}
	if got := summary.RoutingCounts[ur.ResourceRoutingScopeLocalFirst]; got != 1 {
		t.Fatalf("local-first routing count = %d, want 1", got)
	}
	if got := summary.RoutingCounts[ur.ResourceRoutingScopeLocalOnly]; got != 1 {
		t.Fatalf("local-only routing count = %d, want 1", got)
	}
	if got := summary.RedactionCounts[ur.ResourceRedactionHostname]; got == 0 {
		t.Fatal("expected hostname redaction count")
	}

	labels := ur.ResourcePolicyRedactionLabelsFromCounts(summary.RedactionCounts)
	if !reflect.DeepEqual(labels, []string{"Hostname", "IP Address", "Platform ID", "Alias"}) {
		t.Fatalf("redaction labels = %#v, want %#v", labels, []string{"Hostname", "IP Address", "Platform ID", "Alias"})
	}
}

func TestIntelligence_getResourcesAtRisk(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	findings := NewFindingsStore()
	findings.Add(&Finding{
		ID:           "crit-1",
		Key:          "crit-1",
		Severity:     FindingSeverityCritical,
		Category:     FindingCategoryReliability,
		ResourceID:   "res-critical",
		ResourceName: "critical-vm",
		ResourceType: "vm",
		Title:        "Critical outage",
		DetectedAt:   time.Now(),
		LastSeenAt:   time.Now(),
		Source:       "test",
	})
	findings.Add(&Finding{
		ID:           "warn-1",
		Key:          "warn-1",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryPerformance,
		ResourceID:   "res-warning",
		ResourceName: "warning-vm",
		ResourceType: "vm",
		Title:        "Warning issue",
		DetectedAt:   time.Now(),
		LastSeenAt:   time.Now(),
		Source:       "test",
	})
	findings.Add(&Finding{
		ID:           "watch-1",
		Key:          "watch-1",
		Severity:     FindingSeverityWatch,
		Category:     FindingCategoryPerformance,
		ResourceID:   "res-warning",
		ResourceName: "warning-vm",
		ResourceType: "vm",
		Title:        "Watch issue",
		DetectedAt:   time.Now(),
		LastSeenAt:   time.Now(),
		Source:       "test",
	})
	findings.Add(&Finding{
		ID:           "info-1",
		Key:          "info-1",
		Severity:     FindingSeverityInfo,
		Category:     FindingCategoryPerformance,
		ResourceID:   "res-warning",
		ResourceName: "warning-vm",
		ResourceType: "vm",
		Title:        "Info issue",
		DetectedAt:   time.Now(),
		LastSeenAt:   time.Now(),
		Source:       "test",
	})

	intel.SetSubsystems(findings, nil, nil, nil, nil, nil, nil, nil)
	risks := intel.getResourcesAtRisk(1)
	if len(risks) != 1 {
		t.Fatalf("expected 1 risk, got %d", len(risks))
	}
	if risks[0].ResourceID != "res-critical" {
		t.Errorf("expected critical resource first, got %s", risks[0].ResourceID)
	}
	if risks[0].TopIssue != "Critical outage" {
		t.Errorf("expected top issue to be critical, got %s", risks[0].TopIssue)
	}
}

func TestIntelligence_calculateResourceHealth_ClampAndSeverities(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})

	resourceIntel := &ResourceIntelligence{
		ResourceID: "test-vm",
		ActiveFindings: []*Finding{
			nil,
			{Severity: FindingSeverityCritical, Title: "crit-1"},
			{Severity: FindingSeverityCritical, Title: "crit-2"},
			{Severity: FindingSeverityCritical, Title: "crit-3"},
			{Severity: FindingSeverityCritical, Title: "crit-4"},
			{Severity: FindingSeverityWatch, Title: "watch"},
			{Severity: FindingSeverityInfo, Title: "info"},
		},
		Anomalies: []AnomalyReport{
			{Metric: "cpu", Severity: baseline.AnomalyHigh, Description: "high"},
			{Metric: "disk", Severity: baseline.AnomalyLow, Description: "low"},
		},
	}

	health := intel.calculateResourceHealth(resourceIntel)
	if health.Score != 0 {
		t.Errorf("expected score clamped to 0, got %f", health.Score)
	}
}

func TestIntelligence_calculateOverallHealth_Clamps(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	var predictions []patterns.FailurePrediction
	for i := 0; i < 10; i++ {
		predictions = append(predictions, patterns.FailurePrediction{EventType: patterns.EventHighCPU, DaysUntil: 1, Confidence: 0.9})
	}
	negative := intel.calculateOverallHealth(&IntelligenceSummary{
		FindingsCount: FindingsCounts{Critical: 10, Warning: 10},
		UpcomingRisks: predictions,
	}, nil)
	if negative.Score != 0 {
		t.Errorf("expected score clamped to 0, got %f", negative.Score)
	}

	positive := intel.calculateOverallHealth(&IntelligenceSummary{
		Learning: LearningStats{ResourcesWithKnowledge: 10},
	}, nil)
	if positive.Score != 100 {
		t.Errorf("expected score clamped to 100, got %f", positive.Score)
	}
	if len(positive.Factors) == 0 {
		t.Error("expected learning factor for positive health")
	}
}

func TestIntelligence_generateHealthPrediction_Branches(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})

	gradeA := intel.generateHealthPrediction(HealthScore{Grade: HealthGradeA}, &IntelligenceSummary{})
	if !strings.Contains(gradeA, "healthy") {
		t.Errorf("expected healthy prediction, got %q", gradeA)
	}

	critical := intel.generateHealthPrediction(HealthScore{Grade: HealthGradeB}, &IntelligenceSummary{
		FindingsCount: FindingsCounts{Critical: 2},
	})
	if !strings.Contains(critical, "Immediate attention") {
		t.Errorf("expected critical prediction, got %q", critical)
	}

	risk := intel.generateHealthPrediction(HealthScore{Grade: HealthGradeB}, &IntelligenceSummary{
		UpcomingRisks: []patterns.FailurePrediction{{EventType: patterns.EventHighCPU, DaysUntil: 2, Confidence: 0.8}},
	})
	if !strings.Contains(risk, "Predicted") {
		t.Errorf("expected prediction text, got %q", risk)
	}

	stable := intel.generateHealthPrediction(HealthScore{Grade: HealthGradeC}, &IntelligenceSummary{})
	if !strings.Contains(stable, "stable") {
		t.Errorf("expected stable prediction, got %q", stable)
	}
}

func TestIntelligence_FormatContext_AllSubsystems(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	knowledgeStore, _ := knowledge.NewStore(t.TempDir())
	_ = knowledgeStore.SaveNote("vm-ctx", "vm-ctx", "vm", "general", "Note", "Content")

	baselineStore := baseline.NewStore(baseline.StoreConfig{MinSamples: 1})
	_ = baselineStore.Learn("vm-ctx", "vm", "cpu", []baseline.MetricPoint{{Value: 10}})

	patternDetector := patterns.NewDetector(patterns.DetectorConfig{MinOccurrences: 2, PatternWindow: 48 * time.Hour, PredictionLimit: 30 * 24 * time.Hour})
	patternStart := time.Now().Add(-90 * time.Minute)
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-ctx", EventType: patterns.EventHighCPU, Timestamp: patternStart})
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-ctx", EventType: patterns.EventHighCPU, Timestamp: patternStart.Add(60 * time.Minute)})

	correlationDetector := correlation.NewDetector(correlation.Config{MinOccurrences: 1, CorrelationWindow: 2 * time.Hour, RetentionWindow: 24 * time.Hour})
	corrBase := time.Now().Add(-30 * time.Minute)
	correlationDetector.RecordEvent(correlation.Event{ResourceID: "vm-ctx", ResourceName: "vm-ctx", ResourceType: "vm", EventType: correlation.EventHighCPU, Timestamp: corrBase})
	correlationDetector.RecordEvent(correlation.Event{ResourceID: "node-ctx", ResourceName: "node-ctx", ResourceType: "node", EventType: correlation.EventRestart, Timestamp: corrBase.Add(1 * time.Minute)})

	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{MaxIncidents: 10})
	incidentStore.RecordAlertFired(&alerts.Alert{ID: "alert-ctx", ResourceID: "vm-ctx", ResourceName: "vm-ctx", Type: "cpu", StartTime: time.Now()})

	intel.SetSubsystems(nil, patternDetector, correlationDetector, baselineStore, incidentStore, knowledgeStore, nil, nil)
	ctx := intel.FormatContext("vm-ctx")
	if !strings.Contains(ctx, "Learned Baselines") {
		t.Fatalf("expected baseline context, got %q", ctx)
	}
	if !strings.Contains(ctx, "Failure Predictions") {
		t.Fatalf("expected predictions context, got %q", ctx)
	}
	if !strings.Contains(ctx, "Resource Correlations") {
		t.Fatalf("expected correlations context, got %q", ctx)
	}
	if !strings.Contains(ctx, "Incident Memory") {
		t.Fatalf("expected incident context, got %q", ctx)
	}
}

func TestIntelligence_FormatGlobalContext_AllSubsystems(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	knowledgeStore, _ := knowledge.NewStore(t.TempDir())
	_ = knowledgeStore.SaveNote("vm-global", "vm-global", "vm", "general", "Note", "Content")

	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{MaxIncidents: 10})
	incidentStore.RecordAlertFired(&alerts.Alert{ID: "alert-global", ResourceID: "vm-global", ResourceName: "vm-global", Type: "cpu", StartTime: time.Now()})

	correlationDetector := correlation.NewDetector(correlation.Config{MinOccurrences: 1, CorrelationWindow: 2 * time.Hour, RetentionWindow: 24 * time.Hour})
	corrBase := time.Now().Add(-30 * time.Minute)
	for i := 0; i < 2; i++ {
		base := corrBase.Add(time.Duration(i) * 10 * time.Minute)
		correlationDetector.RecordEvent(correlation.Event{ResourceID: "node-a", ResourceName: "node-a", ResourceType: "node", EventType: correlation.EventHighCPU, Timestamp: base})
		correlationDetector.RecordEvent(correlation.Event{ResourceID: "vm-global", ResourceName: "vm-global", ResourceType: "vm", EventType: correlation.EventRestart, Timestamp: base.Add(1 * time.Minute)})
	}

	patternDetector := patterns.NewDetector(patterns.DetectorConfig{MinOccurrences: 2, PatternWindow: 48 * time.Hour, PredictionLimit: 30 * 24 * time.Hour})
	patternStart := time.Now().Add(-90 * time.Minute)
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-global", EventType: patterns.EventHighCPU, Timestamp: patternStart})
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-global", EventType: patterns.EventHighCPU, Timestamp: patternStart.Add(60 * time.Minute)})

	intel.SetSubsystems(nil, patternDetector, correlationDetector, nil, incidentStore, knowledgeStore, nil, nil)
	ctx := intel.FormatGlobalContext()
	if ctx == "" {
		t.Fatal("expected non-empty global context")
	}
	if !strings.Contains(ctx, "Resource Correlations") {
		t.Fatalf("expected correlations in global context, got %q", ctx)
	}
}

func TestIntelligence_GetSummary_WithSubsystems(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	findings := NewFindingsStore()
	findings.Add(&Finding{
		ID:           "f1",
		Key:          "f1",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryPerformance,
		ResourceID:   "vm-sum",
		ResourceName: "vm-sum",
		ResourceType: "vm",
		Title:        "Warning",
		DetectedAt:   time.Now(),
		LastSeenAt:   time.Now(),
		Source:       "test",
	})

	patternDetector := patterns.NewDetector(patterns.DetectorConfig{MinOccurrences: 2, PatternWindow: 48 * time.Hour, PredictionLimit: 30 * 24 * time.Hour})
	patternStart := time.Now().Add(-90 * time.Minute)
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-sum", EventType: patterns.EventHighCPU, Timestamp: patternStart})
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-sum", EventType: patterns.EventHighCPU, Timestamp: patternStart.Add(60 * time.Minute)})

	changes := newLegacyChangeDetector(t, legacyCreatedChange("vm-sum", "vm-sum", "vm"))

	remediations := memory.NewRemediationLog(memory.RemediationLogConfig{MaxRecords: 10})
	_ = remediations.Log(memory.RemediationRecord{ResourceID: "vm-sum", Problem: "cpu", Action: "restart", Outcome: memory.OutcomeResolved})

	intel.SetSubsystems(findings, patternDetector, nil, nil, nil, nil, changes, remediations)

	summary := intel.GetSummary()
	if summary.FindingsCount.Total == 0 {
		t.Error("expected findings in summary")
	}
	if summary.PredictionsCount == 0 {
		t.Error("expected predictions in summary")
	}
	if len(summary.UpcomingRisks) == 0 {
		t.Error("expected upcoming risks in summary")
	}
	if summary.RecentChangesCount == 0 {
		t.Error("expected recent changes in summary")
	}
	if len(summary.RecentChanges) == 0 {
		t.Fatal("expected recent change entries in summary")
	}
	if summary.RecentChanges[0].SourceType != ur.SourceHeuristic {
		t.Fatalf("expected heuristic source type for fallback summary, got %s", summary.RecentChanges[0].SourceType)
	}
	if len(summary.RecentRemediations) == 0 {
		t.Error("expected recent remediations in summary")
	}
	if len(summary.ResourcesAtRisk) == 0 {
		t.Error("expected resources at risk in summary")
	}
}

func TestIntelligence_GetSummary_UsesCanonicalResourceTimeline(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	canonicalStore := ur.NewMemoryStore()
	if err := canonicalStore.RecordChange(ur.ResourceChange{
		ID:            "change-1",
		ObservedAt:    time.Now().Add(-time.Hour),
		ResourceID:    "vm-sum",
		Kind:          ur.ChangeConfigUpdate,
		SourceType:    ur.SourcePulseDiff,
		SourceAdapter: ur.AdapterOpsAgent,
		Reason:        "Config refresh",
	}); err != nil {
		t.Fatalf("record canonical change: %v", err)
	}

	intel.SetResourceTimelineStore(canonicalStore, "org-1")

	summary := intel.GetSummary()
	if summary.RecentChangesCount != 1 {
		t.Fatalf("expected canonical recent change count, got %d", summary.RecentChangesCount)
	}
	if len(summary.RecentChanges) != 1 {
		t.Fatalf("expected canonical recent change slice, got %d", len(summary.RecentChanges))
	}
	if summary.RecentChanges[0].Kind != ur.ChangeConfigUpdate {
		t.Fatalf("expected canonical recent change kind, got %s", summary.RecentChanges[0].Kind)
	}
}

func TestIntelligence_GetResourceIntelligence_WithAllSubsystems(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	findings := NewFindingsStore()
	findings.Add(&Finding{
		ID:           "f1",
		Key:          "f1",
		Severity:     FindingSeverityWarning,
		Category:     FindingCategoryPerformance,
		ResourceID:   "vm-intel",
		ResourceName: "vm-intel",
		ResourceType: "vm",
		Title:        "Warning",
		DetectedAt:   time.Now(),
		LastSeenAt:   time.Now(),
		Source:       "test",
	})

	patternDetector := patterns.NewDetector(patterns.DetectorConfig{MinOccurrences: 2, PatternWindow: 48 * time.Hour, PredictionLimit: 30 * 24 * time.Hour})
	patternStart := time.Now().Add(-90 * time.Minute)
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-intel", EventType: patterns.EventHighCPU, Timestamp: patternStart})
	patternDetector.RecordEvent(patterns.HistoricalEvent{ResourceID: "vm-intel", EventType: patterns.EventHighCPU, Timestamp: patternStart.Add(60 * time.Minute)})

	correlationDetector := correlation.NewDetector(correlation.Config{MinOccurrences: 1, CorrelationWindow: 2 * time.Hour, RetentionWindow: 24 * time.Hour})
	corrBase := time.Now().Add(-30 * time.Minute)
	correlationDetector.RecordEvent(correlation.Event{ResourceID: "node-intel", ResourceName: "node-intel", ResourceType: "node", EventType: correlation.EventHighCPU, Timestamp: corrBase})
	correlationDetector.RecordEvent(correlation.Event{ResourceID: "vm-intel", ResourceName: "vm-intel", ResourceType: "vm", EventType: correlation.EventRestart, Timestamp: corrBase.Add(1 * time.Minute)})
	correlationDetector.RecordEvent(correlation.Event{ResourceID: "vm-intel", ResourceName: "vm-intel", ResourceType: "vm", EventType: correlation.EventHighCPU, Timestamp: corrBase.Add(2 * time.Minute)})
	correlationDetector.RecordEvent(correlation.Event{ResourceID: "vm-child", ResourceName: "vm-child", ResourceType: "vm", EventType: correlation.EventRestart, Timestamp: corrBase.Add(3 * time.Minute)})

	baselineStore := baseline.NewStore(baseline.StoreConfig{MinSamples: 1})
	_ = baselineStore.Learn("vm-intel", "vm", "cpu", []baseline.MetricPoint{{Value: 10}})

	incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{MaxIncidents: 10})
	incidentStore.RecordAlertFired(&alerts.Alert{ID: "alert-intel", ResourceID: "vm-intel", ResourceName: "vm-intel", Type: "cpu", StartTime: time.Now()})

	knowledgeStore, _ := knowledge.NewStore(t.TempDir())
	_ = knowledgeStore.SaveNote("vm-intel", "vm-intel", "vm", "general", "Note", "Content")

	intel.SetSubsystems(findings, patternDetector, correlationDetector, baselineStore, incidentStore, knowledgeStore, nil, nil)
	intel.SetUnifiedResourceProvider(&mockUnifiedResourceProvider{
		getAllFunc: func() []ur.Resource {
			return []ur.Resource{
				{ID: "public-1", Type: ur.ResourceTypeVM, Tags: []string{"public"}},
				{
					ID:     "internal-1",
					Type:   ur.ResourceTypeAgent,
					Agent:  &ur.AgentData{Hostname: "agent-1"},
					Status: ur.StatusOnline,
				},
			}
		},
	})
	canonicalStore := ur.NewMemoryStore()
	if err := canonicalStore.RecordChange(ur.ResourceChange{
		ID:            "change-intel",
		ObservedAt:    time.Now().Add(-30 * time.Minute),
		ResourceID:    "vm-intel",
		Kind:          ur.ChangeRestart,
		SourceType:    ur.SourcePlatformEvent,
		SourceAdapter: ur.AdapterProxmox,
		Reason:        "guest rebooted",
	}); err != nil {
		t.Fatalf("record canonical change: %v", err)
	}
	intel.SetResourceTimelineStore(canonicalStore, "org-1")

	res := intel.GetResourceIntelligence("vm-intel")
	if len(res.ActiveFindings) == 0 {
		t.Error("expected active findings")
	}
	if len(res.Predictions) == 0 {
		t.Error("expected predictions")
	}
	if len(res.Correlations) == 0 {
		t.Error("expected correlations")
	}
	if len(res.Dependents) == 0 || len(res.Dependencies) == 0 {
		t.Error("expected dependencies and dependents")
	}
	if len(res.Baselines) == 0 {
		t.Error("expected baselines")
	}
	if len(res.RecentIncidents) == 0 {
		t.Error("expected incidents")
	}
	if len(res.RecentChanges) == 0 {
		t.Error("expected recent changes")
	}
	if res.RecentChanges[0].Kind != ur.ChangeRestart {
		t.Fatalf("expected restart change kind, got %s", res.RecentChanges[0].Kind)
	}
	if res.Knowledge == nil || res.NoteCount == 0 {
		t.Error("expected knowledge")
	}
}

// TestIntelligence_GetSummary_RecoversAfterTrailingSuccessfulFullRuns
// guards the recovery-suppression introduced after the
// malformed-history Patrol bug: when a Patrol-affecting bug causes a
// run of errors and the fix lands, three consecutive successful full
// runs at the END of the recent-runs window should drop the error
// penalty entirely so the health score reflects current reality. Before
// this, fixing a bug kept the assessment depressed for ~9 hours while
// the historical errors aged out of the trailing-10 ratio.
func TestIntelligence_GetSummary_RecoversAfterTrailingSuccessfulFullRuns(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(20)
	now := time.Now()

	// Six older errored full runs.
	for i := 0; i < 6; i++ {
		runHistory.Add(PatrolRunRecord{
			ID:               fmt.Sprintf("error-%d", i),
			Type:             "full",
			CompletedAt:      now.Add(-time.Duration(20-i) * time.Hour),
			ErrorCount:       1,
			Status:           "error",
			ResourcesChecked: 1,
		})
	}
	// Three trailing successful full runs (the "fix landed" sequence).
	for i := 0; i < 3; i++ {
		runHistory.Add(PatrolRunRecord{
			ID:               fmt.Sprintf("success-%d", i),
			Type:             "full",
			CompletedAt:      now.Add(-time.Duration(6-i*2) * time.Hour),
			ErrorCount:       0,
			Status:           "issues_found",
			ResourcesChecked: 50,
			NewFindings:      1,
		})
	}
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	for _, factor := range summary.OverallHealth.Factors {
		if factor.Category == "coverage" {
			t.Fatalf("expected coverage factor to be suppressed after trailing successes, got %+v", factor)
		}
	}
	if strings.Contains(summary.OverallHealth.Prediction, "encountered errors") {
		t.Fatalf("expected prediction to not mention historical errors after recovery, got %q", summary.OverallHealth.Prediction)
	}
}

// TestIntelligence_GetSummary_KeepsPenaltyWhenRecoveryIncomplete guards
// the inverse: two trailing successes after errors isn't yet "recovered"
// — the threshold is three. This prevents premature trust restoration
// from a transient single-run recovery.
func TestIntelligence_GetSummary_KeepsPenaltyWhenRecoveryIncomplete(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	runHistory := NewPatrolRunHistoryStore(20)
	now := time.Now()

	for i := 0; i < 6; i++ {
		runHistory.Add(PatrolRunRecord{
			ID:               fmt.Sprintf("error-%d", i),
			Type:             "full",
			CompletedAt:      now.Add(-time.Duration(15-i) * time.Hour),
			ErrorCount:       1,
			Status:           "error",
			ResourcesChecked: 1,
		})
	}
	// Only two trailing successful runs — not yet a clean recovery.
	for i := 0; i < 2; i++ {
		runHistory.Add(PatrolRunRecord{
			ID:               fmt.Sprintf("success-%d", i),
			Type:             "full",
			CompletedAt:      now.Add(-time.Duration(4-i*2) * time.Hour),
			ErrorCount:       0,
			Status:           "issues_found",
			ResourcesChecked: 50,
			NewFindings:      1,
		})
	}
	intel.SetRunHistoryStore(runHistory)

	summary := intel.GetSummary()
	foundCoverage := false
	for _, factor := range summary.OverallHealth.Factors {
		if factor.Category == "coverage" {
			foundCoverage = true
			break
		}
	}
	if !foundCoverage {
		t.Fatal("expected coverage penalty to remain when trailing successes are below the recovery threshold")
	}
}

func TestIntelligence_GetResourceIntelligence_KnowledgeFallback(t *testing.T) {
	intel := NewIntelligence(IntelligenceConfig{})
	knowledgeStore, _ := knowledge.NewStore(t.TempDir())
	_ = knowledgeStore.SaveNote("vm-know", "knowledge-vm", "vm", "general", "Note", "Content")

	intel.SetSubsystems(nil, nil, nil, nil, nil, knowledgeStore, nil, nil)
	res := intel.GetResourceIntelligence("vm-know")
	if res.ResourceName != "knowledge-vm" {
		t.Errorf("expected resource name from knowledge, got %q", res.ResourceName)
	}
	if res.ResourceType != "vm" {
		t.Errorf("expected resource type from knowledge, got %q", res.ResourceType)
	}
}