Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

79.2 KB
Raw Blame
package ai

import (
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

// --- filterStateByScope tests ---

func TestFilterStateByScope_NoScope(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
		VMs:   []models.VM{{ID: "vm1", Name: "vm-1"}},
	}
	scope := PatrolScope{} // no resource IDs or types

	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)

	if len(filtered.Nodes) != 1 {
		t.Errorf("expected 1 node with no scope filter, got %d", len(filtered.Nodes))
	}
	if len(filtered.VMs) != 1 {
		t.Errorf("expected 1 VM with no scope filter, got %d", len(filtered.VMs))
	}
}

func TestResolvePatrolScopeExpandsCanonicalUnifiedIDToSourceResource(t *testing.T) {
	snapshot := models.StateSnapshot{VMs: []models.VM{{ID: "source-vm-401", Name: "web", VMID: 401}}}
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(snapshot)
	views := registry.VMs()
	if len(views) != 1 || views[0].ID() == "" {
		t.Fatalf("expected one canonical VM view, got %+v", views)
	}
	state := newPatrolRuntimeStateWithProviders(snapshot, registry, unifiedresources.NewUnifiedAIAdapter(registry))
	resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{views[0].ID()}})
	if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 {
		t.Fatalf("canonical ID did not resolve exactly: %+v", resolution)
	}
	filtered := (&PatrolService{}).filterStateByScopeState(state, resolved)
	if len(filtered.VMs) != 1 || filtered.VMs[0].ID != "source-vm-401" {
		t.Fatalf("resolved scope did not select source VM: %+v", filtered.VMs)
	}
	if len(resolution.EffectiveResourceIDs) == 0 {
		t.Fatalf("expected effective resource IDs, got %+v", resolution)
	}
	if len(resolved.resolvedResourceTypes) != 1 || resolved.resolvedResourceTypes[0] != "vm" {
		t.Fatalf("resolved resource types = %v, want [vm]", resolved.resolvedResourceTypes)
	}
}

func TestResolvePatrolScopePreservesCanonicalAvailabilityEndpoint(t *testing.T) {
	now := time.Now().UTC()
	endpoint := unifiedresources.Resource{
		ID:       "network-endpoint-jellyfin",
		Name:     "Jellyfin",
		Type:     unifiedresources.ResourceTypeNetworkEndpoint,
		Status:   unifiedresources.StatusOnline,
		LastSeen: now,
		Sources:  []unifiedresources.DataSource{unifiedresources.SourceAvailability},
		Availability: &unifiedresources.AvailabilityData{
			TargetID: "jellyfin-check", Enabled: true, ProbeOutcome: "reachable",
		},
		Canonical: &unifiedresources.CanonicalIdentity{
			DisplayName: "Jellyfin playback endpoint",
			Aliases:     []string{"jellyfin.local"},
		},
	}
	unrelated := unifiedresources.Resource{
		ID: "network-endpoint-unrelated", Name: "Unrelated", Type: unifiedresources.ResourceTypeNetworkEndpoint,
		Status: unifiedresources.StatusOnline, LastSeen: now,
	}
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestResources([]unifiedresources.Resource{endpoint, unrelated})
	state := newPatrolRuntimeStateWithProviders(
		models.EmptyStateSnapshot(), registry, unifiedresources.NewUnifiedAIAdapter(registry),
	)

	resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{endpoint.ID}})
	if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 {
		t.Fatalf("canonical availability endpoint did not resolve exactly: %+v", resolution)
	}
	filtered := filterPatrolStateByScopeState(state, resolved)
	resources := filtered.unifiedResourceProvider.GetAll()
	if len(resources) != 1 || resources[0].ID != endpoint.ID {
		t.Fatalf("scoped provider resources = %+v, want only %q", resources, endpoint.ID)
	}
	checks := unifiedresources.AvailabilityChecksForResource(resources[0])
	if len(checks) != 1 || checks[0].TargetID != "jellyfin-check" || checks[0].ProbeOutcome != "reachable" {
		t.Fatalf("availability evidence was not preserved: %+v", checks)
	}
	effectiveContainsEndpoint := false
	for _, id := range resolution.EffectiveResourceIDs {
		if id == endpoint.ID {
			effectiveContainsEndpoint = true
			break
		}
	}
	if !effectiveContainsEndpoint {
		t.Fatalf("effective scope = %v, want canonical endpoint %q", resolution.EffectiveResourceIDs, endpoint.ID)
	}
	counts := patrolRuntimeCountResources(filtered)
	if counts.unified != 1 || counts.total() != 1 {
		t.Fatalf("canonical endpoint counts = %#v, want one unified resource", counts)
	}
	if len(resolved.resolvedResourceTypes) != 1 || resolved.resolvedResourceTypes[0] != "network-endpoint" {
		t.Fatalf("resolved endpoint types = %v, want [network-endpoint]", resolved.resolvedResourceTypes)
	}
}

func TestResolvePatrolScopeRejectsUnmatchedAndAmbiguousAliases(t *testing.T) {
	state := newPatrolRuntimeState(models.StateSnapshot{VMs: []models.VM{
		{ID: "vm-1", Name: "duplicate", VMID: 1},
		{ID: "vm-2", Name: "duplicate", VMID: 2},
	}})
	resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{"duplicate", "missing"}})
	if len(resolved.ResourceIDs) != 0 {
		t.Fatalf("ambiguous or unmatched aliases must not expand: %v", resolved.ResourceIDs)
	}
	if len(resolution.AmbiguousResourceIDs) != 1 || resolution.AmbiguousResourceIDs[0] != "duplicate" {
		t.Fatalf("ambiguous IDs = %v", resolution.AmbiguousResourceIDs)
	}
	if len(resolution.UnmatchedResourceIDs) != 1 || resolution.UnmatchedResourceIDs[0] != "missing" {
		t.Fatalf("unmatched IDs = %v", resolution.UnmatchedResourceIDs)
	}
}

func TestResolvePatrolScopeCanonicalIDOutranksAliasCollision(t *testing.T) {
	state := newPatrolRuntimeState(models.StateSnapshot{VMs: []models.VM{
		{ID: "vm-canonical", Name: "primary", VMID: 1},
		{ID: "vm-other", Name: "vm-canonical", VMID: 2},
	}})
	resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{"vm-canonical"}})
	if len(resolution.AmbiguousResourceIDs) != 0 || len(resolution.UnmatchedResourceIDs) != 0 {
		t.Fatalf("canonical ID was treated as an alias collision: %+v", resolution)
	}
	filtered := filterPatrolStateByScopeState(state, resolved)
	if len(filtered.VMs) != 1 || filtered.VMs[0].ID != "vm-canonical" {
		t.Fatalf("canonical ID selected wrong resource: %+v", filtered.VMs)
	}
}

func TestResolvePatrolScopeResolvesDockerAlertResourceIDs(t *testing.T) {
	// A container reported without an ID alerts under its own name reference,
	// so the host's reference stays unambiguous.
	snapshot := models.StateSnapshot{DockerHosts: []models.DockerHost{{
		ID: "nas", Hostname: "nas",
		Containers: []models.DockerContainer{
			{ID: "abc123def456", Name: "dockhand-hawser-updater-nas", State: "exited"},
			{Name: "legacy-sidecar", State: "exited"},
		},
	}}}
	containerAlertID := alerts.DockerContainerResourceID("nas", "abc123def456", "dockhand-hawser-updater-nas")
	namedContainerAlertID := alerts.DockerContainerResourceID("nas", "", "legacy-sidecar")
	hostAlertID := alerts.DockerHostResourceID("nas")

	snapshotState := newPatrolRuntimeState(snapshot)
	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(snapshot)
	readState := newPatrolRuntimeStateWithProviders(snapshot, registry, unifiedresources.NewUnifiedAIAdapter(registry))

	for _, state := range []patrolRuntimeState{snapshotState, readState} {
		for _, tc := range []struct {
			requested    string
			resourceType string
		}{{containerAlertID, "app-container"}, {namedContainerAlertID, "app-container"}, {hostAlertID, "docker-host"}} {
			if tc.requested == namedContainerAlertID && state.readState == nil {
				// The legacy snapshot record of a container without an ID has
				// no ID to scope by; only the unified read state can.
				continue
			}
			resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{tc.requested}})
			if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 {
				t.Fatalf("alert resource ID %q did not resolve exactly: %+v", tc.requested, resolution)
			}
			if len(resolved.ResourceIDs) == 0 {
				t.Fatalf("alert resource ID %q resolved to an empty scope", tc.requested)
			}
			if len(resolved.resolvedResourceTypes) != 1 || resolved.resolvedResourceTypes[0] != tc.resourceType {
				t.Fatalf("alert resource ID %q resolved types = %v, want [%s]", tc.requested, resolved.resolvedResourceTypes, tc.resourceType)
			}
		}
	}
}

func TestRunScopedPatrolPersistsZeroMatchFailure(t *testing.T) {
	ps := NewPatrolService(nil, &mockStateProvider{state: models.StateSnapshot{
		Nodes: []models.Node{{ID: "node-1", Name: "pve-1"}},
	}})
	ps.SetConfig(PatrolConfig{Enabled: true, AnalyzeNodes: true, Interval: time.Hour})
	ps.runScopedPatrol(t.Context(), PatrolScope{ResourceIDs: []string{"missing-resource"}, Reason: TriggerReasonManual})
	runs := ps.runHistoryStore.GetRecent(1)
	if len(runs) != 1 {
		t.Fatalf("zero-match scope should create one durable run, got %d", len(runs))
	}
	run := runs[0]
	if run.Status != "error" || run.ErrorCount != 1 || run.ResourcesChecked != 0 {
		t.Fatalf("unexpected zero-match run: %+v", run)
	}
	if len(run.ScopeResourceIDs) != 1 || run.ScopeResourceIDs[0] != "missing-resource" || len(run.EffectiveScopeResourceIDs) != 0 {
		t.Fatalf("zero-match scope identity was not preserved: %+v", run)
	}
	if run.FindingsSummary == "All healthy" {
		t.Fatal("zero-match run must never claim all healthy")
	}
}

func TestPatrolFindingSummaryForRunExcludesSupportingResourceFindings(t *testing.T) {
	summary := patrolFindingSummaryForRun([]*Finding{
		{ID: "run-owned", ResourceID: "container-target", Severity: FindingSeverityWarning},
		{ID: "supporting-context", ResourceID: "agent-host", Severity: FindingSeverityCritical},
	}, []string{"run-owned"})
	if summary.Warning != 1 || summary.Critical != 0 || summary.Total != 1 {
		t.Fatalf("run summary = %+v, want one run-owned warning and no supporting-resource finding", summary)
	}
}

func TestFilterStateByScope_ByResourceID(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "n1", Name: "node-1"},
			{ID: "n2", Name: "node-2"},
		},
	}
	scope := PatrolScope{
		ResourceIDs:   []string{"n1"},
		ResourceTypes: []string{"node"},
	}

	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)

	if len(filtered.Nodes) != 1 {
		t.Fatalf("expected 1 node, got %d", len(filtered.Nodes))
	}
	if filtered.Nodes[0].ID != "n1" {
		t.Errorf("expected node n1, got %s", filtered.Nodes[0].ID)
	}
}

func TestFilterStateByScope_ByResourceName(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "n1", Name: "node-1"},
			{ID: "n2", Name: "node-2"},
		},
	}
	scope := PatrolScope{
		ResourceIDs:   []string{"node-1"},
		ResourceTypes: []string{"node"},
	}

	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)

	if len(filtered.Nodes) != 1 {
		t.Fatalf("expected 1 node matched by name, got %d", len(filtered.Nodes))
	}
	if filtered.Nodes[0].Name != "node-1" {
		t.Errorf("expected node-1, got %s", filtered.Nodes[0].Name)
	}
}

func TestFilterStateByScope_ByType_VM(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
		VMs:   []models.VM{{ID: "vm1", Name: "vm-1"}},
	}
	scope := PatrolScope{
		ResourceTypes: []string{"vm"},
	}

	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)

	if len(filtered.Nodes) != 0 {
		t.Errorf("expected 0 nodes when scoped to VM type, got %d", len(filtered.Nodes))
	}
	if len(filtered.VMs) != 1 {
		t.Errorf("expected 1 VM, got %d", len(filtered.VMs))
	}
}

func TestFilterStateByScope_TypeAliasesRejected(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		VMs:        []models.VM{{ID: "vm1", Name: "vm-1"}},
		Containers: []models.Container{{ID: "ct1", Name: "ct-1"}},
		Hosts:      []models.Host{{ID: "h1", Hostname: "host-1"}},
		PhysicalDisks: []models.PhysicalDisk{
			{ID: "disk-1", DevPath: "/dev/sda", Model: "sda"},
		},
		PBSInstances: []models.PBSInstance{
			{ID: "pbs1", Name: "pbs-main", Datastores: []models.PBSDatastore{{Name: "ds1"}}},
		},
		PMGInstances: []models.PMGInstance{
			{ID: "pmg1", Name: "pmg-main", Host: "pmg.local"},
		},
		KubernetesClusters: []models.KubernetesCluster{
			{ID: "k1", Name: "cluster-1"},
		},
		DockerHosts: []models.DockerHost{
			{ID: "dh1", Hostname: "docker-host-1"},
		},
	}

	// Legacy alias should not match canonical VM resources.
	scope := PatrolScope{ResourceTypes: []string{"qemu"}}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.VMs) != 0 {
		t.Errorf("expected legacy 'qemu' alias to be rejected, got %d VMs", len(filtered.VMs))
	}

	// Legacy alias should not match canonical system-container resources.
	scope = PatrolScope{ResourceTypes: []string{"container"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Containers) != 0 {
		t.Errorf("expected legacy 'container' alias to be rejected, got %d containers", len(filtered.Containers))
	}

	// Removed host alias should not match canonical agent resources.
	scope = PatrolScope{ResourceTypes: []string{"host"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Hosts) != 0 {
		t.Errorf("expected legacy 'host' alias to be rejected, got %d hosts", len(filtered.Hosts))
	}

	// Canonical v6 type should match containers.
	scope = PatrolScope{ResourceTypes: []string{"system-container"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Containers) != 1 {
		t.Errorf("expected 'system-container' to match LXC containers, got %d", len(filtered.Containers))
	}

	// Underscore alias should be rejected; hyphenated canonical type is required.
	scope = PatrolScope{ResourceTypes: []string{"system_container"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Containers) != 0 {
		t.Errorf("expected legacy 'system_container' alias to be rejected, got %d containers", len(filtered.Containers))
	}

	// Additional removed aliases should also be rejected.
	scope = PatrolScope{ResourceTypes: []string{"docker_container"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.DockerHosts) != 0 {
		t.Errorf("expected legacy 'docker_container' alias to be rejected, got %d docker hosts", len(filtered.DockerHosts))
	}

	scope = PatrolScope{ResourceTypes: []string{"kubernetes_cluster"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.KubernetesClusters) != 0 {
		t.Errorf("expected legacy 'kubernetes_cluster' alias to be rejected, got %d kubernetes clusters", len(filtered.KubernetesClusters))
	}

	scope = PatrolScope{ResourceTypes: []string{"docker"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.DockerHosts) != 0 {
		t.Errorf("expected non-canonical 'docker' alias to be rejected, got %d docker hosts", len(filtered.DockerHosts))
	}

	scope = PatrolScope{ResourceTypes: []string{"kubernetes"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.KubernetesClusters) != 0 {
		t.Errorf("expected non-canonical 'kubernetes' alias to be rejected, got %d kubernetes clusters", len(filtered.KubernetesClusters))
	}

	scope = PatrolScope{ResourceTypes: []string{"pbs_datastore"}, ResourceIDs: []string{"pbs1:ds1"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.PBSInstances) != 0 {
		t.Errorf("expected non-canonical 'pbs_datastore' alias to be rejected, got %d pbs instances", len(filtered.PBSInstances))
	}

	scope = PatrolScope{ResourceTypes: []string{"agent_raid"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Hosts) != 0 {
		t.Errorf("expected non-canonical 'agent_raid' alias to be rejected, got %d hosts", len(filtered.Hosts))
	}

	scope = PatrolScope{ResourceTypes: []string{"pmg"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.PMGInstances) != 1 {
		t.Errorf("expected canonical 'pmg' to match PMG instances, got %d", len(filtered.PMGInstances))
	}

	scope = PatrolScope{ResourceTypes: []string{"physical_disk"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.PhysicalDisks) != 1 {
		t.Errorf("expected canonical 'physical_disk' to match physical disks, got %d", len(filtered.PhysicalDisks))
	}
}

func TestFilterStateByScope_AppContainerAlias(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		DockerHosts: []models.DockerHost{
			{
				ID:       "dh1",
				Hostname: "docker-host-1",
				Containers: []models.DockerContainer{
					{ID: "c1", Name: "web"},
				},
			},
		},
	}

	// Canonical semantic name should match Docker resources.
	scope := PatrolScope{ResourceTypes: []string{"app-container"}}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.DockerHosts) != 1 {
		t.Errorf("expected 'app-container' to match Docker hosts, got %d", len(filtered.DockerHosts))
	}

	// Underscore alias should be rejected.
	scope = PatrolScope{ResourceTypes: []string{"app_container"}}
	filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.DockerHosts) != 0 {
		t.Errorf("expected legacy 'app_container' alias to be rejected, got %d docker hosts", len(filtered.DockerHosts))
	}
}

func TestFilterStateByScope_DockerHost(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		DockerHosts: []models.DockerHost{
			{
				ID:       "dh1",
				Hostname: "docker-host-1",
				Containers: []models.DockerContainer{
					{
						ID:       "c1",
						Name:     "web",
						Image:    "example/web:latest",
						Networks: []models.DockerContainerNetworkLink{{Name: "frontend"}},
						Mounts:   []models.DockerContainerMount{{Type: "volume", Name: "web-data"}},
					},
					{ID: "c2", Name: "db"},
				},
				Images:   []models.DockerImage{{ID: "sha256:web", RepoTags: []string{"example/web:latest"}}, {ID: "sha256:db", RepoTags: []string{"example/db:latest"}}},
				Volumes:  []models.DockerVolume{{Name: "web-data"}, {Name: "db-data"}},
				Networks: []models.DockerNetwork{{ID: "network-frontend", Name: "frontend"}, {ID: "network-backend", Name: "backend"}},
				Services: []models.DockerService{{ID: "service-web", Name: "web"}},
				Tasks:    []models.DockerTask{{ID: "task-web", ContainerID: "c1"}},
				Nodes:    []models.DockerNode{{ID: "node-1", Hostname: "docker-host-1"}},
				Secrets:  []models.DockerSecret{{ID: "secret-web", Name: "web-secret"}},
				Configs:  []models.DockerConfig{{ID: "config-web", Name: "web-config"}},
				StorageUsage: &models.DockerStorageUsage{
					Images: models.DockerStorageUsageBucket{TotalCount: 2},
				},
				Swarm: &models.DockerSwarmInfo{LocalState: "active"},
			},
		},
	}

	// Match by host ID
	scope := PatrolScope{
		ResourceIDs:   []string{"dh1"},
		ResourceTypes: []string{"docker-host"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.DockerHosts) != 1 {
		t.Fatalf("expected 1 docker host, got %d", len(filtered.DockerHosts))
	}
	// When the host itself matches, all containers should be included
	if len(filtered.DockerHosts[0].Containers) != 2 {
		t.Errorf("expected all 2 containers when host matches, got %d", len(filtered.DockerHosts[0].Containers))
	}
}

func TestFilterStateByScope_DockerContainerOnly(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		DockerHosts: []models.DockerHost{
			{
				ID:       "dh1",
				Hostname: "docker-host-1",
				Containers: []models.DockerContainer{
					{ID: "c1", Name: "web"},
					{ID: "c2", Name: "db"},
				},
			},
		},
	}

	// Match by container ID only
	scope := PatrolScope{
		ResourceIDs:   []string{"c1"},
		ResourceTypes: []string{"app-container"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.DockerHosts) != 1 {
		t.Fatalf("expected 1 docker host, got %d", len(filtered.DockerHosts))
	}
	if len(filtered.DockerHosts[0].Containers) != 1 {
		t.Fatalf("expected 1 container, got %d", len(filtered.DockerHosts[0].Containers))
	}
	if filtered.DockerHosts[0].Containers[0].ID != "c1" {
		t.Errorf("expected container c1, got %s", filtered.DockerHosts[0].Containers[0].ID)
	}
	filteredHost := filtered.DockerHosts[0]
	if len(filteredHost.Images) != 0 || len(filteredHost.Volumes) != 0 || len(filteredHost.Networks) != 0 ||
		len(filteredHost.Services) != 0 || len(filteredHost.Tasks) != 0 || len(filteredHost.Nodes) != 0 ||
		len(filteredHost.Secrets) != 0 || len(filteredHost.Configs) != 0 || filteredHost.StorageUsage != nil || filteredHost.Swarm != nil {
		t.Fatalf("container scope retained unrelated host inventory: %+v", filteredHost)
	}
	for _, resourceType := range []unifiedresources.ResourceType{
		unifiedresources.ResourceTypeDockerImage,
		unifiedresources.ResourceTypeDockerVolume,
		unifiedresources.ResourceTypeDockerNetwork,
		unifiedresources.ResourceTypeDockerService,
		unifiedresources.ResourceTypeDockerTask,
		unifiedresources.ResourceTypeDockerSwarmNode,
		unifiedresources.ResourceTypeDockerSecret,
		unifiedresources.ResourceTypeDockerConfig,
	} {
		if got := len(filtered.unifiedResourceProvider.GetByType(resourceType)); got != 0 {
			t.Fatalf("container scope retained %d unrelated %s resources", got, resourceType)
		}
	}
}

func TestFilterStateByScope_Storage(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Storage: []models.Storage{
			{ID: "s1", Name: "local"},
			{ID: "s2", Name: "ceph"},
		},
	}

	scope := PatrolScope{
		ResourceIDs:   []string{"s1"},
		ResourceTypes: []string{"storage"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Storage) != 1 {
		t.Fatalf("expected 1 storage, got %d", len(filtered.Storage))
	}
	if filtered.Storage[0].ID != "s1" {
		t.Errorf("expected storage s1, got %s", filtered.Storage[0].ID)
	}
}

func TestFilterStateByScope_Hosts(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Hosts: []models.Host{
			{ID: "h1", Hostname: "host-1"},
			{ID: "h2", Hostname: "host-2"},
		},
	}

	scope := PatrolScope{
		ResourceIDs:   []string{"h1"},
		ResourceTypes: []string{"agent"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Hosts) != 1 {
		t.Fatalf("expected 1 host, got %d", len(filtered.Hosts))
	}
	if filtered.Hosts[0].ID != "h1" {
		t.Errorf("expected host h1, got %s", filtered.Hosts[0].ID)
	}
}

func TestFilterStateByScope_PBS(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		PBSInstances: []models.PBSInstance{
			{
				ID:   "pbs1",
				Name: "pbs-main",
				Datastores: []models.PBSDatastore{
					{Name: "ds1"},
				},
			},
			{
				ID:   "pbs2",
				Name: "pbs-secondary",
			},
		},
	}

	scope := PatrolScope{
		ResourceIDs:   []string{"pbs1"},
		ResourceTypes: []string{"pbs"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.PBSInstances) != 1 {
		t.Fatalf("expected 1 PBS instance, got %d", len(filtered.PBSInstances))
	}
	if filtered.PBSInstances[0].ID != "pbs1" {
		t.Errorf("expected PBS pbs1, got %s", filtered.PBSInstances[0].ID)
	}
}

func TestFilterStateByScope_Kubernetes(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		KubernetesClusters: []models.KubernetesCluster{
			{ID: "k1", Name: "cluster-1"},
			{ID: "k2", Name: "cluster-2"},
		},
	}

	scope := PatrolScope{
		ResourceIDs:   []string{"k1"},
		ResourceTypes: []string{"k8s-cluster"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.KubernetesClusters) != 1 {
		t.Fatalf("expected 1 k8s cluster, got %d", len(filtered.KubernetesClusters))
	}
	if filtered.KubernetesClusters[0].ID != "k1" {
		t.Errorf("expected cluster k1, got %s", filtered.KubernetesClusters[0].ID)
	}
}

func TestFilterStateByScope_PreservesScopedMetadataOnly(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	now := time.Now()
	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "node-1", Name: "node-1"},
		},
		VMs: []models.VM{
			{ID: "vm-101", VMID: 101, Name: "vm-101"},
		},
		LastUpdate: now,
		ConnectionHealth: map[string]bool{
			"node-1": true,
			"node-2": false,
		},
		ActiveAlerts: []models.Alert{
			{ID: "a1", ResourceID: "node-1", Message: "scoped"},
			{ID: "a2", ResourceID: "node-2", Message: "global"},
		},
		RecentlyResolved: []models.ResolvedAlert{
			{Alert: models.Alert{ID: "r1", ResourceID: "node-1", Message: "resolved scoped"}},
			{Alert: models.Alert{ID: "r2", ResourceID: "node-2", Message: "resolved global"}},
		},
		PVEBackups: models.PVEBackups{
			BackupTasks: []models.BackupTask{
				{ID: "bt-101", VMID: 101},
				{ID: "bt-202", VMID: 202},
			},
			StorageBackups: []models.StorageBackup{
				{ID: "sb-101", VMID: 101},
				{ID: "sb-202", VMID: 202},
			},
			GuestSnapshots: []models.GuestSnapshot{
				{ID: "gs-101", VMID: 101},
				{ID: "gs-202", VMID: 202},
			},
		},
		PBSBackups: []models.PBSBackup{
			{ID: "pb-101", VMID: "101"},
			{ID: "pb-202", VMID: "202"},
		},
	}
	scope := PatrolScope{ResourceTypes: []string{"node", "vm"}}

	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)

	if len(filtered.ConnectionHealth) != 1 {
		t.Fatalf("expected only scoped ConnectionHealth entries, got %d", len(filtered.ConnectionHealth))
	}
	if !filtered.ConnectionHealth["node-1"] {
		t.Fatal("expected node-1 connection health to be preserved")
	}
	if len(filtered.ActiveAlerts) != 1 || filtered.ActiveAlerts[0].ResourceID != "node-1" {
		t.Fatalf("expected only scoped active alerts, got %+v", filtered.ActiveAlerts)
	}
	if len(filtered.RecentlyResolved) != 1 || filtered.RecentlyResolved[0].ResourceID != "node-1" {
		t.Fatalf("expected only scoped resolved alerts, got %+v", filtered.RecentlyResolved)
	}
	if len(filtered.PVEBackups.BackupTasks) != 1 || filtered.PVEBackups.BackupTasks[0].VMID != 101 {
		t.Fatalf("expected only scoped PVE backup tasks, got %+v", filtered.PVEBackups.BackupTasks)
	}
	if len(filtered.PVEBackups.StorageBackups) != 1 || filtered.PVEBackups.StorageBackups[0].VMID != 101 {
		t.Fatalf("expected only scoped storage backups, got %+v", filtered.PVEBackups.StorageBackups)
	}
	if len(filtered.PVEBackups.GuestSnapshots) != 1 || filtered.PVEBackups.GuestSnapshots[0].VMID != 101 {
		t.Fatalf("expected only scoped guest snapshots, got %+v", filtered.PVEBackups.GuestSnapshots)
	}
	if len(filtered.PBSBackups) != 1 || filtered.PBSBackups[0].VMID != "101" {
		t.Fatalf("expected only scoped PBS backups, got %+v", filtered.PBSBackups)
	}
}

func TestFilterStateByScope_RebuildsScopedProviders(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	state := patrolRuntimeState{
		Nodes: []models.Node{
			{ID: "n1", Name: "node-1"},
			{ID: "n2", Name: "node-2"},
		},
		VMs: []models.VM{
			{ID: "vm1", Name: "vm-1"},
			{ID: "vm2", Name: "vm-2"},
		},
		Storage: []models.Storage{
			{ID: "s1", Name: "local", Usage: 42},
			{ID: "s2", Name: "backup", Usage: 10},
		},
		PhysicalDisks: []models.PhysicalDisk{
			{ID: "disk-1", DevPath: "/dev/sda", Model: "sda"},
			{ID: "disk-2", DevPath: "/dev/sdb", Model: "sdb"},
		},
		PMGInstances: []models.PMGInstance{
			{ID: "pmg1", Name: "pmg-main", Host: "pmg.local"},
			{ID: "pmg2", Name: "pmg-edge", Host: "pmg2.local"},
		},
	}

	filtered := ps.filterStateByScopeState(state, PatrolScope{
		ResourceIDs:   []string{"n1", "s1"},
		ResourceTypes: []string{"node", "storage"},
	})

	if filtered.readState == nil {
		t.Fatal("expected scoped readState to be rebuilt")
	}
	if filtered.unifiedResourceProvider == nil {
		t.Fatal("expected scoped unified resource provider to be rebuilt")
	}
	if got := len(filtered.readState.Nodes()); got != 1 {
		t.Fatalf("expected 1 scoped node in readState, got %d", got)
	}
	if got := len(filtered.readState.VMs()); got != 0 {
		t.Fatalf("expected 0 scoped VMs in readState, got %d", got)
	}
	if got := len(filtered.readState.StoragePools()); got != 1 {
		t.Fatalf("expected 1 scoped storage pool in readState, got %d", got)
	}
	if got := len(filtered.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypeStorage)); got != 1 {
		t.Fatalf("expected 1 scoped storage resource in provider, got %d", got)
	}
	if got := len(filtered.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypePhysicalDisk)); got != 0 {
		t.Fatalf("expected no scoped physical disk resources in provider, got %d", got)
	}

	filteredPMG := ps.filterStateByScopeState(state, PatrolScope{
		ResourceIDs:   []string{"pmg.local"},
		ResourceTypes: []string{"pmg"},
	})
	if got := len(filteredPMG.PMGInstances); got != 1 {
		t.Fatalf("expected 1 scoped PMG instance, got %d", got)
	}
	if filteredPMG.PMGInstances[0].ID != "pmg1" {
		t.Fatalf("expected scoped PMG instance pmg1, got %s", filteredPMG.PMGInstances[0].ID)
	}
	if filteredPMG.readState == nil {
		t.Fatal("expected scoped PMG readState to be rebuilt")
	}
	if got := len(filteredPMG.readState.PMGInstances()); got != 1 {
		t.Fatalf("expected 1 scoped PMG instance in readState, got %d", got)
	}
	if got := len(filteredPMG.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypePMG)); got != 1 {
		t.Fatalf("expected 1 scoped PMG resource in provider, got %d", got)
	}

	filteredDisks := ps.filterStateByScopeState(state, PatrolScope{
		ResourceIDs:   []string{"/dev/sda"},
		ResourceTypes: []string{"physical_disk"},
	})
	if got := len(filteredDisks.PhysicalDisks); got != 1 {
		t.Fatalf("expected 1 scoped physical disk, got %d", got)
	}
	if filteredDisks.PhysicalDisks[0].ID != "disk-1" {
		t.Fatalf("expected scoped physical disk disk-1, got %s", filteredDisks.PhysicalDisks[0].ID)
	}
	if got := len(filteredDisks.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypePhysicalDisk)); got != 1 {
		t.Fatalf("expected 1 scoped physical disk resource in provider, got %d", got)
	}

	counts := patrolRuntimeCountResources(filteredDisks)
	if counts.storage != 1 {
		t.Fatalf("expected scoped runtime counts to include 1 storage resource, got %#v", counts)
	}
	if counts.total() != 1 {
		t.Fatalf("expected scoped runtime total to be 1, got %#v", counts)
	}
}

func TestPatrolRuntimeResourceHelpers_PrepareCanonicalInventory(t *testing.T) {
	nodeView := unifiedresources.NewNodeView(&unifiedresources.Resource{
		ID:   "node-1",
		Name: "node-a",
		Type: unifiedresources.ResourceTypeAgent,
	})
	vmView := newTestVMView("qemu/100", "vm-a", 100, "pve1", "", unifiedresources.StatusOnline, false, nil)
	ctView := newTestContainerView("lxc/200", "ct-a", 200, "pve1", "", unifiedresources.StatusOnline, false, nil)
	storageView := unifiedresources.NewStoragePoolView(&unifiedresources.Resource{
		ID:   "storage-1",
		Name: "local-zfs",
		Type: unifiedresources.ResourceTypeStorage,
	})
	dockerHostView := unifiedresources.NewDockerHostView(&unifiedresources.Resource{
		ID:   "docker-1",
		Name: "docker-a",
		Type: unifiedresources.ResourceTypeAgent,
		Docker: &unifiedresources.DockerData{
			Hostname: "docker.local",
		},
	})
	dockerCtrView := unifiedresources.NewDockerContainerView(&unifiedresources.Resource{
		ID:   "ctr-1",
		Name: "web",
		Type: unifiedresources.ResourceTypeAppContainer,
	})
	hostView := unifiedresources.NewHostView(&unifiedresources.Resource{
		ID:   "agent-1",
		Name: "agent-a",
		Type: unifiedresources.ResourceTypeAgent,
		Agent: &unifiedresources.AgentData{
			Hostname: "agent.local",
		},
	})
	trueNASHostView := unifiedresources.NewHostView(&unifiedresources.Resource{
		ID:   "truenas-1",
		Name: "truenas-main",
		Type: unifiedresources.ResourceTypeAgent,
		Agent: &unifiedresources.AgentData{
			Hostname: "truenas-main.local",
			Platform: "truenas",
		},
	})
	pbsView := unifiedresources.NewPBSInstanceView(&unifiedresources.Resource{
		ID:   "pbs-1",
		Name: "pbs-a",
		Type: unifiedresources.ResourceTypePBS,
	})
	pmgView := unifiedresources.NewPMGInstanceView(&unifiedresources.Resource{
		ID:   "pmg-1",
		Name: "pmg-a",
		Type: unifiedresources.ResourceTypePMG,
	})
	k8sView := unifiedresources.NewK8sClusterView(&unifiedresources.Resource{
		ID:   "k8s-1",
		Name: "cluster-a",
		Type: unifiedresources.ResourceTypeK8sCluster,
	})

	state := patrolRuntimeState{
		readState: &mockReadState{
			nodes:       []*unifiedresources.NodeView{&nodeView},
			vms:         []*unifiedresources.VMView{vmView},
			containers:  []*unifiedresources.ContainerView{ctView},
			hosts:       []*unifiedresources.HostView{&hostView, &trueNASHostView},
			dockerHosts: []*unifiedresources.DockerHostView{&dockerHostView},
			dockerCtrs:  []*unifiedresources.DockerContainerView{&dockerCtrView},
			storage:     []*unifiedresources.StoragePoolView{&storageView},
			pbs:         []*unifiedresources.PBSInstanceView{&pbsView},
			pmg:         []*unifiedresources.PMGInstanceView{&pmgView},
			k8sClusters: []*unifiedresources.K8sClusterView{&k8sView},
		},
		DockerHosts: []models.DockerHost{
			{
				ID:                "docker-1",
				CustomDisplayName: "docker-custom",
				Hostname:          "docker.local",
				Containers: []models.DockerContainer{
					{ID: "ctr-1", Name: "web-snapshot"},
				},
			},
		},
		Hosts: []models.Host{
			{ID: "agent-1", DisplayName: "agent-display", Hostname: "agent.local"},
			{ID: "truenas-1", DisplayName: "truenas-display", Hostname: "truenas-main.local", Platform: "truenas"},
		},
		KubernetesClusters: []models.KubernetesCluster{
			{ID: "k8s-1", CustomDisplayName: "cluster-custom", Name: "cluster-a"},
		},
		unifiedResourceProvider: &mockUnifiedResourceProvider{
			getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource {
				if t != unifiedresources.ResourceTypePhysicalDisk {
					return nil
				}
				return []unifiedresources.Resource{
					{
						ID:   "disk-1",
						Name: "disk-a",
						Type: unifiedresources.ResourceTypePhysicalDisk,
						PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
							DevPath: "/dev/sda",
							Model:   "Samsung SSD",
						},
					},
				}
			},
		},
	}

	counts := patrolRuntimeCountResources(state)
	if counts.nodes != 1 || counts.guests != 2 || counts.storage != 2 || counts.docker != 1 || counts.hosts != 1 || counts.truenas != 1 || counts.pbs != 1 || counts.pmg != 1 || counts.kubernetes != 1 {
		t.Fatalf("unexpected canonical runtime counts: %#v", counts)
	}
	if counts.total() != 11 {
		t.Fatalf("expected canonical runtime total 11, got %#v", counts)
	}

	resourceIDs := patrolRuntimeSortedResourceIDs(state)
	for _, want := range []string{"node-1", "qemu/100", "lxc/200", "storage-1", "docker-1", "ctr-1", "agent-1", "truenas-1", "pbs-1", "pmg-1", "k8s-1", "disk-1", "/dev/sda"} {
		found := false
		for _, got := range resourceIDs {
			if got == want {
				found = true
				break
			}
		}
		if !found {
			t.Fatalf("expected runtime resource IDs to include %q, got %v", want, resourceIDs)
		}
	}

	known := patrolRuntimeKnownResources(state)
	for _, want := range []string{"node-a", "vm-a", "ct-a", "local-zfs", "docker.local", "docker-custom", "web", "web-snapshot", "agent.local", "agent-display", "truenas-main", "truenas-main.local", "truenas-display", "cluster-a", "cluster-custom", "Samsung SSD"} {
		if !known[want] {
			t.Fatalf("expected known runtime resources to include %q", want)
		}
	}
}

func TestPatrolUpdateSafetyDockerHosts_UsesReadStateDockerHosts(t *testing.T) {
	readStateForDigest := func(digest string) unifiedresources.ReadState {
		registry := unifiedresources.NewRegistry(nil)
		registry.IngestSnapshot(models.StateSnapshot{
			DockerHosts: []models.DockerHost{
				{
					ID:       "canonical-host",
					Hostname: "canonical.local",
					Containers: []models.DockerContainer{
						{ID: "app-1", Name: "web", ImageDigest: digest},
					},
				},
			},
		})
		return registry
	}

	legacyUnchangedHost := []models.DockerHost{
		{
			ID:       "legacy-host",
			Hostname: "legacy.local",
			Containers: []models.DockerContainer{
				{ID: "legacy-app", Name: "legacy-web", ImageDigest: "sha256:aaa"},
			},
		},
	}

	watcher := newUpdateSafetyWatcher()
	state := patrolRuntimeState{
		readState:   readStateForDigest("sha256:aaa"),
		DockerHosts: legacyUnchangedHost,
	}
	if emit, resolve := watcher.Observe(patrolUpdateSafetyDockerHosts(state), time.Unix(1, 0)); len(emit) != 0 || len(resolve) != 0 {
		t.Fatalf("expected first read-state observation to establish baseline, got emit=%d resolve=%d", len(emit), len(resolve))
	}

	state.readState = readStateForDigest("sha256:bbb")
	emit, resolve := watcher.Observe(patrolUpdateSafetyDockerHosts(state), time.Unix(2, 0))
	if len(resolve) != 0 {
		t.Fatalf("expected no resolve on digest change, got %d", len(resolve))
	}
	if len(emit) != 1 {
		t.Fatalf("expected read-state digest change to emit one finding, got %d", len(emit))
	}
	if emit[0].Key != UpdateSafetyFindingPrefix+":canonical-host/app-1" {
		t.Fatalf("expected canonical read-state host/container key, got %q", emit[0].Key)
	}
	if emit[0].Node != "canonical.local" {
		t.Fatalf("expected canonical read-state host name, got %q", emit[0].Node)
	}
}

func TestPatrolRuntimeState_WithDerivedProviders_UsesResourceInventoryOnly(t *testing.T) {
	state := patrolRuntimeState{
		Nodes: []models.Node{
			{ID: "node-1", Name: "node-a"},
		},
		ActiveAlerts: []models.Alert{
			{ResourceID: "node-1"},
		},
		ConnectionHealth: map[string]bool{
			"node-1": true,
		},
		PVEBackups: models.PVEBackups{
			BackupTasks: []models.BackupTask{{ID: "bt-1", VMID: 101}},
		},
		PBSBackups: []models.PBSBackup{{ID: "pb-1", VMID: "101"}},
	}

	derived := state.withDerivedProviders()
	if derived.readState == nil || derived.unifiedResourceProvider == nil {
		t.Fatal("expected derived providers to be rebuilt")
	}
	if got := len(derived.readState.Nodes()); got != 1 {
		t.Fatalf("expected 1 node in derived readState, got %d", got)
	}
	if got := len(derived.readState.VMs()); got != 0 {
		t.Fatalf("expected no VM inventory from metadata-only fields, got %d", got)
	}
	if len(derived.ActiveAlerts) != 1 || derived.ActiveAlerts[0].ResourceID != "node-1" {
		t.Fatalf("expected active alerts to remain on patrol runtime state, got %+v", derived.ActiveAlerts)
	}
	if !derived.ConnectionHealth["node-1"] {
		t.Fatalf("expected connection health to remain on patrol runtime state, got %+v", derived.ConnectionHealth)
	}
	if len(derived.PVEBackups.BackupTasks) != 1 || len(derived.PBSBackups) != 1 {
		t.Fatalf("expected backup metadata to remain on patrol runtime state, got %+v / %+v", derived.PVEBackups.BackupTasks, derived.PBSBackups)
	}
	snapshot := state.resourceSnapshot()
	if snapshot.ConnectionHealth == nil {
		t.Fatal("expected derived resource snapshot to preserve normalized connection health map")
	}
	if snapshot.ActiveAlerts == nil {
		t.Fatal("expected derived resource snapshot to preserve normalized active alerts slice")
	}
	if snapshot.RecentlyResolved == nil {
		t.Fatal("expected derived resource snapshot to preserve normalized recently resolved slice")
	}
}

func TestNewPatrolRuntimeState_NormalizesCollections(t *testing.T) {
	state := newPatrolRuntimeState(models.StateSnapshot{})

	if state.Nodes == nil {
		t.Fatal("expected nodes slice to be normalized")
	}
	if state.ConnectionHealth == nil {
		t.Fatal("expected connection health map to be normalized")
	}
	if state.ActiveAlerts == nil {
		t.Fatal("expected active alerts slice to be normalized")
	}
	if state.RecentlyResolved == nil {
		t.Fatal("expected recently resolved slice to be normalized")
	}
}

func TestPatrolService_CurrentPatrolRuntimeState_UsesNormalizedEmptyState(t *testing.T) {
	service := &PatrolService{}

	state := service.currentPatrolRuntimeState()
	if state.Nodes == nil {
		t.Fatal("expected nodes slice to be normalized")
	}
	if state.ConnectionHealth == nil {
		t.Fatal("expected connection health map to be normalized")
	}
}

func TestFilterStateByScope_WhitespaceInIDs(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
	}

	// IDs with whitespace should be trimmed
	scope := PatrolScope{
		ResourceIDs:   []string{"  n1  "},
		ResourceTypes: []string{"node"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Nodes) != 1 {
		t.Errorf("expected whitespace-trimmed ID to match, got %d nodes", len(filtered.Nodes))
	}
}

func TestFilterStateByScope_EmptyIDsIgnored(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
	}

	scope := PatrolScope{
		ResourceIDs:   []string{"", "  ", "n1"},
		ResourceTypes: []string{"node"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Nodes) != 1 {
		t.Errorf("expected empty IDs to be ignored, got %d nodes", len(filtered.Nodes))
	}
}

func TestFilterStateByScope_CaseInsensitiveTypes(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
	}

	scope := PatrolScope{
		ResourceTypes: []string{"NODE"},
	}
	filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
	if len(filtered.Nodes) != 1 {
		t.Errorf("expected case-insensitive type matching, got %d nodes", len(filtered.Nodes))
	}
}

// --- tryStartRun / endRun tests ---

func TestTryStartRun_Basic(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	if !ps.tryStartRun("full") {
		t.Error("expected first tryStartRun to succeed")
	}
	if ps.tryStartRun("full") {
		t.Error("expected second tryStartRun to fail (run in progress)")
	}

	ps.endRun()

	if !ps.tryStartRun("full") {
		t.Error("expected tryStartRun to succeed after endRun")
	}
	ps.endRun()
}

func TestTryStartRun_Concurrent(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	var wg sync.WaitGroup
	var successes int32
	var mu sync.Mutex

	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			if ps.tryStartRun("scoped") {
				mu.Lock()
				successes++
				mu.Unlock()
				// Simulate some work
				time.Sleep(1 * time.Millisecond)
				ps.endRun()
			}
		}()
	}
	wg.Wait()

	if successes == 0 {
		t.Error("expected at least one goroutine to acquire the run")
	}
	// After all goroutines complete, run should not be in progress
	ps.mu.RLock()
	inProgress := ps.runInProgress
	ps.mu.RUnlock()
	if inProgress {
		t.Error("expected runInProgress to be false after all goroutines complete")
	}
}

// --- Subscribe / Unsubscribe ---

func TestSubscribeUnsubscribe(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	ch1 := ps.SubscribeToStream()
	ch2 := ps.SubscribeToStream()

	ps.streamMu.RLock()
	count := len(ps.streamSubscribers)
	ps.streamMu.RUnlock()

	if count != 2 {
		t.Errorf("expected 2 subscribers, got %d", count)
	}

	ps.UnsubscribeFromStream(ch1)

	ps.streamMu.RLock()
	count = len(ps.streamSubscribers)
	ps.streamMu.RUnlock()

	if count != 1 {
		t.Errorf("expected 1 subscriber after unsubscribe, got %d", count)
	}

	ps.UnsubscribeFromStream(ch2)

	ps.streamMu.RLock()
	count = len(ps.streamSubscribers)
	ps.streamMu.RUnlock()

	if count != 0 {
		t.Errorf("expected 0 subscribers after all unsubscribe, got %d", count)
	}
}

func TestSubscribeToStream_ReceivesCurrentState(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	// Set active streaming state
	ps.streamMu.Lock()
	ps.streamPhase = "analyzing"
	ps.currentOutput.WriteString("some output")
	ps.streamMu.Unlock()

	ch := ps.SubscribeToStream()
	defer ps.UnsubscribeFromStream(ch)

	// New subscriber should receive a snapshot of the current state.
	select {
	case event := <-ch:
		if event.Type != "snapshot" {
			t.Errorf("expected snapshot event first, got %s", event.Type)
		}
		if event.Phase != "analyzing" {
			t.Errorf("expected phase 'analyzing', got %q", event.Phase)
		}
		if event.Content != "some output" {
			t.Errorf("expected 'some output', got %q", event.Content)
		}
	case <-time.After(100 * time.Millisecond):
		t.Error("expected to receive snapshot event on subscribe")
	}
}

// --- broadcast ---

func TestBroadcast_MultipleSubscribers(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	ch1 := ps.SubscribeToStream()
	ch2 := ps.SubscribeToStream()
	ch3 := ps.SubscribeToStream()

	event := PatrolStreamEvent{Type: "test", Content: "hello"}
	ps.broadcast(event)

	for i, ch := range []chan PatrolStreamEvent{ch1, ch2, ch3} {
		select {
		case received := <-ch:
			if received.Content != "hello" {
				t.Errorf("subscriber %d: expected 'hello', got %q", i, received.Content)
			}
		case <-time.After(100 * time.Millisecond):
			t.Errorf("subscriber %d: timed out waiting for event", i)
		}
	}

	ps.UnsubscribeFromStream(ch1)
	ps.UnsubscribeFromStream(ch2)
	ps.UnsubscribeFromStream(ch3)
}

func TestBroadcast_StaleSubscriberRemoved(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	// Create a subscriber with a full buffer
	ch := ps.SubscribeToStream()

	// Fill the buffer (100 capacity)
	for i := 0; i < 100; i++ {
		ps.broadcast(PatrolStreamEvent{Type: "fill", Content: "x"})
	}

	// Keep broadcasting without reading; subscriber should eventually be dropped
	// after repeated full-buffer backpressure.
	for i := 0; i < 30; i++ {
		ps.broadcast(PatrolStreamEvent{Type: "overflow"})
	}

	// Give goroutine time to close the channel
	time.Sleep(50 * time.Millisecond)

	ps.streamMu.RLock()
	_, exists := ps.streamSubscribers[ch]
	ps.streamMu.RUnlock()

	if exists {
		t.Error("expected stale subscriber to be removed")
	}
}

func TestSubscribeToStreamFrom_ReplaysBufferedEvents(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.resetStreamForRun("run-1")

	// No subscribers: broadcast should still buffer events for replay.
	ps.broadcast(PatrolStreamEvent{Type: "phase", Phase: "analyzing"})
	ps.broadcast(PatrolStreamEvent{Type: "content", Content: "hello"})
	ps.broadcast(PatrolStreamEvent{Type: "tool_start", ToolName: "uptime"})

	ps.streamMu.RLock()
	lastSeq := ps.streamSeq
	ps.streamMu.RUnlock()
	if lastSeq < 3 {
		t.Fatalf("expected seq to be >= 3, got %d", lastSeq)
	}

	// Subscribe from seq 1; should replay seq 2 and 3.
	ch := ps.SubscribeToStreamFrom(1)
	defer ps.UnsubscribeFromStream(ch)

	// First replayed event: content
	select {
	case ev := <-ch:
		if ev.Type != "content" {
			t.Fatalf("expected content replay, got %s", ev.Type)
		}
		if ev.Content != "hello" {
			t.Fatalf("expected hello content, got %q", ev.Content)
		}
		if ev.Seq <= 1 {
			t.Fatalf("expected seq > 1, got %d", ev.Seq)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("timed out waiting for replayed content event")
	}

	// Second replayed event: tool_start
	select {
	case ev := <-ch:
		if ev.Type != "tool_start" {
			t.Fatalf("expected tool_start replay, got %s", ev.Type)
		}
		if ev.ToolName != "uptime" {
			t.Fatalf("expected tool uptime, got %q", ev.ToolName)
		}
		if ev.Seq <= 1 {
			t.Fatalf("expected seq > 1, got %d", ev.Seq)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("timed out waiting for replayed tool_start event")
	}
}

func TestSubscribeToStreamFrom_BufferRotatedSnapshot(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.resetStreamForRun("run-1")
	ps.setStreamPhase("analyzing")

	// Fill beyond replay buffer to force rotation.
	for i := 0; i < 260; i++ {
		ps.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
	}

	ps.streamMu.RLock()
	start, end := ps.streamBufferWindowLocked()
	ps.streamMu.RUnlock()
	if start == 0 || end == 0 || end <= start {
		t.Fatalf("expected non-empty buffer window, got start=%d end=%d", start, end)
	}

	// Ask to resume from a seq that is behind the buffered window.
	ch := ps.SubscribeToStreamFrom(start - 1)
	defer ps.UnsubscribeFromStream(ch)

	select {
	case ev := <-ch:
		if ev.Type != "snapshot" {
			t.Fatalf("expected snapshot, got %s", ev.Type)
		}
		if ev.ResyncReason != "buffer_rotated" {
			t.Fatalf("expected resync_reason buffer_rotated, got %q", ev.ResyncReason)
		}
		if ev.BufferStart == 0 || ev.BufferEnd == 0 || ev.BufferEnd < ev.BufferStart {
			t.Fatalf("expected buffer window in snapshot, got start=%d end=%d", ev.BufferStart, ev.BufferEnd)
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("timed out waiting for buffer_rotated snapshot")
	}
}

func TestSubscribeToStreamFrom_BufferRotatedSnapshotNotDuplicated(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.resetStreamForRun("run-1")
	ps.setStreamPhase("analyzing")

	for i := 0; i < 260; i++ {
		ps.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
	}

	ps.streamMu.RLock()
	start, _ := ps.streamBufferWindowLocked()
	ps.streamMu.RUnlock()

	ch := ps.SubscribeToStreamFrom(start - 1)
	defer ps.UnsubscribeFromStream(ch)

	snapshotCount := 0
	read := 0
	for read < 25 {
		select {
		case ev := <-ch:
			read++
			if ev.Type == "snapshot" {
				snapshotCount++
			}
		case <-time.After(100 * time.Millisecond):
			read = 25
		}
	}

	if snapshotCount != 1 {
		t.Fatalf("expected exactly 1 snapshot for buffer_rotated resume, got %d", snapshotCount)
	}
}

func TestSubscribeToStreamFrom_ChurnDoesNotLeakSubscribers(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	const workers = 12
	const loopsPerWorker = 60

	var wg sync.WaitGroup
	for w := 0; w < workers; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := 0; i < loopsPerWorker; i++ {
				ch := ps.SubscribeToStreamFrom(0)
				ps.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
				ps.UnsubscribeFromStream(ch)
			}
		}()
	}
	wg.Wait()

	// Allow concurrent close/remove paths to settle.
	time.Sleep(25 * time.Millisecond)

	ps.streamMu.RLock()
	subs := len(ps.streamSubscribers)
	ps.streamMu.RUnlock()
	if subs != 0 {
		t.Fatalf("expected no leaked subscribers after churn, got %d", subs)
	}

	// Sanity: broadcasting after heavy churn should still work.
	ps.broadcast(PatrolStreamEvent{Type: "content", Content: "ok"})
}

func TestSubscribeToStreamFrom_RecordsReplayMetrics(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.resetStreamForRun("run-metrics-replay")
	ps.setStreamPhase("analyzing")

	ps.broadcast(PatrolStreamEvent{Type: "phase", Phase: "analyzing"})
	ps.broadcast(PatrolStreamEvent{Type: "content", Content: "hello"})
	ps.broadcast(PatrolStreamEvent{Type: "tool_start", ToolName: "uptime"})

	m := GetPatrolMetrics()
	beforeOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("replay"))
	beforeEvents := testutil.ToFloat64(m.streamReplayEvents)

	ch := ps.SubscribeToStreamFrom(1)
	defer ps.UnsubscribeFromStream(ch)

	// Drain replayed events to avoid backpressure affecting subsequent tests.
	<-ch
	<-ch

	afterOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("replay"))
	afterEvents := testutil.ToFloat64(m.streamReplayEvents)
	if afterOutcome <= beforeOutcome {
		t.Fatalf("expected replay outcome counter to increase (before=%f after=%f)", beforeOutcome, afterOutcome)
	}
	if afterEvents < beforeEvents+2 {
		t.Fatalf("expected replay event counter to increase by at least 2 (before=%f after=%f)", beforeEvents, afterEvents)
	}
}

func TestSubscribeToStreamFrom_RecordsSnapshotAndMissMetrics(t *testing.T) {
	// Snapshot path (buffer rotated)
	psSnap := NewPatrolService(nil, nil)
	psSnap.resetStreamForRun("run-metrics-snapshot")
	psSnap.setStreamPhase("analyzing")
	for i := 0; i < 260; i++ {
		psSnap.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
	}
	psSnap.streamMu.RLock()
	start, _ := psSnap.streamBufferWindowLocked()
	psSnap.streamMu.RUnlock()

	m := GetPatrolMetrics()
	beforeSnapshotOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("snapshot"))
	beforeReason := testutil.ToFloat64(m.streamResyncReason.WithLabelValues("buffer_rotated"))

	chSnap := psSnap.SubscribeToStreamFrom(start - 1)
	defer psSnap.UnsubscribeFromStream(chSnap)
	<-chSnap // consume snapshot

	afterSnapshotOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("snapshot"))
	afterReason := testutil.ToFloat64(m.streamResyncReason.WithLabelValues("buffer_rotated"))
	if afterSnapshotOutcome <= beforeSnapshotOutcome {
		t.Fatalf("expected snapshot outcome counter to increase (before=%f after=%f)", beforeSnapshotOutcome, afterSnapshotOutcome)
	}
	if afterReason <= beforeReason {
		t.Fatalf("expected buffer_rotated reason counter to increase (before=%f after=%f)", beforeReason, afterReason)
	}

	// Miss path (resume requested while stream idle and no buffered events)
	psMiss := NewPatrolService(nil, nil)
	psMiss.resetStreamForRun("run-metrics-miss")

	beforeMiss := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("miss"))
	chMiss := psMiss.SubscribeToStreamFrom(42)
	defer psMiss.UnsubscribeFromStream(chMiss)
	afterMiss := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("miss"))
	if afterMiss <= beforeMiss {
		t.Fatalf("expected miss outcome counter to increase (before=%f after=%f)", beforeMiss, afterMiss)
	}
}

func TestBroadcast_RecordsBackpressureDropMetric(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ch := ps.SubscribeToStream()
	defer ps.UnsubscribeFromStream(ch)

	m := GetPatrolMetrics()
	beforeDrop := testutil.ToFloat64(m.streamSubscriberDrop.WithLabelValues("backpressure"))

	for i := 0; i < 100; i++ {
		ps.broadcast(PatrolStreamEvent{Type: "fill", Content: "x"})
	}
	for i := 0; i < 30; i++ {
		ps.broadcast(PatrolStreamEvent{Type: "overflow"})
	}

	time.Sleep(25 * time.Millisecond)
	afterDrop := testutil.ToFloat64(m.streamSubscriberDrop.WithLabelValues("backpressure"))
	if afterDrop <= beforeDrop {
		t.Fatalf("expected backpressure drop metric to increase (before=%f after=%f)", beforeDrop, afterDrop)
	}
}

func TestStreamOutput_TruncatesTailAndSnapshotMarksIt(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ps.resetStreamForRun("run-1")
	ps.setStreamPhase("analyzing")

	// Write more than the tail buffer can hold.
	payload := strings.Repeat("a", patrolStreamMaxOutputBytes+123)
	ps.appendStreamContent(payload)

	out, phase := ps.GetCurrentStreamOutput()
	if phase != "analyzing" {
		t.Fatalf("expected phase analyzing, got %q", phase)
	}
	if len(out) != patrolStreamMaxOutputBytes {
		t.Fatalf("expected output len %d, got %d", patrolStreamMaxOutputBytes, len(out))
	}
	expectedTail := payload[len(payload)-patrolStreamMaxOutputBytes:]
	if out != expectedTail {
		t.Fatalf("expected tail output to match last %d bytes", patrolStreamMaxOutputBytes)
	}

	ch := ps.SubscribeToStreamFrom(0)
	defer ps.UnsubscribeFromStream(ch)
	select {
	case ev := <-ch:
		if ev.Type != "snapshot" {
			t.Fatalf("expected snapshot, got %s", ev.Type)
		}
		if ev.ContentTruncated == nil || !*ev.ContentTruncated {
			t.Fatalf("expected snapshot content_truncated to be true")
		}
	case <-time.After(100 * time.Millisecond):
		t.Fatal("timed out waiting for snapshot")
	}
}

// --- generateFindingID ---

func TestGenerateFindingID_Deterministic(t *testing.T) {
	id1 := generateFindingID("res-1", "performance", "high-cpu")
	id2 := generateFindingID("res-1", "performance", "high-cpu")

	if id1 != id2 {
		t.Errorf("expected deterministic ID, got %s and %s", id1, id2)
	}
}

func TestGenerateFindingID_Different(t *testing.T) {
	id1 := generateFindingID("res-1", "performance", "high-cpu")
	id2 := generateFindingID("res-2", "performance", "high-cpu")
	id3 := generateFindingID("res-1", "reliability", "high-cpu")
	id4 := generateFindingID("res-1", "performance", "high-mem")

	if id1 == id2 {
		t.Error("different resource should produce different ID")
	}
	if id1 == id3 {
		t.Error("different category should produce different ID")
	}
	if id1 == id4 {
		t.Error("different issue should produce different ID")
	}
}

func TestGenerateFindingID_Length(t *testing.T) {
	id := generateFindingID("res-1", "performance", "high-cpu")
	// sha256[:8] = 8 bytes = 16 hex chars
	if len(id) != 16 {
		t.Errorf("expected 16-char hex ID, got %d chars: %s", len(id), id)
	}
}

// --- joinParts (already tested in patrol_test.go, but verify here for completeness) ---

func TestJoinParts_Zero(t *testing.T) {
	if joinParts(nil) != "" {
		t.Error("expected empty string for nil")
	}
}

func TestJoinParts_Three(t *testing.T) {
	result := joinParts([]string{"a", "b", "c"})
	if result != "a, b, and c" {
		t.Errorf("expected 'a, b, and c', got %q", result)
	}
}

// --- GetStatus ---

func TestGetStatus_Running(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	// Simulate a run in progress
	ps.mu.Lock()
	ps.runInProgress = true
	ps.mu.Unlock()

	status := ps.GetStatus()
	if !status.Running {
		t.Error("expected Running to be true when runInProgress is true")
	}

	ps.mu.Lock()
	ps.runInProgress = false
	ps.mu.Unlock()

	status = ps.GetStatus()
	if status.Running {
		t.Error("expected Running to be false when runInProgress is false")
	}
}

func TestGetStatus_NextPatrolAt(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	// Set nextScheduledAt so nextPatrolAt is returned
	nextTime := time.Now().Add(30 * time.Minute)
	ps.mu.Lock()
	ps.nextScheduledAt = nextTime
	ps.mu.Unlock()

	status := ps.GetStatus()
	if status.NextPatrolAt == nil {
		t.Fatal("expected NextPatrolAt to be calculated")
	}

	if !status.NextPatrolAt.Equal(nextTime) {
		t.Errorf("expected NextPatrolAt %v, got %v", nextTime, *status.NextPatrolAt)
	}
}

func TestGetStatus_NextPatrolAt_IndependentOfLastPatrol(t *testing.T) {
	// nextScheduledAt should drive NextPatrolAt, not lastFullPatrol + interval.
	// This is the scenario that was previously broken: user changes interval
	// mid-cycle, lastFullPatrol is old, so lastFullPatrol+newInterval could be in the past.
	ps := NewPatrolService(nil, nil)

	// Simulate: patrol ran 45 min ago
	ps.mu.Lock()
	ps.lastFullPatrol = time.Now().Add(-45 * time.Minute)
	// But the ticker was just reset with a 15-min interval, so next fire is ~15 min from now
	expectedNext := time.Now().Add(15 * time.Minute)
	ps.nextScheduledAt = expectedNext
	ps.mu.Unlock()

	status := ps.GetStatus()
	if status.NextPatrolAt == nil {
		t.Fatal("expected NextPatrolAt to be set")
	}
	// NextPatrolAt must be the tracked nextScheduledAt (in the future),
	// NOT lastFullPatrol + interval (which would be 30 min in the past).
	if !status.NextPatrolAt.Equal(expectedNext) {
		t.Errorf("expected NextPatrolAt = %v, got %v", expectedNext, *status.NextPatrolAt)
	}
	if status.NextPatrolAt.Before(time.Now()) {
		t.Errorf("NextPatrolAt should be in the future, got %v", *status.NextPatrolAt)
	}
}

func TestGetStatus_NextPatrolAt_NilWhenNotScheduled(t *testing.T) {
	// Before the patrol loop starts, nextScheduledAt is zero — no NextPatrolAt should be returned.
	ps := NewPatrolService(nil, nil)

	status := ps.GetStatus()
	if status.NextPatrolAt != nil {
		t.Errorf("expected NextPatrolAt to be nil before patrol loop starts, got %v", *status.NextPatrolAt)
	}
}

func TestGetStatus_Disabled(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	ps.SetConfig(PatrolConfig{Enabled: false})

	// Even if nextScheduledAt is set, disabled patrol should not report NextPatrolAt
	ps.mu.Lock()
	ps.nextScheduledAt = time.Now().Add(10 * time.Minute)
	ps.mu.Unlock()

	status := ps.GetStatus()
	if status.Enabled {
		t.Error("expected Enabled to be false")
	}
	if status.NextPatrolAt != nil {
		t.Error("expected NextPatrolAt to be nil when disabled")
	}
}

func TestGetStatus_BlockedReason(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	ps.setBlockedReason("circuit breaker is open")

	status := ps.GetStatus()
	if status.BlockedReason != "circuit breaker is open" {
		t.Errorf("expected blocked reason, got %q", status.BlockedReason)
	}
	if status.BlockedAt == nil {
		t.Error("expected BlockedAt to be set")
	}

	ps.clearBlockedReason()
	status = ps.GetStatus()
	if status.BlockedReason != "" {
		t.Errorf("expected empty blocked reason after clear, got %q", status.BlockedReason)
	}
}

func TestInitialPatrolSchedule_IgnoresRecentScopedErrorRun(t *testing.T) {
	now := time.Now()
	p := NewPatrolService(nil, nil)
	p.runHistoryStore.Add(PatrolRunRecord{
		ID: "scoped-error", Type: "scoped", TriggerReason: "alert_fired",
		CompletedAt: now.Add(-5 * time.Minute), ErrorCount: 1, Status: "error",
	})
	next, trigger := p.initialPatrolSchedule(now, 15*time.Minute)
	if trigger != TriggerReasonStartup || !next.Equal(now) {
		t.Fatalf("scoped error postponed initial full patrol: next=%v trigger=%s", next, trigger)
	}
}

func TestInitialPatrolSchedule_ResumesAfterRecentSuccessfulFullRun(t *testing.T) {
	now := time.Now()
	p := NewPatrolService(nil, nil)
	p.runHistoryStore.Add(PatrolRunRecord{
		ID: "full-success", Type: "patrol", TriggerReason: "scheduled",
		CompletedAt: now.Add(-10 * time.Minute), Status: "healthy",
	})
	next, trigger := p.initialPatrolSchedule(now, 15*time.Minute)
	if trigger != TriggerReasonScheduled || !next.Equal(now.Add(5*time.Minute)) {
		t.Fatalf("full patrol cadence was not resumed: next=%v trigger=%s", next, trigger)
	}
}

// --- appendStreamContent ---

func TestAppendStreamContent(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	ch := ps.SubscribeToStream()
	defer ps.UnsubscribeFromStream(ch)

	ps.appendStreamContent("hello ")
	ps.appendStreamContent("world")

	output, _ := ps.GetCurrentStreamOutput()
	if output != "hello world" {
		t.Errorf("expected 'hello world', got %q", output)
	}

	// Should have broadcast 2 events
	for i := 0; i < 2; i++ {
		select {
		case event := <-ch:
			if event.Type != "content" {
				t.Errorf("expected content event, got %s", event.Type)
			}
		case <-time.After(100 * time.Millisecond):
			t.Errorf("expected broadcast event %d", i)
		}
	}
}

// --- setStreamPhase ---

func TestSetStreamPhase_ResetClearsOutput(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	ps.setStreamPhase("analyzing")
	ps.appendStreamContent("some data")

	output, phase := ps.GetCurrentStreamOutput()
	if phase != "analyzing" {
		t.Errorf("expected phase 'analyzing', got %q", phase)
	}
	if output != "some data" {
		t.Errorf("expected 'some data', got %q", output)
	}

	ps.setStreamPhase("idle")
	output, phase = ps.GetCurrentStreamOutput()
	if phase != "idle" {
		t.Errorf("expected phase 'idle', got %q", phase)
	}
	// Output is no longer cleared on idle; it is cleared when a new run starts.
}

// --- isResourceOnline (heuristic alert resolution) ---

func TestIsResourceOnline_Node(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "n1", Name: "node-1", Status: "online"},
			{ID: "n2", Name: "node-2", Status: "offline"},
		},
	}

	alert := AlertInfo{ResourceID: "n1", ResourceType: "node"}
	if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected node n1 to be online")
	}

	alert = AlertInfo{ResourceID: "n2", ResourceType: "node"}
	if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected node n2 to be offline")
	}
}

func TestIsResourceOnline_VM(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		VMs: []models.VM{
			{ID: "vm1", Name: "vm-1", Status: "running"},
			{ID: "vm2", Name: "vm-2", Status: "stopped"},
		},
	}

	alert := AlertInfo{ResourceID: "vm1", ResourceType: "vm"}
	if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected VM vm1 to be online")
	}

	alert = AlertInfo{ResourceID: "vm2", ResourceType: "vm"}
	if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected VM vm2 to be offline")
	}
}

func TestIsResourceOnline_SystemContainer(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Containers: []models.Container{
			{ID: "ct1", Name: "ct-1", Status: "running"},
		},
	}

	alert := AlertInfo{ResourceID: "ct1", ResourceType: "system-container"}
	if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected container ct1 to be online")
	}
}

func TestIsResourceOnline_AppContainer(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		DockerHosts: []models.DockerHost{
			{
				Containers: []models.DockerContainer{
					{ID: "dc1", Name: "web", State: "running"},
					{ID: "dc2", Name: "db", State: "exited"},
				},
			},
		},
	}

	alert := AlertInfo{ResourceID: "dc1", ResourceType: "app-container"}
	if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected docker container dc1 to be online")
	}

	alert = AlertInfo{ResourceID: "dc2", ResourceType: "app-container"}
	if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected docker container dc2 to be offline")
	}
}

func TestIsResourceOnline_Agent(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Hosts: []models.Host{
			{ID: "h1", Hostname: "host-1", DisplayName: "Host One", Status: "online"},
			{ID: "h2", Hostname: "host-2", Status: "offline"},
		},
	}

	alert := AlertInfo{ResourceID: "h1", ResourceType: "agent"}
	if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected host h1 to be online")
	}

	alert = AlertInfo{ResourceID: "h2", ResourceType: "agent"}
	if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected host h2 to be offline")
	}
}

func TestIsResourceOnline_TrueNAS(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Hosts: []models.Host{
			{ID: "tn1", Hostname: "truenas-1", DisplayName: "TrueNAS One", Platform: "truenas", Status: "online"},
			{ID: "tn2", Hostname: "truenas-2", Platform: "truenas", Status: "offline"},
		},
	}

	alert := AlertInfo{ResourceID: "tn1", ResourceType: "truenas"}
	if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected TrueNAS tn1 to be online")
	}

	alert = AlertInfo{ResourceID: "tn2", ResourceType: "truenas"}
	if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected TrueNAS tn2 to be offline")
	}
}

func TestIsResourceOnline_UnknownType(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{}

	alert := AlertInfo{ResourceID: "x", ResourceType: "unknown"}
	if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected unknown type to return false")
	}
}

// --- getCurrentMetricValue ---

func TestGetCurrentMetricValue_Node(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "n1", Name: "node-1", CPU: 0.75, Memory: models.Memory{Usage: 60.0}},
		},
	}

	alert := AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "cpu"}
	val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 75.0 {
		t.Errorf("expected CPU 75.0 (0.75 * 100), got %f", val)
	}

	alert.Type = "memory"
	val = ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 60.0 {
		t.Errorf("expected memory 60.0, got %f", val)
	}
}

func TestGetCurrentMetricValue_NotFound(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{}

	alert := AlertInfo{ResourceID: "missing", ResourceType: "node", Type: "cpu"}
	val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != -1 {
		t.Errorf("expected -1 for not found, got %f", val)
	}
}

func TestGetCurrentMetricValue_AppContainer(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		DockerHosts: []models.DockerHost{
			{
				Containers: []models.DockerContainer{
					{ID: "dc1", Name: "web", CPUPercent: 45.0, MemoryPercent: 30.0},
				},
			},
		},
	}

	alert := AlertInfo{ResourceID: "dc1", ResourceType: "app-container", Type: "cpu"}
	val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 45.0 {
		t.Errorf("expected docker CPU 45.0, got %f", val)
	}

	alert.Type = "memory"
	val = ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 30.0 {
		t.Errorf("expected docker memory 30.0, got %f", val)
	}
}

func TestGetCurrentMetricValue_Agent(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Hosts: []models.Host{
			{ID: "h1", Hostname: "host-1", CPUUsage: 67.0, Memory: models.Memory{Usage: 54.0}},
		},
	}

	alert := AlertInfo{ResourceID: "h1", ResourceType: "agent", Type: "cpu"}
	val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 67.0 {
		t.Errorf("expected host CPU 67.0, got %f", val)
	}

	alert.Type = "memory"
	val = ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 54.0 {
		t.Errorf("expected host memory 54.0, got %f", val)
	}
}

func TestGetCurrentMetricValue_Storage(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Storage: []models.Storage{
			{ID: "s1", Name: "local", Usage: 72.5},
		},
	}

	alert := AlertInfo{ResourceID: "s1", ResourceType: "storage", Type: "usage"}
	val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
	if val != 72.5 {
		t.Errorf("expected storage usage 72.5, got %f", val)
	}
}

func TestGetCurrentMetricValue_UsesReadStateWhenLegacySlicesEmpty(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := newPatrolRuntimeState(models.StateSnapshot{})
	nodeView := unifiedresources.NewNodeView(&unifiedresources.Resource{
		ID:     "n1",
		Name:   "node-1",
		Type:   unifiedresources.ResourceTypeAgent,
		Status: unifiedresources.StatusOnline,
		Proxmox: &unifiedresources.ProxmoxData{
			NodeName: "node-1",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			CPU:    &unifiedresources.MetricValue{Percent: 42},
			Memory: &unifiedresources.MetricValue{Percent: 33},
		},
	})
	state.readState = &mockReadState{nodes: []*unifiedresources.NodeView{&nodeView}}

	val := ps.getCurrentMetricValueState(AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "cpu"}, state)
	if val != 42.0 {
		t.Fatalf("expected readState-backed CPU 42.0, got %f", val)
	}

	val = ps.getCurrentMetricValueState(AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "memory"}, state)
	if val != 33.0 {
		t.Fatalf("expected readState-backed memory 33.0, got %f", val)
	}
}

func TestIsResourceOnline_UsesReadStateWhenLegacySlicesEmpty(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := newPatrolRuntimeState(models.StateSnapshot{})
	hostView := unifiedresources.NewHostView(&unifiedresources.Resource{
		ID:     "h1",
		Name:   "host-1",
		Type:   unifiedresources.ResourceTypeAgent,
		Status: unifiedresources.StatusOnline,
		Agent: &unifiedresources.AgentData{
			Hostname: "host-1",
		},
	})
	state.readState = &mockReadState{hosts: []*unifiedresources.HostView{&hostView}}

	if !ps.isResourceOnlineState(AlertInfo{ResourceID: "h1", ResourceType: "agent", Type: "offline"}, state) {
		t.Fatal("expected host to be online via readState")
	}
}

func TestIsResourceOnline_UsesReadStateForTrueNASWhenLegacySlicesEmpty(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := newPatrolRuntimeState(models.StateSnapshot{})
	hostView := unifiedresources.NewHostView(&unifiedresources.Resource{
		ID:     "tn1",
		Name:   "truenas-1",
		Type:   unifiedresources.ResourceTypeAgent,
		Status: unifiedresources.StatusOnline,
		Agent: &unifiedresources.AgentData{
			Hostname: "truenas-1",
			Platform: "truenas",
		},
	})
	state.readState = &mockReadState{hosts: []*unifiedresources.HostView{&hostView}}

	if !ps.isResourceOnlineState(AlertInfo{ResourceID: "tn1", ResourceType: "truenas", Type: "offline"}, state) {
		t.Fatal("expected TrueNAS host to be online via readState")
	}
}

func TestGetResourceCurrentState_UsesReadStateWhenLegacySlicesEmpty(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := newPatrolRuntimeState(models.StateSnapshot{})
	storageView := unifiedresources.NewStoragePoolView(&unifiedresources.Resource{
		ID:     "s1",
		Name:   "local",
		Type:   unifiedresources.ResourceTypeStorage,
		Status: unifiedresources.StatusOnline,
		Metrics: &unifiedresources.ResourceMetrics{
			Disk: &unifiedresources.MetricValue{Percent: 72.5},
		},
	})
	state.readState = &mockReadState{storage: []*unifiedresources.StoragePoolView{&storageView}}

	desc := ps.getResourceCurrentStateState(AlertInfo{ResourceID: "s1", ResourceType: "storage"}, state)
	if !strings.Contains(desc, "Storage 'local': 72.5% used") {
		t.Fatalf("expected readState-backed storage description, got %q", desc)
	}
}

func TestAlertResolutionHelpers_UseReadStateForAppContainer(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := newPatrolRuntimeState(models.StateSnapshot{})
	parentID := "docker-1"
	containerView := unifiedresources.NewDockerContainerView(&unifiedresources.Resource{
		ID:       "dc-1",
		Name:     "web",
		Type:     unifiedresources.ResourceTypeAppContainer,
		Status:   unifiedresources.StatusOnline,
		ParentID: &parentID,
		Docker: &unifiedresources.DockerData{
			ContainerState: "running",
		},
		Metrics: &unifiedresources.ResourceMetrics{
			CPU:    &unifiedresources.MetricValue{Percent: 12.3},
			Memory: &unifiedresources.MetricValue{Percent: 45.6},
		},
	})
	state.readState = &mockReadState{dockerCtrs: []*unifiedresources.DockerContainerView{&containerView}}

	alert := AlertInfo{ResourceID: "dc-1", ResourceName: "web", ResourceType: "app-container", Type: "cpu"}
	if got := ps.getCurrentMetricValueState(alert, state); got != 12.3 {
		t.Fatalf("expected readState-backed docker CPU 12.3, got %f", got)
	}
	if !ps.isResourceOnlineState(AlertInfo{ResourceID: "dc-1", ResourceName: "web", ResourceType: "app-container"}, state) {
		t.Fatal("expected docker container to be online via readState")
	}
	desc := ps.getResourceCurrentStateState(AlertInfo{ResourceID: "dc-1", ResourceName: "web", ResourceType: "app-container"}, state)
	if !strings.Contains(desc, "Docker container 'web': CPU 12.3%, Memory 45.6%, State: running") {
		t.Fatalf("expected readState-backed docker container description, got %q", desc)
	}
}

// --- alert auto-resolve review (model-owned, no local auto-resolution without AI) ---

// Alerts whose trigger condition may have cleared go to model review; without
// a model available, nothing resolves — local state never resolves an alert.
func TestReviewAndResolveAlerts_ClearedConditionsDoNotResolveWithoutAI(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	resolver := &mockAlertResolver{
		alerts: []AlertInfo{
			{
				ID:           "a-usage",
				ResourceID:   "s1",
				ResourceType: "usage",
				Type:         "usage",
				Threshold:    85.0,
				Value:        88.0,
				StartTime:    time.Now().Add(-1 * time.Hour),
			},
			{
				ID:           "a-cpu",
				ResourceID:   "n1",
				ResourceType: "node",
				Type:         "cpu",
				Threshold:    90.0,
				Value:        95.0,
				StartTime:    time.Now().Add(-30 * time.Minute),
			},
			{
				ID:           "a-offline",
				ResourceID:   "n1",
				ResourceType: "node",
				Type:         "offline",
				StartTime:    time.Now().Add(-1 * time.Hour),
			},
			{
				ID:           "a-missing",
				ResourceID:   "gone",
				ResourceType: "node",
				Type:         "cpu",
				Threshold:    90.0,
				Value:        95.0,
				StartTime:    time.Now().Add(-30 * time.Minute),
			},
		},
	}
	ps.mu.Lock()
	ps.alertResolver = resolver
	ps.mu.Unlock()

	state := models.StateSnapshot{
		Storage: []models.Storage{{ID: "s1", Name: "local", Usage: 70.0, Status: "active"}},
		Nodes:   []models.Node{{ID: "n1", Name: "node-1", CPU: 0.5, Memory: models.Memory{Usage: 40.0}, Status: "online"}},
	}

	result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
	if result != 0 {
		t.Errorf("expected no alerts resolved without AI review, got %d", result)
	}
	if len(resolver.resolved) != 0 {
		t.Errorf("expected no alerts resolved without AI review, got %v", resolver.resolved)
	}
}

func TestAlertStillFiring_StorageStillHigh(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Storage: []models.Storage{
			{ID: "s1", Name: "local", Usage: 90.0},
		},
	}

	alert := AlertInfo{
		ID:           "a1",
		ResourceID:   "s1",
		ResourceType: "usage",
		Type:         "usage",
		Threshold:    85.0,
		Value:        88.0,
		StartTime:    time.Now().Add(-1 * time.Hour),
	}

	if !ps.alertStillFiringState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected still-above-threshold storage alert to be treated as still firing")
	}
}

// Regression test: node CPU is a 0-1 fraction in the model snapshot and must be
// scaled to a percentage before threshold comparison. Before the fix,
// getCurrentMetricValue returned 0.95 (raw fraction) which was always below the
// 90.0 percentage threshold — under the still-firing gate that would send every
// high-CPU alert to a pointless model review on every run.
func TestAlertStillFiring_CPUScaleRegression(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{
		Nodes: []models.Node{
			{ID: "n1", Name: "node-1", CPU: 0.95, Memory: models.Memory{Usage: 40.0}},
		},
	}

	alert := AlertInfo{
		ID:           "a1",
		ResourceID:   "n1",
		ResourceType: "node",
		Type:         "cpu",
		Threshold:    90.0,
		Value:        95.0,
		StartTime:    time.Now().Add(-30 * time.Minute),
	}

	if !ps.alertStillFiringState(alert, patrolRuntimeStateForTest(ps, state)) {
		t.Error("expected 95% CPU against a 90% threshold to be treated as still firing")
	}
}

func TestGetCurrentMetricValue_CPUScalePercent(t *testing.T) {
	ps := NewPatrolService(nil, nil)

	// Node CPU
	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", CPU: 0.42}},
	}
	val := ps.getCurrentMetricValueState(AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "cpu"}, patrolRuntimeStateForTest(ps, state))
	if val != 42.0 {
		t.Errorf("node CPU: expected 42.0, got %f", val)
	}

	// VM CPU
	state = models.StateSnapshot{
		VMs: []models.VM{{ID: "vm1", CPU: 0.88}},
	}
	val = ps.getCurrentMetricValueState(AlertInfo{ResourceID: "vm1", ResourceType: "vm", Type: "cpu"}, patrolRuntimeStateForTest(ps, state))
	if val != 88.0 {
		t.Errorf("VM CPU: expected 88.0, got %f", val)
	}

	// Container CPU
	state = models.StateSnapshot{
		Containers: []models.Container{{ID: "ct1", CPU: 0.15}},
	}
	val = ps.getCurrentMetricValueState(AlertInfo{ResourceID: "ct1", ResourceType: "system-container", Type: "cpu"}, patrolRuntimeStateForTest(ps, state))
	if val != 15.0 {
		t.Errorf("Container CPU: expected 15.0, got %f", val)
	}
}

// --- reviewAndResolveAlerts ---

func TestReviewAndResolveAlerts_NilResolver(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	state := models.StateSnapshot{}

	result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
	if result != 0 {
		t.Errorf("expected 0 resolved with nil resolver, got %d", result)
	}
}

func TestReviewAndResolveAlerts_NoActiveAlerts(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	resolver := &mockAlertResolver{alerts: nil}
	ps.mu.Lock()
	ps.alertResolver = resolver
	ps.mu.Unlock()

	state := models.StateSnapshot{}
	result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
	if result != 0 {
		t.Errorf("expected 0 resolved with no alerts, got %d", result)
	}
}

func TestReviewAndResolveAlerts_SkipsRecentAlerts(t *testing.T) {
	ps := NewPatrolService(nil, nil)
	resolver := &mockAlertResolver{
		alerts: []AlertInfo{
			{
				ID:           "a1",
				ResourceID:   "n1",
				ResourceType: "node",
				Type:         "offline",
				StartTime:    time.Now().Add(-5 * time.Minute), // only 5 minutes old
			},
		},
	}
	ps.mu.Lock()
	ps.alertResolver = resolver
	ps.mu.Unlock()

	state := models.StateSnapshot{
		Nodes: []models.Node{{ID: "n1", Name: "node-1", Status: "online"}},
	}

	result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
	if result != 0 {
		t.Errorf("expected 0 resolved (alert too recent), got %d", result)
	}
}

// --- mockAlertResolver ---

type mockAlertResolver struct {
	alerts   []AlertInfo
	resolved []string
}

func (m *mockAlertResolver) GetActiveAlerts() []AlertInfo {
	return m.alerts
}

func (m *mockAlertResolver) ResolveAlert(alertID string) bool {
	m.resolved = append(m.resolved, alertID)
	return true
}