Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package ai
import (
"strings"
"sync"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
// --- filterStateByScope tests ---
func TestFilterStateByScope_NoScope(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
VMs: []models.VM{{ID: "vm1", Name: "vm-1"}},
}
scope := PatrolScope{} // no resource IDs or types
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 1 {
t.Errorf("expected 1 node with no scope filter, got %d", len(filtered.Nodes))
}
if len(filtered.VMs) != 1 {
t.Errorf("expected 1 VM with no scope filter, got %d", len(filtered.VMs))
}
}
func TestResolvePatrolScopeExpandsCanonicalUnifiedIDToSourceResource(t *testing.T) {
snapshot := models.StateSnapshot{VMs: []models.VM{{ID: "source-vm-401", Name: "web", VMID: 401}}}
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(snapshot)
views := registry.VMs()
if len(views) != 1 || views[0].ID() == "" {
t.Fatalf("expected one canonical VM view, got %+v", views)
}
state := newPatrolRuntimeStateWithProviders(snapshot, registry, unifiedresources.NewUnifiedAIAdapter(registry))
resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{views[0].ID()}})
if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 {
t.Fatalf("canonical ID did not resolve exactly: %+v", resolution)
}
filtered := (&PatrolService{}).filterStateByScopeState(state, resolved)
if len(filtered.VMs) != 1 || filtered.VMs[0].ID != "source-vm-401" {
t.Fatalf("resolved scope did not select source VM: %+v", filtered.VMs)
}
if len(resolution.EffectiveResourceIDs) == 0 {
t.Fatalf("expected effective resource IDs, got %+v", resolution)
}
if len(resolved.resolvedResourceTypes) != 1 || resolved.resolvedResourceTypes[0] != "vm" {
t.Fatalf("resolved resource types = %v, want [vm]", resolved.resolvedResourceTypes)
}
}
func TestResolvePatrolScopePreservesCanonicalAvailabilityEndpoint(t *testing.T) {
now := time.Now().UTC()
endpoint := unifiedresources.Resource{
ID: "network-endpoint-jellyfin",
Name: "Jellyfin",
Type: unifiedresources.ResourceTypeNetworkEndpoint,
Status: unifiedresources.StatusOnline,
LastSeen: now,
Sources: []unifiedresources.DataSource{unifiedresources.SourceAvailability},
Availability: &unifiedresources.AvailabilityData{
TargetID: "jellyfin-check", Enabled: true, ProbeOutcome: "reachable",
},
Canonical: &unifiedresources.CanonicalIdentity{
DisplayName: "Jellyfin playback endpoint",
Aliases: []string{"jellyfin.local"},
},
}
unrelated := unifiedresources.Resource{
ID: "network-endpoint-unrelated", Name: "Unrelated", Type: unifiedresources.ResourceTypeNetworkEndpoint,
Status: unifiedresources.StatusOnline, LastSeen: now,
}
registry := unifiedresources.NewRegistry(nil)
registry.IngestResources([]unifiedresources.Resource{endpoint, unrelated})
state := newPatrolRuntimeStateWithProviders(
models.EmptyStateSnapshot(), registry, unifiedresources.NewUnifiedAIAdapter(registry),
)
resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{endpoint.ID}})
if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 {
t.Fatalf("canonical availability endpoint did not resolve exactly: %+v", resolution)
}
filtered := filterPatrolStateByScopeState(state, resolved)
resources := filtered.unifiedResourceProvider.GetAll()
if len(resources) != 1 || resources[0].ID != endpoint.ID {
t.Fatalf("scoped provider resources = %+v, want only %q", resources, endpoint.ID)
}
checks := unifiedresources.AvailabilityChecksForResource(resources[0])
if len(checks) != 1 || checks[0].TargetID != "jellyfin-check" || checks[0].ProbeOutcome != "reachable" {
t.Fatalf("availability evidence was not preserved: %+v", checks)
}
effectiveContainsEndpoint := false
for _, id := range resolution.EffectiveResourceIDs {
if id == endpoint.ID {
effectiveContainsEndpoint = true
break
}
}
if !effectiveContainsEndpoint {
t.Fatalf("effective scope = %v, want canonical endpoint %q", resolution.EffectiveResourceIDs, endpoint.ID)
}
counts := patrolRuntimeCountResources(filtered)
if counts.unified != 1 || counts.total() != 1 {
t.Fatalf("canonical endpoint counts = %#v, want one unified resource", counts)
}
if len(resolved.resolvedResourceTypes) != 1 || resolved.resolvedResourceTypes[0] != "network-endpoint" {
t.Fatalf("resolved endpoint types = %v, want [network-endpoint]", resolved.resolvedResourceTypes)
}
}
func TestResolvePatrolScopeRejectsUnmatchedAndAmbiguousAliases(t *testing.T) {
state := newPatrolRuntimeState(models.StateSnapshot{VMs: []models.VM{
{ID: "vm-1", Name: "duplicate", VMID: 1},
{ID: "vm-2", Name: "duplicate", VMID: 2},
}})
resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{"duplicate", "missing"}})
if len(resolved.ResourceIDs) != 0 {
t.Fatalf("ambiguous or unmatched aliases must not expand: %v", resolved.ResourceIDs)
}
if len(resolution.AmbiguousResourceIDs) != 1 || resolution.AmbiguousResourceIDs[0] != "duplicate" {
t.Fatalf("ambiguous IDs = %v", resolution.AmbiguousResourceIDs)
}
if len(resolution.UnmatchedResourceIDs) != 1 || resolution.UnmatchedResourceIDs[0] != "missing" {
t.Fatalf("unmatched IDs = %v", resolution.UnmatchedResourceIDs)
}
}
func TestResolvePatrolScopeCanonicalIDOutranksAliasCollision(t *testing.T) {
state := newPatrolRuntimeState(models.StateSnapshot{VMs: []models.VM{
{ID: "vm-canonical", Name: "primary", VMID: 1},
{ID: "vm-other", Name: "vm-canonical", VMID: 2},
}})
resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{"vm-canonical"}})
if len(resolution.AmbiguousResourceIDs) != 0 || len(resolution.UnmatchedResourceIDs) != 0 {
t.Fatalf("canonical ID was treated as an alias collision: %+v", resolution)
}
filtered := filterPatrolStateByScopeState(state, resolved)
if len(filtered.VMs) != 1 || filtered.VMs[0].ID != "vm-canonical" {
t.Fatalf("canonical ID selected wrong resource: %+v", filtered.VMs)
}
}
func TestResolvePatrolScopeResolvesDockerAlertResourceIDs(t *testing.T) {
// A container reported without an ID alerts under its own name reference,
// so the host's reference stays unambiguous.
snapshot := models.StateSnapshot{DockerHosts: []models.DockerHost{{
ID: "nas", Hostname: "nas",
Containers: []models.DockerContainer{
{ID: "abc123def456", Name: "dockhand-hawser-updater-nas", State: "exited"},
{Name: "legacy-sidecar", State: "exited"},
},
}}}
containerAlertID := alerts.DockerContainerResourceID("nas", "abc123def456", "dockhand-hawser-updater-nas")
namedContainerAlertID := alerts.DockerContainerResourceID("nas", "", "legacy-sidecar")
hostAlertID := alerts.DockerHostResourceID("nas")
snapshotState := newPatrolRuntimeState(snapshot)
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(snapshot)
readState := newPatrolRuntimeStateWithProviders(snapshot, registry, unifiedresources.NewUnifiedAIAdapter(registry))
for _, state := range []patrolRuntimeState{snapshotState, readState} {
for _, tc := range []struct {
requested string
resourceType string
}{{containerAlertID, "app-container"}, {namedContainerAlertID, "app-container"}, {hostAlertID, "docker-host"}} {
if tc.requested == namedContainerAlertID && state.readState == nil {
// The legacy snapshot record of a container without an ID has
// no ID to scope by; only the unified read state can.
continue
}
resolved, resolution := resolvePatrolScopeState(state, PatrolScope{ResourceIDs: []string{tc.requested}})
if len(resolution.UnmatchedResourceIDs) != 0 || len(resolution.AmbiguousResourceIDs) != 0 {
t.Fatalf("alert resource ID %q did not resolve exactly: %+v", tc.requested, resolution)
}
if len(resolved.ResourceIDs) == 0 {
t.Fatalf("alert resource ID %q resolved to an empty scope", tc.requested)
}
if len(resolved.resolvedResourceTypes) != 1 || resolved.resolvedResourceTypes[0] != tc.resourceType {
t.Fatalf("alert resource ID %q resolved types = %v, want [%s]", tc.requested, resolved.resolvedResourceTypes, tc.resourceType)
}
}
}
}
func TestRunScopedPatrolPersistsZeroMatchFailure(t *testing.T) {
ps := NewPatrolService(nil, &mockStateProvider{state: models.StateSnapshot{
Nodes: []models.Node{{ID: "node-1", Name: "pve-1"}},
}})
ps.SetConfig(PatrolConfig{Enabled: true, AnalyzeNodes: true, Interval: time.Hour})
ps.runScopedPatrol(t.Context(), PatrolScope{ResourceIDs: []string{"missing-resource"}, Reason: TriggerReasonManual})
runs := ps.runHistoryStore.GetRecent(1)
if len(runs) != 1 {
t.Fatalf("zero-match scope should create one durable run, got %d", len(runs))
}
run := runs[0]
if run.Status != "error" || run.ErrorCount != 1 || run.ResourcesChecked != 0 {
t.Fatalf("unexpected zero-match run: %+v", run)
}
if len(run.ScopeResourceIDs) != 1 || run.ScopeResourceIDs[0] != "missing-resource" || len(run.EffectiveScopeResourceIDs) != 0 {
t.Fatalf("zero-match scope identity was not preserved: %+v", run)
}
if run.FindingsSummary == "All healthy" {
t.Fatal("zero-match run must never claim all healthy")
}
}
func TestPatrolFindingSummaryForRunExcludesSupportingResourceFindings(t *testing.T) {
summary := patrolFindingSummaryForRun([]*Finding{
{ID: "run-owned", ResourceID: "container-target", Severity: FindingSeverityWarning},
{ID: "supporting-context", ResourceID: "agent-host", Severity: FindingSeverityCritical},
}, []string{"run-owned"})
if summary.Warning != 1 || summary.Critical != 0 || summary.Total != 1 {
t.Fatalf("run summary = %+v, want one run-owned warning and no supporting-resource finding", summary)
}
}
func TestFilterStateByScope_ByResourceID(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{
{ID: "n1", Name: "node-1"},
{ID: "n2", Name: "node-2"},
},
}
scope := PatrolScope{
ResourceIDs: []string{"n1"},
ResourceTypes: []string{"node"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 1 {
t.Fatalf("expected 1 node, got %d", len(filtered.Nodes))
}
if filtered.Nodes[0].ID != "n1" {
t.Errorf("expected node n1, got %s", filtered.Nodes[0].ID)
}
}
func TestFilterStateByScope_ByResourceName(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{
{ID: "n1", Name: "node-1"},
{ID: "n2", Name: "node-2"},
},
}
scope := PatrolScope{
ResourceIDs: []string{"node-1"},
ResourceTypes: []string{"node"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 1 {
t.Fatalf("expected 1 node matched by name, got %d", len(filtered.Nodes))
}
if filtered.Nodes[0].Name != "node-1" {
t.Errorf("expected node-1, got %s", filtered.Nodes[0].Name)
}
}
func TestFilterStateByScope_ByType_VM(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
VMs: []models.VM{{ID: "vm1", Name: "vm-1"}},
}
scope := PatrolScope{
ResourceTypes: []string{"vm"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 0 {
t.Errorf("expected 0 nodes when scoped to VM type, got %d", len(filtered.Nodes))
}
if len(filtered.VMs) != 1 {
t.Errorf("expected 1 VM, got %d", len(filtered.VMs))
}
}
func TestFilterStateByScope_TypeAliasesRejected(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
VMs: []models.VM{{ID: "vm1", Name: "vm-1"}},
Containers: []models.Container{{ID: "ct1", Name: "ct-1"}},
Hosts: []models.Host{{ID: "h1", Hostname: "host-1"}},
PhysicalDisks: []models.PhysicalDisk{
{ID: "disk-1", DevPath: "/dev/sda", Model: "sda"},
},
PBSInstances: []models.PBSInstance{
{ID: "pbs1", Name: "pbs-main", Datastores: []models.PBSDatastore{{Name: "ds1"}}},
},
PMGInstances: []models.PMGInstance{
{ID: "pmg1", Name: "pmg-main", Host: "pmg.local"},
},
KubernetesClusters: []models.KubernetesCluster{
{ID: "k1", Name: "cluster-1"},
},
DockerHosts: []models.DockerHost{
{ID: "dh1", Hostname: "docker-host-1"},
},
}
// Legacy alias should not match canonical VM resources.
scope := PatrolScope{ResourceTypes: []string{"qemu"}}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.VMs) != 0 {
t.Errorf("expected legacy 'qemu' alias to be rejected, got %d VMs", len(filtered.VMs))
}
// Legacy alias should not match canonical system-container resources.
scope = PatrolScope{ResourceTypes: []string{"container"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Containers) != 0 {
t.Errorf("expected legacy 'container' alias to be rejected, got %d containers", len(filtered.Containers))
}
// Removed host alias should not match canonical agent resources.
scope = PatrolScope{ResourceTypes: []string{"host"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Hosts) != 0 {
t.Errorf("expected legacy 'host' alias to be rejected, got %d hosts", len(filtered.Hosts))
}
// Canonical v6 type should match containers.
scope = PatrolScope{ResourceTypes: []string{"system-container"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Containers) != 1 {
t.Errorf("expected 'system-container' to match LXC containers, got %d", len(filtered.Containers))
}
// Underscore alias should be rejected; hyphenated canonical type is required.
scope = PatrolScope{ResourceTypes: []string{"system_container"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Containers) != 0 {
t.Errorf("expected legacy 'system_container' alias to be rejected, got %d containers", len(filtered.Containers))
}
// Additional removed aliases should also be rejected.
scope = PatrolScope{ResourceTypes: []string{"docker_container"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.DockerHosts) != 0 {
t.Errorf("expected legacy 'docker_container' alias to be rejected, got %d docker hosts", len(filtered.DockerHosts))
}
scope = PatrolScope{ResourceTypes: []string{"kubernetes_cluster"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.KubernetesClusters) != 0 {
t.Errorf("expected legacy 'kubernetes_cluster' alias to be rejected, got %d kubernetes clusters", len(filtered.KubernetesClusters))
}
scope = PatrolScope{ResourceTypes: []string{"docker"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.DockerHosts) != 0 {
t.Errorf("expected non-canonical 'docker' alias to be rejected, got %d docker hosts", len(filtered.DockerHosts))
}
scope = PatrolScope{ResourceTypes: []string{"kubernetes"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.KubernetesClusters) != 0 {
t.Errorf("expected non-canonical 'kubernetes' alias to be rejected, got %d kubernetes clusters", len(filtered.KubernetesClusters))
}
scope = PatrolScope{ResourceTypes: []string{"pbs_datastore"}, ResourceIDs: []string{"pbs1:ds1"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.PBSInstances) != 0 {
t.Errorf("expected non-canonical 'pbs_datastore' alias to be rejected, got %d pbs instances", len(filtered.PBSInstances))
}
scope = PatrolScope{ResourceTypes: []string{"agent_raid"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Hosts) != 0 {
t.Errorf("expected non-canonical 'agent_raid' alias to be rejected, got %d hosts", len(filtered.Hosts))
}
scope = PatrolScope{ResourceTypes: []string{"pmg"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.PMGInstances) != 1 {
t.Errorf("expected canonical 'pmg' to match PMG instances, got %d", len(filtered.PMGInstances))
}
scope = PatrolScope{ResourceTypes: []string{"physical_disk"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.PhysicalDisks) != 1 {
t.Errorf("expected canonical 'physical_disk' to match physical disks, got %d", len(filtered.PhysicalDisks))
}
}
func TestFilterStateByScope_AppContainerAlias(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
ID: "dh1",
Hostname: "docker-host-1",
Containers: []models.DockerContainer{
{ID: "c1", Name: "web"},
},
},
},
}
// Canonical semantic name should match Docker resources.
scope := PatrolScope{ResourceTypes: []string{"app-container"}}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.DockerHosts) != 1 {
t.Errorf("expected 'app-container' to match Docker hosts, got %d", len(filtered.DockerHosts))
}
// Underscore alias should be rejected.
scope = PatrolScope{ResourceTypes: []string{"app_container"}}
filtered = ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.DockerHosts) != 0 {
t.Errorf("expected legacy 'app_container' alias to be rejected, got %d docker hosts", len(filtered.DockerHosts))
}
}
func TestFilterStateByScope_DockerHost(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
ID: "dh1",
Hostname: "docker-host-1",
Containers: []models.DockerContainer{
{
ID: "c1",
Name: "web",
Image: "example/web:latest",
Networks: []models.DockerContainerNetworkLink{{Name: "frontend"}},
Mounts: []models.DockerContainerMount{{Type: "volume", Name: "web-data"}},
},
{ID: "c2", Name: "db"},
},
Images: []models.DockerImage{{ID: "sha256:web", RepoTags: []string{"example/web:latest"}}, {ID: "sha256:db", RepoTags: []string{"example/db:latest"}}},
Volumes: []models.DockerVolume{{Name: "web-data"}, {Name: "db-data"}},
Networks: []models.DockerNetwork{{ID: "network-frontend", Name: "frontend"}, {ID: "network-backend", Name: "backend"}},
Services: []models.DockerService{{ID: "service-web", Name: "web"}},
Tasks: []models.DockerTask{{ID: "task-web", ContainerID: "c1"}},
Nodes: []models.DockerNode{{ID: "node-1", Hostname: "docker-host-1"}},
Secrets: []models.DockerSecret{{ID: "secret-web", Name: "web-secret"}},
Configs: []models.DockerConfig{{ID: "config-web", Name: "web-config"}},
StorageUsage: &models.DockerStorageUsage{
Images: models.DockerStorageUsageBucket{TotalCount: 2},
},
Swarm: &models.DockerSwarmInfo{LocalState: "active"},
},
},
}
// Match by host ID
scope := PatrolScope{
ResourceIDs: []string{"dh1"},
ResourceTypes: []string{"docker-host"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.DockerHosts) != 1 {
t.Fatalf("expected 1 docker host, got %d", len(filtered.DockerHosts))
}
// When the host itself matches, all containers should be included
if len(filtered.DockerHosts[0].Containers) != 2 {
t.Errorf("expected all 2 containers when host matches, got %d", len(filtered.DockerHosts[0].Containers))
}
}
func TestFilterStateByScope_DockerContainerOnly(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
ID: "dh1",
Hostname: "docker-host-1",
Containers: []models.DockerContainer{
{ID: "c1", Name: "web"},
{ID: "c2", Name: "db"},
},
},
},
}
// Match by container ID only
scope := PatrolScope{
ResourceIDs: []string{"c1"},
ResourceTypes: []string{"app-container"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.DockerHosts) != 1 {
t.Fatalf("expected 1 docker host, got %d", len(filtered.DockerHosts))
}
if len(filtered.DockerHosts[0].Containers) != 1 {
t.Fatalf("expected 1 container, got %d", len(filtered.DockerHosts[0].Containers))
}
if filtered.DockerHosts[0].Containers[0].ID != "c1" {
t.Errorf("expected container c1, got %s", filtered.DockerHosts[0].Containers[0].ID)
}
filteredHost := filtered.DockerHosts[0]
if len(filteredHost.Images) != 0 || len(filteredHost.Volumes) != 0 || len(filteredHost.Networks) != 0 ||
len(filteredHost.Services) != 0 || len(filteredHost.Tasks) != 0 || len(filteredHost.Nodes) != 0 ||
len(filteredHost.Secrets) != 0 || len(filteredHost.Configs) != 0 || filteredHost.StorageUsage != nil || filteredHost.Swarm != nil {
t.Fatalf("container scope retained unrelated host inventory: %+v", filteredHost)
}
for _, resourceType := range []unifiedresources.ResourceType{
unifiedresources.ResourceTypeDockerImage,
unifiedresources.ResourceTypeDockerVolume,
unifiedresources.ResourceTypeDockerNetwork,
unifiedresources.ResourceTypeDockerService,
unifiedresources.ResourceTypeDockerTask,
unifiedresources.ResourceTypeDockerSwarmNode,
unifiedresources.ResourceTypeDockerSecret,
unifiedresources.ResourceTypeDockerConfig,
} {
if got := len(filtered.unifiedResourceProvider.GetByType(resourceType)); got != 0 {
t.Fatalf("container scope retained %d unrelated %s resources", got, resourceType)
}
}
}
func TestFilterStateByScope_Storage(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Storage: []models.Storage{
{ID: "s1", Name: "local"},
{ID: "s2", Name: "ceph"},
},
}
scope := PatrolScope{
ResourceIDs: []string{"s1"},
ResourceTypes: []string{"storage"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Storage) != 1 {
t.Fatalf("expected 1 storage, got %d", len(filtered.Storage))
}
if filtered.Storage[0].ID != "s1" {
t.Errorf("expected storage s1, got %s", filtered.Storage[0].ID)
}
}
func TestFilterStateByScope_Hosts(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Hosts: []models.Host{
{ID: "h1", Hostname: "host-1"},
{ID: "h2", Hostname: "host-2"},
},
}
scope := PatrolScope{
ResourceIDs: []string{"h1"},
ResourceTypes: []string{"agent"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Hosts) != 1 {
t.Fatalf("expected 1 host, got %d", len(filtered.Hosts))
}
if filtered.Hosts[0].ID != "h1" {
t.Errorf("expected host h1, got %s", filtered.Hosts[0].ID)
}
}
func TestFilterStateByScope_PBS(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
PBSInstances: []models.PBSInstance{
{
ID: "pbs1",
Name: "pbs-main",
Datastores: []models.PBSDatastore{
{Name: "ds1"},
},
},
{
ID: "pbs2",
Name: "pbs-secondary",
},
},
}
scope := PatrolScope{
ResourceIDs: []string{"pbs1"},
ResourceTypes: []string{"pbs"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.PBSInstances) != 1 {
t.Fatalf("expected 1 PBS instance, got %d", len(filtered.PBSInstances))
}
if filtered.PBSInstances[0].ID != "pbs1" {
t.Errorf("expected PBS pbs1, got %s", filtered.PBSInstances[0].ID)
}
}
func TestFilterStateByScope_Kubernetes(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
KubernetesClusters: []models.KubernetesCluster{
{ID: "k1", Name: "cluster-1"},
{ID: "k2", Name: "cluster-2"},
},
}
scope := PatrolScope{
ResourceIDs: []string{"k1"},
ResourceTypes: []string{"k8s-cluster"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.KubernetesClusters) != 1 {
t.Fatalf("expected 1 k8s cluster, got %d", len(filtered.KubernetesClusters))
}
if filtered.KubernetesClusters[0].ID != "k1" {
t.Errorf("expected cluster k1, got %s", filtered.KubernetesClusters[0].ID)
}
}
func TestFilterStateByScope_PreservesScopedMetadataOnly(t *testing.T) {
ps := NewPatrolService(nil, nil)
now := time.Now()
state := models.StateSnapshot{
Nodes: []models.Node{
{ID: "node-1", Name: "node-1"},
},
VMs: []models.VM{
{ID: "vm-101", VMID: 101, Name: "vm-101"},
},
LastUpdate: now,
ConnectionHealth: map[string]bool{
"node-1": true,
"node-2": false,
},
ActiveAlerts: []models.Alert{
{ID: "a1", ResourceID: "node-1", Message: "scoped"},
{ID: "a2", ResourceID: "node-2", Message: "global"},
},
RecentlyResolved: []models.ResolvedAlert{
{Alert: models.Alert{ID: "r1", ResourceID: "node-1", Message: "resolved scoped"}},
{Alert: models.Alert{ID: "r2", ResourceID: "node-2", Message: "resolved global"}},
},
PVEBackups: models.PVEBackups{
BackupTasks: []models.BackupTask{
{ID: "bt-101", VMID: 101},
{ID: "bt-202", VMID: 202},
},
StorageBackups: []models.StorageBackup{
{ID: "sb-101", VMID: 101},
{ID: "sb-202", VMID: 202},
},
GuestSnapshots: []models.GuestSnapshot{
{ID: "gs-101", VMID: 101},
{ID: "gs-202", VMID: 202},
},
},
PBSBackups: []models.PBSBackup{
{ID: "pb-101", VMID: "101"},
{ID: "pb-202", VMID: "202"},
},
}
scope := PatrolScope{ResourceTypes: []string{"node", "vm"}}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.ConnectionHealth) != 1 {
t.Fatalf("expected only scoped ConnectionHealth entries, got %d", len(filtered.ConnectionHealth))
}
if !filtered.ConnectionHealth["node-1"] {
t.Fatal("expected node-1 connection health to be preserved")
}
if len(filtered.ActiveAlerts) != 1 || filtered.ActiveAlerts[0].ResourceID != "node-1" {
t.Fatalf("expected only scoped active alerts, got %+v", filtered.ActiveAlerts)
}
if len(filtered.RecentlyResolved) != 1 || filtered.RecentlyResolved[0].ResourceID != "node-1" {
t.Fatalf("expected only scoped resolved alerts, got %+v", filtered.RecentlyResolved)
}
if len(filtered.PVEBackups.BackupTasks) != 1 || filtered.PVEBackups.BackupTasks[0].VMID != 101 {
t.Fatalf("expected only scoped PVE backup tasks, got %+v", filtered.PVEBackups.BackupTasks)
}
if len(filtered.PVEBackups.StorageBackups) != 1 || filtered.PVEBackups.StorageBackups[0].VMID != 101 {
t.Fatalf("expected only scoped storage backups, got %+v", filtered.PVEBackups.StorageBackups)
}
if len(filtered.PVEBackups.GuestSnapshots) != 1 || filtered.PVEBackups.GuestSnapshots[0].VMID != 101 {
t.Fatalf("expected only scoped guest snapshots, got %+v", filtered.PVEBackups.GuestSnapshots)
}
if len(filtered.PBSBackups) != 1 || filtered.PBSBackups[0].VMID != "101" {
t.Fatalf("expected only scoped PBS backups, got %+v", filtered.PBSBackups)
}
}
func TestFilterStateByScope_RebuildsScopedProviders(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := patrolRuntimeState{
Nodes: []models.Node{
{ID: "n1", Name: "node-1"},
{ID: "n2", Name: "node-2"},
},
VMs: []models.VM{
{ID: "vm1", Name: "vm-1"},
{ID: "vm2", Name: "vm-2"},
},
Storage: []models.Storage{
{ID: "s1", Name: "local", Usage: 42},
{ID: "s2", Name: "backup", Usage: 10},
},
PhysicalDisks: []models.PhysicalDisk{
{ID: "disk-1", DevPath: "/dev/sda", Model: "sda"},
{ID: "disk-2", DevPath: "/dev/sdb", Model: "sdb"},
},
PMGInstances: []models.PMGInstance{
{ID: "pmg1", Name: "pmg-main", Host: "pmg.local"},
{ID: "pmg2", Name: "pmg-edge", Host: "pmg2.local"},
},
}
filtered := ps.filterStateByScopeState(state, PatrolScope{
ResourceIDs: []string{"n1", "s1"},
ResourceTypes: []string{"node", "storage"},
})
if filtered.readState == nil {
t.Fatal("expected scoped readState to be rebuilt")
}
if filtered.unifiedResourceProvider == nil {
t.Fatal("expected scoped unified resource provider to be rebuilt")
}
if got := len(filtered.readState.Nodes()); got != 1 {
t.Fatalf("expected 1 scoped node in readState, got %d", got)
}
if got := len(filtered.readState.VMs()); got != 0 {
t.Fatalf("expected 0 scoped VMs in readState, got %d", got)
}
if got := len(filtered.readState.StoragePools()); got != 1 {
t.Fatalf("expected 1 scoped storage pool in readState, got %d", got)
}
if got := len(filtered.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypeStorage)); got != 1 {
t.Fatalf("expected 1 scoped storage resource in provider, got %d", got)
}
if got := len(filtered.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypePhysicalDisk)); got != 0 {
t.Fatalf("expected no scoped physical disk resources in provider, got %d", got)
}
filteredPMG := ps.filterStateByScopeState(state, PatrolScope{
ResourceIDs: []string{"pmg.local"},
ResourceTypes: []string{"pmg"},
})
if got := len(filteredPMG.PMGInstances); got != 1 {
t.Fatalf("expected 1 scoped PMG instance, got %d", got)
}
if filteredPMG.PMGInstances[0].ID != "pmg1" {
t.Fatalf("expected scoped PMG instance pmg1, got %s", filteredPMG.PMGInstances[0].ID)
}
if filteredPMG.readState == nil {
t.Fatal("expected scoped PMG readState to be rebuilt")
}
if got := len(filteredPMG.readState.PMGInstances()); got != 1 {
t.Fatalf("expected 1 scoped PMG instance in readState, got %d", got)
}
if got := len(filteredPMG.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypePMG)); got != 1 {
t.Fatalf("expected 1 scoped PMG resource in provider, got %d", got)
}
filteredDisks := ps.filterStateByScopeState(state, PatrolScope{
ResourceIDs: []string{"/dev/sda"},
ResourceTypes: []string{"physical_disk"},
})
if got := len(filteredDisks.PhysicalDisks); got != 1 {
t.Fatalf("expected 1 scoped physical disk, got %d", got)
}
if filteredDisks.PhysicalDisks[0].ID != "disk-1" {
t.Fatalf("expected scoped physical disk disk-1, got %s", filteredDisks.PhysicalDisks[0].ID)
}
if got := len(filteredDisks.unifiedResourceProvider.GetByType(unifiedresources.ResourceTypePhysicalDisk)); got != 1 {
t.Fatalf("expected 1 scoped physical disk resource in provider, got %d", got)
}
counts := patrolRuntimeCountResources(filteredDisks)
if counts.storage != 1 {
t.Fatalf("expected scoped runtime counts to include 1 storage resource, got %#v", counts)
}
if counts.total() != 1 {
t.Fatalf("expected scoped runtime total to be 1, got %#v", counts)
}
}
func TestPatrolRuntimeResourceHelpers_PrepareCanonicalInventory(t *testing.T) {
nodeView := unifiedresources.NewNodeView(&unifiedresources.Resource{
ID: "node-1",
Name: "node-a",
Type: unifiedresources.ResourceTypeAgent,
})
vmView := newTestVMView("qemu/100", "vm-a", 100, "pve1", "", unifiedresources.StatusOnline, false, nil)
ctView := newTestContainerView("lxc/200", "ct-a", 200, "pve1", "", unifiedresources.StatusOnline, false, nil)
storageView := unifiedresources.NewStoragePoolView(&unifiedresources.Resource{
ID: "storage-1",
Name: "local-zfs",
Type: unifiedresources.ResourceTypeStorage,
})
dockerHostView := unifiedresources.NewDockerHostView(&unifiedresources.Resource{
ID: "docker-1",
Name: "docker-a",
Type: unifiedresources.ResourceTypeAgent,
Docker: &unifiedresources.DockerData{
Hostname: "docker.local",
},
})
dockerCtrView := unifiedresources.NewDockerContainerView(&unifiedresources.Resource{
ID: "ctr-1",
Name: "web",
Type: unifiedresources.ResourceTypeAppContainer,
})
hostView := unifiedresources.NewHostView(&unifiedresources.Resource{
ID: "agent-1",
Name: "agent-a",
Type: unifiedresources.ResourceTypeAgent,
Agent: &unifiedresources.AgentData{
Hostname: "agent.local",
},
})
trueNASHostView := unifiedresources.NewHostView(&unifiedresources.Resource{
ID: "truenas-1",
Name: "truenas-main",
Type: unifiedresources.ResourceTypeAgent,
Agent: &unifiedresources.AgentData{
Hostname: "truenas-main.local",
Platform: "truenas",
},
})
pbsView := unifiedresources.NewPBSInstanceView(&unifiedresources.Resource{
ID: "pbs-1",
Name: "pbs-a",
Type: unifiedresources.ResourceTypePBS,
})
pmgView := unifiedresources.NewPMGInstanceView(&unifiedresources.Resource{
ID: "pmg-1",
Name: "pmg-a",
Type: unifiedresources.ResourceTypePMG,
})
k8sView := unifiedresources.NewK8sClusterView(&unifiedresources.Resource{
ID: "k8s-1",
Name: "cluster-a",
Type: unifiedresources.ResourceTypeK8sCluster,
})
state := patrolRuntimeState{
readState: &mockReadState{
nodes: []*unifiedresources.NodeView{&nodeView},
vms: []*unifiedresources.VMView{vmView},
containers: []*unifiedresources.ContainerView{ctView},
hosts: []*unifiedresources.HostView{&hostView, &trueNASHostView},
dockerHosts: []*unifiedresources.DockerHostView{&dockerHostView},
dockerCtrs: []*unifiedresources.DockerContainerView{&dockerCtrView},
storage: []*unifiedresources.StoragePoolView{&storageView},
pbs: []*unifiedresources.PBSInstanceView{&pbsView},
pmg: []*unifiedresources.PMGInstanceView{&pmgView},
k8sClusters: []*unifiedresources.K8sClusterView{&k8sView},
},
DockerHosts: []models.DockerHost{
{
ID: "docker-1",
CustomDisplayName: "docker-custom",
Hostname: "docker.local",
Containers: []models.DockerContainer{
{ID: "ctr-1", Name: "web-snapshot"},
},
},
},
Hosts: []models.Host{
{ID: "agent-1", DisplayName: "agent-display", Hostname: "agent.local"},
{ID: "truenas-1", DisplayName: "truenas-display", Hostname: "truenas-main.local", Platform: "truenas"},
},
KubernetesClusters: []models.KubernetesCluster{
{ID: "k8s-1", CustomDisplayName: "cluster-custom", Name: "cluster-a"},
},
unifiedResourceProvider: &mockUnifiedResourceProvider{
getByTypeFunc: func(t unifiedresources.ResourceType) []unifiedresources.Resource {
if t != unifiedresources.ResourceTypePhysicalDisk {
return nil
}
return []unifiedresources.Resource{
{
ID: "disk-1",
Name: "disk-a",
Type: unifiedresources.ResourceTypePhysicalDisk,
PhysicalDisk: &unifiedresources.PhysicalDiskMeta{
DevPath: "/dev/sda",
Model: "Samsung SSD",
},
},
}
},
},
}
counts := patrolRuntimeCountResources(state)
if counts.nodes != 1 || counts.guests != 2 || counts.storage != 2 || counts.docker != 1 || counts.hosts != 1 || counts.truenas != 1 || counts.pbs != 1 || counts.pmg != 1 || counts.kubernetes != 1 {
t.Fatalf("unexpected canonical runtime counts: %#v", counts)
}
if counts.total() != 11 {
t.Fatalf("expected canonical runtime total 11, got %#v", counts)
}
resourceIDs := patrolRuntimeSortedResourceIDs(state)
for _, want := range []string{"node-1", "qemu/100", "lxc/200", "storage-1", "docker-1", "ctr-1", "agent-1", "truenas-1", "pbs-1", "pmg-1", "k8s-1", "disk-1", "/dev/sda"} {
found := false
for _, got := range resourceIDs {
if got == want {
found = true
break
}
}
if !found {
t.Fatalf("expected runtime resource IDs to include %q, got %v", want, resourceIDs)
}
}
known := patrolRuntimeKnownResources(state)
for _, want := range []string{"node-a", "vm-a", "ct-a", "local-zfs", "docker.local", "docker-custom", "web", "web-snapshot", "agent.local", "agent-display", "truenas-main", "truenas-main.local", "truenas-display", "cluster-a", "cluster-custom", "Samsung SSD"} {
if !known[want] {
t.Fatalf("expected known runtime resources to include %q", want)
}
}
}
func TestPatrolUpdateSafetyDockerHosts_UsesReadStateDockerHosts(t *testing.T) {
readStateForDigest := func(digest string) unifiedresources.ReadState {
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
ID: "canonical-host",
Hostname: "canonical.local",
Containers: []models.DockerContainer{
{ID: "app-1", Name: "web", ImageDigest: digest},
},
},
},
})
return registry
}
legacyUnchangedHost := []models.DockerHost{
{
ID: "legacy-host",
Hostname: "legacy.local",
Containers: []models.DockerContainer{
{ID: "legacy-app", Name: "legacy-web", ImageDigest: "sha256:aaa"},
},
},
}
watcher := newUpdateSafetyWatcher()
state := patrolRuntimeState{
readState: readStateForDigest("sha256:aaa"),
DockerHosts: legacyUnchangedHost,
}
if emit, resolve := watcher.Observe(patrolUpdateSafetyDockerHosts(state), time.Unix(1, 0)); len(emit) != 0 || len(resolve) != 0 {
t.Fatalf("expected first read-state observation to establish baseline, got emit=%d resolve=%d", len(emit), len(resolve))
}
state.readState = readStateForDigest("sha256:bbb")
emit, resolve := watcher.Observe(patrolUpdateSafetyDockerHosts(state), time.Unix(2, 0))
if len(resolve) != 0 {
t.Fatalf("expected no resolve on digest change, got %d", len(resolve))
}
if len(emit) != 1 {
t.Fatalf("expected read-state digest change to emit one finding, got %d", len(emit))
}
if emit[0].Key != UpdateSafetyFindingPrefix+":canonical-host/app-1" {
t.Fatalf("expected canonical read-state host/container key, got %q", emit[0].Key)
}
if emit[0].Node != "canonical.local" {
t.Fatalf("expected canonical read-state host name, got %q", emit[0].Node)
}
}
func TestPatrolRuntimeState_WithDerivedProviders_UsesResourceInventoryOnly(t *testing.T) {
state := patrolRuntimeState{
Nodes: []models.Node{
{ID: "node-1", Name: "node-a"},
},
ActiveAlerts: []models.Alert{
{ResourceID: "node-1"},
},
ConnectionHealth: map[string]bool{
"node-1": true,
},
PVEBackups: models.PVEBackups{
BackupTasks: []models.BackupTask{{ID: "bt-1", VMID: 101}},
},
PBSBackups: []models.PBSBackup{{ID: "pb-1", VMID: "101"}},
}
derived := state.withDerivedProviders()
if derived.readState == nil || derived.unifiedResourceProvider == nil {
t.Fatal("expected derived providers to be rebuilt")
}
if got := len(derived.readState.Nodes()); got != 1 {
t.Fatalf("expected 1 node in derived readState, got %d", got)
}
if got := len(derived.readState.VMs()); got != 0 {
t.Fatalf("expected no VM inventory from metadata-only fields, got %d", got)
}
if len(derived.ActiveAlerts) != 1 || derived.ActiveAlerts[0].ResourceID != "node-1" {
t.Fatalf("expected active alerts to remain on patrol runtime state, got %+v", derived.ActiveAlerts)
}
if !derived.ConnectionHealth["node-1"] {
t.Fatalf("expected connection health to remain on patrol runtime state, got %+v", derived.ConnectionHealth)
}
if len(derived.PVEBackups.BackupTasks) != 1 || len(derived.PBSBackups) != 1 {
t.Fatalf("expected backup metadata to remain on patrol runtime state, got %+v / %+v", derived.PVEBackups.BackupTasks, derived.PBSBackups)
}
snapshot := state.resourceSnapshot()
if snapshot.ConnectionHealth == nil {
t.Fatal("expected derived resource snapshot to preserve normalized connection health map")
}
if snapshot.ActiveAlerts == nil {
t.Fatal("expected derived resource snapshot to preserve normalized active alerts slice")
}
if snapshot.RecentlyResolved == nil {
t.Fatal("expected derived resource snapshot to preserve normalized recently resolved slice")
}
}
func TestNewPatrolRuntimeState_NormalizesCollections(t *testing.T) {
state := newPatrolRuntimeState(models.StateSnapshot{})
if state.Nodes == nil {
t.Fatal("expected nodes slice to be normalized")
}
if state.ConnectionHealth == nil {
t.Fatal("expected connection health map to be normalized")
}
if state.ActiveAlerts == nil {
t.Fatal("expected active alerts slice to be normalized")
}
if state.RecentlyResolved == nil {
t.Fatal("expected recently resolved slice to be normalized")
}
}
func TestPatrolService_CurrentPatrolRuntimeState_UsesNormalizedEmptyState(t *testing.T) {
service := &PatrolService{}
state := service.currentPatrolRuntimeState()
if state.Nodes == nil {
t.Fatal("expected nodes slice to be normalized")
}
if state.ConnectionHealth == nil {
t.Fatal("expected connection health map to be normalized")
}
}
func TestFilterStateByScope_WhitespaceInIDs(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
}
// IDs with whitespace should be trimmed
scope := PatrolScope{
ResourceIDs: []string{" n1 "},
ResourceTypes: []string{"node"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 1 {
t.Errorf("expected whitespace-trimmed ID to match, got %d nodes", len(filtered.Nodes))
}
}
func TestFilterStateByScope_EmptyIDsIgnored(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
}
scope := PatrolScope{
ResourceIDs: []string{"", " ", "n1"},
ResourceTypes: []string{"node"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 1 {
t.Errorf("expected empty IDs to be ignored, got %d nodes", len(filtered.Nodes))
}
}
func TestFilterStateByScope_CaseInsensitiveTypes(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", Name: "node-1"}},
}
scope := PatrolScope{
ResourceTypes: []string{"NODE"},
}
filtered := ps.filterStateByScopeState(ps.patrolRuntimeStateForSnapshot(state), scope)
if len(filtered.Nodes) != 1 {
t.Errorf("expected case-insensitive type matching, got %d nodes", len(filtered.Nodes))
}
}
// --- tryStartRun / endRun tests ---
func TestTryStartRun_Basic(t *testing.T) {
ps := NewPatrolService(nil, nil)
if !ps.tryStartRun("full") {
t.Error("expected first tryStartRun to succeed")
}
if ps.tryStartRun("full") {
t.Error("expected second tryStartRun to fail (run in progress)")
}
ps.endRun()
if !ps.tryStartRun("full") {
t.Error("expected tryStartRun to succeed after endRun")
}
ps.endRun()
}
func TestTryStartRun_Concurrent(t *testing.T) {
ps := NewPatrolService(nil, nil)
var wg sync.WaitGroup
var successes int32
var mu sync.Mutex
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if ps.tryStartRun("scoped") {
mu.Lock()
successes++
mu.Unlock()
// Simulate some work
time.Sleep(1 * time.Millisecond)
ps.endRun()
}
}()
}
wg.Wait()
if successes == 0 {
t.Error("expected at least one goroutine to acquire the run")
}
// After all goroutines complete, run should not be in progress
ps.mu.RLock()
inProgress := ps.runInProgress
ps.mu.RUnlock()
if inProgress {
t.Error("expected runInProgress to be false after all goroutines complete")
}
}
// --- Subscribe / Unsubscribe ---
func TestSubscribeUnsubscribe(t *testing.T) {
ps := NewPatrolService(nil, nil)
ch1 := ps.SubscribeToStream()
ch2 := ps.SubscribeToStream()
ps.streamMu.RLock()
count := len(ps.streamSubscribers)
ps.streamMu.RUnlock()
if count != 2 {
t.Errorf("expected 2 subscribers, got %d", count)
}
ps.UnsubscribeFromStream(ch1)
ps.streamMu.RLock()
count = len(ps.streamSubscribers)
ps.streamMu.RUnlock()
if count != 1 {
t.Errorf("expected 1 subscriber after unsubscribe, got %d", count)
}
ps.UnsubscribeFromStream(ch2)
ps.streamMu.RLock()
count = len(ps.streamSubscribers)
ps.streamMu.RUnlock()
if count != 0 {
t.Errorf("expected 0 subscribers after all unsubscribe, got %d", count)
}
}
func TestSubscribeToStream_ReceivesCurrentState(t *testing.T) {
ps := NewPatrolService(nil, nil)
// Set active streaming state
ps.streamMu.Lock()
ps.streamPhase = "analyzing"
ps.currentOutput.WriteString("some output")
ps.streamMu.Unlock()
ch := ps.SubscribeToStream()
defer ps.UnsubscribeFromStream(ch)
// New subscriber should receive a snapshot of the current state.
select {
case event := <-ch:
if event.Type != "snapshot" {
t.Errorf("expected snapshot event first, got %s", event.Type)
}
if event.Phase != "analyzing" {
t.Errorf("expected phase 'analyzing', got %q", event.Phase)
}
if event.Content != "some output" {
t.Errorf("expected 'some output', got %q", event.Content)
}
case <-time.After(100 * time.Millisecond):
t.Error("expected to receive snapshot event on subscribe")
}
}
// --- broadcast ---
func TestBroadcast_MultipleSubscribers(t *testing.T) {
ps := NewPatrolService(nil, nil)
ch1 := ps.SubscribeToStream()
ch2 := ps.SubscribeToStream()
ch3 := ps.SubscribeToStream()
event := PatrolStreamEvent{Type: "test", Content: "hello"}
ps.broadcast(event)
for i, ch := range []chan PatrolStreamEvent{ch1, ch2, ch3} {
select {
case received := <-ch:
if received.Content != "hello" {
t.Errorf("subscriber %d: expected 'hello', got %q", i, received.Content)
}
case <-time.After(100 * time.Millisecond):
t.Errorf("subscriber %d: timed out waiting for event", i)
}
}
ps.UnsubscribeFromStream(ch1)
ps.UnsubscribeFromStream(ch2)
ps.UnsubscribeFromStream(ch3)
}
func TestBroadcast_StaleSubscriberRemoved(t *testing.T) {
ps := NewPatrolService(nil, nil)
// Create a subscriber with a full buffer
ch := ps.SubscribeToStream()
// Fill the buffer (100 capacity)
for i := 0; i < 100; i++ {
ps.broadcast(PatrolStreamEvent{Type: "fill", Content: "x"})
}
// Keep broadcasting without reading; subscriber should eventually be dropped
// after repeated full-buffer backpressure.
for i := 0; i < 30; i++ {
ps.broadcast(PatrolStreamEvent{Type: "overflow"})
}
// Give goroutine time to close the channel
time.Sleep(50 * time.Millisecond)
ps.streamMu.RLock()
_, exists := ps.streamSubscribers[ch]
ps.streamMu.RUnlock()
if exists {
t.Error("expected stale subscriber to be removed")
}
}
func TestSubscribeToStreamFrom_ReplaysBufferedEvents(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.resetStreamForRun("run-1")
// No subscribers: broadcast should still buffer events for replay.
ps.broadcast(PatrolStreamEvent{Type: "phase", Phase: "analyzing"})
ps.broadcast(PatrolStreamEvent{Type: "content", Content: "hello"})
ps.broadcast(PatrolStreamEvent{Type: "tool_start", ToolName: "uptime"})
ps.streamMu.RLock()
lastSeq := ps.streamSeq
ps.streamMu.RUnlock()
if lastSeq < 3 {
t.Fatalf("expected seq to be >= 3, got %d", lastSeq)
}
// Subscribe from seq 1; should replay seq 2 and 3.
ch := ps.SubscribeToStreamFrom(1)
defer ps.UnsubscribeFromStream(ch)
// First replayed event: content
select {
case ev := <-ch:
if ev.Type != "content" {
t.Fatalf("expected content replay, got %s", ev.Type)
}
if ev.Content != "hello" {
t.Fatalf("expected hello content, got %q", ev.Content)
}
if ev.Seq <= 1 {
t.Fatalf("expected seq > 1, got %d", ev.Seq)
}
case <-time.After(100 * time.Millisecond):
t.Fatal("timed out waiting for replayed content event")
}
// Second replayed event: tool_start
select {
case ev := <-ch:
if ev.Type != "tool_start" {
t.Fatalf("expected tool_start replay, got %s", ev.Type)
}
if ev.ToolName != "uptime" {
t.Fatalf("expected tool uptime, got %q", ev.ToolName)
}
if ev.Seq <= 1 {
t.Fatalf("expected seq > 1, got %d", ev.Seq)
}
case <-time.After(100 * time.Millisecond):
t.Fatal("timed out waiting for replayed tool_start event")
}
}
func TestSubscribeToStreamFrom_BufferRotatedSnapshot(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.resetStreamForRun("run-1")
ps.setStreamPhase("analyzing")
// Fill beyond replay buffer to force rotation.
for i := 0; i < 260; i++ {
ps.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
}
ps.streamMu.RLock()
start, end := ps.streamBufferWindowLocked()
ps.streamMu.RUnlock()
if start == 0 || end == 0 || end <= start {
t.Fatalf("expected non-empty buffer window, got start=%d end=%d", start, end)
}
// Ask to resume from a seq that is behind the buffered window.
ch := ps.SubscribeToStreamFrom(start - 1)
defer ps.UnsubscribeFromStream(ch)
select {
case ev := <-ch:
if ev.Type != "snapshot" {
t.Fatalf("expected snapshot, got %s", ev.Type)
}
if ev.ResyncReason != "buffer_rotated" {
t.Fatalf("expected resync_reason buffer_rotated, got %q", ev.ResyncReason)
}
if ev.BufferStart == 0 || ev.BufferEnd == 0 || ev.BufferEnd < ev.BufferStart {
t.Fatalf("expected buffer window in snapshot, got start=%d end=%d", ev.BufferStart, ev.BufferEnd)
}
case <-time.After(100 * time.Millisecond):
t.Fatal("timed out waiting for buffer_rotated snapshot")
}
}
func TestSubscribeToStreamFrom_BufferRotatedSnapshotNotDuplicated(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.resetStreamForRun("run-1")
ps.setStreamPhase("analyzing")
for i := 0; i < 260; i++ {
ps.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
}
ps.streamMu.RLock()
start, _ := ps.streamBufferWindowLocked()
ps.streamMu.RUnlock()
ch := ps.SubscribeToStreamFrom(start - 1)
defer ps.UnsubscribeFromStream(ch)
snapshotCount := 0
read := 0
for read < 25 {
select {
case ev := <-ch:
read++
if ev.Type == "snapshot" {
snapshotCount++
}
case <-time.After(100 * time.Millisecond):
read = 25
}
}
if snapshotCount != 1 {
t.Fatalf("expected exactly 1 snapshot for buffer_rotated resume, got %d", snapshotCount)
}
}
func TestSubscribeToStreamFrom_ChurnDoesNotLeakSubscribers(t *testing.T) {
ps := NewPatrolService(nil, nil)
const workers = 12
const loopsPerWorker = 60
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < loopsPerWorker; i++ {
ch := ps.SubscribeToStreamFrom(0)
ps.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
ps.UnsubscribeFromStream(ch)
}
}()
}
wg.Wait()
// Allow concurrent close/remove paths to settle.
time.Sleep(25 * time.Millisecond)
ps.streamMu.RLock()
subs := len(ps.streamSubscribers)
ps.streamMu.RUnlock()
if subs != 0 {
t.Fatalf("expected no leaked subscribers after churn, got %d", subs)
}
// Sanity: broadcasting after heavy churn should still work.
ps.broadcast(PatrolStreamEvent{Type: "content", Content: "ok"})
}
func TestSubscribeToStreamFrom_RecordsReplayMetrics(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.resetStreamForRun("run-metrics-replay")
ps.setStreamPhase("analyzing")
ps.broadcast(PatrolStreamEvent{Type: "phase", Phase: "analyzing"})
ps.broadcast(PatrolStreamEvent{Type: "content", Content: "hello"})
ps.broadcast(PatrolStreamEvent{Type: "tool_start", ToolName: "uptime"})
m := GetPatrolMetrics()
beforeOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("replay"))
beforeEvents := testutil.ToFloat64(m.streamReplayEvents)
ch := ps.SubscribeToStreamFrom(1)
defer ps.UnsubscribeFromStream(ch)
// Drain replayed events to avoid backpressure affecting subsequent tests.
<-ch
<-ch
afterOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("replay"))
afterEvents := testutil.ToFloat64(m.streamReplayEvents)
if afterOutcome <= beforeOutcome {
t.Fatalf("expected replay outcome counter to increase (before=%f after=%f)", beforeOutcome, afterOutcome)
}
if afterEvents < beforeEvents+2 {
t.Fatalf("expected replay event counter to increase by at least 2 (before=%f after=%f)", beforeEvents, afterEvents)
}
}
func TestSubscribeToStreamFrom_RecordsSnapshotAndMissMetrics(t *testing.T) {
// Snapshot path (buffer rotated)
psSnap := NewPatrolService(nil, nil)
psSnap.resetStreamForRun("run-metrics-snapshot")
psSnap.setStreamPhase("analyzing")
for i := 0; i < 260; i++ {
psSnap.broadcast(PatrolStreamEvent{Type: "content", Content: "x"})
}
psSnap.streamMu.RLock()
start, _ := psSnap.streamBufferWindowLocked()
psSnap.streamMu.RUnlock()
m := GetPatrolMetrics()
beforeSnapshotOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("snapshot"))
beforeReason := testutil.ToFloat64(m.streamResyncReason.WithLabelValues("buffer_rotated"))
chSnap := psSnap.SubscribeToStreamFrom(start - 1)
defer psSnap.UnsubscribeFromStream(chSnap)
<-chSnap // consume snapshot
afterSnapshotOutcome := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("snapshot"))
afterReason := testutil.ToFloat64(m.streamResyncReason.WithLabelValues("buffer_rotated"))
if afterSnapshotOutcome <= beforeSnapshotOutcome {
t.Fatalf("expected snapshot outcome counter to increase (before=%f after=%f)", beforeSnapshotOutcome, afterSnapshotOutcome)
}
if afterReason <= beforeReason {
t.Fatalf("expected buffer_rotated reason counter to increase (before=%f after=%f)", beforeReason, afterReason)
}
// Miss path (resume requested while stream idle and no buffered events)
psMiss := NewPatrolService(nil, nil)
psMiss.resetStreamForRun("run-metrics-miss")
beforeMiss := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("miss"))
chMiss := psMiss.SubscribeToStreamFrom(42)
defer psMiss.UnsubscribeFromStream(chMiss)
afterMiss := testutil.ToFloat64(m.streamResumeOutcome.WithLabelValues("miss"))
if afterMiss <= beforeMiss {
t.Fatalf("expected miss outcome counter to increase (before=%f after=%f)", beforeMiss, afterMiss)
}
}
func TestBroadcast_RecordsBackpressureDropMetric(t *testing.T) {
ps := NewPatrolService(nil, nil)
ch := ps.SubscribeToStream()
defer ps.UnsubscribeFromStream(ch)
m := GetPatrolMetrics()
beforeDrop := testutil.ToFloat64(m.streamSubscriberDrop.WithLabelValues("backpressure"))
for i := 0; i < 100; i++ {
ps.broadcast(PatrolStreamEvent{Type: "fill", Content: "x"})
}
for i := 0; i < 30; i++ {
ps.broadcast(PatrolStreamEvent{Type: "overflow"})
}
time.Sleep(25 * time.Millisecond)
afterDrop := testutil.ToFloat64(m.streamSubscriberDrop.WithLabelValues("backpressure"))
if afterDrop <= beforeDrop {
t.Fatalf("expected backpressure drop metric to increase (before=%f after=%f)", beforeDrop, afterDrop)
}
}
func TestStreamOutput_TruncatesTailAndSnapshotMarksIt(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.resetStreamForRun("run-1")
ps.setStreamPhase("analyzing")
// Write more than the tail buffer can hold.
payload := strings.Repeat("a", patrolStreamMaxOutputBytes+123)
ps.appendStreamContent(payload)
out, phase := ps.GetCurrentStreamOutput()
if phase != "analyzing" {
t.Fatalf("expected phase analyzing, got %q", phase)
}
if len(out) != patrolStreamMaxOutputBytes {
t.Fatalf("expected output len %d, got %d", patrolStreamMaxOutputBytes, len(out))
}
expectedTail := payload[len(payload)-patrolStreamMaxOutputBytes:]
if out != expectedTail {
t.Fatalf("expected tail output to match last %d bytes", patrolStreamMaxOutputBytes)
}
ch := ps.SubscribeToStreamFrom(0)
defer ps.UnsubscribeFromStream(ch)
select {
case ev := <-ch:
if ev.Type != "snapshot" {
t.Fatalf("expected snapshot, got %s", ev.Type)
}
if ev.ContentTruncated == nil || !*ev.ContentTruncated {
t.Fatalf("expected snapshot content_truncated to be true")
}
case <-time.After(100 * time.Millisecond):
t.Fatal("timed out waiting for snapshot")
}
}
// --- generateFindingID ---
func TestGenerateFindingID_Deterministic(t *testing.T) {
id1 := generateFindingID("res-1", "performance", "high-cpu")
id2 := generateFindingID("res-1", "performance", "high-cpu")
if id1 != id2 {
t.Errorf("expected deterministic ID, got %s and %s", id1, id2)
}
}
func TestGenerateFindingID_Different(t *testing.T) {
id1 := generateFindingID("res-1", "performance", "high-cpu")
id2 := generateFindingID("res-2", "performance", "high-cpu")
id3 := generateFindingID("res-1", "reliability", "high-cpu")
id4 := generateFindingID("res-1", "performance", "high-mem")
if id1 == id2 {
t.Error("different resource should produce different ID")
}
if id1 == id3 {
t.Error("different category should produce different ID")
}
if id1 == id4 {
t.Error("different issue should produce different ID")
}
}
func TestGenerateFindingID_Length(t *testing.T) {
id := generateFindingID("res-1", "performance", "high-cpu")
// sha256[:8] = 8 bytes = 16 hex chars
if len(id) != 16 {
t.Errorf("expected 16-char hex ID, got %d chars: %s", len(id), id)
}
}
// --- joinParts (already tested in patrol_test.go, but verify here for completeness) ---
func TestJoinParts_Zero(t *testing.T) {
if joinParts(nil) != "" {
t.Error("expected empty string for nil")
}
}
func TestJoinParts_Three(t *testing.T) {
result := joinParts([]string{"a", "b", "c"})
if result != "a, b, and c" {
t.Errorf("expected 'a, b, and c', got %q", result)
}
}
// --- GetStatus ---
func TestGetStatus_Running(t *testing.T) {
ps := NewPatrolService(nil, nil)
// Simulate a run in progress
ps.mu.Lock()
ps.runInProgress = true
ps.mu.Unlock()
status := ps.GetStatus()
if !status.Running {
t.Error("expected Running to be true when runInProgress is true")
}
ps.mu.Lock()
ps.runInProgress = false
ps.mu.Unlock()
status = ps.GetStatus()
if status.Running {
t.Error("expected Running to be false when runInProgress is false")
}
}
func TestGetStatus_NextPatrolAt(t *testing.T) {
ps := NewPatrolService(nil, nil)
// Set nextScheduledAt so nextPatrolAt is returned
nextTime := time.Now().Add(30 * time.Minute)
ps.mu.Lock()
ps.nextScheduledAt = nextTime
ps.mu.Unlock()
status := ps.GetStatus()
if status.NextPatrolAt == nil {
t.Fatal("expected NextPatrolAt to be calculated")
}
if !status.NextPatrolAt.Equal(nextTime) {
t.Errorf("expected NextPatrolAt %v, got %v", nextTime, *status.NextPatrolAt)
}
}
func TestGetStatus_NextPatrolAt_IndependentOfLastPatrol(t *testing.T) {
// nextScheduledAt should drive NextPatrolAt, not lastFullPatrol + interval.
// This is the scenario that was previously broken: user changes interval
// mid-cycle, lastFullPatrol is old, so lastFullPatrol+newInterval could be in the past.
ps := NewPatrolService(nil, nil)
// Simulate: patrol ran 45 min ago
ps.mu.Lock()
ps.lastFullPatrol = time.Now().Add(-45 * time.Minute)
// But the ticker was just reset with a 15-min interval, so next fire is ~15 min from now
expectedNext := time.Now().Add(15 * time.Minute)
ps.nextScheduledAt = expectedNext
ps.mu.Unlock()
status := ps.GetStatus()
if status.NextPatrolAt == nil {
t.Fatal("expected NextPatrolAt to be set")
}
// NextPatrolAt must be the tracked nextScheduledAt (in the future),
// NOT lastFullPatrol + interval (which would be 30 min in the past).
if !status.NextPatrolAt.Equal(expectedNext) {
t.Errorf("expected NextPatrolAt = %v, got %v", expectedNext, *status.NextPatrolAt)
}
if status.NextPatrolAt.Before(time.Now()) {
t.Errorf("NextPatrolAt should be in the future, got %v", *status.NextPatrolAt)
}
}
func TestGetStatus_NextPatrolAt_NilWhenNotScheduled(t *testing.T) {
// Before the patrol loop starts, nextScheduledAt is zero — no NextPatrolAt should be returned.
ps := NewPatrolService(nil, nil)
status := ps.GetStatus()
if status.NextPatrolAt != nil {
t.Errorf("expected NextPatrolAt to be nil before patrol loop starts, got %v", *status.NextPatrolAt)
}
}
func TestGetStatus_Disabled(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.SetConfig(PatrolConfig{Enabled: false})
// Even if nextScheduledAt is set, disabled patrol should not report NextPatrolAt
ps.mu.Lock()
ps.nextScheduledAt = time.Now().Add(10 * time.Minute)
ps.mu.Unlock()
status := ps.GetStatus()
if status.Enabled {
t.Error("expected Enabled to be false")
}
if status.NextPatrolAt != nil {
t.Error("expected NextPatrolAt to be nil when disabled")
}
}
func TestGetStatus_BlockedReason(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.setBlockedReason("circuit breaker is open")
status := ps.GetStatus()
if status.BlockedReason != "circuit breaker is open" {
t.Errorf("expected blocked reason, got %q", status.BlockedReason)
}
if status.BlockedAt == nil {
t.Error("expected BlockedAt to be set")
}
ps.clearBlockedReason()
status = ps.GetStatus()
if status.BlockedReason != "" {
t.Errorf("expected empty blocked reason after clear, got %q", status.BlockedReason)
}
}
func TestInitialPatrolSchedule_IgnoresRecentScopedErrorRun(t *testing.T) {
now := time.Now()
p := NewPatrolService(nil, nil)
p.runHistoryStore.Add(PatrolRunRecord{
ID: "scoped-error", Type: "scoped", TriggerReason: "alert_fired",
CompletedAt: now.Add(-5 * time.Minute), ErrorCount: 1, Status: "error",
})
next, trigger := p.initialPatrolSchedule(now, 15*time.Minute)
if trigger != TriggerReasonStartup || !next.Equal(now) {
t.Fatalf("scoped error postponed initial full patrol: next=%v trigger=%s", next, trigger)
}
}
func TestInitialPatrolSchedule_ResumesAfterRecentSuccessfulFullRun(t *testing.T) {
now := time.Now()
p := NewPatrolService(nil, nil)
p.runHistoryStore.Add(PatrolRunRecord{
ID: "full-success", Type: "patrol", TriggerReason: "scheduled",
CompletedAt: now.Add(-10 * time.Minute), Status: "healthy",
})
next, trigger := p.initialPatrolSchedule(now, 15*time.Minute)
if trigger != TriggerReasonScheduled || !next.Equal(now.Add(5*time.Minute)) {
t.Fatalf("full patrol cadence was not resumed: next=%v trigger=%s", next, trigger)
}
}
// --- appendStreamContent ---
func TestAppendStreamContent(t *testing.T) {
ps := NewPatrolService(nil, nil)
ch := ps.SubscribeToStream()
defer ps.UnsubscribeFromStream(ch)
ps.appendStreamContent("hello ")
ps.appendStreamContent("world")
output, _ := ps.GetCurrentStreamOutput()
if output != "hello world" {
t.Errorf("expected 'hello world', got %q", output)
}
// Should have broadcast 2 events
for i := 0; i < 2; i++ {
select {
case event := <-ch:
if event.Type != "content" {
t.Errorf("expected content event, got %s", event.Type)
}
case <-time.After(100 * time.Millisecond):
t.Errorf("expected broadcast event %d", i)
}
}
}
// --- setStreamPhase ---
func TestSetStreamPhase_ResetClearsOutput(t *testing.T) {
ps := NewPatrolService(nil, nil)
ps.setStreamPhase("analyzing")
ps.appendStreamContent("some data")
output, phase := ps.GetCurrentStreamOutput()
if phase != "analyzing" {
t.Errorf("expected phase 'analyzing', got %q", phase)
}
if output != "some data" {
t.Errorf("expected 'some data', got %q", output)
}
ps.setStreamPhase("idle")
output, phase = ps.GetCurrentStreamOutput()
if phase != "idle" {
t.Errorf("expected phase 'idle', got %q", phase)
}
// Output is no longer cleared on idle; it is cleared when a new run starts.
}
// --- isResourceOnline (heuristic alert resolution) ---
func TestIsResourceOnline_Node(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{
{ID: "n1", Name: "node-1", Status: "online"},
{ID: "n2", Name: "node-2", Status: "offline"},
},
}
alert := AlertInfo{ResourceID: "n1", ResourceType: "node"}
if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected node n1 to be online")
}
alert = AlertInfo{ResourceID: "n2", ResourceType: "node"}
if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected node n2 to be offline")
}
}
func TestIsResourceOnline_VM(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
VMs: []models.VM{
{ID: "vm1", Name: "vm-1", Status: "running"},
{ID: "vm2", Name: "vm-2", Status: "stopped"},
},
}
alert := AlertInfo{ResourceID: "vm1", ResourceType: "vm"}
if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected VM vm1 to be online")
}
alert = AlertInfo{ResourceID: "vm2", ResourceType: "vm"}
if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected VM vm2 to be offline")
}
}
func TestIsResourceOnline_SystemContainer(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Containers: []models.Container{
{ID: "ct1", Name: "ct-1", Status: "running"},
},
}
alert := AlertInfo{ResourceID: "ct1", ResourceType: "system-container"}
if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected container ct1 to be online")
}
}
func TestIsResourceOnline_AppContainer(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
Containers: []models.DockerContainer{
{ID: "dc1", Name: "web", State: "running"},
{ID: "dc2", Name: "db", State: "exited"},
},
},
},
}
alert := AlertInfo{ResourceID: "dc1", ResourceType: "app-container"}
if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected docker container dc1 to be online")
}
alert = AlertInfo{ResourceID: "dc2", ResourceType: "app-container"}
if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected docker container dc2 to be offline")
}
}
func TestIsResourceOnline_Agent(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Hosts: []models.Host{
{ID: "h1", Hostname: "host-1", DisplayName: "Host One", Status: "online"},
{ID: "h2", Hostname: "host-2", Status: "offline"},
},
}
alert := AlertInfo{ResourceID: "h1", ResourceType: "agent"}
if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected host h1 to be online")
}
alert = AlertInfo{ResourceID: "h2", ResourceType: "agent"}
if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected host h2 to be offline")
}
}
func TestIsResourceOnline_TrueNAS(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Hosts: []models.Host{
{ID: "tn1", Hostname: "truenas-1", DisplayName: "TrueNAS One", Platform: "truenas", Status: "online"},
{ID: "tn2", Hostname: "truenas-2", Platform: "truenas", Status: "offline"},
},
}
alert := AlertInfo{ResourceID: "tn1", ResourceType: "truenas"}
if !ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected TrueNAS tn1 to be online")
}
alert = AlertInfo{ResourceID: "tn2", ResourceType: "truenas"}
if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected TrueNAS tn2 to be offline")
}
}
func TestIsResourceOnline_UnknownType(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{}
alert := AlertInfo{ResourceID: "x", ResourceType: "unknown"}
if ps.isResourceOnlineState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected unknown type to return false")
}
}
// --- getCurrentMetricValue ---
func TestGetCurrentMetricValue_Node(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{
{ID: "n1", Name: "node-1", CPU: 0.75, Memory: models.Memory{Usage: 60.0}},
},
}
alert := AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "cpu"}
val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 75.0 {
t.Errorf("expected CPU 75.0 (0.75 * 100), got %f", val)
}
alert.Type = "memory"
val = ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 60.0 {
t.Errorf("expected memory 60.0, got %f", val)
}
}
func TestGetCurrentMetricValue_NotFound(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{}
alert := AlertInfo{ResourceID: "missing", ResourceType: "node", Type: "cpu"}
val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != -1 {
t.Errorf("expected -1 for not found, got %f", val)
}
}
func TestGetCurrentMetricValue_AppContainer(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
DockerHosts: []models.DockerHost{
{
Containers: []models.DockerContainer{
{ID: "dc1", Name: "web", CPUPercent: 45.0, MemoryPercent: 30.0},
},
},
},
}
alert := AlertInfo{ResourceID: "dc1", ResourceType: "app-container", Type: "cpu"}
val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 45.0 {
t.Errorf("expected docker CPU 45.0, got %f", val)
}
alert.Type = "memory"
val = ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 30.0 {
t.Errorf("expected docker memory 30.0, got %f", val)
}
}
func TestGetCurrentMetricValue_Agent(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Hosts: []models.Host{
{ID: "h1", Hostname: "host-1", CPUUsage: 67.0, Memory: models.Memory{Usage: 54.0}},
},
}
alert := AlertInfo{ResourceID: "h1", ResourceType: "agent", Type: "cpu"}
val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 67.0 {
t.Errorf("expected host CPU 67.0, got %f", val)
}
alert.Type = "memory"
val = ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 54.0 {
t.Errorf("expected host memory 54.0, got %f", val)
}
}
func TestGetCurrentMetricValue_Storage(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Storage: []models.Storage{
{ID: "s1", Name: "local", Usage: 72.5},
},
}
alert := AlertInfo{ResourceID: "s1", ResourceType: "storage", Type: "usage"}
val := ps.getCurrentMetricValueState(alert, patrolRuntimeStateForTest(ps, state))
if val != 72.5 {
t.Errorf("expected storage usage 72.5, got %f", val)
}
}
func TestGetCurrentMetricValue_UsesReadStateWhenLegacySlicesEmpty(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := newPatrolRuntimeState(models.StateSnapshot{})
nodeView := unifiedresources.NewNodeView(&unifiedresources.Resource{
ID: "n1",
Name: "node-1",
Type: unifiedresources.ResourceTypeAgent,
Status: unifiedresources.StatusOnline,
Proxmox: &unifiedresources.ProxmoxData{
NodeName: "node-1",
},
Metrics: &unifiedresources.ResourceMetrics{
CPU: &unifiedresources.MetricValue{Percent: 42},
Memory: &unifiedresources.MetricValue{Percent: 33},
},
})
state.readState = &mockReadState{nodes: []*unifiedresources.NodeView{&nodeView}}
val := ps.getCurrentMetricValueState(AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "cpu"}, state)
if val != 42.0 {
t.Fatalf("expected readState-backed CPU 42.0, got %f", val)
}
val = ps.getCurrentMetricValueState(AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "memory"}, state)
if val != 33.0 {
t.Fatalf("expected readState-backed memory 33.0, got %f", val)
}
}
func TestIsResourceOnline_UsesReadStateWhenLegacySlicesEmpty(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := newPatrolRuntimeState(models.StateSnapshot{})
hostView := unifiedresources.NewHostView(&unifiedresources.Resource{
ID: "h1",
Name: "host-1",
Type: unifiedresources.ResourceTypeAgent,
Status: unifiedresources.StatusOnline,
Agent: &unifiedresources.AgentData{
Hostname: "host-1",
},
})
state.readState = &mockReadState{hosts: []*unifiedresources.HostView{&hostView}}
if !ps.isResourceOnlineState(AlertInfo{ResourceID: "h1", ResourceType: "agent", Type: "offline"}, state) {
t.Fatal("expected host to be online via readState")
}
}
func TestIsResourceOnline_UsesReadStateForTrueNASWhenLegacySlicesEmpty(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := newPatrolRuntimeState(models.StateSnapshot{})
hostView := unifiedresources.NewHostView(&unifiedresources.Resource{
ID: "tn1",
Name: "truenas-1",
Type: unifiedresources.ResourceTypeAgent,
Status: unifiedresources.StatusOnline,
Agent: &unifiedresources.AgentData{
Hostname: "truenas-1",
Platform: "truenas",
},
})
state.readState = &mockReadState{hosts: []*unifiedresources.HostView{&hostView}}
if !ps.isResourceOnlineState(AlertInfo{ResourceID: "tn1", ResourceType: "truenas", Type: "offline"}, state) {
t.Fatal("expected TrueNAS host to be online via readState")
}
}
func TestGetResourceCurrentState_UsesReadStateWhenLegacySlicesEmpty(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := newPatrolRuntimeState(models.StateSnapshot{})
storageView := unifiedresources.NewStoragePoolView(&unifiedresources.Resource{
ID: "s1",
Name: "local",
Type: unifiedresources.ResourceTypeStorage,
Status: unifiedresources.StatusOnline,
Metrics: &unifiedresources.ResourceMetrics{
Disk: &unifiedresources.MetricValue{Percent: 72.5},
},
})
state.readState = &mockReadState{storage: []*unifiedresources.StoragePoolView{&storageView}}
desc := ps.getResourceCurrentStateState(AlertInfo{ResourceID: "s1", ResourceType: "storage"}, state)
if !strings.Contains(desc, "Storage 'local': 72.5% used") {
t.Fatalf("expected readState-backed storage description, got %q", desc)
}
}
func TestAlertResolutionHelpers_UseReadStateForAppContainer(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := newPatrolRuntimeState(models.StateSnapshot{})
parentID := "docker-1"
containerView := unifiedresources.NewDockerContainerView(&unifiedresources.Resource{
ID: "dc-1",
Name: "web",
Type: unifiedresources.ResourceTypeAppContainer,
Status: unifiedresources.StatusOnline,
ParentID: &parentID,
Docker: &unifiedresources.DockerData{
ContainerState: "running",
},
Metrics: &unifiedresources.ResourceMetrics{
CPU: &unifiedresources.MetricValue{Percent: 12.3},
Memory: &unifiedresources.MetricValue{Percent: 45.6},
},
})
state.readState = &mockReadState{dockerCtrs: []*unifiedresources.DockerContainerView{&containerView}}
alert := AlertInfo{ResourceID: "dc-1", ResourceName: "web", ResourceType: "app-container", Type: "cpu"}
if got := ps.getCurrentMetricValueState(alert, state); got != 12.3 {
t.Fatalf("expected readState-backed docker CPU 12.3, got %f", got)
}
if !ps.isResourceOnlineState(AlertInfo{ResourceID: "dc-1", ResourceName: "web", ResourceType: "app-container"}, state) {
t.Fatal("expected docker container to be online via readState")
}
desc := ps.getResourceCurrentStateState(AlertInfo{ResourceID: "dc-1", ResourceName: "web", ResourceType: "app-container"}, state)
if !strings.Contains(desc, "Docker container 'web': CPU 12.3%, Memory 45.6%, State: running") {
t.Fatalf("expected readState-backed docker container description, got %q", desc)
}
}
// --- alert auto-resolve review (model-owned, no local auto-resolution without AI) ---
// Alerts whose trigger condition may have cleared go to model review; without
// a model available, nothing resolves — local state never resolves an alert.
func TestReviewAndResolveAlerts_ClearedConditionsDoNotResolveWithoutAI(t *testing.T) {
ps := NewPatrolService(nil, nil)
resolver := &mockAlertResolver{
alerts: []AlertInfo{
{
ID: "a-usage",
ResourceID: "s1",
ResourceType: "usage",
Type: "usage",
Threshold: 85.0,
Value: 88.0,
StartTime: time.Now().Add(-1 * time.Hour),
},
{
ID: "a-cpu",
ResourceID: "n1",
ResourceType: "node",
Type: "cpu",
Threshold: 90.0,
Value: 95.0,
StartTime: time.Now().Add(-30 * time.Minute),
},
{
ID: "a-offline",
ResourceID: "n1",
ResourceType: "node",
Type: "offline",
StartTime: time.Now().Add(-1 * time.Hour),
},
{
ID: "a-missing",
ResourceID: "gone",
ResourceType: "node",
Type: "cpu",
Threshold: 90.0,
Value: 95.0,
StartTime: time.Now().Add(-30 * time.Minute),
},
},
}
ps.mu.Lock()
ps.alertResolver = resolver
ps.mu.Unlock()
state := models.StateSnapshot{
Storage: []models.Storage{{ID: "s1", Name: "local", Usage: 70.0, Status: "active"}},
Nodes: []models.Node{{ID: "n1", Name: "node-1", CPU: 0.5, Memory: models.Memory{Usage: 40.0}, Status: "online"}},
}
result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
if result != 0 {
t.Errorf("expected no alerts resolved without AI review, got %d", result)
}
if len(resolver.resolved) != 0 {
t.Errorf("expected no alerts resolved without AI review, got %v", resolver.resolved)
}
}
func TestAlertStillFiring_StorageStillHigh(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Storage: []models.Storage{
{ID: "s1", Name: "local", Usage: 90.0},
},
}
alert := AlertInfo{
ID: "a1",
ResourceID: "s1",
ResourceType: "usage",
Type: "usage",
Threshold: 85.0,
Value: 88.0,
StartTime: time.Now().Add(-1 * time.Hour),
}
if !ps.alertStillFiringState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected still-above-threshold storage alert to be treated as still firing")
}
}
// Regression test: node CPU is a 0-1 fraction in the model snapshot and must be
// scaled to a percentage before threshold comparison. Before the fix,
// getCurrentMetricValue returned 0.95 (raw fraction) which was always below the
// 90.0 percentage threshold — under the still-firing gate that would send every
// high-CPU alert to a pointless model review on every run.
func TestAlertStillFiring_CPUScaleRegression(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{
Nodes: []models.Node{
{ID: "n1", Name: "node-1", CPU: 0.95, Memory: models.Memory{Usage: 40.0}},
},
}
alert := AlertInfo{
ID: "a1",
ResourceID: "n1",
ResourceType: "node",
Type: "cpu",
Threshold: 90.0,
Value: 95.0,
StartTime: time.Now().Add(-30 * time.Minute),
}
if !ps.alertStillFiringState(alert, patrolRuntimeStateForTest(ps, state)) {
t.Error("expected 95% CPU against a 90% threshold to be treated as still firing")
}
}
func TestGetCurrentMetricValue_CPUScalePercent(t *testing.T) {
ps := NewPatrolService(nil, nil)
// Node CPU
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", CPU: 0.42}},
}
val := ps.getCurrentMetricValueState(AlertInfo{ResourceID: "n1", ResourceType: "node", Type: "cpu"}, patrolRuntimeStateForTest(ps, state))
if val != 42.0 {
t.Errorf("node CPU: expected 42.0, got %f", val)
}
// VM CPU
state = models.StateSnapshot{
VMs: []models.VM{{ID: "vm1", CPU: 0.88}},
}
val = ps.getCurrentMetricValueState(AlertInfo{ResourceID: "vm1", ResourceType: "vm", Type: "cpu"}, patrolRuntimeStateForTest(ps, state))
if val != 88.0 {
t.Errorf("VM CPU: expected 88.0, got %f", val)
}
// Container CPU
state = models.StateSnapshot{
Containers: []models.Container{{ID: "ct1", CPU: 0.15}},
}
val = ps.getCurrentMetricValueState(AlertInfo{ResourceID: "ct1", ResourceType: "system-container", Type: "cpu"}, patrolRuntimeStateForTest(ps, state))
if val != 15.0 {
t.Errorf("Container CPU: expected 15.0, got %f", val)
}
}
// --- reviewAndResolveAlerts ---
func TestReviewAndResolveAlerts_NilResolver(t *testing.T) {
ps := NewPatrolService(nil, nil)
state := models.StateSnapshot{}
result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
if result != 0 {
t.Errorf("expected 0 resolved with nil resolver, got %d", result)
}
}
func TestReviewAndResolveAlerts_NoActiveAlerts(t *testing.T) {
ps := NewPatrolService(nil, nil)
resolver := &mockAlertResolver{alerts: nil}
ps.mu.Lock()
ps.alertResolver = resolver
ps.mu.Unlock()
state := models.StateSnapshot{}
result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
if result != 0 {
t.Errorf("expected 0 resolved with no alerts, got %d", result)
}
}
func TestReviewAndResolveAlerts_SkipsRecentAlerts(t *testing.T) {
ps := NewPatrolService(nil, nil)
resolver := &mockAlertResolver{
alerts: []AlertInfo{
{
ID: "a1",
ResourceID: "n1",
ResourceType: "node",
Type: "offline",
StartTime: time.Now().Add(-5 * time.Minute), // only 5 minutes old
},
},
}
ps.mu.Lock()
ps.alertResolver = resolver
ps.mu.Unlock()
state := models.StateSnapshot{
Nodes: []models.Node{{ID: "n1", Name: "node-1", Status: "online"}},
}
result := ps.reviewAndResolveAlertsState(nil, patrolRuntimeStateForTest(ps, state), true, "")
if result != 0 {
t.Errorf("expected 0 resolved (alert too recent), got %d", result)
}
}
// --- mockAlertResolver ---
type mockAlertResolver struct {
alerts []AlertInfo
resolved []string
}
func (m *mockAlertResolver) GetActiveAlerts() []AlertInfo {
return m.alerts
}
func (m *mockAlertResolver) ResolveAlert(alertID string) bool {
m.resolved = append(m.resolved, alertID)
return true
}