Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"net/http/httptest"
"os"
"reflect"
"slices"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
gorillaws "github.com/gorilla/websocket"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/api/agentbinding"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
func TestHostAgentStartupWaitsWithinAcceptedReportLease(t *testing.T) {
dir := t.TempDir()
original := issue1654Monitor()
original.hostContinuityStore = config.NewHostContinuityStore(dir, nil)
report := issue1654Report(time.Now().UTC())
report.SequenceID = agentshost.FormatReportSequenceID("startup-report", 1)
host, err := original.ApplyHostReport(report, nil)
if err != nil {
t.Fatal(err)
}
restarted := issue1654Monitor()
restarted.hostContinuityStore = config.NewHostContinuityStore(dir, nil)
if err := restarted.hostContinuityStore.Upsert(config.HostContinuityEntry{
HostID: "removed-agent", Hostname: "removed-host", LastSeen: host.LastSeen,
IntervalSeconds: 30, RemovedAt: host.LastSeen,
}); err != nil {
t.Fatal(err)
}
base := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
restarted.SetResourceStore(base)
if rejected, err := restarted.ApplyHostReport(report, nil); err != nil || rejected.Status != "pending" {
t.Fatalf("duplicate startup report ended the first-report wait: %+v, %v", rejected, err)
}
lease := hostAgentHealthWindow(host.IntervalSeconds)
for _, tc := range []struct {
name string
age time.Duration
want string
}{
{"restart", time.Second, "pending"},
{"deadline", lease, "pending"},
{"no-report-before-deadline", lease + time.Nanosecond, "offline"},
} {
t.Run(tc.name, func(t *testing.T) {
now := host.LastSeen.Add(tc.age)
read := restarted.readStateWithStandaloneHostContinuityAt(base, now)
views := read.Hosts()
if len(views) != 1 {
t.Fatalf("saved hosts = %d, want 1", len(views))
}
status, seen := hostAgentLivenessFromReadStateView(views[0], now)
if status != tc.want || !seen.Equal(host.LastSeen) || views[0].AgentID() != host.ID {
t.Fatalf("restored status/receipt/identity = %s %s %s", status, seen, views[0].AgentID())
}
if tc.want == "pending" {
sighting, ok := views[0].SourceStatus(unifiedresources.SourceAgent)
if !ok || sighting.Status != "pending" || views[0].Status() != unifiedresources.StatusUnknown {
t.Fatalf("saved enrollment became a current observation: %+v", sighting)
}
}
if len(restarted.GetLiveHostsSnapshot()) != 0 || len(base.Hosts()) != 0 {
t.Fatal("read overlay mutated live state")
}
resources := read.(*unifiedresources.MonitorAdapter).GetAll()
if resources[0].Metrics != nil {
t.Fatal("saved enrollment manufactured telemetry before its first report")
}
frontend := models.ConvertResourceToFrontend(monitorResourceToConvertInput(resources[0]))
var delivery map[string]unifiedresources.SourceStatus
if err := json.Unmarshal(frontend.SourceStatus, &delivery); err != nil {
t.Fatal(err)
}
if tc.want == "pending" && (delivery["agent"].Status != "pending" || frontend.Status != "unknown") {
t.Fatalf("realtime projection lost startup delivery: %+v", delivery)
}
if frontend.CPU != nil || frontend.Memory != nil || frontend.Disk != nil || frontend.LastSeen != host.LastSeen.UnixMilli() {
t.Fatal("realtime projection changed the accepted receipt or supplied metrics")
}
})
}
// A real accepted report ends the wait and supplies the live observation.
report.Timestamp = report.Timestamp.Add(time.Second)
report.SequenceID = agentshost.FormatReportSequenceID("startup-report", 2)
if _, err := restarted.ApplyHostReport(report, nil); err != nil {
t.Fatal(err)
}
views := restarted.readStateWithStandaloneHostContinuity(restarted.GetUnifiedReadStateOrSnapshot()).Hosts()
if len(views) != 1 || views[0].Status() != unifiedresources.StatusOnline {
t.Fatal("first accepted report did not bring the saved agent online")
}
if sighting, ok := views[0].SourceStatus(unifiedresources.SourceAgent); !ok || sighting.Status != "online" {
t.Fatalf("first accepted report retained pending delivery: %+v", sighting)
}
}
func TestGuestPollOwnershipRuntimeDoesNotWithdrawIndependentAgentEvidence(t *testing.T) {
c := &guestPollOwnershipClient{cpu: .1}
m := guestPollOwnershipMonitor(t, c)
m.state.UpsertHost(models.Host{ID: "independent", Hostname: "agent", Status: "online", LastSeen: time.Now(), Memory: models.Memory{Total: 1000, Used: 300, Free: 700, Usage: 30}})
hosts := m.state.GetSnapshot().Hosts
key := guestMetadataCacheKey("site-a", "node", 105)
metadata := guestMetadataCacheEntry{osName: "OpenBSD", osInfoFailureCount: guestAgentOSInfoFailureThreshold, osInfoSkip: true, fetchedAt: time.Now().Add(-time.Minute)}
m.guestMetadataCache = map[string]guestMetadataCacheEntry{key: metadata}
memKey := guestMemoryCacheKey("site-a", "node", 105)
memory := agentMemCacheEntry{info: memoryLifecycleSample(4096), fetchedAt: time.Now().Add(-time.Minute)}
deniedKey := guestMemoryCacheKey("site-a", "node", 106)
denied := agentMemCacheEntry{negative: true, permissionDenied: true, fetchedAt: time.Now().Add(-time.Minute)}
apt := pendingUpdatesCache{count: 7, checkedAt: time.Now().Add(-time.Hour), deniedAt: time.Now().Add(-time.Minute)}
m.vmAgentMemCache = map[string]agentMemCacheEntry{memKey: memory, deniedKey: denied}
m.nodePendingUpdatesCache = map[string]pendingUpdatesCache{"site-a-node": apt}
ctx, owner := m.beginPVEGuestPoll(context.Background(), "site-a", c)
if owner == nil {
t.Fatal("registered cycle not admitted")
}
defer m.finishPVEGuestPoll("site-a", owner)
stagePVEGuestEffect(ctx, func() { m.recordGuestSnapshot("site-a", "qemu", "node", 105, GuestMemorySnapshot{Name: "obsolete"}) })
m.setRuntimeContext(context.Background(), nil)
m.publishPVEGuests(ctx, "site-a", nil, nil)
if owner.ctx.Err() == nil || len(m.GetDiagnosticSnapshots().Guests) != 0 || !reflect.DeepEqual(m.state.GetSnapshot().Hosts, hosts) {
t.Fatal("runtime withdrawal admitted obsolete guest evidence or withdrew independent agent memory")
}
if !reflect.DeepEqual(m.guestMetadataCache[key], metadata) || m.vmAgentMemCache[memKey] != memory ||
m.vmAgentMemCache[deniedKey] != denied || m.nodePendingUpdatesCache["site-a-node"] != apt {
t.Fatal("runtime withdrawal erased OS-command safety or renewed accepted cache age")
}
}
func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) {
monitor, err := New(&config.Config{DataPath: t.TempDir()})
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(monitor.Stop)
delivered := make(chan *alerts.Alert, 1)
monitor.SetAlertPushCallback(func(alert *alerts.Alert) {
delivered <- alert
})
manager := monitor.GetAlertManager()
alertConfig := manager.GetConfig()
alertConfig.Enabled = true
alertConfig.ActivationState = alerts.ActivationActive
alertConfig.Schedule.QuietHours.Enabled = false
manager.UpdateConfig(alertConfig)
value := 1.0
manager.CheckHost(models.Host{
ID: "startup-warning-host",
Hostname: "startup-warning-host",
Sensors: models.HostSensorSummary{Custom: []models.HostCustomSensorMetric{{
ID: "apt-upgradable",
Name: "Apt Upgradable",
Value: &value,
Status: "warning",
ObservedAt: time.Now().UTC(),
}}},
})
select {
case alert := <-delivered:
if alert.Type != "custom-sensor" || alert.Level != alerts.AlertLevelWarning {
t.Fatalf("startup alert = %#v, want warning custom-sensor", alert)
}
case <-time.After(time.Second):
t.Fatal("custom sensor warning did not reach external delivery before Monitor.Start")
}
}
func TestRAIDRequiredMembersReportAndReadState(t *testing.T) {
for _, tc := range []struct {
name string
required int
total int
active int
working int
want storagehealth.RiskLevel
}{
{"reported legacy clean spare", 0, 5, 4, 5, storagehealth.RiskHealthy},
{"new mdadm clean spare", 4, 5, 4, 5, storagehealth.RiskHealthy},
{"new mdstat clean spare", 4, 4, 4, 5, storagehealth.RiskHealthy},
{"count deficit despite clean state and spare", 4, 4, 3, 4, storagehealth.RiskCritical},
} {
t.Run(tc.name, func(t *testing.T) {
manager := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(manager.Stop)
cfg := manager.GetConfig()
cfg.Enabled = true
cfg.ActivationState = alerts.ActivationActive
cfg.TimeThresholds = map[string]int{}
manager.UpdateConfig(cfg)
monitor := &Monitor{
state: models.NewState(), alertManager: manager,
hostTokenBindings: make(map[string]string), config: &config.Config{}, rateTracker: NewRateTracker(),
}
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "raid-agent", Version: "test", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "raid-host", Hostname: "linux-raid", Platform: "linux"},
Timestamp: time.Now().UTC(),
RAID: []agentshost.RAIDArray{{Device: "/dev/md1", Level: "raid5", State: "clean", RequiredDevices: tc.required, TotalDevices: tc.total, ActiveDevices: tc.active, WorkingDevices: tc.working, SpareDevices: 1}},
}
payload, err := json.Marshal(report)
if err != nil {
t.Fatal(err)
}
var decoded agentshost.Report
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatal(err)
}
host, err := monitor.ApplyHostReport(decoded, nil)
if err != nil {
t.Fatal(err)
}
stored := monitor.state.GetSnapshot().Hosts
if len(stored) != 1 || len(stored[0].RAID) != 1 {
t.Fatalf("stored host RAID=%+v", stored)
}
record := unifiedresources.HostIngestRecord(stored[0])
view := unifiedresources.NewHostView(&record.Resource)
readback := hostRAIDFromReadStateView(view.RAID())
for _, array := range []models.HostRAIDArray{host.RAID[0], stored[0].RAID[0], readback[0]} {
if array.RequiredDevices != tc.required || array.TotalDevices != tc.total || array.ActiveDevices != tc.active || array.SpareDevices != 1 {
t.Fatalf("ingest/state/canonical readback changed count meaning: %+v", array)
}
if got := storagehealth.AssessHostRAIDArray(array); got.Level != tc.want {
t.Fatalf("assessment after round trip=%+v, want %s", got, tc.want)
}
}
var raidAlerts []alerts.Alert
for _, alert := range manager.GetActiveAlerts() {
if alert.Type == "raid" {
raidAlerts = append(raidAlerts, alert)
}
}
if tc.want == storagehealth.RiskHealthy {
if len(raidAlerts) != 0 || record.Resource.Agent.RAID[0].Risk != nil {
t.Fatalf("healthy spare produced alert or canonical risk: alerts=%+v risk=%+v", raidAlerts, record.Resource.Agent.RAID[0].Risk)
}
} else if len(raidAlerts) != 1 || raidAlerts[0].Level != alerts.AlertLevelCritical || record.Resource.Agent.RAID[0].Risk == nil || record.Resource.Agent.RAID[0].Risk.Level != storagehealth.RiskCritical {
t.Fatalf("real deficit lost alert or canonical risk: alerts=%+v risk=%+v", raidAlerts, record.Resource.Agent.RAID[0].Risk)
}
})
}
}
func TestHostZFSPoolsFromAgentDisksPreservesDatasetFacts(t *testing.T) {
got := hostZFSPoolsFromAgentDisks([]agentshost.Disk{
{Device: "/", Type: "ext4"},
{
Device: "tank",
Type: "zfs",
ZFSDatasets: []agentshost.ZFSDataset{
{
Name: " tank/apps ",
Type: " filesystem ",
Mountpoint: " /tank/apps ",
UsedBytes: 100,
AvailableBytes: 200,
ReferencedBytes: 50,
},
},
},
})
if len(got) != 1 || got[0].Name != "tank" || len(got[0].Datasets) != 1 {
t.Fatalf("host zfs pools = %#v", got)
}
if got[0].Datasets[0].Name != "tank/apps" ||
got[0].Datasets[0].Mountpoint != "/tank/apps" {
t.Fatalf("dataset was not normalized: %#v", got[0].Datasets[0])
}
}
func TestHostZFSPoolsFromAgentDisksRejectsInvalidAndBoundsRows(t *testing.T) {
datasets := []agentshost.ZFSDataset{
{Name: "tank/negative", Type: "filesystem", UsedBytes: -1},
{Name: "other/wrong-pool", Type: "filesystem"},
{Name: strings.Repeat("x", maxHostZFSDatasetNameBytes+1), Type: "filesystem"},
{Name: "tank/long-mount", Type: "filesystem", Mountpoint: strings.Repeat("x", maxHostZFSMountpointBytes+1)},
}
for i := 0; i < maxHostZFSDatasetsPerPool+2; i++ {
datasets = append(datasets, agentshost.ZFSDataset{
Name: fmt.Sprintf("tank/data-%03d", i),
Type: "filesystem",
})
}
got := hostZFSPoolsFromAgentDisks([]agentshost.Disk{{
Device: "tank",
Type: "zfs",
ZFSDatasets: datasets,
}})
if len(got) != 1 || len(got[0].Datasets) != maxHostZFSDatasetsPerPool {
t.Fatalf("bounded host zfs pools = %#v", got)
}
if got[0].Datasets[0].Name != "tank/data-000" {
t.Fatalf("invalid rows were not rejected: %#v", got[0].Datasets[0])
}
}
func TestMonitoringBroadcastCarriesEveryAvailabilityProjection(t *testing.T) {
input := monitorResourceToConvertInput(unifiedresources.Resource{
ID: "agent-core2026",
Type: unifiedresources.ResourceTypeAgent,
Name: "core2026",
Status: unifiedresources.StatusOnline,
Sources: []unifiedresources.DataSource{
unifiedresources.SourceAgent,
unifiedresources.SourceAvailability,
},
AvailabilityChecks: []unifiedresources.AvailabilityData{
{TargetID: "stats-pv", CorrelationState: unifiedresources.AvailabilityCorrelationAttached},
{TargetID: "grafana", CorrelationState: unifiedresources.AvailabilityCorrelationAttached},
},
})
payload := string(input.AvailabilityChecks)
if !strings.Contains(payload, `"targetId":"stats-pv"`) ||
!strings.Contains(payload, `"targetId":"grafana"`) {
t.Fatalf("AvailabilityChecks = %s, want both projected checks", payload)
}
frontend := models.ConvertResourceToFrontend(input)
if string(frontend.AvailabilityChecks) != payload {
t.Fatalf("frontend availabilityChecks = %s, want %s", frontend.AvailabilityChecks, payload)
}
}
func TestBroadcastSlimmingKeepsHostAgentSupersededIdentityResolvable(t *testing.T) {
// A re-enrolled host carries retired canonical spellings. Broadcast
// slimming stops duplicating them into the alias list, but they must stay
// resolvable through supersededIds so alert overrides and identity
// consumers keep following the host across its retired ids.
input := monitorResourceToConvertInput(unifiedresources.Resource{
ID: "agent-core2026",
Type: unifiedresources.ResourceTypeAgent,
Name: "core2026",
Status: unifiedresources.StatusOnline,
Sources: []unifiedresources.DataSource{unifiedresources.SourceAgent},
SupersededCanonicalIDs: []string{"agent-core2026-a1b2c3", "agent-core2026-d4e5f6"},
Agent: &unifiedresources.AgentData{
AgentID: "core2026",
Hostname: "core2026",
},
})
var canonical struct {
Aliases []string `json:"aliases"`
SupersededIDs []string `json:"supersededIds"`
}
if err := json.Unmarshal(input.Canonical, &canonical); err != nil {
t.Fatalf("unmarshal canonicalIdentity: %v (payload %s)", err, input.Canonical)
}
if len(canonical.SupersededIDs) != 2 {
t.Fatalf("supersededIds = %v, want both retired spellings", canonical.SupersededIDs)
}
for _, alias := range canonical.Aliases {
if alias == "agent-core2026-a1b2c3" || alias == "agent-core2026-d4e5f6" {
t.Fatalf("aliases must not duplicate superseded ids, got %v", canonical.Aliases)
}
}
// The live alias vocabulary the host is reachable by must survive the trim.
aliasSet := make(map[string]struct{}, len(canonical.Aliases))
for _, alias := range canonical.Aliases {
aliasSet[alias] = struct{}{}
}
for _, required := range []string{"agent:core2026", "core2026"} {
if _, ok := aliasSet[required]; !ok {
t.Fatalf("aliases = %v, want %q retained", canonical.Aliases, required)
}
}
// The row's own wire id is a self-reference every consumer unions
// independently; it must not travel inside aliases.
if _, ok := aliasSet["agent-core2026"]; ok {
t.Fatalf("aliases = %v, must not duplicate the wire id", canonical.Aliases)
}
}
// The broadcast coalesces an agent that keeps its own registry row into its
// Proxmox node by hostname, and picks each reading's source by freshness
// judged by the configured thresholds, as the registry's own merges are. A
// node polled every two minutes, read ninety seconds after its poll, shows its
// own readings rather than those of its agent, silent for seventy-five
// seconds. Polled at the default cadence, the node is stale at ninety seconds
// too, so the agent's readings stand.
func TestBroadcastCoalescedHostJudgesFreshnessByConfiguredThresholds(t *testing.T) {
now := time.Now().UTC()
for _, tc := range []struct {
name string
interval time.Duration
cpu, memory float64
}{
{"two-minute polls", 2 * time.Minute, 20, 12.5},
{"default polls", 10 * time.Second, 70, 62.5},
} {
t.Run(tc.name, func(t *testing.T) {
m := &Monitor{state: models.NewState(), config: &config.Config{PVEPollingInterval: tc.interval}}
m.state.UpdateNodesForInstance("homelab", []models.Node{{
ID: "homelab-pve3", Name: "pve3", Instance: "homelab", Status: "online", CPU: 0.20,
Memory: models.Memory{Total: 64 << 30, Used: 8 << 30, Free: 56 << 30, Usage: 12.5},
LastSeen: now.Add(-90 * time.Second),
}})
m.state.UpsertHost(models.Host{
ID: "host-pve3", MachineID: "machine-pve3", Hostname: "pve3", Status: "online", CPUUsage: 70,
Memory: models.Memory{Total: 64 << 30, Used: 40 << 30, Free: 24 << 30, Usage: 62.5},
LastSeen: now.Add(-75 * time.Second), IntervalSeconds: 30,
})
m.SetResourceStore(unifiedresources.NewMonitorAdapter(nil))
var rows []models.ResourceFrontend
for _, resource := range m.BuildBroadcastFrontendState().Resources {
if resource.Name == "pve3" {
rows = append(rows, resource)
}
}
if len(rows) != 1 {
t.Fatalf("broadcast shows %d pve3 rows, want the node and its agent coalesced into one", len(rows))
}
if rows[0].CPU == nil || rows[0].Memory == nil {
t.Fatalf("pve3 lost its readings: cpu %+v memory %+v", rows[0].CPU, rows[0].Memory)
}
if rows[0].CPU.Current != tc.cpu || rows[0].Memory.Current != tc.memory {
t.Fatalf("pve3 cpu %.1f%%, memory %.1f%%; want %.1f%% and %.1f%%",
rows[0].CPU.Current, rows[0].Memory.Current, tc.cpu, tc.memory)
}
})
}
}
// A monitor view pairs its listing with the thresholds that registry
// generation was judged by. Thresholds set on the store since its last
// rebuild belong to the next generation, so the view and the seed it hands
// the resources API keep the ones the listing was judged by.
func TestUnifiedViewCarriesItsGenerationsStaleThresholds(t *testing.T) {
now := time.Now().UTC()
m := &Monitor{state: models.NewState(), config: &config.Config{PVEPollingInterval: 2 * time.Minute}}
m.state.UpdateNodesForInstance("homelab", []models.Node{{
ID: "homelab-pve2", Name: "pve2", Instance: "homelab", Status: "online", CPU: 0.30,
Memory: models.Memory{Total: 64 << 30, Used: 32 << 30, Free: 32 << 30, Usage: 50},
LastSeen: now.Add(-90 * time.Second),
}})
adapter := unifiedresources.NewMonitorAdapter(nil)
m.SetResourceStore(adapter)
adapter.PopulateFromSnapshot(m.state.GetSnapshot())
adapter.SetStaleThresholds(map[unifiedresources.DataSource]time.Duration{unifiedresources.SourceProxmox: time.Minute})
resources, _, thresholds := m.UnifiedResourceSnapshotWithStaleThresholds()
if got := thresholds[unifiedresources.SourceProxmox]; got != 4*time.Minute {
t.Fatalf("seed thresholds = %v, want the four minutes its generation was judged by", thresholds)
}
if len(resources) != 1 || resources[0].SourceStatus[unifiedresources.SourceProxmox].Status != "online" {
t.Fatalf("seed = %+v, want pve2 with the Proxmox sighting its generation judged online", resources)
}
}
func TestHostAgentCorrelationKeepsSameNamedProxmoxProvidersDistinct(t *testing.T) {
now := time.Now()
state := &models.State{
Nodes: []models.Node{
{
ID: "node-from-b",
Name: "pve01",
Instance: "instance-b",
ClusterName: "cluster-a",
IsClusterMember: true,
Host: "https://pve01:8006",
LinkedAgentID: "shared-agent-id",
Status: "online",
ConnectionHealth: "healthy",
LastSeen: now,
},
},
}
state.UpdateNodesForInstance("instance-a", []models.Node{
{
ID: "node-from-a",
Name: "pve01",
Instance: "instance-a",
ClusterName: "cluster-a",
IsClusterMember: true,
Host: "https://pve01:8006",
LinkedAgentID: "shared-agent-id",
Status: "unknown",
ConnectionHealth: "unknown",
LastSeen: now.Add(-time.Minute),
},
})
if len(state.Nodes) != 2 {
t.Fatalf("nodes = %#v, want one node per provider instance", state.Nodes)
}
byInstance := make(map[string]models.Node, len(state.Nodes))
for _, node := range state.Nodes {
byInstance[node.Instance] = node
}
if byInstance["instance-a"].ID != "node-from-a" || byInstance["instance-b"].ID != "node-from-b" {
t.Fatalf("nodes = %#v, want stable instance-scoped identities", state.Nodes)
}
}
func TestNormalizeAgentMemoryCacheAwareFallbacks(t *testing.T) {
const gib = int64(1024 * 1024 * 1024)
tests := []struct {
name string
total int64
used int64
free int64
cache int64
usage float64
swapTotal int64
swapUsed int64
wantUsed int64
wantUsage float64
wantUnavailable bool
}{
{
name: "explicit agent usage remains authoritative",
total: 8 * gib,
used: 6 * gib,
free: gib / 2,
cache: 2 * gib,
wantUsed: 6 * gib,
wantUsage: 75,
},
{
name: "explicit bytes correct a conflicting reported percentage",
total: 8 * gib,
used: 4 * gib,
free: gib,
usage: 94,
wantUsed: 4 * gib,
wantUsage: 50,
},
{
name: "complete Linux cache evidence derives used without swap",
total: 8 * gib,
free: gib,
cache: 2 * gib,
wantUsed: 5 * gib,
wantUsage: 62.5,
},
{
name: "total and free alone stay unknown",
total: 8 * gib,
free: gib,
wantUnavailable: true,
},
{
name: "reported percentage can establish usage",
total: 8 * gib,
free: gib,
usage: 76,
wantUsed: 8 * gib * 76 / 100,
wantUsage: 76,
},
{
name: "zero swap does not invalidate cache aware memory",
total: 8 * gib,
free: 2 * gib,
cache: gib,
swapTotal: 0,
swapUsed: 0,
wantUsed: 5 * gib,
wantUsage: 62.5,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := normalizeAgentMemory(tt.total, tt.used, tt.free, tt.cache, tt.usage, tt.swapTotal, tt.swapUsed)
if got.UsageUnavailable != tt.wantUnavailable {
t.Fatalf("UsageUnavailable = %t, want %t", got.UsageUnavailable, tt.wantUnavailable)
}
if tt.wantUnavailable {
if got.HasKnownUsage() {
t.Fatalf("HasKnownUsage() = true for unavailable memory: %+v", got)
}
return
}
if got.Used != tt.wantUsed {
t.Fatalf("Used = %d, want %d", got.Used, tt.wantUsed)
}
if got.Usage != tt.wantUsage {
t.Fatalf("Usage = %.2f, want %.2f", got.Usage, tt.wantUsage)
}
})
}
}
func TestApplyHostReportOperationReceiptProtocolReplacesCapabilityAuthority(t *testing.T) {
now := time.Now().UTC()
baseReport := func(version int) agentshost.Report {
return agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "receipt-agent",
Version: "6.0.6",
Type: "unified",
IntervalSeconds: 30,
CommandsEnabled: true,
OperationReceiptVersion: version,
},
Host: agentshost.HostInfo{
ID: "receipt-machine",
Hostname: "receipt-host",
Platform: "linux",
OSName: "debian",
OSVersion: "12",
PackageUpdates: &agentshost.PackageUpdateStatus{
Supported: true, Manager: "apt", InventoryHash: "sha256:" + strings.Repeat("a", 64), PendingCount: 2, CheckedAt: now,
},
StorageCleanup: &agentshost.StorageCleanupStatus{
Supported: true, Provider: "apt-package-cache", Fingerprint: "sha256:" + strings.Repeat("b", 64), ReclaimableBytes: 512 << 20, CheckedAt: now,
},
},
Disks: []agentshost.Disk{{Mountpoint: "/", TotalBytes: 100 << 30, UsedBytes: 95 << 30, FreeBytes: 5 << 30, Usage: 95}},
Timestamp: now,
}
}
capabilityNames := func(host models.Host) map[string]bool {
names := map[string]bool{}
for _, capability := range unifiedresources.HostIngestRecord(host).Resource.Capabilities {
names[capability.Name] = true
}
return names
}
t.Run("legacy absent", func(t *testing.T) {
encoded, err := json.Marshal(baseReport(0))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(encoded), "operationReceiptVersion") {
t.Fatalf("legacy fixture unexpectedly contains receipt protocol: %s", encoded)
}
var legacy agentshost.Report
if err := json.Unmarshal(encoded, &legacy); err != nil {
t.Fatal(err)
}
monitor := newTestMonitor(t)
host, err := monitor.ApplyHostReport(legacy, &config.APITokenRecord{ID: "receipt-token"})
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.OperationReceiptVersion != 0 || len(capabilityNames(host)) != 0 {
t.Fatalf("legacy absent protocol gained capability authority: version=%d capabilities=%#v", host.OperationReceiptVersion, capabilityNames(host))
}
})
for _, test := range []struct {
name string
version int
want bool
}{
{name: "explicit zero unsupported", version: 0, want: false},
{name: "current supported", version: operationreceipt.ProtocolVersion, want: true},
{name: "future unsupported", version: operationreceipt.ProtocolVersion + 1, want: false},
} {
t.Run(test.name, func(t *testing.T) {
monitor := newTestMonitor(t)
host, err := monitor.ApplyHostReport(baseReport(test.version), &config.APITokenRecord{ID: "receipt-token"})
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.OperationReceiptVersion != test.version {
t.Fatalf("ingested receipt version = %d, want %d", host.OperationReceiptVersion, test.version)
}
names := capabilityNames(host)
got := names["install_os_updates"] && names["clean_package_cache"] && len(names) == 2
if got != test.want {
t.Fatalf("derived capabilities = %#v, want both=%v", names, test.want)
}
})
}
monitor := newTestMonitor(t)
compatible, err := monitor.ApplyHostReport(baseReport(operationreceipt.ProtocolVersion), &config.APITokenRecord{ID: "receipt-token"})
if err != nil || len(capabilityNames(compatible)) != 2 {
t.Fatalf("compatible ingest: capabilities=%#v err=%v", capabilityNames(compatible), err)
}
downgradedReport := baseReport(0)
downgradedReport.Timestamp = now.Add(time.Second)
downgraded, err := monitor.ApplyHostReport(downgradedReport, &config.APITokenRecord{ID: "receipt-token"})
if err != nil {
t.Fatalf("downgrade ApplyHostReport: %v", err)
}
if downgraded.OperationReceiptVersion != 0 || len(capabilityNames(downgraded)) != 0 {
t.Fatalf("downgrade retained capability authority: version=%d capabilities=%#v", downgraded.OperationReceiptVersion, capabilityNames(downgraded))
}
}
func TestMonitor_GetConfiguredHostIPs_ResolvesOutsideMonitorLock(t *testing.T) {
originalLookup := lookupConfiguredHostIP
lookupStarted := make(chan struct{})
releaseLookup := make(chan struct{})
lookupConfiguredHostIP = func(host string) ([]net.IP, error) {
close(lookupStarted)
<-releaseLookup
return []net.IP{net.ParseIP("192.0.2.10")}, nil
}
defer func() {
lookupConfiguredHostIP = originalLookup
}()
m := &Monitor{
config: &config.Config{
PVEInstances: []config.PVEInstance{
{Host: "pve.example.test"},
},
},
}
done := make(chan []string, 1)
go func() {
done <- m.getConfiguredHostIPs()
}()
select {
case <-lookupStarted:
case <-time.After(time.Second):
t.Fatal("timed out waiting for configured host lookup to start")
}
writerAcquired := make(chan struct{})
go func() {
m.mu.Lock()
close(writerAcquired)
m.mu.Unlock()
}()
select {
case <-writerAcquired:
case <-time.After(time.Second):
close(releaseLookup)
t.Fatal("configured host lookup held the monitor lock while resolving hostnames")
}
close(releaseLookup)
ips := <-done
if len(ips) != 1 || ips[0] != "192.0.2.10" {
t.Fatalf("unexpected resolved host IPs: %#v", ips)
}
}
func TestMonitor_DiscoveryConfigSnapshot_MergesConfiguredHostIPs(t *testing.T) {
m := &Monitor{
config: &config.Config{
Discovery: config.DiscoveryConfig{
IPBlocklist: []string{"10.0.0.8"},
},
PVEInstances: []config.PVEInstance{
{Host: "https://192.168.1.10:8006"},
},
PBSInstances: []config.PBSInstance{
{Host: "http://192.168.1.20:8007"},
},
},
}
cfg := m.discoveryConfigSnapshot()
ipMap := make(map[string]bool, len(cfg.IPBlocklist))
for _, ip := range cfg.IPBlocklist {
ipMap[ip] = true
}
for _, expected := range []string{"10.0.0.8", "192.168.1.10", "192.168.1.20"} {
if !ipMap[expected] {
t.Fatalf("expected discovery IP blocklist to include %s, got %#v", expected, cfg.IPBlocklist)
}
}
}
func TestFindLinkedProxmoxEntity_MatchesCanonicalReadStateViews(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
}
monitor.state.UpdateNodes([]models.Node{
{ID: "node-1", Name: "pve-a", Instance: "pve1"},
})
monitor.state.UpdateVMs([]models.VM{
{ID: "vm-100", Name: "vm-a", Instance: "pve1", VMID: 100},
})
monitor.state.UpdateContainers([]models.Container{
{ID: "ct-200", Name: "ct-a", Instance: "pve1", VMID: 200},
})
nodeID, vmID, ctID := monitor.findLinkedProxmoxEntity("pve-a")
if nodeID != "node-1" || vmID != "" || ctID != "" {
t.Fatalf("expected node match only, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
nodeID, vmID, ctID = monitor.findLinkedProxmoxEntity("vm-a")
if nodeID != "" || vmID != "vm-100" || ctID != "" {
t.Fatalf("expected vm match only, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
nodeID, vmID, ctID = monitor.findLinkedProxmoxEntity("ct-a")
if nodeID != "" || vmID != "" || ctID != "ct-200" {
t.Fatalf("expected container match only, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
}
func TestSnapshotBackedUnifiedReadStateUsesConfiguredStaleThreshold(t *testing.T) {
seen := time.Now().UTC().Add(-90 * time.Second).Truncate(time.Millisecond)
state := models.NewState()
state.UpdateVMs([]models.VM{{
ID: "cluster-a:pve-a:101",
Name: "db",
Node: "pve-a",
Instance: "cluster-a",
VMID: 101,
Status: "running",
Type: "qemu",
LastSeen: seen,
}})
monitor := &Monitor{
config: &config.Config{
PVEPollingInterval: 5 * time.Minute,
},
state: state,
}
readState := monitor.snapshotBackedAgentLookupReadState()
if readState == nil {
t.Fatal("expected snapshot-backed read state")
}
vms := readState.VMs()
if len(vms) != 1 {
t.Fatalf("VM count = %d, want 1", len(vms))
}
if vms[0].Status() != unifiedresources.StatusOnline {
t.Fatalf("VM status = %q, want online", vms[0].Status())
}
}
func TestFindLinkedProxmoxEntity_AmbiguousNodeNameReturnsNoLink(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
}
monitor.state.UpdateNodes([]models.Node{
{ID: "node-1", Name: "pve", Instance: "pve-a"},
{ID: "node-2", Name: "pve", Instance: "pve-b"},
})
nodeID, vmID, ctID := monitor.findLinkedProxmoxEntity("pve")
if nodeID != "" || vmID != "" || ctID != "" {
t.Fatalf("expected ambiguous node name to produce no link, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
}
func TestFindLinkedProxmoxEntityWithHints_UsesEndpointIPToDisambiguateNodes(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
}
monitor.state.UpdateNodes([]models.Node{
{ID: "node-1", Name: "pve", Instance: "pve-a", Host: "https://10.0.0.1:8006"},
{ID: "node-2", Name: "pve", Instance: "pve-b", Host: "https://10.0.0.2:8006"},
})
nodeID, vmID, ctID := monitor.findLinkedProxmoxEntityWithHints("pve", "10.0.0.2", nil)
if nodeID != "node-2" || vmID != "" || ctID != "" {
t.Fatalf("expected endpoint IP to disambiguate node-2, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
}
func TestFindLinkedProxmoxEntityWithHints_UsesExactEndpointHostnameBeforeNameFallback(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
}
monitor.state.UpdateNodes([]models.Node{
{ID: "node-1", Name: "pve", Instance: "pve-a", Host: "https://pve-a.lab:8006"},
{ID: "node-2", Name: "pve", Instance: "pve-b", Host: "https://pve-b.lab:8006"},
})
nodeID, vmID, ctID := monitor.findLinkedProxmoxEntityWithHints("pve-b.lab", "", nil)
if nodeID != "node-2" || vmID != "" || ctID != "" {
t.Fatalf("expected endpoint hostname to disambiguate node-2, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
}
// The v6.4.0-rc.12 state reconciliation fix keeps standalone providers with
// the same native node name separate, but the later agent report must still be
// able to attach to the right node. In the common case the configured endpoint
// is an FQDN while Proxmox and the agent both report the short hostname. The
// provider-observed node interface is the remaining machine-local evidence.
// Refs #1753.
func TestFindLinkedProxmoxEntityWithHints_UsesUniqueNodeNetworkToDisambiguateFQDNEndpoints(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
}
monitor.state.UpdateNodes([]models.Node{
{
ID: "staging-pve01",
Name: "pve01",
Instance: "staging",
Host: "https://pve01.staging.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "vmbr0", Addresses: []string{"192.0.2.11/24"}},
{Name: "docker0", Addresses: []string{"172.17.0.1/16"}},
},
},
{
ID: "production-pve01",
Name: "pve01",
Instance: "production",
Host: "https://pve01.production.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "vmbr0", Addresses: []string{"198.51.100.21/24"}},
{Name: "docker0", Addresses: []string{"172.17.0.1/16"}},
},
},
})
nodeID, vmID, ctID := monitor.findLinkedProxmoxEntityWithHints(
"pve01",
"",
[]agentshost.NetworkInterface{
{Name: "vmbr0", Addresses: []string{"198.51.100.21/24"}},
{Name: "docker0", Addresses: []string{"172.17.0.1/16"}},
},
)
if nodeID != "production-pve01" || vmID != "" || ctID != "" {
t.Fatalf("expected unique provider network evidence to select production node, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
}
func TestFindLinkedProxmoxEntityWithHints_SharedNodeNetworkRemainsAmbiguous(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
}
monitor.state.UpdateNodes([]models.Node{
{
ID: "staging-pve01",
Name: "pve01",
Instance: "staging",
Host: "https://pve01.staging.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "vmbr0", Addresses: []string{"192.168.1.11/24"}},
},
},
{
ID: "production-pve01",
Name: "pve01",
Instance: "production",
Host: "https://pve01.production.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "vmbr0", Addresses: []string{"192.168.1.11/24"}},
},
},
})
nodeID, vmID, ctID := monitor.findLinkedProxmoxEntityWithHints(
"pve01",
"",
[]agentshost.NetworkInterface{{Name: "vmbr0", Addresses: []string{"192.168.1.11/24"}}},
)
if nodeID != "" || vmID != "" || ctID != "" {
t.Fatalf("shared cross-site address must remain ambiguous, got node=%q vm=%q ct=%q", nodeID, vmID, ctID)
}
}
func TestHostSensorsFromReadStateViewPreservesTypedSensorData(t *testing.T) {
customValue := 12.5
observedAt := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
eventAt := observedAt.Add(-2 * time.Hour)
sensors := &unifiedresources.HostSensorMeta{
Custom: []unifiedresources.HostCustomSensorMetric{{
ID: "queue_depth",
Name: "Queue depth",
Group: "Main server",
Subgroup: "Backup",
Kind: "timestamp",
Unit: "items",
Value: &customValue,
Status: "warning",
ObservedAt: observedAt,
EventAt: &eventAt,
AlertOnError: true,
}},
SMART: []unifiedresources.HostSMARTMeta{
{
Device: "/dev/sda",
Model: "CT240BX500SSD1",
Serial: "SATA-SERIAL-1",
Type: "sata",
SizeBytes: 240_057_409_536,
Temperature: 32,
Health: "PASSED",
},
},
}
got := hostSensorsFromReadStateView(sensors)
if len(got.SMART) != 1 {
t.Fatalf("SMART row count = %d, want 1", len(got.SMART))
}
if got.SMART[0].SizeBytes != 240_057_409_536 {
t.Fatalf("SMART SizeBytes = %d, want 240057409536", got.SMART[0].SizeBytes)
}
if got.SMART[0].Temperature != 32 {
t.Fatalf("SMART temperature = %d, want 32", got.SMART[0].Temperature)
}
if len(got.Custom) != 1 || got.Custom[0].Value == nil || *got.Custom[0].Value != customValue {
t.Fatalf("custom sensors = %+v, want one typed reading", got.Custom)
}
if got.Custom[0].ObservedAt != observedAt || got.Custom[0].EventAt == nil ||
!got.Custom[0].EventAt.Equal(eventAt) || got.Custom[0].Group != "Main server" ||
got.Custom[0].Subgroup != "Backup" || got.Custom[0].Kind != "timestamp" ||
got.Custom[0].Status != "warning" || !got.Custom[0].AlertOnError {
t.Fatalf("custom sensor metadata was not preserved: %+v", got.Custom[0])
}
customValue = 99
if *got.Custom[0].Value != 12.5 {
t.Fatalf("custom sensor value was not cloned: %v", *got.Custom[0].Value)
}
eventAt = eventAt.Add(time.Hour)
if got.Custom[0].EventAt.Equal(eventAt) {
t.Fatal("custom sensor event time was not cloned")
}
}
func TestApplyDockerReportNormalizesContainerCPUCapacityAcceptedIngestProof(t *testing.T) {
monitor := newTestMonitor(t)
token := &config.APITokenRecord{ID: "token-docker-cpu", Name: "Docker Token"}
host, err := monitor.ApplyDockerReport(agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "docker-cpu-agent",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-cpu-host",
MachineID: "docker-cpu-machine",
TotalCPU: 4,
TotalMemoryBytes: 8 << 30,
},
Containers: []agentsdocker.Container{{
ID: "container-cpu",
Name: "api",
Image: "nginx:latest",
State: "running",
Status: "Up 1 minute",
CPUPercent: 240,
}},
Timestamp: time.Now().UTC(),
}, token)
if err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
if len(host.Containers) != 1 {
t.Fatalf("expected one reported container, got %d", len(host.Containers))
}
if got := host.Containers[0].CPUCapacityPercent; got != 60 {
t.Fatalf("reported container CPU capacity percent = %v, want 60", got)
}
points := monitor.metricsHistory.GetGuestMetrics("docker:"+host.Containers[0].ID, "cpu", time.Hour)
if len(points) != 1 || points[0].Value != 60 {
t.Fatalf("metrics history Docker CPU points = %+v, want one normalized value 60", points)
}
}
func TestApplyDockerReportRefreshesUnifiedReadStateWithoutBroadcast(t *testing.T) {
monitor := newTestMonitor(t)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
monitor.SetResourceStore(adapter)
_, err := monitor.ApplyDockerReport(agentsdocker.Report{
Agent: agentsdocker.AgentInfo{ID: "headless-docker-agent", IntervalSeconds: 30},
Host: agentsdocker.HostInfo{
Hostname: "headless-docker-host",
MachineID: "headless-docker-machine",
DockerVersion: "27.0.0",
TotalCPU: 4,
TotalMemoryBytes: 8 << 30,
},
Containers: []agentsdocker.Container{{
ID: "headless-container-id",
Name: "headless-canary-container",
State: "running",
HealthcheckTargets: []string{"dependency-service"},
}},
Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
var foundHost bool
for _, host := range adapter.DockerHosts() {
if host != nil && host.Hostname() == "headless-docker-host" {
foundHost = true
break
}
}
if !foundHost {
t.Fatal("accepted Docker report did not refresh the canonical headless read state")
}
var foundContainer bool
for _, container := range adapter.DockerContainers() {
if container != nil && container.Name() == "headless-canary-container" {
foundContainer = true
targets := container.HealthcheckTargets()
if len(targets) != 1 || targets[0] != "dependency-service" {
t.Fatalf("canonical health-check targets = %v, want dependency-service", targets)
}
break
}
}
if !foundContainer {
t.Fatal("accepted Docker container report did not refresh the canonical headless read state")
}
}
func TestApplyHostReportRefreshesUnifiedReadStateWithoutBroadcast(t *testing.T) {
monitor := newTestMonitor(t)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
monitor.SetResourceStore(adapter)
_, err := monitor.ApplyHostReport(agentshost.Report{
Agent: agentshost.AgentInfo{ID: "headless-host-agent", IntervalSeconds: 30},
Host: agentshost.HostInfo{
ID: "headless-host-machine",
Hostname: "headless-host",
Platform: "linux",
OSName: "debian",
},
Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
for _, host := range adapter.Hosts() {
if host != nil && host.Hostname() == "headless-host" {
return
}
}
t.Fatal("accepted host report did not refresh the canonical headless read state")
}
// An agent reports its Proxmox cluster siblings' sensors by bare node name,
// which another connection's node can share. Ingest records the reporting
// agent, so the readings serve only nodes of that agent's own connection.
func TestApplyHostReportScopesClusterSensorsToReportersConnection(t *testing.T) {
monitor := newTestMonitor(t)
monitor.clusterSensorsCache = make(map[string]clusterSensorsCacheEntry)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
monitor.SetResourceStore(adapter)
monitor.state.UpdateNodesForInstance("siteA", []models.Node{readStateTestNode("siteA", "px1"), readStateTestNode("siteA", "px2")})
monitor.state.UpdateNodesForInstance("siteB", []models.Node{readStateTestNode("siteB", "px1")})
adapter.PopulateFromSnapshot(monitor.state.GetSnapshot())
host, err := monitor.ApplyHostReport(agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-px2", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "machine-px2", Hostname: "px2", Platform: "linux"},
ClusterSensors: clusterSensorReport("px1", 88),
Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.LinkedNodeID != "siteA-px2" {
t.Fatalf("reporting agent linked to %q, want siteA-px2", host.LinkedNodeID)
}
if temp := monitor.getHostAgentTemperatureForNode(readStateTestNode("siteA", "px1")); temp == nil || temp.CPUPackage != 88 {
t.Fatalf("siteA px1 temperature = %#v, want the sibling reading 88", temp)
}
if temp := monitor.getHostAgentTemperatureForNode(readStateTestNode("siteB", "px1")); temp != nil {
t.Fatalf("siteB px1 took site A's cluster sibling reading: %#v", temp)
}
}
func TestApplyHostReportProjectsAndPreservesLibvirtInventory(t *testing.T) {
monitor := newTestMonitor(t)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
monitor.SetResourceStore(adapter)
now := time.Now().UTC()
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "libvirt-host-agent", IntervalSeconds: 30},
Host: agentshost.HostInfo{
ID: "libvirt-host-machine",
MachineID: "libvirt-host-machine",
Hostname: "kvm-host",
Platform: "linux",
},
Libvirt: &agentshost.LibvirtInventory{Domains: []agentshost.LibvirtDomain{{
Name: "app",
State: "running",
VCPUs: 4,
MemoryCurrentBytes: 2 << 30,
MemoryMaximumBytes: 4 << 30,
}}},
Timestamp: now,
}
first, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if first.Libvirt == nil ||
len(first.Libvirt.Domains) != 1 ||
first.Libvirt.Domains[0].Name != "app" ||
first.Libvirt.Domains[0].ID == "" {
t.Fatalf("normalized libvirt inventory = %+v", first.Libvirt)
}
firstCollectedAt := first.Libvirt.CollectedAt
vms := adapter.VMs()
if len(vms) != 1 ||
vms[0].Name() != "app" ||
vms[0].RuntimeStatus() != "running" ||
vms[0].CPUs() != 4 {
t.Fatalf("canonical libvirt VM views = %+v", vms)
}
report.Libvirt = nil
report.Timestamp = now.Add(time.Second)
second, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport after collection failure: %v", err)
}
if second.Libvirt == nil ||
len(second.Libvirt.Domains) != 1 ||
!second.Libvirt.CollectedAt.Equal(firstCollectedAt) {
t.Fatalf("preserved libvirt inventory = %+v", second.Libvirt)
}
}
func TestUnifiedAgentHostAndDockerReportsShareOneCanonicalMachine(t *testing.T) {
now := time.Now().UTC()
hostReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "dual-mode-agent",
Version: "6.1.1",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "dual-mode-machine",
MachineID: "dual-mode-machine",
Hostname: "docker-lxc.local",
Platform: "linux",
OSName: "debian",
},
Timestamp: now,
}
dockerReport := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "dual-mode-agent",
Version: "6.1.1",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-lxc.local",
MachineID: "dual-mode-machine",
DockerVersion: "27.0.0",
TotalCPU: 4,
TotalMemoryBytes: 8 << 30,
},
Containers: []agentsdocker.Container{{
ID: "workload-1",
Name: "app",
State: "running",
}},
Timestamp: now.Add(time.Second),
}
for _, tc := range []struct {
name string
dockerFirst bool
}{
{name: "host report first"},
{name: "Docker report first", dockerFirst: true},
} {
t.Run(tc.name, func(t *testing.T) {
monitor := newTestMonitor(t)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
monitor.SetResourceStore(adapter)
token := &config.APITokenRecord{ID: "dual-mode-token", Name: "Dual-mode Agent"}
applyHost := func() {
t.Helper()
if _, err := monitor.ApplyHostReport(hostReport, token); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
}
applyDocker := func() {
t.Helper()
if _, err := monitor.ApplyDockerReport(dockerReport, token); err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
}
if tc.dockerFirst {
applyDocker()
applyHost()
} else {
applyHost()
applyDocker()
}
resources := adapter.GetAll()
if len(resources) != 2 {
t.Fatalf("canonical resource count = %d, want one machine and one workload: %#v", len(resources), resources)
}
var machine *unifiedresources.Resource
for i := range resources {
if resources[i].Type == unifiedresources.ResourceTypeAgent {
machine = &resources[i]
break
}
}
if machine == nil {
t.Fatalf("canonical resources did not retain the machine: %#v", resources)
}
if machine.Agent == nil || machine.Docker == nil {
t.Fatalf("machine facets = agent:%v docker:%v, want both", machine.Agent != nil, machine.Docker != nil)
}
if got := unifiedresources.ContractResourceType(*machine); got != unifiedresources.ResourceTypeAgent {
t.Fatalf("machine contract type = %q, want agent", got)
}
hostViews := adapter.Hosts()
dockerViews := adapter.DockerHosts()
if len(hostViews) != 1 || len(dockerViews) != 1 {
t.Fatalf("typed views = hosts:%d docker:%d, want one in each", len(hostViews), len(dockerViews))
}
if hostViews[0].ID() != dockerViews[0].ID() || hostViews[0].ID() != machine.ID {
t.Fatalf("typed view identities diverged: host=%q docker=%q machine=%q", hostViews[0].ID(), dockerViews[0].ID(), machine.ID)
}
})
}
}
func TestDockerOnlyAgentRemainsWorkloadOnly(t *testing.T) {
monitor := newTestMonitor(t)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
monitor.SetResourceStore(adapter)
_, err := monitor.ApplyDockerReport(agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "workload-only-agent",
Version: "6.1.1",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "runtime-only.local",
MachineID: "runtime-only-machine",
DockerVersion: "27.0.0",
TotalCPU: 2,
TotalMemoryBytes: 4 << 30,
},
Timestamp: time.Now().UTC(),
}, &config.APITokenRecord{ID: "workload-only-token"})
if err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
resources := adapter.GetAll()
if len(resources) != 1 {
t.Fatalf("canonical resource count = %d, want one Docker runtime: %#v", len(resources), resources)
}
if resources[0].Agent != nil || resources[0].Docker == nil {
t.Fatalf("workload-only facets = agent:%v docker:%v, want Docker only", resources[0].Agent != nil, resources[0].Docker != nil)
}
if got := unifiedresources.ContractResourceType(resources[0]); got != unifiedresources.ResourceType("docker-host") {
t.Fatalf("workload-only contract type = %q, want docker-host", got)
}
if len(adapter.Hosts()) != 0 || len(adapter.DockerHosts()) != 1 {
t.Fatalf("typed views = hosts:%d docker:%d, want workload-only runtime excluded from Hosts", len(adapter.Hosts()), len(adapter.DockerHosts()))
}
}
func TestApplyDockerReportMigratesAppContainerURLToStableNameAcceptedIngestProof(t *testing.T) {
monitor := newTestMonitor(t)
report := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "docker-url-agent",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-url-host",
MachineID: "docker-url-machine",
},
Containers: []agentsdocker.Container{{
ID: "container-runtime-old",
Name: "/homepage",
}},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyDockerReport(report, nil)
if err != nil {
t.Fatalf("initial ApplyDockerReport: %v", err)
}
legacyKey := dockerAppContainerLegacyResourceID(host.ID, "container-runtime-old")
if err := monitor.guestMetadataStore.Set(legacyKey, &config.GuestMetadata{
CustomURL: "https://homepage.internal",
}); err != nil {
t.Fatalf("seed legacy guest metadata: %v", err)
}
report.Timestamp = report.Timestamp.Add(30 * time.Second)
if _, err := monitor.ApplyDockerReport(report, nil); err != nil {
t.Fatalf("metadata migration ApplyDockerReport: %v", err)
}
stableKey := dockerAppContainerMetadataKey(host.ID, "homepage")
stableMeta := monitor.guestMetadataStore.Get(stableKey)
if stableMeta == nil {
t.Fatalf("expected stable metadata at %q", stableKey)
}
if stableMeta.CustomURL != "https://homepage.internal" {
t.Fatalf("stable CustomURL = %q, want legacy URL", stableMeta.CustomURL)
}
}
func TestMonitor_HostAgentConfigUpdatePreservesReportedCommandStateInHostState(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
hostMetadataStore: config.NewHostMetadataStore(t.TempDir(), nil),
config: &config.Config{},
}
monitor.state.UpsertHost(models.Host{
ID: "host-command-policy",
Hostname: "host-command-policy",
CommandsEnabled: true,
})
desired := false
if err := monitor.UpdateHostAgentConfig("host-command-policy", &desired); err != nil {
t.Fatalf("UpdateHostAgentConfig: %v", err)
}
hosts := monitor.state.GetHosts()
if len(hosts) != 1 {
t.Fatalf("expected one host, got %d", len(hosts))
}
if !hosts[0].CommandsEnabled {
t.Fatalf("reported CommandsEnabled should remain true until the agent applies and reports the desired policy")
}
cfg := monitor.GetHostAgentConfig("host-command-policy")
if cfg.CommandsEnabled == nil || *cfg.CommandsEnabled {
t.Fatalf("desired CommandsEnabled = %#v, want false", cfg.CommandsEnabled)
}
if cfg.DesiredConfig == nil {
t.Fatal("expected desired config metadata for command-policy update")
}
}
func TestApplyDockerReport_RecreatedContainerAgentIDKeepsTokenBinding(t *testing.T) {
monitor := newTestMonitor(t)
token := &config.APITokenRecord{ID: "token-recreated-container", Name: "Docker Token"}
firstReport := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "container-machine-id-a",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-lxc",
MachineID: "container-machine-id-a",
},
Timestamp: time.Now().UTC(),
}
host1, err := monitor.ApplyDockerReport(firstReport, token)
if err != nil {
t.Fatalf("first ApplyDockerReport failed: %v", err)
}
recreatedReport := firstReport
recreatedReport.Agent.ID = "container-machine-id-b"
recreatedReport.Host.MachineID = "container-machine-id-b"
recreatedReport.Timestamp = firstReport.Timestamp.Add(time.Minute)
host2, err := monitor.ApplyDockerReport(recreatedReport, token)
if err != nil {
t.Fatalf("recreated container report should keep the existing token binding: %v", err)
}
if host1.ID != host2.ID {
t.Fatalf("expected recreated container to retain host ID %q, got %q", host1.ID, host2.ID)
}
if got := monitor.dockerTokenBindings[token.ID]; got != host1.ID {
t.Fatalf("token binding = %q, want stable host ID %q", got, host1.ID)
}
}
func TestApplyDockerReportTokenConflictUsesModuleCopy(t *testing.T) {
monitor := newTestMonitor(t)
token := &config.APITokenRecord{ID: "token-conflict", Name: "Docker Token"}
firstReport := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{ID: "module-a", Version: "1.0.0", IntervalSeconds: 30},
Host: agentsdocker.HostInfo{
Hostname: "docker-a",
MachineID: "machine-a",
},
Timestamp: time.Now().UTC(),
}
if _, err := monitor.ApplyDockerReport(firstReport, token); err != nil {
t.Fatalf("first ApplyDockerReport failed: %v", err)
}
conflictingReport := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{ID: "module-b", Version: "1.0.0", IntervalSeconds: 30},
Host: agentsdocker.HostInfo{
Hostname: "docker-b",
MachineID: "machine-b",
},
Timestamp: firstReport.Timestamp.Add(time.Minute),
}
_, err := monitor.ApplyDockerReport(conflictingReport, token)
if err == nil {
t.Fatal("expected token conflict")
}
message := err.Error()
if !strings.Contains(message, "Each Docker / Podman module must use a unique API token") {
t.Fatalf("token conflict must use module copy, got %q", message)
}
if strings.Contains(message, "Docker"+" agent") || strings.Contains(message, "Docker / Podman"+" agent") {
t.Fatalf("token conflict must not describe Docker / Podman as a separate agent product: %q", message)
}
}
func TestApplyDockerReportPreservesDockerSwarmNodes(t *testing.T) {
monitor := newTestMonitor(t)
now := time.Now().UTC()
report := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "docker-agent-1",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-host",
MachineID: "machine-docker-1",
Runtime: "docker",
RuntimeVersion: "27.5.1",
Swarm: &agentsdocker.SwarmInfo{
NodeID: "node-manager",
NodeRole: "manager",
LocalState: "active",
ControlAvailable: true,
ClusterID: "cluster-1",
ClusterName: "prod-swarm",
Scope: "cluster",
},
},
Nodes: []agentsdocker.Node{{
ID: " node-manager ",
Hostname: " manager-1 ",
Role: " manager ",
Availability: " active ",
State: " ready ",
Address: " 192.0.2.10 ",
ManagerReachability: " reachable ",
ManagerAddress: " 192.0.2.10:2377 ",
Leader: true,
EngineVersion: " 27.5.1 ",
OS: " linux ",
Architecture: " amd64 ",
NanoCPUs: 8_000_000_000,
MemoryBytes: 32 * 1024 * 1024 * 1024,
Labels: map[string]string{"zone": "rack-a"},
EngineLabels: map[string]string{"engine": "primary"},
CreatedAt: now.Add(-time.Hour),
UpdatedAt: &now,
}},
Timestamp: now,
}
host, err := monitor.ApplyDockerReport(report, nil)
if err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
if len(host.Nodes) != 1 {
t.Fatalf("expected host node inventory, got %+v", host.Nodes)
}
node := host.Nodes[0]
if node.ID != "node-manager" || node.Hostname != "manager-1" || node.Role != "manager" {
t.Fatalf("unexpected host node identity: %+v", node)
}
if !node.Leader || node.ManagerReachability != "reachable" || node.EngineLabels["engine"] != "primary" {
t.Fatalf("expected manager metadata to be preserved, got %+v", node)
}
snapshot := monitor.state.GetSnapshot()
if len(snapshot.DockerHosts) != 1 || len(snapshot.DockerHosts[0].Nodes) != 1 {
t.Fatalf("expected state snapshot to preserve nodes, got %+v", snapshot.DockerHosts)
}
if got := snapshot.DockerHosts[0].Nodes[0].ID; got != "node-manager" {
t.Fatalf("snapshot node id = %q, want node-manager", got)
}
}
func TestEvaluateHostAgentsTriggersOfflineAlert(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-offline"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "offline.local",
DisplayName: "Offline Host",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now().Add(-10 * time.Minute),
})
now := time.Now()
for i := 0; i < 3; i++ {
monitor.evaluateHostAgents(now.Add(time.Duration(i) * time.Second))
}
snapshot := monitor.state.GetSnapshot()
statusUpdated := false
for _, host := range snapshot.Hosts {
if host.ID == hostID {
statusUpdated = true
if got := host.Status; got != "offline" {
t.Fatalf("expected host status offline, got %q", got)
}
}
}
if !statusUpdated {
t.Fatalf("host %q not found in state snapshot", hostID)
}
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || healthy {
t.Fatalf("expected connection health false, got %v (exists=%v)", healthy, ok)
}
alerts := monitor.alertManager.GetActiveAlerts()
found := false
for _, alert := range alerts {
if alert.Type == "host-offline" && alert.ResourceID == "agent:"+hostID {
found = true
break
}
}
if !found {
t.Fatalf("expected host offline alert to remain active")
}
}
func TestSharedSystemAlertCorrelationForHostRequiresReciprocalNodeLink(t *testing.T) {
host := models.Host{ID: "agent-1", LinkedNodeID: "node-1"}
correlation := sharedSystemAlertCorrelationForHost(host, []models.Node{{
ID: "node-1",
Instance: "delly",
LinkedAgentID: "agent-1",
}})
if correlation == nil || correlation.Key != "pve:delly" {
t.Fatalf("expected reciprocal node-agent correlation, got %+v", correlation)
}
for _, nodes := range [][]models.Node{
nil,
{{ID: "node-1", Instance: "delly"}},
{{ID: "node-1", Instance: "delly", LinkedAgentID: "agent-2"}},
{{ID: "node-1", LinkedAgentID: "agent-1"}},
{
{ID: "node-1", Instance: "delly-a", LinkedAgentID: "agent-1"},
{ID: "node-1", Instance: "delly-b", LinkedAgentID: "agent-1"},
},
} {
if got := sharedSystemAlertCorrelationForHost(host, nodes); got != nil {
t.Fatalf("incomplete or conflicting link must fail open, got %+v", got)
}
}
}
func TestEvaluateHostAgentsClearsAlertWhenHostReturns(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-recover"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "recover.local",
DisplayName: "Recover Host",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now().Add(-10 * time.Minute),
})
for i := 0; i < 3; i++ {
monitor.evaluateHostAgents(time.Now().Add(time.Duration(i) * time.Second))
}
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "recover.local",
DisplayName: "Recover Host",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now(),
})
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true after recovery, got %v (exists=%v)", healthy, ok)
}
for _, alert := range monitor.alertManager.GetActiveAlerts() {
if alert.ID == "host-offline-"+hostID {
t.Fatalf("offline alert still active after recovery")
}
}
}
func TestCleanupTrackingMapsClearsStalePVEBackupInventoryScope(t *testing.T) {
now := time.Now()
stale := now.Add(-25 * time.Hour)
fresh := now.Add(-time.Hour)
staleSubject := pveBackupTemplateSubjectKey("pve-stale", "qemu", "node1", 900)
freshSubject := pveBackupTemplateSubjectKey("pve-fresh", "qemu", "node1", 901)
monitor := &Monitor{
lastPVEBackupPoll: map[string]time.Time{
"pve-stale": stale,
"pve-fresh": fresh,
},
pveBackupInventoryReady: map[string]map[string]bool{
"pve-stale": {"qemu": true},
"pve-fresh": {"qemu": true},
},
pveBackupTemplateSubjects: map[string]map[string]struct{}{
"pve-stale": {staleSubject: {}},
"pve-fresh": {freshSubject: {}},
},
}
monitor.cleanupTrackingMaps(now)
if _, ok := monitor.lastPVEBackupPoll["pve-stale"]; ok {
t.Fatalf("expected stale PVE backup poll marker to be removed")
}
if _, ok := monitor.pveBackupInventoryReady["pve-stale"]; ok {
t.Fatalf("expected stale PVE backup inventory readiness to be removed")
}
if _, ok := monitor.pveBackupTemplateSubjects["pve-stale"]; ok {
t.Fatalf("expected stale PVE backup template subjects to be removed")
}
if _, ok := monitor.lastPVEBackupPoll["pve-fresh"]; !ok {
t.Fatalf("expected fresh PVE backup poll marker to remain")
}
if !monitor.pveBackupInventoryReady["pve-fresh"]["qemu"] {
t.Fatalf("expected fresh PVE backup inventory readiness to remain")
}
if _, ok := monitor.pveBackupTemplateSubjects["pve-fresh"][freshSubject]; !ok {
t.Fatalf("expected fresh PVE backup template subject to remain")
}
}
func TestApplyHostReportAllowsTokenReuseAcrossHosts(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
baseReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-one",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-one",
Hostname: "host-one",
Platform: "linux",
OSName: "debian",
OSVersion: "12",
},
Timestamp: now,
Metrics: agentshost.Metrics{
CPUUsagePercent: 1.0,
},
}
token := &config.APITokenRecord{ID: "token-one", Name: "Token One"}
hostOne, err := monitor.ApplyHostReport(baseReport, token)
if err != nil {
t.Fatalf("ApplyHostReport hostOne: %v", err)
}
if hostOne.ID == "" {
t.Fatalf("expected hostOne to have an identifier")
}
secondReport := baseReport
secondReport.Agent.ID = "agent-two"
secondReport.Host.ID = "machine-two"
secondReport.Host.Hostname = "host-two"
secondReport.Timestamp = now.Add(30 * time.Second)
hostTwo, err := monitor.ApplyHostReport(secondReport, token)
if err != nil {
t.Fatalf("ApplyHostReport hostTwo: %v", err)
}
if hostTwo.ID == "" {
t.Fatalf("expected hostTwo to have an identifier")
}
if hostTwo.ID == hostOne.ID {
t.Fatalf("expected different host IDs for different machines, got %q", hostTwo.ID)
}
snapshot := monitor.state.GetSnapshot()
if got := len(snapshot.Hosts); got != 2 {
t.Fatalf("expected 2 hosts in state, got %d", got)
}
}
func TestApplyHostReportPreservesThermalState(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
warningLevel := 1
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "mac-agent", Version: "1.0.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{
ID: "mac-machine",
Hostname: "mac-mini",
Platform: "macos",
},
Timestamp: time.Now().UTC(),
Sensors: agentshost.Sensors{
ThermalState: &agentshost.ThermalState{
Source: "pmset",
Pressure: agentshost.ThermalPressureConstrained,
ThermalWarningLevel: &warningLevel,
LimitsPercent: map[string]int{"cpu_speed_limit": 72},
},
},
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.Sensors.ThermalState == nil {
t.Fatalf("expected thermal state on applied host: %+v", host.Sensors)
}
report.Sensors.ThermalState.LimitsPercent["cpu_speed_limit"] = 99
*report.Sensors.ThermalState.ThermalWarningLevel = 2
snapshot := monitor.state.GetSnapshot()
if len(snapshot.Hosts) != 1 || snapshot.Hosts[0].Sensors.ThermalState == nil {
t.Fatalf("expected one host with thermal state, got %+v", snapshot.Hosts)
}
state := snapshot.Hosts[0].Sensors.ThermalState
if state.Pressure != agentshost.ThermalPressureConstrained {
t.Fatalf("pressure = %q, want constrained", state.Pressure)
}
if got := state.LimitsPercent["cpu_speed_limit"]; got != 72 {
t.Fatalf("thermal limit = %d, want copied value 72", got)
}
if state.ThermalWarningLevel == nil || *state.ThermalWarningLevel != 1 {
t.Fatalf("thermal warning level = %+v, want copied value 1", state.ThermalWarningLevel)
}
}
func TestApplyHostReportUsesReceiptTimeForSkewedAgentClockLiveness(t *testing.T) {
monitor := newTestMonitor(t)
agentClock := time.Now().UTC().Add(-3 * time.Hour)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "clock-skew-agent",
Version: "6.0.3",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "clock-skew-machine",
MachineID: "clock-skew-machine",
Hostname: "clock-skew.local",
Platform: "linux",
},
Timestamp: agentClock,
Metrics: agentshost.Metrics{
CPUUsagePercent: 12,
},
}
before := time.Now().UTC()
host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "clock-skew-token"})
after := time.Now().UTC()
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.LastSeen.Before(before) || host.LastSeen.After(after.Add(time.Second)) {
t.Fatalf("host LastSeen = %s, want server receipt between %s and %s", host.LastSeen, before, after)
}
if !host.LastSeen.After(agentClock.Add(2 * time.Hour)) {
t.Fatalf("host LastSeen followed skewed agent timestamp %s: got %s", agentClock, host.LastSeen)
}
readState := monitor.snapshotBackedAgentLookupReadState()
if readState == nil {
t.Fatal("expected snapshot-backed read state")
}
hosts := readState.Hosts()
if len(hosts) != 1 {
t.Fatalf("host count = %d, want 1", len(hosts))
}
if hosts[0].Status() != unifiedresources.StatusOnline {
t.Fatalf("canonical host status = %q, want online", hosts[0].Status())
}
}
func TestApplyHostReportPreservesAgentLifecycleEvidence(t *testing.T) {
monitor := newTestMonitor(t)
checkedAt := time.Now().UTC().Add(-time.Minute).Truncate(time.Second)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "lifecycle-agent",
Version: "6.0.4",
UpdatedFrom: "6.0.3",
AppliedConfig: &agentshost.ConfigFingerprint{
Version: "pulse-agent-config-v1",
Hash: "sha256:applied",
},
Update: &agentshost.UpdateStatus{
State: "error",
AutoUpdate: true,
AvailableVersion: "6.0.5",
LastCheckedAt: &checkedAt,
LastError: "signature verification failed",
},
Modules: []agentshost.ModuleStatus{{
Name: "docker", Enabled: true, State: "retrying", LastError: "socket unavailable", UpdatedAt: checkedAt,
}},
},
Host: agentshost.HostInfo{ID: "machine-lifecycle", Hostname: "lifecycle.local", Platform: "linux"},
}
host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "lifecycle-token"})
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.AppliedConfig == nil || host.AppliedConfig.Version != "pulse-agent-config-v1" || host.AppliedConfig.Hash != "sha256:applied" {
t.Fatalf("applied config = %+v", host.AppliedConfig)
}
if host.AgentUpdate == nil || host.AgentUpdate.State != "error" || host.AgentUpdate.LastCheckedAt == nil || !host.AgentUpdate.LastCheckedAt.Equal(checkedAt) {
t.Fatalf("agent update = %+v", host.AgentUpdate)
}
if host.AgentUpdate.LastError != "signature verification failed" {
t.Fatalf("last update error = %q", host.AgentUpdate.LastError)
}
if len(host.AgentModules) != 1 || host.AgentModules[0].Name != "docker" || host.AgentModules[0].LastError != "socket unavailable" {
t.Fatalf("agent modules = %+v", host.AgentModules)
}
if host.AgentUpdate.UpdatedFrom != "6.0.3" || host.AgentUpdate.LastSuccessAt == nil {
t.Fatalf("successful restart evidence = %+v", host.AgentUpdate)
}
report.Agent.UpdatedFrom = ""
report.Agent.Update = &agentshost.UpdateStatus{State: "idle", AutoUpdate: true}
host, err = monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "lifecycle-token"})
if err != nil {
t.Fatalf("ApplyHostReport second report: %v", err)
}
if host.AgentUpdate == nil || host.AgentUpdate.UpdatedFrom != "6.0.3" || host.AgentUpdate.LastSuccessAt == nil {
t.Fatalf("second report lost successful restart evidence: %+v", host.AgentUpdate)
}
}
func TestApplyDockerReportUsesReceiptTimeForSkewedAgentClockLiveness(t *testing.T) {
monitor := newTestMonitor(t)
agentClock := time.Now().UTC().Add(-3 * time.Hour)
before := time.Now().UTC()
host, err := monitor.ApplyDockerReport(agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "docker-clock-skew-agent",
Version: "6.0.3",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-clock-skew.local",
MachineID: "docker-clock-skew-machine",
},
Timestamp: agentClock,
}, &config.APITokenRecord{ID: "docker-clock-skew-token"})
after := time.Now().UTC()
if err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
if host.LastSeen.Before(before) || host.LastSeen.After(after.Add(time.Second)) {
t.Fatalf("docker host LastSeen = %s, want server receipt between %s and %s", host.LastSeen, before, after)
}
if !host.LastSeen.After(agentClock.Add(2 * time.Hour)) {
t.Fatalf("docker host LastSeen followed skewed agent timestamp %s: got %s", agentClock, host.LastSeen)
}
}
func TestApplyDockerReportPreservesNativeRuntimeInventory(t *testing.T) {
monitor := newTestMonitor(t)
now := time.Date(2026, 5, 24, 8, 0, 0, 0, time.UTC)
updatedAt := now.Add(time.Minute)
report := agentsdocker.Report{
Agent: agentsdocker.AgentInfo{ID: "docker-agent-1", Version: "1.0.0", IntervalSeconds: 30},
Host: agentsdocker.HostInfo{
Hostname: "docker-host",
Name: "Docker Host",
Runtime: "podman",
RuntimeVersion: "5.0.0",
},
Images: []agentsdocker.Image{{
ID: "sha256:image1",
RepoTags: []string{"repo/app:latest"},
RepoDigests: []string{"repo/app@sha256:abc"},
SizeBytes: 1024,
SharedSizeBytes: 128,
Containers: 2,
CreatedAt: now,
Labels: map[string]string{"tier": "web"},
}},
Volumes: []agentsdocker.Volume{{
Name: "app-data", Driver: "local", Mountpoint: "/var/lib/volumes/app-data", Scope: "local", SizeBytes: 2048, RefCount: 3,
}},
Networks: []agentsdocker.Network{{
ID: "net1", Name: "app-net", Driver: "bridge", Scope: "local", EnableIPv4: true, Attachable: true, Subnets: []agentsdocker.NetworkSubnet{{Subnet: "10.88.0.0/24", Gateway: "10.88.0.1"}},
}},
StorageUsage: &agentsdocker.StorageUsage{
Images: agentsdocker.StorageUsageBucket{TotalCount: 3, ActiveCount: 2, TotalSizeBytes: 4096, ReclaimableBytes: 512},
Volumes: agentsdocker.StorageUsageBucket{TotalCount: 1, ActiveCount: 1, TotalSizeBytes: 2048},
Containers: agentsdocker.StorageUsageBucket{TotalCount: 2, ActiveCount: 2},
},
Secrets: []agentsdocker.Secret{{
ID: "secret1", Name: "api-token", DriverName: "vault", TemplatingDriver: "", CreatedAt: now, UpdatedAt: &updatedAt, Labels: map[string]string{"stack": "ops"},
}},
Configs: []agentsdocker.Config{{
ID: "config1", Name: "nginx-conf", TemplatingDriver: "golang", CreatedAt: now, UpdatedAt: &updatedAt, Labels: map[string]string{"stack": "frontend"},
}},
Timestamp: now,
}
host, err := monitor.ApplyDockerReport(report, nil)
if err != nil {
t.Fatalf("ApplyDockerReport: %v", err)
}
if len(host.Images) != 1 || host.Images[0].ID != "sha256:image1" || host.Images[0].RepoTags[0] != "repo/app:latest" {
t.Fatalf("expected image inventory, got %+v", host.Images)
}
if len(host.Volumes) != 1 || host.Volumes[0].Name != "app-data" || host.Volumes[0].SizeBytes != 2048 {
t.Fatalf("expected volume inventory, got %+v", host.Volumes)
}
if len(host.Networks) != 1 || host.Networks[0].Name != "app-net" || host.Networks[0].Subnets[0].Gateway != "10.88.0.1" {
t.Fatalf("expected network inventory, got %+v", host.Networks)
}
if host.StorageUsage == nil || host.StorageUsage.Images.ReclaimableBytes != 512 {
t.Fatalf("expected storage usage inventory, got %+v", host.StorageUsage)
}
if len(host.Secrets) != 1 || host.Secrets[0].Name != "api-token" || host.Secrets[0].DriverName != "vault" {
t.Fatalf("expected secret metadata inventory, got %+v", host.Secrets)
}
if len(host.Configs) != 1 || host.Configs[0].Name != "nginx-conf" || host.Configs[0].TemplatingDriver != "golang" {
t.Fatalf("expected config metadata inventory, got %+v", host.Configs)
}
}
func TestStatusOnlyDockerSnapshotPreservesPlatformWithoutReplacingModuleEvidence(t *testing.T) {
previous := models.DockerHost{
ID: "helper-host", DisplayName: "Docker node", OS: "linux", KernelVersion: "6.8.0",
Architecture: "amd64", Runtime: "docker", CollectionMode: "typed-helper-summary",
Containers: []models.DockerContainer{{ID: "existing", Name: "existing"}},
}
current := models.DockerHost{
ID: "helper-host",
AgentModules: []models.AgentModuleStatus{{
Name: "typed-privilege-helper", State: "degraded", LastError: "container.inventory: helper provider unavailable",
}},
}
preserveDockerInventorySnapshot(¤t, previous)
if current.KernelVersion != "6.8.0" || current.Runtime != "docker" || current.CollectionMode != "typed-helper-summary" {
t.Fatalf("status-only snapshot lost platform metadata: %+v", current)
}
if len(current.Containers) != 1 || current.Containers[0].ID != "existing" {
t.Fatalf("status-only snapshot replaced inventory: %+v", current.Containers)
}
if len(current.AgentModules) != 1 || current.AgentModules[0].State != "degraded" {
t.Fatalf("status-only snapshot replaced current module evidence: %+v", current.AgentModules)
}
}
func TestApplyDockerReportDerivesCanonicalDockerSecurityPosture(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
rateTracker: NewRateTracker(),
dockerMetadataStore: config.NewDockerMetadataStore(t.TempDir(), nil),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
host, err := monitor.ApplyDockerReport(agentsdocker.Report{
Agent: agentsdocker.AgentInfo{
ID: "agent-secure",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentsdocker.HostInfo{
Hostname: "docker-secure-host",
Runtime: "docker",
Security: &agentsdocker.HostSecurityInfo{
AuthorizationPlugins: []string{"opa", " audit "},
},
},
Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatalf("ApplyDockerReport returned error: %v", err)
}
if host.Security == nil {
t.Fatalf("expected security posture on docker host")
}
if !host.Security.MutatingCommandsBlocked {
t.Fatalf("expected mutating commands to be blocked")
}
if got := host.Security.AuthorizationPlugins; len(got) != 2 || got[0] != "opa" || got[1] != "audit" {
t.Fatalf("expected normalized authorization plugins, got %#v", got)
}
if !strings.Contains(host.Security.MutatingCommandsBlockedReason, "GO-2026-4887") {
t.Fatalf("expected advisory reason, got %q", host.Security.MutatingCommandsBlockedReason)
}
}
func TestHostReportMatchesKnownIdentityUsesPersistedContinuity(t *testing.T) {
now := time.Now().UTC()
store := config.NewHostContinuityStore(t.TempDir(), nil)
if err := store.Upsert(config.HostContinuityEntry{
HostID: "host-1",
ReportHostID: "machine-1",
AgentReportedID: "agent-1",
Hostname: "host-1.local",
MachineID: "machine-1",
TokenID: "token-1",
LastSeen: now,
}); err != nil {
t.Fatalf("Upsert continuity: %v", err)
}
monitor := &Monitor{
state: models.NewState(),
hostContinuityStore: store,
}
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-1",
Version: "6.0.0-rc.1",
},
Host: agentshost.HostInfo{
ID: "machine-1",
MachineID: "machine-1",
Hostname: "host-1.local",
Platform: "linux",
},
Timestamp: now.Add(time.Minute),
}
if !monitor.HostReportMatchesKnownIdentity(report, &config.APITokenRecord{ID: "token-1"}) {
t.Fatal("expected persisted continuity to count as a known host identity")
}
}
func TestApplyHostReportReusesPersistedContinuityAcrossRestart(t *testing.T) {
dir := t.TempDir()
now := time.Now().UTC()
newMonitor := func() *Monitor {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{DataPath: dir},
rateTracker: NewRateTracker(),
hostContinuityStore: config.NewHostContinuityStore(dir, nil),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
return monitor
}
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-1",
Version: "6.0.0-rc.1",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-1",
MachineID: "machine-1",
Hostname: "host-1.local",
Platform: "linux",
},
Timestamp: now,
}
token := &config.APITokenRecord{ID: "token-1", Name: "Token One"}
firstMonitor := newMonitor()
firstHost, err := firstMonitor.ApplyHostReport(report, token)
if err != nil {
t.Fatalf("first ApplyHostReport: %v", err)
}
restartedMonitor := newMonitor()
restartReport := report
restartReport.Timestamp = now.Add(30 * time.Second)
restartedHost, err := restartedMonitor.ApplyHostReport(restartReport, token)
if err != nil {
t.Fatalf("restarted ApplyHostReport: %v", err)
}
if restartedHost.ID != firstHost.ID {
t.Fatalf("expected persisted continuity to preserve host ID %q, got %q", firstHost.ID, restartedHost.ID)
}
}
func TestApplyHostReportReusesTokenBindingAcrossShortFQDNAfterReload(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
token := &config.APITokenRecord{ID: "token-host-continuity", Name: "Host Token"}
monitor.state.UpsertHost(models.Host{
ID: "host-v5",
Hostname: "docker-lxc.lab",
DisplayName: "docker-lxc",
TokenID: token.ID,
Status: "online",
AgentVersion: "5.1.30",
IntervalSeconds: 30,
LastSeen: now.Add(-time.Minute),
})
monitor.hostTokenBindings[token.ID+":docker-lxc.lab"] = "host-v5"
report := agentshost.Report{
Agent: agentshost.AgentInfo{
Version: "6.0.0-rc.4",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
Hostname: "docker-lxc",
Platform: "linux",
OSName: "debian",
},
Timestamp: now,
}
host, err := monitor.ApplyHostReport(report, token)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.ID != "host-v5" {
t.Fatalf("expected equivalent hostname token binding to preserve host ID host-v5, got %q", host.ID)
}
snapshot := monitor.state.GetSnapshot()
if got := len(snapshot.Hosts); got != 1 {
t.Fatalf("expected host report to update existing host instead of creating duplicate, got %d hosts", got)
}
if got := monitor.hostTokenBindings[token.ID+":docker-lxc"]; got != "host-v5" {
t.Fatalf("expected current hostname alias to bind to host-v5, got %q", got)
}
}
func TestApplyHostReportDisambiguatesCollidingIdentifiersAcrossTokens(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
baseReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-one",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "colliding-machine-id",
Hostname: "nas-one",
Platform: "linux",
OSName: "synology",
OSVersion: "7.0",
},
Timestamp: now,
Metrics: agentshost.Metrics{
CPUUsagePercent: 1.0,
},
}
hostOne, err := monitor.ApplyHostReport(baseReport, &config.APITokenRecord{ID: "token-one"})
if err != nil {
t.Fatalf("ApplyHostReport hostOne: %v", err)
}
if hostOne.ID == "" {
t.Fatalf("expected hostOne to have an identifier")
}
secondReport := baseReport
secondReport.Agent.ID = "agent-two"
secondReport.Host.Hostname = "nas-two"
secondReport.Timestamp = now.Add(30 * time.Second)
hostTwo, err := monitor.ApplyHostReport(secondReport, &config.APITokenRecord{ID: "token-two"})
if err != nil {
t.Fatalf("ApplyHostReport hostTwo: %v", err)
}
if hostTwo.ID == "" {
t.Fatalf("expected hostTwo to have an identifier")
}
if hostTwo.ID == hostOne.ID {
t.Fatalf("expected disambiguated host IDs, got %q", hostTwo.ID)
}
hostTwoRepeat, err := monitor.ApplyHostReport(secondReport, &config.APITokenRecord{ID: "token-two"})
if err != nil {
t.Fatalf("ApplyHostReport hostTwo repeat: %v", err)
}
if hostTwoRepeat.ID != hostTwo.ID {
t.Fatalf("expected stable host ID for repeated reports, got %q want %q", hostTwoRepeat.ID, hostTwo.ID)
}
// Removing the first host should not cause the second host to change identity.
if _, err := monitor.RemoveHostAgent(hostOne.ID); err != nil {
t.Fatalf("RemoveHostAgent hostOne: %v", err)
}
hostTwoAfterRemoval, err := monitor.ApplyHostReport(secondReport, &config.APITokenRecord{ID: "token-two"})
if err != nil {
t.Fatalf("ApplyHostReport hostTwo after removal: %v", err)
}
if hostTwoAfterRemoval.ID != hostTwo.ID {
t.Fatalf("expected stable host ID after removal, got %q want %q", hostTwoAfterRemoval.ID, hostTwo.ID)
}
snapshot := monitor.state.GetSnapshot()
if got := len(snapshot.Hosts); got != 1 {
t.Fatalf("expected 1 host in state after removal, got %d", got)
}
}
func TestRemoveHostAgent_DoesNotBlockDistinctLiveHostWithSameHostname(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
staleReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "old-agent",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "shared-machine",
MachineID: "shared-machine",
Hostname: "pve-node",
Platform: "linux",
},
Timestamp: now,
}
liveReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "new-agent",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "shared-machine",
MachineID: "shared-machine",
Hostname: "pve-node",
Platform: "linux",
},
Timestamp: now.Add(30 * time.Second),
}
staleHost, err := monitor.ApplyHostReport(staleReport, &config.APITokenRecord{ID: "old-token"})
if err != nil {
t.Fatalf("ApplyHostReport stale host: %v", err)
}
liveHost, err := monitor.ApplyHostReport(liveReport, &config.APITokenRecord{ID: "new-token"})
if err != nil {
t.Fatalf("ApplyHostReport live host: %v", err)
}
if liveHost.ID == staleHost.ID {
t.Fatalf("expected distinct duplicate host IDs, got %q", liveHost.ID)
}
if _, err := monitor.RemoveHostAgent(staleHost.ID); err != nil {
t.Fatalf("RemoveHostAgent stale host: %v", err)
}
liveReport.Timestamp = now.Add(time.Minute)
liveHostAfterRemoval, err := monitor.ApplyHostReport(liveReport, &config.APITokenRecord{ID: "new-token"})
if err != nil {
t.Fatalf("live host with same hostname should not be blocked by stale removal: %v", err)
}
if liveHostAfterRemoval.ID != liveHost.ID {
t.Fatalf("expected live host ID %q to remain stable, got %q", liveHost.ID, liveHostAfterRemoval.ID)
}
}
func TestRemoveHostAgent_BlocksRemovedTokenMachineAfterIdentifierChurn(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
hostID := "agent-id-before-reinstall"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "blocked-node",
MachineID: "machine-stable",
TokenID: "host-token",
Status: "offline",
LastSeen: now.Add(-time.Minute),
})
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("RemoveHostAgent: %v", err)
}
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-id-after-reinstall",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-stable",
MachineID: "machine-stable",
Hostname: "blocked-node",
Platform: "linux",
},
Timestamp: now,
}
if _, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "host-token"}); err == nil {
t.Fatal("expected stable token and machine identity to remain blocked after host ID changed")
}
}
func TestRemoveHostAgentUnbindsToken(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-to-remove"
tokenID := "token-remove"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "remove.me",
TokenID: tokenID,
})
monitor.hostTokenBindings[tokenID+":remove.me"] = hostID
monitor.hostTokenBindings[tokenID+":remove.me.local"] = hostID
monitor.hostTokenBindings[tokenID] = hostID
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("RemoveHostAgent: %v", err)
}
if _, exists := monitor.hostTokenBindings[tokenID+":remove.me"]; exists {
t.Fatalf("expected token binding to be cleared after host removal")
}
if _, exists := monitor.hostTokenBindings[tokenID]; exists {
t.Fatalf("expected legacy token binding to be cleared after host removal")
}
if _, exists := monitor.hostTokenBindings[tokenID+":remove.me.local"]; exists {
t.Fatalf("expected equivalent hostname token binding to be cleared after host removal")
}
}
func TestRemoveHostAgent_PreservesLinkedGuestIdentityInRemovedState(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-linked-guest"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "guest-host.local",
DisplayName: "guest-host",
LinkedVMID: "101",
LinkedContainerID: "102",
})
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("RemoveHostAgent: %v", err)
}
removedHosts := monitor.state.GetRemovedHostAgents()
if len(removedHosts) != 1 {
t.Fatalf("expected one removed host entry, got %d", len(removedHosts))
}
if removedHosts[0].LinkedVMID != "101" {
t.Fatalf("expected linked VM id to persist, got %q", removedHosts[0].LinkedVMID)
}
if removedHosts[0].LinkedContainerID != "102" {
t.Fatalf(
"expected linked container id to persist, got %q",
removedHosts[0].LinkedContainerID,
)
}
}
func TestRemoveHostAgent_KeepsSharedTokenUsedByDockerRuntime(t *testing.T) {
t.Helper()
tokenID := "shared-token"
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{
APITokens: []config.APITokenRecord{
{ID: tokenID, Name: "Shared Token"},
},
},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-shared"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "shared-host.local",
TokenID: tokenID,
})
monitor.state.UpsertDockerHost(models.DockerHost{
ID: "docker-shared",
Hostname: "docker-shared.local",
TokenID: tokenID,
Status: "online",
})
monitor.hostTokenBindings[tokenID+":shared-host.local"] = hostID
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("RemoveHostAgent: %v", err)
}
if got := len(monitor.config.APITokens); got != 1 {
t.Fatalf("expected shared API token to remain, got %d tokens", got)
}
if monitor.config.APITokens[0].ID != tokenID {
t.Fatalf("expected shared token %q to remain, got %q", tokenID, monitor.config.APITokens[0].ID)
}
}
func TestApplyHostReport_PreservesPreviousTokenMetadata(t *testing.T) {
t.Helper()
lastUsed := time.Now().UTC().Add(-5 * time.Minute)
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
monitor.state.UpsertHost(models.Host{
ID: "host-prev",
Hostname: "preserve.local",
TokenID: "token-prev",
TokenName: "Previous Token",
TokenHint: "prev_1234",
TokenLastUsedAt: &lastUsed,
})
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-prev",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "host-prev",
Hostname: "preserve.local",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.TokenID != "token-prev" || host.TokenName != "Previous Token" || host.TokenHint != "prev_1234" {
t.Fatalf("expected previous token metadata to be preserved, got id=%q name=%q hint=%q", host.TokenID, host.TokenName, host.TokenHint)
}
if host.TokenLastUsedAt == nil || !host.TokenLastUsedAt.Equal(lastUsed) {
t.Fatalf("expected TokenLastUsedAt %v, got %v", lastUsed, host.TokenLastUsedAt)
}
}
func TestApplyHostReportStoresUnraidTopology(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-tower",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-tower",
Hostname: "tower",
MachineID: "machine-tower",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Unraid: &agentshost.UnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
SyncAction: "check",
SyncProgress: 55,
Disks: []agentshost.UnraidDisk{
{Name: "parity", Device: "/dev/sdb", Role: "parity", Status: "online", RawStatus: "DISK_OK", Serial: "SERIAL-PARITY"},
{
Name: "disk1",
Device: "/dev/sdc",
Role: "data",
Status: "online",
RawStatus: "DISK_OK",
Model: "WDC WD60EFRX",
Serial: "SERIAL-DATA",
Filesystem: "xfs",
Transport: "sata",
SizeBytes: 6_000_000_000_000,
UsedBytes: 4_000,
FreeBytes: 2_000,
Temperature: 31,
SpunDown: true,
ReadCount: 11,
WriteCount: 12,
ErrorCount: 16,
Slot: 1,
},
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.Unraid == nil {
t.Fatal("expected unraid topology on host")
}
if !host.Unraid.ArrayStarted || host.Unraid.SyncAction != "check" {
t.Fatalf("unexpected unraid summary %+v", host.Unraid)
}
if len(host.Unraid.Disks) != 2 || host.Unraid.Disks[0].Role != "parity" {
t.Fatalf("unexpected unraid disks %+v", host.Unraid.Disks)
}
disk := host.Unraid.Disks[1]
if disk.Model != "WDC WD60EFRX" || disk.Transport != "sata" || disk.SizeBytes != 6_000_000_000_000 {
t.Fatalf("expected enriched unraid disk metadata, got %+v", disk)
}
if disk.UsedBytes != 4_000 || disk.FreeBytes != 2_000 || disk.Temperature != 31 || !disk.SpunDown {
t.Fatalf("expected native unraid capacity and state fields, got %+v", disk)
}
if disk.ReadCount != 11 || disk.WriteCount != 12 || disk.ErrorCount != 16 {
t.Fatalf("expected native unraid counters, got %+v", disk)
}
}
func TestApplyHostReportNormalizesLegacyUnraidRawStatuses(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-tower-legacy",
Version: "5.1.27",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-tower-legacy",
Hostname: "tower",
MachineID: "machine-tower-legacy",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Unraid: &agentshost.UnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
SyncAction: "check",
SyncProgress: 55,
NumDisabled: 1,
NumInvalid: 1,
Disks: []agentshost.UnraidDisk{
{Name: "parity", Device: "/dev/sdb", Role: "parity", RawStatus: "DISK_OK", Serial: "SERIAL-PARITY"},
{Name: "disk1", Device: "/dev/sdc", Role: "data", RawStatus: "DISK_OK", Serial: "SERIAL-DATA"},
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.Unraid == nil {
t.Fatal("expected unraid topology on host")
}
for _, disk := range host.Unraid.Disks {
if disk.Status != "online" {
t.Fatalf("expected normalized online status from rawStatus, got %+v", host.Unraid.Disks)
}
}
assessment := storagehealth.AssessUnraidStorage(*host.Unraid)
if assessment.Level != storagehealth.RiskWarning {
t.Fatalf("assessment level = %q, want %q", assessment.Level, storagehealth.RiskWarning)
}
for _, reason := range assessment.Reasons {
if reason.Code == "unraid_disabled_disks" || reason.Code == "unraid_invalid_disks" {
t.Fatalf("unexpected aggregate-count reason after raw-status normalization: %+v", assessment.Reasons)
}
}
}
func TestApplyHostReportFiltersLegacyUnraidEmptySlots(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-tower-empty-slots",
Version: "5.1.27",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-tower-empty-slots",
Hostname: "tower",
MachineID: "machine-tower-empty-slots",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Unraid: &agentshost.UnraidStorage{
ArrayStarted: true,
ArrayState: "STARTED",
NumProtected: 0,
NumDisabled: 1,
NumInvalid: 1,
NumMissing: 2,
Disks: []agentshost.UnraidDisk{
{Name: "parity", Device: "/dev/sdb", Role: "parity", RawStatus: "DISK_OK", SizeBytes: 5860522532},
{Name: "disk1", Device: "/dev/sde", Role: "data", RawStatus: "DISK_OK", SizeBytes: 5860522532},
{Name: "disk6", Role: "data", Status: "missing", RawStatus: "DISK_NP", Model: "ata -", Serial: "ata-_", Filesystem: "auto", Slot: 6},
{Name: "parity2", Role: "parity", Status: "missing", RawStatus: "DISK_NP_DSBL", Model: "ata -", Serial: "ata-_", Filesystem: "auto", Slot: 29},
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.Unraid == nil {
t.Fatal("expected unraid topology on host")
}
if len(host.Unraid.Disks) != 2 {
t.Fatalf("unraid disk count = %d, want only assigned disks: %+v", len(host.Unraid.Disks), host.Unraid.Disks)
}
if host.Unraid.NumMissing != 0 || host.Unraid.NumDisabled != 0 || host.Unraid.NumInvalid != 0 {
t.Fatalf("legacy empty slots affected normalized failure counts: %+v", host.Unraid)
}
if host.Unraid.NumProtected != 1 {
t.Fatalf("NumProtected = %d, want one online parity disk", host.Unraid.NumProtected)
}
for _, disk := range host.Unraid.Disks {
if disk.Status != "online" {
t.Fatalf("unexpected assigned disk: %+v", disk)
}
}
assessment := storagehealth.AssessUnraidStorage(*host.Unraid)
for _, reason := range assessment.Reasons {
if reason.Code == "unraid_missing_disks" || reason.Code == "unraid_disabled_disks" || reason.Code == "unraid_invalid_disks" {
t.Fatalf("named empty slots produced a failure assessment: %+v", assessment.Reasons)
}
}
}
func TestLegacyUnraidEmptySlotPreservesExplicitMissingMember(t *testing.T) {
t.Parallel()
disk := agentshost.UnraidDisk{
Name: "disk6",
Role: "data",
Status: "missing",
RawStatus: "DISK_NP_MISSING",
Model: "ata -",
Serial: "ata-_",
Slot: 6,
}
if isLegacyUnraidEmptySlot(disk, disk.Status) {
t.Fatal("explicit assigned-but-missing member was classified as an empty slot")
}
}
func TestApplyHostReportFiltersVendorManagedSystemRAIDArrays(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
testCases := []struct {
name string
hostInfo agentshost.HostInfo
reportRAID []agentshost.RAIDArray
wantDevices []string
}{
{
name: "synology suppresses md0 and md1",
hostInfo: agentshost.HostInfo{
ID: "machine-synology",
Hostname: "synology.local",
Platform: "linux",
OSName: "Synology DSM",
},
reportRAID: []agentshost.RAIDArray{
{Device: "/dev/md0", Level: "raid1", State: "degraded", FailedDevices: 1},
{Device: "/dev/md1", Level: "raid1", State: "resyncing", RebuildPercent: 42},
{Device: "/dev/md2", Level: "raid5", State: "clean"},
},
wantDevices: []string{"/dev/md2"},
},
{
name: "qnap suppresses md9 and md13",
hostInfo: agentshost.HostInfo{
ID: "machine-qnap",
Hostname: "qnap.local",
Platform: "linux",
OSName: "QNAP QTS",
},
reportRAID: []agentshost.RAIDArray{
{Device: "/dev/md9", Level: "raid1", State: "degraded", FailedDevices: 1},
{Device: "/dev/md13", Level: "raid1", State: "clean"},
{Device: "/dev/md2", Level: "raid5", State: "clean"},
},
wantDevices: []string{"/dev/md2"},
},
{
name: "generic hosts keep md0",
hostInfo: agentshost.HostInfo{
ID: "machine-generic",
Hostname: "generic.local",
Platform: "linux",
OSName: "Ubuntu",
},
reportRAID: []agentshost.RAIDArray{
{Device: "/dev/md0", Level: "raid1", State: "clean"},
{Device: "/dev/md2", Level: "raid5", State: "clean"},
},
wantDevices: []string{"/dev/md0", "/dev/md2"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-" + tc.hostInfo.ID,
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: tc.hostInfo,
Timestamp: now,
Metrics: agentshost.Metrics{
CPUUsagePercent: 5.5,
},
RAID: tc.reportRAID,
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if len(host.RAID) != len(tc.wantDevices) {
t.Fatalf("host RAID count = %d, want %d", len(host.RAID), len(tc.wantDevices))
}
for i, device := range tc.wantDevices {
if host.RAID[i].Device != device {
t.Fatalf("host RAID[%d] = %q, want %q", i, host.RAID[i].Device, device)
}
}
snapshot := monitor.state.GetSnapshot()
var stored *models.Host
for i := range snapshot.Hosts {
if snapshot.Hosts[i].ID == host.ID {
stored = &snapshot.Hosts[i]
break
}
}
if stored == nil {
t.Fatalf("stored host %q not found", host.ID)
}
if len(stored.RAID) != len(tc.wantDevices) {
t.Fatalf("stored RAID count = %d, want %d", len(stored.RAID), len(tc.wantDevices))
}
for i, device := range tc.wantDevices {
if stored.RAID[i].Device != device {
t.Fatalf("stored RAID[%d] = %q, want %q", i, stored.RAID[i].Device, device)
}
}
})
}
}
func TestApplyHostReportKeepsLocalMergerFSMounts(t *testing.T) {
t.Helper()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-mergerfs-host",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-mergerfs-host",
Hostname: "mergerfs-host",
MachineID: "machine-mergerfs-host",
},
Disks: []agentshost.Disk{
{
Device: "mergerfs",
Mountpoint: "/mnt/storage",
Type: "fuse.mergerfs",
TotalBytes: 10_000,
UsedBytes: 4_000,
FreeBytes: 6_000,
Usage: 40,
},
{
Device: "sshfs",
Mountpoint: "/mnt/remote",
Type: "fuse.sshfs",
TotalBytes: 10_000,
UsedBytes: 4_000,
FreeBytes: 6_000,
Usage: 40,
},
},
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if len(host.Disks) != 1 {
t.Fatalf("host disk count = %d, want 1 (%+v)", len(host.Disks), host.Disks)
}
if host.Disks[0].Type != "fuse.mergerfs" || host.Disks[0].Mountpoint != "/mnt/storage" {
t.Fatalf("unexpected retained disk %+v", host.Disks[0])
}
snapshot := monitor.state.GetSnapshot()
if len(snapshot.Hosts) != 1 {
t.Fatalf("snapshot host count = %d, want 1", len(snapshot.Hosts))
}
if len(snapshot.Hosts[0].Disks) != 1 {
t.Fatalf("stored host disk count = %d, want 1 (%+v)", len(snapshot.Hosts[0].Disks), snapshot.Hosts[0].Disks)
}
if snapshot.Hosts[0].Disks[0].Type != "fuse.mergerfs" || snapshot.Hosts[0].Disks[0].Mountpoint != "/mnt/storage" {
t.Fatalf("unexpected stored retained disk %+v", snapshot.Hosts[0].Disks[0])
}
}
func TestApplyHostReportPersistsSMARTMetricsForAgentDisks(t *testing.T) {
t.Helper()
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
powerOnHours := int64(1234)
reallocated := int64(2)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-tower",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-tower",
Hostname: "tower",
MachineID: "machine-tower",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
SMART: []agentshost.DiskSMART{
{
Device: "/dev/sda",
Model: "IronWolf",
Serial: "SERIAL-TOWER-1",
Temperature: 41,
Attributes: &agentshost.SMARTAttributes{
PowerOnHours: &powerOnHours,
ReallocatedSectors: &reallocated,
},
},
},
},
Timestamp: time.Now().UTC(),
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
store.Flush()
points := waitForStoredDiskMetric(t, store, "SERIAL-TOWER-1", "smart_temp")
if len(points) == 0 {
t.Fatal("expected SMART temperature metric for agent disk")
}
points = waitForStoredDiskMetric(t, store, "SERIAL-TOWER-1", "smart_power_on_hours")
if len(points) == 0 || points[len(points)-1].Value != float64(powerOnHours) {
t.Fatalf("expected power-on-hours metric %.0f, got %+v", float64(powerOnHours), points)
}
points = waitForStoredDiskMetric(t, store, "SERIAL-TOWER-1", "smart_reallocated_sectors")
if len(points) == 0 || points[len(points)-1].Value != float64(reallocated) {
t.Fatalf("expected reallocated-sectors metric %.0f, got %+v", float64(reallocated), points)
}
}
// An Unraid member the agent sends no SMART row for (a --disk-exclude match)
// still shows the temperature its Unraid inventory row reports, so the report
// charts it under the disk's serial. A disk with its own SMART reading keeps
// it, and a spun-down member's leftover reading is not charted.
func TestApplyHostReportChartsUnraidTemperatureOfDiskWithoutSMART(t *testing.T) {
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-tower", Version: "1.0.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "machine-tower", Hostname: "tower", MachineID: "machine-tower"},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
SMART: []agentshost.DiskSMART{
{Device: "/dev/sda", Model: "IronWolf", Serial: "SERIAL-TOWER-1", Temperature: 41},
},
},
Unraid: &agentshost.UnraidStorage{ArrayStarted: true, Disks: []agentshost.UnraidDisk{
{Name: "disk1", Device: "sda", Role: "data", Status: "online", Serial: "SERIAL-TOWER-1", Temperature: 40},
{Name: "disk2", Device: "sdb", Role: "data", Status: "online", Serial: "SERIAL-TOWER-2", Temperature: 36},
{Name: "disk3", Device: "sdc", Role: "data", Status: "online", Serial: "SERIAL-TOWER-3", Temperature: 34, SpunDown: true},
}},
Timestamp: time.Now().UTC(),
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
store.Flush()
points := waitForStoredDiskMetric(t, store, "SERIAL-TOWER-2", "smart_temp")
if len(points) != 1 || points[0].Value != 36 {
t.Fatalf("SERIAL-TOWER-2 smart_temp = %+v, want the Unraid reading 36", points)
}
points = waitForStoredDiskMetric(t, store, "SERIAL-TOWER-1", "smart_temp")
if len(points) != 1 || points[0].Value != 41 {
t.Fatalf("SERIAL-TOWER-1 smart_temp = %+v, want its own SMART reading 41", points)
}
now := time.Now().UTC()
points, err = store.Query("disk", "SERIAL-TOWER-3", "smart_temp", now.Add(-time.Hour), now.Add(time.Hour), 60)
if err != nil {
t.Fatalf("Query smart_temp: %v", err)
}
if len(points) != 0 {
t.Fatalf("spun-down SERIAL-TOWER-3 smart_temp = %+v, want none", points)
}
}
func TestApplyHostReportPersistsAgentTemperatureMetric(t *testing.T) {
t.Helper()
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(10, time.Hour),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-thermal",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-thermal",
Hostname: "thermal-node",
MachineID: "machine-thermal",
},
Metrics: agentshost.Metrics{
CPUUsagePercent: 12.5,
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
TemperatureCelsius: map[string]float64{
"cpu_core_0": 59,
"cpu_package": 62.5,
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
store.Flush()
storePoints := waitForStoredMetric(t, store, "agent", host.ID, "temperature")
if got := storePoints[len(storePoints)-1].Value; got != 62.5 {
t.Fatalf("expected persisted temperature 62.5, got %f", got)
}
historyPoints := monitor.GetGuestMetrics("agent:"+host.ID, time.Hour)["temperature"]
if len(historyPoints) == 0 || historyPoints[len(historyPoints)-1].Value != 62.5 {
t.Fatalf("expected in-memory temperature 62.5, got %+v", historyPoints)
}
}
// An agent-linked node's drawer headlines the unified resource temperature and
// draws the agent's "temperature" history beneath it; both must come from one
// rule. The node's own Temperature, built from the same sensors, must agree
// too, because node alerts, the node table and node history read it.
func TestApplyHostReportChartsThePrimaryCPUTemperatureEverySurfaceShows(t *testing.T) {
for _, tc := range []struct {
name string
sensors map[string]float64
want float64
}{
// Package above every core: the node's CPUMax (built from cores)
// sits below the package, which once split node and agent history.
{name: "package hottest", sensors: map[string]float64{"cpu_package": 61, "cpu_core_0": 55, "cpu_core_1": 57}, want: 61},
// A core above the package: the agent rule once kept the package.
{name: "core hottest", sensors: map[string]float64{"cpu_package": 60, "cpu_core_0": 66, "cpu_core_1": 58}, want: 66},
} {
t.Run(tc.name, func(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(10, time.Hour),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
host, err := monitor.ApplyHostReport(agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-primary-temp", Version: "1.0.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "machine-primary-temp", Hostname: "primary-temp", MachineID: "machine-primary-temp"},
Metrics: agentshost.Metrics{CPUUsagePercent: 10},
Sensors: agentshost.Sensors{TemperatureCelsius: tc.sensors},
Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
points := monitor.GetGuestMetrics("agent:"+host.ID, time.Hour)["temperature"]
if len(points) == 0 || points[len(points)-1].Value != tc.want {
t.Fatalf("agent temperature history = %+v, want last point %v", points, tc.want)
}
// The unified resource's agent.temperature (the drawer headline).
if got := models.HostSensorPrimaryCPUTemperatureCelsius(host.Sensors); got == nil || *got != tc.want {
t.Fatalf("agent resource temperature = %v, want %v", got, tc.want)
}
// The linked node's Temperature, as the poller builds it from these sensors.
nodeTemperature := convertHostSensorsToTemperature(host.Sensors, host.LastSeen)
if got := models.PrimaryCPUTemperatureCelsius(nodeTemperature); got == nil || *got != tc.want {
t.Fatalf("linked node temperature = %v, want %v", got, tc.want)
}
})
}
}
func TestApplyHostReportPreservesTypedSensorSummary(t *testing.T) {
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(10, time.Hour),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
temperature := 63.0
utilization := 7.0
usedBytes := int64(2 * 1024 * 1024 * 1024)
totalBytes := int64(48 * 1024 * 1024 * 1024)
queueDepth := 12.5
customObservedAt := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-gpu",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-gpu",
Hostname: "gpu-node",
MachineID: "machine-gpu",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
TemperatureCelsius: map[string]float64{"gpu_nvidia_0": temperature},
Custom: []agentshost.CustomSensorMetric{{
ID: "queue_depth",
Name: "Queue depth",
Unit: "items",
Value: &queueDepth,
Status: agentshost.CustomSensorStatusWarning,
ObservedAt: customObservedAt,
AlertOnError: true,
}},
GPU: []agentshost.GPUSensor{
{
ID: "0",
Name: "NVIDIA RTX A6000",
TemperatureCelsius: &temperature,
UtilizationPercent: &utilization,
MemoryUsedBytes: &usedBytes,
MemoryTotalBytes: &totalBytes,
},
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
store.Flush()
if len(host.Sensors.GPU) != 1 {
t.Fatalf("host GPU sensors = %+v, want one sensor", host.Sensors.GPU)
}
gpu := host.Sensors.GPU[0]
if gpu.ID != "0" || gpu.Name != "NVIDIA RTX A6000" {
t.Fatalf("unexpected GPU identity: %+v", gpu)
}
if gpu.TemperatureCelsius == nil || *gpu.TemperatureCelsius != temperature {
t.Fatalf("GPU temperature = %#v, want %.1f", gpu.TemperatureCelsius, temperature)
}
if gpu.UtilizationPercent == nil || *gpu.UtilizationPercent != utilization {
t.Fatalf("GPU utilization = %#v, want %.1f", gpu.UtilizationPercent, utilization)
}
if gpu.MemoryUsedBytes == nil || *gpu.MemoryUsedBytes != usedBytes {
t.Fatalf("GPU used memory = %#v, want %d", gpu.MemoryUsedBytes, usedBytes)
}
if host.Sensors.TemperatureCelsius["gpu_nvidia_0"] != temperature {
t.Fatalf("legacy GPU temperature compatibility = %+v, want %.1f", host.Sensors.TemperatureCelsius, temperature)
}
if len(host.Sensors.Custom) != 1 || host.Sensors.Custom[0].Value == nil || *host.Sensors.Custom[0].Value != queueDepth {
t.Fatalf("host custom sensors = %+v, want one typed reading", host.Sensors.Custom)
}
custom := host.Sensors.Custom[0]
if custom.ID != "queue_depth" || custom.Unit != "items" || custom.Status != agentshost.CustomSensorStatusWarning || custom.ObservedAt != customObservedAt || !custom.AlertOnError {
t.Fatalf("custom sensor metadata was not preserved: %+v", custom)
}
expectedMetrics := map[string]float64{
"gpu": utilization,
"gpu_memory": (float64(usedBytes) / float64(totalBytes)) * 100,
"gpu_temperature": temperature,
}
history := monitor.GetGuestMetrics("agent:"+host.ID, time.Hour)
for metric, expected := range expectedMetrics {
storePoints := waitForStoredMetric(t, store, "agent", host.ID, metric)
if got := storePoints[len(storePoints)-1].Value; got != expected {
t.Fatalf("persisted %s = %f, want %f", metric, got, expected)
}
historyPoints := history[metric]
if len(historyPoints) == 0 || historyPoints[len(historyPoints)-1].Value != expected {
t.Fatalf("in-memory %s = %+v, want %f", metric, historyPoints, expected)
}
}
projected := hostSensorsFromReadStateView(&unifiedresources.HostSensorMeta{
GPU: []unifiedresources.HostGPUSensor{
{
ID: "0",
Name: "NVIDIA RTX A6000",
TemperatureCelsius: &temperature,
UtilizationPercent: &utilization,
MemoryUsedBytes: &usedBytes,
MemoryTotalBytes: &totalBytes,
},
},
})
if len(projected.GPU) != 1 || projected.GPU[0].MemoryTotalBytes == nil || *projected.GPU[0].MemoryTotalBytes != totalBytes {
t.Fatalf("read-state GPU projection = %+v, want total VRAM %d", projected.GPU, totalBytes)
}
}
func TestApplyHostReportPreservesPowerSensorSummary(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-power",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-power",
Hostname: "power-node",
MachineID: "machine-power",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
PowerWatts: map[string]float64{"cpu_package": 82.4, "dram": 13.2},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if got := host.Sensors.PowerWatts["cpu_package"]; got != 82.4 {
t.Fatalf("host power sensor = %.1f, want 82.4", got)
}
report.Sensors.PowerWatts["cpu_package"] = 1
if got := host.Sensors.PowerWatts["cpu_package"]; got != 82.4 {
t.Fatalf("host power sensors share report map, got %.1f want 82.4", got)
}
projected := hostSensorsFromReadStateView(&unifiedresources.HostSensorMeta{
PowerWatts: map[string]float64{"cpu_package": 82.4},
})
if got := projected.PowerWatts["cpu_package"]; got != 82.4 {
t.Fatalf("read-state power projection = %.1f, want 82.4", got)
}
}
func TestApplyHostReportPreservesNativeSensorInventory(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
battery, unattached, fan := 34.2, -127.0, 6152.0
reportedBattery := battery
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-native", Version: "1.0.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "machine-native", Hostname: "macbook-pve", MachineID: "machine-native"},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
Additional: map[string]float64{"applesmc_tb0t": battery},
Native: []agentshost.NativeSensor{
{ID: " applesmc-isa-0300/temp1 ", Chip: "applesmc-isa-0300", Adapter: "ISA adapter", Channel: "temp1", Label: " TB0T ",
Kind: agentshost.NativeSensorKindTemperature, Unit: "kelvin", Value: &reportedBattery, Status: agentshost.NativeSensorStatusOK},
// The server judges plausibility itself, whatever the agent claims.
{ID: "applesmc-isa-0300/temp4", Chip: "applesmc-isa-0300", Channel: "temp4", Label: "TG1D",
Kind: agentshost.NativeSensorKindTemperature, Value: &unattached, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/fan1", Chip: "applesmc-isa-0300", Channel: "fan1", Label: "Left side",
Kind: agentshost.NativeSensorKindFan, Unit: agentshost.NativeSensorUnitRPM, Value: &fan, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/temp2", Chip: "applesmc-isa-0300", Channel: "temp2", Label: "TB1T",
Kind: agentshost.NativeSensorKindTemperature, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/temp1", Chip: "applesmc-isa-0300", Channel: "temp1", Label: "duplicate",
Kind: agentshost.NativeSensorKindTemperature, Status: agentshost.NativeSensorStatusOK},
// Identities that do not follow "<chip>/<channel>" for a canonical
// channel of the reported kind are rejected.
{ID: "applesmc-isa-0300/temp9", Chip: "applesmc-isa-0300", Channel: "temp8", Label: "mismatch",
Kind: agentshost.NativeSensorKindTemperature, Value: &battery, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/temp01", Chip: "applesmc-isa-0300", Channel: "temp01", Label: "leading zero",
Kind: agentshost.NativeSensorKindTemperature, Value: &battery, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/fan2", Chip: "applesmc-isa-0300", Channel: "fan2", Label: "wrong kind",
Kind: agentshost.NativeSensorKindTemperature, Value: &battery, Status: agentshost.NativeSensorStatusOK},
{ID: "/temp3", Channel: "temp3", Label: "no chip", Kind: agentshost.NativeSensorKindTemperature, Value: &battery, Status: agentshost.NativeSensorStatusOK},
{ID: "", Label: "no identity", Kind: agentshost.NativeSensorKindTemperature, Status: agentshost.NativeSensorStatusOK},
{ID: "board-isa-0000/in0", Label: "Vcore", Kind: "voltage", Status: agentshost.NativeSensorStatusOK},
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
want := []models.HostNativeSensor{
{ID: "applesmc-isa-0300/temp1", Chip: "applesmc-isa-0300", Adapter: "ISA adapter", Channel: "temp1", Label: "TB0T",
Kind: agentshost.NativeSensorKindTemperature, Unit: agentshost.NativeSensorUnitCelsius, Value: &battery, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/temp4", Chip: "applesmc-isa-0300", Channel: "temp4", Label: "TG1D",
Kind: agentshost.NativeSensorKindTemperature, Unit: agentshost.NativeSensorUnitCelsius, Value: &unattached, Status: agentshost.NativeSensorStatusInvalid},
{ID: "applesmc-isa-0300/fan1", Chip: "applesmc-isa-0300", Channel: "fan1", Label: "Left side",
Kind: agentshost.NativeSensorKindFan, Unit: agentshost.NativeSensorUnitRPM, Value: &fan, Status: agentshost.NativeSensorStatusOK},
{ID: "applesmc-isa-0300/temp2", Chip: "applesmc-isa-0300", Channel: "temp2", Label: "TB1T",
Kind: agentshost.NativeSensorKindTemperature, Unit: agentshost.NativeSensorUnitCelsius, Status: agentshost.NativeSensorStatusInvalid},
}
if !reflect.DeepEqual(host.Sensors.Native, want) {
t.Fatalf("native inventory = %+v, want %+v", host.Sensors.Native, want)
}
if host.Sensors.Additional["applesmc_tb0t"] != battery {
t.Fatalf("legacy additional reading = %+v, want it unchanged", host.Sensors.Additional)
}
*report.Sensors.Native[0].Value = 99
if got := *host.Sensors.Native[0].Value; got != battery {
t.Fatalf("host inventory shares the report's value pointer, got %.1f want %.1f", got, battery)
}
frontend := host.ToFrontend()
if frontend.Sensors == nil || len(frontend.Sensors.Native) != len(want) || frontend.Sensors.Native[0].Label != "TB0T" {
t.Fatalf("frontend inventory = %+v, want the four accepted sensors", frontend.Sensors)
}
oversized := make([]agentshost.NativeSensor, 300)
for i := range oversized {
channel := "temp" + strconv.Itoa(i+1)
oversized[i] = agentshost.NativeSensor{ID: "board-isa-0000/" + channel, Chip: "board-isa-0000", Channel: channel,
Kind: agentshost.NativeSensorKindTemperature, Value: &battery}
}
if got := convertAgentNativeSensorsToModels(oversized); len(got) != 256 || got[255].ID != "board-isa-0000/temp256" {
t.Fatalf("server kept %d native sensors, want the first 256", len(got))
}
projected := hostSensorsFromReadStateView(&unifiedresources.HostSensorMeta{
Native: []unifiedresources.HostNativeSensor{{ID: "applesmc-isa-0300/temp1", Chip: "applesmc-isa-0300", Channel: "temp1", Label: "TB0T",
Kind: agentshost.NativeSensorKindTemperature, Unit: agentshost.NativeSensorUnitCelsius, Value: &battery, Status: agentshost.NativeSensorStatusOK}},
})
if len(projected.Native) != 1 || projected.Native[0].Value == nil || *projected.Native[0].Value != battery || projected.Native[0].Label != "TB0T" {
t.Fatalf("read-state inventory projection = %+v, want the battery sensor", projected.Native)
}
}
func TestApplyHostReportPersistsSMARTMetricsForAgentDisksWithFallbackID(t *testing.T) {
t.Helper()
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
mediaErrors := int64(7)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-tower-fallback",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-tower",
Hostname: "tower",
MachineID: "machine-tower",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
SMART: []agentshost.DiskSMART{
{
Device: "/dev/nvme0n1",
Model: "CacheDisk",
Temperature: 39,
Attributes: &agentshost.SMARTAttributes{
MediaErrors: &mediaErrors,
},
},
},
},
Timestamp: time.Now().UTC(),
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
store.Flush()
resourceID := "machine-tower:nvme0n1"
points := waitForStoredDiskMetric(t, store, resourceID, "smart_temp")
if len(points) == 0 {
t.Fatal("expected SMART temperature metric for fallback-id agent disk")
}
points = waitForStoredDiskMetric(t, store, resourceID, "smart_media_errors")
if len(points) == 0 || points[len(points)-1].Value != float64(mediaErrors) {
t.Fatalf("expected media-errors metric %.0f, got %+v", float64(mediaErrors), points)
}
}
func TestApplyHostReportPersistsPhysicalDiskIOMetricsForAgentDisks(t *testing.T) {
t.Helper()
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(1000, 24*time.Hour),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
baseReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-pve2",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "host-pve2",
Hostname: "pve2",
MachineID: "machine-pve2",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
SMART: []agentshost.DiskSMART{
{
Device: "/dev/nvme2",
Model: "Samsung 980 PRO 2TB",
Serial: "SERIAL884006359727",
Temperature: 46,
},
},
},
DiskIO: []agentshost.DiskIO{
{
Device: "nvme2",
ReadBytes: 1_000_000,
WriteBytes: 2_000_000,
ReadOps: 100,
WriteOps: 200,
IOTime: 10_000,
},
},
Timestamp: time.Now().UTC(),
}
if _, err := monitor.ApplyHostReport(baseReport, nil); err != nil {
t.Fatalf("ApplyHostReport initial: %v", err)
}
nextReport := baseReport
nextReport.Timestamp = baseReport.Timestamp.Add(30 * time.Second)
nextReport.DiskIO = []agentshost.DiskIO{
{
Device: "nvme2",
ReadBytes: 4_000_000,
WriteBytes: 5_000_000,
ReadOps: 250,
WriteOps: 350,
IOTime: 22_000,
},
}
if _, err := monitor.ApplyHostReport(nextReport, nil); err != nil {
t.Fatalf("ApplyHostReport second: %v", err)
}
store.Flush()
readPoints := waitForStoredDiskMetric(t, store, "SERIAL884006359727", "diskread")
writePoints := waitForStoredDiskMetric(t, store, "SERIAL884006359727", "diskwrite")
busyPoints := waitForStoredDiskMetric(t, store, "SERIAL884006359727", "disk")
if got := readPoints[len(readPoints)-1].Value; got <= 0 {
t.Fatalf("expected persisted diskread rate > 0, got %+v", readPoints)
}
if got := writePoints[len(writePoints)-1].Value; got <= 0 {
t.Fatalf("expected persisted diskwrite rate > 0, got %+v", writePoints)
}
if got := busyPoints[len(busyPoints)-1].Value; got <= 0 || got > 100 {
t.Fatalf("expected persisted disk busy percent within (0,100], got %+v", busyPoints)
}
if got := monitor.metricsHistory.GetDiskMetrics("SERIAL884006359727", "diskread", time.Hour); len(got) == 0 {
t.Fatal("expected in-memory diskread history for physical disk")
}
if got := monitor.metricsHistory.GetDiskMetrics("SERIAL884006359727", "disk", time.Hour); len(got) == 0 {
t.Fatal("expected in-memory busy history for physical disk")
}
}
func TestApplyHostReportPersistsDiskIOMetricsWithoutSMART(t *testing.T) {
t.Helper()
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(1000, 24*time.Hour),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
baseReport := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-pve3",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "host-pve3",
Hostname: "pve3",
MachineID: "machine-pve3",
},
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
DiskIO: []agentshost.DiskIO{
{
Device: "sda",
ReadBytes: 1_000_000,
WriteBytes: 2_000_000,
ReadOps: 100,
WriteOps: 200,
IOTime: 10_000,
},
},
Timestamp: time.Now().UTC(),
}
if _, err := monitor.ApplyHostReport(baseReport, nil); err != nil {
t.Fatalf("ApplyHostReport initial: %v", err)
}
nextReport := baseReport
nextReport.Timestamp = baseReport.Timestamp.Add(30 * time.Second)
nextReport.DiskIO = []agentshost.DiskIO{
{
Device: "sda",
ReadBytes: 4_000_000,
WriteBytes: 5_000_000,
ReadOps: 250,
WriteOps: 350,
IOTime: 22_000,
},
}
if _, err := monitor.ApplyHostReport(nextReport, nil); err != nil {
t.Fatalf("ApplyHostReport second: %v", err)
}
store.Flush()
fallbackID := "host-pve3:sda"
readPoints := waitForStoredDiskMetric(t, store, fallbackID, "diskread")
writePoints := waitForStoredDiskMetric(t, store, fallbackID, "diskwrite")
busyPoints := waitForStoredDiskMetric(t, store, fallbackID, "disk")
if got := readPoints[len(readPoints)-1].Value; got <= 0 {
t.Fatalf("expected persisted diskread rate > 0 without SMART, got %+v", readPoints)
}
if got := writePoints[len(writePoints)-1].Value; got <= 0 {
t.Fatalf("expected persisted diskwrite rate > 0 without SMART, got %+v", writePoints)
}
if got := busyPoints[len(busyPoints)-1].Value; got <= 0 || got > 100 {
t.Fatalf("expected persisted disk busy percent within (0,100] without SMART, got %+v", busyPoints)
}
}
func TestHostDiskIOMetricResourceIDFallbacks(t *testing.T) {
host := models.Host{
ID: "myhost",
Sensors: models.HostSensorSummary{
SMART: []models.HostDiskSMART{
{Device: "/dev/nvme0n1", Serial: "NVME-SERIAL-123"},
},
},
}
io := models.DiskIO{Device: "nvme0n1"}
got := hostDiskIOMetricResourceID(host, io, nil)
if got != "NVME-SERIAL-123" {
t.Fatalf("SMART match: expected NVME-SERIAL-123, got %q", got)
}
ioNoSMART := models.DiskIO{Device: "sda"}
got = hostDiskIOMetricResourceID(host, ioNoSMART, nil)
if got != "myhost:sda" {
t.Fatalf("no SMART, no proxmox: expected myhost:sda, got %q", got)
}
proxmoxDisks := []proxmoxDiskMatch{
{device: "sda", metricID: "SATA-SERIAL-456"},
}
got = hostDiskIOMetricResourceID(host, ioNoSMART, proxmoxDisks)
if got != "SATA-SERIAL-456" {
t.Fatalf("proxmox fallback: expected SATA-SERIAL-456, got %q", got)
}
// The disk resource carries the serial the host's Unraid inventory reports
// when no SMART reading names the disk, so its I/O is filed under it.
// Controller members behind one block device keep their own identity, so
// the device's counter belongs to no member, even while one is in standby.
// Two inventory rows on one device name no single disk. A legacy SMART
// device label still names its block device.
host.Sensors.SMART = append(host.Sensors.SMART,
models.HostDiskSMART{Device: "sdb"},
models.HostDiskSMART{Device: "sdc", Serial: "STANDBY-SERIAL", Standby: true},
models.HostDiskSMART{Device: "sde", Controller: "ctrl0", Target: "megaraid,0"},
models.HostDiskSMART{Device: "sde", Controller: "ctrl0", Target: "megaraid,1"},
models.HostDiskSMART{Device: "sdg", WWN: "5000c500a1b2c3e0", Controller: "ctrl1", Target: "megaraid,0"},
models.HostDiskSMART{Device: "/dev/sdg", WWN: "5000c500a1b2c3e1", Controller: "ctrl1", Target: "megaraid,1", Standby: true},
models.HostDiskSMART{Device: "sdh [sat]", Serial: "LEGACY-LABEL-SERIAL"},
)
host.Unraid = &models.HostUnraidStorage{Disks: []models.HostUnraidDisk{
{Device: "sda", Serial: "UNRAID-SERIAL-A"},
{Device: "sdb", Serial: "UNRAID-SERIAL-B"},
{Device: "sdc", Serial: "UNRAID-SERIAL-C"},
{Device: "sdd"},
{Device: "sde", Serial: "UNRAID-SERIAL-E"},
{Device: "sdf", Serial: "UNRAID-SERIAL-F1"},
{Device: "sdf", Serial: "UNRAID-SERIAL-F2"},
}}
for _, tc := range []struct {
device string
want string
}{
{"sda", "UNRAID-SERIAL-A"},
{"sdb", "UNRAID-SERIAL-B"},
{"sdc", "UNRAID-SERIAL-C"},
{"sdd", "myhost:sdd"},
{"sde", ""},
{"sdf", "myhost:sdf"},
{"sdg", ""},
{"sdh", "LEGACY-LABEL-SERIAL"},
{"nvme0n1", "NVME-SERIAL-123"},
} {
if got := hostDiskIOMetricResourceID(host, models.DiskIO{Device: tc.device}, proxmoxDisks); got != tc.want {
t.Fatalf("device %s: expected %q, got %q", tc.device, tc.want, got)
}
}
}
func TestApplyHostReportSkipsMetricsAndSMARTWritesInMockMode(t *testing.T) {
previous := mock.IsMockEnabled()
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, previous) })
storeCfg := metrics.DefaultConfig(t.TempDir())
storeCfg.WriteBufferSize = 1
store, err := metrics.NewStore(storeCfg)
if err != nil {
t.Fatalf("NewStore: %v", err)
}
defer store.Close()
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
metricsHistory: NewMetricsHistory(1000, 24*time.Hour),
metricsStore: store,
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-demo",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "machine-demo",
Hostname: "demo-host",
MachineID: "machine-demo",
},
Metrics: agentshost.Metrics{
CPUUsagePercent: 44,
Memory: agentshost.MemoryMetric{TotalBytes: 1024, UsedBytes: 512, FreeBytes: 512, Usage: 50},
},
Sensors: agentshost.Sensors{
SMART: []agentshost.DiskSMART{
{
Device: "/dev/sda",
Serial: "SERIAL-DEMO-1",
Temperature: 39,
},
},
},
Timestamp: time.Now().UTC(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
store.Flush()
if got := monitor.metricsHistory.GetGuestMetrics("agent:"+host.ID, "cpu", time.Hour); len(got) != 0 {
t.Fatalf("expected mock mode to skip host-agent metrics history, got %+v", got)
}
now := time.Now().UTC()
points, err := store.Query("disk", "SERIAL-DEMO-1", "smart_temp", now.Add(-time.Hour), now.Add(time.Hour), 60)
if err != nil {
t.Fatalf("Query smart_temp: %v", err)
}
if len(points) != 0 {
t.Fatalf("expected mock mode to skip SMART metric persistence, got %+v", points)
}
}
func TestMonitorStartupAppliesPersistedInitialNotificationTarget(t *testing.T) {
previous := mock.IsMockEnabled()
mustSetMockEnabled(t, false)
t.Cleanup(func() { mustSetMockEnabled(t, previous) })
dataDir := t.TempDir()
persistence := config.NewConfigPersistence(dataDir)
alertManager := alerts.NewManagerWithDataDir(dataDir)
alertConfig := alertManager.GetConfig()
alertConfig.Enabled = true
alertConfig.ActivationState = alerts.ActivationActive
alertConfig.Schedule.InitialNotify = "email"
alertManager.UpdateConfig(alertConfig)
if err := persistence.SaveAlertConfig(alertManager.GetConfig()); err != nil {
alertManager.Stop()
t.Fatalf("save alert config: %v", err)
}
alertManager.Stop()
monitor, err := New(&config.Config{
ConfigPath: dataDir,
DataPath: dataDir,
EnvOverrides: make(map[string]bool),
})
if err != nil {
t.Fatalf("create monitor: %v", err)
}
t.Cleanup(monitor.Stop)
if actual := monitor.GetNotificationManager().GetInitialNotifyTarget(); actual != "email" {
t.Fatalf("startup initial notification target = %q, want email", actual)
}
}
func waitForStoredDiskMetric(t *testing.T, store *metrics.Store, resourceID, metric string) []metrics.MetricPoint {
t.Helper()
return waitForStoredMetric(t, store, "disk", resourceID, metric)
}
func waitForStoredMetric(t *testing.T, store *metrics.Store, resourceType, resourceID, metric string) []metrics.MetricPoint {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for {
now := time.Now().UTC()
points, err := store.Query(resourceType, resourceID, metric, now.Add(-time.Hour), now.Add(time.Hour), 60)
if err != nil {
t.Fatalf("Query %s: %v", metric, err)
}
if len(points) > 0 {
return points
}
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for disk metric %s for %s", metric, resourceID)
}
time.Sleep(10 * time.Millisecond)
}
}
func TestEvaluateHostAgentsEmptyHostsList(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// No hosts in state - should complete without error or state changes
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
if len(snapshot.Hosts) != 0 {
t.Errorf("expected 0 hosts, got %d", len(snapshot.Hosts))
}
if len(snapshot.ConnectionHealth) != 0 {
t.Errorf("expected 0 connection health entries, got %d", len(snapshot.ConnectionHealth))
}
}
func TestEvaluateHostAgentsZeroIntervalUsesDefault(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-zero-interval"
// IntervalSeconds = 0, LastSeen = now, should use default interval (60s)
// Default window = 60s * 6 = 360s, but minimum is 60s, so window = 60s
// With LastSeen = now, the host should be healthy
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "zero-interval.local",
Status: "unknown",
IntervalSeconds: 0, // Zero interval - should use default
LastSeen: time.Now(),
})
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true for zero-interval host with recent LastSeen, got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "online" {
t.Errorf("expected host status online, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsNegativeIntervalUsesDefault(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-negative-interval"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "negative-interval.local",
Status: "unknown",
IntervalSeconds: -10, // Negative interval - should use default
LastSeen: time.Now(),
})
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true for negative-interval host with recent LastSeen, got %v (exists=%v)", healthy, ok)
}
}
func TestEvaluateHostAgentsWindowClampedToMinimum(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-min-window"
// IntervalSeconds = 1, so window = 1s * 6 = 6s, but minimum is 60s
// Host last seen 55s ago should still be healthy (within 60s minimum window)
now := time.Now()
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "min-window.local",
Status: "unknown",
IntervalSeconds: 1, // Very small interval
LastSeen: now.Add(-55 * time.Second),
})
monitor.evaluateHostAgents(now)
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true (window clamped to minimum 60s), got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "online" {
t.Errorf("expected host status online, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsWindowClampedToMaximum(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-max-window"
// IntervalSeconds = 300 (5 min), so window = 300s * 6 = 1800s (30 min)
// But maximum is 10 min = 600s
// Host last seen 11 minutes ago should be unhealthy (outside 10 min max window)
now := time.Now()
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "max-window.local",
Status: "online",
IntervalSeconds: 300, // 5 minute interval
LastSeen: now.Add(-11 * time.Minute),
})
monitor.evaluateHostAgents(now)
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || healthy {
t.Fatalf("expected connection health false (window clamped to maximum 10m), got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "offline" {
t.Errorf("expected host status offline, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsRecentLastSeenIsHealthy(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-recent"
now := time.Now()
// IntervalSeconds = 30, window = 30s * 6 = 180s (clamped to min 60s is not needed)
// LastSeen = 10s ago, should be healthy
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "recent.local",
Status: "unknown",
IntervalSeconds: 30,
LastSeen: now.Add(-10 * time.Second),
})
monitor.evaluateHostAgents(now)
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true for recent LastSeen, got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "online" {
t.Errorf("expected host status online, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsZeroLastSeenIsUnhealthy(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-zero-lastseen"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "zero-lastseen.local",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Time{}, // Zero time
})
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || healthy {
t.Fatalf("expected connection health false for zero LastSeen, got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "offline" {
t.Errorf("expected host status offline for zero LastSeen, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsOldLastSeenIsUnhealthy(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-old-lastseen"
now := time.Now()
// IntervalSeconds = 30, window = 30s * 6 = 180s
// LastSeen = 5 minutes ago, should be unhealthy
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "old-lastseen.local",
Status: "online",
IntervalSeconds: 30,
LastSeen: now.Add(-5 * time.Minute),
})
monitor.evaluateHostAgents(now)
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || healthy {
t.Fatalf("expected connection health false for old LastSeen, got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "offline" {
t.Errorf("expected host status offline for old LastSeen, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsNilAlertManagerOnline(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: nil, // No alert manager
config: &config.Config{},
}
hostID := "host-nil-am-online"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "nil-am-online.local",
Status: "unknown",
IntervalSeconds: 30,
LastSeen: time.Now(),
})
// Should not panic with nil alertManager
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || !healthy {
t.Fatalf("expected connection health true, got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "online" {
t.Errorf("expected host status online, got %q", host.Status)
}
}
}
func TestEvaluateHostAgentsNilAlertManagerOffline(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: nil, // No alert manager
config: &config.Config{},
}
hostID := "host-nil-am-offline"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "nil-am-offline.local",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Time{}, // Zero time - unhealthy
})
// Should not panic with nil alertManager
monitor.evaluateHostAgents(time.Now())
snapshot := monitor.state.GetSnapshot()
connKey := hostConnectionPrefix + hostID
if healthy, ok := snapshot.ConnectionHealth[connKey]; !ok || healthy {
t.Fatalf("expected connection health false, got %v (exists=%v)", healthy, ok)
}
for _, host := range snapshot.Hosts {
if host.ID == hostID && host.Status != "offline" {
t.Errorf("expected host status offline, got %q", host.Status)
}
}
}
func TestRemoveHostAgent_ClearsConnectionHealth(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-connhealth"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "connhealth.local",
Status: "online",
IntervalSeconds: 30,
LastSeen: time.Now(),
})
// Seed connection health for this host (as evaluateHostAgents would)
monitor.state.SetConnectionHealth(hostConnectionPrefix+hostID, true)
// Verify it's present before removal
snapshot := monitor.state.GetSnapshot()
if _, ok := snapshot.ConnectionHealth[hostConnectionPrefix+hostID]; !ok {
t.Fatalf("expected connection health entry to exist before removal")
}
// Remove the host
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("RemoveHostAgent: %v", err)
}
// Verify connection health entry is gone
snapshot = monitor.state.GetSnapshot()
if _, ok := snapshot.ConnectionHealth[hostConnectionPrefix+hostID]; ok {
t.Fatalf("expected connection health entry to be removed after RemoveHostAgent")
}
}
func TestRemoveHostAgent_EmptyHostID(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// Empty hostID should return an error
_, err := monitor.RemoveHostAgent("")
if err == nil {
t.Error("expected error for empty hostID")
}
if err != nil && err.Error() != "host id is required" {
t.Errorf("expected 'host id is required' error, got: %v", err)
}
// Whitespace-only hostID should also return an error
_, err = monitor.RemoveHostAgent(" ")
if err == nil {
t.Error("expected error for whitespace-only hostID")
}
}
func TestRemoveHostAgent_NotFound(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// Host does not exist in state - should return synthetic host without error
host, err := monitor.RemoveHostAgent("nonexistent-host")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Should return a synthetic host with ID/Hostname matching the requested ID
if host.ID != "nonexistent-host" {
t.Errorf("expected host.ID = 'nonexistent-host', got %q", host.ID)
}
if host.Hostname != "nonexistent-host" {
t.Errorf("expected host.Hostname = 'nonexistent-host', got %q", host.Hostname)
}
}
func TestRemoveHostAgent_NoTokenBinding(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-no-binding"
tokenID := "token-no-binding"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "no-binding.local",
TokenID: tokenID,
})
// Intentionally NOT adding to hostTokenBindings
host, err := monitor.RemoveHostAgent(hostID)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if host.ID != hostID {
t.Errorf("expected host.ID = %q, got %q", hostID, host.ID)
}
}
func TestRemoveHostAgent_NilAlertManager(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: nil, // No alert manager
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
hostID := "host-nil-am-remove"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "nil-am-remove.local",
})
// Should not panic with nil alertManager
host, err := monitor.RemoveHostAgent(hostID)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if host.ID != hostID {
t.Errorf("expected host.ID = %q, got %q", hostID, host.ID)
}
}
func TestRemoveHostAgent_BlocksFutureReportsUntilAllowed(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
removedHostAgents: make(map[string]time.Time),
rateTracker: NewRateTracker(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
canonicalProgress := time.Now().Add(-time.Minute).UTC()
monitor.markDeadManMonitoringProgress(canonicalProgress)
hostID := "host-blocked"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "blocked.local",
})
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("remove host agent: %v", err)
}
report := agentshost.Report{
Host: agentshost.HostInfo{
ID: hostID,
Hostname: "blocked.local",
},
Agent: agentshost.AgentInfo{ID: hostID},
Timestamp: time.Now(),
}
if _, err := monitor.ApplyHostReport(report, nil); err == nil {
t.Fatal("expected removed host agent report to be rejected")
}
if err := monitor.AllowHostAgentReenroll(hostID); err != nil {
t.Fatalf("allow host reenroll: %v", err)
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("expected host report after allow reenroll, got %v", err)
}
// Host-agent deletion, rejected reports, and explicit re-enrollment are
// important lifecycle activity, but none prove that the canonical monitor
// loop is still scheduling. The independent watchdog must only advance from
// Monitor.Start's own select loop.
if got := monitor.deadManMonitoringProgress(); !got.Equal(canonicalProgress) {
t.Fatalf("host-agent lifecycle advanced canonical monitor progress: got %s, want %s", got, canonicalProgress)
}
}
// TestApplyHostReport_FreshTokenClearsRemovalBlock pins the #1581 re-enroll
// path: a report presenting a token created after the removal is explicit
// re-add intent and clears the block, while a pre-removal token stays blocked.
func TestApplyHostReport_FreshTokenClearsRemovalBlock(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
removedHostAgents: make(map[string]time.Time),
rateTracker: NewRateTracker(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-fresh-token"
monitor.state.UpsertHost(models.Host{
ID: hostID,
Hostname: "fresh-token.local",
})
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("remove host agent: %v", err)
}
report := agentshost.Report{
Host: agentshost.HostInfo{
ID: hostID,
Hostname: "fresh-token.local",
},
Agent: agentshost.AgentInfo{ID: hostID},
Timestamp: time.Now(),
}
staleToken := &config.APITokenRecord{
ID: "token-old",
CreatedAt: time.Now().Add(-time.Hour),
}
if _, err := monitor.ApplyHostReport(report, staleToken); err == nil {
t.Fatal("expected report with pre-removal token to stay blocked")
}
freshToken := &config.APITokenRecord{
ID: "token-new",
CreatedAt: time.Now().Add(time.Minute),
}
if _, err := monitor.ApplyHostReport(report, freshToken); err != nil {
t.Fatalf("expected report with post-removal token to clear the block, got %v", err)
}
if len(monitor.state.GetRemovedHostAgents()) != 0 {
t.Fatal("expected state removal block to be cleared")
}
}
// TestAllowHostAgentReenroll_ClearsStateMirrorWithoutMemoryEntry preserves
// compatibility with callers that only populated the models.State mirror.
// The production restart path is covered by TestHostAgentRemovalLifecycleSurvivesMonitorReconstruction.
func TestAllowHostAgentReenroll_ClearsStateMirrorWithoutMemoryEntry(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
removedHostAgents: make(map[string]time.Time),
rateTracker: NewRateTracker(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-restart-block"
monitor.state.AddRemovedHostAgent(models.RemovedHostAgent{
ID: hostID,
Hostname: "restart-block.local",
RemovedAt: time.Now().Add(-time.Hour),
})
if err := monitor.AllowHostAgentReenroll(hostID); err != nil {
t.Fatalf("allow host reenroll: %v", err)
}
if len(monitor.state.GetRemovedHostAgents()) != 0 {
t.Fatal("expected state removal block to be cleared without an in-memory entry")
}
}
// Regression for #1772: a removal block that lives only in persisted monitor
// state (written before the durable continuity store existed) must still be
// clearable when a durable store is attached, or every fresh re-enroll reads
// as already-consumed and the host can never report again.
func TestAllowHostAgentReenroll_ClearsStateMirrorWhenDurableStoreLacksEntry(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
removedHostAgents: make(map[string]time.Time),
rateTracker: NewRateTracker(),
config: &config.Config{},
hostContinuityStore: config.NewHostContinuityStore(t.TempDir(), nil),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
hostID := "host-upgrade-block"
monitor.state.AddRemovedHostAgent(models.RemovedHostAgent{
ID: hostID,
Hostname: "npm.local",
RemovedAt: time.Now().Add(-time.Hour),
})
cleared, err := monitor.allowHostAgentReenrollLocked(hostID, "token-fresh", false)
if err != nil {
t.Fatalf("allowHostAgentReenrollLocked: %v", err)
}
if !cleared {
t.Fatal("expected state-only removal block to report cleared with a durable store attached")
}
if len(monitor.state.GetRemovedHostAgents()) != 0 {
t.Fatal("expected state removal block to be cleared")
}
}
// TestCleanupRemovedHostAgents_ExpiresStateMirrorEntries pins expiry for
// models.State-only compatibility entries when the in-memory map lacks them.
func TestCleanupRemovedHostAgents_ExpiresStateMirrorEntries(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
hostTokenBindings: make(map[string]string),
removedHostAgents: make(map[string]time.Time),
config: &config.Config{},
}
monitor.state.AddRemovedHostAgent(models.RemovedHostAgent{
ID: "host-expired",
Hostname: "expired.local",
RemovedAt: time.Now().Add(-25 * time.Hour),
})
monitor.state.AddRemovedHostAgent(models.RemovedHostAgent{
ID: "host-recent",
Hostname: "recent.local",
RemovedAt: time.Now().Add(-time.Hour),
})
monitor.cleanupRemovedHostAgents(time.Now())
remaining := monitor.state.GetRemovedHostAgents()
if len(remaining) != 1 {
t.Fatalf("expected exactly one persisted block to survive, got %d", len(remaining))
}
if remaining[0].ID != "host-recent" {
t.Fatalf("expected host-recent to survive, got %q", remaining[0].ID)
}
}
// TestMatchHostConfigContinuity pins the #1570 config-fetch fallback: after a
// monitor reload wipes live state, a report-scoped token still resolves its
// bound host from the persisted continuity store, other tokens do not, and a
// manage-scoped (empty token) lookup resolves by host ID.
func TestMatchHostConfigContinuity(t *testing.T) {
dir := t.TempDir()
monitor := &Monitor{
hostContinuityStore: config.NewHostContinuityStore(dir, nil),
}
entry := config.HostContinuityEntry{
HostID: "host-cont-1",
Hostname: "cont.local",
MachineID: "machine-cont-1",
TokenID: "token-cont-1",
LastSeen: time.Now().UTC(),
}
if err := monitor.hostContinuityStore.Upsert(entry); err != nil {
t.Fatalf("upsert continuity entry: %v", err)
}
host, ok := monitor.MatchHostConfigContinuity("host-cont-1", "token-cont-1")
if !ok || host.ID != "host-cont-1" {
t.Fatalf("expected token-bound continuity match, got ok=%v host=%q", ok, host.ID)
}
if _, ok := monitor.MatchHostConfigContinuity("host-cont-1", "token-other"); ok {
t.Fatal("expected no match for a token the host is not bound to")
}
host, ok = monitor.MatchHostConfigContinuity("host-cont-1", "")
if !ok || host.ID != "host-cont-1" {
t.Fatalf("expected ID-based continuity match for manage scope, got ok=%v host=%q", ok, host.ID)
}
if _, ok := monitor.MatchHostConfigContinuity("host-unknown", ""); ok {
t.Fatal("expected no ID-based match for an unknown host")
}
}
func TestApplyHostReport_MissingHostname(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// Report with empty hostname should fail
report := agentshost.Report{
Host: agentshost.HostInfo{
Hostname: "", // Missing hostname
ID: "machine-id",
},
Agent: agentshost.AgentInfo{
ID: "agent-id",
Version: "1.0.0",
},
Timestamp: time.Now(),
}
_, err := monitor.ApplyHostReport(report, nil)
if err == nil {
t.Error("expected error for missing hostname")
}
if err != nil && err.Error() != "host report missing hostname" {
t.Errorf("expected 'host report missing hostname' error, got: %v", err)
}
}
func TestApplyHostReport_WhitespaceHostname(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// Report with whitespace-only hostname should fail
report := agentshost.Report{
Host: agentshost.HostInfo{
Hostname: " ", // Whitespace only
ID: "machine-id",
},
Agent: agentshost.AgentInfo{
ID: "agent-id",
Version: "1.0.0",
},
Timestamp: time.Now(),
}
_, err := monitor.ApplyHostReport(report, nil)
if err == nil {
t.Error("expected error for whitespace-only hostname")
}
}
func TestApplyHostReport_NilTokenBindingsMap(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: nil, // Nil map
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Host: agentshost.HostInfo{
Hostname: "test-host",
ID: "machine-id",
},
Agent: agentshost.AgentInfo{
ID: "agent-id",
Version: "1.0.0",
},
Timestamp: time.Now(),
}
token := &config.APITokenRecord{ID: "token-id", Name: "Test Token"}
// Should not panic with nil hostTokenBindings - map should be initialized
host, err := monitor.ApplyHostReport(report, token)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if host.Hostname != "test-host" {
t.Errorf("expected hostname 'test-host', got %q", host.Hostname)
}
}
func TestApplyHostReport_FallbackIdentifier(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// Report with no ID fields - should generate fallback identifier
report := agentshost.Report{
Host: agentshost.HostInfo{
Hostname: "fallback-host",
// No ID, MachineID
},
Agent: agentshost.AgentInfo{
// No ID
Version: "1.0.0",
},
Timestamp: time.Now(),
}
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Should use hostname as fallback identifier
if host.ID == "" {
t.Error("expected host to have an identifier")
}
if host.Hostname != "fallback-host" {
t.Errorf("expected hostname 'fallback-host', got %q", host.Hostname)
}
}
// Regression for #1341: when Ceph is reported by a Pulse host-agent, pool
// alert evaluation must run even when there is no Proxmox API Ceph source.
// Existing per-pool overrides keyed against the older agent-prefixed pool ID
// still have to resolve after the pool's canonical ID drops the agent routing
// prefix.
func TestApplyHostReportFiresCephPoolAlertsForAgentSourcedCluster(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
overrideTrigger := alerts.HysteresisThreshold{Trigger: 50, Clear: 45}
cfg := monitor.alertManager.GetConfig()
cfg.MinimumDelta = 0
if cfg.TimeThresholds == nil {
cfg.TimeThresholds = make(map[string]int)
}
cfg.TimeThresholds["storage"] = 0
cfg.StorageDefault = alerts.HysteresisThreshold{Trigger: 95, Clear: 90}
cfg.Overrides = map[string]alerts.ThresholdConfig{
"agent:pve5-ceph-pool-data_replication": {Usage: &overrideTrigger},
}
monitor.alertManager.UpdateConfig(cfg)
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-ceph",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "pve5-host",
Hostname: "pve5",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
Ceph: &agentshost.CephCluster{
FSID: "ceph-fsid-1341",
Health: agentshost.CephHealth{Status: "HEALTH_OK"},
Pools: []agentshost.CephPool{
{
ID: 2,
Name: "data_replication",
BytesUsed: 611,
BytesAvailable: 389,
PercentUsed: 61.1,
},
},
},
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
active := monitor.alertManager.GetActiveAlerts()
found := false
for _, alert := range active {
if alert.ResourceID == "pve5-ceph-pool-data_replication" && alert.Type == "usage" {
found = true
if alert.Threshold != 50 {
t.Fatalf("Ceph pool alert fired but threshold = %.1f, want 50 (override)", alert.Threshold)
}
break
}
}
if !found {
ids := make([]string, 0, len(active))
for _, a := range active {
ids = append(ids, a.ID)
}
t.Fatalf("expected agent-sourced Ceph pool alert to fire under legacy override at 61.1%% usage; got %d active alerts: %v", len(active), ids)
}
}
func TestApplyHostReportMergesAgentCephWithProxmoxAPICluster(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
overrideTrigger := alerts.HysteresisThreshold{Trigger: 50, Clear: 45}
cfg := monitor.alertManager.GetConfig()
cfg.MinimumDelta = 0
if cfg.TimeThresholds == nil {
cfg.TimeThresholds = make(map[string]int)
}
cfg.TimeThresholds["storage"] = 0
cfg.StorageDefault = alerts.HysteresisThreshold{Trigger: 95, Clear: 90}
cfg.Overrides = map[string]alerts.ThresholdConfig{
"agent:pve5-ceph-pool-data_replication": {Usage: &overrideTrigger},
}
monitor.alertManager.UpdateConfig(cfg)
monitor.state.UpdateCephClustersForInstance("prod", []models.CephCluster{{
ID: "prod-ceph-fsid-1341",
Instance: "prod",
Source: models.CephClusterSourceProxmoxAPI,
Name: "Ceph",
FSID: "ceph-fsid-1341",
Health: "HEALTH_OK",
}})
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-ceph",
Version: "1.0.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "pve5-host",
Hostname: "pve5",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
Ceph: &agentshost.CephCluster{
FSID: "ceph-fsid-1341",
Health: agentshost.CephHealth{Status: "HEALTH_OK"},
Pools: []agentshost.CephPool{
{
ID: 2,
Name: "data_replication",
BytesUsed: 611,
BytesAvailable: 389,
PercentUsed: 61.1,
},
},
},
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
snapshot := monitor.state.GetSnapshot()
if len(snapshot.CephClusters) != 1 {
t.Fatalf("expected API and agent Ceph reports to reconcile to one cluster, got %#v", snapshot.CephClusters)
}
cluster := snapshot.CephClusters[0]
if cluster.Source != models.CephClusterSourceProxmoxAPI || cluster.Instance != "prod" {
t.Fatalf("expected API cluster to remain canonical, got %+v", cluster)
}
foundAlias := false
for _, alias := range cluster.InstanceAliases {
if alias == "pve5" {
foundAlias = true
break
}
}
if !foundAlias {
t.Fatalf("expected agent hostname to be preserved as an alias, got %#v", cluster.InstanceAliases)
}
active := monitor.alertManager.GetActiveAlerts()
for _, alert := range active {
if alert.ResourceID == "prod-ceph-pool-data_replication" && alert.Type == "usage" {
if alert.Threshold != 50 {
t.Fatalf("Ceph pool alert threshold = %.1f, want legacy agent override 50", alert.Threshold)
}
return
}
}
t.Fatalf("expected canonical API Ceph pool alert using legacy agent override, got active alerts: %#v", active)
}
// Agents before the manager fix sent the Ceph mgr row as "1 of N running"
// whenever a manager was available, counting standby managers as stopped.
// Standbys are running ceph-mgr daemons, so ingest must publish every listed
// manager as running while one is available, whichever agent sent the row, and
// keep zero running when none is available. Other rows pass through unchanged.
func TestApplyHostReportCountsStandbyCephManagersAsRunning(t *testing.T) {
cases := []struct {
name string
available bool
mgr agentshost.CephService
want int
}{
{name: "older agent with an available manager", available: true, mgr: agentshost.CephService{Type: "mgr", Running: 1, Total: 3}, want: 3},
{name: "fixed agent with an available manager", available: true, mgr: agentshost.CephService{Type: "mgr", Running: 3, Total: 3}, want: 3},
{name: "no available manager", available: false, mgr: agentshost.CephService{Type: "mgr", Running: 0, Total: 2}, want: 0},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-ceph-mgr", Version: "1.0.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "pve1-host", Hostname: "pve1", Platform: "linux"},
Timestamp: time.Now().UTC(),
Ceph: &agentshost.CephCluster{
FSID: "ceph-fsid-mgr",
Health: agentshost.CephHealth{Status: "HEALTH_OK"},
MgrMap: agentshost.CephManagerMap{Available: tc.available, NumMgrs: tc.mgr.Total, Standbys: 2},
Services: []agentshost.CephService{
{Type: "mon", Running: 2, Total: 3},
tc.mgr,
{Type: "osd", Running: 5, Total: 6},
},
},
}
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
want := []models.CephServiceStatus{
{Type: "mon", Running: 2, Total: 3},
{Type: "mgr", Running: tc.want, Total: tc.mgr.Total},
{Type: "osd", Running: 5, Total: 6},
}
snapshot := monitor.state.GetSnapshot()
if len(snapshot.CephClusters) != 1 || !reflect.DeepEqual(snapshot.CephClusters[0].Services, want) {
t.Fatalf("Ceph cluster services = %+v, want %+v", snapshot.CephClusters, want)
}
if len(snapshot.Hosts) != 1 || snapshot.Hosts[0].Ceph == nil {
t.Fatalf("expected one host with Ceph data, got %+v", snapshot.Hosts)
}
hostRows := make([]models.CephServiceStatus, 0, len(snapshot.Hosts[0].Ceph.Services))
for _, svc := range snapshot.Hosts[0].Ceph.Services {
hostRows = append(hostRows, models.CephServiceStatus{Type: svc.Type, Running: svc.Running, Total: svc.Total})
}
if !reflect.DeepEqual(hostRows, want) {
t.Fatalf("host Ceph services = %+v, want %+v", hostRows, want)
}
})
}
}
func TestApplyHostReportMapsReclaimableMemoryCache(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-cache", Version: "1.0.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{
ID: "machine-cache",
Hostname: "cache-host",
Platform: "linux",
},
Timestamp: time.Now().UTC(),
Metrics: agentshost.Metrics{
Memory: agentshost.MemoryMetric{
TotalBytes: 16 << 30,
UsedBytes: 6 << 30,
FreeBytes: 4 << 30,
CacheBytes: 6 << 30,
Usage: 37.5,
},
},
}
host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "token-cache", Name: "Token"})
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if got, want := host.Memory.Cache, int64(6<<30); got != want {
t.Fatalf("memory cache = %d, want %d", got, want)
}
// An inconsistent report (cache pushing past total) is clamped so the
// used | cache | free split can never exceed the bar.
report.Metrics.Memory.CacheBytes = 12 << 30
host, err = monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "token-cache", Name: "Token"})
if err != nil {
t.Fatalf("ApplyHostReport clamp: %v", err)
}
if got, want := host.Memory.Cache, int64(10<<30); got != want {
t.Fatalf("clamped memory cache = %d, want %d", got, want)
}
}
func TestApplyHostReportNormalizesLegacyAgentPlatformAcceptedIngestProof(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string),
config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
now := time.Now().UTC()
cases := []struct {
name string
platform string
want string
}{
// Legacy v5 agents report gopsutil's host.Info().Platform verbatim,
// e.g. "microsoft windows 11 pro" on Windows (refs #1555).
{"legacy windows caption", "microsoft windows 11 pro", "windows"},
{"darwin", "darwin", "macos"},
{"linux distro preserved", "Ubuntu", "ubuntu"},
}
for index, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: fmt.Sprintf("agent-platform-%d", index),
Version: "5.1.36",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: fmt.Sprintf("machine-platform-%d", index),
Hostname: fmt.Sprintf("host-platform-%d", index),
Platform: tc.platform,
},
Timestamp: now,
}
host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: fmt.Sprintf("token-platform-%d", index), Name: "Platform Token"})
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.Platform != tc.want {
t.Fatalf("expected ingested platform %q for reported %q, got %q", tc.want, tc.platform, host.Platform)
}
})
}
}
// hostFromReadStateView must carry the fabric's integration-source
// discriminator onto models.Host so ledger consumers (connections aggregator,
// update readiness) can tell real Pulse Agents from integration-monitored
// machines, while agent-sourced hosts stay unmarked.
func TestHostFromReadStateViewMapsIntegrationSource(t *testing.T) {
t.Parallel()
vmwareResource := unifiedresources.Resource{
ID: "vc-1:host:host-101",
Type: unifiedresources.ResourceTypeAgent,
Name: "esxi-01.lab.local",
Sources: []unifiedresources.DataSource{unifiedresources.SourceVMware},
Agent: &unifiedresources.AgentData{AgentID: "vc-1:host:host-101", Platform: "vmware-vsphere"},
}
vmwareView := unifiedresources.NewHostView(&vmwareResource)
if got := hostFromReadStateView(&vmwareView).IntegrationSource; got != "vmware" {
t.Fatalf("vSphere-only host must map IntegrationSource %q, got %q", "vmware", got)
}
agentResource := unifiedresources.Resource{
ID: "host-linux-1",
Type: unifiedresources.ResourceTypeAgent,
Name: "apollo-114",
Sources: []unifiedresources.DataSource{unifiedresources.SourceAgent},
Agent: &unifiedresources.AgentData{AgentID: "host-linux-1", AgentVersion: "6.1.0"},
}
agentView := unifiedresources.NewHostView(&agentResource)
if got := hostFromReadStateView(&agentView).IntegrationSource; got != "" {
t.Fatalf("agent-sourced host must map empty IntegrationSource, got %q", got)
}
}
// Removal must retain the agent's last-known platform on the removed record so
// the fleet doctor can still scope host-side cleanup commands after the live
// host record is gone.
func TestRemoveHostAgentRetainsLastKnownPlatform(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),
config: &config.Config{},
}
now := time.Now().UTC()
monitor.state.UpsertHost(models.Host{
ID: "removed-platform-host",
Hostname: "win-node",
Platform: "Microsoft Windows Server 2022",
Status: "online",
LastSeen: now.Add(-time.Minute),
})
monitor.state.UpsertHost(models.Host{
ID: "removed-osname-host",
Hostname: "debian-node",
OSName: "Debian GNU/Linux",
Status: "online",
LastSeen: now.Add(-time.Minute),
})
for _, hostID := range []string{"removed-platform-host", "removed-osname-host"} {
if _, err := monitor.RemoveHostAgent(hostID); err != nil {
t.Fatalf("RemoveHostAgent(%s): %v", hostID, err)
}
}
got := make(map[string]string)
for _, entry := range monitor.state.GetRemovedHostAgents() {
got[entry.ID] = entry.Platform
}
if got["removed-platform-host"] != "Microsoft Windows Server 2022" {
t.Fatalf("removed record platform = %q, want last-known reported platform", got["removed-platform-host"])
}
if got["removed-osname-host"] != "Debian GNU/Linux" {
t.Fatalf("removed record platform = %q, want OS-name fallback when Platform was never reported", got["removed-osname-host"])
}
}
// Wearout 0 means no endurance remaining, so the absent case has to carry the
// unreported sentinel rather than the struct zero value. The only production
// call site filters nil views, but a zero-valued disk escaping this helper
// would publish a spent disk that never existed.
func TestPhysicalDiskFromReadStateView_AbsentViewReportsUnknownWearout(t *testing.T) {
disk := physicalDiskFromReadStateView(nil)
if disk.Wearout != unifiedresources.WearoutUnreported {
t.Fatalf("absent view wearout = %d, want %d", disk.Wearout, unifiedresources.WearoutUnreported)
}
}
// Discovery suppression exists so Pulse does not probe hosts it already holds
// credentials for and write unauthenticated requests into the operator's own
// PBS/PVE logs (#1616). Taking only the first resolved address left a
// multi-homed host partly suppressed, and discovery reaches whichever address
// it reaches, not the one we happened to keep.
func TestConfiguredHostIPs_CoversEveryResolvedIPv4(t *testing.T) {
originalLookup := lookupConfiguredHostIP
lookupConfiguredHostIP = func(host string) ([]net.IP, error) {
return []net.IP{
net.ParseIP("2001:db8::1"),
net.ParseIP("192.0.2.10"),
net.ParseIP("192.0.2.11"),
}, nil
}
defer func() { lookupConfiguredHostIP = originalLookup }()
ips := configuredHostIPsFromConfig(&config.Config{
PBSInstances: []config.PBSInstance{{Host: "pbs.example.test"}},
})
for _, want := range []string{"192.0.2.10", "192.0.2.11"} {
found := false
for _, got := range ips {
if got == want {
found = true
break
}
}
if !found {
t.Fatalf("configured host IPs %v missing resolved address %s", ips, want)
}
}
}
// A host configured by literal IP needs no resolution and must be suppressed
// whatever DNS is doing.
func TestConfiguredHostIPs_LiteralAddressNeedsNoResolution(t *testing.T) {
originalLookup := lookupConfiguredHostIP
lookupConfiguredHostIP = func(host string) ([]net.IP, error) {
t.Fatalf("literal address must not be resolved, got lookup for %q", host)
return nil, nil
}
defer func() { lookupConfiguredHostIP = originalLookup }()
ips := configuredHostIPsFromConfig(&config.Config{
PBSInstances: []config.PBSInstance{{Host: "https://192.0.2.20:8007"}},
})
if len(ips) != 1 || ips[0] != "192.0.2.20" {
t.Fatalf("configured host IPs = %v, want [192.0.2.20]", ips)
}
}
// A name that does not resolve yields no suppression entry. That is
// unavoidable, but it must not take the other configured hosts down with it.
func TestConfiguredHostIPs_UnresolvableHostDoesNotDropOthers(t *testing.T) {
originalLookup := lookupConfiguredHostIP
lookupConfiguredHostIP = func(host string) ([]net.IP, error) {
if host == "broken.example.test" {
return nil, errors.New("no such host")
}
return []net.IP{net.ParseIP("192.0.2.30")}, nil
}
defer func() { lookupConfiguredHostIP = originalLookup }()
ips := configuredHostIPsFromConfig(&config.Config{
PBSInstances: []config.PBSInstance{
{Host: "broken.example.test"},
{Host: "good.example.test"},
},
})
if len(ips) != 1 || ips[0] != "192.0.2.30" {
t.Fatalf("configured host IPs = %v, want only the resolvable host [192.0.2.30]", ips)
}
}
// cleanupRRDCache is part of monitor housekeeping for agent-derived memory
// evidence. After the dead guest RRD memory cache was removed (#1634), the
// short-lived caches it prunes are the node RRD cache and the guest-agent
// meminfo cache; stale guest-agent entries must age out so removed or
// re-enrolled guests cannot replay old agent memory evidence.
func TestCleanupRRDCachePrunesStaleGuestAgentMemCache(t *testing.T) {
now := time.Now()
m := &Monitor{
nodeRRDMemCache: map[string]rrdMemCacheEntry{
"inst/node-stale": {fetchedAt: now.Add(-3 * nodeRRDCacheTTL)},
"inst/node-fresh": {fetchedAt: now},
},
vmAgentMemCache: map[string]agentMemCacheEntry{
"inst/node/100": {fetchedAt: now.Add(-vmAgentMemCleanupMaxAge - time.Second)},
"inst/node/101": {fetchedAt: now},
},
}
m.cleanupRRDCache(now)
if _, ok := m.nodeRRDMemCache["inst/node-stale"]; ok {
t.Error("stale node RRD cache entry not pruned")
}
if _, ok := m.nodeRRDMemCache["inst/node-fresh"]; !ok {
t.Error("fresh node RRD cache entry pruned")
}
if _, ok := m.vmAgentMemCache["inst/node/100"]; ok {
t.Error("stale guest-agent meminfo cache entry not pruned")
}
if _, ok := m.vmAgentMemCache["inst/node/101"]; !ok {
t.Error("fresh guest-agent meminfo cache entry pruned")
}
}
func issue1654Monitor() *Monitor {
return &Monitor{
state: models.NewState(),
config: &config.Config{},
rateTracker: NewRateTracker(),
hostTokenBindings: make(map[string]string),
hostReportOrders: make(map[string]hostReportOrder),
}
}
func issue1654Report(timestamp time.Time) agentshost.Report {
return agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "new-agent-install",
Version: "6.2.0",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
ID: "new-install-id",
MachineID: "machine-stable",
Hostname: "disk-host.local",
Platform: "linux",
},
Timestamp: timestamp,
}
}
func issue1654InstallToken(id string, createdAt time.Time) *config.APITokenRecord {
return &config.APITokenRecord{
ID: id,
CreatedAt: createdAt,
Metadata: map[string]string{
"install_type": "host",
"issued_via": agentbinding.IssuedViaConfig,
},
}
}
func TestIssue1654FreshInstallReusesStalePhysicalHostIdentity(t *testing.T) {
monitor := issue1654Monitor()
createdAt := time.Now().UTC()
monitor.state.UpsertHost(models.Host{
ID: "original-host-id",
Hostname: "disk-host.local",
MachineID: "machine-stable",
TokenID: "old-token",
LastSeen: createdAt.Add(-time.Minute),
Status: "offline",
})
monitor.hostTokenBindings["old-token:disk-host.local"] = "original-host-id"
host, err := monitor.ApplyHostReport(
issue1654Report(createdAt.Add(time.Minute)),
issue1654InstallToken("fresh-token", createdAt),
)
if err != nil {
t.Fatalf("ApplyHostReport() error = %v", err)
}
if host.ID != "original-host-id" {
t.Fatalf("host ID = %q, want stable original-host-id", host.ID)
}
if got := len(monitor.state.GetHosts()); got != 1 {
t.Fatalf("host count = %d, want 1", got)
}
if got := monitor.hostTokenBindings[hostTokenMachineBindingKey("fresh-token", "disk-host.local", "machine-stable")]; got != "original-host-id" {
t.Fatalf("fresh token binding = %q, want original-host-id", got)
}
if _, ok := monitor.hostTokenBindings["old-token:disk-host.local"]; ok {
t.Fatal("pre-install token binding survived stable-ID re-enrollment")
}
}
// TestIssue1753SharedTokenSameHostnameKeepsMachinesDistinct models two
// standalone Proxmox sites that reuse one short node name and were installed
// with the same pasted install token. The token+hostname binding must not fold
// the second machine into the first one's identity: pre-fix, the two records
// collapsed into a single flip-flopping host, and removing that host revoked
// the shared token, which surfaced as 401 "Unauthorized access attempt" on
// /api/agents/agent/report for the surviving site.
func TestIssue1753SharedTokenSameHostnameKeepsMachinesDistinct(t *testing.T) {
monitor := issue1654Monitor()
createdAt := time.Now().UTC().Add(-time.Hour)
sharedToken := issue1654InstallToken("shared-token", createdAt)
siteReport := func(machineID string, timestamp time.Time) agentshost.Report {
return agentshost.Report{
Agent: agentshost.AgentInfo{
Version: "6.4.2",
IntervalSeconds: 30,
},
Host: agentshost.HostInfo{
MachineID: machineID,
Hostname: "pve01",
Platform: "proxmox",
},
Timestamp: timestamp,
}
}
now := time.Now().UTC()
var siteAID, siteBID string
for round := 0; round < 3; round++ {
hostA, err := monitor.ApplyHostReport(siteReport("machine-site-a", now.Add(time.Duration(round)*time.Minute)), sharedToken)
if err != nil {
t.Fatalf("round %d site A ApplyHostReport() error = %v", round, err)
}
hostB, err := monitor.ApplyHostReport(siteReport("machine-site-b", now.Add(time.Duration(round)*time.Minute+30*time.Second)), sharedToken)
if err != nil {
t.Fatalf("round %d site B ApplyHostReport() error = %v", round, err)
}
if round == 0 {
siteAID, siteBID = hostA.ID, hostB.ID
if siteAID == siteBID {
t.Fatalf("both sites resolved to one identity %q", siteAID)
}
continue
}
if hostA.ID != siteAID || hostB.ID != siteBID {
t.Fatalf("round %d identities flapped: site A %q (want %q), site B %q (want %q)",
round, hostA.ID, siteAID, hostB.ID, siteBID)
}
}
hosts := monitor.state.GetHosts()
if len(hosts) != 2 {
t.Fatalf("host count = %d, want 2 (one per site)", len(hosts))
}
for _, host := range hosts {
if host.TokenID != "shared-token" {
t.Fatalf("host %q token = %q, want shared-token", host.ID, host.TokenID)
}
}
}
func TestInstallHandoffReusesIdentityAcrossFinalInFlightReport(t *testing.T) {
monitor := issue1654Monitor()
createdAt := time.Now().UTC().Add(-2 * time.Minute)
monitor.state.UpsertHost(models.Host{
ID: "original-host-id",
Hostname: "disk-host.local",
MachineID: "machine-stable",
ReportIP: "192.0.2.10",
TokenID: "old-token",
LastSeen: createdAt.Add(time.Minute),
IntervalSeconds: 30,
Status: "online",
})
monitor.hostTokenBindings["old-token:disk-host.local"] = "original-host-id"
report := issue1654Report(time.Now().UTC())
report.Host.ReportIP = "192.0.2.10"
host, err := monitor.ApplyHostReport(report, issue1654InstallToken("fresh-token", createdAt))
if err != nil {
t.Fatalf("ApplyHostReport() error = %v", err)
}
if host.ID != "original-host-id" {
t.Fatalf("host ID = %q, want stable original-host-id", host.ID)
}
if got := len(monitor.state.GetHosts()); got != 1 {
t.Fatalf("host count = %d, want 1", got)
}
if _, ok := monitor.hostTokenBindings["old-token:disk-host.local"]; ok {
t.Fatal("overlapped old-token binding survived install handoff")
}
}
func TestInstallHandoffRequiresTrustedUnambiguousIdentityEvidence(t *testing.T) {
createdAt := time.Now().UTC()
report := issue1654Report(createdAt.Add(time.Minute))
report.Host.ReportIP = "192.0.2.10"
candidate := models.Host{
ID: "original-host-id",
Hostname: "disk-host.local",
MachineID: "machine-stable",
ReportIP: "192.0.2.10",
TokenID: "old-token",
LastSeen: createdAt.Add(time.Minute),
IntervalSeconds: 30,
}
for _, test := range []struct {
name string
token *config.APITokenRecord
mutate func(*agentshost.Report, *models.Host)
secondary *models.Host
}{
{
name: "arbitrary API token",
token: &config.APITokenRecord{ID: "fresh-token", CreatedAt: createdAt},
},
{
name: "report IP changed",
token: issue1654InstallToken("fresh-token", createdAt),
mutate: func(report *agentshost.Report, _ *models.Host) {
report.Host.ReportIP = "192.0.2.11"
},
},
{
name: "identity conflict active",
token: issue1654InstallToken("fresh-token", createdAt),
mutate: func(_ *agentshost.Report, candidate *models.Host) {
candidate.IdentityConflict = &models.HostIdentityConflict{Hostnames: []string{"disk-host.local", "clone.local"}}
},
},
{
name: "overlap exceeds health window",
token: issue1654InstallToken("fresh-token", createdAt),
mutate: func(_ *agentshost.Report, candidate *models.Host) {
candidate.LastSeen = createdAt.Add(4 * time.Minute)
},
},
{
name: "multiple matching identities",
token: issue1654InstallToken("fresh-token", createdAt),
secondary: &models.Host{
ID: "another-host-id",
Hostname: "disk-host.local",
MachineID: "machine-stable",
ReportIP: "192.0.2.10",
TokenID: "another-old-token",
LastSeen: createdAt.Add(time.Minute),
IntervalSeconds: 30,
},
},
} {
t.Run(test.name, func(t *testing.T) {
testReport := report
testCandidate := candidate
if test.mutate != nil {
test.mutate(&testReport, &testCandidate)
}
hosts := []models.Host{testCandidate}
if test.secondary != nil {
hosts = append(hosts, *test.secondary)
}
if got := staleHostIdentityForReenrollment(testReport, test.token, hosts); got != "" {
t.Fatalf("staleHostIdentityForReenrollment() = %q, want no handoff", got)
}
})
}
}
func TestIssue1654ExistingDuplicateGenerationIsSuperseded(t *testing.T) {
monitor := issue1654Monitor()
createdAt := time.Now().UTC()
monitor.state.UpsertHost(models.Host{
ID: "stale-host-id",
Hostname: "disk-host.local",
MachineID: "machine-stable",
TokenID: "old-token",
LastSeen: createdAt.Add(-time.Minute),
Status: "offline",
})
monitor.state.UpsertHost(models.Host{
ID: "current-host-id",
Hostname: "disk-host.local",
MachineID: "machine-stable",
TokenID: "fresh-token",
LastSeen: createdAt.Add(time.Minute),
Status: "online",
})
monitor.hostTokenBindings["old-token:disk-host.local"] = "stale-host-id"
monitor.hostTokenBindings["fresh-token:disk-host.local"] = "current-host-id"
host, err := monitor.ApplyHostReport(
issue1654Report(createdAt.Add(2*time.Minute)),
&config.APITokenRecord{ID: "fresh-token", CreatedAt: createdAt},
)
if err != nil {
t.Fatalf("ApplyHostReport() error = %v", err)
}
if host.ID != "current-host-id" {
t.Fatalf("host ID = %q, want current-host-id", host.ID)
}
hosts := monitor.state.GetHosts()
if len(hosts) != 1 || hosts[0].ID != "current-host-id" {
t.Fatalf("hosts after supersession = %+v", hosts)
}
if _, ok := monitor.hostTokenBindings["old-token:disk-host.local"]; ok {
t.Fatal("stale token binding survived supersession")
}
}
func TestIssue1654LivePreexistingAgentIsNotSuperseded(t *testing.T) {
monitor := issue1654Monitor()
createdAt := time.Now().UTC().Add(-5 * time.Minute)
monitor.state.UpsertHost(models.Host{
ID: "still-live-host",
Hostname: "disk-host.local",
MachineID: "machine-stable",
TokenID: "old-token",
LastSeen: createdAt.Add(4 * time.Minute),
IntervalSeconds: 30,
Status: "online",
})
if got := staleHostIdentityForReenrollment(
issue1654Report(createdAt.Add(5*time.Minute)),
issue1654InstallToken("fresh-token", createdAt),
monitor.state.GetHosts(),
); got != "" {
t.Fatalf("live candidate unexpectedly eligible for install handoff: %q", got)
}
_, err := monitor.ApplyHostReport(
issue1654Report(createdAt.Add(5*time.Minute)),
issue1654InstallToken("fresh-token", createdAt),
)
if err != nil {
t.Fatalf("ApplyHostReport() error = %v", err)
}
if got := len(monitor.state.GetHosts()); got != 2 {
t.Fatalf("host count = %d, want both live identities preserved", got)
}
}
func TestNormalizeAgentXCPNGInventoryBoundsAndPreservesFailures(t *testing.T) {
now := time.Date(2026, 7, 30, 15, 0, 0, 0, time.UTC)
report := &agentshost.XCPNGInventory{
PoolUUID: "11111111-1111-1111-1111-111111111111",
PoolName: " Lab Pool ",
LocalHostUUID: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
VMs: []agentshost.XCPNGVM{
{
UUID: "cccccccc-cccc-cccc-cccc-cccccccccccc",
Name: " Database ",
PowerState: "RUNNING",
VCPUs: 5000,
MemoryActual: 10 << 30,
MemoryStaticMax: 8 << 30,
ResidentHostUUID: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa",
},
{UUID: "cccccccc-cccc-cccc-cccc-cccccccccccc", Name: "duplicate"},
{UUID: "bad", Name: "invalid"},
},
}
got := normalizeAgentXCPNGInventory(report, nil, now)
if got == nil || len(got.VMs) != 1 {
t.Fatalf("inventory = %+v", got)
}
if got.PoolName != "Lab Pool" ||
got.VMs[0].Name != "Database" ||
got.VMs[0].PowerState != "running" ||
got.VMs[0].VCPUs != 4096 ||
got.VMs[0].MemoryActual != 8<<30 {
t.Fatalf("normalized inventory = %+v", got)
}
previous := &models.Host{XCPNG: got}
preserved := normalizeAgentXCPNGInventory(nil, previous, now.Add(time.Minute))
if preserved == nil || len(preserved.VMs) != 1 || !preserved.CollectedAt.Equal(now) {
t.Fatalf("preserved inventory = %+v", preserved)
}
preserved.VMs[0].Name = "mutated"
if previous.XCPNG.VMs[0].Name != "Database" {
t.Fatal("preserved inventory aliases previous host state")
}
}
func TestAgentLXCFilesystemsRequireLinkedNodeAndExactRunningGuest(t *testing.T) {
now := time.Date(2026, 7, 30, 21, 0, 0, 0, time.UTC)
state := models.NewState()
state.UpdateNodes([]models.Node{{
ID: "pve-a:node-a",
Name: "node-a",
Instance: "pve-a",
LinkedAgentID: "agent-a",
}})
monitor := &Monitor{state: state}
monitor.applyAgentLXCFilesystems(
"pve-a:node-a",
"agent-a",
&agentshost.ProxmoxLXCInventory{Containers: []agentshost.ProxmoxLXCContainer{{
VMID: 100,
Name: "web",
Disks: []agentshost.Disk{
{
Device: "local-lvm:vm-100-disk-0",
Mountpoint: "/",
Type: "rootfs",
TotalBytes: 8 << 30,
UsedBytes: 2 << 30,
FreeBytes: 6 << 30,
Usage: 99,
},
{
Device: "tank:subvol-100-disk-0",
Mountpoint: "/srv/data",
Type: "mp0",
TotalBytes: 1 << 40,
UsedBytes: 512 << 30,
FreeBytes: 512 << 30,
},
{
Device: "invalid",
Mountpoint: "/srv/../etc",
Type: "mp1",
TotalBytes: 1,
},
},
}}},
now,
30,
)
container := models.Container{
VMID: 100,
Name: "web",
Status: "running",
Disk: models.Disk{Total: 8 << 30, Used: 1 << 30, Mountpoint: "/"},
}
monitor.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &container, now.Add(time.Minute))
if len(container.Disks) != 2 {
t.Fatalf("agent LXC disks = %+v", container.Disks)
}
if container.Disk.Used != 2<<30 || container.Disk.Usage != 25 {
t.Fatalf("agent root disk = %+v", container.Disk)
}
if container.Disks[1].Mountpoint != "/srv/data" || container.Disks[1].Type != "mp0" {
t.Fatalf("agent mount disk = %+v", container.Disks[1])
}
for _, test := range []struct {
name string
instance string
node string
guestName string
status string
}{
{name: "wrong instance", instance: "pve-b", node: "node-a", guestName: "web", status: "running"},
{name: "wrong node", instance: "pve-a", node: "node-b", guestName: "web", status: "running"},
{name: "renamed guest", instance: "pve-a", node: "node-a", guestName: "web-new", status: "running"},
{name: "stopped guest", instance: "pve-a", node: "node-a", guestName: "web", status: "stopped"},
} {
t.Run(test.name, func(t *testing.T) {
candidate := models.Container{
VMID: 100,
Name: test.guestName,
Status: test.status,
Disk: models.Disk{Used: 1},
}
monitor.enrichContainerWithAgentLXCFilesystems(
test.instance,
test.node,
&candidate,
now.Add(time.Minute),
)
if len(candidate.Disks) != 0 || candidate.Disk.Used != 1 {
t.Fatalf("unmatched guest was enriched: %+v", candidate)
}
})
}
monitor.clearAgentLXCFilesystems("agent-a")
unlinked := models.Container{
VMID: 100,
Name: "web",
Status: "running",
Disk: models.Disk{Used: 1},
}
monitor.enrichContainerWithAgentLXCFilesystems(
"pve-a",
"node-a",
&unlinked,
now.Add(time.Minute),
)
if len(unlinked.Disks) != 0 || unlinked.Disk.Used != 1 {
t.Fatalf("removed agent cache was applied: %+v", unlinked)
}
}
func TestAgentLXCFilesystemsExpireAndRejectAmbiguousNodeLink(t *testing.T) {
now := time.Date(2026, 7, 30, 21, 0, 0, 0, time.UTC)
inventory := &agentshost.ProxmoxLXCInventory{Containers: []agentshost.ProxmoxLXCContainer{{
VMID: 100,
Name: "web",
Disks: []agentshost.Disk{{
Device: "local-lvm:vm-100-disk-0",
Mountpoint: "/",
Type: "rootfs",
TotalBytes: 8 << 30,
UsedBytes: 2 << 30,
FreeBytes: 6 << 30,
}},
}}}
ambiguousState := models.NewState()
ambiguousState.UpdateNodes([]models.Node{
{ID: "duplicate", Name: "node-a", Instance: "pve-a"},
{ID: "duplicate", Name: "node-b", Instance: "pve-b"},
})
ambiguousMonitor := &Monitor{state: ambiguousState}
ambiguousMonitor.applyAgentLXCFilesystems("duplicate", "agent-a", inventory, now, 30)
if len(ambiguousMonitor.proxmoxLXCFilesystemsCache) != 0 {
t.Fatalf("ambiguous node link was admitted: %+v", ambiguousMonitor.proxmoxLXCFilesystemsCache)
}
state := models.NewState()
state.UpdateNodes([]models.Node{{
ID: "node-id",
Name: "node-a",
Instance: "pve-a",
LinkedAgentID: "agent-a",
}})
monitor := &Monitor{state: state}
monitor.applyAgentLXCFilesystems("node-id", "agent-a", inventory, now, 30)
container := models.Container{
VMID: 100,
Name: "web",
Status: "running",
Disk: models.Disk{Used: 1},
}
monitor.enrichContainerWithAgentLXCFilesystems(
"pve-a",
"node-a",
&container,
now.Add(agentLXCFilesystemMinTTL),
)
if len(container.Disks) != 0 || container.Disk.Used != 1 {
t.Fatalf("expired cache enriched guest: %+v", container)
}
if len(monitor.proxmoxLXCFilesystemsCache) != 0 {
t.Fatalf("expired cache entry was not removed: %+v", monitor.proxmoxLXCFilesystemsCache)
}
monitor.proxmoxLXCFilesystemsCache["expired-elsewhere"] = agentLXCFilesystemCacheEntry{
name: "old",
expiresAt: now,
}
monitor.applyAgentLXCFilesystems(
"node-id",
"agent-a",
&agentshost.ProxmoxLXCInventory{},
now.Add(time.Second),
30,
)
if len(monitor.proxmoxLXCFilesystemsCache) != 0 {
t.Fatalf("accepted empty inventory did not prune expired entries: %+v", monitor.proxmoxLXCFilesystemsCache)
}
}
// A stopped monitor must not write into its data directory afterwards.
// Tenant offboarding and any caller that removes a monitor's data directory
// depends on Stop having quiesced disk writes; before the guest metadata store
// tracked its own async writes, a queued write could land after Stop and leave
// a partially-written .tmp behind.
func TestMonitorStopQuiescesMetadataWritesBeforeReturning(t *testing.T) {
dataPath := t.TempDir()
monitor := &Monitor{
state: models.NewState(),
guestMetadataStore: config.NewGuestMetadataStore(dataPath, nil),
}
for i := range 40 {
monitor.guestMetadataStore.SetAsync(fmt.Sprintf("pve1:node1:%d", 100+i), &config.GuestMetadata{
ID: fmt.Sprintf("pve1:node1:%d", 100+i),
LastKnownName: fmt.Sprintf("vm-%d", i),
LastKnownType: "qemu",
})
}
monitor.Stop()
entries, err := os.ReadDir(dataPath)
if err != nil {
t.Fatalf("read data dir: %v", err)
}
for _, entry := range entries {
if strings.HasSuffix(entry.Name(), ".tmp") {
t.Fatalf("write still in flight after Stop: %q", entry.Name())
}
}
// The directory must be removable immediately, which is the property tenant
// teardown relies on.
if err := os.RemoveAll(dataPath); err != nil {
t.Fatalf("data directory not removable after Stop: %v", err)
}
}
// TestMockModeDropsRealHostContinuity pins the mock-mode clean room on the
// standalone host continuity path. Continuity entries are persisted to disk
// from real agent reports and survive the mock toggle, and every consumer
// injects them after the read path has already substituted the mock snapshot,
// so without a guard a real machine that reported before mock mode was enabled
// resurfaces on Machines by its real hostname.
func TestMockModeDropsRealHostContinuity(t *testing.T) {
previous := mock.IsMockEnabled()
if err := mock.SetEnabled(true); err != nil {
t.Fatalf("enable mock mode: %v", err)
}
t.Cleanup(func() { _ = mock.SetEnabled(previous) })
store := config.NewHostContinuityStore(t.TempDir(), nil)
if err := store.Upsert(config.HostContinuityEntry{
HostID: "real-host-1",
ReportHostID: "real-machine-1",
Hostname: "real-machine.lan",
DisplayName: "Real Machine",
MachineID: "real-machine-1",
AgentVersion: "6.2.0",
Platform: "darwin",
LastSeen: time.Now().UTC(),
}); err != nil {
t.Fatalf("Upsert continuity: %v", err)
}
m := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),
hostContinuityStore: store,
}
for _, host := range m.HostsSnapshot() {
if host.Hostname == "real-machine.lan" || host.ID == "real-host-1" {
t.Fatalf("real continuity host leaked into the mock read state: %#v", host)
}
}
// The suppression must be mock-scoped, not a removal of continuity itself:
// with mock mode off the same store still backs the standalone host.
if err := mock.SetEnabled(false); err != nil {
t.Fatalf("disable mock mode: %v", err)
}
hosts := m.HostsSnapshot()
if len(hosts) != 1 || hosts[0].Hostname != "real-machine.lan" {
t.Fatalf("real mode must still restore continuity-backed hosts, got %#v", hosts)
}
}
// TestMockUnifiedViewAppliesOperatorManualLinks pins operator links on the
// mock-mode unified view. The fixture graph is built without the link store,
// so a standalone agent linked into a VM showed merged in the resources API,
// which re-seeds through the store-backed registry, but stayed a separate row
// in the websocket broadcast and /api/state. The view applies the links the
// monitor's resource store last loaded, rebuilds when they change without
// waiting for a fixture tick, and never forwards fixture data to the store.
func TestMockUnifiedViewAppliesOperatorManualLinks(t *testing.T) {
previous := mock.IsMockEnabled()
previousConfig := mock.GetConfig()
if previous {
mustSetMockEnabled(t, false)
}
// Hold the fixture graph still, so only the link can change the view.
testConfig := previousConfig
testConfig.UpdateInterval = 5 * time.Minute
mock.SetMockConfig(testConfig)
mustSetMockEnabled(t, true)
t.Cleanup(func() {
mustSetMockEnabled(t, false)
mock.SetMockConfig(previousConfig)
mustSetMockEnabled(t, previous)
})
store := unifiedresources.NewMemoryStore()
m := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
alertManager: alerts.NewManager(),
}
t.Cleanup(m.alertManager.Stop)
unlinked := m.currentUnifiedStateView()
var vmID, agentID string
for _, resource := range unlinked.resources {
switch {
case vmID == "" && resource.Type == unifiedresources.ResourceTypeVM && resource.Agent == nil:
vmID = resource.ID
case agentID == "" && resource.Type == unifiedresources.ResourceTypeAgent && resource.Agent != nil &&
len(resource.Sources) == 1 && resource.Sources[0] == unifiedresources.SourceAgent:
agentID = resource.ID
}
}
if vmID == "" || agentID == "" {
t.Fatalf("fixture graph needs an agentless VM and a standalone agent, got vm=%q agent=%q", vmID, agentID)
}
if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}); err != nil {
t.Fatalf("AddLink: %v", err)
}
version := mock.FixtureDataVersion()
// The broadcast's read-path refresh rebuilds the resource store, which
// loads the link; the mock view must follow it within the same build.
frontend := m.BuildBroadcastFrontendState()
if got := mock.FixtureDataVersion(); got != version {
t.Fatalf("fixture data version moved from %d to %d; the link alone must refresh the view", version, got)
}
var vmRow *models.ResourceFrontend
for i := range frontend.Resources {
switch frontend.Resources[i].ID {
case vmID:
vmRow = &frontend.Resources[i]
case agentID:
t.Fatalf("broadcast still lists the linked agent %s as its own row", agentID)
}
}
if vmRow == nil || len(vmRow.Agent) == 0 {
t.Fatalf("broadcast VM row %s = %+v, want it to carry the linked agent's facet", vmID, vmRow)
}
linked := m.currentUnifiedStateView()
if linked.readState == unlinked.readState {
t.Fatal("expected the view to rebuild when the operator links changed")
}
// The resources API keys its registry cache on this freshness, so a
// link change must advance it even though no fixture tick did.
if !linked.freshness.After(unlinked.freshness) {
t.Fatalf("view freshness %v did not advance past %v on the link change", linked.freshness, unlinked.freshness)
}
if again := m.currentUnifiedStateView(); again.readState != linked.readState {
t.Fatal("expected the linked view to stay cached while neither the fixtures nor the links change")
}
// The resources API re-seeds the same snapshot through a registry backed
// by the link store. Both payloads must list the same rows.
seed, _ := m.UnifiedResourceSnapshot()
rest := unifiedresources.NewRegistry(store)
rest.IngestResources(seed)
viewIDs := make([]string, 0, len(linked.resources))
for _, resource := range linked.resources {
viewIDs = append(viewIDs, resource.ID)
}
restIDs := make([]string, 0, len(seed))
for _, resource := range rest.List() {
restIDs = append(restIDs, resource.ID)
}
slices.Sort(viewIDs)
slices.Sort(restIDs)
if !slices.Equal(viewIDs, restIDs) {
t.Fatalf("mock view and resources API disagree:\nview %v\nrest %v", viewIDs, restIDs)
}
// The view's registry has no store: history recorded through the read
// state it hands out must not land fixture rows in the operator's store.
recorder, ok := linked.readState.(interface {
RecordChange(unifiedresources.ResourceChange) error
})
if !ok {
t.Fatalf("mock view read state %T does not record changes", linked.readState)
}
before, err := store.GetRecentChanges(vmID, time.Time{}, 1000)
if err != nil {
t.Fatalf("GetRecentChanges: %v", err)
}
if err := recorder.RecordChange(unifiedresources.ResourceChange{
ID: "mock-view-probe",
ObservedAt: time.Now().UTC(),
ResourceID: vmID,
Kind: unifiedresources.ChangeStateTransition,
}); err != nil {
t.Fatalf("RecordChange: %v", err)
}
after, err := store.GetRecentChanges(vmID, time.Time{}, 1000)
if err != nil {
t.Fatalf("GetRecentChanges: %v", err)
}
if len(after) != len(before) {
t.Fatalf("mock view wrote %d change rows into the durable store", len(after)-len(before))
}
// Reconcile the mock link proposal with the current latest-decision
// rule: a fresh monitor must restore the link, and unlink/relink must
// invalidate the seed cache without a fixture tick. Mock seeds retain
// their own default thresholds, not the live adapter's configured ones.
restarted := &Monitor{
state: models.NewState(),
config: &config.Config{PVEPollingInterval: 2 * time.Minute},
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
alertManager: alerts.NewManager(),
}
t.Cleanup(restarted.alertManager.Stop)
adapter := restarted.resourceStore.(*unifiedresources.MonitorAdapter)
assertMockLink := func(wantLinked bool) time.Time {
t.Helper()
frontend := restarted.BuildBroadcastFrontendState()
var vmFound, agentFound, vmHasAgent bool
for _, resource := range frontend.Resources {
if resource.ID == vmID {
vmFound, vmHasAgent = true, len(resource.Agent) > 0
}
if resource.ID == agentID {
agentFound = true
}
}
if !vmFound || vmHasAgent != wantLinked || agentFound == wantLinked {
t.Fatalf("mock link=%t: vm=%t vmAgent=%t standaloneAgent=%t", wantLinked, vmFound, vmHasAgent, agentFound)
}
seed, freshness, thresholds := restarted.UnifiedResourceSnapshotWithStaleThresholds()
if len(thresholds) != 0 {
t.Fatalf("mock seed borrowed live thresholds: %v", thresholds)
}
// The held five-minute mock tick sights every PVE node once per tick,
// so that cadence widens the configured four minutes to ten.
if got := adapter.StaleThresholds()[unifiedresources.SourceProxmox]; got != 10*time.Minute {
t.Fatalf("live adapter threshold = %v, want ten minutes from the mock PVE cadence", got)
}
rest := unifiedresources.NewRegistryWithStaleThresholds(store, thresholds)
rest.IngestResources(seed)
_, hasStandaloneAgent := rest.Get(agentID)
if hasStandaloneAgent == wantLinked {
t.Fatalf("REST seed link=%t: standalone agent=%t", wantLinked, hasStandaloneAgent)
}
if got := mock.FixtureDataVersion(); got != version {
t.Fatalf("fixture version moved from %d to %d during operator lifecycle", version, got)
}
return freshness
}
restartFreshness := assertMockLink(true)
if err := store.AddExclusion(unifiedresources.ResourceExclusion{ResourceA: vmID, ResourceB: agentID}); err != nil {
t.Fatalf("unlink: %v", err)
}
// Reads deliberately reuse a generation younger than two seconds.
// Apply the owning ingest/rebuild boundary instead of waiting for that
// throttle or claiming a recorded decision changes a previous listing.
restarted.updateResourceStore(restarted.currentStateWithScope())
unlinkFreshness := assertMockLink(false)
if !unlinkFreshness.After(restartFreshness) {
t.Fatal("unlink must advance the mock seed's freshness")
}
if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: agentID, ResourceB: vmID, PrimaryID: vmID}); err != nil {
t.Fatalf("relink: %v", err)
}
restarted.updateResourceStore(restarted.currentStateWithScope())
if relinkFreshness := assertMockLink(true); !relinkFreshness.After(unlinkFreshness) {
t.Fatal("relink must advance the mock seed's freshness")
}
}
// TestMockUnifiedViewFollowsALinkCreationTimeChange pins the creation time as
// part of the mock view's cache key. Among otherwise equal candidates a chain
// of links folds into the primary of its earliest-created link, so a link list
// that differs from the cached one only in a creation time can leave a
// different row standing: in a cycle of a Proxmox storage, a TrueNAS VM and a
// vSphere VM the storage never competes, and the TrueNAS VM keeps its row
// while its link is older than the vSphere VM's and gives it to the vSphere VM
// once it is newer. The cache compared resource pairs and primaries only, so
// with the fixture data version held it kept serving the row the old times
// chose while the live rebuild, which reloads the links from the store, served
// the new one. Both readers of the key are held to it: the structure view
// (cachedMockStructureView) and the data-version view
// (currentUnifiedStateView).
func TestMockUnifiedViewFollowsALinkCreationTimeChange(t *testing.T) {
useMockEstate(t, 4, 5*time.Minute)
store := unifiedresources.NewMemoryStore()
m := &Monitor{
state: models.NewState(),
config: &config.Config{PVEPollingInterval: 2 * time.Minute},
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
alertManager: alerts.NewManager(),
}
t.Cleanup(m.alertManager.Stop)
soleSource := func(resource unifiedresources.Resource, source unifiedresources.DataSource) bool {
return len(resource.Sources) == 1 && resource.Sources[0] == source
}
var storageID, nasVMID, guestID string
for _, resource := range m.currentUnifiedStateView().resources {
switch {
case storageID == "" && resource.Type == unifiedresources.ResourceTypeStorage &&
soleSource(resource, unifiedresources.SourceProxmox):
storageID = resource.ID
case nasVMID == "" && resource.Type == unifiedresources.ResourceTypeVM &&
soleSource(resource, unifiedresources.SourceTrueNAS):
nasVMID = resource.ID
case guestID == "" && resource.Type == unifiedresources.ResourceTypeVM &&
soleSource(resource, unifiedresources.SourceVMware):
guestID = resource.ID
}
}
if storageID == "" || nasVMID == "" || guestID == "" {
t.Fatalf("fixture graph needs a Proxmox storage, a TrueNAS VM and a vSphere VM, got storage=%q nas=%q guest=%q",
storageID, nasVMID, guestID)
}
created := time.Date(2026, time.October, 1, 9, 0, 0, 0, time.UTC)
// The link whose primary is the TrueNAS VM is the last of the three, so
// replacing it keeps the list's order and changes only its creation time.
cycle := func(nasLinkCreated time.Time) []unifiedresources.ResourceLink {
return []unifiedresources.ResourceLink{
{ResourceA: nasVMID, ResourceB: storageID, PrimaryID: storageID, CreatedAt: created.Add(time.Minute)},
{ResourceA: storageID, ResourceB: guestID, PrimaryID: guestID, CreatedAt: created.Add(2 * time.Minute)},
{ResourceA: guestID, ResourceB: nasVMID, PrimaryID: nasVMID, CreatedAt: nasLinkCreated},
}
}
oldest, newest := created, created.Add(3*time.Minute)
setNasLinkCreated := func(at time.Time) {
t.Helper()
for _, link := range cycle(at) {
if err := store.AddLink(link); err != nil {
t.Fatalf("AddLink: %v", err)
}
}
// Reads deliberately reuse a generation younger than two seconds, so
// apply the rebuild boundary that loads the links instead of waiting.
m.updateResourceStore(m.currentStateWithScope())
}
// cycleRows is what is left of the three members: ID and sources.
cycleRows := func(resources []unifiedresources.Resource) []string {
var rows []string
for _, resource := range resources {
if resource.ID != storageID && resource.ID != nasVMID && resource.ID != guestID {
continue
}
sources := make([]string, 0, len(resource.Sources))
for _, source := range resource.Sources {
sources = append(sources, string(source))
}
slices.Sort(sources)
rows = append(rows, fmt.Sprintf("%s %v", resource.ID, sources))
}
return rows
}
// mockRows is the reference: the linked build the view is made of.
mockRows := func(links []unifiedresources.ResourceLink) []string {
resources, _ := mock.UnifiedResourceSnapshotWithLinks(links)
return cycleRows(resources)
}
wantNas, wantGuest := mockRows(cycle(oldest)), mockRows(cycle(newest))
if slices.Equal(wantNas, wantGuest) || len(wantNas) != 1 || len(wantGuest) != 1 ||
!strings.HasPrefix(wantNas[0], nasVMID+" ") || !strings.HasPrefix(wantGuest[0], guestID+" ") {
t.Fatalf("a creation time alone must move the cycle between the TrueNAS VM %s and the vSphere VM %s; got %v then %v",
nasVMID, guestID, wantNas, wantGuest)
}
version := mock.FixtureDataVersion()
assertLinksDifferOnlyInTime := func(before, after []unifiedresources.ResourceLink) {
t.Helper()
if len(before) != len(after) || len(before) != 3 {
t.Fatalf("link lists %+v and %+v need three links each", before, after)
}
differing := 0
for i := range before {
if before[i].ResourceA != after[i].ResourceA || before[i].ResourceB != after[i].ResourceB ||
before[i].PrimaryID != after[i].PrimaryID {
t.Fatalf("link %d changed its pair or primary: %+v -> %+v", i, before[i], after[i])
}
if !before[i].CreatedAt.Equal(after[i].CreatedAt) {
differing++
}
}
if differing != 1 {
t.Fatalf("%d links changed their creation time, want exactly one: %+v -> %+v", differing, before, after)
}
}
assertRows := func(label string, view monitorUnifiedStateView, want []string) {
t.Helper()
if got := cycleRows(view.resources); !slices.Equal(got, want) {
t.Fatalf("%s kept %v, want the row a linked mock build keeps: %v", label, got, want)
}
if got := mock.FixtureDataVersion(); got != version {
t.Fatalf("fixture data version moved from %d to %d; a creation time alone must refresh the view", version, got)
}
}
setNasLinkCreated(oldest)
first := m.currentUnifiedStateView()
assertRows("view with the TrueNAS VM's link oldest", first, wantNas)
firstLinks := m.resourceStoreManualLinks()
// The link list changes in its creation time alone. The structure reader
// answers from the cached view while the list is unchanged, so it must
// notice before anything else rebuilds the view.
setNasLinkCreated(newest)
assertLinksDifferOnlyInTime(firstLinks, m.resourceStoreManualLinks())
structure := m.currentStructureUnifiedStateView()
if structure.readState == first.readState {
t.Fatal("structure view served the view built before the creation time changed")
}
assertRows("structure view after the creation time changed", structure, wantGuest)
second := m.currentUnifiedStateView()
assertRows("view after the creation time changed", second, wantGuest)
if !second.freshness.After(first.freshness) {
t.Fatalf("view freshness %v did not advance past %v on a creation time change", second.freshness, first.freshness)
}
// And back: the data-version reader is held to the key on its own, with
// the structure reader no longer the one that rebuilt the view.
setNasLinkCreated(oldest)
assertLinksDifferOnlyInTime(m.resourceStoreManualLinks(), cycle(newest))
third := m.currentUnifiedStateView()
if third.readState == second.readState {
t.Fatal("view served the build made before the creation time changed back")
}
assertRows("view after the creation time changed back", third, wantNas)
if !third.freshness.After(second.freshness) {
t.Fatalf("view freshness %v did not advance past %v on a creation time change", third.freshness, second.freshness)
}
if again := m.currentUnifiedStateView(); again.readState != third.readState {
t.Fatal("expected the view to stay cached while neither the fixtures nor the links change")
}
}
// Alert evaluation resolves a target per resource and windowed metric on every
// pass. A metric tick moves the fixture's data version, not what resources
// exist, so a lookup after a tick must be served from the cached view instead
// of building an estate. Before this was pinned, every lookup after a tick
// rebuilt the view, and a process too starved to finish one build within a tick
// rebuilt it for every lookup of the pass: startup never reached its listener.
func TestMockMetricsTargetLookupsDoNotRebuildTheViewAfterFixtureTicks(t *testing.T) {
useMockEstate(t, 4, time.Second)
m := newMockEstateMonitor()
warm := m.currentUnifiedStateView()
ids := resourceIDsWithTargets(t, warm)
waitForFixtureTicks(t, 2)
for _, id := range ids {
want := warm.metricsTargets.MetricsTargetForResource(id)
got := m.MetricsTargetForResource(id)
if !sameMetricsTarget(got, want) {
t.Fatalf("MetricsTargetForResource(%q) = %+v after fixture ticks, want %+v", id, got, want)
}
}
after, ok := publishedMockView(m)
if !ok || after.readState != warm.readState {
t.Fatal("target lookups rebuilt the unified view after a metric tick; they must reuse the view built for the current fixture structure")
}
}
// The reused view answers for the estate it was built from. A structural
// change (here a larger estate) must reach the lookup, not keep serving the
// old resource set.
func TestMockMetricsTargetFollowsAStructuralFixtureChange(t *testing.T) {
useMockEstate(t, 3, 5*time.Minute)
m := newMockEstateMonitor()
before := m.currentUnifiedStateView()
known := make(map[string]struct{}, len(before.resources))
for _, resource := range before.resources {
known[resource.ID] = struct{}{}
}
grown := mock.GetConfig()
grown.NodeCount++
mock.SetMockConfig(grown)
// Another monitor's view of the grown estate names the resources the
// first one lacks. The first monitor has not been asked for a view since
// the change, so the lookup itself has to notice the structure moved.
var added string
for _, resource := range newMockEstateMonitor().currentUnifiedStateView().resources {
if _, ok := known[resource.ID]; !ok && resource.Type == unifiedresources.ResourceTypeAgent {
added = resource.ID
break
}
}
if added == "" {
t.Fatal("the larger fixture estate lists no agent the first one did not")
}
if target := m.MetricsTargetForResource(added); target == nil {
t.Fatalf("MetricsTargetForResource(%q) = nil after the fixture grew; the cached view outlived its structure", added)
}
}
// Reusing a view across ticks must not change an answer: every resource
// resolves to what the view's own registry resolves, and an ID the estate does
// not list resolves to nothing without building a view for it.
func TestMockMetricsTargetMatchesTheRegistryResolution(t *testing.T) {
useMockEstate(t, 4, 5*time.Minute)
m := newMockEstateMonitor()
view := m.currentUnifiedStateView()
registryResolver, ok := view.readState.(MetricsTargetResourceStore)
if !ok {
t.Fatalf("mock view read state %T does not resolve metrics targets", view.readState)
}
for _, resource := range view.resources {
want := registryResolver.MetricsTargetForResource(resource.ID)
got := m.MetricsTargetForResource(resource.ID)
if !sameMetricsTarget(got, want) {
t.Fatalf("MetricsTargetForResource(%q) = %+v, want the registry's %+v", resource.ID, got, want)
}
if padded := m.MetricsTargetForResource(" " + resource.ID + " "); !sameMetricsTarget(padded, want) {
t.Fatalf("MetricsTargetForResource(%q padded) = %+v, want the canonical ID's %+v", resource.ID, padded, want)
}
}
if got := m.MetricsTargetForResource("not-a-mock-resource"); got != nil {
t.Fatalf("MetricsTargetForResource(unknown) = %+v, want nil", got)
}
if after, _ := publishedMockView(m); after.readState != view.readState {
t.Fatal("resolving an unknown ID rebuilt the cached view")
}
}
// A lookup for an ID the estate does not list must stay cheap after a tick
// too: the view has no more to say about it than the one it replaces.
func TestMockMetricsTargetForUnknownIDDoesNotRebuildTheViewAfterFixtureTicks(t *testing.T) {
useMockEstate(t, 3, time.Second)
m := newMockEstateMonitor()
warm := m.currentUnifiedStateView()
waitForFixtureTicks(t, 2)
for i := 0; i < 50; i++ {
if got := m.MetricsTargetForResource(fmt.Sprintf("unknown-%d", i)); got != nil {
t.Fatalf("MetricsTargetForResource(unknown) = %+v, want nil", got)
}
}
if after, _ := publishedMockView(m); after.readState != warm.readState {
t.Fatal("unknown-ID lookups rebuilt the unified view after a metric tick")
}
}
// An operator link folds one resource into another, which changes what the
// folded ID resolves to. Whatever rebuilds the view first, a lookup after the
// resource store applied the link must not resolve the folded resource.
func TestMockMetricsTargetFollowsAnOperatorLink(t *testing.T) {
useMockEstate(t, 4, 5*time.Minute)
store := unifiedresources.NewMemoryStore()
m := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
alertManager: alerts.NewManager(),
}
t.Cleanup(m.alertManager.Stop)
var vmID, agentID string
for _, resource := range m.currentUnifiedStateView().resources {
switch {
case vmID == "" && resource.Type == unifiedresources.ResourceTypeVM && resource.Agent == nil:
vmID = resource.ID
case agentID == "" && resource.Type == unifiedresources.ResourceTypeAgent && resource.Agent != nil &&
len(resource.Sources) == 1 && resource.Sources[0] == unifiedresources.SourceAgent:
agentID = resource.ID
}
}
if vmID == "" || agentID == "" {
t.Fatalf("fixture graph needs an agentless VM and a standalone agent, got vm=%q agent=%q", vmID, agentID)
}
if m.MetricsTargetForResource(agentID) == nil || m.MetricsTargetForResource(vmID) == nil {
t.Fatal("expected the unlinked VM and agent to resolve metrics targets")
}
if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}); err != nil {
t.Fatalf("AddLink: %v", err)
}
// Rebuild the resource store so it loads the link, without asking for a
// mock view: the lookup itself has to see the link list change.
m.updateResourceStore(models.StateSnapshot{}, m.mockModeFence.begin())
if got := m.MetricsTargetForResource(agentID); got != nil {
t.Fatalf("MetricsTargetForResource(folded agent) = %+v after the operator link, want nothing", got)
}
if m.MetricsTargetForResource(vmID) == nil {
t.Fatal("the linked VM lost its metrics target")
}
}
// The reused view was built under one link list; once the resource store
// applies a different one, a lookup must rebuild rather than answer from it.
// A rebuild by any other consumer would hide a missing check, so the list the
// view was built under is changed directly.
func TestMockMetricsTargetRebuildsWhenTheLinkListMoves(t *testing.T) {
useMockEstate(t, 3, 5*time.Minute)
m := newMockEstateMonitor()
warm := m.currentUnifiedStateView()
id := resourceIDsWithTargets(t, warm)[0]
m.mockUnifiedViewMu.Lock()
m.mockUnifiedViewLinks = []unifiedresources.ResourceLink{{ResourceA: "stale-a", ResourceB: "stale-b", PrimaryID: "stale-a"}}
m.mockUnifiedViewMu.Unlock()
if m.MetricsTargetForResource(id) == nil {
t.Fatalf("MetricsTargetForResource(%q) = nil", id)
}
after, _ := publishedMockView(m)
if after.readState == warm.readState {
t.Fatal("a lookup answered from a view built under a different operator link list")
}
}
// The estate view is keyed on the fixture data version, which advances on
// every metric tick, and a build costs about half a second of CPU on an idle
// core (several seconds on a starved process). Callers that read the estate's
// identity and topology only share the view built for the current fixture
// structure instead (currentStructureUnifiedStateView); startup runs every one
// of them before the listener opens, and on a starved process each used to
// build its own estate.
func TestMockStructureConsumersDoNotRebuildTheViewAfterFixtureTicks(t *testing.T) {
useMockEstate(t, 4, time.Second)
m := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(unifiedresources.NewMemoryStore())),
alertManager: alerts.NewManager(),
}
t.Cleanup(m.alertManager.Stop)
warm := m.currentUnifiedStateView()
waitForFixtureTicks(t, 2)
consumers := []struct {
name string
run func()
}{
{"structure read state", func() { m.GetUnifiedStructureReadState() }},
{"mode structure read state", func() { m.currentModeStructureReadState() }},
{"structure node listing", func() { m.structureNodes() }},
{"docker alert prune", func() { m.pruneStaleDockerAlerts() }},
{"host agent evaluation", func() { m.evaluateHostAgents(time.Now()) }},
{"chart cache prewarm", func() { m.prewarmMockDashboardChartCaches() }},
{"host agent evaluation with an offline linked host", func() {
m.state.UpsertHost(models.Host{ID: "offline-agent", Hostname: "offline-agent", LinkedNodeID: "pve1-node", Status: "online", LastSeen: time.Now().Add(-24 * time.Hour)})
m.evaluateHostAgents(time.Now())
}},
}
for _, consumer := range consumers {
consumer.run()
after, ok := publishedMockView(m)
if !ok || after.readState != warm.readState {
t.Fatalf("%s rebuilt the unified view after a metric tick; it only reads identity and topology, which the view built for the current fixture structure holds", consumer.name)
}
}
}
// The host-agent evaluation lists nodes only for a host that names a linked
// node, and only when it has to.
func TestSharedSystemAlertCorrelationForHostLazyListsNodesOnDemand(t *testing.T) {
listed := 0
nodes := func() []models.Node {
listed++
return []models.Node{{ID: "node-1", Instance: "lab", LinkedAgentID: "host-1"}}
}
if got := sharedSystemAlertCorrelationForHostLazy(models.Host{ID: "host-1"}, nodes); got != nil || listed != 0 {
t.Fatalf("an unlinked host listed nodes %d times and correlated %v", listed, got)
}
if got := sharedSystemAlertCorrelationForHostLazy(models.Host{LinkedNodeID: "node-1"}, nodes); got != nil || listed != 0 {
t.Fatalf("a host without an ID listed nodes %d times and correlated %v", listed, got)
}
got := sharedSystemAlertCorrelationForHostLazy(models.Host{ID: "host-1", LinkedNodeID: "node-1"}, nodes)
if listed != 1 {
t.Fatalf("a linked host listed nodes %d times, want once", listed)
}
want := sharedSystemAlertCorrelationForHost(models.Host{ID: "host-1", LinkedNodeID: "node-1"}, nodes())
if got == nil || want == nil || got.Key != want.Key {
t.Fatalf("a linked host correlated %+v, want %+v", got, want)
}
}
// The data-version view must keep serving the callers that read values: a
// consumer of statuses, sensors and metrics must see the tick, so the
// structure view cannot have replaced it.
func TestMockDataVersionViewStillRebuildsAfterFixtureTicks(t *testing.T) {
useMockEstate(t, 3, time.Second)
m := newMockEstateMonitor()
warm := m.currentUnifiedStateView()
waitForFixtureTicks(t, 1)
if m.GetUnifiedReadStateOrSnapshot() == warm.readState {
t.Fatal("GetUnifiedReadStateOrSnapshot served the view from before a metric tick")
}
}
// The structure view answers for the estate it was built from. A structural
// change must reach its callers, not keep serving the old resource set.
func TestMockStructureReadStateFollowsAStructuralFixtureChange(t *testing.T) {
useMockEstate(t, 3, 5*time.Minute)
m := newMockEstateMonitor()
before := m.GetUnifiedStructureReadState()
if before == nil || len(before.Nodes()) == 0 {
t.Fatal("the mock estate lists no nodes")
}
grown := mock.GetConfig()
grown.NodeCount++
mock.SetMockConfig(grown)
after := m.GetUnifiedStructureReadState()
if after == before {
t.Fatal("the structure view outlived a structural fixture change")
}
if len(after.Nodes()) <= len(before.Nodes()) {
t.Fatalf("the structure view lists %d nodes after the estate grew from %d", len(after.Nodes()), len(before.Nodes()))
}
}
// An operator link folds one resource into another; the structure view must
// stop listing the folded resource once the resource store applies the link.
func TestMockStructureReadStateFollowsAnOperatorLink(t *testing.T) {
useMockEstate(t, 4, 5*time.Minute)
store := unifiedresources.NewMemoryStore()
m := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
alertManager: alerts.NewManager(),
}
t.Cleanup(m.alertManager.Stop)
var vmID, agentID string
for _, resource := range m.currentStructureUnifiedStateView().resources {
switch {
case vmID == "" && resource.Type == unifiedresources.ResourceTypeVM && resource.Agent == nil:
vmID = resource.ID
case agentID == "" && resource.Type == unifiedresources.ResourceTypeAgent && resource.Agent != nil &&
len(resource.Sources) == 1 && resource.Sources[0] == unifiedresources.SourceAgent:
agentID = resource.ID
}
}
if vmID == "" || agentID == "" {
t.Fatalf("fixture graph needs an agentless VM and a standalone agent, got vm=%q agent=%q", vmID, agentID)
}
if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: vmID, ResourceB: agentID, PrimaryID: vmID}); err != nil {
t.Fatalf("AddLink: %v", err)
}
m.updateResourceStore(models.StateSnapshot{}, m.mockModeFence.begin())
for _, resource := range m.currentStructureUnifiedStateView().resources {
if resource.ID == agentID {
t.Fatalf("the structure view still lists the agent %q the operator folded into %q", agentID, vmID)
}
}
}
// TestMockModeDiscardsRealHostReports pins the push-side of the mock clean
// room. Mock mode already suspends pull-based PVE/PBS/PMG collection, but a
// real agent keeps POSTing regardless; ingesting those reports lands a real
// machine in state where it raises alerts and feeds the offline sweep beside
// fixture data.
func TestMockModeDiscardsRealHostReports(t *testing.T) {
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "real-agent-1", Version: "6.2.0"},
Host: agentshost.HostInfo{
ID: "real-agent-1",
Hostname: "real-machine.lan",
Platform: "linux",
OSName: "Unraid",
},
Timestamp: time.Now().UTC(),
}
newMonitor := func(t *testing.T) *Monitor {
t.Helper()
return &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),
hostContinuityStore: config.NewHostContinuityStore(t.TempDir(), nil),
rateTracker: NewRateTracker(),
}
}
t.Run("mock mode acknowledges without ingesting", func(t *testing.T) {
previous := mock.IsMockEnabled()
if err := mock.SetEnabled(true); err != nil {
t.Fatalf("enable mock mode: %v", err)
}
t.Cleanup(func() { _ = mock.SetEnabled(previous) })
m := newMonitor(t)
host, err := m.ApplyHostReport(report, nil)
if err != nil {
t.Fatalf("mock mode must acknowledge rather than reject the report: %v", err)
}
if host.ID != "real-agent-1" || host.Hostname != "real-machine.lan" {
t.Fatalf("acknowledgement must echo the reporting agent identity, got %+v", host)
}
if got := m.GetLiveHostsSnapshot(); len(got) != 0 {
t.Fatalf("real host reached monitor state in mock mode: %#v", got)
}
if got := m.recentStandaloneHostContinuityEntries(); len(got) != 0 {
t.Fatalf("real host continuity persisted in mock mode: %#v", got)
}
})
t.Run("real mode still ingests", func(t *testing.T) {
previous := mock.IsMockEnabled()
if err := mock.SetEnabled(false); err != nil {
t.Fatalf("disable mock mode: %v", err)
}
t.Cleanup(func() { _ = mock.SetEnabled(previous) })
m := newMonitor(t)
if _, err := m.ApplyHostReport(report, nil); err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if got := m.GetLiveHostsSnapshot(); len(got) != 1 {
t.Fatalf("real mode must still ingest host reports, got %#v", got)
}
})
}
// Report ingest records an agent's identity on its token only for a host the
// monitor resolved and holds under that token, judged on the report's own
// hostname. That rests on three properties of the returned host, pinned here:
// an applied report carries the token's ID, a report the ordering watermark
// rejects returns the stored host with the stored hostname rather than its
// own, and mock mode's discard acknowledgement carries no token at all.
func TestApplyHostReportStampsTheTokenOnlyOnHostsItHolds(t *testing.T) {
token := &config.APITokenRecord{ID: "token-a", Name: "agent token"}
newMonitor := func(t *testing.T) *Monitor {
t.Helper()
return &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil)),
hostContinuityStore: config.NewHostContinuityStore(t.TempDir(), nil),
rateTracker: NewRateTracker(),
}
}
report := func(hostname string, at time.Time) agentshost.Report {
return agentshost.Report{
Agent: agentshost.AgentInfo{ID: "machine-a", Version: "6.5.0", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "machine-a", MachineID: "machine-a", Hostname: hostname, Platform: "linux"},
Timestamp: at,
}
}
previous := mock.IsMockEnabled()
mustSetMockEnabled(t, false)
t.Cleanup(func() { _ = mock.SetEnabled(previous) })
m := newMonitor(t)
start := time.Now().UTC().Add(-time.Minute)
first := report("node.site-b", start)
applied, err := m.ApplyHostReport(first, token)
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if applied.ID != "machine-a" || applied.TokenID != token.ID || applied.Hostname != "node.site-b" {
t.Fatalf("applied host = %q token %q hostname %q, want machine-a under %q as node.site-b", applied.ID, applied.TokenID, applied.Hostname, token.ID)
}
if _, err := m.ApplyHostReport(report("node", start.Add(30*time.Second)), token); err != nil {
t.Fatalf("ApplyHostReport newer: %v", err)
}
replayed, err := m.ApplyHostReport(first, token)
if err != nil {
t.Fatalf("ApplyHostReport replay: %v", err)
}
if replayed.TokenID != token.ID || replayed.Hostname != "node" {
t.Fatalf("replayed report returned token %q hostname %q, want the stored host under %q as node", replayed.TokenID, replayed.Hostname, token.ID)
}
mustSetMockEnabled(t, true)
discarded, err := newMonitor(t).ApplyHostReport(report("node", time.Now().UTC()), token)
if err != nil {
t.Fatalf("mock mode ApplyHostReport: %v", err)
}
if discarded.ID != "machine-a" || discarded.TokenID != "" {
t.Fatalf("mock acknowledgement = %q token %q, want the echoed ID with no token", discarded.ID, discarded.TokenID)
}
}
// The agent-authored privilege profile must survive ingest into model state
// exactly, and a report without one must not invent a profile.
func TestApplyHostReportCarriesAgentPrivilegeProfile(t *testing.T) {
report := agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "privilege-agent",
Version: "6.3.0",
Type: "unified",
IntervalSeconds: 30,
Privilege: &agentshost.PrivilegeStatus{
RunningAsRoot: false,
ServiceUser: " pulse-agent ",
TypedHelper: true,
SmartctlHelper: true,
},
},
Host: agentshost.HostInfo{ID: "privilege-machine", Hostname: "privilege-host", Platform: "linux"},
Timestamp: time.Now().UTC(),
}
monitor := newTestMonitor(t)
host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "privilege-token"})
if err != nil {
t.Fatalf("ApplyHostReport: %v", err)
}
if host.AgentPrivilege == nil ||
host.AgentPrivilege.RunningAsRoot ||
host.AgentPrivilege.ServiceUser != "pulse-agent" ||
!host.AgentPrivilege.TypedHelper ||
!host.AgentPrivilege.SmartctlHelper ||
host.AgentPrivilege.PctHelper {
t.Fatalf("ingested privilege = %+v", host.AgentPrivilege)
}
report.Agent.Privilege = nil
report.Agent.ID = "privilege-agent-legacy"
report.Host.ID = "privilege-machine-legacy"
report.Host.Hostname = "privilege-host-legacy"
legacyHost, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "privilege-token"})
if err != nil {
t.Fatalf("ApplyHostReport legacy: %v", err)
}
if legacyHost.AgentPrivilege != nil {
t.Fatalf("legacy report invented a privilege profile: %+v", legacyHost.AgentPrivilege)
}
}
func TestMonitorConstructionWiresNotificationDeliveryReconciliation(t *testing.T) {
monitor, err := New(&config.Config{DataPath: t.TempDir()})
if err != nil {
t.Fatalf("New: %v", err)
}
t.Cleanup(func() { monitor.Stop() })
queue := monitor.GetNotificationManager().GetQueue()
if queue == nil {
t.Fatal("expected notification queue")
}
if err := queue.Enqueue(¬ifications.QueuedNotification{
ID: "delivery-reconcile",
Type: "webhook",
Status: notifications.QueueStatusPending,
MaxAttempts: 8,
Config: json.RawMessage(`{}`),
}); err != nil {
t.Fatalf("Enqueue: %v", err)
}
if err := queue.UpdateStatus("delivery-reconcile", notifications.QueueStatusDLQ, "destination unavailable"); err != nil {
t.Fatalf("mark DLQ: %v", err)
}
found := false
for _, active := range monitor.GetAlertManager().GetActiveAlerts() {
if active.Type == alerts.NotificationDeliveryAlertType {
found = true
break
}
}
if !found {
t.Fatal("expected committed DLQ transition to raise notification delivery alert")
}
if _, err := queue.DismissTerminalFailures(); err != nil {
t.Fatalf("DismissTerminalFailures: %v", err)
}
for _, active := range monitor.GetAlertManager().GetActiveAlerts() {
if active.Type == alerts.NotificationDeliveryAlertType {
t.Fatalf("notification delivery alert remained active after committed recovery: %+v", active)
}
}
}
func TestApplyHostReportPreservesPoolOnlyUnraidCount(t *testing.T) {
monitor := &Monitor{
state: models.NewState(), alertManager: alerts.NewManager(),
hostTokenBindings: make(map[string]string), config: &config.Config{},
rateTracker: NewRateTracker(),
}
t.Cleanup(func() { monitor.alertManager.Stop() })
var report agentshost.Report
// Exercise the agent wire format: explicit zero must survive omitempty.
err := json.Unmarshal([]byte(`{"agent":{"id":"pool-only","version":"test"},"host":{"id":"pool-only","hostname":"pool-only"},"unraid":{"arrayStarted":true,"numDisks":0}}`), &report)
if err != nil {
t.Fatal(err)
}
report.Timestamp = time.Now().UTC()
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatal(err)
}
if host.Unraid == nil || host.Unraid.NumDisks == nil || *host.Unraid.NumDisks != 0 {
t.Fatalf("lost explicit zero: %+v", host.Unraid)
}
record := unifiedresources.HostIngestRecord(host)
restored := hostUnraidFromReadStateView(record.Resource.Agent.Unraid)
if restored == nil || restored.NumDisks == nil || *restored.NumDisks != 0 {
t.Fatalf("canonical round trip lost explicit zero: %+v", restored)
}
if assessment := storagehealth.AssessUnraidStorage(*restored); assessment.Level != storagehealth.RiskHealthy {
t.Fatalf("pool-only host raised storage risk: %+v", assessment)
}
}
// Uniqueness among PVE nodes does not make host-local addresses machine identity:
// a non-PVE host is not included in that ownership count.
func TestFindLinkedProxmoxEntityWithHints_RejectsHostLocalNetworkIdentity(t *testing.T) {
for _, tc := range []struct{ name, nic, address string }{
{"docker", "docker0", "172.17.0.1/16"},
{"generated docker bridge", "br-0123456789ab", "192.0.2.1/24"},
{"loopback", "lo", "127.0.0.1/8"},
{"IPv6 loopback", "lo", "::1/128"},
{"link local", "eth0", "169.254.1.2/16"},
{"IPv6 link local", "eth0", "fe80::1/64"},
{"unspecified", "eth0", "0.0.0.0"},
} {
t.Run(tc.name, func(t *testing.T) {
monitor := &Monitor{state: models.NewState()}
monitor.state.UpdateNodes([]models.Node{{
ID: "pve-node", Name: "pve", Instance: "cluster",
Host: "https://pve.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{{Name: tc.nic, Addresses: []string{tc.address}}},
}})
node, vm, ct := monitor.findLinkedProxmoxEntityWithHints("nas", "", []agentshost.NetworkInterface{
{Name: tc.nic, Addresses: []string{tc.address}},
{Name: "eth1", Addresses: []string{"198.51.100.20/24"}},
})
if node != "" || vm != "" || ct != "" {
t.Fatalf("unrelated NAS linked by host-local address: node=%q vm=%q ct=%q", node, vm, ct)
}
})
}
}
func TestAgentLinkNetworkIdentityFiltering(t *testing.T) {
for _, tc := range []struct {
name, nic, address string
want bool
}{
{"management bridge", "vmbr0", "192.0.2.10/24", true},
{"private management", "eth0", "172.17.0.1/16", true},
{"IPv6 management", "vmbr0", "2001:db8::10/64", true},
{"ULA management", "eth0", "fd00::10/64", true},
{"unnamed legacy interface", "", "192.0.2.10", true},
{"custom management bridge", "br-mgmt", "192.0.2.10", true},
{"docker", "docker0", "172.17.0.1/16", false},
{"generated docker", "br-0123456789ab", "192.0.2.1/24", false},
{"non-hex bridge", "br-0123456789ag", "192.0.2.1/24", true},
{"loopback", "eth0", "127.1.2.3/8", false},
{"IPv6 unspecified", "eth0", "::", false},
{"multicast", "eth0", "224.0.0.1", false},
{"IPv6 link local", "eth0", "fe90::1/64", false},
} {
t.Run(tc.name, func(t *testing.T) {
agent := collectReportedHostIPs("", []agentshost.NetworkInterface{{Name: tc.nic, Addresses: []string{tc.address}}})
node := collectNodeNetworkIPs([]unifiedresources.NetworkInterface{{Name: tc.nic, Addresses: []string{tc.address}}})
if (len(agent) == 1) != tc.want || (len(node) == 1) != tc.want {
t.Fatalf("agent=%v node=%v want accepted=%v", agent, node, tc.want)
}
})
}
if got := collectReportedHostIPs("172.17.0.1", nil); len(got) != 1 {
t.Fatalf("explicit private report IP must remain usable: %v", got)
}
if got := collectReportedHostIPs("::1", nil); len(got) != 0 {
t.Fatalf("explicit loopback cannot identify a machine: %v", got)
}
}
// Exercise ingestion as well as the matcher: no reciprocal PVE link may be
// created from a bridge address shared with an unrelated host.
func TestApplyHostReportDoesNotLinkUnrelatedDockerBridge(t *testing.T) {
monitor := issue1654Monitor()
monitor.state.UpdateNodes([]models.Node{{
ID: "pve-node", Name: "pve", Instance: "cluster",
Host: "https://pve.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: "vmbr0", Addresses: []string{"192.0.2.10/24"}},
{Name: "docker0", Addresses: []string{"172.17.0.1/16"}},
},
}})
report := issue1654Report(time.Now().UTC())
report.Host.Hostname = "nas.example"
report.Network = []agentshost.NetworkInterface{
{Name: "eth0", Addresses: []string{"198.51.100.20/24"}},
{Name: "docker0", Addresses: []string{"172.17.0.1/16"}},
}
for i := 0; i < 2; i++ {
report.Timestamp = report.Timestamp.Add(time.Second)
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatal(err)
}
if host.LinkedNodeID != "" {
t.Fatalf("NAS linked to %q", host.LinkedNodeID)
}
nodes := monitor.state.GetSnapshot().Nodes
if len(nodes) != 1 || nodes[0].LinkedAgentID != "" {
t.Fatalf("PVE node acquired an unrelated agent link: %+v", nodes)
}
}
}
// Exercise each side independently: symmetric fixtures alone would still pass
// if filtering accidentally disappeared from either address inventory.
func TestApplyHostReportBridgeIdentity(t *testing.T) {
for _, tc := range []struct {
name, providerNIC, agentNIC, address string
wantLink bool
}{
{"custom management", "br-mgmt", "br-mgmt", "192.0.2.10/24", true},
{"private management", "vmbr0", "eth0", "172.17.0.1/16", true},
{"ULA management", "br-mgmt", "eth0", "fd00::10/64", true},
{"provider docker only", "docker0", "eth0", "172.17.0.1/16", false},
{"agent docker only", "vmbr0", "docker0", "172.17.0.1/16", false},
{"provider generated bridge only", "br-0123456789ab", "eth0", "192.0.2.10/24", false},
{"agent generated bridge only", "vmbr0", "br-0123456789ab", "192.0.2.10/24", false},
} {
t.Run(tc.name, func(t *testing.T) {
monitor := issue1654Monitor()
monitor.state.UpdateNodes([]models.Node{{
ID: "pve-node", Name: "pve", Instance: "cluster",
Host: "https://pve.example:8006",
NetworkInterfaces: []models.HostNetworkInterface{
{Name: tc.providerNIC, Addresses: []string{tc.address}},
},
}})
report := issue1654Report(time.Now().UTC())
// Different names and a DNS endpoint force network evidence.
report.Host.Hostname = "agent.example"
report.Network = []agentshost.NetworkInterface{
{Name: tc.agentNIC, Addresses: []string{tc.address}},
}
for i := 0; i < 2; i++ {
report.Timestamp = report.Timestamp.Add(time.Second)
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatal(err)
}
wantNode, wantAgent := "", ""
if tc.wantLink {
wantNode, wantAgent = "pve-node", host.ID
}
if host.LinkedNodeID != wantNode {
t.Fatalf("report %d: linked node = %q, want %q", i, host.LinkedNodeID, wantNode)
}
nodes := monitor.state.GetSnapshot().Nodes
if len(nodes) != 1 || nodes[0].LinkedAgentID != wantAgent {
t.Fatalf("report %d: reciprocal link mismatch, want agent %q", i, wantAgent)
}
}
})
}
}
// Explicit includes must survive the wire and both server report paths.
func TestAgentReportsPreserveExplicitDiskIncludes(t *testing.T) {
var disks []agentshost.Disk
if err := json.Unmarshal([]byte(`[
{"device":"log2ram","mountpoint":"/var/log","type":"tmpfs","totalBytes":1024,"usedBytes":768,"explicitlyIncluded":true},
{"device":"tmpfs","mountpoint":"/mnt/ramdisk/plex-transcode","type":"tmpfs","totalBytes":1024,"usedBytes":256,"explicitlyIncluded":true},
{"device":"tmpfs","mountpoint":"/run","type":"tmpfs","totalBytes":1024},
{"device":"/dev/sda","mountpoint":"/","type":"ext4","totalBytes":4096,"usedBytes":1024}
]`), &disks); err != nil {
t.Fatal(err)
}
check := func(t *testing.T, got []models.Disk) {
t.Helper()
if len(got) != 3 {
t.Fatalf("accepted disks = %+v, want both selected tmpfs and root only", got)
}
for i, want := range []string{"/var/log", "/mnt/ramdisk/plex-transcode", "/"} {
if got[i].Mountpoint != want || got[i].Total != []int64{1024, 1024, 4096}[i] {
t.Fatalf("disk %d = %+v", i, got[i])
}
}
}
t.Run("host", func(t *testing.T) {
m := newTestMonitor(t)
host, err := m.ApplyHostReport(agentshost.Report{
Agent: agentshost.AgentInfo{ID: "include-host", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "include-machine", Hostname: "include-host"},
Disks: disks, Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatal(err)
}
check(t, host.Disks)
})
t.Run("docker", func(t *testing.T) {
m := newTestMonitor(t)
host, err := m.ApplyDockerReport(agentsdocker.Report{
Agent: agentsdocker.AgentInfo{ID: "include-docker", IntervalSeconds: 30},
Host: agentsdocker.HostInfo{MachineID: "include-docker-machine", Hostname: "include-docker", Disks: disks},
Timestamp: time.Now().UTC(),
}, nil)
if err != nil {
t.Fatal(err)
}
check(t, host.Disks)
})
}
// Issue #1966: read-side registry rebuilds re-run the unified metric sync as
// often as every two seconds, so one poll observation was re-written many
// times between polls. The store upserts on (resource, metric, tier,
// timestamp), so each replay committed a fresh transaction for no new data and
// churned the SQLite WAL. The replay guard drops exact timestamp+value repeats
// before the batch is enqueued while still writing a corrected value.
func TestDedupeUnifiedMetricWritesDropsExactReplays(t *testing.T) {
monitor := &Monitor{}
observedAt := time.Date(2026, time.September, 19, 12, 0, 0, 0, time.UTC)
write := metrics.WriteMetric{
ResourceType: "storage",
ResourceID: "pool:tank",
MetricType: "usage",
Value: 62,
Timestamp: observedAt,
Tier: metrics.TierRaw,
}
first := monitor.dedupeUnifiedMetricWrites([]metrics.WriteMetric{write})
if len(first) != 1 {
t.Fatalf("first write count = %d, want 1", len(first))
}
// The same observation replayed by a read-side registry rebuild must not
// reach the store again: that UPDATE is the write amplification in #1966.
replay := monitor.dedupeUnifiedMetricWrites([]metrics.WriteMetric{write})
if len(replay) != 0 {
t.Fatalf("replayed write count = %d, want 0", len(replay))
}
// A corrected value at the same observation time is still written.
corrected := write
corrected.Value = 63
if got := monitor.dedupeUnifiedMetricWrites([]metrics.WriteMetric{corrected}); len(got) != 1 {
t.Fatalf("corrected write count = %d, want 1", len(got))
}
// A newer observation for the same series is written.
advanced := write
advanced.Value = 63
advanced.Timestamp = observedAt.Add(time.Minute)
if got := monitor.dedupeUnifiedMetricWrites([]metrics.WriteMetric{advanced}); len(got) != 1 {
t.Fatalf("advanced write count = %d, want 1", len(got))
}
}
// Issue #1966: a host agent and an API-backed TrueNAS connection can report
// the same hostname without resolving to the same canonical resource. This
// fixture checks their disk-write source IDs, metrics targets and writer
// selection, including that TrueNAS pool capacity is a different series. Its
// deliberately identical synthetic pair tests the replay-key boundary, not
// whether the reporter's live samples had equal timestamps or values.
func TestUnifiedDiskWriteMetricsKeepSameHostnameHostAgentAndTrueNASTargetsSeparate(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
const (
hostname = "truenas-iscsi.example.test"
hostAgentID = "host-agent-iscsi"
connectionID = "truenas-connection-2"
hostWriteRate = 128_000
trueNASRate = 3_400_000
)
observedAt := time.Date(2026, time.September, 23, 17, 0, 0, 0, time.UTC)
fixtures := truenas.DefaultFixtures()
fixtures.System.Hostname = hostname
fixtures.System.DiskWriteRate = trueNASRate
fixtures.System.IntervalSeconds = 30
fixtures.System.CollectedAt = observedAt
fixtures.CollectedAt = observedAt
fixtures.Pools = fixtures.Pools[:1]
fixtures.Datasets = nil
fixtures.Disks = nil
fixtures.Apps = nil
fixtures.VMs = nil
fixtures.Shares = nil
provider := truenas.NewLiveProviderForConnection(
&truenas.FixtureFetcher{Snapshot: fixtures}, connectionID,
)
trueNASRecords := provider.RecordsFromSnapshot(&fixtures)
if len(trueNASRecords) == 0 {
t.Fatal("connection-scoped TrueNAS provider returned no records")
}
host := models.Host{
ID: hostAgentID,
Hostname: hostname,
MachineID: "host-agent-machine-iscsi",
DiskWriteRate: hostWriteRate,
Status: "online",
LastSeen: observedAt,
IntervalSeconds: 30,
}
resourceStore := unifiedresources.NewMonitorAdapter(nil)
resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{Hosts: []models.Host{host}}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
unifiedresources.SourceTrueNAS: trueNASRecords,
})
var hostResource, trueNASResource *unifiedresources.Resource
for _, resource := range resourceStore.GetAll() {
if resource.Type != unifiedresources.ResourceTypeAgent || resource.Name != hostname {
continue
}
resourceCopy := resource
switch {
case monitorHasSource(resource.Sources, unifiedresources.SourceAgent):
hostResource = &resourceCopy
case monitorHasSource(resource.Sources, unifiedresources.SourceTrueNAS):
trueNASResource = &resourceCopy
}
}
if hostResource == nil || trueNASResource == nil {
t.Fatalf("same-hostname resources did not remain separately source-owned: host=%+v TrueNAS=%+v", hostResource, trueNASResource)
}
if hostResource.ID == trueNASResource.ID {
t.Fatalf("same-hostname host agent and connection-scoped TrueNAS resolved to one canonical ID %q", hostResource.ID)
}
hostTarget := resourceStore.MetricsTargetForResource(hostResource.ID)
if hostTarget == nil || hostTarget.ResourceType != "agent" || hostTarget.ResourceID != hostAgentID {
t.Fatalf("host-agent metrics target = %+v, want agent/%s", hostTarget, hostAgentID)
}
trueNASTarget := resourceStore.MetricsTargetForResource(trueNASResource.ID)
if trueNASTarget == nil || trueNASTarget.ResourceType != "agent" || trueNASTarget.ResourceID != connectionID {
t.Fatalf("TrueNAS metrics target = %+v, want agent/%s", trueNASTarget, connectionID)
}
if hostResource.Metrics == nil || hostResource.Metrics.DiskWrite == nil ||
hostResource.Metrics.DiskWrite.Source != unifiedresources.SourceAgent ||
hostResource.Metrics.DiskWrite.Value != hostWriteRate {
t.Fatalf("host-agent disk-write observation = %+v, want agent rate %v", hostResource.Metrics, hostWriteRate)
}
if trueNASResource.Metrics == nil || trueNASResource.Metrics.DiskWrite == nil ||
trueNASResource.Metrics.DiskWrite.Source != unifiedresources.SourceTrueNAS ||
trueNASResource.Metrics.DiskWrite.Value != trueNASRate {
t.Fatalf("TrueNAS disk-write observation = %+v, want API rate %v", trueNASResource.Metrics, trueNASRate)
}
var poolTarget *unifiedresources.MetricsTarget
for _, resource := range resourceStore.GetAll() {
if resource.Type == unifiedresources.ResourceTypeStorage && resource.Name == "tank" &&
monitorHasSource(resource.Sources, unifiedresources.SourceTrueNAS) {
poolTarget = resourceStore.MetricsTargetForResource(resource.ID)
break
}
}
if poolTarget == nil || poolTarget.ResourceType != "storage" ||
poolTarget.ResourceID != "system:"+connectionID+"/pool:tank" {
t.Fatalf("TrueNAS pool metrics target = %+v, want connection-scoped storage pool", poolTarget)
}
cfg := metrics.DefaultConfig(t.TempDir())
persistentStore, err := metrics.NewStore(cfg)
if err != nil {
t.Fatalf("metrics.NewStore() error = %v", err)
}
defer func() { _ = persistentStore.Close() }()
monitor := &Monitor{metricsStore: persistentStore}
var writes []metrics.WriteMetric
monitor.syncUnifiedAgentMetrics(resourceStore, &writes)
var trueNASWrite *metrics.WriteMetric
for i := range writes {
write := &writes[i]
if write.MetricType != "diskwrite" {
continue
}
if write.ResourceID == hostTarget.ResourceID {
t.Fatalf("unified sync selected the host-agent target %q; that source uses the direct agent writer", hostTarget.ResourceID)
}
if write.ResourceID == trueNASTarget.ResourceID {
trueNASWrite = write
}
}
if trueNASWrite == nil {
t.Fatalf("unified sync did not select the TrueNAS agent target %q for diskwrite; writes=%+v", trueNASTarget.ResourceID, writes)
}
if trueNASWrite.Value != trueNASRate || !trueNASWrite.Timestamp.Equal(observedAt) {
t.Fatalf("TrueNAS disk-write sample = %+v, want value %v at %s", trueNASWrite, trueNASRate, observedAt)
}
var storageWrites []metrics.WriteMetric
monitor.syncUnifiedStorageMetrics(resourceStore, &storageWrites)
if len(storageWrites) != 4 {
t.Fatalf("TrueNAS pool capacity writes = %+v, want usage/used/total/avail only", storageWrites)
}
poolSeries := make(map[string]bool)
for _, write := range storageWrites {
if write.ResourceType != "storage" || write.ResourceID != poolTarget.ResourceID ||
write.MetricType == "diskwrite" {
t.Fatalf("TrueNAS pool capacity reached an agent disk-write series: %+v", write)
}
poolSeries[write.MetricType] = true
}
for _, metric := range []string{"usage", "used", "total", "avail"} {
if !poolSeries[metric] {
t.Fatalf("TrueNAS pool missing capacity series %q; writes=%+v", metric, storageWrites)
}
}
// The fixture intentionally supplies one identical (type, metric, time,
// value) observation under each proven-distinct source target. The replay
// guard preserves both first writes because its store key includes ID, then
// drops each exact same-target replay. Production-source equality for this
// reporter remains unverified until paired values/timestamps are provided.
hostWrite := *trueNASWrite
hostWrite.ResourceID = hostTarget.ResourceID
pairedMonitor := &Monitor{}
paired := pairedMonitor.dedupeUnifiedMetricWrites([]metrics.WriteMetric{*trueNASWrite, hostWrite})
if len(paired) != 2 {
t.Fatalf("distinct source targets retained %d of the deliberately identical paired samples; want 2", len(paired))
}
if replay := pairedMonitor.dedupeUnifiedMetricWrites([]metrics.WriteMetric{*trueNASWrite, hostWrite}); len(replay) != 0 {
t.Fatalf("exact same-target paired replays retained %d writes, want 0: %+v", len(replay), replay)
}
}
func TestSyncUnifiedStorageMetricsDefersWritesToBatchSink(t *testing.T) {
observedAt := time.Now().UTC().Add(-time.Minute).Truncate(time.Second)
resourceStore := unifiedresources.NewMonitorAdapter(nil)
resourceStore.PopulateFromSnapshot(models.StateSnapshot{
PBSInstances: []models.PBSInstance{{
ID: "pbs-main",
Name: "pbs-main",
Status: "online",
LastSeen: observedAt,
Datastores: []models.PBSDatastore{{
Name: "backups",
Status: "available",
Total: 1000,
Used: 400,
Free: 600,
Usage: 40,
}},
}},
})
var targetID string
for _, resource := range resourceStore.GetAll() {
if resource.Type != unifiedresources.ResourceTypeStorage || resource.Storage == nil || resource.Storage.Platform != "pbs" {
continue
}
target := resourceStore.MetricsTargetForResource(resource.ID)
if target == nil || target.ResourceType != "storage" {
t.Fatalf("unexpected PBS datastore metrics target: %+v", target)
}
targetID = target.ResourceID
break
}
if targetID == "" {
t.Fatal("expected a PBS datastore in the unified resource store")
}
cfg := metrics.DefaultConfig(t.TempDir())
persistentStore, err := metrics.NewStore(cfg)
if err != nil {
t.Fatalf("metrics.NewStore() error = %v", err)
}
defer func() { _ = persistentStore.Close() }()
monitor := &Monitor{
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
metricsStore: persistentStore,
}
// A sink collects the batch so the unified syncs can share one transaction.
// The sink path must not write through to the store itself.
var sink []metrics.WriteMetric
monitor.syncUnifiedStorageMetrics(resourceStore, &sink)
if len(sink) == 0 {
t.Fatal("expected storage writes to be collected in the batch sink")
}
before, err := persistentStore.Query("storage", targetID, "usage", observedAt.Add(-time.Second), observedAt.Add(time.Second), 0)
if err != nil {
t.Fatalf("query persisted PBS usage: %v", err)
}
if len(before) != 0 {
t.Fatalf("sink path must not write to the store directly, got %+v", before)
}
persistentStore.WriteBatchBounded(sink)
after, err := persistentStore.Query("storage", targetID, "usage", observedAt.Add(-time.Second), observedAt.Add(time.Second), 0)
if err != nil {
t.Fatalf("query persisted PBS usage: %v", err)
}
if len(after) != 1 {
t.Fatalf("persisted PBS usage points = %d, want one after the batch flush: %+v", len(after), after)
}
if !after[0].Timestamp.Equal(observedAt) {
t.Fatalf("persisted PBS usage timestamp = %s, want source observation %s", after[0].Timestamp, observedAt)
}
}
func TestSyncUnifiedAgentMetricsUsesSourceObservationTimeAcrossRegistryRebuilds(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
t.Cleanup(func() {
truenas.SetFeatureEnabled(previous)
})
fixtures := truenas.DefaultFixtures()
fixtures.CollectedAt = time.Now().UTC().Truncate(time.Second)
fixtures.System.CollectedAt = fixtures.CollectedAt
observedAt := fixtures.CollectedAt
resourceStore := unifiedresources.NewMonitorAdapter(nil)
resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
unifiedresources.SourceTrueNAS: truenas.NewProvider(fixtures).Records(),
})
var targetID string
for _, resource := range resourceStore.GetAll() {
if resource.Type != unifiedresources.ResourceTypeAgent || resource.Name != "truenas-main" {
continue
}
target := resourceStore.MetricsTargetForResource(resource.ID)
if target == nil || target.ResourceType != "agent" {
t.Fatalf("unexpected agent metrics target: %+v", target)
}
targetID = target.ResourceID
break
}
if targetID == "" {
t.Fatal("expected a TrueNAS agent resource in the unified resource store")
}
cfg := metrics.DefaultConfig(t.TempDir())
persistentStore, err := metrics.NewStore(cfg)
if err != nil {
t.Fatalf("metrics.NewStore() error = %v", err)
}
defer func() { _ = persistentStore.Close() }()
monitor := &Monitor{
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
metricsStore: persistentStore,
}
// Read-side registry rebuilds must not invent a fresh sample for the same
// source observation (the write amplification in #1966).
monitor.syncUnifiedAgentMetrics(resourceStore)
monitor.syncUnifiedAgentMetrics(resourceStore)
inMemory := monitor.GetGuestMetrics("agent:"+targetID, time.Hour)["cpu"]
if len(inMemory) != 1 {
t.Fatalf("in-memory agent cpu points = %d, want one per source observation: %+v", len(inMemory), inMemory)
}
if !inMemory[0].Timestamp.Equal(observedAt) {
t.Fatalf("in-memory agent cpu timestamp = %s, want source observation %s", inMemory[0].Timestamp, observedAt)
}
persisted, err := persistentStore.Query("agent", targetID, "cpu", observedAt.Add(-time.Second), observedAt.Add(time.Second), 0)
if err != nil {
t.Fatalf("query persisted agent cpu: %v", err)
}
if len(persisted) != 1 {
t.Fatalf("persisted agent cpu points = %d, want one per source observation: %+v", len(persisted), persisted)
}
if !persisted[0].Timestamp.Equal(observedAt) {
t.Fatalf("persisted agent cpu timestamp = %s, want source observation %s", persisted[0].Timestamp, observedAt)
}
}
func TestSyncUnifiedPhysicalDiskMetricsUsesSourceObservationTimeAcrossRegistryRebuilds(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
t.Cleanup(func() {
truenas.SetFeatureEnabled(previous)
})
fixtures := truenas.DefaultFixtures()
fixtures.CollectedAt = time.Now().UTC().Truncate(time.Second)
fixtures.System.CollectedAt = fixtures.CollectedAt
observedAt := fixtures.CollectedAt
resourceStore := unifiedresources.NewMonitorAdapter(nil)
resourceStore.PopulateSnapshotAndSupplemental(models.StateSnapshot{}, map[unifiedresources.DataSource][]unifiedresources.IngestRecord{
unifiedresources.SourceTrueNAS: truenas.NewProvider(fixtures).Records(),
})
var diskResourceID string
for _, resource := range resourceStore.GetAll() {
if resource.Type != unifiedresources.ResourceTypePhysicalDisk || resource.PhysicalDisk == nil {
continue
}
if strings.TrimSpace(resource.PhysicalDisk.DevPath) == "/dev/sdc" {
diskResourceID = resource.ID
break
}
}
if diskResourceID == "" {
t.Fatal("expected TrueNAS sdc disk resource in unified store")
}
target := resourceStore.MetricsTargetForResource(diskResourceID)
if target == nil || target.ResourceType != "disk" || target.ResourceID != "WD-WX12A3456" {
t.Fatalf("unexpected physical-disk metrics target %+v", target)
}
cfg := metrics.DefaultConfig(t.TempDir())
persistentStore, err := metrics.NewStore(cfg)
if err != nil {
t.Fatalf("metrics.NewStore() error = %v", err)
}
defer func() { _ = persistentStore.Close() }()
monitor := &Monitor{
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
metricsStore: persistentStore,
}
// Read-side registry rebuilds must not invent a fresh sample for the same
// source observation (the write amplification in #1966).
monitor.syncUnifiedPhysicalDiskMetrics(resourceStore)
monitor.syncUnifiedPhysicalDiskMetrics(resourceStore)
inMemory := monitor.GetDiskMetrics(target.ResourceID, "smart_temp", time.Hour)
if len(inMemory) != 1 {
t.Fatalf("in-memory disk smart_temp points = %d, want one per source observation: %+v", len(inMemory), inMemory)
}
if !inMemory[0].Timestamp.Equal(observedAt) {
t.Fatalf("in-memory disk smart_temp timestamp = %s, want source observation %s", inMemory[0].Timestamp, observedAt)
}
persisted, err := persistentStore.Query("disk", target.ResourceID, "smart_temp", observedAt.Add(-time.Second), observedAt.Add(time.Second), 0)
if err != nil {
t.Fatalf("query persisted disk smart_temp: %v", err)
}
if len(persisted) != 1 {
t.Fatalf("persisted disk smart_temp points = %d, want one per source observation: %+v", len(persisted), persisted)
}
if !persisted[0].Timestamp.Equal(observedAt) {
t.Fatalf("persisted disk smart_temp timestamp = %s, want source observation %s", persisted[0].Timestamp, observedAt)
}
}
// Issue #2113: a removed host agent re-enrolls under a derived suffixed
// identity because identity resolution forks the machine before the removal
// block is consulted. The block is keyed on the base machine identity, so the
// forked ID bypasses it and the machine is silently admitted instead of being
// rejected or cleared. This reproduces the fork trigger (a live base-identity
// record still present in the unified read model with an older token) and
// asserts a fresh install token heals to the base identity.
func TestApplyHostReportHonoursRemovalBlockWhenIdentityWouldFork(t *testing.T) {
now := time.Now().UTC()
const (
baseID = "bf3f9ff3273346d9944c49160dc57288"
hostname = "truenas-iscsi.local"
machineID = "bf3f9ff3273346d9944c49160dc57288"
)
live := models.Host{
ID: baseID,
MachineID: machineID,
Hostname: hostname,
Platform: "linux",
Status: "online",
LastSeen: now,
IntervalSeconds: 30,
TokenID: "token-old",
}
state := models.NewState()
state.Hosts = []models.Host{live}
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(state.GetSnapshot())
adapter := unifiedresources.NewMonitorAdapter(registry)
monitor := &Monitor{
state: state,
resourceStore: adapter,
alertManager: alerts.NewManager(),
hostTokenBindings: map[string]string{"token-old": baseID},
removedHostAgents: make(map[string]time.Time),
rateTracker: NewRateTracker(),
config: &config.Config{},
}
t.Cleanup(func() { monitor.alertManager.Stop() })
// The operator removed the host while the live record was still present in
// the read model: the tombstone is keyed on the base identity.
removedAt := now.Add(-2 * time.Hour)
monitor.state.AddRemovedHostAgent(models.RemovedHostAgent{
ID: baseID,
Hostname: hostname,
MachineID: machineID,
TokenID: "token-old",
RemovedAt: removedAt,
})
monitor.removedHostAgents[baseID] = removedAt
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: baseID, Type: "unified", IntervalSeconds: 30},
Host: agentshost.HostInfo{
ID: baseID,
Hostname: hostname,
MachineID: machineID,
Platform: "linux",
},
Timestamp: now,
}
// The fresh token post-dates the removal, but the colliding live record is
// still reporting, so the stale-identity reuse path declines and identity
// resolution forks.
freshToken := &config.APITokenRecord{
ID: "token-new",
Name: "reinstall",
CreatedAt: now.Add(-time.Hour),
}
host, err := monitor.ApplyHostReport(report, freshToken)
if err != nil {
t.Fatalf("fresh re-enroll should clear the block, got %v", err)
}
if host.ID != baseID {
t.Fatalf("expected re-enroll to heal to base identity %q, got derived %q", baseID, host.ID)
}
if len(monitor.state.GetRemovedHostAgents()) != 0 {
t.Fatalf("expected removal block to be cleared, still have %+v", monitor.state.GetRemovedHostAgents())
}
}
// countingResourceStore counts registry clones handed out by the live store.
type countingResourceStore struct {
*unifiedresources.MonitorAdapter
getAll atomic.Int32
}
func (c *countingResourceStore) GetAll() []unifiedresources.Resource {
c.getAll.Add(1)
return c.MonitorAdapter.GetAll()
}
// plainResourceStore lacks metrics-target resolution.
type plainResourceStore struct {
ResourceStoreInterface
}
func TestResourceSnapshotStoreClonesOncePerPass(t *testing.T) {
now := time.Now().UTC()
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
adapter.PopulateFromSnapshot(models.StateSnapshot{
LastUpdate: now,
VMs: []models.VM{{
ID: "lab:pve1:101", VMID: 101, Name: "db", Node: "pve1", Instance: "lab",
Status: "running", Type: "qemu", LastSeen: now,
}},
})
counting := &countingResourceStore{MonitorAdapter: adapter}
store := newResourceSnapshotStore(counting)
first := store.GetAll()
second := store.GetAll()
if got := counting.getAll.Load(); got != 1 {
t.Fatalf("underlying GetAll calls = %d, want 1", got)
}
if len(first) != 1 || len(second) != 1 || &first[0] != &second[0] {
t.Fatalf("pass consumers did not share one snapshot: %d and %d resources", len(first), len(second))
}
resolver, ok := store.(MetricsTargetResourceStore)
if !ok || resolver.MetricsTargetForResource(first[0].ID) == nil {
t.Fatal("snapshot store does not forward metrics-target resolution")
}
if again := newResourceSnapshotStore(store); again != store {
t.Fatal("wrapping a pass snapshot again must reuse it")
}
plain := plainResourceStore{ResourceStoreInterface: adapter}
if wrapped := newResourceSnapshotStore(plain); wrapped != ResourceStoreInterface(plain) {
t.Fatal("a store without metrics-target resolution must pass through unwrapped")
}
}
// An accepted agent report refreshes the store once. Every metric sync and the
// alert sync in that pass must share a single registry clone (#2199).
func TestAgentReportRefreshClonesRegistryOnce(t *testing.T) {
now := time.Now().UTC()
state := models.NewState()
state.UpdateNodes([]models.Node{{ID: "lab-pve1", Name: "pve1", Instance: "lab", Status: "online", Type: "node", LastSeen: now}})
state.UpdateVMs([]models.VM{{
ID: "lab:pve1:101", VMID: 101, Name: "db", Node: "pve1", Instance: "lab",
Status: "running", Type: "qemu", CPU: 0.2, LastSeen: now,
}})
state.UpsertHost(models.Host{ID: "agent-1", Hostname: "pve1.example", Status: "online", CPUUsage: 3, LastSeen: now})
counting := &countingResourceStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}
m := &Monitor{
state: state,
resourceStore: counting,
metricsHistory: NewMetricsHistory(32, time.Hour),
}
m.refreshUnifiedResourceStoreAfterAgentStateChange()
if got := counting.getAll.Load(); got != 1 {
t.Fatalf("registry clones for one agent report refresh = %d, want 1", got)
}
}
// newAgentReportThrottleMonitor returns a monitor whose report-driven store
// refreshes are throttled to window, with one online host agent.
func newAgentReportThrottleMonitor(t *testing.T, window time.Duration) (*Monitor, *countingResourceStore) {
t.Helper()
state := models.NewState()
state.UpsertHost(models.Host{ID: "agent-1", Hostname: "pve1.example", Status: "online", CPUUsage: 1, LastSeen: time.Now().UTC()})
counting := &countingResourceStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))}
m := &Monitor{
state: state,
resourceStore: counting,
metricsHistory: NewMetricsHistory(32, time.Hour),
agentReportRefreshWindow: window,
}
t.Cleanup(m.stopAgentReportRefresh)
return m, counting
}
func reportAgentCPU(m *Monitor, cpu float64) {
m.state.UpsertHost(models.Host{ID: "agent-1", Hostname: "pve1.example", Status: "online", CPUUsage: cpu, LastSeen: time.Now().UTC()})
m.refreshUnifiedResourceStoreAfterAgentReport()
}
func publishedAgentCPU(t *testing.T, store *countingResourceStore) float64 {
t.Helper()
for _, resource := range store.MonitorAdapter.GetAll() {
if resource.Type == unifiedresources.ResourceTypeAgent && resource.Metrics != nil && resource.Metrics.CPU != nil {
return resource.Metrics.CPU.Value
}
}
t.Fatal("no agent CPU in the canonical store")
return 0
}
// Accepted agent reports refresh the estate-wide store at most once per
// window: the first report after a quiet window is published before the call
// returns, and reports inside the window fold into one trailing refresh that
// publishes the latest state (#2199).
func TestAgentReportRefreshFoldsReportsWithinWindow(t *testing.T) {
const window = 500 * time.Millisecond
m, store := newAgentReportThrottleMonitor(t, window)
reportAgentCPU(m, 10)
if got := store.getAll.Load(); got != 1 {
t.Fatalf("refreshes after the leading report = %d, want 1", got)
}
if got := publishedAgentCPU(t, store); got != 10 {
t.Fatalf("leading report published CPU %v, want 10", got)
}
for cpu := 20.0; cpu <= 60; cpu += 10 {
reportAgentCPU(m, cpu)
}
if got := store.getAll.Load(); got != 1 {
t.Fatalf("refreshes during the window = %d, want the burst deferred", got)
}
deadline := time.Now().Add(5 * time.Second)
for store.getAll.Load() < 2 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if got := store.getAll.Load(); got != 2 {
t.Fatalf("refreshes after the window = %d, want one trailing refresh", got)
}
if got := publishedAgentCPU(t, store); got != 60 {
t.Fatalf("trailing refresh published CPU %v, want the latest report's 60", got)
}
time.Sleep(window + 100*time.Millisecond)
reportAgentCPU(m, 70)
if got := store.getAll.Load(); got != 3 {
t.Fatalf("refreshes after a quiet window = %d, want a synchronous leading refresh", got)
}
if got := publishedAgentCPU(t, store); got != 70 {
t.Fatalf("leading report after a quiet window published CPU %v, want 70", got)
}
}
func TestStopCancelsPendingAgentReportRefresh(t *testing.T) {
const window = 100 * time.Millisecond
m, store := newAgentReportThrottleMonitor(t, window)
reportAgentCPU(m, 10)
reportAgentCPU(m, 20)
m.stopAgentReportRefresh()
time.Sleep(3 * window)
if got := store.getAll.Load(); got != 1 {
t.Fatalf("refreshes after stop = %d, want the pending trailing refresh cancelled", got)
}
}
// A standalone agent offline across a restart is added back to every canonical
// read-state lookup from host continuity. Alert evaluation makes one lookup per
// resource per poll, so the overlay must be reused, not rebuilt per lookup.
func TestStandaloneHostContinuityReadStateReusedAcrossLookups(t *testing.T) {
now := time.Now().UTC()
state := models.NewState()
state.UpsertHost(models.Host{ID: "agent-live", Hostname: "live.example", Status: "online", LastSeen: now})
continuity := config.NewHostContinuityStore(t.TempDir(), nil)
if err := continuity.Upsert(config.HostContinuityEntry{HostID: "agent-gone", Hostname: "gone.example", LastSeen: now.Add(-time.Hour)}); err != nil {
t.Fatalf("seed host continuity: %v", err)
}
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m := &Monitor{state: state, resourceStore: adapter, hostContinuityStore: continuity}
m.refreshUnifiedResourceStoreAfterAgentStateChange()
first := m.GetUnifiedReadStateOrSnapshot()
if first == unifiedresources.ReadState(adapter) {
t.Fatal("read state has no continuity overlay for the offline standalone agent")
}
found := false
for _, host := range first.Hosts() {
found = found || host.Hostname() == "gone.example"
}
if !found {
t.Fatal("continuity overlay does not include the offline standalone agent")
}
if second := m.GetUnifiedReadStateOrSnapshot(); second != first {
t.Fatal("a second read-state lookup rebuilt the continuity overlay")
}
}
// GetLiveHostsSnapshot runs on every agent report, config fetch, and host
// continuity lookup, so it must copy the hosts alone, not the whole state.
func TestGetLiveHostsSnapshotCopiesOnlyHosts(t *testing.T) {
state := models.NewState()
state.UpsertHost(models.Host{ID: "agent-1", Hostname: "pve1.example", Status: "online"})
vms := make([]models.VM, 1000)
for i := range vms {
vms[i] = models.VM{
ID: fmt.Sprintf("lab:pve1:%d", 100+i), VMID: 100 + i, Name: fmt.Sprintf("vm-%d", i),
Node: "pve1", Instance: "lab", Status: "running", Type: "qemu",
Disks: []models.Disk{{Mountpoint: "/", Total: 10 << 30, Used: 1 << 30}},
}
}
state.UpdateVMs(vms)
m := &Monitor{state: state}
if hosts := m.GetLiveHostsSnapshot(); len(hosts) != 1 || hosts[0].ID != "agent-1" {
t.Fatalf("live hosts = %+v, want the one registered agent", hosts)
}
if allocs := testing.AllocsPerRun(10, func() { _ = m.GetLiveHostsSnapshot() }); allocs > 50 {
t.Fatalf("GetLiveHostsSnapshot allocated %.0f times with 1 host and 1000 guests; it is copying guests", allocs)
}
}
// Reads model the completed-ingest boundary without persistence/background work.
// The real adapter still owns clone isolation, mutable facets and identity.
type broadcastProjectionCountingStore struct {
*unifiedresources.MonitorAdapter
reads int
}
func (s *broadcastProjectionCountingStore) GetAll() []unifiedresources.Resource {
s.reads++
return s.MonitorAdapter.GetAll()
}
func (s *broadcastProjectionCountingStore) GetAllWithMetricsTargets() ([]unifiedresources.Resource, map[string]unifiedresources.MetricsTarget) {
s.reads++
return s.MonitorAdapter.GetAllWithMetricsTargets()
}
func (s *broadcastProjectionCountingStore) GetAllWithMetricsTargetsAndStaleThresholds() ([]unifiedresources.Resource, map[string]unifiedresources.MetricsTarget, map[unifiedresources.DataSource]time.Duration) {
s.reads++
return s.MonitorAdapter.GetAllWithMetricsTargetsAndStaleThresholds()
}
func (*broadcastProjectionCountingStore) TryReplaceRegistryForRead(models.StateSnapshot, time.Duration, func() map[unifiedresources.DataSource][]unifiedresources.IngestRecord) bool {
return false
}
func TestBroadcastProjectionListsRegistryOnceAndKeepsLiveChanges(t *testing.T) {
m, adapter, _ := newReadStateCloneTestMonitor(t, 4)
store := &broadcastProjectionCountingStore{MonitorAdapter: adapter}
m.resourceStore = store
first := m.BuildFrontendState()
if store.reads != 1 {
t.Fatalf("GetAll calls=%d, want one owned continuity-aware clone", store.reads)
}
if len(first.Resources) != 4 {
t.Fatalf("resources=%d", len(first.Resources))
}
oldCPU := first.Resources[0].CPU.Current
changed := store.GetAll()
// Keep LastSeen/overall freshness unchanged: a timestamp-only cache would
// miss these metric/status/tag changes.
changed[0].Metrics.CPU.Value = 77
changed[0].Tags = []string{"new-tag"}
changed[0].Status = unifiedresources.StatusOffline
registry := unifiedresources.NewRegistry(nil)
registry.IngestResources(changed)
store.MonitorAdapter = unifiedresources.NewMonitorAdapter(registry)
m.state.UpdateActiveAlerts([]models.Alert{{ID: "live-alert", ResourceID: changed[0].ID, Level: "critical", Type: "cpu"}})
m.state.RemovedHostAgents = []models.RemovedHostAgent{{ID: "ignored", Hostname: "ignored.local", RemovedAt: time.Now()}}
store.reads = 0
second := m.BuildFrontendState()
if store.reads != 1 {
t.Fatalf("next GetAll calls=%d", store.reads)
}
var row *models.ResourceFrontend
for i := range second.Resources {
if second.Resources[i].ID == changed[0].ID {
row = &second.Resources[i]
}
}
if row == nil || row.CPU.Current != 77 || len(row.Tags) != 1 || row.Tags[0] != "new-tag" {
t.Fatalf("mutable row was stale: %#v", row)
}
var health unifiedresources.ResourceHealth
if err := json.Unmarshal(row.Health, &health); err != nil || health.Verdict != unifiedresources.HealthCritical {
t.Fatalf("live critical alert did not reach the freshly projected row: %s (%v)", row.Health, err)
}
if first.Resources[0].CPU.Current != oldCPU {
t.Fatal("later projection mutated an accepted baseline")
}
if len(second.ActiveAlerts) != 1 || second.ActiveAlerts[0].ID != "live-alert" {
t.Fatal("live alert disappeared")
}
foundIgnored := false
for _, item := range second.ConnectedInfrastructure {
if item.Status == "ignored" && item.Name == "ignored.local" {
foundIgnored = true
}
}
if !foundIgnored {
t.Fatal("removed host lifecycle surface disappeared")
}
}
func TestAgentLXCPartialInventoryPreservesSuccessAndInvalidatesOmissions(t *testing.T) {
state := models.NewState()
state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a", LinkedAgentID: "agent-a"}})
m := &Monitor{state: state}
now := time.Date(2026, 10, 5, 16, 0, 0, 0, time.UTC)
row := func(vmid int) agentshost.ProxmoxLXCContainer {
return agentshost.ProxmoxLXCContainer{VMID: vmid, Name: "web", Disks: []agentshost.Disk{{Type: "rootfs", Device: "local:disk", Mountpoint: "/", TotalBytes: 4096, UsedBytes: 1024, FreeBytes: 3072}}}
}
m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete", Containers: []agentshost.ProxmoxLXCContainer{row(100), row(102)}}, now, 30)
foreignKey := agentLXCFilesystemCacheKey("pve-b", "node-a", 102)
m.proxmoxLXCFilesystemsCache[foreignKey] = agentLXCFilesystemCacheEntry{agentID: "agent-b", name: "foreign", expiresAt: now.Add(time.Hour)}
m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "partial", Containers: []agentshost.ProxmoxLXCContainer{row(100)}, OmittedVMIDs: []int{102}}, now.Add(time.Minute), 30)
if _, ok := m.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("pve-a", "node-a", 102)]; ok {
t.Fatal("omitted guest kept a usable prior agent reading")
}
if _, ok := m.proxmoxLXCFilesystemsCache[foreignKey]; !ok {
t.Fatal("foreign instance cache was invalidated")
}
guest := models.Container{VMID: 100, Name: "web", Status: "running"}
m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, now.Add(time.Minute))
if len(guest.Disks) != 1 || guest.Disks[0].Used != 1024 {
t.Fatalf("successful partial guest=%+v", guest)
}
m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete"}, now.Add(2*time.Minute), 30)
if _, ok := m.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("pve-a", "node-a", 100)]; ok {
t.Fatal("complete empty collection retained a removed/stopped guest reading")
}
if _, ok := m.proxmoxLXCFilesystemsCache[foreignKey]; !ok {
t.Fatal("complete empty collection cleared a foreign instance")
}
}
func TestAgentLXCMalformedPartialInventoryCannotRenewOrInvalidate(t *testing.T) {
state := models.NewState()
state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a", LinkedAgentID: "agent-a"}})
m := &Monitor{state: state}
now := time.Now()
key := agentLXCFilesystemCacheKey("pve-a", "node-a", 100)
original := agentLXCFilesystemCacheEntry{agentID: "agent-a", name: "web", expiresAt: now.Add(time.Minute)}
m.proxmoxLXCFilesystemsCache = map[string]agentLXCFilesystemCacheEntry{key: original}
m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete", OmittedVMIDs: []int{100}}, now, 30)
got := m.proxmoxLXCFilesystemsCache[key]
if !got.expiresAt.Equal(original.expiresAt) || got.name != "web" {
t.Fatal("malformed completeness changed cache")
}
}
func TestAgentLXCPartialWireReportReachesLinkedGuestWithoutStaleOmission(t *testing.T) {
m := newTestMonitor(t)
m.state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a"}})
row := func(vmid int) agentshost.ProxmoxLXCContainer {
return agentshost.ProxmoxLXCContainer{VMID: vmid, Name: "web", Disks: []agentshost.Disk{{
Type: "rootfs", Device: "local:disk", Mountpoint: "/", TotalBytes: 4096, UsedBytes: 1024, FreeBytes: 3072,
}}}
}
report := agentshost.Report{
Agent: agentshost.AgentInfo{ID: "agent-a", IntervalSeconds: 30},
Host: agentshost.HostInfo{ID: "agent-a", MachineID: "agent-a", Hostname: "node-a", Platform: "linux"},
ProxmoxLXC: &agentshost.ProxmoxLXCInventory{
Status: agentshost.ProxmoxLXCCollectionComplete, Containers: []agentshost.ProxmoxLXCContainer{row(100), row(102)},
},
}
applyWire := func(sequence uint64) models.Host {
t.Helper()
report.Timestamp = time.Now().UTC()
report.SequenceID = agentshost.FormatReportSequenceID("partial-wire-test", sequence)
encoded, err := json.Marshal(report)
if err != nil {
t.Fatal(err)
}
var decoded agentshost.Report
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatal(err)
}
host, err := m.ApplyHostReport(decoded, &config.APITokenRecord{ID: "partial-wire-token"})
if err != nil || host.LinkedNodeID != "node-id" {
t.Fatalf("wire report did not reach linked node: host=%+v err=%v", host, err)
}
return host
}
applyWire(1)
report.ProxmoxLXC.Status = agentshost.ProxmoxLXCCollectionPartial
report.ProxmoxLXC.Containers = []agentshost.ProxmoxLXCContainer{row(100)}
report.ProxmoxLXC.OmittedVMIDs = []int{102}
report.Agent.Modules = []agentshost.ModuleStatus{{Name: agentshost.ModuleNameTypedPrivilegeHelper, Enabled: true, State: "degraded", LastError: "proxmox.lxc_filesystems: helper Proxmox LXC filesystem inventory is incomplete"}}
host := applyWire(2)
if len(host.AgentModules) != 1 || host.AgentModules[0].State != "degraded" {
t.Fatalf("wire report lost partial helper health: %+v", host.AgentModules)
}
for _, vmid := range []int{100, 102} {
guest := models.Container{VMID: vmid, Name: "web", Status: "running"}
m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, time.Now())
if vmid == 100 && (len(guest.Disks) != 1 || guest.Disks[0].Used != 1024) {
t.Fatalf("successful wire reading was lost: %+v", guest.Disks)
}
if vmid == 102 && len(guest.Disks) != 0 {
t.Fatalf("omitted wire guest kept a prior reading: %+v", guest.Disks)
}
}
report.ProxmoxLXC = &agentshost.ProxmoxLXCInventory{Status: agentshost.ProxmoxLXCCollectionComplete}
report.Agent.Modules[0].State, report.Agent.Modules[0].LastError = "running", ""
host = applyWire(3)
if len(host.AgentModules) != 1 || host.AgentModules[0].State != "running" || host.AgentModules[0].LastError != "" {
t.Fatalf("wire recovery did not clear partial helper health: %+v", host.AgentModules)
}
guest := models.Container{VMID: 100, Name: "web", Status: "running"}
m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, time.Now())
if len(guest.Disks) != 0 {
t.Fatal("complete empty wire report retained a stopped guest reading")
}
}
// presentationFixtureStore keeps the fixture registry: the broadcast's read
// refresh would otherwise rebuild it from the empty snapshot.
type presentationFixtureStore struct {
*unifiedresources.MonitorAdapter
}
func (presentationFixtureStore) TryReplaceRegistryForRead(models.StateSnapshot, time.Duration, func() map[unifiedresources.DataSource][]unifiedresources.IngestRecord) bool {
return false
}
// A host pair the operator split (unlink or report-merge) must be two
// broadcast rows exactly when the resources API lists two. The broadcast used
// to coalesce without the store's exclusions, so the websocket folded the
// pair the API kept apart.
func TestBroadcastPresentationCoalesceHonoursMergeExclusions(t *testing.T) {
now := time.Date(2026, 10, 7, 21, 0, 0, 0, time.UTC)
resources := []unifiedresources.Resource{
{
ID: "agent-runtime-alpha", Type: unifiedresources.ResourceTypeAgent, Name: "alpha",
Status: unifiedresources.StatusOnline, LastSeen: now,
Sources: []unifiedresources.DataSource{unifiedresources.SourceAgent},
Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"alpha"}},
Agent: &unifiedresources.AgentData{AgentID: "agent-alpha", Hostname: "alpha"},
},
{
ID: "agent-docker-alpha", Type: unifiedresources.ResourceTypeAgent, Name: "alpha",
Status: unifiedresources.StatusOnline, LastSeen: now,
Sources: []unifiedresources.DataSource{unifiedresources.SourceDocker},
Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"alpha"}},
Docker: &unifiedresources.DockerData{Hostname: "alpha"},
},
}
for _, split := range []bool{false, true} {
store := unifiedresources.NewMemoryStore()
if split {
if err := store.AddExclusion(unifiedresources.ResourceExclusion{ResourceA: "agent-runtime-alpha", ResourceB: "agent-docker-alpha"}); err != nil {
t.Fatal(err)
}
}
registry := unifiedresources.NewRegistry(store)
registry.IngestResources(resources)
adapter := unifiedresources.NewMonitorAdapter(registry)
m := &Monitor{resourceStore: presentationFixtureStore{adapter}}
state := m.buildBroadcastFrontendStateFromSnapshotWithClock(models.EmptyStateSnapshot(), m.mockModeFence.begin(), func() time.Time { return now })
broadcastRows := 0
for _, resource := range state.Resources {
if resource.Name == "alpha" {
broadcastRows++
}
}
// The resources API seeds a store-backed registry from the monitor's
// listing and presents it through ListForPresentation.
rest := unifiedresources.NewRegistry(store)
rest.IngestResources(adapter.GetAll())
restRows := len(rest.ListForPresentation())
want := 1
if split {
want = 2
}
if broadcastRows != want || restRows != want {
t.Fatalf("split=%v: broadcast rows=%d REST rows=%d, want %d on both", split, broadcastRows, restRows, want)
}
}
}
// The rows a broadcast coalesces were listed by its view's read state, such as
// a host-continuity overlay that loaded the store's exclusions afresh, so its
// exclusions decide even when the resource store's generation predates them.
func TestBroadcastPresentationCoalescePrefersTheListingReadState(t *testing.T) {
now := time.Date(2026, 10, 7, 21, 0, 0, 0, time.UTC)
resources := []unifiedresources.Resource{
{ID: "agent-runtime-alpha", Type: unifiedresources.ResourceTypeAgent, Name: "alpha", Status: unifiedresources.StatusOnline, LastSeen: now,
Sources: []unifiedresources.DataSource{unifiedresources.SourceAgent}, Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"alpha"}},
Agent: &unifiedresources.AgentData{AgentID: "agent-alpha", Hostname: "alpha"}},
{ID: "agent-docker-alpha", Type: unifiedresources.ResourceTypeAgent, Name: "alpha", Status: unifiedresources.StatusOnline, LastSeen: now,
Sources: []unifiedresources.DataSource{unifiedresources.SourceDocker}, Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"alpha"}},
Docker: &unifiedresources.DockerData{Hostname: "alpha"}},
}
store := unifiedresources.NewMemoryStore()
base := unifiedresources.NewRegistry(store)
base.IngestResources(resources)
m := &Monitor{resourceStore: unifiedresources.NewMonitorAdapter(base)}
if err := store.AddExclusion(unifiedresources.ResourceExclusion{ResourceA: "agent-runtime-alpha", ResourceB: "agent-docker-alpha"}); err != nil {
t.Fatal(err)
}
overlay := unifiedresources.NewRegistry(store)
overlay.IngestResources(resources)
listing := unifiedresources.NewMonitorAdapter(overlay)
if got := len(m.coalesceResourcesForPresentation(listing, listing.GetAll(), nil)); got != 2 {
t.Fatalf("coalesced %d rows with the listing's exclusion, want 2", got)
}
if got := len(m.coalesceResourcesForPresentation(nil, listing.GetAll(), nil)); got != 1 {
t.Fatalf("without a listing read state the store generation decides: %d rows, want 1", got)
}
// The mock view lists through a store-less registry, which carries no
// operator decisions, so the resource store's adapter supplies them.
mockView := monitorUnifiedStateViewFromResources(resources, now)
withSplit := &Monitor{resourceStore: listing}
if got := len(withSplit.coalesceResourcesForPresentation(mockView.readState, mockView.resources, mockView.staleThresholds)); got != 2 {
t.Fatalf("mock-style view coalesced %d rows, want the store's split to keep 2", got)
}
}
// A host agent linked to a Proxmox node merges into one read-state row that
// every PVE poll keeps online and fresh. Once the agent stops reporting, its
// retained filesystem summary must stop standing in for the node's disk (and
// through it disk alerts and history), and the projected agent must read as
// offline with its own last report, so its connection shows the missed
// heartbeat instead of the PVE poll time.
func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) {
state := models.NewState()
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter}
node := models.Node{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
ConnectionHealth: "healthy", LinkedAgentID: "agent-1",
}
rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}}
// poll records a PVE poll that just saw the node, with the agent as of its
// last report, and resolves the node's disk the way pollPVENode does.
poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) {
t.Helper()
polledNode := node
polledNode.LastSeen = time.Now()
state.UpdateNodes([]models.Node{polledNode})
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID,
IntervalSeconds: 30, LastSeen: agentLastReport,
Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}},
})
m.evaluateHostAgents(time.Now())
m.refreshUnifiedResourceStoreAfterAgentStateChange()
disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS)
for _, host := range m.HostsSnapshot() {
if host.ID == "agent-1" {
return disk, source, host
}
}
t.Fatal("HostsSnapshot does not list agent-1")
return disk, source, models.Host{}
}
agentReport := time.Now().Add(-10 * time.Second)
disk, source, host := poll(agentReport)
if source != "agent" || disk.Used != 900 {
t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source)
}
if host.Status != "online" || !host.LastSeen.Equal(agentReport) {
t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport)
}
silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute)
disk, source, host = poll(silentSince)
if source != "node-status-rootfs" || disk.Used != 50 {
t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source)
}
if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil {
t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked)
}
if host.Status != "offline" || !host.LastSeen.Equal(silentSince) {
t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince)
}
disk, source, host = poll(time.Now())
if source != "agent" || disk.Used != 900 || host.Status != "online" {
t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status)
}
}
// A disk swapped into a slot must not take its identity from the linked host
// agent's row for the slot's previous occupant. A silent agent keeps its last
// rows, and that row used to match the replacement by device path alone: it
// filled a serial Proxmox reported as "unknown" (recorded as unreported, so
// the agent can fill it) or, for SAS, promoted its serial over the SAS address
// Proxmox reports, and the registry joined a SAS replacement to the agent's
// disk on the same path. Either way the replacement took the old disk's
// serial, canonical resource and history, temperature and failed health.
func TestHostAgentSMARTRowForSwappedOutDiskDoesNotLendItsIdentity(t *testing.T) {
const (
oldSerial = "ZA1OLD0001"
oldAgentWWN = "5-c50-aaaa0001" // smartctl's NAA field spelling
oldPVEWWN = "0x5000c500aaaa0001"
newPVEWWN = "0x5000c500bbbb0002"
)
for _, transport := range []struct {
agentType string
// pveSerial is what Proxmox reports for a disk with the given WWN.
pveSerial func(wwn string) string
// replacementSerial is the serial the replacement's record keeps.
replacementSerial string
serialState diskinventory.FieldState
}{
{
agentType: "sata",
pveSerial: func(string) string { return "unknown" },
replacementSerial: "",
serialState: diskinventory.FieldMissing,
},
{
agentType: "sas",
pveSerial: func(wwn string) string { return strings.TrimPrefix(wwn, "0x") },
replacementSerial: strings.TrimPrefix(newPVEWWN, "0x"),
serialState: diskinventory.FieldAvailable,
},
} {
t.Run(transport.agentType, func(t *testing.T) {
t.Setenv("PULSE_DATA_DIR", t.TempDir())
now := time.Now()
state := models.NewState()
state.UpdateNodesForInstance("pve1", []models.Node{{
ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
LastSeen: now, LinkedAgentID: "agent-1",
}})
state.UpsertHost(models.Host{
ID: "agent-1", Hostname: "node1", LinkedNodeID: "pve1-node1",
Status: "online", IntervalSeconds: 30, LastSeen: now,
Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
Device: "/dev/sdb", Serial: oldSerial, WWN: oldAgentWWN, Type: transport.agentType,
Health: "FAILED", Temperature: 52,
Collection: &diskinventory.CollectionStatus{
Serial: diskinventory.Available("smartctl"),
Temperature: diskinventory.Available("smartctl"),
},
}}},
})
alertManager := alerts.NewManager()
t.Cleanup(alertManager.Stop)
adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
m := &Monitor{
state: state, resourceStore: adapter, alertManager: alertManager,
metricsHistory: NewMetricsHistory(100, time.Hour),
startTime: now.Add(-time.Hour), lastPhysicalDiskPoll: make(map[string]time.Time),
}
client := &slotDiskPVEClient{}
pveDisk := func(wwn string) proxmox.Disk {
return proxmox.Disk{
DevPath: "/dev/sdb", Model: "ST4000NM000A", Serial: transport.pveSerial(wwn), WWN: wwn,
Type: "hdd", Health: "PASSED", Wearout: 100, Size: 4000787030016,
}
}
poll := func() (models.PhysicalDisk, *unifiedresources.PhysicalDiskView) {
t.Helper()
adapter.PopulateFromSnapshot(state.GetSnapshot())
started := time.Now()
delete(m.lastPhysicalDiskPoll, "pve1")
m.maybePollPhysicalDisksAsync(context.Background(), "pve1", &config.PVEInstance{}, client,
[]proxmox.Node{{Node: "node1", Status: "online"}}, map[string]string{"node1": "online"}, nil)
deadline := time.Now().Add(3 * time.Second)
for {
if disks := state.GetSnapshot().PhysicalDisks; len(disks) == 1 && !disks[0].LastChecked.Before(started) {
adapter.PopulateFromSnapshot(state.GetSnapshot())
var proxmoxViews []*unifiedresources.PhysicalDiskView
for _, view := range adapter.PhysicalDisks() {
if _, ok := view.SourceStatus(unifiedresources.SourceProxmox); ok {
proxmoxViews = append(proxmoxViews, view)
}
}
if len(proxmoxViews) != 1 {
t.Fatalf("canonical Proxmox physical disks = %d, want one", len(proxmoxViews))
}
return disks[0], proxmoxViews[0]
}
if time.Now().After(deadline) {
t.Fatal("physical disk poll did not land in state")
}
time.Sleep(10 * time.Millisecond)
}
}
client.setDisk(pveDisk(oldPVEWWN))
original, originalView := poll()
if original.Serial != oldSerial || originalView.Serial() != oldSerial ||
original.Collection == nil || original.Collection.Serial.State != diskinventory.FieldAvailable {
t.Fatalf("agent serial not merged into the disk it describes: state %q, canonical %q, collection %+v",
original.Serial, originalView.Serial(), original.Collection)
}
// The agent stops reporting, keeping its rows, and the disk is replaced.
state.TouchHost("agent-1", now.Add(-hostAgentHealthWindow(30)-time.Minute))
m.evaluateHostAgents(now)
client.setDisk(pveDisk(newPVEWWN))
for round := 1; round <= 2; round++ {
got, view := poll()
if got.Serial != transport.replacementSerial || view.Serial() == oldSerial {
t.Fatalf("poll %d: replacement took the retained row's serial: state %q, canonical %q",
round, got.Serial, view.Serial())
}
if view.ID() == originalView.ID() {
t.Fatalf("poll %d: replacement took over the old disk's canonical resource %q", round, view.ID())
}
if got.Temperature != 0 || got.Health != "PASSED" || view.Temperature() != 0 || view.Health() != "PASSED" {
t.Fatalf("poll %d: replacement took the old disk's readings: state %d/%q, canonical %d/%q",
round, got.Temperature, got.Health, view.Temperature(), view.Health())
}
if got.WWN != newPVEWWN ||
got.Collection == nil || got.Collection.Serial.State != transport.serialState {
t.Fatalf("poll %d: replacement wwn %q, serial collection %+v", round, got.WWN, got.Collection)
}
}
})
}
}
// A Pulse agent linked into a VM fills the guest disk metric when Proxmox has
// no guest filesystems. The next poll must not carry that agent reading
// forward as Proxmox's own last known guest read: with Proxmox outranking the
// agent on guests, the carried copy would freeze over the agent's live disk.
func TestPreviousVMFromViewKeepsLinkedAgentDiskOutOfProxmoxCarry(t *testing.T) {
now := time.Now()
view := func(source unifiedresources.DataSource) *unifiedresources.VMView {
used, total := int64(400), int64(1000)
v := unifiedresources.NewVMView(&unifiedresources.Resource{
ID: "vm-pve-node1-101",
Type: unifiedresources.ResourceTypeVM,
Name: "app-101",
LastSeen: now,
Proxmox: &unifiedresources.ProxmoxData{
Instance: "pve",
NodeName: "node1",
VMID: 101,
RuntimeStatus: "running",
DiskStatusReason: "agent-not-running",
DiskObservation: models.GuestDiskObservation{Source: "guest-agent", ObservedAt: now.Add(-time.Minute)},
},
// The linked agent's addresses count as recent guest evidence.
Identity: unifiedresources.ResourceIdentity{IPAddresses: []string{"10.0.0.5"}},
Metrics: &unifiedresources.ResourceMetrics{
Disk: &unifiedresources.MetricValue{Used: &used, Total: &total, Percent: 40, Source: source},
},
})
return &v
}
carry := func(source unifiedresources.DataSource) (float64, string) {
prev := previousVMFromView(view(source))
_, _, _, usage, _, reason := stabilizeGuestLowTrustDisk(
&prev, "running", 32<<30, 0, 32<<30, -1, nil, "agent-not-running", false, now,
)
return usage, reason
}
if usage, reason := carry(unifiedresources.SourceAgent); usage != -1 || reason != "agent-not-running" {
t.Fatalf("linked agent disk carried as Proxmox's: usage=%v reason=%q", usage, reason)
}
if usage, reason := carry(unifiedresources.SourceProxmox); usage != 40 || reason != "prev-agent-not-running" {
t.Fatalf("Proxmox's own guest disk not carried: usage=%v reason=%q", usage, reason)
}
}
// Endpoint hints are deliberately absent: an operator-saved FQDN endpoint
// does not establish the agent's guest hostname. Test that attribution keeps
// full provider/agent names distinct while preserving short-name compatibility.
func TestFindLinkedProxmoxEntityPreservesDistinctGuestFQDNs(t *testing.T) {
for _, kind := range []string{"node", "vm", "container"} {
t.Run(kind, func(t *testing.T) {
m := &Monitor{state: models.NewState()}
setNames := func(names ...string) {
nodes, vms, containers := []models.Node{}, []models.VM{}, []models.Container{}
for i, name := range names {
id, instance := "home", "Home"
if i > 0 {
id, instance = "remote", "Remote"
}
switch kind {
case "node":
nodes = append(nodes, models.Node{ID: id, Name: name, Instance: instance})
case "vm":
vms = append(vms, models.VM{ID: id, Name: name, Instance: instance, VMID: 100})
case "container":
containers = append(containers, models.Container{ID: id, Name: name, Instance: instance, VMID: 100})
}
}
m.state.UpdateNodes(nodes)
m.state.UpdateVMs(vms)
m.state.UpdateContainers(containers)
}
check := func(hostname, want string) {
t.Helper()
node, vm, container := m.findLinkedProxmoxEntity(hostname)
got := map[string]string{"node": node, "vm": vm, "container": container}
for k, id := range got {
expected := ""
if k == kind {
expected = want
}
if id != expected {
t.Errorf("hostname %q: %s=%q, want %q", hostname, k, id, expected)
}
}
}
setNames("docker.home.example")
check("docker.home.example", "home")
check(" DOCKER.HOME.EXAMPLE. ", "home")
check("docker.remote.example", "") // A sole unrelated FQDN is not identity.
check("docker", "home")
setNames("docker.home.example", "docker.remote.example")
check("docker.home.example", "home")
check("docker.remote.example", "remote")
check("docker", "") // Bare name still cannot choose between estates.
check("docker.third.example", "")
setNames("docker")
check("docker.home.example", "home") // Legacy short provider name remains usable.
})
}
}
// The poller records the hostname a PBS node reports about itself so the
// connections ledger can group the host agent running on that machine under
// the PBS connection, even when the connection was configured by IP or DNS
// alias. The ledger reads the name back through PBSInstancesSnapshot, which
// rebuilds each instance from the unified read state; a conversion that
// drops the name leaves the agent as a separate row with no error anywhere.
func TestMonitorPBSInstancesSnapshotCarriesReportedNodeName(t *testing.T) {
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
PBSInstances: []models.PBSInstance{{
ID: "pbs-1",
Name: "backup",
Host: "https://192.0.2.40:8007",
NodeName: "pbs01",
Status: "online",
LastSeen: time.Date(2026, 10, 10, 9, 0, 0, 0, time.UTC),
}},
})
// The unified resource holds the name, so a missing value below is lost
// in the conversion, not at ingest.
resources := registry.ListByType(unifiedresources.ResourceTypePBS)
if len(resources) != 1 || resources[0].PBS == nil || resources[0].PBS.NodeName != "pbs01" {
t.Fatalf("expected the PBS resource to keep the reported node name, got %+v", resources)
}
m := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(registry),
}
instances := m.PBSInstancesSnapshot()
if len(instances) != 1 {
t.Fatalf("expected one PBS instance, got %d", len(instances))
}
if instances[0].NodeName != "pbs01" {
t.Fatalf("PBSInstancesSnapshot()[0].NodeName = %q, want the reported node name %q", instances[0].NodeName, "pbs01")
}
}
func TestGuestMetadataIdleCleanupContract(t *testing.T) {
now := time.Now()
stale := now.Add(-3 * guestMetadataCacheTTL)
suppressed := guestMetadataCacheEntry{osInfoFailureCount: guestAgentOSInfoFailureThreshold, osInfoSkip: true}
retained := suppressed
retained.osName, retained.windowsGuest = "Microsoft Windows", true
retained.fetchedAt, retained.osInfoObservedAt = stale, stale
m := &Monitor{
guestMetadataCache: map[string]guestMetadataCacheEntry{
"active-empty": suppressed,
"active-retained": retained,
"inactive": suppressed,
"legacy-fresh": metadataObservationFixture(now),
"legacy-stale": metadataObservationFixture(stale),
"inflight": suppressed,
},
guestMetadataLastAccess: map[string]time.Time{"active-empty": now, "active-retained": now, "inactive": stale, "only-active": now, "only-idle": stale, "inflight": stale},
guestMetadataLimiter: map[string]time.Time{"active-empty": now.Add(time.Minute), "inactive": stale, "inflight": stale},
guestMetadataInFlight: map[string]*guestMetadataFetch{"inflight": {}},
}
m.cleanupGuestMetadataCache(now)
for _, key := range []string{"active-empty", "active-retained", "legacy-fresh", "inflight"} {
if _, exists := m.guestMetadataCache[key]; !exists {
t.Errorf("live cache %s pruned", key)
}
}
for _, key := range []string{"inactive", "legacy-stale"} {
if _, exists := m.guestMetadataCache[key]; exists {
t.Errorf("idle cache %s retained", key)
}
}
for _, key := range []string{"inactive", "only-idle"} {
if _, exists := m.guestMetadataLastAccess[key]; exists {
t.Errorf("idle bookkeeping %s retained", key)
}
}
if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || m.guestMetadataInFlight["inflight"] == nil {
t.Error("janitor changed current activity, backoff or in-flight ownership")
}
empty := m.guestMetadataCache["active-empty"]
old := m.guestMetadataCache["active-retained"]
if !reflect.DeepEqual(empty, suppressed) || !reflect.DeepEqual(old, retained) {
t.Error("retention changed safety or observation state")
}
// Evidence consumers use actual observation age, never the activity clock.
key := guestMetadataCacheKey("activity", "node", 105)
m.guestMetadataCache[key] = retained
m.guestMetadataLastAccess[key] = now
m.cleanupGuestMetadataCache(now)
if m.hasRecentGuestMetadataEvidence("activity", "node", 105, now) || m.guestWindowsMeminfoUnsupported("activity", "node", 105, now) || !m.guestMetadataEvidenceTime("activity", "node", 105).Equal(stale) {
t.Error("activity invented a useful/Windows OS observation")
}
delete(m.guestMetadataInFlight, "inflight")
m.cleanupGuestMetadataCache(now)
if _, exists := m.guestMetadataCache["inflight"]; exists {
t.Error("completed idle owner retained cache")
}
if _, exists := m.guestMetadataLastAccess["inflight"]; exists {
t.Error("completed idle owner retained bookkeeping")
}
if _, exists := m.guestMetadataLimiter["inflight"]; exists {
t.Error("completed idle owner retained obsolete deadline")
}
}
func TestGuestMetadataActivityInvalidationContract(t *testing.T) {
key := guestMetadataCacheKey("activity", "node", 105)
sibling := guestMetadataCacheKey("activity-elsewhere", "node", 105)
m := &Monitor{
guestMetadataCache: map[string]guestMetadataCacheEntry{key: {osInfoSkip: true}, sibling: {osInfoSkip: true}},
guestMetadataLimiter: map[string]time.Time{key: time.Now().Add(time.Hour), sibling: time.Now().Add(time.Hour)},
guestMetadataLastAccess: map[string]time.Time{key: time.Now(), sibling: time.Now()},
guestMetadataInFlight: map[string]*guestMetadataFetch{key: {}},
}
m.clearGuestMetadataCache("activity", "node", 105)
if _, exists := m.guestMetadataCache[key]; exists {
t.Error("explicit invalidation retained metadata")
}
if _, exists := m.guestMetadataLimiter[key]; exists {
t.Error("explicit invalidation retained deadline")
}
if _, exists := m.guestMetadataLastAccess[key]; exists {
t.Error("explicit invalidation retained activity")
}
if m.guestMetadataInFlight[key] == nil || !m.guestMetadataInFlight[key].invalidated || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {
t.Error("invalidation discarded ownership or another instance")
}
m.guestMetadataLastAccess[key] = time.Now()
m.guestMetadataLastAccess[guestMetadataCacheKey("activity", "node", 106)] = time.Now()
m.retirePVEInstanceRuntime("activity")
for remaining := range m.guestMetadataLastAccess {
if strings.HasPrefix(remaining, "activity|") {
t.Error("retired instance retained activity bookkeeping")
}
}
if m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataInFlight[key] == nil || !m.guestMetadataInFlight[key].invalidated {
t.Error("retirement crossed instance scope or discarded active ownership")
}
}
func TestGuestMetadataActivityConcurrentCleanupContract(t *testing.T) {
const instance = "activity"
key := guestMetadataCacheKey(instance, "node", 105)
old := metadataObservationFixture(time.Now().Add(-3 * guestMetadataCacheTTL))
old.osInfoFailureCount, old.osInfoSkip = guestAgentOSInfoFailureThreshold, true
m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: old}}
m.recordGuestMetadataAccess(key, time.Now())
// An active backup blocks commands, not cache use. Touches and the janitor
// must coordinate without extending the old useful/OS observation timestamps.
status := &proxmox.VMStatus{Lock: "backup"}
done := make(chan struct{})
go func() {
defer close(done)
for i := 0; i < 200; i++ {
_, _, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), nil, instance, "node", "guest", 105, status, false)
if name != old.osName || version != old.osVersion || agent != old.agentVersion || deferred {
t.Error("locked activity changed retained identity or invented a command deferral")
}
}
}()
for i := 0; i < 200; i++ {
m.cleanupGuestMetadataCache(time.Now())
}
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("cache activity and cleanup did not finish")
}
m.guestMetadataMu.RLock()
retained := m.guestMetadataCache[key]
m.guestMetadataMu.RUnlock()
if retained.osName != old.osName || retained.osInfoSkip != old.osInfoSkip || retained.osInfoFailureCount != old.osInfoFailureCount || !retained.fetchedAt.Equal(old.fetchedAt) || !retained.osInfoObservedAt.Equal(old.osInfoObservedAt) || m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {
t.Error("concurrent cleanup lost suppression or renewed stale useful evidence")
}
}
func TestRetiredQGAMemoryCannotAlterIndependentAgentEvidence(t *testing.T) {
const instance = "site"
entry := agentMemCacheEntry{info: memoryLifecycleSample(4096), fetchedAt: time.Now()}
host := models.Host{ID: "independent-host", LastSeen: time.Now(), Memory: models.Memory{Total: 8192, Used: 1024, Free: 7168, Usage: 12.5}}
m := &Monitor{state: models.NewState(), vmAgentMemCache: map[string]agentMemCacheEntry{
guestMemoryCacheKey(instance, "node", 105): entry,
guestMemoryCacheKey(instance, "other", 106): entry,
guestMemoryCacheKey("site/other", "node", 105): entry,
guestMemoryCacheKey("site-other", "node", 105): entry,
}}
m.state.UpsertHost(host)
before := m.state.GetSnapshot().Hosts // Compare stored, cloned telemetry, not its pre-normalisation input.
m.retirePVEInstanceRuntime(instance)
if len(m.vmAgentMemCache) != 2 || m.vmAgentMemCache[guestMemoryCacheKey("site/other", "node", 105)] != entry || m.vmAgentMemCache[guestMemoryCacheKey("site-other", "node", 105)] != entry {
t.Error("connection retirement retained its own QGA memory or erased a different connection")
}
if got := m.state.GetSnapshot().Hosts; !reflect.DeepEqual(got, before) {
t.Error("QGA memory retirement changed independent Pulse-agent reports")
}
id := makeGuestID(instance, "node", 105)
total, used, source, deferred := m.resolveGuestStatusMemory(context.Background(), nil, instance, "guest", "node", 105, id, &proxmox.VMStatus{MaxMem: 8192, Mem: 8192}, map[string]models.Host{id: host}, 8192, "", nil)
if total != 8192 || used != 1024 || source != "agent" || deferred {
t.Errorf("independent linked-agent fallback changed after QGA retirement: %d/%d/%s/%t", total, used, source, deferred)
}
}
// A tenant monitor's Start runs on its own goroutine while request handlers,
// such as auto-register, already read the discovery service. Under -race an
// unlocked publish fails this test whether or not the two calls overlap.
func TestMonitorStartupDiscoveryPublishIsSafeForConcurrentReaders(t *testing.T) {
for _, enabled := range []bool{false, true} {
t.Run(fmt.Sprintf("discoveryEnabled=%v", enabled), func(t *testing.T) {
monitor := &Monitor{config: &config.Config{DiscoveryEnabled: enabled, DiscoverySubnet: "127.0.0.1/32"}}
t.Cleanup(monitor.StopDiscoveryService)
ctx, cancel := context.WithCancel(context.Background())
cancel()
started := make(chan struct{})
go func() {
defer close(started)
monitor.startModeRuntime(ctx, nil, false)
}()
monitor.GetDiscoveryService()
<-started
if got := monitor.GetDiscoveryService() != nil; got != enabled {
t.Fatalf("discovery service published = %v, want %v", got, enabled)
}
})
}
}
func TestMonitorDiscoveryEventsStayWithOwningOrganization(t *testing.T) {
for _, start := range []string{"startup", "explicit"} {
for _, orgID := range []string{"org-a", "default", ""} {
t.Run(start+"/org="+orgID, func(t *testing.T) {
hub := websocket.NewHub(nil)
go hub.Run()
t.Cleanup(hub.Stop)
server := httptest.NewServer(http.HandlerFunc(hub.HandleWebSocket))
t.Cleanup(server.Close)
connect := func(org string) *gorillaws.Conn {
t.Helper()
conn, _, err := gorillaws.DefaultDialer.Dial(
"ws"+strings.TrimPrefix(server.URL, "http")+"?org_id="+org,
http.Header{"Origin": {server.URL}})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.Close() })
deadline := time.Now().Add(time.Second)
for hub.GetTenantClientCount(org) == 0 {
if time.Now().After(deadline) {
t.Fatalf("client for %q did not register", org)
}
time.Sleep(time.Millisecond)
}
return conn
}
ownerOrg := orgID
if ownerOrg == "" {
ownerOrg = "default"
}
owner, other := connect(ownerOrg), connect("org-b")
const subnet = "127.0.0.1/32"
monitor := &Monitor{config: &config.Config{DiscoveryEnabled: true, DiscoverySubnet: subnet}}
monitor.SetOrgID(orgID)
t.Cleanup(monitor.StopDiscoveryService)
// A canceled scan still emits its start/completion envelopes but
// cannot probe hosts. Exercise the real monitor/service/hub wiring.
ctx, cancel := context.WithCancel(context.Background())
cancel()
if start == "startup" {
monitor.startModeRuntime(ctx, hub, false)
} else {
monitor.StartDiscoveryService(ctx, hub, subnet)
}
if monitor.GetDiscoveryService() == nil {
t.Fatal("discovery service was not initialized")
}
if err := owner.SetReadDeadline(time.Now().Add(3 * time.Second)); err != nil {
t.Fatal(err)
}
readDiscoveryEvent := func() (string, map[string]interface{}) {
t.Helper()
var message websocket.Message
if err := owner.ReadJSON(&message); err != nil {
t.Fatalf("read discovery event for %q: %v", ownerOrg, err)
}
envelope, ok := message.Data.(map[string]interface{})
if message.Type != "custom" || !ok {
t.Fatalf("discovery envelope = %#v", message)
}
kind, ok := envelope["type"].(string)
if !ok {
t.Fatalf("discovery event type = %#v", envelope["type"])
}
data, ok := envelope["data"].(map[string]interface{})
if !ok {
t.Fatalf("discovery event payload = %#v", envelope["data"])
}
return kind, data
}
kind, data := readDiscoveryEvent()
if kind != "discovery_started" || data["subnet"] != subnet || data["scanning"] != true {
t.Fatalf("discovery start = %q %#v", kind, data)
}
for {
kind, data = readDiscoveryEvent()
if kind == "discovery_complete" {
if data["scanning"] != false {
t.Fatalf("discovery completion = %#v", data)
}
break
}
}
if err := other.SetReadDeadline(time.Now().Add(100 * time.Millisecond)); err != nil {
t.Fatal(err)
}
if _, data, err := other.ReadMessage(); err == nil {
t.Fatalf("other organization received discovery event: %s", data)
} else if netErr, ok := err.(net.Error); !ok || !netErr.Timeout() {
t.Fatalf("expected open idle connection, got %v", err)
}
})
}
}
}
func TestHostDiskUsageRecordingSkipsAbsenceAndPreservesZero(t *testing.T) {
cfg := metrics.DefaultConfig(t.TempDir())
cfg.WriteBufferSize = 1
store, err := metrics.NewStore(cfg)
if err != nil {
t.Fatal(err)
}
closed := false
t.Cleanup(func() {
if !closed {
store.Close()
}
})
monitor := issue1654Monitor()
monitor.metricsHistory = NewMetricsHistory(20, time.Hour)
monitor.metricsStore = store
// Close drains ordinary queued writes before the final read; no polling for
// a favourable absence result and no changes to the store or its driver.
report := issue1654Report(time.Now().UTC())
report.Disks = nil
report.Metrics = agentshost.Metrics{CPUUsagePercent: 12, Memory: agentshost.MemoryMetric{TotalBytes: 8192, UsedBytes: 2048, Usage: 25}}
report.SequenceID = agentshost.FormatReportSequenceID("disk-presence", 1)
host, err := monitor.ApplyHostReport(report, nil)
if err != nil {
t.Fatal(err)
}
if points := monitor.metricsHistory.GetGuestMetrics("agent:"+host.ID, "disk", time.Hour); len(points) != 0 {
t.Fatalf("absent inventory recorded disk zero: %+v", points)
}
// The persistent store contract records whole-second observations.
recordedAt := time.Now().Add(-2 * time.Minute).Truncate(time.Second)
monitor.metricsHistory.AddGuestMetric("agent:"+host.ID, "disk", 43, recordedAt)
store.Write("agent", host.ID, "disk", 43, recordedAt)
report.Disks = []agentshost.Disk{{Device: "/dev/sda", Mountpoint: "/", Type: "ext4", TotalBytes: 100, FreeBytes: 100, Usage: 0}}
report.SequenceID = agentshost.FormatReportSequenceID("disk-presence", 2)
report.Timestamp = time.Now().UTC()
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatal(err)
}
report.Disks = nil
report.SequenceID = agentshost.FormatReportSequenceID("disk-presence", 3)
report.Timestamp = time.Now().UTC()
if _, err := monitor.ApplyHostReport(report, nil); err != nil {
t.Fatal(err)
}
points := monitor.metricsHistory.GetGuestMetrics("agent:"+host.ID, "disk", time.Hour)
if len(points) != 2 || points[0].Value != 43 || points[1].Value != 0 || !points[0].Timestamp.Equal(recordedAt) {
t.Fatalf("absence changed recorded/real-zero observations: %+v", points)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
closed = true
reopened, err := metrics.NewStore(cfg)
if err != nil {
t.Fatal(err)
}
defer reopened.Close()
saved, err := reopened.Query("agent", host.ID, "disk", recordedAt.Add(-time.Minute), time.Now().Add(time.Minute), 0)
if err != nil {
t.Fatal(err)
}
if len(saved) != 2 || saved[0].Value != 43 || saved[1].Value != 0 {
t.Fatalf("persisted disk series invented or lost a sample: %+v", saved)
}
if saved[0].Timestamp.UnixMilli() != recordedAt.UnixMilli() {
t.Fatalf("persisted observation time changed: %+v", saved[0])
}
if got := monitor.HostsSnapshot(); len(got) != 1 || len(got[0].Disks) != 0 || got[0].Memory.Total != 8192 {
t.Fatalf("accepted no-disk report was not retained: %+v", got)
}
}