Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve reviewed monitoring repairs with current agent source freshness
pulse-triage[bot]49805f32e32c
diff --cc internal/monitoring/monitor_host_agents_test.goindex dd1d94b476,3ef3414d7d..8816f2ea50--- a/internal/monitoring/monitor_host_agents_test.go+++ b/internal/monitoring/monitor_host_agents_test.go@@@ -6844,114 -6845,70 +6845,182 @@@ func TestBroadcastProjectionListsRegist}}+func TestAgentLXCPartialInventoryPreservesSuccessAndInvalidatesOmissions(t *testing.T) {+ state := models.NewState()+ state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a", LinkedAgentID: "agent-a"}})+ m := &Monitor{state: state}+ now := time.Date(2026, 10, 5, 16, 0, 0, 0, time.UTC)+ row := func(vmid int) agentshost.ProxmoxLXCContainer {+ return agentshost.ProxmoxLXCContainer{VMID: vmid, Name: "web", Disks: []agentshost.Disk{{Type: "rootfs", Device: "local:disk", Mountpoint: "/", TotalBytes: 4096, UsedBytes: 1024, FreeBytes: 3072}}}+ }+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete", Containers: []agentshost.ProxmoxLXCContainer{row(100), row(102)}}, now, 30)+ foreignKey := agentLXCFilesystemCacheKey("pve-b", "node-a", 102)+ m.proxmoxLXCFilesystemsCache[foreignKey] = agentLXCFilesystemCacheEntry{agentID: "agent-b", name: "foreign", expiresAt: now.Add(time.Hour)}+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "partial", Containers: []agentshost.ProxmoxLXCContainer{row(100)}, OmittedVMIDs: []int{102}}, now.Add(time.Minute), 30)+ if _, ok := m.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("pve-a", "node-a", 102)]; ok {+ t.Fatal("omitted guest kept a usable prior agent reading")+ }+ if _, ok := m.proxmoxLXCFilesystemsCache[foreignKey]; !ok {+ t.Fatal("foreign instance cache was invalidated")+ }+ guest := models.Container{VMID: 100, Name: "web", Status: "running"}+ m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, now.Add(time.Minute))+ if len(guest.Disks) != 1 || guest.Disks[0].Used != 1024 {+ t.Fatalf("successful partial guest=%+v", guest)+ }+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete"}, now.Add(2*time.Minute), 30)+ if _, ok := m.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("pve-a", "node-a", 100)]; ok {+ t.Fatal("complete empty collection retained a removed/stopped guest reading")+ }+ if _, ok := m.proxmoxLXCFilesystemsCache[foreignKey]; !ok {+ t.Fatal("complete empty collection cleared a foreign instance")+ }+}++func TestAgentLXCMalformedPartialInventoryCannotRenewOrInvalidate(t *testing.T) {+ state := models.NewState()+ state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a", LinkedAgentID: "agent-a"}})+ m := &Monitor{state: state}+ now := time.Now()+ key := agentLXCFilesystemCacheKey("pve-a", "node-a", 100)+ original := agentLXCFilesystemCacheEntry{agentID: "agent-a", name: "web", expiresAt: now.Add(time.Minute)}+ m.proxmoxLXCFilesystemsCache = map[string]agentLXCFilesystemCacheEntry{key: original}+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete", OmittedVMIDs: []int{100}}, now, 30)+ got := m.proxmoxLXCFilesystemsCache[key]+ if !got.expiresAt.Equal(original.expiresAt) || got.name != "web" {+ t.Fatal("malformed completeness changed cache")+ }+}++func TestAgentLXCPartialWireReportReachesLinkedGuestWithoutStaleOmission(t *testing.T) {+ m := newTestMonitor(t)+ m.state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a"}})+ row := func(vmid int) agentshost.ProxmoxLXCContainer {+ return agentshost.ProxmoxLXCContainer{VMID: vmid, Name: "web", Disks: []agentshost.Disk{{+ Type: "rootfs", Device: "local:disk", Mountpoint: "/", TotalBytes: 4096, UsedBytes: 1024, FreeBytes: 3072,+ }}}+ }+ report := agentshost.Report{+ Agent: agentshost.AgentInfo{ID: "agent-a", IntervalSeconds: 30},+ Host: agentshost.HostInfo{ID: "agent-a", MachineID: "agent-a", Hostname: "node-a", Platform: "linux"},+ ProxmoxLXC: &agentshost.ProxmoxLXCInventory{+ Status: agentshost.ProxmoxLXCCollectionComplete, Containers: []agentshost.ProxmoxLXCContainer{row(100), row(102)},+ },+ }+ applyWire := func(sequence uint64) models.Host {+ t.Helper()+ report.Timestamp = time.Now().UTC()+ report.SequenceID = agentshost.FormatReportSequenceID("partial-wire-test", sequence)+ encoded, err := json.Marshal(report)+ if err != nil {+ t.Fatal(err)+ }+ var decoded agentshost.Report+ if err := json.Unmarshal(encoded, &decoded); err != nil {+ t.Fatal(err)+ }+ host, err := m.ApplyHostReport(decoded, &config.APITokenRecord{ID: "partial-wire-token"})+ if err != nil || host.LinkedNodeID != "node-id" {+ t.Fatalf("wire report did not reach linked node: host=%+v err=%v", host, err)+ }+ return host+ }+ applyWire(1)+ report.ProxmoxLXC.Status = agentshost.ProxmoxLXCCollectionPartial+ report.ProxmoxLXC.Containers = []agentshost.ProxmoxLXCContainer{row(100)}+ report.ProxmoxLXC.OmittedVMIDs = []int{102}+ report.Agent.Modules = []agentshost.ModuleStatus{{Name: agentshost.ModuleNameTypedPrivilegeHelper, Enabled: true, State: "degraded", LastError: "proxmox.lxc_filesystems: helper Proxmox LXC filesystem inventory is incomplete"}}+ host := applyWire(2)+ if len(host.AgentModules) != 1 || host.AgentModules[0].State != "degraded" {+ t.Fatalf("wire report lost partial helper health: %+v", host.AgentModules)+ }+ for _, vmid := range []int{100, 102} {+ guest := models.Container{VMID: vmid, Name: "web", Status: "running"}+ m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, time.Now())+ if vmid == 100 && (len(guest.Disks) != 1 || guest.Disks[0].Used != 1024) {+ t.Fatalf("successful wire reading was lost: %+v", guest.Disks)+ }+ if vmid == 102 && len(guest.Disks) != 0 {+ t.Fatalf("omitted wire guest kept a prior reading: %+v", guest.Disks)+ }+ }+ report.ProxmoxLXC = &agentshost.ProxmoxLXCInventory{Status: agentshost.ProxmoxLXCCollectionComplete}+ report.Agent.Modules[0].State, report.Agent.Modules[0].LastError = "running", ""+ host = applyWire(3)+ if len(host.AgentModules) != 1 || host.AgentModules[0].State != "running" || host.AgentModules[0].LastError != "" {+ t.Fatalf("wire recovery did not clear partial helper health: %+v", host.AgentModules)+ }+ guest := models.Container{VMID: 100, Name: "web", Status: "running"}+ m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, time.Now())+ if len(guest.Disks) != 0 {+ t.Fatal("complete empty wire report retained a stopped guest reading")+ }+}+++ // A host agent linked to a Proxmox node merges into one read-state row that+ // every PVE poll keeps online and fresh. Once the agent stops reporting, its+ // retained filesystem summary must stop standing in for the node's disk (and+ // through it disk alerts and history), and the projected agent must read as+ // offline with its own last report, so its connection shows the missed+ // heartbeat instead of the PVE poll time.+ func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) {+ state := models.NewState()+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter}++ node := models.Node{+ ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",+ ConnectionHealth: "healthy", LinkedAgentID: "agent-1",+ }+ rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}}+ // poll records a PVE poll that just saw the node, with the agent as of its+ // last report, and resolves the node's disk the way pollPVENode does.+ poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) {+ t.Helper()+ polledNode := node+ polledNode.LastSeen = time.Now()+ state.UpdateNodes([]models.Node{polledNode})+ state.UpsertHost(models.Host{+ ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID,+ IntervalSeconds: 30, LastSeen: agentLastReport,+ Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}},+ })+ m.evaluateHostAgents(time.Now())+ m.refreshUnifiedResourceStoreAfterAgentStateChange()++ disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS)+ for _, host := range m.HostsSnapshot() {+ if host.ID == "agent-1" {+ return disk, source, host+ }+ }+ t.Fatal("HostsSnapshot does not list agent-1")+ return disk, source, models.Host{}+ }++ agentReport := time.Now().Add(-10 * time.Second)+ disk, source, host := poll(agentReport)+ if source != "agent" || disk.Used != 900 {+ t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source)+ }+ if host.Status != "online" || !host.LastSeen.Equal(agentReport) {+ t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport)+ }++ silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute)+ disk, source, host = poll(silentSince)+ if source != "node-status-rootfs" || disk.Used != 50 {+ t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source)+ }+ if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil {+ t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked)+ }+ if host.Status != "offline" || !host.LastSeen.Equal(silentSince) {+ t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince)+ }++ disk, source, host = poll(time.Now())+ if source != "agent" || disk.Used != 900 || host.Status != "online" {+ t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status)+ }+ }