Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve reviewed monitoring repairs with current agent source freshness

pulse-triage[bot]49805f32e32c
diff --cc internal/monitoring/monitor_host_agents_test.go
index dd1d94b476,3ef3414d7d..8816f2ea50
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@@ -6844,114 -6845,70 +6845,182 @@@ func TestBroadcastProjectionListsRegist
}
}
+func TestAgentLXCPartialInventoryPreservesSuccessAndInvalidatesOmissions(t *testing.T) {
+ state := models.NewState()
+ state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a", LinkedAgentID: "agent-a"}})
+ m := &Monitor{state: state}
+ now := time.Date(2026, 10, 5, 16, 0, 0, 0, time.UTC)
+ row := func(vmid int) agentshost.ProxmoxLXCContainer {
+ return agentshost.ProxmoxLXCContainer{VMID: vmid, Name: "web", Disks: []agentshost.Disk{{Type: "rootfs", Device: "local:disk", Mountpoint: "/", TotalBytes: 4096, UsedBytes: 1024, FreeBytes: 3072}}}
+ }
+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete", Containers: []agentshost.ProxmoxLXCContainer{row(100), row(102)}}, now, 30)
+ foreignKey := agentLXCFilesystemCacheKey("pve-b", "node-a", 102)
+ m.proxmoxLXCFilesystemsCache[foreignKey] = agentLXCFilesystemCacheEntry{agentID: "agent-b", name: "foreign", expiresAt: now.Add(time.Hour)}
+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "partial", Containers: []agentshost.ProxmoxLXCContainer{row(100)}, OmittedVMIDs: []int{102}}, now.Add(time.Minute), 30)
+ if _, ok := m.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("pve-a", "node-a", 102)]; ok {
+ t.Fatal("omitted guest kept a usable prior agent reading")
+ }
+ if _, ok := m.proxmoxLXCFilesystemsCache[foreignKey]; !ok {
+ t.Fatal("foreign instance cache was invalidated")
+ }
+ guest := models.Container{VMID: 100, Name: "web", Status: "running"}
+ m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, now.Add(time.Minute))
+ if len(guest.Disks) != 1 || guest.Disks[0].Used != 1024 {
+ t.Fatalf("successful partial guest=%+v", guest)
+ }
+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete"}, now.Add(2*time.Minute), 30)
+ if _, ok := m.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("pve-a", "node-a", 100)]; ok {
+ t.Fatal("complete empty collection retained a removed/stopped guest reading")
+ }
+ if _, ok := m.proxmoxLXCFilesystemsCache[foreignKey]; !ok {
+ t.Fatal("complete empty collection cleared a foreign instance")
+ }
+}
+
+func TestAgentLXCMalformedPartialInventoryCannotRenewOrInvalidate(t *testing.T) {
+ state := models.NewState()
+ state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a", LinkedAgentID: "agent-a"}})
+ m := &Monitor{state: state}
+ now := time.Now()
+ key := agentLXCFilesystemCacheKey("pve-a", "node-a", 100)
+ original := agentLXCFilesystemCacheEntry{agentID: "agent-a", name: "web", expiresAt: now.Add(time.Minute)}
+ m.proxmoxLXCFilesystemsCache = map[string]agentLXCFilesystemCacheEntry{key: original}
+ m.applyAgentLXCFilesystems("node-id", "agent-a", &agentshost.ProxmoxLXCInventory{Status: "complete", OmittedVMIDs: []int{100}}, now, 30)
+ got := m.proxmoxLXCFilesystemsCache[key]
+ if !got.expiresAt.Equal(original.expiresAt) || got.name != "web" {
+ t.Fatal("malformed completeness changed cache")
+ }
+}
+
+func TestAgentLXCPartialWireReportReachesLinkedGuestWithoutStaleOmission(t *testing.T) {
+ m := newTestMonitor(t)
+ m.state.UpdateNodes([]models.Node{{ID: "node-id", Name: "node-a", Instance: "pve-a"}})
+ row := func(vmid int) agentshost.ProxmoxLXCContainer {
+ return agentshost.ProxmoxLXCContainer{VMID: vmid, Name: "web", Disks: []agentshost.Disk{{
+ Type: "rootfs", Device: "local:disk", Mountpoint: "/", TotalBytes: 4096, UsedBytes: 1024, FreeBytes: 3072,
+ }}}
+ }
+ report := agentshost.Report{
+ Agent: agentshost.AgentInfo{ID: "agent-a", IntervalSeconds: 30},
+ Host: agentshost.HostInfo{ID: "agent-a", MachineID: "agent-a", Hostname: "node-a", Platform: "linux"},
+ ProxmoxLXC: &agentshost.ProxmoxLXCInventory{
+ Status: agentshost.ProxmoxLXCCollectionComplete, Containers: []agentshost.ProxmoxLXCContainer{row(100), row(102)},
+ },
+ }
+ applyWire := func(sequence uint64) models.Host {
+ t.Helper()
+ report.Timestamp = time.Now().UTC()
+ report.SequenceID = agentshost.FormatReportSequenceID("partial-wire-test", sequence)
+ encoded, err := json.Marshal(report)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var decoded agentshost.Report
+ if err := json.Unmarshal(encoded, &decoded); err != nil {
+ t.Fatal(err)
+ }
+ host, err := m.ApplyHostReport(decoded, &config.APITokenRecord{ID: "partial-wire-token"})
+ if err != nil || host.LinkedNodeID != "node-id" {
+ t.Fatalf("wire report did not reach linked node: host=%+v err=%v", host, err)
+ }
+ return host
+ }
+ applyWire(1)
+ report.ProxmoxLXC.Status = agentshost.ProxmoxLXCCollectionPartial
+ report.ProxmoxLXC.Containers = []agentshost.ProxmoxLXCContainer{row(100)}
+ report.ProxmoxLXC.OmittedVMIDs = []int{102}
+ report.Agent.Modules = []agentshost.ModuleStatus{{Name: agentshost.ModuleNameTypedPrivilegeHelper, Enabled: true, State: "degraded", LastError: "proxmox.lxc_filesystems: helper Proxmox LXC filesystem inventory is incomplete"}}
+ host := applyWire(2)
+ if len(host.AgentModules) != 1 || host.AgentModules[0].State != "degraded" {
+ t.Fatalf("wire report lost partial helper health: %+v", host.AgentModules)
+ }
+ for _, vmid := range []int{100, 102} {
+ guest := models.Container{VMID: vmid, Name: "web", Status: "running"}
+ m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, time.Now())
+ if vmid == 100 && (len(guest.Disks) != 1 || guest.Disks[0].Used != 1024) {
+ t.Fatalf("successful wire reading was lost: %+v", guest.Disks)
+ }
+ if vmid == 102 && len(guest.Disks) != 0 {
+ t.Fatalf("omitted wire guest kept a prior reading: %+v", guest.Disks)
+ }
+ }
+ report.ProxmoxLXC = &agentshost.ProxmoxLXCInventory{Status: agentshost.ProxmoxLXCCollectionComplete}
+ report.Agent.Modules[0].State, report.Agent.Modules[0].LastError = "running", ""
+ host = applyWire(3)
+ if len(host.AgentModules) != 1 || host.AgentModules[0].State != "running" || host.AgentModules[0].LastError != "" {
+ t.Fatalf("wire recovery did not clear partial helper health: %+v", host.AgentModules)
+ }
+ guest := models.Container{VMID: 100, Name: "web", Status: "running"}
+ m.enrichContainerWithAgentLXCFilesystems("pve-a", "node-a", &guest, time.Now())
+ if len(guest.Disks) != 0 {
+ t.Fatal("complete empty wire report retained a stopped guest reading")
+ }
+}
++
+ // A host agent linked to a Proxmox node merges into one read-state row that
+ // every PVE poll keeps online and fresh. Once the agent stops reporting, its
+ // retained filesystem summary must stop standing in for the node's disk (and
+ // through it disk alerts and history), and the projected agent must read as
+ // offline with its own last report, so its connection shows the missed
+ // heartbeat instead of the PVE poll time.
+ func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) {
+ state := models.NewState()
+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter}
+
+ node := models.Node{
+ ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
+ ConnectionHealth: "healthy", LinkedAgentID: "agent-1",
+ }
+ rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}}
+ // poll records a PVE poll that just saw the node, with the agent as of its
+ // last report, and resolves the node's disk the way pollPVENode does.
+ poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) {
+ t.Helper()
+ polledNode := node
+ polledNode.LastSeen = time.Now()
+ state.UpdateNodes([]models.Node{polledNode})
+ state.UpsertHost(models.Host{
+ ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID,
+ IntervalSeconds: 30, LastSeen: agentLastReport,
+ Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}},
+ })
+ m.evaluateHostAgents(time.Now())
+ m.refreshUnifiedResourceStoreAfterAgentStateChange()
+
+ disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS)
+ for _, host := range m.HostsSnapshot() {
+ if host.ID == "agent-1" {
+ return disk, source, host
+ }
+ }
+ t.Fatal("HostsSnapshot does not list agent-1")
+ return disk, source, models.Host{}
+ }
+
+ agentReport := time.Now().Add(-10 * time.Second)
+ disk, source, host := poll(agentReport)
+ if source != "agent" || disk.Used != 900 {
+ t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source)
+ }
+ if host.Status != "online" || !host.LastSeen.Equal(agentReport) {
+ t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport)
+ }
+
+ silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute)
+ disk, source, host = poll(silentSince)
+ if source != "node-status-rootfs" || disk.Used != 50 {
+ t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source)
+ }
+ if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil {
+ t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked)
+ }
+ if host.Status != "offline" || !host.LastSeen.Equal(silentSince) {
+ t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince)
+ }
+
+ disk, source, host = poll(time.Now())
+ if source != "agent" || disk.Used != 900 || host.Status != "online" {
+ t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status)
+ }
+ }