Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Read a linked agent's own heartbeat in host snapshots
rcourtmanf4c304037351
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex ea945c3572..93d863fc62 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -8797,3 +8797,15 @@ no update status, instead of reporting "no update available" (#2353). Agentregistration, enrolment, install, update, removal and report identity areunchanged; the per-image memo lives in the registry checker and is prunedeach collection cycle to the images in use.++### Host snapshots report a linked agent's own heartbeat++`internal/monitoring/monitor.go` changed only so a host produced from the read+state carries the agent source's own last report and reads offline past the+agent's reporting lease, even when the agent's row is merged with a Proxmox+node that PVE polling keeps fresh. The agent connection on the Connections+list and the update-readiness agent-continuity check therefore age a silent+agent from its last report instead of from the PVE poll, and mark it stale at+their own heartbeat cutoff (`fleethealth.AgentStaleThreshold`, which is+separate from the monitoring reporting lease). Agent registration, enrolment,+install, update, removal and report identity are unchanged.diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 17b73c02fd..0353b909c1 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -2638,6 +2638,25 @@ online/running/healthy vocabulary. The diagnostics subject set is unchanged:snapshot (real agents), which now matches what agent-only surfaces show onceintegration-backed ledger rows are excluded.+A host produced by `hostFromReadStateView` describes the agent, so its+`Status` and `LastSeen` are the agent's own. A host agent linked to a Proxmox+node merges into one row whose `LastSeen` follows its freshest source and+whose status stays online while PVE polling continues, so the row's values+would present a silent agent as online and current. The projection reads the+agent source's sighting from `HostView.SourceStatus(SourceAgent)`: the host+carries that report time and reads `offline` once it is past the reporting+lease (`hostAgentReportCurrent`, the `hostAgentHealthWindow` that+`evaluateHostAgents` applies to the host in state). Rows without an agent+sighting, such as vSphere and TrueNAS integration hosts, keep the row's+values. `linkedHostForNode` therefore stops handing a silent agent's retained+filesystem summary and ZFS datasets to its node, whose disk falls back to the+Proxmox rootfs or `/nodes` reading before disk alerts and history read it, and+agent connections built from `HostsSnapshot()` (the Connections list and the+update-readiness agent-continuity check) age the agent from its last report+rather than the PVE poll time, applying their own heartbeat cutoff.+`TestSilentLinkedAgentStopsFeedingNodeDisk` drives the disk and projection+path through `evaluateHostAgents` and the registry-backed store refresh.+Unified Agent host reports now make module readiness and updater/configlifecycle evidence monitoring-owned observed state. Monitoring preserves thelast successful one-shot update transition across subsequent reports, forwardsdiff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex baab181eb9..54b6e634c9 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -3577,6 +3577,7 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() {displayName = trimmed}+ status, lastSeen := hostAgentLivenessFromReadStateView(view, time.Now())return models.Host{ID: firstNonEmptyString(view.AgentID(), view.ID()),@@ -3598,10 +3599,10 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {RAID: hostRAIDFromReadStateView(view.RAID()),Unraid: hostUnraidFromReadStateView(view.Unraid()),Ceph: hostCephFromReadStateView(view.Ceph()),- Status: string(view.Status()),+ Status: status,UptimeSeconds: view.UptimeSeconds(),IntervalSeconds: view.IntervalSeconds(),- LastSeen: view.LastSeen(),+ LastSeen: lastSeen,AgentVersion: view.AgentVersion(),IntegrationSource: view.IntegrationSource(),MachineID: view.MachineID(),@@ -3624,6 +3625,25 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {}}+// hostAgentLivenessFromReadStateView returns the status and last report of the+// agent behind a read-state host row. A row merged with a Proxmox node takes+// LastSeen from its freshest source and stays online while PVE polling+// continues, so a silent agent would look current: its retained disk summary+// and datasets kept feeding the node, and its connection read as active. A+// models.Host describes the agent, so read the agent source's own sighting+// and lease, as evaluateHostAgents does for the host in state.+func hostAgentLivenessFromReadStateView(view *unifiedresources.HostView, now time.Time) (string, time.Time) {+ status, lastSeen := string(view.Status()), view.LastSeen()+ agentSighting, ok := view.SourceStatus(unifiedresources.SourceAgent)+ if !ok || agentSighting.LastSeen.IsZero() {+ return status, lastSeen+ }+ if !hostAgentReportCurrent(agentSighting.LastSeen, view.IntervalSeconds(), now) {+ status = "offline"+ }+ return status, agentSighting.LastSeen+}+func vmFromReadStateView(view *unifiedresources.VMView) models.VM {if view == nil {return models.VM{}diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.goindex f14c2d1d55..3ef3414d7d 100644--- a/internal/monitoring/monitor_host_agents_test.go+++ b/internal/monitoring/monitor_host_agents_test.go@@ -24,6 +24,7 @@ import (agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host""github.com/rcourtman/pulse-go-rewrite/pkg/metrics"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox")func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) {@@ -6843,3 +6844,71 @@ func TestBroadcastProjectionListsRegistryOnceAndKeepsLiveChanges(t *testing.T) {t.Fatal("removed host lifecycle surface disappeared")}}++// A host agent linked to a Proxmox node merges into one read-state row that+// every PVE poll keeps online and fresh. Once the agent stops reporting, its+// retained filesystem summary must stop standing in for the node's disk (and+// through it disk alerts and history), and the projected agent must read as+// offline with its own last report, so its connection shows the missed+// heartbeat instead of the PVE poll time.+func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) {+ state := models.NewState()+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter}++ node := models.Node{+ ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",+ ConnectionHealth: "healthy", LinkedAgentID: "agent-1",+ }+ rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}}+ // poll records a PVE poll that just saw the node, with the agent as of its+ // last report, and resolves the node's disk the way pollPVENode does.+ poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) {+ t.Helper()+ polledNode := node+ polledNode.LastSeen = time.Now()+ state.UpdateNodes([]models.Node{polledNode})+ state.UpsertHost(models.Host{+ ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID,+ IntervalSeconds: 30, LastSeen: agentLastReport,+ Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}},+ })+ m.evaluateHostAgents(time.Now())+ m.refreshUnifiedResourceStoreAfterAgentStateChange()++ disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS)+ for _, host := range m.HostsSnapshot() {+ if host.ID == "agent-1" {+ return disk, source, host+ }+ }+ t.Fatal("HostsSnapshot does not list agent-1")+ return disk, source, models.Host{}+ }++ agentReport := time.Now().Add(-10 * time.Second)+ disk, source, host := poll(agentReport)+ if source != "agent" || disk.Used != 900 {+ t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source)+ }+ if host.Status != "online" || !host.LastSeen.Equal(agentReport) {+ t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport)+ }++ silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute)+ disk, source, host = poll(silentSince)+ if source != "node-status-rootfs" || disk.Used != 50 {+ t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source)+ }+ if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil {+ t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked)+ }+ if host.Status != "offline" || !host.LastSeen.Equal(silentSince) {+ t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince)+ }++ disk, source, host = poll(time.Now())+ if source != "agent" || disk.Used != 900 || host.Status != "online" {+ t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status)+ }+}