Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Read a linked agent's own heartbeat in host snapshots

rcourtmanf4c304037351
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index ea945c3572..93d863fc62 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -8797,3 +8797,15 @@ no update status, instead of reporting "no update available" (#2353). Agent
registration, enrolment, install, update, removal and report identity are
unchanged; the per-image memo lives in the registry checker and is pruned
each collection cycle to the images in use.
+
+### Host snapshots report a linked agent's own heartbeat
+
+`internal/monitoring/monitor.go` changed only so a host produced from the read
+state carries the agent source's own last report and reads offline past the
+agent's reporting lease, even when the agent's row is merged with a Proxmox
+node that PVE polling keeps fresh. The agent connection on the Connections
+list and the update-readiness agent-continuity check therefore age a silent
+agent from its last report instead of from the PVE poll, and mark it stale at
+their own heartbeat cutoff (`fleethealth.AgentStaleThreshold`, which is
+separate from the monitoring reporting lease). Agent registration, enrolment,
+install, update, removal and report identity are unchanged.
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 17b73c02fd..0353b909c1 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -2638,6 +2638,25 @@ online/running/healthy vocabulary. The diagnostics subject set is unchanged:
snapshot (real agents), which now matches what agent-only surfaces show once
integration-backed ledger rows are excluded.
+A host produced by `hostFromReadStateView` describes the agent, so its
+`Status` and `LastSeen` are the agent's own. A host agent linked to a Proxmox
+node merges into one row whose `LastSeen` follows its freshest source and
+whose status stays online while PVE polling continues, so the row's values
+would present a silent agent as online and current. The projection reads the
+agent source's sighting from `HostView.SourceStatus(SourceAgent)`: the host
+carries that report time and reads `offline` once it is past the reporting
+lease (`hostAgentReportCurrent`, the `hostAgentHealthWindow` that
+`evaluateHostAgents` applies to the host in state). Rows without an agent
+sighting, such as vSphere and TrueNAS integration hosts, keep the row's
+values. `linkedHostForNode` therefore stops handing a silent agent's retained
+filesystem summary and ZFS datasets to its node, whose disk falls back to the
+Proxmox rootfs or `/nodes` reading before disk alerts and history read it, and
+agent connections built from `HostsSnapshot()` (the Connections list and the
+update-readiness agent-continuity check) age the agent from its last report
+rather than the PVE poll time, applying their own heartbeat cutoff.
+`TestSilentLinkedAgentStopsFeedingNodeDisk` drives the disk and projection
+path through `evaluateHostAgents` and the registry-backed store refresh.
+
Unified Agent host reports now make module readiness and updater/config
lifecycle evidence monitoring-owned observed state. Monitoring preserves the
last successful one-shot update transition across subsequent reports, forwards
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index baab181eb9..54b6e634c9 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -3577,6 +3577,7 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {
if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() {
displayName = trimmed
}
+ status, lastSeen := hostAgentLivenessFromReadStateView(view, time.Now())
return models.Host{
ID: firstNonEmptyString(view.AgentID(), view.ID()),
@@ -3598,10 +3599,10 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {
RAID: hostRAIDFromReadStateView(view.RAID()),
Unraid: hostUnraidFromReadStateView(view.Unraid()),
Ceph: hostCephFromReadStateView(view.Ceph()),
- Status: string(view.Status()),
+ Status: status,
UptimeSeconds: view.UptimeSeconds(),
IntervalSeconds: view.IntervalSeconds(),
- LastSeen: view.LastSeen(),
+ LastSeen: lastSeen,
AgentVersion: view.AgentVersion(),
IntegrationSource: view.IntegrationSource(),
MachineID: view.MachineID(),
@@ -3624,6 +3625,25 @@ func hostFromReadStateView(view *unifiedresources.HostView) models.Host {
}
}
+// hostAgentLivenessFromReadStateView returns the status and last report of the
+// agent behind a read-state host row. A row merged with a Proxmox node takes
+// LastSeen from its freshest source and stays online while PVE polling
+// continues, so a silent agent would look current: its retained disk summary
+// and datasets kept feeding the node, and its connection read as active. A
+// models.Host describes the agent, so read the agent source's own sighting
+// and lease, as evaluateHostAgents does for the host in state.
+func hostAgentLivenessFromReadStateView(view *unifiedresources.HostView, now time.Time) (string, time.Time) {
+ status, lastSeen := string(view.Status()), view.LastSeen()
+ agentSighting, ok := view.SourceStatus(unifiedresources.SourceAgent)
+ if !ok || agentSighting.LastSeen.IsZero() {
+ return status, lastSeen
+ }
+ if !hostAgentReportCurrent(agentSighting.LastSeen, view.IntervalSeconds(), now) {
+ status = "offline"
+ }
+ return status, agentSighting.LastSeen
+}
+
func vmFromReadStateView(view *unifiedresources.VMView) models.VM {
if view == nil {
return models.VM{}
diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go
index f14c2d1d55..3ef3414d7d 100644
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@ -24,6 +24,7 @@ import (
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
func TestNewMonitorRoutesStartupCustomSensorWarningBeforeStart(t *testing.T) {
@@ -6843,3 +6844,71 @@ func TestBroadcastProjectionListsRegistryOnceAndKeepsLiveChanges(t *testing.T) {
t.Fatal("removed host lifecycle surface disappeared")
}
}
+
+// A host agent linked to a Proxmox node merges into one read-state row that
+// every PVE poll keeps online and fresh. Once the agent stops reporting, its
+// retained filesystem summary must stop standing in for the node's disk (and
+// through it disk alerts and history), and the projected agent must read as
+// offline with its own last report, so its connection shows the missed
+// heartbeat instead of the PVE poll time.
+func TestSilentLinkedAgentStopsFeedingNodeDisk(t *testing.T) {
+ state := models.NewState()
+ adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ m := &Monitor{config: &config.Config{}, state: state, resourceStore: adapter}
+
+ node := models.Node{
+ ID: "pve1-node1", Name: "node1", Instance: "pve1", Status: "online",
+ ConnectionHealth: "healthy", LinkedAgentID: "agent-1",
+ }
+ rootFS := &proxmox.NodeStatus{RootFS: &proxmox.RootFS{Total: 200, Used: 50, Free: 150}}
+ // poll records a PVE poll that just saw the node, with the agent as of its
+ // last report, and resolves the node's disk the way pollPVENode does.
+ poll := func(agentLastReport time.Time) (models.Disk, string, models.Host) {
+ t.Helper()
+ polledNode := node
+ polledNode.LastSeen = time.Now()
+ state.UpdateNodes([]models.Node{polledNode})
+ state.UpsertHost(models.Host{
+ ID: "agent-1", Hostname: "node1", Status: "online", LinkedNodeID: node.ID,
+ IntervalSeconds: 30, LastSeen: agentLastReport,
+ Disks: []models.Disk{{Mountpoint: "/", Type: "ext4", Total: 1000, Used: 900, Free: 100, Usage: 90}},
+ })
+ m.evaluateHostAgents(time.Now())
+ m.refreshUnifiedResourceStoreAfterAgentStateChange()
+
+ disk, source := m.resolveNodeDisk("pve1", node.ID, node.Name, proxmox.Node{Node: node.Name}, rootFS)
+ for _, host := range m.HostsSnapshot() {
+ if host.ID == "agent-1" {
+ return disk, source, host
+ }
+ }
+ t.Fatal("HostsSnapshot does not list agent-1")
+ return disk, source, models.Host{}
+ }
+
+ agentReport := time.Now().Add(-10 * time.Second)
+ disk, source, host := poll(agentReport)
+ if source != "agent" || disk.Used != 900 {
+ t.Fatalf("reporting agent: node disk = %+v from %q, want the agent's summary", disk, source)
+ }
+ if host.Status != "online" || !host.LastSeen.Equal(agentReport) {
+ t.Fatalf("reporting agent projected as %q last seen %s, want online at its report %s", host.Status, host.LastSeen, agentReport)
+ }
+
+ silentSince := time.Now().Add(-hostAgentHealthWindow(30) - time.Minute)
+ disk, source, host = poll(silentSince)
+ if source != "node-status-rootfs" || disk.Used != 50 {
+ t.Fatalf("silent agent: node disk = %+v from %q, want Proxmox rootfs", disk, source)
+ }
+ if linked := m.linkedHostForNode("pve1", node.ID, node.Name); linked != nil {
+ t.Fatalf("silent agent still linked to the node for disk and dataset data: %+v", linked)
+ }
+ if host.Status != "offline" || !host.LastSeen.Equal(silentSince) {
+ t.Fatalf("silent agent projected as %q last seen %s, want offline at its last report %s", host.Status, host.LastSeen, silentSince)
+ }
+
+ disk, source, host = poll(time.Now())
+ if source != "agent" || disk.Used != 900 || host.Status != "online" {
+ t.Fatalf("agent reporting again: node disk = %+v from %q, agent %q; want the agent's summary and online", disk, source, host.Status)
+ }
+}