Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Annotate memory provenance on the per-node LXC fallback

rcourtman9caccd545ee7
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index c66d517759..3fc4a0219c 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -503,6 +503,17 @@ retained one keeps its own state, source and time
`TestFixtureRefreshMovesCurrentGuestMemoryObservationTime`). This is fixture
realism, not a change to live source selection.
+The per-node LXC collector that runs when `cluster/resources` is unavailable
+(`monitor_polling_containers.go`) annotates through the same
+`guestMemoryObservation` builder as the cluster path, passing the selected
+source, the listing's `ObservedAt`, the previous snapshot and the linked agent.
+A running container therefore reads `current` from `cluster-resources` (or
+`agent`) at the listing time and a stopped one `unavailable` from
+`powered-off`, rather than an unannotated value the UI qualifies as freshness
+unknown for every container on that node
+(`TestPerNodeLXCMemoryObservationMatchesClusterPath`). The fallback's numeric
+selection, stabilisation and alerting are unchanged.
+
Low-trust memory stabilisation uses the original memory observation time for
its existing two-minute reconnect and ten-minute healthy-guest windows, not
the diagnostic poll's `RetrievedAt`. Repeated carried-forward polls cannot
diff --git a/internal/monitoring/monitor_polling_containers.go b/internal/monitoring/monitor_polling_containers.go
index 24505ed5ba..d0e32c3e7b 100644
--- a/internal/monitoring/monitor_polling_containers.go
+++ b/internal/monitoring/monitor_polling_containers.go
@@ -164,8 +164,9 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName s
agentHost,
hasAgent,
)
+ previousMemory := m.previousGuestSnapshot(instanceName, "lxc", n.Node, int(container.VMID))
memUsed, memorySource, _ = stabilizeGuestLowTrustMemory(
- m.previousGuestSnapshot(instanceName, "lxc", n.Node, int(container.VMID)),
+ previousMemory,
container.Status,
memorySource,
memTotal,
@@ -193,6 +194,13 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName s
Usage: memUsagePercent,
}
}
+ // The cluster path's provenance builder owns state, source and
+ // observation time here too, so a node polled through this
+ // fallback does not read as freshness unknown.
+ memory.Observation = m.guestMemoryObservation(instanceName, "lxc", n.Node, int(container.VMID), memory, guestMemoryObservationContext{
+ source: memorySource, providerObservedAt: counterObservedAt,
+ previous: previousMemory, linkedAgent: agentHost,
+ }, time.Now())
diskTotalBytes := clampToInt64(container.MaxDisk)
diskUsedBytes := clampToInt64(container.Disk)
diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.go
index 6f7492e721..2cc6a53342 100644
--- a/internal/monitoring/monitor_pve_guest_lxc_test.go
+++ b/internal/monitoring/monitor_pve_guest_lxc_test.go
@@ -1003,3 +1003,88 @@ func TestGuestFilesystemFailureGuidanceUsesObservedEvidence(t *testing.T) {
})
}
}
+
+// TestPerNodeLXCMemoryObservationMatchesClusterPath pins the producer parity
+// for the fallback used when cluster/resources is unavailable: the per-node
+// collector annotates memory through the same builder, so its containers do
+// not read as freshness unknown while the same guest polled via the cluster
+// listing reads current.
+func TestPerNodeLXCMemoryObservationMatchesClusterPath(t *testing.T) {
+ t.Setenv("PULSE_DATA_DIR", t.TempDir())
+
+ const gib = uint64(1024 * 1024 * 1024)
+ listedAt := time.Now().Add(-5 * time.Second).UTC()
+
+ tests := []struct {
+ name string
+ status string
+ want models.MemoryObservation
+ }{
+ {
+ name: "running reads current from the listing time",
+ status: "running",
+ want: models.MemoryObservation{State: "current", Source: "cluster-resources", ObservedAt: listedAt},
+ },
+ {
+ name: "stopped reads unavailable from powered-off",
+ status: "stopped",
+ want: models.MemoryObservation{State: "unavailable", Source: "powered-off"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ client := &vmMemoryTrustStubClient{
+ stubPVEClient: &stubPVEClient{},
+ containers: []proxmox.Container{{
+ VMID: 311, Name: "ct-311", Node: "node1", Status: tt.status,
+ MaxMem: 8 * gib, Mem: 3 * gib, CPUs: 2, ObservedAt: listedAt,
+ }},
+ }
+
+ clusterMon := newTestPVEMonitor("test")
+ defer clusterMon.alertManager.Stop()
+ defer clusterMon.notificationMgr.Stop()
+ clusterContainer, _, _, _, ok := clusterMon.buildContainerFromClusterResource(
+ context.Background(),
+ "test",
+ proxmox.ClusterResource{
+ Type: "lxc", Node: "node1", Name: "ct-311", Status: tt.status, VMID: 311,
+ MaxMem: 8 * gib, Mem: 3 * gib, MaxCPU: 2, ObservedAt: listedAt,
+ },
+ client,
+ map[int]bool{},
+ nil,
+ )
+ if !ok {
+ t.Fatal("expected cluster-path container to be built")
+ }
+
+ nodeMon := newTestPVEMonitor("test")
+ defer nodeMon.alertManager.Stop()
+ defer nodeMon.notificationMgr.Stop()
+ nodeMon.pollContainersWithNodes(
+ context.Background(),
+ "test",
+ "",
+ false,
+ client,
+ []proxmox.Node{{Node: "node1", Status: "online"}},
+ map[string]string{"node1": "online"},
+ )
+ containers := nodeMon.state.GetSnapshot().Containers
+ if len(containers) != 1 {
+ t.Fatalf("expected 1 per-node container, got %d", len(containers))
+ }
+
+ for path, got := range map[string]models.MemoryObservation{
+ "cluster": clusterContainer.Memory.Observation,
+ "per-node": containers[0].Memory.Observation,
+ } {
+ if got.State != tt.want.State || got.Source != tt.want.Source || !got.ObservedAt.Equal(tt.want.ObservedAt) {
+ t.Fatalf("%s memory observation = %+v, want %+v", path, got, tt.want)
+ }
+ }
+ })
+ }
+}