Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Annotate memory provenance on the per-node LXC fallback
rcourtman9caccd545ee7
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex c66d517759..3fc4a0219c 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -503,6 +503,17 @@ retained one keeps its own state, source and time`TestFixtureRefreshMovesCurrentGuestMemoryObservationTime`). This is fixturerealism, not a change to live source selection.+The per-node LXC collector that runs when `cluster/resources` is unavailable+(`monitor_polling_containers.go`) annotates through the same+`guestMemoryObservation` builder as the cluster path, passing the selected+source, the listing's `ObservedAt`, the previous snapshot and the linked agent.+A running container therefore reads `current` from `cluster-resources` (or+`agent`) at the listing time and a stopped one `unavailable` from+`powered-off`, rather than an unannotated value the UI qualifies as freshness+unknown for every container on that node+(`TestPerNodeLXCMemoryObservationMatchesClusterPath`). The fallback's numeric+selection, stabilisation and alerting are unchanged.+Low-trust memory stabilisation uses the original memory observation time forits existing two-minute reconnect and ten-minute healthy-guest windows, notthe diagnostic poll's `RetrievedAt`. Repeated carried-forward polls cannotdiff --git a/internal/monitoring/monitor_polling_containers.go b/internal/monitoring/monitor_polling_containers.goindex 24505ed5ba..d0e32c3e7b 100644--- a/internal/monitoring/monitor_polling_containers.go+++ b/internal/monitoring/monitor_polling_containers.go@@ -164,8 +164,9 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName sagentHost,hasAgent,)+ previousMemory := m.previousGuestSnapshot(instanceName, "lxc", n.Node, int(container.VMID))memUsed, memorySource, _ = stabilizeGuestLowTrustMemory(- m.previousGuestSnapshot(instanceName, "lxc", n.Node, int(container.VMID)),+ previousMemory,container.Status,memorySource,memTotal,@@ -193,6 +194,13 @@ func (m *Monitor) collectContainersWithNodes(ctx context.Context, instanceName sUsage: memUsagePercent,}}+ // The cluster path's provenance builder owns state, source and+ // observation time here too, so a node polled through this+ // fallback does not read as freshness unknown.+ memory.Observation = m.guestMemoryObservation(instanceName, "lxc", n.Node, int(container.VMID), memory, guestMemoryObservationContext{+ source: memorySource, providerObservedAt: counterObservedAt,+ previous: previousMemory, linkedAgent: agentHost,+ }, time.Now())diskTotalBytes := clampToInt64(container.MaxDisk)diskUsedBytes := clampToInt64(container.Disk)diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.goindex 6f7492e721..2cc6a53342 100644--- a/internal/monitoring/monitor_pve_guest_lxc_test.go+++ b/internal/monitoring/monitor_pve_guest_lxc_test.go@@ -1003,3 +1003,88 @@ func TestGuestFilesystemFailureGuidanceUsesObservedEvidence(t *testing.T) {})}}++// TestPerNodeLXCMemoryObservationMatchesClusterPath pins the producer parity+// for the fallback used when cluster/resources is unavailable: the per-node+// collector annotates memory through the same builder, so its containers do+// not read as freshness unknown while the same guest polled via the cluster+// listing reads current.+func TestPerNodeLXCMemoryObservationMatchesClusterPath(t *testing.T) {+ t.Setenv("PULSE_DATA_DIR", t.TempDir())++ const gib = uint64(1024 * 1024 * 1024)+ listedAt := time.Now().Add(-5 * time.Second).UTC()++ tests := []struct {+ name string+ status string+ want models.MemoryObservation+ }{+ {+ name: "running reads current from the listing time",+ status: "running",+ want: models.MemoryObservation{State: "current", Source: "cluster-resources", ObservedAt: listedAt},+ },+ {+ name: "stopped reads unavailable from powered-off",+ status: "stopped",+ want: models.MemoryObservation{State: "unavailable", Source: "powered-off"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ client := &vmMemoryTrustStubClient{+ stubPVEClient: &stubPVEClient{},+ containers: []proxmox.Container{{+ VMID: 311, Name: "ct-311", Node: "node1", Status: tt.status,+ MaxMem: 8 * gib, Mem: 3 * gib, CPUs: 2, ObservedAt: listedAt,+ }},+ }++ clusterMon := newTestPVEMonitor("test")+ defer clusterMon.alertManager.Stop()+ defer clusterMon.notificationMgr.Stop()+ clusterContainer, _, _, _, ok := clusterMon.buildContainerFromClusterResource(+ context.Background(),+ "test",+ proxmox.ClusterResource{+ Type: "lxc", Node: "node1", Name: "ct-311", Status: tt.status, VMID: 311,+ MaxMem: 8 * gib, Mem: 3 * gib, MaxCPU: 2, ObservedAt: listedAt,+ },+ client,+ map[int]bool{},+ nil,+ )+ if !ok {+ t.Fatal("expected cluster-path container to be built")+ }++ nodeMon := newTestPVEMonitor("test")+ defer nodeMon.alertManager.Stop()+ defer nodeMon.notificationMgr.Stop()+ nodeMon.pollContainersWithNodes(+ context.Background(),+ "test",+ "",+ false,+ client,+ []proxmox.Node{{Node: "node1", Status: "online"}},+ map[string]string{"node1": "online"},+ )+ containers := nodeMon.state.GetSnapshot().Containers+ if len(containers) != 1 {+ t.Fatalf("expected 1 per-node container, got %d", len(containers))+ }++ for path, got := range map[string]models.MemoryObservation{+ "cluster": clusterContainer.Memory.Observation,+ "per-node": containers[0].Memory.Observation,+ } {+ if got.State != tt.want.State || got.Source != tt.want.Source || !got.ObservedAt.Equal(tt.want.ObservedAt) {+ t.Fatalf("%s memory observation = %+v, want %+v", path, got, tt.want)+ }+ }+ })+ }+}