Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep retained guest readings out of alert decisions
pulse-triage[bot]5ce63f140022
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex 3c89b76df0..5f7c630e4f 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -15,6 +15,34 @@## Purpose+### Retained guest observations are not alert evidence++Guest memory marked `last-known`, `unavailable` or an unknown observation state+cannot start, advance, refresh or resolve a memory incident. A VM filesystem+read with a nonempty `diskStatusReason`, including retained `prev-*` values,+or unavailable aggregate usage is likewise not current disk evidence. It cannot+evaluate either aggregate or per-filesystem thresholds, replace an aggregate+incident with filesystem incidents, or clear filesystem alerts as if a failed+or expired inventory were authoritative removal. Metric filters do not match+these unavailable memory/disk values, including a fabricated healthy zero.++Existing incidents retain their last trusted value, occurrence, acknowledgement,+observation time and lifecycle/delivery evidence until a current observation+resumes evaluation. Successful current inventory still resolves removed+filesystems. Explicit disablement, suppression and stopped-guest behavior remain+effective independently of telemetry. Legacy unannotated memory and independent+current PVE/Pulse-agent memory/disks keep their existing policy, even while QGA+is deferred; current CPU and I/O continue to evaluate. Producers own freshness;+no new age threshold, QGA request, payload, alert family or UI surface is added.++`guest_observation_test.go` pins typed/pointer VM/LXC observation admission,+delayed breaches, canonical incident/acknowledgement/ledger continuity, filters,+fresh recovery and removal, independent metrics and deliberate disablement.+`TestGuestBackupDeferralAlertObservationLifecycle` connects actual HTTP client,+poll builder, retained snapshot, unified view, History and the production alert+call. These are synthetic source proofs, not native backup/thaw, containing+release, installed delivery or external recipient acceptance.+### Pulse system alerts retain their service scope in the existing UIThe `pulse-system-` identity prefix or explicit `systemAlert: true` metadatadiff --git a/internal/alerts/filter_evaluation.go b/internal/alerts/filter_evaluation.goindex 5741e176a1..d8c0954130 100644--- a/internal/alerts/filter_evaluation.go+++ b/internal/alerts/filter_evaluation.go@@ -19,18 +19,20 @@ func (m *Manager) evaluateFilterCondition(guest any, condition FilterCondition)// guestMetrics holds common metrics for filter evaluationtype guestMetrics struct {- CPU float64 // CPU usage as percentage (0-100)- MemUsage float64 // Memory usage percentage- DiskUsage float64 // Disk usage percentage- DiskRead int64 // Bytes/s- DiskWrite int64 // Bytes/s- NetworkIn int64 // Bytes/s- NetworkOut int64 // Bytes/s- IORateValidity models.IORateValidity- Name string- Node string- ID string- Status string+ CPU float64 // CPU usage as percentage (0-100)+ MemUsage float64 // Memory usage percentage+ DiskUsage float64 // Disk usage percentage+ MemoryUnavailable bool+ DiskUnavailable bool+ DiskRead int64 // Bytes/s+ DiskWrite int64 // Bytes/s+ NetworkIn int64 // Bytes/s+ NetworkOut int64 // Bytes/s+ IORateValidity models.IORateValidity+ Name string+ Node string+ ID string+ Status string}// extractGuestMetrics extracts common metrics from a VM or Container@@ -57,8 +59,14 @@ func evaluateGuestCondition(metrics guestMetrics, condition FilterCondition) boocase "cpu":value = metrics.CPUcase "memory":+ if metrics.MemoryUnavailable {+ return false+ }value = metrics.MemUsagecase "disk":+ if metrics.DiskUnavailable {+ return false+ }value = metrics.DiskUsagecase "diskread":if !validity.DiskRead {diff --git a/internal/alerts/guest.go b/internal/alerts/guest.goindex b185ea1d30..6bc19205af 100644--- a/internal/alerts/guest.go+++ b/internal/alerts/guest.go@@ -259,8 +259,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {memoryMetric = &UnifiedResourceMetric{Percent: memUsage}}diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)- diskMetric := &UnifiedResourceMetric{Percent: diskUsage}- if len(disks) > 0 {+ var diskMetric *UnifiedResourceMetric+ if !snapshot.DiskUnavailable {+ diskMetric = &UnifiedResourceMetric{Percent: diskUsage}+ }+ if !snapshot.DiskUnavailable && len(disks) > 0 {seenAggregateDiskKeys := make(map[string]struct{})for idx, disk := range disks {if disk.Total <= 0 || disk.Usage < 0 {@@ -295,6 +298,14 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {NetworkOut: networkOutMetric,}, thresholds, evalOpts)+ // Retained/failed filesystem evidence is useful to display, but neither+ // its values nor an empty result can establish a breach, recovery or disk+ // removal. CPU/I/O and independent current memory were evaluated above;+ // explicit suppression/disablement and stopped-guest handling stay earlier.+ if snapshot.DiskUnavailable {+ return+ }+if len(disks) > 0 {seenDiskKeys := make(map[string]struct{})seenDiskResources := make(map[string]struct{})diff --git a/internal/alerts/guest_snapshot.go b/internal/alerts/guest_snapshot.goindex f9741f9647..9cbcc805e0 100644--- a/internal/alerts/guest_snapshot.go+++ b/internal/alerts/guest_snapshot.go@@ -28,9 +28,10 @@ type guestSnapshot struct {CPUPercent float64MemUsage float64- // MemoryUnavailable prevents a missing cache-aware sample from clearing a- // real alert or starting a false one.+ // Unavailable observations include retained display values. They cannot+ // supply a new breach, recovery, filter match or filesystem removal.MemoryUnavailable bool+ DiskUnavailable boolDiskUsage float64DiskRead int64DiskWrite int64@@ -81,18 +82,20 @@ func (g guestSnapshot) resourceType() string {func (g guestSnapshot) metrics() guestMetrics {return guestMetrics{- CPU: g.CPUPercent,- MemUsage: g.MemUsage,- DiskUsage: g.DiskUsage,- DiskRead: g.DiskRead,- DiskWrite: g.DiskWrite,- NetworkIn: g.NetworkIn,- NetworkOut: g.NetworkOut,- IORateValidity: g.IORateValidity,- Name: g.Name,- Node: g.Node,- ID: g.ID,- Status: g.Status,+ CPU: g.CPUPercent,+ MemUsage: g.MemUsage,+ DiskUsage: g.DiskUsage,+ MemoryUnavailable: g.MemoryUnavailable,+ DiskUnavailable: g.DiskUnavailable,+ DiskRead: g.DiskRead,+ DiskWrite: g.DiskWrite,+ NetworkIn: g.NetworkIn,+ NetworkOut: g.NetworkOut,+ IORateValidity: g.IORateValidity,+ Name: g.Name,+ Node: g.Node,+ ID: g.ID,+ Status: g.Status,}}@@ -108,7 +111,8 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot {Lock: vm.Lock,CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU),MemUsage: vm.Memory.Usage,- MemoryUnavailable: vm.Memory.UsageUnavailable,+ MemoryUnavailable: guestMemoryUnavailableForAlerts(vm.Memory),+ DiskUnavailable: vm.DiskStatusReason != "" || vm.Disk.Usage < 0,DiskUsage: vm.Disk.Usage,DiskRead: vm.DiskRead,DiskWrite: vm.DiskWrite,@@ -133,7 +137,8 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {Lock: container.Lock,CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU),MemUsage: container.Memory.Usage,- MemoryUnavailable: container.Memory.UsageUnavailable,+ MemoryUnavailable: guestMemoryUnavailableForAlerts(container.Memory),+ DiskUnavailable: container.Disk.Usage < 0,DiskUsage: container.Disk.Usage,DiskRead: container.DiskRead,DiskWrite: container.DiskWrite,@@ -146,6 +151,13 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {}.normalizeCollections()}+func guestMemoryUnavailableForAlerts(memory models.Memory) bool {+ // The producer owns freshness and source selection. Legacy values without+ // an observation retain their existing behavior; do not infer freshness+ // from disk/QGA state, since PVE or a linked Pulse agent can be independent.+ return memory.UsageUnavailable || (memory.Observation.State != "" && memory.Observation.State != "current")+}+func guestKindFromType(guestType string) guestKind {switch strings.ToLower(strings.TrimSpace(guestType)) {case "qemu", "vm":diff --git a/internal/alerts/guest_snapshot_test.go b/internal/alerts/guest_snapshot_test.goindex 28c7e4e65f..863dcec16a 100644--- a/internal/alerts/guest_snapshot_test.go+++ b/internal/alerts/guest_snapshot_test.go@@ -19,6 +19,50 @@ func TestGuestSnapshotResourceTypeUsesCanonicalSystemContainer(t *testing.T) {}}+func TestGuestSnapshotObservationAvailability(t *testing.T) {+ for _, test := range []struct {+ name string+ memory models.Memory+ disk models.Disk+ reason string+ wantMemory bool+ wantDisk bool+ }{+ {name: "legacy measured zero", memory: models.Memory{}, disk: models.Disk{}},+ {name: "current independent read", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "current", Source: "agent"}}, disk: models.Disk{Usage: 98}},+ {name: "retained memory only", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "last-known"}}, disk: models.Disk{Usage: 98}, wantMemory: true},+ {name: "unavailable numeric placeholder", memory: models.Memory{Usage: 0, Observation: models.MemoryObservation{State: "unavailable"}}, wantMemory: true},+ {name: "unknown observation version", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "future-version"}}, wantMemory: true},+ {name: "explicit unavailable overrides current", memory: models.Memory{Usage: 96, UsageUnavailable: true, Observation: models.MemoryObservation{State: "current"}}, wantMemory: true},+ {name: "retained filesystem only", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: 98}, reason: "prev-vm-locked", wantDisk: true},+ {name: "failed empty filesystem inventory", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, reason: "agent-error", wantDisk: true},+ {name: "legacy unavailable aggregate", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, wantDisk: true},+ } {+ t.Run(test.name, func(t *testing.T) {+ // A deferred QGA session does not by itself make independent+ // selected memory or Pulse-agent filesystems unavailable.+ vm := models.VM{Memory: test.memory, Disk: test.disk, DiskStatusReason: test.reason, GuestAgentStatus: "deferred"}+ for _, guest := range []any{vm, &vm} {+ snapshot, ok := extractGuestSnapshot(guest)+ if !ok || snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != test.wantDisk {+ t.Fatalf("observation flags lost: %+v", snapshot)+ }+ metrics := snapshot.metrics()+ if metrics.MemoryUnavailable != test.wantMemory || metrics.DiskUnavailable != test.wantDisk || metrics.MemUsage != test.memory.Usage || metrics.DiskUsage != test.disk.Usage {+ t.Fatal("filter projection lost availability or rewrote retained values")+ }+ }+ ct := models.Container{Memory: test.memory, Disk: test.disk}+ for _, guest := range []any{ct, &ct} {+ snapshot, _ := extractGuestSnapshot(guest)+ if snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != (test.disk.Usage < 0) {+ t.Fatal("LXC observation availability changed with an unrelated VM disk failure")+ }+ }+ })+ }+}+func TestGuestSnapshotUsesCanonicalProxmoxCPUPercent(t *testing.T) {for _, tc := range []struct {name string