Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep retained guest readings out of alert decisions

pulse-triage[bot]5ce63f140022
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 3c89b76df0..5f7c630e4f 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -15,6 +15,34 @@
## Purpose
+### Retained guest observations are not alert evidence
+
+Guest memory marked `last-known`, `unavailable` or an unknown observation state
+cannot start, advance, refresh or resolve a memory incident. A VM filesystem
+read with a nonempty `diskStatusReason`, including retained `prev-*` values,
+or unavailable aggregate usage is likewise not current disk evidence. It cannot
+evaluate either aggregate or per-filesystem thresholds, replace an aggregate
+incident with filesystem incidents, or clear filesystem alerts as if a failed
+or expired inventory were authoritative removal. Metric filters do not match
+these unavailable memory/disk values, including a fabricated healthy zero.
+
+Existing incidents retain their last trusted value, occurrence, acknowledgement,
+observation time and lifecycle/delivery evidence until a current observation
+resumes evaluation. Successful current inventory still resolves removed
+filesystems. Explicit disablement, suppression and stopped-guest behavior remain
+effective independently of telemetry. Legacy unannotated memory and independent
+current PVE/Pulse-agent memory/disks keep their existing policy, even while QGA
+is deferred; current CPU and I/O continue to evaluate. Producers own freshness;
+no new age threshold, QGA request, payload, alert family or UI surface is added.
+
+`guest_observation_test.go` pins typed/pointer VM/LXC observation admission,
+delayed breaches, canonical incident/acknowledgement/ledger continuity, filters,
+fresh recovery and removal, independent metrics and deliberate disablement.
+`TestGuestBackupDeferralAlertObservationLifecycle` connects actual HTTP client,
+poll builder, retained snapshot, unified view, History and the production alert
+call. These are synthetic source proofs, not native backup/thaw, containing
+release, installed delivery or external recipient acceptance.
+
### Pulse system alerts retain their service scope in the existing UI
The `pulse-system-` identity prefix or explicit `systemAlert: true` metadata
diff --git a/internal/alerts/filter_evaluation.go b/internal/alerts/filter_evaluation.go
index 5741e176a1..d8c0954130 100644
--- a/internal/alerts/filter_evaluation.go
+++ b/internal/alerts/filter_evaluation.go
@@ -19,18 +19,20 @@ func (m *Manager) evaluateFilterCondition(guest any, condition FilterCondition)
// guestMetrics holds common metrics for filter evaluation
type guestMetrics struct {
- CPU float64 // CPU usage as percentage (0-100)
- MemUsage float64 // Memory usage percentage
- DiskUsage float64 // Disk usage percentage
- DiskRead int64 // Bytes/s
- DiskWrite int64 // Bytes/s
- NetworkIn int64 // Bytes/s
- NetworkOut int64 // Bytes/s
- IORateValidity models.IORateValidity
- Name string
- Node string
- ID string
- Status string
+ CPU float64 // CPU usage as percentage (0-100)
+ MemUsage float64 // Memory usage percentage
+ DiskUsage float64 // Disk usage percentage
+ MemoryUnavailable bool
+ DiskUnavailable bool
+ DiskRead int64 // Bytes/s
+ DiskWrite int64 // Bytes/s
+ NetworkIn int64 // Bytes/s
+ NetworkOut int64 // Bytes/s
+ IORateValidity models.IORateValidity
+ Name string
+ Node string
+ ID string
+ Status string
}
// extractGuestMetrics extracts common metrics from a VM or Container
@@ -57,8 +59,14 @@ func evaluateGuestCondition(metrics guestMetrics, condition FilterCondition) boo
case "cpu":
value = metrics.CPU
case "memory":
+ if metrics.MemoryUnavailable {
+ return false
+ }
value = metrics.MemUsage
case "disk":
+ if metrics.DiskUnavailable {
+ return false
+ }
value = metrics.DiskUsage
case "diskread":
if !validity.DiskRead {
diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go
index b185ea1d30..6bc19205af 100644
--- a/internal/alerts/guest.go
+++ b/internal/alerts/guest.go
@@ -259,8 +259,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
memoryMetric = &UnifiedResourceMetric{Percent: memUsage}
}
diskReadMetric, diskWriteMetric, networkInMetric, networkOutMetric := guestIORateMetrics(snapshot)
- diskMetric := &UnifiedResourceMetric{Percent: diskUsage}
- if len(disks) > 0 {
+ var diskMetric *UnifiedResourceMetric
+ if !snapshot.DiskUnavailable {
+ diskMetric = &UnifiedResourceMetric{Percent: diskUsage}
+ }
+ if !snapshot.DiskUnavailable && len(disks) > 0 {
seenAggregateDiskKeys := make(map[string]struct{})
for idx, disk := range disks {
if disk.Total <= 0 || disk.Usage < 0 {
@@ -295,6 +298,14 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
NetworkOut: networkOutMetric,
}, thresholds, evalOpts)
+ // Retained/failed filesystem evidence is useful to display, but neither
+ // its values nor an empty result can establish a breach, recovery or disk
+ // removal. CPU/I/O and independent current memory were evaluated above;
+ // explicit suppression/disablement and stopped-guest handling stay earlier.
+ if snapshot.DiskUnavailable {
+ return
+ }
+
if len(disks) > 0 {
seenDiskKeys := make(map[string]struct{})
seenDiskResources := make(map[string]struct{})
diff --git a/internal/alerts/guest_snapshot.go b/internal/alerts/guest_snapshot.go
index f9741f9647..9cbcc805e0 100644
--- a/internal/alerts/guest_snapshot.go
+++ b/internal/alerts/guest_snapshot.go
@@ -28,9 +28,10 @@ type guestSnapshot struct {
CPUPercent float64
MemUsage float64
- // MemoryUnavailable prevents a missing cache-aware sample from clearing a
- // real alert or starting a false one.
+ // Unavailable observations include retained display values. They cannot
+ // supply a new breach, recovery, filter match or filesystem removal.
MemoryUnavailable bool
+ DiskUnavailable bool
DiskUsage float64
DiskRead int64
DiskWrite int64
@@ -81,18 +82,20 @@ func (g guestSnapshot) resourceType() string {
func (g guestSnapshot) metrics() guestMetrics {
return guestMetrics{
- CPU: g.CPUPercent,
- MemUsage: g.MemUsage,
- DiskUsage: g.DiskUsage,
- DiskRead: g.DiskRead,
- DiskWrite: g.DiskWrite,
- NetworkIn: g.NetworkIn,
- NetworkOut: g.NetworkOut,
- IORateValidity: g.IORateValidity,
- Name: g.Name,
- Node: g.Node,
- ID: g.ID,
- Status: g.Status,
+ CPU: g.CPUPercent,
+ MemUsage: g.MemUsage,
+ DiskUsage: g.DiskUsage,
+ MemoryUnavailable: g.MemoryUnavailable,
+ DiskUnavailable: g.DiskUnavailable,
+ DiskRead: g.DiskRead,
+ DiskWrite: g.DiskWrite,
+ NetworkIn: g.NetworkIn,
+ NetworkOut: g.NetworkOut,
+ IORateValidity: g.IORateValidity,
+ Name: g.Name,
+ Node: g.Node,
+ ID: g.ID,
+ Status: g.Status,
}
}
@@ -108,7 +111,8 @@ func guestSnapshotFromVM(vm models.VM) guestSnapshot {
Lock: vm.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(vm.CPU),
MemUsage: vm.Memory.Usage,
- MemoryUnavailable: vm.Memory.UsageUnavailable,
+ MemoryUnavailable: guestMemoryUnavailableForAlerts(vm.Memory),
+ DiskUnavailable: vm.DiskStatusReason != "" || vm.Disk.Usage < 0,
DiskUsage: vm.Disk.Usage,
DiskRead: vm.DiskRead,
DiskWrite: vm.DiskWrite,
@@ -133,7 +137,8 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
Lock: container.Lock,
CPUPercent: unifiedresources.ProxmoxGuestCPUPercent(container.CPU),
MemUsage: container.Memory.Usage,
- MemoryUnavailable: container.Memory.UsageUnavailable,
+ MemoryUnavailable: guestMemoryUnavailableForAlerts(container.Memory),
+ DiskUnavailable: container.Disk.Usage < 0,
DiskUsage: container.Disk.Usage,
DiskRead: container.DiskRead,
DiskWrite: container.DiskWrite,
@@ -146,6 +151,13 @@ func guestSnapshotFromContainer(container models.Container) guestSnapshot {
}.normalizeCollections()
}
+func guestMemoryUnavailableForAlerts(memory models.Memory) bool {
+ // The producer owns freshness and source selection. Legacy values without
+ // an observation retain their existing behavior; do not infer freshness
+ // from disk/QGA state, since PVE or a linked Pulse agent can be independent.
+ return memory.UsageUnavailable || (memory.Observation.State != "" && memory.Observation.State != "current")
+}
+
func guestKindFromType(guestType string) guestKind {
switch strings.ToLower(strings.TrimSpace(guestType)) {
case "qemu", "vm":
diff --git a/internal/alerts/guest_snapshot_test.go b/internal/alerts/guest_snapshot_test.go
index 28c7e4e65f..863dcec16a 100644
--- a/internal/alerts/guest_snapshot_test.go
+++ b/internal/alerts/guest_snapshot_test.go
@@ -19,6 +19,50 @@ func TestGuestSnapshotResourceTypeUsesCanonicalSystemContainer(t *testing.T) {
}
}
+func TestGuestSnapshotObservationAvailability(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ memory models.Memory
+ disk models.Disk
+ reason string
+ wantMemory bool
+ wantDisk bool
+ }{
+ {name: "legacy measured zero", memory: models.Memory{}, disk: models.Disk{}},
+ {name: "current independent read", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "current", Source: "agent"}}, disk: models.Disk{Usage: 98}},
+ {name: "retained memory only", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "last-known"}}, disk: models.Disk{Usage: 98}, wantMemory: true},
+ {name: "unavailable numeric placeholder", memory: models.Memory{Usage: 0, Observation: models.MemoryObservation{State: "unavailable"}}, wantMemory: true},
+ {name: "unknown observation version", memory: models.Memory{Usage: 96, Observation: models.MemoryObservation{State: "future-version"}}, wantMemory: true},
+ {name: "explicit unavailable overrides current", memory: models.Memory{Usage: 96, UsageUnavailable: true, Observation: models.MemoryObservation{State: "current"}}, wantMemory: true},
+ {name: "retained filesystem only", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: 98}, reason: "prev-vm-locked", wantDisk: true},
+ {name: "failed empty filesystem inventory", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, reason: "agent-error", wantDisk: true},
+ {name: "legacy unavailable aggregate", memory: models.Memory{Usage: 96}, disk: models.Disk{Usage: -1}, wantDisk: true},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ // A deferred QGA session does not by itself make independent
+ // selected memory or Pulse-agent filesystems unavailable.
+ vm := models.VM{Memory: test.memory, Disk: test.disk, DiskStatusReason: test.reason, GuestAgentStatus: "deferred"}
+ for _, guest := range []any{vm, &vm} {
+ snapshot, ok := extractGuestSnapshot(guest)
+ if !ok || snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != test.wantDisk {
+ t.Fatalf("observation flags lost: %+v", snapshot)
+ }
+ metrics := snapshot.metrics()
+ if metrics.MemoryUnavailable != test.wantMemory || metrics.DiskUnavailable != test.wantDisk || metrics.MemUsage != test.memory.Usage || metrics.DiskUsage != test.disk.Usage {
+ t.Fatal("filter projection lost availability or rewrote retained values")
+ }
+ }
+ ct := models.Container{Memory: test.memory, Disk: test.disk}
+ for _, guest := range []any{ct, &ct} {
+ snapshot, _ := extractGuestSnapshot(guest)
+ if snapshot.MemoryUnavailable != test.wantMemory || snapshot.DiskUnavailable != (test.disk.Usage < 0) {
+ t.Fatal("LXC observation availability changed with an unrelated VM disk failure")
+ }
+ }
+ })
+ }
+}
+
func TestGuestSnapshotUsesCanonicalProxmoxCPUPercent(t *testing.T) {
for _, tc := range []struct {
name string