Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep unavailable agent filesystems from resolving disk alerts

pulse-triage[bot]6e35e3252b20
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index b20715e0d2..5131cd226e 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -15,6 +15,27 @@
## Purpose
+### Unavailable agent filesystems are not recovered disk alerts
+
+Host filesystem thresholds use the canonical disk-observation admission: a
+finite percentage in range, positive capacity and non-negative used space no
+larger than capacity. A listed unavailable reading or an entirely empty report
+holds the existing occurrence, acknowledgement and last trusted evidence. It
+cannot raise a new alert, resolve one, or complete activation, explicit intent
+grace or recovery timing across the gap. A measured zero remains recovery.
+
+An established linked-node summary identity keeps ownership only while its
+held disk alert remains open; interrupting a pending run returns ownership to
+the node. Current host, inherited and filesystem policy disablement, explicit
+agent removal and missing rows in a non-empty replacement inventory retain
+their ordinary cleanup semantics. No collection, guest operation, schema,
+historical metric, notification transport or provider lease changes.
+
+`host_filesystem_observation_test.go` exercises the real manager, lifecycle
+ledger, exact-host isolation, timing, policy, removal and linked-node controls.
+Its optional exported manager payloads feed the real Alerts API adapter and
+Overview surface in desktop/narrow browser proof, not a native recovery claim.
+
### Listed backups need an observed success date to resolve age warnings
The backup-age evaluator retains an existing subject's occurrence, severity,
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 652172be82..043aa51a6a 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -596,7 +596,14 @@ func (m *Manager) CheckHost(host models.Host) {
continue
}
- evaluated := m.checkMetricWithCanonicalSpec(spec, diskName, nodeName, instanceName, "agent-disk", disk.Usage, effectiveDiskThreshold, &metricOptions{Metadata: diskMetadata})
+ evaluated := false
+ if _, observed := unifiedresources.DiskUsagePercent(disk); !observed && !spec.Disabled {
+ // An unavailable listed filesystem neither recovers an incident
+ // nor completes activation/recovery timing across the gap.
+ m.interruptMetricRun(spec)
+ } else {
+ evaluated = m.checkMetricWithCanonicalSpec(spec, diskName, nodeName, instanceName, "agent-disk", disk.Usage, effectiveDiskThreshold, &metricOptions{Metadata: diskMetadata})
+ }
if diskResourceID == summaryDiskResourceID {
summaryDiskLive = liveHysteresisThreshold(effectiveDiskThreshold)
evaluatesSummaryDisk = evaluated
@@ -622,7 +629,22 @@ func (m *Manager) CheckHost(host models.Host) {
}
}
- m.cleanupHostDiskAlerts(host, seenDisks)
+ m.reconcileHostFilesystemObservationGaps(host, seenDisks)
+ if len(host.Disks) > 0 {
+ // A non-empty replacement inventory retains its existing removal
+ // semantics. An empty report cannot establish every disk's removal.
+ m.cleanupHostDiskAlerts(host, seenDisks)
+ } else if host.LinkedNodeID != "" {
+ // Retain only the already-established summary identity, not a new
+ // reading, while its open alert still owns the linked node's disk.
+ m.mu.RLock()
+ previous := m.hostAgentNodeLinks[host.ID]
+ m.mu.RUnlock()
+ if previous.nodeID == host.LinkedNodeID {
+ summaryDiskResourceID = previous.summaryDiskResourceID
+ summaryDiskLive = previous.disk
+ }
+ }
// The linked node releases the usage metrics this agent covers and keeps
// the rest, so deduplication never leaves the machine unmonitored. A metric
diff --git a/internal/alerts/host_filesystem_observation.go b/internal/alerts/host_filesystem_observation.go
new file mode 100644
index 0000000000..3c5d80bf3f
--- /dev/null
+++ b/internal/alerts/host_filesystem_observation.go
@@ -0,0 +1,55 @@
+package alerts
+
+import (
+ "strings"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+// An empty filesystem inventory is not evidence that every filesystem left.
+// Interrupt pending/recovery timing even for rows no longer in the report,
+// preserving open incidents unless current policy explicitly switches them off.
+// A non-empty inventory still retires absent rows through cleanupHostDiskAlerts.
+func (m *Manager) reconcileHostFilesystemObservationGaps(host models.Host, seen map[string]struct{}) {
+ prefix := hostResourceID(host.ID) + "/disk:"
+ m.mu.Lock()
+ resources := make(map[string]struct{})
+ for _, state := range m.mirrorStatesNoLock() {
+ for _, resourceID := range state.PendingResourceIDs() {
+ resources[resourceID] = struct{}{}
+ }
+ }
+ for _, pending := range m.intentPending {
+ if pending.Signal == MetricAlertIntentSignal("disk") {
+ resources[pending.ResourceID] = struct{}{}
+ }
+ }
+ thresholds := m.resolveHostThresholdsNoLock(host.ID, host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID)
+ override, exists := m.hostThresholdOverrideNoLock(host.ID, host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID)
+ for storageKey, alert := range m.activeAlerts {
+ if alert == nil || alert.Type != "disk" || !strings.HasPrefix(alert.ResourceID, prefix) {
+ continue
+ }
+ resources[alert.ResourceID] = struct{}{}
+ if len(host.Disks) == 0 {
+ threshold, disabled := m.hostDiskUsageThresholdNoLock(alert.ResourceID, metadataStringValue(alert.Metadata, "device"), thresholds.Disk, exists && override.Disk != nil)
+ if disabled || !liveHysteresisThreshold(threshold) {
+ m.clearAlertNoLock(storageKey)
+ }
+ }
+ }
+ intentChanged := false
+ for resourceID := range resources {
+ if !strings.HasPrefix(resourceID, prefix) {
+ continue
+ }
+ if _, observed := seen[resourceID]; observed {
+ continue
+ }
+ intentChanged = m.interruptMetricRunNoLock(resourceID, canonicalMetricSpecID(resourceID, "disk"), canonicalMetricStateID(resourceID, "disk")) || intentChanged
+ }
+ m.mu.Unlock()
+ if intentChanged {
+ m.saveActiveAlertsAsync("host filesystem observation gap")
+ }
+}
diff --git a/internal/alerts/host_filesystem_observation_test.go b/internal/alerts/host_filesystem_observation_test.go
new file mode 100644
index 0000000000..43b05fbe63
--- /dev/null
+++ b/internal/alerts/host_filesystem_observation_test.go
@@ -0,0 +1,284 @@
+package alerts
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "math"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+func hostFilesystemManager(t *testing.T, grace bool) (*Manager, *atomic.Int64) {
+ t.Helper()
+ m, elapsed := continuityManager(t, grace)
+ m.mu.Lock()
+ m.config.AgentDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 70}}
+ m.config.NodeDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 70}}
+ m.config.DiskFillByType = nil
+ if !grace {
+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}}
+ }
+ m.mu.Unlock()
+ return m, elapsed
+}
+
+func hostFilesystemSample(id string, usage float64) models.Host {
+ return models.Host{ID: id, Hostname: "storage-host", DisplayName: "Storage host", Status: "online",
+ Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sda1", Total: 1000, Used: int64(usage * 10), Usage: usage}}}
+}
+
+func hostFilesystemGap(host models.Host, gap string) models.Host {
+ disk := host.Disks[0]
+ switch gap {
+ case "empty":
+ host.Disks = nil
+ return host
+ case "unknown-zero":
+ disk.Total, disk.Used, disk.Usage = 0, 0, 0
+ case "unknown-positive":
+ disk.Total = 0
+ case "negative-capacity":
+ disk.Total = -1
+ case "negative-used":
+ disk.Used = -1
+ case "over-capacity":
+ disk.Used = disk.Total + 1
+ case "negative":
+ disk.Usage = -1
+ case "over-100":
+ disk.Usage = 101
+ case "NaN":
+ disk.Usage = math.NaN()
+ case "+Inf":
+ disk.Usage = math.Inf(1)
+ }
+ host.Disks = []models.Disk{disk}
+ return host
+}
+
+func hostFilesystemID(host models.Host) string {
+ id, _ := hostDiskResourceID(host, host.Disks[0])
+ return id
+}
+
+func hostFilesystemJSON(t *testing.T, value any) []byte {
+ t.Helper()
+ data, err := json.Marshal(value)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return data
+}
+
+func TestHostFilesystemUnknownPreservesOccurrence(t *testing.T) {
+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-capacity", "negative-used", "over-capacity", "negative", "over-100", "NaN", "+Inf"} {
+ t.Run(gap, func(t *testing.T) {
+ m, elapsed := hostFilesystemManager(t, false)
+ host := hostFilesystemSample("host-1", 95)
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("trusted full disk must open an alert")
+ }
+ id := m.GetActiveAlerts()[0].ID
+ if err := m.AcknowledgeAlert(id, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ before := hostFilesystemJSON(t, m.GetActiveAlerts())
+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))
+ elapsed.Store(int64(2 * time.Minute))
+ for range 3 {
+ m.CheckHost(hostFilesystemGap(host, gap))
+ }
+ if !bytes.Equal(before, hostFilesystemJSON(t, m.GetActiveAlerts())) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {
+ t.Fatal("unavailable filesystem changed the occurrence, trusted evidence, acknowledgement or lifecycle ledger")
+ }
+ // Observed zero, not a missing reading, must still recover the original.
+ host.Disks[0].Used, host.Disks[0].Usage = 0, 0
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{AlertID: id, Types: []string{eventlog.TypeResolved}})) != 1 {
+ t.Fatal("an observed empty filesystem must resolve the original exactly once")
+ }
+ })
+ }
+}
+
+func TestHostFilesystemGapRestartsTiming(t *testing.T) {
+ for _, grace := range []bool{false, true} {
+ for _, recovery := range []bool{false, true} {
+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-used", "over-capacity"} {
+ t.Run(fmt.Sprintf("intent=%v/recovery=%v/%s", grace, recovery, gap), func(t *testing.T) {
+ m, elapsed := hostFilesystemManager(t, grace)
+ m.mu.Lock()
+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}}
+ m.mu.Unlock()
+ host := hostFilesystemSample("host-1", 85)
+ id := hostFilesystemID(host)
+ if recovery {
+ host.Disks[0].Used, host.Disks[0].Usage = 950, 95
+ m.CheckHost(host)
+ elapsed.Store(int64(time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("critical control must fire")
+ }
+ host.Disks[0].Used, host.Disks[0].Usage = 100, 10
+ elapsed.Store(int64(70 * time.Second))
+ }
+ m.CheckHost(host)
+ incident, exists := continuityIncident(m, id, "disk")
+ if !exists || (!recovery && incident.State != reducer.StatePending) || (recovery && incident.RecoverySince.IsZero()) {
+ t.Fatal("control must start the real timed run")
+ }
+ // A similarly prefixed host must not be interrupted with this host.
+ other := hostFilesystemSample("host-10", 85)
+ m.CheckHost(other)
+ elapsed.Store(int64(3 * time.Minute))
+ m.CheckHost(hostFilesystemGap(host, gap))
+ incident, exists = continuityIncident(m, id, "disk")
+ if (!recovery && exists) || (recovery && (!exists || !incident.RecoverySince.IsZero())) {
+ t.Fatal("observation gap retained timed evidence")
+ }
+ if _, exists := continuityIncident(m, hostFilesystemID(other), "disk"); !exists {
+ t.Fatal("gap crossed the exact host boundary")
+ }
+ m.CheckHost(host)
+ want := 0
+ if recovery {
+ want = 1
+ }
+ if len(m.GetActiveAlerts()) != want {
+ t.Fatal("first fresh sample completed the pre-gap run")
+ }
+ elapsed.Store(int64(3*time.Minute + 59*time.Second))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != want {
+ t.Fatal("fresh run completed early")
+ }
+ elapsed.Store(int64(4 * time.Minute))
+ m.CheckHost(host)
+ if recovery {
+ want = 0
+ } else {
+ want = 1
+ }
+ if len(m.GetActiveAlerts()) != want {
+ t.Fatal("full continuous fresh run did not complete")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("interruption diverged from the shadow reducer")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestHostFilesystemGapHonoursPolicyAndRemoval(t *testing.T) {
+ for _, gap := range []string{"empty", "unknown-zero"} {
+ for _, policy := range []string{"host-off", "disk-off", "disk-disabled", "inherited-off", "agent-removed", "filesystem-removed"} {
+ t.Run(gap+"/"+policy, func(t *testing.T) {
+ m, _ := hostFilesystemManager(t, false)
+ host := hostFilesystemSample("host-1", 95)
+ host.LinkedNodeID = "node-1"
+ id := hostFilesystemID(host)
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("control must fire")
+ }
+ host = hostFilesystemGap(host, gap)
+ // Change config directly: this proves CheckHost, not config-save cleanup.
+ m.mu.Lock()
+ switch policy {
+ case "host-off":
+ m.config.AgentDefaults.Disk = nil
+ case "disk-off":
+ m.config.Overrides[id] = ThresholdConfig{Disk: &HysteresisThreshold{}}
+ case "disk-disabled":
+ m.config.Overrides[id] = ThresholdConfig{Disabled: true}
+ case "inherited-off":
+ m.config.Overrides[host.LinkedNodeID] = ThresholdConfig{Disk: &HysteresisThreshold{}}
+ }
+ m.mu.Unlock()
+ switch policy {
+ case "agent-removed":
+ m.HandleHostRemoved(host)
+ case "filesystem-removed":
+ host.Disks = hostFilesystemSample(host.ID, 0).Disks
+ host.Disks[0].Mountpoint = "/other"
+ m.CheckHost(host)
+ default:
+ m.CheckHost(host)
+ }
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("unknown telemetry overrode explicit policy or authoritative removal")
+ }
+ })
+ }
+ }
+}
+
+func TestHostFilesystemGapKeepsNodeOwnershipOnlyForHeldAlert(t *testing.T) {
+ for _, gap := range []string{"empty", "unknown-zero"} {
+ for _, firing := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/firing=%v", gap, firing), func(t *testing.T) {
+ m, _ := hostFilesystemManager(t, !firing)
+ host := hostFilesystemSample("host-1", 85)
+ host.LinkedNodeID = "node-1"
+ if firing {
+ host.Disks[0].Used, host.Disks[0].Usage = 950, 95
+ }
+ m.CheckHost(host)
+ m.CheckHost(hostFilesystemGap(host, gap))
+ m.mu.RLock()
+ owner := m.hostAgentNodeLinks[host.ID]
+ m.mu.RUnlock()
+ if owner.disk != firing {
+ t.Fatalf("gap disk ownership=%v, want %v", owner.disk, firing)
+ }
+ m.CheckNode(models.Node{ID: "node-1", Name: "storage-host", Status: "online", Disk: models.Disk{Total: 1000, Used: 950, Usage: 95}})
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("node handover left duplicate alerts or no alert")
+ }
+ })
+ }
+ }
+}
+
+// Real manager payloads are optionally exported for the production UI consumer.
+func TestHostFilesystemBrowserProducer(t *testing.T) {
+ m, elapsed := hostFilesystemManager(t, false)
+ host := hostFilesystemSample("host-browser", 95)
+ frames := map[string]json.RawMessage{}
+ for _, phase := range []string{"breach", "unknown-zero", "empty", "recovered"} {
+ sample := host
+ if phase == "recovered" {
+ sample.Disks = hostFilesystemSample(host.ID, 0).Disks
+ } else if phase != "breach" {
+ sample = hostFilesystemGap(host, phase)
+ }
+ elapsed.Add(int64(time.Second))
+ m.CheckHost(sample)
+ active := m.GetActiveAlerts()
+ want := 1
+ if phase == "recovered" {
+ want = 0
+ }
+ if len(active) != want {
+ t.Fatalf("phase %s: %d active, want %d", phase, len(active), want)
+ }
+ frames[phase] = hostFilesystemJSON(t, active)
+ }
+ if !bytes.Equal(frames["breach"], frames["unknown-zero"]) || !bytes.Equal(frames["breach"], frames["empty"]) {
+ t.Fatal("unknown telemetry changed the exported occurrence")
+ }
+ if os.Getenv("PULSE_HOST_FILESYSTEM_FRAMES") == "1" {
+ fmt.Printf("HOST_FILESYSTEM_FRAMES=%s\n", hostFilesystemJSON(t, frames))
+ }
+}
diff --git a/internal/unifiedresources/host_disk_usage.go b/internal/unifiedresources/host_disk_usage.go
index 3bfe93ba4f..3696a5458b 100644
--- a/internal/unifiedresources/host_disk_usage.go
+++ b/internal/unifiedresources/host_disk_usage.go
@@ -5,10 +5,19 @@ import "github.com/rcourtman/pulse-go-rewrite/internal/models"
// HostDiskUsagePercent uses the same first-filesystem admission as the
// canonical host metric. Missing or unavailable inventory is not measured zero.
func HostDiskUsagePercent(disks []models.Disk) (float64, bool) {
- if len(disks) == 0 || !hasObservedDiskUsage(disks[0]) {
+ if len(disks) == 0 {
return 0, false
}
- return disks[0].Usage, true
+ return DiskUsagePercent(disks[0])
+}
+
+// DiskUsagePercent applies canonical observation admission to one filesystem.
+// A listed filesystem can still have an unavailable or contradictory reading.
+func DiskUsagePercent(disk models.Disk) (float64, bool) {
+ if !hasObservedDiskUsage(disk) {
+ return 0, false
+ }
+ return disk.Usage, true
}
// DiskUsagePercent preserves canonical metric presence, including provider
diff --git a/internal/unifiedresources/host_disk_usage_test.go b/internal/unifiedresources/host_disk_usage_test.go
new file mode 100644
index 0000000000..5245aad71e
--- /dev/null
+++ b/internal/unifiedresources/host_disk_usage_test.go
@@ -0,0 +1,39 @@
+package unifiedresources
+
+import (
+ "math"
+ "testing"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+func TestDiskUsagePercentSharesCanonicalAdmission(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ disk models.Disk
+ observed bool
+ }{
+ {"observed zero", models.Disk{Total: 1000}, true},
+ {"observed fraction", models.Disk{Total: 1000, Used: 1, Usage: 0.1}, true},
+ {"observed full", models.Disk{Total: 1000, Used: 1000, Usage: 100}, true},
+ {"unavailable", models.Disk{}, false},
+ {"capacity absent", models.Disk{Usage: 95}, false},
+ {"used negative", models.Disk{Total: 1000, Used: -1}, false},
+ {"used over capacity", models.Disk{Total: 1000, Used: 1001}, false},
+ {"percentage negative", models.Disk{Total: 1000, Usage: -1}, false},
+ {"percentage over 100", models.Disk{Total: 1000, Usage: 101}, false},
+ {"NaN", models.Disk{Total: 1000, Usage: math.NaN()}, false},
+ {"infinite", models.Disk{Total: 1000, Usage: math.Inf(1)}, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ value, observed := DiskUsagePercent(tc.disk)
+ if observed != tc.observed || (observed && value != tc.disk.Usage) || (!observed && value != 0) {
+ t.Fatalf("DiskUsagePercent = %v, %v; want observed=%v", value, observed, tc.observed)
+ }
+ summary, summaryObserved := HostDiskUsagePercent([]models.Disk{tc.disk})
+ if value != summary || observed != summaryObserved {
+ t.Fatal("summary and filesystem admission diverged")
+ }
+ })
+ }
+}