Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep unavailable agent filesystems from resolving disk alerts
pulse-triage[bot]6e35e3252b20
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex b20715e0d2..5131cd226e 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -15,6 +15,27 @@## Purpose+### Unavailable agent filesystems are not recovered disk alerts++Host filesystem thresholds use the canonical disk-observation admission: a+finite percentage in range, positive capacity and non-negative used space no+larger than capacity. A listed unavailable reading or an entirely empty report+holds the existing occurrence, acknowledgement and last trusted evidence. It+cannot raise a new alert, resolve one, or complete activation, explicit intent+grace or recovery timing across the gap. A measured zero remains recovery.++An established linked-node summary identity keeps ownership only while its+held disk alert remains open; interrupting a pending run returns ownership to+the node. Current host, inherited and filesystem policy disablement, explicit+agent removal and missing rows in a non-empty replacement inventory retain+their ordinary cleanup semantics. No collection, guest operation, schema,+historical metric, notification transport or provider lease changes.++`host_filesystem_observation_test.go` exercises the real manager, lifecycle+ledger, exact-host isolation, timing, policy, removal and linked-node controls.+Its optional exported manager payloads feed the real Alerts API adapter and+Overview surface in desktop/narrow browser proof, not a native recovery claim.+### Listed backups need an observed success date to resolve age warningsThe backup-age evaluator retains an existing subject's occurrence, severity,diff --git a/internal/alerts/host.go b/internal/alerts/host.goindex 652172be82..043aa51a6a 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -596,7 +596,14 @@ func (m *Manager) CheckHost(host models.Host) {continue}- evaluated := m.checkMetricWithCanonicalSpec(spec, diskName, nodeName, instanceName, "agent-disk", disk.Usage, effectiveDiskThreshold, &metricOptions{Metadata: diskMetadata})+ evaluated := false+ if _, observed := unifiedresources.DiskUsagePercent(disk); !observed && !spec.Disabled {+ // An unavailable listed filesystem neither recovers an incident+ // nor completes activation/recovery timing across the gap.+ m.interruptMetricRun(spec)+ } else {+ evaluated = m.checkMetricWithCanonicalSpec(spec, diskName, nodeName, instanceName, "agent-disk", disk.Usage, effectiveDiskThreshold, &metricOptions{Metadata: diskMetadata})+ }if diskResourceID == summaryDiskResourceID {summaryDiskLive = liveHysteresisThreshold(effectiveDiskThreshold)evaluatesSummaryDisk = evaluated@@ -622,7 +629,22 @@ func (m *Manager) CheckHost(host models.Host) {}}- m.cleanupHostDiskAlerts(host, seenDisks)+ m.reconcileHostFilesystemObservationGaps(host, seenDisks)+ if len(host.Disks) > 0 {+ // A non-empty replacement inventory retains its existing removal+ // semantics. An empty report cannot establish every disk's removal.+ m.cleanupHostDiskAlerts(host, seenDisks)+ } else if host.LinkedNodeID != "" {+ // Retain only the already-established summary identity, not a new+ // reading, while its open alert still owns the linked node's disk.+ m.mu.RLock()+ previous := m.hostAgentNodeLinks[host.ID]+ m.mu.RUnlock()+ if previous.nodeID == host.LinkedNodeID {+ summaryDiskResourceID = previous.summaryDiskResourceID+ summaryDiskLive = previous.disk+ }+ }// The linked node releases the usage metrics this agent covers and keeps// the rest, so deduplication never leaves the machine unmonitored. A metricdiff --git a/internal/alerts/host_filesystem_observation.go b/internal/alerts/host_filesystem_observation.gonew file mode 100644index 0000000000..3c5d80bf3f--- /dev/null+++ b/internal/alerts/host_filesystem_observation.go@@ -0,0 +1,55 @@+package alerts++import (+ "strings"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++// An empty filesystem inventory is not evidence that every filesystem left.+// Interrupt pending/recovery timing even for rows no longer in the report,+// preserving open incidents unless current policy explicitly switches them off.+// A non-empty inventory still retires absent rows through cleanupHostDiskAlerts.+func (m *Manager) reconcileHostFilesystemObservationGaps(host models.Host, seen map[string]struct{}) {+ prefix := hostResourceID(host.ID) + "/disk:"+ m.mu.Lock()+ resources := make(map[string]struct{})+ for _, state := range m.mirrorStatesNoLock() {+ for _, resourceID := range state.PendingResourceIDs() {+ resources[resourceID] = struct{}{}+ }+ }+ for _, pending := range m.intentPending {+ if pending.Signal == MetricAlertIntentSignal("disk") {+ resources[pending.ResourceID] = struct{}{}+ }+ }+ thresholds := m.resolveHostThresholdsNoLock(host.ID, host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID)+ override, exists := m.hostThresholdOverrideNoLock(host.ID, host.LinkedNodeID, host.LinkedVMID, host.LinkedContainerID)+ for storageKey, alert := range m.activeAlerts {+ if alert == nil || alert.Type != "disk" || !strings.HasPrefix(alert.ResourceID, prefix) {+ continue+ }+ resources[alert.ResourceID] = struct{}{}+ if len(host.Disks) == 0 {+ threshold, disabled := m.hostDiskUsageThresholdNoLock(alert.ResourceID, metadataStringValue(alert.Metadata, "device"), thresholds.Disk, exists && override.Disk != nil)+ if disabled || !liveHysteresisThreshold(threshold) {+ m.clearAlertNoLock(storageKey)+ }+ }+ }+ intentChanged := false+ for resourceID := range resources {+ if !strings.HasPrefix(resourceID, prefix) {+ continue+ }+ if _, observed := seen[resourceID]; observed {+ continue+ }+ intentChanged = m.interruptMetricRunNoLock(resourceID, canonicalMetricSpecID(resourceID, "disk"), canonicalMetricStateID(resourceID, "disk")) || intentChanged+ }+ m.mu.Unlock()+ if intentChanged {+ m.saveActiveAlertsAsync("host filesystem observation gap")+ }+}diff --git a/internal/alerts/host_filesystem_observation_test.go b/internal/alerts/host_filesystem_observation_test.gonew file mode 100644index 0000000000..43b05fbe63--- /dev/null+++ b/internal/alerts/host_filesystem_observation_test.go@@ -0,0 +1,284 @@+package alerts++import (+ "bytes"+ "encoding/json"+ "fmt"+ "math"+ "os"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++func hostFilesystemManager(t *testing.T, grace bool) (*Manager, *atomic.Int64) {+ t.Helper()+ m, elapsed := continuityManager(t, grace)+ m.mu.Lock()+ m.config.AgentDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 70}}+ m.config.NodeDefaults = ThresholdConfig{Disk: &HysteresisThreshold{Trigger: 80, Clear: 70}}+ m.config.DiskFillByType = nil+ if !grace {+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 0}}+ }+ m.mu.Unlock()+ return m, elapsed+}++func hostFilesystemSample(id string, usage float64) models.Host {+ return models.Host{ID: id, Hostname: "storage-host", DisplayName: "Storage host", Status: "online",+ Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sda1", Total: 1000, Used: int64(usage * 10), Usage: usage}}}+}++func hostFilesystemGap(host models.Host, gap string) models.Host {+ disk := host.Disks[0]+ switch gap {+ case "empty":+ host.Disks = nil+ return host+ case "unknown-zero":+ disk.Total, disk.Used, disk.Usage = 0, 0, 0+ case "unknown-positive":+ disk.Total = 0+ case "negative-capacity":+ disk.Total = -1+ case "negative-used":+ disk.Used = -1+ case "over-capacity":+ disk.Used = disk.Total + 1+ case "negative":+ disk.Usage = -1+ case "over-100":+ disk.Usage = 101+ case "NaN":+ disk.Usage = math.NaN()+ case "+Inf":+ disk.Usage = math.Inf(1)+ }+ host.Disks = []models.Disk{disk}+ return host+}++func hostFilesystemID(host models.Host) string {+ id, _ := hostDiskResourceID(host, host.Disks[0])+ return id+}++func hostFilesystemJSON(t *testing.T, value any) []byte {+ t.Helper()+ data, err := json.Marshal(value)+ if err != nil {+ t.Fatal(err)+ }+ return data+}++func TestHostFilesystemUnknownPreservesOccurrence(t *testing.T) {+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-capacity", "negative-used", "over-capacity", "negative", "over-100", "NaN", "+Inf"} {+ t.Run(gap, func(t *testing.T) {+ m, elapsed := hostFilesystemManager(t, false)+ host := hostFilesystemSample("host-1", 95)+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("trusted full disk must open an alert")+ }+ id := m.GetActiveAlerts()[0].ID+ if err := m.AcknowledgeAlert(id, "operator"); err != nil {+ t.Fatal(err)+ }+ before := hostFilesystemJSON(t, m.GetActiveAlerts())+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))+ elapsed.Store(int64(2 * time.Minute))+ for range 3 {+ m.CheckHost(hostFilesystemGap(host, gap))+ }+ if !bytes.Equal(before, hostFilesystemJSON(t, m.GetActiveAlerts())) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {+ t.Fatal("unavailable filesystem changed the occurrence, trusted evidence, acknowledgement or lifecycle ledger")+ }+ // Observed zero, not a missing reading, must still recover the original.+ host.Disks[0].Used, host.Disks[0].Usage = 0, 0+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{AlertID: id, Types: []string{eventlog.TypeResolved}})) != 1 {+ t.Fatal("an observed empty filesystem must resolve the original exactly once")+ }+ })+ }+}++func TestHostFilesystemGapRestartsTiming(t *testing.T) {+ for _, grace := range []bool{false, true} {+ for _, recovery := range []bool{false, true} {+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-used", "over-capacity"} {+ t.Run(fmt.Sprintf("intent=%v/recovery=%v/%s", grace, recovery, gap), func(t *testing.T) {+ m, elapsed := hostFilesystemManager(t, grace)+ m.mu.Lock()+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}}+ m.mu.Unlock()+ host := hostFilesystemSample("host-1", 85)+ id := hostFilesystemID(host)+ if recovery {+ host.Disks[0].Used, host.Disks[0].Usage = 950, 95+ m.CheckHost(host)+ elapsed.Store(int64(time.Minute))+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("critical control must fire")+ }+ host.Disks[0].Used, host.Disks[0].Usage = 100, 10+ elapsed.Store(int64(70 * time.Second))+ }+ m.CheckHost(host)+ incident, exists := continuityIncident(m, id, "disk")+ if !exists || (!recovery && incident.State != reducer.StatePending) || (recovery && incident.RecoverySince.IsZero()) {+ t.Fatal("control must start the real timed run")+ }+ // A similarly prefixed host must not be interrupted with this host.+ other := hostFilesystemSample("host-10", 85)+ m.CheckHost(other)+ elapsed.Store(int64(3 * time.Minute))+ m.CheckHost(hostFilesystemGap(host, gap))+ incident, exists = continuityIncident(m, id, "disk")+ if (!recovery && exists) || (recovery && (!exists || !incident.RecoverySince.IsZero())) {+ t.Fatal("observation gap retained timed evidence")+ }+ if _, exists := continuityIncident(m, hostFilesystemID(other), "disk"); !exists {+ t.Fatal("gap crossed the exact host boundary")+ }+ m.CheckHost(host)+ want := 0+ if recovery {+ want = 1+ }+ if len(m.GetActiveAlerts()) != want {+ t.Fatal("first fresh sample completed the pre-gap run")+ }+ elapsed.Store(int64(3*time.Minute + 59*time.Second))+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != want {+ t.Fatal("fresh run completed early")+ }+ elapsed.Store(int64(4 * time.Minute))+ m.CheckHost(host)+ if recovery {+ want = 0+ } else {+ want = 1+ }+ if len(m.GetActiveAlerts()) != want {+ t.Fatal("full continuous fresh run did not complete")+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("interruption diverged from the shadow reducer")+ }+ })+ }+ }+ }+}++func TestHostFilesystemGapHonoursPolicyAndRemoval(t *testing.T) {+ for _, gap := range []string{"empty", "unknown-zero"} {+ for _, policy := range []string{"host-off", "disk-off", "disk-disabled", "inherited-off", "agent-removed", "filesystem-removed"} {+ t.Run(gap+"/"+policy, func(t *testing.T) {+ m, _ := hostFilesystemManager(t, false)+ host := hostFilesystemSample("host-1", 95)+ host.LinkedNodeID = "node-1"+ id := hostFilesystemID(host)+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("control must fire")+ }+ host = hostFilesystemGap(host, gap)+ // Change config directly: this proves CheckHost, not config-save cleanup.+ m.mu.Lock()+ switch policy {+ case "host-off":+ m.config.AgentDefaults.Disk = nil+ case "disk-off":+ m.config.Overrides[id] = ThresholdConfig{Disk: &HysteresisThreshold{}}+ case "disk-disabled":+ m.config.Overrides[id] = ThresholdConfig{Disabled: true}+ case "inherited-off":+ m.config.Overrides[host.LinkedNodeID] = ThresholdConfig{Disk: &HysteresisThreshold{}}+ }+ m.mu.Unlock()+ switch policy {+ case "agent-removed":+ m.HandleHostRemoved(host)+ case "filesystem-removed":+ host.Disks = hostFilesystemSample(host.ID, 0).Disks+ host.Disks[0].Mountpoint = "/other"+ m.CheckHost(host)+ default:+ m.CheckHost(host)+ }+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("unknown telemetry overrode explicit policy or authoritative removal")+ }+ })+ }+ }+}++func TestHostFilesystemGapKeepsNodeOwnershipOnlyForHeldAlert(t *testing.T) {+ for _, gap := range []string{"empty", "unknown-zero"} {+ for _, firing := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/firing=%v", gap, firing), func(t *testing.T) {+ m, _ := hostFilesystemManager(t, !firing)+ host := hostFilesystemSample("host-1", 85)+ host.LinkedNodeID = "node-1"+ if firing {+ host.Disks[0].Used, host.Disks[0].Usage = 950, 95+ }+ m.CheckHost(host)+ m.CheckHost(hostFilesystemGap(host, gap))+ m.mu.RLock()+ owner := m.hostAgentNodeLinks[host.ID]+ m.mu.RUnlock()+ if owner.disk != firing {+ t.Fatalf("gap disk ownership=%v, want %v", owner.disk, firing)+ }+ m.CheckNode(models.Node{ID: "node-1", Name: "storage-host", Status: "online", Disk: models.Disk{Total: 1000, Used: 950, Usage: 95}})+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("node handover left duplicate alerts or no alert")+ }+ })+ }+ }+}++// Real manager payloads are optionally exported for the production UI consumer.+func TestHostFilesystemBrowserProducer(t *testing.T) {+ m, elapsed := hostFilesystemManager(t, false)+ host := hostFilesystemSample("host-browser", 95)+ frames := map[string]json.RawMessage{}+ for _, phase := range []string{"breach", "unknown-zero", "empty", "recovered"} {+ sample := host+ if phase == "recovered" {+ sample.Disks = hostFilesystemSample(host.ID, 0).Disks+ } else if phase != "breach" {+ sample = hostFilesystemGap(host, phase)+ }+ elapsed.Add(int64(time.Second))+ m.CheckHost(sample)+ active := m.GetActiveAlerts()+ want := 1+ if phase == "recovered" {+ want = 0+ }+ if len(active) != want {+ t.Fatalf("phase %s: %d active, want %d", phase, len(active), want)+ }+ frames[phase] = hostFilesystemJSON(t, active)+ }+ if !bytes.Equal(frames["breach"], frames["unknown-zero"]) || !bytes.Equal(frames["breach"], frames["empty"]) {+ t.Fatal("unknown telemetry changed the exported occurrence")+ }+ if os.Getenv("PULSE_HOST_FILESYSTEM_FRAMES") == "1" {+ fmt.Printf("HOST_FILESYSTEM_FRAMES=%s\n", hostFilesystemJSON(t, frames))+ }+}diff --git a/internal/unifiedresources/host_disk_usage.go b/internal/unifiedresources/host_disk_usage.goindex 3bfe93ba4f..3696a5458b 100644--- a/internal/unifiedresources/host_disk_usage.go+++ b/internal/unifiedresources/host_disk_usage.go@@ -5,10 +5,19 @@ import "github.com/rcourtman/pulse-go-rewrite/internal/models"// HostDiskUsagePercent uses the same first-filesystem admission as the// canonical host metric. Missing or unavailable inventory is not measured zero.func HostDiskUsagePercent(disks []models.Disk) (float64, bool) {- if len(disks) == 0 || !hasObservedDiskUsage(disks[0]) {+ if len(disks) == 0 {return 0, false}- return disks[0].Usage, true+ return DiskUsagePercent(disks[0])+}++// DiskUsagePercent applies canonical observation admission to one filesystem.+// A listed filesystem can still have an unavailable or contradictory reading.+func DiskUsagePercent(disk models.Disk) (float64, bool) {+ if !hasObservedDiskUsage(disk) {+ return 0, false+ }+ return disk.Usage, true}// DiskUsagePercent preserves canonical metric presence, including providerdiff --git a/internal/unifiedresources/host_disk_usage_test.go b/internal/unifiedresources/host_disk_usage_test.gonew file mode 100644index 0000000000..5245aad71e--- /dev/null+++ b/internal/unifiedresources/host_disk_usage_test.go@@ -0,0 +1,39 @@+package unifiedresources++import (+ "math"+ "testing"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++func TestDiskUsagePercentSharesCanonicalAdmission(t *testing.T) {+ for _, tc := range []struct {+ name string+ disk models.Disk+ observed bool+ }{+ {"observed zero", models.Disk{Total: 1000}, true},+ {"observed fraction", models.Disk{Total: 1000, Used: 1, Usage: 0.1}, true},+ {"observed full", models.Disk{Total: 1000, Used: 1000, Usage: 100}, true},+ {"unavailable", models.Disk{}, false},+ {"capacity absent", models.Disk{Usage: 95}, false},+ {"used negative", models.Disk{Total: 1000, Used: -1}, false},+ {"used over capacity", models.Disk{Total: 1000, Used: 1001}, false},+ {"percentage negative", models.Disk{Total: 1000, Usage: -1}, false},+ {"percentage over 100", models.Disk{Total: 1000, Usage: 101}, false},+ {"NaN", models.Disk{Total: 1000, Usage: math.NaN()}, false},+ {"infinite", models.Disk{Total: 1000, Usage: math.Inf(1)}, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ value, observed := DiskUsagePercent(tc.disk)+ if observed != tc.observed || (observed && value != tc.disk.Usage) || (!observed && value != 0) {+ t.Fatalf("DiskUsagePercent = %v, %v; want observed=%v", value, observed, tc.observed)+ }+ summary, summaryObserved := HostDiskUsagePercent([]models.Disk{tc.disk})+ if value != summary || observed != summaryObserved {+ t.Fatal("summary and filesystem admission diverged")+ }+ })+ }+}