Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep the established disk owner when a listed capacity is unknown

pulse-triage[bot]02aff4ab7aaa
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 5131cd226e..591e36a1c5 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -21,8 +21,9 @@ Host filesystem thresholds use the canonical disk-observation admission: a
finite percentage in range, positive capacity and non-negative used space no
larger than capacity. A listed unavailable reading or an entirely empty report
holds the existing occurrence, acknowledgement and last trusted evidence. It
-cannot raise a new alert, resolve one, or complete activation, explicit intent
-grace or recovery timing across the gap. A measured zero remains recovery.
+cannot raise a new alert, resolve one, or complete activation or explicit intent
+grace across the gap. A measured zero retains ordinary immediate disk recovery;
+no recovery delay is introduced.
An established linked-node summary identity keeps ownership only while its
held disk alert remains open; interrupting a pending run returns ownership to
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 043aa51a6a..17f8094bc5 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -536,6 +536,21 @@ func (m *Manager) CheckHost(host models.Host) {
summaryDiskResourceID := ""
if summary, ok := models.SummaryDisk(host.Disks); ok {
summaryDiskResourceID, _ = hostDiskResourceID(host, summary)
+ } else if host.LinkedNodeID != "" {
+ // SummaryDisk needs capacity. A still-listed filesystem whose
+ // capacity disappeared retains its established identity, not a
+ // usable reading, so its held alert still prevents a node duplicate.
+ m.mu.RLock()
+ previous := m.hostAgentNodeLinks[host.ID]
+ m.mu.RUnlock()
+ if previous.nodeID == host.LinkedNodeID {
+ for _, disk := range host.Disks {
+ if id, _ := hostDiskResourceID(host, disk); id == previous.summaryDiskResourceID {
+ summaryDiskResourceID = id
+ break
+ }
+ }
+ }
}
evaluatesSummaryDisk := false
summaryDiskLive := false
diff --git a/internal/alerts/host_filesystem_observation_test.go b/internal/alerts/host_filesystem_observation_test.go
index 05b356f459..e1833c06c6 100644
--- a/internal/alerts/host_filesystem_observation_test.go
+++ b/internal/alerts/host_filesystem_observation_test.go
@@ -111,71 +111,47 @@ func TestHostFilesystemUnknownPreservesOccurrence(t *testing.T) {
func TestHostFilesystemGapRestartsTiming(t *testing.T) {
for _, grace := range []bool{false, true} {
- for _, recovery := range []bool{false, true} {
- for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-used", "over-capacity"} {
- t.Run(fmt.Sprintf("intent=%v/recovery=%v/%s", grace, recovery, gap), func(t *testing.T) {
- m, elapsed := hostFilesystemManager(t, grace)
- m.mu.Lock()
- m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}}
- m.mu.Unlock()
- host := hostFilesystemSample("host-1", 85)
- id := hostFilesystemID(host)
- if recovery {
- host.Disks[0].Used, host.Disks[0].Usage = 950, 95
- m.CheckHost(host)
- elapsed.Store(int64(time.Minute))
- m.CheckHost(host)
- if len(m.GetActiveAlerts()) != 1 {
- t.Fatal("critical control must fire")
- }
- host.Disks[0].Used, host.Disks[0].Usage = 100, 10
- elapsed.Store(int64(70 * time.Second))
- }
- m.CheckHost(host)
- incident, exists := continuityIncident(m, id, "disk")
- if !exists || (!recovery && incident.State != reducer.StatePending) || (recovery && incident.RecoverySince.IsZero()) {
- t.Fatal("control must start the real timed run")
- }
- // A similarly prefixed host must not be interrupted with this host.
- other := hostFilesystemSample("host-10", 85)
- m.CheckHost(other)
- elapsed.Store(int64(3 * time.Minute))
- m.CheckHost(hostFilesystemGap(host, gap))
- incident, exists = continuityIncident(m, id, "disk")
- if (!recovery && exists) || (recovery && (!exists || !incident.RecoverySince.IsZero())) {
- t.Fatal("observation gap retained timed evidence")
- }
- if _, exists := continuityIncident(m, hostFilesystemID(other), "disk"); !exists {
- t.Fatal("gap crossed the exact host boundary")
- }
- m.CheckHost(host)
- want := 0
- if recovery {
- want = 1
- }
- if len(m.GetActiveAlerts()) != want {
- t.Fatal("first fresh sample completed the pre-gap run")
- }
- elapsed.Store(int64(3*time.Minute + 59*time.Second))
- m.CheckHost(host)
- if len(m.GetActiveAlerts()) != want {
- t.Fatal("fresh run completed early")
- }
- elapsed.Store(int64(4 * time.Minute))
- m.CheckHost(host)
- if recovery {
- want = 0
- } else {
- want = 1
- }
- if len(m.GetActiveAlerts()) != want {
- t.Fatal("full continuous fresh run did not complete")
- }
- if m.ShadowDivergences() != 0 {
- t.Fatal("interruption diverged from the shadow reducer")
- }
- })
- }
+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-used", "over-capacity"} {
+ t.Run(fmt.Sprintf("intent=%v/%s", grace, gap), func(t *testing.T) {
+ m, elapsed := hostFilesystemManager(t, grace)
+ m.mu.Lock()
+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}}
+ m.mu.Unlock()
+ host := hostFilesystemSample("host-1", 85)
+ id := hostFilesystemID(host)
+ m.CheckHost(host)
+ incident, exists := continuityIncident(m, id, "disk")
+ if !exists || incident.State != reducer.StatePending {
+ t.Fatal("control must start the real timed activation run")
+ }
+ other := hostFilesystemSample("host-10", 85)
+ m.CheckHost(other)
+ elapsed.Store(int64(2 * time.Minute))
+ m.CheckHost(hostFilesystemGap(host, gap))
+ if _, exists := continuityIncident(m, id, "disk"); exists {
+ t.Fatal("observation gap retained timed evidence")
+ }
+ if _, exists := continuityIncident(m, hostFilesystemID(other), "disk"); !exists {
+ t.Fatal("gap crossed the exact host boundary")
+ }
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("first fresh sample completed the pre-gap run")
+ }
+ elapsed.Store(int64(2*time.Minute + 59*time.Second))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("fresh run completed early")
+ }
+ elapsed.Store(int64(3 * time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("full continuous fresh run did not complete")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("interruption diverged from the shadow reducer")
+ }
+ })
}
}
}
diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go
index 5d995179d4..d074a380e6 100644
--- a/internal/alerts/host_unraid_lifecycle_test.go
+++ b/internal/alerts/host_unraid_lifecycle_test.go
@@ -1619,7 +1619,9 @@ func TestHostSMARTRiskFilesystemAndHostLifecycleStayIndependent(t *testing.T) {
fsID, _ := hostDiskResourceID(f.host, f.host.Disks[0])
id := canonicalMetricStateID(fsID, "disk")
testRequireActiveAlert(t, f.m, id)
- f.host.Disks = nil
+ // A non-empty replacement establishes filesystem removal;
+ // an empty report alone is now an observation gap.
+ f.host.Disks = []models.Disk{{Mountpoint: "/other", Total: 1000}}
f.host.Sensors.SMART = nil
f.m.CheckHost(f.host)
if testHasActiveAlert(t, f.m, id) {
@@ -1690,7 +1692,9 @@ func TestHostSMARTRiskFilesystemIdentityCannotSubstituteInventory(t *testing.T)
usageID := canonicalMetricStateID(diskResourceID, "disk")
testRequireActiveAlert(t, f.m, usageID)
if direction == "SMART-only" {
- f.host.Disks = nil
+ // A non-empty replacement establishes filesystem removal;
+ // an empty report alone is now an observation gap.
+ f.host.Disks = []models.Disk{{Mountpoint: "/other", Total: 1000}}
f.m.CheckHost(f.host)
if testHasActiveAlert(t, f.m, usageID) {
t.Fatal("SMART identity held missing filesystem usage")
diff --git a/internal/alerts/migration_characterization_test.go b/internal/alerts/migration_characterization_test.go
index d8814b3fa8..785d662a2b 100644
--- a/internal/alerts/migration_characterization_test.go
+++ b/internal/alerts/migration_characterization_test.go
@@ -994,10 +994,10 @@ func TestAlertCharacterizationHeldMetricAlertRefreshesItsClassification(t *testi
name: "agent filesystem",
alertID: canonicalMetricStateID("agent:held-host/disk:srv", "disk"),
fire: func(m *Manager) {
- m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 92}}})
+ m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Total: 1000, Used: 920, Usage: 92}}})
},
hold: func(m *Manager) {
- m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 75}}})
+ m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Total: 1000, Used: 750, Usage: 75}}})
},
stale: staleAlert{resourceType: "agent"},
wantResourceType: "agent-disk",