Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bound filesystem gap bookkeeping and honour full grace in controls

pulse-triage[bot]2724953dc172
diff --git a/internal/alerts/host_filesystem_observation.go b/internal/alerts/host_filesystem_observation.go
index 3c5d80bf3f..212c9dab34 100644
--- a/internal/alerts/host_filesystem_observation.go
+++ b/internal/alerts/host_filesystem_observation.go
@@ -16,11 +16,13 @@ func (m *Manager) reconcileHostFilesystemObservationGaps(host models.Host, seen
resources := make(map[string]struct{})
for _, state := range m.mirrorStatesNoLock() {
for _, resourceID := range state.PendingResourceIDs() {
- resources[resourceID] = struct{}{}
+ if strings.HasPrefix(resourceID, prefix) {
+ resources[resourceID] = struct{}{}
+ }
}
}
for _, pending := range m.intentPending {
- if pending.Signal == MetricAlertIntentSignal("disk") {
+ if pending.Signal == MetricAlertIntentSignal("disk") && strings.HasPrefix(pending.ResourceID, prefix) {
resources[pending.ResourceID] = struct{}{}
}
}
diff --git a/internal/alerts/host_filesystem_observation_test.go b/internal/alerts/host_filesystem_observation_test.go
index 43b05fbe63..05b356f459 100644
--- a/internal/alerts/host_filesystem_observation_test.go
+++ b/internal/alerts/host_filesystem_observation_test.go
@@ -228,7 +228,7 @@ func TestHostFilesystemGapKeepsNodeOwnershipOnlyForHeldAlert(t *testing.T) {
for _, gap := range []string{"empty", "unknown-zero"} {
for _, firing := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/firing=%v", gap, firing), func(t *testing.T) {
- m, _ := hostFilesystemManager(t, !firing)
+ m, elapsed := hostFilesystemManager(t, !firing)
host := hostFilesystemSample("host-1", 85)
host.LinkedNodeID = "node-1"
if firing {
@@ -242,7 +242,10 @@ func TestHostFilesystemGapKeepsNodeOwnershipOnlyForHeldAlert(t *testing.T) {
if owner.disk != firing {
t.Fatalf("gap disk ownership=%v, want %v", owner.disk, firing)
}
- m.CheckNode(models.Node{ID: "node-1", Name: "storage-host", Status: "online", Disk: models.Disk{Total: 1000, Used: 950, Usage: 95}})
+ node := models.Node{ID: "node-1", Name: "storage-host", Status: "online", Disk: models.Disk{Total: 1000, Used: 950, Usage: 95}}
+ m.CheckNode(node)
+ elapsed.Store(int64(time.Minute))
+ m.CheckNode(node)
if len(m.GetActiveAlerts()) != 1 {
t.Fatal("node handover left duplicate alerts or no alert")
}
@@ -282,3 +285,44 @@ func TestHostFilesystemBrowserProducer(t *testing.T) {
fmt.Printf("HOST_FILESYSTEM_FRAMES=%s\n", hostFilesystemJSON(t, frames))
}
}
+
+func TestHostFilesystemUnobservedNeverFires(t *testing.T) {
+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-capacity", "negative-used", "over-capacity", "negative", "over-100", "NaN", "+Inf"} {
+ t.Run(gap, func(t *testing.T) {
+ m, elapsed := hostFilesystemManager(t, false)
+ host := hostFilesystemGap(hostFilesystemSample("host-1", 95), gap)
+ m.CheckHost(host)
+ elapsed.Store(int64(time.Hour))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 {
+ t.Fatal("an unavailable filesystem invented an alert or transition")
+ }
+ })
+ }
+}
+
+func TestHostFilesystemReplacementRetiresPendingRun(t *testing.T) {
+ m, elapsed := hostFilesystemManager(t, true)
+ host := hostFilesystemSample("host-1", 85)
+ id := hostFilesystemID(host)
+ m.CheckHost(host)
+ if _, exists := continuityIncident(m, id, "disk"); !exists {
+ t.Fatal("control must start a pending run")
+ }
+ replacement := hostFilesystemSample(host.ID, 0)
+ replacement.Disks[0].Mountpoint = "/other"
+ elapsed.Store(int64(2 * time.Minute))
+ m.CheckHost(replacement)
+ if _, exists := continuityIncident(m, id, "disk"); exists {
+ t.Fatal("removed filesystem retained timed evidence")
+ }
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("reappearing filesystem fired using retired evidence")
+ }
+ elapsed.Store(int64(3 * time.Minute))
+ m.CheckHost(host)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("fresh full run did not fire")
+ }
+}