Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require observed capacity for storage warnings

pulse-triage[bot]e2684f689369
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index f04ad7c148..8fa70e7f3e 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -15,6 +15,27 @@
## Purpose
+
+### Storage capacity warnings require observed byte counters
+
+Static usage and forecast alerts admit a finite percentage in range only with
+positive capacity and non-negative used space no greater than capacity. An
+empty-store recovery still requires zero used bytes and free bytes equal to
+total. Missing or contradictory capacity holds the existing occurrence,
+acknowledgement and trusted evidence, and interrupts pending static activation,
+forecast confirmations and explicit intent grace. A valid empty store still
+recovers; disabling usage policy releases capacity alerts even when the store
+is unavailable. Connectivity and attached ZFS health remain independently
+evaluated, and the global alerting switch remains a pause.
+
+`storage_capacity_observation_test.go` exercises CheckStorage and its trend
+entry point, static and predictive occurrence retention, false activation,
+exact-store isolation, confirmation/grace interruption, policy and independent
+connectivity controls. Its opt-in manager payloads feed the production Alerts
+API/Overview browser proof at desktop and narrow widths. No collector, guest
+operation, schema, history rewrite or notification transport changes; this is
+not installed recovery or inbox evidence.
+
### Unavailable Proxmox node disks are not recovered alerts
The node evaluator admits the same disk observations as the canonical node
diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go
index 5f5bff62a6..9de52c5154 100644
--- a/internal/alerts/alerts_test.go
+++ b/internal/alerts/alerts_test.go
@@ -5373,6 +5373,9 @@ func TestDisableAllStorageClearsExistingAlerts(t *testing.T) {
})
storage := models.Storage{
+ Total: 1000,
+ Used: 900,
+ Free: 100,
ID: storageID,
Name: "local-lvm",
Usage: 90.0,
@@ -18418,6 +18421,9 @@ func TestCheckStorageComprehensive(t *testing.T) {
m.mu.Unlock()
storage := models.Storage{
+ Total: 1000,
+ Used: 950,
+ Free: 50,
ID: "storage1",
Name: "teststorage",
Status: "active",
@@ -18537,6 +18543,9 @@ func TestCheckStorageComprehensive(t *testing.T) {
m.mu.Unlock()
storage := models.Storage{
+ Total: 1000,
+ Used: 950,
+ Free: 50,
ID: "storage1",
Name: "teststorage",
Node: "node1",
@@ -18569,6 +18578,9 @@ func TestCheckStorageComprehensive(t *testing.T) {
m.mu.Unlock()
storage := models.Storage{
+ Total: 1000,
+ Used: 950,
+ Free: 50,
ID: "storage1",
Name: "teststorage",
Status: "active",
@@ -18596,6 +18608,9 @@ func TestCheckStorageComprehensive(t *testing.T) {
m.mu.Unlock()
storage := models.Storage{
+ Total: 1000,
+ Used: 950,
+ Free: 50,
ID: "storage1",
Name: "teststorage",
Status: "offline",
@@ -18623,6 +18638,9 @@ func TestCheckStorageComprehensive(t *testing.T) {
m.mu.Unlock()
storage := models.Storage{
+ Total: 1000,
+ Used: 950,
+ Free: 50,
ID: "storage1",
Name: "teststorage",
Status: "unavailable",
@@ -21913,7 +21931,7 @@ func TestCheckStorageForecastUsesCanonicalUsageAlertIdentity(t *testing.T) {
m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
- storage := models.Storage{ID: "forecast-proof", Name: "archive", Status: "active", Usage: 72}
+ storage := models.Storage{Total: 1000, Used: 720, Free: 280, ID: "forecast-proof", Name: "archive", Status: "active", Usage: 72}
trend := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 5,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
@@ -22044,7 +22062,7 @@ func TestStorageNormalizedOfflineSkipsCapacityEvaluation(t *testing.T) {
m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
- s := models.Storage{ID: "storage-normalized-offline", Name: "backups", Status: status, Usage: 99}
+ s := models.Storage{Total: 1000, Used: 990, Free: 10, ID: "storage-normalized-offline", Name: "backups", Status: status, Usage: 99}
for range 3 {
m.CheckStorage(s)
}
diff --git a/internal/alerts/capacity_forecast_test.go b/internal/alerts/capacity_forecast_test.go
index 234b0f7bd6..b66ed2d3a9 100644
--- a/internal/alerts/capacity_forecast_test.go
+++ b/internal/alerts/capacity_forecast_test.go
@@ -81,6 +81,9 @@ func TestStorageForecastSharesLifecycleWithStaticUsageAlert(t *testing.T) {
m.SetResolvedCallback(func(string) { resolved++ })
storage := models.Storage{
+ Total: 1000,
+ Used: 720,
+ Free: 280,
ID: "storage-forecast-1",
Name: "archive",
Node: "pve1",
@@ -162,7 +165,7 @@ func TestStorageForecastTreatsUncertainEvidenceAsUnknown(t *testing.T) {
m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
m.mu.Unlock()
- storage := models.Storage{ID: "storage-forecast-unknown", Name: "media", Status: "active", Usage: 70}
+ storage := models.Storage{Total: 1000, Used: 700, Free: 300, ID: "storage-forecast-unknown", Name: "media", Status: "active", Usage: 70}
trusted := CapacityTrendObservation{
Ready: true, Reason: "increasing", ObservedAt: time.Now(), DailyChange: 6,
Confidence: 0.99, SampleCount: 300, BucketCount: 48, CoverageSpan: 48 * time.Hour,
diff --git a/internal/alerts/storage.go b/internal/alerts/storage.go
index 4763eab676..6945a57cf5 100644
--- a/internal/alerts/storage.go
+++ b/internal/alerts/storage.go
@@ -106,11 +106,20 @@ func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend Ca
Float64("clear", thresholds.Usage.Clear).
Msg("Checking storage thresholds")
- // Zero usage is a clearing observation only when the byte counters also
- // confirm an empty store. A default zero from missing capacity telemetry
- // must not resolve an existing incident.
+ // Static and forecast decisions both need an observed capacity, not just
+ // a positive percentage left beside unavailable/contradictory byte counts.
+ // Share filesystem admission while retaining storage's stricter empty-store
+ // confirmation: a default zero must not resolve an existing incident.
+ _, observed := unifiedresources.DiskUsagePercent(models.Disk{Total: storage.Total, Used: storage.Used, Usage: storage.Usage})
confirmedEmpty := storage.Usage == 0 && storage.Total > 0 && storage.Used == 0 && storage.Free == storage.Total
- if connectivityStatus != "offline" && connectivityStatus != "unavailable" && (storage.Usage > 0 || confirmedEmpty) {
+ if !liveHysteresisThreshold(thresholds.Usage) {
+ // Explicit usage-policy disablement is not telemetry recovery. Release
+ // through the disabled evaluator even when capacity is unavailable or
+ // nonfinite; connectivity and pool health keep their separate policies.
+ policyOnly := storage
+ policyOnly.Usage = 0
+ m.evaluateStorageCapacity(policyOnly, thresholds, CapacityTrendObservation{})
+ } else if connectivityStatus != "offline" && connectivityStatus != "unavailable" && observed && (storage.Usage > 0 || confirmedEmpty) {
m.evaluateStorageCapacity(storage, thresholds, trend)
} else {
m.interruptMetricRunIDs(storage.ID, canonicalMetricSpecID(storage.ID, "usage"), canonicalMetricStateID(storage.ID, "usage"))
diff --git a/internal/alerts/storage_capacity_observation_test.go b/internal/alerts/storage_capacity_observation_test.go
new file mode 100644
index 0000000000..0c2777e8eb
--- /dev/null
+++ b/internal/alerts/storage_capacity_observation_test.go
@@ -0,0 +1,297 @@
+package alerts
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "math"
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+)
+
+func storageObservationManager(t *testing.T, intent bool) (*Manager, *atomic.Int64) {
+ t.Helper()
+ m, clock := continuityManager(t, intent)
+ m.mu.Lock()
+ m.config.StorageDefault = HysteresisThreshold{Trigger: 80, Clear: 70}
+ if !intent {
+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"usage": 0}}
+ }
+ m.mu.Unlock()
+ return m, clock
+}
+
+func storageObservationSample(id string, usage float64) models.Storage {
+ return models.Storage{ID: id, Name: "backups", Node: "pve", Instance: "site", Type: "dir", Status: "available", Total: 1000, Used: int64(usage * 10), Free: 1000 - int64(usage*10), Usage: usage}
+}
+
+func storageObservationGap(s models.Storage, gap string) models.Storage {
+ switch gap {
+ case "missing":
+ s.Total, s.Used, s.Free, s.Usage = 0, 0, 0, 0
+ case "no-capacity":
+ s.Total = 0
+ case "negative-capacity":
+ s.Total = -1
+ case "negative-used":
+ s.Used = -1
+ case "over-capacity":
+ s.Used = s.Total + 1
+ case "negative-percent":
+ s.Usage = -1
+ case "over-100":
+ s.Usage = 101
+ case "NaN":
+ s.Usage = math.NaN()
+ case "+Inf":
+ s.Usage = math.Inf(1)
+ case "unconfirmed-zero":
+ s.Usage = 0
+ }
+ return s
+}
+
+func storageObservationJSON(t *testing.T, v any) []byte {
+ t.Helper()
+ b, err := json.Marshal(v)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func TestStorageCapacityUnknownPreservesOccurrence(t *testing.T) {
+ for _, gap := range []string{"missing", "no-capacity", "negative-capacity", "negative-used", "over-capacity", "negative-percent", "over-100", "NaN", "+Inf", "unconfirmed-zero"} {
+ t.Run(gap, func(t *testing.T) {
+ m, clock := storageObservationManager(t, false)
+ s := storageObservationSample("site-pve-backups", 95)
+ m.CheckStorage(s)
+ alert := testRequireActiveAlert(t, m, canonicalMetricStateID(s.ID, "usage"))
+ if err := m.AcknowledgeAlert(alert.ID, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ before := storageObservationJSON(t, m.GetActiveAlerts())
+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))
+ clock.Store(int64(2 * time.Minute))
+ // A contradictory low percentage must not recover; a high one must not
+ // refresh the occurrence's trusted evidence or severity either.
+ for _, value := range []float64{10, 98} {
+ for range 3 {
+ m.CheckStorage(storageObservationGap(storageObservationSample(s.ID, value), gap))
+ }
+ if !bytes.Equal(before, storageObservationJSON(t, m.GetActiveAlerts())) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {
+ t.Fatalf("%s at %.0f%% changed occurrence, acknowledgement, evidence or lifecycle", gap, value)
+ }
+ }
+ m.CheckStorage(storageObservationSample(s.ID, 0))
+ if testHasActiveAlert(t, m, canonicalMetricStateID(s.ID, "usage")) || len(queryAlertEvents(t, m, eventlog.Filter{AlertID: alert.ID, Types: []string{eventlog.TypeResolved}})) != 1 {
+ t.Fatal("confirmed empty storage must resolve the original exactly once")
+ }
+ })
+ }
+}
+
+func TestStorageCapacityUnknownCannotActivateStaticOrForecast(t *testing.T) {
+ for _, forecast := range []bool{false, true} {
+ for _, gap := range []string{"missing", "no-capacity", "negative-capacity", "negative-used", "over-capacity", "negative-percent", "over-100", "NaN", "+Inf", "unconfirmed-zero"} {
+ t.Run(fmt.Sprintf("forecast=%v/%s", forecast, gap), func(t *testing.T) {
+ m, clock := storageObservationManager(t, false)
+ value := 95.0
+ if forecast {
+ value = 72
+ }
+ s := storageObservationGap(storageObservationSample("site-pve-backups", value), gap)
+ trend := CapacityTrendObservation{}
+ if forecast {
+ trend = storageObservationTrend()
+ }
+ for range 3 {
+ m.CheckStorageWithCapacityTrend(s, trend)
+ clock.Add(int64(time.Minute))
+ }
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 {
+ t.Fatal("unobserved capacity invented an incident")
+ }
+ })
+ }
+ }
+}
+
+func storageObservationTrend() CapacityTrendObservation {
+ return CapacityTrendObservation{Ready: true, Reason: "increasing", DailyChange: 5, Confidence: .98, ObservedAt: time.Now(), SampleCount: 400, BucketCount: 72, CoverageSpan: 72 * time.Hour}
+}
+
+func TestStorageCapacityGapRestartsActivation(t *testing.T) {
+ for _, intent := range []bool{false, true} {
+ for _, forecast := range []bool{false, true} {
+ for _, gap := range []string{"missing", "no-capacity", "negative-used"} {
+ t.Run(fmt.Sprintf("intent=%v/forecast=%v/%s", intent, forecast, gap), func(t *testing.T) {
+ m, clock := storageObservationManager(t, intent)
+ m.mu.Lock()
+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"usage": 60}}
+ m.mu.Unlock()
+ value := 85.0
+ trend := CapacityTrendObservation{}
+ if forecast {
+ value = 72
+ trend = storageObservationTrend()
+ }
+ s := storageObservationSample("site-pve-backups", value)
+ other := storageObservationSample("other-pve-backups", value)
+ observe := func(sample models.Storage) { m.CheckStorageWithCapacityTrend(sample, trend) }
+ observe(s)
+ observe(other)
+ if i, ok := continuityIncident(m, s.ID, "usage"); !ok || i.State != reducer.StatePending {
+ t.Fatal("control must start pending activation")
+ }
+ clock.Store(int64(2 * time.Minute))
+ observe(storageObservationGap(s, gap))
+ if _, ok := continuityIncident(m, s.ID, "usage"); ok {
+ t.Fatal("gap retained confirmation or timed activation")
+ }
+ if _, ok := continuityIncident(m, other.ID, "usage"); !ok {
+ t.Fatal("gap crossed storage identity")
+ }
+ observe(s)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("first fresh observation fired across the gap")
+ }
+ if !forecast || intent {
+ clock.Store(int64(2*time.Minute + 59*time.Second))
+ observe(s)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("activation completed before fresh grace")
+ }
+ }
+ clock.Store(int64(3 * time.Minute))
+ observe(s)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("full fresh activation did not fire once")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("shadow diverged")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestStorageCapacityUnknownHonoursPolicyAndConnectivity(t *testing.T) {
+ for _, policy := range []string{"usage-off", "alias-off", "resource-off", "all-storage-off", "pause"} {
+ for _, gap := range []string{"missing", "NaN"} {
+ t.Run(policy+"/"+gap, func(t *testing.T) {
+ m, _ := storageObservationManager(t, false)
+ s := storageObservationSample("site-pve-backups", 95)
+ s.AliasIDs = []string{"site/pve/backups"}
+ m.CheckStorage(s)
+ id := canonicalMetricStateID(s.ID, "usage")
+ testRequireActiveAlert(t, m, id)
+ m.mu.Lock()
+ switch policy {
+ case "usage-off":
+ m.config.StorageDefault = HysteresisThreshold{}
+ case "alias-off":
+ m.config.Overrides[s.AliasIDs[0]] = ThresholdConfig{Usage: &HysteresisThreshold{}}
+ case "resource-off":
+ m.config.Overrides[s.ID] = ThresholdConfig{Disabled: true}
+ case "all-storage-off":
+ m.config.DisableAllStorage = true
+ case "pause":
+ m.config.Enabled = false
+ }
+ m.mu.Unlock()
+ missing := storageObservationGap(s, gap)
+ missing.Status = "unavailable"
+ m.CheckStorage(missing)
+ if testHasActiveAlert(t, m, id) != (policy == "pause") {
+ t.Fatal("observation gap overrode explicit policy")
+ }
+ if policy == "usage-off" || policy == "alias-off" {
+ for range 3 {
+ m.CheckStorage(missing)
+ }
+ testRequireActiveAlert(t, m, canonicalConnectivityStateID(s.ID))
+ }
+ })
+ }
+ }
+}
+
+func TestStorageCapacityUnknownHoldsForecastAndIndependentHealth(t *testing.T) {
+ m, clock := storageObservationManager(t, false)
+ s := storageObservationSample("site-pve-backups", 72)
+ trend := storageObservationTrend()
+ for range 2 {
+ m.CheckStorageWithCapacityTrend(s, trend)
+ }
+ alert := testRequireActiveAlert(t, m, canonicalMetricStateID(s.ID, "usage"))
+ if alert.Metadata[capacityAlertOriginKey] != capacityAlertOriginForecast {
+ t.Fatal("control must be a forecast")
+ }
+ before := storageObservationJSON(t, m.GetActiveAlerts())
+ clock.Store(int64(2 * time.Minute))
+ gap := storageObservationGap(storageObservationSample(s.ID, 10), "negative-used")
+ for range 3 {
+ m.CheckStorageWithCapacityTrend(gap, CapacityTrendObservation{Ready: true, Reason: "not-increasing"})
+ }
+ if !bytes.Equal(before, storageObservationJSON(t, m.GetActiveAlerts())) {
+ t.Fatal("unobserved low capacity resolved the forecast")
+ }
+ // Missing capacity does not suppress independent connectivity evaluation.
+ gap.Status = "unavailable"
+ for range 3 {
+ m.CheckStorage(gap)
+ }
+ testRequireActiveAlert(t, m, canonicalConnectivityStateID(s.ID))
+ testRequireActiveAlert(t, m, canonicalMetricStateID(s.ID, "usage"))
+ // A real low reading still clears capacity while preserving the outage.
+ gap = storageObservationSample(s.ID, 0)
+ gap.Status = "unknown"
+ m.CheckStorage(gap)
+ if testHasActiveAlert(t, m, canonicalMetricStateID(s.ID, "usage")) {
+ t.Fatal("confirmed empty store did not recover capacity")
+ }
+ testRequireActiveAlert(t, m, canonicalConnectivityStateID(s.ID))
+}
+
+// Only synthetic manager payloads; no transport or installed recovery claim.
+func TestStorageCapacityBrowserProducer(t *testing.T) {
+ m, clock := storageObservationManager(t, false)
+ s := storageObservationSample("site-browser-backups", 95)
+ frames := map[string]json.RawMessage{}
+ for _, phase := range []string{"breach", "missing", "over-capacity", "recovered"} {
+ sample := s
+ switch phase {
+ case "missing":
+ sample = storageObservationGap(storageObservationSample(s.ID, 10), "no-capacity")
+ case "over-capacity":
+ sample = storageObservationGap(storageObservationSample(s.ID, 10), phase)
+ case "recovered":
+ sample = storageObservationSample(s.ID, 0)
+ }
+ clock.Add(int64(time.Second))
+ m.CheckStorage(sample)
+ active := m.GetActiveAlerts()
+ want := 1
+ if phase == "recovered" {
+ want = 0
+ }
+ if len(active) != want {
+ t.Fatalf("phase %s: %d active, want %d", phase, len(active), want)
+ }
+ frames[phase] = storageObservationJSON(t, active)
+ }
+ if !bytes.Equal(frames["breach"], frames["missing"]) || !bytes.Equal(frames["breach"], frames["over-capacity"]) {
+ t.Fatal("gap changed exported occurrence")
+ }
+ if os.Getenv("PULSE_STORAGE_CAPACITY_FRAMES") == "1" {
+ fmt.Printf("STORAGE_CAPACITY_FRAMES=%s\n", storageObservationJSON(t, frames))
+ }
+}