Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep the established disk owner when a listed capacity is unknown
pulse-triage[bot]02aff4ab7aaa
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex 5131cd226e..591e36a1c5 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -21,8 +21,9 @@ Host filesystem thresholds use the canonical disk-observation admission: afinite percentage in range, positive capacity and non-negative used space nolarger than capacity. A listed unavailable reading or an entirely empty reportholds the existing occurrence, acknowledgement and last trusted evidence. It-cannot raise a new alert, resolve one, or complete activation, explicit intent-grace or recovery timing across the gap. A measured zero remains recovery.+cannot raise a new alert, resolve one, or complete activation or explicit intent+grace across the gap. A measured zero retains ordinary immediate disk recovery;+no recovery delay is introduced.An established linked-node summary identity keeps ownership only while itsheld disk alert remains open; interrupting a pending run returns ownership todiff --git a/internal/alerts/host.go b/internal/alerts/host.goindex 043aa51a6a..17f8094bc5 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -536,6 +536,21 @@ func (m *Manager) CheckHost(host models.Host) {summaryDiskResourceID := ""if summary, ok := models.SummaryDisk(host.Disks); ok {summaryDiskResourceID, _ = hostDiskResourceID(host, summary)+ } else if host.LinkedNodeID != "" {+ // SummaryDisk needs capacity. A still-listed filesystem whose+ // capacity disappeared retains its established identity, not a+ // usable reading, so its held alert still prevents a node duplicate.+ m.mu.RLock()+ previous := m.hostAgentNodeLinks[host.ID]+ m.mu.RUnlock()+ if previous.nodeID == host.LinkedNodeID {+ for _, disk := range host.Disks {+ if id, _ := hostDiskResourceID(host, disk); id == previous.summaryDiskResourceID {+ summaryDiskResourceID = id+ break+ }+ }+ }}evaluatesSummaryDisk := falsesummaryDiskLive := falsediff --git a/internal/alerts/host_filesystem_observation_test.go b/internal/alerts/host_filesystem_observation_test.goindex 05b356f459..e1833c06c6 100644--- a/internal/alerts/host_filesystem_observation_test.go+++ b/internal/alerts/host_filesystem_observation_test.go@@ -111,71 +111,47 @@ func TestHostFilesystemUnknownPreservesOccurrence(t *testing.T) {func TestHostFilesystemGapRestartsTiming(t *testing.T) {for _, grace := range []bool{false, true} {- for _, recovery := range []bool{false, true} {- for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-used", "over-capacity"} {- t.Run(fmt.Sprintf("intent=%v/recovery=%v/%s", grace, recovery, gap), func(t *testing.T) {- m, elapsed := hostFilesystemManager(t, grace)- m.mu.Lock()- m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}}- m.mu.Unlock()- host := hostFilesystemSample("host-1", 85)- id := hostFilesystemID(host)- if recovery {- host.Disks[0].Used, host.Disks[0].Usage = 950, 95- m.CheckHost(host)- elapsed.Store(int64(time.Minute))- m.CheckHost(host)- if len(m.GetActiveAlerts()) != 1 {- t.Fatal("critical control must fire")- }- host.Disks[0].Used, host.Disks[0].Usage = 100, 10- elapsed.Store(int64(70 * time.Second))- }- m.CheckHost(host)- incident, exists := continuityIncident(m, id, "disk")- if !exists || (!recovery && incident.State != reducer.StatePending) || (recovery && incident.RecoverySince.IsZero()) {- t.Fatal("control must start the real timed run")- }- // A similarly prefixed host must not be interrupted with this host.- other := hostFilesystemSample("host-10", 85)- m.CheckHost(other)- elapsed.Store(int64(3 * time.Minute))- m.CheckHost(hostFilesystemGap(host, gap))- incident, exists = continuityIncident(m, id, "disk")- if (!recovery && exists) || (recovery && (!exists || !incident.RecoverySince.IsZero())) {- t.Fatal("observation gap retained timed evidence")- }- if _, exists := continuityIncident(m, hostFilesystemID(other), "disk"); !exists {- t.Fatal("gap crossed the exact host boundary")- }- m.CheckHost(host)- want := 0- if recovery {- want = 1- }- if len(m.GetActiveAlerts()) != want {- t.Fatal("first fresh sample completed the pre-gap run")- }- elapsed.Store(int64(3*time.Minute + 59*time.Second))- m.CheckHost(host)- if len(m.GetActiveAlerts()) != want {- t.Fatal("fresh run completed early")- }- elapsed.Store(int64(4 * time.Minute))- m.CheckHost(host)- if recovery {- want = 0- } else {- want = 1- }- if len(m.GetActiveAlerts()) != want {- t.Fatal("full continuous fresh run did not complete")- }- if m.ShadowDivergences() != 0 {- t.Fatal("interruption diverged from the shadow reducer")- }- })- }+ for _, gap := range []string{"empty", "unknown-zero", "unknown-positive", "negative-used", "over-capacity"} {+ t.Run(fmt.Sprintf("intent=%v/%s", grace, gap), func(t *testing.T) {+ m, elapsed := hostFilesystemManager(t, grace)+ m.mu.Lock()+ m.config.MetricTimeThresholds = map[string]map[string]int{"all": {"disk": 60}}+ m.mu.Unlock()+ host := hostFilesystemSample("host-1", 85)+ id := hostFilesystemID(host)+ m.CheckHost(host)+ incident, exists := continuityIncident(m, id, "disk")+ if !exists || incident.State != reducer.StatePending {+ t.Fatal("control must start the real timed activation run")+ }+ other := hostFilesystemSample("host-10", 85)+ m.CheckHost(other)+ elapsed.Store(int64(2 * time.Minute))+ m.CheckHost(hostFilesystemGap(host, gap))+ if _, exists := continuityIncident(m, id, "disk"); exists {+ t.Fatal("observation gap retained timed evidence")+ }+ if _, exists := continuityIncident(m, hostFilesystemID(other), "disk"); !exists {+ t.Fatal("gap crossed the exact host boundary")+ }+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("first fresh sample completed the pre-gap run")+ }+ elapsed.Store(int64(2*time.Minute + 59*time.Second))+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("fresh run completed early")+ }+ elapsed.Store(int64(3 * time.Minute))+ m.CheckHost(host)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("full continuous fresh run did not complete")+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("interruption diverged from the shadow reducer")+ }+ })}}}diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.goindex 5d995179d4..d074a380e6 100644--- a/internal/alerts/host_unraid_lifecycle_test.go+++ b/internal/alerts/host_unraid_lifecycle_test.go@@ -1619,7 +1619,9 @@ func TestHostSMARTRiskFilesystemAndHostLifecycleStayIndependent(t *testing.T) {fsID, _ := hostDiskResourceID(f.host, f.host.Disks[0])id := canonicalMetricStateID(fsID, "disk")testRequireActiveAlert(t, f.m, id)- f.host.Disks = nil+ // A non-empty replacement establishes filesystem removal;+ // an empty report alone is now an observation gap.+ f.host.Disks = []models.Disk{{Mountpoint: "/other", Total: 1000}}f.host.Sensors.SMART = nilf.m.CheckHost(f.host)if testHasActiveAlert(t, f.m, id) {@@ -1690,7 +1692,9 @@ func TestHostSMARTRiskFilesystemIdentityCannotSubstituteInventory(t *testing.T)usageID := canonicalMetricStateID(diskResourceID, "disk")testRequireActiveAlert(t, f.m, usageID)if direction == "SMART-only" {- f.host.Disks = nil+ // A non-empty replacement establishes filesystem removal;+ // an empty report alone is now an observation gap.+ f.host.Disks = []models.Disk{{Mountpoint: "/other", Total: 1000}}f.m.CheckHost(f.host)if testHasActiveAlert(t, f.m, usageID) {t.Fatal("SMART identity held missing filesystem usage")diff --git a/internal/alerts/migration_characterization_test.go b/internal/alerts/migration_characterization_test.goindex d8814b3fa8..785d662a2b 100644--- a/internal/alerts/migration_characterization_test.go+++ b/internal/alerts/migration_characterization_test.go@@ -994,10 +994,10 @@ func TestAlertCharacterizationHeldMetricAlertRefreshesItsClassification(t *testiname: "agent filesystem",alertID: canonicalMetricStateID("agent:held-host/disk:srv", "disk"),fire: func(m *Manager) {- m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 92}}})+ m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Total: 1000, Used: 920, Usage: 92}}})},hold: func(m *Manager) {- m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 75}}})+ m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Total: 1000, Used: 750, Usage: 75}}})},stale: staleAlert{resourceType: "agent"},wantResourceType: "agent-disk",