Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Judge disk risk only on a temperature collected now
rcourtman28d5f4425712
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 6522d39af6..4804448c5e 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -4327,6 +4327,17 @@ emit structured storage topology such as Unraid per-disk state, the sharedassessment layer must derive canonical risk and alert severity from thatricher disk topology instead of letting coarser aggregate counters override itand flap the operator-facing storage alert surface.+That shared assessment judges only a disk temperature that was collected now.+`storagehealth.CollectedTemperature` passes a reading through when its+`collection.temperature` state is `available`, or when the source predates+collection state, and otherwise drops it before `AssessPhysicalDisk` and+`AssessHostSMARTDisk` build their sample. A last known value retained for a+disk in standby or a host agent past its reporting lease therefore raises no+`temperature_high` reason and cannot turn the disk's verdict or status into+a warning, while the value itself stays on the resource as last-known+history. Proof: `TestDiskAssessmentsIgnoreRetainedTemperature` and+`TestCollectedTemperatureKeepsOnlyCurrentReadings` in+`internal/storagehealth/risk_test.go`.That same monitoring-owned storage polling boundary also owns cluster-sharedProxmox storage status coherence. `internal/monitoring/monitor_polling_storage.go`must merge shared storage observations across nodes into one cluster-scopeddiff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex 5c1dfdb1bb..cffb750018 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -4123,6 +4123,14 @@ incidents during cross-source merges. A provider alert such as TrueNAS`truenas_smart` is not presentation-only context; it must become a canonical`physicalDisk.risk.reasons` entry so hybrid agent/API disk resources keep oneshared disk-health truth after deduplication.+That shared risk contract judges only a collected temperature. Both the+adapter assessment and the registry's recompute from merged metadata+(`physicalDiskAssessmentFromMeta`) pass the temperature through+`storagehealth.CollectedTemperature`, so a retained `unavailable`,+`unsupported` or `missing` temperature stays visible as last-known evidence+but adds no `temperature_high` reason and leaves the disk `online` rather+than `warning`. Proof: `TestPhysicalDiskRiskIgnoresRetainedTemperature` in+`internal/unifiedresources/registry_test.go`.That same canonical disk contract now also owns recent aggregate temperaturehistory. When a provider such as TrueNAS can supply `disk.temperature_agg`min/avg/max readings, it must project those ontodiff --git a/internal/storagehealth/risk.go b/internal/storagehealth/risk.goindex 1c618b8bc4..de165bbb5b 100644--- a/internal/storagehealth/risk.go+++ b/internal/storagehealth/risk.go@@ -6,6 +6,7 @@ import ("strings""github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory")type RiskLevel string@@ -76,11 +77,29 @@ type Sample struct {UnsafeShutdowns int64}+// CollectedTemperature returns a disk temperature only when its collection+// state marks it as observed now. Normalization may keep the last known value+// when the current observation is not available, such as a disk in standby or+// a host agent past its reporting lease. That retained value is history, not+// evidence that the disk is hot now, so risk must not judge it. A source that+// predates collection state counts as collected.+func CollectedTemperature(temperature int, collection *diskinventory.CollectionStatus) int {+ if collection == nil {+ return temperature+ }+ switch collection.Temperature.State {+ case "", diskinventory.FieldAvailable:+ return temperature+ default:+ return 0+ }+}+func AssessPhysicalDisk(disk models.PhysicalDisk) Assessment {sample := Sample{Model: disk.Model,Health: disk.Health,- Temperature: disk.Temperature,+ Temperature: CollectedTemperature(disk.Temperature, disk.Collection),Wearout: disk.Wearout,WearoutKnown: WearoutReported(disk.Wearout, disk.Type),}@@ -96,7 +115,7 @@ func AssessHostSMARTDiskWithThresholds(disk models.HostDiskSMART, thresholds SMAsample := Sample{Model: disk.Model,Health: disk.Health,- Temperature: disk.Temperature,+ Temperature: CollectedTemperature(disk.Temperature, disk.Collection),Wearout: -1,}applySMARTAttributes(&sample, disk.Attributes)diff --git a/internal/storagehealth/risk_test.go b/internal/storagehealth/risk_test.goindex 44bdde444d..e044189efd 100644--- a/internal/storagehealth/risk_test.go+++ b/internal/storagehealth/risk_test.go@@ -4,6 +4,7 @@ import ("testing""github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory")// --- AssessSample: comprehensive branch coverage ---@@ -538,3 +539,59 @@ func TestAssessPhysicalDisk_RotationalZeroIsNotRisk(t *testing.T) {t.Error("a rotational disk reporting 0 wearout reports no endurance at all and must not assess critical")}}++func TestCollectedTemperatureKeepsOnlyCurrentReadings(t *testing.T) {+ cases := []struct {+ name string+ collection *diskinventory.CollectionStatus+ want int+ }{+ {name: "no collection state", collection: nil, want: 72},+ {name: "empty state", collection: &diskinventory.CollectionStatus{}, want: 72},+ {name: "available", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, want: 72},+ {name: "unavailable", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, want: 0},+ {name: "unsupported", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "")}, want: 0},+ {name: "missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, want: 0},+ }+ for _, tc := range cases {+ if got := CollectedTemperature(72, tc.collection); got != tc.want {+ t.Errorf("%s: CollectedTemperature = %d, want %d", tc.name, got, tc.want)+ }+ }+}++func TestDiskAssessmentsIgnoreRetainedTemperature(t *testing.T) {+ retained := &diskinventory.CollectionStatus{+ Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting"),+ }+ collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}++ hasTemperatureReason := func(assessment Assessment) bool {+ for _, reason := range assessment.Reasons {+ if reason.Code == "temperature_high" {+ return true+ }+ }+ return false+ }++ disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 72}+ disk.Collection = collected+ if got := AssessPhysicalDisk(disk); got.Level != RiskCritical || !hasTemperatureReason(got) {+ t.Fatalf("collected 72C physical disk = %+v, want critical temperature_high", got)+ }+ disk.Collection = retained+ if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || hasTemperatureReason(got) {+ t.Fatalf("retained 72C physical disk = %+v, want healthy", got)+ }++ smart := models.HostDiskSMART{Device: "/dev/sda", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Temperature: 72}+ smart.Collection = collected+ if got := AssessHostSMARTDisk(smart); got.Level != RiskCritical || !hasTemperatureReason(got) {+ t.Fatalf("collected 72C host SMART disk = %+v, want critical temperature_high", got)+ }+ smart.Collection = retained+ if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || hasTemperatureReason(got) {+ t.Fatalf("retained 72C host SMART disk = %+v, want healthy", got)+ }+}diff --git a/internal/unifiedresources/physical_disk_risk.go b/internal/unifiedresources/physical_disk_risk.goindex 2cb8007a46..444a62af1d 100644--- a/internal/unifiedresources/physical_disk_risk.go+++ b/internal/unifiedresources/physical_disk_risk.go@@ -82,7 +82,7 @@ func physicalDiskAssessmentFromMeta(meta *PhysicalDiskMeta) storagehealth.Assesssample := storagehealth.Sample{Model: meta.Model,Health: meta.Health,- Temperature: meta.Temperature,+ Temperature: storagehealth.CollectedTemperature(meta.Temperature, meta.Collection),Wearout: meta.Wearout,WearoutKnown: storagehealth.WearoutReported(meta.Wearout, meta.DiskType),}diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.goindex 8eacc8f37b..0256304351 100644--- a/internal/unifiedresources/registry_test.go+++ b/internal/unifiedresources/registry_test.go@@ -10,6 +10,7 @@ import ("github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory")func TestRegistry_CachedReadsUseSharedLock(t *testing.T) {@@ -7033,3 +7034,155 @@ func TestRegistryProjectionSnapshotDetachesAndTracksMappings(t *testing.T) {t.Fatal("empty projection invented resources or targets")}}++// A retained temperature is last-known history, not evidence that the disk is+// hot now. The risk verdict (and the warning status it drives) must judge only+// a temperature whose collection state says it was observed now.+func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) {+ // A fresh check time keeps the registry's source-staleness sweep out of+ // the status this test reads.+ now := time.Now()+ retained := diskinventory.Unavailable("host_agent", "host agent stopped reporting")++ cases := []struct {+ name string+ collection *diskinventory.CollectionStatus+ wantHot bool+ }{+ {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}, wantHot: true},+ {name: "source predates collection state", collection: nil, wantHot: true},+ {name: "state left empty", collection: &diskinventory.CollectionStatus{}, wantHot: true},+ {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}, wantHot: false},+ {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, wantHot: false},+ {name: "retained after the source stopped exposing it", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, wantHot: false},+ {name: "retained while unexpectedly missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, wantHot: false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ adapter := NewMonitorAdapter(NewRegistry(nil))+ adapter.PopulateFromSnapshot(models.StateSnapshot{+ PhysicalDisks: []models.PhysicalDisk{{+ ID: "homelab-pve--dev-sda",+ Node: "pve",+ Instance: "homelab",+ DevPath: "/dev/sda",+ Model: "WDC WD80EFAX",+ Serial: "SERIAL-SDA",+ Type: "sata",+ Health: "PASSED",+ Wearout: -1,+ Temperature: 72,+ Collection: tc.collection,+ LastChecked: now,+ }},+ })+ var disks []Resource+ for _, resource := range adapter.GetAll() {+ if resource.Type == ResourceTypePhysicalDisk {+ disks = append(disks, resource)+ }+ }+ if len(disks) != 1 || disks[0].PhysicalDisk == nil {+ t.Fatalf("physical disks = %+v, want one", disks)+ }+ disk := disks[0]+ if disk.PhysicalDisk.Temperature != 72 {+ t.Fatalf("temperature = %d, want the reading kept at 72", disk.PhysicalDisk.Temperature)+ }+ hot := false+ if disk.PhysicalDisk.Risk != nil {+ for _, reason := range disk.PhysicalDisk.Risk.Reasons {+ hot = hot || reason.Code == "temperature_high"+ }+ }+ if hot != tc.wantHot {+ t.Fatalf("temperature_high = %v, want %v (risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Risk)+ }+ if tc.wantHot && disk.Status != StatusWarning {+ t.Fatalf("status = %q, want warning for a collected 72C", disk.Status)+ }+ if !tc.wantHot && (disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline) {+ t.Fatalf("retained 72C: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk)+ }+ })+ }++ // A host agent's SMART row merged with the Proxmox inventory row for the+ // same disk: the agent's retained reading keeps its unavailable state+ // through the registry merge and recompute, so it stays out of risk.+ for _, tc := range []struct {+ name string+ state diskinventory.FieldStatus+ wantHot bool+ }{+ {name: "agent reading collected", state: diskinventory.Available("host_agent"), wantHot: true},+ {name: "agent reading retained", state: retained, wantHot: false},+ } {+ t.Run("merged "+tc.name, func(t *testing.T) {+ adapter := NewMonitorAdapter(NewRegistry(nil))+ adapter.PopulateFromSnapshot(models.StateSnapshot{+ Hosts: []models.Host{{+ ID: "host-pve",+ Hostname: "pve",+ Status: "online",+ LastSeen: now,+ Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{+ Device: "/dev/sda",+ Model: "WDC WD80EFAX",+ Serial: "SERIAL-SDA",+ Type: "sata",+ Temperature: 72,+ Health: "PASSED",+ Collection: &diskinventory.CollectionStatus{Temperature: tc.state},+ }}},+ }},+ PhysicalDisks: []models.PhysicalDisk{{+ ID: "homelab-pve--dev-sda",+ Node: "pve",+ Instance: "homelab",+ DevPath: "/dev/sda",+ Model: "WDC WD80EFAX",+ Serial: "SERIAL-SDA",+ Type: "sata",+ Health: "PASSED",+ Wearout: -1,+ Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")},+ LastChecked: now,+ }},+ })+ var disks []Resource+ for _, resource := range adapter.GetAll() {+ if resource.Type == ResourceTypePhysicalDisk {+ disks = append(disks, resource)+ }+ }+ if len(disks) != 1 || disks[0].PhysicalDisk == nil {+ t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(disks))+ }+ disk := disks[0]+ if disk.PhysicalDisk.Temperature != 72 {+ t.Fatalf("merged temperature = %d, want 72", disk.PhysicalDisk.Temperature)+ }+ hot := false+ if disk.PhysicalDisk.Risk != nil {+ for _, reason := range disk.PhysicalDisk.Risk.Reasons {+ hot = hot || reason.Code == "temperature_high"+ }+ }+ if hot != tc.wantHot {+ t.Fatalf("merged temperature_high = %v, want %v (collection %+v, risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Collection, disk.PhysicalDisk.Risk)+ }+ })+ }++ // The registry recomputes risk from merged metadata after a source merge;+ // that path follows the same rule.+ meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: retained}}+ if risk := physicalDiskRiskFromMeta(meta, nil); risk != nil {+ t.Fatalf("meta recompute risk = %+v, want none for a retained temperature", risk)+ }+ meta.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}+ if risk := physicalDiskRiskFromMeta(meta, nil); risk == nil || risk.Level != storagehealth.RiskCritical {+ t.Fatalf("meta recompute risk = %+v, want critical for a collected 72C", risk)+ }+}