Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Judge disk risk only on a temperature collected now

rcourtman28d5f4425712
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 6522d39af6..4804448c5e 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -4327,6 +4327,17 @@ emit structured storage topology such as Unraid per-disk state, the shared
assessment layer must derive canonical risk and alert severity from that
richer disk topology instead of letting coarser aggregate counters override it
and flap the operator-facing storage alert surface.
+That shared assessment judges only a disk temperature that was collected now.
+`storagehealth.CollectedTemperature` passes a reading through when its
+`collection.temperature` state is `available`, or when the source predates
+collection state, and otherwise drops it before `AssessPhysicalDisk` and
+`AssessHostSMARTDisk` build their sample. A last known value retained for a
+disk in standby or a host agent past its reporting lease therefore raises no
+`temperature_high` reason and cannot turn the disk's verdict or status into
+a warning, while the value itself stays on the resource as last-known
+history. Proof: `TestDiskAssessmentsIgnoreRetainedTemperature` and
+`TestCollectedTemperatureKeepsOnlyCurrentReadings` in
+`internal/storagehealth/risk_test.go`.
That same monitoring-owned storage polling boundary also owns cluster-shared
Proxmox storage status coherence. `internal/monitoring/monitor_polling_storage.go`
must merge shared storage observations across nodes into one cluster-scoped
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index 5c1dfdb1bb..cffb750018 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -4123,6 +4123,14 @@ incidents during cross-source merges. A provider alert such as TrueNAS
`truenas_smart` is not presentation-only context; it must become a canonical
`physicalDisk.risk.reasons` entry so hybrid agent/API disk resources keep one
shared disk-health truth after deduplication.
+That shared risk contract judges only a collected temperature. Both the
+adapter assessment and the registry's recompute from merged metadata
+(`physicalDiskAssessmentFromMeta`) pass the temperature through
+`storagehealth.CollectedTemperature`, so a retained `unavailable`,
+`unsupported` or `missing` temperature stays visible as last-known evidence
+but adds no `temperature_high` reason and leaves the disk `online` rather
+than `warning`. Proof: `TestPhysicalDiskRiskIgnoresRetainedTemperature` in
+`internal/unifiedresources/registry_test.go`.
That same canonical disk contract now also owns recent aggregate temperature
history. When a provider such as TrueNAS can supply `disk.temperature_agg`
min/avg/max readings, it must project those onto
diff --git a/internal/storagehealth/risk.go b/internal/storagehealth/risk.go
index 1c618b8bc4..de165bbb5b 100644
--- a/internal/storagehealth/risk.go
+++ b/internal/storagehealth/risk.go
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
type RiskLevel string
@@ -76,11 +77,29 @@ type Sample struct {
UnsafeShutdowns int64
}
+// CollectedTemperature returns a disk temperature only when its collection
+// state marks it as observed now. Normalization may keep the last known value
+// when the current observation is not available, such as a disk in standby or
+// a host agent past its reporting lease. That retained value is history, not
+// evidence that the disk is hot now, so risk must not judge it. A source that
+// predates collection state counts as collected.
+func CollectedTemperature(temperature int, collection *diskinventory.CollectionStatus) int {
+ if collection == nil {
+ return temperature
+ }
+ switch collection.Temperature.State {
+ case "", diskinventory.FieldAvailable:
+ return temperature
+ default:
+ return 0
+ }
+}
+
func AssessPhysicalDisk(disk models.PhysicalDisk) Assessment {
sample := Sample{
Model: disk.Model,
Health: disk.Health,
- Temperature: disk.Temperature,
+ Temperature: CollectedTemperature(disk.Temperature, disk.Collection),
Wearout: disk.Wearout,
WearoutKnown: WearoutReported(disk.Wearout, disk.Type),
}
@@ -96,7 +115,7 @@ func AssessHostSMARTDiskWithThresholds(disk models.HostDiskSMART, thresholds SMA
sample := Sample{
Model: disk.Model,
Health: disk.Health,
- Temperature: disk.Temperature,
+ Temperature: CollectedTemperature(disk.Temperature, disk.Collection),
Wearout: -1,
}
applySMARTAttributes(&sample, disk.Attributes)
diff --git a/internal/storagehealth/risk_test.go b/internal/storagehealth/risk_test.go
index 44bdde444d..e044189efd 100644
--- a/internal/storagehealth/risk_test.go
+++ b/internal/storagehealth/risk_test.go
@@ -4,6 +4,7 @@ import (
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// --- AssessSample: comprehensive branch coverage ---
@@ -538,3 +539,59 @@ func TestAssessPhysicalDisk_RotationalZeroIsNotRisk(t *testing.T) {
t.Error("a rotational disk reporting 0 wearout reports no endurance at all and must not assess critical")
}
}
+
+func TestCollectedTemperatureKeepsOnlyCurrentReadings(t *testing.T) {
+ cases := []struct {
+ name string
+ collection *diskinventory.CollectionStatus
+ want int
+ }{
+ {name: "no collection state", collection: nil, want: 72},
+ {name: "empty state", collection: &diskinventory.CollectionStatus{}, want: 72},
+ {name: "available", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, want: 72},
+ {name: "unavailable", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, want: 0},
+ {name: "unsupported", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "")}, want: 0},
+ {name: "missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, want: 0},
+ }
+ for _, tc := range cases {
+ if got := CollectedTemperature(72, tc.collection); got != tc.want {
+ t.Errorf("%s: CollectedTemperature = %d, want %d", tc.name, got, tc.want)
+ }
+ }
+}
+
+func TestDiskAssessmentsIgnoreRetainedTemperature(t *testing.T) {
+ retained := &diskinventory.CollectionStatus{
+ Temperature: diskinventory.Unavailable("host_agent", "host agent stopped reporting"),
+ }
+ collected := &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}
+
+ hasTemperatureReason := func(assessment Assessment) bool {
+ for _, reason := range assessment.Reasons {
+ if reason.Code == "temperature_high" {
+ return true
+ }
+ }
+ return false
+ }
+
+ disk := models.PhysicalDisk{Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 72}
+ disk.Collection = collected
+ if got := AssessPhysicalDisk(disk); got.Level != RiskCritical || !hasTemperatureReason(got) {
+ t.Fatalf("collected 72C physical disk = %+v, want critical temperature_high", got)
+ }
+ disk.Collection = retained
+ if got := AssessPhysicalDisk(disk); got.Level != RiskHealthy || hasTemperatureReason(got) {
+ t.Fatalf("retained 72C physical disk = %+v, want healthy", got)
+ }
+
+ smart := models.HostDiskSMART{Device: "/dev/sda", Model: "WDC WD80EFAX", Type: "sata", Health: "PASSED", Temperature: 72}
+ smart.Collection = collected
+ if got := AssessHostSMARTDisk(smart); got.Level != RiskCritical || !hasTemperatureReason(got) {
+ t.Fatalf("collected 72C host SMART disk = %+v, want critical temperature_high", got)
+ }
+ smart.Collection = retained
+ if got := AssessHostSMARTDisk(smart); got.Level != RiskHealthy || hasTemperatureReason(got) {
+ t.Fatalf("retained 72C host SMART disk = %+v, want healthy", got)
+ }
+}
diff --git a/internal/unifiedresources/physical_disk_risk.go b/internal/unifiedresources/physical_disk_risk.go
index 2cb8007a46..444a62af1d 100644
--- a/internal/unifiedresources/physical_disk_risk.go
+++ b/internal/unifiedresources/physical_disk_risk.go
@@ -82,7 +82,7 @@ func physicalDiskAssessmentFromMeta(meta *PhysicalDiskMeta) storagehealth.Assess
sample := storagehealth.Sample{
Model: meta.Model,
Health: meta.Health,
- Temperature: meta.Temperature,
+ Temperature: storagehealth.CollectedTemperature(meta.Temperature, meta.Collection),
Wearout: meta.Wearout,
WearoutKnown: storagehealth.WearoutReported(meta.Wearout, meta.DiskType),
}
diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.go
index 8eacc8f37b..0256304351 100644
--- a/internal/unifiedresources/registry_test.go
+++ b/internal/unifiedresources/registry_test.go
@@ -10,6 +10,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
func TestRegistry_CachedReadsUseSharedLock(t *testing.T) {
@@ -7033,3 +7034,155 @@ func TestRegistryProjectionSnapshotDetachesAndTracksMappings(t *testing.T) {
t.Fatal("empty projection invented resources or targets")
}
}
+
+// A retained temperature is last-known history, not evidence that the disk is
+// hot now. The risk verdict (and the warning status it drives) must judge only
+// a temperature whose collection state says it was observed now.
+func TestPhysicalDiskRiskIgnoresRetainedTemperature(t *testing.T) {
+ // A fresh check time keeps the registry's source-staleness sweep out of
+ // the status this test reads.
+ now := time.Now()
+ retained := diskinventory.Unavailable("host_agent", "host agent stopped reporting")
+
+ cases := []struct {
+ name string
+ collection *diskinventory.CollectionStatus
+ wantHot bool
+ }{
+ {name: "collected now", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("proxmox_node_smart")}, wantHot: true},
+ {name: "source predates collection state", collection: nil, wantHot: true},
+ {name: "state left empty", collection: &diskinventory.CollectionStatus{}, wantHot: true},
+ {name: "retained after the agent went silent", collection: &diskinventory.CollectionStatus{Temperature: retained}, wantHot: false},
+ {name: "retained while in standby", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}, wantHot: false},
+ {name: "retained after the source stopped exposing it", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")}, wantHot: false},
+ {name: "retained while unexpectedly missing", collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Missing("smartctl", "")}, wantHot: false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ adapter := NewMonitorAdapter(NewRegistry(nil))
+ adapter.PopulateFromSnapshot(models.StateSnapshot{
+ PhysicalDisks: []models.PhysicalDisk{{
+ ID: "homelab-pve--dev-sda",
+ Node: "pve",
+ Instance: "homelab",
+ DevPath: "/dev/sda",
+ Model: "WDC WD80EFAX",
+ Serial: "SERIAL-SDA",
+ Type: "sata",
+ Health: "PASSED",
+ Wearout: -1,
+ Temperature: 72,
+ Collection: tc.collection,
+ LastChecked: now,
+ }},
+ })
+ var disks []Resource
+ for _, resource := range adapter.GetAll() {
+ if resource.Type == ResourceTypePhysicalDisk {
+ disks = append(disks, resource)
+ }
+ }
+ if len(disks) != 1 || disks[0].PhysicalDisk == nil {
+ t.Fatalf("physical disks = %+v, want one", disks)
+ }
+ disk := disks[0]
+ if disk.PhysicalDisk.Temperature != 72 {
+ t.Fatalf("temperature = %d, want the reading kept at 72", disk.PhysicalDisk.Temperature)
+ }
+ hot := false
+ if disk.PhysicalDisk.Risk != nil {
+ for _, reason := range disk.PhysicalDisk.Risk.Reasons {
+ hot = hot || reason.Code == "temperature_high"
+ }
+ }
+ if hot != tc.wantHot {
+ t.Fatalf("temperature_high = %v, want %v (risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Risk)
+ }
+ if tc.wantHot && disk.Status != StatusWarning {
+ t.Fatalf("status = %q, want warning for a collected 72C", disk.Status)
+ }
+ if !tc.wantHot && (disk.PhysicalDisk.Risk != nil || disk.Status != StatusOnline) {
+ t.Fatalf("retained 72C: status = %q risk = %+v, want online with no risk", disk.Status, disk.PhysicalDisk.Risk)
+ }
+ })
+ }
+
+ // A host agent's SMART row merged with the Proxmox inventory row for the
+ // same disk: the agent's retained reading keeps its unavailable state
+ // through the registry merge and recompute, so it stays out of risk.
+ for _, tc := range []struct {
+ name string
+ state diskinventory.FieldStatus
+ wantHot bool
+ }{
+ {name: "agent reading collected", state: diskinventory.Available("host_agent"), wantHot: true},
+ {name: "agent reading retained", state: retained, wantHot: false},
+ } {
+ t.Run("merged "+tc.name, func(t *testing.T) {
+ adapter := NewMonitorAdapter(NewRegistry(nil))
+ adapter.PopulateFromSnapshot(models.StateSnapshot{
+ Hosts: []models.Host{{
+ ID: "host-pve",
+ Hostname: "pve",
+ Status: "online",
+ LastSeen: now,
+ Sensors: models.HostSensorSummary{SMART: []models.HostDiskSMART{{
+ Device: "/dev/sda",
+ Model: "WDC WD80EFAX",
+ Serial: "SERIAL-SDA",
+ Type: "sata",
+ Temperature: 72,
+ Health: "PASSED",
+ Collection: &diskinventory.CollectionStatus{Temperature: tc.state},
+ }}},
+ }},
+ PhysicalDisks: []models.PhysicalDisk{{
+ ID: "homelab-pve--dev-sda",
+ Node: "pve",
+ Instance: "homelab",
+ DevPath: "/dev/sda",
+ Model: "WDC WD80EFAX",
+ Serial: "SERIAL-SDA",
+ Type: "sata",
+ Health: "PASSED",
+ Wearout: -1,
+ Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Unsupported("proxmox_disks", "Proxmox disk inventory does not expose temperature")},
+ LastChecked: now,
+ }},
+ })
+ var disks []Resource
+ for _, resource := range adapter.GetAll() {
+ if resource.Type == ResourceTypePhysicalDisk {
+ disks = append(disks, resource)
+ }
+ }
+ if len(disks) != 1 || disks[0].PhysicalDisk == nil {
+ t.Fatalf("physical disks = %d, want the agent and Proxmox rows merged into one", len(disks))
+ }
+ disk := disks[0]
+ if disk.PhysicalDisk.Temperature != 72 {
+ t.Fatalf("merged temperature = %d, want 72", disk.PhysicalDisk.Temperature)
+ }
+ hot := false
+ if disk.PhysicalDisk.Risk != nil {
+ for _, reason := range disk.PhysicalDisk.Risk.Reasons {
+ hot = hot || reason.Code == "temperature_high"
+ }
+ }
+ if hot != tc.wantHot {
+ t.Fatalf("merged temperature_high = %v, want %v (collection %+v, risk %+v)", hot, tc.wantHot, disk.PhysicalDisk.Collection, disk.PhysicalDisk.Risk)
+ }
+ })
+ }
+
+ // The registry recomputes risk from merged metadata after a source merge;
+ // that path follows the same rule.
+ meta := &PhysicalDiskMeta{Health: "PASSED", Temperature: 72, Collection: &diskinventory.CollectionStatus{Temperature: retained}}
+ if risk := physicalDiskRiskFromMeta(meta, nil); risk != nil {
+ t.Fatalf("meta recompute risk = %+v, want none for a retained temperature", risk)
+ }
+ meta.Collection = &diskinventory.CollectionStatus{Temperature: diskinventory.Available("host_agent")}
+ if risk := physicalDiskRiskFromMeta(meta, nil); risk == nil || risk.Level != storagehealth.RiskCritical {
+ t.Fatalf("meta recompute risk = %+v, want critical for a collected 72C", risk)
+ }
+}