Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep a disk heat finding open without a current reading
rcourtman396bb5d517f1
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.mdindex 216666a720..c6402b3694 100644--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md@@ -315,7 +315,8 @@ configuration applies. A disk is hot from its trigger: a triage warning, a diskissue and an AI chat "needing attention" entry, as its temperature alert andPhysical Disks Running Hot verdict are. A disk-high finding recovers at orbelow the clear value (under the trigger when there is no band below it), as-the alert does. Scoped runs keep the provider, and a+the alert does. With no current reading, a disk last seen hot leaves that+verification unknown rather than recovered. Scoped runs keep the provider, and aprovider set before Patrol starts is handed to it. Below the trigger Patrol flags nothing, so an NVMe at 63C staysquiet and a SATA disk at 56C is flagged. Proofs:`internal/ai/patrol_disk_temperature_test.go` anddiff --git a/internal/ai/patrol_disk_temperature_test.go b/internal/ai/patrol_disk_temperature_test.goindex e38358380e..d8c433eae4 100644--- a/internal/ai/patrol_disk_temperature_test.go+++ b/internal/ai/patrol_disk_temperature_test.go@@ -1,12 +1,15 @@package aiimport (+ "errors""strings""testing""time""github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory")// raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger@@ -226,3 +229,21 @@ func TestDiskHeatRecoversAtTheClearValue(t *testing.T) {t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand)}}++// With no current reading (standby, a silent agent) a disk last seen hot has+// not shown that it cooled, so heat recovery is unknown rather than proven.+func TestDiskHeatRecoveryNeedsACurrentReading(t *testing.T) {+ retained := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}+ state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{+ {ID: "sata-retained-hot", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, Collection: retained},+ {ID: "sata-retained-cool", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 40, Collection: retained},+ }})++ if _, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-hot", "physical_disk"); !errors.Is(err, aicontracts.ErrVerificationUnknown) {+ t.Fatalf("retained 56C: err = %v, want ErrVerificationUnknown", err)+ }+ recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-cool", "physical_disk")+ if err != nil || !recovered {+ t.Fatalf("retained 40C: recovered = %v, err = %v, want recovered", recovered, err)+ }+}diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.goindex 1bb5e87af0..d4f5fbc2b7 100644--- a/internal/ai/patrol_findings.go+++ b/internal/ai/patrol_findings.go@@ -2296,6 +2296,11 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThreshif !disk.temperatureLimits.cooled(disk.temperature) {return false, nil}+ // A disk last seen hot with no current reading has not+ // shown that it cooled, so its recovery is unknown.+ if disk.temperature <= 0 && !disk.temperatureLimits.cooled(disk.lastKnownTemperature) {+ return false, fmt.Errorf("%w: no current temperature reading for %s", aicontracts.ErrVerificationUnknown, rid)+ }return true, nil}break@@ -2317,10 +2322,13 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh}type patrolPhysicalDiskVerification struct {- health string- wearout int- temperature int- temperatureLimits diskTemperatureLimits+ health string+ wearout int+ temperature int+ // lastKnownTemperature is a retained reading the current observation+ // did not collect (a disk in standby, a silent agent).+ lastKnownTemperature int+ temperatureLimits diskTemperatureLimits}type patrolPhysicalDiskVisitor func(identifiers []string, verification patrolPhysicalDiskVerification) bool@@ -2464,10 +2472,11 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPhrows := patrolPhysicalDiskRows(snap, nil)for _, disk := range rows {if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{- health: strings.TrimSpace(disk.health),- wearout: disk.wearout,- temperature: disk.temperature.Collected,- temperatureLimits: disk.temperatureLimits,+ health: strings.TrimSpace(disk.health),+ wearout: disk.wearout,+ temperature: disk.temperature.Collected,+ lastKnownTemperature: disk.temperature.LastKnown,+ temperatureLimits: disk.temperatureLimits,}) {return true}