Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep a disk heat finding open without a current reading

rcourtman396bb5d517f1
diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md
index 216666a720..c6402b3694 100644
--- a/docs/release-control/v6/internal/subsystems/ai-runtime.md
+++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md
@@ -315,7 +315,8 @@ configuration applies. A disk is hot from its trigger: a triage warning, a disk
issue and an AI chat "needing attention" entry, as its temperature alert and
Physical Disks Running Hot verdict are. A disk-high finding recovers at or
below the clear value (under the trigger when there is no band below it), as
-the alert does. Scoped runs keep the provider, and a
+the alert does. With no current reading, a disk last seen hot leaves that
+verification unknown rather than recovered. Scoped runs keep the provider, and a
provider set before Patrol starts is handed to it. Below the trigger Patrol flags nothing, so an NVMe at 63C stays
quiet and a SATA disk at 56C is flagged. Proofs:
`internal/ai/patrol_disk_temperature_test.go` and
diff --git a/internal/ai/patrol_disk_temperature_test.go b/internal/ai/patrol_disk_temperature_test.go
index e38358380e..d8c433eae4 100644
--- a/internal/ai/patrol_disk_temperature_test.go
+++ b/internal/ai/patrol_disk_temperature_test.go
@@ -1,12 +1,15 @@
package ai
import (
+ "errors"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
)
// raisedNVMeThresholds is a user who raised the NVMe disk temperature trigger
@@ -226,3 +229,21 @@ func TestDiskHeatRecoversAtTheClearValue(t *testing.T) {
t.Fatalf("no-band limits %+v: want 70C hot and not cooled, 69C cooled", noBand)
}
}
+
+// With no current reading (standby, a silent agent) a disk last seen hot has
+// not shown that it cooled, so heat recovery is unknown rather than proven.
+func TestDiskHeatRecoveryNeedsACurrentReading(t *testing.T) {
+ retained := &diskinventory.CollectionStatus{Temperature: diskinventory.Unavailable("smartctl", "disk is in standby")}
+ state := newPatrolRuntimeState(models.StateSnapshot{PhysicalDisks: []models.PhysicalDisk{
+ {ID: "sata-retained-hot", DevPath: "/dev/sda", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 56, Collection: retained},
+ {ID: "sata-retained-cool", DevPath: "/dev/sdb", Type: "sata", Health: "PASSED", Wearout: -1, Temperature: 40, Collection: retained},
+ }})
+
+ if _, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-hot", "physical_disk"); !errors.Is(err, aicontracts.ErrVerificationUnknown) {
+ t.Fatalf("retained 56C: err = %v, want ErrVerificationUnknown", err)
+ }
+ recovered, err := verifyMetricRecoveredState(state, PatrolThresholds{}, "disk-high", "sata-retained-cool", "physical_disk")
+ if err != nil || !recovered {
+ t.Fatalf("retained 40C: recovered = %v, err = %v, want recovered", recovered, err)
+ }
+}
diff --git a/internal/ai/patrol_findings.go b/internal/ai/patrol_findings.go
index 1bb5e87af0..d4f5fbc2b7 100644
--- a/internal/ai/patrol_findings.go
+++ b/internal/ai/patrol_findings.go
@@ -2296,6 +2296,11 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh
if !disk.temperatureLimits.cooled(disk.temperature) {
return false, nil
}
+ // A disk last seen hot with no current reading has not
+ // shown that it cooled, so its recovery is unknown.
+ if disk.temperature <= 0 && !disk.temperatureLimits.cooled(disk.lastKnownTemperature) {
+ return false, fmt.Errorf("%w: no current temperature reading for %s", aicontracts.ErrVerificationUnknown, rid)
+ }
return true, nil
}
break
@@ -2317,10 +2322,13 @@ func verifyMetricRecoveredState(snap patrolRuntimeState, thresholds PatrolThresh
}
type patrolPhysicalDiskVerification struct {
- health string
- wearout int
- temperature int
- temperatureLimits diskTemperatureLimits
+ health string
+ wearout int
+ temperature int
+ // lastKnownTemperature is a retained reading the current observation
+ // did not collect (a disk in standby, a silent agent).
+ lastKnownTemperature int
+ temperatureLimits diskTemperatureLimits
}
type patrolPhysicalDiskVisitor func(identifiers []string, verification patrolPhysicalDiskVerification) bool
@@ -2464,10 +2472,11 @@ func patrolVisitPhysicalDiskVerification(snap patrolRuntimeState, visit patrolPh
rows := patrolPhysicalDiskRows(snap, nil)
for _, disk := range rows {
if !visit([]string{disk.id, disk.name, disk.devPath, disk.model}, patrolPhysicalDiskVerification{
- health: strings.TrimSpace(disk.health),
- wearout: disk.wearout,
- temperature: disk.temperature.Collected,
- temperatureLimits: disk.temperatureLimits,
+ health: strings.TrimSpace(disk.health),
+ wearout: disk.wearout,
+ temperature: disk.temperature.Collected,
+ lastKnownTemperature: disk.temperature.LastKnown,
+ temperatureLimits: disk.temperatureLimits,
}) {
return true
}