Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep typed node temperature outside usage gap detection

pulse-triage[bot]796da48cf994
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 7d1ae4a7be..951bad977d 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -58,7 +58,10 @@ identities, without implying filesystem removal. Different guests, instances,
metrics and independently current CPU/I/O remain independent. Host memory also
interrupts timing when its usage is unknown; explicit rule disablement still
uses its normal clearing path independently of telemetry. Node temperature's
-existing interruption rule is unchanged.
+existing interruption rule is unchanged. Typed metric dispatchers must exclude
+metrics they evaluate separately from their own missing-value checks: node
+usage dispatch does not infer a temperature gap before the node's independent
+temperature evaluator observes its actual reading.
`metric_observation_continuity_test.go` verifies actual legacy/canonical,
unified, host and VM entry points, full-delay fresh activation/recovery,
diff --git a/internal/alerts/node.go b/internal/alerts/node.go
index 56fb559735..978fa9aad2 100644
--- a/internal/alerts/node.go
+++ b/internal/alerts/node.go
@@ -112,7 +112,7 @@ func (m *Manager) CheckNode(node models.Node) {
CPU: &UnifiedResourceMetric{Percent: node.CPU * 100},
Memory: memoryMetric,
Disk: &UnifiedResourceMetric{Percent: node.Disk.Usage},
- }, thresholds, nil)
+ }, thresholds, nil, "temperature") // checkNodeTemperature owns its observation gaps.
}
// CPU temperature stays with the node even when a host agent runs on it:
diff --git a/internal/alerts/unified_eval.go b/internal/alerts/unified_eval.go
index 1fe2d5e09c..74796bd896 100644
--- a/internal/alerts/unified_eval.go
+++ b/internal/alerts/unified_eval.go
@@ -275,7 +275,7 @@ func (m *Manager) resolveResourceThresholds(typeKey, resourceID string) Threshol
}
// evaluateUnifiedMetrics runs the common metric dispatch path for unified resources.
-func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds ThresholdConfig, opts *metricOptions) {
+func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds ThresholdConfig, opts *metricOptions, separatelyEvaluated ...string) {
if input == nil {
return
}
@@ -301,7 +301,7 @@ func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds
opts = &merged
}
candidates := buildUnifiedMetricCandidates(input, thresholds)
- m.interruptUnobservedUnifiedMetrics(input, candidates)
+ m.interruptUnobservedUnifiedMetrics(input, candidates, separatelyEvaluated)
for _, candidate := range candidates {
m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)
}
@@ -310,7 +310,7 @@ func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds
// An omitted/rejected metric is unknown, not a pause in its sustained-for
// clock. Build candidates stays pure; dispatch owns interruption for metrics
// this resource supports but cannot currently observe.
-func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput, candidates []unifiedMetricCandidate) {
+func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput, candidates []unifiedMetricCandidate, separatelyEvaluated []string) {
if _, ok := unifiedMetricResourceType(input.Type); !ok {
return
}
@@ -318,6 +318,16 @@ func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput,
m.mu.Lock()
intentChanged := false
for _, metric := range metrics {
+ separateOwner := false
+ for _, separateMetric := range separatelyEvaluated {
+ if metric == separateMetric {
+ separateOwner = true
+ break
+ }
+ }
+ if separateOwner {
+ continue
+ }
switch metric {
case "diskRead", "diskWrite", "networkIn", "networkOut":
if !supportsUnifiedIOMetrics(input.Type) {
diff --git a/internal/alerts/unified_eval_test.go b/internal/alerts/unified_eval_test.go
index dbf4db3508..f918a87796 100644
--- a/internal/alerts/unified_eval_test.go
+++ b/internal/alerts/unified_eval_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
@@ -1593,3 +1594,37 @@ func TestUnifiedMetricNoisyWarningWaitsButCriticalFiresImmediately(t *testing.T)
t.Fatalf("stabilityWindowSeconds = %v, want %d", got, defaultNoisyGaugeStabilitySeconds)
}
}
+
+func TestNodeUsageDispatchPreservesSeparatelyEvaluatedTemperature(t *testing.T) {
+ m, elapsed := continuityManager(t, false)
+ m.mu.Lock()
+ m.config.NodeDefaults.Temperature = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.mu.Unlock()
+ node, _ := testNodeWithHostAgent()
+ node.Temperature = &models.Temperature{Available: true, CPUPackage: 85}
+ m.CheckNode(node)
+ elapsed.Store(int64(40 * time.Second))
+ m.CheckNode(node)
+ incident, ok := continuityIncident(m, node.ID, "temperature")
+ if !ok || incident.State != reducer.StatePending {
+ t.Fatal("temperature control did not remain pending")
+ }
+ originalStart := incident.PendingSince
+ elapsed.Store(int64(59 * time.Second))
+ m.CheckNode(node)
+ incident, _ = continuityIncident(m, node.ID, "temperature")
+ if !incident.PendingSince.Equal(originalStart) {
+ t.Fatal("usage dispatch interrupted a separately observed temperature")
+ }
+ elapsed.Store(int64(time.Minute))
+ m.CheckNode(node)
+ if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "temperature")) {
+ t.Fatal("continuous node temperature did not complete its delay")
+ }
+ node.Temperature = nil
+ elapsed.Store(int64(2 * time.Minute))
+ m.CheckNode(node)
+ if !testHasActiveAlert(t, m, canonicalMetricStateID(node.ID, "temperature")) {
+ t.Fatal("actual missing temperature became recovery")
+ }
+}