Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Restart metric timing after unavailable observations

pulse-triage[bot]2d2ae4c8a1fb
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index 6aa2d738a2..7d1ae4a7be 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -40,6 +40,33 @@ and exactly-once callbacks with shadow parity. `TestInterruptDiscreteRun` in
acknowledgement, no fabricated resolution and idempotent interruption. These
synthetic source checks do not establish native outage or destination delivery.
+### Metric timing requires continuous observed evidence
+
+Unknown metric evidence interrupts timing, not the occurrence. Missing or
+rejected unified values, non-finite current readings, and unavailable,
+incomplete or invalid rolling windows drop a pending activation run and reset
+an open incident's recovery run. Explicit metric-intent grace and its durable
+pending checkpoint are reset as well. Fresh evidence must complete a new full
+configured delay; unknown time cannot fulfil a sustained-for or recovery rule.
+Existing alert values, identity, start time, acknowledgement, notification and
+lifecycle records are not refreshed or resolved by that interruption.
+
+Guest backup/QGA deferrals apply this boundary to retained memory and aggregate
+and per-filesystem disk readings. Failed or expired empty disk inventory
+interrupts tracked filesystem runs, including a guest's former node-scoped
+identities, without implying filesystem removal. Different guests, instances,
+metrics and independently current CPU/I/O remain independent. Host memory also
+interrupts timing when its usage is unknown; explicit rule disablement still
+uses its normal clearing path independently of telemetry. Node temperature's
+existing interruption rule is unchanged.
+
+`metric_observation_continuity_test.go` verifies actual legacy/canonical,
+unified, host and VM entry points, full-delay fresh activation/recovery,
+explicit grace, empty/migrated filesystem inventory and rejected-value/source
+isolation. `guest_observation_test.go` no longer treats a retained-reading gap
+as elapsed breach evidence. These controls establish source lifecycle
+truthfulness, not native freeze/thaw safety or external notification delivery.
+
### Retained guest observations are not alert evidence
Guest memory marked `last-known`, `unavailable` or an unknown observation state
diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go
index fcfa75baa5..95dd73c31a 100644
--- a/internal/alerts/canonical_metric.go
+++ b/internal/alerts/canonical_metric.go
@@ -105,6 +105,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
observedAt := m.policyNow()
windowed := m.evaluateMetricWindow(spec.ResourceID, resourceType, metricType, value, observedAt)
if !windowed.Ready {
+ m.interruptMetricRun(spec)
return
}
value = windowed.Value
diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go
index 25963829ef..6ebb43345e 100644
--- a/internal/alerts/guest.go
+++ b/internal/alerts/guest.go
@@ -303,6 +303,9 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
// removal. CPU/I/O and independent current memory were evaluated above.
// Still apply explicit per-filesystem disablement to retained identities;
// this is operator policy, not recovery inferred from an old value.
+ if snapshot.DiskUnavailable {
+ m.interruptGuestFilesystemMetricRuns(guestID)
+ }
if len(disks) > 0 {
seenDiskKeys := make(map[string]struct{})
diff --git a/internal/alerts/guest_observation_test.go b/internal/alerts/guest_observation_test.go
index d231e727a3..e1c388fcde 100644
--- a/internal/alerts/guest_observation_test.go
+++ b/internal/alerts/guest_observation_test.go
@@ -125,7 +125,7 @@ func TestGuestAlertUnavailableDiskInventoryIsNotRecovery(t *testing.T) {
}
}
-func TestGuestAlertRetainedAggregateAndPendingCannotAdvance(t *testing.T) {
+func TestGuestAlertRetainedAggregateInterruptsPendingRun(t *testing.T) {
m := guestObservationManager(t)
vm := guestObservationVM()
vm.Disks = nil
@@ -149,8 +149,15 @@ func TestGuestAlertRetainedAggregateAndPendingCannotAdvance(t *testing.T) {
}
vm.Memory.Observation.State, vm.DiskStatusReason = "current", ""
m.CheckGuest(vm, "site")
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("fresh evidence must begin a new continuous delay")
+ }
+ m.mu.Lock()
+ m.core.ShiftPending(-2 * time.Minute)
+ m.mu.Unlock()
+ m.CheckGuest(vm, "site")
if len(m.GetActiveAlerts()) != 2 {
- t.Fatal("fresh evidence must resume the existing delay policy")
+ t.Fatal("fresh continuous evidence must still complete the configured delay")
}
}
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 8e0e8c5516..6c3d8b2807 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -316,7 +316,7 @@ func (m *Manager) CheckHost(host models.Host) {
m.clearHostMetricAlerts(host.ID, "cpu")
}
- if thresholds.Memory != nil && host.Memory.HasKnownUsage() {
+ if thresholds.Memory != nil {
memMetadata := cloneMetadata(baseMetadata)
memMetadata["metric"] = "memory"
memMetadata["memoryUsagePercent"] = host.Memory.Usage
@@ -332,6 +332,8 @@ func (m *Manager) CheckHost(host models.Host) {
Str("resourceID", resourceID).
Str("host", resourceName).
Msg("Skipping invalid canonical host memory metric spec")
+ } else if !host.Memory.HasKnownUsage() && !spec.Disabled {
+ m.interruptMetricRun(spec)
} else {
m.checkMetricWithCanonicalSpec(spec, resourceName, nodeName, instanceName, "agent", host.Memory.Usage, thresholds.Memory, &metricOptions{Metadata: memMetadata})
}
diff --git a/internal/alerts/metric_observation.go b/internal/alerts/metric_observation.go
new file mode 100644
index 0000000000..6e720ed617
--- /dev/null
+++ b/internal/alerts/metric_observation.go
@@ -0,0 +1,71 @@
+package alerts
+
+import (
+ "strings"
+
+ alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
+)
+
+// interruptMetricRun records a missing observation for a metric spec. The
+// incident and any open alert are kept, but a pending activation run is
+// dropped and a recovery run restarts, so a sustained-for or recovery delay
+// never completes across a gap in evidence.
+func (m *Manager) interruptMetricRun(spec alertspecs.ResourceAlertSpec) {
+ m.interruptMetricRunIDs(spec.ResourceID, spec.ID, canonicalTrackingKeyForSpec(spec, spec.ID))
+}
+
+func (m *Manager) interruptMetricRunIDs(resourceID, specID, trackingKey string) {
+ m.mu.Lock()
+ intentChanged := m.interruptMetricRunNoLock(resourceID, specID, trackingKey)
+ m.mu.Unlock()
+ if intentChanged {
+ m.saveActiveAlertsAsync("metric observation gap")
+ }
+}
+
+func (m *Manager) interruptMetricRunNoLock(resourceID, specID, trackingKey string) bool {
+ for _, state := range m.mirrorStatesNoLock() {
+ state.InterruptMetricRun(resourceID, specID)
+ }
+ return m.clearIntentPendingNoLock(trackingKey)
+}
+
+// Failed/expired guest disk inventory can be empty. Interrupt the tracked
+// identities, not only rows still present in the retained display snapshot.
+// Match the same logical guest across node moves; do not resolve its alerts
+// or mistake an unknown inventory for authoritative filesystem removal.
+func (m *Manager) interruptGuestFilesystemMetricRuns(guestID string) {
+ if guestID == "" {
+ return
+ }
+ m.mu.Lock()
+ resources := make(map[string]struct{})
+ for _, state := range m.mirrorStatesNoLock() {
+ for _, resourceID := range state.PendingResourceIDs() {
+ resources[resourceID] = struct{}{}
+ }
+ }
+ for _, alert := range m.activeAlerts {
+ if alert != nil && alert.Type == "disk" {
+ resources[alert.ResourceID] = struct{}{}
+ }
+ }
+ for _, pending := range m.intentPending {
+ if pending.Signal == MetricAlertIntentSignal("disk") {
+ resources[pending.ResourceID] = struct{}{}
+ }
+ }
+ intentChanged := false
+ for resourceID := range resources {
+ if !strings.HasPrefix(resourceID, guestID+"-disk-") && !guestDiskAlertBelongsToGuest(resourceID, guestID) {
+ continue
+ }
+ if m.interruptMetricRunNoLock(resourceID, canonicalMetricSpecID(resourceID, "disk"), canonicalMetricStateID(resourceID, "disk")) {
+ intentChanged = true
+ }
+ }
+ m.mu.Unlock()
+ if intentChanged {
+ m.saveActiveAlertsAsync("guest filesystem observation gap")
+ }
+}
diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.go
new file mode 100644
index 0000000000..ee9a372651
--- /dev/null
+++ b/internal/alerts/metric_observation_continuity_test.go
@@ -0,0 +1,360 @@
+package alerts
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "math"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+)
+
+func continuityManager(t *testing.T, explicit bool) (*Manager, *atomic.Int64) {
+ t.Helper()
+ m := newEventLogManager(t)
+ cfg := characterizationBaseConfig()
+ cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"cpu": 60, "memory": 60, "disk": 60, "temperature": 60, "diskRead": 60, "diskWrite": 60, "networkIn": 60, "networkOut": 60, "usage": 60}}
+ cfg.MetricEvaluationWindows = nil
+ cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 99, Clear: 98}
+ cfg.GuestDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ cfg.GuestDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ cfg.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.UpdateConfig(cfg)
+ elapsed := &atomic.Int64{}
+ origin := time.Now().UTC()
+ m.now = func() time.Time { return origin.Add(time.Duration(elapsed.Load())) }
+ m.intentClock = func() time.Duration { return time.Duration(elapsed.Load()) }
+ if explicit {
+ doc := NewAlertIntentPolicyDocument()
+ for _, metric := range []string{"cpu", "memory", "disk", "temperature", "diskWrite", "networkIn", "networkOut", "diskRead", "usage"} {
+ doc.Defaults[MetricAlertIntentSignal(metric)] = AlertIntentRule{GraceSeconds: intPointer(60)}
+ }
+ if err := m.LoadIntentPolicies(doc); err != nil {
+ t.Fatal(err)
+ }
+ }
+ m.EnableShadowFeed()
+ return m, elapsed
+}
+
+func continuityIncident(m *Manager, resourceID, metric string) (reducer.Incident, bool) {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ return m.core.Incident(resourceID, canonicalMetricSpecID(resourceID, metric))
+}
+
+// Actual production entry points, not only a reducer interruption unit test.
+func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, string, func(float64, bool)) {
+ t.Helper()
+ threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}
+ id, metric := "site:node:105", "memory"
+ switch route {
+ case "legacy", "canonical":
+ spec, err := buildCanonicalMetricSpec(id, "guest", unifiedresources.ResourceTypeVM, metric, threshold)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if gap == "history-error" || gap == "history-empty" || gap == "history-gap" || gap == "history-NaN" {
+ m.mu.Lock()
+ m.config.MetricEvaluationWindows = map[string]map[string]int{"all": {metric: 300}}
+ m.mu.Unlock()
+ }
+ return id, metric, func(value float64, missing bool) {
+ if missing {
+ switch gap {
+ case "NaN":
+ value = math.NaN()
+ case "+Inf":
+ value = math.Inf(1)
+ case "-Inf":
+ value = math.Inf(-1)
+ }
+ }
+ m.SetMetricWindowProvider(func(r MetricWindowRequest) ([]MetricWindowPoint, error) {
+ if missing {
+ switch gap {
+ case "history-error":
+ return nil, errors.New("history unavailable")
+ case "history-empty":
+ return nil, nil
+ case "history-gap":
+ return []MetricWindowPoint{{r.Start, value}}, nil
+ case "history-NaN":
+ return []MetricWindowPoint{{r.Start, value}, {r.Start.Add(2 * time.Minute), math.NaN()}, {r.Start.Add(4 * time.Minute), value}}, nil
+ }
+ }
+ return []MetricWindowPoint{{r.Start, value}, {r.Start.Add(2 * time.Minute), value}, {r.Start.Add(4 * time.Minute), value}}, nil
+ })
+ if route == "legacy" {
+ m.checkMetric(id, "guest", "node", "site", "vm", metric, value, threshold, nil)
+ } else {
+ m.checkMetricWithCanonicalSpec(spec, "guest", "node", "site", "vm", value, threshold, nil)
+ }
+ }
+ case "unified":
+ return id, metric, func(value float64, missing bool) {
+ sample := &UnifiedResourceMetric{Percent: value}
+ if missing {
+ sample = nil
+ }
+ m.evaluateUnifiedMetrics(&UnifiedResourceInput{ID: id, Type: "vm", Name: "guest", Memory: sample}, ThresholdConfig{Memory: threshold}, nil)
+ }
+ case "host":
+ id = "agent:host-105"
+ return id, metric, func(value float64, missing bool) {
+ host := models.Host{ID: "host-105", Hostname: "host", Status: "online", Memory: models.Memory{Total: 1000, Used: 850, Usage: value}}
+ if missing {
+ host.Memory = models.UnavailableMemory(1000)
+ }
+ m.CheckHost(host)
+ }
+ case "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired":
+ if route != "guest-memory" {
+ metric = "disk"
+ }
+ resourceID := id
+ if route == "guest-filesystem" || route == "guest-filesystem-expired" {
+ resourceID += "-disk-root"
+ }
+ return resourceID, metric, func(value float64, missing bool) {
+ vm := models.VM{ID: id, VMID: 105, Name: "guest", Node: "node", Instance: "site", Type: "qemu", Status: "running",
+ Memory: models.Memory{Total: 1000, Used: 100, Usage: 10, Observation: models.MemoryObservation{State: "current"}},
+ Disk: models.Disk{Total: 1000, Usage: 10}, IORateValidity: models.IORateValidity{Explicit: true}}
+ if route == "guest-memory" {
+ vm.Memory.Usage = value
+ if missing {
+ vm.Memory.Observation.State = "last-known"
+ }
+ } else {
+ vm.Disk.Usage = value
+ if route != "guest-aggregate" {
+ vm.Disks = []models.Disk{{Total: 1000, Usage: value, Mountpoint: "/"}}
+ }
+ if missing {
+ vm.DiskStatusReason, vm.GuestAgentStatus = "prev-vm-locked", "deferred"
+ if route == "guest-filesystem-expired" {
+ vm.Disks = nil
+ vm.Disk.Usage = -1
+ }
+ }
+ }
+ m.CheckGuest(vm, "site")
+ }
+ }
+ t.Fatalf("unknown observer %s", route)
+ return "", "", nil
+}
+
+func TestMetricObservationGapRestartsActivation(t *testing.T) {
+ for _, explicit := range []bool{false, true} {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired"} {
+ gaps := []string{"missing"}
+ if route == "legacy" || route == "canonical" {
+ gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
+ }
+ for _, gap := range gaps {
+ t.Run(route+"/"+gap+"/explicit="+map[bool]string{false: "false", true: "true"}[explicit], func(t *testing.T) {
+ m, elapsed := continuityManager(t, explicit)
+ id, metric, observe := continuityObserver(t, m, route, gap)
+ observe(85, false)
+ incident, exists := continuityIncident(m, id, metric)
+ if !exists || incident.State != reducer.StatePending {
+ t.Fatalf("expected a real pending run at %s, got %+v %v", id, incident, exists)
+ }
+ m.checkMetric("unrelated", "other", "node", "site", "vm", "memory", 85, &HysteresisThreshold{Trigger: 80, Clear: 70}, nil)
+ elapsed.Store(int64(2 * time.Minute))
+ observe(85, true)
+ if _, exists := continuityIncident(m, id, metric); exists {
+ t.Error("unknown observation retained pending evidence")
+ }
+ if _, exists := continuityIncident(m, "unrelated", "memory"); !exists {
+ t.Error("interruption leaked to another resource")
+ }
+ if len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}})) != 0 {
+ t.Error("unknown observation emitted a lifecycle transition")
+ }
+ observe(85, false)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("fresh sample fired using time accrued before the unknown interval")
+ }
+ elapsed.Store(int64(2*time.Minute + 59*time.Second))
+ observe(85, false)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("restarted delay fired early")
+ }
+ elapsed.Store(int64(3 * time.Minute))
+ observe(85, false)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("continuous fresh evidence did not fire exactly once after the full delay")
+ }
+ if m.ShadowDivergences() != 0 {
+ t.Fatal("timing interruption diverged from shadow")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestMetricObservationGapRestartsRecovery(t *testing.T) {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory"} {
+ gaps := []string{"missing"}
+ if route == "legacy" || route == "canonical" {
+ gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}
+ }
+ for _, gap := range gaps {
+ t.Run(route+"/"+gap, func(t *testing.T) {
+ m, elapsed := continuityManager(t, false)
+ id, metric, observe := continuityObserver(t, m, route, gap)
+ observe(95, false) // Critical evidence retains its legacy immediate activation.
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("critical control did not fire immediately")
+ }
+ if err := m.AcknowledgeAlert(m.GetActiveAlerts()[0].ID, "operator"); err != nil {
+ t.Fatal(err)
+ }
+ elapsed.Store(int64(10 * time.Second))
+ observe(10, false)
+ incident, _ := continuityIncident(m, id, metric)
+ if incident.RecoverySince.IsZero() {
+ t.Fatal("known healthy evidence did not start a recovery run")
+ }
+ before, _ := json.Marshal(m.GetActiveAlerts())
+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))
+ elapsed.Store(int64(2 * time.Minute))
+ observe(10, true)
+ incident, _ = continuityIncident(m, id, metric)
+ if !incident.RecoverySince.IsZero() {
+ t.Error("unknown observation retained recovery evidence")
+ }
+ after, _ := json.Marshal(m.GetActiveAlerts())
+ if !bytes.Equal(before, after) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {
+ t.Fatal("unknown observation changed the acknowledged incident or transition ledger")
+ }
+ observe(10, false)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("fresh healthy sample resolved across an unknown interval")
+ }
+ elapsed.Store(int64(2*time.Minute + 59*time.Second))
+ observe(10, false)
+ if len(m.GetActiveAlerts()) != 1 {
+ t.Fatal("restarted recovery completed early")
+ }
+ elapsed.Store(int64(3 * time.Minute))
+ observe(10, false)
+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {
+ t.Fatal("continuous healthy evidence did not resolve the original once")
+ }
+ })
+ }
+ }
+}
+
+func TestUnifiedMetricObservationGapCoversRejectedAndAbsentValues(t *testing.T) {
+ cases := []struct{ resourceType, metric string }{
+ {"vm", "cpu"}, {"vm", "memory"}, {"vm", "disk"}, {"vm", "diskRead"}, {"vm", "diskWrite"}, {"vm", "networkIn"}, {"vm", "networkOut"},
+ {"node", "temperature"}, {"agent", "memory"}, {"pbs", "memory"}, {"storage", "usage"}, {"truenas-system", "memory"},
+ {"truenas-pool", "usage"}, {"vmware-vm", "memory"}, {"k8s-node", "memory"}, {"pod", "networkIn"},
+ }
+ for _, tc := range cases {
+ gaps := []string{"nil", "NaN", "+Inf"}
+ if isPercentageMetric(tc.metric) {
+ gaps = append(gaps, "negative", "over-100")
+ }
+ for _, gap := range gaps {
+ t.Run(tc.resourceType+"/"+tc.metric+"/"+gap, func(t *testing.T) {
+ m, elapsed := continuityManager(t, true)
+ threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}
+ input := &UnifiedResourceInput{ID: "target", Type: tc.resourceType, Name: "target"}
+ thresholds := ThresholdConfig{CPU: threshold, Memory: threshold, Disk: threshold, DiskRead: threshold, DiskWrite: threshold, NetworkIn: threshold, NetworkOut: threshold, Temperature: threshold, Usage: threshold}
+ set := func(sample *UnifiedResourceMetric) {
+ switch tc.metric {
+ case "cpu":
+ input.CPU = sample
+ case "memory":
+ input.Memory = sample
+ case "disk", "usage":
+ input.Disk = sample
+ case "temperature":
+ input.Temperature = sample
+ case "diskRead":
+ input.DiskRead = sample
+ case "diskWrite":
+ input.DiskWrite = sample
+ case "networkIn":
+ input.NetworkIn = sample
+ case "networkOut":
+ input.NetworkOut = sample
+ }
+ }
+ set(&UnifiedResourceMetric{Value: 85, Percent: 85})
+ m.evaluateUnifiedMetrics(input, thresholds, nil)
+ if _, ok := continuityIncident(m, input.ID, tc.metric); !ok {
+ t.Fatal("known observation did not start a run")
+ }
+ elapsed.Store(int64(2 * time.Minute))
+ invalid := &UnifiedResourceMetric{}
+ switch gap {
+ case "nil":
+ invalid = nil
+ case "NaN":
+ invalid.Value = math.NaN()
+ invalid.Percent = math.NaN()
+ case "+Inf":
+ invalid.Value = math.Inf(1)
+ invalid.Percent = math.Inf(1)
+ case "negative":
+ invalid.Value = -1
+ invalid.Percent = -1
+ case "over-100":
+ invalid.Value = 101
+ invalid.Percent = 101
+ }
+ set(invalid)
+ m.evaluateUnifiedMetrics(input, thresholds, nil)
+ if _, ok := continuityIncident(m, input.ID, tc.metric); ok {
+ t.Fatal("absent or rejected observation retained a pending run")
+ }
+ set(&UnifiedResourceMetric{Value: 85, Percent: 85})
+ m.evaluateUnifiedMetrics(input, thresholds, nil)
+ if len(m.GetActiveAlerts()) != 0 {
+ t.Fatal("a fresh observation completed grace across missing telemetry")
+ }
+ })
+ }
+ }
+}
+
+func TestGuestFilesystemGapMatchesMigratedAndEmptyInventory(t *testing.T) {
+ m, elapsed := continuityManager(t, true)
+ threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}
+ ids := []string{"site:oldnode:105-disk-root", "site:othernode:105-disk-data", "site:oldnode:1050-disk-root", "elsewhere:oldnode:105-disk-root"}
+ for _, id := range ids {
+ m.checkMetric(id, "filesystem", "oldnode", "site", "vm", "disk", 85, threshold, nil)
+ }
+ elapsed.Store(int64(2 * time.Minute))
+ vm := models.VM{ID: "site:newnode:105", VMID: 105, Name: "guest", Node: "newnode", Instance: "site", Type: "qemu", Status: "running", Memory: models.Memory{Usage: 10}, Disk: models.Disk{Usage: -1}, DiskStatusReason: "prev-vm-locked"}
+ m.CheckGuest(vm, "site")
+ for i, id := range ids {
+ _, exists := continuityIncident(m, id, "disk")
+ if exists != (i >= 2) {
+ t.Fatalf("migration/exact identity boundary at %s: exists=%v", id, exists)
+ }
+ }
+ m.mu.RLock()
+ for _, pending := range m.intentPending {
+ if pending.ResourceID == ids[0] || pending.ResourceID == ids[1] {
+ t.Error("migrated filesystem grace remained durable")
+ }
+ }
+ m.mu.RUnlock()
+ if len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 {
+ t.Fatal("unknown empty inventory emitted lifecycle evidence")
+ }
+}
diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.go
index 6d1ad3cba2..2242901eec 100644
--- a/internal/alerts/metric_runtime.go
+++ b/internal/alerts/metric_runtime.go
@@ -271,6 +271,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource
windowed := m.evaluateMetricWindow(resourceID, resourceType, metricType, value, m.policyNow())
if !windowed.Ready {
+ m.interruptMetricRunIDs(resourceID, canonicalSpecID, canonicalStateID)
return
}
value = windowed.Value
diff --git a/internal/alerts/node.go b/internal/alerts/node.go
index 5f9e0500b2..56fb559735 100644
--- a/internal/alerts/node.go
+++ b/internal/alerts/node.go
@@ -153,22 +153,6 @@ func (m *Manager) checkNodeTemperature(node models.Node, threshold *HysteresisTh
m.checkMetricWithCanonicalSpec(spec, node.Name, node.Name, node.Instance, "node", temp, threshold, nil)
}
-// interruptMetricRun records a missing observation for a metric spec. The
-// incident and any open alert are kept, but a pending activation run is
-// dropped and a recovery run restarts, so a sustained-for or recovery delay
-// never completes across a gap in evidence.
-func (m *Manager) interruptMetricRun(spec alertspecs.ResourceAlertSpec) {
- m.mu.Lock()
- for _, state := range m.mirrorStatesNoLock() {
- state.InterruptMetricRun(spec.ResourceID, spec.ID)
- }
- intentChanged := m.clearIntentPendingNoLock(canonicalTrackingKeyForSpec(spec, spec.ID))
- m.mu.Unlock()
- if intentChanged {
- m.saveActiveAlertsAsync("canonical metric intent pending state")
- }
-}
-
// releaseNodeMetricAlerts stops node-side evaluation of the given metrics the
// same way a disabled threshold does: any pending run is dropped and any open
// node alert is resolved. Without this, a node alert that was open when a host
diff --git a/internal/alerts/unified_eval.go b/internal/alerts/unified_eval.go
index c7120cf079..1fe2d5e09c 100644
--- a/internal/alerts/unified_eval.go
+++ b/internal/alerts/unified_eval.go
@@ -300,11 +300,53 @@ func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholds
merged.Metadata = metadata
opts = &merged
}
- for _, candidate := range buildUnifiedMetricCandidates(input, thresholds) {
+ candidates := buildUnifiedMetricCandidates(input, thresholds)
+ m.interruptUnobservedUnifiedMetrics(input, candidates)
+ for _, candidate := range candidates {
m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)
}
}
+// An omitted/rejected metric is unknown, not a pause in its sustained-for
+// clock. Build candidates stays pure; dispatch owns interruption for metrics
+// this resource supports but cannot currently observe.
+func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput, candidates []unifiedMetricCandidate) {
+ if _, ok := unifiedMetricResourceType(input.Type); !ok {
+ return
+ }
+ metrics := [...]string{"cpu", "memory", "disk", "temperature", "diskRead", "diskWrite", "networkIn", "networkOut", "usage"}
+ m.mu.Lock()
+ intentChanged := false
+ for _, metric := range metrics {
+ switch metric {
+ case "diskRead", "diskWrite", "networkIn", "networkOut":
+ if !supportsUnifiedIOMetrics(input.Type) {
+ continue
+ }
+ case "usage":
+ if !unifiedStorageUsageResourceType(input.Type) {
+ continue
+ }
+ }
+ observed := false
+ for _, candidate := range candidates {
+ if candidate.Spec.MetricThreshold.Metric == metric {
+ observed = true
+ break
+ }
+ }
+ if !observed {
+ if m.interruptMetricRunNoLock(input.ID, canonicalMetricSpecID(input.ID, metric), canonicalMetricStateID(input.ID, metric)) {
+ intentChanged = true
+ }
+ }
+ }
+ m.mu.Unlock()
+ if intentChanged {
+ m.saveActiveAlertsAsync("unified metric observation gap")
+ }
+}
+
func metricOptionsWithTags(opts *metricOptions, tags []string) *metricOptions {
if len(tags) == 0 {
return opts
diff --git a/internal/alerts/windowed_metric_test.go b/internal/alerts/windowed_metric_test.go
index 402ae8c89c..fc216db290 100644
--- a/internal/alerts/windowed_metric_test.go
+++ b/internal/alerts/windowed_metric_test.go
@@ -4,6 +4,8 @@ import (
"bytes"
"encoding/json"
"math"
+ "os"
+ "path/filepath"
"sync/atomic"
"testing"
"time"
@@ -373,3 +375,56 @@ func TestNonFiniteMetricDoesNotPreventExplicitDisable(t *testing.T) {
}
}
}
+
+// A skipped window must not leave grace on disk for a restart to resume.
+func TestUnknownMetricObservationDropsDurableIntentCheckpoint(t *testing.T) {
+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-filesystem-expired"} {
+ t.Run(route, func(t *testing.T) {
+ m, elapsed := continuityManager(t, true)
+ gap := "missing"
+ if route == "legacy" || route == "canonical" {
+ gap = "history-empty"
+ }
+ _, _, observe := continuityObserver(t, m, route, gap)
+ observe(85, false)
+ elapsed.Store(int64(40 * time.Second))
+ observe(85, false)
+ if err := m.SaveActiveAlerts(); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(m.getAlertsDir(), intentPendingFileName)
+ read := func() []IntentPendingState {
+ data, err := os.ReadFile(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var saved []IntentPendingState
+ if err := json.Unmarshal(data, &saved); err != nil {
+ t.Fatal(err)
+ }
+ return saved
+ }
+ if len(read()) != 1 {
+ t.Fatal("control did not retain a real in-progress grace checkpoint")
+ }
+ elapsed.Store(int64(2 * time.Minute))
+ observe(85, true)
+ if err := m.SaveActiveAlerts(); err != nil {
+ t.Fatal(err)
+ }
+ if len(read()) != 0 {
+ t.Fatal("unknown observation left grace for a restart to reuse")
+ }
+ dir := filepath.Dir(m.getAlertsDir())
+ m.Stop()
+ restarted := NewManagerWithDataDir(dir)
+ t.Cleanup(restarted.Stop)
+ restarted.mu.RLock()
+ pending := len(restarted.intentPending)
+ restarted.mu.RUnlock()
+ if pending != 0 {
+ t.Fatal("restart restored interrupted grace")
+ }
+ })
+ }
+}