Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Restart metric timing after unavailable observations
pulse-triage[bot]2d2ae4c8a1fb
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex 6aa2d738a2..7d1ae4a7be 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -40,6 +40,33 @@ and exactly-once callbacks with shadow parity. `TestInterruptDiscreteRun` inacknowledgement, no fabricated resolution and idempotent interruption. Thesesynthetic source checks do not establish native outage or destination delivery.+### Metric timing requires continuous observed evidence++Unknown metric evidence interrupts timing, not the occurrence. Missing or+rejected unified values, non-finite current readings, and unavailable,+incomplete or invalid rolling windows drop a pending activation run and reset+an open incident's recovery run. Explicit metric-intent grace and its durable+pending checkpoint are reset as well. Fresh evidence must complete a new full+configured delay; unknown time cannot fulfil a sustained-for or recovery rule.+Existing alert values, identity, start time, acknowledgement, notification and+lifecycle records are not refreshed or resolved by that interruption.++Guest backup/QGA deferrals apply this boundary to retained memory and aggregate+and per-filesystem disk readings. Failed or expired empty disk inventory+interrupts tracked filesystem runs, including a guest's former node-scoped+identities, without implying filesystem removal. Different guests, instances,+metrics and independently current CPU/I/O remain independent. Host memory also+interrupts timing when its usage is unknown; explicit rule disablement still+uses its normal clearing path independently of telemetry. Node temperature's+existing interruption rule is unchanged.++`metric_observation_continuity_test.go` verifies actual legacy/canonical,+unified, host and VM entry points, full-delay fresh activation/recovery,+explicit grace, empty/migrated filesystem inventory and rejected-value/source+isolation. `guest_observation_test.go` no longer treats a retained-reading gap+as elapsed breach evidence. These controls establish source lifecycle+truthfulness, not native freeze/thaw safety or external notification delivery.+### Retained guest observations are not alert evidenceGuest memory marked `last-known`, `unavailable` or an unknown observation statediff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.goindex fcfa75baa5..95dd73c31a 100644--- a/internal/alerts/canonical_metric.go+++ b/internal/alerts/canonical_metric.go@@ -105,6 +105,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpecobservedAt := m.policyNow()windowed := m.evaluateMetricWindow(spec.ResourceID, resourceType, metricType, value, observedAt)if !windowed.Ready {+ m.interruptMetricRun(spec)return}value = windowed.Valuediff --git a/internal/alerts/guest.go b/internal/alerts/guest.goindex 25963829ef..6ebb43345e 100644--- a/internal/alerts/guest.go+++ b/internal/alerts/guest.go@@ -303,6 +303,9 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {// removal. CPU/I/O and independent current memory were evaluated above.// Still apply explicit per-filesystem disablement to retained identities;// this is operator policy, not recovery inferred from an old value.+ if snapshot.DiskUnavailable {+ m.interruptGuestFilesystemMetricRuns(guestID)+ }if len(disks) > 0 {seenDiskKeys := make(map[string]struct{})diff --git a/internal/alerts/guest_observation_test.go b/internal/alerts/guest_observation_test.goindex d231e727a3..e1c388fcde 100644--- a/internal/alerts/guest_observation_test.go+++ b/internal/alerts/guest_observation_test.go@@ -125,7 +125,7 @@ func TestGuestAlertUnavailableDiskInventoryIsNotRecovery(t *testing.T) {}}-func TestGuestAlertRetainedAggregateAndPendingCannotAdvance(t *testing.T) {+func TestGuestAlertRetainedAggregateInterruptsPendingRun(t *testing.T) {m := guestObservationManager(t)vm := guestObservationVM()vm.Disks = nil@@ -149,8 +149,15 @@ func TestGuestAlertRetainedAggregateAndPendingCannotAdvance(t *testing.T) {}vm.Memory.Observation.State, vm.DiskStatusReason = "current", ""m.CheckGuest(vm, "site")+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("fresh evidence must begin a new continuous delay")+ }+ m.mu.Lock()+ m.core.ShiftPending(-2 * time.Minute)+ m.mu.Unlock()+ m.CheckGuest(vm, "site")if len(m.GetActiveAlerts()) != 2 {- t.Fatal("fresh evidence must resume the existing delay policy")+ t.Fatal("fresh continuous evidence must still complete the configured delay")}}diff --git a/internal/alerts/host.go b/internal/alerts/host.goindex 8e0e8c5516..6c3d8b2807 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -316,7 +316,7 @@ func (m *Manager) CheckHost(host models.Host) {m.clearHostMetricAlerts(host.ID, "cpu")}- if thresholds.Memory != nil && host.Memory.HasKnownUsage() {+ if thresholds.Memory != nil {memMetadata := cloneMetadata(baseMetadata)memMetadata["metric"] = "memory"memMetadata["memoryUsagePercent"] = host.Memory.Usage@@ -332,6 +332,8 @@ func (m *Manager) CheckHost(host models.Host) {Str("resourceID", resourceID).Str("host", resourceName).Msg("Skipping invalid canonical host memory metric spec")+ } else if !host.Memory.HasKnownUsage() && !spec.Disabled {+ m.interruptMetricRun(spec)} else {m.checkMetricWithCanonicalSpec(spec, resourceName, nodeName, instanceName, "agent", host.Memory.Usage, thresholds.Memory, &metricOptions{Metadata: memMetadata})}diff --git a/internal/alerts/metric_observation.go b/internal/alerts/metric_observation.gonew file mode 100644index 0000000000..6e720ed617--- /dev/null+++ b/internal/alerts/metric_observation.go@@ -0,0 +1,71 @@+package alerts++import (+ "strings"++ alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"+)++// interruptMetricRun records a missing observation for a metric spec. The+// incident and any open alert are kept, but a pending activation run is+// dropped and a recovery run restarts, so a sustained-for or recovery delay+// never completes across a gap in evidence.+func (m *Manager) interruptMetricRun(spec alertspecs.ResourceAlertSpec) {+ m.interruptMetricRunIDs(spec.ResourceID, spec.ID, canonicalTrackingKeyForSpec(spec, spec.ID))+}++func (m *Manager) interruptMetricRunIDs(resourceID, specID, trackingKey string) {+ m.mu.Lock()+ intentChanged := m.interruptMetricRunNoLock(resourceID, specID, trackingKey)+ m.mu.Unlock()+ if intentChanged {+ m.saveActiveAlertsAsync("metric observation gap")+ }+}++func (m *Manager) interruptMetricRunNoLock(resourceID, specID, trackingKey string) bool {+ for _, state := range m.mirrorStatesNoLock() {+ state.InterruptMetricRun(resourceID, specID)+ }+ return m.clearIntentPendingNoLock(trackingKey)+}++// Failed/expired guest disk inventory can be empty. Interrupt the tracked+// identities, not only rows still present in the retained display snapshot.+// Match the same logical guest across node moves; do not resolve its alerts+// or mistake an unknown inventory for authoritative filesystem removal.+func (m *Manager) interruptGuestFilesystemMetricRuns(guestID string) {+ if guestID == "" {+ return+ }+ m.mu.Lock()+ resources := make(map[string]struct{})+ for _, state := range m.mirrorStatesNoLock() {+ for _, resourceID := range state.PendingResourceIDs() {+ resources[resourceID] = struct{}{}+ }+ }+ for _, alert := range m.activeAlerts {+ if alert != nil && alert.Type == "disk" {+ resources[alert.ResourceID] = struct{}{}+ }+ }+ for _, pending := range m.intentPending {+ if pending.Signal == MetricAlertIntentSignal("disk") {+ resources[pending.ResourceID] = struct{}{}+ }+ }+ intentChanged := false+ for resourceID := range resources {+ if !strings.HasPrefix(resourceID, guestID+"-disk-") && !guestDiskAlertBelongsToGuest(resourceID, guestID) {+ continue+ }+ if m.interruptMetricRunNoLock(resourceID, canonicalMetricSpecID(resourceID, "disk"), canonicalMetricStateID(resourceID, "disk")) {+ intentChanged = true+ }+ }+ m.mu.Unlock()+ if intentChanged {+ m.saveActiveAlertsAsync("guest filesystem observation gap")+ }+}diff --git a/internal/alerts/metric_observation_continuity_test.go b/internal/alerts/metric_observation_continuity_test.gonew file mode 100644index 0000000000..ee9a372651--- /dev/null+++ b/internal/alerts/metric_observation_continuity_test.go@@ -0,0 +1,360 @@+package alerts++import (+ "bytes"+ "encoding/json"+ "errors"+ "math"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++func continuityManager(t *testing.T, explicit bool) (*Manager, *atomic.Int64) {+ t.Helper()+ m := newEventLogManager(t)+ cfg := characterizationBaseConfig()+ cfg.MetricTimeThresholds = map[string]map[string]int{"all": {"cpu": 60, "memory": 60, "disk": 60, "temperature": 60, "diskRead": 60, "diskWrite": 60, "networkIn": 60, "networkOut": 60, "usage": 60}}+ cfg.MetricEvaluationWindows = nil+ cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 99, Clear: 98}+ cfg.GuestDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}+ cfg.GuestDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}+ cfg.AgentDefaults.Memory = &HysteresisThreshold{Trigger: 80, Clear: 70}+ m.UpdateConfig(cfg)+ elapsed := &atomic.Int64{}+ origin := time.Now().UTC()+ m.now = func() time.Time { return origin.Add(time.Duration(elapsed.Load())) }+ m.intentClock = func() time.Duration { return time.Duration(elapsed.Load()) }+ if explicit {+ doc := NewAlertIntentPolicyDocument()+ for _, metric := range []string{"cpu", "memory", "disk", "temperature", "diskWrite", "networkIn", "networkOut", "diskRead", "usage"} {+ doc.Defaults[MetricAlertIntentSignal(metric)] = AlertIntentRule{GraceSeconds: intPointer(60)}+ }+ if err := m.LoadIntentPolicies(doc); err != nil {+ t.Fatal(err)+ }+ }+ m.EnableShadowFeed()+ return m, elapsed+}++func continuityIncident(m *Manager, resourceID, metric string) (reducer.Incident, bool) {+ m.mu.RLock()+ defer m.mu.RUnlock()+ return m.core.Incident(resourceID, canonicalMetricSpecID(resourceID, metric))+}++// Actual production entry points, not only a reducer interruption unit test.+func continuityObserver(t *testing.T, m *Manager, route, gap string) (string, string, func(float64, bool)) {+ t.Helper()+ threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}+ id, metric := "site:node:105", "memory"+ switch route {+ case "legacy", "canonical":+ spec, err := buildCanonicalMetricSpec(id, "guest", unifiedresources.ResourceTypeVM, metric, threshold)+ if err != nil {+ t.Fatal(err)+ }+ if gap == "history-error" || gap == "history-empty" || gap == "history-gap" || gap == "history-NaN" {+ m.mu.Lock()+ m.config.MetricEvaluationWindows = map[string]map[string]int{"all": {metric: 300}}+ m.mu.Unlock()+ }+ return id, metric, func(value float64, missing bool) {+ if missing {+ switch gap {+ case "NaN":+ value = math.NaN()+ case "+Inf":+ value = math.Inf(1)+ case "-Inf":+ value = math.Inf(-1)+ }+ }+ m.SetMetricWindowProvider(func(r MetricWindowRequest) ([]MetricWindowPoint, error) {+ if missing {+ switch gap {+ case "history-error":+ return nil, errors.New("history unavailable")+ case "history-empty":+ return nil, nil+ case "history-gap":+ return []MetricWindowPoint{{r.Start, value}}, nil+ case "history-NaN":+ return []MetricWindowPoint{{r.Start, value}, {r.Start.Add(2 * time.Minute), math.NaN()}, {r.Start.Add(4 * time.Minute), value}}, nil+ }+ }+ return []MetricWindowPoint{{r.Start, value}, {r.Start.Add(2 * time.Minute), value}, {r.Start.Add(4 * time.Minute), value}}, nil+ })+ if route == "legacy" {+ m.checkMetric(id, "guest", "node", "site", "vm", metric, value, threshold, nil)+ } else {+ m.checkMetricWithCanonicalSpec(spec, "guest", "node", "site", "vm", value, threshold, nil)+ }+ }+ case "unified":+ return id, metric, func(value float64, missing bool) {+ sample := &UnifiedResourceMetric{Percent: value}+ if missing {+ sample = nil+ }+ m.evaluateUnifiedMetrics(&UnifiedResourceInput{ID: id, Type: "vm", Name: "guest", Memory: sample}, ThresholdConfig{Memory: threshold}, nil)+ }+ case "host":+ id = "agent:host-105"+ return id, metric, func(value float64, missing bool) {+ host := models.Host{ID: "host-105", Hostname: "host", Status: "online", Memory: models.Memory{Total: 1000, Used: 850, Usage: value}}+ if missing {+ host.Memory = models.UnavailableMemory(1000)+ }+ m.CheckHost(host)+ }+ case "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired":+ if route != "guest-memory" {+ metric = "disk"+ }+ resourceID := id+ if route == "guest-filesystem" || route == "guest-filesystem-expired" {+ resourceID += "-disk-root"+ }+ return resourceID, metric, func(value float64, missing bool) {+ vm := models.VM{ID: id, VMID: 105, Name: "guest", Node: "node", Instance: "site", Type: "qemu", Status: "running",+ Memory: models.Memory{Total: 1000, Used: 100, Usage: 10, Observation: models.MemoryObservation{State: "current"}},+ Disk: models.Disk{Total: 1000, Usage: 10}, IORateValidity: models.IORateValidity{Explicit: true}}+ if route == "guest-memory" {+ vm.Memory.Usage = value+ if missing {+ vm.Memory.Observation.State = "last-known"+ }+ } else {+ vm.Disk.Usage = value+ if route != "guest-aggregate" {+ vm.Disks = []models.Disk{{Total: 1000, Usage: value, Mountpoint: "/"}}+ }+ if missing {+ vm.DiskStatusReason, vm.GuestAgentStatus = "prev-vm-locked", "deferred"+ if route == "guest-filesystem-expired" {+ vm.Disks = nil+ vm.Disk.Usage = -1+ }+ }+ }+ m.CheckGuest(vm, "site")+ }+ }+ t.Fatalf("unknown observer %s", route)+ return "", "", nil+}++func TestMetricObservationGapRestartsActivation(t *testing.T) {+ for _, explicit := range []bool{false, true} {+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-aggregate", "guest-filesystem", "guest-filesystem-expired"} {+ gaps := []string{"missing"}+ if route == "legacy" || route == "canonical" {+ gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}+ }+ for _, gap := range gaps {+ t.Run(route+"/"+gap+"/explicit="+map[bool]string{false: "false", true: "true"}[explicit], func(t *testing.T) {+ m, elapsed := continuityManager(t, explicit)+ id, metric, observe := continuityObserver(t, m, route, gap)+ observe(85, false)+ incident, exists := continuityIncident(m, id, metric)+ if !exists || incident.State != reducer.StatePending {+ t.Fatalf("expected a real pending run at %s, got %+v %v", id, incident, exists)+ }+ m.checkMetric("unrelated", "other", "node", "site", "vm", "memory", 85, &HysteresisThreshold{Trigger: 80, Clear: 70}, nil)+ elapsed.Store(int64(2 * time.Minute))+ observe(85, true)+ if _, exists := continuityIncident(m, id, metric); exists {+ t.Error("unknown observation retained pending evidence")+ }+ if _, exists := continuityIncident(m, "unrelated", "memory"); !exists {+ t.Error("interruption leaked to another resource")+ }+ if len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeResolved}})) != 0 {+ t.Error("unknown observation emitted a lifecycle transition")+ }+ observe(85, false)+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("fresh sample fired using time accrued before the unknown interval")+ }+ elapsed.Store(int64(2*time.Minute + 59*time.Second))+ observe(85, false)+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("restarted delay fired early")+ }+ elapsed.Store(int64(3 * time.Minute))+ observe(85, false)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("continuous fresh evidence did not fire exactly once after the full delay")+ }+ if m.ShadowDivergences() != 0 {+ t.Fatal("timing interruption diverged from shadow")+ }+ })+ }+ }+ }+}++func TestMetricObservationGapRestartsRecovery(t *testing.T) {+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory"} {+ gaps := []string{"missing"}+ if route == "legacy" || route == "canonical" {+ gaps = []string{"NaN", "+Inf", "-Inf", "history-error", "history-empty", "history-gap", "history-NaN"}+ }+ for _, gap := range gaps {+ t.Run(route+"/"+gap, func(t *testing.T) {+ m, elapsed := continuityManager(t, false)+ id, metric, observe := continuityObserver(t, m, route, gap)+ observe(95, false) // Critical evidence retains its legacy immediate activation.+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("critical control did not fire immediately")+ }+ if err := m.AcknowledgeAlert(m.GetActiveAlerts()[0].ID, "operator"); err != nil {+ t.Fatal(err)+ }+ elapsed.Store(int64(10 * time.Second))+ observe(10, false)+ incident, _ := continuityIncident(m, id, metric)+ if incident.RecoverySince.IsZero() {+ t.Fatal("known healthy evidence did not start a recovery run")+ }+ before, _ := json.Marshal(m.GetActiveAlerts())+ events := len(queryAlertEvents(t, m, eventlog.Filter{}))+ elapsed.Store(int64(2 * time.Minute))+ observe(10, true)+ incident, _ = continuityIncident(m, id, metric)+ if !incident.RecoverySince.IsZero() {+ t.Error("unknown observation retained recovery evidence")+ }+ after, _ := json.Marshal(m.GetActiveAlerts())+ if !bytes.Equal(before, after) || len(queryAlertEvents(t, m, eventlog.Filter{})) != events {+ t.Fatal("unknown observation changed the acknowledged incident or transition ledger")+ }+ observe(10, false)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("fresh healthy sample resolved across an unknown interval")+ }+ elapsed.Store(int64(2*time.Minute + 59*time.Second))+ observe(10, false)+ if len(m.GetActiveAlerts()) != 1 {+ t.Fatal("restarted recovery completed early")+ }+ elapsed.Store(int64(3 * time.Minute))+ observe(10, false)+ if len(m.GetActiveAlerts()) != 0 || len(queryAlertEvents(t, m, eventlog.Filter{Types: []string{eventlog.TypeResolved}})) != 1 {+ t.Fatal("continuous healthy evidence did not resolve the original once")+ }+ })+ }+ }+}++func TestUnifiedMetricObservationGapCoversRejectedAndAbsentValues(t *testing.T) {+ cases := []struct{ resourceType, metric string }{+ {"vm", "cpu"}, {"vm", "memory"}, {"vm", "disk"}, {"vm", "diskRead"}, {"vm", "diskWrite"}, {"vm", "networkIn"}, {"vm", "networkOut"},+ {"node", "temperature"}, {"agent", "memory"}, {"pbs", "memory"}, {"storage", "usage"}, {"truenas-system", "memory"},+ {"truenas-pool", "usage"}, {"vmware-vm", "memory"}, {"k8s-node", "memory"}, {"pod", "networkIn"},+ }+ for _, tc := range cases {+ gaps := []string{"nil", "NaN", "+Inf"}+ if isPercentageMetric(tc.metric) {+ gaps = append(gaps, "negative", "over-100")+ }+ for _, gap := range gaps {+ t.Run(tc.resourceType+"/"+tc.metric+"/"+gap, func(t *testing.T) {+ m, elapsed := continuityManager(t, true)+ threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}+ input := &UnifiedResourceInput{ID: "target", Type: tc.resourceType, Name: "target"}+ thresholds := ThresholdConfig{CPU: threshold, Memory: threshold, Disk: threshold, DiskRead: threshold, DiskWrite: threshold, NetworkIn: threshold, NetworkOut: threshold, Temperature: threshold, Usage: threshold}+ set := func(sample *UnifiedResourceMetric) {+ switch tc.metric {+ case "cpu":+ input.CPU = sample+ case "memory":+ input.Memory = sample+ case "disk", "usage":+ input.Disk = sample+ case "temperature":+ input.Temperature = sample+ case "diskRead":+ input.DiskRead = sample+ case "diskWrite":+ input.DiskWrite = sample+ case "networkIn":+ input.NetworkIn = sample+ case "networkOut":+ input.NetworkOut = sample+ }+ }+ set(&UnifiedResourceMetric{Value: 85, Percent: 85})+ m.evaluateUnifiedMetrics(input, thresholds, nil)+ if _, ok := continuityIncident(m, input.ID, tc.metric); !ok {+ t.Fatal("known observation did not start a run")+ }+ elapsed.Store(int64(2 * time.Minute))+ invalid := &UnifiedResourceMetric{}+ switch gap {+ case "nil":+ invalid = nil+ case "NaN":+ invalid.Value = math.NaN()+ invalid.Percent = math.NaN()+ case "+Inf":+ invalid.Value = math.Inf(1)+ invalid.Percent = math.Inf(1)+ case "negative":+ invalid.Value = -1+ invalid.Percent = -1+ case "over-100":+ invalid.Value = 101+ invalid.Percent = 101+ }+ set(invalid)+ m.evaluateUnifiedMetrics(input, thresholds, nil)+ if _, ok := continuityIncident(m, input.ID, tc.metric); ok {+ t.Fatal("absent or rejected observation retained a pending run")+ }+ set(&UnifiedResourceMetric{Value: 85, Percent: 85})+ m.evaluateUnifiedMetrics(input, thresholds, nil)+ if len(m.GetActiveAlerts()) != 0 {+ t.Fatal("a fresh observation completed grace across missing telemetry")+ }+ })+ }+ }+}++func TestGuestFilesystemGapMatchesMigratedAndEmptyInventory(t *testing.T) {+ m, elapsed := continuityManager(t, true)+ threshold := &HysteresisThreshold{Trigger: 80, Clear: 70}+ ids := []string{"site:oldnode:105-disk-root", "site:othernode:105-disk-data", "site:oldnode:1050-disk-root", "elsewhere:oldnode:105-disk-root"}+ for _, id := range ids {+ m.checkMetric(id, "filesystem", "oldnode", "site", "vm", "disk", 85, threshold, nil)+ }+ elapsed.Store(int64(2 * time.Minute))+ vm := models.VM{ID: "site:newnode:105", VMID: 105, Name: "guest", Node: "newnode", Instance: "site", Type: "qemu", Status: "running", Memory: models.Memory{Usage: 10}, Disk: models.Disk{Usage: -1}, DiskStatusReason: "prev-vm-locked"}+ m.CheckGuest(vm, "site")+ for i, id := range ids {+ _, exists := continuityIncident(m, id, "disk")+ if exists != (i >= 2) {+ t.Fatalf("migration/exact identity boundary at %s: exists=%v", id, exists)+ }+ }+ m.mu.RLock()+ for _, pending := range m.intentPending {+ if pending.ResourceID == ids[0] || pending.ResourceID == ids[1] {+ t.Error("migrated filesystem grace remained durable")+ }+ }+ m.mu.RUnlock()+ if len(queryAlertEvents(t, m, eventlog.Filter{})) != 0 {+ t.Fatal("unknown empty inventory emitted lifecycle evidence")+ }+}diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.goindex 6d1ad3cba2..2242901eec 100644--- a/internal/alerts/metric_runtime.go+++ b/internal/alerts/metric_runtime.go@@ -271,6 +271,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resourcewindowed := m.evaluateMetricWindow(resourceID, resourceType, metricType, value, m.policyNow())if !windowed.Ready {+ m.interruptMetricRunIDs(resourceID, canonicalSpecID, canonicalStateID)return}value = windowed.Valuediff --git a/internal/alerts/node.go b/internal/alerts/node.goindex 5f9e0500b2..56fb559735 100644--- a/internal/alerts/node.go+++ b/internal/alerts/node.go@@ -153,22 +153,6 @@ func (m *Manager) checkNodeTemperature(node models.Node, threshold *HysteresisThm.checkMetricWithCanonicalSpec(spec, node.Name, node.Name, node.Instance, "node", temp, threshold, nil)}-// interruptMetricRun records a missing observation for a metric spec. The-// incident and any open alert are kept, but a pending activation run is-// dropped and a recovery run restarts, so a sustained-for or recovery delay-// never completes across a gap in evidence.-func (m *Manager) interruptMetricRun(spec alertspecs.ResourceAlertSpec) {- m.mu.Lock()- for _, state := range m.mirrorStatesNoLock() {- state.InterruptMetricRun(spec.ResourceID, spec.ID)- }- intentChanged := m.clearIntentPendingNoLock(canonicalTrackingKeyForSpec(spec, spec.ID))- m.mu.Unlock()- if intentChanged {- m.saveActiveAlertsAsync("canonical metric intent pending state")- }-}-// releaseNodeMetricAlerts stops node-side evaluation of the given metrics the// same way a disabled threshold does: any pending run is dropped and any open// node alert is resolved. Without this, a node alert that was open when a hostdiff --git a/internal/alerts/unified_eval.go b/internal/alerts/unified_eval.goindex c7120cf079..1fe2d5e09c 100644--- a/internal/alerts/unified_eval.go+++ b/internal/alerts/unified_eval.go@@ -300,11 +300,53 @@ func (m *Manager) evaluateUnifiedMetrics(input *UnifiedResourceInput, thresholdsmerged.Metadata = metadataopts = &merged}- for _, candidate := range buildUnifiedMetricCandidates(input, thresholds) {+ candidates := buildUnifiedMetricCandidates(input, thresholds)+ m.interruptUnobservedUnifiedMetrics(input, candidates)+ for _, candidate := range candidates {m.checkMetricWithCanonicalSpec(candidate.Spec, input.Name, input.Node, input.Instance, unifiedAlertType(input.Type), candidate.Value, candidate.Threshold, opts)}}+// An omitted/rejected metric is unknown, not a pause in its sustained-for+// clock. Build candidates stays pure; dispatch owns interruption for metrics+// this resource supports but cannot currently observe.+func (m *Manager) interruptUnobservedUnifiedMetrics(input *UnifiedResourceInput, candidates []unifiedMetricCandidate) {+ if _, ok := unifiedMetricResourceType(input.Type); !ok {+ return+ }+ metrics := [...]string{"cpu", "memory", "disk", "temperature", "diskRead", "diskWrite", "networkIn", "networkOut", "usage"}+ m.mu.Lock()+ intentChanged := false+ for _, metric := range metrics {+ switch metric {+ case "diskRead", "diskWrite", "networkIn", "networkOut":+ if !supportsUnifiedIOMetrics(input.Type) {+ continue+ }+ case "usage":+ if !unifiedStorageUsageResourceType(input.Type) {+ continue+ }+ }+ observed := false+ for _, candidate := range candidates {+ if candidate.Spec.MetricThreshold.Metric == metric {+ observed = true+ break+ }+ }+ if !observed {+ if m.interruptMetricRunNoLock(input.ID, canonicalMetricSpecID(input.ID, metric), canonicalMetricStateID(input.ID, metric)) {+ intentChanged = true+ }+ }+ }+ m.mu.Unlock()+ if intentChanged {+ m.saveActiveAlertsAsync("unified metric observation gap")+ }+}+func metricOptionsWithTags(opts *metricOptions, tags []string) *metricOptions {if len(tags) == 0 {return optsdiff --git a/internal/alerts/windowed_metric_test.go b/internal/alerts/windowed_metric_test.goindex 402ae8c89c..fc216db290 100644--- a/internal/alerts/windowed_metric_test.go+++ b/internal/alerts/windowed_metric_test.go@@ -4,6 +4,8 @@ import ("bytes""encoding/json""math"+ "os"+ "path/filepath""sync/atomic""testing""time"@@ -373,3 +375,56 @@ func TestNonFiniteMetricDoesNotPreventExplicitDisable(t *testing.T) {}}}++// A skipped window must not leave grace on disk for a restart to resume.+func TestUnknownMetricObservationDropsDurableIntentCheckpoint(t *testing.T) {+ for _, route := range []string{"legacy", "canonical", "unified", "host", "guest-memory", "guest-filesystem-expired"} {+ t.Run(route, func(t *testing.T) {+ m, elapsed := continuityManager(t, true)+ gap := "missing"+ if route == "legacy" || route == "canonical" {+ gap = "history-empty"+ }+ _, _, observe := continuityObserver(t, m, route, gap)+ observe(85, false)+ elapsed.Store(int64(40 * time.Second))+ observe(85, false)+ if err := m.SaveActiveAlerts(); err != nil {+ t.Fatal(err)+ }+ file := filepath.Join(m.getAlertsDir(), intentPendingFileName)+ read := func() []IntentPendingState {+ data, err := os.ReadFile(file)+ if err != nil {+ t.Fatal(err)+ }+ var saved []IntentPendingState+ if err := json.Unmarshal(data, &saved); err != nil {+ t.Fatal(err)+ }+ return saved+ }+ if len(read()) != 1 {+ t.Fatal("control did not retain a real in-progress grace checkpoint")+ }+ elapsed.Store(int64(2 * time.Minute))+ observe(85, true)+ if err := m.SaveActiveAlerts(); err != nil {+ t.Fatal(err)+ }+ if len(read()) != 0 {+ t.Fatal("unknown observation left grace for a restart to reuse")+ }+ dir := filepath.Dir(m.getAlertsDir())+ m.Stop()+ restarted := NewManagerWithDataDir(dir)+ t.Cleanup(restarted.Stop)+ restarted.mu.RLock()+ pending := len(restarted.intentPending)+ restarted.mu.RUnlock()+ if pending != 0 {+ t.Fatal("restart restored interrupted grace")+ }+ })+ }+}