Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
fix(monitoring): read alert windows with their writer identities
rcourtman9f325b29fd88
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 00595cf343..6a2b7de514 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -3932,6 +3932,36 @@ lifecycle decisions. Missing target, query failure, shallow history, or gappedhistory must remain unknown at the alerts boundary rather than being replacedby a synthetic healthy value.+An alert subject named by alert reference or source ID rather than registry+ID (`agent:<host>`, `docker:<host>/<container>`, a PVE guest or node ID) binds+to its canonical resource through the registry's alert-reference resolution+before the target lookup. The binding is used only when the resolved target+has the subject's own type: a merged resource (a PVE node with its host agent,+a VM with the agent inside it) prefers another source's target, and reading it+would evaluate one source's current value against another source's history.+Component, conflicting, unresolved and self bindings keep the subject's own+series. Legacy `agent:<id>` subjects keep that in-memory key and use the bare+source ID for durable history even when a merged owner prefers VM history.+A resolved target reads the keys its writers own: the `agent:`,+`dockerHost:` and `docker:` in-memory prefixes, and the `container`,+`dockerHost` and `dockerContainer` store families. Writer-boundary tests pin+these monitoring-owned coordinates. The chart API separately normalizes its+read aliases and historical compatibility families. A valid single durable+observation is returned even though its time coverage is zero; it remains+shallow evidence at the alert boundary. Reading the target's bare type and ID+left every Docker container CPU window empty under the factory five-minute CPU+rule, so container CPU alerts could never become ready.+`internal/monitoring/metric_window_target_keys_test.go` records history+through the Docker, host-agent and PVE writers and the real registry, reads+each window with the identity its alert evaluator sends (live tail, and SQLite+after the tail is lost), pins the refused binding shapes, and drives+`TestFactoryCPUWindowFiresAndRecoversForDockerContainer` from no history to an+active rolling-average CPU alert, through an unknown window that keeps the+incident, to recovery, retaining the factory grace and incident identity.+Merged-source and single-observation regressions also cover loss of the live+tail. The node alert-pass benchmark measures the source-resolution cost on+already-readable live windows without relaxing their readiness policy.+Monitoring ingest keeps mock mode hermetic. The unified read path alreadysubstitutes the mock snapshot wholesale, so anything that runs after thatsubstitution has to be suppressed explicitly rather than assumed hidden. Serverdiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 4eee42a668..62243dbd5f 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -7016,6 +7016,21 @@"internal/monitoring/unified_inventory_read_refresh_test.go"]},+ {+ "id": "metric-window-runtime",+ "label": "rolling alert writer identity, durable windows and bounded source resolution",+ "match_prefixes": [],+ "match_files": [+ "internal/monitoring/metric_window_provider.go"+ ],+ "allow_same_subsystem_tests": false,+ "test_prefixes": [],+ "exact_files": [+ "internal/monitoring/metric_window_cache_bounds_test.go",+ "internal/monitoring/metric_window_provider_test.go",+ "internal/monitoring/metric_window_target_keys_test.go"+ ]+ },{"id": "monitoring-runtime","label": "monitoring runtime proof",diff --git a/internal/monitoring/metric_window_provider.go b/internal/monitoring/metric_window_provider.goindex bb01b77d0b..75aecbba79 100644--- a/internal/monitoring/metric_window_provider.go+++ b/internal/monitoring/metric_window_provider.go@@ -7,6 +7,7 @@ import ("time""github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rs/zerolog/log")@@ -50,13 +51,22 @@ func (m *Monitor) metricWindowPoints(request alerts.MetricWindowRequest) ([]alerm.mu.RUnlock()resourceType := strings.TrimSpace(request.ResourceType)resourceID := strings.TrimSpace(request.ResourceID)- if target := m.MetricsTargetForResource(request.ResourceID); target != nil {+ memoryID, storeType, storeID := resourceID, resourceType, resourceID+ // Legacy host alerts name agent:<source ID>, while durable host history+ // is written under the bare source ID. This remains the subject's own+ // series when a merged canonical owner prefers a different source.+ if strings.EqualFold(resourceType, "agent") {+ storeID = strings.TrimPrefix(resourceID, "agent:")+ }+ if target := m.metricWindowTarget(request.ResourceID, request.ResourceType); target != nil {if strings.TrimSpace(target.ResourceType) != "" {resourceType = strings.TrimSpace(target.ResourceType)}if strings.TrimSpace(target.ResourceID) != "" {resourceID = strings.TrimSpace(target.ResourceID)}+ memoryID, storeType = metricWindowHistoryKeys(resourceType, resourceID)+ storeID = resourceID}if resourceType == "" || resourceID == "" {return nil, fmt.Errorf("metric target unavailable for %q", request.ResourceID)@@ -67,9 +77,9 @@ func (m *Monitor) metricWindowPoints(request alerts.MetricWindowRequest) ([]alerif duration <= 0 {return nil, fmt.Errorf("invalid metric window")}- points := m.inMemoryMetricWindow(history, resourceType, resourceID, metric, duration)+ points := m.inMemoryMetricWindow(history, resourceType, memoryID, metric, duration)if metricWindowCoverage(points) < duration*8/10 {- stored := m.persistentMetricWindow(history, resourceType, resourceID, metric, request.Start, request.End)+ stored := m.persistentMetricWindow(history, storeType, storeID, metric, request.Start, request.End)// Append the fresh in-memory tail last so it remains authoritative when// SQLite contains an older value at the same timestamp.points = mergeMetricWindowPoints(stored, points, request.Start, request.End)@@ -82,6 +92,59 @@ func (m *Monitor) metricWindowPoints(request alerts.MetricWindowRequest) ([]alerreturn result, nil}+// metricWindowTarget resolves the metrics target for an alert subject. Some+// evaluators name their subject by alert reference or source ID rather than+// registry ID (agent:<host>, docker:<host>/<container>, a PVE guest or node+// ID); those bind to their canonical resource through the registry's+// alert-reference resolution. A binding is used only when its target has the+// subject's own type. A canonical resource can merge sources (a PVE node with+// its host agent, a VM with the agent inside it), and its preferred target+// would then read another source's series than the one being evaluated. A+// component reference names part of another resource, so it is refused too.+// A refused or unresolved subject keeps the request's own coordinates.+func (m *Monitor) metricWindowTarget(resourceID, resourceType string) *unifiedresources.MetricsTarget {+ if target := m.MetricsTargetForResource(resourceID); target != nil {+ return target+ }+ m.mu.RLock()+ store := m.resourceStore+ m.mu.RUnlock()+ resolver, ok := store.(unifiedresources.AlertResourceReferenceResolver)+ if !ok {+ return nil+ }+ reference, claimed := resolver.ResolveAlertResourceReference(resourceID)+ canonicalID := strings.TrimSpace(reference.ResourceID)+ if !claimed || reference.Component || canonicalID == "" || canonicalID == strings.TrimSpace(resourceID) {+ return nil+ }+ target := m.MetricsTargetForResource(canonicalID)+ if target == nil || !strings.EqualFold(strings.TrimSpace(target.ResourceType), strings.TrimSpace(resourceType)) {+ return nil+ }+ return target+}++// metricWindowHistoryKeys maps a canonical metrics target onto the keys its+// writers record. In-memory history prefixes host-agent and Docker series,+// and the store keeps PVE system containers and Docker series under their+// own families. Writer-boundary tests pin these monitoring-owned coordinates.+// Chart API aliases have their own compatibility policy at the read boundary.+func metricWindowHistoryKeys(resourceType, resourceID string) (memoryID, storeType string) {+ switch strings.ToLower(resourceType) {+ case "system-container":+ return resourceID, "container"+ case "agent":+ return "agent:" + resourceID, "agent"+ case "docker-host":+ return "dockerHost:" + resourceID, "dockerHost"+ case "app-container":+ return "docker:" + resourceID, "dockerContainer"+ default:+ return resourceID, resourceType+ }+}+func (m *Monitor) inMemoryMetricWindow(history *MetricsHistory, resourceType, resourceID, metric string, duration time.Duration) []MetricPoint {if history == nil {return nil@@ -121,7 +184,9 @@ func (m *Monitor) persistentMetricWindow(history *MetricsHistory, resourceType,for i, point := range stored {converted[i] = MetricPoint{Timestamp: point.Timestamp, Value: point.Value}}- if metricWindowCoverage(converted) > metricWindowCoverage(best) {+ // A single observation has zero span, but still wins over no data.+ // Readiness belongs to the alert evaluator, not this source selector.+ if len(converted) > 0 && (len(best) == 0 || metricWindowCoverage(converted) > metricWindowCoverage(best)) {best = converted}}diff --git a/internal/monitoring/metric_window_target_keys_test.go b/internal/monitoring/metric_window_target_keys_test.gonew file mode 100644index 0000000000..28f3ce526f--- /dev/null+++ b/internal/monitoring/metric_window_target_keys_test.go@@ -0,0 +1,670 @@+package monitoring++import (+ "encoding/json"+ "fmt"+ "os"+ "strings"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"+ agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"+)++// These tests record history through the production writers and registry,+// then read the rolling alert window with the identity each alert evaluator+// sends. None of them writes a key the provider maps to, so they tie the+// provider to its writers rather than to its own lookup table.++const metricWindowContainerName = "app"++func useRealMetricWindowMode(t *testing.T) {+ t.Helper()+ previous := mock.IsMockEnabled()+ mustSetMockEnabled(t, false)+ t.Cleanup(func() { mustSetMockEnabled(t, previous) })+}++// newMetricWindowRegistryMonitor returns a report-capable monitor. With+// persistent set, writers also commit to SQLite so a test can lose the+// in-memory tail the way a restart does.+func newMetricWindowRegistryMonitor(t *testing.T, persistent bool) *Monitor {+ t.Helper()+ m := newTestMonitor(t)+ if persistent {+ m.metricsStore = newChartFallbackTestMonitor(t).metricsStore+ }+ return m+}++func newMetricWindowAdapter() *unifiedresources.MonitorAdapter {+ return unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+}++func dropInMemoryTail(m *Monitor) {+ m.metricsStore.Flush()+ m.metricsHistory.Reset()+}++func readCPUWindow(t *testing.T, m *Monitor, resourceID, resourceType string) ([]alerts.MetricWindowPoint, time.Time) {+ t.Helper()+ end := time.Now()+ points, err := m.metricWindowPoints(alerts.MetricWindowRequest{+ ResourceID: resourceID,+ ResourceType: resourceType,+ Metric: "cpu",+ Start: end.Add(-5 * time.Minute),+ End: end,+ })+ if err != nil {+ t.Fatalf("metricWindowPoints(%q, %q): %v", resourceID, resourceType, err)+ }+ return points, end+}++// assertOneCPUSample requires exactly the writer's observation. SQLite keeps+// whole seconds, so the lower bound is the second the sample was taken in.+func assertOneCPUSample(t *testing.T, label string, points []alerts.MetricWindowPoint, want float64, takenAfter, readAt time.Time) {+ t.Helper()+ if len(points) != 1 || points[0].Value != want ||+ points[0].Timestamp.Before(takenAfter.Truncate(time.Second)) || points[0].Timestamp.After(readAt) {+ t.Fatalf("%s window = %+v, want one %.2f sample taken after %s", label, points, want, takenAfter.Format(time.RFC3339Nano))+ }+}++func persistenceName(persistent bool) string {+ if persistent {+ return "persistent history after the in-memory tail is lost"+ }+ return "live in-memory tail"+}++func applyMetricWindowDockerReport(t *testing.T, m *Monitor, containerID string, containerCPU, hostCPU float64) models.DockerHost {+ t.Helper()+ host, err := m.ApplyDockerReport(agentsdocker.Report{+ Agent: agentsdocker.AgentInfo{ID: "docker-agent", Version: "test", IntervalSeconds: 30},+ Host: agentsdocker.HostInfo{MachineID: "machine", Hostname: "worker", TotalCPU: 4, CPUUsagePercent: hostCPU},+ Containers: []agentsdocker.Container{{ID: containerID, Name: metricWindowContainerName, State: "running", CPUPercent: containerCPU}},+ Timestamp: time.Now().UTC(),+ }, nil)+ if err != nil {+ t.Fatalf("ApplyDockerReport: %v", err)+ }+ if len(host.Containers) != 1 || host.Containers[0].ID != containerID {+ t.Fatalf("report did not record the container: %+v", host.Containers)+ }+ return host+}++// Docker container CPU alerts name their subject docker:<host>/<container>+// with type app-container (alerts/docker.go) and evaluate the factory+// five-minute CPU window. Docker reports full 64-character container IDs;+// the registry binds a container reference only by that durable ID.+func TestMetricWindowPointsReadsDockerContainerHistoryFromAlertReference(t *testing.T) {+ useRealMetricWindowMode(t)+ containerID := strings.Repeat("ab", 32)++ for _, persistent := range []bool{false, true} {+ t.Run(persistenceName(persistent), func(t *testing.T) {+ m := newMetricWindowRegistryMonitor(t, persistent)+ m.SetResourceStore(newMetricWindowAdapter())+ reportedAt := time.Now()+ host := applyMetricWindowDockerReport(t, m, containerID, 380, 0)+ want := models.DockerContainerCPUCapacityPercent(host.Containers[0], host.CPUs)+ if persistent {+ dropInMemoryTail(m)+ }++ reference := alerts.DockerContainerResourceID(host.ID, containerID, metricWindowContainerName)+ points, readAt := readCPUWindow(t, m, reference, "app-container")+ assertOneCPUSample(t, "container CPU", points, want, reportedAt, readAt)+ })+ }++ t.Run("empty history stays unknown", func(t *testing.T) {+ m := newMetricWindowRegistryMonitor(t, false)+ m.SetResourceStore(newMetricWindowAdapter())+ host := applyMetricWindowDockerReport(t, m, containerID, 380, 0)+ m.metricsHistory.Reset()++ reference := alerts.DockerContainerResourceID(host.ID, containerID, metricWindowContainerName)+ if points, _ := readCPUWindow(t, m, reference, "app-container"); len(points) != 0 {+ t.Fatalf("window without history = %+v, want no observations", points)+ }+ })++ t.Run("a shortened container ID does not bind", func(t *testing.T) {+ m := newMetricWindowRegistryMonitor(t, false)+ m.SetResourceStore(newMetricWindowAdapter())+ host := applyMetricWindowDockerReport(t, m, containerID, 380, 0)++ reference := alerts.DockerContainerResourceID(host.ID, containerID[:12], metricWindowContainerName)+ if points, _ := readCPUWindow(t, m, reference, "app-container"); len(points) != 0 {+ t.Fatalf("short container reference read history: %+v", points)+ }+ })++ t.Run("a replaced resource store is resolved afresh", func(t *testing.T) {+ m := newMetricWindowRegistryMonitor(t, false)+ m.SetResourceStore(newMetricWindowAdapter())+ reportedAt := time.Now()+ host := applyMetricWindowDockerReport(t, m, containerID, 380, 0)+ want := models.DockerContainerCPUCapacityPercent(host.Containers[0], host.CPUs)+ reference := alerts.DockerContainerResourceID(host.ID, containerID, metricWindowContainerName)++ // SetResourceStore rebuilds the new adapter from current state.+ m.SetResourceStore(newMetricWindowAdapter())+ points, readAt := readCPUWindow(t, m, reference, "app-container")+ assertOneCPUSample(t, "container CPU after store replacement", points, want, reportedAt, readAt)++ // A store that cannot bind the reference leaves no earlier binding behind.+ m.mu.Lock()+ m.resourceStore = &metricWindowReferenceStore{}+ m.mu.Unlock()+ if points, _ := readCPUWindow(t, m, reference, "app-container"); len(points) != 0 {+ t.Fatalf("window kept a binding from the replaced store: %+v", points)+ }+ })+}++// Host-agent CPU alerts name their subject agent:<host ID> with type agent+// (alerts/host.go hostResourceID).+func TestMetricWindowPointsReadsHostAgentHistoryFromAlertReference(t *testing.T) {+ useRealMetricWindowMode(t)+ for _, persistent := range []bool{false, true} {+ t.Run(persistenceName(persistent), func(t *testing.T) {+ m := newMetricWindowRegistryMonitor(t, persistent)+ m.SetResourceStore(newMetricWindowAdapter())+ reportedAt := time.Now()+ host, err := m.ApplyHostReport(agentshost.Report{+ Agent: agentshost.AgentInfo{ID: "host-agent", Version: "test", IntervalSeconds: 30},+ Host: agentshost.HostInfo{ID: "host-agent", MachineID: "machine", Hostname: "worker", Platform: "linux"},+ Metrics: agentshost.Metrics{CPUUsagePercent: 61},+ Timestamp: time.Now().UTC(),+ }, nil)+ if err != nil || host.ID == "" {+ t.Fatalf("ApplyHostReport: host %q, err %v", host.ID, err)+ }+ if persistent {+ dropInMemoryTail(m)+ }++ points, readAt := readCPUWindow(t, m, "agent:"+host.ID, "agent")+ assertOneCPUSample(t, "host agent CPU", points, host.CPUUsage, reportedAt, readAt)+ })+ }+}++// No alert evaluates a Docker host's CPU window today, but a canonical+// docker-host target must still read the series its report records.+func TestMetricWindowPointsReadsDockerHostHistoryFromCanonicalTarget(t *testing.T) {+ useRealMetricWindowMode(t)+ for _, persistent := range []bool{false, true} {+ t.Run(persistenceName(persistent), func(t *testing.T) {+ m := newMetricWindowRegistryMonitor(t, persistent)+ adapter := newMetricWindowAdapter()+ m.SetResourceStore(adapter)+ reportedAt := time.Now()+ host := applyMetricWindowDockerReport(t, m, strings.Repeat("ef", 32), 0, 37)++ hostResourceID := ""+ for _, resource := range adapter.GetAll() {+ if target := adapter.MetricsTargetForResource(resource.ID); target != nil &&+ target.ResourceType == "docker-host" && target.ResourceID == host.ID {+ hostResourceID = resource.ID+ }+ }+ if hostResourceID == "" {+ t.Fatalf("registry exposes no docker-host target for %q", host.ID)+ }+ if persistent {+ dropInMemoryTail(m)+ }++ points, readAt := readCPUWindow(t, m, hostResourceID, "docker-host")+ assertOneCPUSample(t, "Docker host CPU", points, host.CPUUsage, reportedAt, readAt)+ })+ }+}++// PVE guest and node alerts send the PVE source ID with a display type+// ("VM", "Container", "Node" from alerts unifiedAlertType); the registry's+// own form of an LXC is a canonical system-container target.+func TestMetricWindowPointsReadsPVEGuestAndNodeHistory(t *testing.T) {+ useRealMetricWindowMode(t)+ vm := models.VM{ID: "pve1:node1:100", VMID: 100, Name: "web", Node: "node1", Instance: "pve1"}+ ct := models.Container{ID: "pve1:node1:101", VMID: 101, Name: "cache", Node: "node1", Instance: "pve1"}+ node := models.Node{ID: "pve1-node1", Name: "node1", Instance: "pve1"}++ setup := func(t *testing.T, persistent bool) (*Monitor, *unifiedresources.MonitorAdapter, time.Time) {+ t.Helper()+ m := newMetricWindowRegistryMonitor(t, persistent)+ m.state.UpdateNodesForInstance("pve1", []models.Node{node})+ m.state.UpdateVMsForInstance("pve1", []models.VM{vm})+ m.state.UpdateContainersForInstance("pve1", []models.Container{ct})+ adapter := newMetricWindowAdapter()+ m.SetResourceStore(adapter) // rebuilt from the state above+ takenAt := time.Now()+ m.recordGuestMetric("vm", vm.ID, 41, -1, -1, -1, -1, -1, -1, -1, takenAt)+ m.recordGuestMetric("container", ct.ID, 42, -1, -1, -1, -1, -1, -1, -1, takenAt)+ // The node poller writes inline (monitor_pve_storage.go): node.ID in+ // node history and the node store family.+ m.metricsHistory.AddNodeMetric(node.ID, "cpu", 43, takenAt)+ if persistent {+ m.metricsStore.Write("node", node.ID, "cpu", 43, takenAt)+ dropInMemoryTail(m)+ }+ return m, adapter, takenAt+ }++ cases := []struct {+ name string+ resourceID string+ resourceType string+ canonical unifiedresources.ResourceType+ want float64+ }{+ {name: "VM alert subject", resourceID: vm.ID, resourceType: "VM", want: 41},+ {name: "LXC alert subject", resourceID: ct.ID, resourceType: "Container", want: 42},+ {name: "canonical system-container target", resourceType: "system-container", canonical: unifiedresources.ResourceTypeSystemContainer, want: 42},+ {name: "node alert subject", resourceID: node.ID, resourceType: "Node", want: 43},+ }+ for _, tc := range cases {+ for _, persistent := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/%s", tc.name, persistenceName(persistent)), func(t *testing.T) {+ m, adapter, takenAt := setup(t, persistent)+ resourceID := tc.resourceID+ if tc.canonical != "" {+ for _, resource := range adapter.GetAll() {+ if resource.Type == tc.canonical {+ resourceID = resource.ID+ }+ }+ if resourceID == "" {+ t.Fatalf("registry has no %s resource", tc.canonical)+ }+ }++ points, readAt := readCPUWindow(t, m, resourceID, tc.resourceType)+ assertOneCPUSample(t, tc.name, points, tc.want, takenAt, readAt)+ })+ }+ }+}++// metricWindowReferenceStore answers target and alert-reference lookups from+// fixed tables, so a test can present each binding shape the registry returns.+type metricWindowReferenceStore struct {+ targets map[string]*unifiedresources.MetricsTarget+ references map[string]metricWindowReference+}++type metricWindowReference struct {+ binding unifiedresources.AlertResourceReference+ claimed bool+}++func (*metricWindowReferenceStore) ShouldSkipAPIPolling(string) bool { return false }+func (*metricWindowReferenceStore) GetPollingRecommendations() map[string]float64 { return nil }+func (*metricWindowReferenceStore) GetAll() []unifiedresources.Resource { return nil }+func (*metricWindowReferenceStore) PopulateFromSnapshot(models.StateSnapshot) {}+func (s *metricWindowReferenceStore) MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget {+ return s.targets[resourceID]+}+func (s *metricWindowReferenceStore) ResolveAlertResourceReference(ref string) (unifiedresources.AlertResourceReference, bool) {+ reference := s.references[ref]+ return reference.binding, reference.claimed+}++// A binding that names another source of a merged resource, a component's+// owner, nothing at all, or the subject itself must leave the subject reading+// its own series.+func TestMetricWindowPointsKeepsSubjectSeriesForUnusableBindings(t *testing.T) {+ useRealMetricWindowMode(t)+ bound := func(resourceType unifiedresources.ResourceType, id string, component bool) metricWindowReference {+ return metricWindowReference{+ binding: unifiedresources.AlertResourceReference{ResourceID: id, ResourceType: resourceType, Component: component},+ claimed: true,+ }+ }+ guest := func(h *MetricsHistory, id string, value float64, at time.Time) {+ h.AddGuestMetric(id, "cpu", value, at)+ }+ cases := []struct {+ name string+ resourceID string+ resourceType string+ reference metricWindowReference+ targets map[string]*unifiedresources.MetricsTarget+ record func(*MetricsHistory, string, float64, time.Time)+ }{+ {+ name: "PVE node merged with its host agent",+ resourceID: "pve1-node1",+ resourceType: "Node",+ reference: bound(unifiedresources.ResourceTypeAgent, "machine-1", false),+ targets: map[string]*unifiedresources.MetricsTarget{"machine-1": {ResourceType: "agent", ResourceID: "host-agent-1"}},+ record: func(h *MetricsHistory, id string, value float64, at time.Time) { h.AddNodeMetric(id, "cpu", value, at) },+ },+ {+ name: "host agent merged into the VM it runs in",+ resourceID: "agent:host-agent-1",+ resourceType: "agent",+ reference: bound(unifiedresources.ResourceTypeVM, "vm-canonical", false),+ targets: map[string]*unifiedresources.MetricsTarget{"vm-canonical": {ResourceType: "vm", ResourceID: "pve1:node1:100"}},+ record: guest,+ },+ {+ name: "component of another resource",+ resourceID: "agent:host-agent-1/disk/sda",+ resourceType: "agent",+ reference: bound(unifiedresources.ResourceTypeAgent, "machine-1", true),+ targets: map[string]*unifiedresources.MetricsTarget{"machine-1": {ResourceType: "agent", ResourceID: "host-agent-1"}},+ record: guest,+ },+ {+ name: "conflicting inventory",+ resourceID: "docker:host-1/" + strings.Repeat("ab", 32),+ resourceType: "app-container",+ reference: metricWindowReference{claimed: true},+ record: guest,+ },+ {+ name: "unknown subject",+ resourceID: "docker:host-1/" + strings.Repeat("cd", 32),+ resourceType: "app-container",+ record: guest,+ },+ {+ name: "binding to itself",+ resourceID: "pve1:node1:100",+ resourceType: "VM",+ reference: bound(unifiedresources.ResourceTypeVM, "pve1:node1:100", false),+ record: guest,+ },+ }+ for i, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ history := NewMetricsHistory(32, time.Hour)+ at := time.Now().Add(-time.Minute)+ value := float64(20 + i)+ tc.record(history, tc.resourceID, value, at)+ store := &metricWindowReferenceStore{+ targets: tc.targets,+ references: map[string]metricWindowReference{tc.resourceID: tc.reference},+ }+ m := &Monitor{metricsHistory: history, resourceStore: store}++ points, _ := readCPUWindow(t, m, tc.resourceID, tc.resourceType)+ if len(points) != 1 || points[0].Value != value || !points[0].Timestamp.Equal(at) {+ t.Fatalf("window = %+v, want the subject's own %.0f sample", points, value)+ }+ })+ }+}++// A single durable observation has zero time coverage, but it is still an+// observation. The alert evaluator, rather than the reader, rejects shallow+// evidence for a rolling average.+func TestMetricWindowPointsPreservesSinglePersistentObservation(t *testing.T) {+ useRealMetricWindowMode(t)+ m := newChartFallbackTestMonitor(t)+ at := time.Now().Add(-time.Minute).Truncate(time.Second)+ m.recordGuestMetric("vm", "pve1:node1:100", 95, -1, -1, -1, -1, -1, -1, -1, at)+ dropInMemoryTail(m)+ stored, err := m.metricsStore.Query("vm", "pve1:node1:100", "cpu", at.Add(-time.Minute), time.Now(), 0)+ if err != nil || len(stored) != 1 || stored[0].Value != 95 {+ t.Fatalf("writer's durable observation = %+v, err %v", stored, err)+ }+ points, readAt := readCPUWindow(t, m, "pve1:node1:100", "VM")+ assertOneCPUSample(t, "single durable observation", points, 95, at, readAt)+}++// The real writers retain independent node/agent histories even when a+// canonical owner prefers another source. Losing the live tail must preserve+// that same source boundary, including the agent reference's store ID.+func TestMetricWindowPointsKeepsMergedSourceHistoryAfterTailLoss(t *testing.T) {+ useRealMetricWindowMode(t)+ writer := newMetricWindowRegistryMonitor(t, true)+ reportedAt := time.Now()+ host, err := writer.ApplyHostReport(agentshost.Report{+ Agent: agentshost.AgentInfo{ID: "linked-agent", Version: "test", IntervalSeconds: 30},+ Host: agentshost.HostInfo{ID: "linked-agent", MachineID: "linked-machine", Hostname: "linked-worker", Platform: "linux"},+ Metrics: agentshost.Metrics{CPUUsagePercent: 61},+ Timestamp: time.Now().UTC(),+ }, nil)+ if err != nil {+ t.Fatal(err)+ }+ const nodeID, vmID = "pve1-node1", "pve1:node1:100"+ at := time.Now()+ // These are the node poller's inline writer calls, as in the PVE guard.+ writer.metricsHistory.AddNodeMetric(nodeID, "cpu", 43, at)+ writer.metricsStore.Write("node", nodeID, "cpu", 43, at)+ writer.recordGuestMetric("vm", vmID, 99, -1, -1, -1, -1, -1, -1, -1, at)+ dropInMemoryTail(writer)+ agentReference := "agent:" + host.ID+ bindings := &metricWindowReferenceStore{+ targets: map[string]*unifiedresources.MetricsTarget{+ "agent-canonical": {ResourceType: "agent", ResourceID: host.ID},+ "vm-canonical": {ResourceType: "vm", ResourceID: vmID},+ },+ references: map[string]metricWindowReference{+ nodeID: {binding: unifiedresources.AlertResourceReference{ResourceID: "agent-canonical", ResourceType: unifiedresources.ResourceTypeAgent}, claimed: true},+ agentReference: {binding: unifiedresources.AlertResourceReference{ResourceID: "vm-canonical", ResourceType: unifiedresources.ResourceTypeVM}, claimed: true},+ },+ }+ provider := &Monitor{metricsHistory: writer.metricsHistory, metricsStore: writer.metricsStore, resourceStore: bindings}+ for _, tc := range []struct {+ name, reference, subjectType, storeType, storeID string+ want float64+ }{+ {"node merged with agent", nodeID, "Node", "node", nodeID, 43},+ {"agent merged into VM", agentReference, "agent", "agent", host.ID, 61},+ } {+ t.Run(tc.name, func(t *testing.T) {+ stored, err := writer.metricsStore.Query(tc.storeType, tc.storeID, "cpu", reportedAt.Add(-time.Minute), time.Now(), 0)+ if err != nil || len(stored) != 1 || stored[0].Value != tc.want {+ t.Fatalf("own writer observation = %+v, err %v", stored, err)+ }+ points, readAt := readCPUWindow(t, provider, tc.reference, tc.subjectType)+ assertOneCPUSample(t, "merged source", points, tc.want, reportedAt, readAt)+ })+ }+}++// The factory five-minute CPU rule must be able to fire for a Docker container+// and later recover through the production provider. The alert manager's+// clock is not reachable from this package, so window contents are placed+// relative to the real clock instead: the report path supplies the+// evaluation, the history key is learned from the real writer, and earlier+// samples are added under that key.+func TestFactoryCPUWindowFiresAndRecoversForDockerContainer(t *testing.T) {+ useRealMetricWindowMode(t)+ m := newMetricWindowRegistryMonitor(t, false)+ m.alertManager.Stop()+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir())+ m.SetResourceStore(newMetricWindowAdapter())++ cfg := m.alertManager.GetConfig()+ cfg.ActivationState = alerts.ActivationActive+ m.alertManager.UpdateConfig(cfg)+ if window := m.alertManager.GetConfig().MetricEvaluationWindows["all"]["cpu"]; window != 300 {+ t.Fatalf("factory CPU window = %ds, want 300", window)+ }+ m.alertManager.SetMetricWindowProvider(m.metricWindowPoints) // as New wires it++ containerID := strings.Repeat("cd", 32)+ host := applyMetricWindowDockerReport(t, m, containerID, 380, 0)+ busy := models.DockerContainerCPUCapacityPercent(host.Containers[0], host.CPUs)+ if busy < 80 {+ t.Fatalf("fixture container CPU = %.1f%%, want above the 80%% factory trigger", busy)+ }+ cpuAlert := func() *alerts.Alert {+ for _, alert := range m.alertManager.GetActiveAlerts() {+ if alert.Type == "cpu" && strings.Contains(alert.ResourceID, containerID) {+ return &alert+ }+ }+ return nil+ }+ reference := alerts.DockerContainerResourceID(host.ID, containerID, metricWindowContainerName)+ policy := m.alertManager.ResolveEffectiveIntentPolicy(reference, "app-container", alerts.MetricAlertIntentSignal("cpu"))+ if policy.GraceSeconds != 5 {+ t.Fatalf("factory CPU grace = %ds, want 5", policy.GraceSeconds)+ }+ proof := func(phase string) {+ t.Helper()+ if os.Getenv("PULSE_METRIC_WINDOW_PROOF") == "" {+ return+ }+ observed := make([]alerts.Alert, 0, 1)+ if alert := cpuAlert(); alert != nil {+ observed = append(observed, *alert)+ }+ data, err := json.Marshal(map[string]any{"phase": phase, "alerts": observed, "windowSeconds": 300, "graceSeconds": policy.GraceSeconds, "hostID": host.ID, "containerID": containerID})+ if err != nil {+ t.Fatal(err)+ }+ fmt.Printf("METRIC_WINDOW_PROOF %s\n", data)+ }+ if alert := cpuAlert(); alert != nil {+ t.Fatalf("CPU alert opened without window history: %+v", alert)+ }+ proof("shallow")++ key := ""+ m.metricsHistory.mu.RLock()+ for candidate, series := range m.metricsHistory.guestMetrics {+ if n := len(series.CPU); n > 0 && series.CPU[n-1].Value == busy {+ key = candidate+ }+ }+ m.metricsHistory.mu.RUnlock()+ if key == "" {+ t.Fatal("the Docker report recorded no container CPU history")+ }+ // Five minutes of history at one level: three samples 90 s apart, then+ // the evaluation's own current value.+ seed := func(value float64) {+ m.metricsHistory.Reset()+ now := time.Now()+ for _, age := range []time.Duration{270 * time.Second, 180 * time.Second, 90 * time.Second} {+ m.metricsHistory.AddGuestMetric(key, "cpu", value, now.Add(-age))+ }+ }++ seed(busy)+ applyMetricWindowDockerReport(t, m, containerID, 380, 0)+ if alert := cpuAlert(); alert != nil {+ t.Fatalf("CPU alert bypassed the factory grace: %+v", alert)+ }+ proof("pending")+ time.Sleep(time.Duration(policy.GraceSeconds)*time.Second + 100*time.Millisecond)+ applyMetricWindowDockerReport(t, m, containerID, 380, 0)+ alert := cpuAlert()+ if alert == nil {+ t.Fatalf("factory CPU window never fired for a container held at %.1f%%", busy)+ }+ samples := 0+ switch count := alert.Metadata["evaluationSampleCount"].(type) {+ case int:+ samples = count+ case float64:+ samples = int(count)+ }+ if alert.Metadata["evaluationMode"] != "rolling_average" || alert.Metadata["evaluationWindowSeconds"] != 300 || samples < 4 {+ t.Fatalf("alert was not raised from the rolling window: %+v", alert.Metadata)+ }+ incidentID, startedAt := alert.ID, alert.StartTime+ proof("active")++ // An idle reading without history is unknown, not recovery.+ m.metricsHistory.Reset()+ applyMetricWindowDockerReport(t, m, containerID, 0, 0)+ if unknown := cpuAlert(); unknown == nil || unknown.ID != incidentID || !unknown.StartTime.Equal(startedAt) {+ t.Fatalf("unknown history changed the incident: %+v", unknown)+ }+ proof("unknown")++ seed(10)+ applyMetricWindowDockerReport(t, m, containerID, 0, 0)+ if alert := cpuAlert(); alert != nil {+ t.Fatalf("CPU alert did not recover after a quiet five-minute window: %+v", alert)+ }+ proof("recovered")+}++// PVE node alerts already had readable live windows on the baseline. Measure+// complete healthy alert passes over those same observations so the extra+// source-reference resolution is compared without changing readiness.+func BenchmarkMetricWindowNodeAlertPass(b *testing.B) {+ previous := mock.IsMockEnabled()+ if err := mock.SetEnabled(false); err != nil {+ b.Fatal(err)+ }+ b.Cleanup(func() {+ if err := mock.SetEnabled(previous); err != nil {+ b.Error(err)+ }+ })+ for _, count := range []int{100, 1000} {+ b.Run(fmt.Sprintf("nodes-%d", count), func(b *testing.B) {+ now := time.Now()+ history := NewMetricsHistory(16, time.Hour)+ nodes := make([]models.Node, count)+ for i := range nodes {+ nodes[i] = models.Node{ID: fmt.Sprintf("bench-node-%d", i), Name: fmt.Sprintf("node-%d", i), Instance: "bench", Status: "online", CPU: 0.1}+ for _, age := range []time.Duration{270 * time.Second, 180 * time.Second, 90 * time.Second, 0} {+ history.AddNodeMetric(nodes[i].ID, "cpu", 0.1, now.Add(-age))+ }+ }+ adapter := newMetricWindowAdapter()+ adapter.PopulateFromSnapshot(models.StateSnapshot{Nodes: nodes})+ monitor := &Monitor{metricsHistory: history, resourceStore: adapter}+ manager := alerts.NewManagerWithDataDir(b.TempDir())+ b.Cleanup(manager.Stop)+ cfg := manager.GetConfig()+ cfg.ActivationState = alerts.ActivationActive+ manager.UpdateConfig(cfg)+ windowCalls, readyWindows := 0, 0+ manager.SetMetricWindowProvider(func(request alerts.MetricWindowRequest) ([]alerts.MetricWindowPoint, error) {+ windowCalls+++ points, err := monitor.metricWindowPoints(request)+ if err == nil && len(points) == 4 && points[3].Timestamp.Sub(points[0].Timestamp) >= 240*time.Second {+ readyWindows+++ }+ return points, err+ })+ if binding, claimed := adapter.ResolveAlertResourceReference(nodes[0].ID); !claimed || binding.ResourceID == "" {+ b.Fatalf("benchmark node reference does not bind: %+v", binding)+ }+ // Warm the generation-scoped registry indexes outside the timed pass.+ manager.CheckNode(nodes[0])+ if windowCalls == 0 || readyWindows == 0 {+ b.Fatal("benchmark must evaluate a ready CPU window")+ }+ before := readyWindows+ b.ReportAllocs()+ b.ResetTimer()+ for pass := 0; pass < b.N; pass++ {+ for _, node := range nodes {+ manager.CheckNode(node)+ }+ }+ b.StopTimer()+ b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N*count), "ns/subject")+ if readyWindows-before != b.N*count {+ b.Fatalf("ready CPU windows = %d, want %d", readyWindows-before, b.N*count)+ }+ if got := manager.GetActiveAlerts(); len(got) != 0 {+ b.Fatalf("healthy benchmark opened incidents: %+v", got)+ }+ })+ }+}