Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Compare complete broadcast health at one evaluation instant
pulse-triage[bot]9cc8523881fb
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex fd23f74cc1..0e47f9e058 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -41,6 +41,12 @@ the exact parent's complete conversion and final JSON encoding, with cold,list-only, typed-clean and dirty inputs; they do not establish installed CPU/RSSrelief, native identity refresh throughput or completion of #2199.+Complete-content comparisons use one health-evaluation instant through a+private clock seam. Production still samples real UTC at the same post-read+boundary. The minute-boundary control requires age labels to advance without+renewing LastSeen; comparisons retain every health field instead of stripping+the reproduced `1m`/`2m` clock difference from JSON.+### Report identity lookups do not rebuild unrelated Kubernetes metadataThe producer's two temporary host/Docker-host admission registries use adiff --git a/internal/monitoring/broadcast_metrics_snapshot_test.go b/internal/monitoring/broadcast_metrics_snapshot_test.goindex d238907642..02e18e99cc 100644--- a/internal/monitoring/broadcast_metrics_snapshot_test.go+++ b/internal/monitoring/broadcast_metrics_snapshot_test.go@@ -79,17 +79,18 @@ func TestBroadcastMetricsSnapshotConnectedContent(t *testing.T) {}assertEqual := func(stage string) {t.Helper()+ evaluationTime := time.Now().UTC()before, err := json.Marshal(adapter.GetAll())if err != nil {t.Fatal(err)}- want := legacyFrontendProjectionForTest(m, graph.State, adapter.GetAll(), adapter, time.Now().UTC())+ want := legacyFrontendProjectionForTest(m, graph.State, adapter.GetAll(), adapter, evaluationTime)// The parent's unified view fills a missing store watermark from the// listed resources. Preserve that full-state header in the oracle too.if freshness := latestUnifiedResourceLastSeen(adapter.GetAll()); !freshness.IsZero() {want.LastUpdate = freshness.UnixMilli()}- got := m.buildBroadcastFrontendStateFromSnapshot(graph.State)+ got := m.buildBroadcastFrontendStateFromSnapshotWithClock(graph.State, func() time.Time { return evaluationTime })wantJSON, err := json.Marshal(want)if err != nil {t.Fatal(err)@@ -125,6 +126,39 @@ func TestBroadcastMetricsSnapshotConnectedContent(t *testing.T) {assertEqual("dirty-same-time")}+func TestBroadcastMetricsSnapshotHealthClockBoundary(t *testing.T) {+ m, adapter, id := newReadStateCloneTestMonitor(t, 1)+ rows := adapter.GetAll()+ original := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)+ rows[0].LastSeen = original+ rows[0].SourceStatus = map[unifiedresources.DataSource]unifiedresources.SourceStatus{+ unifiedresources.SourceProxmox: {Status: "stale", LastSeen: original},+ }+ rr := unifiedresources.NewRegistry(nil)+ rr.IngestResources(rows)+ m.resourceStore = &broadcastProjectionCountingStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(rr)}+ for _, tc := range []struct {+ age time.Duration+ detail string+ }{{59 * time.Second, "0m"}, {61 * time.Second, "1m"}} {+ calls := 0+ got := m.buildBroadcastFrontendStateFromSnapshotWithClock(models.EmptyStateSnapshot(), func() time.Time {+ calls+++ return original.Add(tc.age)+ })+ if calls != 1 || len(got.Resources) != 1 || got.Resources[0].ID != id || got.Resources[0].LastSeen != original.UnixMilli() {+ t.Fatal("health clock changed capture identity/original time or was not sampled once")+ }+ var health unifiedresources.ResourceHealth+ if err := json.Unmarshal(got.Resources[0].Health, &health); err != nil {+ t.Fatal(err)+ }+ if health.Verdict != unifiedresources.HealthStale || len(health.Reasons) != 1 || health.Reasons[0].Code != "telemetry_stale" || health.Reasons[0].Detail != tc.detail {+ t.Fatalf("health age must advance across the minute boundary: %+v", health)+ }+ }+}+func TestBroadcastMetricsSnapshotOwnsTargetsBeforeLiveReplacement(t *testing.T) {m, adapter, _ := newReadStateCloneTestMonitor(t, 4)view := m.currentUnifiedStateView()diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 9e11cb4f7e..f65c96057a 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -2958,7 +2958,7 @@ func TestBroadcastProjectionMatchesPreviousPipeline(t *testing.T) {if !view.freshness.IsZero() {want.LastUpdate = view.freshness.UnixMilli()}- got := m.buildBroadcastFrontendStateFromSnapshot(snapshot)+ got := m.buildBroadcastFrontendStateFromSnapshotWithClock(snapshot, func() time.Time { return now })wantJSON, err := json.Marshal(want)if err != nil {t.Fatal(err)diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 162d189642..c722fc70f0 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -4333,6 +4333,13 @@ func buildFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateF}func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend {+ return m.buildBroadcastFrontendStateFromSnapshotWithClock(snapshot, time.Now)+}++// The clock is sampled at the same post-read health-evaluation boundary as+// ordinary broadcasts. Tests can compare complete projections at one instant+// without mistaking a naturally advancing health-age label for lost content.+func (m *Monitor) buildBroadcastFrontendStateFromSnapshotWithClock(snapshot models.StateSnapshot, healthClock func() time.Time) models.StateFrontend {frontendState := buildFrontendStateFromSnapshot(snapshot)m.updateResourceStoreForRead(snapshot)if m != nil && m.alertManager != nil {@@ -4349,7 +4356,7 @@ func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateS// Coalescing owns the outer slice. Decorate that one projection in place,// not three full-resource copies; nested store data is still read-only.healthAlerts := resourceHealthAlerts(frontendState.ActiveAlerts)- now := time.Now().UTC()+ now := healthClock().UTC()for i := range broadcastResources {m.applyPersistedMetadataToUnifiedResource(&broadcastResources[i])health := unifiedresources.EvaluateResourceHealth(broadcastResources[i], healthAlerts, now)