Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Compare complete broadcast health at one evaluation instant

pulse-triage[bot]9cc8523881fb
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index fd23f74cc1..0e47f9e058 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -41,6 +41,12 @@ the exact parent's complete conversion and final JSON encoding, with cold,
list-only, typed-clean and dirty inputs; they do not establish installed CPU/RSS
relief, native identity refresh throughput or completion of #2199.
+Complete-content comparisons use one health-evaluation instant through a
+private clock seam. Production still samples real UTC at the same post-read
+boundary. The minute-boundary control requires age labels to advance without
+renewing LastSeen; comparisons retain every health field instead of stripping
+the reproduced `1m`/`2m` clock difference from JSON.
+
### Report identity lookups do not rebuild unrelated Kubernetes metadata
The producer's two temporary host/Docker-host admission registries use a
diff --git a/internal/monitoring/broadcast_metrics_snapshot_test.go b/internal/monitoring/broadcast_metrics_snapshot_test.go
index d238907642..02e18e99cc 100644
--- a/internal/monitoring/broadcast_metrics_snapshot_test.go
+++ b/internal/monitoring/broadcast_metrics_snapshot_test.go
@@ -79,17 +79,18 @@ func TestBroadcastMetricsSnapshotConnectedContent(t *testing.T) {
}
assertEqual := func(stage string) {
t.Helper()
+ evaluationTime := time.Now().UTC()
before, err := json.Marshal(adapter.GetAll())
if err != nil {
t.Fatal(err)
}
- want := legacyFrontendProjectionForTest(m, graph.State, adapter.GetAll(), adapter, time.Now().UTC())
+ want := legacyFrontendProjectionForTest(m, graph.State, adapter.GetAll(), adapter, evaluationTime)
// The parent's unified view fills a missing store watermark from the
// listed resources. Preserve that full-state header in the oracle too.
if freshness := latestUnifiedResourceLastSeen(adapter.GetAll()); !freshness.IsZero() {
want.LastUpdate = freshness.UnixMilli()
}
- got := m.buildBroadcastFrontendStateFromSnapshot(graph.State)
+ got := m.buildBroadcastFrontendStateFromSnapshotWithClock(graph.State, func() time.Time { return evaluationTime })
wantJSON, err := json.Marshal(want)
if err != nil {
t.Fatal(err)
@@ -125,6 +126,39 @@ func TestBroadcastMetricsSnapshotConnectedContent(t *testing.T) {
assertEqual("dirty-same-time")
}
+func TestBroadcastMetricsSnapshotHealthClockBoundary(t *testing.T) {
+ m, adapter, id := newReadStateCloneTestMonitor(t, 1)
+ rows := adapter.GetAll()
+ original := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
+ rows[0].LastSeen = original
+ rows[0].SourceStatus = map[unifiedresources.DataSource]unifiedresources.SourceStatus{
+ unifiedresources.SourceProxmox: {Status: "stale", LastSeen: original},
+ }
+ rr := unifiedresources.NewRegistry(nil)
+ rr.IngestResources(rows)
+ m.resourceStore = &broadcastProjectionCountingStore{MonitorAdapter: unifiedresources.NewMonitorAdapter(rr)}
+ for _, tc := range []struct {
+ age time.Duration
+ detail string
+ }{{59 * time.Second, "0m"}, {61 * time.Second, "1m"}} {
+ calls := 0
+ got := m.buildBroadcastFrontendStateFromSnapshotWithClock(models.EmptyStateSnapshot(), func() time.Time {
+ calls++
+ return original.Add(tc.age)
+ })
+ if calls != 1 || len(got.Resources) != 1 || got.Resources[0].ID != id || got.Resources[0].LastSeen != original.UnixMilli() {
+ t.Fatal("health clock changed capture identity/original time or was not sampled once")
+ }
+ var health unifiedresources.ResourceHealth
+ if err := json.Unmarshal(got.Resources[0].Health, &health); err != nil {
+ t.Fatal(err)
+ }
+ if health.Verdict != unifiedresources.HealthStale || len(health.Reasons) != 1 || health.Reasons[0].Code != "telemetry_stale" || health.Reasons[0].Detail != tc.detail {
+ t.Fatalf("health age must advance across the minute boundary: %+v", health)
+ }
+ }
+}
+
func TestBroadcastMetricsSnapshotOwnsTargetsBeforeLiveReplacement(t *testing.T) {
m, adapter, _ := newReadStateCloneTestMonitor(t, 4)
view := m.currentUnifiedStateView()
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 9e11cb4f7e..f65c96057a 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -2958,7 +2958,7 @@ func TestBroadcastProjectionMatchesPreviousPipeline(t *testing.T) {
if !view.freshness.IsZero() {
want.LastUpdate = view.freshness.UnixMilli()
}
- got := m.buildBroadcastFrontendStateFromSnapshot(snapshot)
+ got := m.buildBroadcastFrontendStateFromSnapshotWithClock(snapshot, func() time.Time { return now })
wantJSON, err := json.Marshal(want)
if err != nil {
t.Fatal(err)
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 162d189642..c722fc70f0 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -4333,6 +4333,13 @@ func buildFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateF
}
func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend {
+ return m.buildBroadcastFrontendStateFromSnapshotWithClock(snapshot, time.Now)
+}
+
+// The clock is sampled at the same post-read health-evaluation boundary as
+// ordinary broadcasts. Tests can compare complete projections at one instant
+// without mistaking a naturally advancing health-age label for lost content.
+func (m *Monitor) buildBroadcastFrontendStateFromSnapshotWithClock(snapshot models.StateSnapshot, healthClock func() time.Time) models.StateFrontend {
frontendState := buildFrontendStateFromSnapshot(snapshot)
m.updateResourceStoreForRead(snapshot)
if m != nil && m.alertManager != nil {
@@ -4349,7 +4356,7 @@ func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateS
// Coalescing owns the outer slice. Decorate that one projection in place,
// not three full-resource copies; nested store data is still read-only.
healthAlerts := resourceHealthAlerts(frontendState.ActiveAlerts)
- now := time.Now().UTC()
+ now := healthClock().UTC()
for i := range broadcastResources {
m.applyPersistedMetadataToUnifiedResource(&broadcastResources[i])
health := unifiedresources.EvaluateResourceHealth(broadcastResources[i], healthAlerts, now)