Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep restarts from re-recording every resource as discovered
rcourtman8b6c93d3e5a9
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex d4db08a560..6d92dd0171 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -2081,6 +2081,13 @@ of an unchanged snapshot must not append duplicate change records.cover the publish and blocked-persistence boundaries. Synthetic performanceimprovement alone does not establish field CPU relief.+The adapter also keeps restart continuity: before its first diff it reads+each resource's last recorded state, and a known resource's first appearance+after a Pulse restart or update records only a real difference from that+state, never a fresh discovery (see the unified-resources "Bounded generation+change comparison" clause).+`TestMonitorAdapterRestartDoesNotRediscoverKnownResources` covers it.+### One registry snapshot per store refresh pass — issue #2199Each store refresh pass (`updateResourceStore`, its read-path twin, and sodiff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex ab3af050b9..10afaba743 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -775,6 +775,29 @@ compares the old and new paths, whileboundary. The 1,000-resource benchmark is component evidence, not installedfleet CPU attribution or a release acceptance claim.+A restart is not a discovery. The first diff after Pulse starts compares+against an empty generation, so before it the monitor adapter reads each+resource's last recorded state (`LastResourceStates`, an optional store read:+the to-state of its newest state transition). A known resource's first+appearance in the run then records nothing when that state still holds, a+transition from it marked "resource state changed while Pulse was stopped"+(no `occurredAt`) when it differs, and an ordinary discovery when it was last+recorded as removed. After that appearance its changes record normally, and a+resource first seen while Pulse was down records its discovery. The+while-stopped transition keeps availability reports' boundary for a resource+that recovered or failed during the gap, and carries `changedFields: status`+so the agent event stream announces it like any status change. The newest+state is chosen on parsed observation times (stored text mixes formats), with+the greater change ID winning a tie in both stores. Before this, each restart or update+re-recorded the whole fleet as new (1,776 rows on the mock fleet; now only real+differences), burying the night's changes in `list_recent_changes`. Removals+during the gap are still not recorded, and a resource whose rows have all aged+out of retention, or whose canonical ID changed across the restart (for example+an identity succession on upgrade), is discovered once more.+`TestMonitorAdapterRestartDoesNotRediscoverKnownResources` covers memory and+SQLite stores; `TestLastResourceStatesReadsNewestTransitionAcrossStores` pins+the read.+### Node-indexed guest parent resolution during ingest — issue #2199Guest ingest resolves each Proxmox guest's parent node, and that lookup candiff --git a/internal/unifiedresources/change_emission.go b/internal/unifiedresources/change_emission.goindex 8996a74572..836fcbe74f 100644--- a/internal/unifiedresources/change_emission.go+++ b/internal/unifiedresources/change_emission.go@@ -15,6 +15,13 @@ import (// returns the ones the store accepted, so the caller can announce exactly what// was recorded.func recordRegistryChanges(store ResourceStore, before, after []Resource, observedAt time.Time, occurredAt *time.Time, sourceType ChangeSourceType, sourceAdapterHint ChangeSourceAdapter) []ResourceChange {+ return recordRegistryChangesSkipping(store, before, after, observedAt, occurredAt, sourceType, sourceAdapterHint, nil)+}++// recordRegistryChangesSkipping is recordRegistryChanges with a filter: a+// change skip reports true for is neither persisted nor returned, and skip may+// rewrite a change it keeps. A nil skip keeps every change unchanged.+func recordRegistryChangesSkipping(store ResourceStore, before, after []Resource, observedAt time.Time, occurredAt *time.Time, sourceType ChangeSourceType, sourceAdapterHint ChangeSourceAdapter, skip func(*ResourceChange) bool) []ResourceChange {if store == nil {return nil}@@ -48,7 +55,7 @@ func recordRegistryChanges(store ResourceStore, before, after []Resource, observbeforeResource, beforeOK := beforeByID[id]afterResource, afterOK := afterByID[id]change := buildResourceChange(beforeResource, beforeOK, afterResource, afterOK, observedAt, occurredAt, sourceType, sourceAdapterHint)- if change == nil {+ if change == nil || (skip != nil && skip(change)) {continue}if err := store.RecordChange(*change); err != nil {@@ -68,6 +75,14 @@ func recordRegistryChanges(store ResourceStore, before, after []Resource, observ// returns (MonitorAdapter's mutationMu does so during a replacement). It// returns the changes the store accepted.func recordRegistryChangesBetweenGenerations(before, after *ResourceRegistry, observedAt time.Time, occurredAt *time.Time, sourceType ChangeSourceType, sourceAdapterHint ChangeSourceAdapter) []ResourceChange {+ return recordRegistryChangesBetweenGenerationsSkipping(before, after, observedAt, occurredAt, sourceType, sourceAdapterHint, nil)+}++// recordRegistryChangesBetweenGenerationsSkipping is+// recordRegistryChangesBetweenGenerations with the same skip filter as+// recordRegistryChangesSkipping. skip runs after both registry locks are+// released.+func recordRegistryChangesBetweenGenerationsSkipping(before, after *ResourceRegistry, observedAt time.Time, occurredAt *time.Time, sourceType ChangeSourceType, sourceAdapterHint ChangeSourceAdapter, skip func(*ResourceChange) bool) []ResourceChange {if before == nil || after == nil || before.store == nil {return nil}@@ -110,6 +125,9 @@ func recordRegistryChangesBetweenGenerations(before, after *ResourceRegistry, obrecorded := pending[:0]for _, change := range pending {+ if skip != nil && skip(&change) {+ continue+ }if err := before.store.RecordChange(change); err != nil {log.Printf("unifiedresources: failed to record change for %s: %v", change.ResourceID, err)continuediff --git a/internal/unifiedresources/history_identity_test.go b/internal/unifiedresources/history_identity_test.goindex f3d22ac01e..1990091b71 100644--- a/internal/unifiedresources/history_identity_test.go+++ b/internal/unifiedresources/history_identity_test.go@@ -1070,3 +1070,106 @@ func TestMonitorAdapterChangeObserverHearsOnlyRecordedLiveChanges(t *testing.T)require.NoError(t, adapter.RecordLiveChange(ResourceChange{ID: "unobserved", ResourceID: "lab:pve1:101", Kind: ChangeAlertResolved, SourceType: SourceHeuristic, ObservedAt: now}))require.Len(t, heard, 1)}++// A Pulse restart builds its first registry from nothing, so without the+// history baseline every resource it already knew was recorded as+// "resource discovered" again, burying a night's real changes under one+// entry per resource whenever Pulse restarted or updated. The first+// appearance is compared with the last recorded state instead.+func TestMonitorAdapterRestartDoesNotRediscoverKnownResources(t *testing.T) {+ for _, tc := range []struct {+ name string+ store func(t *testing.T) ResourceStore+ }{+ {name: "memory", store: func(*testing.T) ResourceStore { return NewMemoryStore() }},+ {name: "sqlite", store: func(t *testing.T) ResourceStore {+ store, err := NewSQLiteResourceStore(t.TempDir(), "default")+ if err != nil {+ t.Fatalf("NewSQLiteResourceStore: %v", err)+ }+ t.Cleanup(func() { _ = store.Close() })+ return store+ }},+ } {+ t.Run(tc.name, func(t *testing.T) {+ store := tc.store(t)+ at := time.Date(2026, 10, 29, 2, 0, 0, 0, time.UTC)+ ids := map[string]string{}+ populate := func(adapter *MonitorAdapter, vms ...models.VM) {+ t.Helper()+ at = at.Add(time.Minute)+ for i := range vms {+ vms[i].LastSeen = at+ }+ adapter.PopulateFromSnapshot(models.StateSnapshot{VMs: vms, LastUpdate: at})+ for _, resource := range adapter.currentRegistry().List() {+ ids[resource.Name] = resource.ID+ }+ }+ vm := func(id int, name, status string) models.VM {+ return models.VM{ID: fmt.Sprintf("lab:node-a:%d", id), VMID: id, Name: name, Node: "node-a", Instance: "lab", Status: status}+ }+ history := func(name string) []ResourceChange {+ t.Helper()+ changes, err := store.GetRecentChanges(ids[name], time.Time{}, 100)+ if err != nil {+ t.Fatalf("GetRecentChanges(%s): %v", name, err)+ }+ var transitions []ResourceChange+ for _, change := range changes {+ if change.Kind == ChangeStateTransition {+ transitions = append(transitions, change)+ }+ }+ return transitions+ }++ first := NewMonitorAdapter(NewRegistry(store))+ populate(first, vm(101, "database", "running"), vm(102, "archive", "running"), vm(103, "cache", "running"))+ populate(first, vm(101, "database", "running"), vm(103, "cache", "stopped"))+ if got := history("archive"); len(got) != 2 {+ t.Fatalf("archive history before restart = %+v, want discovery and removal", got)+ }+ cacheBefore := history("cache")+ if len(cacheBefore) != 2 || cacheBefore[0].From == "absent" {+ t.Fatalf("cache history before restart = %+v, want its stop newest", cacheBefore)+ }+ stopped, running := cacheBefore[0].To, cacheBefore[0].From++ // Restart: database is unchanged, backup appeared while Pulse was+ // down, archive was recorded as removed and is back, and cache+ // started again while Pulse was down.+ restarted := NewMonitorAdapter(NewRegistry(store))+ populate(restarted, vm(101, "database", "running"), vm(102, "archive", "running"), vm(103, "cache", "running"), vm(104, "backup", "running"))++ if got := history("database"); len(got) != 1 {+ t.Fatalf("database history after restart = %+v, want only its first discovery", got)+ }+ if got := history("backup"); len(got) != 1 || got[0].From != "absent" {+ t.Fatalf("backup history = %+v, want one discovery", got)+ }+ if got := history("archive"); len(got) != 3 || got[0].From != "absent" {+ t.Fatalf("archive history = %+v, want its return after removal recorded", got)+ }+ cache := history("cache")+ if len(cache) != 3 {+ t.Fatalf("cache history = %+v, want the start while Pulse was stopped recorded", cache)+ }+ if got := cache[0]; got.From != stopped || got.To != running || got.OccurredAt != nil || got.Reason != "resource state changed while Pulse was stopped" {+ t.Fatalf("cache change across restart = %+v, want %s -> %s marked as while stopped", got, stopped, running)+ }+ // It is a status change, so the live agent event stream hears it.+ if fields := fmt.Sprint(cache[0].Metadata["changedFields"]); !strings.Contains(fields, "status") {+ t.Fatalf("cache change across restart changedFields = %s, want status", fields)+ }++ // Once seen in this run, a resource that leaves and returns is a+ // real change and records as discovered again.+ populate(restarted, vm(102, "archive", "running"), vm(103, "cache", "running"), vm(104, "backup", "running"))+ populate(restarted, vm(101, "database", "running"), vm(102, "archive", "running"), vm(103, "cache", "running"), vm(104, "backup", "running"))+ if got := history("database"); len(got) != 3 || got[0].From != "absent" {+ t.Fatalf("database history after leaving and returning = %+v, want discovery, removal and discovery", got)+ }+ })+ }+}diff --git a/internal/unifiedresources/monitor_adapter.go b/internal/unifiedresources/monitor_adapter.goindex 03903492b9..3e5854799f 100644--- a/internal/unifiedresources/monitor_adapter.go+++ b/internal/unifiedresources/monitor_adapter.go@@ -4,6 +4,7 @@ import ("crypto/sha256""encoding/json""fmt"+ "log""sort""strings""sync"@@ -37,6 +38,14 @@ type MonitorAdapter struct {// changeObserver hears each live change after the store accepted it.// Guarded by mu.changeObserver ResourceChangeObserver++ // restartBaseline maps each resource the change history recorded before+ // this process's first diff to its last recorded state. A resource's first+ // appearance in this run is compared with that state rather than with+ // nothing, so restarting or updating Pulse does not record the whole fleet+ // as new. Guarded by mutationMu.+ restartBaseline map[string]string+ restartBaselineLoaded bool}// ResourceChangeObserver is told about a change the adapter just recorded,@@ -603,7 +612,7 @@ func (a *MonitorAdapter) replaceRegistryLocked(snapshot models.StateSnapshot, reocc := snapshot.LastUpdateoccurredAt = &occ}- recorded := recordRegistryChangesBetweenGenerations(registry, rebuilt, rebuiltAt, occurredAt, SourcePulseDiff, "")+ recorded := recordRegistryChangesBetweenGenerationsSkipping(registry, rebuilt, rebuiltAt, occurredAt, SourcePulseDiff, "", a.restartRediscoveryFilterLocked(registry.store))rebuilt.PersistIdentityPins()a.mu.Lock()@@ -618,6 +627,55 @@ func (a *MonitorAdapter) replaceRegistryLocked(snapshot models.StateSnapshot, rea.notifyChanges(recorded)}+// restartRediscoveryFilterLocked returns the skip filter for a registry diff.+// A resource's first appearance in this run is a "resource discovered" change+// built against an empty generation; for a resource the history already knew+// it becomes:+// - nothing, when its last recorded state matches (or none was recorded);+// - a transition from its last recorded state, when that state changed while+// Pulse was stopped;+// - the discovery unchanged, when it was last recorded as removed.+//+// The resource then leaves the baseline, so later changes record normally.+// The baseline is read from store on the first diff. Callers must hold+// mutationMu.+func (a *MonitorAdapter) restartRediscoveryFilterLocked(store ResourceStore) func(*ResourceChange) bool {+ if !a.restartBaselineLoaded {+ a.restartBaselineLoaded = true+ if reader, ok := store.(lastResourceStateReader); ok {+ states, err := reader.LastResourceStates()+ if err != nil {+ log.Printf("unifiedresources: cannot read change history before the first diff; this run may record known resources as discovered: %v", err)+ }+ if len(states) > 0 {+ a.restartBaseline = states+ }+ }+ }+ if len(a.restartBaseline) == 0 {+ return nil+ }+ return func(change *ResourceChange) bool {+ id := CanonicalResourceID(change.ResourceID)+ last, known := a.restartBaseline[id]+ if !known {+ return false+ }+ delete(a.restartBaseline, id)+ if change.Kind != ChangeStateTransition || change.From != "absent" || last == "absent" {+ return false+ }+ if last == "" || last == change.To {+ return true+ }+ change.From = last+ change.OccurredAt = nil+ change.Reason = "resource state changed while Pulse was stopped"+ change.Metadata = map[string]any{"changeType": "state_changed_while_stopped", "changedFields": []string{"status"}}+ return false+ }+}+// ShouldSkipAPIPolling returns true when agent coverage indicates API// polling for the hostname should be skipped entirely.func (a *MonitorAdapter) ShouldSkipAPIPolling(hostname string) bool {@@ -822,7 +880,7 @@ func (a *MonitorAdapter) PopulateSupplementalRecords(source DataSource, recordsbefore := registry.List()registry.IngestRecordsWithStaleThresholds(source, records, thresholds)rebuiltAt := time.Now().UTC()- recorded := recordRegistryChanges(registry.store, before, registry.List(), rebuiltAt, nil, SourcePlatformEvent, changeSourceAdapterForDataSource(source))+ recorded := recordRegistryChangesSkipping(registry.store, before, registry.List(), rebuiltAt, nil, SourcePlatformEvent, changeSourceAdapterForDataSource(source), a.restartRediscoveryFilterLocked(registry.store))registry.PersistIdentityPins()a.mu.Lock()a.lastRebuiltAt = rebuiltAtdiff --git a/internal/unifiedresources/store.go b/internal/unifiedresources/store.goindex d1160754d3..6154978d61 100644--- a/internal/unifiedresources/store.go+++ b/internal/unifiedresources/store.go@@ -2162,6 +2162,84 @@ func parseResourceChangeTimeString(value string) (time.Time, error) {return time.Time{}, fmt.Errorf("unsupported time value %q", value)}+// lastResourceStateReader is the optional store read behind restart+// continuity. It maps every canonical resource ID the change history holds a+// record for to the to-state of its newest state transition ("" when it has+// none). Stores without it make every resource look new after a restart.+type lastResourceStateReader interface {+ LastResourceStates() (map[string]string, error)+}++// LastResourceStates maps each canonical resource ID with at least one+// recorded change to the to-state of its newest state transition, or "" when+// it has no state transition. Newest is judged on parsed observation times,+// because stored timestamps mix text formats; equal times prefer the greater+// change ID.+func (s *SQLiteResourceStore) LastResourceStates() (map[string]string, error) {+ s.mu.Lock()+ defer s.mu.Unlock()++ states := make(map[string]string)+ idRows, err := s.db.Query(`SELECT DISTINCT canonical_id FROM resource_changes`)+ if err != nil {+ return nil, fmt.Errorf("read changed resources: %w", err)+ }+ for idRows.Next() {+ var id string+ if err := idRows.Scan(&id); err != nil {+ idRows.Close()+ return nil, fmt.Errorf("scan changed resource: %w", err)+ }+ states[id] = ""+ }+ if err := idRows.Err(); err != nil {+ idRows.Close()+ return nil, fmt.Errorf("read changed resources: %w", err)+ }+ idRows.Close()++ rows, err := s.db.Query(`SELECT canonical_id, COALESCE(to_state, ''), `+s.resourceChangesObservedAtExpr()+`, id FROM resource_changes WHERE kind = ?`, string(ChangeStateTransition))+ if err != nil {+ return nil, fmt.Errorf("read last resource states: %w", err)+ }+ defer rows.Close()+ newest := make(map[string]resourceStateMark, len(states))+ for rows.Next() {+ var canonicalID, state, changeID string+ var observedRaw any+ if err := rows.Scan(&canonicalID, &state, &observedRaw, &changeID); err != nil {+ return nil, fmt.Errorf("scan last resource state: %w", err)+ }+ observedAt, err := parseResourceChangeTime(observedRaw)+ if err != nil {+ return nil, fmt.Errorf("parse resource change time: %w", err)+ }+ mark := resourceStateMark{observedAt: observedAt, changeID: changeID}+ if current, ok := newest[canonicalID]; ok && !mark.newerThan(current) {+ continue+ }+ newest[canonicalID] = mark+ states[canonicalID] = state+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("read last resource states: %w", err)+ }+ return states, nil+}++// resourceStateMark orders state transitions for LastResourceStates.+type resourceStateMark struct {+ observedAt time.Time+ changeID string+}++func (m resourceStateMark) newerThan(other resourceStateMark) bool {+ if !m.observedAt.Equal(other.observedAt) {+ return m.observedAt.After(other.observedAt)+ }+ return m.changeID > other.changeID+}+func (s *SQLiteResourceStore) CountRecentChanges(canonicalID string, since time.Time) (int, error) {return s.CountRecentChangesFiltered(canonicalID, since, ResourceChangeFilters{})}@@ -3900,6 +3978,36 @@ func (m *MemoryStore) recordChangeLocked(change ResourceChange) error {return nil}+// LastResourceStates maps each canonical resource ID with at least one+// recorded change to the to-state of its newest state transition, or "" when+// it has no state transition. Equal times prefer the greater change ID, as the+// SQLite store does.+func (m *MemoryStore) LastResourceStates() (map[string]string, error) {+ m.mu.RLock()+ defer m.mu.RUnlock()+ states := make(map[string]string, len(m.changes))+ newest := make(map[string]resourceStateMark, len(m.changes))+ for _, change := range m.changes {+ id := CanonicalResourceID(change.ResourceID)+ if id == "" {+ continue+ }+ if _, known := states[id]; !known {+ states[id] = ""+ }+ if change.Kind != ChangeStateTransition {+ continue+ }+ mark := resourceStateMark{observedAt: change.ObservedAt, changeID: change.ID}+ if current, ok := newest[id]; ok && !mark.newerThan(current) {+ continue+ }+ newest[id] = mark+ states[id] = change.To+ }+ return states, nil+}+func (m *MemoryStore) GetRecentChanges(canonicalID string, since time.Time, limit int) ([]ResourceChange, error) {return m.GetRecentChangesFiltered(canonicalID, since, limit, ResourceChangeFilters{})}diff --git a/internal/unifiedresources/store_test.go b/internal/unifiedresources/store_test.goindex f4e16b35e4..877590800f 100644--- a/internal/unifiedresources/store_test.go+++ b/internal/unifiedresources/store_test.go@@ -4554,3 +4554,57 @@ func assertSharedPageCacheBound(t *testing.T, name string, conn interface {t.Fatalf("%s: cache_size = %d, want %d from pdb.PageCachePragma", name, size, -pdb.MaxPageCacheKiB)}}++func TestLastResourceStatesReadsNewestTransitionAcrossStores(t *testing.T) {+ sqlite, err := NewSQLiteResourceStore(t.TempDir(), "default")+ if err != nil {+ t.Fatalf("NewSQLiteResourceStore: %v", err)+ }+ t.Cleanup(func() { _ = sqlite.Close() })+ at := time.Date(2026, 10, 29, 3, 0, 0, 0, time.UTC)+ changes := []ResourceChange{+ {ResourceID: "vm-a", ObservedAt: at, Kind: ChangeStateTransition, From: "absent", To: "online"},+ {ResourceID: "vm-a", ObservedAt: at.Add(2 * time.Minute), Kind: ChangeStateTransition, From: "online", To: "offline"},+ {ResourceID: "vm-a", ObservedAt: at.Add(3 * time.Minute), Kind: ChangeAlertFired},+ {ResourceID: "vm-b", ObservedAt: at, Kind: ChangeAlertFired},+ {ResourceID: "vm-c", ObservedAt: at, Kind: ChangeStateTransition, From: "online", To: "absent"},+ // Equal times: the greater change ID wins in both stores.+ {ID: "z", ResourceID: "vm-d", ObservedAt: at, Kind: ChangeStateTransition, From: "absent", To: "online"},+ {ID: "a", ResourceID: "vm-d", ObservedAt: at, Kind: ChangeStateTransition, From: "online", To: "absent"},+ }+ for name, store := range map[string]ResourceStore{"sqlite": sqlite, "memory": NewMemoryStore()} {+ for i, change := range changes {+ if change.ID == "" {+ change.ID = fmt.Sprintf("%s-%d", name, i)+ } else {+ change.ID = name + "-" + change.ID+ }+ if err := store.RecordChange(change); err != nil {+ t.Fatalf("%s RecordChange: %v", name, err)+ }+ }+ states, err := store.(lastResourceStateReader).LastResourceStates()+ if err != nil {+ t.Fatalf("%s LastResourceStates: %v", name, err)+ }+ want := map[string]string{"vm-a": "offline", "vm-b": "", "vm-c": "absent", "vm-d": "online"}+ if !reflect.DeepEqual(states, want) {+ t.Fatalf("%s last resource states = %v, want %v", name, states, want)+ }+ }++ // Stored timestamps mix text formats; newest is the later instant, not+ // the greater string.+ if _, err := sqlite.db.Exec(`INSERT INTO resource_changes (id, canonical_id, observed_at, kind, from_state, to_state, source_type, confidence) VALUES (?, ?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?, ?)`,+ "legacy-old", "vm-e", "2026-10-29T03:00:00Z", string(ChangeStateTransition), "online", "offline", string(SourcePulseDiff), string(ConfidenceHigh),+ "current-new", "vm-e", "2026-10-29 04:00:00 +0000 UTC", string(ChangeStateTransition), "offline", "online", string(SourcePulseDiff), string(ConfidenceHigh)); err != nil {+ t.Fatalf("insert mixed-format rows: %v", err)+ }+ states, err := sqlite.LastResourceStates()+ if err != nil {+ t.Fatalf("LastResourceStates: %v", err)+ }+ if states["vm-e"] != "online" {+ t.Fatalf("vm-e last state = %q, want online from the later instant", states["vm-e"])+ }+}