Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Read backup and job snapshots without cloning the estate
pulse-triage[bot]72fc83c7ed64
diff --git a/internal/models/metadata_snapshot.go b/internal/models/metadata_snapshot.gonew file mode 100644index 0000000000..1c26213141--- /dev/null+++ b/internal/models/metadata_snapshot.go@@ -0,0 +1,51 @@+package models++// These projections serve backup/job/connection consumers that do not need a+// resource inventory. Clone only their primary fields under the same state+// lock as GetSnapshot; do not read the separately cached Backups aggregate.+// Keep the complete snapshot's normalization and caller-owned collections.+func (s *State) GetBackups() Backups {+ s.mu.RLock()+ defer s.mu.RUnlock()+ return Backups{+ PVE: clonePVEBackups(s.PVEBackups),+ PBS: clonePBSBackups(s.PBSBackups),+ PMG: clonePMGBackups(s.PMGBackups),+ }.NormalizeCollections()+}++func (s *State) GetPVEBackups() PVEBackups {+ s.mu.RLock()+ defer s.mu.RUnlock()+ return clonePVEBackups(s.PVEBackups)+}++func (s *State) GetPBSBackups() []PBSBackup {+ s.mu.RLock()+ defer s.mu.RUnlock()+ backups := clonePBSBackups(s.PBSBackups)+ if backups == nil {+ backups = []PBSBackup{}+ }+ return backups+}++func (s *State) GetReplicationJobs() []ReplicationJob {+ s.mu.RLock()+ defer s.mu.RUnlock()+ jobs := cloneReplicationJobs(s.ReplicationJobs)+ if jobs == nil {+ jobs = []ReplicationJob{}+ }+ return jobs+}++func (s *State) GetConnectionHealth() map[string]bool {+ s.mu.RLock()+ defer s.mu.RUnlock()+ health := make(map[string]bool, len(s.ConnectionHealth))+ for key, healthy := range s.ConnectionHealth {+ health[key] = healthy+ }+ return health+}diff --git a/internal/models/metadata_snapshot_test.go b/internal/models/metadata_snapshot_test.gonew file mode 100644index 0000000000..c7e99cd124--- /dev/null+++ b/internal/models/metadata_snapshot_test.go@@ -0,0 +1,139 @@+package models++import (+ "bytes"+ "encoding/json"+ "fmt"+ "reflect"+ "sync"+ "testing"+ "time"+)++func metadataStateFixture() *State {+ s := NewState()+ now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)+ rate := 64.0+ s.PVEBackups = PVEBackups{+ BackupTasks: []BackupTask{{ID: "task", Instance: "pve", VMID: 100, Status: "OK"}},+ StorageBackups: []StorageBackup{{ID: "archive", Instance: "pve", VMID: "100"}},+ GuestSnapshots: []GuestSnapshot{{ID: "snapshot", Instance: "pve", VMID: 100}},+ }+ s.PBSBackups = []PBSBackup{{ID: "pbs", Instance: "pbs", Namespace: "team", VMID: "100", BackupTime: now,+ Protected: true, InProgress: true, WriteActivityObserved: true, WriteActive: true,+ Files: []string{"index.json.blob"}, VerificationRaw: map[string]interface{}{"state": "ok"}, Owner: "fixture@pbs"}}+ s.PMGBackups = []PMGBackup{{ID: "pmg", Instance: "pmg", BackupTime: now}}+ s.ReplicationJobs = []ReplicationJob{{ID: "replication", LastSyncTime: &now, NextSyncTime: &now, RateLimitMbps: &rate}}+ s.ConnectionHealth = map[string]bool{"pve": true, "pbs": false}+ // The complete snapshot derives backups from the primary collections, not+ // this cache. A scoped optimization must do exactly the same.+ s.Backups = Backups{PBS: []PBSBackup{{ID: "stale-aggregate"}}}+ return s+}++func assertMetadataSnapshotParity(t *testing.T, s *State) {+ t.Helper()+ want := s.GetSnapshot()+ for _, pair := range []struct {+ name string+ got, want any+ }{+ {"backups", s.GetBackups(), want.Backups},+ {"pve", s.GetPVEBackups(), want.PVEBackups},+ {"pbs", s.GetPBSBackups(), want.PBSBackups},+ {"replication", s.GetReplicationJobs(), want.ReplicationJobs},+ {"health", s.GetConnectionHealth(), want.ConnectionHealth},+ } {+ if !reflect.DeepEqual(pair.got, pair.want) {+ t.Fatalf("%s projection differs from complete snapshot", pair.name)+ }+ gotJSON, err := json.Marshal(pair.got)+ if err != nil {+ t.Fatal(err)+ }+ wantJSON, err := json.Marshal(pair.want)+ if err != nil || !bytes.Equal(gotJSON, wantJSON) {+ t.Fatalf("%s serialization changed", pair.name)+ }+ }+}++func TestMetadataSnapshotsParityAndOwnership(t *testing.T) {+ for _, state := range []*State{{}, NewState(), metadataStateFixture()} {+ assertMetadataSnapshotParity(t, state)+ }+ s := metadataStateFixture()+ before, err := json.Marshal(s.GetSnapshot())+ if err != nil {+ t.Fatal(err)+ }+ all := s.GetBackups()+ all.PVE.BackupTasks[0].Status = "changed"+ all.PVE.StorageBackups[0].ID = "changed"+ all.PVE.GuestSnapshots[0].ID = "changed"+ all.PBS[0].Files[0] = "changed"+ all.PBS[0].VerificationRaw.(map[string]interface{})["state"] = "changed"+ all.PMG[0].ID = "changed"+ pve := s.GetPVEBackups()+ pve.BackupTasks[0].ID = "changed"+ pve.StorageBackups[0].ID = "changed"+ pve.GuestSnapshots[0].ID = "changed"+ pbs := s.GetPBSBackups()+ pbs[0].ID = "changed"+ pbs[0].Files[0] = "changed"+ pbs[0].VerificationRaw.(map[string]interface{})["state"] = "changed"+ jobs := s.GetReplicationJobs()+ jobs[0].ID = "changed"+ *jobs[0].LastSyncTime = time.Time{}+ *jobs[0].NextSyncTime = time.Time{}+ *jobs[0].RateLimitMbps = 0+ health := s.GetConnectionHealth()+ health["pve"] = false+ health["foreign"] = true+ after, err := json.Marshal(s.GetSnapshot())+ if err != nil || !bytes.Equal(before, after) {+ t.Fatal("scoped projection shares mutable source fields")+ }+ assertMetadataSnapshotParity(t, s)+}++func TestMetadataSnapshotsConcurrentTransactions(t *testing.T) {+ s := metadataStateFixture()+ var workers sync.WaitGroup+ workers.Add(2)+ go func() {+ defer workers.Done()+ for i := 0; i < 100; i++ {+ id := fmt.Sprint(i)+ s.mu.Lock()+ s.PVEBackups.BackupTasks[0].Instance = id+ s.PBSBackups[0].Instance = id+ s.PMGBackups[0].Instance = id+ s.PBSBackups[0].Files[0] = id+ s.PBSBackups[0].VerificationRaw.(map[string]interface{})["state"] = id+ s.ReplicationJobs[0].ID = id+ s.ConnectionHealth["pve"] = i%2 == 0+ s.mu.Unlock()+ }+ }()+ go func() {+ defer workers.Done()+ for i := 0; i < 100; i++ {+ all := s.GetBackups()+ // The initial fixture has different source instances; every writer+ // transaction after it uses one generation across all three sources.+ id := all.PVE.BackupTasks[0].Instance+ if id != "pve" && (all.PBS[0].Instance != id || all.PMG[0].Instance != id) {+ t.Error("aggregate projection mixed writer generations")+ return+ }+ all.PBS[0].Files[0] = "reader"+ s.GetPVEBackups().BackupTasks[0].ID = "reader"+ s.GetPBSBackups()[0].Files[0] = "reader"+ s.GetReplicationJobs()[0].ID = "reader"+ s.GetConnectionHealth()["pve"] = false+ }+ }()+ workers.Wait()+ assertMetadataSnapshotParity(t, s)+}diff --git a/internal/monitoring/metadata_snapshot_test.go b/internal/monitoring/metadata_snapshot_test.gonew file mode 100644index 0000000000..ab70733ac9--- /dev/null+++ b/internal/monitoring/metadata_snapshot_test.go@@ -0,0 +1,151 @@+package monitoring++import (+ "encoding/json"+ "fmt"+ "reflect"+ "sync"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/mock"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+)++func metadataMonitorFixture(size int) *Monitor {+ state := models.NewState()+ stamp := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)+ rate := 64.0+ state.PVEBackups = models.PVEBackups{BackupTasks: []models.BackupTask{{ID: "task", VMID: 100, Status: "OK"}}}+ state.PBSBackups = []models.PBSBackup{{ID: "pbs", Files: []string{"index.json.blob"}, Protected: true, BackupTime: stamp}}+ state.PMGBackups = []models.PMGBackup{{ID: "pmg", BackupTime: stamp}}+ state.ReplicationJobs = []models.ReplicationJob{{ID: "job", LastSyncTime: &stamp, NextSyncTime: &stamp, RateLimitMbps: &rate}}+ state.ConnectionHealth = map[string]bool{"pve": true, "pbs": false}+ cluster := models.KubernetesCluster{ID: "cluster"}+ for i := 0; i < size; i++ {+ id := fmt.Sprint(i)+ state.VMs = append(state.VMs, models.VM{ID: id, Tags: []string{"production"}, Disks: []models.Disk{{Mountpoint: "/", Total: 100}}})+ cluster.ConfigMaps = append(cluster.ConfigMaps, models.KubernetesConfigMap{UID: id, DataKeys: []string{"key"}, Labels: map[string]string{"app": "test"}})+ }+ if size > 0 {+ state.KubernetesClusters = []models.KubernetesCluster{cluster}+ }+ return &Monitor{state: state}+}++var metadataReaders = []struct {+ name string+ current func(*Monitor) any+ previous func(*Monitor) any+}{+ {"backups", func(m *Monitor) any { return m.BackupsSnapshot() }, func(m *Monitor) any { return m.GetState().Backups }},+ {"pve", func(m *Monitor) any { return m.PVEBackupsSnapshot() }, func(m *Monitor) any { return m.GetState().PVEBackups }},+ {"pbs", func(m *Monitor) any { return m.PBSBackupsSnapshot() }, func(m *Monitor) any { return m.GetState().PBSBackups }},+ {"replication", func(m *Monitor) any { return m.ReplicationJobsSnapshot() }, func(m *Monitor) any { return m.GetState().ReplicationJobs }},+ {"health", func(m *Monitor) any { return m.ConnectionHealthSnapshot() }, func(m *Monitor) any { return m.GetState().ConnectionHealth }},+}++func TestMetadataMonitorProjectionParity(t *testing.T) {+ previous := mock.IsMockEnabled()+ defer mustSetMockEnabled(t, previous)+ for _, enabled := range []bool{false, true} {+ mustSetMockEnabled(t, enabled)+ for _, monitor := range []*Monitor{nil, {}, {state: models.NewState()}, metadataMonitorFixture(100)} {+ for _, reader := range metadataReaders {+ got, want := reader.current(monitor), reader.previous(monitor)+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("%s differs in mock=%t (nil=%t)", reader.name, enabled, monitor == nil)+ }+ gotJSON, err := json.Marshal(got)+ if err != nil {+ t.Fatal(err)+ }+ wantJSON, err := json.Marshal(want)+ if err != nil || string(gotJSON) != string(wantJSON) {+ t.Fatalf("%s changed serialized output", reader.name)+ }+ }+ }+ }+}++func TestMetadataMonitorProjectionConcurrentReset(t *testing.T) {+ previous := mock.IsMockEnabled()+ mustSetMockEnabled(t, false)+ defer mustSetMockEnabled(t, previous)+ m := metadataMonitorFixture(0)+ retained := m.BackupsSnapshot()+ var workers sync.WaitGroup+ workers.Add(2)+ go func() {+ defer workers.Done()+ for i := 0; i < 100; i++ {+ m.mu.Lock()+ m.resetStateLocked()+ m.mu.Unlock()+ }+ }()+ go func() {+ defer workers.Done()+ for i := 0; i < 100; i++ {+ for _, reader := range metadataReaders {+ _ = reader.current(m)+ }+ }+ }()+ workers.Wait()+ if len(retained.PBS) != 1 || retained.PBS[0].ID != "pbs" {+ t.Fatal("reset invalidated caller-owned backup data")+ }+ for _, reader := range metadataReaders {+ if !reflect.DeepEqual(reader.current(m), reader.previous(m)) {+ t.Fatalf("%s changed reset semantics", reader.name)+ }+ }+}++func TestMetadataMonitorProjectionAllocationScaling(t *testing.T) {+ previous := mock.IsMockEnabled()+ mustSetMockEnabled(t, false)+ defer mustSetMockEnabled(t, previous)+ small, large := metadataMonitorFixture(0), metadataMonitorFixture(1000)+ for _, reader := range metadataReaders {+ baseline := testing.AllocsPerRun(10, func() { metadataBenchmarkResult = reader.current(small) })+ grown := testing.AllocsPerRun(10, func() { metadataBenchmarkResult = reader.current(large) })+ if grown != baseline {+ t.Errorf("%s unrelated estate increased allocations: small=%.0f large=%.0f", reader.name, baseline, grown)+ }+ old := testing.AllocsPerRun(10, func() { metadataBenchmarkResult = reader.previous(large) })+ if grown >= old {+ t.Errorf("%s scoped=%.0f did not reduce full-estate=%.0f allocations", reader.name, grown, old)+ }+ }+}++var metadataBenchmarkResult any++// Fixed backup/job/health payloads; only unrelated guests and Kubernetes+// metadata grow. Compare the actual old/current monitor accessors, not a+// substitute micro-kernel, database path or reporter's running process.+func BenchmarkMetadataMonitorProjection(b *testing.B) {+ previous := mock.IsMockEnabled()+ mustSetMockEnabled(b, false)+ b.Cleanup(func() { mustSetMockEnabled(b, previous) })+ for _, size := range []int{0, 100, 1000} {+ m := metadataMonitorFixture(size)+ for _, reader := range metadataReaders {+ for _, method := range []struct {+ name string+ read func(*Monitor) any+ }{{"full", reader.previous}, {"scoped", reader.current}} {+ b.Run(fmt.Sprintf("%s/%d/%s", reader.name, size, method.name), func(b *testing.B) {+ b.ReportAllocs()+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ metadataBenchmarkResult = method.read(m)+ }+ })+ }+ }+ }+}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 902e1d4db1..2305afeb4f 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -3529,7 +3529,27 @@ func (m *Monitor) ReadSnapshot() models.StateSnapshot {// BackupsSnapshot returns the current backup state.func (m *Monitor) BackupsSnapshot() models.Backups {- return m.GetState().Backups+ return readMetadataSnapshot(m, (*models.State).GetBackups, func(s models.StateSnapshot) models.Backups { return s.Backups })+}++// Backup/job/connection metadata is not a unified resource inventory. Capture+// the replaceable state pointer once, then clone only the requested fields.+// Mock mode keeps GetState's fixture and lazy alert-maintenance behaviour;+// absent monitors/states keep the previous zero-value projections.+func readMetadataSnapshot[T any](m *Monitor, read func(*models.State) T, project func(models.StateSnapshot) T) T {+ if m != nil {+ if mock.IsMockEnabled() {+ return project(m.GetState())+ }+ m.mu.RLock()+ state := m.state+ m.mu.RUnlock()+ if state != nil {+ return read(state)+ }+ }+ var empty T+ return empty}// PBSInstancesSnapshot returns the current PBS instances.@@ -3559,12 +3579,12 @@ func (m *Monitor) PBSInstancesSnapshot() []models.PBSInstance {// ReplicationJobsSnapshot returns the current replication jobs.func (m *Monitor) ReplicationJobsSnapshot() []models.ReplicationJob {- return m.GetState().ReplicationJobs+ return readMetadataSnapshot(m, (*models.State).GetReplicationJobs, func(s models.StateSnapshot) []models.ReplicationJob { return s.ReplicationJobs })}// ConnectionHealthSnapshot returns the current connection health map.func (m *Monitor) ConnectionHealthSnapshot() map[string]bool {- return m.GetState().ConnectionHealth+ return readMetadataSnapshot(m, (*models.State).GetConnectionHealth, func(s models.StateSnapshot) map[string]bool { return s.ConnectionHealth })}// HostsSnapshot returns the current hosts.@@ -4577,12 +4597,12 @@ func (m *Monitor) RecentlyResolvedSnapshot() []models.ResolvedAlert {// PVEBackupsSnapshot returns the current PVE backups.func (m *Monitor) PVEBackupsSnapshot() models.PVEBackups {- return m.GetState().PVEBackups+ return readMetadataSnapshot(m, (*models.State).GetPVEBackups, func(s models.StateSnapshot) models.PVEBackups { return s.PVEBackups })}// PBSBackupsSnapshot returns the current PBS recovery artifacts.func (m *Monitor) PBSBackupsSnapshot() []models.PBSBackup {- return m.GetState().PBSBackups+ return readMetadataSnapshot(m, (*models.State).GetPBSBackups, func(s models.StateSnapshot) []models.PBSBackup { return s.PBSBackups })}// BuildFrontendState returns the current state converted to frontend format.