Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep PBS protection truthful through incomplete polls
pulse-triage[bot]75ddede1b2cf
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex d5475857eb..919fdf5008 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -1558,8 +1558,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clienretainedGroups := make(map[pbsBackupGroupKey]struct{}, len(existingGroups))datastoreCount := len(datastores) // Number of datastores to querydatastoreFetches := 0 // Number of successful datastore fetches- datastoreErrors := 0 // Number of failed datastore fetches+ datastoreErrors := 0 // Datastores with incomplete historydatastoreTerminalFailures := 0 // Number of datastores that failed only with terminal errors+ collectionTerminalFailures := 0 // Datastores with any rejected namespace or snapshot request// Process each datastorefor _, ds := range datastores {@@ -1621,6 +1622,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clienStr("namespace", namespace).Int("limit", pbsBackupLiveStateLimit).Msg("PBS backup live-state limit reached; skipping remaining groups")+ // Skipped groups are not a complete inventory, even when+ // every request we did make succeeded.+ datastoreNamespaceErrors++break}@@ -1688,26 +1692,27 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, cliengroupsRequested += len(requests)fetched := m.fetchPBSBackupSnapshots(ctx, client, instanceName, requests)- if len(fetched) > 0 {- allBackups = appendPBSBackupsWithinLimit(allBackups, fetched)- }+ allBackups = appendPBSBackupsWithinLimit(allBackups, fetched.backups)+ datastoreNamespaceErrors += fetched.errors+ datastoreTerminalNamespaceErrors += fetched.terminalErrors- // Record fetch time for each requested group so the TTL tracks freshness.- // We record for all requested groups — on fetch failure, fetchPBSBackupSnapshots- // falls back to cached data, so the timestamp prevents hammering a failing- // endpoint. The TTL ensures we retry within a bounded window.+ // Only a successful snapshot read renews cache freshness. Retrying a+ // lost read on the next ordinary poll is preferable to treating its+ // fallback row as a newly successful collection during the TTL.fetchedAt := time.Now()- for _, req := range requests {- reqKey := pbsBackupGroupKey{- datastore: req.datastore,- namespace: req.namespace,- backupType: req.group.BackupType,- backupID: req.group.BackupID,- }- m.setPBSBackupCacheTime(instanceName, reqKey, fetchedAt)+ for _, key := range fetched.refreshed {+ m.setPBSBackupCacheTime(instanceName, key, fetchedAt)}}+ // Listing groups is not proof that their snapshots or every namespace+ // were readable. Carry those losses into provider-wide evidence.+ if datastoreNamespaceErrors > 0 {+ datastoreErrors+++ }+ if datastoreTerminalNamespaceErrors > 0 {+ collectionTerminalFailures+++ }if datastoreHadSuccess {datastoreFetches++log.Info().@@ -1741,10 +1746,14 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clienretainedGroups[key] = struct{}{}}}- datastoreErrors++}}+ if ctx.Err() != nil {+ // An interrupted enumeration cannot authoritatively replace history.+ return+ }+log.Info().Str("instance", instanceName).Int("count", len(allBackups)).@@ -1757,7 +1766,7 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, cliendatastoreCount,datastoreFetches,datastoreErrors,- datastoreTerminalFailures,+ collectionTerminalFailures,protectionObservedAt,)if protectionObservationErr != nil {@@ -1835,15 +1844,20 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clienif protectionObservationErr == nil {observations = append(observations, protectionObservation)}- m.ingestAndReconcileRecoveryPointsWithObservationsAsync(- points,- observations,- recoveryReconcileScope{+ batch := recoveryIngestBatch{points: points, observations: observations}+ // Only an authoritative complete enumeration can delete absent points.+ // Unreadable history remains historical evidence, qualified by this poll's+ // partial/unavailable provider observation, until an ordinary complete poll+ // recovers or confirms deletion. Never infer absence from a failed request.+ if protectionObservationErr == nil &&+ protectionObservation.HistoryCompleteness == recovery.ProtectionHistoryComplete {+ batch.reconcile = &recoveryReconcileScope{provider: string(recovery.ProviderProxmoxPBS),idPrefix: "pbs-backup:",instance: instanceName,- },- )+ }+ }+ m.enqueueRecoveryIngest(batch)// Sync backup times to VMs/Containers and republish them to canonical resources.m.syncGuestBackupTimesAndResourceStore()@@ -2012,9 +2026,18 @@ func namespacePathsForDatastore(ds models.PBSDatastore) []string {return paths}-func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Client, instanceName string, requests []pbsBackupFetchRequest) []models.PBSBackup {+// pbsBackupSnapshotFetchResult retains request quality alongside the artifacts.+// A cached row is not evidence that its snapshot request succeeded this poll.+type pbsBackupSnapshotFetchResult struct {+ backups []models.PBSBackup+ errors int+ terminalErrors int+ refreshed []pbsBackupGroupKey+}++func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Client, instanceName string, requests []pbsBackupFetchRequest) pbsBackupSnapshotFetchResult {if len(requests) == 0 {- return nil+ return pbsBackupSnapshotFetchResult{}}workerCount := pbsBackupSnapshotFetchWorkers@@ -2023,7 +2046,7 @@ func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Clien}jobs := make(chan pbsBackupFetchRequest)- results := make(chan []models.PBSBackup, workerCount)+ results := make(chan pbsBackupSnapshotFetchResult, workerCount)var wg sync.WaitGroupfor i := 0; i < workerCount; i++ {@@ -2051,18 +2074,26 @@ func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.ClienStr("id", req.group.BackupID).Msg("Failed to list PBS backup snapshots")- if len(req.cached.snapshots) > 0 {- select {- case results <- req.cached.snapshots:- case <-ctx.Done():- }+ result := pbsBackupSnapshotFetchResult{errors: 1}+ if shouldReuseCachedPBSBackups(err) {+ result.backups = req.cached.snapshots+ } else {+ // Access rejection cannot renew a cached raw row.+ result.terminalErrors = 1}+ results <- resultcontinue}backups := convertPBSSnapshots(instanceName, req.datastore, req.namespace, snapshots)select {- case results <- backups:+ case results <- pbsBackupSnapshotFetchResult{+ backups: backups,+ refreshed: []pbsBackupGroupKey{{+ datastore: req.datastore, namespace: req.namespace,+ backupType: req.group.BackupType, backupID: req.group.BackupID,+ }},+ }:case <-ctx.Done():return}@@ -2086,12 +2117,12 @@ func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Clienclose(results)}()- var combined []models.PBSBackup- for backups := range results {- if len(backups) == 0 {- continue- }- combined = appendPBSBackupsWithinLimit(combined, backups)+ var combined pbsBackupSnapshotFetchResult+ for result := range results {+ combined.backups = appendPBSBackupsWithinLimit(combined.backups, result.backups)+ combined.errors += result.errors+ combined.terminalErrors += result.terminalErrors+ combined.refreshed = append(combined.refreshed, result.refreshed...)}return combineddiff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.goindex bd2b881d65..d910e1bb77 100644--- a/internal/monitoring/monitor_backups_readstate_test.go+++ b/internal/monitoring/monitor_backups_readstate_test.go@@ -280,7 +280,11 @@ func TestFetchPBSBackupSnapshotsUsesBoundedWorkerPool(t *testing.T) {}m := &Monitor{}- backups := m.fetchPBSBackupSnapshots(context.Background(), client, "pbs1", requests)+ result := m.fetchPBSBackupSnapshots(context.Background(), client, "pbs1", requests)+ if result.errors != 0 || result.terminalErrors != 0 {+ t.Fatalf("successful snapshot fetch lost quality: %+v", result)+ }+ backups := result.backupsif len(backups) != requestCount {t.Fatalf("expected %d fetched backups, got %d", requestCount, len(backups))}diff --git a/internal/monitoring/pbs_poll_posture_test.go b/internal/monitoring/pbs_poll_posture_test.gonew file mode 100644index 0000000000..ad6dda9f99--- /dev/null+++ b/internal/monitoring/pbs_poll_posture_test.go@@ -0,0 +1,318 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ proxmoxmapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"+ recoverystore "github.com/rcourtman/pulse-go-rewrite/internal/recovery/store"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"+)++type pbsPosturePollFixture struct {+ m *Monitor+ store *recoverystore.Store+ client *pbs.Client+ mode atomic.Int32+ backupAt int64+ ids []string+}++func newPBSPosturePollFixture(t *testing.T, count int) *pbsPosturePollFixture {+ t.Helper()+ m, manager := recoveryIngestTestMonitor(t)+ m.state = models.NewState()+ f := &pbsPosturePollFixture{m: m, backupAt: time.Now().Add(-14 * time.Hour).Truncate(time.Second).Unix()}+ var vms []models.VM+ for i := 0; i < count; i++ {+ id := 100 + i+ vms = append(vms, models.VM{ID: makeGuestID("cluster", fmt.Sprintf("node-%d", i%4), id), VMID: id, Instance: "cluster", Node: fmt.Sprintf("node-%d", i%4), Name: fmt.Sprintf("guest-%d", id), Status: "running"})+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", id))+ }+ m.state.UpdateVMs(vms)+ var err error+ f.store, err = manager.StoreForOrg("default")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { f.wait(t); _ = f.store.Close() })+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ mode := f.mode.Load()+ if strings.HasSuffix(r.URL.Path, "/groups") {+ if mode == 4 {+ http.Error(w, "unavailable", http.StatusServiceUnavailable)+ return+ }+ if mode == 5 {+ http.Error(w, "denied", http.StatusForbidden)+ return+ }+ if mode == 6 && r.URL.Query().Get("ns") == "unreadable" {+ http.Error(w, "denied", http.StatusForbidden)+ return+ }+ var groups []map[string]any+ if mode != 1 {+ for i := 0; i < count; i++ {+ groups = append(groups, map[string]any{"backup-type": "vm", "backup-id": strconv.Itoa(100 + i), "last-backup": f.backupAt, "backup-count": 1})+ }+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": groups})+ } else if strings.HasSuffix(r.URL.Path, "/snapshots") {+ id := r.URL.Query().Get("backup-id")+ if id == "100" && (mode == 2 || mode == 3) {+ status := http.StatusServiceUnavailable+ if mode == 3 {+ status = http.StatusForbidden+ }+ http.Error(w, "unreadable", status)+ return+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"backup-type": "vm", "backup-id": id, "backup-time": f.backupAt, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}})+ } else {+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(server.Close)+ f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})+ if err != nil {+ t.Fatal(err)+ }+ return f+}++func (f *pbsPosturePollFixture) wait(t *testing.T) {+ t.Helper()+ deadline := time.Now().Add(10 * time.Second)+ for {+ f.m.recoveryIngestMu.Lock()+ running := f.m.recoveryIngestRunning+ f.m.recoveryIngestMu.Unlock()+ if !running {+ return+ }+ if time.Now().After(deadline) {+ t.Fatal("ordinary PBS ingest did not finish")+ }+ time.Sleep(time.Millisecond)+ }+}++func (f *pbsPosturePollFixture) poll(t *testing.T, instance string, mode int32, datastores []models.PBSDatastore) {+ t.Helper()+ f.mode.Store(mode)+ // Deliberately expire snapshot reuse: group-count/time are unchanged when+ // only snapshot access or verification changes between ordinary polls.+ f.m.mu.Lock()+ clear(f.m.pbsBackupCacheTime)+ f.m.mu.Unlock()+ f.m.pollPBSBackups(context.Background(), instance, f.client, datastores)+ f.wait(t)+}++func (f *pbsPosturePollFixture) points(t *testing.T, instance string) []recovery.RecoveryPoint {+ t.Helper()+ points, _, err := f.store.ListPoints(context.Background(), recovery.ListPointsOptions{Limit: 1000})+ if err != nil {+ t.Fatal(err)+ }+ var out []recovery.RecoveryPoint+ for _, p := range points {+ if p.RepositoryRef != nil && p.RepositoryRef.Namespace == instance {+ out = append(out, p)+ }+ }+ return out+}++func (f *pbsPosturePollFixture) postures(t *testing.T) []recovery.ProtectionPosture {+ t.Helper()+ postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{SubjectResourceIDs: f.ids})+ if err != nil || total != len(f.ids) {+ t.Fatalf("postures total=%d err=%v", total, err)+ }+ return postures+}++func (f *pbsPosturePollFixture) assertProtected(t *testing.T, instance string) {+ t.Helper()+ var raw []models.PBSBackup+ for _, b := range f.m.state.GetSnapshot().PBSBackups {+ if b.Instance == instance {+ raw = append(raw, b)+ }+ }+ mapped, err := proxmoxmapper.FromPBSBackupsWithEvidence(raw, buildPBSGuestCandidates(f.m.GetUnifiedReadStateOrSnapshot()), time.Now())+ if err != nil || len(mapped) != len(f.ids) {+ t.Fatalf("raw/mapped=%d/%d err=%v", len(raw), len(mapped), err)+ }+ for _, p := range mapped {+ vmid, _ := strconv.Atoi(p.Details["vmid"].(string))+ expected := unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", vmid)+ if p.SubjectResourceID != expected || p.Outcome != recovery.OutcomeSuccess {+ t.Fatalf("mapped subject=%s outcome=%s, want %s success", p.SubjectResourceID, p.Outcome, expected)+ }+ }+ persisted := f.points(t, instance)+ if len(persisted) != len(mapped) {+ t.Fatalf("persisted=%d mapped=%d", len(persisted), len(mapped))+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected || p.Verification != recovery.ProtectionVerificationVerified || p.LastSuccessfulPointAt == nil || p.LastSuccessfulPointAt.Unix() != f.backupAt {+ t.Fatalf("canonical posture=%+v", p)+ }+ }+ t.Logf("raw=%d mapped=%d persisted=%d protected=%d; original backup timestamp retained", len(raw), len(mapped), len(persisted), len(f.ids))+}++func TestPBSOrdinaryPollRootNamespacePostureLifecycle(t *testing.T) {+ f := newPBSPosturePollFixture(t, 40)+ ds := []models.PBSDatastore{{Name: "archive"}}+ f.poll(t, "pbs-main", 0, ds)+ f.assertProtected(t, "pbs-main")+ f.poll(t, "pbs-main", 1, ds)+ if len(f.m.state.GetSnapshot().PBSBackups) != 0 || len(f.points(t, "pbs-main")) != 0 {+ t.Fatal("successful empty poll retained deleted backups")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateUnknown {+ t.Fatalf("empty posture=%+v, want unknown without a subject-linked source", p)+ }+ }+ f.poll(t, "pbs-main", 0, ds)+ f.assertProtected(t, "pbs-main")+ f.poll(t, "pbs-main", 4, ds)+ if len(f.points(t, "pbs-main")) != 40 || len(f.m.state.GetSnapshot().PBSBackups) != 40 {+ t.Fatal("transient group failure lost historical backups")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 1 || p.ProviderStates[0].JobState != recovery.OutcomeFailed || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryUnavailable {+ t.Fatalf("failed posture=%+v", p)+ }+ }+ f.poll(t, "pbs-main", 0, ds)+ f.assertProtected(t, "pbs-main")+ f.poll(t, "pbs-main", 5, ds)+ if len(f.m.state.GetSnapshot().PBSBackups) != 0 || len(f.points(t, "pbs-main")) != 40 {+ t.Fatal("access denial must withdraw raw rows without deleting historical recovery facts")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 1 || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsDenied {+ t.Fatalf("denied posture=%+v", p)+ }+ }+ f.poll(t, "pbs-main", 0, ds)+ f.assertProtected(t, "pbs-main")+ // One connection's authoritative empty poll must not reconcile another.+ f.poll(t, "pbs-other", 0, ds)+ f.poll(t, "pbs-main", 1, ds)+ if len(f.points(t, "pbs-main")) != 0 || len(f.points(t, "pbs-other")) != 40 {+ t.Fatal("empty enumeration crossed the PBS instance boundary")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Fatalf("other readable source lost protection: %+v", p)+ }+ }+}++func TestPBSOrdinaryPollSnapshotFailuresDoNotClaimCompleteHistory(t *testing.T) {+ for _, tc := range []struct {+ name string+ mode int32+ permissions operationaltrust.EvidencePermissions+ state recovery.ProtectionState+ }{+ {"transient", 2, operationaltrust.EvidencePermissionsUnknown, recovery.ProtectionStateUnknown},+ {"denied", 3, operationaltrust.EvidencePermissionsPartial, recovery.ProtectionStateAttention},+ } {+ t.Run(tc.name, func(t *testing.T) {+ f := newPBSPosturePollFixture(t, 2)+ ds := []models.PBSDatastore{{Name: "archive"}}+ f.poll(t, "pbs-main", 0, ds)+ f.assertProtected(t, "pbs-main")+ f.poll(t, "pbs-main", tc.mode, ds)+ for attempt := 0; attempt < 2; attempt++ {+ points := f.points(t, "pbs-main")+ if len(points) != 2 {+ t.Errorf("incomplete snapshot enumeration reconciled away historical points: %d", len(points))+ }+ wantRaw := 2+ if tc.mode == 3 {+ wantRaw = 1+ }+ if got := len(f.m.state.GetSnapshot().PBSBackups); got != wantRaw {+ t.Errorf("raw snapshot rows=%d, want %d", got, wantRaw)+ }+ for _, p := range f.postures(t) {+ if p.State != tc.state || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != tc.permissions || p.ProviderStates[0].JobState != recovery.OutcomeWarning {+ t.Errorf("snapshot failure posture=%+v, want %s/partial/%s/warning", p, tc.state, tc.permissions)+ }+ }+ // No TTL manipulation on the next poll: a failed refresh must not+ // become a healthy cached read while snapshot access is still lost.+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds)+ f.wait(t)+ }+ f.poll(t, "pbs-main", 0, ds)+ f.assertProtected(t, "pbs-main")+ })+ }+}++func TestPBSOrdinaryPollPartialNamespaceRetainsUnseenHistory(t *testing.T) {+ f := newPBSPosturePollFixture(t, 2)+ ds := []models.PBSDatastore{{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}, {Path: "unreadable"}}}}+ f.poll(t, "pbs-main", 0, ds)+ if len(f.points(t, "pbs-main")) != 4 {+ t.Fatal("fixture did not enumerate both namespaces")+ }+ f.poll(t, "pbs-main", 6, ds)+ if len(f.points(t, "pbs-main")) != 4 {+ t.Error("partial namespace poll deleted unseen historical points")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateAttention || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsPartial {+ t.Errorf("partial namespace posture=%+v", p)+ }+ }+ f.poll(t, "pbs-main", 0, ds)+ if len(f.points(t, "pbs-main")) != 4 {+ t.Fatal("recovery did not restore both namespaces")+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Errorf("recovered posture=%+v", p)+ }+ }+}++func TestPBSOrdinaryPollAmbiguousVMIDsStayUnlinked(t *testing.T) {+ f := newPBSPosturePollFixture(t, 1)+ vms := f.m.state.GetSnapshot().VMs+ vms = append(vms, models.VM{ID: makeGuestID("other-cluster", "node-0", 100), VMID: 100, Instance: "other-cluster", Node: "node-0", Name: "guest-100", Status: "running"})+ f.m.state.UpdateVMs(vms)+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "other-cluster", 100))+ f.poll(t, "pbs-main", 0, []models.PBSDatastore{{Name: "archive"}})+ points := f.points(t, "pbs-main")+ if len(points) != 1 || points[0].SubjectResourceID != "" {+ t.Fatalf("ambiguous root-namespace backup was linked: %+v", points)+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateUnknown {+ t.Fatalf("ambiguous VMID painted protected: %+v", p)+ }+ }+}diff --git a/internal/monitoring/recovery_ingest.go b/internal/monitoring/recovery_ingest.goindex 84533459f3..a11d581cda 100644--- a/internal/monitoring/recovery_ingest.go+++ b/internal/monitoring/recovery_ingest.go@@ -53,7 +53,13 @@ func (m *Monitor) coalescePendingRecoveryIngest(batch recoveryIngestBatch) bool}for i := len(m.recoveryIngestPending) - 1; i >= 0; i-- {pendingKey, pendingOK := m.recoveryIngestPending[i].coalesceKey()- if pendingOK && pendingKey == key {+ if !pendingOK {+ // An incomplete poll/event is an ordered fact, not a replaceable+ // full snapshot. Moving a later complete (possibly empty) inventory+ // before it could reinsert old points after authoritative deletion.+ return false+ }+ if pendingKey == key {m.recoveryIngestPending[i] = batchreturn true}diff --git a/internal/monitoring/recovery_ingest_test.go b/internal/monitoring/recovery_ingest_test.goindex df4cdd1d8c..2d4df96266 100644--- a/internal/monitoring/recovery_ingest_test.go+++ b/internal/monitoring/recovery_ingest_test.go@@ -323,3 +323,32 @@ func TestRecoveryIngestPendingKeepsDistinctScopesAndEventBatches(t *testing.T) {t.Fatalf("pending batches = %d, want two source enumerations and one event batch", got)}}++// Partial PBS batches must stay before the complete recovery that supersedes+// them: coalescing across a non-authoritative batch would resurrect deleted+// points after a later successful empty enumeration.+func TestRecoveryIngestCompletePollCannotOvertakePartialBatch(t *testing.T) {+ m, manager := recoveryIngestTestMonitor(t)+ store, err := manager.StoreForOrg("default")+ if err != nil {+ t.Fatal(err)+ }+ defer store.Close()+ m.recoveryIngestRunning = true+ scope := recoveryReconcileScope{provider: string(recovery.ProviderProxmoxPVE), idPrefix: "pve-backup:", instance: "pve1"}+ point := recoveryIngestTestPoint("pve-backup:old", "pve1", time.Now().UTC())+ m.enqueueRecoveryIngest(recoveryIngestBatch{points: []recovery.RecoveryPoint{point}, reconcile: &scope})+ m.enqueueRecoveryIngest(recoveryIngestBatch{points: []recovery.RecoveryPoint{point}})+ m.enqueueRecoveryIngest(recoveryIngestBatch{reconcile: &scope})+ // Repeated complete recovery may still coalesce on its own side of the barrier.+ m.enqueueRecoveryIngest(recoveryIngestBatch{reconcile: &scope})+ if got := len(m.recoveryIngestPending); got != 3 {+ t.Fatalf("pending=%d, want complete/partial/complete in source order", got)+ }+ for _, batch := range m.recoveryIngestPending {+ m.ingestRecoveryPointsBestEffort(context.Background(), batch)+ }+ if ids := listRecoveryPointIDs(t, manager); len(ids) != 0 {+ t.Fatalf("partial history resurrected after authoritative empty recovery: %v", ids)+ }+}