Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep PBS protection truthful through incomplete polls

pulse-triage[bot]75ddede1b2cf
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index d5475857eb..919fdf5008 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -1558,8 +1558,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
retainedGroups := make(map[pbsBackupGroupKey]struct{}, len(existingGroups))
datastoreCount := len(datastores) // Number of datastores to query
datastoreFetches := 0 // Number of successful datastore fetches
- datastoreErrors := 0 // Number of failed datastore fetches
+ datastoreErrors := 0 // Datastores with incomplete history
datastoreTerminalFailures := 0 // Number of datastores that failed only with terminal errors
+ collectionTerminalFailures := 0 // Datastores with any rejected namespace or snapshot request
// Process each datastore
for _, ds := range datastores {
@@ -1621,6 +1622,9 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
Str("namespace", namespace).
Int("limit", pbsBackupLiveStateLimit).
Msg("PBS backup live-state limit reached; skipping remaining groups")
+ // Skipped groups are not a complete inventory, even when
+ // every request we did make succeeded.
+ datastoreNamespaceErrors++
break
}
@@ -1688,26 +1692,27 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
groupsRequested += len(requests)
fetched := m.fetchPBSBackupSnapshots(ctx, client, instanceName, requests)
- if len(fetched) > 0 {
- allBackups = appendPBSBackupsWithinLimit(allBackups, fetched)
- }
+ allBackups = appendPBSBackupsWithinLimit(allBackups, fetched.backups)
+ datastoreNamespaceErrors += fetched.errors
+ datastoreTerminalNamespaceErrors += fetched.terminalErrors
- // Record fetch time for each requested group so the TTL tracks freshness.
- // We record for all requested groups — on fetch failure, fetchPBSBackupSnapshots
- // falls back to cached data, so the timestamp prevents hammering a failing
- // endpoint. The TTL ensures we retry within a bounded window.
+ // Only a successful snapshot read renews cache freshness. Retrying a
+ // lost read on the next ordinary poll is preferable to treating its
+ // fallback row as a newly successful collection during the TTL.
fetchedAt := time.Now()
- for _, req := range requests {
- reqKey := pbsBackupGroupKey{
- datastore: req.datastore,
- namespace: req.namespace,
- backupType: req.group.BackupType,
- backupID: req.group.BackupID,
- }
- m.setPBSBackupCacheTime(instanceName, reqKey, fetchedAt)
+ for _, key := range fetched.refreshed {
+ m.setPBSBackupCacheTime(instanceName, key, fetchedAt)
}
}
+ // Listing groups is not proof that their snapshots or every namespace
+ // were readable. Carry those losses into provider-wide evidence.
+ if datastoreNamespaceErrors > 0 {
+ datastoreErrors++
+ }
+ if datastoreTerminalNamespaceErrors > 0 {
+ collectionTerminalFailures++
+ }
if datastoreHadSuccess {
datastoreFetches++
log.Info().
@@ -1741,10 +1746,14 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
retainedGroups[key] = struct{}{}
}
}
- datastoreErrors++
}
}
+ if ctx.Err() != nil {
+ // An interrupted enumeration cannot authoritatively replace history.
+ return
+ }
+
log.Info().
Str("instance", instanceName).
Int("count", len(allBackups)).
@@ -1757,7 +1766,7 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
datastoreCount,
datastoreFetches,
datastoreErrors,
- datastoreTerminalFailures,
+ collectionTerminalFailures,
protectionObservedAt,
)
if protectionObservationErr != nil {
@@ -1835,15 +1844,20 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
if protectionObservationErr == nil {
observations = append(observations, protectionObservation)
}
- m.ingestAndReconcileRecoveryPointsWithObservationsAsync(
- points,
- observations,
- recoveryReconcileScope{
+ batch := recoveryIngestBatch{points: points, observations: observations}
+ // Only an authoritative complete enumeration can delete absent points.
+ // Unreadable history remains historical evidence, qualified by this poll's
+ // partial/unavailable provider observation, until an ordinary complete poll
+ // recovers or confirms deletion. Never infer absence from a failed request.
+ if protectionObservationErr == nil &&
+ protectionObservation.HistoryCompleteness == recovery.ProtectionHistoryComplete {
+ batch.reconcile = &recoveryReconcileScope{
provider: string(recovery.ProviderProxmoxPBS),
idPrefix: "pbs-backup:",
instance: instanceName,
- },
- )
+ }
+ }
+ m.enqueueRecoveryIngest(batch)
// Sync backup times to VMs/Containers and republish them to canonical resources.
m.syncGuestBackupTimesAndResourceStore()
@@ -2012,9 +2026,18 @@ func namespacePathsForDatastore(ds models.PBSDatastore) []string {
return paths
}
-func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Client, instanceName string, requests []pbsBackupFetchRequest) []models.PBSBackup {
+// pbsBackupSnapshotFetchResult retains request quality alongside the artifacts.
+// A cached row is not evidence that its snapshot request succeeded this poll.
+type pbsBackupSnapshotFetchResult struct {
+ backups []models.PBSBackup
+ errors int
+ terminalErrors int
+ refreshed []pbsBackupGroupKey
+}
+
+func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Client, instanceName string, requests []pbsBackupFetchRequest) pbsBackupSnapshotFetchResult {
if len(requests) == 0 {
- return nil
+ return pbsBackupSnapshotFetchResult{}
}
workerCount := pbsBackupSnapshotFetchWorkers
@@ -2023,7 +2046,7 @@ func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Clien
}
jobs := make(chan pbsBackupFetchRequest)
- results := make(chan []models.PBSBackup, workerCount)
+ results := make(chan pbsBackupSnapshotFetchResult, workerCount)
var wg sync.WaitGroup
for i := 0; i < workerCount; i++ {
@@ -2051,18 +2074,26 @@ func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Clien
Str("id", req.group.BackupID).
Msg("Failed to list PBS backup snapshots")
- if len(req.cached.snapshots) > 0 {
- select {
- case results <- req.cached.snapshots:
- case <-ctx.Done():
- }
+ result := pbsBackupSnapshotFetchResult{errors: 1}
+ if shouldReuseCachedPBSBackups(err) {
+ result.backups = req.cached.snapshots
+ } else {
+ // Access rejection cannot renew a cached raw row.
+ result.terminalErrors = 1
}
+ results <- result
continue
}
backups := convertPBSSnapshots(instanceName, req.datastore, req.namespace, snapshots)
select {
- case results <- backups:
+ case results <- pbsBackupSnapshotFetchResult{
+ backups: backups,
+ refreshed: []pbsBackupGroupKey{{
+ datastore: req.datastore, namespace: req.namespace,
+ backupType: req.group.BackupType, backupID: req.group.BackupID,
+ }},
+ }:
case <-ctx.Done():
return
}
@@ -2086,12 +2117,12 @@ func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Clien
close(results)
}()
- var combined []models.PBSBackup
- for backups := range results {
- if len(backups) == 0 {
- continue
- }
- combined = appendPBSBackupsWithinLimit(combined, backups)
+ var combined pbsBackupSnapshotFetchResult
+ for result := range results {
+ combined.backups = appendPBSBackupsWithinLimit(combined.backups, result.backups)
+ combined.errors += result.errors
+ combined.terminalErrors += result.terminalErrors
+ combined.refreshed = append(combined.refreshed, result.refreshed...)
}
return combined
diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.go
index bd2b881d65..d910e1bb77 100644
--- a/internal/monitoring/monitor_backups_readstate_test.go
+++ b/internal/monitoring/monitor_backups_readstate_test.go
@@ -280,7 +280,11 @@ func TestFetchPBSBackupSnapshotsUsesBoundedWorkerPool(t *testing.T) {
}
m := &Monitor{}
- backups := m.fetchPBSBackupSnapshots(context.Background(), client, "pbs1", requests)
+ result := m.fetchPBSBackupSnapshots(context.Background(), client, "pbs1", requests)
+ if result.errors != 0 || result.terminalErrors != 0 {
+ t.Fatalf("successful snapshot fetch lost quality: %+v", result)
+ }
+ backups := result.backups
if len(backups) != requestCount {
t.Fatalf("expected %d fetched backups, got %d", requestCount, len(backups))
}
diff --git a/internal/monitoring/pbs_poll_posture_test.go b/internal/monitoring/pbs_poll_posture_test.go
new file mode 100644
index 0000000000..ad6dda9f99
--- /dev/null
+++ b/internal/monitoring/pbs_poll_posture_test.go
@@ -0,0 +1,318 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ proxmoxmapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"
+ recoverystore "github.com/rcourtman/pulse-go-rewrite/internal/recovery/store"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+)
+
+type pbsPosturePollFixture struct {
+ m *Monitor
+ store *recoverystore.Store
+ client *pbs.Client
+ mode atomic.Int32
+ backupAt int64
+ ids []string
+}
+
+func newPBSPosturePollFixture(t *testing.T, count int) *pbsPosturePollFixture {
+ t.Helper()
+ m, manager := recoveryIngestTestMonitor(t)
+ m.state = models.NewState()
+ f := &pbsPosturePollFixture{m: m, backupAt: time.Now().Add(-14 * time.Hour).Truncate(time.Second).Unix()}
+ var vms []models.VM
+ for i := 0; i < count; i++ {
+ id := 100 + i
+ vms = append(vms, models.VM{ID: makeGuestID("cluster", fmt.Sprintf("node-%d", i%4), id), VMID: id, Instance: "cluster", Node: fmt.Sprintf("node-%d", i%4), Name: fmt.Sprintf("guest-%d", id), Status: "running"})
+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", id))
+ }
+ m.state.UpdateVMs(vms)
+ var err error
+ f.store, err = manager.StoreForOrg("default")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { f.wait(t); _ = f.store.Close() })
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ mode := f.mode.Load()
+ if strings.HasSuffix(r.URL.Path, "/groups") {
+ if mode == 4 {
+ http.Error(w, "unavailable", http.StatusServiceUnavailable)
+ return
+ }
+ if mode == 5 {
+ http.Error(w, "denied", http.StatusForbidden)
+ return
+ }
+ if mode == 6 && r.URL.Query().Get("ns") == "unreadable" {
+ http.Error(w, "denied", http.StatusForbidden)
+ return
+ }
+ var groups []map[string]any
+ if mode != 1 {
+ for i := 0; i < count; i++ {
+ groups = append(groups, map[string]any{"backup-type": "vm", "backup-id": strconv.Itoa(100 + i), "last-backup": f.backupAt, "backup-count": 1})
+ }
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": groups})
+ } else if strings.HasSuffix(r.URL.Path, "/snapshots") {
+ id := r.URL.Query().Get("backup-id")
+ if id == "100" && (mode == 2 || mode == 3) {
+ status := http.StatusServiceUnavailable
+ if mode == 3 {
+ status = http.StatusForbidden
+ }
+ http.Error(w, "unreadable", status)
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"backup-type": "vm", "backup-id": id, "backup-time": f.backupAt, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}})
+ } else {
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(server.Close)
+ f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return f
+}
+
+func (f *pbsPosturePollFixture) wait(t *testing.T) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ f.m.recoveryIngestMu.Lock()
+ running := f.m.recoveryIngestRunning
+ f.m.recoveryIngestMu.Unlock()
+ if !running {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("ordinary PBS ingest did not finish")
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+func (f *pbsPosturePollFixture) poll(t *testing.T, instance string, mode int32, datastores []models.PBSDatastore) {
+ t.Helper()
+ f.mode.Store(mode)
+ // Deliberately expire snapshot reuse: group-count/time are unchanged when
+ // only snapshot access or verification changes between ordinary polls.
+ f.m.mu.Lock()
+ clear(f.m.pbsBackupCacheTime)
+ f.m.mu.Unlock()
+ f.m.pollPBSBackups(context.Background(), instance, f.client, datastores)
+ f.wait(t)
+}
+
+func (f *pbsPosturePollFixture) points(t *testing.T, instance string) []recovery.RecoveryPoint {
+ t.Helper()
+ points, _, err := f.store.ListPoints(context.Background(), recovery.ListPointsOptions{Limit: 1000})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var out []recovery.RecoveryPoint
+ for _, p := range points {
+ if p.RepositoryRef != nil && p.RepositoryRef.Namespace == instance {
+ out = append(out, p)
+ }
+ }
+ return out
+}
+
+func (f *pbsPosturePollFixture) postures(t *testing.T) []recovery.ProtectionPosture {
+ t.Helper()
+ postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{SubjectResourceIDs: f.ids})
+ if err != nil || total != len(f.ids) {
+ t.Fatalf("postures total=%d err=%v", total, err)
+ }
+ return postures
+}
+
+func (f *pbsPosturePollFixture) assertProtected(t *testing.T, instance string) {
+ t.Helper()
+ var raw []models.PBSBackup
+ for _, b := range f.m.state.GetSnapshot().PBSBackups {
+ if b.Instance == instance {
+ raw = append(raw, b)
+ }
+ }
+ mapped, err := proxmoxmapper.FromPBSBackupsWithEvidence(raw, buildPBSGuestCandidates(f.m.GetUnifiedReadStateOrSnapshot()), time.Now())
+ if err != nil || len(mapped) != len(f.ids) {
+ t.Fatalf("raw/mapped=%d/%d err=%v", len(raw), len(mapped), err)
+ }
+ for _, p := range mapped {
+ vmid, _ := strconv.Atoi(p.Details["vmid"].(string))
+ expected := unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", vmid)
+ if p.SubjectResourceID != expected || p.Outcome != recovery.OutcomeSuccess {
+ t.Fatalf("mapped subject=%s outcome=%s, want %s success", p.SubjectResourceID, p.Outcome, expected)
+ }
+ }
+ persisted := f.points(t, instance)
+ if len(persisted) != len(mapped) {
+ t.Fatalf("persisted=%d mapped=%d", len(persisted), len(mapped))
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected || p.Verification != recovery.ProtectionVerificationVerified || p.LastSuccessfulPointAt == nil || p.LastSuccessfulPointAt.Unix() != f.backupAt {
+ t.Fatalf("canonical posture=%+v", p)
+ }
+ }
+ t.Logf("raw=%d mapped=%d persisted=%d protected=%d; original backup timestamp retained", len(raw), len(mapped), len(persisted), len(f.ids))
+}
+
+func TestPBSOrdinaryPollRootNamespacePostureLifecycle(t *testing.T) {
+ f := newPBSPosturePollFixture(t, 40)
+ ds := []models.PBSDatastore{{Name: "archive"}}
+ f.poll(t, "pbs-main", 0, ds)
+ f.assertProtected(t, "pbs-main")
+ f.poll(t, "pbs-main", 1, ds)
+ if len(f.m.state.GetSnapshot().PBSBackups) != 0 || len(f.points(t, "pbs-main")) != 0 {
+ t.Fatal("successful empty poll retained deleted backups")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateUnknown {
+ t.Fatalf("empty posture=%+v, want unknown without a subject-linked source", p)
+ }
+ }
+ f.poll(t, "pbs-main", 0, ds)
+ f.assertProtected(t, "pbs-main")
+ f.poll(t, "pbs-main", 4, ds)
+ if len(f.points(t, "pbs-main")) != 40 || len(f.m.state.GetSnapshot().PBSBackups) != 40 {
+ t.Fatal("transient group failure lost historical backups")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 1 || p.ProviderStates[0].JobState != recovery.OutcomeFailed || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryUnavailable {
+ t.Fatalf("failed posture=%+v", p)
+ }
+ }
+ f.poll(t, "pbs-main", 0, ds)
+ f.assertProtected(t, "pbs-main")
+ f.poll(t, "pbs-main", 5, ds)
+ if len(f.m.state.GetSnapshot().PBSBackups) != 0 || len(f.points(t, "pbs-main")) != 40 {
+ t.Fatal("access denial must withdraw raw rows without deleting historical recovery facts")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 1 || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsDenied {
+ t.Fatalf("denied posture=%+v", p)
+ }
+ }
+ f.poll(t, "pbs-main", 0, ds)
+ f.assertProtected(t, "pbs-main")
+ // One connection's authoritative empty poll must not reconcile another.
+ f.poll(t, "pbs-other", 0, ds)
+ f.poll(t, "pbs-main", 1, ds)
+ if len(f.points(t, "pbs-main")) != 0 || len(f.points(t, "pbs-other")) != 40 {
+ t.Fatal("empty enumeration crossed the PBS instance boundary")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Fatalf("other readable source lost protection: %+v", p)
+ }
+ }
+}
+
+func TestPBSOrdinaryPollSnapshotFailuresDoNotClaimCompleteHistory(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ mode int32
+ permissions operationaltrust.EvidencePermissions
+ state recovery.ProtectionState
+ }{
+ {"transient", 2, operationaltrust.EvidencePermissionsUnknown, recovery.ProtectionStateUnknown},
+ {"denied", 3, operationaltrust.EvidencePermissionsPartial, recovery.ProtectionStateAttention},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newPBSPosturePollFixture(t, 2)
+ ds := []models.PBSDatastore{{Name: "archive"}}
+ f.poll(t, "pbs-main", 0, ds)
+ f.assertProtected(t, "pbs-main")
+ f.poll(t, "pbs-main", tc.mode, ds)
+ for attempt := 0; attempt < 2; attempt++ {
+ points := f.points(t, "pbs-main")
+ if len(points) != 2 {
+ t.Errorf("incomplete snapshot enumeration reconciled away historical points: %d", len(points))
+ }
+ wantRaw := 2
+ if tc.mode == 3 {
+ wantRaw = 1
+ }
+ if got := len(f.m.state.GetSnapshot().PBSBackups); got != wantRaw {
+ t.Errorf("raw snapshot rows=%d, want %d", got, wantRaw)
+ }
+ for _, p := range f.postures(t) {
+ if p.State != tc.state || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != tc.permissions || p.ProviderStates[0].JobState != recovery.OutcomeWarning {
+ t.Errorf("snapshot failure posture=%+v, want %s/partial/%s/warning", p, tc.state, tc.permissions)
+ }
+ }
+ // No TTL manipulation on the next poll: a failed refresh must not
+ // become a healthy cached read while snapshot access is still lost.
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds)
+ f.wait(t)
+ }
+ f.poll(t, "pbs-main", 0, ds)
+ f.assertProtected(t, "pbs-main")
+ })
+ }
+}
+
+func TestPBSOrdinaryPollPartialNamespaceRetainsUnseenHistory(t *testing.T) {
+ f := newPBSPosturePollFixture(t, 2)
+ ds := []models.PBSDatastore{{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}, {Path: "unreadable"}}}}
+ f.poll(t, "pbs-main", 0, ds)
+ if len(f.points(t, "pbs-main")) != 4 {
+ t.Fatal("fixture did not enumerate both namespaces")
+ }
+ f.poll(t, "pbs-main", 6, ds)
+ if len(f.points(t, "pbs-main")) != 4 {
+ t.Error("partial namespace poll deleted unseen historical points")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateAttention || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsPartial {
+ t.Errorf("partial namespace posture=%+v", p)
+ }
+ }
+ f.poll(t, "pbs-main", 0, ds)
+ if len(f.points(t, "pbs-main")) != 4 {
+ t.Fatal("recovery did not restore both namespaces")
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Errorf("recovered posture=%+v", p)
+ }
+ }
+}
+
+func TestPBSOrdinaryPollAmbiguousVMIDsStayUnlinked(t *testing.T) {
+ f := newPBSPosturePollFixture(t, 1)
+ vms := f.m.state.GetSnapshot().VMs
+ vms = append(vms, models.VM{ID: makeGuestID("other-cluster", "node-0", 100), VMID: 100, Instance: "other-cluster", Node: "node-0", Name: "guest-100", Status: "running"})
+ f.m.state.UpdateVMs(vms)
+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "other-cluster", 100))
+ f.poll(t, "pbs-main", 0, []models.PBSDatastore{{Name: "archive"}})
+ points := f.points(t, "pbs-main")
+ if len(points) != 1 || points[0].SubjectResourceID != "" {
+ t.Fatalf("ambiguous root-namespace backup was linked: %+v", points)
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateUnknown {
+ t.Fatalf("ambiguous VMID painted protected: %+v", p)
+ }
+ }
+}
diff --git a/internal/monitoring/recovery_ingest.go b/internal/monitoring/recovery_ingest.go
index 84533459f3..a11d581cda 100644
--- a/internal/monitoring/recovery_ingest.go
+++ b/internal/monitoring/recovery_ingest.go
@@ -53,7 +53,13 @@ func (m *Monitor) coalescePendingRecoveryIngest(batch recoveryIngestBatch) bool
}
for i := len(m.recoveryIngestPending) - 1; i >= 0; i-- {
pendingKey, pendingOK := m.recoveryIngestPending[i].coalesceKey()
- if pendingOK && pendingKey == key {
+ if !pendingOK {
+ // An incomplete poll/event is an ordered fact, not a replaceable
+ // full snapshot. Moving a later complete (possibly empty) inventory
+ // before it could reinsert old points after authoritative deletion.
+ return false
+ }
+ if pendingKey == key {
m.recoveryIngestPending[i] = batch
return true
}
diff --git a/internal/monitoring/recovery_ingest_test.go b/internal/monitoring/recovery_ingest_test.go
index df4cdd1d8c..2d4df96266 100644
--- a/internal/monitoring/recovery_ingest_test.go
+++ b/internal/monitoring/recovery_ingest_test.go
@@ -323,3 +323,32 @@ func TestRecoveryIngestPendingKeepsDistinctScopesAndEventBatches(t *testing.T) {
t.Fatalf("pending batches = %d, want two source enumerations and one event batch", got)
}
}
+
+// Partial PBS batches must stay before the complete recovery that supersedes
+// them: coalescing across a non-authoritative batch would resurrect deleted
+// points after a later successful empty enumeration.
+func TestRecoveryIngestCompletePollCannotOvertakePartialBatch(t *testing.T) {
+ m, manager := recoveryIngestTestMonitor(t)
+ store, err := manager.StoreForOrg("default")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer store.Close()
+ m.recoveryIngestRunning = true
+ scope := recoveryReconcileScope{provider: string(recovery.ProviderProxmoxPVE), idPrefix: "pve-backup:", instance: "pve1"}
+ point := recoveryIngestTestPoint("pve-backup:old", "pve1", time.Now().UTC())
+ m.enqueueRecoveryIngest(recoveryIngestBatch{points: []recovery.RecoveryPoint{point}, reconcile: &scope})
+ m.enqueueRecoveryIngest(recoveryIngestBatch{points: []recovery.RecoveryPoint{point}})
+ m.enqueueRecoveryIngest(recoveryIngestBatch{reconcile: &scope})
+ // Repeated complete recovery may still coalesce on its own side of the barrier.
+ m.enqueueRecoveryIngest(recoveryIngestBatch{reconcile: &scope})
+ if got := len(m.recoveryIngestPending); got != 3 {
+ t.Fatalf("pending=%d, want complete/partial/complete in source order", got)
+ }
+ for _, batch := range m.recoveryIngestPending {
+ m.ingestRecoveryPointsBestEffort(context.Background(), batch)
+ }
+ if ids := listRecoveryPointIDs(t, manager); len(ids) != 0 {
+ t.Fatalf("partial history resurrected after authoritative empty recovery: %v", ids)
+ }
+}