Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep failed PBS namespace discovery out of complete backup evidence

pulse-triage[bot]7152d2b7fffe
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 782ffeeb9d..7c25ec01a6 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -6907,6 +6907,21 @@ without deleting retained artifacts. The observation is persisted before
reconciliation, so cached backup files cannot continue presenting current
protection after collection becomes incomplete or unauthorized.
+PBS namespace discovery is part of that collection evidence. An unavailable,
+malformed or denied namespace listing must travel with the matching datastore
+snapshot into background backup polling. A readable root namespace may still
+supply accepted artifacts, but cannot turn failed child-namespace discovery
+into a complete root-only inventory. Such a poll records partial history with
+unknown or partial access, preserves unseen durable history and never guesses
+child namespace requests. Recovery and a later successful namespace removal
+resume ordinary complete, connection-scoped reconciliation. The PBS client's
+existing legacy root-only response for an absent namespace endpoint is unchanged.
+`TestPBSNamespaceDiscoveryFailureRetainsRecoveryHistory` crosses real HTTP
+namespace discovery, background polling, durable ingest and canonical posture
+on warm and replacement inventories; recovery, scoped removal, first-read and
+legacy controls cover both safe retention and authoritative absence. No public
+model, endpoint, policy, guest operation or restorable-backup claim is added.
+
Direct PBS backup enumeration emits two separate storage/recovery inputs:
subject-linked recovery points and one typed provider observation for the
polled PBS instance. A complete poll records complete history with sufficient
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index b64eb24ef1..0f42ba3186 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -1579,7 +1579,7 @@ type pbsBackupFetchRequest struct {
}
// pollPBSBackups fetches all backups from PBS datastores
-func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, client *pbs.Client, datastores []models.PBSDatastore) {
+func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, client *pbs.Client, datastores []models.PBSDatastore, namespaceErrors map[string]error) {
log.Debug().Str("instance", instanceName).Msg("polling PBS backups")
// Cache existing PBS backups so we can avoid redundant API calls when no changes occurred.
@@ -1615,6 +1615,15 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, clien
datastoreHadSuccess := false
datastoreNamespaceErrors := 0
datastoreTerminalNamespaceErrors := 0
+ // The root namespace remains readable when discovering child namespaces
+ // fails. Its accepted rows are useful, but they cannot establish that
+ // undiscovered namespaces are empty or renew complete-provider evidence.
+ if err := namespaceErrors[ds.Name]; err != nil {
+ datastoreNamespaceErrors++
+ if !shouldReuseCachedPBSBackups(err) {
+ datastoreTerminalNamespaceErrors++
+ }
+ }
groupsReused := 0
groupsRequested := 0
diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.go
index d910e1bb77..a64405b742 100644
--- a/internal/monitoring/monitor_backups_readstate_test.go
+++ b/internal/monitoring/monitor_backups_readstate_test.go
@@ -348,7 +348,7 @@ func TestPollPBSBackupsFetchesRealSnapshotsForLargeGroups(t *testing.T) {
}
m := &Monitor{state: models.NewState()}
- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})
+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)
if got := atomic.LoadInt64(&snapshotCalls); got != 1 {
t.Fatalf("large PBS backup group fetched snapshots %d times, want 1", got)
@@ -383,7 +383,7 @@ func TestPollPBSBackupsFetchesRealSnapshotsForLargeGroups(t *testing.T) {
}
// A second poll must reuse the bounded cache instead of refetching.
- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})
+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)
if got := atomic.LoadInt64(&snapshotCalls); got != 1 {
t.Fatalf("second poll refetched snapshots (%d calls), want cached reuse", got)
}
@@ -448,7 +448,7 @@ func TestPollPBSBackupsBoundsLargeTopologyAcrossPolls(t *testing.T) {
m := &Monitor{state: models.NewState()}
for cycle := 0; cycle < 3; cycle++ {
- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})
+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)
snapshot := m.state.GetSnapshot()
if got := len(snapshot.PBSBackups); got != pbsBackupLiveStateLimit {
@@ -615,7 +615,7 @@ func TestPollPBSBackupsPrunesCacheTimesForDeletedGroups(t *testing.T) {
m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{
{Name: "archive"},
- })
+ }, nil)
m.mu.RLock()
perGroup := m.pbsBackupCacheTime["pbs1"]
@@ -1039,7 +1039,7 @@ func TestPollPBSBackups_DropsStaleCacheOnTerminalDatastoreError(t *testing.T) {
m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{
{Name: "archive"},
- })
+ }, nil)
snapshot := m.state.GetSnapshot()
for _, backup := range snapshot.PBSBackups {
@@ -1097,7 +1097,7 @@ func TestPollPBSBackups_PreservesCacheOnTransientDatastoreError(t *testing.T) {
m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{
{Name: "archive"},
- })
+ }, nil)
snapshot := m.state.GetSnapshot()
var found bool
diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.go
index d151305fc4..c5d6a0ec05 100644
--- a/internal/monitoring/monitor_extra_coverage_test.go
+++ b/internal/monitoring/monitor_extra_coverage_test.go
@@ -2394,7 +2394,7 @@ func TestMonitor_PollPBSBackups_Extra(t *testing.T) {
t.Fatal(err)
}
ds := []models.PBSDatastore{{Name: "ds1"}}
- m.pollPBSBackups(context.Background(), "pbs1", client, ds)
+ m.pollPBSBackups(context.Background(), "pbs1", client, ds, nil)
}
func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {
diff --git a/internal/monitoring/monitor_full_coverage_test.go b/internal/monitoring/monitor_full_coverage_test.go
index d91a46f41a..a7956a8e57 100644
--- a/internal/monitoring/monitor_full_coverage_test.go
+++ b/internal/monitoring/monitor_full_coverage_test.go
@@ -222,7 +222,7 @@ func TestPollPBSBackups(t *testing.T) {
}
// Execute
- m.pollPBSBackups(context.Background(), "pbs1", client, datastores)
+ m.pollPBSBackups(context.Background(), "pbs1", client, datastores, nil)
// Verify
found := false
@@ -1115,7 +1115,7 @@ func TestPollPBSBackupsCachedIncompleteSnapshotLosesStaleTaskObservation(t *test
WriteActive: false,
}})
- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})
+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)
if snapshotRequests.Load() != 1 {
t.Fatalf("snapshot requests = %d, want 1 cached fallback attempt", snapshotRequests.Load())
diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.go
index 20f1674150..c362f7c6f4 100644
--- a/internal/monitoring/monitor_pbs_coverage_test.go
+++ b/internal/monitoring/monitor_pbs_coverage_test.go
@@ -3,6 +3,8 @@ package monitoring
import (
"context"
"encoding/json"
+ "fmt"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"net/http"
"net/http/httptest"
"os"
@@ -809,3 +811,38 @@ func TestPBSPolledCapacityRequiresObservedRecovery(t *testing.T) {
t.Fatalf("raised policy did not clear high-usage alert: %+v", active)
}
}
+
+func TestPBSNamespaceDiscoveryFirstPollDoesNotInventScope(t *testing.T) {
+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusForbidden} {
+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {
+ f := newPBSNamespacePollFixture(t)
+ f.namespaceStatus.Store(int32(status))
+ f.poll(t, "pbs-main")
+ if len(f.points(t, "pbs-main")) != 1 || f.childReads.Load() != 0 {
+ t.Fatal("failed first discovery invented a child namespace or lost readable root history")
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected {
+ t.Errorf("incomplete first discovery claimed current protection: %+v", p)
+ }
+ }
+ })
+ }
+}
+
+func TestPBSNamespaceDiscoveryLegacyRootOnlyRemainsComplete(t *testing.T) {
+ f := newPBSNamespacePollFixture(t)
+ // The existing PBS client treats an absent namespace endpoint on legacy
+ // versions as a successful root-only inventory. Preserve that support.
+ f.namespaceStatus.Store(http.StatusNotFound)
+ f.poll(t, "pbs-main")
+ if len(f.points(t, "pbs-main")) != 1 || f.childReads.Load() != 0 {
+ t.Fatal("legacy root-only inventory was not collected")
+ }
+ for _, p := range f.postures(t) {
+ if p.SubjectResourceID == f.ids[0] && (p.State != recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||
+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryComplete) {
+ t.Errorf("legacy root-only support was marked incomplete: %+v", p)
+ }
+ }
+}
diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.go
index 1508a9217e..a72f8c6bef 100644
--- a/internal/monitoring/monitor_pbs_pmg.go
+++ b/internal/monitoring/monitor_pbs_pmg.go
@@ -358,6 +358,9 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
Msg("PBS node status retrieved")
}
+ // Keep discovery quality with the immutable input to the background backup
+ // poll. Public datastore presentation is not an authoritative scope receipt.
+ namespaceErrors := make(map[string]error)
// Poll datastores if enabled
if instanceCfg.MonitorDatastores {
datastores, err := client.GetDatastoresFiltered(ctx, includeDatastore)
@@ -417,6 +420,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
namespaces, err := client.ListNamespaces(ctx, ds.Store, "", 0)
if err != nil {
+ namespaceErrors[ds.Store] = err
log.Warn().Err(err).
Str("instance", instanceName).
Str("datastore", ds.Store).
@@ -570,7 +574,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
m.lastPBSBackupPoll[instanceName] = newLast
m.mu.Unlock()
- go func(ds []models.PBSDatastore, inst string, start time.Time, pbsClient *pbs.Client) {
+ go func(ds []models.PBSDatastore, discoveryErrors map[string]error, inst string, start time.Time, pbsClient *pbs.Client) {
defer recoverFromPanic(fmt.Sprintf("pollPBSBackups-%s", inst))
defer func() {
m.mu.Lock()
@@ -593,13 +597,13 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
backupCtx, cancel := context.WithTimeout(parentCtx, 5*time.Minute)
defer cancel()
- m.pollPBSBackups(backupCtx, inst, pbsClient, ds)
+ m.pollPBSBackups(backupCtx, inst, pbsClient, ds, discoveryErrors)
log.Info().
Str("instance", inst).
Dur("duration", time.Since(start)).
Msg("Completed background PBS backup polling")
- }(datastoreSnapshot, instanceName, now, client)
+ }(datastoreSnapshot, namespaceErrors, instanceName, now, client)
}
}
}
diff --git a/internal/monitoring/pbs_poll_cancel_test.go b/internal/monitoring/pbs_poll_cancel_test.go
index fbbc7d0cf5..148b2ef0a8 100644
--- a/internal/monitoring/pbs_poll_cancel_test.go
+++ b/internal/monitoring/pbs_poll_cancel_test.go
@@ -74,7 +74,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {
case "during-task-correlation":
datastores = []models.PBSDatastore{{Name: "archive"}}
}
- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores)
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)
f.wait(t)
f.assertProtected(t, "pbs-main")
key := pbsBackupGroupKey{datastore: "archive", backupType: "vm", backupID: "100"}
@@ -86,7 +86,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {
done := make(chan struct{})
go func() {
defer close(done)
- f.m.pollPBSBackups(ctx, "pbs-main", f.client, datastores)
+ f.m.pollPBSBackups(ctx, "pbs-main", f.client, datastores, nil)
}()
select {
case <-blocked:
@@ -116,7 +116,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {
// Do not clear or expire cache again. The next ordinary poll must
// independently fetch the verification change the aborted poll lost.
phase.Store(2)
- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores)
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)
f.wait(t)
raw = f.m.state.GetSnapshot().PBSBackups
mapped, err := proxmoxmapper.FromPBSBackupsWithEvidence(raw, buildPBSGuestCandidates(f.m.GetUnifiedReadStateOrSnapshot()), time.Now())
@@ -137,7 +137,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {
}
// Once the matching failed row has been published, normal TTL reuse
// must avoid an unnecessary fourth read without restoring verified.
- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores)
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)
f.wait(t)
if snapshotReads.Load() != 3 || !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(publishedAt) || f.postures(t)[0].Verification != recovery.ProtectionVerificationUnverified {
t.Error("cache-hit poll reread or renewed snapshots, or restored stale verification")
diff --git a/internal/monitoring/pbs_poll_posture_test.go b/internal/monitoring/pbs_poll_posture_test.go
index acfc4b7dcf..9cdcc553ed 100644
--- a/internal/monitoring/pbs_poll_posture_test.go
+++ b/internal/monitoring/pbs_poll_posture_test.go
@@ -131,7 +131,7 @@ func (f *pbsPosturePollFixture) poll(t *testing.T, instance string, mode int32,
f.m.mu.Lock()
clear(f.m.pbsBackupCacheTime)
f.m.mu.Unlock()
- f.m.pollPBSBackups(context.Background(), instance, f.client, datastores)
+ f.m.pollPBSBackups(context.Background(), instance, f.client, datastores, nil)
f.wait(t)
}
@@ -289,7 +289,7 @@ func TestPBSOrdinaryPollSnapshotFailuresDoNotClaimCompleteHistory(t *testing.T)
}
// No TTL manipulation on the next poll: a failed refresh must not
// become a healthy cached read while snapshot access is still lost.
- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds)
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds, nil)
f.wait(t)
}
f.poll(t, "pbs-main", 0, ds)
diff --git a/internal/monitoring/pbs_protection_observation_test.go b/internal/monitoring/pbs_protection_observation_test.go
index 4667265c3c..93719797a1 100644
--- a/internal/monitoring/pbs_protection_observation_test.go
+++ b/internal/monitoring/pbs_protection_observation_test.go
@@ -8,14 +8,18 @@ import (
"net/http"
"net/http/httptest"
"os"
+ "reflect"
"strings"
"sync/atomic"
"testing"
"time"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ recoverystore "github.com/rcourtman/pulse-go-rewrite/internal/recovery/store"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
)
@@ -186,7 +190,7 @@ func TestPBSOrdinaryPollNewestRemovalInvalidatesCappedCache(t *testing.T) {
phase.Store(mode)
// No cache clearing, TTL expiry or monitor replacement: these
// are consecutive ordinary polls of the same connection.
- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds)
+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds, nil)
f.wait(t)
}
frames := map[string]any{}
@@ -284,3 +288,242 @@ func TestPBSOrdinaryPollNewestRemovalInvalidatesCappedCache(t *testing.T) {
}
}
}
+
+// Namespace discovery is part of backup enumeration, not just optional drawer
+// metadata. Drive the public poll entry point and its background backup worker
+// so a missing namespace cannot masquerade as an authoritative root-only scan.
+type pbsNamespacePollFixture struct {
+ m *Monitor
+ store *recoverystore.Store
+ client *pbs.Client
+ namespaceStatus atomic.Int32
+ namespaceRemoved atomic.Bool
+ rootEmpty atomic.Bool
+ backupAt atomic.Int64
+ childReads atomic.Int32
+ ids []string
+}
+
+func newPBSNamespacePollFixture(t *testing.T) *pbsNamespacePollFixture {
+ t.Helper()
+ m, manager := recoveryIngestTestMonitor(t)
+ f := &pbsNamespacePollFixture{m: m}
+ f.backupAt.Store(time.Now().UTC().Add(-time.Hour).Truncate(time.Second).Unix())
+ m.state = models.NewState()
+ m.state.UpdateVMs([]models.VM{
+ {ID: makeGuestID("cluster", "node-a", 100), VMID: 100, Instance: "cluster", Node: "node-a", Name: "guest-100", Status: "running"},
+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "guest-101", Status: "running"},
+ })
+ for _, id := range []int{100, 101} {
+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", id))
+ }
+ var err error
+ f.store, err = manager.StoreForOrg("default")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { f.wait(t); _ = f.store.Close() })
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ var data any = []any{}
+ switch r.URL.Path {
+ case "/api2/json/version":
+ data = map[string]any{"version": "3.4.2"}
+ case "/api2/json/admin/datastore":
+ data = []map[string]any{{"store": "archive"}}
+ case "/api2/json/admin/datastore/archive/status":
+ data = map[string]any{"total": 4096, "used": 1024, "avail": 3072}
+ case "/api2/json/admin/datastore/archive/namespace":
+ status := int(f.namespaceStatus.Load())
+ if status == 200 {
+ _, _ = w.Write([]byte(`{"data":`)) // incomplete, not a complete empty list
+ return
+ }
+ if status != 0 {
+ http.Error(w, "fixture namespace discovery unavailable", status)
+ return
+ }
+ if !f.namespaceRemoved.Load() {
+ data = []map[string]any{{"ns": "tenant"}}
+ }
+ case "/api2/json/admin/datastore/archive/groups":
+ id := "100"
+ if r.URL.Query().Get("ns") == "tenant" {
+ id = "101"
+ f.childReads.Add(1)
+ }
+ if id != "100" || !f.rootEmpty.Load() {
+ data = []map[string]any{{"backup-type": "vm", "backup-id": id, "last-backup": f.backupAt.Load(), "backup-count": 1}}
+ }
+ case "/api2/json/admin/datastore/archive/snapshots":
+ id := r.URL.Query().Get("backup-id")
+ if r.URL.Query().Get("ns") == "tenant" {
+ f.childReads.Add(1)
+ }
+ data = []map[string]any{{"backup-type": "vm", "backup-id": id, "backup-time": f.backupAt.Load(), "size": 1024,
+ "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": data})
+ }))
+ t.Cleanup(server.Close)
+ f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ m.config = &config.Config{EnableBackupPolling: true, BackupPollingCycles: 1, PBSInstances: []config.PBSInstance{
+ {Name: "pbs-main", Host: server.URL, MonitorDatastores: true, MonitorBackups: true},
+ {Name: "pbs-other", Host: server.URL, MonitorDatastores: true, MonitorBackups: true},
+ }}
+ m.lastPBSBackupPoll = make(map[string]time.Time)
+ return f
+}
+
+func (f *pbsNamespacePollFixture) wait(t *testing.T) {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ f.m.mu.RLock()
+ polling := len(f.m.pbsBackupPollers) > 0
+ f.m.mu.RUnlock()
+ f.m.recoveryIngestMu.Lock()
+ ingesting := f.m.recoveryIngestRunning
+ f.m.recoveryIngestMu.Unlock()
+ if !polling && !ingesting {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("PBS namespace poll/ingest did not finish")
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+func (f *pbsNamespacePollFixture) poll(t *testing.T, instance string) {
+ t.Helper()
+ f.m.pollPBSInstance(context.Background(), instance, f.client)
+ f.wait(t)
+}
+
+func (f *pbsNamespacePollFixture) points(t *testing.T, instance string) map[string]recovery.RecoveryPoint {
+ t.Helper()
+ points, _, err := f.store.ListPoints(context.Background(), recovery.ListPointsOptions{Limit: 1000})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := make(map[string]recovery.RecoveryPoint)
+ for _, point := range points {
+ if point.RepositoryRef != nil && point.RepositoryRef.Namespace == instance {
+ out[point.ID] = point
+ }
+ }
+ return out
+}
+
+func (f *pbsNamespacePollFixture) postures(t *testing.T) []recovery.ProtectionPosture {
+ t.Helper()
+ postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{SubjectResourceIDs: f.ids})
+ if err != nil || total != len(f.ids) {
+ t.Fatalf("postures total=%d err=%v", total, err)
+ }
+ return postures
+}
+
+func TestPBSNamespaceDiscoveryFailureRetainsRecoveryHistory(t *testing.T) {
+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusForbidden, http.StatusOK} {
+ for _, cold := range []bool{false, true} {
+ t.Run(fmt.Sprintf("status-%d/cold-%t", status, cold), func(t *testing.T) {
+ f := newPBSNamespacePollFixture(t)
+ f.poll(t, "pbs-main")
+ before := f.points(t, "pbs-main")
+ if len(before) != 2 {
+ t.Fatalf("initial points=%d, want root and tenant", len(before))
+ }
+ for _, p := range f.postures(t) {
+ if p.State != recovery.ProtectionStateProtected {
+ t.Fatalf("healthy control=%+v", p)
+ }
+ }
+ if cold {
+ // Monitor replacement removes live inventory, not durable history.
+ vms := f.m.state.GetSnapshot().VMs
+ f.m.state = models.NewState()
+ f.m.state.UpdateVMs(vms)
+ }
+ f.namespaceStatus.Store(int32(status))
+ f.backupAt.Add(15 * 60)
+ reads := f.childReads.Load()
+ for range 2 {
+ f.poll(t, "pbs-main")
+ after := f.points(t, "pbs-main")
+ if len(after) != 2 {
+ t.Errorf("failed namespace discovery deleted durable history: %d points", len(after))
+ }
+ for id, point := range before {
+ if point.Details["namespace"] == "tenant" && !reflect.DeepEqual(after[id], point) {
+ t.Error("unseen namespace point was deleted or renewed without an observation")
+ }
+ }
+ for _, point := range after {
+ if point.Details["namespace"] == "" && (point.CompletedAt == nil || point.CompletedAt.Unix() != f.backupAt.Load()) {
+ t.Error("independent successful root read did not update its point")
+ }
+ }
+ permissions := operationaltrust.EvidencePermissionsUnknown
+ if status == http.StatusForbidden {
+ permissions = operationaltrust.EvidencePermissionsPartial
+ }
+ for _, p := range f.postures(t) {
+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||
+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial ||
+ p.ProviderStates[0].Permissions != permissions || p.ProviderStates[0].JobState != recovery.OutcomeWarning {
+ t.Errorf("discovery failure presented as complete/healthy: %+v", p)
+ }
+ }
+ if f.childReads.Load() != reads {
+ t.Error("failed discovery guessed a hidden namespace read")
+ }
+ }
+ })
+ }
+ }
+}
+
+func TestPBSNamespaceDiscoveryRecoveryAndScopedRemoval(t *testing.T) {
+ f := newPBSNamespacePollFixture(t)
+ f.poll(t, "pbs-other")
+ other := f.points(t, "pbs-other")
+ f.poll(t, "pbs-main")
+ f.namespaceStatus.Store(http.StatusServiceUnavailable)
+ f.poll(t, "pbs-main")
+ if len(f.points(t, "pbs-main")) != 2 {
+ t.Error("transient discovery failure removed tenant history")
+ }
+ f.namespaceStatus.Store(0)
+ f.poll(t, "pbs-main")
+ if len(f.points(t, "pbs-main")) != 2 {
+ t.Fatal("recovery did not restore namespace scope")
+ }
+ for _, p := range f.postures(t) {
+ for _, provider := range p.ProviderStates {
+ if provider.HistoryCompleteness != recovery.ProtectionHistoryComplete {
+ t.Errorf("recovered discovery still incomplete: %+v", p)
+ }
+ }
+ }
+ f.namespaceRemoved.Store(true)
+ f.poll(t, "pbs-main")
+ if got := len(f.points(t, "pbs-main")); got != 1 {
+ t.Errorf("authoritative namespace removal retained %d points, want root only", got)
+ }
+ if !reflect.DeepEqual(other, f.points(t, "pbs-other")) {
+ t.Error("namespace removal crossed connection scope")
+ }
+ f.rootEmpty.Store(true)
+ f.poll(t, "pbs-main")
+ if len(f.points(t, "pbs-main")) != 0 {
+ t.Error("authoritative complete empty scan retained points")
+ }
+ if !reflect.DeepEqual(other, f.points(t, "pbs-other")) {
+ t.Error("empty scan crossed connection scope")
+ }
+}