Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep failed PBS namespace discovery out of complete backup evidence
pulse-triage[bot]7152d2b7fffe
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 782ffeeb9d..7c25ec01a6 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -6907,6 +6907,21 @@ without deleting retained artifacts. The observation is persisted beforereconciliation, so cached backup files cannot continue presenting currentprotection after collection becomes incomplete or unauthorized.+PBS namespace discovery is part of that collection evidence. An unavailable,+malformed or denied namespace listing must travel with the matching datastore+snapshot into background backup polling. A readable root namespace may still+supply accepted artifacts, but cannot turn failed child-namespace discovery+into a complete root-only inventory. Such a poll records partial history with+unknown or partial access, preserves unseen durable history and never guesses+child namespace requests. Recovery and a later successful namespace removal+resume ordinary complete, connection-scoped reconciliation. The PBS client's+existing legacy root-only response for an absent namespace endpoint is unchanged.+`TestPBSNamespaceDiscoveryFailureRetainsRecoveryHistory` crosses real HTTP+namespace discovery, background polling, durable ingest and canonical posture+on warm and replacement inventories; recovery, scoped removal, first-read and+legacy controls cover both safe retention and authoritative absence. No public+model, endpoint, policy, guest operation or restorable-backup claim is added.+Direct PBS backup enumeration emits two separate storage/recovery inputs:subject-linked recovery points and one typed provider observation for thepolled PBS instance. A complete poll records complete history with sufficientdiff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex b64eb24ef1..0f42ba3186 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -1579,7 +1579,7 @@ type pbsBackupFetchRequest struct {}// pollPBSBackups fetches all backups from PBS datastores-func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, client *pbs.Client, datastores []models.PBSDatastore) {+func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, client *pbs.Client, datastores []models.PBSDatastore, namespaceErrors map[string]error) {log.Debug().Str("instance", instanceName).Msg("polling PBS backups")// Cache existing PBS backups so we can avoid redundant API calls when no changes occurred.@@ -1615,6 +1615,15 @@ func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, cliendatastoreHadSuccess := falsedatastoreNamespaceErrors := 0datastoreTerminalNamespaceErrors := 0+ // The root namespace remains readable when discovering child namespaces+ // fails. Its accepted rows are useful, but they cannot establish that+ // undiscovered namespaces are empty or renew complete-provider evidence.+ if err := namespaceErrors[ds.Name]; err != nil {+ datastoreNamespaceErrors+++ if !shouldReuseCachedPBSBackups(err) {+ datastoreTerminalNamespaceErrors+++ }+ }groupsReused := 0groupsRequested := 0diff --git a/internal/monitoring/monitor_backups_readstate_test.go b/internal/monitoring/monitor_backups_readstate_test.goindex d910e1bb77..a64405b742 100644--- a/internal/monitoring/monitor_backups_readstate_test.go+++ b/internal/monitoring/monitor_backups_readstate_test.go@@ -348,7 +348,7 @@ func TestPollPBSBackupsFetchesRealSnapshotsForLargeGroups(t *testing.T) {}m := &Monitor{state: models.NewState()}- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)if got := atomic.LoadInt64(&snapshotCalls); got != 1 {t.Fatalf("large PBS backup group fetched snapshots %d times, want 1", got)@@ -383,7 +383,7 @@ func TestPollPBSBackupsFetchesRealSnapshotsForLargeGroups(t *testing.T) {}// A second poll must reuse the bounded cache instead of refetching.- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)if got := atomic.LoadInt64(&snapshotCalls); got != 1 {t.Fatalf("second poll refetched snapshots (%d calls), want cached reuse", got)}@@ -448,7 +448,7 @@ func TestPollPBSBackupsBoundsLargeTopologyAcrossPolls(t *testing.T) {m := &Monitor{state: models.NewState()}for cycle := 0; cycle < 3; cycle++ {- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)snapshot := m.state.GetSnapshot()if got := len(snapshot.PBSBackups); got != pbsBackupLiveStateLimit {@@ -615,7 +615,7 @@ func TestPollPBSBackupsPrunesCacheTimesForDeletedGroups(t *testing.T) {m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"},- })+ }, nil)m.mu.RLock()perGroup := m.pbsBackupCacheTime["pbs1"]@@ -1039,7 +1039,7 @@ func TestPollPBSBackups_DropsStaleCacheOnTerminalDatastoreError(t *testing.T) {m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"},- })+ }, nil)snapshot := m.state.GetSnapshot()for _, backup := range snapshot.PBSBackups {@@ -1097,7 +1097,7 @@ func TestPollPBSBackups_PreservesCacheOnTransientDatastoreError(t *testing.T) {m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"},- })+ }, nil)snapshot := m.state.GetSnapshot()var found booldiff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.goindex d151305fc4..c5d6a0ec05 100644--- a/internal/monitoring/monitor_extra_coverage_test.go+++ b/internal/monitoring/monitor_extra_coverage_test.go@@ -2394,7 +2394,7 @@ func TestMonitor_PollPBSBackups_Extra(t *testing.T) {t.Fatal(err)}ds := []models.PBSDatastore{{Name: "ds1"}}- m.pollPBSBackups(context.Background(), "pbs1", client, ds)+ m.pollPBSBackups(context.Background(), "pbs1", client, ds, nil)}func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {diff --git a/internal/monitoring/monitor_full_coverage_test.go b/internal/monitoring/monitor_full_coverage_test.goindex d91a46f41a..a7956a8e57 100644--- a/internal/monitoring/monitor_full_coverage_test.go+++ b/internal/monitoring/monitor_full_coverage_test.go@@ -222,7 +222,7 @@ func TestPollPBSBackups(t *testing.T) {}// Execute- m.pollPBSBackups(context.Background(), "pbs1", client, datastores)+ m.pollPBSBackups(context.Background(), "pbs1", client, datastores, nil)// Verifyfound := false@@ -1115,7 +1115,7 @@ func TestPollPBSBackupsCachedIncompleteSnapshotLosesStaleTaskObservation(t *testWriteActive: false,}})- m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}})+ m.pollPBSBackups(context.Background(), "pbs1", client, []models.PBSDatastore{{Name: "archive"}}, nil)if snapshotRequests.Load() != 1 {t.Fatalf("snapshot requests = %d, want 1 cached fallback attempt", snapshotRequests.Load())diff --git a/internal/monitoring/monitor_pbs_coverage_test.go b/internal/monitoring/monitor_pbs_coverage_test.goindex 20f1674150..c362f7c6f4 100644--- a/internal/monitoring/monitor_pbs_coverage_test.go+++ b/internal/monitoring/monitor_pbs_coverage_test.go@@ -3,6 +3,8 @@ package monitoringimport ("context""encoding/json"+ "fmt"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery""net/http""net/http/httptest""os"@@ -809,3 +811,38 @@ func TestPBSPolledCapacityRequiresObservedRecovery(t *testing.T) {t.Fatalf("raised policy did not clear high-usage alert: %+v", active)}}++func TestPBSNamespaceDiscoveryFirstPollDoesNotInventScope(t *testing.T) {+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusForbidden} {+ t.Run(fmt.Sprintf("status-%d", status), func(t *testing.T) {+ f := newPBSNamespacePollFixture(t)+ f.namespaceStatus.Store(int32(status))+ f.poll(t, "pbs-main")+ if len(f.points(t, "pbs-main")) != 1 || f.childReads.Load() != 0 {+ t.Fatal("failed first discovery invented a child namespace or lost readable root history")+ }+ for _, p := range f.postures(t) {+ if p.State == recovery.ProtectionStateProtected {+ t.Errorf("incomplete first discovery claimed current protection: %+v", p)+ }+ }+ })+ }+}++func TestPBSNamespaceDiscoveryLegacyRootOnlyRemainsComplete(t *testing.T) {+ f := newPBSNamespacePollFixture(t)+ // The existing PBS client treats an absent namespace endpoint on legacy+ // versions as a successful root-only inventory. Preserve that support.+ f.namespaceStatus.Store(http.StatusNotFound)+ f.poll(t, "pbs-main")+ if len(f.points(t, "pbs-main")) != 1 || f.childReads.Load() != 0 {+ t.Fatal("legacy root-only inventory was not collected")+ }+ for _, p := range f.postures(t) {+ if p.SubjectResourceID == f.ids[0] && (p.State != recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryComplete) {+ t.Errorf("legacy root-only support was marked incomplete: %+v", p)+ }+ }+}diff --git a/internal/monitoring/monitor_pbs_pmg.go b/internal/monitoring/monitor_pbs_pmg.goindex 1508a9217e..a72f8c6bef 100644--- a/internal/monitoring/monitor_pbs_pmg.go+++ b/internal/monitoring/monitor_pbs_pmg.go@@ -358,6 +358,9 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clieMsg("PBS node status retrieved")}+ // Keep discovery quality with the immutable input to the background backup+ // poll. Public datastore presentation is not an authoritative scope receipt.+ namespaceErrors := make(map[string]error)// Poll datastores if enabledif instanceCfg.MonitorDatastores {datastores, err := client.GetDatastoresFiltered(ctx, includeDatastore)@@ -417,6 +420,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clienamespaces, err := client.ListNamespaces(ctx, ds.Store, "", 0)if err != nil {+ namespaceErrors[ds.Store] = errlog.Warn().Err(err).Str("instance", instanceName).Str("datastore", ds.Store).@@ -570,7 +574,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, cliem.lastPBSBackupPoll[instanceName] = newLastm.mu.Unlock()- go func(ds []models.PBSDatastore, inst string, start time.Time, pbsClient *pbs.Client) {+ go func(ds []models.PBSDatastore, discoveryErrors map[string]error, inst string, start time.Time, pbsClient *pbs.Client) {defer recoverFromPanic(fmt.Sprintf("pollPBSBackups-%s", inst))defer func() {m.mu.Lock()@@ -593,13 +597,13 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, cliebackupCtx, cancel := context.WithTimeout(parentCtx, 5*time.Minute)defer cancel()- m.pollPBSBackups(backupCtx, inst, pbsClient, ds)+ m.pollPBSBackups(backupCtx, inst, pbsClient, ds, discoveryErrors)log.Info().Str("instance", inst).Dur("duration", time.Since(start)).Msg("Completed background PBS backup polling")- }(datastoreSnapshot, instanceName, now, client)+ }(datastoreSnapshot, namespaceErrors, instanceName, now, client)}}}diff --git a/internal/monitoring/pbs_poll_cancel_test.go b/internal/monitoring/pbs_poll_cancel_test.goindex fbbc7d0cf5..148b2ef0a8 100644--- a/internal/monitoring/pbs_poll_cancel_test.go+++ b/internal/monitoring/pbs_poll_cancel_test.go@@ -74,7 +74,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {case "during-task-correlation":datastores = []models.PBSDatastore{{Name: "archive"}}}- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores)+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)f.wait(t)f.assertProtected(t, "pbs-main")key := pbsBackupGroupKey{datastore: "archive", backupType: "vm", backupID: "100"}@@ -86,7 +86,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {done := make(chan struct{})go func() {defer close(done)- f.m.pollPBSBackups(ctx, "pbs-main", f.client, datastores)+ f.m.pollPBSBackups(ctx, "pbs-main", f.client, datastores, nil)}()select {case <-blocked:@@ -116,7 +116,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {// Do not clear or expire cache again. The next ordinary poll must// independently fetch the verification change the aborted poll lost.phase.Store(2)- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores)+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)f.wait(t)raw = f.m.state.GetSnapshot().PBSBackupsmapped, err := proxmoxmapper.FromPBSBackupsWithEvidence(raw, buildPBSGuestCandidates(f.m.GetUnifiedReadStateOrSnapshot()), time.Now())@@ -137,7 +137,7 @@ func TestPBSOrdinaryPollCancellationDoesNotRenewUnpublishedCache(t *testing.T) {}// Once the matching failed row has been published, normal TTL reuse// must avoid an unnecessary fourth read without restoring verified.- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores)+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, datastores, nil)f.wait(t)if snapshotReads.Load() != 3 || !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(publishedAt) || f.postures(t)[0].Verification != recovery.ProtectionVerificationUnverified {t.Error("cache-hit poll reread or renewed snapshots, or restored stale verification")diff --git a/internal/monitoring/pbs_poll_posture_test.go b/internal/monitoring/pbs_poll_posture_test.goindex acfc4b7dcf..9cdcc553ed 100644--- a/internal/monitoring/pbs_poll_posture_test.go+++ b/internal/monitoring/pbs_poll_posture_test.go@@ -131,7 +131,7 @@ func (f *pbsPosturePollFixture) poll(t *testing.T, instance string, mode int32,f.m.mu.Lock()clear(f.m.pbsBackupCacheTime)f.m.mu.Unlock()- f.m.pollPBSBackups(context.Background(), instance, f.client, datastores)+ f.m.pollPBSBackups(context.Background(), instance, f.client, datastores, nil)f.wait(t)}@@ -289,7 +289,7 @@ func TestPBSOrdinaryPollSnapshotFailuresDoNotClaimCompleteHistory(t *testing.T)}// No TTL manipulation on the next poll: a failed refresh must not// become a healthy cached read while snapshot access is still lost.- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds)+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds, nil)f.wait(t)}f.poll(t, "pbs-main", 0, ds)diff --git a/internal/monitoring/pbs_protection_observation_test.go b/internal/monitoring/pbs_protection_observation_test.goindex 4667265c3c..93719797a1 100644--- a/internal/monitoring/pbs_protection_observation_test.go+++ b/internal/monitoring/pbs_protection_observation_test.go@@ -8,14 +8,18 @@ import ("net/http""net/http/httptest""os"+ "reflect""strings""sync/atomic""testing""time"+ "github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/models""github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust""github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ recoverystore "github.com/rcourtman/pulse-go-rewrite/internal/recovery/store"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources""github.com/rcourtman/pulse-go-rewrite/pkg/pbs")@@ -186,7 +190,7 @@ func TestPBSOrdinaryPollNewestRemovalInvalidatesCappedCache(t *testing.T) {phase.Store(mode)// No cache clearing, TTL expiry or monitor replacement: these// are consecutive ordinary polls of the same connection.- f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds)+ f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds, nil)f.wait(t)}frames := map[string]any{}@@ -284,3 +288,242 @@ func TestPBSOrdinaryPollNewestRemovalInvalidatesCappedCache(t *testing.T) {}}}++// Namespace discovery is part of backup enumeration, not just optional drawer+// metadata. Drive the public poll entry point and its background backup worker+// so a missing namespace cannot masquerade as an authoritative root-only scan.+type pbsNamespacePollFixture struct {+ m *Monitor+ store *recoverystore.Store+ client *pbs.Client+ namespaceStatus atomic.Int32+ namespaceRemoved atomic.Bool+ rootEmpty atomic.Bool+ backupAt atomic.Int64+ childReads atomic.Int32+ ids []string+}++func newPBSNamespacePollFixture(t *testing.T) *pbsNamespacePollFixture {+ t.Helper()+ m, manager := recoveryIngestTestMonitor(t)+ f := &pbsNamespacePollFixture{m: m}+ f.backupAt.Store(time.Now().UTC().Add(-time.Hour).Truncate(time.Second).Unix())+ m.state = models.NewState()+ m.state.UpdateVMs([]models.VM{+ {ID: makeGuestID("cluster", "node-a", 100), VMID: 100, Instance: "cluster", Node: "node-a", Name: "guest-100", Status: "running"},+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "guest-101", Status: "running"},+ })+ for _, id := range []int{100, 101} {+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", id))+ }+ var err error+ f.store, err = manager.StoreForOrg("default")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { f.wait(t); _ = f.store.Close() })+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ var data any = []any{}+ switch r.URL.Path {+ case "/api2/json/version":+ data = map[string]any{"version": "3.4.2"}+ case "/api2/json/admin/datastore":+ data = []map[string]any{{"store": "archive"}}+ case "/api2/json/admin/datastore/archive/status":+ data = map[string]any{"total": 4096, "used": 1024, "avail": 3072}+ case "/api2/json/admin/datastore/archive/namespace":+ status := int(f.namespaceStatus.Load())+ if status == 200 {+ _, _ = w.Write([]byte(`{"data":`)) // incomplete, not a complete empty list+ return+ }+ if status != 0 {+ http.Error(w, "fixture namespace discovery unavailable", status)+ return+ }+ if !f.namespaceRemoved.Load() {+ data = []map[string]any{{"ns": "tenant"}}+ }+ case "/api2/json/admin/datastore/archive/groups":+ id := "100"+ if r.URL.Query().Get("ns") == "tenant" {+ id = "101"+ f.childReads.Add(1)+ }+ if id != "100" || !f.rootEmpty.Load() {+ data = []map[string]any{{"backup-type": "vm", "backup-id": id, "last-backup": f.backupAt.Load(), "backup-count": 1}}+ }+ case "/api2/json/admin/datastore/archive/snapshots":+ id := r.URL.Query().Get("backup-id")+ if r.URL.Query().Get("ns") == "tenant" {+ f.childReads.Add(1)+ }+ data = []map[string]any{{"backup-type": "vm", "backup-id": id, "backup-time": f.backupAt.Load(), "size": 1024,+ "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": data})+ }))+ t.Cleanup(server.Close)+ f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})+ if err != nil {+ t.Fatal(err)+ }+ m.config = &config.Config{EnableBackupPolling: true, BackupPollingCycles: 1, PBSInstances: []config.PBSInstance{+ {Name: "pbs-main", Host: server.URL, MonitorDatastores: true, MonitorBackups: true},+ {Name: "pbs-other", Host: server.URL, MonitorDatastores: true, MonitorBackups: true},+ }}+ m.lastPBSBackupPoll = make(map[string]time.Time)+ return f+}++func (f *pbsNamespacePollFixture) wait(t *testing.T) {+ t.Helper()+ deadline := time.Now().Add(10 * time.Second)+ for {+ f.m.mu.RLock()+ polling := len(f.m.pbsBackupPollers) > 0+ f.m.mu.RUnlock()+ f.m.recoveryIngestMu.Lock()+ ingesting := f.m.recoveryIngestRunning+ f.m.recoveryIngestMu.Unlock()+ if !polling && !ingesting {+ return+ }+ if time.Now().After(deadline) {+ t.Fatal("PBS namespace poll/ingest did not finish")+ }+ time.Sleep(time.Millisecond)+ }+}++func (f *pbsNamespacePollFixture) poll(t *testing.T, instance string) {+ t.Helper()+ f.m.pollPBSInstance(context.Background(), instance, f.client)+ f.wait(t)+}++func (f *pbsNamespacePollFixture) points(t *testing.T, instance string) map[string]recovery.RecoveryPoint {+ t.Helper()+ points, _, err := f.store.ListPoints(context.Background(), recovery.ListPointsOptions{Limit: 1000})+ if err != nil {+ t.Fatal(err)+ }+ out := make(map[string]recovery.RecoveryPoint)+ for _, point := range points {+ if point.RepositoryRef != nil && point.RepositoryRef.Namespace == instance {+ out[point.ID] = point+ }+ }+ return out+}++func (f *pbsNamespacePollFixture) postures(t *testing.T) []recovery.ProtectionPosture {+ t.Helper()+ postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{SubjectResourceIDs: f.ids})+ if err != nil || total != len(f.ids) {+ t.Fatalf("postures total=%d err=%v", total, err)+ }+ return postures+}++func TestPBSNamespaceDiscoveryFailureRetainsRecoveryHistory(t *testing.T) {+ for _, status := range []int{http.StatusServiceUnavailable, http.StatusForbidden, http.StatusOK} {+ for _, cold := range []bool{false, true} {+ t.Run(fmt.Sprintf("status-%d/cold-%t", status, cold), func(t *testing.T) {+ f := newPBSNamespacePollFixture(t)+ f.poll(t, "pbs-main")+ before := f.points(t, "pbs-main")+ if len(before) != 2 {+ t.Fatalf("initial points=%d, want root and tenant", len(before))+ }+ for _, p := range f.postures(t) {+ if p.State != recovery.ProtectionStateProtected {+ t.Fatalf("healthy control=%+v", p)+ }+ }+ if cold {+ // Monitor replacement removes live inventory, not durable history.+ vms := f.m.state.GetSnapshot().VMs+ f.m.state = models.NewState()+ f.m.state.UpdateVMs(vms)+ }+ f.namespaceStatus.Store(int32(status))+ f.backupAt.Add(15 * 60)+ reads := f.childReads.Load()+ for range 2 {+ f.poll(t, "pbs-main")+ after := f.points(t, "pbs-main")+ if len(after) != 2 {+ t.Errorf("failed namespace discovery deleted durable history: %d points", len(after))+ }+ for id, point := range before {+ if point.Details["namespace"] == "tenant" && !reflect.DeepEqual(after[id], point) {+ t.Error("unseen namespace point was deleted or renewed without an observation")+ }+ }+ for _, point := range after {+ if point.Details["namespace"] == "" && (point.CompletedAt == nil || point.CompletedAt.Unix() != f.backupAt.Load()) {+ t.Error("independent successful root read did not update its point")+ }+ }+ permissions := operationaltrust.EvidencePermissionsUnknown+ if status == http.StatusForbidden {+ permissions = operationaltrust.EvidencePermissionsPartial+ }+ for _, p := range f.postures(t) {+ if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||+ p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial ||+ p.ProviderStates[0].Permissions != permissions || p.ProviderStates[0].JobState != recovery.OutcomeWarning {+ t.Errorf("discovery failure presented as complete/healthy: %+v", p)+ }+ }+ if f.childReads.Load() != reads {+ t.Error("failed discovery guessed a hidden namespace read")+ }+ }+ })+ }+ }+}++func TestPBSNamespaceDiscoveryRecoveryAndScopedRemoval(t *testing.T) {+ f := newPBSNamespacePollFixture(t)+ f.poll(t, "pbs-other")+ other := f.points(t, "pbs-other")+ f.poll(t, "pbs-main")+ f.namespaceStatus.Store(http.StatusServiceUnavailable)+ f.poll(t, "pbs-main")+ if len(f.points(t, "pbs-main")) != 2 {+ t.Error("transient discovery failure removed tenant history")+ }+ f.namespaceStatus.Store(0)+ f.poll(t, "pbs-main")+ if len(f.points(t, "pbs-main")) != 2 {+ t.Fatal("recovery did not restore namespace scope")+ }+ for _, p := range f.postures(t) {+ for _, provider := range p.ProviderStates {+ if provider.HistoryCompleteness != recovery.ProtectionHistoryComplete {+ t.Errorf("recovered discovery still incomplete: %+v", p)+ }+ }+ }+ f.namespaceRemoved.Store(true)+ f.poll(t, "pbs-main")+ if got := len(f.points(t, "pbs-main")); got != 1 {+ t.Errorf("authoritative namespace removal retained %d points, want root only", got)+ }+ if !reflect.DeepEqual(other, f.points(t, "pbs-other")) {+ t.Error("namespace removal crossed connection scope")+ }+ f.rootEmpty.Store(true)+ f.poll(t, "pbs-main")+ if len(f.points(t, "pbs-main")) != 0 {+ t.Error("authoritative complete empty scan retained points")+ }+ if !reflect.DeepEqual(other, f.points(t, "pbs-other")) {+ t.Error("empty scan crossed connection scope")+ }+}