Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Count node-restricted backup storage as read when another node lists it

rcourtman835eddbff415
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index d0e779f1de..18cac33cc4 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -134,6 +134,13 @@ unknown access rather than claiming a successful empty enumeration. Independent
readable targets still publish accepted rows. An ordinary re-enable allows the
existing read path again; a successful empty reply can then establish removal.
+Proxmox lists storage restricted to other nodes as disabled on every node it
+is excluded from, so a skip is judged per storage after all nodes are walked: a
+target skipped on one node but listed on another was read, and does not make
+the provider history partial. Only a target no node listed counts as unread.
+`TestPVEProtectionNodeRestrictedStorageReadElsewhereIsComplete` covers both
+node orders and verifies no content read on an excluded node.
+
`TestPVEProtectionDisabledStorageIsNotAnEmptyRead` drives storage polling,
asynchronous recovery ingest and posture reads with warm/cold inventory, one/all
disabled targets, repeated skips, re-enable, confirmed absence and recovery.
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index 6bba5d68f7..dbe75d1863 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -190,6 +190,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
permissionFailureCount := 0 // Number of node/storage reads rejected by authorization
storagePreserveNeeded := map[string]struct{}{}
storageSuccess := map[string]struct{}{}
+ unqueryableStorages := map[string]struct{}{}
readState := m.backupReadStateForInstance(instanceName)
// Build guest lookup map to find actual node for each VMID
@@ -282,8 +283,10 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
// Configured but unreadable is not an empty content listing.
// Preserve the no-query boundary (notably disabled storage)
// without letting this skipped scope delete known history or
- // qualify the provider-wide observation as complete.
- contentFailures++
+ // qualify the provider-wide observation as complete. PVE also
+ // reports storage restricted to other nodes as disabled here,
+ // so whether it went unread is decided after every node.
+ unqueryableStorages[storage.Storage] = struct{}{}
if _, ok := storageSuccess[storage.Storage]; !ok {
storagePreserveNeeded[storage.Storage] = struct{}{}
}
@@ -449,6 +452,13 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
return
}
+ // A storage skipped on one node is unread only if no node listed it.
+ for storageName := range unqueryableStorages {
+ if _, ok := storageSuccess[storageName]; !ok {
+ contentFailures++
+ }
+ }
+
// Carry-forward serves the live inventory, not a new provider observation.
// Only rows actually returned this poll may renew point-level evidence.
observedBackups := allBackups
diff --git a/internal/monitoring/pve_protection_observation_test.go b/internal/monitoring/pve_protection_observation_test.go
index cf2c9945ed..4fb38760b2 100644
--- a/internal/monitoring/pve_protection_observation_test.go
+++ b/internal/monitoring/pve_protection_observation_test.go
@@ -325,3 +325,46 @@ func TestPVEProtectionDisabledStorageIsNotAnEmptyRead(t *testing.T) {
}
}
}
+
+// nodeRestrictedBackupStorageClient mirrors a cluster whose backup storage is
+// limited to one node: PVE lists it on every other node as disabled.
+type nodeRestrictedBackupStorageClient struct {
+ *pveReconciliationClient
+ contentQueries map[string]int
+}
+
+func (c *nodeRestrictedBackupStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
+ storages, err := c.pveReconciliationClient.GetStorage(ctx, node)
+ for _, other := range []string{"node-a", "node-b"} {
+ if other != node {
+ storages = append(storages, proxmox.Storage{Storage: "archive-" + other, Type: "dir", Content: "backup"})
+ }
+ }
+ return storages, err
+}
+
+func (c *nodeRestrictedBackupStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
+ c.contentQueries[node+"/"+storage]++
+ return c.pveReconciliationClient.GetStorageContent(ctx, node, storage)
+}
+
+func TestPVEProtectionNodeRestrictedStorageReadElsewhereIsComplete(t *testing.T) {
+ for _, reversed := range []bool{false, true} {
+ t.Run(fmt.Sprintf("reversed=%t", reversed), func(t *testing.T) {
+ f := newPVEReconciliationFixture(t)
+ if reversed {
+ f.nodes = []proxmox.Node{f.nodes[1], f.nodes[0]}
+ }
+ client := &nodeRestrictedBackupStorageClient{pveReconciliationClient: f.client, contentQueries: make(map[string]int)}
+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)
+ f.wait(t)
+ if len(f.points(t)) != 2 {
+ t.Fatal("node-restricted storage lost its backups")
+ }
+ if client.contentQueries["node-a/archive-node-b"] != 0 || client.contentQueries["node-b/archive-node-a"] != 0 {
+ t.Fatalf("storage was queried on a node it is not available on: %+v", client.contentQueries)
+ }
+ f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)
+ })
+ }
+}