Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Count node-restricted backup storage as read when another node lists it
rcourtman835eddbff415
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex d0e779f1de..18cac33cc4 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -134,6 +134,13 @@ unknown access rather than claiming a successful empty enumeration. Independentreadable targets still publish accepted rows. An ordinary re-enable allows theexisting read path again; a successful empty reply can then establish removal.+Proxmox lists storage restricted to other nodes as disabled on every node it+is excluded from, so a skip is judged per storage after all nodes are walked: a+target skipped on one node but listed on another was read, and does not make+the provider history partial. Only a target no node listed counts as unread.+`TestPVEProtectionNodeRestrictedStorageReadElsewhereIsComplete` covers both+node orders and verifies no content read on an excluded node.+`TestPVEProtectionDisabledStorageIsNotAnEmptyRead` drives storage polling,asynchronous recovery ingest and posture reads with warm/cold inventory, one/alldisabled targets, repeated skips, re-enable, confirmed absence and recovery.diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex 6bba5d68f7..dbe75d1863 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -190,6 +190,7 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamepermissionFailureCount := 0 // Number of node/storage reads rejected by authorizationstoragePreserveNeeded := map[string]struct{}{}storageSuccess := map[string]struct{}{}+ unqueryableStorages := map[string]struct{}{}readState := m.backupReadStateForInstance(instanceName)// Build guest lookup map to find actual node for each VMID@@ -282,8 +283,10 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName// Configured but unreadable is not an empty content listing.// Preserve the no-query boundary (notably disabled storage)// without letting this skipped scope delete known history or- // qualify the provider-wide observation as complete.- contentFailures+++ // qualify the provider-wide observation as complete. PVE also+ // reports storage restricted to other nodes as disabled here,+ // so whether it went unread is decided after every node.+ unqueryableStorages[storage.Storage] = struct{}{}if _, ok := storageSuccess[storage.Storage]; !ok {storagePreserveNeeded[storage.Storage] = struct{}{}}@@ -449,6 +452,13 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamereturn}+ // A storage skipped on one node is unread only if no node listed it.+ for storageName := range unqueryableStorages {+ if _, ok := storageSuccess[storageName]; !ok {+ contentFailures+++ }+ }+// Carry-forward serves the live inventory, not a new provider observation.// Only rows actually returned this poll may renew point-level evidence.observedBackups := allBackupsdiff --git a/internal/monitoring/pve_protection_observation_test.go b/internal/monitoring/pve_protection_observation_test.goindex cf2c9945ed..4fb38760b2 100644--- a/internal/monitoring/pve_protection_observation_test.go+++ b/internal/monitoring/pve_protection_observation_test.go@@ -325,3 +325,46 @@ func TestPVEProtectionDisabledStorageIsNotAnEmptyRead(t *testing.T) {}}}++// nodeRestrictedBackupStorageClient mirrors a cluster whose backup storage is+// limited to one node: PVE lists it on every other node as disabled.+type nodeRestrictedBackupStorageClient struct {+ *pveReconciliationClient+ contentQueries map[string]int+}++func (c *nodeRestrictedBackupStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {+ storages, err := c.pveReconciliationClient.GetStorage(ctx, node)+ for _, other := range []string{"node-a", "node-b"} {+ if other != node {+ storages = append(storages, proxmox.Storage{Storage: "archive-" + other, Type: "dir", Content: "backup"})+ }+ }+ return storages, err+}++func (c *nodeRestrictedBackupStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {+ c.contentQueries[node+"/"+storage]+++ return c.pveReconciliationClient.GetStorageContent(ctx, node, storage)+}++func TestPVEProtectionNodeRestrictedStorageReadElsewhereIsComplete(t *testing.T) {+ for _, reversed := range []bool{false, true} {+ t.Run(fmt.Sprintf("reversed=%t", reversed), func(t *testing.T) {+ f := newPVEReconciliationFixture(t)+ if reversed {+ f.nodes = []proxmox.Node{f.nodes[1], f.nodes[0]}+ }+ client := &nodeRestrictedBackupStorageClient{pveReconciliationClient: f.client, contentQueries: make(map[string]int)}+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)+ f.wait(t)+ if len(f.points(t)) != 2 {+ t.Fatal("node-restricted storage lost its backups")+ }+ if client.contentQueries["node-a/archive-node-b"] != 0 || client.contentQueries["node-b/archive-node-a"] != 0 {+ t.Fatalf("storage was queried on a node it is not available on: %+v", client.contentQueries)+ }+ f.assertHistory(t, recovery.ProtectionHistoryComplete, operationaltrust.EvidencePermissionsSufficient)+ })+ }+}