Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep incomplete guest filesystem readings out of recovery
pulse-triage[bot]615fa2991859
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 5bb69533b2..390cb2c376 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -6753,3 +6753,28 @@ generation younger than two seconds. The decision takes effect in the nextloaded generation, not retroactively in the listing already served. Thefixture seed keeps default thresholds while the configured live adapterretains its four-minute Proxmox threshold.++### Incomplete guest filesystem responses do not establish recovery++`GetVMFSInfo` preserves valid peer rows from a malformed array but returns an+error with them, through both single and cluster clients. The surviving rows+cannot establish total guest capacity, usage, or removal of a rejected volume.+A completed response with invalid counters is not a command-completion failure:+serial admission, same-target lock checks and single-attempt guest requests+remain unchanged, and there is no failover/replay or new cooldown. Alternate+clients' contradictory or out-of-range counters enforce the same completeness+boundary before publishing a current aggregate.++Monitoring retains only the prior complete disk tuple, mount inventory and+original observation time within the existing age bound; otherwise disk usage+is unavailable. Partial responses do not create disk History, resolve genuine+filesystem alerts, or remove unknown volumes. Independent current CPU/memory+continues. A later complete ordinary inventory can still remove a volume and+publish an explicitly measured zero, append History and recover alerts.++`pkg/proxmox/guest_filesystem_completeness_test.go` and+`internal/monitoring/guest_filesystem_completeness_test.go` cover valid peers plus+malformed records, both client/collector paths, canonical and served state,+original age/expiry, History, breach continuity and complete removal/recovery.+These are synthetic source controls, not native thaw, the cause of #2619/#2439,+or installed/released acceptance.diff --git a/internal/monitoring/guest_filesystem_completeness_test.go b/internal/monitoring/guest_filesystem_completeness_test.gonew file mode 100644index 0000000000..ee2c6e308f--- /dev/null+++ b/internal/monitoring/guest_filesystem_completeness_test.go@@ -0,0 +1,246 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "math"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++type incompleteFilesystemClient struct {+ mockPVEClientExtended+ filesystems []proxmox.VMFileSystem+}++func (c *incompleteFilesystemClient) GetVMFSInfo(context.Context, string, int) ([]proxmox.VMFileSystem, error) {+ return c.filesystems, nil+}++func TestVMFilesystemIncompleteAlternateClientCannotPublishRemainder(t *testing.T) {+ for name, bad := range map[string]proxmox.VMFileSystem{+ "overused": {TotalBytes: 9000, UsedBytes: 9001},+ "signed-overflow": {TotalBytes: uint64(math.MaxInt64) + 1, UsedBytes: 0},+ } {+ t.Run(name, func(t *testing.T) {+ bad.Mountpoint, bad.Type, bad.Disk = "/data", "ext4", "/dev/vdb1"+ peer := proxmox.VMFileSystem{Mountpoint: "/", Type: "ext4", Disk: "/dev/vda1", TotalBytes: 1000, UsedBytes: 0}+ c := &incompleteFilesystemClient{filesystems: []proxmox.VMFileSystem{peer, bad}}+ m := &Monitor{}+ res := proxmox.ClusterResource{Node: "node", VMID: 105, MaxDisk: 10000}+ total, used, free, usage, disks, fresh, reason := m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "partial", res, c, 10000, 0, -1)+ if total != 10000 || used != 0 || free != 10000 || usage != -1 || len(disks) != 0 || fresh || reason != "agent-error" {+ t.Fatalf("invalid peer became a complete current aggregate: %d/%d/%d usage=%v rows=%+v fresh=%t reason=%q", total, used, free, usage, disks, fresh, reason)+ }+ // A real subsequent complete inventory may remove the other volume.+ c.filesystems = []proxmox.VMFileSystem{peer}+ total, used, free, usage, disks, fresh, reason = m.updateVMDisksFromGuestAgentFSInfo(context.Background(), "partial", res, c, 10000, 0, -1)+ if total != 1000 || used != 0 || free != 1000 || usage != 0 || len(disks) != 1 || !fresh || reason != "" {+ t.Fatal("complete inventory removal/zero was treated as partial")+ }+ })+ }+}++// Ordinary QEMU-only collection must not turn a malformed large volume into+// recovery/removal merely because a smaller valid volume now reports zero.+// Exercise both collectors, canonical previous state, served JSON, History and+// genuine filesystem alerts. No guest freeze, command bypass or native probe.+func TestVMFilesystemIncompletePollingPreservesHistoryAndBreach(t *testing.T) {+ for _, path := range []string{"cluster", "node"} {+ t.Run(path, func(t *testing.T) {+ var phase, fsCalls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":1000,"mem":400,"meminfo":{"total":1000,"available":600}}}`)+ case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):+ fsCalls.Add(1)+ rows := `{"mountpoint":"/","type":"ext4","disk":[{"dev":"/dev/vda1"}],"total-bytes":1000,"used-bytes":980},{"mountpoint":"/data","type":"ext4","disk":[{"dev":"/dev/vdb1"}],"total-bytes":9000,"used-bytes":8820}`+ switch phase.Load() {+ case 1:+ rows = `{"mountpoint":"/","type":"ext4","disk":[{"dev":"/dev/vda1"}],"total-bytes":1000,"used-bytes":0},{"mountpoint":"/data","type":"ext4","total-bytes":9000}`+ case 2:+ rows = `{"mountpoint":"/","type":"ext4","disk":[{"dev":"/dev/vda1"}],"total-bytes":1000,"used-bytes":0}`+ }+ fmt.Fprintf(w, `{"data":{"result":[%s]}}`, rows)+ case strings.Contains(r.URL.Path, "/agent/"):+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ default:+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ m := guestHistoryObservationMonitor(t)+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(m.alertManager.Stop)+ cfg := m.alertManager.GetConfig()+ cfg.Enabled = true+ cfg.GuestDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}+ cfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"disk": 0}}+ m.alertManager.UpdateConfig(cfg)+ m.config, m.state, m.rateTracker = &config.Config{}, models.NewState(), NewRateTracker()+ m.guestMetadataLimiter = make(map[string]time.Time)+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 1000, Mem: 400, MaxDisk: 10000, CPU: .2}+ id := makeGuestID("partial-filesystems", "node", 105)+ publish := func(vm models.VM) {+ m.state.UpdateVMs([]models.VM{vm})+ m.updateResourceStore(m.currentStateWithScope())+ }+ build := func() models.VM {+ t.Helper()+ previous := m.previousGuestContextForInstance("partial-filesystems").vmsByID[id]+ var vm models.VM+ if path == "node" {+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "partial-filesystems", "node", []proxmox.VM{{VMID: 105, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, Mem: res.Mem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, client, map[string]models.VM{id: previous}, nil)+ if len(vms) != 1 {+ t.Fatal("node collector lost guest")+ }+ vm = vms[0]+ } else {+ var ok bool+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), "partial-filesystems", res, client, id, nil, &previous)+ if !ok {+ t.Fatal("cluster builder lost guest")+ }+ m.checkGuestAlertsForVM("partial-filesystems", vm)+ }+ publish(vm)+ m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))+ view := m.currentModeReadState().VMs()[0]+ if view.DiskStatusReason() != vm.DiskStatusReason || view.DiskObservation() != vm.DiskObservation || !reflect.DeepEqual(previousVMFromView(view).Disks, vm.Disks) {+ t.Fatalf("canonical projection changed completeness, original age or mount rows: vm=%+v view=%+v", vm, previousVMFromView(view))+ }+ if vm.Disk.Usage >= 0 && (view.DiskTotal() != vm.Disk.Total || view.DiskUsed() != vm.Disk.Used || view.DiskPercent() != vm.Disk.Usage) {+ t.Fatal("canonical projection changed a known disk tuple")+ }+ // Allocated capacity without usage is not a canonical metric.+ // Check absence on the wire below, not equality with raw allocation.+ if vm.Disk.Usage < 0 && (view.DiskTotal() != 0 || view.DiskUsed() != 0 || view.DiskPercent() != 0) {+ t.Fatal("canonical projection retained expired disk metrics")+ }+ front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())+ wire, err := json.Marshal(front.Resources)+ var served []struct {+ Proxmox unifiedresources.ProxmoxData+ CPU *models.ResourceMetricFrontend+ Memory *models.ResourceMetricFrontend+ Disk *models.ResourceMetricFrontend+ }+ if err != nil || json.Unmarshal(wire, &served) != nil || len(served) != 1 || served[0].Proxmox.DiskStatusReason != vm.DiskStatusReason || strings.Contains(string(wire), "diskObservation") {+ t.Fatalf("served completeness differs or private observation leaked: %s / %v", wire, err)+ }+ if served[0].CPU == nil || served[0].CPU.Current != 20 || served[0].Memory == nil || served[0].Memory.Current != 40 {+ t.Fatalf("independent current CPU/memory metrics disappeared: %s", wire)+ }+ metric := served[0].Disk+ if vm.Disk.Usage < 0 {+ if metric != nil {+ t.Fatal("unavailable disk reading became a measured wire zero")+ }+ } else if metric == nil || metric.Total == nil || metric.Used == nil || *metric.Total != vm.Disk.Total || *metric.Used != vm.Disk.Used || metric.Current != vm.Disk.Usage {+ t.Fatalf("known disk tuple or genuine measured zero lost on the wire: %s", wire)+ }+ return vm+ }+ initial := build()+ if initial.Disk.Total != 10000 || initial.Disk.Used != 9800 || initial.Disk.Usage != 98 || len(initial.Disks) != 2 || initial.DiskStatusReason != "" || initial.DiskObservation.ObservedAt.IsZero() {+ t.Fatalf("initial complete reading missing: %+v", initial)+ }+ active := m.alertManager.GetActiveAlerts()+ if len(active) != 2 {+ t.Fatalf("complete high readings did not establish two filesystem breaches: %+v", active)+ }+ breaches := make(map[string]time.Time)+ for _, alert := range active {+ breaches[alert.ID] = alert.StartTime+ }+ originalHistory := guestHistoryStoredPoints(t, m, "vm", id, "disk")+ if len(originalHistory) != 1 || originalHistory[0].Value != 98 {+ t.Fatal("complete reading did not establish History")+ }+ phase.Store(1)+ for i := 0; i < 3; i++ {+ retained := build()+ if retained.Disk != initial.Disk || !reflect.DeepEqual(retained.Disks, initial.Disks) || retained.DiskStatusReason != "prev-agent-error" || retained.DiskObservation != initial.DiskObservation {+ t.Fatalf("partial poll %d became fresh or removed an unknown volume: %+v", i, retained)+ }+ }+ aged := m.previousGuestContextForInstance("partial-filesystems").vmsByID[id]+ aged.DiskObservation.ObservedAt = time.Now().Add(-recentGuestAgentEvidenceMaxAge - time.Second)+ publish(aged)+ expired := build()+ if expired.Disk.Usage != -1 || expired.DiskStatusReason != "agent-error" || len(expired.Disks) != 0 || !expired.DiskObservation.ObservedAt.IsZero() {+ t.Fatal("repeated partial responses renewed expired complete evidence")+ }+ active = m.alertManager.GetActiveAlerts()+ if len(active) != 2 {+ t.Fatalf("partial/expired evidence cleared a genuine filesystem breach: %+v", active)+ }+ for _, alert := range active {+ if !alert.StartTime.Equal(breaches[alert.ID]) {+ t.Fatal("partial reading replaced a breach occurrence")+ }+ }+ if !reflect.DeepEqual(guestHistoryStoredPoints(t, m, "vm", id, "disk"), originalHistory) || len(m.metricsHistory.GetGuestMetrics(id, "disk", time.Hour)) != 1 || len(m.metricsHistory.GetGuestMetrics(id, "cpu", time.Hour)) != 5 || fsCalls.Load() != 5 {+ t.Fatal("partial evidence created disk History, stopped independent CPU History or retried a command")+ }+ // Complete ordinary replacement (the large volume really absent) and+ // measured zero must still remove/recover. History has second keys.+ time.Sleep(1100 * time.Millisecond)+ phase.Store(2)+ recovered := build()+ if recovered.Disk.Total != 1000 || recovered.Disk.Used != 0 || recovered.Disk.Usage != 0 || len(recovered.Disks) != 1 || recovered.DiskStatusReason != "" || !recovered.DiskObservation.ObservedAt.After(initial.DiskObservation.ObservedAt) || len(m.alertManager.GetActiveAlerts()) != 0 || fsCalls.Load() != 6 {+ t.Fatalf("complete ordinary removal/zero failed to recover: %+v", recovered)+ }+ points := guestHistoryStoredPoints(t, m, "vm", id, "disk")+ if len(points) != 2 || points[0] != originalHistory[0] || points[1].Value != 0 {+ t.Fatal("History lost high reading or failed to append genuine zero")+ }+ })+ }+}++func TestVMFilesystemIncompleteColdPollRemainsUnavailable(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):+ fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0},{"mountpoint":"/data","type":"ext4","total-bytes":9000}]}}`)+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ res := proxmox.ClusterResource{Node: "node", VMID: 105, MaxDisk: 10000}+ _, _, _, usage, disks, fresh, reason := (&Monitor{}).updateVMDisksFromGuestAgentFSInfo(context.Background(), "cold-partial", res, client, 10000, 0, -1)+ if usage != -1 || fresh || len(disks) != 0 || reason != "agent-error" {+ t.Fatalf("cold partial result became healthy usage: %v/%+v/%t/%q", usage, disks, fresh, reason)+ }+}diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex 37899bdbe8..0895e3b761 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -890,8 +890,10 @@ func (m *Monitor) updateVMDisksFromGuestAgentFSInfo(summary := m.summarizeVMFSInfo(instanceName, res, fsInfo)- // If we got valid data from guest agent, use it- if summary.totalBytes > 0 {+ // Valid peer rows are not complete guest usage if any reading was rejected.+ // The wire client reports that distinction as an error; alternate clients+ // can supply invalid counters directly, so enforce it at this boundary too.+ if summary.totalBytes > 0 && !summary.invalidBytes {// Sanity check: if the reported disk is way larger than allocated disk,// we might be getting host disk info somehowallocatedDiskGB := float64(res.MaxDisk) / 1073741824diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex 9d80472c11..ccca7b3c56 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -1706,13 +1706,21 @@ func (fs *VMFileSystem) UnmarshalJSON(data []byte) error {if !valid {return fmt.Errorf("guest filesystem record is ambiguous or malformed")}+ recognized := falsefor name := range fields {+ switch name {+ case "name", "type", "mountpoint", "total-bytes", "total-bytes-privileged", "used-bytes", "disk":+ recognized = true+ }for _, canonical := range []string{"total-bytes", "total-bytes-privileged", "used-bytes"} {if name != canonical && strings.EqualFold(name, canonical) {return fmt.Errorf("guest filesystem byte count has a noncanonical field name")}}}+ if !recognized {+ return fmt.Errorf("guest filesystem record has no recognized fields")+ }type rawVMFileSystem struct {Name string `json:"name"`Type string `json:"type"`@@ -2023,7 +2031,9 @@ func coerceString(value interface{}) string {}}-// GetVMFSInfo returns filesystem information from QEMU guest agent+// GetVMFSInfo returns filesystem information from QEMU guest agent. If any+// record is malformed, valid peers are returned with an error: they cannot+// establish complete guest capacity or removal of the rejected filesystem.func (c *Client) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFileSystem, error) {resp, err := c.get(ctx, fmt.Sprintf("/nodes/%s/qemu/%d/agent/get-fsinfo", node, vmid))if err != nil {@@ -2045,8 +2055,8 @@ func (c *Client) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFiStr("response", string(bodyBytes)).Msg("Raw response from guest agent get-fsinfo")- // Decode array payloads entry-by-entry so one malformed filesystem record- // does not wipe valid guest-agent disk data for the whole VM.+ // Decode entry-by-entry to retain valid peers, but never mistake that+ // remainder for a complete inventory after dropping a malformed record.var arrayResult struct {Data struct {Result []json.RawMessage `json:"result"`@@ -2054,9 +2064,11 @@ func (c *Client) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFi}if err := json.Unmarshal(bodyBytes, &arrayResult); err == nil && arrayResult.Data.Result != nil {filesystems := make([]VMFileSystem, 0, len(arrayResult.Data.Result))+ invalidEntries := 0for idx, rawFS := range arrayResult.Data.Result {var fs VMFileSystemif err := json.Unmarshal(rawFS, &fs); err != nil {+ invalidEntries++log.Warn().Err(err).Str("node", node).@@ -2068,6 +2080,9 @@ func (c *Client) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]VMFifilesystems = append(filesystems, fs)}postProcessVMFilesystems(node, vmid, filesystems)+ if invalidEntries > 0 {+ return filesystems, fmt.Errorf("guest filesystem inventory is incomplete: %d malformed entries", invalidEntries)+ }return filesystems, nil}diff --git a/pkg/proxmox/client_api_more3_test.go b/pkg/proxmox/client_api_more3_test.goindex 240dce4cef..db03647575 100644--- a/pkg/proxmox/client_api_more3_test.go+++ b/pkg/proxmox/client_api_more3_test.go@@ -261,8 +261,8 @@ func TestClientVMFSInfoSkipsMalformedEntries(t *testing.T) {ctx := context.Background()filesystems, err := client.GetVMFSInfo(ctx, "node1", 100)- if err != nil {- t.Fatalf("GetVMFSInfo error: %v", err)+ if err == nil {+ t.Fatal("malformed peer must prevent a complete filesystem inventory")}if len(filesystems) != 1 {t.Fatalf("expected malformed entry to be skipped, got %+v", filesystems)diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.goindex fde1cffbec..b30569170c 100644--- a/pkg/proxmox/cluster_client.go+++ b/pkg/proxmox/cluster_client.go@@ -1466,11 +1466,10 @@ func (cc *ClusterClient) GetVMFSInfo(ctx context.Context, node string, vmid int)var result []VMFileSystemerr := cc.executeGuestAgent(ctx, func(client *Client) error {info, err := client.GetVMFSInfo(ctx, node, vmid)- if err != nil {- return err- }+ // Preserve valid peers and the incomplete-inventory error together.+ // Neither a partial response nor its error grants another endpoint read.result = info- return nil+ return err})return result, err}diff --git a/pkg/proxmox/guest_filesystem_bytes_test.go b/pkg/proxmox/guest_filesystem_bytes_test.goindex c0aa9c4379..82f9b17340 100644--- a/pkg/proxmox/guest_filesystem_bytes_test.go+++ b/pkg/proxmox/guest_filesystem_bytes_test.go@@ -89,7 +89,7 @@ func TestVMFilesystemBytesWireAdmissionKeepsValidPeersAndObjectPrecision(t *testtotal, used uint64wantError bool}{- "array-peers": {`[{"mountpoint":"/bad","type":"ext4","total-bytes":1000},{"mountpoint":"/negative","type":"ext4","total-bytes":1000,"used-bytes":-1},{"mountpoint":"/overfull","type":"ext4","total-bytes":1000,"used-bytes":1001},{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0}]`, 1000, 0, false},+ "array-peers": {`[{"mountpoint":"/bad","type":"ext4","total-bytes":1000},{"mountpoint":"/negative","type":"ext4","total-bytes":1000,"used-bytes":-1},{"mountpoint":"/overfull","type":"ext4","total-bytes":1000,"used-bytes":1001},{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0}]`, 1000, 0, true},"object-precision": {`{"mountpoint":"C:\\","type":"ntfs","total-bytes":9007199254740993,"used-bytes":9007199254740991,"disk":{"bus-type":"scsi","target":2}}`, 9007199254740993, 9007199254740991, false},"object-duplicate": {`{"mountpoint":"C:\\","type":"ntfs","total-bytes":1000,"used-bytes":1001,"used-bytes":0}`, 0, 0, true},"object-missing": {`{"mountpoint":"C:\\","type":"ntfs","total-bytes":1000}`, 0, 0, true},@@ -112,9 +112,16 @@ func TestVMFilesystemBytesWireAdmissionKeepsValidPeersAndObjectPrecision(t *testc := backupTestClient(t, server.URL)readings, err := c.GetVMFSInfo(context.Background(), "node", 105)if tc.wantError {- if err == nil || len(readings) != 0 {+ if err == nil {t.Fatalf("invalid object admitted: %+v %v", readings, err)}+ if name == "array-peers" {+ if len(readings) != 1 || readings[0].TotalBytes != tc.total || readings[0].UsedBytes != tc.used {+ t.Fatalf("valid peers lost with incomplete inventory error: %+v %v", readings, err)+ }+ } else if len(readings) != 0 {+ t.Fatalf("invalid object returned readings: %+v", readings)+ }} else {if err != nil || len(readings) != 1 {t.Fatalf("valid readings lost: %+v %v", readings, err)diff --git a/pkg/proxmox/guest_filesystem_completeness_test.go b/pkg/proxmox/guest_filesystem_completeness_test.gonew file mode 100644index 0000000000..0226d6a304--- /dev/null+++ b/pkg/proxmox/guest_filesystem_completeness_test.go@@ -0,0 +1,88 @@+package proxmox++import (+ "context"+ "errors"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++// A malformed row is not an absent volume. Keep the valid peers, but make the+// completeness failure visible through both ordinary client paths. This is a+// completed synthetic QGA reply, not a native freeze/recovery experiment.+func TestVMFilesystemIncompleteInventoryKeepsPeersAndSingleAttempt(t *testing.T) {+ for _, path := range []string{"single", "cluster"} {+ for name, bad := range map[string]string{+ "missing-used": `{"mountpoint":"/data","type":"ext4","total-bytes":9000}`,+ "overused": `{"mountpoint":"/data","type":"ext4","total-bytes":9000,"used-bytes":9001}`,+ "duplicate": `{"mountpoint":"/data","type":"ext4","total-bytes":9000,"used-bytes":8800,"used-bytes":0}`,+ "null-record": `null`,+ "scalar-record": `42`,+ "empty-record": `{}`,+ "unknown-record": `{"unknown":"value"}`,+ } {+ t.Run(path+"/"+name, func(t *testing.T) {+ var commands, configs atomic.Int32+ var corrected atomic.Bool+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[]}`)+ case strings.HasSuffix(r.URL.Path, "/config"):+ configs.Add(1)+ fmt.Fprint(w, `{"data":{}}`)+ case strings.Contains(r.URL.Path, "/agent/"):+ commands.Add(1)+ if strings.HasSuffix(r.URL.Path, "/get-fsinfo") {+ peer := `{"mountpoint":"/","type":"ext4","total-bytes":1000,"used-bytes":0}`+ if corrected.Load() {+ fmt.Fprintf(w, `{"data":{"result":[%s]}}`, peer)+ } else {+ fmt.Fprintf(w, `{"data":{"result":[%s,%s]}}`, bad, peer)+ }+ } else {+ backupAgentPayload(w, r)+ }+ default:+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ })+ a, b := httptest.NewServer(handler), httptest.NewServer(handler)+ defer a.Close()+ defer b.Close()+ var client interface {+ GetVMFSInfo(context.Context, string, int) ([]VMFileSystem, error)+ GetVMAgentInfo(context.Context, string, int) (map[string]interface{}, error)+ }+ if path == "cluster" {+ client = NewClusterClient("partial-filesystems", ClientConfig{Host: a.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}, []string{a.URL, b.URL}, nil)+ } else {+ client = backupTestClient(t, a.URL)+ }+ peers, err := client.GetVMFSInfo(context.Background(), "node", 105)+ if err == nil || errors.Is(err, ErrGuestAgentDeferred) || len(peers) != 1 || peers[0].TotalBytes != 1000 || peers[0].UsedBytes != 0 || peers[0].Disk != "root-filesystem" {+ t.Fatalf("partial inventory lost failure or valid peer: %+v / %v", peers, err)+ }+ if commands.Load() != 1 || configs.Load() != 2 {+ t.Fatalf("incomplete payload repeated/bypassed guest admission: commands=%d config=%d", commands.Load(), configs.Load())+ }+ // Parsing failure is not uncertain command completion. Another+ // ordinary metadata read remains eligible, not a forced probe.+ if _, err := client.GetVMAgentInfo(context.Background(), "node", 105); err != nil {+ t.Fatalf("completed malformed reading started a command cooldown: %v", err)+ }+ corrected.Store(true)+ peers, err = client.GetVMFSInfo(context.Background(), "migrated", 105)+ if err != nil || len(peers) != 1 || peers[0].UsedBytes != 0 || commands.Load() != 3 || configs.Load() != 6 {+ t.Fatalf("complete ordinary removal/zero did not recover once: %+v / %v / commands=%d config=%d", peers, err, commands.Load(), configs.Load())+ }+ })+ }+ }+}