Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep equal-sized Windows guest volumes in disk capacity
pulse-triage[bot]006f61d8b993
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 405c5a251d..4584f5700e 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,27 @@## Purpose+### QEMU Windows capacity is owned by the volume, not the physical drive++The guest filesystem aggregate uses the reported Windows volume GUID when+present, or a normalised drive/mount path without one. QGA `disk.dev` names+the backing `PhysicalDriveN`, so equal-sized partitions on that drive must not+be treated as one filesystem. Multiple mount paths for one GUID still count+capacity once; each admitted mount row and its original device metadata remain+visible. Non-Windows device/subvolume deduplication, byte-validation and the+signed aggregate-overflow fence are unchanged. These temporary capacity keys+are not hardware identity, guest identity or History keys.++`guest_filesystem_identity_test.go` checks separate equal-size volumes, alias+mounts, missing GUIDs, Linux bind/device peers and overflow. Its synthetic+QGA-only polling lifecycle carries complete capacity, volume removal and+explicit zero through the native VM builder, canonical read state and both+in-memory/persistent History. It establishes source acceptance, not native+Windows/PVE recovery or a diagnosis of issue #2619. The optional-read lifecycle fixture also carries the same captured mock-mode+scope through poll publication and broadcast after the mode-fence API change,+rather than recapturing an epoch for stale state. Guest commands, operation+locks, shared admission, retries, timeout and backup policy are unchanged.+### Sustained poll failures preserve the capped retry fenceScheduler circuit-breaker backoff saturates before duration arithmetic candiff --git a/internal/monitoring/guest_agent_optional_read_integration_test.go b/internal/monitoring/guest_agent_optional_read_integration_test.goindex c0178cf109..0aac05f645 100644--- a/internal/monitoring/guest_agent_optional_read_integration_test.go+++ b/internal/monitoring/guest_agent_optional_read_integration_test.go@@ -131,6 +131,7 @@ func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {}}build := func() models.VM {+ scope := m.mockModeFence.begin()vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "optional", res, client, id, nil, previous)if !ok {t.Fatal("guest disappeared")@@ -152,12 +153,12 @@ func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {// Read-only hydration may deliberately reuse a generation for// two seconds; it is not a replacement for poll publication.snapshot := models.StateSnapshot{VMs: []models.VM{vm}, LastUpdate: vm.LastSeen}- m.updateResourceStore(snapshot)+ m.updateResourceStore(snapshot, scope)live := m.GetUnifiedReadState().VMs()[0]if live.ID() != canonicalID || (knownDisk && live.DiskUsed() != vm.Disk.Used) || live.DiskStatusReason() != vm.DiskStatusReason {t.Fatal("accepted poll did not replace the production read generation")}- encoded, err := json.Marshal(m.buildBroadcastFrontendStateFromSnapshot(snapshot).Resources)+ encoded, err := json.Marshal(m.buildBroadcastFrontendStateFromSnapshot(snapshot, scope).Resources)if err != nil || !strings.Contains(string(encoded), canonicalID) {t.Fatal("guest identity missing from JSON read projection")}diff --git a/internal/monitoring/guest_filesystem_identity.go b/internal/monitoring/guest_filesystem_identity.gonew file mode 100644index 0000000000..f10d785ac1--- /dev/null+++ b/internal/monitoring/guest_filesystem_identity.go@@ -0,0 +1,26 @@+package monitoring++import (+ "strings"++ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// windowsVMFilesystemCapacityKey identifies a filesystem, not its backing+// physical disk. Windows QGA reports disk.dev as PhysicalDriveN for every+// partition on that drive. A volume GUID is stronger than its mount path;+// without one, keep distinct drive/mount paths rather than guess equivalence.+// This key is local to capacity aggregation, never hardware or History identity.+func windowsVMFilesystemCapacityKey(fs proxmox.VMFileSystem) string {+ name := strings.ToLower(strings.TrimSpace(fs.Name))+ if strings.HasPrefix(name, `\\?\volume{`) && strings.HasSuffix(strings.TrimRight(name, `\/`), "}") {+ return "volume:" + strings.TrimRight(name, `\/`)+ }+ mount := strings.TrimSpace(fs.Mountpoint)+ if len(mount) >= 2 && mount[1] == ':' &&+ ((mount[0] >= 'a' && mount[0] <= 'z') || (mount[0] >= 'A' && mount[0] <= 'Z')) &&+ (len(mount) == 2 || mount[2] == '\\' || mount[2] == '/') {+ return "mount:" + strings.TrimRight(strings.ToLower(strings.ReplaceAll(mount, "/", `\`)), `\`)+ }+ return ""+}diff --git a/internal/monitoring/guest_filesystem_identity_test.go b/internal/monitoring/guest_filesystem_identity_test.gonew file mode 100644index 0000000000..e477a00a40--- /dev/null+++ b/internal/monitoring/guest_filesystem_identity_test.go@@ -0,0 +1,159 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "math"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++const (+ windowsVolumeC = "\\\\?\\Volume{c65410ae-abf1-4829-8d8e-81d4b7581949}\\"+ windowsVolumeD = "\\\\?\\Volume{5743c199-8613-4953-94a2-574d75a27bfc}\\"+ windowsPhysicalDrive = "\\\\.\\PhysicalDrive0"+)++func TestVMFilesystemCapacityIdentity(t *testing.T) {+ windows := func(name, mount string, used uint64) proxmox.VMFileSystem {+ return proxmox.VMFileSystem{Name: name, Mountpoint: mount, Type: "NTFS", Disk: windowsPhysicalDrive, TotalBytes: 1000, UsedBytes: used}+ }+ linux := func(device, mount string) proxmox.VMFileSystem {+ return proxmox.VMFileSystem{Name: device, Mountpoint: mount, Type: "ext4", Disk: device, TotalBytes: 1000, UsedBytes: 400}+ }+ for _, tc := range []struct {+ name string+ fs []proxmox.VMFileSystem+ total, used uint64+ }{+ {"equal Windows partitions", []proxmox.VMFileSystem{windows(windowsVolumeC, "C:\\", 100), windows(windowsVolumeD, "D:\\", 900)}, 2000, 1000},+ {"no GUID distinct mount paths", []proxmox.VMFileSystem{windows("", "C:\\", 100), windows("", "D:\\", 900)}, 2000, 1000},+ {"one GUID multiple mount paths", []proxmox.VMFileSystem{windows(windowsVolumeC, "C:\\", 400), windows(strings.ToUpper(windowsVolumeC), "D:\\mount\\", 400)}, 1000, 400},+ {"same mount case slash aliases", []proxmox.VMFileSystem{windows("", "C:\\", 0), windows("", "c:/", 0)}, 1000, 0},+ {"Linux bind mount", []proxmox.VMFileSystem{linux("/dev/vda1", "/"), linux("/dev/vda1", "/bind")}, 1000, 400},+ {"Linux distinct devices", []proxmox.VMFileSystem{linux("/dev/vda1", "/"), linux("/dev/vda2", "/data")}, 2000, 800},+ {"Windows identity does not hide a Linux peer", []proxmox.VMFileSystem{windows(windowsVolumeC, "C:\\", 100), linux(windowsPhysicalDrive, "/data")}, 2000, 500},+ {"different GUID same mount label", []proxmox.VMFileSystem{windows(windowsVolumeC, "Data", 100), windows(windowsVolumeD, "Data", 900)}, 2000, 1000},+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := (&Monitor{}).summarizeVMFSInfo("fixture", proxmox.ClusterResource{}, tc.fs)+ if summary.totalBytes != tc.total || summary.usedBytes != tc.used || len(summary.individualDisks) != len(tc.fs) || summary.invalidBytes {+ t.Fatalf("filesystem identity changed capacity or visible mount rows: %+v, want %d/%d and %d rows", summary, tc.total, tc.used, len(tc.fs))+ }+ for i, disk := range summary.individualDisks {+ if disk.Mountpoint != tc.fs[i].Mountpoint || disk.Device != tc.fs[i].Disk || disk.Used != int64(tc.fs[i].UsedBytes) {+ t.Fatalf("capacity key leaked into row metadata: %+v", disk)+ }+ }+ })+ }+}++func TestVMFilesystemDistinctWindowsVolumesRetainOverflowFence(t *testing.T) {+ fs := []proxmox.VMFileSystem{+ {Name: windowsVolumeC, Mountpoint: "C:\\", Type: "NTFS", Disk: windowsPhysicalDrive, TotalBytes: math.MaxInt64},+ {Name: windowsVolumeD, Mountpoint: "D:\\", Type: "NTFS", Disk: windowsPhysicalDrive, TotalBytes: math.MaxInt64},+ }+ summary := (&Monitor{}).summarizeVMFSInfo("fixture", proxmox.ClusterResource{}, fs)+ if !summary.invalidBytes || summary.totalBytes != 0 || len(summary.individualDisks) != 0 {+ t.Fatalf("deduplication concealed an unrepresentable aggregate: %+v", summary)+ }+}++// Synthetic existing QGA wire shapes (including issue #1319's volume GUID and+// PhysicalDrive form), not a reproduction of #2619's native failure. No Pulse+// host agent participates: ordinary QEMU polling must publish every volume.+func TestVMFilesystemWindowsVolumePollingHistory(t *testing.T) {+ var poll, fsCalls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, "{\"data\":{}}")+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, "{\"data\":{\"status\":\"running\",\"agent\":1,\"maxmem\":1000,\"mem\":400,\"meminfo\":{\"total\":1000,\"available\":600}}}")+ case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):+ fsCalls.Add(1)+ used := 100+ if poll.Load() == 2 {+ used = 0+ }+ row := func(name, mount string, used int) map[string]any {+ return map[string]any{"name": name, "mountpoint": mount, "type": "NTFS", "total-bytes": 1000, "used-bytes": used, "disk": []any{map[string]any{"dev": windowsPhysicalDrive, "serial": "QM00005"}}}+ }+ rows := []any{row(windowsVolumeC, "C:\\", used), row(strings.ToUpper(windowsVolumeC), "C:\\mount\\", used)}+ if poll.Load() == 0 {+ rows = append(rows, row(windowsVolumeD, "D:\\", 900))+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"result": rows}})+ case strings.HasSuffix(r.URL.Path, "/agent/network-get-interfaces"):+ fmt.Fprint(w, "{\"data\":{\"result\":[]}}")+ case strings.HasSuffix(r.URL.Path, "/agent/get-osinfo"):+ fmt.Fprint(w, "{\"data\":{\"name\":\"Microsoft Windows Server 2022\",\"version\":\"fixture\"}}")+ case strings.HasSuffix(r.URL.Path, "/agent/info"):+ fmt.Fprint(w, "{\"data\":{\"result\":{\"version\":\"fixture\"}}}")+ default:+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})+ if err != nil {+ t.Fatal(err)+ }+ m := guestHistoryObservationMonitor(t)+ m.rateTracker = NewRateTracker()+ m.guestMetadataLimiter = make(map[string]time.Time)+ registry := unifiedresources.NewRegistry(nil)+ m.resourceStore = unifiedresources.NewMonitorAdapter(registry)+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "windows", VMID: 105, Status: "running", MaxMem: 1000, Mem: 400, MaxDisk: 2000}+ id := makeGuestID("volumes", "node", 105)+ var previous *models.VM+ for phase, want := range []struct {+ total, used int64+ usage float64+ rows int+ }{{2000, 1000, 50, 3}, {1000, 100, 10, 2}, {1000, 0, 0, 2}} {+ if phase > 0 {+ time.Sleep(1100 * time.Millisecond)+ }+ poll.Store(int32(phase))+ scope := m.mockModeFence.begin()+ vm, _, _, _, _, ok := m.buildVMFromClusterResource(context.Background(), "volumes", res, client, id, nil, previous)+ if !ok || vm.Disk.Total != want.total || vm.Disk.Used != want.used || vm.Disk.Free != want.total-want.used || vm.Disk.Usage != want.usage || len(vm.Disks) != want.rows || vm.DiskStatusReason != "" {+ t.Fatalf("poll %d did not publish complete current volume usage: %+v", phase, vm)+ }+ m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))+ m.updateResourceStore(models.StateSnapshot{VMs: []models.VM{vm}, LastUpdate: vm.LastSeen}, scope)+ views := m.GetUnifiedReadState().VMs()+ if len(views) != 1 || views[0].DiskTotal() != want.total || views[0].DiskUsed() != want.used || len(views[0].Disks()) != want.rows {+ t.Fatalf("poll %d lost complete volume capacity at the canonical read boundary", phase)+ }+ next := previousVMFromView(views[0])+ previous = &next+ }+ points := []float64{50, 10, 0}+ memory := m.metricsHistory.GetGuestMetrics(id, "disk", time.Hour)+ stored := guestHistoryStoredPoints(t, m, "vm", id, "disk")+ if len(memory) != len(points) || len(stored) != len(points) {+ t.Fatalf("volume lifecycle lost History: memory=%+v stored=%+v", memory, stored)+ }+ for i, want := range points {+ if memory[i].Value != want || stored[i].Value != want {+ t.Fatalf("History[%d] = %v/%v, want %v", i, memory[i].Value, stored[i].Value, want)+ }+ }+ if fsCalls.Load() != 3 {+ t.Fatalf("filesystem read was repeated or skipped: %d commands", fsCalls.Load())+ }+}diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.goindex e6b1f60c8a..e3ed5e553d 100644--- a/internal/monitoring/monitor_pve_guest_builders.go+++ b/internal/monitoring/monitor_pve_guest_builders.go@@ -687,8 +687,15 @@ func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterReso// In both cases, only count the device's capacity once.fsTypeLower := strings.ToLower(fs.Type)countThisFS := true- if fs.Disk != "" {- // Same device at multiple mount paths → count once+ if volumeKey := windowsVMFilesystemCapacityKey(fs); volumeKey != "" {+ // QGA's disk.dev is the backing physical drive on Windows,+ // not the volume. Equal-sized partitions must remain separate.+ // A GUID can still identify one volume mounted at several paths.+ dedupeKey := fmt.Sprintf("windows:%s:%d", volumeKey, fs.TotalBytes)+ countThisFS = !seenFilesystems[dedupeKey]+ seenFilesystems[dedupeKey] = true+ } else if fs.Disk != "" {+ // Same non-Windows device at multiple mount paths → count oncededupeKey := fmt.Sprintf("%s:%d", fs.Disk, fs.TotalBytes)if seenFilesystems[dedupeKey] {countThisFS = false