Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep real runtime data disks visible in filesystem monitoring
pulse-triage[bot]993e12c972ce
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 864ca64c3f..56dd5ea027 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -1153,6 +1153,26 @@ through QGA aggregation and both report/state/canonical-read paths, and hostusage alert onset/recovery. These synthetic controls do not establish nativeguest polling, installed agent recovery or completion of a reported incident.+### Runtime data directories retain real disk capacity++`/var/lib/containers` and `/var/lib/docker` can be independently mounted data+disks, not just container plumbing (#3444). The shared filesystem filter admits+their real backing mounts and persistent data volumes without an include+override. Only the runtime's specific layer trees remain path-excluded, with+directory-boundary matching; virtual, read-only and remote filesystem types+remain excluded everywhere. Existing device/volume deduplication still counts+aliases once and retains their own per-mount observations. Explicit operator+exclusions, guest-read admission, backup policy and backup exclusions do not+change.++`runtime_data_mount_test.go` covers eligible root/data mounts and filtered+layers/types. Host collector and report-boundary controls cover pre-usage+admission, state and canonical views. `TestVMRuntimeDataMountPolling` exercises+the real Proxmox HTTP client and both ordinary VM builders for ext4/xfs/btrfs,+near-full/zero readings, reordered aliases, canonical/served counters and+filesystem alert onset/recovery. These synthetic endpoints are not a native+guest probe, the reporter's actual fsinfo payload or installed recovery proof.+### QEMU Windows capacity is owned by the volume, not the physical driveThe guest filesystem aggregate uses the reported Windows volume GUID whendiff --git a/internal/hostmetrics/mount_boundary_test.go b/internal/hostmetrics/mount_boundary_test.goindex 44ec3c2d29..b93af19557 100644--- a/internal/hostmetrics/mount_boundary_test.go+++ b/internal/hostmetrics/mount_boundary_test.go@@ -20,6 +20,10 @@ func TestCollectDisksKeepsSystemNamedSiblingVolumes(t *testing.T) {{Device: "runtime", Mountpoint: "/run/worker", Fstype: "ext4"},{Device: "remote", Mountpoint: "/snapshots-remote", Fstype: "nfs4"},{Device: "/dev/vdd1", Mountpoint: "/snapshots-private", Fstype: "ext4"},+ {Device: "/dev/vde1", Mountpoint: "/var/lib/containers", Fstype: "btrfs"},+ {Device: "/dev/vdf1", Mountpoint: "/var/lib/docker", Fstype: "xfs"},+ {Device: "layer", Mountpoint: "/var/lib/docker/overlay2/layer/merged", Fstype: "ext4"},+ {Device: "podman-layer", Mountpoint: "/var/lib/containers/storage/overlay/layer/merged", Fstype: "xfs"},}, nil}var calls []string@@ -28,7 +32,7 @@ func TestCollectDisksKeepsSystemNamedSiblingVolumes(t *testing.T) {return &godisk.UsageStat{Total: 1000, Used: 950, Free: 50, UsedPercent: 95}, nil}disks := collectDisks(context.Background(), []string{"/snapshots-private"})- want := []string{"/snapshots", "/runtime", "/var/lib/docker-data"}+ want := []string{"/snapshots", "/runtime", "/var/lib/docker-data", "/var/lib/containers", "/var/lib/docker"}if !reflect.DeepEqual(calls, want) {t.Fatalf("usage probes = %v, want only eligible local volumes %v", calls, want)}diff --git a/internal/monitoring/filesystem_mount_boundary_test.go b/internal/monitoring/filesystem_mount_boundary_test.goindex 34fbc175bc..0b99177101 100644--- a/internal/monitoring/filesystem_mount_boundary_test.go+++ b/internal/monitoring/filesystem_mount_boundary_test.go@@ -50,9 +50,13 @@ func TestAgentReportsKeepSystemNamedSiblingVolumes(t *testing.T) {{"device":"/dev/vda1","mountpoint":"/snapshots","type":"ext4","totalBytes":1000,"usedBytes":950,"freeBytes":50,"usage":95},{"device":"/dev/vdb1","mountpoint":"/runtime","type":"xfs","totalBytes":1000,"usedBytes":400,"freeBytes":600,"usage":40},{"device":"/dev/vdc1","mountpoint":"/var/lib/docker-data","type":"ext4","totalBytes":1000,"usedBytes":700,"freeBytes":300,"usage":70},+ {"device":"/dev/vdd1","mountpoint":"/var/lib/containers","type":"btrfs","totalBytes":1000,"usedBytes":700,"freeBytes":300,"usage":70},+ {"device":"/dev/vde1","mountpoint":"/var/lib/docker","type":"xfs","totalBytes":1000,"usedBytes":500,"freeBytes":500,"usage":50},{"device":"image","mountpoint":"/snap/image","type":"squashfs","totalBytes":1000,"usedBytes":1000,"usage":100},{"device":"runtime","mountpoint":"/run/worker","type":"ext4","totalBytes":1000,"usedBytes":1000,"usage":100},- {"device":"remote","mountpoint":"/snapshots-remote","type":"nfs4","totalBytes":1000,"usedBytes":1000,"usage":100}+ {"device":"remote","mountpoint":"/snapshots-remote","type":"nfs4","totalBytes":1000,"usedBytes":1000,"usage":100},+ {"device":"layer","mountpoint":"/var/lib/docker/overlay2/layer/merged","type":"ext4","totalBytes":1000,"usedBytes":1000,"usage":100},+ {"device":"podman-layer","mountpoint":"/var/lib/containers/storage/overlay/layer/merged","type":"xfs","totalBytes":1000,"usedBytes":1000,"usage":100}]`), &disks); err != nil {t.Fatal(err)}@@ -101,10 +105,10 @@ func TestAgentReportsKeepSystemNamedSiblingVolumes(t *testing.T) {stored = m.state.GetSnapshot().DockerHosts[0].Diskscanonical = m.GetUnifiedReadState().DockerHosts()[0].Disks()}- if len(native) != 3 || len(stored) != 3 || len(canonical) != 3 {+ if len(native) != 5 || len(stored) != 5 || len(canonical) != 5 {t.Fatalf("local volumes missing at report/state/read boundaries: %v / %v / %v", native, stored, canonical)}- for i := 0; i < 3; i++ {+ for i := 0; i < 5; i++ {want := disks[i]if native[i].Mountpoint != want.Mountpoint || native[i].Total != want.TotalBytes || native[i].Used != want.UsedBytes || native[i].Usage != want.Usage ||stored[i] != native[i] || canonical[i].Mountpoint != want.Mountpoint || canonical[i].Total != want.TotalBytes || canonical[i].Used != want.UsedBytes || canonical[i].Usage != want.Usage {diff --git a/internal/monitoring/guest_runtime_data_mount_test.go b/internal/monitoring/guest_runtime_data_mount_test.gonew file mode 100644index 0000000000..455070e2f1--- /dev/null+++ b/internal/monitoring/guest_runtime_data_mount_test.go@@ -0,0 +1,184 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++const runtimeMountMiB = int64(1024 * 1024)++type runtimeDataMountFixture struct {+ monitor *Monitor+ client *proxmox.Client+ phase atomic.Int32+ fsCalls atomic.Int32+}++// Only synthetic, guest-local HTTP is used. No real guest, Pulse agent,+// backup, mount operation or guest-command safety setting participates.+func newRuntimeDataMountFixture(t *testing.T, fsType, mount string) *runtimeDataMountFixture {+ t.Helper()+ f := &runtimeDataMountFixture{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if r.Method != http.MethodGet {+ t.Errorf("unexpected modifying request: %s %s", r.Method, r.URL.Path)+ http.Error(w, "read-only fixture", http.StatusMethodNotAllowed)+ return+ }+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":1000,"mem":400,"meminfo":{"total":1000,"available":600}}}`)+ case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):+ f.fsCalls.Add(1)+ row := func(path, kind, device string, total, used int64) map[string]any {+ return map[string]any{"mountpoint": path, "type": kind, "disk": []any{map[string]any{"dev": device}}, "total-bytes": total * runtimeMountMiB, "used-bytes": used * runtimeMountMiB}+ }+ used, aliasUsed := int64(8820), int64(6000)+ phase := f.phase.Load()+ if phase == 2 {+ used, aliasUsed = 0, 0+ }+ rows := []any{+ row("/", "ext4", "/dev/vda1", 1000, 200),+ row(mount, fsType, "/dev/vdb1", 9000, used),+ row("/data-alias", fsType, "/dev/vdb1", 9000, aliasUsed),+ row("/var/lib/docker/overlay2/layer/merged", "ext4", "/dev/vda1", 1000, 1000),+ row("/var/lib/containers/storage/overlay/layer/merged", "xfs", "/dev/vda1", 1000, 1000),+ row(mount+"/virtual", "overlay", "overlay", 9000, 9000),+ row(mount+"/remote", "nfs4", "remote:/data", 9000, 9000),+ row(mount+"/memory", "tmpfs", "tmpfs", 9000, 9000),+ }+ if phase == 1 {+ rows[1], rows[2] = rows[2], rows[1]+ }+ _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"result": rows}})+ case strings.HasSuffix(r.URL.Path, "/agent/get-osinfo"), strings.HasSuffix(r.URL.Path, "/agent/info"):+ fmt.Fprint(w, `{"data":{"result":{}}}`)+ case strings.HasSuffix(r.URL.Path, "/agent/network-get-interfaces"):+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ default:+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(server.Close)+ var err error+ f.client, err = proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})+ if err != nil {+ t.Fatal(err)+ }+ m := newTestMonitor(t)+ m.config = &config.Config{}+ m.guestMetadataLimiter = make(map[string]time.Time)+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ cfg := m.alertManager.GetConfig()+ cfg.Enabled, cfg.ActivationState = true, alerts.ActivationActive+ cfg.GuestDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}+ cfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"disk": 0}}+ m.alertManager.UpdateConfig(cfg)+ f.monitor = m+ return f+}++func (f *runtimeDataMountFixture) poll(t *testing.T, collector string) models.VM {+ t.Helper()+ m := f.monitor+ id := makeGuestID("runtime-data", "node", 105)+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "podman-vm", VMID: 105, Status: "running", MaxMem: 1000, Mem: 400, MaxDisk: uint64(10000 * runtimeMountMiB), CPU: .2}+ previous := m.previousGuestContextForInstance("runtime-data").vmsByID[id]+ var vm models.VM+ if collector == "node" {+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "runtime-data", "node", []proxmox.VM{{VMID: 105, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, Mem: res.Mem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, f.client, map[string]models.VM{id: previous}, nil)+ if len(vms) != 1 {+ t.Fatal("node collector lost guest")+ }+ vm = vms[0]+ } else {+ var ok bool+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), "runtime-data", res, f.client, id, nil, &previous)+ if !ok {+ t.Fatal("cluster builder lost guest")+ }+ m.checkGuestAlertsForVM("runtime-data", vm)+ }+ m.state.UpdateVMs([]models.VM{vm})+ m.updateResourceStore(m.currentStateWithScope())+ return vm+}++func TestVMRuntimeDataMountPolling(t *testing.T) {+ for _, fsType := range []string{"ext4", "xfs", "btrfs"} {+ for _, mount := range []string{"/var/lib/containers", "/var/lib/docker"} {+ for _, collector := range []string{"cluster", "node"} {+ t.Run(fsType+mount+"/"+collector, func(t *testing.T) {+ f := newRuntimeDataMountFixture(t, fsType, mount)+ var breachID string+ for phase := 0; phase < 3; phase++ {+ f.phase.Store(int32(phase))+ vm := f.poll(t, collector)+ wantUsed := int64(9020 * runtimeMountMiB)+ wantMountUsage := float64(98)+ if phase == 2 {+ wantUsed = 200 * runtimeMountMiB+ wantMountUsage = 0+ }+ if vm.Disk.Total != 10000*runtimeMountMiB || vm.Disk.Used != wantUsed || vm.Disk.Free != vm.Disk.Total-wantUsed || len(vm.Disks) != 3 || vm.DiskStatusReason != "" || vm.DiskObservation.ObservedAt.IsZero() {+ t.Fatalf("real data disk omitted or alias counted twice: %+v", vm)+ }+ view := f.monitor.currentModeReadState().VMs()[0]+ if view.DiskTotal() != vm.Disk.Total || view.DiskUsed() != wantUsed || !reflect.DeepEqual(previousVMFromView(view).Disks, vm.Disks) {+ t.Fatal("canonical state lost data mount or changed observations")+ }+ front := f.monitor.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, f.monitor.mockModeFence.begin())+ wire, err := json.Marshal(front.Resources)+ var served []struct {+ Disk *models.ResourceMetricFrontend+ Proxmox struct{ Disks []unifiedresources.DiskInfo }+ }+ if err != nil || json.Unmarshal(wire, &served) != nil || len(served) != 1 || served[0].Disk == nil || served[0].Disk.Total == nil || *served[0].Disk.Total != vm.Disk.Total || served[0].Disk.Used == nil || *served[0].Disk.Used != wantUsed || len(served[0].Proxmox.Disks) != 3 {+ t.Fatalf("served disk differs from poll: %s / %v", wire, err)+ }+ for _, disk := range served[0].Proxmox.Disks {+ if disk.Mountpoint == mount && (disk.Device != "/dev/vdb1" || disk.Filesystem != fsType || disk.Usage != wantMountUsage) {+ t.Fatalf("data mount lost its own identity/usage: %+v", disk)+ }+ }+ active := f.monitor.alertManager.GetActiveAlerts()+ if phase < 2 {+ if len(active) != 1 || active[0].Value != 98 {+ t.Fatalf("real near-full mount failed ordinary alerting: %+v", active)+ }+ if phase == 0 {+ breachID = active[0].ID+ } else if active[0].ID != breachID {+ t.Fatal("alias reordering replaced the breach")+ }+ } else if len(active) != 0 {+ t.Fatal("real zero data usage failed to clear the alert")+ }+ }+ if f.fsCalls.Load() != 3 {+ t.Fatalf("unexpected filesystem read count: %d", f.fsCalls.Load())+ }+ })+ }+ }+ }+}diff --git a/pkg/fsfilters/filters.go b/pkg/fsfilters/filters.goindex 7023b480e9..2eca6900f0 100644--- a/pkg/fsfilters/filters.go+++ b/pkg/fsfilters/filters.go@@ -96,8 +96,6 @@ var specialMountPrefixes = []string{"/sys","/run","/var/run/", // FreeBSD: /var/run is not a symlink to /run- "/var/lib/docker",- "/var/lib/containers","/snap","/System/Volumes/", // macOS APFS system/data/preboot/update companion volumes"/Library/Developer/CoreSimulator/Volumes/", // Xcode simulator runtime disk images@@ -130,6 +128,25 @@ var containerPathPrefixes = []string{"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts}+// A runtime's data directory can be a separately mounted real disk (#3444).+// Exclude its layer trees, not the backing mount or persistent data volumes.+// Keep path checks as well as type checks: bind-mounted layers can report the+// underlying ext4/xfs/btrfs type instead of "overlay".+var containerLayerMountPrefixes = []string{+ "/var/lib/docker/overlay2",+ "/var/lib/docker/overlay",+ "/var/lib/docker/aufs",+ "/var/lib/docker/devicemapper/mnt",+ "/var/lib/docker/btrfs/subvolumes",+ "/var/lib/docker/zfs/graph",+ "/var/lib/docker/vfs/dir",+ "/var/lib/containers/storage/overlay",+ "/var/lib/containers/storage/overlay2",+ "/var/lib/containers/storage/btrfs/subvolumes",+ "/var/lib/containers/storage/zfs/graph",+ "/var/lib/containers/storage/vfs/dir",+}+// transientMountPatterns identify OS-managed mounts whose lifetime follows an// application launch rather than operator-managed storage. They must not// participate in capacity inventory or alerting.@@ -182,6 +199,14 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6reasons = append(reasons, "special-mountpoint")}+ for _, directory := range containerLayerMountPrefixes {+ if strings.HasPrefix(mountpoint, directory) &&+ (len(mountpoint) == len(directory) || mountpoint[len(directory)] == '/') {+ reasons = append(reasons, "container-overlay")+ break+ }+ }+for _, pattern := range transientMountPatterns {if strings.Contains(mountpoint, pattern) {reasons = append(reasons, "transient-mountpoint")diff --git a/pkg/fsfilters/mount_allocation_test.go b/pkg/fsfilters/mount_allocation_test.goindex e4956e32ff..142326c0f2 100644--- a/pkg/fsfilters/mount_allocation_test.go+++ b/pkg/fsfilters/mount_allocation_test.go@@ -6,7 +6,7 @@ import ()func TestEligibleFilesystemFilteringDoesNotAllocate(t *testing.T) {- paths := []string{"", "/", "/data", "C:\\", " /runtime "}+ paths := []string{"", "/", "/data", "C:\\", " /runtime ", "/var/lib/containers", "/var/lib/docker"}for _, prefix := range specialMountPrefixes {directory := strings.TrimSuffix(prefix, "/")// Exercise short and long mismatches, including the macOS prefixesdiff --git a/pkg/fsfilters/mount_boundary_test.go b/pkg/fsfilters/mount_boundary_test.goindex 5a92f1d288..65ee716f82 100644--- a/pkg/fsfilters/mount_boundary_test.go+++ b/pkg/fsfilters/mount_boundary_test.go@@ -9,8 +9,7 @@ func TestSystemMountFiltersRespectDirectoryBoundaries(t *testing.T) {// Include the roots as well as descendants. Paths that merely share a// directory's spelling must remain eligible, even when they are full.for _, directory := range []string{- "/dev", "/proc", "/sys", "/run", "/var/run", "/var/lib/docker",- "/var/lib/containers", "/snap", "/System/Volumes",+ "/dev", "/proc", "/sys", "/run", "/var/run", "/snap", "/System/Volumes","/Library/Developer/CoreSimulator/Volumes", "/Volumes/.timemachine",} {t.Run(directory, func(t *testing.T) {diff --git a/pkg/fsfilters/runtime_data_mount_test.go b/pkg/fsfilters/runtime_data_mount_test.gonew file mode 100644index 0000000000..7c97b12cc3--- /dev/null+++ b/pkg/fsfilters/runtime_data_mount_test.go@@ -0,0 +1,44 @@+package fsfilters++import "testing"++func TestRuntimeDataMountsRemainEligible(t *testing.T) {+ for _, fsType := range []string{"ext4", "xfs", "btrfs", "zfs", ""} {+ for _, mount := range []string{+ "/var/lib/containers", "/var/lib/containers/", "/var/lib/containers/storage",+ "/var/lib/containers/storage/volumes/data", "/var/lib/docker", "/var/lib/docker/",+ "/var/lib/docker/volumes/data/_data", "/var/lib/docker-data",+ "/var/lib/containers/storage/overlay-backups", "/var/lib/docker/overlay2-data",+ } {+ t.Run(fsType+mount, func(t *testing.T) {+ for _, used := range []uint64{0, 980, 1000} {+ if skip, reasons := ShouldSkipFilesystem(fsType, mount, 1000, used); skip {+ t.Fatalf("real runtime data disk hidden at %d/1000: %v", used, reasons)+ }+ }+ })+ }+ }+}++func TestRuntimeDataMountsStillExcludeLayersAndSpecialFilesystems(t *testing.T) {+ for _, mount := range []string{"/var/lib/containers", "/var/lib/docker"} {+ for _, fsType := range []string{"overlay", "overlayfs", "tmpfs", "squashfs", "nfs4", "cifs"} {+ if skip, _ := ShouldSkipFilesystem(fsType, mount, 1000, 980); !skip {+ t.Errorf("runtime directory bypassed %q filtering at %q", fsType, mount)+ }+ }+ }+ for _, directory := range containerLayerMountPrefixes {+ for _, fsType := range []string{"ext4", "xfs", "btrfs"} {+ for _, mount := range []string{directory, directory + "/", directory + "/layer/merged"} {+ if skip, _ := ShouldSkipFilesystem(fsType, mount, 1000, 980); !skip {+ t.Errorf("runtime layer admitted at %q (%s)", mount, fsType)+ }+ }+ if skip, reasons := ShouldSkipFilesystem(fsType, directory+"-data", 1000, 980); skip {+ t.Errorf("layer directory swallowed sibling data mount %q: %v", directory+"-data", reasons)+ }+ }+ }+}