Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep real runtime data disks visible in filesystem monitoring

pulse-triage[bot]993e12c972ce
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 864ca64c3f..56dd5ea027 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -1153,6 +1153,26 @@ through QGA aggregation and both report/state/canonical-read paths, and host
usage alert onset/recovery. These synthetic controls do not establish native
guest polling, installed agent recovery or completion of a reported incident.
+### Runtime data directories retain real disk capacity
+
+`/var/lib/containers` and `/var/lib/docker` can be independently mounted data
+disks, not just container plumbing (#3444). The shared filesystem filter admits
+their real backing mounts and persistent data volumes without an include
+override. Only the runtime's specific layer trees remain path-excluded, with
+directory-boundary matching; virtual, read-only and remote filesystem types
+remain excluded everywhere. Existing device/volume deduplication still counts
+aliases once and retains their own per-mount observations. Explicit operator
+exclusions, guest-read admission, backup policy and backup exclusions do not
+change.
+
+`runtime_data_mount_test.go` covers eligible root/data mounts and filtered
+layers/types. Host collector and report-boundary controls cover pre-usage
+admission, state and canonical views. `TestVMRuntimeDataMountPolling` exercises
+the real Proxmox HTTP client and both ordinary VM builders for ext4/xfs/btrfs,
+near-full/zero readings, reordered aliases, canonical/served counters and
+filesystem alert onset/recovery. These synthetic endpoints are not a native
+guest probe, the reporter's actual fsinfo payload or installed recovery proof.
+
### QEMU Windows capacity is owned by the volume, not the physical drive
The guest filesystem aggregate uses the reported Windows volume GUID when
diff --git a/internal/hostmetrics/mount_boundary_test.go b/internal/hostmetrics/mount_boundary_test.go
index 44ec3c2d29..b93af19557 100644
--- a/internal/hostmetrics/mount_boundary_test.go
+++ b/internal/hostmetrics/mount_boundary_test.go
@@ -20,6 +20,10 @@ func TestCollectDisksKeepsSystemNamedSiblingVolumes(t *testing.T) {
{Device: "runtime", Mountpoint: "/run/worker", Fstype: "ext4"},
{Device: "remote", Mountpoint: "/snapshots-remote", Fstype: "nfs4"},
{Device: "/dev/vdd1", Mountpoint: "/snapshots-private", Fstype: "ext4"},
+ {Device: "/dev/vde1", Mountpoint: "/var/lib/containers", Fstype: "btrfs"},
+ {Device: "/dev/vdf1", Mountpoint: "/var/lib/docker", Fstype: "xfs"},
+ {Device: "layer", Mountpoint: "/var/lib/docker/overlay2/layer/merged", Fstype: "ext4"},
+ {Device: "podman-layer", Mountpoint: "/var/lib/containers/storage/overlay/layer/merged", Fstype: "xfs"},
}, nil
}
var calls []string
@@ -28,7 +32,7 @@ func TestCollectDisksKeepsSystemNamedSiblingVolumes(t *testing.T) {
return &godisk.UsageStat{Total: 1000, Used: 950, Free: 50, UsedPercent: 95}, nil
}
disks := collectDisks(context.Background(), []string{"/snapshots-private"})
- want := []string{"/snapshots", "/runtime", "/var/lib/docker-data"}
+ want := []string{"/snapshots", "/runtime", "/var/lib/docker-data", "/var/lib/containers", "/var/lib/docker"}
if !reflect.DeepEqual(calls, want) {
t.Fatalf("usage probes = %v, want only eligible local volumes %v", calls, want)
}
diff --git a/internal/monitoring/filesystem_mount_boundary_test.go b/internal/monitoring/filesystem_mount_boundary_test.go
index 34fbc175bc..0b99177101 100644
--- a/internal/monitoring/filesystem_mount_boundary_test.go
+++ b/internal/monitoring/filesystem_mount_boundary_test.go
@@ -50,9 +50,13 @@ func TestAgentReportsKeepSystemNamedSiblingVolumes(t *testing.T) {
{"device":"/dev/vda1","mountpoint":"/snapshots","type":"ext4","totalBytes":1000,"usedBytes":950,"freeBytes":50,"usage":95},
{"device":"/dev/vdb1","mountpoint":"/runtime","type":"xfs","totalBytes":1000,"usedBytes":400,"freeBytes":600,"usage":40},
{"device":"/dev/vdc1","mountpoint":"/var/lib/docker-data","type":"ext4","totalBytes":1000,"usedBytes":700,"freeBytes":300,"usage":70},
+ {"device":"/dev/vdd1","mountpoint":"/var/lib/containers","type":"btrfs","totalBytes":1000,"usedBytes":700,"freeBytes":300,"usage":70},
+ {"device":"/dev/vde1","mountpoint":"/var/lib/docker","type":"xfs","totalBytes":1000,"usedBytes":500,"freeBytes":500,"usage":50},
{"device":"image","mountpoint":"/snap/image","type":"squashfs","totalBytes":1000,"usedBytes":1000,"usage":100},
{"device":"runtime","mountpoint":"/run/worker","type":"ext4","totalBytes":1000,"usedBytes":1000,"usage":100},
- {"device":"remote","mountpoint":"/snapshots-remote","type":"nfs4","totalBytes":1000,"usedBytes":1000,"usage":100}
+ {"device":"remote","mountpoint":"/snapshots-remote","type":"nfs4","totalBytes":1000,"usedBytes":1000,"usage":100},
+ {"device":"layer","mountpoint":"/var/lib/docker/overlay2/layer/merged","type":"ext4","totalBytes":1000,"usedBytes":1000,"usage":100},
+ {"device":"podman-layer","mountpoint":"/var/lib/containers/storage/overlay/layer/merged","type":"xfs","totalBytes":1000,"usedBytes":1000,"usage":100}
]`), &disks); err != nil {
t.Fatal(err)
}
@@ -101,10 +105,10 @@ func TestAgentReportsKeepSystemNamedSiblingVolumes(t *testing.T) {
stored = m.state.GetSnapshot().DockerHosts[0].Disks
canonical = m.GetUnifiedReadState().DockerHosts()[0].Disks()
}
- if len(native) != 3 || len(stored) != 3 || len(canonical) != 3 {
+ if len(native) != 5 || len(stored) != 5 || len(canonical) != 5 {
t.Fatalf("local volumes missing at report/state/read boundaries: %v / %v / %v", native, stored, canonical)
}
- for i := 0; i < 3; i++ {
+ for i := 0; i < 5; i++ {
want := disks[i]
if native[i].Mountpoint != want.Mountpoint || native[i].Total != want.TotalBytes || native[i].Used != want.UsedBytes || native[i].Usage != want.Usage ||
stored[i] != native[i] || canonical[i].Mountpoint != want.Mountpoint || canonical[i].Total != want.TotalBytes || canonical[i].Used != want.UsedBytes || canonical[i].Usage != want.Usage {
diff --git a/internal/monitoring/guest_runtime_data_mount_test.go b/internal/monitoring/guest_runtime_data_mount_test.go
new file mode 100644
index 0000000000..455070e2f1
--- /dev/null
+++ b/internal/monitoring/guest_runtime_data_mount_test.go
@@ -0,0 +1,184 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+const runtimeMountMiB = int64(1024 * 1024)
+
+type runtimeDataMountFixture struct {
+ monitor *Monitor
+ client *proxmox.Client
+ phase atomic.Int32
+ fsCalls atomic.Int32
+}
+
+// Only synthetic, guest-local HTTP is used. No real guest, Pulse agent,
+// backup, mount operation or guest-command safety setting participates.
+func newRuntimeDataMountFixture(t *testing.T, fsType, mount string) *runtimeDataMountFixture {
+ t.Helper()
+ f := &runtimeDataMountFixture{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.Method != http.MethodGet {
+ t.Errorf("unexpected modifying request: %s %s", r.Method, r.URL.Path)
+ http.Error(w, "read-only fixture", http.StatusMethodNotAllowed)
+ return
+ }
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":1000,"mem":400,"meminfo":{"total":1000,"available":600}}}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):
+ f.fsCalls.Add(1)
+ row := func(path, kind, device string, total, used int64) map[string]any {
+ return map[string]any{"mountpoint": path, "type": kind, "disk": []any{map[string]any{"dev": device}}, "total-bytes": total * runtimeMountMiB, "used-bytes": used * runtimeMountMiB}
+ }
+ used, aliasUsed := int64(8820), int64(6000)
+ phase := f.phase.Load()
+ if phase == 2 {
+ used, aliasUsed = 0, 0
+ }
+ rows := []any{
+ row("/", "ext4", "/dev/vda1", 1000, 200),
+ row(mount, fsType, "/dev/vdb1", 9000, used),
+ row("/data-alias", fsType, "/dev/vdb1", 9000, aliasUsed),
+ row("/var/lib/docker/overlay2/layer/merged", "ext4", "/dev/vda1", 1000, 1000),
+ row("/var/lib/containers/storage/overlay/layer/merged", "xfs", "/dev/vda1", 1000, 1000),
+ row(mount+"/virtual", "overlay", "overlay", 9000, 9000),
+ row(mount+"/remote", "nfs4", "remote:/data", 9000, 9000),
+ row(mount+"/memory", "tmpfs", "tmpfs", 9000, 9000),
+ }
+ if phase == 1 {
+ rows[1], rows[2] = rows[2], rows[1]
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"result": rows}})
+ case strings.HasSuffix(r.URL.Path, "/agent/get-osinfo"), strings.HasSuffix(r.URL.Path, "/agent/info"):
+ fmt.Fprint(w, `{"data":{"result":{}}}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/network-get-interfaces"):
+ fmt.Fprint(w, `{"data":{"result":[]}}`)
+ default:
+ t.Errorf("unexpected request: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(server.Close)
+ var err error
+ f.client, err = proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := newTestMonitor(t)
+ m.config = &config.Config{}
+ m.guestMetadataLimiter = make(map[string]time.Time)
+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ cfg := m.alertManager.GetConfig()
+ cfg.Enabled, cfg.ActivationState = true, alerts.ActivationActive
+ cfg.GuestDefaults.Disk = &alerts.HysteresisThreshold{Trigger: 90, Clear: 85}
+ cfg.MetricTimeThresholds = map[string]map[string]int{"guest": {"disk": 0}}
+ m.alertManager.UpdateConfig(cfg)
+ f.monitor = m
+ return f
+}
+
+func (f *runtimeDataMountFixture) poll(t *testing.T, collector string) models.VM {
+ t.Helper()
+ m := f.monitor
+ id := makeGuestID("runtime-data", "node", 105)
+ res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "podman-vm", VMID: 105, Status: "running", MaxMem: 1000, Mem: 400, MaxDisk: uint64(10000 * runtimeMountMiB), CPU: .2}
+ previous := m.previousGuestContextForInstance("runtime-data").vmsByID[id]
+ var vm models.VM
+ if collector == "node" {
+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "runtime-data", "node", []proxmox.VM{{VMID: 105, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, Mem: res.Mem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, f.client, map[string]models.VM{id: previous}, nil)
+ if len(vms) != 1 {
+ t.Fatal("node collector lost guest")
+ }
+ vm = vms[0]
+ } else {
+ var ok bool
+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), "runtime-data", res, f.client, id, nil, &previous)
+ if !ok {
+ t.Fatal("cluster builder lost guest")
+ }
+ m.checkGuestAlertsForVM("runtime-data", vm)
+ }
+ m.state.UpdateVMs([]models.VM{vm})
+ m.updateResourceStore(m.currentStateWithScope())
+ return vm
+}
+
+func TestVMRuntimeDataMountPolling(t *testing.T) {
+ for _, fsType := range []string{"ext4", "xfs", "btrfs"} {
+ for _, mount := range []string{"/var/lib/containers", "/var/lib/docker"} {
+ for _, collector := range []string{"cluster", "node"} {
+ t.Run(fsType+mount+"/"+collector, func(t *testing.T) {
+ f := newRuntimeDataMountFixture(t, fsType, mount)
+ var breachID string
+ for phase := 0; phase < 3; phase++ {
+ f.phase.Store(int32(phase))
+ vm := f.poll(t, collector)
+ wantUsed := int64(9020 * runtimeMountMiB)
+ wantMountUsage := float64(98)
+ if phase == 2 {
+ wantUsed = 200 * runtimeMountMiB
+ wantMountUsage = 0
+ }
+ if vm.Disk.Total != 10000*runtimeMountMiB || vm.Disk.Used != wantUsed || vm.Disk.Free != vm.Disk.Total-wantUsed || len(vm.Disks) != 3 || vm.DiskStatusReason != "" || vm.DiskObservation.ObservedAt.IsZero() {
+ t.Fatalf("real data disk omitted or alias counted twice: %+v", vm)
+ }
+ view := f.monitor.currentModeReadState().VMs()[0]
+ if view.DiskTotal() != vm.Disk.Total || view.DiskUsed() != wantUsed || !reflect.DeepEqual(previousVMFromView(view).Disks, vm.Disks) {
+ t.Fatal("canonical state lost data mount or changed observations")
+ }
+ front := f.monitor.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, f.monitor.mockModeFence.begin())
+ wire, err := json.Marshal(front.Resources)
+ var served []struct {
+ Disk *models.ResourceMetricFrontend
+ Proxmox struct{ Disks []unifiedresources.DiskInfo }
+ }
+ if err != nil || json.Unmarshal(wire, &served) != nil || len(served) != 1 || served[0].Disk == nil || served[0].Disk.Total == nil || *served[0].Disk.Total != vm.Disk.Total || served[0].Disk.Used == nil || *served[0].Disk.Used != wantUsed || len(served[0].Proxmox.Disks) != 3 {
+ t.Fatalf("served disk differs from poll: %s / %v", wire, err)
+ }
+ for _, disk := range served[0].Proxmox.Disks {
+ if disk.Mountpoint == mount && (disk.Device != "/dev/vdb1" || disk.Filesystem != fsType || disk.Usage != wantMountUsage) {
+ t.Fatalf("data mount lost its own identity/usage: %+v", disk)
+ }
+ }
+ active := f.monitor.alertManager.GetActiveAlerts()
+ if phase < 2 {
+ if len(active) != 1 || active[0].Value != 98 {
+ t.Fatalf("real near-full mount failed ordinary alerting: %+v", active)
+ }
+ if phase == 0 {
+ breachID = active[0].ID
+ } else if active[0].ID != breachID {
+ t.Fatal("alias reordering replaced the breach")
+ }
+ } else if len(active) != 0 {
+ t.Fatal("real zero data usage failed to clear the alert")
+ }
+ }
+ if f.fsCalls.Load() != 3 {
+ t.Fatalf("unexpected filesystem read count: %d", f.fsCalls.Load())
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/pkg/fsfilters/filters.go b/pkg/fsfilters/filters.go
index 7023b480e9..2eca6900f0 100644
--- a/pkg/fsfilters/filters.go
+++ b/pkg/fsfilters/filters.go
@@ -96,8 +96,6 @@ var specialMountPrefixes = []string{
"/sys",
"/run",
"/var/run/", // FreeBSD: /var/run is not a symlink to /run
- "/var/lib/docker",
- "/var/lib/containers",
"/snap",
"/System/Volumes/", // macOS APFS system/data/preboot/update companion volumes
"/Library/Developer/CoreSimulator/Volumes/", // Xcode simulator runtime disk images
@@ -130,6 +128,25 @@ var containerPathPrefixes = []string{
"/mnt/.ix-apps/docker/", // TrueNAS SCALE Docker overlay mounts
}
+// A runtime's data directory can be a separately mounted real disk (#3444).
+// Exclude its layer trees, not the backing mount or persistent data volumes.
+// Keep path checks as well as type checks: bind-mounted layers can report the
+// underlying ext4/xfs/btrfs type instead of "overlay".
+var containerLayerMountPrefixes = []string{
+ "/var/lib/docker/overlay2",
+ "/var/lib/docker/overlay",
+ "/var/lib/docker/aufs",
+ "/var/lib/docker/devicemapper/mnt",
+ "/var/lib/docker/btrfs/subvolumes",
+ "/var/lib/docker/zfs/graph",
+ "/var/lib/docker/vfs/dir",
+ "/var/lib/containers/storage/overlay",
+ "/var/lib/containers/storage/overlay2",
+ "/var/lib/containers/storage/btrfs/subvolumes",
+ "/var/lib/containers/storage/zfs/graph",
+ "/var/lib/containers/storage/vfs/dir",
+}
+
// transientMountPatterns identify OS-managed mounts whose lifetime follows an
// application launch rather than operator-managed storage. They must not
// participate in capacity inventory or alerting.
@@ -182,6 +199,14 @@ func ShouldSkipFilesystem(fsType, mountpoint string, totalBytes, usedBytes uint6
reasons = append(reasons, "special-mountpoint")
}
+ for _, directory := range containerLayerMountPrefixes {
+ if strings.HasPrefix(mountpoint, directory) &&
+ (len(mountpoint) == len(directory) || mountpoint[len(directory)] == '/') {
+ reasons = append(reasons, "container-overlay")
+ break
+ }
+ }
+
for _, pattern := range transientMountPatterns {
if strings.Contains(mountpoint, pattern) {
reasons = append(reasons, "transient-mountpoint")
diff --git a/pkg/fsfilters/mount_allocation_test.go b/pkg/fsfilters/mount_allocation_test.go
index e4956e32ff..142326c0f2 100644
--- a/pkg/fsfilters/mount_allocation_test.go
+++ b/pkg/fsfilters/mount_allocation_test.go
@@ -6,7 +6,7 @@ import (
)
func TestEligibleFilesystemFilteringDoesNotAllocate(t *testing.T) {
- paths := []string{"", "/", "/data", "C:\\", " /runtime "}
+ paths := []string{"", "/", "/data", "C:\\", " /runtime ", "/var/lib/containers", "/var/lib/docker"}
for _, prefix := range specialMountPrefixes {
directory := strings.TrimSuffix(prefix, "/")
// Exercise short and long mismatches, including the macOS prefixes
diff --git a/pkg/fsfilters/mount_boundary_test.go b/pkg/fsfilters/mount_boundary_test.go
index 5a92f1d288..65ee716f82 100644
--- a/pkg/fsfilters/mount_boundary_test.go
+++ b/pkg/fsfilters/mount_boundary_test.go
@@ -9,8 +9,7 @@ func TestSystemMountFiltersRespectDirectoryBoundaries(t *testing.T) {
// Include the roots as well as descendants. Paths that merely share a
// directory's spelling must remain eligible, even when they are full.
for _, directory := range []string{
- "/dev", "/proc", "/sys", "/run", "/var/run", "/var/lib/docker",
- "/var/lib/containers", "/snap", "/System/Volumes",
+ "/dev", "/proc", "/sys", "/run", "/var/run", "/snap", "/System/Volumes",
"/Library/Developer/CoreSimulator/Volumes", "/Volumes/.timemachine",
} {
t.Run(directory, func(t *testing.T) {
diff --git a/pkg/fsfilters/runtime_data_mount_test.go b/pkg/fsfilters/runtime_data_mount_test.go
new file mode 100644
index 0000000000..7c97b12cc3
--- /dev/null
+++ b/pkg/fsfilters/runtime_data_mount_test.go
@@ -0,0 +1,44 @@
+package fsfilters
+
+import "testing"
+
+func TestRuntimeDataMountsRemainEligible(t *testing.T) {
+ for _, fsType := range []string{"ext4", "xfs", "btrfs", "zfs", ""} {
+ for _, mount := range []string{
+ "/var/lib/containers", "/var/lib/containers/", "/var/lib/containers/storage",
+ "/var/lib/containers/storage/volumes/data", "/var/lib/docker", "/var/lib/docker/",
+ "/var/lib/docker/volumes/data/_data", "/var/lib/docker-data",
+ "/var/lib/containers/storage/overlay-backups", "/var/lib/docker/overlay2-data",
+ } {
+ t.Run(fsType+mount, func(t *testing.T) {
+ for _, used := range []uint64{0, 980, 1000} {
+ if skip, reasons := ShouldSkipFilesystem(fsType, mount, 1000, used); skip {
+ t.Fatalf("real runtime data disk hidden at %d/1000: %v", used, reasons)
+ }
+ }
+ })
+ }
+ }
+}
+
+func TestRuntimeDataMountsStillExcludeLayersAndSpecialFilesystems(t *testing.T) {
+ for _, mount := range []string{"/var/lib/containers", "/var/lib/docker"} {
+ for _, fsType := range []string{"overlay", "overlayfs", "tmpfs", "squashfs", "nfs4", "cifs"} {
+ if skip, _ := ShouldSkipFilesystem(fsType, mount, 1000, 980); !skip {
+ t.Errorf("runtime directory bypassed %q filtering at %q", fsType, mount)
+ }
+ }
+ }
+ for _, directory := range containerLayerMountPrefixes {
+ for _, fsType := range []string{"ext4", "xfs", "btrfs"} {
+ for _, mount := range []string{directory, directory + "/", directory + "/layer/merged"} {
+ if skip, _ := ShouldSkipFilesystem(fsType, mount, 1000, 980); !skip {
+ t.Errorf("runtime layer admitted at %q (%s)", mount, fsType)
+ }
+ }
+ if skip, reasons := ShouldSkipFilesystem(fsType, directory+"-data", 1000, 980); skip {
+ t.Errorf("layer directory swallowed sibling data mount %q: %v", directory+"-data", reasons)
+ }
+ }
+ }
+}