Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Inspect LXC init before wide cgroup siblings exhaust discovery

pulse-triage[bot]e1bba156e39e
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index ea001419d8..f67e30f7fb 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -620,6 +620,17 @@ admissible, but discovery must inspect the rest of the bounded config listing:
a 129th running guest or an unestablished later guest makes discovery unavailable,
not a truncated complete inventory. The existing bounded `pct list` fallback
then either establishes the full node or remains degraded under PrivateNetwork.
+The init search checks the preferred `ns/init.scope` path before unrelated
+workload cgroups, even when either level has hundreds of siblings. Its bounded
+priority frontier permits at most 64 cgroup process reads and 4,096 candidate
+PIDs; reaching either limit without a verified init remains unknown, not stopped.
+Unreadable or cancelled traversal still fails closed. Width alone cannot prevent
+a verified init from supporting the existing pinned filesystem observation.
+`TestHelperProxmoxLXCInitSearchPrioritizesBoundedReads` connects wide-tree discovery
+to complete measured collection without IPC commands;
+`TestProxmoxLXCInitSearchKeepsBoundsAndIdentity` pins exact directory/PID limits,
+foreign VM and nested namespace rejection, unreadable input and cancellation.
+These are synthetic source controls, not installed helper or native PVE recovery.
`TestSocketFreeProxmoxLXCDiscoveryCannotTruncateCompleteInventory` covers 127/128/129
running guests and stopped/unknown guests after the limit through discovery and
collection. The fixed `lxc-info -n <vmid> -p`
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index d9105c18bf..fc631be6bf 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -973,7 +973,15 @@ the existing bounded fallback; a stopped later guest does not invalidate 128
established guests. This preserves the consumer's distinction between complete
inventory replacement and explicit partial omissions. The producer boundary is
covered by `TestSocketFreeProxmoxLXCDiscoveryCannotTruncateCompleteInventory`;
-partial-wire, omitted-reading and complete-empty cache tests remain authoritative. An `lxc` request names the decimal VMID; the pinned process
+partial-wire, omitted-reading and complete-empty cache tests remain authoritative.
+Wide cgroup trees do not by themselves disqualify a verified init: the producer
+inspects `ns/init.scope` before unrelated workload siblings using at most 64
+cgroup process reads and 4,096 candidate PIDs. Exhausted, unreadable, cancelled
+or identity-invalid searches remain unknown/degraded; no omitted guest becomes
+healthy empty telemetry. The wide-tree helper collection and search-boundary
+controls in `internal/hostagent/privilege_helper_client_test.go` preserve measured
+capacity and exact identity without IPC commands. They are not native recovery.
+An `lxc` request names the decimal VMID; the pinned process
must sit in that guest's exact `lxc/<vmid>` cgroup and be its init (`NSpid` is
exactly `<pid> 1`, so the PID 1 of a namespace nested inside the guest, such
as Docker in LXC, cannot stand in for it), checked before and after the read.
diff --git a/internal/hostagent/privilege_helper_client_test.go b/internal/hostagent/privilege_helper_client_test.go
index d70155212a..97b4cb7367 100644
--- a/internal/hostagent/privilege_helper_client_test.go
+++ b/internal/hostagent/privilege_helper_client_test.go
@@ -9,6 +9,7 @@ import (
"io"
"net"
"os/exec"
+ "path"
"path/filepath"
"strconv"
"strings"
@@ -660,3 +661,137 @@ func TestSocketFreeProxmoxLXCDiscoveryCannotTruncateCompleteInventory(t *testing
})
}
}
+
+// A wide delegated cgroup tree must not prevent the helper from inspecting
+// the exact init it already prioritises. All observations here are synthetic;
+// the collector's real identity/prober contract and PrivateNetwork stay intact.
+func TestHelperProxmoxLXCInitSearchPrioritizesBoundedReads(t *testing.T) {
+ for _, wideHost := range []bool{false, true} {
+ name := "wide_payload"
+ if wideHost {
+ name = "wide_host_and_payload"
+ }
+ t.Run(name, func(t *testing.T) {
+ h := newFakeProxmoxLXCHost()
+ h.config(126, "hostname: busy-host\nrootfs: local:vm-126-disk-0,size=1G\n")
+ h.cgroup("126/ns/init.scope", 4000)
+ h.proc(4000, "4000\t1", "/lxc/126/ns/init.scope")
+ for n := 0; n < 4*proxmoxLXCMaxCgroupDirs; n++ {
+ h.cgroup(fmt.Sprintf("126/ns/workload%03d", n))
+ if wideHost {
+ h.cgroup(fmt.Sprintf("126/auxiliary%03d", n))
+ }
+ }
+ h.observations[4000] = map[string]lxcObservation{"/": lxcUsage(4096, 1024, 3072)}
+ collector, commands := h.collector(t)
+ readFile := collector.readFileFn
+ var examined []string
+ collector.readFileFn = func(name string) ([]byte, error) {
+ if strings.HasPrefix(name, proxmoxLXCCgroupRoot+"/") && strings.HasSuffix(name, "/cgroup.procs") {
+ examined = append(examined, name)
+ }
+ return readFile(name)
+ }
+ result := (&Agent{logger: zerolog.Nop(), collector: collector}).collectProxmoxLXCFilesystemsResult(t.Context())
+ if !result.Applicable || result.Degraded || result.Inventory == nil || result.Inventory.Status != "complete" || result.Inventory.ValidateCollection() != nil || *commands != 0 {
+ t.Fatalf("wide-tree helper collection = %+v; command attempts=%d", result, *commands)
+ }
+ rows := result.Inventory.Containers
+ if len(rows) != 1 || rows[0].VMID != 126 || rows[0].Name != "busy-host" || len(rows[0].Disks) != 1 || rows[0].Disks[0].UsedBytes != 1024 {
+ t.Fatalf("wide-tree source/name/capacity lost: %+v", rows)
+ }
+ want := []string{
+ path.Join(proxmoxLXCCgroupRoot, "126/cgroup.procs"),
+ path.Join(proxmoxLXCCgroupRoot, "126/ns/cgroup.procs"),
+ path.Join(proxmoxLXCCgroupRoot, "126/ns/init.scope/cgroup.procs"),
+ }
+ if fmt.Sprint(examined) != fmt.Sprint(want) {
+ t.Fatalf("examined cgroups=%v, want only %v", examined, want)
+ }
+ t.Logf("%d payload siblings: complete measured inventory, three cgroup process reads, no pct/lxc-info", 4*proxmoxLXCMaxCgroupDirs)
+ })
+ }
+}
+
+func TestProxmoxLXCInitSearchKeepsBoundsAndIdentity(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ depth int
+ state proxmoxLXCInitState
+ }{
+ {"last_permitted_directory", proxmoxLXCMaxCgroupDirs, proxmoxLXCRunning},
+ {"beyond_directory_budget", proxmoxLXCMaxCgroupDirs + 1, proxmoxLXCUnknown},
+ {"foreign_vm", 3, proxmoxLXCUnknown},
+ {"nested_namespace", 3, proxmoxLXCUnknown},
+ {"unreadable_preferred", 3, proxmoxLXCUnknown},
+ {"cancelled_before_init", 3, proxmoxLXCUnknown},
+ {"wide_tree_without_init", 2, proxmoxLXCUnknown},
+ {"last_permitted_pid", 3, proxmoxLXCRunning},
+ {"beyond_pid_budget", 3, proxmoxLXCUnknown},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ h := newFakeProxmoxLXCHost()
+ rel := "126"
+ for n := 1; n < tc.depth; n++ {
+ rel += "/step"
+ }
+ h.cgroup(rel, 4000)
+ h.proc(4000, "4000\t1", "/lxc/"+rel)
+ switch tc.name {
+ case "foreign_vm":
+ h.proc(4000, "4000\t1", "/lxc/127/ns")
+ case "nested_namespace":
+ h.proc(4000, "4000\t2\t1", "/lxc/"+rel)
+ case "unreadable_preferred":
+ delete(h.files, path.Join(proxmoxLXCCgroupRoot, rel, "cgroup.procs"))
+ case "wide_tree_without_init":
+ h.proc(4000, "4000\t2", "/lxc/"+rel)
+ for n := 0; n < 4*proxmoxLXCMaxCgroupDirs; n++ {
+ h.cgroup(fmt.Sprintf("126/workload%03d", n))
+ }
+ case "last_permitted_pid", "beyond_pid_budget":
+ count := proxmoxLXCMaxCgroupPIDs
+ if tc.name == "beyond_pid_budget" {
+ count++
+ }
+ pids := make([]int, count)
+ for n := range pids {
+ pids[n] = 10000 + n // absent /proc entries cannot establish init
+ }
+ pids[count-1] = 4000
+ h.cgroup(rel, pids...)
+ }
+ collector, commands := h.collector(t)
+ readFile := collector.readFileFn
+ directories, processes := 0, 0
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ collector.readFileFn = func(name string) ([]byte, error) {
+ if strings.HasPrefix(name, proxmoxLXCCgroupRoot+"/") && strings.HasSuffix(name, "/cgroup.procs") {
+ directories++
+ if tc.name == "cancelled_before_init" && directories == 2 {
+ cancel()
+ }
+ }
+ if strings.HasPrefix(name, "/proc/") && strings.HasSuffix(name, "/status") {
+ processes++
+ }
+ return readFile(name)
+ }
+ pid, state := (&Agent{logger: zerolog.Nop(), collector: collector}).proxmoxLXCInitPID(ctx, collector, 126)
+ if state != tc.state || (state == proxmoxLXCRunning && pid != 4000) || (state != proxmoxLXCRunning && pid != 0) || *commands != 0 {
+ t.Fatalf("pid/state=%d/%d want state=%d; commands=%d", pid, state, tc.state, *commands)
+ }
+ if directories > proxmoxLXCMaxCgroupDirs || processes > proxmoxLXCMaxCgroupPIDs {
+ t.Fatalf("search exceeded bounds: dirs=%d pids=%d", directories, processes)
+ }
+ if (tc.name == "last_permitted_directory" || tc.name == "beyond_directory_budget" || tc.name == "wide_tree_without_init") && directories != proxmoxLXCMaxCgroupDirs {
+ t.Fatalf("directory budget exercised only %d reads", directories)
+ }
+ if (tc.name == "last_permitted_pid" || tc.name == "beyond_pid_budget") && processes != proxmoxLXCMaxCgroupPIDs {
+ t.Fatalf("PID budget exercised only %d reads", processes)
+ }
+ t.Logf("cgroup process reads=%d; checked processes=%d; state=%d", directories, processes, state)
+ })
+ }
+}
diff --git a/internal/hostagent/proxmox_lxc_running.go b/internal/hostagent/proxmox_lxc_running.go
index 65888f61e1..606139776b 100644
--- a/internal/hostagent/proxmox_lxc_running.go
+++ b/internal/hostagent/proxmox_lxc_running.go
@@ -154,7 +154,7 @@ func (a *Agent) proxmoxLXCInitPID(ctx context.Context, lister proxmoxLXCDirListe
}
queue := []string{root}
- queued := 1
+ visited := 0
checked := 0
for len(queue) > 0 {
if ctx.Err() != nil {
@@ -162,6 +162,7 @@ func (a *Agent) proxmoxLXCInitPID(ctx context.Context, lister proxmoxLXCDirListe
}
dir := queue[0]
queue = queue[1:]
+ visited++
raw, err := a.collector.ReadFile(path.Join(dir, "cgroup.procs"))
if err != nil || len(raw) > proxmoxLXCMaxCgroupProcsBytes {
@@ -191,19 +192,45 @@ func (a *Agent) proxmoxLXCInitPID(ctx context.Context, lister proxmoxLXCDirListe
children = append(children, entry.Name())
}
}
- // systemd guests run init in init.scope; look there first.
- sort.SliceStable(children, func(i, j int) bool {
- return children[i] == "init.scope" && children[j] != "init.scope"
+ // PVE puts the payload under ns/; systemd puts its init in
+ // init.scope. Inspect those before unrelated workloads, regardless of
+ // directory-list ordering. The limit bounds reads, not the number of
+ // siblings that can exist alongside a verified init.
+ priority := func(name string) int {
+ switch name {
+ case "init.scope":
+ return 0
+ case "ns":
+ return 1
+ default:
+ return 2
+ }
+ }
+ sort.Slice(children, func(i, j int) bool {
+ if a, b := priority(children[i]), priority(children[j]); a != b {
+ return a < b
+ }
+ return children[i] < children[j]
})
+ // Depth-first priority must survive a wide parent too: queued siblings
+ // cannot crowd out ns/init.scope before it is inspected. Keep at most
+ // the remaining read budget in the frontier. If no exact init is found
+ // within that budget, the result below stays unknown, never stopped.
+ remaining := proxmoxLXCMaxCgroupDirs - visited
+ next := make([]string, 0, remaining)
for _, child := range children {
- if queued == proxmoxLXCMaxCgroupDirs {
- // A guest can create cgroups in its delegated subtree; give up
- // rather than walk an unbounded tree.
- return 0, proxmoxLXCUnknown
+ if len(next) == remaining {
+ break
+ }
+ next = append(next, path.Join(dir, child))
+ }
+ for _, pending := range queue {
+ if len(next) == remaining {
+ break
}
- queued++
- queue = append(queue, path.Join(dir, child))
+ next = append(next, pending)
}
+ queue = next
}
// A cgroup without an init is a guest starting or stopping.
return 0, proxmoxLXCUnknown