Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Inspect LXC init before wide cgroup siblings exhaust discovery
pulse-triage[bot]e1bba156e39e
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex ea001419d8..f67e30f7fb 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -620,6 +620,17 @@ admissible, but discovery must inspect the rest of the bounded config listing:a 129th running guest or an unestablished later guest makes discovery unavailable,not a truncated complete inventory. The existing bounded `pct list` fallbackthen either establishes the full node or remains degraded under PrivateNetwork.+The init search checks the preferred `ns/init.scope` path before unrelated+workload cgroups, even when either level has hundreds of siblings. Its bounded+priority frontier permits at most 64 cgroup process reads and 4,096 candidate+PIDs; reaching either limit without a verified init remains unknown, not stopped.+Unreadable or cancelled traversal still fails closed. Width alone cannot prevent+a verified init from supporting the existing pinned filesystem observation.+`TestHelperProxmoxLXCInitSearchPrioritizesBoundedReads` connects wide-tree discovery+to complete measured collection without IPC commands;+`TestProxmoxLXCInitSearchKeepsBoundsAndIdentity` pins exact directory/PID limits,+foreign VM and nested namespace rejection, unreadable input and cancellation.+These are synthetic source controls, not installed helper or native PVE recovery.`TestSocketFreeProxmoxLXCDiscoveryCannotTruncateCompleteInventory` covers 127/128/129running guests and stopped/unknown guests after the limit through discovery andcollection. The fixed `lxc-info -n <vmid> -p`diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex d9105c18bf..fc631be6bf 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -973,7 +973,15 @@ the existing bounded fallback; a stopped later guest does not invalidate 128established guests. This preserves the consumer's distinction between completeinventory replacement and explicit partial omissions. The producer boundary iscovered by `TestSocketFreeProxmoxLXCDiscoveryCannotTruncateCompleteInventory`;-partial-wire, omitted-reading and complete-empty cache tests remain authoritative. An `lxc` request names the decimal VMID; the pinned process+partial-wire, omitted-reading and complete-empty cache tests remain authoritative.+Wide cgroup trees do not by themselves disqualify a verified init: the producer+inspects `ns/init.scope` before unrelated workload siblings using at most 64+cgroup process reads and 4,096 candidate PIDs. Exhausted, unreadable, cancelled+or identity-invalid searches remain unknown/degraded; no omitted guest becomes+healthy empty telemetry. The wide-tree helper collection and search-boundary+controls in `internal/hostagent/privilege_helper_client_test.go` preserve measured+capacity and exact identity without IPC commands. They are not native recovery.+An `lxc` request names the decimal VMID; the pinned processmust sit in that guest's exact `lxc/<vmid>` cgroup and be its init (`NSpid` isexactly `<pid> 1`, so the PID 1 of a namespace nested inside the guest, suchas Docker in LXC, cannot stand in for it), checked before and after the read.diff --git a/internal/hostagent/privilege_helper_client_test.go b/internal/hostagent/privilege_helper_client_test.goindex d70155212a..97b4cb7367 100644--- a/internal/hostagent/privilege_helper_client_test.go+++ b/internal/hostagent/privilege_helper_client_test.go@@ -9,6 +9,7 @@ import ("io""net""os/exec"+ "path""path/filepath""strconv""strings"@@ -660,3 +661,137 @@ func TestSocketFreeProxmoxLXCDiscoveryCannotTruncateCompleteInventory(t *testing})}}++// A wide delegated cgroup tree must not prevent the helper from inspecting+// the exact init it already prioritises. All observations here are synthetic;+// the collector's real identity/prober contract and PrivateNetwork stay intact.+func TestHelperProxmoxLXCInitSearchPrioritizesBoundedReads(t *testing.T) {+ for _, wideHost := range []bool{false, true} {+ name := "wide_payload"+ if wideHost {+ name = "wide_host_and_payload"+ }+ t.Run(name, func(t *testing.T) {+ h := newFakeProxmoxLXCHost()+ h.config(126, "hostname: busy-host\nrootfs: local:vm-126-disk-0,size=1G\n")+ h.cgroup("126/ns/init.scope", 4000)+ h.proc(4000, "4000\t1", "/lxc/126/ns/init.scope")+ for n := 0; n < 4*proxmoxLXCMaxCgroupDirs; n++ {+ h.cgroup(fmt.Sprintf("126/ns/workload%03d", n))+ if wideHost {+ h.cgroup(fmt.Sprintf("126/auxiliary%03d", n))+ }+ }+ h.observations[4000] = map[string]lxcObservation{"/": lxcUsage(4096, 1024, 3072)}+ collector, commands := h.collector(t)+ readFile := collector.readFileFn+ var examined []string+ collector.readFileFn = func(name string) ([]byte, error) {+ if strings.HasPrefix(name, proxmoxLXCCgroupRoot+"/") && strings.HasSuffix(name, "/cgroup.procs") {+ examined = append(examined, name)+ }+ return readFile(name)+ }+ result := (&Agent{logger: zerolog.Nop(), collector: collector}).collectProxmoxLXCFilesystemsResult(t.Context())+ if !result.Applicable || result.Degraded || result.Inventory == nil || result.Inventory.Status != "complete" || result.Inventory.ValidateCollection() != nil || *commands != 0 {+ t.Fatalf("wide-tree helper collection = %+v; command attempts=%d", result, *commands)+ }+ rows := result.Inventory.Containers+ if len(rows) != 1 || rows[0].VMID != 126 || rows[0].Name != "busy-host" || len(rows[0].Disks) != 1 || rows[0].Disks[0].UsedBytes != 1024 {+ t.Fatalf("wide-tree source/name/capacity lost: %+v", rows)+ }+ want := []string{+ path.Join(proxmoxLXCCgroupRoot, "126/cgroup.procs"),+ path.Join(proxmoxLXCCgroupRoot, "126/ns/cgroup.procs"),+ path.Join(proxmoxLXCCgroupRoot, "126/ns/init.scope/cgroup.procs"),+ }+ if fmt.Sprint(examined) != fmt.Sprint(want) {+ t.Fatalf("examined cgroups=%v, want only %v", examined, want)+ }+ t.Logf("%d payload siblings: complete measured inventory, three cgroup process reads, no pct/lxc-info", 4*proxmoxLXCMaxCgroupDirs)+ })+ }+}++func TestProxmoxLXCInitSearchKeepsBoundsAndIdentity(t *testing.T) {+ for _, tc := range []struct {+ name string+ depth int+ state proxmoxLXCInitState+ }{+ {"last_permitted_directory", proxmoxLXCMaxCgroupDirs, proxmoxLXCRunning},+ {"beyond_directory_budget", proxmoxLXCMaxCgroupDirs + 1, proxmoxLXCUnknown},+ {"foreign_vm", 3, proxmoxLXCUnknown},+ {"nested_namespace", 3, proxmoxLXCUnknown},+ {"unreadable_preferred", 3, proxmoxLXCUnknown},+ {"cancelled_before_init", 3, proxmoxLXCUnknown},+ {"wide_tree_without_init", 2, proxmoxLXCUnknown},+ {"last_permitted_pid", 3, proxmoxLXCRunning},+ {"beyond_pid_budget", 3, proxmoxLXCUnknown},+ } {+ t.Run(tc.name, func(t *testing.T) {+ h := newFakeProxmoxLXCHost()+ rel := "126"+ for n := 1; n < tc.depth; n++ {+ rel += "/step"+ }+ h.cgroup(rel, 4000)+ h.proc(4000, "4000\t1", "/lxc/"+rel)+ switch tc.name {+ case "foreign_vm":+ h.proc(4000, "4000\t1", "/lxc/127/ns")+ case "nested_namespace":+ h.proc(4000, "4000\t2\t1", "/lxc/"+rel)+ case "unreadable_preferred":+ delete(h.files, path.Join(proxmoxLXCCgroupRoot, rel, "cgroup.procs"))+ case "wide_tree_without_init":+ h.proc(4000, "4000\t2", "/lxc/"+rel)+ for n := 0; n < 4*proxmoxLXCMaxCgroupDirs; n++ {+ h.cgroup(fmt.Sprintf("126/workload%03d", n))+ }+ case "last_permitted_pid", "beyond_pid_budget":+ count := proxmoxLXCMaxCgroupPIDs+ if tc.name == "beyond_pid_budget" {+ count+++ }+ pids := make([]int, count)+ for n := range pids {+ pids[n] = 10000 + n // absent /proc entries cannot establish init+ }+ pids[count-1] = 4000+ h.cgroup(rel, pids...)+ }+ collector, commands := h.collector(t)+ readFile := collector.readFileFn+ directories, processes := 0, 0+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ collector.readFileFn = func(name string) ([]byte, error) {+ if strings.HasPrefix(name, proxmoxLXCCgroupRoot+"/") && strings.HasSuffix(name, "/cgroup.procs") {+ directories+++ if tc.name == "cancelled_before_init" && directories == 2 {+ cancel()+ }+ }+ if strings.HasPrefix(name, "/proc/") && strings.HasSuffix(name, "/status") {+ processes+++ }+ return readFile(name)+ }+ pid, state := (&Agent{logger: zerolog.Nop(), collector: collector}).proxmoxLXCInitPID(ctx, collector, 126)+ if state != tc.state || (state == proxmoxLXCRunning && pid != 4000) || (state != proxmoxLXCRunning && pid != 0) || *commands != 0 {+ t.Fatalf("pid/state=%d/%d want state=%d; commands=%d", pid, state, tc.state, *commands)+ }+ if directories > proxmoxLXCMaxCgroupDirs || processes > proxmoxLXCMaxCgroupPIDs {+ t.Fatalf("search exceeded bounds: dirs=%d pids=%d", directories, processes)+ }+ if (tc.name == "last_permitted_directory" || tc.name == "beyond_directory_budget" || tc.name == "wide_tree_without_init") && directories != proxmoxLXCMaxCgroupDirs {+ t.Fatalf("directory budget exercised only %d reads", directories)+ }+ if (tc.name == "last_permitted_pid" || tc.name == "beyond_pid_budget") && processes != proxmoxLXCMaxCgroupPIDs {+ t.Fatalf("PID budget exercised only %d reads", processes)+ }+ t.Logf("cgroup process reads=%d; checked processes=%d; state=%d", directories, processes, state)+ })+ }+}diff --git a/internal/hostagent/proxmox_lxc_running.go b/internal/hostagent/proxmox_lxc_running.goindex 65888f61e1..606139776b 100644--- a/internal/hostagent/proxmox_lxc_running.go+++ b/internal/hostagent/proxmox_lxc_running.go@@ -154,7 +154,7 @@ func (a *Agent) proxmoxLXCInitPID(ctx context.Context, lister proxmoxLXCDirListe}queue := []string{root}- queued := 1+ visited := 0checked := 0for len(queue) > 0 {if ctx.Err() != nil {@@ -162,6 +162,7 @@ func (a *Agent) proxmoxLXCInitPID(ctx context.Context, lister proxmoxLXCDirListe}dir := queue[0]queue = queue[1:]+ visited++raw, err := a.collector.ReadFile(path.Join(dir, "cgroup.procs"))if err != nil || len(raw) > proxmoxLXCMaxCgroupProcsBytes {@@ -191,19 +192,45 @@ func (a *Agent) proxmoxLXCInitPID(ctx context.Context, lister proxmoxLXCDirListechildren = append(children, entry.Name())}}- // systemd guests run init in init.scope; look there first.- sort.SliceStable(children, func(i, j int) bool {- return children[i] == "init.scope" && children[j] != "init.scope"+ // PVE puts the payload under ns/; systemd puts its init in+ // init.scope. Inspect those before unrelated workloads, regardless of+ // directory-list ordering. The limit bounds reads, not the number of+ // siblings that can exist alongside a verified init.+ priority := func(name string) int {+ switch name {+ case "init.scope":+ return 0+ case "ns":+ return 1+ default:+ return 2+ }+ }+ sort.Slice(children, func(i, j int) bool {+ if a, b := priority(children[i]), priority(children[j]); a != b {+ return a < b+ }+ return children[i] < children[j]})+ // Depth-first priority must survive a wide parent too: queued siblings+ // cannot crowd out ns/init.scope before it is inspected. Keep at most+ // the remaining read budget in the frontier. If no exact init is found+ // within that budget, the result below stays unknown, never stopped.+ remaining := proxmoxLXCMaxCgroupDirs - visited+ next := make([]string, 0, remaining)for _, child := range children {- if queued == proxmoxLXCMaxCgroupDirs {- // A guest can create cgroups in its delegated subtree; give up- // rather than walk an unbounded tree.- return 0, proxmoxLXCUnknown+ if len(next) == remaining {+ break+ }+ next = append(next, path.Join(dir, child))+ }+ for _, pending := range queue {+ if len(next) == remaining {+ break}- queued++- queue = append(queue, path.Join(dir, child))+ next = append(next, pending)}+ queue = next}// A cgroup without an init is a guest starting or stopping.return 0, proxmoxLXCUnknown