Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep Podman CPU interval evidence when one-shot percentages are zero
pulse-triage[bot]ab978b6cfc4a
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex 4754a53d0d..f7aba26ab8 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -15,6 +15,21 @@## Purpose+### Podman interval CPU evidence — issue #2447++Every one-shot container stats response retains a cumulative CPU baseline,+even when a positive Podman compatibility percentage takes precedence. Zero,+absent or invalid percentages use the valid nanosecond/wall-clock interval,+without Docker's host-core multiplier. First samples, counter resets and changes+of container start time rebaseline rather than presenting lifetime CPU as current+usage. Repeated or older read timestamps cannot move the baseline backwards.+Positive native percentages keep the units established by #1391; a genuine idle+counter interval remains zero. Report shapes, enrollment, runtime transport trust+and command authority are unchanged. `TestCollectContainerPodmanCPUIntervals`+in `internal/dockeragent/agent_internal_test.go` exercises decoding through the+real collector, idle/advancing counters, start changes and Docker isolation.+Synthetic controls do not establish a native Podman payload or reporter recovery.+### MD RAID required members and spares — issue #2369Host RAID reports carry optional `requiredDevices`: the configured member countdiff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 26ce03a228..38bca29d81 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,20 @@## Purpose+### Podman zero-percentage fallback — issue #2447++One-shot compatibility stats with `cpu:0` do not suppress advancing cumulative+CPU samples. The collector records every interval and uses Podman's CPU-time+nanoseconds over elapsed wall-clock nanoseconds, independent of host core count.+A positive finite reported percentage remains authoritative, preserving #1391.+Counter resets or a changed container start establish a new baseline; duplicate+and out-of-order reads cannot create subsequent spikes. No timestamp means no+measurable counter interval. The existing report CPU field, graph/alert units,+Docker system-counter calculation, memory and permission scopes are unchanged.+`TestCollectContainerPodmanCPUIntervals` in `agent_internal_test.go` covers the+actual JSON/collector boundary, including changing percentage availability.+Native Podman shape and graph/alert acceptance remain separate evidence.+### Guest-agent coordination around backups — issue #2439All PVE QEMU guest-agent reads, including disk diagnostics and legacy meminfodiff --git a/internal/dockeragent/agent.go b/internal/dockeragent/agent.goindex 0c268a91a3..4872bdc4fe 100644--- a/internal/dockeragent/agent.go+++ b/internal/dockeragent/agent.go@@ -229,6 +229,7 @@ type cpuSample struct {systemUsage uint64onlineCPUs uint32read time.Time+ startedAt time.Time}// New creates a new Docker / Podman module instance.diff --git a/internal/dockeragent/agent_cpu_test.go b/internal/dockeragent/agent_cpu_test.goindex f719bff24c..3bbfefee98 100644--- a/internal/dockeragent/agent_cpu_test.go+++ b/internal/dockeragent/agent_cpu_test.go@@ -28,7 +28,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {OnlineCPUs: 2,},}- got := agent.calculateContainerCPUPercent("container-123456", stats1)+ got := agent.calculateContainerCPUPercent("container-123456", stats1, time.Time{})if got != 0 {t.Fatalf("first call: expected 0, got %f", got)}@@ -45,7 +45,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {OnlineCPUs: 2,},}- got = agent.calculateContainerCPUPercent("container-123456", stats2)+ got = agent.calculateContainerCPUPercent("container-123456", stats2, time.Time{})if got <= 0 {t.Fatalf("second call: expected percent > 0, got %f", got)}@@ -68,7 +68,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if got != 0 {t.Fatalf("expected 0, got %f", got)}@@ -103,7 +103,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if got <= 0 {t.Fatalf("expected percent > 0, got %f", got)}@@ -136,7 +136,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if got <= 0 {t.Fatalf("expected percent > 0, got %f", got)}@@ -168,7 +168,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if got <= 0 {t.Fatalf("expected percent > 0, got %f", got)}@@ -200,7 +200,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {+ if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {t.Fatalf("expected 0, got %f", got)}})@@ -232,7 +232,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {+ if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {t.Fatalf("expected 0, got %f", got)}})@@ -263,7 +263,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {+ if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {t.Fatalf("expected 0, got %f", got)}})@@ -291,7 +291,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {PreCPUStats: containertypes.CPUStats{},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if got != 0 {t.Fatalf("expected 0, got %f", got)}@@ -319,7 +319,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if got != 0 {t.Fatalf("expected 0 on first call (manual tracking), got %f", got)}@@ -349,7 +349,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {},}- got := agent.calculateContainerCPUPercent("container-123456", stats)+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})if math.Abs(got-25.0) > 0.01 {t.Fatalf("expected podman cpu percent near 25.0, got %f", got)}diff --git a/internal/dockeragent/agent_internal_test.go b/internal/dockeragent/agent_internal_test.goindex 193caaf5bb..c4bffdc700 100644--- a/internal/dockeragent/agent_internal_test.go+++ b/internal/dockeragent/agent_internal_test.go@@ -1840,3 +1840,137 @@ func TestCollectorRuntimeBoundaryLossIsDaemonUnavailable(t *testing.T) {}}}++// Exercise the compatibility JSON through the actual container collector, not+// just the delta helper. These are synthetic inputs, not a native Podman trace.+func TestCollectContainerPodmanCPUIntervals(t *testing.T) {+ type sample struct {+ second int+ total uint64+ percent string // Empty means absent; other values are literal JSON.+ start int+ stopped bool+ want float64+ }+ tests := []struct {+ name string+ runtime RuntimeKind+ cpus int+ samples []sample+ }{+ {"zero with advancing and idle counters", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0"},+ {second: 1, total: 130_000_000, percent: "0", want: 3},+ {second: 2, total: 130_000_000, percent: "0"},+ {second: 3, total: 1_630_000_000, percent: "0", want: 150},+ }},+ {"absent percentage uses nanosecond wall clock units", RuntimePodman, 1, []sample{+ {second: 0, total: 100_000_000},+ {second: 1, total: 130_000_000, want: 3},+ }},+ {"null percentage", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "null"},+ {second: 1, total: 130_000_000, percent: "null", want: 3},+ }},+ {"positive percentages keep units and retain the baseline", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0.32", want: 0.32},+ {second: 1, total: 130_000_000, percent: "0.64", want: 0.64},+ {second: 2, total: 160_000_000, percent: "0", want: 3},+ {second: 3, total: 190_000_000, want: 3},+ }},+ {"absent then reported then zero", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000},+ {second: 1, total: 130_000_000, percent: "0.32", want: 0.32},+ {second: 2, total: 160_000_000, percent: "0", want: 3},+ }},+ {"invalid negative percentage", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "-1"},+ {second: 1, total: 130_000_000, percent: "-1", want: 3},+ }},+ {"unparseable optional percentage", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: `"unknown"`},+ {second: 1, total: 130_000_000, percent: `"unknown"`, want: 3},+ }},+ {"counter reset rebaselines rather than using new lifetime", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0"},+ {second: 1, total: 10_000_000, percent: "0"},+ {second: 2, total: 40_000_000, percent: "0", want: 3},+ }},+ {"restart with an already larger counter", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0"},+ {second: 1, total: 260_000_000, percent: "0", start: 1},+ {second: 2, total: 290_000_000, percent: "0", start: 1, want: 3},+ }},+ {"repeated and old timestamps do not move the baseline", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0"},+ {second: 1, total: 130_000_000, percent: "0", want: 3},+ {second: 1, total: 150_000_000, percent: "0"},+ {second: 0, total: 170_000_000, percent: "0"},+ {second: 2, total: 160_000_000, percent: "0", want: 3},+ }},+ {"missing timestamp cannot invent an interval", RuntimePodman, 16, []sample{+ {second: -1, total: 100_000_000, percent: "0"},+ {second: 1, total: 130_000_000, percent: "0"},+ {second: 2, total: 160_000_000, percent: "0", want: 3},+ }},+ {"stop clears the baseline", RuntimePodman, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0.32", want: 0.32},+ {second: 1, stopped: true},+ {second: 2, total: 260_000_000, percent: "0"},+ {second: 3, total: 290_000_000, percent: "0", want: 3},+ }},+ {"docker ignores compat percentage and retains system delta", RuntimeDocker, 16, []sample{+ {second: 0, total: 100_000_000, percent: "0.32"},+ {second: 1, total: 130_000_000, percent: "0.32", want: 48},+ }},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ epoch := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC)+ var current sample+ agent := &Agent{+ runtime: tt.runtime,+ logger: zerolog.Nop(),+ cpuCount: tt.cpus,+ docker: &fakeDockerClient{+ containerInspectWithRawFn: func(context.Context, string, bool) (containertypes.InspectResponse, []byte, error) {+ inspect := baseInspect()+ inspect.State = &containertypes.State{+ Running: !current.stopped,+ StartedAt: epoch.Add(time.Duration(current.start-60) * time.Second).Format(time.RFC3339Nano),+ }+ return inspect, nil, nil+ },+ containerStatsOneShotFn: func(context.Context, string) (dockerStatsResponseReader, error) {+ read := "null"+ if current.second >= 0 {+ read = fmt.Sprintf("%q", epoch.Add(time.Duration(current.second)*time.Second).Format(time.RFC3339Nano))+ }+ percent := ""+ if current.percent != "" {+ percent = `,"cpu":` + current.percent+ }+ payload := fmt.Sprintf(`{"read":%s,"cpu_stats":{"cpu_usage":{"total_usage":%d},"system_cpu_usage":%d,"online_cpus":%d%s},"memory_stats":{"usage":1000000,"limit":4000000}}`,+ read, current.total, uint64(current.second+2)*1_000_000_000, tt.cpus, percent)+ return dockerStatsResponseReader{Body: io.NopCloser(strings.NewReader(payload))}, nil+ },+ },+ }+ for i, s := range tt.samples {+ current = s+ container, err := agent.collectContainer(context.Background(), containertypes.Summary{+ ID: "container-123456", Names: []string{"/cpu-fixture"}, Image: "fixture:local",+ })+ if err != nil {+ t.Fatalf("sample %d: %v", i, err)+ }+ if math.Abs(container.CPUPercent-s.want) > 1e-9 {+ t.Errorf("sample %d: CPU = %g%%, want %g%%", i, container.CPUPercent, s.want)+ }+ if !s.stopped && (container.MemoryUsageBytes != 1_000_000 || container.MemoryLimitBytes != 4_000_000) {+ t.Fatalf("sample %d: memory collection changed: %+v", i, container)+ }+ }+ })+ }+}diff --git a/internal/dockeragent/collect.go b/internal/dockeragent/collect.goindex 28c8b92874..9ca434c9f8 100644--- a/internal/dockeragent/collect.go+++ b/internal/dockeragent/collect.go@@ -601,10 +601,14 @@ func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Sumreturn agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)}- if a.runtime == RuntimePodman && podmanCPUPercent != nil {- cpuPercent = safeFloat(*podmanCPUPercent)- } else {- cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)+ // Keep a baseline even when Podman supplies a percentage: one-shot+ // compatibility responses can report cpu:0 while total_usage advances.+ // Preserve positive native percentages (and their units, #1391), but+ // use our interval delta for an absent, zero or invalid percentage.+ cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats, parseTime(inspect.State.StartedAt))+ if a.runtime == RuntimePodman && podmanCPUPercent != nil &&+ *podmanCPUPercent > 0 && !math.IsInf(*podmanCPUPercent, 0) {+ cpuPercent = *podmanCPUPercent}memUsage, memLimit, memPercent = calculateMemoryUsage(stats)blockIO = summarizeBlockIO(stats)@@ -1138,7 +1142,7 @@ func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse,return stats, probe.CPUStats.CPU, nil}-func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse) float64 {+func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse, startedAt time.Time) float64 {a.cpuMu.Lock()defer a.cpuMu.Unlock()@@ -1147,6 +1151,7 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats containersystemUsage: stats.CPUStats.SystemUsage,onlineCPUs: stats.CPUStats.OnlineCPUs,read: stats.Read,+ startedAt: startedAt,}// Always use manual delta tracking. Docker's PreCPUStats is unreliable for@@ -1155,6 +1160,14 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container// (from container start) and producing a constant lifetime-average CPU%// instead of a current value.prev, ok := a.prevContainerCPU[containerID]+ if a.runtime == RuntimePodman && ok && !prev.read.IsZero() && !current.read.IsZero() && !current.read.After(prev.read) {+ // Repeated or out-of-order responses must not move the baseline back+ // and manufacture usage in the next interval.+ return 0+ }+ if a.prevContainerCPU == nil {+ a.prevContainerCPU = make(map[string]cpuSample)+ }if !ok {// First time seeing this container - store current sample and return 0// On next collection cycle we'll have a previous sample to compare against@@ -1169,6 +1182,12 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container// We have a previous sample - update it after calculationa.prevContainerCPU[containerID] = current+ if a.runtime == RuntimePodman && (current.totalUsage < prev.totalUsage ||+ (!prev.startedAt.IsZero() && !current.startedAt.IsZero() && !current.startedAt.Equal(prev.startedAt))) {+ // A restart is not an interval sample, even if the new lifetime counter+ // has already exceeded the old one. Start again from this baseline.+ return 0+ }var totalDelta float64if current.totalUsage >= prev.totalUsage {