Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep Podman CPU interval evidence when one-shot percentages are zero

pulse-triage[bot]ab978b6cfc4a
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 4754a53d0d..f7aba26ab8 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -15,6 +15,21 @@
## Purpose
+### Podman interval CPU evidence — issue #2447
+
+Every one-shot container stats response retains a cumulative CPU baseline,
+even when a positive Podman compatibility percentage takes precedence. Zero,
+absent or invalid percentages use the valid nanosecond/wall-clock interval,
+without Docker's host-core multiplier. First samples, counter resets and changes
+of container start time rebaseline rather than presenting lifetime CPU as current
+usage. Repeated or older read timestamps cannot move the baseline backwards.
+Positive native percentages keep the units established by #1391; a genuine idle
+counter interval remains zero. Report shapes, enrollment, runtime transport trust
+and command authority are unchanged. `TestCollectContainerPodmanCPUIntervals`
+in `internal/dockeragent/agent_internal_test.go` exercises decoding through the
+real collector, idle/advancing counters, start changes and Docker isolation.
+Synthetic controls do not establish a native Podman payload or reporter recovery.
+
### MD RAID required members and spares — issue #2369
Host RAID reports carry optional `requiredDevices`: the configured member count
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 26ce03a228..38bca29d81 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,20 @@
## Purpose
+### Podman zero-percentage fallback — issue #2447
+
+One-shot compatibility stats with `cpu:0` do not suppress advancing cumulative
+CPU samples. The collector records every interval and uses Podman's CPU-time
+nanoseconds over elapsed wall-clock nanoseconds, independent of host core count.
+A positive finite reported percentage remains authoritative, preserving #1391.
+Counter resets or a changed container start establish a new baseline; duplicate
+and out-of-order reads cannot create subsequent spikes. No timestamp means no
+measurable counter interval. The existing report CPU field, graph/alert units,
+Docker system-counter calculation, memory and permission scopes are unchanged.
+`TestCollectContainerPodmanCPUIntervals` in `agent_internal_test.go` covers the
+actual JSON/collector boundary, including changing percentage availability.
+Native Podman shape and graph/alert acceptance remain separate evidence.
+
### Guest-agent coordination around backups — issue #2439
All PVE QEMU guest-agent reads, including disk diagnostics and legacy meminfo
diff --git a/internal/dockeragent/agent.go b/internal/dockeragent/agent.go
index 0c268a91a3..4872bdc4fe 100644
--- a/internal/dockeragent/agent.go
+++ b/internal/dockeragent/agent.go
@@ -229,6 +229,7 @@ type cpuSample struct {
systemUsage uint64
onlineCPUs uint32
read time.Time
+ startedAt time.Time
}
// New creates a new Docker / Podman module instance.
diff --git a/internal/dockeragent/agent_cpu_test.go b/internal/dockeragent/agent_cpu_test.go
index f719bff24c..3bbfefee98 100644
--- a/internal/dockeragent/agent_cpu_test.go
+++ b/internal/dockeragent/agent_cpu_test.go
@@ -28,7 +28,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
OnlineCPUs: 2,
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats1)
+ got := agent.calculateContainerCPUPercent("container-123456", stats1, time.Time{})
if got != 0 {
t.Fatalf("first call: expected 0, got %f", got)
}
@@ -45,7 +45,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
OnlineCPUs: 2,
},
}
- got = agent.calculateContainerCPUPercent("container-123456", stats2)
+ got = agent.calculateContainerCPUPercent("container-123456", stats2, time.Time{})
if got <= 0 {
t.Fatalf("second call: expected percent > 0, got %f", got)
}
@@ -68,7 +68,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got != 0 {
t.Fatalf("expected 0, got %f", got)
}
@@ -103,7 +103,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got <= 0 {
t.Fatalf("expected percent > 0, got %f", got)
}
@@ -136,7 +136,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got <= 0 {
t.Fatalf("expected percent > 0, got %f", got)
}
@@ -168,7 +168,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got <= 0 {
t.Fatalf("expected percent > 0, got %f", got)
}
@@ -200,7 +200,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {
+ if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {
t.Fatalf("expected 0, got %f", got)
}
})
@@ -232,7 +232,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {
+ if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {
t.Fatalf("expected 0, got %f", got)
}
})
@@ -263,7 +263,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- if got := agent.calculateContainerCPUPercent("container-123456", stats); got != 0 {
+ if got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{}); got != 0 {
t.Fatalf("expected 0, got %f", got)
}
})
@@ -291,7 +291,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
PreCPUStats: containertypes.CPUStats{},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got != 0 {
t.Fatalf("expected 0, got %f", got)
}
@@ -319,7 +319,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if got != 0 {
t.Fatalf("expected 0 on first call (manual tracking), got %f", got)
}
@@ -349,7 +349,7 @@ func TestCalculateContainerCPUPercent(t *testing.T) {
},
}
- got := agent.calculateContainerCPUPercent("container-123456", stats)
+ got := agent.calculateContainerCPUPercent("container-123456", stats, time.Time{})
if math.Abs(got-25.0) > 0.01 {
t.Fatalf("expected podman cpu percent near 25.0, got %f", got)
}
diff --git a/internal/dockeragent/agent_internal_test.go b/internal/dockeragent/agent_internal_test.go
index 193caaf5bb..c4bffdc700 100644
--- a/internal/dockeragent/agent_internal_test.go
+++ b/internal/dockeragent/agent_internal_test.go
@@ -1840,3 +1840,137 @@ func TestCollectorRuntimeBoundaryLossIsDaemonUnavailable(t *testing.T) {
}
}
}
+
+// Exercise the compatibility JSON through the actual container collector, not
+// just the delta helper. These are synthetic inputs, not a native Podman trace.
+func TestCollectContainerPodmanCPUIntervals(t *testing.T) {
+ type sample struct {
+ second int
+ total uint64
+ percent string // Empty means absent; other values are literal JSON.
+ start int
+ stopped bool
+ want float64
+ }
+ tests := []struct {
+ name string
+ runtime RuntimeKind
+ cpus int
+ samples []sample
+ }{
+ {"zero with advancing and idle counters", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0"},
+ {second: 1, total: 130_000_000, percent: "0", want: 3},
+ {second: 2, total: 130_000_000, percent: "0"},
+ {second: 3, total: 1_630_000_000, percent: "0", want: 150},
+ }},
+ {"absent percentage uses nanosecond wall clock units", RuntimePodman, 1, []sample{
+ {second: 0, total: 100_000_000},
+ {second: 1, total: 130_000_000, want: 3},
+ }},
+ {"null percentage", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "null"},
+ {second: 1, total: 130_000_000, percent: "null", want: 3},
+ }},
+ {"positive percentages keep units and retain the baseline", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0.32", want: 0.32},
+ {second: 1, total: 130_000_000, percent: "0.64", want: 0.64},
+ {second: 2, total: 160_000_000, percent: "0", want: 3},
+ {second: 3, total: 190_000_000, want: 3},
+ }},
+ {"absent then reported then zero", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000},
+ {second: 1, total: 130_000_000, percent: "0.32", want: 0.32},
+ {second: 2, total: 160_000_000, percent: "0", want: 3},
+ }},
+ {"invalid negative percentage", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "-1"},
+ {second: 1, total: 130_000_000, percent: "-1", want: 3},
+ }},
+ {"unparseable optional percentage", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: `"unknown"`},
+ {second: 1, total: 130_000_000, percent: `"unknown"`, want: 3},
+ }},
+ {"counter reset rebaselines rather than using new lifetime", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0"},
+ {second: 1, total: 10_000_000, percent: "0"},
+ {second: 2, total: 40_000_000, percent: "0", want: 3},
+ }},
+ {"restart with an already larger counter", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0"},
+ {second: 1, total: 260_000_000, percent: "0", start: 1},
+ {second: 2, total: 290_000_000, percent: "0", start: 1, want: 3},
+ }},
+ {"repeated and old timestamps do not move the baseline", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0"},
+ {second: 1, total: 130_000_000, percent: "0", want: 3},
+ {second: 1, total: 150_000_000, percent: "0"},
+ {second: 0, total: 170_000_000, percent: "0"},
+ {second: 2, total: 160_000_000, percent: "0", want: 3},
+ }},
+ {"missing timestamp cannot invent an interval", RuntimePodman, 16, []sample{
+ {second: -1, total: 100_000_000, percent: "0"},
+ {second: 1, total: 130_000_000, percent: "0"},
+ {second: 2, total: 160_000_000, percent: "0", want: 3},
+ }},
+ {"stop clears the baseline", RuntimePodman, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0.32", want: 0.32},
+ {second: 1, stopped: true},
+ {second: 2, total: 260_000_000, percent: "0"},
+ {second: 3, total: 290_000_000, percent: "0", want: 3},
+ }},
+ {"docker ignores compat percentage and retains system delta", RuntimeDocker, 16, []sample{
+ {second: 0, total: 100_000_000, percent: "0.32"},
+ {second: 1, total: 130_000_000, percent: "0.32", want: 48},
+ }},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ epoch := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC)
+ var current sample
+ agent := &Agent{
+ runtime: tt.runtime,
+ logger: zerolog.Nop(),
+ cpuCount: tt.cpus,
+ docker: &fakeDockerClient{
+ containerInspectWithRawFn: func(context.Context, string, bool) (containertypes.InspectResponse, []byte, error) {
+ inspect := baseInspect()
+ inspect.State = &containertypes.State{
+ Running: !current.stopped,
+ StartedAt: epoch.Add(time.Duration(current.start-60) * time.Second).Format(time.RFC3339Nano),
+ }
+ return inspect, nil, nil
+ },
+ containerStatsOneShotFn: func(context.Context, string) (dockerStatsResponseReader, error) {
+ read := "null"
+ if current.second >= 0 {
+ read = fmt.Sprintf("%q", epoch.Add(time.Duration(current.second)*time.Second).Format(time.RFC3339Nano))
+ }
+ percent := ""
+ if current.percent != "" {
+ percent = `,"cpu":` + current.percent
+ }
+ payload := fmt.Sprintf(`{"read":%s,"cpu_stats":{"cpu_usage":{"total_usage":%d},"system_cpu_usage":%d,"online_cpus":%d%s},"memory_stats":{"usage":1000000,"limit":4000000}}`,
+ read, current.total, uint64(current.second+2)*1_000_000_000, tt.cpus, percent)
+ return dockerStatsResponseReader{Body: io.NopCloser(strings.NewReader(payload))}, nil
+ },
+ },
+ }
+ for i, s := range tt.samples {
+ current = s
+ container, err := agent.collectContainer(context.Background(), containertypes.Summary{
+ ID: "container-123456", Names: []string{"/cpu-fixture"}, Image: "fixture:local",
+ })
+ if err != nil {
+ t.Fatalf("sample %d: %v", i, err)
+ }
+ if math.Abs(container.CPUPercent-s.want) > 1e-9 {
+ t.Errorf("sample %d: CPU = %g%%, want %g%%", i, container.CPUPercent, s.want)
+ }
+ if !s.stopped && (container.MemoryUsageBytes != 1_000_000 || container.MemoryLimitBytes != 4_000_000) {
+ t.Fatalf("sample %d: memory collection changed: %+v", i, container)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/dockeragent/collect.go b/internal/dockeragent/collect.go
index 28c8b92874..9ca434c9f8 100644
--- a/internal/dockeragent/collect.go
+++ b/internal/dockeragent/collect.go
@@ -601,10 +601,14 @@ func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Sum
return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
}
- if a.runtime == RuntimePodman && podmanCPUPercent != nil {
- cpuPercent = safeFloat(*podmanCPUPercent)
- } else {
- cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats)
+ // Keep a baseline even when Podman supplies a percentage: one-shot
+ // compatibility responses can report cpu:0 while total_usage advances.
+ // Preserve positive native percentages (and their units, #1391), but
+ // use our interval delta for an absent, zero or invalid percentage.
+ cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats, parseTime(inspect.State.StartedAt))
+ if a.runtime == RuntimePodman && podmanCPUPercent != nil &&
+ *podmanCPUPercent > 0 && !math.IsInf(*podmanCPUPercent, 0) {
+ cpuPercent = *podmanCPUPercent
}
memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
blockIO = summarizeBlockIO(stats)
@@ -1138,7 +1142,7 @@ func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse,
return stats, probe.CPUStats.CPU, nil
}
-func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse) float64 {
+func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse, startedAt time.Time) float64 {
a.cpuMu.Lock()
defer a.cpuMu.Unlock()
@@ -1147,6 +1151,7 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container
systemUsage: stats.CPUStats.SystemUsage,
onlineCPUs: stats.CPUStats.OnlineCPUs,
read: stats.Read,
+ startedAt: startedAt,
}
// Always use manual delta tracking. Docker's PreCPUStats is unreliable for
@@ -1155,6 +1160,14 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container
// (from container start) and producing a constant lifetime-average CPU%
// instead of a current value.
prev, ok := a.prevContainerCPU[containerID]
+ if a.runtime == RuntimePodman && ok && !prev.read.IsZero() && !current.read.IsZero() && !current.read.After(prev.read) {
+ // Repeated or out-of-order responses must not move the baseline back
+ // and manufacture usage in the next interval.
+ return 0
+ }
+ if a.prevContainerCPU == nil {
+ a.prevContainerCPU = make(map[string]cpuSample)
+ }
if !ok {
// First time seeing this container - store current sample and return 0
// On next collection cycle we'll have a previous sample to compare against
@@ -1169,6 +1182,12 @@ func (a *Agent) calculateContainerCPUPercent(containerID string, stats container
// We have a previous sample - update it after calculation
a.prevContainerCPU[containerID] = current
+ if a.runtime == RuntimePodman && (current.totalUsage < prev.totalUsage ||
+ (!prev.startedAt.IsZero() && !current.startedAt.IsZero() && !current.startedAt.Equal(prev.startedAt))) {
+ // A restart is not an interval sample, even if the new lifetime counter
+ // has already exceeded the old one. Start again from this baseline.
+ return 0
+ }
var totalDelta float64
if current.totalUsage >= prev.totalUsage {