Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

fix(agent): use time-based CPU calc for Podman containers (#2131)

Pavel Artsishevsky2df1f722e49d
diff --git a/agent/docker.go b/agent/docker.go
index 0bac24395..f10fffa51 100644
--- a/agent/docker.go
+++ b/agent/docker.go
@@ -544,11 +544,18 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
// Get previous CPU values
prevCpuContainer, prevCpuSystem := dm.getCpuPreviousValues(cacheTimeMs, ctr.IdShort)
- // Calculate CPU percentage based on platform
+ // Calculate CPU percentage based on platform.
+ // Podman reports system_cpu_usage from cgroup cpu.stat (not /proc/stat), so it reflects
+ // only cgroup-tracked activity rather than total host capacity. Use a time-based method
+ // instead so the result is comparable to host CPU utilization. See:
+ // https://github.com/henrygd/beszel/issues/2049
var cpuPct float64
if dm.isWindows {
prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
cpuPct = res.CalculateCpuPercentWindows(prevCpuContainer, prevRead)
+ } else if dm.usingPodman && res.CPUStats.OnlineCPUs > 0 {
+ prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]
+ cpuPct = res.CalculateCpuPercentPodman(prevCpuContainer, prevRead)
} else {
cpuPct = res.CalculateCpuPercentLinux(prevCpuContainer, prevCpuSystem)
}
diff --git a/agent/docker_test.go b/agent/docker_test.go
index 77a965f30..0ba67aff5 100644
--- a/agent/docker_test.go
+++ b/agent/docker_test.go
@@ -1059,6 +1059,162 @@ func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) {
assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)
}
+func TestCalculateCpuPercentPodman(t *testing.T) {
+ baseTime := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)
+
+ tests := []struct {
+ name string
+ prevCpuContainer uint64
+ prevRead time.Time
+ currentUsage uint64
+ currentRead time.Time
+ onlineCPUs uint32
+ expectedPct float64
+ }{
+ {
+ name: "normal calculation",
+ // container used 2ms of CPU over 1s with 2 CPUs → 0.1%
+ prevCpuContainer: 1_000_000_000,
+ prevRead: baseTime,
+ currentUsage: 1_002_000_000, // +2ms CPU time
+ currentRead: baseTime.Add(time.Second),
+ onlineCPUs: 2,
+ expectedPct: 0.1, // 2e6 / (1e9 * 2) * 100
+ },
+ {
+ name: "first run returns zero",
+ prevCpuContainer: 0,
+ prevRead: baseTime,
+ currentUsage: 5_000_000,
+ currentRead: baseTime.Add(time.Second),
+ onlineCPUs: 4,
+ expectedPct: 0.0,
+ },
+ {
+ name: "zero online cpus returns zero",
+ prevCpuContainer: 1_000_000_000,
+ prevRead: baseTime,
+ currentUsage: 1_010_000_000,
+ currentRead: baseTime.Add(time.Second),
+ onlineCPUs: 0,
+ expectedPct: 0.0,
+ },
+ {
+ name: "same read time returns zero",
+ prevCpuContainer: 1_000_000_000,
+ prevRead: baseTime,
+ currentUsage: 1_010_000_000,
+ currentRead: baseTime, // no elapsed time
+ onlineCPUs: 2,
+ expectedPct: 0.0,
+ },
+ {
+ name: "counter rollback returns zero",
+ prevCpuContainer: 2_000_000_000,
+ prevRead: baseTime,
+ currentUsage: 1_000_000_000,
+ currentRead: baseTime.Add(time.Second),
+ onlineCPUs: 2,
+ expectedPct: 0.0,
+ },
+ {
+ name: "100% single cpu",
+ // container consumed a full CPU-second over 1s on a 1-CPU host → 100%
+ prevCpuContainer: 1_000_000_000,
+ prevRead: baseTime,
+ currentUsage: 2_000_000_000, // +1s CPU time
+ currentRead: baseTime.Add(time.Second),
+ onlineCPUs: 1,
+ expectedPct: 100.0, // 1e9 / (1e9 * 1) * 100
+ },
+ {
+ name: "high utilization on multi-cpu host",
+ // container used 800ms on a 4-CPU host over 1s → 20%
+ prevCpuContainer: 10_000_000_000,
+ prevRead: baseTime,
+ currentUsage: 10_800_000_000,
+ currentRead: baseTime.Add(time.Second),
+ onlineCPUs: 4,
+ expectedPct: 20.0, // 800e6 / (1e9 * 4) * 100
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := &container.ApiStats{
+ Read: tt.currentRead,
+ CPUStats: container.CPUStats{
+ CPUUsage: container.CPUUsage{TotalUsage: tt.currentUsage},
+ OnlineCPUs: tt.onlineCPUs,
+ },
+ }
+ got := s.CalculateCpuPercentPodman(tt.prevCpuContainer, tt.prevRead)
+ assert.InDelta(t, tt.expectedPct, got, 0.001, "test %q", tt.name)
+ })
+ }
+}
+
+func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
+ // Verify that Podman containers use the time-based CPU calculation
+ // when online_cpus is provided in the stats response.
+ // container used 20ms CPU over 1s with 2 CPUs → 1%
+ prevReadTime := time.Date(2026, 3, 15, 21, 26, 58, 0, time.UTC) // 1 second before stats read
+ const prevCpuUsage = uint64(5_000_000_000)
+
+ dm := &dockerManager{
+ client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ switch req.URL.EscapedPath() {
+ case "/containers/0123456789ab/stats":
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Status: "200 OK",
+ Header: make(http.Header),
+ Body: io.NopCloser(strings.NewReader(`{
+ "read":"2026-03-15T21:26:59Z",
+ "cpu_stats":{"cpu_usage":{"total_usage":5020000000},"system_cpu_usage":9999999,"online_cpus":2},
+ "memory_stats":{"usage":1048576,"stats":{"inactive_file":262144}},
+ "networks":{"eth0":{"rx_bytes":0,"tx_bytes":0}}
+ }`)),
+ Request: req,
+ }, nil
+ default:
+ return nil, fmt.Errorf("unexpected path: %s", req.URL.EscapedPath())
+ }
+ })},
+ containerStatsMap: make(map[string]*container.Stats),
+ apiStats: &container.ApiStats{},
+ usingPodman: true,
+ lastCpuContainer: map[uint16]map[string]uint64{
+ defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
+ },
+ lastCpuSystem: map[uint16]map[string]uint64{
+ defaultCacheTimeMs: {"0123456789ab": 1}, // intentionally tiny — should NOT be used
+ },
+ lastCpuReadTime: map[uint16]map[string]time.Time{
+ defaultCacheTimeMs: {"0123456789ab": prevReadTime},
+ },
+ networkSentTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
+ networkRecvTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),
+ lastNetworkReadTime: make(map[uint16]map[string]time.Time),
+ }
+
+ ctr := &container.ApiInfo{
+ IdShort: "0123456789ab",
+ Names: []string{"/myapp"},
+ Status: "Up 5 minutes",
+ Image: "myapp:latest",
+ }
+
+ err := dm.updateContainerStats(ctr, defaultCacheTimeMs)
+ require.NoError(t, err)
+
+ // cpu delta = 5020000000 - 5000000000 = 20000000 ns (20ms)
+ // elapsed = 1s = 1000000000 ns, online_cpus = 2
+ // expected = 20000000 / (1000000000 * 2) * 100 = 1.0%
+ expectedCpu := 1.0
+ assert.InDelta(t, expectedCpu, dm.containerStatsMap[ctr.IdShort].Cpu, 0.01)
+}
+
func TestNetworkStatsCalculationWithRealData(t *testing.T) {
// Create synthetic test data to avoid timing issues
apiStats1 := &container.ApiStats{
diff --git a/internal/entities/container/container.go b/internal/entities/container/container.go
index 989d4b2b2..ef3090172 100644
--- a/internal/entities/container/container.go
+++ b/internal/entities/container/container.go
@@ -74,6 +74,30 @@ func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSyst
return float64(cpuDelta) / float64(systemDelta) * 100.0
}
+// CalculateCpuPercentPodman calculates CPU percentage for Podman containers.
+// Podman populates system_cpu_usage from cgroup cpu.stat rather than /proc/stat, so it
+// represents only cgroup-accounted activity, not total host CPU capacity. Using it as
+// a denominator inflates the result. Instead we use elapsed wall-clock time × online_cpus,
+// matching the approach used for Windows and recommended in:
+// https://github.com/henrygd/beszel/issues/2049
+func (s *ApiStats) CalculateCpuPercentPodman(prevCpuContainer uint64, prevRead time.Time) float64 {
+ if prevCpuContainer == 0 || s.CPUStats.OnlineCPUs == 0 {
+ return 0.0
+ }
+ // Treat a reset or out-of-order counter as a new baseline instead of
+ // allowing unsigned subtraction to wrap to an enormous percentage.
+ if s.CPUStats.CPUUsage.TotalUsage < prevCpuContainer {
+ return 0.0
+ }
+ cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer
+ elapsedNs := uint64(s.Read.Sub(prevRead).Nanoseconds())
+ systemCapacity := elapsedNs * uint64(s.CPUStats.OnlineCPUs)
+ if systemCapacity == 0 {
+ return 0.0
+ }
+ return float64(cpuDelta) / float64(systemCapacity) * 100.0
+}
+
// from: https://github.com/docker/cli/blob/master/cli/command/container/stats_helpers.go#L185
func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time.Time) float64 {
// Max number of 100ns intervals between the previous time read and now
@@ -98,8 +122,10 @@ func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time
type CPUStats struct {
// CPU Usage. Linux and Windows.
CPUUsage CPUUsage `json:"cpu_usage"`
- // System Usage. Linux only.
+ // System Usage. Linux only. Populated from /proc/stat on Docker; from cgroup cpu.stat on Podman.
SystemUsage uint64 `json:"system_cpu_usage,omitempty"`
+ // Number of online CPUs. Linux only. Used by Podman for time-based CPU calculation.
+ OnlineCPUs uint32 `json:"online_cpus,omitempty"`
}
type CPUUsage struct {