Lightweight server monitoring with historical data, docker stats, and alerts.
fix(agent): use time-based CPU calc for Podman containers (#2131)
Pavel Artsishevsky2df1f722e49d
diff --git a/agent/docker.go b/agent/docker.goindex 0bac24395..f10fffa51 100644--- a/agent/docker.go+++ b/agent/docker.go@@ -544,11 +544,18 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM// Get previous CPU valuesprevCpuContainer, prevCpuSystem := dm.getCpuPreviousValues(cacheTimeMs, ctr.IdShort)- // Calculate CPU percentage based on platform+ // Calculate CPU percentage based on platform.+ // Podman reports system_cpu_usage from cgroup cpu.stat (not /proc/stat), so it reflects+ // only cgroup-tracked activity rather than total host capacity. Use a time-based method+ // instead so the result is comparable to host CPU utilization. See:+ // https://github.com/henrygd/beszel/issues/2049var cpuPct float64if dm.isWindows {prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]cpuPct = res.CalculateCpuPercentWindows(prevCpuContainer, prevRead)+ } else if dm.usingPodman && res.CPUStats.OnlineCPUs > 0 {+ prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort]+ cpuPct = res.CalculateCpuPercentPodman(prevCpuContainer, prevRead)} else {cpuPct = res.CalculateCpuPercentLinux(prevCpuContainer, prevCpuSystem)}diff --git a/agent/docker_test.go b/agent/docker_test.goindex 77a965f30..0ba67aff5 100644--- a/agent/docker_test.go+++ b/agent/docker_test.go@@ -1059,6 +1059,162 @@ func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) {assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0)}+func TestCalculateCpuPercentPodman(t *testing.T) {+ baseTime := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC)++ tests := []struct {+ name string+ prevCpuContainer uint64+ prevRead time.Time+ currentUsage uint64+ currentRead time.Time+ onlineCPUs uint32+ expectedPct float64+ }{+ {+ name: "normal calculation",+ // container used 2ms of CPU over 1s with 2 CPUs → 0.1%+ prevCpuContainer: 1_000_000_000,+ prevRead: baseTime,+ currentUsage: 1_002_000_000, // +2ms CPU time+ currentRead: baseTime.Add(time.Second),+ onlineCPUs: 2,+ expectedPct: 0.1, // 2e6 / (1e9 * 2) * 100+ },+ {+ name: "first run returns zero",+ prevCpuContainer: 0,+ prevRead: baseTime,+ currentUsage: 5_000_000,+ currentRead: baseTime.Add(time.Second),+ onlineCPUs: 4,+ expectedPct: 0.0,+ },+ {+ name: "zero online cpus returns zero",+ prevCpuContainer: 1_000_000_000,+ prevRead: baseTime,+ currentUsage: 1_010_000_000,+ currentRead: baseTime.Add(time.Second),+ onlineCPUs: 0,+ expectedPct: 0.0,+ },+ {+ name: "same read time returns zero",+ prevCpuContainer: 1_000_000_000,+ prevRead: baseTime,+ currentUsage: 1_010_000_000,+ currentRead: baseTime, // no elapsed time+ onlineCPUs: 2,+ expectedPct: 0.0,+ },+ {+ name: "counter rollback returns zero",+ prevCpuContainer: 2_000_000_000,+ prevRead: baseTime,+ currentUsage: 1_000_000_000,+ currentRead: baseTime.Add(time.Second),+ onlineCPUs: 2,+ expectedPct: 0.0,+ },+ {+ name: "100% single cpu",+ // container consumed a full CPU-second over 1s on a 1-CPU host → 100%+ prevCpuContainer: 1_000_000_000,+ prevRead: baseTime,+ currentUsage: 2_000_000_000, // +1s CPU time+ currentRead: baseTime.Add(time.Second),+ onlineCPUs: 1,+ expectedPct: 100.0, // 1e9 / (1e9 * 1) * 100+ },+ {+ name: "high utilization on multi-cpu host",+ // container used 800ms on a 4-CPU host over 1s → 20%+ prevCpuContainer: 10_000_000_000,+ prevRead: baseTime,+ currentUsage: 10_800_000_000,+ currentRead: baseTime.Add(time.Second),+ onlineCPUs: 4,+ expectedPct: 20.0, // 800e6 / (1e9 * 4) * 100+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := &container.ApiStats{+ Read: tt.currentRead,+ CPUStats: container.CPUStats{+ CPUUsage: container.CPUUsage{TotalUsage: tt.currentUsage},+ OnlineCPUs: tt.onlineCPUs,+ },+ }+ got := s.CalculateCpuPercentPodman(tt.prevCpuContainer, tt.prevRead)+ assert.InDelta(t, tt.expectedPct, got, 0.001, "test %q", tt.name)+ })+ }+}++func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {+ // Verify that Podman containers use the time-based CPU calculation+ // when online_cpus is provided in the stats response.+ // container used 20ms CPU over 1s with 2 CPUs → 1%+ prevReadTime := time.Date(2026, 3, 15, 21, 26, 58, 0, time.UTC) // 1 second before stats read+ const prevCpuUsage = uint64(5_000_000_000)++ dm := &dockerManager{+ client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {+ switch req.URL.EscapedPath() {+ case "/containers/0123456789ab/stats":+ return &http.Response{+ StatusCode: http.StatusOK,+ Status: "200 OK",+ Header: make(http.Header),+ Body: io.NopCloser(strings.NewReader(`{+ "read":"2026-03-15T21:26:59Z",+ "cpu_stats":{"cpu_usage":{"total_usage":5020000000},"system_cpu_usage":9999999,"online_cpus":2},+ "memory_stats":{"usage":1048576,"stats":{"inactive_file":262144}},+ "networks":{"eth0":{"rx_bytes":0,"tx_bytes":0}}+ }`)),+ Request: req,+ }, nil+ default:+ return nil, fmt.Errorf("unexpected path: %s", req.URL.EscapedPath())+ }+ })},+ containerStatsMap: make(map[string]*container.Stats),+ apiStats: &container.ApiStats{},+ usingPodman: true,+ lastCpuContainer: map[uint16]map[string]uint64{+ defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},+ },+ lastCpuSystem: map[uint16]map[string]uint64{+ defaultCacheTimeMs: {"0123456789ab": 1}, // intentionally tiny — should NOT be used+ },+ lastCpuReadTime: map[uint16]map[string]time.Time{+ defaultCacheTimeMs: {"0123456789ab": prevReadTime},+ },+ networkSentTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),+ networkRecvTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]),+ lastNetworkReadTime: make(map[uint16]map[string]time.Time),+ }++ ctr := &container.ApiInfo{+ IdShort: "0123456789ab",+ Names: []string{"/myapp"},+ Status: "Up 5 minutes",+ Image: "myapp:latest",+ }++ err := dm.updateContainerStats(ctr, defaultCacheTimeMs)+ require.NoError(t, err)++ // cpu delta = 5020000000 - 5000000000 = 20000000 ns (20ms)+ // elapsed = 1s = 1000000000 ns, online_cpus = 2+ // expected = 20000000 / (1000000000 * 2) * 100 = 1.0%+ expectedCpu := 1.0+ assert.InDelta(t, expectedCpu, dm.containerStatsMap[ctr.IdShort].Cpu, 0.01)+}+func TestNetworkStatsCalculationWithRealData(t *testing.T) {// Create synthetic test data to avoid timing issuesapiStats1 := &container.ApiStats{diff --git a/internal/entities/container/container.go b/internal/entities/container/container.goindex 989d4b2b2..ef3090172 100644--- a/internal/entities/container/container.go+++ b/internal/entities/container/container.go@@ -74,6 +74,30 @@ func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSystreturn float64(cpuDelta) / float64(systemDelta) * 100.0}+// CalculateCpuPercentPodman calculates CPU percentage for Podman containers.+// Podman populates system_cpu_usage from cgroup cpu.stat rather than /proc/stat, so it+// represents only cgroup-accounted activity, not total host CPU capacity. Using it as+// a denominator inflates the result. Instead we use elapsed wall-clock time × online_cpus,+// matching the approach used for Windows and recommended in:+// https://github.com/henrygd/beszel/issues/2049+func (s *ApiStats) CalculateCpuPercentPodman(prevCpuContainer uint64, prevRead time.Time) float64 {+ if prevCpuContainer == 0 || s.CPUStats.OnlineCPUs == 0 {+ return 0.0+ }+ // Treat a reset or out-of-order counter as a new baseline instead of+ // allowing unsigned subtraction to wrap to an enormous percentage.+ if s.CPUStats.CPUUsage.TotalUsage < prevCpuContainer {+ return 0.0+ }+ cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer+ elapsedNs := uint64(s.Read.Sub(prevRead).Nanoseconds())+ systemCapacity := elapsedNs * uint64(s.CPUStats.OnlineCPUs)+ if systemCapacity == 0 {+ return 0.0+ }+ return float64(cpuDelta) / float64(systemCapacity) * 100.0+}+// from: https://github.com/docker/cli/blob/master/cli/command/container/stats_helpers.go#L185func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time.Time) float64 {// Max number of 100ns intervals between the previous time read and now@@ -98,8 +122,10 @@ func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead timetype CPUStats struct {// CPU Usage. Linux and Windows.CPUUsage CPUUsage `json:"cpu_usage"`- // System Usage. Linux only.+ // System Usage. Linux only. Populated from /proc/stat on Docker; from cgroup cpu.stat on Podman.SystemUsage uint64 `json:"system_cpu_usage,omitempty"`+ // Number of online CPUs. Linux only. Used by Podman for time-based CPU calculation.+ OnlineCPUs uint32 `json:"online_cpus,omitempty"`}type CPUUsage struct {