Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

feat(agent): collect nvtop video engine utilization (#2749)

Aditya rai325fbd47d45d
diff --git a/agent/gpu.go b/agent/gpu.go
index 2655c4f6c..04cb9f8db 100644
--- a/agent/gpu.go
+++ b/agent/gpu.go
@@ -458,7 +458,8 @@ func (gm *GPUManager) calculateGPUAverage(id string, gpu *system.GPUData, cacheK
// make fresh map for averaged engine metrics to avoid mutating
// the accumulator map stored in gm.GpuDataMap
gpuAvg.Engines = make(map[string]float64, len(gpu.Engines))
- gpuAvg.Usage = gm.calculateIntelGPUUsage(&gpuAvg, gpu, lastSnapshot, deltaCount)
+ // Usage is the busiest of overall utilization and any engine (e.g. nvtop media engines).
+ gpuAvg.Usage = max(utils.TwoDecimals(deltaUsage/float64(deltaCount)), gm.calculateEngineUsage(&gpuAvg, gpu, lastSnapshot, deltaCount))
} else {
gpuAvg.Usage = utils.TwoDecimals(deltaUsage / float64(deltaCount))
}
@@ -485,8 +486,8 @@ func (gm *GPUManager) calculateDeltas(gpu *system.GPUData, lastSnapshot *gpuSnap
return gpu.Usage, gpu.Power, gpu.PowerPkg
}
-// calculateIntelGPUUsage computes Intel GPU usage from engine metrics and returns max engine usage
-func (gm *GPUManager) calculateIntelGPUUsage(gpuAvg, gpu *system.GPUData, lastSnapshot *gpuSnapshot, deltaCount uint32) float64 {
+// calculateEngineUsage averages per-engine metrics into gpuAvg and returns the max engine usage
+func (gm *GPUManager) calculateEngineUsage(gpuAvg, gpu *system.GPUData, lastSnapshot *gpuSnapshot, deltaCount uint32) float64 {
maxEngineUsage := 0.0
for name, engine := range gpu.Engines {
var deltaEngine float64
diff --git a/agent/gpu_nvtop.go b/agent/gpu_nvtop.go
index 28136e8c6..0e991b7d9 100644
--- a/agent/gpu_nvtop.go
+++ b/agent/gpu_nvtop.go
@@ -15,12 +15,15 @@ import (
)
type nvtopSnapshot struct {
- DeviceName string `json:"device_name"`
- Temp *string `json:"temp"`
- PowerDraw *string `json:"power_draw"`
- GpuUtil *string `json:"gpu_util"`
- MemTotal *string `json:"mem_total"`
- MemUsed *string `json:"mem_used"`
+ DeviceName string `json:"device_name"`
+ Temp *string `json:"temp"`
+ PowerDraw *string `json:"power_draw"`
+ GpuUtil *string `json:"gpu_util"`
+ EncodeDecode *string `json:"encode_decode"` // devices with a shared media engine
+ Encode *string `json:"encode"` // devices with separate encoder and decoder
+ Decode *string `json:"decode"`
+ MemTotal *string `json:"mem_total"`
+ MemUsed *string `json:"mem_used"`
}
// parseNvtopNumber parses nvtop numeric strings with units (C/W/%).
@@ -95,6 +98,9 @@ func (gm *GPUManager) updateNvtopSnapshots(snapshots []nvtopSnapshot) bool {
if sample.GpuUtil != nil {
gpu.Usage += parseNvtopNumber(*sample.GpuUtil)
}
+ addNvtopEngine(gpu, "ENC/DEC", sample.EncodeDecode)
+ addNvtopEngine(gpu, "ENC", sample.Encode)
+ addNvtopEngine(gpu, "DEC", sample.Decode)
if sample.PowerDraw != nil {
gpu.Power += parseNvtopNumber(*sample.PowerDraw)
}
@@ -105,6 +111,17 @@ func (gm *GPUManager) updateNvtopSnapshots(snapshots []nvtopSnapshot) bool {
return valid
}
+// addNvtopEngine accumulates a media engine reading if nvtop reported one.
+func addNvtopEngine(gpu *system.GPUData, name string, value *string) {
+ if value == nil {
+ return
+ }
+ if gpu.Engines == nil {
+ gpu.Engines = make(map[string]float64)
+ }
+ gpu.Engines[name] += parseNvtopNumber(*value)
+}
+
// collectNvtopStats runs nvtop loop mode and continuously decodes JSON snapshots.
func (gm *GPUManager) collectNvtopStats(interval string) error {
cmd := exec.Command(nvtopCmd, "-lP", "-d", interval)
diff --git a/agent/gpu_nvtop_test.go b/agent/gpu_nvtop_test.go
new file mode 100644
index 000000000..d17b29d2d
--- /dev/null
+++ b/agent/gpu_nvtop_test.go
@@ -0,0 +1,73 @@
+package agent
+
+import (
+ "testing"
+
+ "github.com/henrygd/beszel/internal/entities/system"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNvtopEncodeDecode(t *testing.T) {
+ gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
+ // The device-level value from #2740 must be used, not the process value.
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"0%","encode_decode":"100%","processes":[{"encode_decode":"106%"}]}]`)))
+ first := gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 100.0, first.Usage)
+ assert.Equal(t, map[string]float64{"ENC/DEC": 100}, first.Engines)
+
+ // Average new samples per cache key, keeping 3D usage when it is busier.
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"80%","encode_decode":"20%"}]`)))
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"40%","encode_decode":"0%"}]`)))
+ next := gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 60.0, next.Usage)
+ assert.Equal(t, map[string]float64{"ENC/DEC": 10}, next.Engines)
+ assert.Equal(t, 100.0, first.Engines["ENC/DEC"], "averaging must not mutate earlier results")
+ other := gm.GetCurrentData(2)["n0"]
+ assert.Equal(t, 40.0, other.Usage)
+ assert.Equal(t, 40.0, other.Engines["ENC/DEC"])
+
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"0%","encode_decode":"0%"}]`)))
+ idle := gm.GetCurrentData(1)["n0"]
+ assert.Zero(t, idle.Usage)
+ assert.Zero(t, idle.Engines["ENC/DEC"])
+}
+
+func TestNvtopWithoutEncodeDecode(t *testing.T) {
+ for _, payload := range []string{
+ `[{"device_name":"GPU","gpu_util":"30%"}]`,
+ `[{"device_name":"GPU","gpu_util":"30%","encode_decode":null}]`,
+ } {
+ gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
+ require.True(t, gm.parseNvtopData([]byte(payload)))
+ gpu := gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 30.0, gpu.Usage)
+ assert.Nil(t, gpu.Engines)
+
+ // A later sample may start reporting the media engine.
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"GPU","gpu_util":"10%","encode_decode":"80%"}]`)))
+ gpu = gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 80.0, gpu.Usage)
+ assert.Equal(t, 80.0, gpu.Engines["ENC/DEC"])
+
+ // Missing or null readings must not reuse the previous media load.
+ require.True(t, gm.parseNvtopData([]byte(payload)))
+ gpu = gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 30.0, gpu.Usage)
+ assert.Zero(t, gpu.Engines["ENC/DEC"])
+ }
+}
+
+func TestNvtopSeparateEncodeDecode(t *testing.T) {
+ gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}
+ // nvtop emits separate encode/decode fields when the device's media engines are not shared.
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"NVIDIA GeForce RTX 3050","gpu_util":"5%","encode":"30%","decode":"70%"}]`)))
+ gpu := gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 70.0, gpu.Usage)
+ assert.Equal(t, map[string]float64{"ENC": 30, "DEC": 70}, gpu.Engines)
+
+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"NVIDIA GeForce RTX 3050","gpu_util":"50%","encode":"10%","decode":null}]`)))
+ gpu = gm.GetCurrentData(1)["n0"]
+ assert.Equal(t, 50.0, gpu.Usage)
+ assert.Equal(t, map[string]float64{"ENC": 10, "DEC": 0}, gpu.Engines)
+}
diff --git a/agent/gpu_test.go b/agent/gpu_test.go
index c1cbf1eb8..4f2f2bc4c 100644
--- a/agent/gpu_test.go
+++ b/agent/gpu_test.go
@@ -722,7 +722,7 @@ func TestCalculateDeltas(t *testing.T) {
})
}
-func TestCalculateIntelGPUUsage(t *testing.T) {
+func TestCalculateEngineUsage(t *testing.T) {
gm := &GPUManager{}
t.Run("with no previous snapshot", func(t *testing.T) {
@@ -736,7 +736,7 @@ func TestCalculateIntelGPUUsage(t *testing.T) {
"Compute": 60.0,
},
}
- maxUsage := gm.calculateIntelGPUUsage(gpuAvg, gpu, nil, 2)
+ maxUsage := gm.calculateEngineUsage(gpuAvg, gpu, nil, 2)
assert.Equal(t, 40.0, maxUsage, "Should return max engine usage (80/2=40)")
assert.Equal(t, 40.0, gpuAvg.Engines["Render/3D"])
@@ -762,7 +762,7 @@ func TestCalculateIntelGPUUsage(t *testing.T) {
"Compute": 60.0,
},
}
- maxUsage := gm.calculateIntelGPUUsage(gpuAvg, gpu, snapshot, 5)
+ maxUsage := gm.calculateEngineUsage(gpuAvg, gpu, snapshot, 5)
// Deltas: Render/3D=100, Video=60, Compute=80 over 5 counts
assert.Equal(t, 20.0, maxUsage, "Should return max engine delta (100/5=20)")
@@ -787,7 +787,7 @@ func TestCalculateIntelGPUUsage(t *testing.T) {
// NewEngine doesn't exist in snapshot
},
}
- maxUsage := gm.calculateIntelGPUUsage(gpuAvg, gpu, snapshot, 2)
+ maxUsage := gm.calculateEngineUsage(gpuAvg, gpu, snapshot, 2)
assert.Equal(t, 25.0, maxUsage)
assert.Equal(t, 10.0, gpuAvg.Engines["Render/3D"], "Should use delta for existing engine")
diff --git a/supplemental/CHANGELOG.md b/supplemental/CHANGELOG.md
index 331271e47..745336a22 100644
--- a/supplemental/CHANGELOG.md
+++ b/supplemental/CHANGELOG.md
@@ -10,6 +10,8 @@
- Add warnings for missing automatic login users and trusted proxy configuration (#2824, #2812)
+- Add video encode/decode engine usage to GPU metrics when using nvtop (#2749)
+
- Limit sign-in attempts to 10 per minute per address (#2642)
- Improve GPU collection retries and back off slow or repeatedly failing reads (#2705, #2687, #2709, #2656)