Lightweight server monitoring with historical data, docker stats, and alerts.
feat(agent): collect nvtop video engine utilization (#2749)
Aditya rai325fbd47d45d
diff --git a/agent/gpu.go b/agent/gpu.goindex 2655c4f6c..04cb9f8db 100644--- a/agent/gpu.go+++ b/agent/gpu.go@@ -458,7 +458,8 @@ func (gm *GPUManager) calculateGPUAverage(id string, gpu *system.GPUData, cacheK// make fresh map for averaged engine metrics to avoid mutating// the accumulator map stored in gm.GpuDataMapgpuAvg.Engines = make(map[string]float64, len(gpu.Engines))- gpuAvg.Usage = gm.calculateIntelGPUUsage(&gpuAvg, gpu, lastSnapshot, deltaCount)+ // Usage is the busiest of overall utilization and any engine (e.g. nvtop media engines).+ gpuAvg.Usage = max(utils.TwoDecimals(deltaUsage/float64(deltaCount)), gm.calculateEngineUsage(&gpuAvg, gpu, lastSnapshot, deltaCount))} else {gpuAvg.Usage = utils.TwoDecimals(deltaUsage / float64(deltaCount))}@@ -485,8 +486,8 @@ func (gm *GPUManager) calculateDeltas(gpu *system.GPUData, lastSnapshot *gpuSnapreturn gpu.Usage, gpu.Power, gpu.PowerPkg}-// calculateIntelGPUUsage computes Intel GPU usage from engine metrics and returns max engine usage-func (gm *GPUManager) calculateIntelGPUUsage(gpuAvg, gpu *system.GPUData, lastSnapshot *gpuSnapshot, deltaCount uint32) float64 {+// calculateEngineUsage averages per-engine metrics into gpuAvg and returns the max engine usage+func (gm *GPUManager) calculateEngineUsage(gpuAvg, gpu *system.GPUData, lastSnapshot *gpuSnapshot, deltaCount uint32) float64 {maxEngineUsage := 0.0for name, engine := range gpu.Engines {var deltaEngine float64diff --git a/agent/gpu_nvtop.go b/agent/gpu_nvtop.goindex 28136e8c6..0e991b7d9 100644--- a/agent/gpu_nvtop.go+++ b/agent/gpu_nvtop.go@@ -15,12 +15,15 @@ import ()type nvtopSnapshot struct {- DeviceName string `json:"device_name"`- Temp *string `json:"temp"`- PowerDraw *string `json:"power_draw"`- GpuUtil *string `json:"gpu_util"`- MemTotal *string `json:"mem_total"`- MemUsed *string `json:"mem_used"`+ DeviceName string `json:"device_name"`+ Temp *string `json:"temp"`+ PowerDraw *string `json:"power_draw"`+ GpuUtil *string `json:"gpu_util"`+ EncodeDecode *string `json:"encode_decode"` // devices with a shared media engine+ Encode *string `json:"encode"` // devices with separate encoder and decoder+ Decode *string `json:"decode"`+ MemTotal *string `json:"mem_total"`+ MemUsed *string `json:"mem_used"`}// parseNvtopNumber parses nvtop numeric strings with units (C/W/%).@@ -95,6 +98,9 @@ func (gm *GPUManager) updateNvtopSnapshots(snapshots []nvtopSnapshot) bool {if sample.GpuUtil != nil {gpu.Usage += parseNvtopNumber(*sample.GpuUtil)}+ addNvtopEngine(gpu, "ENC/DEC", sample.EncodeDecode)+ addNvtopEngine(gpu, "ENC", sample.Encode)+ addNvtopEngine(gpu, "DEC", sample.Decode)if sample.PowerDraw != nil {gpu.Power += parseNvtopNumber(*sample.PowerDraw)}@@ -105,6 +111,17 @@ func (gm *GPUManager) updateNvtopSnapshots(snapshots []nvtopSnapshot) bool {return valid}+// addNvtopEngine accumulates a media engine reading if nvtop reported one.+func addNvtopEngine(gpu *system.GPUData, name string, value *string) {+ if value == nil {+ return+ }+ if gpu.Engines == nil {+ gpu.Engines = make(map[string]float64)+ }+ gpu.Engines[name] += parseNvtopNumber(*value)+}+// collectNvtopStats runs nvtop loop mode and continuously decodes JSON snapshots.func (gm *GPUManager) collectNvtopStats(interval string) error {cmd := exec.Command(nvtopCmd, "-lP", "-d", interval)diff --git a/agent/gpu_nvtop_test.go b/agent/gpu_nvtop_test.gonew file mode 100644index 000000000..d17b29d2d--- /dev/null+++ b/agent/gpu_nvtop_test.go@@ -0,0 +1,73 @@+package agent++import (+ "testing"++ "github.com/henrygd/beszel/internal/entities/system"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestNvtopEncodeDecode(t *testing.T) {+ gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}+ // The device-level value from #2740 must be used, not the process value.+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"0%","encode_decode":"100%","processes":[{"encode_decode":"106%"}]}]`)))+ first := gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 100.0, first.Usage)+ assert.Equal(t, map[string]float64{"ENC/DEC": 100}, first.Engines)++ // Average new samples per cache key, keeping 3D usage when it is busier.+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"80%","encode_decode":"20%"}]`)))+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"40%","encode_decode":"0%"}]`)))+ next := gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 60.0, next.Usage)+ assert.Equal(t, map[string]float64{"ENC/DEC": 10}, next.Engines)+ assert.Equal(t, 100.0, first.Engines["ENC/DEC"], "averaging must not mutate earlier results")+ other := gm.GetCurrentData(2)["n0"]+ assert.Equal(t, 40.0, other.Usage)+ assert.Equal(t, 40.0, other.Engines["ENC/DEC"])++ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"Arrow Lake-S (Intel Graphics)","gpu_util":"0%","encode_decode":"0%"}]`)))+ idle := gm.GetCurrentData(1)["n0"]+ assert.Zero(t, idle.Usage)+ assert.Zero(t, idle.Engines["ENC/DEC"])+}++func TestNvtopWithoutEncodeDecode(t *testing.T) {+ for _, payload := range []string{+ `[{"device_name":"GPU","gpu_util":"30%"}]`,+ `[{"device_name":"GPU","gpu_util":"30%","encode_decode":null}]`,+ } {+ gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}+ require.True(t, gm.parseNvtopData([]byte(payload)))+ gpu := gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 30.0, gpu.Usage)+ assert.Nil(t, gpu.Engines)++ // A later sample may start reporting the media engine.+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"GPU","gpu_util":"10%","encode_decode":"80%"}]`)))+ gpu = gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 80.0, gpu.Usage)+ assert.Equal(t, 80.0, gpu.Engines["ENC/DEC"])++ // Missing or null readings must not reuse the previous media load.+ require.True(t, gm.parseNvtopData([]byte(payload)))+ gpu = gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 30.0, gpu.Usage)+ assert.Zero(t, gpu.Engines["ENC/DEC"])+ }+}++func TestNvtopSeparateEncodeDecode(t *testing.T) {+ gm := &GPUManager{GpuDataMap: make(map[string]*system.GPUData)}+ // nvtop emits separate encode/decode fields when the device's media engines are not shared.+ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"NVIDIA GeForce RTX 3050","gpu_util":"5%","encode":"30%","decode":"70%"}]`)))+ gpu := gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 70.0, gpu.Usage)+ assert.Equal(t, map[string]float64{"ENC": 30, "DEC": 70}, gpu.Engines)++ require.True(t, gm.parseNvtopData([]byte(`[{"device_name":"NVIDIA GeForce RTX 3050","gpu_util":"50%","encode":"10%","decode":null}]`)))+ gpu = gm.GetCurrentData(1)["n0"]+ assert.Equal(t, 50.0, gpu.Usage)+ assert.Equal(t, map[string]float64{"ENC": 10, "DEC": 0}, gpu.Engines)+}diff --git a/agent/gpu_test.go b/agent/gpu_test.goindex c1cbf1eb8..4f2f2bc4c 100644--- a/agent/gpu_test.go+++ b/agent/gpu_test.go@@ -722,7 +722,7 @@ func TestCalculateDeltas(t *testing.T) {})}-func TestCalculateIntelGPUUsage(t *testing.T) {+func TestCalculateEngineUsage(t *testing.T) {gm := &GPUManager{}t.Run("with no previous snapshot", func(t *testing.T) {@@ -736,7 +736,7 @@ func TestCalculateIntelGPUUsage(t *testing.T) {"Compute": 60.0,},}- maxUsage := gm.calculateIntelGPUUsage(gpuAvg, gpu, nil, 2)+ maxUsage := gm.calculateEngineUsage(gpuAvg, gpu, nil, 2)assert.Equal(t, 40.0, maxUsage, "Should return max engine usage (80/2=40)")assert.Equal(t, 40.0, gpuAvg.Engines["Render/3D"])@@ -762,7 +762,7 @@ func TestCalculateIntelGPUUsage(t *testing.T) {"Compute": 60.0,},}- maxUsage := gm.calculateIntelGPUUsage(gpuAvg, gpu, snapshot, 5)+ maxUsage := gm.calculateEngineUsage(gpuAvg, gpu, snapshot, 5)// Deltas: Render/3D=100, Video=60, Compute=80 over 5 countsassert.Equal(t, 20.0, maxUsage, "Should return max engine delta (100/5=20)")@@ -787,7 +787,7 @@ func TestCalculateIntelGPUUsage(t *testing.T) {// NewEngine doesn't exist in snapshot},}- maxUsage := gm.calculateIntelGPUUsage(gpuAvg, gpu, snapshot, 2)+ maxUsage := gm.calculateEngineUsage(gpuAvg, gpu, snapshot, 2)assert.Equal(t, 25.0, maxUsage)assert.Equal(t, 10.0, gpuAvg.Engines["Render/3D"], "Should use delta for existing engine")diff --git a/supplemental/CHANGELOG.md b/supplemental/CHANGELOG.mdindex 331271e47..745336a22 100644--- a/supplemental/CHANGELOG.md+++ b/supplemental/CHANGELOG.md@@ -10,6 +10,8 @@- Add warnings for missing automatic login users and trusted proxy configuration (#2824, #2812)+- Add video encode/decode engine usage to GPU metrics when using nvtop (#2749)+- Limit sign-in attempts to 10 per minute per address (#2642)- Improve GPU collection retries and back off slow or repeatedly failing reads (#2705, #2687, #2709, #2656)