Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

5.8 KB
Raw Blame
package agent

import (
	"encoding/json"
	"io"
	"log/slog"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/henrygd/beszel/agent/utils"
	"github.com/henrygd/beszel/internal/entities/system"
)

type nvtopSnapshot struct {
	DeviceName   string  `json:"device_name"`
	Temp         *string `json:"temp"`
	PowerDraw    *string `json:"power_draw"`
	GpuUtil      *string `json:"gpu_util"`
	EncodeDecode *string `json:"encode_decode"` // devices with a shared media engine
	Encode       *string `json:"encode"`        // devices with separate encoder and decoder
	Decode       *string `json:"decode"`
	MemTotal     *string `json:"mem_total"`
	MemUsed      *string `json:"mem_used"`
}

// parseNvtopNumber parses nvtop numeric strings with units (C/W/%).
func parseNvtopNumber(raw string) float64 {
	cleaned := strings.TrimSpace(raw)
	cleaned = strings.TrimSuffix(cleaned, "C")
	cleaned = strings.TrimSuffix(cleaned, "W")
	cleaned = strings.TrimSuffix(cleaned, "%")
	val, _ := strconv.ParseFloat(cleaned, 64)
	return val
}

// parseNvtopData parses a single nvtop JSON snapshot payload.
func (gm *GPUManager) parseNvtopData(output []byte) bool {
	var snapshots []nvtopSnapshot
	if err := json.Unmarshal(output, &snapshots); err != nil || len(snapshots) == 0 {
		return false
	}
	return gm.updateNvtopSnapshots(snapshots)
}

// updateNvtopSnapshots applies one decoded nvtop snapshot batch to GPU accumulators.
func (gm *GPUManager) updateNvtopSnapshots(snapshots []nvtopSnapshot) bool {
	gm.Lock()
	defer gm.Unlock()

	valid := false
	usedIDs := make(map[string]struct{}, len(snapshots))
	var xeName string
	for i, sample := range snapshots {
		// nvtop leaves device_name unset on xe devices.
		if sample.DeviceName == "" {
			if xeName == "" {
				xeName = xeGpuName()
			}
			sample.DeviceName = xeName
		}
		indexID := "n" + strconv.Itoa(i)
		id := indexID

		// nvtop ordering can change, so prefer reusing an existing slot with matching device name.
		if existingByIndex, ok := gm.GpuDataMap[indexID]; ok && existingByIndex.Name != "" && existingByIndex.Name != sample.DeviceName {
			for existingID, gpu := range gm.GpuDataMap {
				if !strings.HasPrefix(existingID, "n") {
					continue
				}
				if _, taken := usedIDs[existingID]; taken {
					continue
				}
				if gpu.Name == sample.DeviceName {
					id = existingID
					break
				}
			}
		}

		if _, ok := gm.GpuDataMap[id]; !ok {
			gm.GpuDataMap[id] = &system.GPUData{Name: sample.DeviceName}
		}
		gpu := gm.GpuDataMap[id]
		gpu.Name = sample.DeviceName

		if sample.Temp != nil {
			gpu.Temperature = parseNvtopNumber(*sample.Temp)
		}
		if sample.MemUsed != nil {
			gpu.MemoryUsed = utils.BytesToMegabytes(parseNvtopNumber(*sample.MemUsed))
		}
		if sample.MemTotal != nil {
			gpu.MemoryTotal = utils.BytesToMegabytes(parseNvtopNumber(*sample.MemTotal))
		}
		if sample.GpuUtil != nil {
			gpu.Usage += parseNvtopNumber(*sample.GpuUtil)
		}
		addNvtopEngine(gpu, "ENC/DEC", sample.EncodeDecode)
		addNvtopEngine(gpu, "ENC", sample.Encode)
		addNvtopEngine(gpu, "DEC", sample.Decode)
		if sample.PowerDraw != nil {
			gpu.Power += parseNvtopNumber(*sample.PowerDraw)
		}
		gpu.Count++
		usedIDs[id] = struct{}{}
		valid = true
	}
	return valid
}

// addNvtopEngine accumulates a media engine reading if nvtop reported one.
func addNvtopEngine(gpu *system.GPUData, name string, value *string) {
	if value == nil {
		return
	}
	if gpu.Engines == nil {
		gpu.Engines = make(map[string]float64)
	}
	gpu.Engines[name] += parseNvtopNumber(*value)
}

// collectNvtopStats runs nvtop loop mode and continuously decodes JSON snapshots.
func (gm *GPUManager) collectNvtopStats(interval string) error {
	cmd := exec.Command(nvtopCmd, "-lP", "-d", interval)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	defer func() {
		_ = stdout.Close()
		if cmd.ProcessState == nil || !cmd.ProcessState.Exited() {
			_ = cmd.Process.Kill()
		}
		_ = cmd.Wait()
	}()

	decoder := json.NewDecoder(stdout)
	foundValid := false
	for {
		var snapshots []nvtopSnapshot
		if err := decoder.Decode(&snapshots); err != nil {
			if err == io.EOF {
				if foundValid {
					return nil
				}
				return errNoValidData
			}
			return err
		}
		if gm.updateNvtopSnapshots(snapshots) {
			foundValid = true
		}
	}
}

// startNvtopCollector starts nvtop collection with retry or fallback callback handling.
func (gm *GPUManager) startNvtopCollector(interval string, onFailure func()) {
	go func() {
		failures := 0
		for {
			if err := gm.collectNvtopStats(interval); err != nil {
				if onFailure != nil {
					slog.Warn("Error collecting GPU data via nvtop", "err", err)
					onFailure()
					return
				}
				failures++
				if failures > maxFailureRetries {
					break
				}
				slog.Warn("Error collecting GPU data via nvtop", "err", err)
				time.Sleep(retryWaitTime)
				continue
			}
		}
	}()
}

// xeDevicePath returns the sysfs device path of the first xe GPU, or "".
func xeDevicePath() string {
	cards, err := filepath.Glob("/sys/class/drm/card*")
	if err != nil {
		return ""
	}
	for _, card := range cards {
		if strings.Contains(filepath.Base(card), "-") {
			continue
		}
		if uevent, err := utils.ReadStringFileLimited(filepath.Join(card, "device", "uevent"), 4096); err == nil && strings.Contains(uevent, "DRIVER=xe") {
			return filepath.Join(card, "device")
		}
	}
	return ""
}

func (gm *GPUManager) hasXe() bool {
	return xeDevicePath() != ""
}

// xeGpuName names an xe GPU from its PCI device id; nvtop leaves device_name unset on xe.
func xeGpuName() string {
	devicePath := xeDevicePath()
	if devicePath == "" {
		return "GPU"
	}
	id, err := utils.ReadStringFileLimited(filepath.Join(devicePath, "device"), 64)
	if err != nil {
		return "GPU"
	}
	id = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(id, "0x")))
	return "Intel GPU (" + id + ")"
}