Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

28.3 KB
Raw Blame
package agent

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"log/slog"
	"maps"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/henrygd/beszel/agent/utils"
	"github.com/henrygd/beszel/internal/entities/system"
)

const (
	// Commands
	nvidiaSmiCmd    string = "nvidia-smi"
	rocmSmiCmd      string = "rocm-smi"
	tegraStatsCmd   string = "tegrastats"
	nvtopCmd        string = "nvtop"
	powermetricsCmd string = "powermetrics"
	macmonCmd       string = "macmon"
	noGPUFoundMsg   string = "no GPU found - see https://beszel.dev/guide/gpu"

	// Command retry and timeout constants
	retryWaitTime     time.Duration = 5 * time.Second
	maxRetryBackoff   time.Duration = 5 * time.Minute
	maxFailureRetries int           = 5

	// Unit Conversions
	mebibytesInAMegabyte float64 = 1.024  // nvidia-smi reports memory in MiB
	milliwattsInAWatt    float64 = 1000.0 // tegrastats reports power in mW
)

// GPUManager manages data collection for GPUs (either Nvidia or AMD)
type GPUManager struct {
	sync.Mutex
	GpuDataMap map[string]*system.GPUData
	// lastAvgData stores the last calculated averages for each GPU
	// Used when a collection happens before new data arrives (Count == 0)
	lastAvgData map[string]system.GPUData
	// Per-cache-key tracking for delta calculations
	// cacheKey -> gpuId -> snapshot of last count/usage/power values
	lastSnapshots map[uint16]map[string]*gpuSnapshot
	// Per-card energy snapshots for Intel sysfs power calculation.
	intelSysfsEnergySnapshots map[string]intelSysfsEnergySnapshot
}

// gpuSnapshot stores the last observed incremental values for delta tracking
type gpuSnapshot struct {
	count    uint32
	usage    float64
	power    float64
	powerPkg float64
	engines  map[string]float64
}

// RocmSmiJson represents the JSON structure of rocm-smi output
type RocmSmiJson struct {
	ID           string `json:"GUID"`
	Name         string `json:"Card series"`
	Temperature  string `json:"Temperature (Sensor edge) (C)"`
	MemoryUsed   string `json:"VRAM Total Used Memory (B)"`
	MemoryTotal  string `json:"VRAM Total Memory (B)"`
	Usage        string `json:"GPU use (%)"`
	PowerPackage string `json:"Average Graphics Package Power (W)"`
	PowerSocket  string `json:"Current Socket Graphics Package Power (W)"`
}

// gpuCollector defines a collector for a specific GPU management utility (nvidia-smi or rocm-smi)
type gpuCollector struct {
	name    string
	cmdArgs []string
	parse   func([]byte) bool // returns true if valid data was found
	buf     []byte
	bufSize uint16
	sleep   func(time.Duration) // waits between runs; time.Sleep if nil
}

var errNoValidData = fmt.Errorf("no valid GPU data found") // Error for missing data

// collectorSource identifies a selectable GPU collector in GPU_COLLECTOR.
type collectorSource string

const (
	collectorSourceNVTop        collectorSource = collectorSource(nvtopCmd)
	collectorSourceNVML         collectorSource = "nvml"
	collectorSourceNvidiaSMI    collectorSource = collectorSource(nvidiaSmiCmd)
	collectorSourceIntelGpuTop  collectorSource = collectorSource(intelGpuStatsCmd)
	collectorSourceIntelSysfs   collectorSource = "intel_sysfs"
	collectorSourceAmdSysfs     collectorSource = "amd_sysfs"
	collectorSourceRocmSMI      collectorSource = collectorSource(rocmSmiCmd)
	collectorSourceMacmon       collectorSource = collectorSource(macmonCmd)
	collectorSourcePowermetrics collectorSource = collectorSource(powermetricsCmd)
	collectorGroupNvidia        string          = "nvidia"
	collectorGroupIntel         string          = "intel"
	collectorGroupAmd           string          = "amd"
	collectorGroupApple         string          = "apple"
)

func isValidCollectorSource(source collectorSource) bool {
	switch source {
	case collectorSourceNVTop,
		collectorSourceNVML,
		collectorSourceNvidiaSMI,
		collectorSourceIntelGpuTop,
		collectorSourceIntelSysfs,
		collectorSourceAmdSysfs,
		collectorSourceRocmSMI,
		collectorSourceMacmon,
		collectorSourcePowermetrics:
		return true
	}
	return false
}

// gpuCapabilities describes detected GPU tooling and sysfs support on the host.
type gpuCapabilities struct {
	hasNvidiaSmi    bool
	hasRocmSmi      bool
	hasAmdSysfs     bool
	hasTegrastats   bool
	hasIntelGpuTop  bool
	hasXe           bool
	hasIntelSysfs   bool
	hasNvtop        bool
	hasMacmon       bool
	hasPowermetrics bool
}

type collectorDefinition struct {
	group              string
	available          bool
	start              func(onFailure func()) bool
	deprecationWarning string
}

// starts and manages the ongoing collection of GPU data for the specified GPU management utility.
// The tool is restarted whenever it exits. A run without valid data counts as a failure, since the
// tool may have started before the GPU driver was ready (e.g. right after boot) or may be broken,
// and each consecutive failure doubles the delay up to maxRetryBackoff. A run that produces valid
// data resets the count whatever its exit status, because tools started with -l or --interval
// normally only exit when something ends them. Collection never stops for good, so it recovers
// without an agent restart.
func (c *gpuCollector) start() {
	sleep := c.sleep
	if sleep == nil {
		sleep = time.Sleep
	}
	consecutiveFailures := 0
	for {
		hasValidData, err := c.collect()
		if hasValidData {
			consecutiveFailures = 0
		} else {
			consecutiveFailures++
		}
		wait := retryDelay(retryWaitTime, consecutiveFailures)
		slog.Warn(c.name+" stopped, restarting", "err", err, "failures", consecutiveFailures, "wait", wait.String())
		sleep(wait)
	}
}

// retryDelay doubles the base wait after each consecutive failure, capping at maxRetryBackoff
func retryDelay(base time.Duration, consecutiveFailures int) time.Duration {
	backoff := base
	for range consecutiveFailures - 1 {
		backoff *= 2
		if backoff >= maxRetryBackoff {
			return maxRetryBackoff
		}
	}
	return backoff
}

// maxLoggedLineLen limits how much of an unparseable output line is included in errors
const maxLoggedLineLen = 200

// collect executes the command, parses output with the assigned parser function
// and reports whether any valid data was found. Blank lines are ignored. Lines the parser
// rejects are skipped, and the first one is logged at WARN if valid data came before it or
// more output after it. If the command printed output but none of it was valid, collect
// returns errNoValidData with the first rejected line.
func (c *gpuCollector) collect() (hasValidData bool, err error) {
	cmd := exec.Command(c.name, c.cmdArgs...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return false, err
	}
	if err := cmd.Start(); err != nil {
		return false, err
	}

	scanner := bufio.NewScanner(stdout)
	if c.buf == nil {
		c.buf = make([]byte, 0, c.bufSize)
	}
	scanner.Buffer(c.buf, bufio.MaxScanTokenSize)

	var (
		invalidLine       string // first line of this run the parser rejected
		hasInvalidLine    bool
		warnedInvalidLine bool
	)
	// warnInvalidLine logs the run's first rejected line at WARN, once per run. It is only
	// called once the run is more than a tool that printed an error and exited, which start()
	// reports when the run ends: valid data came before the rejected line, or more output
	// followed it. With -l the tool may never exit, so this is how a stream that stops
	// producing valid data mid-run shows up above debug level.
	warnInvalidLine := func() {
		if hasInvalidLine && !warnedInvalidLine {
			warnedInvalidLine = true
			slog.Warn(c.name+" skipping unparseable output", "line", invalidLine)
		}
	}
	for scanner.Scan() {
		if len(bytes.TrimSpace(scanner.Bytes())) == 0 {
			// a separator, not output to parse or report; nvidia-smi -l has been seen
			// printing these between samples (#1803)
			continue
		}
		warnInvalidLine() // output kept coming after a rejected line
		if c.parse(scanner.Bytes()) {
			hasValidData = true
			continue
		}
		if hasInvalidLine {
			slog.Debug(c.name+" skipping unparseable output", "line", scanner.Text())
			continue
		}
		hasInvalidLine = true
		invalidLine = scanner.Text()
		if len(invalidLine) > maxLoggedLineLen {
			invalidLine = invalidLine[:maxLoggedLineLen] + "..."
		}
		if hasValidData {
			warnInvalidLine() // valid data came before it
		}
	}

	if err := scanner.Err(); err != nil {
		// we stopped reading, so the command may never exit on its own
		_ = cmd.Process.Kill()
		_ = cmd.Wait()
		return hasValidData, fmt.Errorf("scanner error: %w", err)
	}
	// always wait so the process is reaped, even if its output was unusable
	waitErr := cmd.Wait()
	if !hasValidData && hasInvalidLine {
		return false, fmt.Errorf("%w: %q", errNoValidData, invalidLine)
	}
	return hasValidData, waitErr
}

// getJetsonParser returns a function to parse the output of tegrastats and update the GPUData map
func (gm *GPUManager) getJetsonParser() func(output []byte) bool {
	// use closure to avoid recompiling the regex
	ramPattern := regexp.MustCompile(`RAM (\d+)/(\d+)MB`)
	gr3dPattern := regexp.MustCompile(`GR3D_FREQ (\d+)%`)
	tempPattern := regexp.MustCompile(`(?:tj|GPU)@(\d+\.?\d*)C`)
	// Orin Nano / NX do not have GPU specific power monitor
	// TODO: Maybe use VDD_IN for Nano / NX and add a total system power chart
	powerPattern := regexp.MustCompile(`(GPU_SOC|CPU_GPU_CV)\s+(\d+)mW|VDD_SYS_GPU\s+(\d+)/\d+`)

	// jetson devices have only one gpu so we'll just initialize here
	gpuData := &system.GPUData{Name: "GPU"}
	gm.GpuDataMap["0"] = gpuData

	return func(output []byte) bool {
		gm.Lock()
		defer gm.Unlock()
		// Parse RAM usage
		ramMatches := ramPattern.FindSubmatch(output)
		if ramMatches != nil {
			gpuData.MemoryUsed, _ = strconv.ParseFloat(string(ramMatches[1]), 64)
			gpuData.MemoryTotal, _ = strconv.ParseFloat(string(ramMatches[2]), 64)
		}
		// Parse GR3D (GPU) usage
		gr3dMatches := gr3dPattern.FindSubmatch(output)
		if gr3dMatches != nil {
			gr3dUsage, _ := strconv.ParseFloat(string(gr3dMatches[1]), 64)
			gpuData.Usage += gr3dUsage
		}
		// Parse temperature
		tempMatches := tempPattern.FindSubmatch(output)
		if tempMatches != nil {
			gpuData.Temperature, _ = strconv.ParseFloat(string(tempMatches[1]), 64)
		}
		// Parse power usage
		powerMatches := powerPattern.FindSubmatch(output)
		if powerMatches != nil {
			// powerMatches[2] is the "(GPU_SOC|CPU_GPU_CV) <N>mW" capture
			// powerMatches[3] is the "VDD_SYS_GPU <N>/<N>" capture
			powerStr := string(powerMatches[2])
			if powerStr == "" {
				powerStr = string(powerMatches[3])
			}
			power, _ := strconv.ParseFloat(powerStr, 64)
			gpuData.Power += power / milliwattsInAWatt
		}
		gpuData.Count++
		return true
	}
}

// parseNvidiaData parses the output of nvidia-smi and updates the GPUData map
func (gm *GPUManager) parseNvidiaData(output []byte) bool {
	gm.Lock()
	defer gm.Unlock()
	scanner := bufio.NewScanner(bytes.NewReader(output))
	var valid bool
	for scanner.Scan() {
		line := scanner.Text() // Or use scanner.Bytes() for []byte
		fields := strings.Split(strings.TrimSpace(line), ", ")
		if len(fields) < 7 {
			continue
		}
		valid = true
		id := fields[0]
		temp, _ := strconv.ParseFloat(fields[2], 64)
		memoryUsage, _ := strconv.ParseFloat(fields[3], 64)
		totalMemory, _ := strconv.ParseFloat(fields[4], 64)
		usage, _ := strconv.ParseFloat(fields[5], 64)
		power, _ := strconv.ParseFloat(fields[6], 64)
		// add gpu if not exists
		if _, ok := gm.GpuDataMap[id]; !ok {
			name := strings.TrimPrefix(fields[1], "NVIDIA ")
			gm.GpuDataMap[id] = &system.GPUData{Name: strings.TrimSuffix(name, " Laptop GPU")}
		}
		// update gpu data
		gpu := gm.GpuDataMap[id]
		gpu.Temperature = temp
		gpu.MemoryUsed = memoryUsage / mebibytesInAMegabyte
		gpu.MemoryTotal = totalMemory / mebibytesInAMegabyte
		gpu.Usage += usage
		gpu.Power += power
		gpu.Count++
	}
	return valid
}

// parseAmdData parses the output of rocm-smi and updates the GPUData map
func (gm *GPUManager) parseAmdData(output []byte) bool {
	var rocmSmiInfo map[string]RocmSmiJson
	if err := json.Unmarshal(output, &rocmSmiInfo); err != nil || len(rocmSmiInfo) == 0 {
		return false
	}
	gm.Lock()
	defer gm.Unlock()
	for _, v := range rocmSmiInfo {
		var power float64
		if v.PowerPackage != "" {
			power, _ = strconv.ParseFloat(v.PowerPackage, 64)
		} else {
			power, _ = strconv.ParseFloat(v.PowerSocket, 64)
		}
		memoryUsage, _ := strconv.ParseFloat(v.MemoryUsed, 64)
		totalMemory, _ := strconv.ParseFloat(v.MemoryTotal, 64)
		usage, _ := strconv.ParseFloat(v.Usage, 64)

		id := v.ID
		if _, ok := gm.GpuDataMap[id]; !ok {
			gm.GpuDataMap[id] = &system.GPUData{Name: v.Name}
		}
		gpu := gm.GpuDataMap[id]
		gpu.Temperature, _ = strconv.ParseFloat(v.Temperature, 64)
		gpu.MemoryUsed = utils.BytesToMegabytes(memoryUsage)
		gpu.MemoryTotal = utils.BytesToMegabytes(totalMemory)
		gpu.Usage += usage
		gpu.Power += power
		gpu.Count++
	}
	return true
}

// GetCurrentData returns GPU utilization data averaged since the last call with this cacheKey
func (gm *GPUManager) GetCurrentData(cacheKey uint16) map[string]system.GPUData {
	gm.Lock()
	defer gm.Unlock()

	gm.initializeSnapshots(cacheKey)
	nameCounts := gm.countGPUNames()

	gpuData := make(map[string]system.GPUData, len(gm.GpuDataMap))
	for id, gpu := range gm.GpuDataMap {
		gpuAvg := gm.calculateGPUAverage(id, gpu, cacheKey)
		gm.updateInstantaneousValues(&gpuAvg, gpu)
		gm.storeSnapshot(id, gpu, cacheKey)

		// Append id to name if there are multiple GPUs with the same name
		if nameCounts[gpu.Name] > 1 {
			gpuAvg.Name = fmt.Sprintf("%s %s", gpu.Name, id)
		}
		gpuData[id] = gpuAvg
	}
	slog.Debug("GPU", "data", gpuData)
	return gpuData
}

// initializeSnapshots ensures snapshot maps are initialized for the given cache key
func (gm *GPUManager) initializeSnapshots(cacheKey uint16) {
	if gm.lastAvgData == nil {
		gm.lastAvgData = make(map[string]system.GPUData)
	}
	if gm.lastSnapshots == nil {
		gm.lastSnapshots = make(map[uint16]map[string]*gpuSnapshot)
	}
	if gm.lastSnapshots[cacheKey] == nil {
		gm.lastSnapshots[cacheKey] = make(map[string]*gpuSnapshot)
	}
}

// countGPUNames returns a map of GPU names to their occurrence count
func (gm *GPUManager) countGPUNames() map[string]int {
	nameCounts := make(map[string]int)
	for _, gpu := range gm.GpuDataMap {
		nameCounts[gpu.Name]++
	}
	return nameCounts
}

// calculateGPUAverage computes the average GPU metrics since the last snapshot for this cache key
func (gm *GPUManager) calculateGPUAverage(id string, gpu *system.GPUData, cacheKey uint16) system.GPUData {
	lastSnapshot := gm.lastSnapshots[cacheKey][id]
	currentCount := uint32(gpu.Count)
	deltaCount := gm.calculateDeltaCount(currentCount, lastSnapshot)

	// If no new data arrived
	if deltaCount == 0 {
		// Only discrete GPUs report temp/memory, so treat all-zero as suspended (return zeros).
		// Engine-based (Intel) GPUs don't, so carry the last average forward across sample gaps.
		if gpu.Engines == nil && gpu.Temperature == 0 && gpu.MemoryUsed == 0 {
			return system.GPUData{Name: gpu.Name}
		}
		lastAvg := gm.lastAvgData[id] // zero value if not found
		if lastAvg.Name == "" {
			lastAvg.Name = gpu.Name
		}
		return lastAvg
	}

	// Calculate new average
	gpuAvg := *gpu
	deltaUsage, deltaPower, deltaPowerPkg := gm.calculateDeltas(gpu, lastSnapshot)

	gpuAvg.Power = utils.TwoDecimals(deltaPower / float64(deltaCount))

	gpuAvg.PowerPkg = utils.TwoDecimals(deltaPowerPkg / float64(deltaCount))

	if gpu.Engines != nil {
		// make fresh map for averaged engine metrics to avoid mutating
		// the accumulator map stored in gm.GpuDataMap
		gpuAvg.Engines = make(map[string]float64, len(gpu.Engines))
		// Usage is the busiest of overall utilization and any engine (e.g. nvtop media engines).
		gpuAvg.Usage = max(utils.TwoDecimals(deltaUsage/float64(deltaCount)), gm.calculateEngineUsage(&gpuAvg, gpu, lastSnapshot, deltaCount))
	} else {
		gpuAvg.Usage = utils.TwoDecimals(deltaUsage / float64(deltaCount))
	}

	gm.lastAvgData[id] = gpuAvg
	return gpuAvg
}

// calculateDeltaCount returns the change in count since the last snapshot
func (gm *GPUManager) calculateDeltaCount(currentCount uint32, lastSnapshot *gpuSnapshot) uint32 {
	if lastSnapshot != nil {
		return currentCount - lastSnapshot.count
	}
	return currentCount
}

// calculateDeltas computes the change in usage, power, and powerPkg since the last snapshot
func (gm *GPUManager) calculateDeltas(gpu *system.GPUData, lastSnapshot *gpuSnapshot) (deltaUsage, deltaPower, deltaPowerPkg float64) {
	if lastSnapshot != nil {
		return gpu.Usage - lastSnapshot.usage,
			gpu.Power - lastSnapshot.power,
			gpu.PowerPkg - lastSnapshot.powerPkg
	}
	return gpu.Usage, gpu.Power, gpu.PowerPkg
}

// calculateEngineUsage averages per-engine metrics into gpuAvg and returns the max engine usage
func (gm *GPUManager) calculateEngineUsage(gpuAvg, gpu *system.GPUData, lastSnapshot *gpuSnapshot, deltaCount uint32) float64 {
	maxEngineUsage := 0.0
	for name, engine := range gpu.Engines {
		var deltaEngine float64
		if lastSnapshot != nil && lastSnapshot.engines != nil {
			deltaEngine = engine - lastSnapshot.engines[name]
		} else {
			deltaEngine = engine
		}
		gpuAvg.Engines[name] = utils.TwoDecimals(deltaEngine / float64(deltaCount))
		maxEngineUsage = max(maxEngineUsage, deltaEngine/float64(deltaCount))
	}
	return utils.TwoDecimals(maxEngineUsage)
}

// updateInstantaneousValues updates values that should reflect current state, not averages
func (gm *GPUManager) updateInstantaneousValues(gpuAvg *system.GPUData, gpu *system.GPUData) {
	gpuAvg.Temperature = utils.TwoDecimals(gpu.Temperature)
	gpuAvg.MemoryUsed = utils.TwoDecimals(gpu.MemoryUsed)
	gpuAvg.MemoryTotal = utils.TwoDecimals(gpu.MemoryTotal)
}

// storeSnapshot saves the current GPU state for this cache key
func (gm *GPUManager) storeSnapshot(id string, gpu *system.GPUData, cacheKey uint16) {
	snapshot := &gpuSnapshot{
		count:    uint32(gpu.Count),
		usage:    gpu.Usage,
		power:    gpu.Power,
		powerPkg: gpu.PowerPkg,
	}
	if gpu.Engines != nil {
		snapshot.engines = make(map[string]float64, len(gpu.Engines))
		maps.Copy(snapshot.engines, gpu.Engines)
	}
	gm.lastSnapshots[cacheKey][id] = snapshot
}

// discoverGpuCapabilities checks for available GPU tooling and sysfs support.
// It only reports capability presence and does not apply policy decisions.
func (gm *GPUManager) discoverGpuCapabilities() gpuCapabilities {
	caps := gpuCapabilities{
		hasAmdSysfs:   gm.hasAmdSysfs(),
		hasXe:         gm.hasXe(),
		hasIntelSysfs: gm.hasIntelSysfs(),
	}
	if _, err := exec.LookPath(nvidiaSmiCmd); err == nil {
		caps.hasNvidiaSmi = true
	}
	if _, err := exec.LookPath(rocmSmiCmd); err == nil {
		caps.hasRocmSmi = true
	}
	if _, err := exec.LookPath(tegraStatsCmd); err == nil {
		caps.hasTegrastats = true
	}
	if _, err := exec.LookPath(intelGpuStatsCmd); err == nil {
		caps.hasIntelGpuTop = true
	}
	if _, err := exec.LookPath(nvtopCmd); err == nil {
		caps.hasNvtop = true
	}
	if runtime.GOOS == "darwin" {
		if _, err := utils.LookPathHomebrew(macmonCmd); err == nil {
			caps.hasMacmon = true
		}
		if _, err := exec.LookPath(powermetricsCmd); err == nil {
			caps.hasPowermetrics = true
		}
	}
	return caps
}

func hasAnyGpuCollector(caps gpuCapabilities) bool {
	return caps.hasNvidiaSmi || caps.hasRocmSmi || caps.hasAmdSysfs || caps.hasTegrastats || caps.hasIntelGpuTop || caps.hasIntelSysfs || caps.hasNvtop || caps.hasMacmon || caps.hasPowermetrics
}

func (gm *GPUManager) startIntelCollector() {
	go func() {
		failures := 0
		for {
			if err := gm.collectIntelStats(); err != nil {
				failures++
				if failures > maxFailureRetries {
					break
				}
				slog.Warn("Error collecting Intel GPU data; see https://beszel.dev/guide/gpu", "err", err)
				time.Sleep(retryWaitTime)
				continue
			}
		}
	}()
}

func (gm *GPUManager) startNvidiaSmiCollector(intervalSeconds string) {
	collector := gpuCollector{
		name:    nvidiaSmiCmd,
		bufSize: 10 * 1024,
		cmdArgs: []string{
			"-l", intervalSeconds,
			"--query-gpu=index,name,temperature.gpu,memory.used,memory.total,utilization.gpu,power.draw",
			"--format=csv,noheader,nounits",
		},
		parse: gm.parseNvidiaData,
	}
	go collector.start()
}

func (gm *GPUManager) startTegraStatsCollector(intervalMilliseconds string) {
	collector := gpuCollector{
		name:    tegraStatsCmd,
		bufSize: 10 * 1024,
		cmdArgs: []string{"--interval", intervalMilliseconds},
		parse:   gm.getJetsonParser(),
	}
	go collector.start()
}

func (gm *GPUManager) startRocmSmiCollector(pollInterval time.Duration) {
	collector := gpuCollector{
		name:    rocmSmiCmd,
		bufSize: 10 * 1024,
		cmdArgs: []string{"--showid", "--showtemp", "--showuse", "--showpower", "--showproductname", "--showmeminfo", "vram", "--json"},
		parse:   gm.parseAmdData,
	}
	go func() {
		failures := 0
		for {
			if _, err := collector.collect(); err != nil {
				failures++
				if failures > maxFailureRetries {
					break
				}
				slog.Warn("Error collecting AMD GPU data via rocm-smi", "err", err)
			}
			time.Sleep(pollInterval)
		}
	}()
}

func (gm *GPUManager) collectorDefinitions(caps gpuCapabilities) map[collectorSource]collectorDefinition {
	return map[collectorSource]collectorDefinition{
		collectorSourceNVML: {
			group:     collectorGroupNvidia,
			available: true,
			start: func(_ func()) bool {
				return gm.startNvmlCollector()
			},
		},
		collectorSourceNvidiaSMI: {
			group:     collectorGroupNvidia,
			available: caps.hasNvidiaSmi,
			start: func(_ func()) bool {
				gm.startNvidiaSmiCollector("4") // seconds
				return true
			},
		},
		collectorSourceIntelGpuTop: {
			group:     collectorGroupIntel,
			available: caps.hasIntelGpuTop,
			start: func(_ func()) bool {
				gm.startIntelCollector()
				return true
			},
		},
		collectorSourceIntelSysfs: {
			group:     collectorGroupIntel,
			available: caps.hasIntelSysfs,
			start: func(_ func()) bool {
				return gm.startIntelSysfsCollector()
			},
		},
		collectorSourceAmdSysfs: {
			group:     collectorGroupAmd,
			available: caps.hasAmdSysfs,
			start: func(_ func()) bool {
				return gm.startAmdSysfsCollector()
			},
		},
		collectorSourceRocmSMI: {
			group:              collectorGroupAmd,
			available:          caps.hasRocmSmi,
			deprecationWarning: "rocm-smi is deprecated and may be removed in a future release",
			start: func(_ func()) bool {
				gm.startRocmSmiCollector(4300 * time.Millisecond)
				return true
			},
		},
		collectorSourceNVTop: {
			available: caps.hasNvtop,
			start: func(onFailure func()) bool {
				gm.startNvtopCollector("30", onFailure) // tens of milliseconds
				return true
			},
		},
		collectorSourceMacmon: {
			group:     collectorGroupApple,
			available: caps.hasMacmon,
			start: func(_ func()) bool {
				gm.startMacmonCollector()
				return true
			},
		},
		collectorSourcePowermetrics: {
			group:     collectorGroupApple,
			available: caps.hasPowermetrics,
			start: func(_ func()) bool {
				gm.startPowermetricsCollector()
				return true
			},
		},
	}
}

// parseCollectorPriority parses GPU_COLLECTOR and returns valid ordered entries.
func parseCollectorPriority(value string) []collectorSource {
	parts := strings.Split(value, ",")
	priorities := make([]collectorSource, 0, len(parts))
	for _, raw := range parts {
		name := collectorSource(strings.TrimSpace(strings.ToLower(raw)))
		if !isValidCollectorSource(name) {
			if name != "" {
				slog.Warn("Ignoring unknown GPU collector", "collector", name)
			}
			continue
		}
		priorities = append(priorities, name)
	}
	return priorities
}

// startNvmlCollector initializes NVML and starts its polling loop.
func (gm *GPUManager) startNvmlCollector() bool {
	collector := &nvmlCollector{gm: gm}
	if err := collector.init(); err != nil {
		slog.Warn("Failed to initialize NVML", "err", err)
		return false
	}
	go collector.start()
	return true
}

// startAmdSysfsCollector starts AMD GPU collection via sysfs.
func (gm *GPUManager) startAmdSysfsCollector() bool {
	go func() {
		if err := gm.collectAmdStats(); err != nil {
			slog.Warn("Error collecting AMD GPU data via sysfs", "err", err)
		}
	}()
	return true
}

// startCollectorsByPriority starts collectors in order with one source per vendor group.
func (gm *GPUManager) startCollectorsByPriority(priorities []collectorSource, caps gpuCapabilities) int {
	definitions := gm.collectorDefinitions(caps)
	selectedGroups := make(map[string]bool, 3)
	started := 0
	for i, source := range priorities {
		definition, ok := definitions[source]
		if !ok || !definition.available {
			continue
		}
		// nvtop is not a vendor-specific collector, so should only be used if no other collectors are selected or it is first in GPU_COLLECTOR.
		if source == collectorSourceNVTop {
			if len(selectedGroups) > 0 {
				slog.Warn("Skipping nvtop because other collectors are selected")
				continue
			}
			// if nvtop fails, fall back to remaining collectors.
			remaining := append([]collectorSource(nil), priorities[i+1:]...)
			if definition.start(func() {
				gm.startCollectorsByPriority(remaining, caps)
			}) {
				started++
				return started
			}
		}
		group := definition.group
		if group == "" || selectedGroups[group] {
			continue
		}
		if definition.deprecationWarning != "" {
			slog.Warn(definition.deprecationWarning)
		}
		if definition.start(nil) {
			selectedGroups[group] = true
			started++
		}
	}
	return started
}

// resolveLegacyCollectorPriority builds the default collector order when GPU_COLLECTOR is unset.
func (gm *GPUManager) resolveLegacyCollectorPriority(caps gpuCapabilities) []collectorSource {
	priorities := make([]collectorSource, 0, 4)

	if caps.hasNvidiaSmi && !caps.hasTegrastats {
		if nvml, _ := utils.GetEnv("NVML"); nvml == "true" {
			priorities = append(priorities, collectorSourceNVML, collectorSourceNvidiaSMI)
		} else {
			priorities = append(priorities, collectorSourceNvidiaSMI)
		}
	}

	if caps.hasRocmSmi {
		if val, _ := utils.GetEnv("AMD_SYSFS"); val == "true" {
			priorities = append(priorities, collectorSourceAmdSysfs)
		} else {
			priorities = append(priorities, collectorSourceRocmSMI)
		}
	} else if caps.hasAmdSysfs {
		priorities = append(priorities, collectorSourceAmdSysfs)
	}

	if caps.hasIntelGpuTop && !caps.hasXe {
		priorities = append(priorities, collectorSourceIntelGpuTop)
	}
	if caps.hasIntelSysfs {
		priorities = append(priorities, collectorSourceIntelSysfs)
	}

	// Apple collectors are currently opt-in only for testing.
	// Enable them with GPU_COLLECTOR=macmon or GPU_COLLECTOR=powermetrics.
	// TODO: uncomment below when Apple collectors are confirmed to be working.
	//
	// Prefer macmon on macOS (no sudo). Fall back to powermetrics if present.
	// if caps.hasMacmon {
	// 	priorities = append(priorities, collectorSourceMacmon)
	// } else if caps.hasPowermetrics {
	// 	priorities = append(priorities, collectorSourcePowermetrics)
	// }

	// Keep nvtop as a last resort only when no vendor collector exists.
	if len(priorities) == 0 && caps.hasNvtop {
		priorities = append(priorities, collectorSourceNVTop)
	}
	return priorities
}

// gpuHwmonChips are hwmon chip names belonging to GPUs. Sensor reads on some
// of these drivers (notably Intel Xe, where each read is a runtime PM resume)
// wake the card, so SKIP_GPU must avoid touching them, not just hide them.
var gpuHwmonChips = []string{"xe", "i915", "amdgpu", "radeon", "nvidia", "nouveau"}

func isGpuChipName(name string) bool {
	name = strings.ToLower(strings.TrimSpace(name))
	for _, chip := range gpuHwmonChips {
		if name == chip {
			return true
		}
	}
	return false
}

// SensorKeys are "<chip>" or "<chip>_<label>".
func isGpuSensorKey(key string) bool {
	key = strings.ToLower(strings.TrimSpace(key))
	for _, chip := range gpuHwmonChips {
		if key == chip || strings.HasPrefix(key, chip+"_") {
			return true
		}
	}
	return false
}

// NewGPUManager creates and initializes a new GPUManager
func NewGPUManager() (*GPUManager, error) {
	if skipGPU, _ := utils.GetEnv("SKIP_GPU"); skipGPU == "true" {
		slog.Info("SKIP_GPU enabled, skipping GPU monitoring (collectors, temperatures, and fans)")
		return nil, nil
	}
	var gm GPUManager
	caps := gm.discoverGpuCapabilities()
	gm.GpuDataMap = make(map[string]*system.GPUData)

	// Jetson devices should always use tegrastats (ignore GPU_COLLECTOR).
	if caps.hasTegrastats {
		gm.startTegraStatsCollector("3700")
		return &gm, nil
	}

	// Respect explicit collector selection before capability auto-detection.
	if collectorConfig, ok := utils.GetEnv("GPU_COLLECTOR"); ok && strings.TrimSpace(collectorConfig) != "" {
		priorities := parseCollectorPriority(collectorConfig)
		if gm.startCollectorsByPriority(priorities, caps) == 0 {
			return nil, fmt.Errorf("no configured GPU collectors are available")
		}
		return &gm, nil
	}

	if !hasAnyGpuCollector(caps) {
		return nil, fmt.Errorf(noGPUFoundMsg)
	}

	// auto-detect and start collectors when GPU_COLLECTOR is unset.
	if gm.startCollectorsByPriority(gm.resolveLegacyCollectorPriority(caps), caps) == 0 {
		return nil, fmt.Errorf(noGPUFoundMsg)
	}

	return &gm, nil
}