Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"math"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/sensors"
"github.com/rcourtman/pulse-go-rewrite/internal/system"
"github.com/rs/zerolog/log"
)
const maxTemperatureCommandOutputSize = 1 << 20 // 1 MiB
const defaultSSHCommandTimeout = 15 * time.Second
const pulseSensorsSSHCommand = "if command -v /usr/local/sbin/pulse-sensors >/dev/null 2>&1; then /usr/local/sbin/pulse-sensors; else sensors -j 2>/dev/null || true; fi"
var errTemperatureCommandOutputTooLarge = errors.New("temperature command output exceeded limit")
// CommandRunner abstracts command execution for testing
type CommandRunner interface {
Run(ctx context.Context, name string, args ...string) ([]byte, error)
}
type defaultCommandRunner struct{}
type limitedTemperatureBuffer struct {
buf bytes.Buffer
maxBytes int
exceeded bool
}
func (b *limitedTemperatureBuffer) Write(p []byte) (int, error) {
remaining := b.maxBytes - b.buf.Len()
if remaining <= 0 {
b.exceeded = true
return 0, errTemperatureCommandOutputTooLarge
}
if len(p) > remaining {
b.exceeded = true
written, _ := b.buf.Write(p[:remaining])
return written, errTemperatureCommandOutputTooLarge
}
return b.buf.Write(p)
}
func (b *limitedTemperatureBuffer) Bytes() []byte {
return b.buf.Bytes()
}
func (r *defaultCommandRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...)
stdout := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
stderr := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if stdout.exceeded || stderr.exceeded {
return nil, errTemperatureCommandOutputTooLarge
}
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
// Preserve bounded stderr details for upstream error wrapping.
exitErr.Stderr = append([]byte(nil), stderr.Bytes()...)
}
return stdout.Bytes(), err
}
return stdout.Bytes(), nil
}
// TemperatureCollector handles SSH-based temperature collection from Proxmox nodes
type TemperatureCollector struct {
sshUser string // SSH user (typically "root" or "pulse-monitor")
sshKeyPath string // Path to SSH private key
sshPort int // SSH port (default 22)
hostKeys KnownHostsManager
missingKeyWarned atomic.Bool
runner CommandRunner
sshFailureMu sync.Mutex
sshFailures map[string]*temperatureSSHFailure
sshKeyIdentity string
}
// temperatureSSHFailure remembers a host whose SSH collection failed so
// subsequent 10s poll cycles don't re-exec ssh (twice, with the RPi fallback)
// against a host that keeps failing identically (#1638). Backoff doubles per
// consecutive failure; any successful collection clears it.
type temperatureSSHFailure struct {
retryAt time.Time
backoff time.Duration
}
const (
temperatureSSHFailureInitialBackoff = 30 * time.Second
temperatureSSHFailureMaxBackoff = 15 * time.Minute
)
// sshFailureOutcome describes what a failed collection attempt tells us about
// the host, which decides how the per-host backoff moves.
type sshFailureOutcome int
const (
// sshFailureHard is a refusal, auth failure, or unusable output: the host
// answered (or actively did not) and the backoff should compound.
sshFailureHard sshFailureOutcome = iota
// sshFailureTransient is our own deadline expiring. That says nothing about
// the host, so hold the backoff at the floor instead of doubling it.
sshFailureTransient
// sshFailureNotAttempted means no ssh ran at all — the knownhosts manager
// suppressed the call inside its own backoff. Nothing was learned and
// nothing was spent, so the backoff must not move.
sshFailureNotAttempted
)
// classifySSHFailure maps a failed attempt onto its backoff treatment (#1638).
func classifySSHFailure(err error) sshFailureOutcome {
switch {
case err == nil:
return sshFailureHard
case errors.Is(err, ErrKeyscanSuppressed):
return sshFailureNotAttempted
case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
return sshFailureTransient
default:
return sshFailureHard
}
}
func (tc *TemperatureCollector) inSSHFailureBackoff(host string) bool {
tc.sshFailureMu.Lock()
defer tc.sshFailureMu.Unlock()
failure := tc.sshFailures[host]
return failure != nil && sshBackoffNow().Before(failure.retryAt)
}
func (tc *TemperatureCollector) recordSSHFailure(host string, outcome sshFailureOutcome) {
if outcome == sshFailureNotAttempted {
return
}
tc.sshFailureMu.Lock()
defer tc.sshFailureMu.Unlock()
if tc.sshFailures == nil {
tc.sshFailures = make(map[string]*temperatureSSHFailure)
}
now := sshBackoffNow()
var previousRetryAt time.Time
var previousBackoff time.Duration
if existing := tc.sshFailures[host]; existing != nil {
previousRetryAt = existing.retryAt
previousBackoff = existing.backoff
}
backoff := nextSSHBackoff(now, previousRetryAt, previousBackoff, temperatureSSHFailureInitialBackoff, temperatureSSHFailureMaxBackoff, outcome == sshFailureHard)
tc.sshFailures[host] = &temperatureSSHFailure{
retryAt: now.Add(backoff),
backoff: backoff,
}
}
func (tc *TemperatureCollector) clearSSHFailure(host string) {
tc.sshFailureMu.Lock()
defer tc.sshFailureMu.Unlock()
delete(tc.sshFailures, host)
}
// ResetSSHFailures drops every recorded SSH backoff, including the knownhosts
// manager's keyscan backoff, so an operator who has just repaired an SSH key or
// host key sees the next poll cycle retry rather than waiting out a window that
// has compounded to a quarter of an hour (#1638).
func (tc *TemperatureCollector) ResetSSHFailures() {
if tc == nil {
return
}
tc.sshFailureMu.Lock()
tc.sshFailures = nil
hostKeys := tc.hostKeys
tc.sshFailureMu.Unlock()
if hostKeys != nil {
hostKeys.ResetFailures()
}
}
// resetSSHFailuresOnKeyChange clears the backoff maps when the SSH key on disk
// has been replaced since the last cycle.
//
// Repairing that key is the usual fix for the failures that opened these
// backoff windows in the first place, and nothing else notices it happening, so
// without this the operator waits out a window that has already compounded
// toward fifteen minutes before Pulse tries the repaired key (#1638).
func (tc *TemperatureCollector) resetSSHFailuresOnKeyChange() {
identity := tc.currentSSHKeyIdentity()
tc.sshFailureMu.Lock()
changed := tc.sshKeyIdentity != "" && tc.sshKeyIdentity != identity
tc.sshKeyIdentity = identity
tc.sshFailureMu.Unlock()
if changed {
log.Info().
Str("sshKeyPath", tc.sshKeyPath).
Msg("Temperature SSH key changed on disk; clearing per-host SSH backoff")
tc.ResetSSHFailures()
}
}
// currentSSHKeyIdentity fingerprints the key file cheaply enough to run every
// poll cycle. Content is deliberately not read: this is change detection, not
// validation.
func (tc *TemperatureCollector) currentSSHKeyIdentity() string {
info, err := os.Stat(strings.TrimSpace(tc.sshKeyPath))
if err != nil {
return ""
}
return fmt.Sprintf("%d:%d", info.ModTime().UnixNano(), info.Size())
}
// NewTemperatureCollectorWithPort creates a new temperature collector with custom SSH port
func NewTemperatureCollectorWithPort(sshUser, sshKeyPath string, sshPort int) *TemperatureCollector {
if sshPort <= 0 {
sshPort = 22 // Default to standard SSH port
}
tc := &TemperatureCollector{
sshUser: sshUser,
sshKeyPath: sshKeyPath,
sshPort: sshPort,
runner: &defaultCommandRunner{},
}
homeDir := os.Getenv("HOME")
if homeDir == "" {
homeDir = "/home/pulse"
}
knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts_sensors")
if manager, err := NewKnownHostsManager(knownHostsPath); err != nil {
log.Warn().Err(err).Str("path", knownHostsPath).Msg("failed to initialize temperature known_hosts manager")
} else {
tc.hostKeys = manager
}
return tc
}
// CollectTemperature collects temperature data from a node via SSH
func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost, nodeName string) (*models.Temperature, error) {
// Extract hostname/IP from the host URL (might be https://hostname:8006)
host := extractHostname(nodeHost)
// SECURITY: Warn about legacy SSH collection in containers; this is not a block.
// Container compromise = SSH key compromise = root access to infrastructure
devModeAllowSSH := os.Getenv("PULSE_DEV_ALLOW_CONTAINER_SSH") == "true"
isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer()
if isContainer && devModeAllowSSH {
// Log when dev override is active so operators understand the security posture
log.Info().
Str("node", nodeName).
Msg("Temperature collection using direct SSH (dev mode override active - not for production)")
}
if isContainer && !devModeAllowSSH {
// Warn but allow if key is present (legacy behavior restoration)
// We don't return here, allowing the code to fall through to the SSH key check
log.Warn().
Str("node", nodeName).
Msg("Temperature collection using direct SSH from container. This is insecure for production deployments.")
}
if strings.TrimSpace(tc.sshKeyPath) == "" {
tc.logMissingSSHKey(nil)
return &models.Temperature{Available: false}, nil
}
if keyErr := tc.validateSSHKeyPath(); keyErr != nil {
tc.logMissingSSHKey(keyErr)
return &models.Temperature{Available: false}, nil
}
tc.resetSSHFailuresOnKeyChange()
if tc.inSSHFailureBackoff(host) {
log.Debug().
Str("node", nodeName).
Str("host", host).
Msg("Skipping SSH temperature collection while failure backoff is active")
return &models.Temperature{Available: false}, nil
}
// Direct SSH (legacy method, including containers with an existing usable key).
// New setup scripts restrict the key to /usr/local/sbin/pulse-sensors, which emits
// the canonical {sensors, smart} payload. Older keys ignore the requested command
// and still force sensors -j, so keep the parser backward-compatible.
output, err := tc.runSSHCommand(ctx, host, pulseSensorsSSHCommand)
if err != nil || strings.TrimSpace(output) == "" {
// A suppressed keyscan or an expired deadline would fail the RPi
// fallback identically, so stop here rather than paying for a second
// probe, and do not compound a backoff the host has not earned (#1638).
if outcome := classifySSHFailure(err); outcome != sshFailureHard {
tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
tc.recordSSHFailure(host, outcome)
return &models.Temperature{Available: false}, nil
}
if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
tc.recordSSHFailure(host, sshFailureHard)
return &models.Temperature{Available: false}, nil
}
// Try Raspberry Pi temperature method
output, err = tc.runSSHCommand(ctx, host, "cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null")
if err == nil && strings.TrimSpace(output) != "" {
// Parse RPi temperature format
temp, parseErr := tc.parseRPiTemperature(output)
if parseErr == nil {
tc.clearSSHFailure(host)
return temp, nil
}
}
if outcome := classifySSHFailure(err); outcome != sshFailureHard {
tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
tc.recordSSHFailure(host, outcome)
return &models.Temperature{Available: false}, nil
}
if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
tc.recordSSHFailure(host, sshFailureHard)
return &models.Temperature{Available: false}, nil
}
log.Debug().
Str("node", nodeName).
Str("host", host).
Err(err).
Msg("Failed to collect temperature data via SSH (tried both lm-sensors and RPi methods)")
tc.recordSSHFailure(host, sshFailureHard)
return &models.Temperature{Available: false}, nil
}
tc.clearSSHFailure(host)
// Parse sensors JSON output
temp, err := tc.parseSensorsJSON(output)
if err != nil {
log.Debug().
Str("node", nodeName).
Err(err).
Msg("Failed to parse sensors output")
return &models.Temperature{Available: false}, nil
}
if !temp.Available {
return temp, nil
}
temp.LastUpdate = time.Now()
return temp, nil
}
// logSkippedSSHAttempt records why an attempt is being abandoned without
// escalating the per-host backoff.
func (tc *TemperatureCollector) logSkippedSSHAttempt(outcome sshFailureOutcome, nodeName, host string, err error) {
event := log.Debug().Str("node", nodeName).Str("host", host).Err(err)
if outcome == sshFailureNotAttempted {
event.Msg("Skipping SSH temperature collection; host key scan is in its own backoff window")
return
}
event.Msg("Abandoning SSH temperature collection after the collection deadline expired")
}
func (tc *TemperatureCollector) runSSHCommand(ctx context.Context, host, command string) (string, error) {
if err := tc.validateSSHKeyPath(); err != nil {
return "", err
}
runCtx, cancel := withDefaultTimeout(ctx, defaultSSHCommandTimeout)
defer cancel()
if err := tc.ensureHostKey(runCtx, host); err != nil {
return "", err
}
// Build SSH command with appropriate options
sshArgs := []string{
"-o", "StrictHostKeyChecking=yes",
"-o", "BatchMode=yes",
"-o", "LogLevel=ERROR", // Suppress host key warnings that break JSON parsing
"-o", "ConnectTimeout=5",
"-p", strconv.Itoa(tc.sshPort), // Use configured SSH port
}
if tc.hostKeys != nil && tc.hostKeys.Path() != "" {
sshArgs = append(sshArgs,
"-o", fmt.Sprintf("UserKnownHostsFile=%s", tc.hostKeys.Path()),
"-o", "GlobalKnownHostsFile=/dev/null",
)
}
// Explicitly use SSH config file if it exists (for ProxyJump configuration)
homeDir := os.Getenv("HOME")
if homeDir == "" {
homeDir = "/home/pulse"
}
sshConfigPath := filepath.Join(homeDir, ".ssh/config")
if _, err := os.Stat(sshConfigPath); err == nil {
sshArgs = append(sshArgs, "-F", sshConfigPath)
}
// Add key if specified
if tc.sshKeyPath != "" {
sshArgs = append(sshArgs, "-i", tc.sshKeyPath)
}
// Add user@host and command
sshArgs = append(sshArgs, fmt.Sprintf("%s@%s", tc.sshUser, host), command)
output, err := tc.runner.Run(runCtx, "ssh", sshArgs...)
if err != nil {
if errors.Is(err, errTemperatureCommandOutputTooLarge) {
return "", fmt.Errorf("ssh command output exceeded %d bytes", maxTemperatureCommandOutputSize)
}
// Our own budget running out is not evidence about the host, and
// sanitizeSSHCommandError would flatten the signal into an exit status,
// so report the context error directly (#1638).
if ctxErr := runCtx.Err(); ctxErr != nil {
return "", fmt.Errorf("ssh command aborted: %w", ctxErr)
}
return "", sanitizeSSHCommandError(err)
}
outputStr := strings.TrimSpace(string(output))
// Strip any leading SSH noise (e.g., "Warning: Permanently added ...") so sensors JSON parses cleanly.
if idx := strings.Index(outputStr, "{"); idx > 0 {
outputStr = outputStr[idx:]
}
if idx := strings.LastIndex(outputStr, "}"); idx != -1 && idx < len(outputStr)-1 {
outputStr = outputStr[:idx+1]
}
return outputStr, nil
}
func withDefaultTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
if ctx == nil {
return context.WithTimeout(context.Background(), timeout)
}
if _, hasDeadline := ctx.Deadline(); hasDeadline {
return ctx, func() {}
}
return context.WithTimeout(ctx, timeout)
}
func sanitizeSSHCommandError(err error) error {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
stderrLower := strings.ToLower(string(exitErr.Stderr))
switch {
case strings.Contains(stderrLower, "permission denied"),
strings.Contains(stderrLower, "publickey"),
strings.Contains(stderrLower, "authentication failed"):
return errors.New("ssh command failed: authentication failed")
case strings.Contains(stderrLower, "host key verification failed"):
return errors.New("ssh command failed: host key verification failed")
case strings.Contains(stderrLower, "connection timed out"),
strings.Contains(stderrLower, "operation timed out"),
strings.Contains(stderrLower, "connection timeout"):
return errors.New("ssh command failed: timeout")
default:
if code := exitErr.ExitCode(); code >= 0 {
return fmt.Errorf("ssh command failed with exit status %d", code)
}
return errors.New("ssh command failed")
}
}
return fmt.Errorf("ssh command failed: %w", err)
}
func (tc *TemperatureCollector) validateSSHKeyPath() error {
keyPath := strings.TrimSpace(tc.sshKeyPath)
if keyPath == "" {
return errors.New("temperature SSH key unavailable")
}
info, err := os.Stat(keyPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return os.ErrNotExist
}
return errors.New("temperature SSH key unavailable")
}
if info.IsDir() {
return errors.New("temperature SSH key path is not a file")
}
if err := validatePrivateKeyPermissions(info.Mode().Perm()); err != nil {
return err
}
return nil
}
func validatePrivateKeyPermissions(mode os.FileMode) error {
perm := mode.Perm()
if perm&0o077 != 0 {
return fmt.Errorf("temperature SSH key has insecure permissions (%03o): expected no group/other access", perm)
}
return nil
}
func (tc *TemperatureCollector) logMissingSSHKey(cause error) {
if tc.missingKeyWarned.Load() {
return
}
if tc.missingKeyWarned.CompareAndSwap(false, true) {
event := log.Debug().
Str("sshKeyPath", tc.sshKeyPath)
if cause != nil && !errors.Is(cause, os.ErrNotExist) {
event = event.Err(cause)
}
event.Msg("temperature SSH key not available; skipping legacy SSH collection")
}
}
func (tc *TemperatureCollector) disableLegacySSHOnAuthFailure(err error, nodeName, host string) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
authFailure := strings.Contains(msg, "permission denied") ||
strings.Contains(msg, "authentication failed") ||
strings.Contains(msg, "publickey")
if !authFailure {
return false
}
// Do not disable globally on single node failure
log.Warn().
Str("node", nodeName).
Str("host", host).
Err(err).
Msg("SSH temperature collection failed due to authentication error; check SSH keys")
return true
}
type smartEntryRaw struct {
Device string `json:"device"`
Serial string `json:"serial,omitempty"`
WWN string `json:"wwn,omitempty"`
Model string `json:"model,omitempty"`
Type string `json:"type,omitempty"`
Temperature *int `json:"temperature"`
LastUpdated string `json:"lastUpdated,omitempty"`
StandbySkipped bool `json:"standbySkipped,omitempty"`
}
func normalizeSMARTEntries(raw []smartEntryRaw) []models.DiskTemp {
if len(raw) == 0 {
return nil
}
normalized := make([]models.DiskTemp, 0, len(raw))
for _, entry := range raw {
dev := strings.TrimSpace(entry.Device)
if dev == "" {
continue
}
var lastUpdated time.Time
if entry.LastUpdated != "" {
if parsed, err := time.Parse(time.RFC3339, entry.LastUpdated); err == nil {
lastUpdated = parsed
}
}
tempVal := 0
if entry.Temperature != nil {
tempVal = *entry.Temperature
}
normalized = append(normalized, models.DiskTemp{
Device: dev,
Serial: strings.TrimSpace(entry.Serial),
WWN: strings.TrimSpace(entry.WWN),
Model: strings.TrimSpace(entry.Model),
Type: strings.TrimSpace(entry.Type),
Temperature: tempVal,
LastUpdated: lastUpdated,
StandbySkipped: entry.StandbySkipped,
})
}
return normalized
}
// parseSensorsJSON parses the JSON output from the sensor wrapper
func (tc *TemperatureCollector) parseSensorsJSON(jsonStr string) (*models.Temperature, error) {
if strings.TrimSpace(jsonStr) == "" {
return nil, fmt.Errorf("empty sensors output")
}
// Try to parse as wrapper format first: {sensors: {...}, smart: [...]}
// Fall back to legacy format for backward compatibility
var wrapperData struct {
Sensors map[string]interface{} `json:"sensors"`
SMART []smartEntryRaw `json:"smart"`
}
var sensorsData map[string]interface{}
var smartRaw []smartEntryRaw
var parsedWrapper bool
if err := json.Unmarshal([]byte(jsonStr), &wrapperData); err == nil && wrapperData.Sensors != nil {
// New wrapper format
sensorsData = wrapperData.Sensors
smartRaw = wrapperData.SMART
parsedWrapper = true
} else {
// Legacy format: direct sensors -j output
if err := json.Unmarshal([]byte(jsonStr), &sensorsData); err != nil {
return nil, fmt.Errorf("failed to parse sensors JSON: %w", err)
}
log.Debug().Msg("parsed legacy sensors format (no SMART data)")
}
smartData := normalizeSMARTEntries(smartRaw)
if parsedWrapper {
log.Debug().
Int("smartDisks", len(smartData)).
Msg("Parsed new wrapper format with SMART data")
}
temp := &models.Temperature{
Cores: []models.CoreTemp{},
NVMe: []models.NVMeTemp{},
SMART: smartData,
LegacySensorsFormat: !parsedWrapper,
}
foundCPUChip := false
// Parse each sensor chip
for chipName, chipData := range sensorsData {
chipMap, ok := chipData.(map[string]interface{})
if !ok {
continue
}
// Handle CPU temperature sensors
chipLower := strings.ToLower(chipName)
if strings.Contains(chipLower, "coretemp") ||
strings.Contains(chipLower, "k10temp") ||
strings.Contains(chipLower, "zenpower") ||
strings.Contains(chipLower, "k8temp") ||
strings.Contains(chipLower, "acpitz") ||
strings.Contains(chipLower, "it87") ||
strings.Contains(chipLower, "nct6687") || // Nuvoton NCT6687 SuperIO
strings.Contains(chipLower, "nct6775") || // Nuvoton NCT6775 SuperIO
strings.Contains(chipLower, "nct6776") || // Nuvoton NCT6776 SuperIO
strings.Contains(chipLower, "nct6779") || // Nuvoton NCT6779 SuperIO
strings.Contains(chipLower, "nct6791") || // Nuvoton NCT6791 SuperIO
strings.Contains(chipLower, "nct6792") || // Nuvoton NCT6792 SuperIO
strings.Contains(chipLower, "nct6793") || // Nuvoton NCT6793 SuperIO
strings.Contains(chipLower, "nct6795") || // Nuvoton NCT6795 SuperIO
strings.Contains(chipLower, "nct6796") || // Nuvoton NCT6796 SuperIO
strings.Contains(chipLower, "nct6797") || // Nuvoton NCT6797 SuperIO
strings.Contains(chipLower, "nct6798") || // Nuvoton NCT6798 SuperIO
strings.Contains(chipLower, "w83627") || // Winbond W83627 SuperIO series
strings.Contains(chipLower, "f71882") || // Fintek F71882 SuperIO
strings.Contains(chipLower, "cpu_thermal") || // Raspberry Pi CPU temperature
strings.Contains(chipLower, "rp1_adc") || // Raspberry Pi RP1 ADC
strings.Contains(chipLower, "rpitemp") {
foundCPUChip = true
log.Debug().
Str("chip", chipName).
Msg("Detected CPU temperature chip")
tc.parseCPUTemps(chipMap, temp)
}
// Handle NVMe temperature sensors
if strings.Contains(chipName, "nvme") {
tc.parseNVMeTemps(chipName, chipMap, temp)
}
// Handle GPU temperature sensors
if strings.Contains(chipLower, "amdgpu") {
log.Debug().
Str("chip", chipName).
Msg("Detected AMD GPU temperature chip")
tc.parseGPUTemps(chipName, chipMap, temp)
}
// Handle NVIDIA GPU temperature sensors (nouveau driver)
if strings.Contains(chipLower, "nouveau") {
log.Debug().
Str("chip", chipName).
Msg("Detected NVIDIA GPU temperature chip (nouveau)")
tc.parseNouveauGPUTemps(chipName, chipMap, temp)
}
}
// If we got CPU temps, calculate max from cores if package not available
if temp.CPUPackage == 0 && len(temp.Cores) > 0 {
for _, core := range temp.Cores {
if core.Temp > temp.CPUMax {
temp.CPUMax = core.Temp
}
}
}
// Set individual sensor type flags based on chip presence, not value thresholds
// This prevents false negatives when sensors report 0°C during resets or temporarily
temp.HasCPU = foundCPUChip
temp.HasNVMe = len(temp.NVMe) > 0
temp.HasGPU = len(temp.GPU) > 0
temp.HasSMART = len(temp.SMART) > 0
// Available means any temperature data exists (backward compatibility)
temp.Available = temp.HasCPU || temp.HasNVMe || temp.HasGPU || temp.HasSMART
// Log summary of what was detected
if !foundCPUChip {
// List all chip names found for debugging
chipNames := make([]string, 0, len(sensorsData))
for chipName := range sensorsData {
chipNames = append(chipNames, chipName)
}
log.Debug().
Strs("chips", chipNames).
Msg("No recognized CPU temperature chip found in sensors output")
} else {
log.Debug().
Bool("hasCPU", temp.HasCPU).
Bool("hasNVMe", temp.HasNVMe).
Bool("hasGPU", temp.HasGPU).
Bool("hasSMART", temp.HasSMART).
Float64("cpuPackage", temp.CPUPackage).
Float64("cpuMax", temp.CPUMax).
Int("coreCount", len(temp.Cores)).
Int("nvmeCount", len(temp.NVMe)).
Int("gpuCount", len(temp.GPU)).
Int("smartCount", len(temp.SMART)).
Msg("Temperature data parsed successfully")
}
return temp, nil
}
// parseCPUTemps extracts CPU temperature data from a sensor chip
func (tc *TemperatureCollector) parseCPUTemps(chipMap map[string]interface{}, temp *models.Temperature) {
foundPackageTemp := false
var chipletTemps []float64 // Store AMD Tccd chiplet temps for fallback
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
sensorNameLower := strings.ToLower(sensorName)
// Look for Package id (Intel) or Tdie/Tctl (AMD control loop temperature)
if strings.Contains(sensorName, "Package id") ||
strings.Contains(sensorName, "Tdie") ||
strings.Contains(sensorNameLower, "tctl") {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
temp.CPUPackage = tempVal
foundPackageTemp = true
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Float64("temp", tempVal).
Msg("Found CPU package temperature")
}
}
// Look for AMD chiplet temperatures (Tccd1, Tccd2, etc.) as fallback
if strings.HasPrefix(sensorName, "Tccd") {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
chipletTemps = append(chipletTemps, tempVal)
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Float64("temp", tempVal).
Msg("Found AMD chiplet temperature")
}
}
// Look for SuperIO chip CPU temperature fields (CPUTIN, CPU Temperature, etc.)
if strings.Contains(sensorNameLower, "cputin") ||
strings.Contains(sensorNameLower, "cpu temperature") ||
(strings.Contains(sensorNameLower, "temp") && strings.Contains(sensorNameLower, "cpu")) {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
if !foundPackageTemp {
temp.CPUPackage = tempVal
foundPackageTemp = true
}
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Float64("temp", tempVal).
Msg("Found SuperIO CPU temperature")
}
}
// Look for individual cores
if strings.HasPrefix(sensorName, "Core ") {
coreNum := extractCoreNumber(sensorName)
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
temp.Cores = append(temp.Cores, models.CoreTemp{
Core: coreNum,
Temp: tempVal,
})
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
log.Debug().
Str("sensor", sensorName).
Int("core", coreNum).
Float64("temp", tempVal).
Msg("Found core temperature")
}
}
}
// If no package temperature found, use highest chiplet temp (AMD Ryzen)
if !foundPackageTemp && len(chipletTemps) > 0 {
for _, chipletTemp := range chipletTemps {
if chipletTemp > temp.CPUPackage {
temp.CPUPackage = chipletTemp
}
}
foundPackageTemp = true
log.Debug().
Float64("temp", temp.CPUPackage).
Msg("Using highest chiplet temperature as CPU package temperature")
}
// If no package temperature was found (e.g., Raspberry Pi), look for generic temp sensors
if !foundPackageTemp {
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
// Look for generic temperature sensors (e.g., "temp1" on Raspberry Pi)
if strings.HasPrefix(sensorName, "temp") || strings.HasPrefix(sensorName, "Temp") {
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
if temp.CPUPackage <= 0 {
temp.CPUPackage = tempVal
}
if tempVal > temp.CPUMax {
temp.CPUMax = tempVal
}
break // Use the first valid generic temp sensor
}
}
}
}
}
// parseNVMeTemps extracts NVMe temperature data from a sensor chip
func (tc *TemperatureCollector) parseNVMeTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
// Extract device name from chip name (e.g., "nvme-pci-0400" -> "nvme0")
device := "nvme" + strings.TrimPrefix(chipName, "nvme-pci-")
// Try "Composite" first (preferred sensor name for NVMe temps)
for sensorName, sensorData := range chipMap {
if !strings.Contains(sensorName, "Composite") {
continue
}
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
Device: device,
Temp: tempVal,
})
return
}
}
// Fall back to "Sensor 1" if no valid Composite found
for sensorName, sensorData := range chipMap {
if !strings.Contains(sensorName, "Sensor 1") {
continue
}
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
temp.NVMe = append(temp.NVMe, models.NVMeTemp{
Device: device,
Temp: tempVal,
})
return
}
}
}
// parseGPUTemps extracts GPU temperature data from a sensor chip
func (tc *TemperatureCollector) parseGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
gpuTemp := models.GPUTemp{
Device: chipName,
}
// AMD GPU sensors typically have: edge, junction (hotspot), mem
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
sensorLower := strings.ToLower(sensorName)
tempVal := extractTempInput(sensorMap)
if math.IsNaN(tempVal) || tempVal <= 0 {
continue
}
// Map sensor names to struct fields
if strings.Contains(sensorLower, "edge") {
gpuTemp.Edge = tempVal
} else if strings.Contains(sensorLower, "junction") || strings.Contains(sensorLower, "hotspot") {
gpuTemp.Junction = tempVal
} else if strings.Contains(sensorLower, "mem") {
gpuTemp.Mem = tempVal
}
}
// Only add GPU entry if we got at least one valid temperature
if gpuTemp.Edge > 0 || gpuTemp.Junction > 0 || gpuTemp.Mem > 0 {
temp.GPU = append(temp.GPU, gpuTemp)
log.Debug().
Str("device", chipName).
Float64("edge", gpuTemp.Edge).
Float64("junction", gpuTemp.Junction).
Float64("mem", gpuTemp.Mem).
Msg("Parsed GPU temperatures")
}
}
// parseNouveauGPUTemps extracts NVIDIA GPU temperature data from nouveau driver sensors
func (tc *TemperatureCollector) parseNouveauGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
gpuTemp := models.GPUTemp{
Device: chipName,
}
// Nouveau driver typically exposes "GPU core" sensor
for sensorName, sensorData := range chipMap {
sensorMap, ok := sensorData.(map[string]interface{})
if !ok {
continue
}
sensorLower := strings.ToLower(sensorName)
tempVal := extractTempInput(sensorMap)
if math.IsNaN(tempVal) || tempVal <= 0 {
continue
}
// Nouveau typically has "GPU core" sensor - map to edge temperature
if strings.Contains(sensorLower, "gpu") || strings.Contains(sensorLower, "core") {
gpuTemp.Edge = tempVal
}
}
// Only add GPU entry if we got a valid temperature
if gpuTemp.Edge > 0 {
temp.GPU = append(temp.GPU, gpuTemp)
log.Debug().
Str("device", chipName).
Float64("edge", gpuTemp.Edge).
Msg("Parsed NVIDIA GPU (nouveau) temperature")
}
}
// extractTempInput extracts temperature value from sensor data
func extractTempInput(sensorMap map[string]interface{}) float64 {
// Look for temp*_input fields
for key, val := range sensorMap {
if sensors.IsTemperatureInput(key) {
switch v := val.(type) {
case float64:
return v
case int:
return float64(v)
case string:
if parsed, ok := parseStringTemperature(v); ok {
return parsed
}
}
}
}
return math.NaN()
}
func parseStringTemperature(value string) (float64, bool) {
value = strings.TrimSpace(value)
if value == "" {
return 0, false
}
parsed, err := strconv.ParseFloat(value, 64)
if err != nil {
if _, scanErr := fmt.Sscanf(value, "%f", &parsed); scanErr != nil {
return 0, false
}
}
if math.Abs(parsed) >= 1000 {
parsed = parsed / 1000.0
}
return parsed, true
}
// extractCoreNumber extracts the core number from a sensor name like "Core 0"
func extractCoreNumber(name string) int {
parts := strings.Fields(name)
if len(parts) >= 2 {
if num, err := strconv.Atoi(parts[len(parts)-1]); err == nil {
return num
}
}
return 0
}
// parseRPiTemperature parses Raspberry Pi temperature from /sys/class/thermal/thermal_zone0/temp
// Format: integer representing millidegrees Celsius (e.g., "45678" = 45.678°C)
func (tc *TemperatureCollector) parseRPiTemperature(output string) (*models.Temperature, error) {
millidegrees := strings.TrimSpace(output)
if millidegrees == "" {
return nil, fmt.Errorf("empty RPi temperature output")
}
tempMilliC, err := strconv.ParseFloat(millidegrees, 64)
if err != nil {
return nil, fmt.Errorf("failed to parse RPi temperature: %w", err)
}
// Convert millidegrees to degrees Celsius
tempC := tempMilliC / 1000.0
temp := &models.Temperature{
Available: true,
HasCPU: true,
CPUPackage: tempC,
CPUMax: tempC,
Cores: []models.CoreTemp{},
NVMe: []models.NVMeTemp{},
LastUpdate: time.Now(),
}
return temp, nil
}
// extractHostname extracts hostname/IP from a Proxmox host URL
func extractHostname(hostURL string) string {
// Remove protocol
host := strings.TrimPrefix(hostURL, "https://")
host = strings.TrimPrefix(host, "http://")
// Remove port
if idx := strings.Index(host, ":"); idx != -1 {
host = host[:idx]
}
// Remove path
if idx := strings.Index(host, "/"); idx != -1 {
host = host[:idx]
}
return host
}
func (tc *TemperatureCollector) ensureHostKey(ctx context.Context, host string) error {
if tc.hostKeys == nil {
return nil
}
if ctx == nil {
ctx = context.Background()
}
return tc.hostKeys.EnsureWithPort(ctx, host, tc.sshPort)
}