Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

34.1 KB
Raw Blame
package monitoring

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/sensors"
	"github.com/rcourtman/pulse-go-rewrite/internal/system"
	"github.com/rs/zerolog/log"
)

const maxTemperatureCommandOutputSize = 1 << 20 // 1 MiB
const defaultSSHCommandTimeout = 15 * time.Second
const pulseSensorsSSHCommand = "if command -v /usr/local/sbin/pulse-sensors >/dev/null 2>&1; then /usr/local/sbin/pulse-sensors; else sensors -j 2>/dev/null || true; fi"

var errTemperatureCommandOutputTooLarge = errors.New("temperature command output exceeded limit")

// CommandRunner abstracts command execution for testing
type CommandRunner interface {
	Run(ctx context.Context, name string, args ...string) ([]byte, error)
}

type defaultCommandRunner struct{}

type limitedTemperatureBuffer struct {
	buf      bytes.Buffer
	maxBytes int
	exceeded bool
}

func (b *limitedTemperatureBuffer) Write(p []byte) (int, error) {
	remaining := b.maxBytes - b.buf.Len()
	if remaining <= 0 {
		b.exceeded = true
		return 0, errTemperatureCommandOutputTooLarge
	}
	if len(p) > remaining {
		b.exceeded = true
		written, _ := b.buf.Write(p[:remaining])
		return written, errTemperatureCommandOutputTooLarge
	}
	return b.buf.Write(p)
}

func (b *limitedTemperatureBuffer) Bytes() []byte {
	return b.buf.Bytes()
}

func (r *defaultCommandRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	stdout := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
	stderr := limitedTemperatureBuffer{maxBytes: maxTemperatureCommandOutputSize}
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr

	err := cmd.Run()
	if stdout.exceeded || stderr.exceeded {
		return nil, errTemperatureCommandOutputTooLarge
	}
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			// Preserve bounded stderr details for upstream error wrapping.
			exitErr.Stderr = append([]byte(nil), stderr.Bytes()...)
		}
		return stdout.Bytes(), err
	}

	return stdout.Bytes(), nil
}

// TemperatureCollector handles SSH-based temperature collection from Proxmox nodes
type TemperatureCollector struct {
	sshUser          string // SSH user (typically "root" or "pulse-monitor")
	sshKeyPath       string // Path to SSH private key
	sshPort          int    // SSH port (default 22)
	hostKeys         KnownHostsManager
	missingKeyWarned atomic.Bool
	runner           CommandRunner
	sshFailureMu     sync.Mutex
	sshFailures      map[string]*temperatureSSHFailure
	sshKeyIdentity   string
}

// temperatureSSHFailure remembers a host whose SSH collection failed so
// subsequent 10s poll cycles don't re-exec ssh (twice, with the RPi fallback)
// against a host that keeps failing identically (#1638). Backoff doubles per
// consecutive failure; any successful collection clears it.
type temperatureSSHFailure struct {
	retryAt time.Time
	backoff time.Duration
}

const (
	temperatureSSHFailureInitialBackoff = 30 * time.Second
	temperatureSSHFailureMaxBackoff     = 15 * time.Minute
)

// sshFailureOutcome describes what a failed collection attempt tells us about
// the host, which decides how the per-host backoff moves.
type sshFailureOutcome int

const (
	// sshFailureHard is a refusal, auth failure, or unusable output: the host
	// answered (or actively did not) and the backoff should compound.
	sshFailureHard sshFailureOutcome = iota
	// sshFailureTransient is our own deadline expiring. That says nothing about
	// the host, so hold the backoff at the floor instead of doubling it.
	sshFailureTransient
	// sshFailureNotAttempted means no ssh ran at all — the knownhosts manager
	// suppressed the call inside its own backoff. Nothing was learned and
	// nothing was spent, so the backoff must not move.
	sshFailureNotAttempted
)

// classifySSHFailure maps a failed attempt onto its backoff treatment (#1638).
func classifySSHFailure(err error) sshFailureOutcome {
	switch {
	case err == nil:
		return sshFailureHard
	case errors.Is(err, ErrKeyscanSuppressed):
		return sshFailureNotAttempted
	case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled):
		return sshFailureTransient
	default:
		return sshFailureHard
	}
}

func (tc *TemperatureCollector) inSSHFailureBackoff(host string) bool {
	tc.sshFailureMu.Lock()
	defer tc.sshFailureMu.Unlock()
	failure := tc.sshFailures[host]
	return failure != nil && sshBackoffNow().Before(failure.retryAt)
}

func (tc *TemperatureCollector) recordSSHFailure(host string, outcome sshFailureOutcome) {
	if outcome == sshFailureNotAttempted {
		return
	}

	tc.sshFailureMu.Lock()
	defer tc.sshFailureMu.Unlock()
	if tc.sshFailures == nil {
		tc.sshFailures = make(map[string]*temperatureSSHFailure)
	}

	now := sshBackoffNow()
	var previousRetryAt time.Time
	var previousBackoff time.Duration
	if existing := tc.sshFailures[host]; existing != nil {
		previousRetryAt = existing.retryAt
		previousBackoff = existing.backoff
	}
	backoff := nextSSHBackoff(now, previousRetryAt, previousBackoff, temperatureSSHFailureInitialBackoff, temperatureSSHFailureMaxBackoff, outcome == sshFailureHard)

	tc.sshFailures[host] = &temperatureSSHFailure{
		retryAt: now.Add(backoff),
		backoff: backoff,
	}
}

func (tc *TemperatureCollector) clearSSHFailure(host string) {
	tc.sshFailureMu.Lock()
	defer tc.sshFailureMu.Unlock()
	delete(tc.sshFailures, host)
}

// ResetSSHFailures drops every recorded SSH backoff, including the knownhosts
// manager's keyscan backoff, so an operator who has just repaired an SSH key or
// host key sees the next poll cycle retry rather than waiting out a window that
// has compounded to a quarter of an hour (#1638).
func (tc *TemperatureCollector) ResetSSHFailures() {
	if tc == nil {
		return
	}

	tc.sshFailureMu.Lock()
	tc.sshFailures = nil
	hostKeys := tc.hostKeys
	tc.sshFailureMu.Unlock()

	if hostKeys != nil {
		hostKeys.ResetFailures()
	}
}

// resetSSHFailuresOnKeyChange clears the backoff maps when the SSH key on disk
// has been replaced since the last cycle.
//
// Repairing that key is the usual fix for the failures that opened these
// backoff windows in the first place, and nothing else notices it happening, so
// without this the operator waits out a window that has already compounded
// toward fifteen minutes before Pulse tries the repaired key (#1638).
func (tc *TemperatureCollector) resetSSHFailuresOnKeyChange() {
	identity := tc.currentSSHKeyIdentity()

	tc.sshFailureMu.Lock()
	changed := tc.sshKeyIdentity != "" && tc.sshKeyIdentity != identity
	tc.sshKeyIdentity = identity
	tc.sshFailureMu.Unlock()

	if changed {
		log.Info().
			Str("sshKeyPath", tc.sshKeyPath).
			Msg("Temperature SSH key changed on disk; clearing per-host SSH backoff")
		tc.ResetSSHFailures()
	}
}

// currentSSHKeyIdentity fingerprints the key file cheaply enough to run every
// poll cycle. Content is deliberately not read: this is change detection, not
// validation.
func (tc *TemperatureCollector) currentSSHKeyIdentity() string {
	info, err := os.Stat(strings.TrimSpace(tc.sshKeyPath))
	if err != nil {
		return ""
	}
	return fmt.Sprintf("%d:%d", info.ModTime().UnixNano(), info.Size())
}

// NewTemperatureCollectorWithPort creates a new temperature collector with custom SSH port
func NewTemperatureCollectorWithPort(sshUser, sshKeyPath string, sshPort int) *TemperatureCollector {
	if sshPort <= 0 {
		sshPort = 22 // Default to standard SSH port
	}

	tc := &TemperatureCollector{
		sshUser:    sshUser,
		sshKeyPath: sshKeyPath,
		sshPort:    sshPort,
		runner:     &defaultCommandRunner{},
	}

	homeDir := os.Getenv("HOME")
	if homeDir == "" {
		homeDir = "/home/pulse"
	}
	knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts_sensors")
	if manager, err := NewKnownHostsManager(knownHostsPath); err != nil {
		log.Warn().Err(err).Str("path", knownHostsPath).Msg("failed to initialize temperature known_hosts manager")
	} else {
		tc.hostKeys = manager
	}

	return tc
}

// CollectTemperature collects temperature data from a node via SSH
func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost, nodeName string) (*models.Temperature, error) {
	// Extract hostname/IP from the host URL (might be https://hostname:8006)
	host := extractHostname(nodeHost)

	// SECURITY: Warn about legacy SSH collection in containers; this is not a block.
	// Container compromise = SSH key compromise = root access to infrastructure
	devModeAllowSSH := os.Getenv("PULSE_DEV_ALLOW_CONTAINER_SSH") == "true"
	isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer()

	if isContainer && devModeAllowSSH {
		// Log when dev override is active so operators understand the security posture
		log.Info().
			Str("node", nodeName).
			Msg("Temperature collection using direct SSH (dev mode override active - not for production)")
	}

	if isContainer && !devModeAllowSSH {
		// Warn but allow if key is present (legacy behavior restoration)
		// We don't return here, allowing the code to fall through to the SSH key check
		log.Warn().
			Str("node", nodeName).
			Msg("Temperature collection using direct SSH from container. This is insecure for production deployments.")
	}

	if strings.TrimSpace(tc.sshKeyPath) == "" {
		tc.logMissingSSHKey(nil)
		return &models.Temperature{Available: false}, nil
	}

	if keyErr := tc.validateSSHKeyPath(); keyErr != nil {
		tc.logMissingSSHKey(keyErr)
		return &models.Temperature{Available: false}, nil
	}

	tc.resetSSHFailuresOnKeyChange()

	if tc.inSSHFailureBackoff(host) {
		log.Debug().
			Str("node", nodeName).
			Str("host", host).
			Msg("Skipping SSH temperature collection while failure backoff is active")
		return &models.Temperature{Available: false}, nil
	}

	// Direct SSH (legacy method, including containers with an existing usable key).
	// New setup scripts restrict the key to /usr/local/sbin/pulse-sensors, which emits
	// the canonical {sensors, smart} payload. Older keys ignore the requested command
	// and still force sensors -j, so keep the parser backward-compatible.
	output, err := tc.runSSHCommand(ctx, host, pulseSensorsSSHCommand)
	if err != nil || strings.TrimSpace(output) == "" {
		// A suppressed keyscan or an expired deadline would fail the RPi
		// fallback identically, so stop here rather than paying for a second
		// probe, and do not compound a backoff the host has not earned (#1638).
		if outcome := classifySSHFailure(err); outcome != sshFailureHard {
			tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
			tc.recordSSHFailure(host, outcome)
			return &models.Temperature{Available: false}, nil
		}

		if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
			tc.recordSSHFailure(host, sshFailureHard)
			return &models.Temperature{Available: false}, nil
		}

		// Try Raspberry Pi temperature method
		output, err = tc.runSSHCommand(ctx, host, "cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null")
		if err == nil && strings.TrimSpace(output) != "" {
			// Parse RPi temperature format
			temp, parseErr := tc.parseRPiTemperature(output)
			if parseErr == nil {
				tc.clearSSHFailure(host)
				return temp, nil
			}
		}

		if outcome := classifySSHFailure(err); outcome != sshFailureHard {
			tc.logSkippedSSHAttempt(outcome, nodeName, host, err)
			tc.recordSSHFailure(host, outcome)
			return &models.Temperature{Available: false}, nil
		}

		if tc.disableLegacySSHOnAuthFailure(err, nodeName, host) {
			tc.recordSSHFailure(host, sshFailureHard)
			return &models.Temperature{Available: false}, nil
		}

		log.Debug().
			Str("node", nodeName).
			Str("host", host).
			Err(err).
			Msg("Failed to collect temperature data via SSH (tried both lm-sensors and RPi methods)")
		tc.recordSSHFailure(host, sshFailureHard)
		return &models.Temperature{Available: false}, nil
	}

	tc.clearSSHFailure(host)

	// Parse sensors JSON output
	temp, err := tc.parseSensorsJSON(output)
	if err != nil {
		log.Debug().
			Str("node", nodeName).
			Err(err).
			Msg("Failed to parse sensors output")
		return &models.Temperature{Available: false}, nil
	}

	if !temp.Available {
		return temp, nil
	}

	temp.LastUpdate = time.Now()

	return temp, nil
}

// logSkippedSSHAttempt records why an attempt is being abandoned without
// escalating the per-host backoff.
func (tc *TemperatureCollector) logSkippedSSHAttempt(outcome sshFailureOutcome, nodeName, host string, err error) {
	event := log.Debug().Str("node", nodeName).Str("host", host).Err(err)
	if outcome == sshFailureNotAttempted {
		event.Msg("Skipping SSH temperature collection; host key scan is in its own backoff window")
		return
	}
	event.Msg("Abandoning SSH temperature collection after the collection deadline expired")
}

func (tc *TemperatureCollector) runSSHCommand(ctx context.Context, host, command string) (string, error) {
	if err := tc.validateSSHKeyPath(); err != nil {
		return "", err
	}

	runCtx, cancel := withDefaultTimeout(ctx, defaultSSHCommandTimeout)
	defer cancel()

	if err := tc.ensureHostKey(runCtx, host); err != nil {
		return "", err
	}

	// Build SSH command with appropriate options
	sshArgs := []string{
		"-o", "StrictHostKeyChecking=yes",
		"-o", "BatchMode=yes",
		"-o", "LogLevel=ERROR", // Suppress host key warnings that break JSON parsing
		"-o", "ConnectTimeout=5",
		"-p", strconv.Itoa(tc.sshPort), // Use configured SSH port
	}

	if tc.hostKeys != nil && tc.hostKeys.Path() != "" {
		sshArgs = append(sshArgs,
			"-o", fmt.Sprintf("UserKnownHostsFile=%s", tc.hostKeys.Path()),
			"-o", "GlobalKnownHostsFile=/dev/null",
		)
	}

	// Explicitly use SSH config file if it exists (for ProxyJump configuration)
	homeDir := os.Getenv("HOME")
	if homeDir == "" {
		homeDir = "/home/pulse"
	}
	sshConfigPath := filepath.Join(homeDir, ".ssh/config")
	if _, err := os.Stat(sshConfigPath); err == nil {
		sshArgs = append(sshArgs, "-F", sshConfigPath)
	}

	// Add key if specified
	if tc.sshKeyPath != "" {
		sshArgs = append(sshArgs, "-i", tc.sshKeyPath)
	}

	// Add user@host and command
	sshArgs = append(sshArgs, fmt.Sprintf("%s@%s", tc.sshUser, host), command)

	output, err := tc.runner.Run(runCtx, "ssh", sshArgs...)
	if err != nil {
		if errors.Is(err, errTemperatureCommandOutputTooLarge) {
			return "", fmt.Errorf("ssh command output exceeded %d bytes", maxTemperatureCommandOutputSize)
		}
		// Our own budget running out is not evidence about the host, and
		// sanitizeSSHCommandError would flatten the signal into an exit status,
		// so report the context error directly (#1638).
		if ctxErr := runCtx.Err(); ctxErr != nil {
			return "", fmt.Errorf("ssh command aborted: %w", ctxErr)
		}
		return "", sanitizeSSHCommandError(err)
	}

	outputStr := strings.TrimSpace(string(output))

	// Strip any leading SSH noise (e.g., "Warning: Permanently added ...") so sensors JSON parses cleanly.
	if idx := strings.Index(outputStr, "{"); idx > 0 {
		outputStr = outputStr[idx:]
	}
	if idx := strings.LastIndex(outputStr, "}"); idx != -1 && idx < len(outputStr)-1 {
		outputStr = outputStr[:idx+1]
	}

	return outputStr, nil
}

func withDefaultTimeout(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) {
	if ctx == nil {
		return context.WithTimeout(context.Background(), timeout)
	}
	if _, hasDeadline := ctx.Deadline(); hasDeadline {
		return ctx, func() {}
	}
	return context.WithTimeout(ctx, timeout)
}

func sanitizeSSHCommandError(err error) error {
	var exitErr *exec.ExitError
	if errors.As(err, &exitErr) {
		stderrLower := strings.ToLower(string(exitErr.Stderr))
		switch {
		case strings.Contains(stderrLower, "permission denied"),
			strings.Contains(stderrLower, "publickey"),
			strings.Contains(stderrLower, "authentication failed"):
			return errors.New("ssh command failed: authentication failed")
		case strings.Contains(stderrLower, "host key verification failed"):
			return errors.New("ssh command failed: host key verification failed")
		case strings.Contains(stderrLower, "connection timed out"),
			strings.Contains(stderrLower, "operation timed out"),
			strings.Contains(stderrLower, "connection timeout"):
			return errors.New("ssh command failed: timeout")
		default:
			if code := exitErr.ExitCode(); code >= 0 {
				return fmt.Errorf("ssh command failed with exit status %d", code)
			}
			return errors.New("ssh command failed")
		}
	}

	return fmt.Errorf("ssh command failed: %w", err)
}

func (tc *TemperatureCollector) validateSSHKeyPath() error {
	keyPath := strings.TrimSpace(tc.sshKeyPath)
	if keyPath == "" {
		return errors.New("temperature SSH key unavailable")
	}

	info, err := os.Stat(keyPath)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return os.ErrNotExist
		}
		return errors.New("temperature SSH key unavailable")
	}
	if info.IsDir() {
		return errors.New("temperature SSH key path is not a file")
	}
	if err := validatePrivateKeyPermissions(info.Mode().Perm()); err != nil {
		return err
	}
	return nil
}

func validatePrivateKeyPermissions(mode os.FileMode) error {
	perm := mode.Perm()
	if perm&0o077 != 0 {
		return fmt.Errorf("temperature SSH key has insecure permissions (%03o): expected no group/other access", perm)
	}
	return nil
}

func (tc *TemperatureCollector) logMissingSSHKey(cause error) {
	if tc.missingKeyWarned.Load() {
		return
	}
	if tc.missingKeyWarned.CompareAndSwap(false, true) {
		event := log.Debug().
			Str("sshKeyPath", tc.sshKeyPath)
		if cause != nil && !errors.Is(cause, os.ErrNotExist) {
			event = event.Err(cause)
		}
		event.Msg("temperature SSH key not available; skipping legacy SSH collection")
	}
}

func (tc *TemperatureCollector) disableLegacySSHOnAuthFailure(err error, nodeName, host string) bool {
	if err == nil {
		return false
	}

	msg := strings.ToLower(err.Error())
	authFailure := strings.Contains(msg, "permission denied") ||
		strings.Contains(msg, "authentication failed") ||
		strings.Contains(msg, "publickey")

	if !authFailure {
		return false
	}

	// Do not disable globally on single node failure
	log.Warn().
		Str("node", nodeName).
		Str("host", host).
		Err(err).
		Msg("SSH temperature collection failed due to authentication error; check SSH keys")

	return true
}

type smartEntryRaw struct {
	Device         string `json:"device"`
	Serial         string `json:"serial,omitempty"`
	WWN            string `json:"wwn,omitempty"`
	Model          string `json:"model,omitempty"`
	Type           string `json:"type,omitempty"`
	Temperature    *int   `json:"temperature"`
	LastUpdated    string `json:"lastUpdated,omitempty"`
	StandbySkipped bool   `json:"standbySkipped,omitempty"`
}

func normalizeSMARTEntries(raw []smartEntryRaw) []models.DiskTemp {
	if len(raw) == 0 {
		return nil
	}

	normalized := make([]models.DiskTemp, 0, len(raw))
	for _, entry := range raw {
		dev := strings.TrimSpace(entry.Device)
		if dev == "" {
			continue
		}

		var lastUpdated time.Time
		if entry.LastUpdated != "" {
			if parsed, err := time.Parse(time.RFC3339, entry.LastUpdated); err == nil {
				lastUpdated = parsed
			}
		}

		tempVal := 0
		if entry.Temperature != nil {
			tempVal = *entry.Temperature
		}

		normalized = append(normalized, models.DiskTemp{
			Device:         dev,
			Serial:         strings.TrimSpace(entry.Serial),
			WWN:            strings.TrimSpace(entry.WWN),
			Model:          strings.TrimSpace(entry.Model),
			Type:           strings.TrimSpace(entry.Type),
			Temperature:    tempVal,
			LastUpdated:    lastUpdated,
			StandbySkipped: entry.StandbySkipped,
		})
	}

	return normalized
}

// parseSensorsJSON parses the JSON output from the sensor wrapper
func (tc *TemperatureCollector) parseSensorsJSON(jsonStr string) (*models.Temperature, error) {
	if strings.TrimSpace(jsonStr) == "" {
		return nil, fmt.Errorf("empty sensors output")
	}

	// Try to parse as wrapper format first: {sensors: {...}, smart: [...]}
	// Fall back to legacy format for backward compatibility
	var wrapperData struct {
		Sensors map[string]interface{} `json:"sensors"`
		SMART   []smartEntryRaw        `json:"smart"`
	}

	var sensorsData map[string]interface{}
	var smartRaw []smartEntryRaw
	var parsedWrapper bool

	if err := json.Unmarshal([]byte(jsonStr), &wrapperData); err == nil && wrapperData.Sensors != nil {
		// New wrapper format
		sensorsData = wrapperData.Sensors
		smartRaw = wrapperData.SMART
		parsedWrapper = true
	} else {
		// Legacy format: direct sensors -j output
		if err := json.Unmarshal([]byte(jsonStr), &sensorsData); err != nil {
			return nil, fmt.Errorf("failed to parse sensors JSON: %w", err)
		}
		log.Debug().Msg("parsed legacy sensors format (no SMART data)")
	}

	smartData := normalizeSMARTEntries(smartRaw)
	if parsedWrapper {
		log.Debug().
			Int("smartDisks", len(smartData)).
			Msg("Parsed new wrapper format with SMART data")
	}

	temp := &models.Temperature{
		Cores:               []models.CoreTemp{},
		NVMe:                []models.NVMeTemp{},
		SMART:               smartData,
		LegacySensorsFormat: !parsedWrapper,
	}

	foundCPUChip := false

	// Parse each sensor chip
	for chipName, chipData := range sensorsData {
		chipMap, ok := chipData.(map[string]interface{})
		if !ok {
			continue
		}

		// Handle CPU temperature sensors
		chipLower := strings.ToLower(chipName)
		if strings.Contains(chipLower, "coretemp") ||
			strings.Contains(chipLower, "k10temp") ||
			strings.Contains(chipLower, "zenpower") ||
			strings.Contains(chipLower, "k8temp") ||
			strings.Contains(chipLower, "acpitz") ||
			strings.Contains(chipLower, "it87") ||
			strings.Contains(chipLower, "nct6687") || // Nuvoton NCT6687 SuperIO
			strings.Contains(chipLower, "nct6775") || // Nuvoton NCT6775 SuperIO
			strings.Contains(chipLower, "nct6776") || // Nuvoton NCT6776 SuperIO
			strings.Contains(chipLower, "nct6779") || // Nuvoton NCT6779 SuperIO
			strings.Contains(chipLower, "nct6791") || // Nuvoton NCT6791 SuperIO
			strings.Contains(chipLower, "nct6792") || // Nuvoton NCT6792 SuperIO
			strings.Contains(chipLower, "nct6793") || // Nuvoton NCT6793 SuperIO
			strings.Contains(chipLower, "nct6795") || // Nuvoton NCT6795 SuperIO
			strings.Contains(chipLower, "nct6796") || // Nuvoton NCT6796 SuperIO
			strings.Contains(chipLower, "nct6797") || // Nuvoton NCT6797 SuperIO
			strings.Contains(chipLower, "nct6798") || // Nuvoton NCT6798 SuperIO
			strings.Contains(chipLower, "w83627") || // Winbond W83627 SuperIO series
			strings.Contains(chipLower, "f71882") || // Fintek F71882 SuperIO
			strings.Contains(chipLower, "cpu_thermal") || // Raspberry Pi CPU temperature
			strings.Contains(chipLower, "rp1_adc") || // Raspberry Pi RP1 ADC
			strings.Contains(chipLower, "rpitemp") {
			foundCPUChip = true
			log.Debug().
				Str("chip", chipName).
				Msg("Detected CPU temperature chip")
			tc.parseCPUTemps(chipMap, temp)
		}

		// Handle NVMe temperature sensors
		if strings.Contains(chipName, "nvme") {
			tc.parseNVMeTemps(chipName, chipMap, temp)
		}

		// Handle GPU temperature sensors
		if strings.Contains(chipLower, "amdgpu") {
			log.Debug().
				Str("chip", chipName).
				Msg("Detected AMD GPU temperature chip")
			tc.parseGPUTemps(chipName, chipMap, temp)
		}

		// Handle NVIDIA GPU temperature sensors (nouveau driver)
		if strings.Contains(chipLower, "nouveau") {
			log.Debug().
				Str("chip", chipName).
				Msg("Detected NVIDIA GPU temperature chip (nouveau)")
			tc.parseNouveauGPUTemps(chipName, chipMap, temp)
		}
	}

	// If we got CPU temps, calculate max from cores if package not available
	if temp.CPUPackage == 0 && len(temp.Cores) > 0 {
		for _, core := range temp.Cores {
			if core.Temp > temp.CPUMax {
				temp.CPUMax = core.Temp
			}
		}
	}

	// Set individual sensor type flags based on chip presence, not value thresholds
	// This prevents false negatives when sensors report 0°C during resets or temporarily
	temp.HasCPU = foundCPUChip
	temp.HasNVMe = len(temp.NVMe) > 0
	temp.HasGPU = len(temp.GPU) > 0
	temp.HasSMART = len(temp.SMART) > 0

	// Available means any temperature data exists (backward compatibility)
	temp.Available = temp.HasCPU || temp.HasNVMe || temp.HasGPU || temp.HasSMART

	// Log summary of what was detected
	if !foundCPUChip {
		// List all chip names found for debugging
		chipNames := make([]string, 0, len(sensorsData))
		for chipName := range sensorsData {
			chipNames = append(chipNames, chipName)
		}
		log.Debug().
			Strs("chips", chipNames).
			Msg("No recognized CPU temperature chip found in sensors output")
	} else {
		log.Debug().
			Bool("hasCPU", temp.HasCPU).
			Bool("hasNVMe", temp.HasNVMe).
			Bool("hasGPU", temp.HasGPU).
			Bool("hasSMART", temp.HasSMART).
			Float64("cpuPackage", temp.CPUPackage).
			Float64("cpuMax", temp.CPUMax).
			Int("coreCount", len(temp.Cores)).
			Int("nvmeCount", len(temp.NVMe)).
			Int("gpuCount", len(temp.GPU)).
			Int("smartCount", len(temp.SMART)).
			Msg("Temperature data parsed successfully")
	}

	return temp, nil
}

// parseCPUTemps extracts CPU temperature data from a sensor chip
func (tc *TemperatureCollector) parseCPUTemps(chipMap map[string]interface{}, temp *models.Temperature) {
	foundPackageTemp := false
	var chipletTemps []float64 // Store AMD Tccd chiplet temps for fallback

	for sensorName, sensorData := range chipMap {
		sensorMap, ok := sensorData.(map[string]interface{})
		if !ok {
			continue
		}

		sensorNameLower := strings.ToLower(sensorName)

		// Look for Package id (Intel) or Tdie/Tctl (AMD control loop temperature)
		if strings.Contains(sensorName, "Package id") ||
			strings.Contains(sensorName, "Tdie") ||
			strings.Contains(sensorNameLower, "tctl") {
			if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
				temp.CPUPackage = tempVal
				foundPackageTemp = true
				if tempVal > temp.CPUMax {
					temp.CPUMax = tempVal
				}
				log.Debug().
					Str("sensor", sensorName).
					Float64("temp", tempVal).
					Msg("Found CPU package temperature")
			}
		}

		// Look for AMD chiplet temperatures (Tccd1, Tccd2, etc.) as fallback
		if strings.HasPrefix(sensorName, "Tccd") {
			if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
				chipletTemps = append(chipletTemps, tempVal)
				if tempVal > temp.CPUMax {
					temp.CPUMax = tempVal
				}
				log.Debug().
					Str("sensor", sensorName).
					Float64("temp", tempVal).
					Msg("Found AMD chiplet temperature")
			}
		}

		// Look for SuperIO chip CPU temperature fields (CPUTIN, CPU Temperature, etc.)
		if strings.Contains(sensorNameLower, "cputin") ||
			strings.Contains(sensorNameLower, "cpu temperature") ||
			(strings.Contains(sensorNameLower, "temp") && strings.Contains(sensorNameLower, "cpu")) {
			if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
				if !foundPackageTemp {
					temp.CPUPackage = tempVal
					foundPackageTemp = true
				}
				if tempVal > temp.CPUMax {
					temp.CPUMax = tempVal
				}
				log.Debug().
					Str("sensor", sensorName).
					Float64("temp", tempVal).
					Msg("Found SuperIO CPU temperature")
			}
		}

		// Look for individual cores
		if strings.HasPrefix(sensorName, "Core ") {
			coreNum := extractCoreNumber(sensorName)
			if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) {
				temp.Cores = append(temp.Cores, models.CoreTemp{
					Core: coreNum,
					Temp: tempVal,
				})
				if tempVal > temp.CPUMax {
					temp.CPUMax = tempVal
				}
				log.Debug().
					Str("sensor", sensorName).
					Int("core", coreNum).
					Float64("temp", tempVal).
					Msg("Found core temperature")
			}
		}
	}

	// If no package temperature found, use highest chiplet temp (AMD Ryzen)
	if !foundPackageTemp && len(chipletTemps) > 0 {
		for _, chipletTemp := range chipletTemps {
			if chipletTemp > temp.CPUPackage {
				temp.CPUPackage = chipletTemp
			}
		}
		foundPackageTemp = true
		log.Debug().
			Float64("temp", temp.CPUPackage).
			Msg("Using highest chiplet temperature as CPU package temperature")
	}

	// If no package temperature was found (e.g., Raspberry Pi), look for generic temp sensors
	if !foundPackageTemp {
		for sensorName, sensorData := range chipMap {
			sensorMap, ok := sensorData.(map[string]interface{})
			if !ok {
				continue
			}

			// Look for generic temperature sensors (e.g., "temp1" on Raspberry Pi)
			if strings.HasPrefix(sensorName, "temp") || strings.HasPrefix(sensorName, "Temp") {
				if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
					if temp.CPUPackage <= 0 {
						temp.CPUPackage = tempVal
					}
					if tempVal > temp.CPUMax {
						temp.CPUMax = tempVal
					}
					break // Use the first valid generic temp sensor
				}
			}
		}
	}
}

// parseNVMeTemps extracts NVMe temperature data from a sensor chip
func (tc *TemperatureCollector) parseNVMeTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
	// Extract device name from chip name (e.g., "nvme-pci-0400" -> "nvme0")
	device := "nvme" + strings.TrimPrefix(chipName, "nvme-pci-")

	// Try "Composite" first (preferred sensor name for NVMe temps)
	for sensorName, sensorData := range chipMap {
		if !strings.Contains(sensorName, "Composite") {
			continue
		}
		sensorMap, ok := sensorData.(map[string]interface{})
		if !ok {
			continue
		}
		if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
			temp.NVMe = append(temp.NVMe, models.NVMeTemp{
				Device: device,
				Temp:   tempVal,
			})
			return
		}
	}

	// Fall back to "Sensor 1" if no valid Composite found
	for sensorName, sensorData := range chipMap {
		if !strings.Contains(sensorName, "Sensor 1") {
			continue
		}
		sensorMap, ok := sensorData.(map[string]interface{})
		if !ok {
			continue
		}
		if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
			temp.NVMe = append(temp.NVMe, models.NVMeTemp{
				Device: device,
				Temp:   tempVal,
			})
			return
		}
	}
}

// parseGPUTemps extracts GPU temperature data from a sensor chip
func (tc *TemperatureCollector) parseGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
	gpuTemp := models.GPUTemp{
		Device: chipName,
	}

	// AMD GPU sensors typically have: edge, junction (hotspot), mem
	for sensorName, sensorData := range chipMap {
		sensorMap, ok := sensorData.(map[string]interface{})
		if !ok {
			continue
		}

		sensorLower := strings.ToLower(sensorName)
		tempVal := extractTempInput(sensorMap)

		if math.IsNaN(tempVal) || tempVal <= 0 {
			continue
		}

		// Map sensor names to struct fields
		if strings.Contains(sensorLower, "edge") {
			gpuTemp.Edge = tempVal
		} else if strings.Contains(sensorLower, "junction") || strings.Contains(sensorLower, "hotspot") {
			gpuTemp.Junction = tempVal
		} else if strings.Contains(sensorLower, "mem") {
			gpuTemp.Mem = tempVal
		}
	}

	// Only add GPU entry if we got at least one valid temperature
	if gpuTemp.Edge > 0 || gpuTemp.Junction > 0 || gpuTemp.Mem > 0 {
		temp.GPU = append(temp.GPU, gpuTemp)
		log.Debug().
			Str("device", chipName).
			Float64("edge", gpuTemp.Edge).
			Float64("junction", gpuTemp.Junction).
			Float64("mem", gpuTemp.Mem).
			Msg("Parsed GPU temperatures")
	}
}

// parseNouveauGPUTemps extracts NVIDIA GPU temperature data from nouveau driver sensors
func (tc *TemperatureCollector) parseNouveauGPUTemps(chipName string, chipMap map[string]interface{}, temp *models.Temperature) {
	gpuTemp := models.GPUTemp{
		Device: chipName,
	}

	// Nouveau driver typically exposes "GPU core" sensor
	for sensorName, sensorData := range chipMap {
		sensorMap, ok := sensorData.(map[string]interface{})
		if !ok {
			continue
		}

		sensorLower := strings.ToLower(sensorName)
		tempVal := extractTempInput(sensorMap)

		if math.IsNaN(tempVal) || tempVal <= 0 {
			continue
		}

		// Nouveau typically has "GPU core" sensor - map to edge temperature
		if strings.Contains(sensorLower, "gpu") || strings.Contains(sensorLower, "core") {
			gpuTemp.Edge = tempVal
		}
	}

	// Only add GPU entry if we got a valid temperature
	if gpuTemp.Edge > 0 {
		temp.GPU = append(temp.GPU, gpuTemp)
		log.Debug().
			Str("device", chipName).
			Float64("edge", gpuTemp.Edge).
			Msg("Parsed NVIDIA GPU (nouveau) temperature")
	}
}

// extractTempInput extracts temperature value from sensor data
func extractTempInput(sensorMap map[string]interface{}) float64 {
	// Look for temp*_input fields
	for key, val := range sensorMap {
		if sensors.IsTemperatureInput(key) {
			switch v := val.(type) {
			case float64:
				return v
			case int:
				return float64(v)
			case string:
				if parsed, ok := parseStringTemperature(v); ok {
					return parsed
				}
			}
		}
	}
	return math.NaN()
}

func parseStringTemperature(value string) (float64, bool) {
	value = strings.TrimSpace(value)
	if value == "" {
		return 0, false
	}

	parsed, err := strconv.ParseFloat(value, 64)
	if err != nil {
		if _, scanErr := fmt.Sscanf(value, "%f", &parsed); scanErr != nil {
			return 0, false
		}
	}

	if math.Abs(parsed) >= 1000 {
		parsed = parsed / 1000.0
	}

	return parsed, true
}

// extractCoreNumber extracts the core number from a sensor name like "Core 0"
func extractCoreNumber(name string) int {
	parts := strings.Fields(name)
	if len(parts) >= 2 {
		if num, err := strconv.Atoi(parts[len(parts)-1]); err == nil {
			return num
		}
	}
	return 0
}

// parseRPiTemperature parses Raspberry Pi temperature from /sys/class/thermal/thermal_zone0/temp
// Format: integer representing millidegrees Celsius (e.g., "45678" = 45.678°C)
func (tc *TemperatureCollector) parseRPiTemperature(output string) (*models.Temperature, error) {
	millidegrees := strings.TrimSpace(output)
	if millidegrees == "" {
		return nil, fmt.Errorf("empty RPi temperature output")
	}

	tempMilliC, err := strconv.ParseFloat(millidegrees, 64)
	if err != nil {
		return nil, fmt.Errorf("failed to parse RPi temperature: %w", err)
	}

	// Convert millidegrees to degrees Celsius
	tempC := tempMilliC / 1000.0

	temp := &models.Temperature{
		Available:  true,
		HasCPU:     true,
		CPUPackage: tempC,
		CPUMax:     tempC,
		Cores:      []models.CoreTemp{},
		NVMe:       []models.NVMeTemp{},
		LastUpdate: time.Now(),
	}

	return temp, nil
}

// extractHostname extracts hostname/IP from a Proxmox host URL
func extractHostname(hostURL string) string {
	// Remove protocol
	host := strings.TrimPrefix(hostURL, "https://")
	host = strings.TrimPrefix(host, "http://")

	// Remove port
	if idx := strings.Index(host, ":"); idx != -1 {
		host = host[:idx]
	}

	// Remove path
	if idx := strings.Index(host, "/"); idx != -1 {
		host = host[:idx]
	}

	return host
}

func (tc *TemperatureCollector) ensureHostKey(ctx context.Context, host string) error {
	if tc.hostKeys == nil {
		return nil
	}
	if ctx == nil {
		ctx = context.Background()
	}
	return tc.hostKeys.EnsureWithPort(ctx, host, tc.sshPort)
}