Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

61.5 KB
Raw Blame
package hostagent

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/agenttarget"
	"github.com/rs/zerolog"
)

// ProxmoxSetup handles one-time Proxmox API token creation and node registration.
type ProxmoxSetup struct {
	logger             zerolog.Logger
	httpClient         *http.Client
	pulseURL           string
	apiToken           string
	proxmoxType        proxmoxProductType // pve, pbs, or unknown for auto-detect
	hostname           string
	reportIP           string
	insecureSkipVerify bool
	collector          SystemCollector
	stateDir           string          // directory for registration state files
	retryBackoffs      []time.Duration // overridable for testing; nil uses defaults
	monitorToken       string          // optional destination-scoped token name
	authoritative      bool            // primary destinations retain local-network HTTP compatibility
	allowPlaintextHTTP bool            // explicit observer plaintext transport policy
}

// ProxmoxSetupResult contains the result of a successful Proxmox setup.
type ProxmoxSetupResult struct {
	ProxmoxType string // "pve" or "pbs"
	TokenID     string // Full token ID (e.g., "pulse-monitor@pve!pulse-pulse-example-com")
	TokenValue  string // The secret token value
	NodeHost    string // The host URL for auto-registration
	NodeName    string // The canonical stored node identity returned by Pulse
	Registered  bool   // Whether the node was successfully registered with Pulse
}

type proxmoxProductType string

const (
	proxmoxProductUnknown proxmoxProductType = ""
	proxmoxProductPVE     proxmoxProductType = "pve"
	proxmoxProductPBS     proxmoxProductType = "pbs"
)

type autoRegisterSource string

const autoRegisterSourceAgent autoRegisterSource = "agent"

type autoRegisterRequest struct {
	Type              proxmoxProductType `json:"type"`
	Host              string             `json:"host"`
	CandidateHosts    []string           `json:"candidateHosts,omitempty"`
	ServerName        string             `json:"serverName"`
	AuthToken         string             `json:"authToken,omitempty"`
	TokenID           string             `json:"tokenId,omitempty"`
	TokenValue        string             `json:"tokenValue,omitempty"`
	Source            autoRegisterSource `json:"source"`
	CheckRegistration bool               `json:"checkRegistration,omitempty"`
}

type autoRegisterResponse struct {
	Status     string `json:"status"`
	Message    string `json:"message"`
	Action     string `json:"action"`
	Type       string `json:"type"`
	Source     string `json:"source"`
	Host       string `json:"host"`
	NodeID     string `json:"nodeId"`
	NodeName   string `json:"nodeName"`
	TokenID    string `json:"tokenId"`
	TokenValue string `json:"tokenValue"`
}

type autoRegisterCheckResponse struct {
	Registered   bool  `json:"registered"`
	SourceExists *bool `json:"sourceExists,omitempty"`
	CanRegister  *bool `json:"canRegister,omitempty"`
}

type registrationState struct {
	registered   bool
	sourceExists bool
	canRegister  bool
}

const maxRegistrationCheckResponseBytes = 8 * 1024

type setupScriptURLRequest struct {
	Type        string `json:"type"`
	Host        string `json:"host"`
	BackupPerms bool   `json:"backupPerms"`
}

type setupScriptURLResponse struct {
	Type              string `json:"type"`
	Host              string `json:"host"`
	URL               string `json:"url"`
	DownloadURL       string `json:"downloadURL"`
	ScriptFileName    string `json:"scriptFileName"`
	Command           string `json:"command"`
	CommandWithEnv    string `json:"commandWithEnv"`
	CommandWithoutEnv string `json:"commandWithoutEnv"`
	SetupToken        string `json:"setupToken"`
	TokenHint         string `json:"tokenHint"`
	Expires           int64  `json:"expires"`
}

func validateSetupScriptCommand(command string, expectedScriptURL string, setupToken string, tokenRequired bool) error {
	trimmedCommand := strings.TrimSpace(command)
	if trimmedCommand == "" {
		return fmt.Errorf("missing command")
	}
	if !strings.Contains(trimmedCommand, expectedScriptURL) {
		return fmt.Errorf("command url mismatch")
	}
	if !strings.Contains(trimmedCommand, `if [ "$(id -u)" -eq 0 ]; then`) {
		return fmt.Errorf("command missing direct-root path")
	}
	if !strings.Contains(trimmedCommand, `elif command -v sudo >/dev/null 2>&1; then`) {
		return fmt.Errorf("command missing sudo path")
	}
	// New servers separate reveal from private input. Accept the old coherent
	// artifact only for rolling upgrades against an older server; never mix
	// its token-bearing URL with the new command format.
	if strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN_FILE=") && !strings.Contains(trimmedCommand, setupToken) && !strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN=") {
		return nil
	}
	if tokenRequired {
		if !strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN=") || !strings.Contains(trimmedCommand, setupToken) {
			return fmt.Errorf("command missing setup token transport")
		}
		return nil
	}
	if strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN=") || strings.Contains(trimmedCommand, setupToken) {
		return fmt.Errorf("command leaked setup token transport")
	}
	return nil
}

const (
	proxmoxUser    = "pulse-monitor"
	proxmoxUserPVE = "pulse-monitor@pve"
	proxmoxUserPBS = "pulse-monitor@pbs"
	proxmoxComment = "Pulse monitoring service"
)

const proxmoxMonitorRole = "PulseMonitor"

var proxmoxTokenSlugPattern = regexp.MustCompile(`[^a-z0-9]+`)

func proxmoxTokenHostCandidate(candidate string) string {
	raw := strings.TrimSpace(candidate)
	if raw == "" {
		return ""
	}

	if strings.Contains(raw, "://") {
		if parsed, err := url.Parse(raw); err == nil && parsed.Hostname() != "" {
			return parsed.Hostname()
		}
	}

	if host, _, err := net.SplitHostPort(raw); err == nil {
		return strings.Trim(host, "[]")
	}

	if parsed, err := url.Parse("https://" + raw); err == nil && parsed.Hostname() != "" {
		return parsed.Hostname()
	}

	return strings.Trim(raw, "[]")
}

func proxmoxTokenSlug(raw string) string {
	trimmed := strings.Trim(strings.ToLower(strings.TrimSpace(raw)), ".")
	if trimmed == "" {
		return ""
	}

	slug := proxmoxTokenSlugPattern.ReplaceAllString(trimmed, "-")
	slug = strings.Trim(slug, "-")
	if slug == "" {
		return ""
	}

	const maxSlugLen = 48
	if len(slug) > maxSlugLen {
		slug = strings.Trim(slug[:maxSlugLen], "-")
	}
	return slug
}

func proxmoxTokenScope(candidates ...string) string {
	for _, candidate := range candidates {
		host := proxmoxTokenHostCandidate(candidate)
		if host == "" {
			continue
		}
		if slug := proxmoxTokenSlug(host); slug != "" {
			return slug
		}
	}
	return "server"
}

func (p *ProxmoxSetup) monitorTokenName() string {
	if tokenName := strings.TrimSpace(p.monitorToken); tokenName != "" {
		return tokenName
	}
	return "pulse-" + proxmoxTokenScope(p.hostname, p.pulseURL)
}

func tokenAlreadyExists(err error, output string) bool {
	lowerErr := strings.ToLower(fmt.Sprintf("%v", err))
	lowerOutput := strings.ToLower(output)
	return strings.Contains(lowerErr, "already exists") || strings.Contains(lowerOutput, "already exists")
}

func privProbeRoleName(priv string) string {
	// Keep role name deterministic (helps tests) and valid for pveum.
	// Replace characters that are likely to cause issues in role names.
	safe := strings.NewReplacer(".", "_", ":", "_", "/", "_", " ", "_", ",", "_").Replace(priv)
	return "PulseTmpPrivCheck_" + safe
}

func (p *ProxmoxSetup) probePVEPrivilege(ctx context.Context, privilege string) bool {
	roleName := privProbeRoleName(privilege)

	// If privilege doesn't exist on this PVE version, pveum will fail with a non-zero exit code.
	if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "add", roleName, "-privs", privilege); err != nil {
		return false
	}
	if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "delete", roleName); err != nil {
		p.logger.Debug().Err(err).Str("role", roleName).Msg("Failed to cleanup temporary PVE privilege probe role")
	}
	return true
}

func (p *ProxmoxSetup) applyPVEACL(ctx context.Context, path string, principalFlag string, principal string, role string) {
	if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "aclmod", path, principalFlag, principal, "-role", role); err != nil {
		p.logger.Warn().Err(err).Str("path", path).Str("principal", principal).Str("role", role).Msg("Failed to apply PVE role")
	}
}

func (p *ProxmoxSetup) configurePVEPermissions(ctx context.Context, tokenID string) {
	// Baseline: read-only access.
	p.applyPVEACL(ctx, "/", "-user", proxmoxUserPVE, "PVEAuditor")
	if tokenID != "" {
		p.applyPVEACL(ctx, "/", "-token", tokenID, "PVEAuditor")
	}

	// Extra privileges are optional, but enable additional features:
	// - Sys.Audit: required for pending apt updates + some cluster/ceph visibility
	// - VM.GuestAgent.Audit/FileRead (PVE 9+) or VM.Monitor (PVE 8): guest agent data
	// - Datastore.Audit: improved storage visibility
	var extraPrivs []string

	if p.probePVEPrivilege(ctx, "Sys.Audit") {
		extraPrivs = append(extraPrivs, "Sys.Audit")
	}

	if p.probePVEPrivilege(ctx, "VM.GuestAgent.Audit") {
		extraPrivs = append(extraPrivs, "VM.GuestAgent.Audit")
		if p.probePVEPrivilege(ctx, "VM.GuestAgent.FileRead") {
			extraPrivs = append(extraPrivs, "VM.GuestAgent.FileRead")
		}
	} else if p.probePVEPrivilege(ctx, "VM.Monitor") {
		extraPrivs = append(extraPrivs, "VM.Monitor")
	}

	if p.probePVEPrivilege(ctx, "Datastore.Audit") {
		extraPrivs = append(extraPrivs, "Datastore.Audit")
	}

	if len(extraPrivs) > 0 {
		privString := strings.Join(extraPrivs, ",")

		// Prefer modify (non-destructive) in case the role already exists.
		if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "modify", proxmoxMonitorRole, "-privs", privString); err != nil {
			if _, addErr := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "add", proxmoxMonitorRole, "-privs", privString); addErr != nil {
				p.logger.Warn().Err(addErr).Str("privs", privString).Msg("Failed to configure PulseMonitor role")
				return
			}
		}

		p.applyPVEACL(ctx, "/", "-user", proxmoxUserPVE, proxmoxMonitorRole)
		if tokenID != "" {
			p.applyPVEACL(ctx, "/", "-token", tokenID, proxmoxMonitorRole)
		}
	}

	// Add PVEDatastoreAdmin on /storage for backup visibility (issue #1139)
	p.applyPVEACL(ctx, "/storage", "-user", proxmoxUserPVE, "PVEDatastoreAdmin")
	if tokenID != "" {
		p.applyPVEACL(ctx, "/storage", "-token", tokenID, "PVEDatastoreAdmin")
	}
}

const (
	proxmoxStateDirPerm  = 0700
	proxmoxStateFilePerm = 0600
)

// State file path methods — derive from the configurable stateDir.
func (p *ProxmoxSetup) legacyStateFilePath() string {
	return filepath.Join(p.stateDir, "proxmox-registered")
}

func (p *ProxmoxSetup) pveStateFilePath() string {
	return filepath.Join(p.stateDir, "proxmox-pve-registered")
}

func (p *ProxmoxSetup) pbsStateFilePath() string {
	return filepath.Join(p.stateDir, "proxmox-pbs-registered")
}

// registrationBlockedFilePath is the marker the installer reads to surface a
// denied registration grant in its own output instead of leaving the failure
// buried in the agent journal (#1644).
func (p *ProxmoxSetup) registrationBlockedFilePath(ptype proxmoxProductType) string {
	return filepath.Join(p.stateDir, fmt.Sprintf("proxmox-%s-registration-blocked", ptype))
}

// detectedTypesFilePath names the marker RunAll writes listing the canonical
// Proxmox products it is registering on this host, one per line.
//
// Without it the installer cannot tell a PVE-only host from a combined PVE+PBS
// host, so it has to report the first outcome it sees and call the install
// done. On a combined host — an officially supported deployment that now
// consumes one bootstrap grant per type — that produced a partial verdict:
// either a success banner before PBS had landed, or an error banner for PBS
// that buried a perfectly good PVE registration (#1644).
func (p *ProxmoxSetup) detectedTypesFilePath() string {
	return filepath.Join(p.stateDir, "proxmox-detected-types")
}

// writeDetectedTypesMarker publishes the types RunAll is about to register so
// the installer waits for an outcome from each of them, and only from them.
func (p *ProxmoxSetup) writeDetectedTypesMarker(types []proxmoxProductType) {
	if len(types) == 0 {
		return
	}
	if err := p.collector.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
		p.logger.Debug().Err(err).Str("dir", p.stateDir).Msg("Failed to create state dir for detected-types marker")
		return
	}
	contents := strings.Join(proxmoxProductTypesToStrings(types), "\n") + "\n"
	path := p.detectedTypesFilePath()
	if err := p.collector.WriteFile(path, []byte(contents), proxmoxStateFilePerm); err != nil {
		p.logger.Debug().Err(err).Msg("Failed to write detected-types marker")
		return
	}
	if err := p.collector.Chmod(path, proxmoxStateFilePerm); err != nil {
		p.logger.Debug().Err(err).Msg("Failed to enforce detected-types marker permissions")
	}
}

func (p *ProxmoxSetup) writeRegistrationBlockedMarker(ptype proxmoxProductType, reason string) {
	if err := os.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
		p.logger.Debug().Err(err).Str("dir", p.stateDir).Msg("Failed to create state dir for registration-blocked marker")
		return
	}
	if err := os.WriteFile(p.registrationBlockedFilePath(ptype), []byte(reason+"\n"), proxmoxStateFilePerm); err != nil {
		p.logger.Debug().Err(err).Str("type", string(ptype)).Msg("Failed to write registration-blocked marker")
	}
}

func (p *ProxmoxSetup) clearRegistrationBlockedMarker(ptype proxmoxProductType) {
	if err := os.Remove(p.registrationBlockedFilePath(ptype)); err != nil && !os.IsNotExist(err) {
		p.logger.Debug().Err(err).Str("type", string(ptype)).Msg("Failed to remove registration-blocked marker")
	}
}

func parseProxmoxProductType(rawType string) proxmoxProductType {
	switch strings.ToLower(strings.TrimSpace(rawType)) {
	case string(proxmoxProductPVE):
		return proxmoxProductPVE
	case string(proxmoxProductPBS):
		return proxmoxProductPBS
	default:
		return proxmoxProductUnknown
	}
}

func proxmoxProductTypesToStrings(types []proxmoxProductType) []string {
	if len(types) == 0 {
		return nil
	}

	result := make([]string, 0, len(types))
	for _, t := range types {
		result = append(result, string(t))
	}

	return result
}

// NewProxmoxSetup creates a new ProxmoxSetup instance.
func NewProxmoxSetup(logger zerolog.Logger, httpClient *http.Client, collector SystemCollector, pulseURL, apiToken, proxmoxType, hostname, reportIP, stateDir string, insecure bool) *ProxmoxSetup {
	if stateDir == "" {
		stateDir = defaultStateDir
	}
	return &ProxmoxSetup{
		logger:             logger,
		httpClient:         httpClient,
		collector:          collector,
		pulseURL:           strings.TrimRight(pulseURL, "/"),
		apiToken:           apiToken,
		proxmoxType:        proxmoxProductType(proxmoxType),
		hostname:           hostname,
		reportIP:           reportIP,
		stateDir:           stateDir,
		insecureSkipVerify: insecure,
		authoritative:      true,
	}
}

// WithObserverPlaintextPolicy marks this setup as an observer transport. Unlike
// the primary, every non-loopback plaintext observer requires its own explicit
// opt-in and cannot inherit process-wide primary transport consent.
func (p *ProxmoxSetup) WithObserverPlaintextPolicy(allowPlaintext bool) *ProxmoxSetup {
	p.authoritative = false
	p.allowPlaintextHTTP = allowPlaintext
	return p
}

// WithMonitorTokenName isolates Proxmox credentials for an additional Pulse
// destination. The primary destination deliberately keeps the legacy token
// name for upgrade compatibility.
func (p *ProxmoxSetup) WithMonitorTokenName(tokenName string) *ProxmoxSetup {
	p.monitorToken = strings.TrimSpace(tokenName)
	return p
}

// Run executes the Proxmox setup process:
// 1. Detects Proxmox type (if not specified)
// 2. Creates the monitoring user and API token
// 3. Registers the node with Pulse via auto-register
func (p *ProxmoxSetup) Run(ctx context.Context) (*ProxmoxSetupResult, error) {
	pulseURL, err := agenttarget.NormalizePulseURL(p.pulseURL, p.authoritative, p.allowPlaintextHTTP)
	if err != nil {
		return nil, fmt.Errorf("invalid pulse URL: %w", err)
	}
	p.pulseURL = pulseURL

	ptypeStr, err := normalizeProxmoxType(string(p.proxmoxType))
	if err != nil {
		return nil, err
	}
	ptype := proxmoxProductType(ptypeStr)

	// Detect Proxmox type
	if ptype == proxmoxProductUnknown {
		detected := p.detectProxmoxType()
		if detected == proxmoxProductUnknown {
			return nil, fmt.Errorf("this system does not appear to be a Proxmox VE or PBS node")
		}
		ptype = detected
		p.logger.Info().Str("type", string(ptype)).Msg("Auto-detected Proxmox type")
	}

	result, err := p.runForType(ctx, ptype)
	if result != nil && result.Registered {
		p.markAsRegistered()
	}
	return result, err
}

// RunAll detects and registers ALL Proxmox products on this system.
// This supports hosts with both PVE and PBS installed (a common and officially
// supported configuration). Each type gets its own registration and state tracking,
// and the server grants one bootstrap create per canonical type, so a combined
// host registers both from a single install token (#1644).
// Returns results for all types that were processed (skipping already-registered ones).
//
// One type failing does not abandon the others: the remaining types are still
// attempted and their results returned. An error comes back only when every
// detected type failed, so a fully failed Proxmox setup is still loud while a
// partial one leaves the successful registrations reportable.
func (p *ProxmoxSetup) RunAll(ctx context.Context) ([]*ProxmoxSetupResult, error) {
	pulseURL, err := agenttarget.NormalizePulseURL(p.pulseURL, p.authoritative, p.allowPlaintextHTTP)
	if err != nil {
		return nil, fmt.Errorf("invalid pulse URL: %w", err)
	}
	p.pulseURL = pulseURL

	forcedTypeStr, err := normalizeProxmoxType(string(p.proxmoxType))
	if err != nil {
		return nil, err
	}
	forcedType := proxmoxProductType(forcedTypeStr)

	var results []*ProxmoxSetupResult

	// If a specific type is forced, only run that
	if forcedType != "" {
		p.writeDetectedTypesMarker([]proxmoxProductType{forcedType})
		result, err := p.runForType(ctx, forcedType)
		if err != nil {
			return nil, fmt.Errorf("setup proxmox type %s: %w", p.proxmoxType, err)
		}
		if result != nil {
			results = append(results, result)
		}
		return results, nil
	}

	// Detect all Proxmox products
	types := p.detectProxmoxTypes()
	if len(types) == 0 {
		return nil, fmt.Errorf("this system does not appear to be a Proxmox VE or PBS node")
	}

	p.logger.Info().Strs("types", proxmoxProductTypesToStrings(types)).Msg("Auto-detected Proxmox products")
	p.writeDetectedTypesMarker(types)

	// Register each type
	var failures []error
	var failedTypes []string
	for _, ptype := range types {
		result, err := p.runForType(ctx, ptype)
		if err != nil {
			p.logger.Error().Err(err).Str("type", string(ptype)).Msg("Failed to setup Proxmox type")
			failures = append(failures, fmt.Errorf("setup proxmox type %s: %w", ptype, err))
			failedTypes = append(failedTypes, string(ptype))
			continue // Don't fail completely, try other types
		}
		if result != nil {
			results = append(results, result)
		}
	}

	if len(failures) == len(types) {
		return results, errors.Join(failures...)
	}
	if len(failures) > 0 {
		p.logger.Warn().
			Strs("failed_types", failedTypes).
			Strs("detected_types", proxmoxProductTypesToStrings(types)).
			Msg("Some Proxmox products failed to set up; the remaining products were registered")
	}

	return results, nil
}

// RunHealthCheck verifies that every previously-registered Proxmox type is
// still connected to Pulse. If a type's registration is confirmed stale by the
// server (node exists in config but connection is broken), RunHealthCheck
// re-registers it with a fresh token.
//
// Types with no local registration marker are intentionally skipped. Those
// need a full RunAll cycle (typically at next agent startup) rather than a
// health-check rotation, which would cause uncontrolled token churn if Pulse
// is temporarily unreachable.
func (p *ProxmoxSetup) RunHealthCheck(ctx context.Context) ([]*ProxmoxSetupResult, error) {
	pulseURL, err := agenttarget.NormalizePulseURL(p.pulseURL, p.authoritative, p.allowPlaintextHTTP)
	if err != nil {
		return nil, fmt.Errorf("invalid pulse URL: %w", err)
	}
	p.pulseURL = pulseURL

	forcedTypeStr, err := normalizeProxmoxType(string(p.proxmoxType))
	if err != nil {
		return nil, err
	}
	forcedType := proxmoxProductType(forcedTypeStr)

	var typesToCheck []proxmoxProductType
	if forcedType != "" {
		typesToCheck = []proxmoxProductType{forcedType}
	} else {
		typesToCheck = p.detectProxmoxTypes()
	}

	var results []*ProxmoxSetupResult
	for _, ptype := range typesToCheck {
		// Skip types with no local marker — they need a full startup setup
		// cycle, not a periodic health-check rotation.
		if !p.isTypeRegistered(ptype) {
			continue
		}

		hostURL := p.getHostURL(ctx, ptype)
		registration, err := p.checkRegistrationWithPulse(ctx, ptype, hostURL)
		if err != nil {
			p.logger.Warn().Err(err).Str("type", string(ptype)).
				Msg("Proxmox health check: could not verify registration with Pulse")
			continue
		}
		if registration.registered {
			continue // still healthy
		}
		if !registration.canRegister {
			p.logger.Warn().
				Str("type", string(ptype)).
				Str("host", hostURL).
				Bool("source_exists", registration.sourceExists).
				Msg("Proxmox health check: Pulse has no matching source and this agent token cannot create one; re-run the Proxmox agent install command")
			continue
		}

		p.logger.Info().Str("type", string(ptype)).Str("host", hostURL).
			Msg("Proxmox health check: node disconnected, re-registering")
		result, err := p.runForType(ctx, ptype)
		if err != nil {
			p.logger.Error().Err(err).Str("type", string(ptype)).
				Msg("Proxmox health check: re-registration failed")
			continue
		}
		if result != nil {
			results = append(results, result)
		}
	}
	return results, nil
}

// runForType executes setup for a specific Proxmox type.
func (p *ProxmoxSetup) runForType(ctx context.Context, ptype proxmoxProductType) (*ProxmoxSetupResult, error) {
	normalizedTypeStr, err := normalizeProxmoxType(string(ptype))
	if err != nil || normalizedTypeStr == "" {
		return nil, fmt.Errorf("invalid proxmox type %q: must be pve or pbs", ptype)
	}
	ptype = proxmoxProductType(normalizedTypeStr)
	hostURL := p.getHostURL(ctx, ptype)

	// Always establish destination reachability and registration state before
	// mutating a Proxmox token. This prevents a missing local marker or a Pulse
	// startup outage from rotating credentials that a healthy Pulse instance is
	// still using.
	registration, err := p.checkRegistrationWithPulse(ctx, ptype, hostURL)
	if err != nil {
		p.logger.Warn().
			Err(err).
			Str("type", string(ptype)).
			Msg("Failed to verify Proxmox registration state with Pulse; leaving credentials unchanged")
		return nil, nil
	}
	if registration.registered {
		if !p.isTypeRegistered(ptype) {
			p.markTypeAsRegistered(ptype)
		}
		p.clearRegistrationBlockedMarker(ptype)
		p.logger.Info().Str("type", string(ptype)).Msg("Proxmox type already registered, skipping")
		return nil, nil
	}
	if !registration.canRegister {
		reason := fmt.Sprintf(
			"Pulse at %s has no %s source for %s and this agent's token cannot create one. Local Proxmox credentials were left unchanged. Generate a fresh agent install command in Pulse Settings -> Infrastructure and re-run it on this host.",
			p.pulseURL, strings.ToUpper(string(ptype)), hostURL,
		)
		p.writeRegistrationBlockedMarker(ptype, reason)
		p.logger.Error().
			Str("type", string(ptype)).
			Str("host", hostURL).
			Bool("source_exists", registration.sourceExists).
			Msg("Proxmox registration blocked: Pulse has no matching source and this agent token cannot create one")
		return nil, fmt.Errorf("proxmox %s registration blocked: %s", ptype, reason)
	}
	p.clearRegistrationBlockedMarker(ptype)
	if p.isTypeRegistered(ptype) {
		p.logger.Info().
			Str("type", string(ptype)).
			Str("host", hostURL).
			Msg("Local Proxmox registration marker exists but Pulse has no matching node; re-registering")
	}

	p.logger.Info().Str("type", string(ptype)).Msg("Setting up Proxmox type")

	// Clear the registration marker before token operations so that a failed
	// Pulse update (rotation succeeds but registration fails) leaves the system
	// retriable on the next startup rather than stuck on a stale state file.
	p.clearTypeRegisteredMarker(ptype)

	// Create monitoring user and token
	tokenID, tokenValue, err := p.setupToken(ctx, ptype)
	if err != nil {
		return nil, fmt.Errorf("failed to create %s API token: %w", ptype, err)
	}

	p.logger.Info().Str("type", string(ptype)).Str("token_id", tokenID).Msg("Created Proxmox API token")

	// Register with Pulse
	registered := false
	registerResp, err := p.registerWithPulse(ctx, ptype, hostURL, tokenID, tokenValue)
	if err != nil {
		p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to register with Pulse (node may already exist)")
	} else {
		registered = true
		p.markTypeAsRegistered(ptype)
		if strings.TrimSpace(registerResp.Host) != "" {
			hostURL = registerResp.Host
		}
		p.logger.Info().Str("type", string(ptype)).Str("host", hostURL).Str("node", registerResp.NodeName).Msg("Successfully registered Proxmox node with Pulse")
	}

	return &ProxmoxSetupResult{
		ProxmoxType: string(ptype),
		TokenID:     tokenID,
		TokenValue:  tokenValue,
		NodeHost:    hostURL,
		NodeName:    registerResp.NodeName,
		Registered:  registered,
	}, nil
}

func (p *ProxmoxSetup) orderedCandidateHosts(ctx context.Context, ptype proxmoxProductType, hostURL string) []string {
	candidateHosts := p.candidateHostURLs(ctx, ptype)
	if len(candidateHosts) == 0 || strings.TrimSpace(candidateHosts[0]) != strings.TrimSpace(hostURL) {
		seen := make(map[string]struct{}, len(candidateHosts)+1)
		reordered := make([]string, 0, len(candidateHosts)+1)
		reordered = appendUniqueHostCandidate(reordered, seen, hostURL)
		for _, candidate := range candidateHosts {
			reordered = appendUniqueHostCandidate(reordered, seen, candidate)
		}
		candidateHosts = reordered
	}
	return candidateHosts
}

func (p *ProxmoxSetup) checkRegistrationWithPulse(ctx context.Context, ptype proxmoxProductType, hostURL string) (registrationState, error) {
	// Runtime agent tokens authenticate directly against the canonical
	// auto-register endpoint. The setup-script-url endpoint is an operator
	// control-plane surface protected by settings:write and must never be used
	// as a runtime token-discovery probe.
	setupToken := ""
	if strings.TrimSpace(p.apiToken) == "" {
		var err error
		setupToken, err = p.fetchSetupToken(ctx, ptype, hostURL)
		if err != nil {
			return registrationState{}, fmt.Errorf("fetch setup token: %w", err)
		}
	}

	payload := autoRegisterRequest{
		Type:              ptype,
		Host:              hostURL,
		CandidateHosts:    p.orderedCandidateHosts(ctx, ptype, hostURL),
		ServerName:        p.hostname,
		AuthToken:         setupToken,
		Source:            autoRegisterSourceAgent,
		CheckRegistration: true,
	}

	body, err := json.Marshal(payload)
	if err != nil {
		return registrationState{}, fmt.Errorf("marshal registration-check payload: %w", err)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.pulseURL+"/api/auto-register", bytes.NewReader(body))
	if err != nil {
		return registrationState{}, fmt.Errorf("create registration-check request: %w", err)
	}
	req.Header.Set("Content-Type", "application/json")
	if token := autoRegisterAPITokenHeader(p.apiToken, setupToken); token != "" {
		req.Header.Set("X-API-Token", token)
	}

	resp, err := p.httpClient.Do(req)
	if err != nil {
		return registrationState{}, fmt.Errorf("registration-check request failed: %w", err)
	}
	defer func() {
		if closeErr := resp.Body.Close(); closeErr != nil {
			p.logger.Debug().Err(closeErr).Msg("Failed to close registration-check response body")
		}
	}()

	bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxRegistrationCheckResponseBytes))
	if err != nil {
		return registrationState{}, fmt.Errorf("read registration-check response: %w", err)
	}
	if resp.StatusCode >= http.StatusBadRequest {
		return registrationState{}, &clientError{
			statusCode: resp.StatusCode,
			body:       strings.TrimSpace(string(bodyBytes)),
		}
	}

	var parsed autoRegisterCheckResponse
	if err := json.Unmarshal(bodyBytes, &parsed); err != nil {
		return registrationState{}, fmt.Errorf("decode registration-check response: %w", err)
	}
	// Older Pulse servers returned only registered. Preserve their repair
	// behavior during rolling updates by treating omitted capability fields as
	// permissive; current servers always send both fields explicitly.
	sourceExists := parsed.Registered
	if parsed.SourceExists != nil {
		sourceExists = *parsed.SourceExists
	}
	canRegister := true
	if parsed.CanRegister != nil {
		canRegister = *parsed.CanRegister
	}
	return registrationState{
		registered:   parsed.Registered,
		sourceExists: sourceExists,
		canRegister:  canRegister,
	}, nil
}

// detectProxmoxType checks for pvesh (PVE) or proxmox-backup-manager (PBS).
// For backward compatibility, returns the first detected type.
// Use detectProxmoxTypes() to get all detected types.
func (p *ProxmoxSetup) detectProxmoxType() proxmoxProductType {
	types := p.detectProxmoxTypes()
	if len(types) > 0 {
		return types[0]
	}
	return proxmoxProductUnknown
}

// detectProxmoxTypes checks for ALL Proxmox products on this system.
// Returns a slice of detected types (e.g., ["pve", "pbs"] if both are installed).
// This is common when PBS is installed directly on a PVE host.
// detectProxmoxTypes checks for ALL Proxmox products on this system.
// Returns a slice of detected types (e.g., ["pve", "pbs"] if both are installed).
// This is common when PBS is installed directly on a PVE host.
func (p *ProxmoxSetup) detectProxmoxTypes() []proxmoxProductType {
	var types []proxmoxProductType

	// Check for PVE
	if _, err := p.collector.LookPath("pvesh"); err == nil {
		types = append(types, proxmoxProductPVE)
	}

	// Check for PBS
	if _, err := p.collector.LookPath("proxmox-backup-manager"); err == nil {
		types = append(types, proxmoxProductPBS)
	}

	return types
}

// setupToken creates the monitoring user and API token.
func (p *ProxmoxSetup) setupToken(ctx context.Context, ptype proxmoxProductType) (string, string, error) {
	tokenName := p.monitorTokenName()

	switch ptype {
	case proxmoxProductPVE:
		return p.setupPVEToken(ctx, tokenName)
	case proxmoxProductPBS:
		return p.setupPBSToken(ctx, tokenName)
	default:
		return "", "", fmt.Errorf("unsupported Proxmox type %q", ptype)
	}
}

// setupPVEToken creates a PVE monitoring user and token.
func (p *ProxmoxSetup) setupPVEToken(ctx context.Context, tokenName string) (string, string, error) {
	// Create user (ignore error if already exists)
	if out, err := p.collector.CommandCombinedOutput(ctx, "pveum", "user", "add", proxmoxUserPVE, "--comment", proxmoxComment); err != nil {
		if isAlreadyExistsOutput(out) {
			p.logger.Debug().Msg("Proxmox PVE monitor user already exists")
		} else {
			p.logger.Warn().Err(err).Str("output", strings.TrimSpace(out)).Msg("Failed to create Proxmox PVE monitor user")
		}
	}

	// Create token with privilege separation enabled, then mirror ACLs onto the token.
	output, err := p.collector.CommandCombinedOutput(ctx, "pveum", "user", "token", "add", proxmoxUserPVE, tokenName, "--privsep", "1")
	if err != nil {
		if tokenAlreadyExists(err, output) {
			p.logger.Info().Str("token", tokenName).Msg("PVE monitoring token already exists; rotating token in place")
			if _, deleteErr := p.collector.CommandCombinedOutput(ctx, "pveum", "user", "token", "remove", proxmoxUserPVE, tokenName); deleteErr != nil {
				return "", "", fmt.Errorf("failed to delete existing token before rotation: %w", deleteErr)
			}
			output, err = p.collector.CommandCombinedOutput(ctx, "pveum", "user", "token", "add", proxmoxUserPVE, tokenName, "--privsep", "1")
		}
		if err != nil {
			return "", "", fmt.Errorf("failed to create token: %w", err)
		}
	}

	// Parse token value from output
	tokenValue := p.parseTokenValue(output)
	if tokenValue == "" {
		return "", "", fmt.Errorf("failed to parse token value from pveum output")
	}

	tokenID := fmt.Sprintf("%s!%s", proxmoxUserPVE, tokenName)
	p.configurePVEPermissions(ctx, tokenID)

	return tokenID, tokenValue, nil
}

// setupPBSToken creates a PBS monitoring user and token.
func (p *ProxmoxSetup) setupPBSToken(ctx context.Context, tokenName string) (string, string, error) {
	// Create user (ignore error if already exists)
	if out, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "create", proxmoxUserPBS); err != nil {
		if isAlreadyExistsOutput(out) {
			p.logger.Debug().Msg("Proxmox PBS monitor user already exists")
		} else {
			p.logger.Warn().Err(err).Str("output", strings.TrimSpace(out)).Msg("Failed to create Proxmox PBS monitor user")
		}
	}

	// Add Audit role
	if out, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "acl", "update", "/", "Audit", "--auth-id", proxmoxUserPBS); err != nil {
		p.logger.Warn().Err(err).Str("output", strings.TrimSpace(out)).Msg("Failed to apply PBS Audit role to monitor user")
	}

	// Create token
	output, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "generate-token", proxmoxUserPBS, tokenName)
	if err != nil {
		if tokenAlreadyExists(err, output) {
			p.logger.Info().Str("token", tokenName).Msg("PBS monitoring token already exists; rotating token in place")
			if _, deleteErr := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "delete-token", proxmoxUserPBS, tokenName); deleteErr != nil {
				return "", "", fmt.Errorf("failed to delete existing token before rotation: %w", deleteErr)
			}
			output, err = p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "generate-token", proxmoxUserPBS, tokenName)
		}
		if err != nil {
			return "", "", fmt.Errorf("failed to create token: %w", err)
		}
	}

	// Parse token value from JSON output
	tokenValue := p.parsePBSTokenValue(output)
	if tokenValue == "" {
		return "", "", fmt.Errorf("failed to parse token value from PBS output")
	}

	// Add Audit role for the token itself
	tokenID := fmt.Sprintf("%s!%s", proxmoxUserPBS, tokenName)
	if out, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "acl", "update", "/", "Audit", "--auth-id", tokenID); err != nil {
		p.logger.Warn().Err(err).Str("token_id", tokenID).Str("output", strings.TrimSpace(out)).Msg("Failed to apply PBS Audit role to token")
	}

	return tokenID, tokenValue, nil
}

// parseTokenValue extracts the token value from PVE pveum output.
// The output format is typically a table with columns: │ key │ value │
// Example output:
// ┌──────────────┬──────────────────────────────────────┐
// │ key          │ value                                │
// ╞══════════════╪══════════════════════════════════════╡
// │ full-tokenid │ pulse-monitor@pve!pulse-token        │
// ├──────────────┼──────────────────────────────────────┤
// │ info         │ {"privsep":"1"}                      │
// ├──────────────┼──────────────────────────────────────┤
// │ value        │ xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx │
// └──────────────┴──────────────────────────────────────┘
func (p *ProxmoxSetup) parseTokenValue(output string) string {
	// Look for the "value" row in the table output - where "value" is the KEY column
	// We need to find the row where parts[1] (key column) is "value", not any row
	// containing the word "value" (which would match the header row)
	lines := strings.Split(output, "\n")
	for _, line := range lines {
		// Split by the box drawing character │ (U+2502)
		parts := strings.Split(line, "│")
		if len(parts) >= 3 {
			key := strings.TrimSpace(parts[1])
			if key == "value" {
				return strings.TrimSpace(parts[2])
			}
		}
	}

	// Fallback: try regex for UUID-like token
	re := regexp.MustCompile(`[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`)
	if match := re.FindString(output); match != "" {
		return match
	}

	return ""
}

// parsePBSTokenValue extracts the token value from PBS JSON output.
func (p *ProxmoxSetup) parsePBSTokenValue(output string) string {
	// PBS outputs JSON like: {"tokenid": "...", "value": "..."}
	var result struct {
		Value string `json:"value"`
	}
	if err := json.Unmarshal([]byte(output), &result); err == nil && result.Value != "" {
		return result.Value
	}

	// Fallback: extract from "value": "..." pattern
	re := regexp.MustCompile(`"value"\s*:\s*"([^"]+)"`)
	if matches := re.FindStringSubmatch(output); len(matches) >= 2 {
		return matches[1]
	}

	return ""
}

func appendUniqueHostCandidate(candidates []string, seen map[string]struct{}, candidate string) []string {
	normalized := strings.TrimSpace(candidate)
	if normalized == "" {
		return candidates
	}
	if _, exists := seen[normalized]; exists {
		return candidates
	}
	seen[normalized] = struct{}{}
	return append(candidates, normalized)
}

func (p *ProxmoxSetup) candidateHostURLs(ctx context.Context, ptype proxmoxProductType) []string {
	port := "8006"
	if ptype == proxmoxProductPBS {
		port = "8007"
	}

	if ctx == nil {
		ctx = context.Background()
	}

	candidates := make([]string, 0, 4)
	seen := make(map[string]struct{}, 4)
	buildURL := func(host string) string {
		if strings.TrimSpace(host) == "" {
			return ""
		}
		return formatHTTPSURL(host, port)
	}

	// Priority 1: User-specified ReportIP override from configuration.
	// This allows users to manually specify which IP should be used for Proxmox API
	// connections when auto-detection picks the wrong one (e.g., Issue #1061).
	if p.reportIP != "" {
		p.logger.Info().Str("ip", p.reportIP).Msg("Using user-specified ReportIP for Proxmox registration")
		return append(candidates, buildURL(p.reportIP))
	}

	if p.collector == nil {
		hostname := strings.TrimSpace(p.hostname)
		if hostname == "" {
			hostname = "localhost"
		}
		return append(candidates, buildURL(hostname))
	}

	// Priority 2: Prefer the route-aware local IP used to reach Pulse. This keeps
	// generated Proxmox config aligned with the reachable endpoint shown to users,
	// while preserving hostname as a fallback candidate below.
	if reachableIP := p.getIPThatReachesPulse(); reachableIP != "" {
		p.logger.Debug().Str("ip", reachableIP).Msg("Using IP that can reach Pulse server")
		candidates = appendUniqueHostCandidate(candidates, seen, buildURL(reachableIP))
	}

	// Priority 3: Keep the system hostname as a fallback when it resolves to a
	// non-loopback, non-link-local address. The server receives candidateHosts
	// and may still choose it if it is the correct stable endpoint.
	if hostname := strings.TrimSpace(p.hostname); hostname != "" {
		if hostnameIP := p.getIPForHostname(); hostnameIP != "" && !isLoopbackOrLinkLocalIP(hostnameIP) {
			p.logger.Debug().
				Str("hostname", hostname).
				Str("ip", hostnameIP).
				Msg("Adding resolvable hostname fallback for Proxmox registration")
			candidates = appendUniqueHostCandidate(candidates, seen, buildURL(hostname))
		}
	}

	// Priority 4: Get all IPs and select the best one based on heuristics.
	hostnameCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
	defer cancel()

	if out, err := p.collector.CommandCombinedOutput(hostnameCtx, "hostname", "-I"); err == nil {
		ips := strings.Fields(out)
		if len(ips) > 0 {
			// Get the IP that the system hostname currently resolves to
			hostnameIP := p.getIPForHostname()

			bestIP := selectBestIP(ips, hostnameIP)
			if bestIP != "" {
				candidates = appendUniqueHostCandidate(candidates, seen, buildURL(bestIP))
			}
		}
	}

	if len(candidates) > 0 {
		return candidates
	}

	// Final fallback to hostname if IP detection failed
	hostname := p.hostname
	if hostname == "" {
		hostname = "localhost"
	}
	return append(candidates, buildURL(hostname))
}

// getHostURL constructs the host URL for this Proxmox node.
// Prefers explicit reportIP, then route-aware IP detection, then hostname and
// heuristic local IP fallbacks.
func (p *ProxmoxSetup) getHostURL(ctx context.Context, ptype proxmoxProductType) string {
	candidates := p.candidateHostURLs(ctx, ptype)
	if len(candidates) == 0 {
		return ""
	}
	return candidates[0]
}

// getIPThatReachesPulse determines which local IP is used to connect to the Pulse server.
// This handles cases where multiple network interfaces exist (e.g., management, Ceph, cluster ring)
// and ensures we pick the one that can actually reach Pulse. Related to #929.
func (p *ProxmoxSetup) getIPThatReachesPulse() string {
	if p.collector == nil {
		return ""
	}
	if p.pulseURL == "" {
		return ""
	}

	// Parse the Pulse URL to get host:port
	pulseServerURL, err := url.Parse(p.pulseURL)
	if err != nil {
		return ""
	}

	host := pulseServerURL.Hostname()
	if host == "" {
		return ""
	}

	port := pulseServerURL.Port()
	if port == "" {
		switch pulseServerURL.Scheme {
		case "https":
			port = "443"
		case "http":
			port = "80"
		default:
			port = "7655"
		}
	}
	target := net.JoinHostPort(host, port)

	// Create a UDP "connection" to determine local address (doesn't actually send data)
	// We use a short timeout as this is just a routing table lookup.
	conn, err := p.collector.DialTimeout("udp", target, 500*time.Millisecond)
	if err != nil {
		p.logger.Debug().Err(err).Str("target", target).Msg("Could not determine local IP for Pulse connection (routing check failed)")
		return ""
	}
	defer func() {
		if closeErr := conn.Close(); closeErr != nil {
			p.logger.Debug().Err(closeErr).Msg("Failed to close local route probe connection")
		}
	}()

	localAddr, ok := conn.LocalAddr().(*net.UDPAddr)
	if !ok || localAddr == nil {
		return ""
	}
	ip := localAddr.IP.String()

	// If we found an IP that can reach Pulse, it's the most reliable IP fallback
	// after a canonical hostname probe. Reject loopback and link-local addresses.
	if ip != "" && !isLoopbackOrLinkLocalIP(ip) {
		return ip
	}
	return ""
}

func formatHTTPSURL(host, port string) string {
	return (&url.URL{
		Scheme: "https",
		Host:   net.JoinHostPort(host, port),
	}).String()
}

// getIPForHostname resolves the system hostname to an IP address.
func (p *ProxmoxSetup) getIPForHostname() string {
	if p.collector == nil {
		return ""
	}
	hostname := p.hostname
	if hostname == "" {
		var err error
		hostname, err = p.collector.Hostname()
		if err != nil {
			p.logger.Debug().Err(err).Msg("Could not resolve system hostname")
			return ""
		}
	}
	if hostname == "" {
		return ""
	}

	ips, err := p.collector.LookupIP(hostname)
	if err != nil {
		p.logger.Debug().Err(err).Str("hostname", hostname).Msg("Could not resolve hostname IP")
		return ""
	}

	for _, ip := range ips {
		if ipv4 := ip.To4(); ipv4 != nil {
			return ipv4.String()
		}
	}
	return ""
}

func isLoopbackOrLinkLocalIP(raw string) bool {
	ip := net.ParseIP(strings.TrimSpace(raw))
	if ip == nil {
		return false
	}
	return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast()
}

// selectBestIP picks the most likely externally-reachable IP from a list.
// Prefers common LAN subnets (192.168.x.x, 10.x.x.x) and avoids internal
// cluster networks (like corosync's 172.20.x.x) and link-local addresses.
// If hostnameIP is provided and present in the list, it is given the highest priority.
func selectBestIP(ips []string, hostnameIP string) string {
	type scoredIP struct {
		ip    string
		score int
	}

	var candidates []scoredIP

	for _, ip := range ips {
		// Skip loopback and IPv6 link-local
		if ip == "127.0.0.1" || ip == "::1" || strings.HasPrefix(ip, "fe80:") {
			continue
		}

		// Skip IPv6 for simplicity (most Proxmox setups use IPv4)
		if strings.Contains(ip, ":") {
			continue
		}

		score := scoreIPv4(ip)

		// If this IP matches the system hostname, give it a significant bonus.
		// A bonus of +40 ensures that a 172.16.x.x (score 50) match (50+40=90)
		// correctly loses to a 192.168.x.x (score 100) interface if both are present.
		// However, it correctly breaks ties between equal-scored ranges.
		if hostnameIP != "" && ip == hostnameIP {
			score += 40
		}

		if score > 0 {
			candidates = append(candidates, scoredIP{ip: ip, score: score})
		}
	}

	if len(candidates) == 0 {
		// Fallback: return first non-loopback IP if no good candidates
		for _, ip := range ips {
			if ip != "127.0.0.1" && ip != "::1" && !strings.HasPrefix(ip, "fe80:") {
				return ip
			}
		}
		return ""
	}

	// Return highest scored IP
	best := candidates[0]
	for _, c := range candidates[1:] {
		if c.score > best.score {
			best = c
		}
	}
	return best.ip
}

// scoreIPv4 assigns a preference score to an IPv4 address.
// Higher score = more likely to be externally reachable.
func scoreIPv4(ip string) int {
	parts := strings.Split(ip, ".")
	if len(parts) != 4 {
		return 0
	}

	// Parse first two octets
	first := 0
	second := 0
	if _, err := fmt.Sscanf(parts[0], "%d", &first); err != nil {
		return 0
	}
	if _, err := fmt.Sscanf(parts[1], "%d", &second); err != nil {
		return 0
	}

	// Scoring logic:
	// - 192.168.x.x: Very common home/office LAN, highest priority (score 100)
	// - 10.0.x.x - 10.31.x.x: Common corporate LAN ranges (score 90)
	// - 100.64.x.x: Tailscale / CGNAT (score 85)
	// - 10.x.x.x (other): Less common, but still likely LAN (score 70)
	// - 172.16-31.x.x: Private range, often used for internal clusters (score 50)
	//   Corosync often uses 172.20.x.x or similar for ring0/ring1
	// - 169.254.x.x: Link-local, skip (score 0)
	// - Other private/public: Unknown, low priority (score 30)

	switch {
	case first == 192 && second == 168:
		return 100 // Most common home/office LAN
	case first == 10 && second <= 31:
		return 90 // Common corporate LAN
	case first == 100 && (second >= 64 && second <= 127):
		return 85 // Tailscale / CGNAT
	case first == 10:
		return 70 // Other 10.x.x.x
	case first == 172 && second >= 16 && second <= 31:
		// This is private 172.16-31.x.x range, often used for internal clusters
		// Corosync commonly uses 172.20.x.x for cluster communication
		return 50 // Lower priority - often internal cluster
	case first == 169 && second == 254:
		return 0 // Link-local, skip
	default:
		return 30 // Unknown/public - low priority
	}
}

// clientError represents a non-retryable HTTP client error (4xx).
type clientError struct {
	statusCode int
	body       string
}

func (e *clientError) Error() string {
	return fmt.Sprintf("auto-register returned HTTP %d: %s", e.statusCode, e.body)
}

// permanentError wraps an error that should not be retried (e.g. malformed URL).
type permanentError struct {
	err error
}

func (e *permanentError) Error() string { return e.err.Error() }
func (e *permanentError) Unwrap() error { return e.err }

func autoRegisterResponseMatchesCandidateHost(responseHost string, candidateHosts []string) bool {
	trimmedResponseHost := strings.TrimSpace(responseHost)
	if trimmedResponseHost == "" {
		return false
	}
	for _, candidate := range candidateHosts {
		if trimmedResponseHost == strings.TrimSpace(candidate) {
			return true
		}
	}
	return false
}

func autoRegisterAPITokenHeader(apiToken, setupToken string) string {
	if strings.TrimSpace(setupToken) != "" {
		return ""
	}
	return strings.TrimSpace(apiToken)
}

func (p *ProxmoxSetup) doRegisterRequest(ctx context.Context, body []byte, setupToken string, expectedType proxmoxProductType, expectedSource autoRegisterSource, expectedHosts []string, expectedTokenID, expectedTokenValue string) (autoRegisterResponse, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.pulseURL+"/api/auto-register", bytes.NewReader(body))
	if err != nil {
		return autoRegisterResponse{}, &permanentError{fmt.Errorf("create request: %w", err)}
	}
	req.Header.Set("Content-Type", "application/json")
	if token := autoRegisterAPITokenHeader(p.apiToken, setupToken); token != "" {
		req.Header.Set("X-API-Token", token)
	}

	resp, err := p.httpClient.Do(req)
	if err != nil {
		return autoRegisterResponse{}, fmt.Errorf("request failed: %w", err)
	}
	defer func() {
		if closeErr := resp.Body.Close(); closeErr != nil {
			p.logger.Debug().Err(closeErr).Msg("Failed to close auto-register response body")
		}
	}()

	if resp.StatusCode >= 400 {
		bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		bodyStr := strings.TrimSpace(string(bodyBytes))
		if resp.StatusCode < 500 {
			return autoRegisterResponse{}, &clientError{statusCode: resp.StatusCode, body: bodyStr}
		}
		return autoRegisterResponse{}, fmt.Errorf("auto-register returned HTTP %d: %s", resp.StatusCode, bodyStr)
	}

	var parsed autoRegisterResponse
	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
		return autoRegisterResponse{}, fmt.Errorf("decode auto-register response: %w", err)
	}
	if strings.TrimSpace(parsed.Status) != "success" {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response missing success status")
	}
	if strings.TrimSpace(parsed.Action) != "use_token" {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response missing use_token action")
	}
	if strings.TrimSpace(parsed.Type) != strings.TrimSpace(string(expectedType)) {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response type mismatch")
	}
	if strings.TrimSpace(parsed.Source) != strings.TrimSpace(string(expectedSource)) {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response source mismatch")
	}
	if !autoRegisterResponseMatchesCandidateHost(parsed.Host, expectedHosts) {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response host mismatch")
	}
	if strings.TrimSpace(parsed.NodeID) == "" {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response missing nodeId")
	}
	if strings.TrimSpace(parsed.NodeName) == "" {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response missing nodeName")
	}
	if strings.TrimSpace(parsed.NodeName) != strings.TrimSpace(parsed.NodeID) {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response node identity mismatch")
	}
	if strings.TrimSpace(parsed.TokenID) != strings.TrimSpace(expectedTokenID) {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response tokenId mismatch")
	}
	if strings.TrimSpace(expectedTokenValue) != "" && strings.TrimSpace(parsed.TokenValue) != strings.TrimSpace(expectedTokenValue) {
		return autoRegisterResponse{}, fmt.Errorf("auto-register response tokenValue mismatch")
	}
	return parsed, nil
}

func (p *ProxmoxSetup) fetchSetupToken(ctx context.Context, ptype proxmoxProductType, hostURL string) (string, error) {
	payload := setupScriptURLRequest{
		Type:        string(ptype),
		Host:        hostURL,
		BackupPerms: false,
	}

	body, err := json.Marshal(payload)
	if err != nil {
		return "", fmt.Errorf("marshal setup token payload: %w", err)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.pulseURL+"/api/setup-script-url", bytes.NewReader(body))
	if err != nil {
		return "", &permanentError{fmt.Errorf("create setup token request: %w", err)}
	}
	req.Header.Set("Content-Type", "application/json")
	if token := strings.TrimSpace(p.apiToken); token != "" {
		req.Header.Set("X-API-Token", token)
	}

	resp, err := p.httpClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("setup token request failed: %w", err)
	}
	defer func() {
		if closeErr := resp.Body.Close(); closeErr != nil {
			p.logger.Debug().Err(closeErr).Msg("Failed to close setup token response body")
		}
	}()

	if resp.StatusCode >= 400 {
		bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
		bodyStr := strings.TrimSpace(string(bodyBytes))
		if resp.StatusCode < 500 {
			return "", &clientError{statusCode: resp.StatusCode, body: bodyStr}
		}
		return "", fmt.Errorf("setup token endpoint returned HTTP %d: %s", resp.StatusCode, bodyStr)
	}

	var parsed setupScriptURLResponse
	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
		return "", fmt.Errorf("decode setup token response: %w", err)
	}
	if strings.TrimSpace(parsed.Type) != strings.TrimSpace(string(ptype)) {
		return "", fmt.Errorf("setup token response type mismatch")
	}
	if strings.TrimSpace(parsed.Host) != strings.TrimSpace(hostURL) {
		return "", fmt.Errorf("setup token response host mismatch")
	}
	expectedScriptURL := fmt.Sprintf(
		"%s/api/setup-script?host=%s&pulse_url=%s&type=%s",
		p.pulseURL,
		url.QueryEscape(hostURL),
		url.QueryEscape(p.pulseURL),
		url.QueryEscape(string(ptype)),
	)
	if strings.TrimSpace(parsed.URL) != expectedScriptURL {
		return "", fmt.Errorf("setup token response url mismatch")
	}
	expectedFileName := fmt.Sprintf("pulse-setup-%s.sh", strings.TrimSpace(string(ptype)))
	if strings.TrimSpace(parsed.ScriptFileName) != expectedFileName {
		return "", fmt.Errorf("setup token response scriptFileName mismatch")
	}
	if strings.TrimSpace(parsed.SetupToken) == "" {
		return "", fmt.Errorf("setup token response missing setupToken")
	}
	expectedDownloadURL := expectedScriptURL
	if parsedURL, err := url.Parse(expectedScriptURL); err == nil {
		query := parsedURL.Query()
		query.Set("setup_token", strings.TrimSpace(parsed.SetupToken))
		parsedURL.RawQuery = query.Encode()
		expectedDownloadURL = parsedURL.String()
	}
	if strings.TrimSpace(parsed.DownloadURL) != expectedDownloadURL && strings.TrimSpace(parsed.DownloadURL) != expectedScriptURL {
		return "", fmt.Errorf("setup token response downloadURL mismatch")
	}
	if strings.TrimSpace(parsed.TokenHint) == "" {
		return "", fmt.Errorf("setup token response missing tokenHint")
	}
	if strings.TrimSpace(parsed.TokenHint) == strings.TrimSpace(parsed.SetupToken) {
		return "", fmt.Errorf("setup token response tokenHint must mask setupToken")
	}
	modern := strings.TrimSpace(parsed.DownloadURL) == expectedScriptURL
	for _, command := range []string{parsed.Command, parsed.CommandWithEnv, parsed.CommandWithoutEnv} {
		if modern != strings.Contains(command, "PULSE_SETUP_TOKEN_FILE=") {
			return "", fmt.Errorf("setup-script command and download credential transports do not match")
		}
	}

	if err := validateSetupScriptCommand(parsed.Command, expectedScriptURL, strings.TrimSpace(parsed.SetupToken), true); err != nil {
		return "", fmt.Errorf("setup token response command invalid: %w", err)
	}
	if err := validateSetupScriptCommand(parsed.CommandWithEnv, expectedScriptURL, strings.TrimSpace(parsed.SetupToken), true); err != nil {
		return "", fmt.Errorf("setup token response commandWithEnv invalid: %w", err)
	}
	if err := validateSetupScriptCommand(parsed.CommandWithoutEnv, expectedScriptURL, strings.TrimSpace(parsed.SetupToken), false); err != nil {
		return "", fmt.Errorf("setup token response commandWithoutEnv invalid: %w", err)
	}
	if parsed.Expires <= time.Now().Unix() {
		return "", fmt.Errorf("setup token response missing live expiry")
	}
	return strings.TrimSpace(parsed.SetupToken), nil
}

// registerWithPulse calls the auto-register endpoint to add the node.
func (p *ProxmoxSetup) registerWithPulse(ctx context.Context, ptype proxmoxProductType, hostURL, tokenID, tokenValue string) (autoRegisterResponse, error) {
	candidateHosts := p.orderedCandidateHosts(ctx, ptype, hostURL)

	backoffs := p.retryBackoffs
	if backoffs == nil {
		backoffs = []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, 60 * time.Second}
	}

	var lastErr error
	for attempt := 0; attempt <= len(backoffs); attempt++ {
		if attempt > 0 {
			p.logger.Info().
				Int("attempt", attempt+1).
				Int("max_attempts", len(backoffs)+1).
				Str("type", string(ptype)).
				Msg("Retrying Proxmox auto-registration with Pulse")
			select {
			case <-ctx.Done():
				return autoRegisterResponse{}, ctx.Err()
			case <-time.After(backoffs[attempt-1]):
			}
		}

		setupToken := ""
		if strings.TrimSpace(p.apiToken) == "" {
			var err error
			setupToken, err = p.fetchSetupToken(ctx, ptype, hostURL)
			if err != nil {
				lastErr = fmt.Errorf("fetch setup token: %w", err)
				var ce *clientError
				var pe *permanentError
				if errors.As(lastErr, &ce) || errors.As(lastErr, &pe) {
					return autoRegisterResponse{}, lastErr
				}
				p.logger.Warn().Err(lastErr).
					Int("attempt", attempt+1).
					Int("max_attempts", len(backoffs)+1).
					Str("type", string(ptype)).
					Msg("Proxmox setup token fetch failed")
				continue
			}
		}

		payload := autoRegisterRequest{
			Type:           ptype,
			Host:           hostURL,
			CandidateHosts: candidateHosts,
			ServerName:     p.hostname,
			AuthToken:      setupToken,
			TokenID:        tokenID,
			TokenValue:     tokenValue,
			Source:         autoRegisterSourceAgent,
		}

		body, err := json.Marshal(payload)
		if err != nil {
			return autoRegisterResponse{}, fmt.Errorf("marshal payload: %w", err)
		}

		parsed, err := p.doRegisterRequest(ctx, body, setupToken, ptype, autoRegisterSourceAgent, candidateHosts, tokenID, tokenValue)
		if err == nil {
			if attempt > 0 {
				p.logger.Info().Int("attempt", attempt+1).Str("type", string(ptype)).Msg("Proxmox auto-registration succeeded after retry")
			}
			return parsed, nil
		}
		lastErr = err

		// Don't retry client errors (4xx) or permanent errors (e.g. malformed URL).
		var ce *clientError
		var pe *permanentError
		if errors.As(err, &ce) || errors.As(err, &pe) {
			return autoRegisterResponse{}, err
		}

		p.logger.Warn().Err(err).
			Int("attempt", attempt+1).
			Int("max_attempts", len(backoffs)+1).
			Str("type", string(ptype)).
			Msg("Proxmox auto-registration attempt failed")
	}

	return autoRegisterResponse{}, fmt.Errorf("all %d registration attempts failed: %w", len(backoffs)+1, lastErr)
}

func isAlreadyExistsOutput(output string) bool {
	text := strings.ToLower(strings.TrimSpace(output))
	return strings.Contains(text, "already exists")
}

// isAlreadyRegistered checks if we've already done Proxmox setup.
// This uses the legacy single state file for backward compatibility.
func (p *ProxmoxSetup) isAlreadyRegistered() bool {
	_, err := p.collector.Stat(p.legacyStateFilePath())
	return err == nil
}

// isTypeRegistered checks if a specific Proxmox type has been registered.
func (p *ProxmoxSetup) isTypeRegistered(ptype proxmoxProductType) bool {
	// Check per-type state file first (new behavior)
	stateFile := p.stateFileForType(ptype)
	if _, err := p.collector.Stat(stateFile); err == nil {
		return true
	}

	// Check legacy state file for backward compat
	if _, err := p.collector.Stat(p.legacyStateFilePath()); err == nil {
		// Legacy file exists. The old detection logic was:
		// 1. If pvesh exists → registered "pve"
		// 2. Else if proxmox-backup-manager exists → registered "pbs"
		//
		// So we need to figure out what was likely registered.
		// If PVE is currently installed, treat PVE as registered.
		// If only PBS is installed (no PVE), treat PBS as registered.
		types := p.detectProxmoxTypes()
		if len(types) == 0 {
			return false
		}

		// The old code registered the "first" type it found
		// If PVE is installed, it was always detected first
		if ptype == proxmoxProductPVE {
			for _, t := range types {
				if t == proxmoxProductPVE {
					return true
				}
			}
		}
		// If PBS-only host (no PVE), PBS was what was registered
		if ptype == proxmoxProductPBS {
			hasPVE := false
			hasPBS := false
			for _, t := range types {
				if t == proxmoxProductPVE {
					hasPVE = true
				}
				if t == proxmoxProductPBS {
					hasPBS = true
				}
			}
			if hasPBS && !hasPVE {
				return true
			}
		}
	}

	return false
}

// stateFileForType returns the state file path for a specific Proxmox type.
func (p *ProxmoxSetup) stateFileForType(ptype proxmoxProductType) string {
	switch ptype {
	case proxmoxProductPVE:
		return p.pveStateFilePath()
	case proxmoxProductPBS:
		return p.pbsStateFilePath()
	default:
		return p.legacyStateFilePath()
	}
}

// markAsRegistered creates a state file to indicate setup is complete.
func (p *ProxmoxSetup) markAsRegistered() {
	if err := p.collector.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
		p.logger.Warn().Err(err).Msg("Failed to create state directory")
		return
	}
	if err := p.collector.Chmod(p.stateDir, proxmoxStateDirPerm); err != nil {
		p.logger.Warn().Err(err).Msg("Failed to enforce state directory permissions")
	}

	legacyPath := p.legacyStateFilePath()
	if err := p.collector.WriteFile(legacyPath, []byte(time.Now().Format(time.RFC3339)), proxmoxStateFilePerm); err != nil {
		p.logger.Warn().Err(err).Msg("Failed to write state file")
		return
	}
	if err := p.collector.Chmod(legacyPath, proxmoxStateFilePerm); err != nil {
		p.logger.Warn().Err(err).Msg("Failed to enforce state file permissions")
	}
}

// markTypeAsRegistered creates a state file for a specific Proxmox type.
func (p *ProxmoxSetup) markTypeAsRegistered(ptype proxmoxProductType) {
	if err := p.collector.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
		p.logger.Warn().Err(err).Msg("Failed to create state directory")
		return
	}
	if err := p.collector.Chmod(p.stateDir, proxmoxStateDirPerm); err != nil {
		p.logger.Warn().Err(err).Msg("Failed to enforce state directory permissions")
	}

	stateFile := p.stateFileForType(ptype)
	if err := p.collector.WriteFile(stateFile, []byte(time.Now().Format(time.RFC3339)), proxmoxStateFilePerm); err != nil {
		p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to write type state file")
		return
	}
	if err := p.collector.Chmod(stateFile, proxmoxStateFilePerm); err != nil {
		p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to enforce type state file permissions")
	}
}

// clearTypeRegisteredMarker removes the registration state file for a specific
// Proxmox type. Called before token rotation so that a failed Pulse update
// leaves the system in a retriable state rather than silently stale.
func (p *ProxmoxSetup) clearTypeRegisteredMarker(ptype proxmoxProductType) {
	stateFile := p.stateFileForType(ptype)
	if err := os.Remove(stateFile); err != nil && !os.IsNotExist(err) {
		p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to clear type registration marker")
	}
}