Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package hostagent
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/agenttarget"
"github.com/rs/zerolog"
)
// ProxmoxSetup handles one-time Proxmox API token creation and node registration.
type ProxmoxSetup struct {
logger zerolog.Logger
httpClient *http.Client
pulseURL string
apiToken string
proxmoxType proxmoxProductType // pve, pbs, or unknown for auto-detect
hostname string
reportIP string
insecureSkipVerify bool
collector SystemCollector
stateDir string // directory for registration state files
retryBackoffs []time.Duration // overridable for testing; nil uses defaults
monitorToken string // optional destination-scoped token name
authoritative bool // primary destinations retain local-network HTTP compatibility
allowPlaintextHTTP bool // explicit observer plaintext transport policy
}
// ProxmoxSetupResult contains the result of a successful Proxmox setup.
type ProxmoxSetupResult struct {
ProxmoxType string // "pve" or "pbs"
TokenID string // Full token ID (e.g., "pulse-monitor@pve!pulse-pulse-example-com")
TokenValue string // The secret token value
NodeHost string // The host URL for auto-registration
NodeName string // The canonical stored node identity returned by Pulse
Registered bool // Whether the node was successfully registered with Pulse
}
type proxmoxProductType string
const (
proxmoxProductUnknown proxmoxProductType = ""
proxmoxProductPVE proxmoxProductType = "pve"
proxmoxProductPBS proxmoxProductType = "pbs"
)
type autoRegisterSource string
const autoRegisterSourceAgent autoRegisterSource = "agent"
type autoRegisterRequest struct {
Type proxmoxProductType `json:"type"`
Host string `json:"host"`
CandidateHosts []string `json:"candidateHosts,omitempty"`
ServerName string `json:"serverName"`
AuthToken string `json:"authToken,omitempty"`
TokenID string `json:"tokenId,omitempty"`
TokenValue string `json:"tokenValue,omitempty"`
Source autoRegisterSource `json:"source"`
CheckRegistration bool `json:"checkRegistration,omitempty"`
}
type autoRegisterResponse struct {
Status string `json:"status"`
Message string `json:"message"`
Action string `json:"action"`
Type string `json:"type"`
Source string `json:"source"`
Host string `json:"host"`
NodeID string `json:"nodeId"`
NodeName string `json:"nodeName"`
TokenID string `json:"tokenId"`
TokenValue string `json:"tokenValue"`
}
type autoRegisterCheckResponse struct {
Registered bool `json:"registered"`
SourceExists *bool `json:"sourceExists,omitempty"`
CanRegister *bool `json:"canRegister,omitempty"`
}
type registrationState struct {
registered bool
sourceExists bool
canRegister bool
}
const maxRegistrationCheckResponseBytes = 8 * 1024
type setupScriptURLRequest struct {
Type string `json:"type"`
Host string `json:"host"`
BackupPerms bool `json:"backupPerms"`
}
type setupScriptURLResponse struct {
Type string `json:"type"`
Host string `json:"host"`
URL string `json:"url"`
DownloadURL string `json:"downloadURL"`
ScriptFileName string `json:"scriptFileName"`
Command string `json:"command"`
CommandWithEnv string `json:"commandWithEnv"`
CommandWithoutEnv string `json:"commandWithoutEnv"`
SetupToken string `json:"setupToken"`
TokenHint string `json:"tokenHint"`
Expires int64 `json:"expires"`
}
func validateSetupScriptCommand(command string, expectedScriptURL string, setupToken string, tokenRequired bool) error {
trimmedCommand := strings.TrimSpace(command)
if trimmedCommand == "" {
return fmt.Errorf("missing command")
}
if !strings.Contains(trimmedCommand, expectedScriptURL) {
return fmt.Errorf("command url mismatch")
}
if !strings.Contains(trimmedCommand, `if [ "$(id -u)" -eq 0 ]; then`) {
return fmt.Errorf("command missing direct-root path")
}
if !strings.Contains(trimmedCommand, `elif command -v sudo >/dev/null 2>&1; then`) {
return fmt.Errorf("command missing sudo path")
}
// New servers separate reveal from private input. Accept the old coherent
// artifact only for rolling upgrades against an older server; never mix
// its token-bearing URL with the new command format.
if strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN_FILE=") && !strings.Contains(trimmedCommand, setupToken) && !strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN=") {
return nil
}
if tokenRequired {
if !strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN=") || !strings.Contains(trimmedCommand, setupToken) {
return fmt.Errorf("command missing setup token transport")
}
return nil
}
if strings.Contains(trimmedCommand, "PULSE_SETUP_TOKEN=") || strings.Contains(trimmedCommand, setupToken) {
return fmt.Errorf("command leaked setup token transport")
}
return nil
}
const (
proxmoxUser = "pulse-monitor"
proxmoxUserPVE = "pulse-monitor@pve"
proxmoxUserPBS = "pulse-monitor@pbs"
proxmoxComment = "Pulse monitoring service"
)
const proxmoxMonitorRole = "PulseMonitor"
var proxmoxTokenSlugPattern = regexp.MustCompile(`[^a-z0-9]+`)
func proxmoxTokenHostCandidate(candidate string) string {
raw := strings.TrimSpace(candidate)
if raw == "" {
return ""
}
if strings.Contains(raw, "://") {
if parsed, err := url.Parse(raw); err == nil && parsed.Hostname() != "" {
return parsed.Hostname()
}
}
if host, _, err := net.SplitHostPort(raw); err == nil {
return strings.Trim(host, "[]")
}
if parsed, err := url.Parse("https://" + raw); err == nil && parsed.Hostname() != "" {
return parsed.Hostname()
}
return strings.Trim(raw, "[]")
}
func proxmoxTokenSlug(raw string) string {
trimmed := strings.Trim(strings.ToLower(strings.TrimSpace(raw)), ".")
if trimmed == "" {
return ""
}
slug := proxmoxTokenSlugPattern.ReplaceAllString(trimmed, "-")
slug = strings.Trim(slug, "-")
if slug == "" {
return ""
}
const maxSlugLen = 48
if len(slug) > maxSlugLen {
slug = strings.Trim(slug[:maxSlugLen], "-")
}
return slug
}
func proxmoxTokenScope(candidates ...string) string {
for _, candidate := range candidates {
host := proxmoxTokenHostCandidate(candidate)
if host == "" {
continue
}
if slug := proxmoxTokenSlug(host); slug != "" {
return slug
}
}
return "server"
}
func (p *ProxmoxSetup) monitorTokenName() string {
if tokenName := strings.TrimSpace(p.monitorToken); tokenName != "" {
return tokenName
}
return "pulse-" + proxmoxTokenScope(p.hostname, p.pulseURL)
}
func tokenAlreadyExists(err error, output string) bool {
lowerErr := strings.ToLower(fmt.Sprintf("%v", err))
lowerOutput := strings.ToLower(output)
return strings.Contains(lowerErr, "already exists") || strings.Contains(lowerOutput, "already exists")
}
func privProbeRoleName(priv string) string {
// Keep role name deterministic (helps tests) and valid for pveum.
// Replace characters that are likely to cause issues in role names.
safe := strings.NewReplacer(".", "_", ":", "_", "/", "_", " ", "_", ",", "_").Replace(priv)
return "PulseTmpPrivCheck_" + safe
}
func (p *ProxmoxSetup) probePVEPrivilege(ctx context.Context, privilege string) bool {
roleName := privProbeRoleName(privilege)
// If privilege doesn't exist on this PVE version, pveum will fail with a non-zero exit code.
if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "add", roleName, "-privs", privilege); err != nil {
return false
}
if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "delete", roleName); err != nil {
p.logger.Debug().Err(err).Str("role", roleName).Msg("Failed to cleanup temporary PVE privilege probe role")
}
return true
}
func (p *ProxmoxSetup) applyPVEACL(ctx context.Context, path string, principalFlag string, principal string, role string) {
if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "aclmod", path, principalFlag, principal, "-role", role); err != nil {
p.logger.Warn().Err(err).Str("path", path).Str("principal", principal).Str("role", role).Msg("Failed to apply PVE role")
}
}
func (p *ProxmoxSetup) configurePVEPermissions(ctx context.Context, tokenID string) {
// Baseline: read-only access.
p.applyPVEACL(ctx, "/", "-user", proxmoxUserPVE, "PVEAuditor")
if tokenID != "" {
p.applyPVEACL(ctx, "/", "-token", tokenID, "PVEAuditor")
}
// Extra privileges are optional, but enable additional features:
// - Sys.Audit: required for pending apt updates + some cluster/ceph visibility
// - VM.GuestAgent.Audit/FileRead (PVE 9+) or VM.Monitor (PVE 8): guest agent data
// - Datastore.Audit: improved storage visibility
var extraPrivs []string
if p.probePVEPrivilege(ctx, "Sys.Audit") {
extraPrivs = append(extraPrivs, "Sys.Audit")
}
if p.probePVEPrivilege(ctx, "VM.GuestAgent.Audit") {
extraPrivs = append(extraPrivs, "VM.GuestAgent.Audit")
if p.probePVEPrivilege(ctx, "VM.GuestAgent.FileRead") {
extraPrivs = append(extraPrivs, "VM.GuestAgent.FileRead")
}
} else if p.probePVEPrivilege(ctx, "VM.Monitor") {
extraPrivs = append(extraPrivs, "VM.Monitor")
}
if p.probePVEPrivilege(ctx, "Datastore.Audit") {
extraPrivs = append(extraPrivs, "Datastore.Audit")
}
if len(extraPrivs) > 0 {
privString := strings.Join(extraPrivs, ",")
// Prefer modify (non-destructive) in case the role already exists.
if _, err := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "modify", proxmoxMonitorRole, "-privs", privString); err != nil {
if _, addErr := p.collector.CommandCombinedOutput(ctx, "pveum", "role", "add", proxmoxMonitorRole, "-privs", privString); addErr != nil {
p.logger.Warn().Err(addErr).Str("privs", privString).Msg("Failed to configure PulseMonitor role")
return
}
}
p.applyPVEACL(ctx, "/", "-user", proxmoxUserPVE, proxmoxMonitorRole)
if tokenID != "" {
p.applyPVEACL(ctx, "/", "-token", tokenID, proxmoxMonitorRole)
}
}
// Add PVEDatastoreAdmin on /storage for backup visibility (issue #1139)
p.applyPVEACL(ctx, "/storage", "-user", proxmoxUserPVE, "PVEDatastoreAdmin")
if tokenID != "" {
p.applyPVEACL(ctx, "/storage", "-token", tokenID, "PVEDatastoreAdmin")
}
}
const (
proxmoxStateDirPerm = 0700
proxmoxStateFilePerm = 0600
)
// State file path methods — derive from the configurable stateDir.
func (p *ProxmoxSetup) legacyStateFilePath() string {
return filepath.Join(p.stateDir, "proxmox-registered")
}
func (p *ProxmoxSetup) pveStateFilePath() string {
return filepath.Join(p.stateDir, "proxmox-pve-registered")
}
func (p *ProxmoxSetup) pbsStateFilePath() string {
return filepath.Join(p.stateDir, "proxmox-pbs-registered")
}
// registrationBlockedFilePath is the marker the installer reads to surface a
// denied registration grant in its own output instead of leaving the failure
// buried in the agent journal (#1644).
func (p *ProxmoxSetup) registrationBlockedFilePath(ptype proxmoxProductType) string {
return filepath.Join(p.stateDir, fmt.Sprintf("proxmox-%s-registration-blocked", ptype))
}
// detectedTypesFilePath names the marker RunAll writes listing the canonical
// Proxmox products it is registering on this host, one per line.
//
// Without it the installer cannot tell a PVE-only host from a combined PVE+PBS
// host, so it has to report the first outcome it sees and call the install
// done. On a combined host — an officially supported deployment that now
// consumes one bootstrap grant per type — that produced a partial verdict:
// either a success banner before PBS had landed, or an error banner for PBS
// that buried a perfectly good PVE registration (#1644).
func (p *ProxmoxSetup) detectedTypesFilePath() string {
return filepath.Join(p.stateDir, "proxmox-detected-types")
}
// writeDetectedTypesMarker publishes the types RunAll is about to register so
// the installer waits for an outcome from each of them, and only from them.
func (p *ProxmoxSetup) writeDetectedTypesMarker(types []proxmoxProductType) {
if len(types) == 0 {
return
}
if err := p.collector.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
p.logger.Debug().Err(err).Str("dir", p.stateDir).Msg("Failed to create state dir for detected-types marker")
return
}
contents := strings.Join(proxmoxProductTypesToStrings(types), "\n") + "\n"
path := p.detectedTypesFilePath()
if err := p.collector.WriteFile(path, []byte(contents), proxmoxStateFilePerm); err != nil {
p.logger.Debug().Err(err).Msg("Failed to write detected-types marker")
return
}
if err := p.collector.Chmod(path, proxmoxStateFilePerm); err != nil {
p.logger.Debug().Err(err).Msg("Failed to enforce detected-types marker permissions")
}
}
func (p *ProxmoxSetup) writeRegistrationBlockedMarker(ptype proxmoxProductType, reason string) {
if err := os.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
p.logger.Debug().Err(err).Str("dir", p.stateDir).Msg("Failed to create state dir for registration-blocked marker")
return
}
if err := os.WriteFile(p.registrationBlockedFilePath(ptype), []byte(reason+"\n"), proxmoxStateFilePerm); err != nil {
p.logger.Debug().Err(err).Str("type", string(ptype)).Msg("Failed to write registration-blocked marker")
}
}
func (p *ProxmoxSetup) clearRegistrationBlockedMarker(ptype proxmoxProductType) {
if err := os.Remove(p.registrationBlockedFilePath(ptype)); err != nil && !os.IsNotExist(err) {
p.logger.Debug().Err(err).Str("type", string(ptype)).Msg("Failed to remove registration-blocked marker")
}
}
func parseProxmoxProductType(rawType string) proxmoxProductType {
switch strings.ToLower(strings.TrimSpace(rawType)) {
case string(proxmoxProductPVE):
return proxmoxProductPVE
case string(proxmoxProductPBS):
return proxmoxProductPBS
default:
return proxmoxProductUnknown
}
}
func proxmoxProductTypesToStrings(types []proxmoxProductType) []string {
if len(types) == 0 {
return nil
}
result := make([]string, 0, len(types))
for _, t := range types {
result = append(result, string(t))
}
return result
}
// NewProxmoxSetup creates a new ProxmoxSetup instance.
func NewProxmoxSetup(logger zerolog.Logger, httpClient *http.Client, collector SystemCollector, pulseURL, apiToken, proxmoxType, hostname, reportIP, stateDir string, insecure bool) *ProxmoxSetup {
if stateDir == "" {
stateDir = defaultStateDir
}
return &ProxmoxSetup{
logger: logger,
httpClient: httpClient,
collector: collector,
pulseURL: strings.TrimRight(pulseURL, "/"),
apiToken: apiToken,
proxmoxType: proxmoxProductType(proxmoxType),
hostname: hostname,
reportIP: reportIP,
stateDir: stateDir,
insecureSkipVerify: insecure,
authoritative: true,
}
}
// WithObserverPlaintextPolicy marks this setup as an observer transport. Unlike
// the primary, every non-loopback plaintext observer requires its own explicit
// opt-in and cannot inherit process-wide primary transport consent.
func (p *ProxmoxSetup) WithObserverPlaintextPolicy(allowPlaintext bool) *ProxmoxSetup {
p.authoritative = false
p.allowPlaintextHTTP = allowPlaintext
return p
}
// WithMonitorTokenName isolates Proxmox credentials for an additional Pulse
// destination. The primary destination deliberately keeps the legacy token
// name for upgrade compatibility.
func (p *ProxmoxSetup) WithMonitorTokenName(tokenName string) *ProxmoxSetup {
p.monitorToken = strings.TrimSpace(tokenName)
return p
}
// Run executes the Proxmox setup process:
// 1. Detects Proxmox type (if not specified)
// 2. Creates the monitoring user and API token
// 3. Registers the node with Pulse via auto-register
func (p *ProxmoxSetup) Run(ctx context.Context) (*ProxmoxSetupResult, error) {
pulseURL, err := agenttarget.NormalizePulseURL(p.pulseURL, p.authoritative, p.allowPlaintextHTTP)
if err != nil {
return nil, fmt.Errorf("invalid pulse URL: %w", err)
}
p.pulseURL = pulseURL
ptypeStr, err := normalizeProxmoxType(string(p.proxmoxType))
if err != nil {
return nil, err
}
ptype := proxmoxProductType(ptypeStr)
// Detect Proxmox type
if ptype == proxmoxProductUnknown {
detected := p.detectProxmoxType()
if detected == proxmoxProductUnknown {
return nil, fmt.Errorf("this system does not appear to be a Proxmox VE or PBS node")
}
ptype = detected
p.logger.Info().Str("type", string(ptype)).Msg("Auto-detected Proxmox type")
}
result, err := p.runForType(ctx, ptype)
if result != nil && result.Registered {
p.markAsRegistered()
}
return result, err
}
// RunAll detects and registers ALL Proxmox products on this system.
// This supports hosts with both PVE and PBS installed (a common and officially
// supported configuration). Each type gets its own registration and state tracking,
// and the server grants one bootstrap create per canonical type, so a combined
// host registers both from a single install token (#1644).
// Returns results for all types that were processed (skipping already-registered ones).
//
// One type failing does not abandon the others: the remaining types are still
// attempted and their results returned. An error comes back only when every
// detected type failed, so a fully failed Proxmox setup is still loud while a
// partial one leaves the successful registrations reportable.
func (p *ProxmoxSetup) RunAll(ctx context.Context) ([]*ProxmoxSetupResult, error) {
pulseURL, err := agenttarget.NormalizePulseURL(p.pulseURL, p.authoritative, p.allowPlaintextHTTP)
if err != nil {
return nil, fmt.Errorf("invalid pulse URL: %w", err)
}
p.pulseURL = pulseURL
forcedTypeStr, err := normalizeProxmoxType(string(p.proxmoxType))
if err != nil {
return nil, err
}
forcedType := proxmoxProductType(forcedTypeStr)
var results []*ProxmoxSetupResult
// If a specific type is forced, only run that
if forcedType != "" {
p.writeDetectedTypesMarker([]proxmoxProductType{forcedType})
result, err := p.runForType(ctx, forcedType)
if err != nil {
return nil, fmt.Errorf("setup proxmox type %s: %w", p.proxmoxType, err)
}
if result != nil {
results = append(results, result)
}
return results, nil
}
// Detect all Proxmox products
types := p.detectProxmoxTypes()
if len(types) == 0 {
return nil, fmt.Errorf("this system does not appear to be a Proxmox VE or PBS node")
}
p.logger.Info().Strs("types", proxmoxProductTypesToStrings(types)).Msg("Auto-detected Proxmox products")
p.writeDetectedTypesMarker(types)
// Register each type
var failures []error
var failedTypes []string
for _, ptype := range types {
result, err := p.runForType(ctx, ptype)
if err != nil {
p.logger.Error().Err(err).Str("type", string(ptype)).Msg("Failed to setup Proxmox type")
failures = append(failures, fmt.Errorf("setup proxmox type %s: %w", ptype, err))
failedTypes = append(failedTypes, string(ptype))
continue // Don't fail completely, try other types
}
if result != nil {
results = append(results, result)
}
}
if len(failures) == len(types) {
return results, errors.Join(failures...)
}
if len(failures) > 0 {
p.logger.Warn().
Strs("failed_types", failedTypes).
Strs("detected_types", proxmoxProductTypesToStrings(types)).
Msg("Some Proxmox products failed to set up; the remaining products were registered")
}
return results, nil
}
// RunHealthCheck verifies that every previously-registered Proxmox type is
// still connected to Pulse. If a type's registration is confirmed stale by the
// server (node exists in config but connection is broken), RunHealthCheck
// re-registers it with a fresh token.
//
// Types with no local registration marker are intentionally skipped. Those
// need a full RunAll cycle (typically at next agent startup) rather than a
// health-check rotation, which would cause uncontrolled token churn if Pulse
// is temporarily unreachable.
func (p *ProxmoxSetup) RunHealthCheck(ctx context.Context) ([]*ProxmoxSetupResult, error) {
pulseURL, err := agenttarget.NormalizePulseURL(p.pulseURL, p.authoritative, p.allowPlaintextHTTP)
if err != nil {
return nil, fmt.Errorf("invalid pulse URL: %w", err)
}
p.pulseURL = pulseURL
forcedTypeStr, err := normalizeProxmoxType(string(p.proxmoxType))
if err != nil {
return nil, err
}
forcedType := proxmoxProductType(forcedTypeStr)
var typesToCheck []proxmoxProductType
if forcedType != "" {
typesToCheck = []proxmoxProductType{forcedType}
} else {
typesToCheck = p.detectProxmoxTypes()
}
var results []*ProxmoxSetupResult
for _, ptype := range typesToCheck {
// Skip types with no local marker — they need a full startup setup
// cycle, not a periodic health-check rotation.
if !p.isTypeRegistered(ptype) {
continue
}
hostURL := p.getHostURL(ctx, ptype)
registration, err := p.checkRegistrationWithPulse(ctx, ptype, hostURL)
if err != nil {
p.logger.Warn().Err(err).Str("type", string(ptype)).
Msg("Proxmox health check: could not verify registration with Pulse")
continue
}
if registration.registered {
continue // still healthy
}
if !registration.canRegister {
p.logger.Warn().
Str("type", string(ptype)).
Str("host", hostURL).
Bool("source_exists", registration.sourceExists).
Msg("Proxmox health check: Pulse has no matching source and this agent token cannot create one; re-run the Proxmox agent install command")
continue
}
p.logger.Info().Str("type", string(ptype)).Str("host", hostURL).
Msg("Proxmox health check: node disconnected, re-registering")
result, err := p.runForType(ctx, ptype)
if err != nil {
p.logger.Error().Err(err).Str("type", string(ptype)).
Msg("Proxmox health check: re-registration failed")
continue
}
if result != nil {
results = append(results, result)
}
}
return results, nil
}
// runForType executes setup for a specific Proxmox type.
func (p *ProxmoxSetup) runForType(ctx context.Context, ptype proxmoxProductType) (*ProxmoxSetupResult, error) {
normalizedTypeStr, err := normalizeProxmoxType(string(ptype))
if err != nil || normalizedTypeStr == "" {
return nil, fmt.Errorf("invalid proxmox type %q: must be pve or pbs", ptype)
}
ptype = proxmoxProductType(normalizedTypeStr)
hostURL := p.getHostURL(ctx, ptype)
// Always establish destination reachability and registration state before
// mutating a Proxmox token. This prevents a missing local marker or a Pulse
// startup outage from rotating credentials that a healthy Pulse instance is
// still using.
registration, err := p.checkRegistrationWithPulse(ctx, ptype, hostURL)
if err != nil {
p.logger.Warn().
Err(err).
Str("type", string(ptype)).
Msg("Failed to verify Proxmox registration state with Pulse; leaving credentials unchanged")
return nil, nil
}
if registration.registered {
if !p.isTypeRegistered(ptype) {
p.markTypeAsRegistered(ptype)
}
p.clearRegistrationBlockedMarker(ptype)
p.logger.Info().Str("type", string(ptype)).Msg("Proxmox type already registered, skipping")
return nil, nil
}
if !registration.canRegister {
reason := fmt.Sprintf(
"Pulse at %s has no %s source for %s and this agent's token cannot create one. Local Proxmox credentials were left unchanged. Generate a fresh agent install command in Pulse Settings -> Infrastructure and re-run it on this host.",
p.pulseURL, strings.ToUpper(string(ptype)), hostURL,
)
p.writeRegistrationBlockedMarker(ptype, reason)
p.logger.Error().
Str("type", string(ptype)).
Str("host", hostURL).
Bool("source_exists", registration.sourceExists).
Msg("Proxmox registration blocked: Pulse has no matching source and this agent token cannot create one")
return nil, fmt.Errorf("proxmox %s registration blocked: %s", ptype, reason)
}
p.clearRegistrationBlockedMarker(ptype)
if p.isTypeRegistered(ptype) {
p.logger.Info().
Str("type", string(ptype)).
Str("host", hostURL).
Msg("Local Proxmox registration marker exists but Pulse has no matching node; re-registering")
}
p.logger.Info().Str("type", string(ptype)).Msg("Setting up Proxmox type")
// Clear the registration marker before token operations so that a failed
// Pulse update (rotation succeeds but registration fails) leaves the system
// retriable on the next startup rather than stuck on a stale state file.
p.clearTypeRegisteredMarker(ptype)
// Create monitoring user and token
tokenID, tokenValue, err := p.setupToken(ctx, ptype)
if err != nil {
return nil, fmt.Errorf("failed to create %s API token: %w", ptype, err)
}
p.logger.Info().Str("type", string(ptype)).Str("token_id", tokenID).Msg("Created Proxmox API token")
// Register with Pulse
registered := false
registerResp, err := p.registerWithPulse(ctx, ptype, hostURL, tokenID, tokenValue)
if err != nil {
p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to register with Pulse (node may already exist)")
} else {
registered = true
p.markTypeAsRegistered(ptype)
if strings.TrimSpace(registerResp.Host) != "" {
hostURL = registerResp.Host
}
p.logger.Info().Str("type", string(ptype)).Str("host", hostURL).Str("node", registerResp.NodeName).Msg("Successfully registered Proxmox node with Pulse")
}
return &ProxmoxSetupResult{
ProxmoxType: string(ptype),
TokenID: tokenID,
TokenValue: tokenValue,
NodeHost: hostURL,
NodeName: registerResp.NodeName,
Registered: registered,
}, nil
}
func (p *ProxmoxSetup) orderedCandidateHosts(ctx context.Context, ptype proxmoxProductType, hostURL string) []string {
candidateHosts := p.candidateHostURLs(ctx, ptype)
if len(candidateHosts) == 0 || strings.TrimSpace(candidateHosts[0]) != strings.TrimSpace(hostURL) {
seen := make(map[string]struct{}, len(candidateHosts)+1)
reordered := make([]string, 0, len(candidateHosts)+1)
reordered = appendUniqueHostCandidate(reordered, seen, hostURL)
for _, candidate := range candidateHosts {
reordered = appendUniqueHostCandidate(reordered, seen, candidate)
}
candidateHosts = reordered
}
return candidateHosts
}
func (p *ProxmoxSetup) checkRegistrationWithPulse(ctx context.Context, ptype proxmoxProductType, hostURL string) (registrationState, error) {
// Runtime agent tokens authenticate directly against the canonical
// auto-register endpoint. The setup-script-url endpoint is an operator
// control-plane surface protected by settings:write and must never be used
// as a runtime token-discovery probe.
setupToken := ""
if strings.TrimSpace(p.apiToken) == "" {
var err error
setupToken, err = p.fetchSetupToken(ctx, ptype, hostURL)
if err != nil {
return registrationState{}, fmt.Errorf("fetch setup token: %w", err)
}
}
payload := autoRegisterRequest{
Type: ptype,
Host: hostURL,
CandidateHosts: p.orderedCandidateHosts(ctx, ptype, hostURL),
ServerName: p.hostname,
AuthToken: setupToken,
Source: autoRegisterSourceAgent,
CheckRegistration: true,
}
body, err := json.Marshal(payload)
if err != nil {
return registrationState{}, fmt.Errorf("marshal registration-check payload: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.pulseURL+"/api/auto-register", bytes.NewReader(body))
if err != nil {
return registrationState{}, fmt.Errorf("create registration-check request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if token := autoRegisterAPITokenHeader(p.apiToken, setupToken); token != "" {
req.Header.Set("X-API-Token", token)
}
resp, err := p.httpClient.Do(req)
if err != nil {
return registrationState{}, fmt.Errorf("registration-check request failed: %w", err)
}
defer func() {
if closeErr := resp.Body.Close(); closeErr != nil {
p.logger.Debug().Err(closeErr).Msg("Failed to close registration-check response body")
}
}()
bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxRegistrationCheckResponseBytes))
if err != nil {
return registrationState{}, fmt.Errorf("read registration-check response: %w", err)
}
if resp.StatusCode >= http.StatusBadRequest {
return registrationState{}, &clientError{
statusCode: resp.StatusCode,
body: strings.TrimSpace(string(bodyBytes)),
}
}
var parsed autoRegisterCheckResponse
if err := json.Unmarshal(bodyBytes, &parsed); err != nil {
return registrationState{}, fmt.Errorf("decode registration-check response: %w", err)
}
// Older Pulse servers returned only registered. Preserve their repair
// behavior during rolling updates by treating omitted capability fields as
// permissive; current servers always send both fields explicitly.
sourceExists := parsed.Registered
if parsed.SourceExists != nil {
sourceExists = *parsed.SourceExists
}
canRegister := true
if parsed.CanRegister != nil {
canRegister = *parsed.CanRegister
}
return registrationState{
registered: parsed.Registered,
sourceExists: sourceExists,
canRegister: canRegister,
}, nil
}
// detectProxmoxType checks for pvesh (PVE) or proxmox-backup-manager (PBS).
// For backward compatibility, returns the first detected type.
// Use detectProxmoxTypes() to get all detected types.
func (p *ProxmoxSetup) detectProxmoxType() proxmoxProductType {
types := p.detectProxmoxTypes()
if len(types) > 0 {
return types[0]
}
return proxmoxProductUnknown
}
// detectProxmoxTypes checks for ALL Proxmox products on this system.
// Returns a slice of detected types (e.g., ["pve", "pbs"] if both are installed).
// This is common when PBS is installed directly on a PVE host.
// detectProxmoxTypes checks for ALL Proxmox products on this system.
// Returns a slice of detected types (e.g., ["pve", "pbs"] if both are installed).
// This is common when PBS is installed directly on a PVE host.
func (p *ProxmoxSetup) detectProxmoxTypes() []proxmoxProductType {
var types []proxmoxProductType
// Check for PVE
if _, err := p.collector.LookPath("pvesh"); err == nil {
types = append(types, proxmoxProductPVE)
}
// Check for PBS
if _, err := p.collector.LookPath("proxmox-backup-manager"); err == nil {
types = append(types, proxmoxProductPBS)
}
return types
}
// setupToken creates the monitoring user and API token.
func (p *ProxmoxSetup) setupToken(ctx context.Context, ptype proxmoxProductType) (string, string, error) {
tokenName := p.monitorTokenName()
switch ptype {
case proxmoxProductPVE:
return p.setupPVEToken(ctx, tokenName)
case proxmoxProductPBS:
return p.setupPBSToken(ctx, tokenName)
default:
return "", "", fmt.Errorf("unsupported Proxmox type %q", ptype)
}
}
// setupPVEToken creates a PVE monitoring user and token.
func (p *ProxmoxSetup) setupPVEToken(ctx context.Context, tokenName string) (string, string, error) {
// Create user (ignore error if already exists)
if out, err := p.collector.CommandCombinedOutput(ctx, "pveum", "user", "add", proxmoxUserPVE, "--comment", proxmoxComment); err != nil {
if isAlreadyExistsOutput(out) {
p.logger.Debug().Msg("Proxmox PVE monitor user already exists")
} else {
p.logger.Warn().Err(err).Str("output", strings.TrimSpace(out)).Msg("Failed to create Proxmox PVE monitor user")
}
}
// Create token with privilege separation enabled, then mirror ACLs onto the token.
output, err := p.collector.CommandCombinedOutput(ctx, "pveum", "user", "token", "add", proxmoxUserPVE, tokenName, "--privsep", "1")
if err != nil {
if tokenAlreadyExists(err, output) {
p.logger.Info().Str("token", tokenName).Msg("PVE monitoring token already exists; rotating token in place")
if _, deleteErr := p.collector.CommandCombinedOutput(ctx, "pveum", "user", "token", "remove", proxmoxUserPVE, tokenName); deleteErr != nil {
return "", "", fmt.Errorf("failed to delete existing token before rotation: %w", deleteErr)
}
output, err = p.collector.CommandCombinedOutput(ctx, "pveum", "user", "token", "add", proxmoxUserPVE, tokenName, "--privsep", "1")
}
if err != nil {
return "", "", fmt.Errorf("failed to create token: %w", err)
}
}
// Parse token value from output
tokenValue := p.parseTokenValue(output)
if tokenValue == "" {
return "", "", fmt.Errorf("failed to parse token value from pveum output")
}
tokenID := fmt.Sprintf("%s!%s", proxmoxUserPVE, tokenName)
p.configurePVEPermissions(ctx, tokenID)
return tokenID, tokenValue, nil
}
// setupPBSToken creates a PBS monitoring user and token.
func (p *ProxmoxSetup) setupPBSToken(ctx context.Context, tokenName string) (string, string, error) {
// Create user (ignore error if already exists)
if out, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "create", proxmoxUserPBS); err != nil {
if isAlreadyExistsOutput(out) {
p.logger.Debug().Msg("Proxmox PBS monitor user already exists")
} else {
p.logger.Warn().Err(err).Str("output", strings.TrimSpace(out)).Msg("Failed to create Proxmox PBS monitor user")
}
}
// Add Audit role
if out, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "acl", "update", "/", "Audit", "--auth-id", proxmoxUserPBS); err != nil {
p.logger.Warn().Err(err).Str("output", strings.TrimSpace(out)).Msg("Failed to apply PBS Audit role to monitor user")
}
// Create token
output, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "generate-token", proxmoxUserPBS, tokenName)
if err != nil {
if tokenAlreadyExists(err, output) {
p.logger.Info().Str("token", tokenName).Msg("PBS monitoring token already exists; rotating token in place")
if _, deleteErr := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "delete-token", proxmoxUserPBS, tokenName); deleteErr != nil {
return "", "", fmt.Errorf("failed to delete existing token before rotation: %w", deleteErr)
}
output, err = p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "user", "generate-token", proxmoxUserPBS, tokenName)
}
if err != nil {
return "", "", fmt.Errorf("failed to create token: %w", err)
}
}
// Parse token value from JSON output
tokenValue := p.parsePBSTokenValue(output)
if tokenValue == "" {
return "", "", fmt.Errorf("failed to parse token value from PBS output")
}
// Add Audit role for the token itself
tokenID := fmt.Sprintf("%s!%s", proxmoxUserPBS, tokenName)
if out, err := p.collector.CommandCombinedOutput(ctx, "proxmox-backup-manager", "acl", "update", "/", "Audit", "--auth-id", tokenID); err != nil {
p.logger.Warn().Err(err).Str("token_id", tokenID).Str("output", strings.TrimSpace(out)).Msg("Failed to apply PBS Audit role to token")
}
return tokenID, tokenValue, nil
}
// parseTokenValue extracts the token value from PVE pveum output.
// The output format is typically a table with columns: │ key │ value │
// Example output:
// ┌──────────────┬──────────────────────────────────────┐
// │ key │ value │
// ╞══════════════╪══════════════════════════════════════╡
// │ full-tokenid │ pulse-monitor@pve!pulse-token │
// ├──────────────┼──────────────────────────────────────┤
// │ info │ {"privsep":"1"} │
// ├──────────────┼──────────────────────────────────────┤
// │ value │ xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx │
// └──────────────┴──────────────────────────────────────┘
func (p *ProxmoxSetup) parseTokenValue(output string) string {
// Look for the "value" row in the table output - where "value" is the KEY column
// We need to find the row where parts[1] (key column) is "value", not any row
// containing the word "value" (which would match the header row)
lines := strings.Split(output, "\n")
for _, line := range lines {
// Split by the box drawing character │ (U+2502)
parts := strings.Split(line, "│")
if len(parts) >= 3 {
key := strings.TrimSpace(parts[1])
if key == "value" {
return strings.TrimSpace(parts[2])
}
}
}
// Fallback: try regex for UUID-like token
re := regexp.MustCompile(`[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`)
if match := re.FindString(output); match != "" {
return match
}
return ""
}
// parsePBSTokenValue extracts the token value from PBS JSON output.
func (p *ProxmoxSetup) parsePBSTokenValue(output string) string {
// PBS outputs JSON like: {"tokenid": "...", "value": "..."}
var result struct {
Value string `json:"value"`
}
if err := json.Unmarshal([]byte(output), &result); err == nil && result.Value != "" {
return result.Value
}
// Fallback: extract from "value": "..." pattern
re := regexp.MustCompile(`"value"\s*:\s*"([^"]+)"`)
if matches := re.FindStringSubmatch(output); len(matches) >= 2 {
return matches[1]
}
return ""
}
func appendUniqueHostCandidate(candidates []string, seen map[string]struct{}, candidate string) []string {
normalized := strings.TrimSpace(candidate)
if normalized == "" {
return candidates
}
if _, exists := seen[normalized]; exists {
return candidates
}
seen[normalized] = struct{}{}
return append(candidates, normalized)
}
func (p *ProxmoxSetup) candidateHostURLs(ctx context.Context, ptype proxmoxProductType) []string {
port := "8006"
if ptype == proxmoxProductPBS {
port = "8007"
}
if ctx == nil {
ctx = context.Background()
}
candidates := make([]string, 0, 4)
seen := make(map[string]struct{}, 4)
buildURL := func(host string) string {
if strings.TrimSpace(host) == "" {
return ""
}
return formatHTTPSURL(host, port)
}
// Priority 1: User-specified ReportIP override from configuration.
// This allows users to manually specify which IP should be used for Proxmox API
// connections when auto-detection picks the wrong one (e.g., Issue #1061).
if p.reportIP != "" {
p.logger.Info().Str("ip", p.reportIP).Msg("Using user-specified ReportIP for Proxmox registration")
return append(candidates, buildURL(p.reportIP))
}
if p.collector == nil {
hostname := strings.TrimSpace(p.hostname)
if hostname == "" {
hostname = "localhost"
}
return append(candidates, buildURL(hostname))
}
// Priority 2: Prefer the route-aware local IP used to reach Pulse. This keeps
// generated Proxmox config aligned with the reachable endpoint shown to users,
// while preserving hostname as a fallback candidate below.
if reachableIP := p.getIPThatReachesPulse(); reachableIP != "" {
p.logger.Debug().Str("ip", reachableIP).Msg("Using IP that can reach Pulse server")
candidates = appendUniqueHostCandidate(candidates, seen, buildURL(reachableIP))
}
// Priority 3: Keep the system hostname as a fallback when it resolves to a
// non-loopback, non-link-local address. The server receives candidateHosts
// and may still choose it if it is the correct stable endpoint.
if hostname := strings.TrimSpace(p.hostname); hostname != "" {
if hostnameIP := p.getIPForHostname(); hostnameIP != "" && !isLoopbackOrLinkLocalIP(hostnameIP) {
p.logger.Debug().
Str("hostname", hostname).
Str("ip", hostnameIP).
Msg("Adding resolvable hostname fallback for Proxmox registration")
candidates = appendUniqueHostCandidate(candidates, seen, buildURL(hostname))
}
}
// Priority 4: Get all IPs and select the best one based on heuristics.
hostnameCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
if out, err := p.collector.CommandCombinedOutput(hostnameCtx, "hostname", "-I"); err == nil {
ips := strings.Fields(out)
if len(ips) > 0 {
// Get the IP that the system hostname currently resolves to
hostnameIP := p.getIPForHostname()
bestIP := selectBestIP(ips, hostnameIP)
if bestIP != "" {
candidates = appendUniqueHostCandidate(candidates, seen, buildURL(bestIP))
}
}
}
if len(candidates) > 0 {
return candidates
}
// Final fallback to hostname if IP detection failed
hostname := p.hostname
if hostname == "" {
hostname = "localhost"
}
return append(candidates, buildURL(hostname))
}
// getHostURL constructs the host URL for this Proxmox node.
// Prefers explicit reportIP, then route-aware IP detection, then hostname and
// heuristic local IP fallbacks.
func (p *ProxmoxSetup) getHostURL(ctx context.Context, ptype proxmoxProductType) string {
candidates := p.candidateHostURLs(ctx, ptype)
if len(candidates) == 0 {
return ""
}
return candidates[0]
}
// getIPThatReachesPulse determines which local IP is used to connect to the Pulse server.
// This handles cases where multiple network interfaces exist (e.g., management, Ceph, cluster ring)
// and ensures we pick the one that can actually reach Pulse. Related to #929.
func (p *ProxmoxSetup) getIPThatReachesPulse() string {
if p.collector == nil {
return ""
}
if p.pulseURL == "" {
return ""
}
// Parse the Pulse URL to get host:port
pulseServerURL, err := url.Parse(p.pulseURL)
if err != nil {
return ""
}
host := pulseServerURL.Hostname()
if host == "" {
return ""
}
port := pulseServerURL.Port()
if port == "" {
switch pulseServerURL.Scheme {
case "https":
port = "443"
case "http":
port = "80"
default:
port = "7655"
}
}
target := net.JoinHostPort(host, port)
// Create a UDP "connection" to determine local address (doesn't actually send data)
// We use a short timeout as this is just a routing table lookup.
conn, err := p.collector.DialTimeout("udp", target, 500*time.Millisecond)
if err != nil {
p.logger.Debug().Err(err).Str("target", target).Msg("Could not determine local IP for Pulse connection (routing check failed)")
return ""
}
defer func() {
if closeErr := conn.Close(); closeErr != nil {
p.logger.Debug().Err(closeErr).Msg("Failed to close local route probe connection")
}
}()
localAddr, ok := conn.LocalAddr().(*net.UDPAddr)
if !ok || localAddr == nil {
return ""
}
ip := localAddr.IP.String()
// If we found an IP that can reach Pulse, it's the most reliable IP fallback
// after a canonical hostname probe. Reject loopback and link-local addresses.
if ip != "" && !isLoopbackOrLinkLocalIP(ip) {
return ip
}
return ""
}
func formatHTTPSURL(host, port string) string {
return (&url.URL{
Scheme: "https",
Host: net.JoinHostPort(host, port),
}).String()
}
// getIPForHostname resolves the system hostname to an IP address.
func (p *ProxmoxSetup) getIPForHostname() string {
if p.collector == nil {
return ""
}
hostname := p.hostname
if hostname == "" {
var err error
hostname, err = p.collector.Hostname()
if err != nil {
p.logger.Debug().Err(err).Msg("Could not resolve system hostname")
return ""
}
}
if hostname == "" {
return ""
}
ips, err := p.collector.LookupIP(hostname)
if err != nil {
p.logger.Debug().Err(err).Str("hostname", hostname).Msg("Could not resolve hostname IP")
return ""
}
for _, ip := range ips {
if ipv4 := ip.To4(); ipv4 != nil {
return ipv4.String()
}
}
return ""
}
func isLoopbackOrLinkLocalIP(raw string) bool {
ip := net.ParseIP(strings.TrimSpace(raw))
if ip == nil {
return false
}
return ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast()
}
// selectBestIP picks the most likely externally-reachable IP from a list.
// Prefers common LAN subnets (192.168.x.x, 10.x.x.x) and avoids internal
// cluster networks (like corosync's 172.20.x.x) and link-local addresses.
// If hostnameIP is provided and present in the list, it is given the highest priority.
func selectBestIP(ips []string, hostnameIP string) string {
type scoredIP struct {
ip string
score int
}
var candidates []scoredIP
for _, ip := range ips {
// Skip loopback and IPv6 link-local
if ip == "127.0.0.1" || ip == "::1" || strings.HasPrefix(ip, "fe80:") {
continue
}
// Skip IPv6 for simplicity (most Proxmox setups use IPv4)
if strings.Contains(ip, ":") {
continue
}
score := scoreIPv4(ip)
// If this IP matches the system hostname, give it a significant bonus.
// A bonus of +40 ensures that a 172.16.x.x (score 50) match (50+40=90)
// correctly loses to a 192.168.x.x (score 100) interface if both are present.
// However, it correctly breaks ties between equal-scored ranges.
if hostnameIP != "" && ip == hostnameIP {
score += 40
}
if score > 0 {
candidates = append(candidates, scoredIP{ip: ip, score: score})
}
}
if len(candidates) == 0 {
// Fallback: return first non-loopback IP if no good candidates
for _, ip := range ips {
if ip != "127.0.0.1" && ip != "::1" && !strings.HasPrefix(ip, "fe80:") {
return ip
}
}
return ""
}
// Return highest scored IP
best := candidates[0]
for _, c := range candidates[1:] {
if c.score > best.score {
best = c
}
}
return best.ip
}
// scoreIPv4 assigns a preference score to an IPv4 address.
// Higher score = more likely to be externally reachable.
func scoreIPv4(ip string) int {
parts := strings.Split(ip, ".")
if len(parts) != 4 {
return 0
}
// Parse first two octets
first := 0
second := 0
if _, err := fmt.Sscanf(parts[0], "%d", &first); err != nil {
return 0
}
if _, err := fmt.Sscanf(parts[1], "%d", &second); err != nil {
return 0
}
// Scoring logic:
// - 192.168.x.x: Very common home/office LAN, highest priority (score 100)
// - 10.0.x.x - 10.31.x.x: Common corporate LAN ranges (score 90)
// - 100.64.x.x: Tailscale / CGNAT (score 85)
// - 10.x.x.x (other): Less common, but still likely LAN (score 70)
// - 172.16-31.x.x: Private range, often used for internal clusters (score 50)
// Corosync often uses 172.20.x.x or similar for ring0/ring1
// - 169.254.x.x: Link-local, skip (score 0)
// - Other private/public: Unknown, low priority (score 30)
switch {
case first == 192 && second == 168:
return 100 // Most common home/office LAN
case first == 10 && second <= 31:
return 90 // Common corporate LAN
case first == 100 && (second >= 64 && second <= 127):
return 85 // Tailscale / CGNAT
case first == 10:
return 70 // Other 10.x.x.x
case first == 172 && second >= 16 && second <= 31:
// This is private 172.16-31.x.x range, often used for internal clusters
// Corosync commonly uses 172.20.x.x for cluster communication
return 50 // Lower priority - often internal cluster
case first == 169 && second == 254:
return 0 // Link-local, skip
default:
return 30 // Unknown/public - low priority
}
}
// clientError represents a non-retryable HTTP client error (4xx).
type clientError struct {
statusCode int
body string
}
func (e *clientError) Error() string {
return fmt.Sprintf("auto-register returned HTTP %d: %s", e.statusCode, e.body)
}
// permanentError wraps an error that should not be retried (e.g. malformed URL).
type permanentError struct {
err error
}
func (e *permanentError) Error() string { return e.err.Error() }
func (e *permanentError) Unwrap() error { return e.err }
func autoRegisterResponseMatchesCandidateHost(responseHost string, candidateHosts []string) bool {
trimmedResponseHost := strings.TrimSpace(responseHost)
if trimmedResponseHost == "" {
return false
}
for _, candidate := range candidateHosts {
if trimmedResponseHost == strings.TrimSpace(candidate) {
return true
}
}
return false
}
func autoRegisterAPITokenHeader(apiToken, setupToken string) string {
if strings.TrimSpace(setupToken) != "" {
return ""
}
return strings.TrimSpace(apiToken)
}
func (p *ProxmoxSetup) doRegisterRequest(ctx context.Context, body []byte, setupToken string, expectedType proxmoxProductType, expectedSource autoRegisterSource, expectedHosts []string, expectedTokenID, expectedTokenValue string) (autoRegisterResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.pulseURL+"/api/auto-register", bytes.NewReader(body))
if err != nil {
return autoRegisterResponse{}, &permanentError{fmt.Errorf("create request: %w", err)}
}
req.Header.Set("Content-Type", "application/json")
if token := autoRegisterAPITokenHeader(p.apiToken, setupToken); token != "" {
req.Header.Set("X-API-Token", token)
}
resp, err := p.httpClient.Do(req)
if err != nil {
return autoRegisterResponse{}, fmt.Errorf("request failed: %w", err)
}
defer func() {
if closeErr := resp.Body.Close(); closeErr != nil {
p.logger.Debug().Err(closeErr).Msg("Failed to close auto-register response body")
}
}()
if resp.StatusCode >= 400 {
bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
bodyStr := strings.TrimSpace(string(bodyBytes))
if resp.StatusCode < 500 {
return autoRegisterResponse{}, &clientError{statusCode: resp.StatusCode, body: bodyStr}
}
return autoRegisterResponse{}, fmt.Errorf("auto-register returned HTTP %d: %s", resp.StatusCode, bodyStr)
}
var parsed autoRegisterResponse
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return autoRegisterResponse{}, fmt.Errorf("decode auto-register response: %w", err)
}
if strings.TrimSpace(parsed.Status) != "success" {
return autoRegisterResponse{}, fmt.Errorf("auto-register response missing success status")
}
if strings.TrimSpace(parsed.Action) != "use_token" {
return autoRegisterResponse{}, fmt.Errorf("auto-register response missing use_token action")
}
if strings.TrimSpace(parsed.Type) != strings.TrimSpace(string(expectedType)) {
return autoRegisterResponse{}, fmt.Errorf("auto-register response type mismatch")
}
if strings.TrimSpace(parsed.Source) != strings.TrimSpace(string(expectedSource)) {
return autoRegisterResponse{}, fmt.Errorf("auto-register response source mismatch")
}
if !autoRegisterResponseMatchesCandidateHost(parsed.Host, expectedHosts) {
return autoRegisterResponse{}, fmt.Errorf("auto-register response host mismatch")
}
if strings.TrimSpace(parsed.NodeID) == "" {
return autoRegisterResponse{}, fmt.Errorf("auto-register response missing nodeId")
}
if strings.TrimSpace(parsed.NodeName) == "" {
return autoRegisterResponse{}, fmt.Errorf("auto-register response missing nodeName")
}
if strings.TrimSpace(parsed.NodeName) != strings.TrimSpace(parsed.NodeID) {
return autoRegisterResponse{}, fmt.Errorf("auto-register response node identity mismatch")
}
if strings.TrimSpace(parsed.TokenID) != strings.TrimSpace(expectedTokenID) {
return autoRegisterResponse{}, fmt.Errorf("auto-register response tokenId mismatch")
}
if strings.TrimSpace(expectedTokenValue) != "" && strings.TrimSpace(parsed.TokenValue) != strings.TrimSpace(expectedTokenValue) {
return autoRegisterResponse{}, fmt.Errorf("auto-register response tokenValue mismatch")
}
return parsed, nil
}
func (p *ProxmoxSetup) fetchSetupToken(ctx context.Context, ptype proxmoxProductType, hostURL string) (string, error) {
payload := setupScriptURLRequest{
Type: string(ptype),
Host: hostURL,
BackupPerms: false,
}
body, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("marshal setup token payload: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.pulseURL+"/api/setup-script-url", bytes.NewReader(body))
if err != nil {
return "", &permanentError{fmt.Errorf("create setup token request: %w", err)}
}
req.Header.Set("Content-Type", "application/json")
if token := strings.TrimSpace(p.apiToken); token != "" {
req.Header.Set("X-API-Token", token)
}
resp, err := p.httpClient.Do(req)
if err != nil {
return "", fmt.Errorf("setup token request failed: %w", err)
}
defer func() {
if closeErr := resp.Body.Close(); closeErr != nil {
p.logger.Debug().Err(closeErr).Msg("Failed to close setup token response body")
}
}()
if resp.StatusCode >= 400 {
bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
bodyStr := strings.TrimSpace(string(bodyBytes))
if resp.StatusCode < 500 {
return "", &clientError{statusCode: resp.StatusCode, body: bodyStr}
}
return "", fmt.Errorf("setup token endpoint returned HTTP %d: %s", resp.StatusCode, bodyStr)
}
var parsed setupScriptURLResponse
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
return "", fmt.Errorf("decode setup token response: %w", err)
}
if strings.TrimSpace(parsed.Type) != strings.TrimSpace(string(ptype)) {
return "", fmt.Errorf("setup token response type mismatch")
}
if strings.TrimSpace(parsed.Host) != strings.TrimSpace(hostURL) {
return "", fmt.Errorf("setup token response host mismatch")
}
expectedScriptURL := fmt.Sprintf(
"%s/api/setup-script?host=%s&pulse_url=%s&type=%s",
p.pulseURL,
url.QueryEscape(hostURL),
url.QueryEscape(p.pulseURL),
url.QueryEscape(string(ptype)),
)
if strings.TrimSpace(parsed.URL) != expectedScriptURL {
return "", fmt.Errorf("setup token response url mismatch")
}
expectedFileName := fmt.Sprintf("pulse-setup-%s.sh", strings.TrimSpace(string(ptype)))
if strings.TrimSpace(parsed.ScriptFileName) != expectedFileName {
return "", fmt.Errorf("setup token response scriptFileName mismatch")
}
if strings.TrimSpace(parsed.SetupToken) == "" {
return "", fmt.Errorf("setup token response missing setupToken")
}
expectedDownloadURL := expectedScriptURL
if parsedURL, err := url.Parse(expectedScriptURL); err == nil {
query := parsedURL.Query()
query.Set("setup_token", strings.TrimSpace(parsed.SetupToken))
parsedURL.RawQuery = query.Encode()
expectedDownloadURL = parsedURL.String()
}
if strings.TrimSpace(parsed.DownloadURL) != expectedDownloadURL && strings.TrimSpace(parsed.DownloadURL) != expectedScriptURL {
return "", fmt.Errorf("setup token response downloadURL mismatch")
}
if strings.TrimSpace(parsed.TokenHint) == "" {
return "", fmt.Errorf("setup token response missing tokenHint")
}
if strings.TrimSpace(parsed.TokenHint) == strings.TrimSpace(parsed.SetupToken) {
return "", fmt.Errorf("setup token response tokenHint must mask setupToken")
}
modern := strings.TrimSpace(parsed.DownloadURL) == expectedScriptURL
for _, command := range []string{parsed.Command, parsed.CommandWithEnv, parsed.CommandWithoutEnv} {
if modern != strings.Contains(command, "PULSE_SETUP_TOKEN_FILE=") {
return "", fmt.Errorf("setup-script command and download credential transports do not match")
}
}
if err := validateSetupScriptCommand(parsed.Command, expectedScriptURL, strings.TrimSpace(parsed.SetupToken), true); err != nil {
return "", fmt.Errorf("setup token response command invalid: %w", err)
}
if err := validateSetupScriptCommand(parsed.CommandWithEnv, expectedScriptURL, strings.TrimSpace(parsed.SetupToken), true); err != nil {
return "", fmt.Errorf("setup token response commandWithEnv invalid: %w", err)
}
if err := validateSetupScriptCommand(parsed.CommandWithoutEnv, expectedScriptURL, strings.TrimSpace(parsed.SetupToken), false); err != nil {
return "", fmt.Errorf("setup token response commandWithoutEnv invalid: %w", err)
}
if parsed.Expires <= time.Now().Unix() {
return "", fmt.Errorf("setup token response missing live expiry")
}
return strings.TrimSpace(parsed.SetupToken), nil
}
// registerWithPulse calls the auto-register endpoint to add the node.
func (p *ProxmoxSetup) registerWithPulse(ctx context.Context, ptype proxmoxProductType, hostURL, tokenID, tokenValue string) (autoRegisterResponse, error) {
candidateHosts := p.orderedCandidateHosts(ctx, ptype, hostURL)
backoffs := p.retryBackoffs
if backoffs == nil {
backoffs = []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, 60 * time.Second}
}
var lastErr error
for attempt := 0; attempt <= len(backoffs); attempt++ {
if attempt > 0 {
p.logger.Info().
Int("attempt", attempt+1).
Int("max_attempts", len(backoffs)+1).
Str("type", string(ptype)).
Msg("Retrying Proxmox auto-registration with Pulse")
select {
case <-ctx.Done():
return autoRegisterResponse{}, ctx.Err()
case <-time.After(backoffs[attempt-1]):
}
}
setupToken := ""
if strings.TrimSpace(p.apiToken) == "" {
var err error
setupToken, err = p.fetchSetupToken(ctx, ptype, hostURL)
if err != nil {
lastErr = fmt.Errorf("fetch setup token: %w", err)
var ce *clientError
var pe *permanentError
if errors.As(lastErr, &ce) || errors.As(lastErr, &pe) {
return autoRegisterResponse{}, lastErr
}
p.logger.Warn().Err(lastErr).
Int("attempt", attempt+1).
Int("max_attempts", len(backoffs)+1).
Str("type", string(ptype)).
Msg("Proxmox setup token fetch failed")
continue
}
}
payload := autoRegisterRequest{
Type: ptype,
Host: hostURL,
CandidateHosts: candidateHosts,
ServerName: p.hostname,
AuthToken: setupToken,
TokenID: tokenID,
TokenValue: tokenValue,
Source: autoRegisterSourceAgent,
}
body, err := json.Marshal(payload)
if err != nil {
return autoRegisterResponse{}, fmt.Errorf("marshal payload: %w", err)
}
parsed, err := p.doRegisterRequest(ctx, body, setupToken, ptype, autoRegisterSourceAgent, candidateHosts, tokenID, tokenValue)
if err == nil {
if attempt > 0 {
p.logger.Info().Int("attempt", attempt+1).Str("type", string(ptype)).Msg("Proxmox auto-registration succeeded after retry")
}
return parsed, nil
}
lastErr = err
// Don't retry client errors (4xx) or permanent errors (e.g. malformed URL).
var ce *clientError
var pe *permanentError
if errors.As(err, &ce) || errors.As(err, &pe) {
return autoRegisterResponse{}, err
}
p.logger.Warn().Err(err).
Int("attempt", attempt+1).
Int("max_attempts", len(backoffs)+1).
Str("type", string(ptype)).
Msg("Proxmox auto-registration attempt failed")
}
return autoRegisterResponse{}, fmt.Errorf("all %d registration attempts failed: %w", len(backoffs)+1, lastErr)
}
func isAlreadyExistsOutput(output string) bool {
text := strings.ToLower(strings.TrimSpace(output))
return strings.Contains(text, "already exists")
}
// isAlreadyRegistered checks if we've already done Proxmox setup.
// This uses the legacy single state file for backward compatibility.
func (p *ProxmoxSetup) isAlreadyRegistered() bool {
_, err := p.collector.Stat(p.legacyStateFilePath())
return err == nil
}
// isTypeRegistered checks if a specific Proxmox type has been registered.
func (p *ProxmoxSetup) isTypeRegistered(ptype proxmoxProductType) bool {
// Check per-type state file first (new behavior)
stateFile := p.stateFileForType(ptype)
if _, err := p.collector.Stat(stateFile); err == nil {
return true
}
// Check legacy state file for backward compat
if _, err := p.collector.Stat(p.legacyStateFilePath()); err == nil {
// Legacy file exists. The old detection logic was:
// 1. If pvesh exists → registered "pve"
// 2. Else if proxmox-backup-manager exists → registered "pbs"
//
// So we need to figure out what was likely registered.
// If PVE is currently installed, treat PVE as registered.
// If only PBS is installed (no PVE), treat PBS as registered.
types := p.detectProxmoxTypes()
if len(types) == 0 {
return false
}
// The old code registered the "first" type it found
// If PVE is installed, it was always detected first
if ptype == proxmoxProductPVE {
for _, t := range types {
if t == proxmoxProductPVE {
return true
}
}
}
// If PBS-only host (no PVE), PBS was what was registered
if ptype == proxmoxProductPBS {
hasPVE := false
hasPBS := false
for _, t := range types {
if t == proxmoxProductPVE {
hasPVE = true
}
if t == proxmoxProductPBS {
hasPBS = true
}
}
if hasPBS && !hasPVE {
return true
}
}
}
return false
}
// stateFileForType returns the state file path for a specific Proxmox type.
func (p *ProxmoxSetup) stateFileForType(ptype proxmoxProductType) string {
switch ptype {
case proxmoxProductPVE:
return p.pveStateFilePath()
case proxmoxProductPBS:
return p.pbsStateFilePath()
default:
return p.legacyStateFilePath()
}
}
// markAsRegistered creates a state file to indicate setup is complete.
func (p *ProxmoxSetup) markAsRegistered() {
if err := p.collector.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
p.logger.Warn().Err(err).Msg("Failed to create state directory")
return
}
if err := p.collector.Chmod(p.stateDir, proxmoxStateDirPerm); err != nil {
p.logger.Warn().Err(err).Msg("Failed to enforce state directory permissions")
}
legacyPath := p.legacyStateFilePath()
if err := p.collector.WriteFile(legacyPath, []byte(time.Now().Format(time.RFC3339)), proxmoxStateFilePerm); err != nil {
p.logger.Warn().Err(err).Msg("Failed to write state file")
return
}
if err := p.collector.Chmod(legacyPath, proxmoxStateFilePerm); err != nil {
p.logger.Warn().Err(err).Msg("Failed to enforce state file permissions")
}
}
// markTypeAsRegistered creates a state file for a specific Proxmox type.
func (p *ProxmoxSetup) markTypeAsRegistered(ptype proxmoxProductType) {
if err := p.collector.MkdirAll(p.stateDir, proxmoxStateDirPerm); err != nil {
p.logger.Warn().Err(err).Msg("Failed to create state directory")
return
}
if err := p.collector.Chmod(p.stateDir, proxmoxStateDirPerm); err != nil {
p.logger.Warn().Err(err).Msg("Failed to enforce state directory permissions")
}
stateFile := p.stateFileForType(ptype)
if err := p.collector.WriteFile(stateFile, []byte(time.Now().Format(time.RFC3339)), proxmoxStateFilePerm); err != nil {
p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to write type state file")
return
}
if err := p.collector.Chmod(stateFile, proxmoxStateFilePerm); err != nil {
p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to enforce type state file permissions")
}
}
// clearTypeRegisteredMarker removes the registration state file for a specific
// Proxmox type. Called before token rotation so that a failed Pulse update
// leaves the system in a retriable state rather than silently stale.
func (p *ProxmoxSetup) clearTypeRegisteredMarker(ptype proxmoxProductType) {
stateFile := p.stateFileForType(ptype)
if err := os.Remove(stateFile); err != nil && !os.IsNotExist(err) {
p.logger.Warn().Err(err).Str("type", string(ptype)).Msg("Failed to clear type registration marker")
}
}