Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

43.3 KB
Raw Blame
package monitoring

import (
	"context"
	stderrors "errors"
	"fmt"
	"math"
	"sort"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/logging"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
	"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
	"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
)

func matchesDatastoreExclude(datastoreName string, excludePatterns []string) bool {
	if len(excludePatterns) == 0 {
		return false
	}

	for _, pattern := range excludePatterns {
		pattern = strings.TrimSpace(pattern)
		if pattern == "" {
			continue
		}

		// Contains pattern: *substring*
		if strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*") && len(pattern) > 2 {
			substring := strings.ToLower(pattern[1 : len(pattern)-1])
			if strings.Contains(strings.ToLower(datastoreName), substring) {
				return true
			}
			continue
		}

		// Suffix pattern: *suffix
		if strings.HasPrefix(pattern, "*") && len(pattern) > 1 {
			suffix := strings.ToLower(pattern[1:])
			if strings.HasSuffix(strings.ToLower(datastoreName), suffix) {
				return true
			}
			continue
		}

		// Prefix pattern: prefix*
		if strings.HasSuffix(pattern, "*") && len(pattern) > 1 {
			prefix := strings.ToLower(pattern[:len(pattern)-1])
			if strings.HasPrefix(strings.ToLower(datastoreName), prefix) {
				return true
			}
			continue
		}

		// Exact match (case-insensitive)
		if strings.EqualFold(pattern, datastoreName) {
			return true
		}
	}

	return false
}

func pbsDatastoreStatus(ds pbs.Datastore) string {
	status := strings.TrimSpace(ds.Status)
	if status != "" {
		return status
	}
	if strings.TrimSpace(ds.Error) != "" {
		return "unavailable"
	}
	return "available"
}

func pbsJobHealthEvidenceFromFacts(facts []pbs.JobHealthEvidence, observedAt time.Time) []models.PBSJobHealthEvidence {
	if len(facts) == 0 {
		return []models.PBSJobHealthEvidence{}
	}
	out := make([]models.PBSJobHealthEvidence, 0, len(facts))
	for _, fact := range facts {
		evidence := models.PBSJobHealthEvidence{
			ID:             fact.ID,
			Family:         fact.Family,
			Store:          fact.Store,
			Remote:         fact.Remote,
			Namespace:      fact.Namespace,
			Schedule:       fact.Schedule,
			Comment:        fact.Comment,
			Enabled:        fact.Enabled,
			LastRunState:   fact.LastRunState,
			LastRunUPID:    fact.LastRunUPID,
			LastRunEndtime: fact.LastRunEndtime,
			NextRun:        fact.NextRun,
			UPID:           fact.UPID,
			WorkerType:     fact.WorkerType,
			WorkerID:       fact.WorkerID,
			TaskStatus:     fact.TaskStatus,
			TaskStartTime:  fact.TaskStartTime,
			TaskEndTime:    fact.TaskEndTime,
			Confidence:     fact.Confidence,
			EvidenceSource: fact.EvidenceSource,
			EvidenceScope:  fact.EvidenceScope,
			Error:          fact.Error,
		}
		evidence.Freshness = pbsJobHealthFreshness(fact, observedAt)
		evidence.Posture, evidence.PostureReason = pbsJobHealthPosture(fact, evidence.Freshness)
		out = append(out, evidence)
	}
	return out
}

func pbsJobHealthFreshness(fact pbs.JobHealthEvidence, observedAt time.Time) models.PBSJobHealthFreshness {
	freshness := models.PBSJobHealthFreshness{
		ObservedAt: observedAt.UTC(),
		State:      "unknown",
	}
	if fact.LastRunEndtime > 0 {
		freshness.LastRunEndTime = time.Unix(fact.LastRunEndtime, 0).UTC()
	} else if fact.TaskEndTime > 0 {
		freshness.LastRunEndTime = time.Unix(fact.TaskEndTime, 0).UTC()
	}
	if fact.NextRun > 0 {
		freshness.NextRun = time.Unix(fact.NextRun, 0).UTC()
	}
	if !freshness.LastRunEndTime.IsZero() {
		freshness.AgeSeconds = int64(observedAt.Sub(freshness.LastRunEndTime).Seconds())
		if freshness.AgeSeconds < 0 {
			freshness.AgeSeconds = 0
		}
		freshness.State = "observed"
	}
	if !freshness.NextRun.IsZero() {
		if observedAt.After(freshness.NextRun) && (freshness.LastRunEndTime.IsZero() || freshness.LastRunEndTime.Before(freshness.NextRun)) {
			freshness.State = "overdue"
		} else if freshness.State == "unknown" {
			freshness.State = "scheduled"
		}
	}
	return freshness
}

func pbsJobHealthPosture(fact pbs.JobHealthEvidence, freshness models.PBSJobHealthFreshness) (string, string) {
	if fact.Error != "" {
		return "unknown", strings.TrimSpace(fact.Confidence)
	}
	outcome := pbsTaskStatusOutcome(firstNonEmptyString(fact.TaskStatus, fact.LastRunState))
	// A failed last run outranks a missed next one: an overdue job whose last
	// run failed is still failed, not merely late.
	if outcome == "failed" {
		return "critical", "last-run-failed"
	}
	if freshness.State == "overdue" {
		return "warning", "job-overdue"
	}
	switch outcome {
	case "":
		return "unknown", "no-run-observed"
	case "ok":
		return "healthy", "last-run-success"
	case "running":
		return "healthy", "running"
	case "warning":
		return "warning", "last-run-warning"
	default:
		return "unknown", "last-run-state-unknown"
	}
}

// pbsTaskStatusOutcome reads a PBS task or job status the way Proxmox's own UI
// does (proxmox-widget-toolkit parse_task_status). PBS writes "OK",
// "WARNINGS: <n>", "unknown", or, for a failure, the error message itself with
// no prefix, such as "remote 'offsite' does not exist" or "broken pipe".
// Matching words like "error" missed failures that lack them, and matching
// "ok" called "broken pipe" a success.
func pbsTaskStatusOutcome(status string) string {
	lower := strings.ToLower(strings.TrimSpace(status))
	switch {
	case lower == "":
		return ""
	case lower == "ok":
		return "ok"
	case strings.HasPrefix(lower, "warnings:"):
		return "warning"
	case lower == "unknown":
		return "unknown"
	case lower == "running":
		return "running"
	default:
		return "failed"
	}
}

// pbsJobKey identifies one job across polls. PBS job ids are only unique
// within a family, so a sync and a verify job may share one.
func pbsJobKey(entry models.PBSJobHealthEvidence) string {
	return strings.ToLower(strings.TrimSpace(entry.Family)) + "\x00" + strings.TrimSpace(entry.ID)
}

// carryPBSJobFailures keeps a job's last completed failure known while PBS
// cannot report a newer outcome. PBS's /admin job status drops the previous
// result while the next run is in progress, an overdue or unreadable job
// reports none, and a failed read lists no jobs at all; treating any of those
// as recovery would clear a failed-job alert and raise it again on the next
// poll. A carried failure clears only when the job's newest run is observed to
// succeed or warn, or when a complete read no longer lists the job (removed),
// or lists it disabled.
func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFailed bool) []models.PBSJobHealthEvidence {
	failed := make(map[string]models.PBSJobHealthEvidence)
	for _, entry := range previous {
		if entry.Posture == "critical" && entry.PostureReason == "last-run-failed" && strings.TrimSpace(entry.ID) != "" {
			failed[pbsJobKey(entry)] = entry
		}
	}
	if len(failed) == 0 {
		return current
	}

	unreadFamilies := make(map[string]bool)
	for _, entry := range current {
		family := strings.ToLower(strings.TrimSpace(entry.Family))
		if strings.TrimSpace(entry.Error) != "" && entry.ID == family+":partial" &&
			entry.EvidenceSource == pbs.JobEvidenceSourcePBSPartialRead {
			unreadFamilies[family] = true
		}
	}

	out := make([]models.PBSJobHealthEvidence, 0, len(current)+len(failed))
	seen := make(map[string]bool, len(current))
	for _, entry := range current {
		key := pbsJobKey(entry)
		seen[key] = true
		if prior, ok := failed[key]; ok && entry.Enabled && !pbsJobReportsOutcome(entry) {
			out = append(out, prior)
			continue
		}
		out = append(out, entry)
	}
	for key, prior := range failed {
		if seen[key] {
			continue
		}
		if readFailed || unreadFamilies[strings.ToLower(strings.TrimSpace(prior.Family))] {
			out = append(out, prior)
		}
	}
	return out
}

// carryPBSJobFailuresForInstance carries failures from the instance's last
// stored results, or, on the first poll after a restart when state holds none,
// from the persisted failure record of the same server, and records what is
// still failing. Records of connections no longer configured are dropped.
func (m *Monitor) carryPBSJobFailuresForInstance(instanceID, host string, current []models.PBSJobHealthEvidence, readFailed bool) []models.PBSJobHealthEvidence {
	previous := m.state.PBSJobHealthEvidence(instanceID)
	if len(previous) == 0 {
		previous = m.pbsJobFailureStore.Get(instanceID, host)
	}
	carried := carryPBSJobFailures(previous, current, readFailed)
	if err := m.pbsJobFailureStore.Set(instanceID, host, carried); err != nil {
		log.Warn().Err(err).Str("instance", instanceID).Msg("Failed to record PBS job failures")
	}
	if m.config != nil {
		configured := make([]string, 0, len(m.config.PBSInstances))
		for _, cfg := range m.config.PBSInstances {
			configured = append(configured, PBSMonitorResourceID(cfg.Name))
		}
		if err := m.pbsJobFailureStore.Retain(configured); err != nil {
			log.Warn().Err(err).Msg("Failed to prune PBS job failure record")
		}
	}
	return carried
}

// pbsJobReportsOutcome says whether this poll observed how the job's newest
// run ended: success, warning or failure, rather than running or an
// unreadable state. It reads the run's own status, not the posture, so a
// successful run first seen after its next schedule passed still counts.
func pbsJobReportsOutcome(entry models.PBSJobHealthEvidence) bool {
	if strings.TrimSpace(entry.Error) != "" {
		return false
	}
	switch pbsTaskStatusOutcome(firstNonEmptyString(entry.TaskStatus, entry.LastRunState)) {
	case "ok", "warning", "failed":
		return true
	default:
		return false
	}
}

// pollPBSInstance polls a single PBS instance
func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, client *pbs.Client) {
	start := time.Now()
	debugEnabled := logging.IsLevelEnabled(zerolog.DebugLevel)
	var pollErr error
	var pbsInst models.PBSInstance
	publishResult := false
	if m.pollMetrics != nil {
		m.pollMetrics.IncInFlight("pbs")
	}
	defer func() {
		if recovered := recover(); recovered != nil {
			pollErr = fmt.Errorf("panic while polling PBS instance %q: %v", instanceName, recovered)
			log.Error().
				Str("goroutine", fmt.Sprintf("pollPBSInstance-%s", instanceName)).
				Interface("panic", recovered).
				Stack().
				Msg("Recovered from panic in monitoring goroutine")
		}

		m.recordTaskResult(InstanceTypePBS, instanceName, pollErr)
		if m.stalenessTracker != nil {
			if pollErr == nil {
				m.stalenessTracker.UpdateSuccess(InstanceTypePBS, instanceName, nil)
			} else {
				m.stalenessTracker.UpdateError(InstanceTypePBS, instanceName)
			}
		}
		if publishResult {
			m.publishPBSConnectionOutcome(pbsInst, pollErr)
		}
		if m.pollMetrics != nil {
			m.pollMetrics.DecInFlight("pbs")
			m.pollMetrics.RecordResult(PollResult{
				InstanceName: instanceName,
				InstanceType: "pbs",
				Success:      pollErr == nil,
				Error:        pollErr,
				StartTime:    start,
				EndTime:      time.Now(),
			})
		}
	}()

	// Get instance config
	var instanceCfg *config.PBSInstance
	for _, cfg := range m.config.PBSInstances {
		if cfg.Name == instanceName {
			instanceCfg = &cfg
			if debugEnabled {
				log.Debug().
					Str("instance", instanceName).
					Bool("monitorDatastores", cfg.MonitorDatastores).
					Msg("Found PBS instance config")
			}
			break
		}
	}
	if instanceCfg == nil {
		pollErr = fmt.Errorf("PBS instance config %q not found", instanceName)
		log.Error().Str("instance", instanceName).Msg("PBS instance config not found")
		return
	}
	if instanceCfg.Disabled {
		if debugEnabled {
			log.Debug().Str("instance", instanceName).Msg("Skipping PBS poll: instance is paused")
		}
		return
	}
	includeDatastore := func(datastoreName string) bool {
		return !matchesDatastoreExclude(datastoreName, instanceCfg.ExcludeDatastores)
	}

	// Initialize PBS instance with default values
	pbsInst = models.PBSInstance{
		NodeMetricsUnavailable: true,
		ID:                     PBSMonitorResourceID(instanceName),
		Name:                   instanceName,
		Host:                   instanceCfg.Host,
		GuestURL:               instanceCfg.GuestURL,
		Status:                 "offline",
		Version:                "unknown",
		ConnectionHealth:       "unhealthy",
		LastSeen:               time.Now(),
	}
	publishResult = true

	// Check if context is cancelled after the configured instance projection is
	// available, so cancellation still publishes the same failed poll outcome.
	select {
	case <-ctx.Done():
		pollErr = ctx.Err()
		if debugEnabled {
			log.Debug().Str("instance", instanceName).Msg("polling cancelled")
		}
		return
	default:
	}

	if debugEnabled {
		log.Debug().Str("instance", instanceName).Msg("polling PBS instance")
	}

	// Try to get version first
	version, versionErr := client.GetVersion(ctx)
	if versionErr == nil {
		pbsInst.Status = "online"
		pbsInst.Version = version.Version
		pbsInst.ConnectionHealth = "healthy"
		m.resetAuthFailures(instanceName, "pbs")

		if debugEnabled {
			log.Debug().
				Str("instance", instanceName).
				Str("version", version.Version).
				Bool("monitorDatastores", instanceCfg.MonitorDatastores).
				Msg("PBS version retrieved successfully")
		}
	} else {
		if debugEnabled {
			log.Debug().Err(versionErr).Str("instance", instanceName).Msg("failed to get PBS version, trying fallback")
		}

		// Use parent context for proper cancellation chain
		ctx2, cancel2 := context.WithTimeout(ctx, 10*time.Second)
		defer cancel2()
		_, datastoreErr := client.GetDatastoresFiltered(ctx2, includeDatastore)
		if datastoreErr == nil {
			pbsInst.Status = "online"
			pbsInst.Version = "connected"
			pbsInst.ConnectionHealth = "healthy"
			m.resetAuthFailures(instanceName, "pbs")

			log.Info().
				Str("instance", instanceName).
				Msg("PBS connected (version unavailable but datastores accessible)")
		} else {
			pbsInst.Status = "offline"
			pbsInst.ConnectionHealth = "error"
			connectionErr := versionErr
			if errors.IsAuthError(datastoreErr) {
				connectionErr = datastoreErr
			}
			monErr := errors.WrapConnectionError("get_pbs_version", instanceName, connectionErr)
			pollErr = monErr
			log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to connect to PBS")

			if errors.IsAuthError(versionErr) || errors.IsAuthError(datastoreErr) {
				m.recordAuthFailure(instanceName, "pbs")
			}
			// The connectivity fallback also failed datastore discovery. Qualify
			// retained backup evidence through the ordinary serialized admission.
			if !stderrors.Is(datastoreErr, context.Canceled) {
				m.pollPBSBackupsAsync(instanceName, client, instanceCfg, nil, nil, datastoreErr)
			}
			return
		}
	}

	// The PBS-reported node hostname is machine-identity evidence: connected
	// systems grouping merges the host agent running on this machine into
	// this PBS connection through it, even when the configured connection
	// address is an IP or DNS alias the agent never reports.
	if nodeName, nodeNameErr := client.GetNodeName(ctx); nodeNameErr == nil {
		pbsInst.NodeName = strings.TrimSpace(nodeName)
	} else if !stderrors.Is(nodeNameErr, pbs.ErrNodeNameUnavailableForAuth) && debugEnabled {
		log.Debug().Err(nodeNameErr).Str("instance", instanceName).Msg("could not get PBS node name")
	}

	// Get node status (CPU, memory, etc.)
	nodeStatus, err := client.GetNodeStatus(ctx)
	if err != nil {
		if debugEnabled {
			log.Debug().Err(err).Str("instance", instanceName).Msg("could not get PBS node status (may need Sys.Audit permission)")
		}
	} else if nodeStatus != nil {
		pbsInst.NodeMetricsUnavailable = false
		pbsInst.CPU = nodeStatus.CPU
		if nodeStatus.Memory.Total > 0 {
			pbsInst.Memory = float64(nodeStatus.Memory.Used) / float64(nodeStatus.Memory.Total) * 100
			pbsInst.MemoryUsed = nodeStatus.Memory.Used
			pbsInst.MemoryTotal = nodeStatus.Memory.Total
		}
		pbsInst.Uptime = nodeStatus.Uptime
		m.recordPBSNodeHistory(pbsInst.ID, nodeStatus, time.Now())

		log.Debug().
			Str("instance", instanceName).
			Float64("cpu", pbsInst.CPU).
			Float64("memory", pbsInst.Memory).
			Int64("uptime", pbsInst.Uptime).
			Msg("PBS node status retrieved")
	}

	// Keep discovery quality with the immutable input to the background backup
	// poll. Public datastore presentation is not an authoritative scope receipt.
	namespaceErrors := make(map[string]error)
	var datastoreDiscoveryErr error
	// Poll datastores if enabled
	if instanceCfg.MonitorDatastores {
		datastores, err := client.GetDatastoresFiltered(ctx, includeDatastore)
		if err != nil {
			datastoreDiscoveryErr = err
			monErr := errors.WrapAPIError("get_datastores", instanceName, err, 0)
			log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get datastores")
		} else {
			log.Info().
				Str("instance", instanceName).
				Int("count", len(datastores)).
				Msg("Got PBS datastores")

			for _, ds := range datastores {
				// Skip excluded datastores (for removable/unmounted datastores)
				if matchesDatastoreExclude(ds.Store, instanceCfg.ExcludeDatastores) {
					log.Debug().
						Str("instance", instanceName).
						Str("datastore", ds.Store).
						Msg("Skipping excluded datastore")
					continue
				}
				total := ds.Total
				if total == 0 && ds.TotalSpace > 0 {
					total = ds.TotalSpace
				}
				used := ds.Used
				if used == 0 && ds.UsedSpace > 0 {
					used = ds.UsedSpace
				}
				avail := ds.Avail
				if avail == 0 && ds.AvailSpace > 0 {
					avail = ds.AvailSpace
				}
				if total == 0 && used > 0 && avail > 0 {
					total = used + avail
				}

				log.Debug().
					Str("store", ds.Store).
					Int64("total", total).
					Int64("used", used).
					Int64("avail", avail).
					Int64("orig_total", ds.Total).
					Int64("orig_total_space", ds.TotalSpace).
					Msg("PBS datastore details")

				modelDS := models.PBSDatastore{
					Name:                ds.Store,
					Total:               total,
					Used:                used,
					Free:                avail,
					Usage:               safePercentage(float64(used), float64(total)),
					Status:              pbsDatastoreStatus(ds),
					Error:               ds.Error,
					DeduplicationFactor: ds.DeduplicationFactor,
				}

				namespaces, err := client.ListNamespaces(ctx, ds.Store, "", 0)
				if err != nil {
					namespaceErrors[ds.Store] = err
					log.Warn().Err(err).
						Str("instance", instanceName).
						Str("datastore", ds.Store).
						Msg("Failed to list namespaces")
				} else {
					for _, ns := range namespaces {
						nsPath := ns.NS
						if nsPath == "" {
							nsPath = ns.Path
						}
						if nsPath == "" {
							nsPath = ns.Name
						}

						modelNS := models.PBSNamespace{
							Path:   nsPath,
							Parent: ns.Parent,
							Depth:  strings.Count(nsPath, "/"),
						}
						modelDS.Namespaces = append(modelDS.Namespaces, modelNS)
					}

					hasRoot := false
					for _, ns := range modelDS.Namespaces {
						if ns.Path == "" {
							hasRoot = true
							break
						}
					}
					if !hasRoot {
						modelDS.Namespaces = append([]models.PBSNamespace{{Path: "", Depth: 0}}, modelDS.Namespaces...)
					}
				}

				pbsInst.Datastores = append(pbsInst.Datastores, modelDS)
			}
		}
	}

	if instanceCfg.MonitorBackups || instanceCfg.MonitorSyncJobs || instanceCfg.MonitorVerifyJobs || instanceCfg.MonitorPruneJobs || instanceCfg.MonitorGarbageJobs {
		datastoreNames := make([]string, 0, len(pbsInst.Datastores))
		for _, ds := range pbsInst.Datastores {
			datastoreNames = append(datastoreNames, ds.Name)
		}
		jobFacts, err := client.GetJobHealthEvidence(ctx, datastoreNames, pbs.JobHealthOptions{
			MonitorBackups:     instanceCfg.MonitorBackups,
			MonitorSyncJobs:    instanceCfg.MonitorSyncJobs,
			MonitorVerifyJobs:  instanceCfg.MonitorVerifyJobs,
			MonitorPruneJobs:   instanceCfg.MonitorPruneJobs,
			MonitorGarbageJobs: instanceCfg.MonitorGarbageJobs,
		})
		if err != nil {
			log.Warn().Err(err).
				Str("instance", instanceName).
				Msg("Failed to collect PBS job health evidence")
		}
		pbsInst.JobHealthEvidence = m.carryPBSJobFailuresForInstance(
			pbsInst.ID,
			pbsInst.Host,
			pbsJobHealthEvidenceFromFacts(jobFacts, time.Now()),
			err != nil,
		)
	}

	// Convert PBS datastores to Storage entries for unified storage view
	if len(pbsInst.Datastores) > 0 && instanceCfg.MonitorDatastores {
		var pbsStorages []models.Storage
		for _, ds := range pbsInst.Datastores {
			// Create a storage entry for this PBS datastore
			storageID := fmt.Sprintf("pbs-%s-%s", instanceName, ds.Name)
			pbsStorage := models.Storage{
				ID: storageID,
				// The unified registry and thresholds UI key this datastore by
				// its canonical "<instance-id>/<name>" ID; carrying it as an
				// alias lets alert override lookups accept either format (#1591).
				AliasIDs: []string{pbsInst.ID + "/" + ds.Name},
				Name:     ds.Name,
				Node:     instanceName, // Use PBS instance name as "node"
				Instance: PBSMonitorResourceID(instanceName),
				Type:     "pbs",
				Status:   ds.Status,
				Total:    ds.Total,
				Used:     ds.Used,
				Free:     ds.Free,
				Usage:    ds.Usage,
				Content:  "backup", // PBS datastores are for backups
				Shared:   true,     // PBS datastores are typically shared/network storage
				Enabled:  true,
				Active:   pbsInst.Status == "online",
				LastSeen: pbsInst.LastSeen,
			}
			pbsStorages = append(pbsStorages, pbsStorage)
		}
		m.state.UpdateStorageForInstance("pbs-"+instanceName, pbsStorages)
		// PBS storage is not evaluated by the unified metric path (which
		// handles TrueNAS and VMware storage). Evaluate fresh poll observations
		// here, as the PVE storage poller does, rather than relying on mock ticks.
		if m.alertManager != nil {
			for _, storage := range pbsStorages {
				m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now()))
			}
		}
		log.Debug().
			Str("instance", instanceName).
			Int("storageEntries", len(pbsStorages)).
			Msg("Added PBS datastores to unified storage view")
	}

	// A successful empty scope and a failed discovery both need an accepted
	// backup-poll result; neither may silently keep the last good protection.
	if !stderrors.Is(datastoreDiscoveryErr, context.Canceled) {
		m.pollPBSBackupsAsync(instanceName, client, instanceCfg, pbsInst.Datastores, namespaceErrors, datastoreDiscoveryErr)
	}
}

// pollPBSBackupsAsync admits all backup-scope outcomes through one worker.
// Failed discovery never becomes an empty enumeration, and a skipped/disabled
// discovery is not absence evidence. Keeping failure/empty publication inside
// this admission prevents it from racing an older in-flight backup worker.
func (m *Monitor) pollPBSBackupsAsync(instanceName string, client *pbs.Client, instanceCfg *config.PBSInstance, datastores []models.PBSDatastore, namespaceErrors map[string]error, discoveryErr error) {
	if !instanceCfg.MonitorBackups || !instanceCfg.MonitorDatastores || !m.backupPollingEnabledSetting() {
		return
	}
	now := time.Now()
	m.mu.RLock()
	lastPoll := m.lastPBSBackupPoll[instanceName]
	m.mu.RUnlock()
	shouldPoll, reason, newLast := m.shouldRunBackupPoll(lastPoll, now)
	if !shouldPoll {
		log.Debug().Str("instance", instanceName).Str("reason", reason).Msg("Skipping PBS backup polling this cycle")
		return
	}

	datastoreSnapshot := append([]models.PBSDatastore(nil), datastores...)
	for i := range datastoreSnapshot {
		datastoreSnapshot[i].Namespaces = append([]models.PBSNamespace(nil), datastores[i].Namespaces...)
	}
	namespaceErrorSnapshot := make(map[string]error, len(namespaceErrors))
	for name, err := range namespaceErrors {
		namespaceErrorSnapshot[name] = err
	}
	owner := m.beginPBSBackupPoll(instanceName, client, lastPoll, newLast)
	if owner == nil {
		log.Debug().Str("instance", instanceName).Msg("PBS backup polling already in progress")
		return
	}

	go func() {
		defer recoverFromPanic(fmt.Sprintf("pollPBSBackups-%s", instanceName))
		defer m.finishPBSBackupPoll(instanceName, owner)

		// The main polling loop cancels its per-cycle context after returning.
		// Backup work keeps the existing independent, runtime-scoped deadline.
		parentCtx := context.WithValue(owner.ctx, pbsBackupPollContextKey{}, owner)
		backupCtx, cancel := context.WithTimeout(parentCtx, 5*time.Minute)
		defer cancel()
		if backupCtx.Err() != nil {
			return
		}
		if discoveryErr != nil {
			unlock, current := m.beginPBSBackupPublication(backupCtx, instanceName)
			if !current {
				return
			}
			defer unlock()
			// One unreadable discovery scope, not a guessed datastore count.
			// Only actual authentication/authorization status establishes denial.
			denied := 0
			if status, ok := pbs.HTTPStatus(discoveryErr); ok && (status == 401 || status == 403) {
				denied = 1
			}
			observation, err := buildPBSProtectionProviderObservation(instanceName, 1, 0, 1, denied, time.Now().UTC())
			if err != nil {
				log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to build PBS discovery failure observation")
				return
			}
			m.ingestProtectionProviderObservationsAsync([]recovery.ProtectionProviderObservation{observation})
		} else {
			// The explicit empty list is authoritative and reconciles only this
			// connection; the ordinary poller also drops its live rows/cache.
			m.pollPBSBackups(backupCtx, instanceName, client, datastoreSnapshot, namespaceErrorSnapshot)
		}
		log.Info().Str("instance", instanceName).Dur("duration", time.Since(now)).Msg("Completed background PBS backup polling")
	}()
}

// recordPBSNodeHistory writes the API-observed CPU and memory measurements to
// the PBS service's existing metrics target (agent:<PBS source ID>). When no
// host agent can be corroborated, the Backups drawer still has useful history
// instead of querying an unwritten key. Host-agent history remains separate and
// takes precedence in the drawer when a unique host is linked. A failed or
// incomplete node-status response must never produce an apparent zero sample.
func (m *Monitor) recordPBSNodeHistory(serviceID string, status *pbs.NodeStatus, observedAt time.Time) {
	serviceID = strings.TrimSpace(serviceID)
	if m == nil || status == nil || serviceID == "" || status.Memory.Total <= 0 ||
		status.Memory.Used < 0 || status.CPU < 0 || math.IsNaN(status.CPU) || math.IsInf(status.CPU, 0) {
		return
	}

	cpuPercent := status.CPU
	if cpuPercent <= 1 {
		cpuPercent *= 100
	}
	memoryPercent := float64(status.Memory.Used) / float64(status.Memory.Total) * 100
	if m.metricsHistory != nil {
		key := "agent:" + serviceID
		m.metricsHistory.AddGuestMetric(key, "cpu", cpuPercent, observedAt)
		m.metricsHistory.AddGuestMetric(key, "memory", memoryPercent, observedAt)
	}
	if m.metricsStore != nil {
		m.metricsStore.WriteBatchBounded([]metrics.WriteMetric{
			{ResourceType: "agent", ResourceID: serviceID, MetricType: "cpu", Value: cpuPercent, Timestamp: observedAt, Tier: metrics.TierRaw},
			{ResourceType: "agent", ResourceID: serviceID, MetricType: "memory", Value: memoryPercent, Timestamp: observedAt, Tier: metrics.TierRaw},
		})
	}
}

// publishPBSConnectionOutcome projects the final connectivity result into every
// legacy PBS health consumer. Inventory collection may be partial, but the
// connection map, dashboard resource, and legacy alert evaluator must all use
// the same poll error that is recorded in the scheduler ledger.
func (m *Monitor) publishPBSConnectionOutcome(instance models.PBSInstance, pollErr error) {
	// A poll that failed before reading jobs knows nothing new about them;
	// keep the last results so a failed job is still carried when PBS returns.
	if pollErr != nil && len(instance.JobHealthEvidence) == 0 {
		instance.JobHealthEvidence = m.state.PBSJobHealthEvidence(instance.ID)
	}
	connected := pollErr == nil
	if connected {
		instance.Status = "online"
		instance.ConnectionHealth = "healthy"
	} else {
		instance.Status = "offline"
		instance.ConnectionHealth = "error"
	}

	m.setProviderConnectionHealth(InstanceTypePBS, instance.Name, connected)
	m.state.UpdatePBSInstance(instance)
	log.Info().
		Str("instance", instance.Name).
		Str("id", instance.ID).
		Bool("connected", connected).
		Int("datastores", len(instance.Datastores)).
		Msg("PBS instance updated in state")

	if m.alertManager != nil {
		m.alertManager.CheckPBS(instance)
	}
}

// pmgPollClient is the existing read-only PMG collection boundary.
type pmgPollClient interface {
	GetVersion(context.Context) (*pmg.VersionInfo, error)
	GetClusterStatus(context.Context, bool) ([]pmg.ClusterStatusEntry, error)
	GetQueueStatus(context.Context, string) (*pmg.QueueStatusEntry, error)
	ListBackups(context.Context, string) ([]pmg.BackupEntry, error)
	GetMailStatistics(context.Context, string) (*pmg.MailStatistics, error)
	GetMailCount(context.Context, int) ([]pmg.MailCountEntry, error)
	GetSpamScores(context.Context) ([]pmg.SpamScore, error)
	GetQuarantineStatus(context.Context, string) (*pmg.QuarantineStatus, error)
	ListRelayDomains(context.Context) ([]pmg.RelayDomainEntry, error)
	GetDomainStatistics(context.Context, int64, int64) ([]pmg.DomainStatisticsEntry, error)
}

var _ pmgPollClient = (*pmg.Client)(nil)

// pollPMGInstance polls a single Proxmox Mail Gateway instance
func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, client pmgPollClient) {
	defer recoverFromPanic(fmt.Sprintf("pollPMGInstance-%s", instanceName))

	start := time.Now()
	debugEnabled := logging.IsLevelEnabled(zerolog.DebugLevel)
	// pollErr is a connection, version or whole-poll failure and drives
	// scheduler backoff. readErr collects downstream reads that failed after
	// the connection succeeded; see recordTaskOutcome.
	var pollErr, readErr error
	if m.pollMetrics != nil {
		m.pollMetrics.IncInFlight("pmg")
		defer m.pollMetrics.DecInFlight("pmg")
		defer func() {
			m.pollMetrics.RecordResult(PollResult{
				InstanceName: instanceName,
				InstanceType: "pmg",
				Success:      pollErr == nil,
				Error:        pollErr,
				StartTime:    start,
				EndTime:      time.Now(),
			})
		}()
	}
	if m.stalenessTracker != nil {
		defer func() {
			if pollErr == nil {
				m.stalenessTracker.UpdateSuccess(InstanceTypePMG, instanceName, nil)
			} else {
				m.stalenessTracker.UpdateError(InstanceTypePMG, instanceName)
			}
		}()
	}
	defer func() {
		m.recordTaskOutcome(InstanceTypePMG, instanceName, pollErr, readErr)
	}()

	select {
	case <-ctx.Done():
		pollErr = ctx.Err()
		if debugEnabled {
			log.Debug().Str("instance", instanceName).Msg("PMG polling cancelled by context")
		}
		return
	default:
	}

	if debugEnabled {
		log.Debug().Str("instance", instanceName).Msg("polling PMG instance")
	}

	// Each poll uses one saved scope snapshot. A later edit/pause applies to
	// the next poll and does not revoke requests already in flight.
	var instanceCfg *config.PMGInstance
	config.Mu.RLock()
	if m.config != nil {
		for _, cfg := range m.config.PMGInstances {
			if cfg.Name == instanceName {
				snapshot := cfg
				instanceCfg = &snapshot
				break
			}
		}
	}
	config.Mu.RUnlock()

	if instanceCfg == nil {
		log.Error().Str("instance", instanceName).Msg("PMG instance config not found")
		pollErr = fmt.Errorf("pmg instance config not found for %s", instanceName)
		return
	}
	if instanceCfg.Disabled {
		if debugEnabled {
			log.Debug().Str("instance", instanceName).Msg("Skipping PMG poll: instance is paused")
		}
		return
	}

	now := time.Now()
	pmgInst := models.PMGInstance{
		ID:               PMGMonitorResourceID(instanceName),
		Name:             instanceName,
		Host:             instanceCfg.Host,
		GuestURL:         instanceCfg.GuestURL,
		Status:           "offline",
		ConnectionHealth: "unhealthy",
		LastSeen:         now,
		LastUpdated:      now,
	}

	version, err := client.GetVersion(ctx)
	if err != nil {
		monErr := errors.WrapConnectionError("pmg_get_version", instanceName, err)
		pollErr = monErr
		log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to connect to PMG instance")
		m.setProviderConnectionHealth(InstanceTypePMG, instanceName, false)
		m.state.UpdatePMGInstance(pmgInst)

		// Check PMG offline status against alert thresholds
		if m.alertManager != nil {
			m.alertManager.CheckPMG(pmgInst)
		}

		if errors.IsAuthError(err) {
			m.recordAuthFailure(instanceName, "pmg")
		}
		return
	}

	pmgInst.Status = "online"
	pmgInst.ConnectionHealth = "healthy"
	if version != nil {
		pmgInst.Version = strings.TrimSpace(version.Version)
	}
	m.setProviderConnectionHealth(InstanceTypePMG, instanceName, true)
	m.resetAuthFailures(instanceName, "pmg")

	// A successful version read establishes connection health, not complete
	// downstream collection. Preserve missing readings and report failed reads
	// without backing off the readings that succeeded.
	recordReadError := func(operation string, err error) {
		if err != nil {
			readErr = stderrors.Join(readErr, errors.WrapAPIError(operation, instanceName, err, 0))
		}
	}

	cluster, clusterErr := client.GetClusterStatus(ctx, true)
	recordReadError("pmg_cluster_status", clusterErr)
	if clusterErr != nil {
		if debugEnabled {
			log.Debug().Err(clusterErr).Str("instance", instanceName).Msg("failed to retrieve PMG cluster status")
		}
	}

	backupNodes := make(map[string]struct{})

	if len(cluster) > 0 {
		nodes := make([]models.PMGNodeStatus, 0, len(cluster))
		for _, entry := range cluster {
			status := strings.ToLower(strings.TrimSpace(entry.Type))
			if status == "" {
				status = "online"
			}
			node := models.PMGNodeStatus{
				Name:   entry.Name,
				Status: status,
				Role:   entry.Type,
			}

			backupNodes[entry.Name] = struct{}{}

			if instanceCfg.MonitorQueues {
				// Fetch queue status for this node
				if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil {
					recordReadError("pmg_queue_status", qErr)
					if debugEnabled {
						log.Debug().Err(qErr).
							Str("instance", instanceName).
							Str("node", entry.Name).
							Msg("Failed to fetch PMG queue status")
					}
				} else if queueData != nil {
					total := queueData.Active.Int64() + queueData.Deferred.Int64() + queueData.Hold.Int64() + queueData.Incoming.Int64()
					node.QueueStatus = &models.PMGQueueStatus{
						Active:    queueData.Active.Int(),
						Deferred:  queueData.Deferred.Int(),
						Hold:      queueData.Hold.Int(),
						Incoming:  queueData.Incoming.Int(),
						Total:     int(total),
						OldestAge: queueData.OldestAge.Int64(),
						UpdatedAt: time.Now(),
					}
				}
			}

			nodes = append(nodes, node)
		}
		pmgInst.Nodes = nodes
	}

	// Fall back to the connection name only when PMG answered with no cluster
	// entries. After a failed cluster read the name is a display label, not a
	// node: requesting backups with it fails with a misleading hostname-lookup
	// error on PMG (#2960) and cannot return backups.
	if len(backupNodes) == 0 && clusterErr == nil {
		trimmed := strings.TrimSpace(instanceName)
		if trimmed != "" {
			backupNodes[trimmed] = struct{}{}
		}
	}

	pmgBackups := make([]models.PMGBackup, 0)
	seenBackupIDs := make(map[string]struct{})
	readBackupNodes := make(map[string]struct{}, len(backupNodes))

	for nodeName := range backupNodes {
		if ctx.Err() != nil {
			break
		}

		backups, backupErr := client.ListBackups(ctx, nodeName)
		if backupErr != nil {
			recordReadError("pmg_backups", backupErr)
			if debugEnabled {
				log.Debug().Err(backupErr).
					Str("instance", instanceName).
					Str("node", nodeName).
					Msg("Failed to list PMG configuration backups")
			}
			continue
		}
		readBackupNodes[nodeName] = struct{}{}

		for _, b := range backups {
			timestamp := b.Timestamp.Int64()
			backupTime := time.Unix(timestamp, 0)
			backupID := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, timestamp)
			if _, exists := seenBackupIDs[backupID]; exists {
				continue
			}
			seenBackupIDs[backupID] = struct{}{}
			pmgBackups = append(pmgBackups, models.PMGBackup{
				ID:         backupID,
				Instance:   instanceName,
				Node:       nodeName,
				Filename:   b.Filename,
				BackupTime: backupTime,
				Size:       b.Size.Int64(),
			})
		}
	}

	// Only a node whose backup list was read is authoritative this cycle. A
	// failed cluster read, a failed per-node read or a cancelled poll observed
	// nothing, so those nodes keep their last-known backups instead of being
	// published as having none; the failed read is already in readErr.
	if clusterErr != nil || len(readBackupNodes) < len(backupNodes) {
		var preservedBackupNodes []string
		pmgBackups, preservedBackupNodes = preserveUnreadPMGNodeBackups(
			instanceName, m.state.GetSnapshot().PMGBackups, pmgBackups,
			clusterErr != nil, backupNodes, readBackupNodes)
		if len(preservedBackupNodes) > 0 {
			log.Warn().
				Str("instance", instanceName).
				Strs("nodes", preservedBackupNodes).
				Bool("clusterReadFailed", clusterErr != nil).
				Msg("PMG backup list unavailable for some nodes; keeping their previous backups")
		}
	}

	if debugEnabled {
		log.Debug().
			Str("instance", instanceName).
			Int("backupCount", len(pmgBackups)).
			Msg("PMG backups polled")
	}

	if instanceCfg.MailStatsEnabled() {
		if stats, err := client.GetMailStatistics(ctx, ""); err != nil {
			recordReadError("pmg_mail_statistics", err)
			log.Warn().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG mail statistics")
		} else if stats != nil {
			pmgInst.MailStats = &models.PMGMailStats{
				Timeframe:            "day",
				CountTotal:           stats.Count.Float64(),
				CountIn:              stats.CountIn.Float64(),
				CountOut:             stats.CountOut.Float64(),
				SpamIn:               stats.SpamIn.Float64(),
				SpamOut:              stats.SpamOut.Float64(),
				VirusIn:              stats.VirusIn.Float64(),
				VirusOut:             stats.VirusOut.Float64(),
				BouncesIn:            stats.BouncesIn.Float64(),
				BouncesOut:           stats.BouncesOut.Float64(),
				BytesIn:              stats.BytesIn.Float64(),
				BytesOut:             stats.BytesOut.Float64(),
				GreylistCount:        stats.GreylistCount.Float64(),
				JunkIn:               stats.JunkIn.Float64(),
				AverageProcessTimeMs: stats.AvgProcessSec.Float64() * 1000,
				RBLRejects:           stats.RBLRejects.Float64(),
				PregreetRejects:      stats.Pregreet.Float64(),
				UpdatedAt:            time.Now(),
			}
		}

		if counts, err := client.GetMailCount(ctx, 86400); err != nil {
			recordReadError("pmg_mail_count", err)
			if debugEnabled {
				log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG mail count data")
			}
		} else if len(counts) > 0 {
			points := make([]models.PMGMailCountPoint, 0, len(counts))
			for _, entry := range counts {
				ts := time.Unix(entry.Time.Int64(), 0)
				points = append(points, models.PMGMailCountPoint{
					Timestamp:   ts,
					Count:       entry.Count.Float64(),
					CountIn:     entry.CountIn.Float64(),
					CountOut:    entry.CountOut.Float64(),
					SpamIn:      entry.SpamIn.Float64(),
					SpamOut:     entry.SpamOut.Float64(),
					VirusIn:     entry.VirusIn.Float64(),
					VirusOut:    entry.VirusOut.Float64(),
					RBLRejects:  entry.RBLRejects.Float64(),
					Pregreet:    entry.PregreetReject.Float64(),
					BouncesIn:   entry.BouncesIn.Float64(),
					BouncesOut:  entry.BouncesOut.Float64(),
					Greylist:    entry.GreylistCount.Float64(),
					Index:       entry.Index.Int(),
					Timeframe:   "hour",
					WindowStart: ts,
				})
			}
			pmgInst.MailCount = points
		}

		if scores, err := client.GetSpamScores(ctx); err != nil {
			recordReadError("pmg_spam_scores", err)
			if debugEnabled {
				log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG spam score distribution")
			}
		} else if len(scores) > 0 {
			buckets := make([]models.PMGSpamBucket, 0, len(scores))
			for _, bucket := range scores {
				buckets = append(buckets, models.PMGSpamBucket{
					Score: bucket.Level,
					Count: float64(bucket.Count.Int()),
				})
			}
			pmgInst.SpamDistribution = buckets
		}

	}

	if instanceCfg.MonitorQuarantine {
		spamStatus, spamErr := client.GetQuarantineStatus(ctx, "spam")
		virusStatus, virusErr := client.GetQuarantineStatus(ctx, "virus")
		recordReadError("pmg_spam_quarantine", spamErr)
		recordReadError("pmg_virus_quarantine", virusErr)
		// Totals and growth require both categories. Never publish a failed
		// category as zero or clear its alert using a partial observation.
		if spamErr == nil && virusErr == nil && spamStatus != nil && virusStatus != nil {
			pmgInst.Quarantine = &models.PMGQuarantineTotals{
				Spam: int(spamStatus.Count.Int64()), Virus: int(virusStatus.Count.Int64()),
			}
		}
	}

	if instanceCfg.MonitorDomainStats {
		if domains, err := client.ListRelayDomains(ctx); err != nil {
			recordReadError("pmg_relay_domains", err)
			if debugEnabled {
				log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG relay domains")
			}
		} else if len(domains) > 0 {
			relayDomains := make([]models.PMGRelayDomain, 0, len(domains))
			for _, domain := range domains {
				relayDomains = append(relayDomains, models.PMGRelayDomain{
					Domain:  strings.TrimSpace(domain.Domain),
					Comment: strings.TrimSpace(domain.Comment),
				})
			}
			pmgInst.RelayDomains = relayDomains
		}

		end := time.Now()
		start := end.Add(-24 * time.Hour)
		if stats, err := client.GetDomainStatistics(ctx, start.Unix(), end.Unix()); err != nil {
			recordReadError("pmg_domain_statistics", err)
			if debugEnabled {
				log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG domain statistics")
			}
		} else if len(stats) > 0 {
			domainStats := make([]models.PMGDomainStat, 0, len(stats))
			for _, entry := range stats {
				d := strings.TrimSpace(entry.Domain)
				if d == "" {
					continue
				}
				domainStats = append(domainStats, models.PMGDomainStat{
					Domain:     d,
					MailCount:  entry.Count.Float64(),
					SpamCount:  entry.SpamCount.Float64(),
					VirusCount: entry.VirusCount.Float64(),
					Bytes:      entry.Bytes.Float64(),
				})
			}
			pmgInst.DomainStats = domainStats
			pmgInst.DomainStatsAsOf = end
		}
	}

	// Reads cut short by the poll's own deadline or cancellation are a
	// whole-poll failure, not one broken endpoint, and keep their backoff.
	if ctxErr := ctx.Err(); ctxErr != nil && readErr != nil {
		pollErr = stderrors.Join(ctxErr, readErr)
	}

	m.state.UpdatePMGBackups(instanceName, pmgBackups)
	m.state.UpdatePMGInstance(pmgInst)
	log.Info().
		Str("instance", instanceName).
		Str("status", pmgInst.Status).
		Int("nodes", len(pmgInst.Nodes)).
		Msg("PMG instance updated in state")

	// Check PMG metrics against alert thresholds
	if m.alertManager != nil {
		m.alertManager.CheckPMG(pmgInst)
	}
}

// preserveUnreadPMGNodeBackups carries forward previous backups for nodes
// whose backup list was not read this cycle, mirroring
// preserveFailedStorageBackups for PVE storages. When the cluster read failed
// no node was enumerated, so every previous node is unread. When the cluster
// read succeeded, a node it no longer lists is gone and its backups drop.
func preserveUnreadPMGNodeBackups(instanceName string, previous, current []models.PMGBackup, clusterUnread bool, enumeratedNodes, readNodes map[string]struct{}) ([]models.PMGBackup, []string) {
	existing := make(map[string]struct{}, len(current))
	for _, backup := range current {
		existing[backup.ID] = struct{}{}
	}

	preserved := make(map[string]struct{})
	for _, backup := range previous {
		if backup.Instance != instanceName {
			continue
		}
		if _, read := readNodes[backup.Node]; read {
			continue
		}
		if _, enumerated := enumeratedNodes[backup.Node]; !enumerated && !clusterUnread {
			continue
		}
		if _, duplicate := existing[backup.ID]; duplicate {
			continue
		}
		current = append(current, backup)
		existing[backup.ID] = struct{}{}
		preserved[backup.Node] = struct{}{}
	}

	if len(preserved) == 0 {
		return current, nil
	}
	nodes := make([]string, 0, len(preserved))
	for node := range preserved {
		nodes = append(nodes, node)
	}
	sort.Strings(nodes)
	return current, nodes
}

// GetState returns the current state