Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
stderrors "errors"
"fmt"
"math"
"sort"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
func matchesDatastoreExclude(datastoreName string, excludePatterns []string) bool {
if len(excludePatterns) == 0 {
return false
}
for _, pattern := range excludePatterns {
pattern = strings.TrimSpace(pattern)
if pattern == "" {
continue
}
// Contains pattern: *substring*
if strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*") && len(pattern) > 2 {
substring := strings.ToLower(pattern[1 : len(pattern)-1])
if strings.Contains(strings.ToLower(datastoreName), substring) {
return true
}
continue
}
// Suffix pattern: *suffix
if strings.HasPrefix(pattern, "*") && len(pattern) > 1 {
suffix := strings.ToLower(pattern[1:])
if strings.HasSuffix(strings.ToLower(datastoreName), suffix) {
return true
}
continue
}
// Prefix pattern: prefix*
if strings.HasSuffix(pattern, "*") && len(pattern) > 1 {
prefix := strings.ToLower(pattern[:len(pattern)-1])
if strings.HasPrefix(strings.ToLower(datastoreName), prefix) {
return true
}
continue
}
// Exact match (case-insensitive)
if strings.EqualFold(pattern, datastoreName) {
return true
}
}
return false
}
func pbsDatastoreStatus(ds pbs.Datastore) string {
status := strings.TrimSpace(ds.Status)
if status != "" {
return status
}
if strings.TrimSpace(ds.Error) != "" {
return "unavailable"
}
return "available"
}
func pbsJobHealthEvidenceFromFacts(facts []pbs.JobHealthEvidence, observedAt time.Time) []models.PBSJobHealthEvidence {
if len(facts) == 0 {
return []models.PBSJobHealthEvidence{}
}
out := make([]models.PBSJobHealthEvidence, 0, len(facts))
for _, fact := range facts {
evidence := models.PBSJobHealthEvidence{
ID: fact.ID,
Family: fact.Family,
Store: fact.Store,
Remote: fact.Remote,
Namespace: fact.Namespace,
Schedule: fact.Schedule,
Comment: fact.Comment,
Enabled: fact.Enabled,
LastRunState: fact.LastRunState,
LastRunUPID: fact.LastRunUPID,
LastRunEndtime: fact.LastRunEndtime,
NextRun: fact.NextRun,
UPID: fact.UPID,
WorkerType: fact.WorkerType,
WorkerID: fact.WorkerID,
TaskStatus: fact.TaskStatus,
TaskStartTime: fact.TaskStartTime,
TaskEndTime: fact.TaskEndTime,
Confidence: fact.Confidence,
EvidenceSource: fact.EvidenceSource,
EvidenceScope: fact.EvidenceScope,
Error: fact.Error,
}
evidence.Freshness = pbsJobHealthFreshness(fact, observedAt)
evidence.Posture, evidence.PostureReason = pbsJobHealthPosture(fact, evidence.Freshness)
out = append(out, evidence)
}
return out
}
func pbsJobHealthFreshness(fact pbs.JobHealthEvidence, observedAt time.Time) models.PBSJobHealthFreshness {
freshness := models.PBSJobHealthFreshness{
ObservedAt: observedAt.UTC(),
State: "unknown",
}
if fact.LastRunEndtime > 0 {
freshness.LastRunEndTime = time.Unix(fact.LastRunEndtime, 0).UTC()
} else if fact.TaskEndTime > 0 {
freshness.LastRunEndTime = time.Unix(fact.TaskEndTime, 0).UTC()
}
if fact.NextRun > 0 {
freshness.NextRun = time.Unix(fact.NextRun, 0).UTC()
}
if !freshness.LastRunEndTime.IsZero() {
freshness.AgeSeconds = int64(observedAt.Sub(freshness.LastRunEndTime).Seconds())
if freshness.AgeSeconds < 0 {
freshness.AgeSeconds = 0
}
freshness.State = "observed"
}
if !freshness.NextRun.IsZero() {
if observedAt.After(freshness.NextRun) && (freshness.LastRunEndTime.IsZero() || freshness.LastRunEndTime.Before(freshness.NextRun)) {
freshness.State = "overdue"
} else if freshness.State == "unknown" {
freshness.State = "scheduled"
}
}
return freshness
}
func pbsJobHealthPosture(fact pbs.JobHealthEvidence, freshness models.PBSJobHealthFreshness) (string, string) {
if fact.Error != "" {
return "unknown", strings.TrimSpace(fact.Confidence)
}
outcome := pbsTaskStatusOutcome(firstNonEmptyString(fact.TaskStatus, fact.LastRunState))
// A failed last run outranks a missed next one: an overdue job whose last
// run failed is still failed, not merely late.
if outcome == "failed" {
return "critical", "last-run-failed"
}
if freshness.State == "overdue" {
return "warning", "job-overdue"
}
switch outcome {
case "":
return "unknown", "no-run-observed"
case "ok":
return "healthy", "last-run-success"
case "running":
return "healthy", "running"
case "warning":
return "warning", "last-run-warning"
default:
return "unknown", "last-run-state-unknown"
}
}
// pbsTaskStatusOutcome reads a PBS task or job status the way Proxmox's own UI
// does (proxmox-widget-toolkit parse_task_status). PBS writes "OK",
// "WARNINGS: <n>", "unknown", or, for a failure, the error message itself with
// no prefix, such as "remote 'offsite' does not exist" or "broken pipe".
// Matching words like "error" missed failures that lack them, and matching
// "ok" called "broken pipe" a success.
func pbsTaskStatusOutcome(status string) string {
lower := strings.ToLower(strings.TrimSpace(status))
switch {
case lower == "":
return ""
case lower == "ok":
return "ok"
case strings.HasPrefix(lower, "warnings:"):
return "warning"
case lower == "unknown":
return "unknown"
case lower == "running":
return "running"
default:
return "failed"
}
}
// pbsJobKey identifies one job across polls. PBS job ids are only unique
// within a family, so a sync and a verify job may share one.
func pbsJobKey(entry models.PBSJobHealthEvidence) string {
return strings.ToLower(strings.TrimSpace(entry.Family)) + "\x00" + strings.TrimSpace(entry.ID)
}
// carryPBSJobFailures keeps a job's last completed failure known while PBS
// cannot report a newer outcome. PBS's /admin job status drops the previous
// result while the next run is in progress, an overdue or unreadable job
// reports none, and a failed read lists no jobs at all; treating any of those
// as recovery would clear a failed-job alert and raise it again on the next
// poll. A carried failure clears only when the job's newest run is observed to
// succeed or warn, or when a complete read no longer lists the job (removed),
// or lists it disabled.
func carryPBSJobFailures(previous, current []models.PBSJobHealthEvidence, readFailed bool) []models.PBSJobHealthEvidence {
failed := make(map[string]models.PBSJobHealthEvidence)
for _, entry := range previous {
if entry.Posture == "critical" && entry.PostureReason == "last-run-failed" && strings.TrimSpace(entry.ID) != "" {
failed[pbsJobKey(entry)] = entry
}
}
if len(failed) == 0 {
return current
}
unreadFamilies := make(map[string]bool)
for _, entry := range current {
family := strings.ToLower(strings.TrimSpace(entry.Family))
if strings.TrimSpace(entry.Error) != "" && entry.ID == family+":partial" &&
entry.EvidenceSource == pbs.JobEvidenceSourcePBSPartialRead {
unreadFamilies[family] = true
}
}
out := make([]models.PBSJobHealthEvidence, 0, len(current)+len(failed))
seen := make(map[string]bool, len(current))
for _, entry := range current {
key := pbsJobKey(entry)
seen[key] = true
if prior, ok := failed[key]; ok && entry.Enabled && !pbsJobReportsOutcome(entry) {
out = append(out, prior)
continue
}
out = append(out, entry)
}
for key, prior := range failed {
if seen[key] {
continue
}
if readFailed || unreadFamilies[strings.ToLower(strings.TrimSpace(prior.Family))] {
out = append(out, prior)
}
}
return out
}
// carryPBSJobFailuresForInstance carries failures from the instance's last
// stored results, or, on the first poll after a restart when state holds none,
// from the persisted failure record of the same server, and records what is
// still failing. Records of connections no longer configured are dropped.
func (m *Monitor) carryPBSJobFailuresForInstance(instanceID, host string, current []models.PBSJobHealthEvidence, readFailed bool) []models.PBSJobHealthEvidence {
previous := m.state.PBSJobHealthEvidence(instanceID)
if len(previous) == 0 {
previous = m.pbsJobFailureStore.Get(instanceID, host)
}
carried := carryPBSJobFailures(previous, current, readFailed)
if err := m.pbsJobFailureStore.Set(instanceID, host, carried); err != nil {
log.Warn().Err(err).Str("instance", instanceID).Msg("Failed to record PBS job failures")
}
if m.config != nil {
configured := make([]string, 0, len(m.config.PBSInstances))
for _, cfg := range m.config.PBSInstances {
configured = append(configured, PBSMonitorResourceID(cfg.Name))
}
if err := m.pbsJobFailureStore.Retain(configured); err != nil {
log.Warn().Err(err).Msg("Failed to prune PBS job failure record")
}
}
return carried
}
// pbsJobReportsOutcome says whether this poll observed how the job's newest
// run ended: success, warning or failure, rather than running or an
// unreadable state. It reads the run's own status, not the posture, so a
// successful run first seen after its next schedule passed still counts.
func pbsJobReportsOutcome(entry models.PBSJobHealthEvidence) bool {
if strings.TrimSpace(entry.Error) != "" {
return false
}
switch pbsTaskStatusOutcome(firstNonEmptyString(entry.TaskStatus, entry.LastRunState)) {
case "ok", "warning", "failed":
return true
default:
return false
}
}
// pollPBSInstance polls a single PBS instance
func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, client *pbs.Client) {
start := time.Now()
debugEnabled := logging.IsLevelEnabled(zerolog.DebugLevel)
var pollErr error
var pbsInst models.PBSInstance
publishResult := false
if m.pollMetrics != nil {
m.pollMetrics.IncInFlight("pbs")
}
defer func() {
if recovered := recover(); recovered != nil {
pollErr = fmt.Errorf("panic while polling PBS instance %q: %v", instanceName, recovered)
log.Error().
Str("goroutine", fmt.Sprintf("pollPBSInstance-%s", instanceName)).
Interface("panic", recovered).
Stack().
Msg("Recovered from panic in monitoring goroutine")
}
m.recordTaskResult(InstanceTypePBS, instanceName, pollErr)
if m.stalenessTracker != nil {
if pollErr == nil {
m.stalenessTracker.UpdateSuccess(InstanceTypePBS, instanceName, nil)
} else {
m.stalenessTracker.UpdateError(InstanceTypePBS, instanceName)
}
}
if publishResult {
m.publishPBSConnectionOutcome(pbsInst, pollErr)
}
if m.pollMetrics != nil {
m.pollMetrics.DecInFlight("pbs")
m.pollMetrics.RecordResult(PollResult{
InstanceName: instanceName,
InstanceType: "pbs",
Success: pollErr == nil,
Error: pollErr,
StartTime: start,
EndTime: time.Now(),
})
}
}()
// Get instance config
var instanceCfg *config.PBSInstance
for _, cfg := range m.config.PBSInstances {
if cfg.Name == instanceName {
instanceCfg = &cfg
if debugEnabled {
log.Debug().
Str("instance", instanceName).
Bool("monitorDatastores", cfg.MonitorDatastores).
Msg("Found PBS instance config")
}
break
}
}
if instanceCfg == nil {
pollErr = fmt.Errorf("PBS instance config %q not found", instanceName)
log.Error().Str("instance", instanceName).Msg("PBS instance config not found")
return
}
if instanceCfg.Disabled {
if debugEnabled {
log.Debug().Str("instance", instanceName).Msg("Skipping PBS poll: instance is paused")
}
return
}
includeDatastore := func(datastoreName string) bool {
return !matchesDatastoreExclude(datastoreName, instanceCfg.ExcludeDatastores)
}
// Initialize PBS instance with default values
pbsInst = models.PBSInstance{
NodeMetricsUnavailable: true,
ID: PBSMonitorResourceID(instanceName),
Name: instanceName,
Host: instanceCfg.Host,
GuestURL: instanceCfg.GuestURL,
Status: "offline",
Version: "unknown",
ConnectionHealth: "unhealthy",
LastSeen: time.Now(),
}
publishResult = true
// Check if context is cancelled after the configured instance projection is
// available, so cancellation still publishes the same failed poll outcome.
select {
case <-ctx.Done():
pollErr = ctx.Err()
if debugEnabled {
log.Debug().Str("instance", instanceName).Msg("polling cancelled")
}
return
default:
}
if debugEnabled {
log.Debug().Str("instance", instanceName).Msg("polling PBS instance")
}
// Try to get version first
version, versionErr := client.GetVersion(ctx)
if versionErr == nil {
pbsInst.Status = "online"
pbsInst.Version = version.Version
pbsInst.ConnectionHealth = "healthy"
m.resetAuthFailures(instanceName, "pbs")
if debugEnabled {
log.Debug().
Str("instance", instanceName).
Str("version", version.Version).
Bool("monitorDatastores", instanceCfg.MonitorDatastores).
Msg("PBS version retrieved successfully")
}
} else {
if debugEnabled {
log.Debug().Err(versionErr).Str("instance", instanceName).Msg("failed to get PBS version, trying fallback")
}
// Use parent context for proper cancellation chain
ctx2, cancel2 := context.WithTimeout(ctx, 10*time.Second)
defer cancel2()
_, datastoreErr := client.GetDatastoresFiltered(ctx2, includeDatastore)
if datastoreErr == nil {
pbsInst.Status = "online"
pbsInst.Version = "connected"
pbsInst.ConnectionHealth = "healthy"
m.resetAuthFailures(instanceName, "pbs")
log.Info().
Str("instance", instanceName).
Msg("PBS connected (version unavailable but datastores accessible)")
} else {
pbsInst.Status = "offline"
pbsInst.ConnectionHealth = "error"
connectionErr := versionErr
if errors.IsAuthError(datastoreErr) {
connectionErr = datastoreErr
}
monErr := errors.WrapConnectionError("get_pbs_version", instanceName, connectionErr)
pollErr = monErr
log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to connect to PBS")
if errors.IsAuthError(versionErr) || errors.IsAuthError(datastoreErr) {
m.recordAuthFailure(instanceName, "pbs")
}
// The connectivity fallback also failed datastore discovery. Qualify
// retained backup evidence through the ordinary serialized admission.
if !stderrors.Is(datastoreErr, context.Canceled) {
m.pollPBSBackupsAsync(instanceName, client, instanceCfg, nil, nil, datastoreErr)
}
return
}
}
// The PBS-reported node hostname is machine-identity evidence: connected
// systems grouping merges the host agent running on this machine into
// this PBS connection through it, even when the configured connection
// address is an IP or DNS alias the agent never reports.
if nodeName, nodeNameErr := client.GetNodeName(ctx); nodeNameErr == nil {
pbsInst.NodeName = strings.TrimSpace(nodeName)
} else if !stderrors.Is(nodeNameErr, pbs.ErrNodeNameUnavailableForAuth) && debugEnabled {
log.Debug().Err(nodeNameErr).Str("instance", instanceName).Msg("could not get PBS node name")
}
// Get node status (CPU, memory, etc.)
nodeStatus, err := client.GetNodeStatus(ctx)
if err != nil {
if debugEnabled {
log.Debug().Err(err).Str("instance", instanceName).Msg("could not get PBS node status (may need Sys.Audit permission)")
}
} else if nodeStatus != nil {
pbsInst.NodeMetricsUnavailable = false
pbsInst.CPU = nodeStatus.CPU
if nodeStatus.Memory.Total > 0 {
pbsInst.Memory = float64(nodeStatus.Memory.Used) / float64(nodeStatus.Memory.Total) * 100
pbsInst.MemoryUsed = nodeStatus.Memory.Used
pbsInst.MemoryTotal = nodeStatus.Memory.Total
}
pbsInst.Uptime = nodeStatus.Uptime
m.recordPBSNodeHistory(pbsInst.ID, nodeStatus, time.Now())
log.Debug().
Str("instance", instanceName).
Float64("cpu", pbsInst.CPU).
Float64("memory", pbsInst.Memory).
Int64("uptime", pbsInst.Uptime).
Msg("PBS node status retrieved")
}
// Keep discovery quality with the immutable input to the background backup
// poll. Public datastore presentation is not an authoritative scope receipt.
namespaceErrors := make(map[string]error)
var datastoreDiscoveryErr error
// Poll datastores if enabled
if instanceCfg.MonitorDatastores {
datastores, err := client.GetDatastoresFiltered(ctx, includeDatastore)
if err != nil {
datastoreDiscoveryErr = err
monErr := errors.WrapAPIError("get_datastores", instanceName, err, 0)
log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get datastores")
} else {
log.Info().
Str("instance", instanceName).
Int("count", len(datastores)).
Msg("Got PBS datastores")
for _, ds := range datastores {
// Skip excluded datastores (for removable/unmounted datastores)
if matchesDatastoreExclude(ds.Store, instanceCfg.ExcludeDatastores) {
log.Debug().
Str("instance", instanceName).
Str("datastore", ds.Store).
Msg("Skipping excluded datastore")
continue
}
total := ds.Total
if total == 0 && ds.TotalSpace > 0 {
total = ds.TotalSpace
}
used := ds.Used
if used == 0 && ds.UsedSpace > 0 {
used = ds.UsedSpace
}
avail := ds.Avail
if avail == 0 && ds.AvailSpace > 0 {
avail = ds.AvailSpace
}
if total == 0 && used > 0 && avail > 0 {
total = used + avail
}
log.Debug().
Str("store", ds.Store).
Int64("total", total).
Int64("used", used).
Int64("avail", avail).
Int64("orig_total", ds.Total).
Int64("orig_total_space", ds.TotalSpace).
Msg("PBS datastore details")
modelDS := models.PBSDatastore{
Name: ds.Store,
Total: total,
Used: used,
Free: avail,
Usage: safePercentage(float64(used), float64(total)),
Status: pbsDatastoreStatus(ds),
Error: ds.Error,
DeduplicationFactor: ds.DeduplicationFactor,
}
namespaces, err := client.ListNamespaces(ctx, ds.Store, "", 0)
if err != nil {
namespaceErrors[ds.Store] = err
log.Warn().Err(err).
Str("instance", instanceName).
Str("datastore", ds.Store).
Msg("Failed to list namespaces")
} else {
for _, ns := range namespaces {
nsPath := ns.NS
if nsPath == "" {
nsPath = ns.Path
}
if nsPath == "" {
nsPath = ns.Name
}
modelNS := models.PBSNamespace{
Path: nsPath,
Parent: ns.Parent,
Depth: strings.Count(nsPath, "/"),
}
modelDS.Namespaces = append(modelDS.Namespaces, modelNS)
}
hasRoot := false
for _, ns := range modelDS.Namespaces {
if ns.Path == "" {
hasRoot = true
break
}
}
if !hasRoot {
modelDS.Namespaces = append([]models.PBSNamespace{{Path: "", Depth: 0}}, modelDS.Namespaces...)
}
}
pbsInst.Datastores = append(pbsInst.Datastores, modelDS)
}
}
}
if instanceCfg.MonitorBackups || instanceCfg.MonitorSyncJobs || instanceCfg.MonitorVerifyJobs || instanceCfg.MonitorPruneJobs || instanceCfg.MonitorGarbageJobs {
datastoreNames := make([]string, 0, len(pbsInst.Datastores))
for _, ds := range pbsInst.Datastores {
datastoreNames = append(datastoreNames, ds.Name)
}
jobFacts, err := client.GetJobHealthEvidence(ctx, datastoreNames, pbs.JobHealthOptions{
MonitorBackups: instanceCfg.MonitorBackups,
MonitorSyncJobs: instanceCfg.MonitorSyncJobs,
MonitorVerifyJobs: instanceCfg.MonitorVerifyJobs,
MonitorPruneJobs: instanceCfg.MonitorPruneJobs,
MonitorGarbageJobs: instanceCfg.MonitorGarbageJobs,
})
if err != nil {
log.Warn().Err(err).
Str("instance", instanceName).
Msg("Failed to collect PBS job health evidence")
}
pbsInst.JobHealthEvidence = m.carryPBSJobFailuresForInstance(
pbsInst.ID,
pbsInst.Host,
pbsJobHealthEvidenceFromFacts(jobFacts, time.Now()),
err != nil,
)
}
// Convert PBS datastores to Storage entries for unified storage view
if len(pbsInst.Datastores) > 0 && instanceCfg.MonitorDatastores {
var pbsStorages []models.Storage
for _, ds := range pbsInst.Datastores {
// Create a storage entry for this PBS datastore
storageID := fmt.Sprintf("pbs-%s-%s", instanceName, ds.Name)
pbsStorage := models.Storage{
ID: storageID,
// The unified registry and thresholds UI key this datastore by
// its canonical "<instance-id>/<name>" ID; carrying it as an
// alias lets alert override lookups accept either format (#1591).
AliasIDs: []string{pbsInst.ID + "/" + ds.Name},
Name: ds.Name,
Node: instanceName, // Use PBS instance name as "node"
Instance: PBSMonitorResourceID(instanceName),
Type: "pbs",
Status: ds.Status,
Total: ds.Total,
Used: ds.Used,
Free: ds.Free,
Usage: ds.Usage,
Content: "backup", // PBS datastores are for backups
Shared: true, // PBS datastores are typically shared/network storage
Enabled: true,
Active: pbsInst.Status == "online",
LastSeen: pbsInst.LastSeen,
}
pbsStorages = append(pbsStorages, pbsStorage)
}
m.state.UpdateStorageForInstance("pbs-"+instanceName, pbsStorages)
// PBS storage is not evaluated by the unified metric path (which
// handles TrueNAS and VMware storage). Evaluate fresh poll observations
// here, as the PVE storage poller does, rather than relying on mock ticks.
if m.alertManager != nil {
for _, storage := range pbsStorages {
m.alertManager.CheckStorageWithCapacityTrend(storage, m.storageCapacityTrend(storage, time.Now()))
}
}
log.Debug().
Str("instance", instanceName).
Int("storageEntries", len(pbsStorages)).
Msg("Added PBS datastores to unified storage view")
}
// A successful empty scope and a failed discovery both need an accepted
// backup-poll result; neither may silently keep the last good protection.
if !stderrors.Is(datastoreDiscoveryErr, context.Canceled) {
m.pollPBSBackupsAsync(instanceName, client, instanceCfg, pbsInst.Datastores, namespaceErrors, datastoreDiscoveryErr)
}
}
// pollPBSBackupsAsync admits all backup-scope outcomes through one worker.
// Failed discovery never becomes an empty enumeration, and a skipped/disabled
// discovery is not absence evidence. Keeping failure/empty publication inside
// this admission prevents it from racing an older in-flight backup worker.
func (m *Monitor) pollPBSBackupsAsync(instanceName string, client *pbs.Client, instanceCfg *config.PBSInstance, datastores []models.PBSDatastore, namespaceErrors map[string]error, discoveryErr error) {
if !instanceCfg.MonitorBackups || !instanceCfg.MonitorDatastores || !m.backupPollingEnabledSetting() {
return
}
now := time.Now()
m.mu.RLock()
lastPoll := m.lastPBSBackupPoll[instanceName]
m.mu.RUnlock()
shouldPoll, reason, newLast := m.shouldRunBackupPoll(lastPoll, now)
if !shouldPoll {
log.Debug().Str("instance", instanceName).Str("reason", reason).Msg("Skipping PBS backup polling this cycle")
return
}
datastoreSnapshot := append([]models.PBSDatastore(nil), datastores...)
for i := range datastoreSnapshot {
datastoreSnapshot[i].Namespaces = append([]models.PBSNamespace(nil), datastores[i].Namespaces...)
}
namespaceErrorSnapshot := make(map[string]error, len(namespaceErrors))
for name, err := range namespaceErrors {
namespaceErrorSnapshot[name] = err
}
owner := m.beginPBSBackupPoll(instanceName, client, lastPoll, newLast)
if owner == nil {
log.Debug().Str("instance", instanceName).Msg("PBS backup polling already in progress")
return
}
go func() {
defer recoverFromPanic(fmt.Sprintf("pollPBSBackups-%s", instanceName))
defer m.finishPBSBackupPoll(instanceName, owner)
// The main polling loop cancels its per-cycle context after returning.
// Backup work keeps the existing independent, runtime-scoped deadline.
parentCtx := context.WithValue(owner.ctx, pbsBackupPollContextKey{}, owner)
backupCtx, cancel := context.WithTimeout(parentCtx, 5*time.Minute)
defer cancel()
if backupCtx.Err() != nil {
return
}
if discoveryErr != nil {
unlock, current := m.beginPBSBackupPublication(backupCtx, instanceName)
if !current {
return
}
defer unlock()
// One unreadable discovery scope, not a guessed datastore count.
// Only actual authentication/authorization status establishes denial.
denied := 0
if status, ok := pbs.HTTPStatus(discoveryErr); ok && (status == 401 || status == 403) {
denied = 1
}
observation, err := buildPBSProtectionProviderObservation(instanceName, 1, 0, 1, denied, time.Now().UTC())
if err != nil {
log.Warn().Err(err).Str("instance", instanceName).Msg("Failed to build PBS discovery failure observation")
return
}
m.ingestProtectionProviderObservationsAsync([]recovery.ProtectionProviderObservation{observation})
} else {
// The explicit empty list is authoritative and reconciles only this
// connection; the ordinary poller also drops its live rows/cache.
m.pollPBSBackups(backupCtx, instanceName, client, datastoreSnapshot, namespaceErrorSnapshot)
}
log.Info().Str("instance", instanceName).Dur("duration", time.Since(now)).Msg("Completed background PBS backup polling")
}()
}
// recordPBSNodeHistory writes the API-observed CPU and memory measurements to
// the PBS service's existing metrics target (agent:<PBS source ID>). When no
// host agent can be corroborated, the Backups drawer still has useful history
// instead of querying an unwritten key. Host-agent history remains separate and
// takes precedence in the drawer when a unique host is linked. A failed or
// incomplete node-status response must never produce an apparent zero sample.
func (m *Monitor) recordPBSNodeHistory(serviceID string, status *pbs.NodeStatus, observedAt time.Time) {
serviceID = strings.TrimSpace(serviceID)
if m == nil || status == nil || serviceID == "" || status.Memory.Total <= 0 ||
status.Memory.Used < 0 || status.CPU < 0 || math.IsNaN(status.CPU) || math.IsInf(status.CPU, 0) {
return
}
cpuPercent := status.CPU
if cpuPercent <= 1 {
cpuPercent *= 100
}
memoryPercent := float64(status.Memory.Used) / float64(status.Memory.Total) * 100
if m.metricsHistory != nil {
key := "agent:" + serviceID
m.metricsHistory.AddGuestMetric(key, "cpu", cpuPercent, observedAt)
m.metricsHistory.AddGuestMetric(key, "memory", memoryPercent, observedAt)
}
if m.metricsStore != nil {
m.metricsStore.WriteBatchBounded([]metrics.WriteMetric{
{ResourceType: "agent", ResourceID: serviceID, MetricType: "cpu", Value: cpuPercent, Timestamp: observedAt, Tier: metrics.TierRaw},
{ResourceType: "agent", ResourceID: serviceID, MetricType: "memory", Value: memoryPercent, Timestamp: observedAt, Tier: metrics.TierRaw},
})
}
}
// publishPBSConnectionOutcome projects the final connectivity result into every
// legacy PBS health consumer. Inventory collection may be partial, but the
// connection map, dashboard resource, and legacy alert evaluator must all use
// the same poll error that is recorded in the scheduler ledger.
func (m *Monitor) publishPBSConnectionOutcome(instance models.PBSInstance, pollErr error) {
// A poll that failed before reading jobs knows nothing new about them;
// keep the last results so a failed job is still carried when PBS returns.
if pollErr != nil && len(instance.JobHealthEvidence) == 0 {
instance.JobHealthEvidence = m.state.PBSJobHealthEvidence(instance.ID)
}
connected := pollErr == nil
if connected {
instance.Status = "online"
instance.ConnectionHealth = "healthy"
} else {
instance.Status = "offline"
instance.ConnectionHealth = "error"
}
m.setProviderConnectionHealth(InstanceTypePBS, instance.Name, connected)
m.state.UpdatePBSInstance(instance)
log.Info().
Str("instance", instance.Name).
Str("id", instance.ID).
Bool("connected", connected).
Int("datastores", len(instance.Datastores)).
Msg("PBS instance updated in state")
if m.alertManager != nil {
m.alertManager.CheckPBS(instance)
}
}
// pmgPollClient is the existing read-only PMG collection boundary.
type pmgPollClient interface {
GetVersion(context.Context) (*pmg.VersionInfo, error)
GetClusterStatus(context.Context, bool) ([]pmg.ClusterStatusEntry, error)
GetQueueStatus(context.Context, string) (*pmg.QueueStatusEntry, error)
ListBackups(context.Context, string) ([]pmg.BackupEntry, error)
GetMailStatistics(context.Context, string) (*pmg.MailStatistics, error)
GetMailCount(context.Context, int) ([]pmg.MailCountEntry, error)
GetSpamScores(context.Context) ([]pmg.SpamScore, error)
GetQuarantineStatus(context.Context, string) (*pmg.QuarantineStatus, error)
ListRelayDomains(context.Context) ([]pmg.RelayDomainEntry, error)
GetDomainStatistics(context.Context, int64, int64) ([]pmg.DomainStatisticsEntry, error)
}
var _ pmgPollClient = (*pmg.Client)(nil)
// pollPMGInstance polls a single Proxmox Mail Gateway instance
func (m *Monitor) pollPMGInstance(ctx context.Context, instanceName string, client pmgPollClient) {
defer recoverFromPanic(fmt.Sprintf("pollPMGInstance-%s", instanceName))
start := time.Now()
debugEnabled := logging.IsLevelEnabled(zerolog.DebugLevel)
// pollErr is a connection, version or whole-poll failure and drives
// scheduler backoff. readErr collects downstream reads that failed after
// the connection succeeded; see recordTaskOutcome.
var pollErr, readErr error
if m.pollMetrics != nil {
m.pollMetrics.IncInFlight("pmg")
defer m.pollMetrics.DecInFlight("pmg")
defer func() {
m.pollMetrics.RecordResult(PollResult{
InstanceName: instanceName,
InstanceType: "pmg",
Success: pollErr == nil,
Error: pollErr,
StartTime: start,
EndTime: time.Now(),
})
}()
}
if m.stalenessTracker != nil {
defer func() {
if pollErr == nil {
m.stalenessTracker.UpdateSuccess(InstanceTypePMG, instanceName, nil)
} else {
m.stalenessTracker.UpdateError(InstanceTypePMG, instanceName)
}
}()
}
defer func() {
m.recordTaskOutcome(InstanceTypePMG, instanceName, pollErr, readErr)
}()
select {
case <-ctx.Done():
pollErr = ctx.Err()
if debugEnabled {
log.Debug().Str("instance", instanceName).Msg("PMG polling cancelled by context")
}
return
default:
}
if debugEnabled {
log.Debug().Str("instance", instanceName).Msg("polling PMG instance")
}
// Each poll uses one saved scope snapshot. A later edit/pause applies to
// the next poll and does not revoke requests already in flight.
var instanceCfg *config.PMGInstance
config.Mu.RLock()
if m.config != nil {
for _, cfg := range m.config.PMGInstances {
if cfg.Name == instanceName {
snapshot := cfg
instanceCfg = &snapshot
break
}
}
}
config.Mu.RUnlock()
if instanceCfg == nil {
log.Error().Str("instance", instanceName).Msg("PMG instance config not found")
pollErr = fmt.Errorf("pmg instance config not found for %s", instanceName)
return
}
if instanceCfg.Disabled {
if debugEnabled {
log.Debug().Str("instance", instanceName).Msg("Skipping PMG poll: instance is paused")
}
return
}
now := time.Now()
pmgInst := models.PMGInstance{
ID: PMGMonitorResourceID(instanceName),
Name: instanceName,
Host: instanceCfg.Host,
GuestURL: instanceCfg.GuestURL,
Status: "offline",
ConnectionHealth: "unhealthy",
LastSeen: now,
LastUpdated: now,
}
version, err := client.GetVersion(ctx)
if err != nil {
monErr := errors.WrapConnectionError("pmg_get_version", instanceName, err)
pollErr = monErr
log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to connect to PMG instance")
m.setProviderConnectionHealth(InstanceTypePMG, instanceName, false)
m.state.UpdatePMGInstance(pmgInst)
// Check PMG offline status against alert thresholds
if m.alertManager != nil {
m.alertManager.CheckPMG(pmgInst)
}
if errors.IsAuthError(err) {
m.recordAuthFailure(instanceName, "pmg")
}
return
}
pmgInst.Status = "online"
pmgInst.ConnectionHealth = "healthy"
if version != nil {
pmgInst.Version = strings.TrimSpace(version.Version)
}
m.setProviderConnectionHealth(InstanceTypePMG, instanceName, true)
m.resetAuthFailures(instanceName, "pmg")
// A successful version read establishes connection health, not complete
// downstream collection. Preserve missing readings and report failed reads
// without backing off the readings that succeeded.
recordReadError := func(operation string, err error) {
if err != nil {
readErr = stderrors.Join(readErr, errors.WrapAPIError(operation, instanceName, err, 0))
}
}
cluster, clusterErr := client.GetClusterStatus(ctx, true)
recordReadError("pmg_cluster_status", clusterErr)
if clusterErr != nil {
if debugEnabled {
log.Debug().Err(clusterErr).Str("instance", instanceName).Msg("failed to retrieve PMG cluster status")
}
}
backupNodes := make(map[string]struct{})
if len(cluster) > 0 {
nodes := make([]models.PMGNodeStatus, 0, len(cluster))
for _, entry := range cluster {
status := strings.ToLower(strings.TrimSpace(entry.Type))
if status == "" {
status = "online"
}
node := models.PMGNodeStatus{
Name: entry.Name,
Status: status,
Role: entry.Type,
}
backupNodes[entry.Name] = struct{}{}
if instanceCfg.MonitorQueues {
// Fetch queue status for this node
if queueData, qErr := client.GetQueueStatus(ctx, entry.Name); qErr != nil {
recordReadError("pmg_queue_status", qErr)
if debugEnabled {
log.Debug().Err(qErr).
Str("instance", instanceName).
Str("node", entry.Name).
Msg("Failed to fetch PMG queue status")
}
} else if queueData != nil {
total := queueData.Active.Int64() + queueData.Deferred.Int64() + queueData.Hold.Int64() + queueData.Incoming.Int64()
node.QueueStatus = &models.PMGQueueStatus{
Active: queueData.Active.Int(),
Deferred: queueData.Deferred.Int(),
Hold: queueData.Hold.Int(),
Incoming: queueData.Incoming.Int(),
Total: int(total),
OldestAge: queueData.OldestAge.Int64(),
UpdatedAt: time.Now(),
}
}
}
nodes = append(nodes, node)
}
pmgInst.Nodes = nodes
}
// Fall back to the connection name only when PMG answered with no cluster
// entries. After a failed cluster read the name is a display label, not a
// node: requesting backups with it fails with a misleading hostname-lookup
// error on PMG (#2960) and cannot return backups.
if len(backupNodes) == 0 && clusterErr == nil {
trimmed := strings.TrimSpace(instanceName)
if trimmed != "" {
backupNodes[trimmed] = struct{}{}
}
}
pmgBackups := make([]models.PMGBackup, 0)
seenBackupIDs := make(map[string]struct{})
readBackupNodes := make(map[string]struct{}, len(backupNodes))
for nodeName := range backupNodes {
if ctx.Err() != nil {
break
}
backups, backupErr := client.ListBackups(ctx, nodeName)
if backupErr != nil {
recordReadError("pmg_backups", backupErr)
if debugEnabled {
log.Debug().Err(backupErr).
Str("instance", instanceName).
Str("node", nodeName).
Msg("Failed to list PMG configuration backups")
}
continue
}
readBackupNodes[nodeName] = struct{}{}
for _, b := range backups {
timestamp := b.Timestamp.Int64()
backupTime := time.Unix(timestamp, 0)
backupID := fmt.Sprintf("pmg-%s-%s-%d", instanceName, nodeName, timestamp)
if _, exists := seenBackupIDs[backupID]; exists {
continue
}
seenBackupIDs[backupID] = struct{}{}
pmgBackups = append(pmgBackups, models.PMGBackup{
ID: backupID,
Instance: instanceName,
Node: nodeName,
Filename: b.Filename,
BackupTime: backupTime,
Size: b.Size.Int64(),
})
}
}
// Only a node whose backup list was read is authoritative this cycle. A
// failed cluster read, a failed per-node read or a cancelled poll observed
// nothing, so those nodes keep their last-known backups instead of being
// published as having none; the failed read is already in readErr.
if clusterErr != nil || len(readBackupNodes) < len(backupNodes) {
var preservedBackupNodes []string
pmgBackups, preservedBackupNodes = preserveUnreadPMGNodeBackups(
instanceName, m.state.GetSnapshot().PMGBackups, pmgBackups,
clusterErr != nil, backupNodes, readBackupNodes)
if len(preservedBackupNodes) > 0 {
log.Warn().
Str("instance", instanceName).
Strs("nodes", preservedBackupNodes).
Bool("clusterReadFailed", clusterErr != nil).
Msg("PMG backup list unavailable for some nodes; keeping their previous backups")
}
}
if debugEnabled {
log.Debug().
Str("instance", instanceName).
Int("backupCount", len(pmgBackups)).
Msg("PMG backups polled")
}
if instanceCfg.MailStatsEnabled() {
if stats, err := client.GetMailStatistics(ctx, ""); err != nil {
recordReadError("pmg_mail_statistics", err)
log.Warn().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG mail statistics")
} else if stats != nil {
pmgInst.MailStats = &models.PMGMailStats{
Timeframe: "day",
CountTotal: stats.Count.Float64(),
CountIn: stats.CountIn.Float64(),
CountOut: stats.CountOut.Float64(),
SpamIn: stats.SpamIn.Float64(),
SpamOut: stats.SpamOut.Float64(),
VirusIn: stats.VirusIn.Float64(),
VirusOut: stats.VirusOut.Float64(),
BouncesIn: stats.BouncesIn.Float64(),
BouncesOut: stats.BouncesOut.Float64(),
BytesIn: stats.BytesIn.Float64(),
BytesOut: stats.BytesOut.Float64(),
GreylistCount: stats.GreylistCount.Float64(),
JunkIn: stats.JunkIn.Float64(),
AverageProcessTimeMs: stats.AvgProcessSec.Float64() * 1000,
RBLRejects: stats.RBLRejects.Float64(),
PregreetRejects: stats.Pregreet.Float64(),
UpdatedAt: time.Now(),
}
}
if counts, err := client.GetMailCount(ctx, 86400); err != nil {
recordReadError("pmg_mail_count", err)
if debugEnabled {
log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG mail count data")
}
} else if len(counts) > 0 {
points := make([]models.PMGMailCountPoint, 0, len(counts))
for _, entry := range counts {
ts := time.Unix(entry.Time.Int64(), 0)
points = append(points, models.PMGMailCountPoint{
Timestamp: ts,
Count: entry.Count.Float64(),
CountIn: entry.CountIn.Float64(),
CountOut: entry.CountOut.Float64(),
SpamIn: entry.SpamIn.Float64(),
SpamOut: entry.SpamOut.Float64(),
VirusIn: entry.VirusIn.Float64(),
VirusOut: entry.VirusOut.Float64(),
RBLRejects: entry.RBLRejects.Float64(),
Pregreet: entry.PregreetReject.Float64(),
BouncesIn: entry.BouncesIn.Float64(),
BouncesOut: entry.BouncesOut.Float64(),
Greylist: entry.GreylistCount.Float64(),
Index: entry.Index.Int(),
Timeframe: "hour",
WindowStart: ts,
})
}
pmgInst.MailCount = points
}
if scores, err := client.GetSpamScores(ctx); err != nil {
recordReadError("pmg_spam_scores", err)
if debugEnabled {
log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG spam score distribution")
}
} else if len(scores) > 0 {
buckets := make([]models.PMGSpamBucket, 0, len(scores))
for _, bucket := range scores {
buckets = append(buckets, models.PMGSpamBucket{
Score: bucket.Level,
Count: float64(bucket.Count.Int()),
})
}
pmgInst.SpamDistribution = buckets
}
}
if instanceCfg.MonitorQuarantine {
spamStatus, spamErr := client.GetQuarantineStatus(ctx, "spam")
virusStatus, virusErr := client.GetQuarantineStatus(ctx, "virus")
recordReadError("pmg_spam_quarantine", spamErr)
recordReadError("pmg_virus_quarantine", virusErr)
// Totals and growth require both categories. Never publish a failed
// category as zero or clear its alert using a partial observation.
if spamErr == nil && virusErr == nil && spamStatus != nil && virusStatus != nil {
pmgInst.Quarantine = &models.PMGQuarantineTotals{
Spam: int(spamStatus.Count.Int64()), Virus: int(virusStatus.Count.Int64()),
}
}
}
if instanceCfg.MonitorDomainStats {
if domains, err := client.ListRelayDomains(ctx); err != nil {
recordReadError("pmg_relay_domains", err)
if debugEnabled {
log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG relay domains")
}
} else if len(domains) > 0 {
relayDomains := make([]models.PMGRelayDomain, 0, len(domains))
for _, domain := range domains {
relayDomains = append(relayDomains, models.PMGRelayDomain{
Domain: strings.TrimSpace(domain.Domain),
Comment: strings.TrimSpace(domain.Comment),
})
}
pmgInst.RelayDomains = relayDomains
}
end := time.Now()
start := end.Add(-24 * time.Hour)
if stats, err := client.GetDomainStatistics(ctx, start.Unix(), end.Unix()); err != nil {
recordReadError("pmg_domain_statistics", err)
if debugEnabled {
log.Debug().Err(err).Str("instance", instanceName).Msg("failed to fetch PMG domain statistics")
}
} else if len(stats) > 0 {
domainStats := make([]models.PMGDomainStat, 0, len(stats))
for _, entry := range stats {
d := strings.TrimSpace(entry.Domain)
if d == "" {
continue
}
domainStats = append(domainStats, models.PMGDomainStat{
Domain: d,
MailCount: entry.Count.Float64(),
SpamCount: entry.SpamCount.Float64(),
VirusCount: entry.VirusCount.Float64(),
Bytes: entry.Bytes.Float64(),
})
}
pmgInst.DomainStats = domainStats
pmgInst.DomainStatsAsOf = end
}
}
// Reads cut short by the poll's own deadline or cancellation are a
// whole-poll failure, not one broken endpoint, and keep their backoff.
if ctxErr := ctx.Err(); ctxErr != nil && readErr != nil {
pollErr = stderrors.Join(ctxErr, readErr)
}
m.state.UpdatePMGBackups(instanceName, pmgBackups)
m.state.UpdatePMGInstance(pmgInst)
log.Info().
Str("instance", instanceName).
Str("status", pmgInst.Status).
Int("nodes", len(pmgInst.Nodes)).
Msg("PMG instance updated in state")
// Check PMG metrics against alert thresholds
if m.alertManager != nil {
m.alertManager.CheckPMG(pmgInst)
}
}
// preserveUnreadPMGNodeBackups carries forward previous backups for nodes
// whose backup list was not read this cycle, mirroring
// preserveFailedStorageBackups for PVE storages. When the cluster read failed
// no node was enumerated, so every previous node is unread. When the cluster
// read succeeded, a node it no longer lists is gone and its backups drop.
func preserveUnreadPMGNodeBackups(instanceName string, previous, current []models.PMGBackup, clusterUnread bool, enumeratedNodes, readNodes map[string]struct{}) ([]models.PMGBackup, []string) {
existing := make(map[string]struct{}, len(current))
for _, backup := range current {
existing[backup.ID] = struct{}{}
}
preserved := make(map[string]struct{})
for _, backup := range previous {
if backup.Instance != instanceName {
continue
}
if _, read := readNodes[backup.Node]; read {
continue
}
if _, enumerated := enumeratedNodes[backup.Node]; !enumerated && !clusterUnread {
continue
}
if _, duplicate := existing[backup.ID]; duplicate {
continue
}
current = append(current, backup)
existing[backup.ID] = struct{}{}
preserved[backup.Node] = struct{}{}
}
if len(preserved) == 0 {
return current, nil
}
nodes := make([]string, 0, len(preserved))
for node := range preserved {
nodes = append(nodes, node)
}
sort.Strings(nodes)
return current, nodes
}
// GetState returns the current state