Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"crypto/sha256"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
proxmoxrecoverymapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
const (
pbsBackupSnapshotFetchWorkers = 5
pveGuestSnapshotFetchWorkers = 8
// runningVzdumpClockSkew is the slack applied when deciding whether a
// backup file's ctime falls inside a running vzdump task's window, so a
// small clock offset between Pulse and the node cannot misclassify the
// task's own partial output as a completed backup.
runningVzdumpClockSkew = 2 * time.Minute
// pbsBackupSnapshotsPerGroupLimit bounds how many real snapshots are
// retained per backup group (newest first). It must comfortably exceed
// common PBS keep policies: groups are always represented by real fetched
// snapshots, never synthesized group metadata, because a synthesized entry
// has no verification, size, file, or per-snapshot time data (issue #1541).
pbsBackupSnapshotsPerGroupLimit = 100
// pbsBackupLiveStateLimit bounds the per-instance PBS backup list as a
// whole. Groups are processed newest-first so the newest restore points
// win when the limit is hit (issue #1524).
pbsBackupLiveStateLimit = 5000
)
func pveBackupTemplateSubjectKey(instance, guestType, node string, vmid int) string {
return alerts.BuildBackupPVETemplateSubjectKey(instance, guestType, node, vmid)
}
func (m *Monitor) updatePVEBackupTemplateSubjectsForType(instanceName, guestType string, subjects map[string]struct{}) {
if m == nil {
return
}
instanceName = strings.TrimSpace(instanceName)
guestType = strings.TrimSpace(guestType)
if instanceName == "" || guestType == "" {
return
}
m.mu.Lock()
defer m.mu.Unlock()
if m.pveBackupInventoryReady == nil {
m.pveBackupInventoryReady = make(map[string]map[string]bool)
}
if m.pveBackupInventoryReady[instanceName] == nil {
m.pveBackupInventoryReady[instanceName] = make(map[string]bool)
}
m.pveBackupInventoryReady[instanceName][guestType] = true
if m.pveBackupTemplateSubjects == nil {
m.pveBackupTemplateSubjects = make(map[string]map[string]struct{})
}
existing := m.pveBackupTemplateSubjects[instanceName]
if existing == nil {
existing = make(map[string]struct{})
}
prefix := instanceName + "\x00" + guestType + "\x00"
for key := range existing {
if strings.HasPrefix(key, prefix) {
delete(existing, key)
}
}
for key := range subjects {
if key != "" {
existing[key] = struct{}{}
}
}
m.pveBackupTemplateSubjects[instanceName] = existing
}
func (m *Monitor) updatePVEBackupTemplateSubjectsFromClusterResources(instanceName string, resources []proxmox.ClusterResource) {
qemuTemplates := make(map[string]struct{})
lxcTemplates := make(map[string]struct{})
for _, res := range resources {
if res.Template != 1 {
continue
}
switch strings.TrimSpace(res.Type) {
case "qemu":
if key := pveBackupTemplateSubjectKey(instanceName, "qemu", res.Node, res.VMID); key != "" {
qemuTemplates[key] = struct{}{}
}
case "lxc":
if key := pveBackupTemplateSubjectKey(instanceName, "lxc", res.Node, res.VMID); key != "" {
lxcTemplates[key] = struct{}{}
}
}
}
m.updatePVEBackupTemplateSubjectsForType(instanceName, "qemu", qemuTemplates)
m.updatePVEBackupTemplateSubjectsForType(instanceName, "lxc", lxcTemplates)
}
func pveBackupPermissionWarning(instanceCfg *config.PVEInstance) string {
identity := "the configured user and API token in the saved PVE connection"
if instanceCfg != nil && strings.TrimSpace(instanceCfg.TokenName) == "" {
identity = "the configured login credentials or manually supplied API token in the saved PVE connection"
}
return "Check " + identity + ". For a privilege-separated token, check both user and token scopes without disabling privilege separation. Verify the permissions required by the rejected endpoint on your installed PVE version; a failed read alone does not identify a missing role. Do not delete nodes, remove registration state or rotate credentials to diagnose an empty backup table."
}
func pveBackupAccessWarning(instanceCfg *config.PVEInstance, endpoint string, err error) string {
status, known := proxmox.APIErrorStatus(err)
cause := "PVE backup inventory access failed; authentication or permission cause is unconfirmed."
if known && status == 401 {
cause = "PVE rejected backup inventory authentication (HTTP 401); this is not evidence of a missing role."
} else if known && status == 403 {
cause = "PVE denied backup inventory access (HTTP 403); the required permission and scope need verification."
}
// Endpoint comes from the actual attempted node/storage operation, never
// from provider error text. Bound and quote it to keep controls out of logs
// and warnings. No credential, token ID or provider body is reproduced.
endpointRunes := []rune(strings.ToValidUTF8(endpoint, "?"))
if len(endpointRunes) > 256 {
endpoint = string(endpointRunes[:256]) + "…"
} else {
endpoint = string(endpointRunes)
}
return cause + " Rejected endpoint: " + strconv.Quote(endpoint) + ". " + pveBackupPermissionWarning(instanceCfg)
}
func (m *Monitor) recordPVEBackupAccessWarning(ctx context.Context, instanceName, endpoint string, err error) {
unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
if !current {
return
}
defer unlock()
warning := pveBackupAccessWarning(m.getInstanceConfig(instanceName), endpoint, err)
m.mu.Lock()
if m.backupPermissionWarnings == nil {
m.backupPermissionWarnings = make(map[string]string)
}
m.backupPermissionWarnings[instanceName] = warning
m.mu.Unlock()
log.Warn().Str("instance", instanceName).Str("guidance", warning).Msg("PVE backup inventory access failed")
}
func (m *Monitor) backupInventoryScopeForAlerts() *alerts.BackupInventoryScope {
if m == nil {
return nil
}
scope := &alerts.BackupInventoryScope{
PVEOrphanInventoryReady: make(map[string]map[string]bool),
PVETemplateSubjects: make(map[string]struct{}),
}
m.mu.RLock()
defer m.mu.RUnlock()
for instance, readyByType := range m.pveBackupInventoryReady {
if len(readyByType) == 0 {
continue
}
scope.PVEOrphanInventoryReady[instance] = make(map[string]bool, len(readyByType))
for guestType, ready := range readyByType {
scope.PVEOrphanInventoryReady[instance][guestType] = ready
}
}
for _, subjects := range m.pveBackupTemplateSubjects {
for key := range subjects {
scope.PVETemplateSubjects[key] = struct{}{}
}
}
return scope
}
// A volume ID names a path inside a storage, not an independent node's disk.
// Shared repositories keep their existing IDs; local IDs use the exact read
// origin, never the guest's current node. Length-prefix each UTF-8 byte string
// before hashing so delimiters, case and field boundaries remain significant.
func pveStorageBackupID(instance, storageNode, storage, volume string) string {
if storageNode == "" {
return fmt.Sprintf("%s-%s", instance, volume)
}
tuple := fmt.Sprintf("%d:%s%d:%s%d:%s%d:%s", len(instance), instance,
len(storageNode), storageNode, len(storage), storage, len(volume), volume)
// No hyphen in this namespace/digest: it cannot equal a legacy shared ID,
// whose connection and volume are always separated by a hyphen.
return fmt.Sprintf("pvelocal1:%x", sha256.Sum256([]byte(tuple)))
}
// Guest kind belongs to the artifact discriminator, not the repository label
// or incidental filename text. In particular, "qemu-archive" may contain LXC
// backups. Keep unknown artifacts unknown rather than borrowing a label's type.
func pveStorageBackupType(content proxmox.StorageContent) string {
if content.VMID == 0 {
return "host"
}
volume := content.Volid
// Strip an optional storage prefix, but not a colon in a prefixless PBS
// timestamp (backup/ct/101/2026-10-27T00:00:00Z).
if colon := strings.IndexByte(volume, ':'); colon >= 0 && !strings.Contains(volume[:colon], "/") {
volume = volume[colon+1:]
}
volume = strings.TrimPrefix(volume, "backup/")
var kind, encodedID string
switch {
case strings.HasPrefix(volume, "vzdump-qemu-"):
kind, encodedID = "qemu", strings.TrimPrefix(volume, "vzdump-qemu-")
case strings.HasPrefix(volume, "vzdump-lxc-"):
kind, encodedID = "lxc", strings.TrimPrefix(volume, "vzdump-lxc-")
case strings.HasPrefix(volume, "vm/"):
kind, encodedID = "qemu", strings.TrimPrefix(volume, "vm/")
case strings.HasPrefix(volume, "ct/"):
kind, encodedID = "lxc", strings.TrimPrefix(volume, "ct/")
}
if kind != "" {
if strings.HasPrefix(volume, "vzdump-") {
if end := strings.IndexAny(encodedID, "-."); end >= 0 {
encodedID = encodedID[:end]
}
} else {
encodedID, _, _ = strings.Cut(encodedID, "/")
}
// A kind prefix is not sufficient subject evidence. Inconsistent
// metadata must not credit either guest, lend its current placement,
// or confirm a direct PBS snapshot. Keep the raw archive as unknown;
// a format fallback cannot overrule a malformed/conflicting subject.
for _, digit := range encodedID {
if digit < '0' || digit > '9' {
return "unknown"
}
}
id, err := strconv.Atoi(encodedID)
if err != nil || id <= 0 || id != content.VMID {
return "unknown"
}
return kind
}
// Explicit PBS formats remain a fallback for paths without a recognised
// discriminator. A substring of an unrelated format is not type evidence.
switch content.Format {
case "pbs-vm":
return "qemu"
case "pbs-ct":
return "lxc"
default:
return "unknown"
}
}
func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node, nodeEffectiveStatus map[string]string) {
if ctx.Err() != nil {
return
}
var allBackups []models.StorageBackup
var pbsConfirmations []models.PBSGuestConfirmation
hasPBSDirectConnection := m.config != nil && len(m.config.PBSInstances) > 0
seenBackups := make(map[string]bool) // Collapse repeated views of the same repository only
hadSuccessfulNode := false // Track if at least one node responded successfully
successfulNodeQueries := 0 // Number of nodes whose storage list was enumerated
storagesWithBackup := 0 // Number of storages that should contain backups
contentSuccess := 0 // Number of successful storage content fetches
contentFailures := 0 // Number of failed storage content fetches
storageQueryErrors := 0 // Number of nodes where storage list could not be queried
hadPermissionError := false // Track if any permission errors occurred this cycle
permissionFailureCount := 0 // Number of node/storage reads rejected by authorization
storagePreserveNeeded := map[string]struct{}{}
confirmationPreserveNeeded := map[string]struct{}{}
unreadNodes := map[string]struct{}{}
failedLocalStorages := map[pveLocalBackupStorage]struct{}{}
storageSuccess := map[string]struct{}{}
sharedStorageSuccess := map[string]struct{}{}
unqueryableStorages := map[string]struct{}{}
skippedLocalStorages := map[pveLocalBackupStorage]struct{}{}
// Where unread reads were attempted, so the provider observation can name
// them. A denied entry with no storage is a rejected node storage list.
sharedUnreadReads := map[pveLocalBackupStorage]struct{}{}
deniedReads := map[pveLocalBackupStorage]struct{}{}
readState := m.backupReadStateForInstance(instanceName)
// Build guest lookup map to find actual node for each VMID
snapshot := m.state.GetSnapshot()
guestInfo := buildProxmoxGuestInfoIndex(readState)
guestNodeMap := make(map[int]string) // VMID -> actual node name
populateGuestNodeMapFromReadState(readState, instanceName, guestNodeMap)
// vzdump writes archives under their final name, so a backup that is
// still running is already listed by the storage content API as a
// growing partial file. Correlate content entries with the live vzdump
// tasks polled just before this pass (pollBackupTasks runs first in the
// cycle): a guest backup file created at or after a running task's start
// is that task's partial output, not a completed backup.
// Local archive paths belong to the disk on the task's executing node,
// not to the guest's current placement. Shared/remote repositories and
// tasks without a node retain the conservative instance/VMID fallback.
runningVzdumpStart := make(map[int]map[string]time.Time) // VMID -> task node -> earliest start
for _, task := range snapshot.PVEBackups.BackupTasks {
if task.Instance != instanceName || task.VMID <= 0 || !task.EndTime.IsZero() {
continue
}
status := strings.ToLower(strings.TrimSpace(task.Status))
if status != "" && status != "running" {
continue
}
if runningVzdumpStart[task.VMID] == nil {
runningVzdumpStart[task.VMID] = make(map[string]time.Time)
}
byNode := runningVzdumpStart[task.VMID]
if existing, ok := byNode[task.Node]; !ok || task.StartTime.Before(existing) {
byNode[task.Node] = task.StartTime
}
}
// For each node, get storage and check content
for _, node := range nodes {
if ctx.Err() != nil || nodeEffectiveStatus[node.Node] != "online" {
// A spent child budget is unread scope, not an empty listing.
// Finish classifying the remaining nodes without another API call.
unreadNodes[node.Node] = struct{}{}
for _, storageName := range storageNamesForNode(readState, instanceName, node.Node) {
confirmationPreserveNeeded[storageName] = struct{}{}
}
continue
}
// Get storage for this node - retry once on timeout
var storages []proxmox.Storage
var err error
storageAttempts:
for attempt := 1; attempt <= 2; attempt++ {
if ctx.Err() != nil {
err = ctx.Err()
break
}
storages, err = client.GetStorage(ctx, node.Node)
if ctx.Err() != nil {
err = ctx.Err()
break
}
if err == nil {
break // Success
}
// Check if it's a timeout error
errStr := err.Error()
if strings.Contains(errStr, "timeout") || strings.Contains(errStr, "deadline exceeded") {
if attempt == 1 {
log.Warn().
Str("node", node.Node).
Str("instance", instanceName).
Msg("Storage query timed out, retrying within the remaining backup poll budget")
// Do not add two seconds per remaining node after the
// shared budget expires or the monitor is stopped.
timer := time.NewTimer(2 * time.Second)
select {
case <-ctx.Done():
timer.Stop()
err = ctx.Err()
break storageAttempts
case <-timer.C:
}
continue
}
}
// Non-timeout error or second attempt failed
break
}
if err != nil {
if isPVEBackupPermissionError(err) {
hadPermissionError = true
permissionFailureCount++
deniedReads[pveLocalBackupStorage{node: node.Node}] = struct{}{}
m.recordPVEBackupAccessWarning(ctx, instanceName, fmt.Sprintf("/nodes/%s/storage", node.Node), err)
}
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_for_backups", instanceName, err).WithNode(node.Node)
log.Warn().Err(monErr).Str("node", node.Node).Msg("failed to get storage for backups - skipping node")
for _, storageName := range storageNamesForNode(readState, instanceName, node.Node) {
confirmationPreserveNeeded[storageName] = struct{}{}
}
unreadNodes[node.Node] = struct{}{}
storageQueryErrors++
continue
}
hadSuccessfulNode = true
successfulNodeQueries++
// For each storage that can contain backups or templates
for _, storage := range storages {
// Check if storage supports backup content
if !strings.Contains(storage.Content, "backup") {
continue
}
storagesWithBackup++
// Establish repository scope before admission: a disabled local
// disk is no more covered by a peer than a failed content read.
storageNode := ""
if storage.Shared == 0 && !isInherentlySharedStorageType(storage.Type) {
storageNode = node.Node
}
if !storageContentQueryable(storage) {
// Do not query disabled content to resolve its membership. A
// shared read may cover a skipped peer, but local assignment
// needs independent evidence after every node has been read.
if storageNode != "" {
skippedLocalStorages[pveLocalBackupStorage{node: storageNode, storage: storage.Storage}] = struct{}{}
} else {
unqueryableStorages[storage.Storage] = struct{}{}
storagePreserveNeeded[storage.Storage] = struct{}{}
sharedUnreadReads[pveLocalBackupStorage{node: node.Node, storage: storage.Storage}] = struct{}{}
}
continue
}
// Classify every unattempted storage after expiry, but never start a
// late read or accept a nil-error response returned after its budget.
var contents []proxmox.StorageContent
err := ctx.Err()
if err == nil {
contents, err = client.GetStorageContent(ctx, node.Node, storage.Storage)
if ctx.Err() != nil {
err = ctx.Err()
}
}
if err != nil {
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_storage_content", instanceName, err).WithNode(node.Node)
// Check if this is a permission error
if isPVEBackupPermissionError(err) {
hadPermissionError = true
permissionFailureCount++
deniedReads[pveLocalBackupStorage{node: node.Node, storage: storage.Storage}] = struct{}{}
m.recordPVEBackupAccessWarning(ctx, instanceName, fmt.Sprintf("/nodes/%s/storage/%s/content", node.Node, storage.Storage), err)
} else {
log.Debug().Err(monErr).
Str("node", node.Node).
Str("storage", storage.Storage).
Msg("Failed to get storage content")
}
if storageNode != "" {
failedLocalStorages[pveLocalBackupStorage{node: storageNode, storage: storage.Storage}] = struct{}{}
} else {
storagePreserveNeeded[storage.Storage] = struct{}{}
sharedUnreadReads[pveLocalBackupStorage{node: node.Node, storage: storage.Storage}] = struct{}{}
}
contentFailures++
continue
}
contentSuccess++
storageSuccess[storage.Storage] = struct{}{}
if storageNode == "" {
sharedStorageSuccess[storage.Storage] = struct{}{}
}
delete(storagePreserveNeeded, storage.Storage)
// Convert to models
for _, content := range contents {
backupID := pveStorageBackupID(instanceName, storageNode, storage.Storage, content.Volid)
// Shared views collapse, but matching paths on independent local
// disks must survive both live inventory and recovery ingestion.
if seenBackups[backupID] {
continue
}
seenBackups[backupID] = true
// Skip templates and ISOs - they're not backups
if content.Content == "vztmpl" || content.Content == "iso" {
continue
}
backupType := pveStorageBackupType(content)
// Determine the correct node: for guest backups (VMID > 0), use the actual guest's node
// For host backups (VMID == 0), use the node where the backup was found
backupNode := node.Node
if content.VMID > 0 {
if actualNode, found := guestNodeMap[content.VMID]; found {
currentKind := guestInfo[proxmoxmapperKey(instanceName, actualNode, content.VMID)].ResourceType
// A reused VMID does not move a removed guest's archive
// to the opposite-kind occupant's current placement.
matchingKind := backupType == "qemu" && currentKind == unifiedresources.ResourceTypeVM ||
backupType == "lxc" && currentKind == unifiedresources.ResourceTypeSystemContainer
if matchingKind {
backupNode = actualNode
}
}
// If not found in map, fall back to queried node (shouldn't happen normally)
}
// Classify by storage type only. A dir/NFS storage is free to be
// NAMED "pbs-…" (e.g. "pbs-backup"), and treating the name prefix
// as PBS silently dropped its vzdump backups whenever a direct PBS
// connection was also configured (#1592).
isPBSStorage := storage.Type == "pbs"
if isPBSStorage && hasPBSDirectConnection {
// The direct PBS connection is authoritative for the
// backup list, but which storage view listed a snapshot
// is evidence only this connection has: it helps
// attribute a snapshot whose VMID exists on several
// clusters (#1639). Keep that association, tagged with
// the storage it came from, even though the raw entry is
// dropped - a listing only proves this connection can see
// the snapshot, so consumers need the storage to tell an
// exclusive view from a shared or synced datastore.
if confirmationType := pbsGuestConfirmationType(backupType); confirmationType != "" && content.VMID > 0 && content.CTime > 0 {
pbsConfirmations = append(pbsConfirmations, models.PBSGuestConfirmation{
Storage: storage.Storage,
BackupType: confirmationType,
VMID: content.VMID,
Time: content.CTime,
})
}
log.Debug().
Str("instance", instanceName).
Str("node", node.Node).
Str("storage", storage.Storage).
Str("volid", content.Volid).
Msg("Skipping PBS backup from PVE storage - PBS direct connection is authoritative")
continue
}
// Check verification status for PBS backups
verified := false
verificationInfo := ""
if isPBSStorage {
// Check if verified flag is set
if content.Verified > 0 {
verified = true
}
// Also check verification map if available
if content.Verification != nil {
if state, ok := content.Verification["state"].(string); ok {
verified = (state == "ok")
verificationInfo = state
}
}
}
// Partial-output window: the file belongs to a vzdump task
// that is still running. PBS-type storages additionally list
// in-flight snapshots without a size (no manifest yet), which
// covers backups submitted by another cluster or host.
inProgress := false
if content.VMID > 0 {
for taskNode, start := range runningVzdumpStart[content.VMID] {
if storageNode != "" && taskNode != "" && taskNode != storageNode {
continue
}
if !time.Unix(content.CTime, 0).Before(start.Add(-runningVzdumpClockSkew)) {
inProgress = true
break
}
}
if isPBSStorage && content.Size == 0 {
inProgress = true
}
}
backup := models.StorageBackup{
StorageNode: storageNode,
ID: backupID,
Storage: storage.Storage,
Node: backupNode,
Instance: instanceName,
Type: backupType,
VMID: content.VMID,
Time: time.Unix(content.CTime, 0),
CTime: content.CTime,
Size: int64(content.Size),
Format: content.Format,
Notes: content.Notes,
Protected: content.Protected > 0,
Volid: content.Volid,
IsPBS: isPBSStorage,
Verified: verified,
Verification: verificationInfo,
InProgress: inProgress,
}
allBackups = append(allBackups, backup)
}
}
}
if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
return
}
// PVE also lists node-restricted local storage as disabled on excluded
// nodes. A peer's matching name alone cannot distinguish that from an
// unread local disk. Resolve only that ambiguity with the ordinary bounded
// datacenter catalogue, once per poll; failed/missing config is not absence.
storageAssignments := make(map[string]string)
for scope := range skippedLocalStorages {
if ctx.Err() != nil {
break
}
if _, readElsewhere := storageSuccess[scope.storage]; !readElsewhere {
continue
}
if configured, err := client.GetAllStorage(ctx); err == nil && ctx.Err() == nil {
for _, storage := range configured {
storageAssignments[storage.Storage] = storage.Nodes
}
}
break
}
if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
// Explicit cancellation still withdraws the entire enumeration.
return
}
// A later storage/assignment timeout cannot discard earlier accepted
// listings. Drop only the spent child budget: the owner value still fences
// runtime shutdown, connection replacement and retirement atomically.
publicationCtx := ctx
if ctx.Err() == context.DeadlineExceeded {
publicationCtx = context.WithoutCancel(ctx)
}
unlock, current := m.beginPVEBackupPublication(publicationCtx, instanceName)
if !current {
return
}
defer unlock()
for scope := range skippedLocalStorages {
if assignment, known := storageAssignments[scope.storage]; known &&
clusterStorageRestrictedToOtherNodes(assignment, scope.node) {
continue
}
failedLocalStorages[scope] = struct{}{}
contentFailures++
}
// A shared storage skipped on one node is unread only if no node listed it.
for storageName := range unqueryableStorages {
if _, ok := storageSuccess[storageName]; !ok {
contentFailures++
}
}
// Resolve name-wide skips/shared failures after every node, regardless of
// order. A local disk failure remains scoped to its own node below.
if len(unreadNodes) > 0 {
// Confirmation rows have no node provenance. An unread node cannot
// establish their absence, even after its live storage inventory was
// replaced. A successful read of that storage below still wins.
for _, confirmation := range m.state.PBSGuestConfirmationsForInstance(instanceName) {
confirmationPreserveNeeded[confirmation.Storage] = struct{}{}
}
}
for storageName := range storageSuccess {
delete(storagePreserveNeeded, storageName)
}
for storageName := range storagePreserveNeeded {
confirmationPreserveNeeded[storageName] = struct{}{}
}
for storageName := range storageSuccess {
delete(confirmationPreserveNeeded, storageName)
}
unreadScopes := pveUnreadProtectionScopes(pveUnreadReads{
nodes: unreadNodes,
local: failedLocalStorages,
shared: storagePreserveNeeded,
sharedReads: sharedUnreadReads,
denied: deniedReads,
})
// Carry-forward serves the live inventory, not a new provider observation.
// Only rows actually returned this poll may renew point-level evidence.
observedBackups := allBackups
allBackups, preservedStorages := preserveFailedStorageBackups(instanceName, snapshot, pveBackupReadFailures{
storages: storagePreserveNeeded,
nodes: unreadNodes,
local: failedLocalStorages,
sharedSuccess: sharedStorageSuccess,
}, allBackups)
if len(preservedStorages) > 0 {
log.Warn().
Str("instance", instanceName).
Strs("storages", preservedStorages).
Msg("Preserving previous storage backup data due to partial failures")
}
pbsConfirmations, preservedConfirmationStorages := preserveFailedPBSGuestConfirmations(
m.state.PBSGuestConfirmationsForInstance(instanceName), confirmationPreserveNeeded, pbsConfirmations)
if len(preservedConfirmationStorages) > 0 {
log.Warn().
Str("instance", instanceName).
Strs("storages", preservedConfirmationStorages).
Msg("Preserving previous PBS guest confirmation evidence due to partial failures")
}
protectionObservedAt := time.Now().UTC()
protectionObservation, protectionObservationErr := buildPVEProtectionProviderObservation(
instanceName,
pveProtectionCollectionStats{
NodeCount: len(nodes),
SuccessfulNodeQueries: successfulNodeQueries,
BackupStorageCount: storagesWithBackup,
ContentSuccesses: contentSuccess,
ContentFailures: contentFailures,
PermissionFailureCount: permissionFailureCount,
UnreadScopes: unreadScopes,
},
protectionObservedAt,
)
if protectionObservationErr != nil {
log.Warn().
Err(protectionObservationErr).
Str("instance", instanceName).
Msg("Failed to build PVE protection provider observation")
}
// Decide whether to keep existing backups when every query failed
if shouldPreserveBackups(len(nodes), hadSuccessfulNode, storagesWithBackup, contentSuccess) {
if protectionObservationErr == nil {
m.ingestProtectionProviderObservationsAsync(
[]recovery.ProtectionProviderObservation{protectionObservation},
)
}
if len(nodes) > 0 && !hadSuccessfulNode {
log.Warn().
Str("instance", instanceName).
Int("nodes", len(nodes)).
Int("errors", storageQueryErrors).
Msg("Failed to query storage on all nodes; keeping previous backup list")
} else if storagesWithBackup > 0 && contentSuccess == 0 {
log.Warn().
Str("instance", instanceName).
Int("storages", storagesWithBackup).
Int("failures", contentFailures).
Msg("All storage content queries failed; keeping previous backup list")
}
return
}
// Update state with storage backups for this instance
m.state.UpdateStorageBackupsForInstance(instanceName, allBackups)
m.state.UpdatePBSGuestConfirmationsForInstance(instanceName, pbsConfirmations)
// Best-effort ingestion into recovery store (for rollups / unified backups UX).
points, evidenceErr := proxmoxrecoverymapper.FromPVEStorageBackupsWithEvidence(
observedBackups,
guestInfo,
protectionObservedAt,
)
if evidenceErr != nil {
log.Warn().
Err(evidenceErr).
Str("instance", instanceName).
Msg("Failed to attach PVE recovery evidence; preserving existing points")
if protectionObservationErr == nil {
m.ingestProtectionProviderObservationsAsync(
[]recovery.ProtectionProviderObservation{protectionObservation},
)
}
} else {
observations := []recovery.ProtectionProviderObservation{}
if protectionObservationErr == nil {
observations = append(observations, protectionObservation)
}
batch := recoveryIngestBatch{points: points, observations: observations}
// Live inventory can be empty after a monitor replacement even when
// durable recovery history exists. A partial scan may add observations,
// but cannot establish absence anywhere in the instance-wide scope.
if protectionObservationErr == nil &&
protectionObservation.HistoryCompleteness == recovery.ProtectionHistoryComplete {
batch.reconcile = &recoveryReconcileScope{
provider: string(recovery.ProviderProxmoxPVE),
idPrefix: "pve-backup:",
instance: instanceName,
}
}
m.enqueueRecoveryIngest(batch)
}
// Sync backup times to VMs/Containers and republish them to canonical resources.
m.syncGuestBackupTimesAndResourceStore()
// Optional collection expiry is not a failed recovery-store read. Keep
// existing alerts for the next ordinary evaluation rather than issuing an
// already-expired read that would manufacture an evaluation-outage alert.
if ctx.Err() == nil {
m.checkBackupAlerts(ctx)
}
// Only a complete accepted enumeration can clear an earlier access
// warning. An unread scope has not demonstrated permission recovery.
if !hadPermissionError && protectionObservationErr == nil &&
protectionObservation.HistoryCompleteness == recovery.ProtectionHistoryComplete {
m.mu.Lock()
delete(m.backupPermissionWarnings, instanceName)
m.mu.Unlock()
}
log.Debug().
Str("instance", instanceName).
Int("count", len(allBackups)).
Msg("Storage backups polled")
// Immediately broadcast the updated state so frontend sees new backups
m.broadcastStateUpdate()
}
// pbsGuestConfirmationType maps a PVE storage-content backup type to PBS
// snapshot nomenclature. Host and unclassifiable entries return "" and are
// not usable as guest attribution evidence.
func pbsGuestConfirmationType(backupType string) string {
switch backupType {
case "qemu":
return "vm"
case "lxc":
return "ct"
default:
return ""
}
}
func shouldPreserveBackups(nodeCount int, hadSuccessfulNode bool, storagesWithBackup, contentSuccess int) bool {
if nodeCount > 0 && !hadSuccessfulNode {
return true
}
if storagesWithBackup > 0 && contentSuccess == 0 {
return true
}
return false
}
func (m *Monitor) syncGuestBackupTimesAndResourceStore() {
if m == nil || m.state == nil {
return
}
m.state.SyncGuestBackupTimes()
scope := m.mockModeFence.begin()
m.updateResourceStore(m.state.GetSnapshot(), scope)
}
func (m *Monitor) backupReadStateForInstance(instanceName string) unifiedresources.ReadState {
if m == nil {
return nil
}
readState := m.currentModeReadState()
if backupReadStateHasGuestForInstance(readState, instanceName) || m.state == nil {
return readState
}
scope := m.mockModeFence.begin()
snapshot := m.state.GetSnapshot()
if !backupSnapshotHasGuestForInstance(snapshot, instanceName) {
return readState
}
m.updateResourceStore(snapshot, scope)
readState = m.currentModeReadState()
if backupReadStateHasGuestForInstance(readState, instanceName) {
return readState
}
return monitorUnifiedStateViewFromSnapshot(snapshot).readState
}
func backupReadStateHasGuestForInstance(readState unifiedresources.ReadState, instanceName string) bool {
if readState == nil {
return false
}
for _, vm := range readState.VMs() {
if vm != nil && vm.Instance() == instanceName {
return true
}
}
for _, ct := range readState.Containers() {
if ct != nil && ct.Instance() == instanceName {
return true
}
}
return false
}
func backupSnapshotHasGuestForInstance(snapshot models.StateSnapshot, instanceName string) bool {
for _, vm := range snapshot.VMs {
if vm.Instance == instanceName {
return true
}
}
for _, ct := range snapshot.Containers {
if ct.Instance == instanceName {
return true
}
}
return false
}
func storageNamesForNode(readState unifiedresources.ReadState, instanceName, nodeName string) []string {
if readState == nil || nodeName == "" {
return nil
}
var storages []string
for _, storage := range readState.StoragePools() {
if storage == nil || storage.Instance() != instanceName {
continue
}
if storage.Name() == "" {
continue
}
if !strings.Contains(storage.Content(), "backup") {
continue
}
if storage.Node() == nodeName {
storages = append(storages, storage.Name())
continue
}
for _, node := range storage.AccessibleNodes() {
if node == nodeName {
storages = append(storages, storage.Name())
break
}
}
}
return storages
}
// preserveFailedPBSGuestConfirmations carries forward the PBS guest
// confirmation evidence of storages whose content query failed this cycle,
// mirroring preserveFailedStorageBackups. Publishing the partial (possibly
// empty) set instead would evict attribution evidence on a transient poll
// failure and flip collision-VMID guests between cycles (#1639).
func preserveFailedPBSGuestConfirmations(previous []models.PBSGuestConfirmation, storagesToPreserve map[string]struct{}, current []models.PBSGuestConfirmation) ([]models.PBSGuestConfirmation, []string) {
if len(storagesToPreserve) == 0 || len(previous) == 0 {
return current, nil
}
existing := make(map[models.PBSGuestConfirmation]struct{}, len(current))
for _, confirmation := range current {
existing[confirmation] = struct{}{}
}
preserved := make(map[string]struct{})
for _, confirmation := range previous {
if _, ok := storagesToPreserve[confirmation.Storage]; !ok {
continue
}
if _, duplicate := existing[confirmation]; duplicate {
continue
}
current = append(current, confirmation)
existing[confirmation] = struct{}{}
preserved[confirmation.Storage] = struct{}{}
}
if len(preserved) == 0 {
return current, nil
}
storages := make([]string, 0, len(preserved))
for storage := range preserved {
storages = append(storages, storage)
}
sort.Strings(storages)
return current, storages
}
type pveLocalBackupStorage struct {
node string
storage string
}
// Name-wide failures cover shared storage. Local failed/skipped content reads
// and unread nodes must survive another node's success.
// Legacy rows without provenance are retained conservatively until a complete
// read replaces them; guest location is not evidence of storage read origin.
type pveBackupReadFailures struct {
storages map[string]struct{}
nodes map[string]struct{}
local map[pveLocalBackupStorage]struct{}
sharedSuccess map[string]struct{}
}
func (f pveBackupReadFailures) preserves(backup models.StorageBackup) bool {
if _, ok := f.storages[backup.Storage]; ok {
return true
}
if backup.StorageNode != "" {
if _, ok := f.nodes[backup.StorageNode]; ok {
return true
}
_, ok := f.local[pveLocalBackupStorage{node: backup.StorageNode, storage: backup.Storage}]
return ok
}
if _, ok := f.sharedSuccess[backup.Storage]; ok {
return false
}
if len(f.nodes) > 0 {
return true
}
for scope := range f.local {
if scope.storage == backup.Storage {
return true
}
}
return false
}
func preserveFailedStorageBackups(instanceName string, snapshot models.StateSnapshot, failures pveBackupReadFailures, current []models.StorageBackup) ([]models.StorageBackup, []string) {
if len(failures.storages) == 0 && len(failures.nodes) == 0 && len(failures.local) == 0 {
return current, nil
}
existing := make(map[string]struct{}, len(current))
for _, backup := range current {
existing[backup.ID] = struct{}{}
}
preserved := make(map[string]struct{})
for _, backup := range snapshot.PVEBackups.StorageBackups {
if backup.Instance != instanceName {
continue
}
if !failures.preserves(backup) {
continue
}
if _, duplicate := existing[backup.ID]; duplicate {
continue
}
current = append(current, backup)
existing[backup.ID] = struct{}{}
preserved[backup.Storage] = struct{}{}
}
if len(preserved) == 0 {
return current, nil
}
storages := make([]string, 0, len(preserved))
for storage := range preserved {
storages = append(storages, storage)
}
sort.Strings(storages)
return current, storages
}
func buildGuestLookupsFromReadState(readState unifiedresources.ReadState, metadataStore *config.GuestMetadataStore) (map[string]alerts.GuestLookup, map[string][]alerts.GuestLookup) {
byKey := make(map[string]alerts.GuestLookup)
byVMID := make(map[string][]alerts.GuestLookup)
if readState == nil {
if metadataStore != nil {
enrichWithPersistedMetadata(metadataStore, byVMID)
}
return byKey, byVMID
}
for _, vm := range readState.VMs() {
if vm == nil {
continue
}
info := alerts.GuestLookup{
ResourceID: makeGuestID(vm.Instance(), vm.Node(), vm.VMID()),
Name: vm.Name(),
Instance: vm.Instance(),
Node: vm.Node(),
Type: "qemu",
VMID: vm.VMID(),
Tags: vm.Tags(),
}
key := alerts.BuildGuestKey(vm.Instance(), vm.Node(), vm.VMID())
byKey[key] = info
vmidKey := strconv.Itoa(vm.VMID())
byVMID[vmidKey] = append(byVMID[vmidKey], info)
// Persist last-known name and type for this guest
if metadataStore != nil && vm.Name() != "" {
persistGuestIdentity(metadataStore, key, vm.Name(), info.Type)
}
}
for _, ct := range readState.Containers() {
if ct == nil {
continue
}
guestType := firstNonEmptyString(ct.ContainerType(), "lxc")
info := alerts.GuestLookup{
ResourceID: makeGuestID(ct.Instance(), ct.Node(), ct.VMID()),
Name: ct.Name(),
Instance: ct.Instance(),
Node: ct.Node(),
Type: guestType,
VMID: ct.VMID(),
Tags: ct.Tags(),
}
key := alerts.BuildGuestKey(ct.Instance(), ct.Node(), ct.VMID())
if _, exists := byKey[key]; !exists {
byKey[key] = info
}
vmidKey := strconv.Itoa(ct.VMID())
byVMID[vmidKey] = append(byVMID[vmidKey], info)
// Persist last-known name and type for this guest
if metadataStore != nil && ct.Name() != "" {
persistGuestIdentity(metadataStore, key, ct.Name(), guestType)
}
}
// Augment byVMID with persisted metadata for deleted guests
if metadataStore != nil {
enrichWithPersistedMetadata(metadataStore, byVMID)
}
return byKey, byVMID
}
func populateGuestNodeMapFromReadState(readState unifiedresources.ReadState, instanceName string, guestNodeMap map[int]string) {
if readState == nil {
return
}
for _, vm := range readState.VMs() {
if vm == nil || vm.Instance() != instanceName {
continue
}
guestNodeMap[vm.VMID()] = vm.Node()
}
for _, ct := range readState.Containers() {
if ct == nil || ct.Instance() != instanceName {
continue
}
guestNodeMap[ct.VMID()] = ct.Node()
}
}
// enrichWithPersistedMetadata adds entries from the metadata store for guests
// that no longer exist in the live inventory but have persisted identity data
func enrichWithPersistedMetadata(metadataStore *config.GuestMetadataStore, byVMID map[string][]alerts.GuestLookup) {
allMetadata := metadataStore.GetAll()
for guestKey, meta := range allMetadata {
if meta.LastKnownName == "" {
continue // No name persisted, skip
}
// Parse the guest key (format: instance:node:vmid)
// We need to extract instance, node, and vmid
var instance, node string
parts := strings.Split(guestKey, ":")
if len(parts) != 3 {
continue
}
instance, node = parts[0], parts[1]
vmid, err := strconv.Atoi(parts[2])
if err != nil {
continue
}
vmidKey := strconv.Itoa(vmid)
// Check if we already have a live entry for this exact guest
hasLiveEntry := false
for _, existing := range byVMID[vmidKey] {
if existing.Instance == instance && existing.Node == node && existing.VMID == vmid {
hasLiveEntry = true
break
}
}
// Only add persisted metadata if no live entry exists
if !hasLiveEntry {
byVMID[vmidKey] = append(byVMID[vmidKey], alerts.GuestLookup{
Name: meta.LastKnownName,
Instance: instance,
Node: node,
Type: meta.LastKnownType,
VMID: vmid,
})
}
}
}
// persistGuestIdentity updates the metadata store with the last-known name and type for a guest
func persistGuestIdentity(metadataStore *config.GuestMetadataStore, guestKey, name, guestType string) {
if err := metadataStore.RememberIdentity(guestKey, name, guestType); err != nil {
log.Error().Err(err).Str("guestKey", guestKey).Msg("failed to admit guest identity update")
}
}
func (m *Monitor) calculateBackupOperationTimeout(instanceName string) time.Duration {
const (
minTimeout = 2 * time.Minute
maxTimeout = 5 * time.Minute
timeoutPerGuest = 2 * time.Second
)
timeout := minTimeout
readState := m.backupReadStateForInstance(instanceName)
guestCount := 0
for _, vm := range readState.VMs() {
if vm != nil && vm.Instance() == instanceName && !vm.Template() {
guestCount++
}
}
for _, ct := range readState.Containers() {
if ct != nil && ct.Instance() == instanceName && !ct.Template() {
guestCount++
}
}
if guestCount > 0 {
dynamic := time.Duration(guestCount) * timeoutPerGuest
if dynamic > timeout {
timeout = dynamic
}
}
if timeout > maxTimeout {
return maxTimeout
}
return timeout
}
// pollPVEBackupsAndSnapshots runs the two backup-inventory scans under a shared
// bounded budget, then polls guest snapshots on the parent context so they get
// their own independent budget. A slow storage/backup scan can no longer starve
// snapshot discovery by exhausting the shared timeout before snapshots run.
func (m *Monitor) pollPVEBackupsAndSnapshots(parentCtx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node, nodeEffectiveStatus map[string]string, timeout time.Duration) {
if parentCtx == nil {
parentCtx = context.Background()
}
backupCtx, cancel := context.WithTimeout(parentCtx, timeout)
// Poll backup tasks
m.pollBackupTasks(backupCtx, instanceName, client)
// Poll storage backups - pass nodes to avoid duplicate API calls
m.pollStorageBackupsWithNodes(backupCtx, instanceName, client, nodes, nodeEffectiveStatus)
backupErr := backupCtx.Err()
cancel()
if backupErr != nil && parentCtx.Err() == nil {
log.Warn().
Str("instance", instanceName).
Err(backupErr).
Msg("Backup storage polling budget was exhausted before guest snapshot polling; continuing snapshots with their own bounded poll budget")
}
// Snapshots are independent backup inventory. Passing parentCtx (no deadline)
// lets pollGuestSnapshots establish its own bounded budget instead of
// inheriting an already-exhausted backup deadline and skipping entirely.
m.pollGuestSnapshots(parentCtx, instanceName, client)
}
// pollGuestSnapshots polls snapshots for all VMs and containers
func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, client PVEClientInterface) {
if ctx.Err() != nil {
return
}
log.Debug().Str("instance", instanceName).Msg("polling guest snapshots")
// The guest inventory comes from the fixture read view in mock mode.
scope := m.mockModeFence.begin()
readState := m.backupReadStateForInstance(instanceName)
var vms []models.VM
for _, vm := range readState.VMs() {
if vm == nil || vm.Instance() != instanceName {
continue
}
vms = append(vms, vmFromReadStateView(vm))
}
var containers []models.Container
for _, ct := range readState.Containers() {
if ct == nil || ct.Instance() != instanceName {
continue
}
containers = append(containers, containerFromReadStateView(ct))
}
previousSnapshots := make([]models.GuestSnapshot, 0)
if m.state != nil {
snapshot := m.state.GetSnapshot()
for _, snap := range snapshot.PVEBackups.GuestSnapshots {
if snap.Instance == instanceName {
previousSnapshots = append(previousSnapshots, snap)
}
}
}
guestKey := func(instance, node string, vmid int) string {
if instance == node {
return fmt.Sprintf("%s-%d", node, vmid)
}
return fmt.Sprintf("%s-%s-%d", instance, node, vmid)
}
guestLookups := make(map[string]alerts.GuestLookup, len(vms)+len(containers))
for _, vm := range vms {
key := alerts.BuildGuestKey(vm.Instance, vm.Node, vm.VMID)
guestLookups[key] = alerts.GuestLookup{
ResourceID: key,
Name: vm.Name,
Instance: vm.Instance,
Node: vm.Node,
Type: "qemu",
VMID: vm.VMID,
Tags: append([]string(nil), vm.Tags...),
}
}
for _, ct := range containers {
key := alerts.BuildGuestKey(ct.Instance, ct.Node, ct.VMID)
guestLookups[key] = alerts.GuestLookup{
ResourceID: key,
Name: ct.Name,
Instance: ct.Instance,
Node: ct.Node,
Type: firstNonEmptyString(ct.Type, "lxc"),
VMID: ct.VMID,
Tags: append([]string(nil), ct.Tags...),
}
}
activeGuests := 0
for _, vm := range vms {
if !vm.Template {
activeGuests++
}
}
for _, ct := range containers {
if !ct.Template {
activeGuests++
}
}
const (
minSnapshotTimeout = 60 * time.Second
maxSnapshotTimeout = 4 * time.Minute
snapshotTimeoutPerGuest = 2 * time.Second
)
timeout := minSnapshotTimeout
if activeGuests > 0 {
dynamic := time.Duration(activeGuests) * snapshotTimeoutPerGuest
if dynamic > timeout {
timeout = dynamic
}
}
if timeout > maxSnapshotTimeout {
timeout = maxSnapshotTimeout
}
if deadline, ok := ctx.Deadline(); ok {
remaining := time.Until(deadline)
if remaining <= 0 {
log.Warn().
Str("instance", instanceName).
Msg("Skipping guest snapshot polling; backup context deadline exceeded")
return
}
if timeout > remaining {
timeout = remaining
}
}
snapshotCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
log.Debug().
Str("instance", instanceName).
Int("guestCount", activeGuests).
Dur("timeout", timeout).
Msg("Guest snapshot polling budget established")
var allSnapshots []models.GuestSnapshot
deadlineExceeded := false
polledGuestKeys := make(map[string]struct{})
type guestSnapshotPollTarget struct {
key string
node string
guestType string
vmid int
vmState bool
}
targets := make([]guestSnapshotPollTarget, 0, activeGuests)
for _, vm := range vms {
if vm.Template {
continue
}
targets = append(targets, guestSnapshotPollTarget{
key: guestKey(instanceName, vm.Node, vm.VMID),
node: vm.Node,
guestType: "qemu",
vmid: vm.VMID,
vmState: true,
})
}
for _, ct := range containers {
if ct.Template {
continue
}
targets = append(targets, guestSnapshotPollTarget{
key: guestKey(instanceName, ct.Node, ct.VMID),
node: ct.Node,
guestType: "lxc",
vmid: ct.VMID,
vmState: false,
})
}
type guestSnapshotPollResult struct {
target guestSnapshotPollTarget
snapshots []models.GuestSnapshot
err error
polled bool
}
buildSnapshot := func(target guestSnapshotPollTarget, snap proxmox.Snapshot) models.GuestSnapshot {
return models.GuestSnapshot{
ID: fmt.Sprintf("%s-%s-%d-%s", instanceName, target.node, target.vmid, snap.Name),
Name: snap.Name,
Node: target.node,
Instance: instanceName,
Type: target.guestType,
VMID: target.vmid,
Time: time.Unix(snap.SnapTime, 0),
Description: snap.Description,
Parent: snap.Parent,
VMState: target.vmState,
}
}
fetchSnapshots := func(target guestSnapshotPollTarget) ([]proxmox.Snapshot, error) {
if target.guestType == "lxc" {
return client.GetContainerSnapshots(snapshotCtx, target.node, target.vmid)
}
return client.GetVMSnapshots(snapshotCtx, target.node, target.vmid)
}
// Bound waiting goroutines as well as active provider reads. A semaphore
// inside one goroutine per guest still leaves thousands of suspended
// collectors on a large estate. The dispatcher stops on cancellation, and
// unattempted targets remain unpolled so their prior listings survive.
queue := make(chan guestSnapshotPollTarget)
results := make(chan guestSnapshotPollResult, pveGuestSnapshotFetchWorkers)
var wg sync.WaitGroup
for i := 0; i < min(len(targets), pveGuestSnapshotFetchWorkers); i++ {
wg.Add(1)
go func() {
defer wg.Done()
for target := range queue {
// Dispatch and cancellation can become ready together. A queued
// target is not permission to start another expired provider read.
if snapshotCtx.Err() != nil {
return
}
snapshots, err := fetchSnapshots(target)
// A provider returning nil error after cancellation did not supply
// an accepted listing. Optional sizing can exhaust its budget later,
// but cannot retroactively invalidate a listing accepted here.
if err == nil {
err = snapshotCtx.Err()
}
result := guestSnapshotPollResult{target: target, err: err}
if err == nil {
result.polled = true
result.snapshots = make([]models.GuestSnapshot, 0, len(snapshots))
for _, snap := range snapshots {
result.snapshots = append(result.snapshots, buildSnapshot(target, snap))
}
// Only an accepted listing establishes membership, including an
// empty one. LXC 596/unavailable errors are not deletion evidence;
// leave failed guests unpolled so their last listing is retained.
}
results <- result
}
}()
}
wg.Add(1)
go func() {
defer wg.Done()
defer close(queue)
for _, target := range targets {
select {
case queue <- target:
case <-snapshotCtx.Done():
return
}
}
}()
go func() {
wg.Wait()
close(results)
}()
for result := range results {
if result.polled {
polledGuestKeys[result.target.key] = struct{}{}
allSnapshots = append(allSnapshots, result.snapshots...)
continue
}
if result.err == nil {
continue
}
if snapshotCtx.Err() != nil {
deadlineExceeded = true
log.Warn().
Str("instance", instanceName).
Str("node", result.target.node).
Str("type", result.target.guestType).
Int("vmid", result.target.vmid).
Err(snapshotCtx.Err()).
Msg("Guest snapshot polling context expired before all guests completed")
continue
}
if result.target.guestType == "lxc" {
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_container_snapshots", instanceName, result.err).WithNode(result.target.node)
log.Debug().
Err(monErr).
Str("node", result.target.node).
Int("vmid", result.target.vmid).
Msg("Failed to get container snapshots")
} else {
monErr := errors.NewMonitorError(errors.ErrorTypeAPI, "get_vm_snapshots", instanceName, result.err).WithNode(result.target.node)
log.Debug().
Err(monErr).
Str("node", result.target.node).
Int("vmid", result.target.vmid).
Msg("Failed to get VM snapshots")
}
}
// The dispatcher can stop without a result for each unattempted guest.
// Those targets are unknown, not successful empty listings.
deadlineExceeded = deadlineExceeded || snapshotCtx.Err() != nil
// Retained live rows are continuity, not newly accepted provider evidence.
// Keep their count separate before merging so a failed listing cannot renew
// durable point evidence merely because the raw row survived in memory.
observedSnapshotCount := len(allSnapshots)
carriedForward := 0
for _, prev := range previousSnapshots {
if _, polled := polledGuestKeys[guestKey(instanceName, prev.Node, prev.VMID)]; polled {
continue
}
allSnapshots = append(allSnapshots, prev)
carriedForward++
}
if deadlineExceeded {
log.Warn().
Str("instance", instanceName).
Int("freshlyPolled", len(polledGuestKeys)).
Int("carriedForward", carriedForward).
Msg("Guest snapshot polling timed out before completion; merged fresh results with previously-known snapshots for unpolled guests")
} else if carriedForward > 0 {
log.Debug().
Str("instance", instanceName).
Int("carriedForward", carriedForward).
Msg("Guest snapshot polling completed; carried forward previous snapshots for guests with per-call errors")
}
if len(allSnapshots) > 0 && !deadlineExceeded {
sizeMap := m.collectSnapshotSizes(snapshotCtx, instanceName, client, allSnapshots)
if len(sizeMap) > 0 {
for i := range allSnapshots {
if size, ok := sizeMap[allSnapshots[i].ID]; ok && size > 0 {
allSnapshots[i].SizeBytes = size
}
}
}
}
// Keep listings accepted before optional sizing exhausted the child budget.
// WithoutCancel retains the owner value: runtime cancellation, retirement
// and connection replacement still fence every local effect. A cancelled
// listing above is not observed evidence and cannot authorise removal.
unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)
if !current {
return
}
defer unlock()
// Update state with guest snapshots for this instance
m.state.UpdateGuestSnapshotsForInstance(instanceName, allSnapshots)
// Best-effort ingestion into recovery store (for rollups / unified backups UX).
guestInfo := buildProxmoxGuestInfoIndex(readState)
points, evidenceErr := proxmoxrecoverymapper.FromPVEGuestSnapshotsWithEvidence(
allSnapshots[:observedSnapshotCount],
guestInfo,
time.Now().UTC(),
)
if evidenceErr != nil {
log.Warn().
Err(evidenceErr).
Str("instance", instanceName).
Msg("Failed to attach PVE snapshot recovery evidence; preserving existing points")
} else if len(targets) > 0 && len(polledGuestKeys) == len(targets) &&
carriedForward == 0 && snapshotCtx.Err() == nil {
// Only an accepted complete listing can establish instance-wide
// absence. In particular, after a monitor replacement the live cache
// can be empty while durable snapshots still exist for a failed guest.
// No observed guest inventory is not a successful empty enumeration.
m.ingestAndReconcileRecoveryPointsAsync(
points,
recoveryReconcileScope{
provider: string(recovery.ProviderProxmoxPVE),
idPrefix: "pve-snapshot:",
instance: instanceName,
},
)
} else {
m.ingestRecoveryPointsAsync(points)
}
if m.alertManager != nil {
scope.run(func() { m.alertManager.CheckSnapshotsForInstance(instanceName, allSnapshots, guestLookups) })
}
log.Debug().
Str("instance", instanceName).
Int("count", len(allSnapshots)).
Msg("Guest snapshots polled")
// Immediately broadcast the updated state so frontend sees new snapshots
m.broadcastStateUpdate()
}
func (m *Monitor) collectSnapshotSizes(ctx context.Context, instanceName string, client PVEClientInterface, snapshots []models.GuestSnapshot) map[string]int64 {
sizes := make(map[string]int64, len(snapshots))
if len(snapshots) == 0 {
return sizes
}
validSnapshots := make(map[string]struct{}, len(snapshots))
nodes := make(map[string]struct{})
for _, snap := range snapshots {
validSnapshots[snap.ID] = struct{}{}
if snap.Node != "" {
nodes[snap.Node] = struct{}{}
}
}
if len(nodes) == 0 {
return sizes
}
seenVolids := make(map[string]struct{})
for nodeName := range nodes {
if ctx.Err() != nil {
break
}
storages, err := client.GetStorage(ctx, nodeName)
if err != nil {
log.Debug().
Err(err).
Str("node", nodeName).
Str("instance", instanceName).
Msg("Failed to get storage list for snapshot sizing")
continue
}
for _, storage := range storages {
if ctx.Err() != nil {
break
}
contentTypes := strings.ToLower(storage.Content)
if !strings.Contains(contentTypes, "images") && !strings.Contains(contentTypes, "rootdir") {
continue
}
if !storageContentQueryable(storage) {
continue
}
contents, err := client.GetStorageContent(ctx, nodeName, storage.Storage)
if err != nil {
log.Debug().
Err(err).
Str("node", nodeName).
Str("storage", storage.Storage).
Str("instance", instanceName).
Msg("Failed to get storage content for snapshot sizing")
continue
}
for _, item := range contents {
if item.VMID <= 0 {
continue
}
if _, seen := seenVolids[item.Volid]; seen {
continue
}
snapName := extractSnapshotName(item.Volid)
if snapName == "" {
continue
}
key := fmt.Sprintf("%s-%s-%d-%s", instanceName, nodeName, item.VMID, snapName)
if _, ok := validSnapshots[key]; !ok {
continue
}
seenVolids[item.Volid] = struct{}{}
size := int64(item.Size)
if size < 0 {
size = 0
}
sizes[key] += size
}
}
}
return sizes
}
func (m *Monitor) recordAuthFailure(instanceName string, nodeType string) {
nodeID := instanceName
if nodeType != "" {
nodeID = nodeType + "-" + instanceName
}
m.mu.Lock()
m.authFailures[nodeID]++
failures := m.authFailures[nodeID]
m.lastAuthAttempt[nodeID] = time.Now()
m.mu.Unlock()
log.Warn().
Str("node", nodeID).
Int("failures", failures).
Msg("Authentication failure recorded")
const maxAuthFailures = 5
if failures >= maxAuthFailures {
// Clear tracking first, then perform removal outside the monitor lock.
// Removal updates state/health and may need to acquire monitor locks internally.
m.mu.Lock()
delete(m.authFailures, nodeID)
delete(m.lastAuthAttempt, nodeID)
m.mu.Unlock()
log.Error().
Str("node", nodeID).
Int("failures", failures).
Msg("Maximum authentication failures reached, removing node from state")
// Remove from state based on type
if nodeType == "pve" {
m.removeFailedPVENode(instanceName)
} else if nodeType == "pbs" {
m.removeFailedPBSNode(instanceName)
} else if nodeType == "pmg" {
m.removeFailedPMGInstance(instanceName)
}
}
}
// resetAuthFailures resets the failure count for a node after successful auth
func (m *Monitor) resetAuthFailures(instanceName string, nodeType string) {
m.mu.Lock()
defer m.mu.Unlock()
nodeID := instanceName
if nodeType != "" {
nodeID = nodeType + "-" + instanceName
}
if count, exists := m.authFailures[nodeID]; exists && count > 0 {
log.Info().
Str("node", nodeID).
Int("previousFailures", count).
Msg("Authentication succeeded, resetting failure count")
delete(m.authFailures, nodeID)
delete(m.lastAuthAttempt, nodeID)
}
}
// removeFailedPVENode updates a PVE node to show failed authentication status
func (m *Monitor) removeFailedPVENode(instanceName string) {
// Get instance config to get host URL
var hostURL string
m.mu.RLock()
if m.config != nil {
for _, cfg := range m.config.PVEInstances {
if cfg.Name == instanceName {
hostURL = cfg.Host
break
}
}
}
m.mu.RUnlock()
// Create a failed node entry to show in UI with error status
failedNode := models.Node{
ID: instanceName + "-failed",
Name: instanceName,
DisplayName: instanceName,
Instance: instanceName,
Host: hostURL, // Include host URL even for failed nodes
Status: "offline",
Type: "node",
ConnectionHealth: "error",
LastSeen: time.Now(),
// Set other fields to zero values to indicate no data
CPU: 0,
Memory: models.Memory{},
Disk: models.Disk{},
}
// Update with just the failed node
m.state.UpdateNodesForInstance(instanceName, []models.Node{failedNode})
// Remove all other resources associated with this instance
m.state.UpdateVMsForInstance(instanceName, []models.VM{})
m.state.UpdateContainersForInstance(instanceName, []models.Container{})
m.state.UpdateStorageForInstance(instanceName, []models.Storage{})
m.state.UpdateCephClustersForInstance(instanceName, []models.CephCluster{})
m.state.UpdateBackupTasksForInstance(instanceName, []models.BackupTask{})
m.state.UpdateStorageBackupsForInstance(instanceName, []models.StorageBackup{})
m.state.UpdatePBSGuestConfirmationsForInstance(instanceName, nil)
m.state.UpdateGuestSnapshotsForInstance(instanceName, []models.GuestSnapshot{})
// Set connection health to false
m.setProviderConnectionHealth(InstanceTypePVE, instanceName, false)
}
// removeFailedPBSNode removes a PBS node and all its resources from state
func (m *Monitor) removeFailedPBSNode(instanceName string) {
// Remove PBS instance by passing empty array
currentInstances := m.state.PBSInstances
var updatedInstances []models.PBSInstance
for _, inst := range currentInstances {
if inst.Name != instanceName {
updatedInstances = append(updatedInstances, inst)
}
}
m.state.UpdatePBSInstances(updatedInstances)
// Remove PBS backups
m.state.UpdatePBSBackups(instanceName, []models.PBSBackup{})
// Set connection health to false
m.setProviderConnectionHealth(InstanceTypePBS, instanceName, false)
}
// removeFailedPMGInstance removes PMG data from state when authentication fails repeatedly
func (m *Monitor) removeFailedPMGInstance(instanceName string) {
currentInstances := m.state.PMGInstances
updated := make([]models.PMGInstance, 0, len(currentInstances))
for _, inst := range currentInstances {
if inst.Name != instanceName {
updated = append(updated, inst)
}
}
m.state.UpdatePMGInstances(updated)
m.state.UpdatePMGBackups(instanceName, nil)
m.setProviderConnectionHealth(InstanceTypePMG, instanceName, false)
}
// pbsBackupCacheTTL controls how long cached PBS backup snapshots are reused
// before forcing a re-fetch. This ensures verification status changes (which
// don't alter backup count or timestamp) are picked up periodically.
const pbsBackupCacheTTL = 10 * time.Minute
type pbsBackupGroupKey struct {
datastore string
namespace string
backupType string
backupID string
}
type pbsBackupNamespaceKey struct {
datastore string
namespace string
}
type cachedPBSGroup struct {
snapshots []models.PBSBackup
latest time.Time
}
type pbsBackupFetchRequest struct {
datastore string
namespace string
group pbs.BackupGroup
cached cachedPBSGroup
}
// pollPBSBackups fetches all backups from PBS datastores
// pbsReadDenied reports an authorization rejection. Other terminal errors,
// such as a namespace that no longer exists, are not missing permission.
func pbsReadDenied(err error) bool {
status, ok := pbs.HTTPStatus(err)
return ok && (status == 401 || status == 403)
}
func (m *Monitor) pollPBSBackups(ctx context.Context, instanceName string, client *pbs.Client, datastores []models.PBSDatastore, namespaceErrors map[string]error) {
log.Debug().Str("instance", instanceName).Msg("polling PBS backups")
// Cache existing PBS backups so we can avoid redundant API calls when no changes occurred.
existingGroups := m.buildPBSBackupCache(instanceName)
var allBackups []models.PBSBackup
observedGroups := make(map[pbsBackupGroupKey]struct{})
retainedGroups := make(map[pbsBackupGroupKey]struct{}, len(existingGroups))
refreshedGroups := make(map[pbsBackupGroupKey]time.Time)
queriedNamespaces := make(map[pbsBackupNamespaceKey]struct{})
transientNamespaces := make(map[pbsBackupNamespaceKey]struct{})
transientDiscovery := make(map[string]struct{})
// Names what went unread for the provider observation; it never decides
// which points this poll confirmed.
var unreadScopes []recovery.ProtectionUnreadScope
unreadNamespace := func(datastore, namespace string, denied bool) {
unreadScopes = append(unreadScopes, recovery.ProtectionUnreadScope{
Kind: recovery.ProtectionUnreadScopeNamespace,
Storage: datastore,
Namespace: namespace,
AccessDenied: denied,
})
}
datastoreCount := len(datastores) // Number of datastores to query
datastoreFetches := 0 // Number of successful datastore fetches
datastoreErrors := 0 // Datastores with incomplete history
datastoreTerminalFailures := 0 // Number of datastores that failed only with terminal errors
collectionTerminalFailures := 0 // Datastores with any rejected namespace or snapshot request
// Process each datastore
for _, ds := range datastores {
namespacePaths := namespacePathsForDatastore(ds)
log.Info().
Str("instance", instanceName).
Str("datastore", ds.Name).
Int("namespaces", len(namespacePaths)).
Strs("namespace_paths", namespacePaths).
Msg("Processing datastore namespaces")
datastoreHadSuccess := false
datastoreNamespaceErrors := 0
datastoreTerminalNamespaceErrors := 0
// The root namespace remains readable when discovering child namespaces
// fails. Its accepted rows are useful, but they cannot establish that
// undiscovered namespaces are empty or renew complete-provider evidence.
if err := namespaceErrors[ds.Name]; err != nil {
datastoreNamespaceErrors++
if !shouldReuseCachedPBSBackups(err) {
datastoreTerminalNamespaceErrors++
} else {
transientDiscovery[ds.Name] = struct{}{}
}
unreadScopes = append(unreadScopes, recovery.ProtectionUnreadScope{
Kind: recovery.ProtectionUnreadScopeDatastore,
Storage: ds.Name,
AccessDenied: pbsReadDenied(err),
})
}
groupsReused := 0
groupsRequested := 0
for _, namespace := range namespacePaths {
namespaceKey := pbsBackupNamespaceKey{datastore: ds.Name, namespace: namespace}
if ctx.Err() != nil {
// Classify every unread namespace without starting an expired
// request. Earlier accepted namespaces remain independently useful.
if ctx.Err() != context.DeadlineExceeded {
return
}
datastoreNamespaceErrors++
transientNamespaces[namespaceKey] = struct{}{}
unreadNamespace(ds.Name, namespace, false)
continue
}
groups, err := client.ListBackupGroups(ctx, ds.Name, namespace)
queriedNamespaces[namespaceKey] = struct{}{}
if ctx.Err() != nil {
// A late nil-error response is not an accepted empty listing.
err = ctx.Err()
}
if err != nil {
datastoreNamespaceErrors++
if !shouldReuseCachedPBSBackups(err) {
datastoreTerminalNamespaceErrors++
} else {
transientNamespaces[namespaceKey] = struct{}{}
}
unreadNamespace(ds.Name, namespace, pbsReadDenied(err))
log.Error().
Err(err).
Str("instance", instanceName).
Str("datastore", ds.Name).
Str("namespace", namespace).
Msg("Failed to list PBS backup groups")
continue
}
datastoreHadSuccess = true
sortPBSBackupGroupsByLatest(groups)
requests := make([]pbsBackupFetchRequest, 0, len(groups))
projectedBackups := len(allBackups)
for _, group := range groups {
if projectedBackups >= pbsBackupLiveStateLimit {
log.Warn().
Str("instance", instanceName).
Str("datastore", ds.Name).
Str("namespace", namespace).
Int("limit", pbsBackupLiveStateLimit).
Msg("PBS backup live-state limit reached; skipping remaining groups")
// Skipped groups are not a complete inventory, even when
// every request we did make succeeded.
datastoreNamespaceErrors++
unreadNamespace(ds.Name, namespace, false)
break
}
key := pbsBackupGroupKey{
datastore: ds.Name,
namespace: namespace,
backupType: group.BackupType,
backupID: group.BackupID,
}
cached := existingGroups[key]
// Group deleted (no backups left) - ensure cached data is dropped.
if group.BackupCount == 0 {
continue
}
retainedGroups[key] = struct{}{}
lastBackupTime := time.Unix(group.LastBackup, 0)
hasCachedData := len(cached.snapshots) > 0
cacheCountMatches := len(cached.snapshots) == retainedPBSSnapshotCount(group.BackupCount)
// Check if the cached data is still within its TTL.
cacheAge := time.Since(m.pbsBackupCacheTimeFor(instanceName, key))
cacheStillFresh := cacheAge < pbsBackupCacheTTL
// A cached in-flight snapshot must be re-fetched every poll:
// when it completes, neither the group's last-backup time nor
// its count changes (the listing already included it), so the
// normal reuse conditions would keep showing "backup running"
// until the TTL expired.
cachedHasInProgress := false
for _, cachedSnapshot := range cached.snapshots {
if cachedSnapshot.InProgress {
cachedHasInProgress = true
break
}
}
// Reuse only while both retained count and newest time still match.
// Pruning can move last-backup backwards without changing the
// capped cache size; that is removal evidence, not an unchanged group.
// The TTL also bounds reuse when only verification changes.
if hasCachedData &&
cacheStillFresh &&
cacheCountMatches &&
!cachedHasInProgress &&
lastBackupTime.Equal(cached.latest) {
allBackups = appendPBSBackupsWithinLimit(allBackups, cached.snapshots)
observedGroups[key] = struct{}{}
groupsReused++
projectedBackups = len(allBackups)
continue
}
requests = append(requests, pbsBackupFetchRequest{
datastore: ds.Name,
namespace: namespace,
group: group,
cached: cached,
})
projectedBackups += retainedPBSSnapshotCount(group.BackupCount)
}
if len(requests) == 0 {
continue
}
groupsRequested += len(requests)
fetched := m.fetchPBSBackupSnapshots(ctx, client, instanceName, requests)
allBackups = appendPBSBackupsWithinLimit(allBackups, fetched.backups)
datastoreNamespaceErrors += fetched.errors
datastoreTerminalNamespaceErrors += fetched.terminalErrors
if fetched.errors > 0 {
// Snapshot failures carry no status here; do not call them denials.
unreadNamespace(ds.Name, namespace, false)
}
// Stage successful read times until their matching rows are published.
// A later cancellation must not freshen the previously published rows
// when this poll's refreshed payload never reaches state.
for key, fetchedAt := range fetched.refreshed {
observedGroups[key] = struct{}{}
refreshedGroups[key] = fetchedAt
}
}
// Listing groups is not proof that their snapshots or every namespace
// were readable. Carry those losses into provider-wide evidence.
if datastoreNamespaceErrors > 0 {
datastoreErrors++
}
if datastoreTerminalNamespaceErrors > 0 {
collectionTerminalFailures++
}
if datastoreHadSuccess {
datastoreFetches++
log.Info().
Str("instance", instanceName).
Str("datastore", ds.Name).
Int("namespaces", len(namespacePaths)).
Int("groups_reused", groupsReused).
Int("groups_refreshed", groupsRequested).
Msg("PBS datastore processed")
} else {
allNamespaceErrorsTerminal := datastoreNamespaceErrors > 0 &&
datastoreTerminalNamespaceErrors == datastoreNamespaceErrors
if allNamespaceErrorsTerminal {
datastoreTerminalFailures++
log.Warn().
Str("instance", instanceName).
Str("datastore", ds.Name).
Int("namespace_errors", datastoreNamespaceErrors).
Msg("No namespaces succeeded for PBS datastore due to terminal errors; clearing cached backups")
} else {
// Preserve cached data for this datastore when failures are transient.
log.Warn().
Str("instance", instanceName).
Str("datastore", ds.Name).
Msg("No namespaces succeeded for PBS datastore; using cached backups")
}
}
}
if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
// Explicit cancellation still withdraws the enumeration.
return
}
// A readable sibling namespace cannot establish absence in an unread one.
// Carry only the transiently unread scope, after accepted rows so old
// inventory cannot displace fresh results at the existing live-state bound.
// Discovery failure may hide old namespaces, but never overrides a request
// we actually made (notably an access rejection or a successful empty list).
var carriedBackups []models.PBSBackup
for key, entry := range existingGroups {
namespaceKey := pbsBackupNamespaceKey{datastore: key.datastore, namespace: key.namespace}
_, unread := transientNamespaces[namespaceKey]
_, undiscovered := transientDiscovery[key.datastore]
_, queried := queriedNamespaces[namespaceKey]
if !unread && !(undiscovered && !queried) {
continue
}
carriedBackups = append(carriedBackups, entry.snapshots...)
retainedGroups[key] = struct{}{}
}
sortPBSBackupsByLatest(carriedBackups)
allBackups = appendPBSBackupsWithinLimit(allBackups, carriedBackups)
log.Info().
Str("instance", instanceName).
Int("count", len(allBackups)).
Msg("PBS backups fetched")
// A manifestless snapshot is incomplete, but it is not necessarily still
// being written: failed backup and PBS-to-PBS sync tasks can leave that
// shape behind. Correlate incomplete artifacts with PBS's live task view
// before they feed guest "Backup Running" state. If task visibility is
// denied or transiently unavailable, keep the previous conservative
// inference rather than claiming that no write is active.
if hasIncompletePBSBackup(allBackups) {
// Write-activity evidence belongs to this poll, not to the snapshot
// cache. A failed snapshot refresh may have reused an incomplete
// artifact whose previous poll authoritatively found no writer. Clear
// that old observation before asking for the current task view so a
// task-query failure leaves visibility unknown instead of publishing a
// stale terminal-failure conclusion.
clearPBSWriteActivityObservation(allBackups)
var runningDataTasks []pbs.RunningDataTask
runningTasksErr := ctx.Err()
if runningTasksErr == nil {
runningDataTasks, runningTasksErr = client.ListRunningDataTasks(ctx)
if ctx.Err() != nil {
runningTasksErr = ctx.Err()
}
}
if runningTasksErr != nil {
log.Warn().Err(runningTasksErr).
Str("instance", instanceName).
Msg("Could not correlate incomplete PBS snapshots with running tasks")
} else {
applyPBSWriteActivity(allBackups, runningDataTasks)
}
}
if ctx.Err() != nil && ctx.Err() != context.DeadlineExceeded {
return
}
if ctx.Err() == context.DeadlineExceeded && datastoreErrors == 0 {
// The final live-writer read can spend the remaining budget too.
// Its unknown result cannot turn a partial collection into protection.
datastoreErrors = 1
}
publicationCtx := ctx
if ctx.Err() == context.DeadlineExceeded {
// Drop only the child budget, not its runtime/client owner value.
publicationCtx = context.WithoutCancel(ctx)
}
unlock, current := m.beginPBSBackupPublication(publicationCtx, instanceName)
if !current {
return
}
defer unlock()
// With no accepted group listing, a mixture of rejected and transiently
// unread namespaces does not establish that every datastore was denied.
// Partial successful collection still reports any actual access rejection.
observationTerminalFailures := collectionTerminalFailures
if datastoreFetches == 0 {
observationTerminalFailures = datastoreTerminalFailures
}
protectionObservedAt := time.Now().UTC()
protectionObservation, protectionObservationErr :=
buildPBSProtectionProviderObservation(
instanceName,
datastoreCount,
datastoreFetches,
datastoreErrors,
observationTerminalFailures,
protectionObservedAt,
)
if protectionObservationErr != nil {
log.Warn().
Err(protectionObservationErr).
Str("instance", instanceName).
Msg("Failed to build PBS protection provider observation")
} else {
protectionObservation = protectionObservation.WithUnreadScopes(unreadScopes)
}
// Decide whether to keep existing backups when all queries failed
if collectionTerminalFailures == 0 &&
shouldPreservePBSBackupsWithTerminal(datastoreCount, datastoreFetches, datastoreTerminalFailures) {
if protectionObservationErr == nil {
m.ingestProtectionProviderObservationsAsync(
[]recovery.ProtectionProviderObservation{protectionObservation},
)
}
log.Warn().
Str("instance", instanceName).
Int("datastores", datastoreCount).
Int("errors", datastoreErrors).
Int("terminal_failures", datastoreTerminalFailures).
Msg("All PBS datastore queries failed; keeping previous backup list")
return
}
// Publication is the commit boundary: no cancellation return may split a
// successful read time from its matching payload. Per-instance poller
// admission serializes this publication with the next ordinary poll.
m.state.UpdatePBSBackups(instanceName, allBackups)
m.prunePBSBackupCacheTimes(instanceName, retainedGroups)
for key, fetchedAt := range refreshedGroups {
m.setPBSBackupCacheTime(instanceName, key, fetchedAt)
}
// Failed reads may retain raw artifacts for continuity, but cannot renew
// their point evidence or use them to infer another backup's guest identity.
// Successful, unchanged group listings retain the existing bounded TTL reuse.
observedBackups := make([]models.PBSBackup, 0, len(allBackups))
for _, backup := range allBackups {
key := pbsBackupGroupKey{
datastore: backup.Datastore, namespace: normalizePBSNamespacePath(backup.Namespace),
backupType: backup.BackupType, backupID: backup.VMID,
}
if _, observed := observedGroups[key]; observed {
observedBackups = append(observedBackups, backup)
}
}
// Best-effort ingestion into recovery store (for rollups / unified backups UX).
candidates := buildPBSGuestCandidates(m.GetUnifiedReadStateOrSnapshot())
points, evidenceErr := proxmoxrecoverymapper.FromPBSBackupsWithEvidence(
observedBackups,
candidates,
protectionObservedAt,
)
if evidenceErr != nil {
log.Warn().
Err(evidenceErr).
Str("instance", instanceName).
Msg("Failed to attach PBS recovery evidence; preserving existing points")
if protectionObservationErr == nil {
m.ingestProtectionProviderObservationsAsync(
[]recovery.ProtectionProviderObservation{protectionObservation},
)
}
return
}
observations := []recovery.ProtectionProviderObservation{}
if protectionObservationErr == nil {
observations = append(observations, protectionObservation)
}
batch := recoveryIngestBatch{points: points, observations: observations}
// Only an authoritative complete enumeration can delete absent points.
// Unreadable history remains historical evidence, qualified by this poll's
// partial/unavailable provider observation, until an ordinary complete poll
// recovers or confirms deletion. Never infer absence from a failed request.
if protectionObservationErr == nil &&
protectionObservation.HistoryCompleteness == recovery.ProtectionHistoryComplete {
batch.reconcile = &recoveryReconcileScope{
provider: string(recovery.ProviderProxmoxPBS),
idPrefix: "pbs-backup:",
instance: instanceName,
}
}
m.enqueueRecoveryIngest(batch)
// Sync backup times to VMs/Containers and republish them to canonical resources.
m.syncGuestBackupTimesAndResourceStore()
if ctx.Err() == nil {
// Collection expiry is not a failed alert-store read or permission
// to clear an existing wanted alert. The next ordinary check remains.
m.checkBackupAlerts(ctx)
}
// Immediately broadcast the updated state so frontend sees new backups
m.broadcastStateUpdate()
}
func hasIncompletePBSBackup(backups []models.PBSBackup) bool {
for _, backup := range backups {
if backup.InProgress {
return true
}
}
return false
}
func clearPBSWriteActivityObservation(backups []models.PBSBackup) {
for i := range backups {
backups[i].WriteActivityObserved = false
backups[i].WriteActive = false
}
}
// applyPBSWriteActivity marks whether each incomplete snapshot is accounted
// for by a task which PBS currently reports as running. Backup task worker IDs
// normally identify "<store>:<type>/<id>". If a server version returns a less
// specific backup ID, the task remains a conservative instance-wide writer.
// Sync tasks can touch many groups and therefore apply instance-wide.
func applyPBSWriteActivity(backups []models.PBSBackup, tasks []pbs.RunningDataTask) {
for i := range backups {
if !backups[i].InProgress {
continue
}
backups[i].WriteActivityObserved = true
backups[i].WriteActive = runningPBSTaskMatchesBackup(tasks, backups[i])
}
}
func runningPBSTaskMatchesBackup(tasks []pbs.RunningDataTask, backup models.PBSBackup) bool {
subject := strings.ToLower(strings.TrimSpace(backup.BackupType + "/" + backup.VMID))
store := strings.ToLower(strings.TrimSpace(backup.Datastore))
for _, task := range tasks {
switch strings.ToLower(strings.TrimSpace(task.WorkerType)) {
case "syncjob":
return true
case "backup":
workerID := strings.ToLower(strings.TrimSpace(task.WorkerID))
if workerID == "" {
return true
}
if !strings.Contains(workerID, "/") {
// Unknown legacy/new worker-ID shape: do not hide a real write.
return true
}
if !strings.HasSuffix(workerID, subject) {
continue
}
if store == "" || strings.HasPrefix(workerID, store+":") || strings.Contains(workerID, ":"+store+":") {
return true
}
}
}
return false
}
func (m *Monitor) buildPBSBackupCache(instanceName string) map[pbsBackupGroupKey]cachedPBSGroup {
snapshot := m.state.GetSnapshot()
cache := make(map[pbsBackupGroupKey]cachedPBSGroup)
for _, backup := range snapshot.PBSBackups {
if backup.Instance != instanceName {
continue
}
key := pbsBackupGroupKey{
datastore: backup.Datastore,
namespace: normalizePBSNamespacePath(backup.Namespace),
backupType: backup.BackupType,
backupID: backup.VMID,
}
entry := cache[key]
entry.snapshots = append(entry.snapshots, backup)
if backup.BackupTime.After(entry.latest) {
entry.latest = backup.BackupTime
}
cache[key] = entry
}
for key, entry := range cache {
sortPBSBackupsByLatest(entry.snapshots)
if len(entry.snapshots) > pbsBackupSnapshotsPerGroupLimit {
entry.snapshots = entry.snapshots[:pbsBackupSnapshotsPerGroupLimit]
}
cache[key] = entry
}
return cache
}
// pbsBackupCacheTimeFor returns the last fetch time for a PBS backup group.
func (m *Monitor) pbsBackupCacheTimeFor(instanceName string, key pbsBackupGroupKey) time.Time {
m.mu.RLock()
defer m.mu.RUnlock()
if perGroup, ok := m.pbsBackupCacheTime[instanceName]; ok {
return perGroup[key]
}
return time.Time{}
}
// setPBSBackupCacheTime records when a PBS backup group was last fetched.
func (m *Monitor) setPBSBackupCacheTime(instanceName string, key pbsBackupGroupKey, t time.Time) {
m.mu.Lock()
defer m.mu.Unlock()
if m.pbsBackupCacheTime == nil {
m.pbsBackupCacheTime = make(map[string]map[pbsBackupGroupKey]time.Time)
}
if m.pbsBackupCacheTime[instanceName] == nil {
m.pbsBackupCacheTime[instanceName] = make(map[pbsBackupGroupKey]time.Time)
}
m.pbsBackupCacheTime[instanceName][key] = t
}
func (m *Monitor) prunePBSBackupCacheTimes(instanceName string, retained map[pbsBackupGroupKey]struct{}) {
m.mu.Lock()
defer m.mu.Unlock()
if m.pbsBackupCacheTime == nil {
return
}
perGroup := m.pbsBackupCacheTime[instanceName]
if len(perGroup) == 0 {
return
}
for key := range perGroup {
if _, ok := retained[key]; !ok {
delete(perGroup, key)
}
}
if len(perGroup) == 0 {
delete(m.pbsBackupCacheTime, instanceName)
}
}
func normalizePBSNamespacePath(ns string) string {
if ns == "/" {
return ""
}
return ns
}
func namespacePathsForDatastore(ds models.PBSDatastore) []string {
if len(ds.Namespaces) == 0 {
return []string{""}
}
seen := make(map[string]struct{}, len(ds.Namespaces))
var paths []string
for _, ns := range ds.Namespaces {
path := normalizePBSNamespacePath(ns.Path)
if _, ok := seen[path]; ok {
continue
}
seen[path] = struct{}{}
paths = append(paths, path)
}
return paths
}
// pbsBackupSnapshotFetchResult retains request quality alongside the artifacts.
// A cached row is not evidence that its snapshot request succeeded this poll.
type pbsBackupSnapshotFetchResult struct {
backups []models.PBSBackup
errors int
terminalErrors int
refreshed map[pbsBackupGroupKey]time.Time
}
func (m *Monitor) fetchPBSBackupSnapshots(ctx context.Context, client *pbs.Client, instanceName string, requests []pbsBackupFetchRequest) pbsBackupSnapshotFetchResult {
if len(requests) == 0 {
return pbsBackupSnapshotFetchResult{}
}
workerCount := pbsBackupSnapshotFetchWorkers
if len(requests) < workerCount {
workerCount = len(requests)
}
jobs := make(chan pbsBackupFetchRequest)
results := make(chan pbsBackupSnapshotFetchResult, workerCount)
var wg sync.WaitGroup
for i := 0; i < workerCount; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for req := range jobs {
log.Debug().
Str("instance", instanceName).
Str("datastore", req.datastore).
Str("namespace", req.namespace).
Str("type", req.group.BackupType).
Str("id", req.group.BackupID).
Msg("Refreshing PBS backup group")
// Drain queued work as unread after expiry, without provider I/O.
var snapshots []pbs.BackupSnapshot
err := ctx.Err()
if err == nil {
snapshots, err = client.ListBackupSnapshots(ctx, req.datastore, req.namespace, req.group.BackupType, req.group.BackupID)
if ctx.Err() != nil {
err = ctx.Err()
}
}
if err != nil {
log.Error().
Err(err).
Str("instance", instanceName).
Str("datastore", req.datastore).
Str("namespace", req.namespace).
Str("type", req.group.BackupType).
Str("id", req.group.BackupID).
Msg("Failed to list PBS backup snapshots")
result := pbsBackupSnapshotFetchResult{errors: 1}
if shouldReuseCachedPBSBackups(err) {
result.backups = req.cached.snapshots
} else {
// Access rejection cannot renew a cached raw row.
result.terminalErrors = 1
}
results <- result
continue
}
fetchedAt := time.Now()
backups := convertPBSSnapshots(instanceName, req.datastore, req.namespace, snapshots)
results <- pbsBackupSnapshotFetchResult{
backups: backups,
refreshed: map[pbsBackupGroupKey]time.Time{{
datastore: req.datastore, namespace: req.namespace,
backupType: req.group.BackupType, backupID: req.group.BackupID,
}: fetchedAt},
}
}
}()
}
go func() {
defer close(jobs)
for _, req := range requests {
// Every planned group receives a result, including cached rows for
// unread groups; only successful, on-time reads are refreshed.
jobs <- req
}
}()
go func() {
wg.Wait()
close(results)
}()
combined := pbsBackupSnapshotFetchResult{refreshed: make(map[pbsBackupGroupKey]time.Time)}
for result := range results {
combined.backups = appendPBSBackupsWithinLimit(combined.backups, result.backups)
combined.errors += result.errors
combined.terminalErrors += result.terminalErrors
for key, fetchedAt := range result.refreshed {
combined.refreshed[key] = fetchedAt
}
}
return combined
}
func convertPBSSnapshots(instanceName, datastore, namespace string, snapshots []pbs.BackupSnapshot) []models.PBSBackup {
sort.SliceStable(snapshots, func(i, j int) bool {
return snapshots[i].BackupTime > snapshots[j].BackupTime
})
if len(snapshots) > pbsBackupSnapshotsPerGroupLimit {
snapshots = snapshots[:pbsBackupSnapshotsPerGroupLimit]
}
backups := make([]models.PBSBackup, 0, len(snapshots))
for _, snapshot := range snapshots {
backupTime := time.Unix(snapshot.BackupTime, 0)
backupID := fmt.Sprintf("pbs-%s-%s-%s-%s-%s-%d",
instanceName, datastore, namespace,
snapshot.BackupType, snapshot.BackupID,
snapshot.BackupTime)
var fileNames []string
for _, file := range snapshot.Files {
switch f := file.(type) {
case string:
fileNames = append(fileNames, f)
case map[string]interface{}:
if filename, ok := f["filename"].(string); ok {
fileNames = append(fileNames, filename)
}
}
}
// A snapshot the PBS API lists while the backup is still being
// written has no manifest yet: no size field (decoded as 0) and no
// index.json.blob in files - typically only the guest config blob.
// Treat it as in-progress, not as a completed backup.
hasManifest := false
for _, name := range fileNames {
if name == "index.json.blob" {
hasManifest = true
break
}
}
inProgress := snapshot.Size == 0 && !hasManifest
verified := false
if snapshot.Verification != nil {
switch v := snapshot.Verification.(type) {
case string:
verified = v == "ok"
case map[string]interface{}:
if state, ok := v["state"].(string); ok {
verified = state == "ok"
}
}
log.Debug().
Str("vmid", snapshot.BackupID).
Int64("time", snapshot.BackupTime).
Interface("verification", snapshot.Verification).
Bool("verified", verified).
Msg("PBS backup verification status")
}
backups = append(backups, models.PBSBackup{
ID: backupID,
Instance: instanceName,
Datastore: datastore,
Namespace: namespace,
BackupType: snapshot.BackupType,
VMID: snapshot.BackupID,
BackupTime: backupTime,
Size: snapshot.Size,
Protected: snapshot.Protected,
Verified: verified,
InProgress: inProgress,
VerificationRaw: snapshot.Verification,
Comment: snapshot.Comment,
Files: fileNames,
Owner: snapshot.Owner,
})
}
return backups
}
func sortPBSBackupGroupsByLatest(groups []pbs.BackupGroup) {
sort.SliceStable(groups, func(i, j int) bool {
if groups[i].LastBackup == groups[j].LastBackup {
if groups[i].BackupType == groups[j].BackupType {
return groups[i].BackupID < groups[j].BackupID
}
return groups[i].BackupType < groups[j].BackupType
}
return groups[i].LastBackup > groups[j].LastBackup
})
}
func sortPBSBackupsByLatest(backups []models.PBSBackup) {
sort.SliceStable(backups, func(i, j int) bool {
if backups[i].BackupTime.Equal(backups[j].BackupTime) {
return backups[i].ID < backups[j].ID
}
return backups[i].BackupTime.After(backups[j].BackupTime)
})
}
// retainedPBSSnapshotCount is how many snapshots poll retention keeps for a
// group with the given backup count: the full group, capped at the per-group
// limit. It doubles as the expected cache size when deciding whether cached
// snapshots for a group are still complete.
func retainedPBSSnapshotCount(backupCount int) int {
if backupCount <= 0 {
return 0
}
if backupCount > pbsBackupSnapshotsPerGroupLimit {
return pbsBackupSnapshotsPerGroupLimit
}
return backupCount
}
func appendPBSBackupsWithinLimit(backups []models.PBSBackup, additions []models.PBSBackup) []models.PBSBackup {
if len(backups) >= pbsBackupLiveStateLimit || len(additions) == 0 {
return backups
}
remaining := pbsBackupLiveStateLimit - len(backups)
if len(additions) > remaining {
additions = additions[:remaining]
}
return append(backups, additions...)
}
// pollBackupTasks polls backup tasks from a PVE instance
func (m *Monitor) pollBackupTasks(ctx context.Context, instanceName string, client PVEClientInterface) {
if ctx.Err() != nil {
return
}
log.Debug().Str("instance", instanceName).Msg("polling backup tasks")
history, collection, err := m.readBackupTaskHistory(ctx, instanceName, client)
if err != nil {
if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
m.persistBackupTaskHistory(ctx, nil, collection)
unlock()
}
monErr := errors.WrapAPIError("get_backup_tasks", instanceName, err, 0)
log.Error().Err(monErr).Str("instance", instanceName).Msg("failed to get backup tasks")
return
}
// A read returning after cancellation is not accepted history. Once the
// listing is accepted, later log-budget exhaustion must not hide it.
if ctx.Err() != nil {
return
}
var backupTasks []models.BackupTask
var liveTasks []models.BackupTask
liveByNode := make(map[string]int)
liveJobIDs := make(map[string]bool)
var jobTasks []models.BackupTask
jobUPIDs := make(map[string]string)
for _, task := range history.Tasks {
// Extract VMID from task ID (format: "UPID:node:pid:starttime:type:vmid:user@realm:")
vmid := 0
if task.ID != "" {
if vmidInt, err := strconv.Atoi(task.ID); err == nil {
vmid = vmidInt
}
}
taskID := fmt.Sprintf("%s-%s", instanceName, task.UPID)
backupTask := models.BackupTask{
ID: taskID,
Node: task.Node,
Instance: instanceName,
Type: task.Type,
VMID: vmid,
Status: task.Status,
StartTime: time.Unix(task.StartTime, 0),
ObservedAt: time.Now().UTC(),
}
if task.EndTime > 0 {
backupTask.EndTime = time.Unix(task.EndTime, 0)
}
backupTasks = append(backupTasks, backupTask)
// Durable reports backfill more history than the live API historically
// exposed. Preserve its 200 provider tasks per node rather than inflating
// every state/WebSocket payload with the entire retained task history.
if liveByNode[task.Node] < 200 {
liveTasks = append(liveTasks, backupTask)
liveByNode[task.Node]++
liveJobIDs[taskID] = true
}
// Scheduled multi-guest backup jobs run under a single UPID with an
// empty VMID slot, so the task alone says nothing about which guests
// it covered (#1452 area, regressed with the guest-centric redesign).
if vmid == 0 && task.Type == "vzdump" && task.UPID != "" {
jobUPIDs[backupTask.ID] = task.UPID
jobTasks = append(jobTasks, backupTask)
}
}
guestTasks := m.synthesizeVzdumpJobGuestTasks(ctx, instanceName, client, jobTasks, jobUPIDs)
backupTasks = append(backupTasks, guestTasks...)
for _, guest := range guestTasks {
parentID := strings.TrimSuffix(guest.ID, fmt.Sprintf("-vm%d", guest.VMID))
if liveJobIDs[parentID] {
liveTasks = append(liveTasks, guest)
}
}
// Preserve the accepted listing and earlier log reads if a later log used
// the shared budget. WithoutCancel drops only that child budget: the owner
// value still fences explicit connection/runtime withdrawal below.
unlock, current := m.beginPVEBackupPublication(context.WithoutCancel(ctx), instanceName)
if !current {
return
}
defer unlock()
// Update state with new backup tasks for this instance
m.state.UpdateBackupTasksForInstance(instanceName, m.retainUnreadBackupTasks(instanceName, history.Nodes, liveTasks))
// Retain execution history before qualifying collection coverage. Task rows
// are not reconciled against the provider's shrinking inventory window.
guestInfo := buildProxmoxGuestInfoIndex(m.backupReadStateForInstance(instanceName))
points, evidenceErr := proxmoxrecoverymapper.FromPVEBackupTasksWithEvidence(
backupTasks,
guestInfo,
time.Now().UTC(),
)
if evidenceErr != nil {
log.Warn().
Err(evidenceErr).
Str("instance", instanceName).
Msg("Failed to attach PVE backup-task recovery evidence; preserving existing points")
return
}
m.qualifyBackupGuestTaskDetails(instanceName, jobTasks, jobUPIDs, points)
m.persistBackupTaskHistory(ctx, points, collection)
}
// vzdumpJobTaskCacheEntry caches the per-guest tasks synthesized from one
// vzdump job run's log so finished jobs are fetched at most once.
type vzdumpJobTaskCacheEntry struct {
tasks []models.BackupTask
final bool
historyComplete bool
lastAttempt time.Time // scheduling metadata, not an accepted log observation
}
// maxVzdumpJobLogFetchesPerPoll bounds task-log requests per poll cycle so a
// large backlog of historical job runs (e.g. right after startup) trickles in
// over a few cycles instead of stalling the shared backup-poll budget. Pending
// reads rotate by last attempt so errors and incomplete logs cannot consume the
// entire allowance every cycle and permanently hide older guest results.
const maxVzdumpJobLogFetchesPerPoll = 25
// synthesizeVzdumpJobGuestTasks expands multi-guest vzdump job runs into one
// synthetic per-guest task each by parsing the job's task log. Results are
// cached by instance|UPID; logs of finished jobs are immutable, so those are
// fetched only once. Running jobs are re-parsed every cycle to pick up guests
// as the job progresses.
func (m *Monitor) synthesizeVzdumpJobGuestTasks(ctx context.Context, instanceName string, client PVEClientInterface, jobTasks []models.BackupTask, jobUPIDs map[string]string) []models.BackupTask {
if len(jobTasks) == 0 {
if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
m.pruneVzdumpJobTaskCache(instanceName, nil)
unlock()
}
return nil
}
now := time.Now().UTC()
seen := make(map[string]struct{}, len(jobTasks))
fetchJobs := m.selectVzdumpJobLogReads(instanceName, jobTasks, jobUPIDs)
order := make([]int, len(jobTasks))
for i := range order {
order[i] = i
}
// Execute selected reads in scheduling order as well: an early slow read
// can exhaust the shared context before the request-count cap is reached.
sort.SliceStable(order, func(i, j int) bool {
left := fetchJobs[instanceName+"|"+jobUPIDs[jobTasks[order[i]].ID]]
right := fetchJobs[instanceName+"|"+jobUPIDs[jobTasks[order[j]].ID]]
return left > 0 && (right == 0 || left < right)
})
byJob := make([][]models.BackupTask, len(jobTasks))
fetches := 0
for _, index := range order {
jobTask := jobTasks[index]
upid := jobUPIDs[jobTask.ID]
key := instanceName + "|" + upid
seen[key] = struct{}{}
m.mu.RLock()
entry, cached := m.vzdumpJobTaskCache[key]
m.mu.RUnlock()
if cached && entry.final {
byJob[index] = refreshObservedAt(entry.tasks, now)
continue
}
if fetchJobs[key] == 0 || fetches >= maxVzdumpJobLogFetchesPerPoll || ctx.Err() != nil {
// Serve a stale non-final entry if we have one; the next cycle
// will catch up. The current parent can end a cached running
// guest, but cannot supply its missing verdict or a fresh log read.
if cached {
byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
}
continue
}
unlock, current := m.beginPVEBackupPublication(ctx, instanceName)
if !current {
if cached {
byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
}
continue
}
fetches++
delete(fetchJobs, key) // duplicate provider rows cannot spend the same turn twice
// A failed or incomplete read still spent the allowance. Keep attempt
// order separate from accepted observations so failure never freshens evidence.
m.mu.Lock()
if m.vzdumpJobTaskCache == nil {
m.vzdumpJobTaskCache = make(map[string]vzdumpJobTaskCacheEntry)
}
attempted := m.vzdumpJobTaskCache[key]
attempted.lastAttempt = now
m.vzdumpJobTaskCache[key] = attempted
if m.vzdumpJobLogHistoryFirst == nil {
m.vzdumpJobLogHistoryFirst = make(map[string]bool)
}
m.vzdumpJobLogHistoryFirst[instanceName] = jobTask.EndTime.IsZero()
m.mu.Unlock()
unlock()
logLines, err := client.GetTaskLog(ctx, jobTask.Node, upid)
if err != nil {
log.Debug().Err(err).
Str("instance", instanceName).
Str("upid", upid).
Msg("failed to fetch vzdump job task log; per-guest status unavailable this cycle")
if cached {
byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
}
continue
}
unlock, current = m.beginPVEBackupPublication(ctx, instanceName)
if !current {
if cached {
byJob[index] = cachedVzdumpGuestTasks(entry.tasks, jobTask)
}
continue
}
guestTasks := refreshObservedAt(parseVzdumpJobLog(jobTask, upid, logLines), now)
historyComplete := backupGuestLogComplete(jobTask, logLines, guestTasks)
m.mu.Lock()
if m.vzdumpJobTaskCache == nil {
m.vzdumpJobTaskCache = make(map[string]vzdumpJobTaskCacheEntry)
}
m.vzdumpJobTaskCache[key] = vzdumpJobTaskCacheEntry{
tasks: guestTasks,
final: historyComplete,
historyComplete: historyComplete,
lastAttempt: now,
}
m.mu.Unlock()
byJob[index] = guestTasks
unlock()
}
if unlock, current := m.beginPVEBackupPublication(ctx, instanceName); current {
m.pruneVzdumpJobTaskCache(instanceName, seen)
unlock()
}
var synthesized []models.BackupTask
for _, tasks := range byJob {
synthesized = append(synthesized, tasks...)
}
return synthesized
}
// Select without reordering the returned guest tasks or mutating the caller's
// history. Keep ordinary running jobs current, but reserve at least one turn
// for pending ended jobs even when live work fills the allowance. Within each
// group, never-read then least recently attempted reads win; ties retain provider
// order. The queue not served by the last actual request gets the first turn,
// so either queue can progress even when one read exhausts the shared context.
// Only actual calls advance scheduling, not selection or cancellation.
func (m *Monitor) selectVzdumpJobLogReads(instance string, jobs []models.BackupTask, upids map[string]string) map[string]int {
type pendingRead struct {
key string
attempt time.Time
running bool
}
pending := make([]pendingRead, 0, len(jobs))
seen := make(map[string]bool, len(jobs))
m.mu.RLock()
historyFirst := m.vzdumpJobLogHistoryFirst[instance]
for _, job := range jobs {
key := instance + "|" + upids[job.ID]
if seen[key] || m.vzdumpJobTaskCache[key].final {
continue
}
seen[key] = true
pending = append(pending, pendingRead{key: key, attempt: m.vzdumpJobTaskCache[key].lastAttempt, running: job.EndTime.IsZero()})
}
m.mu.RUnlock()
sort.SliceStable(pending, func(i, j int) bool {
return pending[i].attempt.Before(pending[j].attempt)
})
// Positive ranks bind both admission and execution order; zero is unselected.
selected := make(map[string]int, min(len(pending), maxVzdumpJobLogFetchesPerPoll))
for _, read := range pending {
if read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll-1 {
selected[read.key] = len(selected) + 1
}
}
for _, read := range pending {
if !read.running && len(selected) < maxVzdumpJobLogFetchesPerPoll {
selected[read.key] = len(selected) + 1
}
}
for _, read := range pending {
if len(selected) >= maxVzdumpJobLogFetchesPerPoll {
break
}
if selected[read.key] == 0 {
selected[read.key] = len(selected) + 1
}
}
// Admission reserves live capacity and an ended turn; execution must share
// the context too. Move only the oldest selected ended read ahead of live
// work when the last actual request was live. If either queue exhausts the
// context, the other starts next poll, rather than waiting behind an entire
// slow backlog. The shared time budget can still defer admitted live reads.
if historyFirst {
for _, read := range pending {
if rank := selected[read.key]; !read.running && rank > 0 {
for key, priority := range selected {
if priority < rank {
selected[key] = priority + 1
}
}
selected[read.key] = 1
break
}
}
}
return selected
}
// Retain the last successful log observation when a non-final log is unread or
// unavailable. A freshly ended parent proves the guest is no longer running,
// not whether it succeeded. Do not mutate accepted task evidence: only another
// accepted log read can complete attribution and make the entry final.
func cachedVzdumpGuestTasks(tasks []models.BackupTask, job models.BackupTask) []models.BackupTask {
out := append([]models.BackupTask(nil), tasks...)
if job.EndTime.IsZero() {
return out
}
for i := range out {
if out[i].Status == "running" {
out[i].Status = "unknown"
out[i].Error = "backup job ended without an observed guest result"
}
if out[i].EndTime.IsZero() {
out[i].EndTime = job.EndTime
}
}
return out
}
// pruneVzdumpJobTaskCache drops cached job entries for this instance whose
// UPIDs are no longer present in the task list (fell out of the window).
func (m *Monitor) pruneVzdumpJobTaskCache(instanceName string, seen map[string]struct{}) {
prefix := instanceName + "|"
m.mu.Lock()
if len(seen) == 0 {
delete(m.vzdumpJobLogHistoryFirst, instanceName)
}
for key := range m.vzdumpJobTaskCache {
if !strings.HasPrefix(key, prefix) {
continue
}
if _, ok := seen[key]; !ok {
delete(m.vzdumpJobTaskCache, key)
}
}
m.mu.Unlock()
}
func refreshObservedAt(tasks []models.BackupTask, now time.Time) []models.BackupTask {
if len(tasks) == 0 {
return nil
}
out := make([]models.BackupTask, len(tasks))
copy(out, tasks)
for i := range out {
out[i].ObservedAt = now
}
return out
}
// vzdump job logs mark each guest's section with these lines.
var (
vzdumpLogGuestStartRe = regexp.MustCompile(`^INFO: Starting Backup of VM (\d+)`)
vzdumpLogGuestFinishRe = regexp.MustCompile(`^INFO: Finished Backup of VM (\d+) \((\d+):(\d{2}):(\d{2})\)$`)
vzdumpLogGuestFailRe = regexp.MustCompile(`^ERROR: Backup of VM (\d+) failed -\s*(.*)$`)
)
// parseVzdumpJobLog turns a multi-guest vzdump job run into synthetic
// per-guest BackupTask entries. Log lines carry no timestamps, so per-guest
// times are reconstructed from the job start plus the durations vzdump prints
// on each "Finished Backup" line. Task IDs embed the parent UPID, keeping them
// stable across polls and distinct from individually-run guest backups.
func parseVzdumpJobLog(jobTask models.BackupTask, upid string, logLines []proxmox.TaskLogLine) []models.BackupTask {
type guestSection struct {
vmid int
start time.Time
end time.Time
status string
errMsg string
}
var order []int
sections := make(map[int]*guestSection)
cursor := jobTask.StartTime
ensureSection := func(vmid int) *guestSection {
if sec, ok := sections[vmid]; ok {
return sec
}
sec := &guestSection{vmid: vmid, start: cursor, status: "running"}
sections[vmid] = sec
order = append(order, vmid)
return sec
}
for _, line := range logLines {
text := strings.TrimSpace(line.Text)
if match := vzdumpLogGuestStartRe.FindStringSubmatch(text); match != nil {
vmid, err := strconv.Atoi(match[1])
if err != nil || vmid <= 0 {
continue
}
sec := ensureSection(vmid)
sec.start = cursor
sec.status = "running"
} else if match := vzdumpLogGuestFinishRe.FindStringSubmatch(text); match != nil {
vmid, err := strconv.Atoi(match[1])
if err != nil || vmid <= 0 {
continue
}
sec := ensureSection(vmid)
if sec.status == "running" {
sec.status = "OK"
}
hours, _ := strconv.Atoi(match[2])
minutes, _ := strconv.Atoi(match[3])
seconds, _ := strconv.Atoi(match[4])
duration := time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second
sec.end = sec.start.Add(duration)
cursor = sec.end
} else if match := vzdumpLogGuestFailRe.FindStringSubmatch(text); match != nil {
vmid, err := strconv.Atoi(match[1])
if err != nil || vmid <= 0 {
continue
}
sec := ensureSection(vmid)
sec.status = "error"
sec.errMsg = strings.TrimSpace(match[2])
if sec.errMsg == "" {
sec.errMsg = "backup failed"
}
}
}
if len(order) == 0 {
return nil
}
jobFinished := !jobTask.EndTime.IsZero()
tasks := make([]models.BackupTask, 0, len(order))
for _, vmid := range order {
sec := sections[vmid]
if jobFinished {
if sec.status == "running" {
// The parent ended, but the returned log may be truncated or
// missing a guest verdict. Neither a failed nor an OK parent
// establishes this guest's result. Keep explicit finish/failure
// lines above authoritative; do not invent either outcome here.
sec.status = "unknown"
sec.errMsg = "backup job ended without an observed guest result"
}
if sec.end.IsZero() {
sec.end = jobTask.EndTime
}
}
tasks = append(tasks, models.BackupTask{
ID: fmt.Sprintf("%s-%s-vm%d", jobTask.Instance, upid, vmid),
Node: jobTask.Node,
Instance: jobTask.Instance,
Type: jobTask.Type,
VMID: vmid,
Status: sec.status,
Error: sec.errMsg,
StartTime: sec.start,
EndTime: sec.end,
})
}
return tasks
}
func (m *Monitor) pollPVEBackupsAsync(
instanceName string,
instanceCfg *config.PVEInstance,
client PVEClientInterface,
nodes []proxmox.Node,
nodeEffectiveStatus map[string]string,
) {
// Poll backups if enabled - respect configured interval or cycle gating
if !instanceCfg.MonitorBackups {
return
}
if !m.backupPollingEnabledSetting() {
log.Debug().
Str("instance", instanceName).
Msg("Skipping backup polling - globally disabled")
return
}
now := time.Now()
m.mu.RLock()
lastPoll := m.lastPVEBackupPoll[instanceName]
m.mu.RUnlock()
shouldPoll, reason, newLast := m.shouldRunBackupPoll(lastPoll, now)
if !shouldPoll {
if reason != "" {
log.Debug().
Str("instance", instanceName).
Str("reason", reason).
Msg("Skipping PVE backup polling this cycle")
}
return
}
// Atomically claim this connection before dispatch. A timestamp is cadence,
// not an in-flight fence: a scan can outlast the next eligible core cycle.
owner := m.beginPVEBackupPoll(instanceName, client, lastPoll, newLast)
if owner == nil {
return
}
// Core-cycle cancellation must not cancel optional background inventory.
ctx := context.WithValue(owner.ctx, pveBackupPollContextKey{}, owner)
// The caller may reuse its inventory slices/maps on the next core cycle.
nodes = append([]proxmox.Node(nil), nodes...)
statuses := make(map[string]string, len(nodeEffectiveStatus))
for node, status := range nodeEffectiveStatus {
statuses[node] = status
}
// Run backup polling in a separate goroutine to avoid blocking real-time stats
go func(startTime time.Time, inst string, pveClient PVEClientInterface) {
defer m.finishPVEBackupPoll(inst, owner)
defer recoverFromPanic(fmt.Sprintf("pollPVEBackups-%s", inst))
timeout := m.calculateBackupOperationTimeout(inst)
log.Info().
Str("instance", inst).
Dur("timeout", timeout).
Msg("Starting background backup/snapshot polling")
m.pollPVEBackupsAndSnapshots(ctx, inst, pveClient, nodes, statuses, timeout)
duration := time.Since(startTime)
log.Info().
Str("instance", inst).
Dur("duration", duration).
Msg("Completed background backup/snapshot polling")
}(now, instanceName, client)
}
// checkMockAlerts checks alerts for mock data