Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep explicitly unknown PVE archives off guest protection

pulse-triage[bot]676e94c9b115
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 32a473c27a..29249eb14c 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -4627,6 +4627,17 @@ named and default organizations. The custom event envelope stays unchanged.
## Current State
+PVE backup collection keeps an explicitly unclassified archive separate from
+the live occupant of its VMID. Only matching recognised guest kinds can borrow
+current placement; model date selection and recovery subject mapping cannot
+upgrade `unknown`/unsupported non-empty kinds through the legacy empty-type
+fallback. The raw archive remains available with its original facts. The
+normal-suite `TestPVEUnknownArchiveKindLifecycle` exercises polling, asynchronous
+ingestion, canonical dates and postures across unknown-only, typed, partial,
+recovered and absent inventories, including a namesake in another connection.
+This preserves existing artifact-kind safety, not an explanation of #3248 or
+proof of installed guest recovery. No host-agent lifecycle authority changes.
+
### Tenant monitors read agent credentials from both token inventories
A non-default organization's monitor config is a deep copy whose `APITokens`
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 7d1909e3e4..68d5692537 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -213,6 +213,21 @@ A disk named `qemu-archive` may hold LXC backups without hiding them behind the
VM filter or emitting VM-typed PBS guest confirmations. VMID-zero host handling
and template/ISO admission stay unchanged.
+An explicit `unknown` or unsupported non-empty kind is not an absent legacy
+discriminator. Keep that archive's ID, read node, repository, date, size and
+completion visible, but do not borrow a live VM/LXC's kind, name, canonical
+subject or placement merely because its VMID matches. It cannot advance that
+guest's LastBackup or protected posture, or hide an older correctly typed
+backup. Only the empty-type legacy path retains best-effort matching.
+
+`TestPVEUnknownArchiveKindLifecycle` follows opaque and recognised files through
+polling, live dates, asynchronous ingestion and resource-ID postures, including
+ordinary typed recovery, failed reads, return to unknown-only inventory and
+accepted absence. It preserves a namesake in another connection. Mapper and
+model tests also cover explicit unsupported/host kinds and the empty legacy
+control. Actual lifecycle responses drive desktop and narrow Coverage, By date
+and workload backup-status browser verification; no guest operation is used.
+
The normal-suite `pve_backup_content_type_test.go` exercises named-disk and
suffix collisions, prefixed/prefixless local and PBS paths, exact-format
fallback, unknowns, host/non-backup controls and typed direct-PBS confirmations
diff --git a/internal/models/backup_in_progress_test.go b/internal/models/backup_in_progress_test.go
index 8b578e8eb4..c5a290efe2 100644
--- a/internal/models/backup_in_progress_test.go
+++ b/internal/models/backup_in_progress_test.go
@@ -330,7 +330,7 @@ func TestSyncGuestBackupTimesLocalArchiveGuestKinds(t *testing.T) {
}
func TestSyncGuestBackupTimesUntypedLocalArchiveCompatibility(t *testing.T) {
- for _, kind := range []string{"", "unknown", "legacy"} {
+ for _, kind := range []string{"", " "} {
t.Run(kind, func(t *testing.T) {
state := NewState()
now := time.Now().UTC().Truncate(time.Second)
@@ -343,3 +343,33 @@ func TestSyncGuestBackupTimesUntypedLocalArchiveCompatibility(t *testing.T) {
})
}
}
+
+func TestSyncGuestBackupTimesExplicitUnknownArchiveIsNotGuestEvidence(t *testing.T) {
+ for _, kind := range []string{"unknown", " UNKNOWN ", "legacy", "host"} {
+ t.Run(kind, func(t *testing.T) {
+ state := NewState()
+ now := time.Now().UTC().Truncate(time.Second)
+ state.UpdateVMs([]VM{{VMID: 100, Instance: "a"}, {VMID: 100, Instance: "b"}})
+ state.UpdateContainers([]Container{{VMID: 100, Instance: "a"}})
+ for _, withTyped := range []bool{false, true} {
+ rows := []StorageBackup{{ID: "opaque", Instance: "a", VMID: 100, Type: kind, Time: now}}
+ wantVM, wantCT := time.Time{}, time.Time{}
+ if withTyped {
+ wantVM, wantCT = now.Add(-time.Hour), now.Add(-2*time.Hour)
+ rows = append(rows,
+ StorageBackup{ID: "vm", Instance: "a", VMID: 100, Type: "qemu", Time: wantVM},
+ StorageBackup{ID: "ct", Instance: "a", VMID: 100, Type: "lxc", Time: wantCT})
+ }
+ state.UpdateStorageBackupsForInstance("a", rows)
+ state.SyncGuestBackupTimes()
+ snap := state.GetSnapshot()
+ if !snap.VMs[0].LastBackup.Equal(wantVM) || !snap.Containers[0].LastBackup.Equal(wantCT) || !snap.VMs[1].LastBackup.IsZero() {
+ t.Fatalf("unknown archive supplied or hid a guest date: vms=%+v containers=%+v", snap.VMs, snap.Containers)
+ }
+ if snap.VMs[0].BackupInProgress || snap.Containers[0].BackupInProgress || len(snap.PVEBackups.StorageBackups) != len(rows) {
+ t.Fatal("unknown identity changed activity or hid an artifact")
+ }
+ }
+ })
+ }
+}
diff --git a/internal/models/models.go b/internal/models/models.go
index 780497aae5..b9c9596f51 100644
--- a/internal/models/models.go
+++ b/internal/models/models.go
@@ -4632,15 +4632,18 @@ type localBackupSubjectKey struct {
// localBackupGuestType keeps explicit archive kinds consistent between date
// selection and placement alignment. An untyped legacy row keeps its existing
-// best-effort VMID lookup; a known contradictory kind never uses that fallback.
+// best-effort VMID lookup; an explicitly unknown or contradictory kind never
+// uses that fallback.
func localBackupGuestType(kind string) string {
switch strings.ToLower(strings.TrimSpace(kind)) {
case "qemu", "vm":
return "vm"
case "lxc", "ct":
return "ct"
- default:
+ case "":
return ""
+ default:
+ return "unknown"
}
}
@@ -4780,8 +4783,8 @@ func (s *State) SyncGuestBackupTimes() {
}
latestLocalBackup := func(instance string, vmid int, guestType string) time.Time {
latest := latestBackup[localBackupSubjectKey{instance, guestType, vmid}]
- // Preserve best-effort matching of older/untyped rows. Only an explicit
- // artifact discriminator can establish a contradictory guest kind.
+ // Preserve best-effort matching only for an absent legacy discriminator.
+ // An explicitly unknown kind cannot establish either guest's identity.
if untyped := latestBackup[localBackupSubjectKey{instance, "", vmid}]; untyped.After(latest) {
latest = untyped
}
diff --git a/internal/models/state_host_test.go b/internal/models/state_host_test.go
index d23545fa81..8e1faaab4f 100644
--- a/internal/models/state_host_test.go
+++ b/internal/models/state_host_test.go
@@ -2267,7 +2267,7 @@ func TestUpdateNodesForInstanceHoldsASplitAgentBackBeforeItFoldsNodeViews(t *tes
}
func TestUpdateStorageBackupsForInstanceKeepsArchiveGuestKind(t *testing.T) {
- for _, kind := range []string{"qemu", "vm", " QEMU ", "lxc", "ct", " LXC ", "", "unknown"} {
+ for _, kind := range []string{"qemu", "vm", " QEMU ", "lxc", "ct", " LXC ", "", "unknown", " UNKNOWN ", "unsupported", "host"} {
t.Run(kind, func(t *testing.T) {
state := NewState()
state.UpdateVMs([]VM{{VMID: 100, Instance: "a", Node: "vm-node"}, {VMID: 100, Instance: "b", Node: "other-cluster"}})
@@ -2280,6 +2280,8 @@ func TestUpdateStorageBackupsForInstanceKeepsArchiveGuestKind(t *testing.T) {
want101 = "read-node"
case "lxc", "ct", " LXC ":
want100 = "read-node"
+ case "unknown", " UNKNOWN ", "unsupported", "host":
+ want100, want101 = "read-node", "read-node"
}
snapshot := state.GetSnapshot()
if len(snapshot.PVEBackups.StorageBackups) != 2 {
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index d353c7c4b1..39f62a150a 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -435,9 +435,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceName
currentKind := guestInfo[proxmoxmapperKey(instanceName, actualNode, content.VMID)].ResourceType
// A reused VMID does not move a removed guest's archive
// to the opposite-kind occupant's current placement.
- contradictoryKind := backupType == "qemu" && currentKind == unifiedresources.ResourceTypeSystemContainer ||
- backupType == "lxc" && currentKind == unifiedresources.ResourceTypeVM
- if !contradictoryKind {
+ matchingKind := backupType == "qemu" && currentKind == unifiedresources.ResourceTypeVM ||
+ backupType == "lxc" && currentKind == unifiedresources.ResourceTypeSystemContainer
+ if matchingKind {
backupNode = actualNode
}
}
diff --git a/internal/monitoring/pve_unknown_archive_kind_test.go b/internal/monitoring/pve_unknown_archive_kind_test.go
new file mode 100644
index 0000000000..d8c26c2c89
--- /dev/null
+++ b/internal/monitoring/pve_unknown_archive_kind_test.go
@@ -0,0 +1,124 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// A positive VMID alone cannot turn an explicitly unclassified file into the
+// current occupant's backup. Exercise real polling, live state, asynchronous
+// ingestion and canonical posture, without a guest or backup operation.
+func TestPVEUnknownArchiveKindLifecycle(t *testing.T) {
+ f := newPVEReconciliationFixture(t)
+ f.nodes = f.nodes[:1]
+ f.ids = []string{
+ unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", 101),
+ unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeSystemContainer, "cluster", 102),
+ unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "other", 101),
+ }
+ f.m.state.UpdateVMs([]models.VM{
+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "current-vm", Status: "running"},
+ {ID: makeGuestID("other", "node-b", 101), VMID: 101, Instance: "other", Node: "node-b", Name: "current-vm", Status: "running"},
+ })
+ f.m.state.UpdateContainers([]models.Container{{ID: makeGuestID("cluster", "node-b", 102), VMID: 102, Instance: "cluster", Node: "node-b", Name: "current-lxc", Status: "running"}})
+ unknown := []proxmox.StorageContent{
+ {Volid: "qemu-archive:backup/opaque-101", VMID: 101, CTime: f.client.at.Unix(), Size: 2048, Content: "backup", Format: "tar.zst"},
+ {Volid: "qemu-archive:backup/opaque-102", VMID: 102, CTime: f.client.at.Unix(), Size: 4096, Content: "backup", Format: "not-pbs-ct"},
+ }
+ typedAt := f.client.at.Add(-time.Hour)
+ typed := []proxmox.StorageContent{
+ {Volid: "qemu-archive:backup/vzdump-qemu-101.vma.zst", VMID: 101, CTime: typedAt.Unix(), Size: 8192, Content: "backup", Format: "vma.zst"},
+ {Volid: "qemu-archive:backup/vzdump-lxc-102.tar.zst", VMID: 102, CTime: typedAt.Unix(), Size: 16384, Content: "backup", Format: "tar.zst"},
+ }
+ client := &backupTypeLifecycleClient{backupContentTypeClient: backupContentTypeClient{storage: "qemu-archive"}, rows: unknown}
+ frames := map[string]any{}
+ var accepted map[string]recovery.RecoveryPoint
+ for _, phase := range []string{"unknown", "typed", "partial", "recovered", "unknown-again", "empty"} {
+ switch phase {
+ case "typed":
+ client.rows = append(append([]proxmox.StorageContent(nil), unknown...), typed...)
+ case "partial":
+ client.err = fmt.Errorf("503 unavailable")
+ case "recovered":
+ client.err = nil
+ case "unknown-again":
+ client.rows = unknown
+ case "empty":
+ client.rows = nil
+ }
+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)
+ f.wait(t)
+ snap, points, postures := f.m.state.GetSnapshot(), f.points(t), f.postures(t)
+ rows := snap.PVEBackups.StorageBackups
+ if len(rows) != len(client.rows) || len(points) != len(rows) {
+ t.Fatalf("phase=%s lost artifacts: live=%d points=%d", phase, len(rows), len(points))
+ }
+ for _, row := range rows {
+ p, ok := points["pve-backup:"+row.ID]
+ if !ok || p.CompletedAt == nil || !p.CompletedAt.Equal(row.Time) || p.SizeBytes == nil || *p.SizeBytes != row.Size || p.Details["volid"] != row.Volid || p.Details["type"] != row.Type || row.StorageNode != "node-a" || row.InProgress {
+ t.Fatalf("phase=%s archive facts changed: row=%+v point=%+v", phase, row, p)
+ }
+ if row.Type == "unknown" {
+ if row.Node != "node-a" || p.SubjectResourceID != "" || p.SubjectRef != nil || p.Evidence == nil || p.Evidence.Subject.ResourceID != "" {
+ t.Fatalf("phase=%s unknown archive borrowed occupant: row=%+v point=%+v", phase, row, p)
+ }
+ } else {
+ i := row.VMID - 101
+ if row.Node != "node-b" || p.SubjectResourceID != f.ids[i] || p.SubjectRef == nil || p.SubjectRef.Class != "node-b" {
+ t.Fatalf("phase=%s typed attribution lost: row=%+v point=%+v", phase, row, p)
+ }
+ }
+ }
+ withTyped := phase == "typed" || phase == "partial" || phase == "recovered"
+ want := time.Time{}
+ if withTyped {
+ want = typedAt
+ }
+ if !snap.VMs[0].LastBackup.Equal(want) || !snap.Containers[0].LastBackup.Equal(want) || !snap.VMs[1].LastBackup.IsZero() {
+ t.Fatalf("phase=%s unknown file supplied/hid a canonical date: vms=%+v containers=%+v", phase, snap.VMs, snap.Containers)
+ }
+ for _, p := range postures {
+ own := p.SubjectResourceID == f.ids[0] || p.SubjectResourceID == f.ids[1]
+ if own && withTyped {
+ if p.LastSuccessfulPointAt == nil || !p.LastSuccessfulPointAt.Equal(typedAt) || (phase != "partial" && p.State != recovery.ProtectionStateProtected) || (phase == "partial" && p.State == recovery.ProtectionStateProtected) {
+ t.Fatalf("phase=%s wrong typed posture: %+v", phase, p)
+ }
+ } else if p.LastSuccessfulPointAt != nil || p.State == recovery.ProtectionStateProtected {
+ t.Fatalf("phase=%s unknown file protected a guest: %+v", phase, p)
+ }
+ }
+ if phase == "typed" {
+ accepted = points
+ }
+ if phase == "partial" && !reflect.DeepEqual(accepted, points) {
+ t.Fatal("unread inventory rewrote/renewed archive evidence")
+ }
+ frames[phase] = map[string]any{"pve": map[string]any{"data": snap.PVEBackups}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points, "vms": snap.VMs, "containers": snap.Containers}
+ }
+ if os.Getenv("PULSE_UNKNOWN_ARCHIVE_FRAMES") == "1" {
+ hashes := map[string]string{}
+ for _, path := range []string{"../models/models.go", "../recovery/mapper/proxmox/mapper.go", "monitor_backups.go"} {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hashes[path] = fmt.Sprintf("%x", sha256.Sum256(b))
+ }
+ b, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log("UNKNOWN_ARCHIVE_FRAMES=" + string(b))
+ }
+}
diff --git a/internal/recovery/mapper/proxmox/mapper.go b/internal/recovery/mapper/proxmox/mapper.go
index 836b0cb9ae..7e13f145d4 100644
--- a/internal/recovery/mapper/proxmox/mapper.go
+++ b/internal/recovery/mapper/proxmox/mapper.go
@@ -269,8 +269,10 @@ func FromPVEStorageBackups(backups []models.StorageBackup, guestInfoByKey map[st
key := guestLookupKey(instanceName, nodeName, vmid)
info := guestInfoByKey[key]
resourceType := resourceTypeFromGuestType(b.Type)
- if resourceType == "" {
- // Untyped legacy rows keep the existing best-effort lookup.
+ if resourceType == "" && strings.TrimSpace(b.Type) == "" {
+ // Only an absent legacy discriminator uses best-effort lookup.
+ // An explicitly unknown/unsupported archive kind cannot borrow
+ // the current occupant's type and claim its protection.
resourceType = info.ResourceType
} else if info.ResourceType != "" && info.ResourceType != resourceType {
// VMIDs can be reused across guest kinds. Keep the historical
diff --git a/internal/recovery/mapper/proxmox/mapper_test.go b/internal/recovery/mapper/proxmox/mapper_test.go
index 1c2ae73293..238bd51b0c 100644
--- a/internal/recovery/mapper/proxmox/mapper_test.go
+++ b/internal/recovery/mapper/proxmox/mapper_test.go
@@ -878,7 +878,6 @@ func TestFromPVEStorageBackupsRetainsExplicitGuestKind(t *testing.T) {
{"matching-vm", "qemu", unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeVM},
{"matching-lxc", "lxc", unifiedresources.ResourceTypeSystemContainer, unifiedresources.ResourceTypeSystemContainer},
{"untyped-legacy", "", unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeVM},
- {"unknown-legacy", "unknown", unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeVM},
} {
t.Run(tc.name, func(t *testing.T) {
now := time.Now().UTC().Truncate(time.Second)
@@ -917,3 +916,32 @@ func TestFromPVEStorageBackupsRetainsExplicitGuestKind(t *testing.T) {
})
}
}
+
+func TestFromPVEStorageBackupsExplicitUnknownKindCannotBorrowSubject(t *testing.T) {
+ for _, kind := range []string{"unknown", " UNKNOWN ", "unsupported", "host"} {
+ for _, liveType := range []unifiedresources.ResourceType{unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeSystemContainer} {
+ t.Run(kind+"/"+string(liveType), func(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ b := models.StorageBackup{ID: "opaque", Instance: "cluster", Node: "read-node", VMID: 100, Type: kind, Time: now, Size: 2048, Storage: "local", Volid: "local:backup/opaque"}
+ info := map[string]GuestInfo{"cluster|read-node|100": {ResourceID: "current-canonical", SourceID: "current-source", ResourceType: liveType, Name: "current-occupant"}}
+ for _, running := range []bool{false, true} {
+ b.InProgress = running
+ points, err := FromPVEStorageBackupsWithEvidence([]models.StorageBackup{b}, info, now)
+ if err != nil || len(points) != 1 {
+ t.Fatalf("points=%+v err=%v", points, err)
+ }
+ p := points[0]
+ if p.SubjectResourceID != "" || p.SubjectRef != nil || p.Evidence == nil || p.Evidence.Subject.ResourceID != "" {
+ t.Fatalf("unclassified archive borrowed a current subject: %+v", p)
+ }
+ if p.ID != "pve-backup:opaque" || p.StartedAt == nil || !p.StartedAt.Equal(now) || p.SizeBytes == nil || *p.SizeBytes != b.Size || p.Details["type"] != kind || p.Details["volid"] != b.Volid || p.ProviderScope != "cluster" {
+ t.Fatalf("lost independent archive facts: %+v", p)
+ }
+ if running && (p.Outcome != recovery.OutcomeRunning || p.CompletedAt != nil) || !running && (p.Outcome != recovery.OutcomeSuccess || p.CompletedAt == nil || !p.CompletedAt.Equal(now)) {
+ t.Fatalf("unknown identity changed completion: %+v", p)
+ }
+ }
+ })
+ }
+ }
+}