Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep explicitly unknown PVE archives off guest protection
pulse-triage[bot]676e94c9b115
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 32a473c27a..29249eb14c 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -4627,6 +4627,17 @@ named and default organizations. The custom event envelope stays unchanged.## Current State+PVE backup collection keeps an explicitly unclassified archive separate from+the live occupant of its VMID. Only matching recognised guest kinds can borrow+current placement; model date selection and recovery subject mapping cannot+upgrade `unknown`/unsupported non-empty kinds through the legacy empty-type+fallback. The raw archive remains available with its original facts. The+normal-suite `TestPVEUnknownArchiveKindLifecycle` exercises polling, asynchronous+ingestion, canonical dates and postures across unknown-only, typed, partial,+recovered and absent inventories, including a namesake in another connection.+This preserves existing artifact-kind safety, not an explanation of #3248 or+proof of installed guest recovery. No host-agent lifecycle authority changes.+### Tenant monitors read agent credentials from both token inventoriesA non-default organization's monitor config is a deep copy whose `APITokens`diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 7d1909e3e4..68d5692537 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -213,6 +213,21 @@ A disk named `qemu-archive` may hold LXC backups without hiding them behind theVM filter or emitting VM-typed PBS guest confirmations. VMID-zero host handlingand template/ISO admission stay unchanged.+An explicit `unknown` or unsupported non-empty kind is not an absent legacy+discriminator. Keep that archive's ID, read node, repository, date, size and+completion visible, but do not borrow a live VM/LXC's kind, name, canonical+subject or placement merely because its VMID matches. It cannot advance that+guest's LastBackup or protected posture, or hide an older correctly typed+backup. Only the empty-type legacy path retains best-effort matching.++`TestPVEUnknownArchiveKindLifecycle` follows opaque and recognised files through+polling, live dates, asynchronous ingestion and resource-ID postures, including+ordinary typed recovery, failed reads, return to unknown-only inventory and+accepted absence. It preserves a namesake in another connection. Mapper and+model tests also cover explicit unsupported/host kinds and the empty legacy+control. Actual lifecycle responses drive desktop and narrow Coverage, By date+and workload backup-status browser verification; no guest operation is used.+The normal-suite `pve_backup_content_type_test.go` exercises named-disk andsuffix collisions, prefixed/prefixless local and PBS paths, exact-formatfallback, unknowns, host/non-backup controls and typed direct-PBS confirmationsdiff --git a/internal/models/backup_in_progress_test.go b/internal/models/backup_in_progress_test.goindex 8b578e8eb4..c5a290efe2 100644--- a/internal/models/backup_in_progress_test.go+++ b/internal/models/backup_in_progress_test.go@@ -330,7 +330,7 @@ func TestSyncGuestBackupTimesLocalArchiveGuestKinds(t *testing.T) {}func TestSyncGuestBackupTimesUntypedLocalArchiveCompatibility(t *testing.T) {- for _, kind := range []string{"", "unknown", "legacy"} {+ for _, kind := range []string{"", " "} {t.Run(kind, func(t *testing.T) {state := NewState()now := time.Now().UTC().Truncate(time.Second)@@ -343,3 +343,33 @@ func TestSyncGuestBackupTimesUntypedLocalArchiveCompatibility(t *testing.T) {})}}++func TestSyncGuestBackupTimesExplicitUnknownArchiveIsNotGuestEvidence(t *testing.T) {+ for _, kind := range []string{"unknown", " UNKNOWN ", "legacy", "host"} {+ t.Run(kind, func(t *testing.T) {+ state := NewState()+ now := time.Now().UTC().Truncate(time.Second)+ state.UpdateVMs([]VM{{VMID: 100, Instance: "a"}, {VMID: 100, Instance: "b"}})+ state.UpdateContainers([]Container{{VMID: 100, Instance: "a"}})+ for _, withTyped := range []bool{false, true} {+ rows := []StorageBackup{{ID: "opaque", Instance: "a", VMID: 100, Type: kind, Time: now}}+ wantVM, wantCT := time.Time{}, time.Time{}+ if withTyped {+ wantVM, wantCT = now.Add(-time.Hour), now.Add(-2*time.Hour)+ rows = append(rows,+ StorageBackup{ID: "vm", Instance: "a", VMID: 100, Type: "qemu", Time: wantVM},+ StorageBackup{ID: "ct", Instance: "a", VMID: 100, Type: "lxc", Time: wantCT})+ }+ state.UpdateStorageBackupsForInstance("a", rows)+ state.SyncGuestBackupTimes()+ snap := state.GetSnapshot()+ if !snap.VMs[0].LastBackup.Equal(wantVM) || !snap.Containers[0].LastBackup.Equal(wantCT) || !snap.VMs[1].LastBackup.IsZero() {+ t.Fatalf("unknown archive supplied or hid a guest date: vms=%+v containers=%+v", snap.VMs, snap.Containers)+ }+ if snap.VMs[0].BackupInProgress || snap.Containers[0].BackupInProgress || len(snap.PVEBackups.StorageBackups) != len(rows) {+ t.Fatal("unknown identity changed activity or hid an artifact")+ }+ }+ })+ }+}diff --git a/internal/models/models.go b/internal/models/models.goindex 780497aae5..b9c9596f51 100644--- a/internal/models/models.go+++ b/internal/models/models.go@@ -4632,15 +4632,18 @@ type localBackupSubjectKey struct {// localBackupGuestType keeps explicit archive kinds consistent between date// selection and placement alignment. An untyped legacy row keeps its existing-// best-effort VMID lookup; a known contradictory kind never uses that fallback.+// best-effort VMID lookup; an explicitly unknown or contradictory kind never+// uses that fallback.func localBackupGuestType(kind string) string {switch strings.ToLower(strings.TrimSpace(kind)) {case "qemu", "vm":return "vm"case "lxc", "ct":return "ct"- default:+ case "":return ""+ default:+ return "unknown"}}@@ -4780,8 +4783,8 @@ func (s *State) SyncGuestBackupTimes() {}latestLocalBackup := func(instance string, vmid int, guestType string) time.Time {latest := latestBackup[localBackupSubjectKey{instance, guestType, vmid}]- // Preserve best-effort matching of older/untyped rows. Only an explicit- // artifact discriminator can establish a contradictory guest kind.+ // Preserve best-effort matching only for an absent legacy discriminator.+ // An explicitly unknown kind cannot establish either guest's identity.if untyped := latestBackup[localBackupSubjectKey{instance, "", vmid}]; untyped.After(latest) {latest = untyped}diff --git a/internal/models/state_host_test.go b/internal/models/state_host_test.goindex d23545fa81..8e1faaab4f 100644--- a/internal/models/state_host_test.go+++ b/internal/models/state_host_test.go@@ -2267,7 +2267,7 @@ func TestUpdateNodesForInstanceHoldsASplitAgentBackBeforeItFoldsNodeViews(t *tes}func TestUpdateStorageBackupsForInstanceKeepsArchiveGuestKind(t *testing.T) {- for _, kind := range []string{"qemu", "vm", " QEMU ", "lxc", "ct", " LXC ", "", "unknown"} {+ for _, kind := range []string{"qemu", "vm", " QEMU ", "lxc", "ct", " LXC ", "", "unknown", " UNKNOWN ", "unsupported", "host"} {t.Run(kind, func(t *testing.T) {state := NewState()state.UpdateVMs([]VM{{VMID: 100, Instance: "a", Node: "vm-node"}, {VMID: 100, Instance: "b", Node: "other-cluster"}})@@ -2280,6 +2280,8 @@ func TestUpdateStorageBackupsForInstanceKeepsArchiveGuestKind(t *testing.T) {want101 = "read-node"case "lxc", "ct", " LXC ":want100 = "read-node"+ case "unknown", " UNKNOWN ", "unsupported", "host":+ want100, want101 = "read-node", "read-node"}snapshot := state.GetSnapshot()if len(snapshot.PVEBackups.StorageBackups) != 2 {diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.goindex d353c7c4b1..39f62a150a 100644--- a/internal/monitoring/monitor_backups.go+++ b/internal/monitoring/monitor_backups.go@@ -435,9 +435,9 @@ func (m *Monitor) pollStorageBackupsWithNodes(ctx context.Context, instanceNamecurrentKind := guestInfo[proxmoxmapperKey(instanceName, actualNode, content.VMID)].ResourceType// A reused VMID does not move a removed guest's archive// to the opposite-kind occupant's current placement.- contradictoryKind := backupType == "qemu" && currentKind == unifiedresources.ResourceTypeSystemContainer ||- backupType == "lxc" && currentKind == unifiedresources.ResourceTypeVM- if !contradictoryKind {+ matchingKind := backupType == "qemu" && currentKind == unifiedresources.ResourceTypeVM ||+ backupType == "lxc" && currentKind == unifiedresources.ResourceTypeSystemContainer+ if matchingKind {backupNode = actualNode}}diff --git a/internal/monitoring/pve_unknown_archive_kind_test.go b/internal/monitoring/pve_unknown_archive_kind_test.gonew file mode 100644index 0000000000..d8c26c2c89--- /dev/null+++ b/internal/monitoring/pve_unknown_archive_kind_test.go@@ -0,0 +1,124 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/json"+ "fmt"+ "os"+ "reflect"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// A positive VMID alone cannot turn an explicitly unclassified file into the+// current occupant's backup. Exercise real polling, live state, asynchronous+// ingestion and canonical posture, without a guest or backup operation.+func TestPVEUnknownArchiveKindLifecycle(t *testing.T) {+ f := newPVEReconciliationFixture(t)+ f.nodes = f.nodes[:1]+ f.ids = []string{+ unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "cluster", 101),+ unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeSystemContainer, "cluster", 102),+ unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "other", 101),+ }+ f.m.state.UpdateVMs([]models.VM{+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "current-vm", Status: "running"},+ {ID: makeGuestID("other", "node-b", 101), VMID: 101, Instance: "other", Node: "node-b", Name: "current-vm", Status: "running"},+ })+ f.m.state.UpdateContainers([]models.Container{{ID: makeGuestID("cluster", "node-b", 102), VMID: 102, Instance: "cluster", Node: "node-b", Name: "current-lxc", Status: "running"}})+ unknown := []proxmox.StorageContent{+ {Volid: "qemu-archive:backup/opaque-101", VMID: 101, CTime: f.client.at.Unix(), Size: 2048, Content: "backup", Format: "tar.zst"},+ {Volid: "qemu-archive:backup/opaque-102", VMID: 102, CTime: f.client.at.Unix(), Size: 4096, Content: "backup", Format: "not-pbs-ct"},+ }+ typedAt := f.client.at.Add(-time.Hour)+ typed := []proxmox.StorageContent{+ {Volid: "qemu-archive:backup/vzdump-qemu-101.vma.zst", VMID: 101, CTime: typedAt.Unix(), Size: 8192, Content: "backup", Format: "vma.zst"},+ {Volid: "qemu-archive:backup/vzdump-lxc-102.tar.zst", VMID: 102, CTime: typedAt.Unix(), Size: 16384, Content: "backup", Format: "tar.zst"},+ }+ client := &backupTypeLifecycleClient{backupContentTypeClient: backupContentTypeClient{storage: "qemu-archive"}, rows: unknown}+ frames := map[string]any{}+ var accepted map[string]recovery.RecoveryPoint+ for _, phase := range []string{"unknown", "typed", "partial", "recovered", "unknown-again", "empty"} {+ switch phase {+ case "typed":+ client.rows = append(append([]proxmox.StorageContent(nil), unknown...), typed...)+ case "partial":+ client.err = fmt.Errorf("503 unavailable")+ case "recovered":+ client.err = nil+ case "unknown-again":+ client.rows = unknown+ case "empty":+ client.rows = nil+ }+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)+ f.wait(t)+ snap, points, postures := f.m.state.GetSnapshot(), f.points(t), f.postures(t)+ rows := snap.PVEBackups.StorageBackups+ if len(rows) != len(client.rows) || len(points) != len(rows) {+ t.Fatalf("phase=%s lost artifacts: live=%d points=%d", phase, len(rows), len(points))+ }+ for _, row := range rows {+ p, ok := points["pve-backup:"+row.ID]+ if !ok || p.CompletedAt == nil || !p.CompletedAt.Equal(row.Time) || p.SizeBytes == nil || *p.SizeBytes != row.Size || p.Details["volid"] != row.Volid || p.Details["type"] != row.Type || row.StorageNode != "node-a" || row.InProgress {+ t.Fatalf("phase=%s archive facts changed: row=%+v point=%+v", phase, row, p)+ }+ if row.Type == "unknown" {+ if row.Node != "node-a" || p.SubjectResourceID != "" || p.SubjectRef != nil || p.Evidence == nil || p.Evidence.Subject.ResourceID != "" {+ t.Fatalf("phase=%s unknown archive borrowed occupant: row=%+v point=%+v", phase, row, p)+ }+ } else {+ i := row.VMID - 101+ if row.Node != "node-b" || p.SubjectResourceID != f.ids[i] || p.SubjectRef == nil || p.SubjectRef.Class != "node-b" {+ t.Fatalf("phase=%s typed attribution lost: row=%+v point=%+v", phase, row, p)+ }+ }+ }+ withTyped := phase == "typed" || phase == "partial" || phase == "recovered"+ want := time.Time{}+ if withTyped {+ want = typedAt+ }+ if !snap.VMs[0].LastBackup.Equal(want) || !snap.Containers[0].LastBackup.Equal(want) || !snap.VMs[1].LastBackup.IsZero() {+ t.Fatalf("phase=%s unknown file supplied/hid a canonical date: vms=%+v containers=%+v", phase, snap.VMs, snap.Containers)+ }+ for _, p := range postures {+ own := p.SubjectResourceID == f.ids[0] || p.SubjectResourceID == f.ids[1]+ if own && withTyped {+ if p.LastSuccessfulPointAt == nil || !p.LastSuccessfulPointAt.Equal(typedAt) || (phase != "partial" && p.State != recovery.ProtectionStateProtected) || (phase == "partial" && p.State == recovery.ProtectionStateProtected) {+ t.Fatalf("phase=%s wrong typed posture: %+v", phase, p)+ }+ } else if p.LastSuccessfulPointAt != nil || p.State == recovery.ProtectionStateProtected {+ t.Fatalf("phase=%s unknown file protected a guest: %+v", phase, p)+ }+ }+ if phase == "typed" {+ accepted = points+ }+ if phase == "partial" && !reflect.DeepEqual(accepted, points) {+ t.Fatal("unread inventory rewrote/renewed archive evidence")+ }+ frames[phase] = map[string]any{"pve": map[string]any{"data": snap.PVEBackups}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points, "vms": snap.VMs, "containers": snap.Containers}+ }+ if os.Getenv("PULSE_UNKNOWN_ARCHIVE_FRAMES") == "1" {+ hashes := map[string]string{}+ for _, path := range []string{"../models/models.go", "../recovery/mapper/proxmox/mapper.go", "monitor_backups.go"} {+ b, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ hashes[path] = fmt.Sprintf("%x", sha256.Sum256(b))+ }+ b, err := json.Marshal(map[string]any{"runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})+ if err != nil {+ t.Fatal(err)+ }+ t.Log("UNKNOWN_ARCHIVE_FRAMES=" + string(b))+ }+}diff --git a/internal/recovery/mapper/proxmox/mapper.go b/internal/recovery/mapper/proxmox/mapper.goindex 836b0cb9ae..7e13f145d4 100644--- a/internal/recovery/mapper/proxmox/mapper.go+++ b/internal/recovery/mapper/proxmox/mapper.go@@ -269,8 +269,10 @@ func FromPVEStorageBackups(backups []models.StorageBackup, guestInfoByKey map[stkey := guestLookupKey(instanceName, nodeName, vmid)info := guestInfoByKey[key]resourceType := resourceTypeFromGuestType(b.Type)- if resourceType == "" {- // Untyped legacy rows keep the existing best-effort lookup.+ if resourceType == "" && strings.TrimSpace(b.Type) == "" {+ // Only an absent legacy discriminator uses best-effort lookup.+ // An explicitly unknown/unsupported archive kind cannot borrow+ // the current occupant's type and claim its protection.resourceType = info.ResourceType} else if info.ResourceType != "" && info.ResourceType != resourceType {// VMIDs can be reused across guest kinds. Keep the historicaldiff --git a/internal/recovery/mapper/proxmox/mapper_test.go b/internal/recovery/mapper/proxmox/mapper_test.goindex 1c2ae73293..238bd51b0c 100644--- a/internal/recovery/mapper/proxmox/mapper_test.go+++ b/internal/recovery/mapper/proxmox/mapper_test.go@@ -878,7 +878,6 @@ func TestFromPVEStorageBackupsRetainsExplicitGuestKind(t *testing.T) {{"matching-vm", "qemu", unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeVM},{"matching-lxc", "lxc", unifiedresources.ResourceTypeSystemContainer, unifiedresources.ResourceTypeSystemContainer},{"untyped-legacy", "", unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeVM},- {"unknown-legacy", "unknown", unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeVM},} {t.Run(tc.name, func(t *testing.T) {now := time.Now().UTC().Truncate(time.Second)@@ -917,3 +916,32 @@ func TestFromPVEStorageBackupsRetainsExplicitGuestKind(t *testing.T) {})}}++func TestFromPVEStorageBackupsExplicitUnknownKindCannotBorrowSubject(t *testing.T) {+ for _, kind := range []string{"unknown", " UNKNOWN ", "unsupported", "host"} {+ for _, liveType := range []unifiedresources.ResourceType{unifiedresources.ResourceTypeVM, unifiedresources.ResourceTypeSystemContainer} {+ t.Run(kind+"/"+string(liveType), func(t *testing.T) {+ now := time.Now().UTC().Truncate(time.Second)+ b := models.StorageBackup{ID: "opaque", Instance: "cluster", Node: "read-node", VMID: 100, Type: kind, Time: now, Size: 2048, Storage: "local", Volid: "local:backup/opaque"}+ info := map[string]GuestInfo{"cluster|read-node|100": {ResourceID: "current-canonical", SourceID: "current-source", ResourceType: liveType, Name: "current-occupant"}}+ for _, running := range []bool{false, true} {+ b.InProgress = running+ points, err := FromPVEStorageBackupsWithEvidence([]models.StorageBackup{b}, info, now)+ if err != nil || len(points) != 1 {+ t.Fatalf("points=%+v err=%v", points, err)+ }+ p := points[0]+ if p.SubjectResourceID != "" || p.SubjectRef != nil || p.Evidence == nil || p.Evidence.Subject.ResourceID != "" {+ t.Fatalf("unclassified archive borrowed a current subject: %+v", p)+ }+ if p.ID != "pve-backup:opaque" || p.StartedAt == nil || !p.StartedAt.Equal(now) || p.SizeBytes == nil || *p.SizeBytes != b.Size || p.Details["type"] != kind || p.Details["volid"] != b.Volid || p.ProviderScope != "cluster" {+ t.Fatalf("lost independent archive facts: %+v", p)+ }+ if running && (p.Outcome != recovery.OutcomeRunning || p.CompletedAt != nil) || !running && (p.Outcome != recovery.OutcomeSuccess || p.CompletedAt == nil || !p.CompletedAt.Equal(now)) {+ t.Fatalf("unknown identity changed completion: %+v", p)+ }+ }+ })+ }+ }+}