Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep conflicting archive subjects off guest protection

pulse-triage[bot]babd8af8a8d9
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 63394cbbbc..ee7ead4d74 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -307,6 +307,30 @@ A disk named `qemu-archive` may hold LXC backups without hiding them behind the
VM filter or emitting VM-typed PBS guest confirmations. VMID-zero host handling
and template/ISO admission stay unchanged.
+A recognised archive path must encode a positive numeric guest ID that agrees
+with the returned VMID before supplying that kind. Malformed, overflowing or
+conflicting path subjects stay explicitly unknown even when a PBS format would
+otherwise supply a fallback. Retain the raw volume ID, returned VMID, date,
+size and read origin without choosing either subject. Such a row supplies no
+live guest date, placement, recovery subject or direct-PBS guest confirmation.
+Ordinary corrected re-observation restores attribution under the same archive
+ID; a failed read does not renew or reinterpret prior evidence. Opaque paths
+with exact PBS formats keep their existing fallback, and VMID-zero host and
+non-backup admission remain unchanged.
+
+`TestPVEArchiveSubjectRequiresAgreement` covers both archive kinds and PBS
+paths, optional storage/backup prefixes and malformed numeric tokens.
+`TestPVEArchiveSubjectCannotConfirmAnotherPBSGuest` checks confirmation
+withdrawal and recovery for both kinds. The normal-suite
+`TestPVEArchiveSubjectAgreementLifecycle` follows conflicts, corrected metadata,
+failed reads, recurrence, recovery and accepted absence through live inventory,
+canonical dates, asynchronous recovery points and protection postures. An older
+correctly attributed backup, independent LXC and another connection's namesake
+stay isolated. Its actual frames drive desktop and narrow Backups proof; this
+is attribution safety, not original guest recovery or a diagnosis of stale
+protection reports. No guest, backup, restore, store/schema or access operation
+is added.
+
An explicit `unknown` or unsupported non-empty kind is not an absent legacy
discriminator. Keep that archive's ID, read node, repository, date, size and
completion visible, but do not borrow a live VM/LXC's kind, name, canonical
diff --git a/internal/monitoring/monitor_backups.go b/internal/monitoring/monitor_backups.go
index e388863939..e2217c0451 100644
--- a/internal/monitoring/monitor_backups.go
+++ b/internal/monitoring/monitor_backups.go
@@ -204,11 +204,39 @@ func pveStorageBackupType(content proxmox.StorageContent) string {
volume = volume[colon+1:]
}
volume = strings.TrimPrefix(volume, "backup/")
+ var kind, encodedID string
switch {
- case strings.HasPrefix(volume, "vzdump-qemu-"), strings.HasPrefix(volume, "vm/"):
- return "qemu"
- case strings.HasPrefix(volume, "vzdump-lxc-"), strings.HasPrefix(volume, "ct/"):
- return "lxc"
+ case strings.HasPrefix(volume, "vzdump-qemu-"):
+ kind, encodedID = "qemu", strings.TrimPrefix(volume, "vzdump-qemu-")
+ case strings.HasPrefix(volume, "vzdump-lxc-"):
+ kind, encodedID = "lxc", strings.TrimPrefix(volume, "vzdump-lxc-")
+ case strings.HasPrefix(volume, "vm/"):
+ kind, encodedID = "qemu", strings.TrimPrefix(volume, "vm/")
+ case strings.HasPrefix(volume, "ct/"):
+ kind, encodedID = "lxc", strings.TrimPrefix(volume, "ct/")
+ }
+ if kind != "" {
+ if strings.HasPrefix(volume, "vzdump-") {
+ if end := strings.IndexAny(encodedID, "-."); end >= 0 {
+ encodedID = encodedID[:end]
+ }
+ } else {
+ encodedID, _, _ = strings.Cut(encodedID, "/")
+ }
+ // A kind prefix is not sufficient subject evidence. Inconsistent
+ // metadata must not credit either guest, lend its current placement,
+ // or confirm a direct PBS snapshot. Keep the raw archive as unknown;
+ // a format fallback cannot overrule a malformed/conflicting subject.
+ for _, digit := range encodedID {
+ if digit < '0' || digit > '9' {
+ return "unknown"
+ }
+ }
+ id, err := strconv.Atoi(encodedID)
+ if err != nil || id <= 0 || id != content.VMID {
+ return "unknown"
+ }
+ return kind
}
// Explicit PBS formats remain a fallback for paths without a recognised
// discriminator. A substring of an unrelated format is not type evidence.
diff --git a/internal/monitoring/pve_archive_subject_agreement_test.go b/internal/monitoring/pve_archive_subject_agreement_test.go
new file mode 100644
index 0000000000..4eb7f09bec
--- /dev/null
+++ b/internal/monitoring/pve_archive_subject_agreement_test.go
@@ -0,0 +1,205 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/recovery"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// An artifact's encoded subject and its returned VMID must agree before the
+// collector may lend that archive to a current workload. Keep conflicting raw
+// evidence visible rather than guessing which independent field is correct.
+func TestPVEArchiveSubjectRequiresAgreement(t *testing.T) {
+ for _, tc := range []struct{ name, path, format string }{
+ {"other-vm", "vzdump-qemu-102-2026_10_29.vma.zst", "vma.zst"},
+ {"other-lxc", "vzdump-lxc-102-2026_10_29.tar.zst", "tar.zst"},
+ {"bare-other-vm", "vzdump-qemu-102.vma.zst", "vma.zst"},
+ {"pbs-other-vm", "vm/102/2026-10-29T00:00:00Z", "pbs-vm"},
+ {"pbs-other-ct", "ct/102/2026-10-29T00:00:00Z", "pbs-ct"},
+ {"empty-vzdump-id", "vzdump-qemu-.vma.zst", "pbs-vm"},
+ {"negative-vzdump-id", "vzdump-qemu--101.vma.zst", "pbs-vm"},
+ {"nonnumeric-vzdump-id", "vzdump-lxc-unknown.tar.zst", "pbs-ct"},
+ {"numeric-prefix-not-id", "vzdump-qemu-101other.vma.zst", "pbs-vm"},
+ {"overflow-vzdump-id", "vzdump-qemu-999999999999999999999999.vma.zst", "pbs-vm"},
+ {"empty-pbs-id", "vm//2026-10-29T00:00:00Z", "pbs-vm"},
+ {"negative-pbs-id", "ct/-101/2026-10-29T00:00:00Z", "pbs-ct"},
+ {"zero-pbs-id", "vm/0/2026-10-29T00:00:00Z", "pbs-vm"},
+ {"nonnumeric-pbs-id", "ct/101other/2026-10-29T00:00:00Z", "pbs-ct"},
+ } {
+ for _, prefix := range []string{"archive:backup/", "backup/", ""} {
+ t.Run(tc.name+"/"+prefix, func(t *testing.T) {
+ row := proxmox.StorageContent{Volid: prefix + tc.path, VMID: 101, Content: "backup", CTime: 1700000000, Size: 4096, Format: tc.format}
+ client := &backupContentTypeClient{storage: "archive", row: row}
+ m := &Monitor{state: models.NewState()}
+ m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, []proxmox.Node{{Node: "node"}}, map[string]string{"node": "online"})
+ rows := m.state.GetSnapshot().PVEBackups.StorageBackups
+ if len(rows) != 1 || rows[0].Type != "unknown" {
+ t.Fatalf("inconsistent archive gained a guest kind: %+v", rows)
+ }
+ got := rows[0]
+ if got.Volid != row.Volid || got.VMID != row.VMID || got.Format != row.Format || got.CTime != row.CTime || got.Size != int64(row.Size) || got.Node != "node" || got.StorageNode != "node" || got.Storage != "archive" || got.Instance != "cluster" {
+ t.Fatalf("raw archive facts changed: %+v", got)
+ }
+ })
+ }
+ }
+}
+
+func TestPVEArchiveSubjectCannotConfirmAnotherPBSGuest(t *testing.T) {
+ for _, kind := range []string{"vm", "ct"} {
+ t.Run(kind, func(t *testing.T) {
+ at := time.Now().UTC().Truncate(time.Second)
+ client := &backupContentTypeClient{storage: "pbs", storageType: "pbs", row: proxmox.StorageContent{Volid: fmt.Sprintf("pbs:backup/%s/102/%s", kind, at.Format(time.RFC3339)), VMID: 102, Content: "backup", CTime: at.Unix(), Size: 4096, Format: "pbs-" + kind}}
+ m := &Monitor{state: models.NewState(), config: &config.Config{PBSInstances: []config.PBSInstance{{Name: "direct-pbs"}}}}
+ for _, vmid := range []int{102, 101, 102} {
+ client.row.VMID = vmid
+ m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, []proxmox.Node{{Node: "node"}}, map[string]string{"node": "online"})
+ confirmations := m.state.PBSGuestConfirmationsForInstance("cluster")
+ want := []models.PBSGuestConfirmation{{Storage: "pbs", BackupType: kind, VMID: 102, Time: at.Unix()}}
+ if vmid == 101 {
+ want = nil
+ }
+ if len(confirmations) != len(want) || len(want) > 0 && !reflect.DeepEqual(confirmations, want) {
+ t.Fatalf("metadata VMID=%d confirmations=%+v want=%+v", vmid, confirmations, want)
+ }
+ if len(m.state.GetSnapshot().PVEBackups.StorageBackups) != 0 {
+ t.Fatal("direct PBS inventory exclusion changed")
+ }
+ }
+ })
+ }
+}
+
+func TestPVEArchiveSubjectAgreementLifecycle(t *testing.T) {
+ f := newPVEReconciliationFixture(t)
+ f.nodes = f.nodes[:1]
+ f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeSystemContainer, "cluster", 103), unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "other", 101))
+ f.m.state.UpdateVMs([]models.VM{
+ {ID: makeGuestID("cluster", "node-b", 101), VMID: 101, Instance: "cluster", Node: "node-b", Name: "original-vm", Status: "running"},
+ {ID: makeGuestID("cluster", "node-b", 102), VMID: 102, Instance: "cluster", Node: "node-b", Name: "archive-vm", Status: "running"},
+ {ID: makeGuestID("other", "node-b", 101), VMID: 101, Instance: "other", Node: "node-b", Name: "namesake-vm", Status: "running"},
+ })
+ f.m.state.UpdateContainers([]models.Container{{ID: makeGuestID("cluster", "node-b", 103), VMID: 103, Instance: "cluster", Node: "node-b", Name: "independent-lxc", Status: "running"}})
+ at := f.client.at
+ client := &backupTypeLifecycleClient{backupContentTypeClient: backupContentTypeClient{storage: "archive"}, rows: []proxmox.StorageContent{
+ {Volid: "archive:backup/vzdump-qemu-102-2026_10_29.vma.zst", VMID: 101, Content: "backup", CTime: at.Unix(), Size: 4096, Format: "vma.zst"},
+ {Volid: "archive:backup/vzdump-qemu-101-older.vma.zst", VMID: 101, Content: "backup", CTime: at.Add(-time.Hour).Unix(), Size: 2048, Format: "vma.zst"},
+ {Volid: "archive:backup/vzdump-lxc-103-2026_10_29.tar.zst", VMID: 103, Content: "backup", CTime: at.Unix(), Size: 8192, Format: "tar.zst"},
+ }}
+ frames := map[string]any{}
+ var accepted map[string]recovery.RecoveryPoint
+ var archiveID string
+ for _, phase := range []string{"conflict", "corrected", "unread", "conflict-again", "recovered", "empty"} {
+ switch phase {
+ case "corrected", "recovered":
+ client.rows[0].VMID = 102
+ client.err = nil
+ case "unread":
+ client.err = fmt.Errorf("503 unavailable")
+ case "conflict-again":
+ client.err = nil
+ client.rows[0].VMID = 101
+ case "empty":
+ client.rows = nil
+ }
+ f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)
+ f.wait(t)
+ snap, points, postures := f.m.state.GetSnapshot(), f.points(t), f.postures(t)
+ empty := phase == "empty"
+ conflict := phase == "conflict" || phase == "conflict-again"
+ want := 3
+ if empty {
+ want = 0
+ }
+ if len(snap.PVEBackups.StorageBackups) != want || len(points) != want {
+ t.Fatalf("phase=%s raw inventory lost: live=%d points=%d", phase, len(snap.PVEBackups.StorageBackups), len(points))
+ }
+ if !empty {
+ row := snap.PVEBackups.StorageBackups[0]
+ for _, r := range snap.PVEBackups.StorageBackups {
+ if r.Volid == client.rows[0].Volid {
+ row = r
+ }
+ }
+ if archiveID == "" {
+ archiveID = row.ID
+ }
+ if row.ID != archiveID {
+ t.Fatal("metadata correction renamed the raw archive")
+ }
+ point := points["pve-backup:"+archiveID]
+ if row.Volid != client.rows[0].Volid || row.CTime != at.Unix() || row.Size != 4096 || point.CompletedAt == nil || !point.CompletedAt.Equal(at) || point.Details["volid"] != row.Volid || row.StorageNode != "node-a" {
+ t.Fatalf("phase=%s raw evidence changed: row=%+v point=%+v", phase, row, point)
+ }
+ if conflict {
+ if row.Type != "unknown" || row.Node != "node-a" || point.SubjectResourceID != "" || point.SubjectRef != nil || point.Evidence != nil {
+ t.Fatalf("phase=%s conflicting archive borrowed a guest: row=%+v point=%+v", phase, row, point)
+ }
+ } else if row.Type != "qemu" || row.Node != "node-b" || point.SubjectResourceID != f.ids[1] {
+ t.Fatalf("phase=%s matching subject lost: row=%+v point=%+v", phase, row, point)
+ }
+ }
+ expected := []time.Time{at.Add(-time.Hour), at, {}, at}
+ if conflict {
+ expected[1] = time.Time{}
+ }
+ if empty {
+ expected = make([]time.Time, 4)
+ }
+ dates := []time.Time{snap.VMs[0].LastBackup, snap.VMs[1].LastBackup, snap.VMs[2].LastBackup, snap.Containers[0].LastBackup}
+ for i := range dates {
+ if !dates[i].Equal(expected[i]) {
+ t.Errorf("phase=%s guest%d LastBackup=%v want=%v", phase, i, dates[i], expected[i])
+ }
+ }
+ for _, posture := range postures {
+ index := -1
+ for i, id := range f.ids {
+ if id == posture.SubjectResourceID {
+ index = i
+ }
+ }
+ if index < 0 {
+ t.Fatal("unexpected subject")
+ }
+ datesByPosture := []time.Time{expected[0], expected[1], expected[3], expected[2]}
+ expectedDate := datesByPosture[index]
+ if expectedDate.IsZero() {
+ if posture.LastSuccessfulPointAt != nil || posture.State == recovery.ProtectionStateProtected {
+ t.Fatalf("phase=%s unsupported protected subject: %+v", phase, posture)
+ }
+ } else if posture.LastSuccessfulPointAt == nil || !posture.LastSuccessfulPointAt.Equal(expectedDate) || phase != "unread" && posture.State != recovery.ProtectionStateProtected || phase == "unread" && posture.State == recovery.ProtectionStateProtected {
+ t.Fatalf("phase=%s wrong dated posture: %+v", phase, posture)
+ }
+ }
+ if phase == "corrected" {
+ accepted = points
+ }
+ if phase == "unread" && !reflect.DeepEqual(accepted, points) {
+ t.Fatal("failed read renewed or rewrote the accepted evidence")
+ }
+ frames[phase] = map[string]any{"pve": map[string]any{"data": snap.PVEBackups}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": points, "vms": snap.VMs, "containers": snap.Containers}
+ }
+ if os.Getenv("PULSE_ARCHIVE_SUBJECT_FRAMES") == "1" {
+ b, err := os.ReadFile("monitor_backups.go")
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := json.Marshal(map[string]any{"runtime_sha256": fmt.Sprintf("%x", sha256.Sum256(b)), "resource_ids": f.ids, "frames": frames})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Log("ARCHIVE_SUBJECT_FRAMES=" + string(data))
+ }
+}