Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.0 KB
Raw Blame
package models

import (
	"testing"
	"time"
)

// An in-flight PBS snapshot (no manifest yet) was previously counted as a
// completed backup: the guest's LastBackup jumped to "now" the moment vzdump
// started, and the badge showed a healthy backup that did not exist.
// These tests pin the corrected behaviour: in-progress evidence feeds
// BackupInProgress only, and LastBackup tracks completed backups.

func TestSyncGuestBackupTimesIgnoresInProgressPBSSnapshot(t *testing.T) {
	state := NewState()
	now := time.Now()
	completedTime := now.Add(-48 * time.Hour)
	inFlightTime := now.Add(-5 * time.Minute)

	state.UpdateVMs([]VM{
		{VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{ID: "done-117", VMID: "117", BackupType: "vm", BackupTime: completedTime,
			Instance: "verdeclose", Datastore: "main", Namespace: "pve-pc"},
		{ID: "flight-117", VMID: "117", BackupType: "vm", BackupTime: inFlightTime,
			Instance: "verdeclose", Datastore: "main", Namespace: "pve-pc", InProgress: true},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	if len(snapshot.VMs) != 1 {
		t.Fatalf("expected 1 VM, got %d", len(snapshot.VMs))
	}
	vm := snapshot.VMs[0]
	if !vm.LastBackup.Equal(completedTime) {
		t.Errorf("LastBackup = %v, want the completed snapshot %v (in-flight one must not advance it)",
			vm.LastBackup, completedTime)
	}
	if !vm.BackupInProgress {
		t.Error("BackupInProgress = false, want true while an in-flight snapshot exists")
	}
}

func TestSyncGuestBackupTimesInProgressOnlyNeverSetsLastBackup(t *testing.T) {
	state := NewState()

	state.UpdateVMs([]VM{
		{VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{ID: "flight-117", VMID: "117", BackupType: "vm", BackupTime: time.Now(),
			Instance: "verdeclose", Datastore: "main", Namespace: "pve-pc", InProgress: true},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	vm := snapshot.VMs[0]
	if !vm.LastBackup.IsZero() {
		t.Errorf("LastBackup = %v, want zero: the guest has no completed backup", vm.LastBackup)
	}
	if !vm.BackupInProgress {
		t.Error("BackupInProgress = false, want true")
	}
}

func TestSyncGuestBackupTimesTerminalIncompletePBSSnapshotIsNotRunningOrComplete(t *testing.T) {
	state := NewState()
	state.UpdateVMs([]VM{
		{VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{{
		ID:                    "failed-sync-117",
		VMID:                  "117",
		BackupType:            "vm",
		BackupTime:            time.Now().Add(-time.Hour),
		Instance:              "offsite",
		Datastore:             "main",
		Namespace:             "pve-pc",
		InProgress:            true,
		WriteActivityObserved: true,
		WriteActive:           false,
	}}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	vm := state.GetSnapshot().VMs[0]
	if vm.BackupInProgress {
		t.Error("BackupInProgress = true for an incomplete snapshot after its PBS task terminated")
	}
	if !vm.LastBackup.IsZero() {
		t.Errorf("LastBackup = %v, want zero because a failed sync artifact is not complete", vm.LastBackup)
	}
}

func TestSyncGuestBackupTimesRunningVzdumpTaskSetsBackupInProgress(t *testing.T) {
	state := NewState()
	now := time.Now()

	state.UpdateContainers([]Container{
		{VMID: 105, Name: "media", Instance: "homelab", Node: "minipc"},
	})

	state.mu.Lock()
	state.PVEBackups.BackupTasks = []BackupTask{
		{ID: "homelab-upid1", Instance: "homelab", Node: "minipc", Type: "vzdump",
			VMID: 105, Status: "running", StartTime: now.Add(-10 * time.Minute),
			ObservedAt: now},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	ct := snapshot.Containers[0]
	if !ct.BackupInProgress {
		t.Error("BackupInProgress = false, want true while a vzdump task is running")
	}
	if !ct.LastBackup.IsZero() {
		t.Errorf("LastBackup = %v, want zero", ct.LastBackup)
	}
}

func TestSyncGuestBackupTimesIgnoresStaleRunningTask(t *testing.T) {
	state := NewState()
	now := time.Now()

	state.UpdateVMs([]VM{
		{VMID: 200, Name: "old", Instance: "homelab", Node: "delly"},
	})

	state.mu.Lock()
	state.PVEBackups.BackupTasks = []BackupTask{
		// Unfinished task last observed two days ago: the instance stopped
		// refreshing, so this must not pin the guest as backing up.
		{ID: "homelab-upid2", Instance: "homelab", Node: "delly", Type: "vzdump",
			VMID: 200, Status: "running", StartTime: now.Add(-49 * time.Hour),
			ObservedAt: now.Add(-48 * time.Hour)},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	if snapshot.VMs[0].BackupInProgress {
		t.Error("BackupInProgress = true, want false for a stale running task")
	}
}

func TestSyncGuestBackupTimesIgnoresInProgressStorageBackup(t *testing.T) {
	state := NewState()
	now := time.Now()
	completedTime := now.Add(-24 * time.Hour)

	state.UpdateVMs([]VM{
		{VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	state.mu.Lock()
	state.PVEBackups.StorageBackups = []StorageBackup{
		{ID: "pve-pc-old", Instance: "pve-pc", Node: "pve-pc", Type: "qemu",
			VMID: 117, Time: completedTime, Size: 42 << 30},
		// Partial archive a running vzdump is still writing.
		{ID: "pve-pc-partial", Instance: "pve-pc", Node: "pve-pc", Type: "qemu",
			VMID: 117, Time: now.Add(-3 * time.Minute), Size: 7 << 30, InProgress: true},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	vm := snapshot.VMs[0]
	if !vm.LastBackup.Equal(completedTime) {
		t.Errorf("LastBackup = %v, want completed archive time %v", vm.LastBackup, completedTime)
	}
}

// Guest polls know nothing about backups; the instance merge must carry the
// backup-running flag forward alongside LastBackup so it does not flap off
// between backup sync passes.
func TestUpdateVMsForInstancePreservesBackupInProgress(t *testing.T) {
	state := NewState()

	state.UpdateVMs([]VM{
		{ID: "pve-pc-117", VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc",
			LastBackup: time.Now().Add(-24 * time.Hour), BackupInProgress: true},
	})

	// Fresh poll data: no backup knowledge at all.
	state.UpdateVMsForInstance("pve-pc", []VM{
		{ID: "pve-pc-117", VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	snapshot := state.GetSnapshot()
	vm := snapshot.VMs[0]
	if vm.LastBackup.IsZero() {
		t.Error("LastBackup lost across instance update")
	}
	if !vm.BackupInProgress {
		t.Error("BackupInProgress lost across instance update")
	}
}

// A PBS-to-PBS sync interrupted mid-snapshot leaves a manifest-less copy of
// an already-completed backup on the target datastore. That partial copy
// carries the original backup time, so it is never newer than its completed
// twin and must not report the guest as backing up until the sync is
// repaired (#1815).
func TestSyncGuestBackupTimesIgnoresPartialSyncCopyOfCompletedBackup(t *testing.T) {
	state := NewState()
	now := time.Now()
	backupTime := now.Add(-2 * time.Hour)

	state.UpdateVMs([]VM{
		{VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		// Completed snapshot on the primary PBS.
		{ID: "done-117", VMID: "117", BackupType: "vm", BackupTime: backupTime,
			Instance: "verdeclose", Datastore: "main", Namespace: "pve-pc"},
		// Partial copy of the same snapshot on the sync target.
		{ID: "partial-117", VMID: "117", BackupType: "vm", BackupTime: backupTime,
			Instance: "offsite", Datastore: "sync", Namespace: "pve-pc", InProgress: true},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	vm := snapshot.VMs[0]
	if !vm.LastBackup.Equal(backupTime) {
		t.Errorf("LastBackup = %v, want %v", vm.LastBackup, backupTime)
	}
	if vm.BackupInProgress {
		t.Error("BackupInProgress = true, want false for a partial sync copy of a completed backup")
	}
}

// A manifest-less snapshot with no completed twin left must age out the same
// way a stale running task does: whatever was writing it is long gone.
func TestSyncGuestBackupTimesIgnoresAbandonedInFlightSnapshot(t *testing.T) {
	state := NewState()
	now := time.Now()

	state.UpdateVMs([]VM{
		{VMID: 117, Name: "win11-pvepc", Instance: "pve-pc", Node: "pve-pc"},
	})

	state.mu.Lock()
	state.PBSBackups = []PBSBackup{
		{ID: "flight-117", VMID: "117", BackupType: "vm", BackupTime: now.Add(-72 * time.Hour),
			Instance: "verdeclose", Datastore: "main", Namespace: "pve-pc", InProgress: true},
	}
	state.mu.Unlock()

	state.SyncGuestBackupTimes()
	snapshot := state.GetSnapshot()

	if snapshot.VMs[0].BackupInProgress {
		t.Error("BackupInProgress = true, want false for an abandoned in-flight snapshot")
	}
	if !snapshot.VMs[0].LastBackup.IsZero() {
		t.Errorf("LastBackup = %v, want zero", snapshot.VMs[0].LastBackup)
	}
}

// Local archives retain their explicit guest kind after VMID reuse. PBS has
// always used typed subjects; local files must not bypass that safety property.
func TestSyncGuestBackupTimesLocalArchiveGuestKinds(t *testing.T) {
	for _, kind := range []string{"qemu", "vm", " QEMU ", "lxc", "ct", " LXC "} {
		t.Run(kind, func(t *testing.T) {
			state := NewState()
			now := time.Now().UTC().Truncate(time.Second)
			matching, opposite := "qemu", "lxc"
			isVM := kind == "qemu" || kind == "vm" || kind == " QEMU "
			if !isVM {
				matching, opposite = "lxc", "qemu"
			}
			state.UpdateVMs([]VM{{VMID: 100, Instance: "a"}, {VMID: 100, Instance: "b"}})
			state.UpdateContainers([]Container{{VMID: 100, Instance: "a"}, {VMID: 100, Instance: "b"}})
			state.UpdateStorageBackupsForInstance("a", []StorageBackup{{ID: "artifact", Instance: "a", VMID: 100, Type: kind, Time: now}})
			state.SyncGuestBackupTimes()
			snap := state.GetSnapshot()
			right, wrong := snap.VMs[0].LastBackup, snap.Containers[0].LastBackup
			if !isVM {
				right, wrong = wrong, right
			}
			if !right.Equal(now) || !wrong.IsZero() {
				t.Errorf("explicit %q archive right=%s wrong=%s", kind, right, wrong)
			}
			if !snap.VMs[1].LastBackup.IsZero() || !snap.Containers[1].LastBackup.IsZero() {
				t.Fatal("another connection inherited backup time")
			}
			// Recency is considered within a kind, never before attribution.
			older := now.Add(-time.Hour)
			state.UpdateStorageBackupsForInstance("a", []StorageBackup{
				{ID: "correct", Instance: "a", VMID: 100, Type: matching, Time: older},
				{ID: "opposite", Instance: "a", VMID: 100, Type: opposite, Time: now},
				{ID: "partial", Instance: "a", VMID: 100, Type: matching, Time: now.Add(time.Minute), InProgress: true},
			})
			state.SyncGuestBackupTimes()
			snap = state.GetSnapshot()
			right = snap.VMs[0].LastBackup
			if !isVM {
				right = snap.Containers[0].LastBackup
			}
			if !right.Equal(older) {
				t.Errorf("newer opposite/partial file hid correct archive: %s want %s", right, older)
			}
			state.UpdateStorageBackupsForInstance("a", nil)
			state.SyncGuestBackupTimes()
			snap = state.GetSnapshot()
			if !snap.VMs[0].LastBackup.IsZero() || !snap.Containers[0].LastBackup.IsZero() {
				t.Error("accepted absence retained typed dates")
			}
		})
	}
}

func TestSyncGuestBackupTimesUntypedLocalArchiveCompatibility(t *testing.T) {
	for _, kind := range []string{"", "  "} {
		t.Run(kind, func(t *testing.T) {
			state := NewState()
			now := time.Now().UTC().Truncate(time.Second)
			state.UpdateVMs([]VM{{VMID: 100, Instance: "a"}})
			state.UpdateStorageBackupsForInstance("a", []StorageBackup{{ID: "legacy", Instance: "a", VMID: 100, Type: kind, Time: now}})
			state.SyncGuestBackupTimes()
			if got := state.GetSnapshot().VMs[0].LastBackup; !got.Equal(now) {
				t.Errorf("untyped legacy behavior changed: %s", got)
			}
		})
	}
}

func TestSyncGuestBackupTimesExplicitUnknownArchiveIsNotGuestEvidence(t *testing.T) {
	for _, kind := range []string{"unknown", " UNKNOWN ", "legacy", "host"} {
		t.Run(kind, func(t *testing.T) {
			state := NewState()
			now := time.Now().UTC().Truncate(time.Second)
			state.UpdateVMs([]VM{{VMID: 100, Instance: "a"}, {VMID: 100, Instance: "b"}})
			state.UpdateContainers([]Container{{VMID: 100, Instance: "a"}})
			for _, withTyped := range []bool{false, true} {
				rows := []StorageBackup{{ID: "opaque", Instance: "a", VMID: 100, Type: kind, Time: now}}
				wantVM, wantCT := time.Time{}, time.Time{}
				if withTyped {
					wantVM, wantCT = now.Add(-time.Hour), now.Add(-2*time.Hour)
					rows = append(rows,
						StorageBackup{ID: "vm", Instance: "a", VMID: 100, Type: "qemu", Time: wantVM},
						StorageBackup{ID: "ct", Instance: "a", VMID: 100, Type: "lxc", Time: wantCT})
				}
				state.UpdateStorageBackupsForInstance("a", rows)
				state.SyncGuestBackupTimes()
				snap := state.GetSnapshot()
				if !snap.VMs[0].LastBackup.Equal(wantVM) || !snap.Containers[0].LastBackup.Equal(wantCT) || !snap.VMs[1].LastBackup.IsZero() {
					t.Fatalf("unknown archive supplied or hid a guest date: vms=%+v containers=%+v", snap.VMs, snap.Containers)
				}
				if snap.VMs[0].BackupInProgress || snap.Containers[0].BackupInProgress || len(snap.PVEBackups.StorageBackups) != len(rows) {
					t.Fatal("unknown identity changed activity or hid an artifact")
				}
			}
		})
	}
}