Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

15.6 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"os"
	"sync"
	"sync/atomic"
	"testing"
	"testing/synctest"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/stretchr/testify/require"
)

// No guest-agent, task operation, external connection or installed target.
// The same behavioural tests also compile against the exact original source.
type backupLifecycleClient struct {
	mockPVEClientExtra
	label string
	read  func(context.Context, string)
	calls atomic.Int32
}

func (c *backupLifecycleClient) before(ctx context.Context, phase string) {
	c.calls.Add(1)
	if c.read != nil {
		c.read(ctx, phase)
	}
}
func (c *backupLifecycleClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
	c.before(ctx, "tasks")
	return []proxmox.Task{{UPID: testJobUPID, Node: "node-a", Type: "vzdump", StartTime: time.Now().Add(-time.Hour).Unix(), EndTime: time.Now().Add(-time.Minute).Unix(), Status: "job errors"}}, nil
}
func (c *backupLifecycleClient) GetTaskLog(ctx context.Context, _, _ string) ([]proxmox.TaskLogLine, error) {
	c.before(ctx, "log")
	return []proxmox.TaskLogLine{{LineNumber: 1, Text: "INFO: Starting Backup of VM 101 (qemu)"}, {LineNumber: 2, Text: "ERROR: Backup of VM 101 failed - " + c.label + " result"}, {LineNumber: 3, Text: "TASK ERROR: job errors"}}, nil
}
func (c *backupLifecycleClient) GetStorage(ctx context.Context, _ string) ([]proxmox.Storage, error) {
	c.before(ctx, "storage")
	return []proxmox.Storage{{Storage: "archive", Type: "dir", Content: "backup", Active: 1, Enabled: 1}}, nil
}
func (c *backupLifecycleClient) GetStorageContent(ctx context.Context, _, _ string) ([]proxmox.StorageContent, error) {
	c.before(ctx, "content")
	return []proxmox.StorageContent{{Volid: "archive:backup/vzdump-qemu-101-" + c.label + ".vma.zst", VMID: 101, CTime: time.Now().Add(-2 * time.Hour).Unix(), Size: 1024, Content: "backup"}}, nil
}
func (c *backupLifecycleClient) GetVMSnapshots(ctx context.Context, _ string, _ int) ([]proxmox.Snapshot, error) {
	c.before(ctx, "snapshots")
	return []proxmox.Snapshot{{Name: c.label, SnapTime: time.Now().Add(-time.Hour).Unix()}}, nil
}
func (c *backupLifecycleClient) GetContainerSnapshots(ctx context.Context, node string, id int) ([]proxmox.Snapshot, error) {
	return c.GetVMSnapshots(ctx, node, id)
}
func newBackupLifecycleMonitor(c PVEClientInterface, kind string) *Monitor {
	m := &Monitor{state: models.NewState(), pveClients: map[string]PVEClientInterface{"site-a": c}, config: &config.Config{EnableBackupPolling: true, BackupPollingCycles: 1}, lastPVEBackupPoll: map[string]time.Time{}, runtimeCtx: context.Background()}
	if kind == "qemu" {
		m.state.UpdateVMsForInstance("site-a", []models.VM{{ID: "site-a-node-a-101", VMID: 101, Node: "node-a", Instance: "site-a", Name: "workload", Status: "running"}})
	} else {
		m.state.UpdateContainersForInstance("site-a", []models.Container{{ID: "site-a-node-a-101", VMID: 101, Node: "node-a", Instance: "site-a", Name: "workload", Status: "running"}})
	}
	m.state.UpdateStorageBackupsForInstance("site-b", []models.StorageBackup{{ID: "peer-file", Instance: "site-b", Node: "node-a", Type: "qemu", VMID: 101, Storage: "archive", Time: time.Now().Add(-time.Hour)}})
	m.state.UpdateGuestSnapshotsForInstance("site-b", []models.GuestSnapshot{{ID: "peer-snap", Instance: "site-b", Node: "node-a", Type: "qemu", VMID: 101, Name: "independent", Time: time.Now().Add(-time.Hour)}})
	return m
}
func scheduleBackupLifecycle(m *Monitor, c PVEClientInterface) {
	m.pollPVEBackupsAsync("site-a", &config.PVEInstance{MonitorBackups: true}, c, []proxmox.Node{{Node: "node-a", Status: "online"}}, map[string]string{"node-a": "online"})
}

func TestPVEBackupLifecycleSingleFlight(t *testing.T) {
	for _, interval := range []time.Duration{0, time.Minute} {
		t.Run(interval.String(), func(t *testing.T) {
			synctest.Test(t, func(t *testing.T) {
				release := make(chan struct{})
				var tasks atomic.Int32
				c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, phase string) {
					if phase == "tasks" {
						tasks.Add(1)
						<-release
					}
				}}
				m := newBackupLifecycleMonitor(c, "qemu")
				m.config.BackupPollingInterval = interval
				scheduleBackupLifecycle(m, c)
				synctest.Wait()
				// A slow scan outlasts cadence, but it still owns the connection.
				time.Sleep(90 * time.Second)
				for i := 0; i < 8; i++ {
					scheduleBackupLifecycle(m, c)
				}
				synctest.Wait()
				got := tasks.Load()
				close(release)
				synctest.Wait()
				require.Equal(t, int32(1), got, "cadence must not stack scans")
				completed := m.lastPVEBackupPoll["site-a"]
				require.Equal(t, time.Now(), completed, "cadence starts at accepted completion")
				time.Sleep(interval + time.Second)
				scheduleBackupLifecycle(m, c)
				synctest.Wait()
				require.Equal(t, int32(2), tasks.Load(), "ordinary next eligible cycle recovers")
			})
		})
	}
}

func TestPVEBackupLifecycleLateReadsCannotReplaceCurrentEvidence(t *testing.T) {
	for _, kind := range []string{"qemu", "lxc"} {
		for _, phase := range []string{"tasks", "log", "storage", "content", "snapshots", "sizing"} {
			for _, transition := range []string{"client", "runtime", "retire"} {
				t.Run(kind+"/"+phase+"/"+transition, func(t *testing.T) {
					synctest.Test(t, func(t *testing.T) {
						release := make(chan struct{})
						released := false
						defer func() {
							if !released {
								close(release)
							}
						}()
						var reached atomic.Bool
						var storageReads atomic.Int32
						var oldCtx context.Context
						old := &backupLifecycleClient{label: "old", read: func(ctx context.Context, p string) {
							sizing := p == "storage" && storageReads.Add(1) == 2
							if p == phase || (phase == "sizing" && sizing) {
								oldCtx = ctx
								reached.Store(true)
								<-release
							}
						}}
						m := newBackupLifecycleMonitor(old, kind)
						scheduleBackupLifecycle(m, old)
						synctest.Wait()
						require.True(t, reached.Load())
						fresh := &backupLifecycleClient{label: "current"}
						switch transition {
						case "retire":
							m.retirePVEInstanceRuntime("site-a")
						case "runtime":
							m.setRuntimeContext(context.Background(), nil) // Deliberately don't cancel the old parent.
						}
						m.replacePVEClient("site-a", fresh)
						if transition == "retire" {
							// Fresh configuration rebuilds its guest inventory without a live guest operation.
							source := newBackupLifecycleMonitor(fresh, kind).state.GetSnapshot()
							m.state.UpdateVMsForInstance("site-a", source.VMs)
							m.state.UpdateContainersForInstance("site-a", source.Containers)
						}
						scheduleBackupLifecycle(m, fresh)
						synctest.Wait()
						before := m.state.GetSnapshot().PVEBackups
						require.NotZero(t, fresh.calls.Load(), "replacement must not wait for the late client")
						currentStamp := m.lastPVEBackupPoll["site-a"]
						cache := m.vzdumpJobTaskCache["site-a|"+testJobUPID]
						close(release)
						released = true
						synctest.Wait()
						require.NotNil(t, oldCtx.Err(), "binding transition must cancel the old read")
						require.Equal(t, before, m.state.GetSnapshot().PVEBackups, "late read changed accepted inventories")
						require.Equal(t, currentStamp, m.lastPVEBackupPoll["site-a"], "old finish rewrote replacement cadence")
						require.Equal(t, cache, m.vzdumpJobTaskCache["site-a|"+testJobUPID], "old result poisoned the immutable cache")
						require.Len(t, before.StorageBackups, 2)
						require.Len(t, before.GuestSnapshots, 2)
						found := false
						for _, task := range before.BackupTasks {
							if task.VMID == 101 {
								require.Equal(t, "current result", task.Error)
								found = true
							}
						}
						require.True(t, found, "new client must read its own guest log even for a reused UPID")
						if os.Getenv("PULSE_BACKUP_LIFECYCLE_FRAMES") == "1" && kind == "qemu" && phase == "tasks" && transition == "client" {
							data, err := json.Marshal(map[string]any{"accepted": before, "after_late": m.state.GetSnapshot().PVEBackups, "vms": m.state.GetSnapshot().VMs, "containers": m.state.GetSnapshot().Containers})
							require.NoError(t, err)
							t.Log("BACKUP_LIFECYCLE_FRAME=" + string(data))
						}
					})
				})
			}
		}
	}
}

func TestPVEBackupLifecycleRetirementDoesNotResurrectState(t *testing.T) {
	synctest.Test(t, func(t *testing.T) {
		release := make(chan struct{})
		old := &backupLifecycleClient{label: "retired", read: func(_ context.Context, p string) {
			if p == "tasks" {
				<-release
			}
		}}
		m := newBackupLifecycleMonitor(old, "qemu")
		scheduleBackupLifecycle(m, old)
		synctest.Wait()
		m.retirePVEInstanceRuntime("site-a")
		before := m.state.GetSnapshot().PVEBackups
		close(release)
		synctest.Wait()
		require.Equal(t, before, m.state.GetSnapshot().PVEBackups)
		_, exists := m.lastPVEBackupPoll["site-a"]
		require.False(t, exists, "retired completion recreated scheduling state")
		require.Empty(t, m.vzdumpJobTaskCache)
	})
}

func TestPVEBackupLifecycleConcurrentAdmission(t *testing.T) {
	synctest.Test(t, func(t *testing.T) {
		release := make(chan struct{})
		var tasks atomic.Int32
		c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, p string) {
			if p == "tasks" {
				tasks.Add(1)
				<-release
			}
		}}
		m := newBackupLifecycleMonitor(c, "qemu")
		var callers sync.WaitGroup
		for i := 0; i < 32; i++ {
			callers.Add(1)
			go func() { defer callers.Done(); scheduleBackupLifecycle(m, c) }()
		}
		callers.Wait()
		synctest.Wait()
		count := tasks.Load()
		close(release)
		synctest.Wait()
		require.Equal(t, int32(1), count)
	})
}

func TestPVEBackupLifecycleCopiesSchedulingInventory(t *testing.T) {
	synctest.Test(t, func(t *testing.T) {
		release := make(chan struct{})
		c := &backupLifecycleClient{label: "accepted", read: func(_ context.Context, p string) {
			if p == "tasks" {
				<-release
			}
		}}
		m := newBackupLifecycleMonitor(c, "qemu")
		nodes := []proxmox.Node{{Node: "node-a", Status: "online"}}
		statuses := map[string]string{"node-a": "online"}
		m.pollPVEBackupsAsync("site-a", &config.PVEInstance{MonitorBackups: true}, c, nodes, statuses)
		synctest.Wait()
		nodes[0].Node = "reused-core-buffer"
		statuses["node-a"] = "offline"
		close(release)
		synctest.Wait()
		found := false
		for _, row := range m.state.GetSnapshot().PVEBackups.StorageBackups {
			if row.Instance == "site-a" {
				found = true
				require.Equal(t, "node-a", row.StorageNode)
			}
		}
		require.True(t, found, "next core cycle mutated the owned storage origin")
	})
}

func TestPVEBackupLifecycleCanceledCollectorsKeepEvidence(t *testing.T) {
	for _, phase := range []string{"tasks", "log", "snapshots"} {
		for _, kind := range []string{"qemu", "lxc"} {
			t.Run(phase+"/"+kind, func(t *testing.T) {
				c := &backupLifecycleClient{label: "accepted"}
				m := newBackupLifecycleMonitor(c, kind)
				m.pollBackupTasks(context.Background(), "site-a", c)
				m.pollGuestSnapshots(context.Background(), "site-a", c)
				before := m.state.GetSnapshot().PVEBackups
				ctx, cancel := context.WithCancel(context.Background())
				defer cancel()
				c.label = "late"
				c.read = func(_ context.Context, p string) {
					if p == phase {
						cancel()
					}
				}
				// Non-finalise the prior cache to force another real log read.
				key := "site-a|" + testJobUPID
				entry := m.vzdumpJobTaskCache[key]
				entry.final = false
				m.vzdumpJobTaskCache[key] = entry
				cache := m.vzdumpJobTaskCache[key]
				if phase == "snapshots" {
					m.pollGuestSnapshots(ctx, "site-a", c)
				} else {
					m.pollBackupTasks(ctx, "site-a", c)
				}
				observed := m.state.GetSnapshot().PVEBackups
				if phase == "log" {
					// The parent listing was accepted before this log exhausted
					// the budget. Its observation may advance; cached guest facts
					// may not. A canceled read supplied no new guest verdict.
					var beforeGuests, afterGuests []models.BackupTask
					for _, row := range before.BackupTasks {
						if row.VMID > 0 {
							beforeGuests = append(beforeGuests, row)
						}
					}
					for _, row := range observed.BackupTasks {
						if row.VMID > 0 {
							afterGuests = append(afterGuests, row)
						}
					}
					require.Equal(t, beforeGuests, afterGuests)
					require.Equal(t, before.StorageBackups, observed.StorageBackups)
					require.Equal(t, before.GuestSnapshots, observed.GuestSnapshots)
				} else {
					require.Equal(t, before, observed)
				}
				after := m.vzdumpJobTaskCache[key]
				require.Equal(t, cache.tasks, after.tasks, "cancellation replaced accepted log evidence")
				require.Equal(t, cache.final, after.final)
			})
		}
	}
}

func TestPVEBackupLifecycleSizingBudgetKeepsAcceptedListings(t *testing.T) {
	for _, kind := range []string{"qemu", "lxc"} {
		t.Run(kind, func(t *testing.T) {
			synctest.Test(t, func(t *testing.T) {
				var storageReads atomic.Int32
				c := &backupLifecycleClient{label: "accepted", read: func(ctx context.Context, phase string) {
					if phase == "storage" && storageReads.Add(1) == 2 {
						// The listing succeeded before this optional enrichment.
						// Exhaust only the snapshot child, not its runtime owner.
						<-ctx.Done()
					}
				}}
				m := newBackupLifecycleMonitor(c, kind)
				scheduleBackupLifecycle(m, c)
				synctest.Wait()
				require.Equal(t, int32(2), storageReads.Load(), "fixture must reach snapshot sizing")
				time.Sleep(61 * time.Second)
				synctest.Wait()
				state := m.state.GetSnapshot()
				require.Len(t, state.PVEBackups.GuestSnapshots, 2, "accepted listing and independent peer survive sizing timeout")
				for _, snap := range state.PVEBackups.GuestSnapshots {
					if snap.Instance == "site-a" {
						require.Equal(t, "accepted", snap.Name)
						require.Zero(t, snap.SizeBytes, "missing enrichment must not invent a size")
					}
				}
				require.Empty(t, m.pveBackupPolls, "timed-out sizing must release the completed owner")
				if os.Getenv("PULSE_BACKUP_LIFECYCLE_FRAMES") == "1" && kind == "qemu" {
					data, err := json.Marshal(map[string]any{"accepted": state.PVEBackups, "vms": state.VMs, "containers": state.Containers})
					require.NoError(t, err)
					t.Log("BACKUP_SNAPSHOT_BUDGET_FRAME=" + string(data))
				}
			})
		})
	}
}

func TestPVEBackupLifecycleCompletedReplacementRefreshesImmediately(t *testing.T) {
	synctest.Test(t, func(t *testing.T) {
		old := &backupLifecycleClient{label: "old"}
		m := newBackupLifecycleMonitor(old, "qemu")
		m.config.BackupPollingInterval = 24 * time.Hour
		scheduleBackupLifecycle(m, old)
		synctest.Wait()
		fresh := &backupLifecycleClient{label: "current"}
		m.replacePVEClient("site-a", fresh)
		scheduleBackupLifecycle(m, fresh)
		synctest.Wait()
		require.NotZero(t, fresh.calls.Load(), "a completed previous binding must not delay the new client's first scan")
		for _, row := range m.state.GetSnapshot().PVEBackups.GuestSnapshots {
			if row.Instance == "site-a" {
				require.Equal(t, "current", row.Name)
			}
		}
	})
}

func TestPVEBackupLifecycleRuntimeReplacementKeepsSameClientSafe(t *testing.T) {
	synctest.Test(t, func(t *testing.T) {
		release := make(chan struct{})
		var taskReads atomic.Int32
		var oldCtx context.Context
		c := &backupLifecycleClient{label: "accepted", read: func(ctx context.Context, p string) {
			if p == "tasks" && taskReads.Add(1) == 1 {
				oldCtx = ctx
				<-release
			}
		}}
		m := newBackupLifecycleMonitor(c, "qemu")
		scheduleBackupLifecycle(m, c)
		synctest.Wait()
		m.setRuntimeContext(context.Background(), nil)
		scheduleBackupLifecycle(m, c)
		synctest.Wait()
		before := m.state.GetSnapshot().PVEBackups
		require.Equal(t, int32(2), taskReads.Load())
		close(release)
		synctest.Wait()
		require.Error(t, oldCtx.Err())
		require.Equal(t, before, m.state.GetSnapshot().PVEBackups)
	})
}