Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
const testJobUPID = "UPID:pve1:000E9F2C:0AC734B2:68A1B2C3:vzdump::root@pam:"
func testLogLines(lines ...string) []proxmox.TaskLogLine {
out := make([]proxmox.TaskLogLine, 0, len(lines))
for i, line := range lines {
out = append(out, proxmox.TaskLogLine{LineNumber: int64(i + 1), Text: line})
}
return out
}
func TestParseVzdumpJobLogFinishedJob(t *testing.T) {
start := time.Date(2026, 7, 25, 2, 0, 0, 0, time.UTC)
jobTask := models.BackupTask{
ID: "pve-cluster-" + testJobUPID,
Node: "pve1",
Instance: "pve-cluster",
Type: "vzdump",
Status: "job errors",
StartTime: start,
EndTime: start.Add(10 * time.Minute),
}
tasks := parseVzdumpJobLog(jobTask, testJobUPID, testLogLines(
"INFO: starting new backup job: vzdump 101 102 103 --mode snapshot --storage pbs",
"INFO: Starting Backup of VM 101 (qemu)",
"INFO: creating Proxmox Backup Server archive 'vm/101/2026-07-25T02:00:01Z'",
"INFO: Finished Backup of VM 101 (00:02:30)",
"INFO: Starting Backup of VM 102 (lxc)",
"ERROR: Backup of VM 102 failed - command 'rsync' failed: exit code 23",
"INFO: Starting Backup of VM 103 (qemu)",
"INFO: Finished Backup of VM 103 (00:01:00)",
"INFO: Backup job finished with errors",
))
if len(tasks) != 3 {
t.Fatalf("expected 3 synthetic guest tasks, got %d", len(tasks))
}
vm101 := tasks[0]
if vm101.VMID != 101 || vm101.Status != "OK" || vm101.Error != "" {
t.Errorf("vm101 = %+v, want VMID 101 status OK with no error", vm101)
}
if vm101.ID != "pve-cluster-"+testJobUPID+"-vm101" {
t.Errorf("vm101 ID = %q, want parent UPID embedded", vm101.ID)
}
if !vm101.StartTime.Equal(start) || !vm101.EndTime.Equal(start.Add(2*time.Minute+30*time.Second)) {
t.Errorf("vm101 times = %v..%v, want job start plus printed duration", vm101.StartTime, vm101.EndTime)
}
vm102 := tasks[1]
if vm102.VMID != 102 || vm102.Status != "error" {
t.Errorf("vm102 = %+v, want VMID 102 status error", vm102)
}
if vm102.Error != "command 'rsync' failed: exit code 23" {
t.Errorf("vm102 error = %q, want the reason from the log line", vm102.Error)
}
if !vm102.EndTime.Equal(jobTask.EndTime) {
t.Errorf("vm102 end = %v, want parent job end for failed guest", vm102.EndTime)
}
vm103 := tasks[2]
if vm103.VMID != 103 || vm103.Status != "OK" {
t.Errorf("vm103 = %+v, want VMID 103 status OK", vm103)
}
// vm102 failed without a duration line, so vm103 starts from vm101's end.
wantStart := start.Add(2*time.Minute + 30*time.Second)
if !vm103.StartTime.Equal(wantStart) || !vm103.EndTime.Equal(wantStart.Add(time.Minute)) {
t.Errorf("vm103 times = %v..%v, want %v..%v", vm103.StartTime, vm103.EndTime, wantStart, wantStart.Add(time.Minute))
}
for _, task := range tasks {
if task.Node != "pve1" || task.Instance != "pve-cluster" || task.Type != "vzdump" {
t.Errorf("task %d inherits wrong parent metadata: %+v", task.VMID, task)
}
}
}
func TestParseVzdumpJobLogRunningJob(t *testing.T) {
start := time.Date(2026, 7, 25, 2, 0, 0, 0, time.UTC)
jobTask := models.BackupTask{
ID: "pve-cluster-" + testJobUPID,
Node: "pve1",
Instance: "pve-cluster",
Type: "vzdump",
StartTime: start,
}
tasks := parseVzdumpJobLog(jobTask, testJobUPID, testLogLines(
"INFO: Starting Backup of VM 101 (qemu)",
"INFO: Finished Backup of VM 101 (00:00:45)",
"INFO: Starting Backup of VM 102 (lxc)",
))
if len(tasks) != 2 {
t.Fatalf("expected 2 synthetic guest tasks, got %d", len(tasks))
}
if tasks[0].Status != "OK" || tasks[0].EndTime.IsZero() {
t.Errorf("finished guest in running job = %+v, want status OK with end time", tasks[0])
}
if tasks[1].Status != "running" || !tasks[1].EndTime.IsZero() {
t.Errorf("in-progress guest = %+v, want status running with zero end time", tasks[1])
}
}
func TestParseVzdumpJobLogEndedWithoutGuestVerdict(t *testing.T) {
start := time.Date(2026, 7, 25, 2, 0, 0, 0, time.UTC)
jobTask := models.BackupTask{
ID: "pve-cluster-" + testJobUPID,
Node: "pve1",
Instance: "pve-cluster",
Type: "vzdump",
Status: "unexpected status",
StartTime: start,
EndTime: start.Add(time.Minute),
}
tasks := parseVzdumpJobLog(jobTask, testJobUPID, testLogLines(
"INFO: Starting Backup of VM 101 (qemu)",
))
if len(tasks) != 1 {
t.Fatalf("expected 1 synthetic guest task, got %d", len(tasks))
}
if tasks[0].Status != "unknown" || tasks[0].Error != "backup job ended without an observed guest result" {
t.Errorf("guest without a verdict = %+v, want unknown with an evidence explanation", tasks[0])
}
if !tasks[0].EndTime.Equal(jobTask.EndTime) {
t.Errorf("guest end = %v, want parent end time", tasks[0].EndTime)
}
}
func TestParseVzdumpJobLogNoGuestMarkers(t *testing.T) {
jobTask := models.BackupTask{
ID: "pve-cluster-" + testJobUPID,
Instance: "pve-cluster",
Type: "vzdump",
StartTime: time.Now(),
}
if tasks := parseVzdumpJobLog(jobTask, testJobUPID, testLogLines("INFO: nothing to do")); tasks != nil {
t.Errorf("expected no synthetic tasks, got %+v", tasks)
}
}
// jobBackupTaskClient serves a multi-guest vzdump job task plus its log.
// Only the two methods pollBackupTasks uses are implemented; the embedded
// interface satisfies the rest.
type jobBackupTaskClient struct {
PVEClientInterface
task proxmox.Task
logLines []proxmox.TaskLogLine
logRequests int
}
func (c *jobBackupTaskClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
return []proxmox.Task{c.task}, nil
}
func (c *jobBackupTaskClient) GetTaskLog(ctx context.Context, node, upid string) ([]proxmox.TaskLogLine, error) {
c.logRequests++
return c.logLines, nil
}
func TestPollBackupTasksSynthesizesJobGuestTasks(t *testing.T) {
start := time.Now().Add(-time.Hour)
client := &jobBackupTaskClient{
task: proxmox.Task{
UPID: testJobUPID,
Node: "pve1",
Type: "vzdump",
ID: "", // job-run UPIDs carry no VMID
Status: "OK",
StartTime: start.Unix(),
EndTime: start.Add(5 * time.Minute).Unix(),
},
logLines: testLogLines(
"INFO: Starting Backup of VM 101 (qemu)",
"INFO: Finished Backup of VM 101 (00:02:00)",
"INFO: Starting Backup of VM 102 (lxc)",
"INFO: Finished Backup of VM 102 (00:01:00)",
"TASK OK",
),
}
m := &Monitor{state: models.NewState()}
m.pollBackupTasks(context.Background(), "pve-test", client)
state := m.state.GetSnapshot()
if len(state.PVEBackups.BackupTasks) != 3 {
t.Fatalf("expected job task plus 2 synthetic guest tasks, got %d: %+v", len(state.PVEBackups.BackupTasks), state.PVEBackups.BackupTasks)
}
byVMID := make(map[int]models.BackupTask)
for _, task := range state.PVEBackups.BackupTasks {
byVMID[task.VMID] = task
}
for _, vmid := range []int{101, 102} {
task, ok := byVMID[vmid]
if !ok {
t.Fatalf("no synthetic task for VM %d", vmid)
}
if task.Status != "OK" {
t.Errorf("VM %d status = %q, want OK", vmid, task.Status)
}
if task.ObservedAt.IsZero() {
t.Errorf("VM %d has no observation time", vmid)
}
}
if _, ok := byVMID[0]; !ok {
t.Error("parent job task should remain in state")
}
if client.logRequests != 1 {
t.Fatalf("expected 1 log fetch, got %d", client.logRequests)
}
// A finished job's log is immutable: the second poll must serve the
// synthetic tasks from cache without refetching, refreshing ObservedAt.
m.pollBackupTasks(context.Background(), "pve-test", client)
if client.logRequests != 1 {
t.Errorf("expected cached synthesis on second poll, got %d log fetches", client.logRequests)
}
state = m.state.GetSnapshot()
if len(state.PVEBackups.BackupTasks) != 3 {
t.Errorf("expected stable task count after second poll, got %d", len(state.PVEBackups.BackupTasks))
}
}
func TestPollBackupTasksRunningJobRefetchesAndSuppressesAlerts(t *testing.T) {
start := time.Now().Add(-2 * time.Minute)
client := &jobBackupTaskClient{
task: proxmox.Task{
UPID: testJobUPID,
Node: "pve1",
Type: "vzdump",
ID: "",
StartTime: start.Unix(),
// running: no EndTime, no Status
},
logLines: testLogLines(
"INFO: Starting Backup of VM 101 (qemu)",
),
}
m := &Monitor{state: models.NewState()}
m.pollBackupTasks(context.Background(), "pve-test", client)
m.pollBackupTasks(context.Background(), "pve-test", client)
if client.logRequests != 2 {
t.Errorf("running job should be re-parsed every poll, got %d log fetches", client.logRequests)
}
state := m.state.GetSnapshot()
var running *models.BackupTask
for i := range state.PVEBackups.BackupTasks {
if state.PVEBackups.BackupTasks[i].VMID == 101 {
running = &state.PVEBackups.BackupTasks[i]
}
}
if running == nil {
t.Fatal("no synthetic task for VM 101")
}
if running.Status != "running" {
t.Errorf("status = %q, want running", running.Status)
}
intent, ok := m.resolveBackupIntentContext("", "pve-test", "pve1", 101, time.Now())
if !ok || !intent.Active {
t.Errorf("expected active backup intent for guest covered by running job, got ok=%v intent=%+v", ok, intent)
}
if _, ok := m.resolveBackupIntentContext("", "pve-test", "pve1", 999, time.Now()); ok {
t.Error("guest not covered by the job should have no backup intent")
}
}