Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.2 KB
Raw Blame
package monitoring

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	pveapi "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

// dirStorageNamedPBSClient serves a dir-type storage whose NAME starts with
// "pbs-" alongside a real pbs-type storage. Only the latter may be treated as
// PBS-backed when a direct PBS connection exists (#1592).
type dirStorageNamedPBSClient struct {
	mockPVEClientExtra
}

func (c *dirStorageNamedPBSClient) GetStorage(ctx context.Context, node string) ([]pveapi.Storage, error) {
	return []pveapi.Storage{
		{Storage: "pbs-backup", Content: "backup", Type: "dir", Enabled: 1, Active: 1},
		{Storage: "pbs-real", Content: "backup", Type: "pbs", Enabled: 1, Active: 1},
	}, nil
}

func (c *dirStorageNamedPBSClient) GetStorageContent(ctx context.Context, node, storage string) ([]pveapi.StorageContent, error) {
	switch storage {
	case "pbs-backup":
		return []pveapi.StorageContent{{
			Volid:   "pbs-backup:backup/vzdump-qemu-106-2026_07_17-01_00_00.vma.zst",
			VMID:    106,
			Size:    2048,
			CTime:   time.Date(2026, 7, 17, 1, 0, 0, 0, time.UTC).Unix(),
			Content: "backup",
		}}, nil
	case "pbs-real":
		return []pveapi.StorageContent{{
			Volid:   "pbs-real:backup/vm/107/2026-07-17T01:00:00Z",
			VMID:    107,
			Size:    2048,
			CTime:   time.Date(2026, 7, 17, 1, 0, 0, 0, time.UTC).Unix(),
			Content: "backup",
			Format:  "pbs-vm",
		}}, nil
	}
	return nil, nil
}

func TestPollStorageBackups_KeepsDirStorageNamedLikePBS(t *testing.T) {
	m := &Monitor{
		state: models.NewState(),
		config: &config.Config{
			PBSInstances: []config.PBSInstance{{Name: "pbs-1", Host: "https://pbs.example:8007"}},
		},
	}

	client := &dirStorageNamedPBSClient{}
	nodes := []pveapi.Node{{Node: "node1", Status: "online"}, {Node: "node2", Status: "online"}}

	m.pollStorageBackupsWithNodes(context.Background(), "pve1", client, nodes, map[string]string{"node1": "online", "node2": "online"})

	backups := m.state.GetSnapshot().PVEBackups.StorageBackups
	if len(backups) != 2 {
		t.Fatalf("expected each independent dir-storage copy to survive, got %+v", backups)
	}
	if backups[0].ID == backups[1].ID || backups[0].StorageNode == backups[1].StorageNode {
		t.Fatalf("local disks named like PBS must have independent identities: %+v", backups)
	}
	for _, backup := range backups {
		if backup.Storage != "pbs-backup" || backup.VMID != 106 {
			t.Fatalf("expected only vzdump from dir storage named pbs-backup, got %+v", backup)
		}
	}
}

// pbsPartialFailureClient serves two pbs-type storages and can be told to
// fail one storage's content query, standing in for a partial poll failure.
type pbsPartialFailureClient struct {
	mockPVEClientExtra
	snapshotTime time.Time
	failStorage  string
}

func (c *pbsPartialFailureClient) GetStorage(ctx context.Context, node string) ([]pveapi.Storage, error) {
	return []pveapi.Storage{
		{Storage: "pbs-one", Content: "backup", Type: "pbs", Enabled: 1, Active: 1},
		{Storage: "pbs-two", Content: "backup", Type: "pbs", Enabled: 1, Active: 1},
	}, nil
}

func (c *pbsPartialFailureClient) GetStorageContent(ctx context.Context, node, storage string) ([]pveapi.StorageContent, error) {
	if storage == c.failStorage {
		return nil, fmt.Errorf("500 internal error")
	}
	vmid := 173
	if storage == "pbs-two" {
		vmid = 174
	}
	return []pveapi.StorageContent{{
		Volid:   fmt.Sprintf("%s:backup/vm/%d/2026-07-27T01:00:00Z", storage, vmid),
		VMID:    vmid,
		Size:    2048,
		CTime:   c.snapshotTime.Unix(),
		Content: "backup",
		Format:  "pbs-vm",
	}}, nil
}

// TestPollStorageBackups_Issue1639PreservesConfirmationsOnPartialFailure
// verifies that a storage whose content query fails keeps its previous PBS
// guest confirmations instead of having them evicted by the partial set.
// Losing them flips collision-VMID attribution from cycle to cycle.
func TestPollStorageBackups_Issue1639PreservesConfirmationsOnPartialFailure(t *testing.T) {
	snapshotTime := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)

	m := &Monitor{
		state: models.NewState(),
		config: &config.Config{
			PBSInstances: []config.PBSInstance{{Name: "pbs-1", Host: "https://pbs.example:8007"}},
		},
	}
	nodes := []pveapi.Node{{Node: "node-a", Status: "online"}}
	online := map[string]string{"node-a": "online"}

	// Healthy cycle: both storages list one snapshot each.
	m.pollStorageBackupsWithNodes(context.Background(), "cluster-a",
		&pbsPartialFailureClient{snapshotTime: snapshotTime}, nodes, online)

	healthy := m.state.PBSGuestConfirmationsForInstance("cluster-a")
	if len(healthy) != 2 {
		t.Fatalf("expected both storages to contribute confirmations, got %+v", healthy)
	}

	// Degraded cycle: pbs-one's content query fails. Its evidence must
	// survive, and pbs-two's must be refreshed as usual.
	m.pollStorageBackupsWithNodes(context.Background(), "cluster-a",
		&pbsPartialFailureClient{snapshotTime: snapshotTime, failStorage: "pbs-one"}, nodes, online)

	degraded := m.state.PBSGuestConfirmationsForInstance("cluster-a")
	if len(degraded) != 2 {
		t.Fatalf("partial failure evicted confirmation evidence, got %+v", degraded)
	}
	byStorage := map[string]models.PBSGuestConfirmation{}
	for _, confirmation := range degraded {
		byStorage[confirmation.Storage] = confirmation
	}
	if got, ok := byStorage["pbs-one"]; !ok || got.VMID != 173 {
		t.Fatalf("pbs-one confirmation not preserved across the failed query, got %+v", degraded)
	}
	if got, ok := byStorage["pbs-two"]; !ok || got.VMID != 174 {
		t.Fatalf("pbs-two confirmation missing after successful query, got %+v", degraded)
	}
}

// pbsCollisionStorageClient serves one pbs-type storage listing a single
// snapshot for VMID 173, standing in for cluster-a's own PBS storage view.
type pbsCollisionStorageClient struct {
	mockPVEClientExtra
	snapshotTime time.Time
}

func (c *pbsCollisionStorageClient) GetStorage(ctx context.Context, node string) ([]pveapi.Storage, error) {
	return []pveapi.Storage{
		{Storage: "pbs-store", Content: "backup", Type: "pbs", Enabled: 1, Active: 1},
	}, nil
}

func (c *pbsCollisionStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]pveapi.StorageContent, error) {
	return []pveapi.StorageContent{{
		Volid:   "pbs-store:backup/vm/173/2026-07-27T01:00:00Z",
		VMID:    173,
		Size:    2048,
		CTime:   c.snapshotTime.Unix(),
		Content: "backup",
		Format:  "pbs-vm",
	}}, nil
}

// TestPollStorageBackups_Issue1639HarvestsPBSGuestConfirmations verifies
// that snapshots skipped in favor of the direct PBS connection still leave
// PVE-side attribution evidence: the cluster whose storage listed a
// snapshot gets it as LastBackup even though the VMID exists on another
// cluster too, and the other cluster's unconfirmed snapshot is not guessed.
func TestPollStorageBackups_Issue1639HarvestsPBSGuestConfirmations(t *testing.T) {
	timeA := time.Date(2026, 7, 27, 1, 0, 0, 0, time.UTC)
	timeB := time.Date(2026, 7, 27, 2, 0, 0, 0, time.UTC)

	state := models.NewState()
	state.UpdateVMs([]models.VM{
		{VMID: 173, Name: "web", Instance: "cluster-a", Node: "node-a", Status: "running"},
		{VMID: 173, Name: "web", Instance: "cluster-b", Node: "node-b", Status: "running"},
	})
	state.UpdatePBSBackups("pbs-1", []models.PBSBackup{
		{ID: "a-173", VMID: "173", BackupType: "vm", BackupTime: timeA, Instance: "pbs-1", Datastore: "backups", Owner: "shared@pbs!token"},
		{ID: "b-173", VMID: "173", BackupType: "vm", BackupTime: timeB, Instance: "pbs-1", Datastore: "backups", Owner: "shared@pbs!token"},
	})

	m := &Monitor{
		state: state,
		config: &config.Config{
			PBSInstances: []config.PBSInstance{{Name: "pbs-1", Host: "https://pbs.example:8007"}},
		},
	}

	client := &pbsCollisionStorageClient{snapshotTime: timeA}
	nodes := []pveapi.Node{{Node: "node-a", Status: "online"}}

	m.pollStorageBackupsWithNodes(context.Background(), "cluster-a", client, nodes, map[string]string{"node-a": "online"})

	snapshot := m.state.GetSnapshot()
	if len(snapshot.PVEBackups.StorageBackups) != 0 {
		t.Fatalf("pbs-type storage contents must stay out of StorageBackups, got %+v", snapshot.PVEBackups.StorageBackups)
	}
	for _, vm := range snapshot.VMs {
		switch vm.Instance {
		case "cluster-a":
			if !vm.LastBackup.Equal(timeA) {
				t.Errorf("cluster-a VM 173 LastBackup = %v, want confirmed snapshot %v", vm.LastBackup, timeA)
			}
		case "cluster-b":
			if !vm.LastBackup.IsZero() {
				t.Errorf("cluster-b VM 173 LastBackup = %v, want zero (its snapshot is unconfirmed)", vm.LastBackup)
			}
		}
	}
}

func TestPollPVEBackupsAsyncRejectsStoppedRuntime(t *testing.T) {
	runtimeCtx, cancelRuntime := context.WithCancel(context.Background())
	cancelRuntime()
	m := &Monitor{
		state:             models.NewState(),
		runtimeCtx:        runtimeCtx,
		lastPVEBackupPoll: make(map[string]time.Time),
		config: &config.Config{
			EnableBackupPolling: true,
			BackupPollingCycles: 1,
		},
	}
	client := &mockPVEClientExtra{}
	m.pveClients = map[string]PVEClientInterface{"cluster-a": client}
	m.pollPVEBackupsAsync(
		"cluster-a",
		&config.PVEInstance{MonitorBackups: true},
		client,
		nil,
		map[string]string{},
	)
	m.mu.RLock()
	scheduledAt := m.lastPVEBackupPoll["cluster-a"]
	m.mu.RUnlock()
	if !scheduledAt.IsZero() {
		t.Fatal("stopped runtime scheduled a new backup poll")
	}
}

// A task running on node-b cannot be writing a file on node-a's independent
// local disk, even when both paths and VMIDs are identical. Shared views and
// missing task origins remain conservative: a peer may be writing that file.
func TestPVEStorageRunningArchiveOrigin(t *testing.T) {
	for _, tc := range []struct {
		name, storageType, taskNode string
		shared, otherInstance       bool
		ended, failed, oldArchive   bool
		earlierPeer                 bool
		want                        map[string]bool
	}{
		{name: "writer-a", taskNode: "node-a", want: map[string]bool{"node-a": true, "node-b": false}},
		{name: "writer-b", taskNode: "node-b", want: map[string]bool{"node-a": false, "node-b": true}},
		{name: "writer-elsewhere", taskNode: "node-c", want: map[string]bool{"node-a": false, "node-b": false}},
		{name: "separate-node-windows", taskNode: "node-a", earlierPeer: true, want: map[string]bool{"node-a": false, "node-b": true}},
		{name: "unknown-writer", want: map[string]bool{"node-a": true, "node-b": true}},
		{name: "shared-flag", shared: true, taskNode: "node-b", want: map[string]bool{"": true}},
		{name: "nfs-peer-view", storageType: "nfs", taskNode: "node-b", want: map[string]bool{"": true}},
		{name: "cifs-peer-view", storageType: "cifs", taskNode: "node-b", want: map[string]bool{"": true}},
		{name: "pbs-peer-view", storageType: "pbs", taskNode: "node-b", want: map[string]bool{"": true}},
		{name: "other-connection", taskNode: "node-a", otherInstance: true, want: map[string]bool{"node-a": false, "node-b": false}},
		{name: "ended-task", taskNode: "node-a", ended: true, want: map[string]bool{"node-a": false, "node-b": false}},
		{name: "failed-task", taskNode: "node-a", failed: true, want: map[string]bool{"node-a": false, "node-b": false}},
		{name: "archive-before-window", taskNode: "node-a", oldArchive: true, want: map[string]bool{"node-a": false, "node-b": false}},
	} {
		for _, reverse := range []bool{false, true} {
			t.Run(fmt.Sprintf("%s/reverse=%t", tc.name, reverse), func(t *testing.T) {
				f, client := localArchiveIdentityFixture(t, "qemu")
				now := time.Now().UTC().Truncate(time.Second)
				task := models.BackupTask{ID: "running", Instance: "cluster", Node: tc.taskNode, VMID: 101,
					Type: "vzdump", Status: "running", StartTime: now.Add(-time.Minute), ObservedAt: now}
				// Inside the existing clock-skew window, including a file just
				// older than the task. The change must not widen that window.
				f.client.at = task.StartTime.Add(-time.Minute)
				if tc.oldArchive {
					f.client.at = task.StartTime.Add(-runningVzdumpClockSkew - time.Second)
				}
				if tc.otherInstance {
					task.Instance = "other"
				}
				if tc.ended {
					task.EndTime = now
				}
				if tc.failed {
					task.Status = "error"
				}
				client.shared = tc.shared
				if tc.storageType != "" {
					client.storageType = tc.storageType
				}
				if reverse {
					f.nodes[0], f.nodes[1] = f.nodes[1], f.nodes[0]
				}
				tasks := []models.BackupTask{task}
				if tc.earlierPeer {
					peer := task
					peer.ID, peer.Node, peer.StartTime = "peer", "node-b", now.Add(-time.Hour)
					tasks = append(tasks, peer)
					f.client.at = now.Add(-10 * time.Minute)
				}
				f.m.state.UpdateBackupTasksForInstance(task.Instance, tasks)
				f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)
				f.wait(t)
				rows := f.m.state.GetSnapshot().PVEBackups.StorageBackups
				if len(rows) != len(tc.want) {
					t.Fatalf("archive count=%d want=%d", len(rows), len(tc.want))
				}
				for _, row := range rows {
					want, ok := tc.want[row.StorageNode]
					if !ok || row.InProgress != want {
						t.Errorf("disk=%q partial=%t want=%t (known=%t)", row.StorageNode, row.InProgress, want, ok)
					}
				}
			})
		}
	}
}

// Drive the actual poller, live guest dates, asynchronous recovery ingestion
// and canonical posture through a running task, read failure and ordinary
// completion. No guest, backup operation, schema or data repair is involved.
func TestPVEStorageRunningArchiveOriginLifecycle(t *testing.T) {
	for _, kind := range []string{"qemu", "lxc"} {
		t.Run(kind, func(t *testing.T) {
			f, client := localArchiveIdentityFixture(t, kind)
			now := time.Now().UTC().Truncate(time.Second)
			f.client.at = now.Add(-time.Minute)
			task := models.BackupTask{ID: "running", Instance: "cluster", Node: "node-b", VMID: 101,
				Type: "vzdump", Status: "running", StartTime: now.Add(-10 * time.Minute), ObservedAt: now}
			frames := map[string]any{}
			var idA, idB string
			for _, phase := range []string{"running", "partial-read", "completed"} {
				if phase == "partial-read" {
					f.client.contentErr = fmt.Errorf("503 unavailable")
				}
				if phase == "completed" {
					f.client.contentErr = nil
					task.Status, task.EndTime = "OK", now
				}
				f.m.state.UpdateBackupTasksForInstance("cluster", []models.BackupTask{task})
				f.m.pollStorageBackupsWithNodes(context.Background(), "cluster", client, f.nodes, f.online)
				f.wait(t)
				snap, points, postures := f.m.state.GetSnapshot(), f.points(t), f.postures(t)
				if len(snap.PVEBackups.StorageBackups) != 2 || len(points) != 2 {
					t.Fatal("independent archive lost from live inventory or recovery")
				}
				for _, row := range snap.PVEBackups.StorageBackups {
					wantPartial := row.StorageNode == "node-b" && phase != "completed"
					point := points["pve-backup:"+row.ID]
					if row.InProgress != wantPartial || point.SubjectResourceID != f.ids[0] || row.Node != "node-b" ||
						row.Type != kind || !row.Time.Equal(f.client.at) || row.VMID != 101 || row.Storage != "local" {
						t.Fatalf("phase=%s wrong archive facts: row=%+v point=%+v", phase, row, point)
					}
					if wantPartial {
						if point.Outcome != recovery.OutcomeRunning || point.CompletedAt != nil {
							t.Fatal("the writer's partial output claimed completion")
						}
					} else if point.Outcome != recovery.OutcomeSuccess || point.CompletedAt == nil || !point.CompletedAt.Equal(row.Time) {
						t.Fatal("the independent completed archive lost its completion")
					}
					if row.StorageNode == "node-a" {
						if idA != "" && idA != row.ID {
							t.Fatal("task state renamed independent archive")
						}
						idA = row.ID
					} else {
						if idB != "" && idB != row.ID {
							t.Fatal("task state renamed writer archive")
						}
						idB = row.ID
					}
				}
				last, running := time.Time{}, false
				if kind == "qemu" {
					last, running = snap.VMs[0].LastBackup, snap.VMs[0].BackupInProgress
				} else {
					last, running = snap.Containers[0].LastBackup, snap.Containers[0].BackupInProgress
				}
				if !last.Equal(f.client.at) || running != (phase != "completed") {
					t.Fatalf("phase=%s guest completion/running=%s/%t", phase, last, running)
				}
				// Node-a's completed archive is re-read in every phase, so it keeps
				// the guest protected; node-b's unread disk only limits coverage.
				posture := postures[0]
				if posture.LastSuccessfulPointAt == nil || !posture.LastSuccessfulPointAt.Equal(f.client.at) ||
					posture.State != recovery.ProtectionStateProtected ||
					(phase == "partial-read") != (posture.Coverage == recovery.ProtectionCoveragePartial) {
					t.Fatalf("phase=%s wrong scoped protection: %+v", phase, posture)
				}
				frames[phase] = map[string]any{"pve": map[string]any{"data": snap.PVEBackups}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "vms": snap.VMs, "containers": snap.Containers}
			}
			if os.Getenv("PULSE_RUNNING_ARCHIVE_FRAMES") == "1" {
				hashes := map[string]string{}
				for _, path := range []string{"monitor_backups.go", "../models/models.go", "../recovery/mapper/proxmox/mapper.go"} {
					b, err := os.ReadFile(path)
					if err != nil {
						t.Fatal(err)
					}
					hashes[path] = fmt.Sprintf("%x", sha256.Sum256(b))
				}
				b, err := json.Marshal(map[string]any{"kind": kind, "runtime_sha256": hashes, "resource_ids": f.ids, "frames": frames})
				if err != nil {
					t.Fatal(err)
				}
				t.Log("RUNNING_ARCHIVE_FRAMES=" + string(b))
			}
		})
	}
}