Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.5 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
	"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
	proxmoxmapper "github.com/rcourtman/pulse-go-rewrite/internal/recovery/mapper/proxmox"
	recoverystore "github.com/rcourtman/pulse-go-rewrite/internal/recovery/store"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
)

type pbsPosturePollFixture struct {
	m        *Monitor
	store    *recoverystore.Store
	client   *pbs.Client
	mode     atomic.Int32
	backupAt int64
	ids      []string
}

func newPBSPosturePollFixture(t *testing.T, count int, backupTypes ...string) *pbsPosturePollFixture {
	t.Helper()
	m, manager := recoveryIngestTestMonitor(t)
	m.state = models.NewState()
	f := &pbsPosturePollFixture{m: m, backupAt: time.Now().Add(-14 * time.Hour).Truncate(time.Second).Unix()}
	backupType := "vm"
	if len(backupTypes) > 0 {
		backupType = backupTypes[0]
	}
	var vms []models.VM
	var containers []models.Container
	for i := 0; i < count; i++ {
		id := 100 + i
		node := fmt.Sprintf("node-%d", i%4)
		resourceType := unifiedresources.ResourceTypeVM
		if backupType == "ct" {
			resourceType = unifiedresources.ResourceTypeSystemContainer
			containers = append(containers, models.Container{ID: makeGuestID("cluster", node, id), VMID: id, Instance: "cluster", Node: node, Name: fmt.Sprintf("guest-%d", id), Status: "running"})
		} else {
			vms = append(vms, models.VM{ID: makeGuestID("cluster", node, id), VMID: id, Instance: "cluster", Node: node, Name: fmt.Sprintf("guest-%d", id), Status: "running"})
		}
		f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(resourceType, "cluster", id))
	}
	m.state.UpdateVMs(vms)
	m.state.UpdateContainers(containers)
	var err error
	f.store, err = manager.StoreForOrg("default")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { f.wait(t); _ = f.store.Close() })
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mode := f.mode.Load()
		if strings.HasSuffix(r.URL.Path, "/groups") {
			if mode == 4 {
				http.Error(w, "unavailable", http.StatusServiceUnavailable)
				return
			}
			if mode == 5 {
				http.Error(w, "denied", http.StatusForbidden)
				return
			}
			if mode == 6 && r.URL.Query().Get("ns") == "unreadable" {
				http.Error(w, "denied", http.StatusForbidden)
				return
			}
			groups := make([]map[string]any, 0)
			if mode != 1 {
				for i := 0; i < count; i++ {
					groups = append(groups, map[string]any{"backup-type": backupType, "backup-id": strconv.Itoa(100 + i), "last-backup": f.backupAt, "backup-count": 1})
				}
			}
			_ = json.NewEncoder(w).Encode(map[string]any{"data": groups})
		} else if strings.HasSuffix(r.URL.Path, "/snapshots") {
			id := r.URL.Query().Get("backup-id")
			if id == "100" && (mode == 2 || mode == 3) {
				status := http.StatusServiceUnavailable
				if mode == 3 {
					status = http.StatusForbidden
				}
				http.Error(w, "unreadable", status)
				return
			}
			_ = json.NewEncoder(w).Encode(map[string]any{"data": []map[string]any{{"backup-type": backupType, "backup-id": id, "backup-time": f.backupAt, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}})
		} else {
			http.NotFound(w, r)
		}
	}))
	t.Cleanup(server.Close)
	f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
	if err != nil {
		t.Fatal(err)
	}
	return f
}

func (f *pbsPosturePollFixture) wait(t *testing.T) {
	t.Helper()
	deadline := time.Now().Add(10 * time.Second)
	for {
		f.m.recoveryIngestMu.Lock()
		running := f.m.recoveryIngestRunning
		f.m.recoveryIngestMu.Unlock()
		if !running {
			return
		}
		if time.Now().After(deadline) {
			t.Fatal("ordinary PBS ingest did not finish")
		}
		time.Sleep(time.Millisecond)
	}
}

func (f *pbsPosturePollFixture) poll(t *testing.T, instance string, mode int32, datastores []models.PBSDatastore) {
	t.Helper()
	f.mode.Store(mode)
	// Deliberately expire snapshot reuse: group-count/time are unchanged when
	// only snapshot access or verification changes between ordinary polls.
	f.m.mu.Lock()
	clear(f.m.pbsBackupCacheTime)
	f.m.mu.Unlock()
	f.m.pollPBSBackups(context.Background(), instance, f.client, datastores, nil)
	f.wait(t)
}

func (f *pbsPosturePollFixture) points(t *testing.T, instance string) []recovery.RecoveryPoint {
	t.Helper()
	points, _, err := f.store.ListPoints(context.Background(), recovery.ListPointsOptions{Limit: 1000})
	if err != nil {
		t.Fatal(err)
	}
	var out []recovery.RecoveryPoint
	for _, p := range points {
		if p.RepositoryRef != nil && p.RepositoryRef.Namespace == instance {
			out = append(out, p)
		}
	}
	return out
}

func (f *pbsPosturePollFixture) postures(t *testing.T) []recovery.ProtectionPosture {
	t.Helper()
	postures, total, err := f.store.ListProtectionPostures(context.Background(), recovery.ProtectionPostureQuery{SubjectResourceIDs: f.ids})
	if err != nil || total != len(f.ids) {
		t.Fatalf("postures total=%d err=%v", total, err)
	}
	return postures
}

func (f *pbsPosturePollFixture) assertProtected(t *testing.T, instance string) {
	t.Helper()
	var raw []models.PBSBackup
	for _, b := range f.m.state.GetSnapshot().PBSBackups {
		if b.Instance == instance {
			raw = append(raw, b)
		}
	}
	mapped, err := proxmoxmapper.FromPBSBackupsWithEvidence(raw, buildPBSGuestCandidates(f.m.GetUnifiedReadStateOrSnapshot()), time.Now())
	if err != nil || len(mapped) != len(f.ids) {
		t.Fatalf("raw/mapped=%d/%d err=%v", len(raw), len(mapped), err)
	}
	for _, p := range mapped {
		vmid, _ := strconv.Atoi(p.Details["vmid"].(string))
		resourceType := unifiedresources.ResourceTypeVM
		if p.Details["backupType"] == "ct" {
			resourceType = unifiedresources.ResourceTypeSystemContainer
		}
		expected := unifiedresources.ProxmoxGuestCanonicalID(resourceType, "cluster", vmid)
		if p.SubjectResourceID != expected || p.Outcome != recovery.OutcomeSuccess {
			t.Fatalf("mapped subject=%s outcome=%s, want %s success", p.SubjectResourceID, p.Outcome, expected)
		}
	}
	persisted := f.points(t, instance)
	if len(persisted) != len(mapped) {
		t.Fatalf("persisted=%d mapped=%d", len(persisted), len(mapped))
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateProtected || p.Verification != recovery.ProtectionVerificationVerified || p.LastSuccessfulPointAt == nil || p.LastSuccessfulPointAt.Unix() != f.backupAt {
			t.Fatalf("canonical posture=%+v", p)
		}
	}
	t.Logf("raw=%d mapped=%d persisted=%d protected=%d; original backup timestamp retained", len(raw), len(mapped), len(persisted), len(f.ids))
}

func TestPBSOrdinaryPollRootNamespacePostureLifecycle(t *testing.T) {
	exercisePBSRootNamespacePostureLifecycle(t, "vm")
}

func TestPBSOrdinaryPollRootNamespaceContainerPostureLifecycle(t *testing.T) {
	exercisePBSRootNamespacePostureLifecycle(t, "ct")
}

func exercisePBSRootNamespacePostureLifecycle(t *testing.T, backupType string) {
	t.Helper()
	f := newPBSPosturePollFixture(t, 40, backupType)
	ds := []models.PBSDatastore{{Name: "archive"}}
	f.poll(t, "pbs-main", 0, ds)
	f.assertProtected(t, "pbs-main")
	f.poll(t, "pbs-main", 1, ds)
	if len(f.m.state.GetSnapshot().PBSBackups) != 0 || len(f.points(t, "pbs-main")) != 0 {
		t.Fatal("successful empty poll retained deleted backups")
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateUnknown {
			t.Fatalf("empty posture=%+v, want unknown without a subject-linked source", p)
		}
	}
	f.poll(t, "pbs-main", 0, ds)
	f.assertProtected(t, "pbs-main")
	f.poll(t, "pbs-main", 4, ds)
	if len(f.points(t, "pbs-main")) != 40 || len(f.m.state.GetSnapshot().PBSBackups) != 40 {
		t.Fatal("transient group failure lost historical backups")
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 1 || p.ProviderStates[0].JobState != recovery.OutcomeFailed || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryUnavailable {
			t.Fatalf("failed posture=%+v", p)
		}
	}
	f.poll(t, "pbs-main", 0, ds)
	f.assertProtected(t, "pbs-main")
	f.poll(t, "pbs-main", 5, ds)
	if len(f.m.state.GetSnapshot().PBSBackups) != 0 || len(f.points(t, "pbs-main")) != 40 {
		t.Fatal("access denial must withdraw raw rows without deleting historical recovery facts")
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateUnknown || len(p.ProviderStates) != 1 || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsDenied {
			t.Fatalf("denied posture=%+v", p)
		}
	}
	f.poll(t, "pbs-main", 0, ds)
	f.assertProtected(t, "pbs-main")
	// One connection's authoritative empty poll must not reconcile another.
	f.poll(t, "pbs-other", 0, ds)
	f.poll(t, "pbs-main", 1, ds)
	if len(f.points(t, "pbs-main")) != 0 || len(f.points(t, "pbs-other")) != 40 {
		t.Fatal("empty enumeration crossed the PBS instance boundary")
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateProtected {
			t.Fatalf("other readable source lost protection: %+v", p)
		}
	}
}

func TestPBSOrdinaryPollSnapshotFailuresDoNotClaimCompleteHistory(t *testing.T) {
	for _, tc := range []struct {
		name        string
		mode        int32
		permissions operationaltrust.EvidencePermissions
		state       recovery.ProtectionState
	}{
		{"transient", 2, operationaltrust.EvidencePermissionsUnknown, recovery.ProtectionStateAttention},
		{"denied", 3, operationaltrust.EvidencePermissionsPartial, recovery.ProtectionStateAttention},
	} {
		t.Run(tc.name, func(t *testing.T) {
			f := newPBSPosturePollFixture(t, 2)
			ds := []models.PBSDatastore{{Name: "archive"}}
			f.poll(t, "pbs-main", 0, ds)
			f.assertProtected(t, "pbs-main")
			f.poll(t, "pbs-main", tc.mode, ds)
			for attempt := 0; attempt < 2; attempt++ {
				points := f.points(t, "pbs-main")
				if len(points) != 2 {
					t.Errorf("incomplete snapshot enumeration reconciled away historical points: %d", len(points))
				}
				wantRaw := 2
				if tc.mode == 3 {
					wantRaw = 1
				}
				if got := len(f.m.state.GetSnapshot().PBSBackups); got != wantRaw {
					t.Errorf("raw snapshot rows=%d, want %d", got, wantRaw)
				}
				for _, p := range f.postures(t) {
					// Only guest 100's snapshot read fails; guest 101's snapshot
					// is re-read and stays protected under partial coverage.
					state := tc.state
					if p.SubjectResourceID == f.ids[1] {
						state = recovery.ProtectionStateProtected
					}
					if p.State != state || p.Coverage != recovery.ProtectionCoveragePartial || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != tc.permissions || p.ProviderStates[0].JobState != recovery.OutcomeWarning {
						t.Errorf("snapshot failure posture=%+v, want %s/partial/%s/warning", p, state, tc.permissions)
					}
				}
				// No TTL manipulation on the next poll: a failed refresh must not
				// become a healthy cached read while snapshot access is still lost.
				f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, ds, nil)
				f.wait(t)
			}
			f.poll(t, "pbs-main", 0, ds)
			f.assertProtected(t, "pbs-main")
		})
	}
}

func TestPBSOrdinaryPollPartialNamespaceRetainsUnseenHistory(t *testing.T) {
	f := newPBSPosturePollFixture(t, 2)
	ds := []models.PBSDatastore{{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}, {Path: "unreadable"}}}}
	f.poll(t, "pbs-main", 0, ds)
	if len(f.points(t, "pbs-main")) != 4 {
		t.Fatal("fixture did not enumerate both namespaces")
	}
	f.poll(t, "pbs-main", 6, ds)
	if len(f.points(t, "pbs-main")) != 4 {
		t.Error("partial namespace poll deleted unseen historical points")
	}
	// Both guests' root snapshots were re-read; only the unreadable
	// namespace's retained points went unconfirmed, and it is named.
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateProtected || p.Coverage != recovery.ProtectionCoveragePartial || len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsPartial ||
			!reflect.DeepEqual(p.ProviderStates[0].UnreadScopes, []recovery.ProtectionUnreadScope{{Kind: recovery.ProtectionUnreadScopeNamespace, Storage: "archive", Namespace: "unreadable", AccessDenied: true}}) {
			t.Errorf("partial namespace posture=%+v", p)
		}
	}
	f.poll(t, "pbs-main", 0, ds)
	if len(f.points(t, "pbs-main")) != 4 {
		t.Fatal("recovery did not restore both namespaces")
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateProtected {
			t.Errorf("recovered posture=%+v", p)
		}
	}
}

func TestPBSOrdinaryPollAmbiguousVMIDsStayUnlinked(t *testing.T) {
	f := newPBSPosturePollFixture(t, 1)
	vms := f.m.state.GetSnapshot().VMs
	vms = append(vms, models.VM{ID: makeGuestID("other-cluster", "node-0", 100), VMID: 100, Instance: "other-cluster", Node: "node-0", Name: "guest-100", Status: "running"})
	f.m.state.UpdateVMs(vms)
	f.ids = append(f.ids, unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "other-cluster", 100))
	f.poll(t, "pbs-main", 0, []models.PBSDatastore{{Name: "archive"}})
	points := f.points(t, "pbs-main")
	if len(points) != 1 || points[0].SubjectResourceID != "" {
		t.Fatalf("ambiguous root-namespace backup was linked: %+v", points)
	}
	for _, p := range f.postures(t) {
		if p.State != recovery.ProtectionStateUnknown {
			t.Fatalf("ambiguous VMID painted protected: %+v", p)
		}
	}
}