Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"reflect"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
)
type pbsNamespaceContinuityFixture struct {
*pbsPosturePollFixture
rootStatus, tenantStatus atomic.Int32
rootEmpty, tenantEmpty atomic.Bool
rootAt, tenantAt atomic.Int64
rootReads, tenantReads atomic.Int32
tenantSnapshotStatus, namespaceStatus atomic.Int32
failedReads atomic.Int32
inventoryResponse atomic.Pointer[pbsInventoryResponse]
}
func newPBSNamespaceContinuityFixture(t *testing.T, kind string) *pbsNamespaceContinuityFixture {
t.Helper()
f := &pbsNamespaceContinuityFixture{pbsPosturePollFixture: newPBSPosturePollFixture(t, 2, kind)}
f.rootAt.Store(f.backupAt)
f.tenantAt.Store(f.backupAt)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if response := f.inventoryResponse.Load(); response != nil &&
r.URL.Path == "/api2/json/admin/datastore/archive/"+response.stage &&
(response.stage == "namespace" || r.URL.Query().Get("ns") == "tenant") {
f.failedReads.Add(1)
_, _ = w.Write([]byte(response.body))
return
}
id, at, status, empty := "100", f.rootAt.Load(), f.rootStatus.Load(), f.rootEmpty.Load()
if r.URL.Query().Get("ns") == "tenant" {
id, at, status, empty = "101", f.tenantAt.Load(), f.tenantStatus.Load(), f.tenantEmpty.Load()
f.tenantReads.Add(1)
} else {
f.rootReads.Add(1)
}
fail := func(status int32) {
f.failedReads.Add(1)
w.Header().Set("Retry-After", "3600")
http.Error(w, "upstream API error 404: fixture unavailable", int(status))
}
var data any = []any{}
switch r.URL.Path {
case "/api2/json/admin/datastore/archive/namespace":
if status := f.namespaceStatus.Load(); status != 0 {
fail(status)
return
}
data = []map[string]any{{"ns": "tenant"}}
case "/api2/json/admin/datastore/archive/groups":
if status == http.StatusOK {
_, _ = w.Write([]byte(`{"data":`))
return
}
if status != 0 {
fail(status)
return
}
if !empty {
data = []map[string]any{{"backup-type": kind, "backup-id": id, "last-backup": at, "backup-count": 1}}
}
case "/api2/json/admin/datastore/archive/snapshots":
if id == "101" && f.tenantSnapshotStatus.Load() != 0 {
fail(f.tenantSnapshotStatus.Load())
return
}
data = []map[string]any{{"backup-type": kind, "backup-id": id, "backup-time": at, "size": 1024, "files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}}
default:
http.NotFound(w, r)
return
}
_ = json.NewEncoder(w).Encode(map[string]any{"data": data})
}))
t.Cleanup(server.Close)
var err error
f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
if err != nil {
t.Fatal(err)
}
return f
}
func (f *pbsNamespaceContinuityFixture) collect(t *testing.T, discovered bool, discoveryErr error) {
t.Helper()
ds := models.PBSDatastore{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}}}
if discovered {
ds.Namespaces = append(ds.Namespaces, models.PBSNamespace{Path: "tenant"})
}
errs := map[string]error{}
if discoveryErr != nil {
errs["archive"] = discoveryErr
}
f.m.pollPBSBackups(context.Background(), "pbs-main", f.client, []models.PBSDatastore{ds}, errs)
f.wait(t)
}
func (f *pbsNamespaceContinuityFixture) live() map[string]models.PBSBackup {
out := map[string]models.PBSBackup{}
for _, b := range f.m.state.GetSnapshot().PBSBackups {
if b.Instance == "pbs-main" {
out[b.VMID] = b
}
}
return out
}
func TestPBSNamespaceTransientGroupReadKeepsLiveInventory(t *testing.T) {
for _, kind := range []string{"vm", "ct"} {
for _, ns := range []string{"", "tenant"} {
for _, status := range []int{http.StatusServiceUnavailable, http.StatusOK, http.StatusRequestTimeout, http.StatusTooManyRequests} {
t.Run(fmt.Sprintf("%s/ns=%s/status=%d", kind, ns, status), func(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, kind)
f.collect(t, true, nil)
before, points := f.live(), f.points(t, "pbs-main")
if len(before) != 2 || len(points) != 2 {
t.Fatal("healthy control did not list both namespaces")
}
failed, accepted := "100", "101"
if ns == "tenant" {
failed, accepted = "101", "100"
f.tenantStatus.Store(int32(status))
f.rootAt.Add(15 * 60)
} else {
f.rootStatus.Store(int32(status))
f.tenantAt.Add(15 * 60)
}
key := pbsBackupGroupKey{datastore: "archive", namespace: ns, backupType: kind, backupID: failed}
cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
for range 2 {
f.collect(t, true, nil)
got := f.live()
if len(got) != 2 || !reflect.DeepEqual(got[failed], before[failed]) {
t.Errorf("transient namespace failure evicted or changed accepted live history: got=%+v", got)
}
if got[accepted].BackupTime.Unix() != before[accepted].BackupTime.Add(15*time.Minute).Unix() {
t.Error("accepted sibling did not advance")
}
if !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
t.Error("unread namespace cache time changed")
}
for _, prior := range points {
if prior.Details["namespace"] == ns {
found := false
for _, now := range f.points(t, "pbs-main") {
if now.ID == prior.ID {
found = reflect.DeepEqual(prior, now)
}
}
if !found {
t.Error("carried namespace renewed or lost recovery evidence")
}
}
}
// The accepted sibling was re-read; the failed namespace's
// retained point was not and cannot count.
acceptedID := f.ids[0]
if accepted == "101" {
acceptedID = f.ids[1]
}
for _, p := range f.postures(t) {
if len(p.ProviderStates) != 1 || p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown || p.Coverage != recovery.ProtectionCoveragePartial {
t.Errorf("incomplete history claimed complete coverage: %+v", p)
}
if (p.State == recovery.ProtectionStateProtected) != (p.SubjectResourceID == acceptedID) {
t.Errorf("protection does not follow the re-read namespace: %+v", p)
}
}
}
f.rootStatus.Store(0)
f.tenantStatus.Store(0)
f.collect(t, true, nil)
for _, p := range f.postures(t) {
if p.State != recovery.ProtectionStateProtected {
t.Errorf("ordinary read failed to recover: %+v", p)
}
}
f.rootEmpty.Store(true)
f.tenantEmpty.Store(true)
f.collect(t, true, nil)
if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
t.Fatal("complete empty read failed to clear confirmed absence")
}
})
}
}
}
}
func TestPBSNamespaceDiscoveryContinuityAndAccessWithdrawal(t *testing.T) {
for _, kind := range []string{"vm", "ct"} {
for _, denied := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/denied=%t", kind, denied), func(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, kind)
f.collect(t, true, nil)
prior := f.live()["101"]
key := pbsBackupGroupKey{datastore: "archive", namespace: "tenant", backupType: kind, backupID: "101"}
cacheAt := f.m.pbsBackupCacheTimeFor("pbs-main", key)
childReads := f.tenantReads.Load()
f.rootAt.Add(15 * 60)
discoveryErr := fmt.Errorf("fixture namespace discovery unavailable")
if denied {
discoveryErr = fmt.Errorf("api error 403: fixture denied")
}
for range 2 {
f.collect(t, false, discoveryErr)
got := f.live()
if denied {
if len(got) != 1 {
t.Errorf("discovery denial restored unread raw inventory: %+v", got)
}
} else if len(got) != 2 || !reflect.DeepEqual(got["101"], prior) {
t.Errorf("transient discovery erased or changed unread namespace: %+v", got)
}
if !denied && !f.m.pbsBackupCacheTimeFor("pbs-main", key).Equal(cacheAt) {
t.Error("failed discovery renewed snapshot reuse")
}
if f.tenantReads.Load() != childReads {
t.Error("discovery failure guessed a hidden namespace read")
}
// The root namespace was still read; only the tenant is unread.
for _, p := range f.postures(t) {
if (p.State == recovery.ProtectionStateProtected) != (p.SubjectResourceID == f.ids[0]) ||
(p.State == recovery.ProtectionStateProtected && p.Coverage != recovery.ProtectionCoveragePartial) {
t.Errorf("failed discovery protection does not follow the read root: %+v", p)
}
}
}
f.collect(t, true, nil)
if len(f.live()) != 2 {
t.Fatal("ordinary discovery did not recover inventory")
}
// A complete discovery that removes the child is authoritative absence.
f.collect(t, false, nil)
if len(f.live()) != 1 || len(f.points(t, "pbs-main")) != 1 {
t.Fatal("complete namespace removal retained absent history")
}
if !f.m.pbsBackupCacheTimeFor("pbs-main", key).IsZero() {
t.Error("removed namespace kept snapshot cache time")
}
})
}
}
}
func TestPBSNamespaceMixedFailuresDoNotResurrectDeniedScope(t *testing.T) {
for _, rootDenied := range []bool{false, true} {
for _, discoveryFailed := range []bool{false, true} {
t.Run(fmt.Sprintf("root-denied=%t/discovery-failed=%t", rootDenied, discoveryFailed), func(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, "vm")
f.collect(t, true, nil)
before := f.live()
other := before["100"]
other.ID = "other-instance-backup"
other.Instance = "pbs-other"
f.m.state.UpdatePBSBackups("pbs-other", []models.PBSBackup{other})
denied, retained := "101", "100"
f.rootStatus.Store(http.StatusServiceUnavailable)
f.tenantStatus.Store(http.StatusForbidden)
if rootDenied {
denied, retained = "100", "101"
f.rootStatus.Store(http.StatusUnauthorized)
f.tenantStatus.Store(http.StatusServiceUnavailable)
}
var discoveryErr error
if discoveryFailed {
discoveryErr = fmt.Errorf("fixture transient discovery failure")
}
for range 2 {
f.collect(t, true, discoveryErr)
got := f.live()
if len(got) != 1 || !reflect.DeepEqual(got[retained], before[retained]) {
t.Errorf("mixed failure lost transient scope or retained denied scope %s: %+v", denied, got)
}
if len(f.points(t, "pbs-main")) != 2 {
t.Error("mixed access failure deleted historical recovery facts")
}
for _, row := range f.m.state.GetSnapshot().PBSBackups {
if row.Instance == "pbs-other" && !reflect.DeepEqual(row, other) {
t.Error("failure crossed instance scope")
}
}
if len(f.m.state.GetSnapshot().PBSBackups) != 2 {
t.Error("other instance inventory disappeared or denied row was resurrected")
}
for _, p := range f.postures(t) {
if p.State == recovery.ProtectionStateProtected || len(p.ProviderStates) != 1 ||
p.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown ||
p.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryUnavailable {
t.Errorf("mixed rejected/unread scopes invented protection or universal denial: %+v", p)
}
}
}
f.rootStatus.Store(0)
f.tenantStatus.Store(0)
f.collect(t, true, nil)
if len(f.live()) != 2 {
t.Fatal("ordinary mixed-failure recovery did not restore both namespaces")
}
})
}
}
}
func TestPBSNamespaceSuccessfulEmptySiblingStaysEmpty(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, "vm")
f.collect(t, true, nil)
before := f.live()["101"]
f.rootEmpty.Store(true)
f.tenantStatus.Store(http.StatusServiceUnavailable)
for range 2 {
// A failed discovery cannot override the root's actual successful empty read.
f.collect(t, true, fmt.Errorf("fixture discovery unavailable"))
got := f.live()
if len(got) != 1 || !reflect.DeepEqual(got["101"], before) {
t.Errorf("carry-forward resurrected a confirmed-absent sibling or dropped unread history: %+v", got)
}
}
f.tenantStatus.Store(0)
f.tenantEmpty.Store(true)
f.collect(t, true, nil)
if len(f.live()) != 0 || len(f.points(t, "pbs-main")) != 0 {
t.Fatal("complete empty recovery retained absent files")
}
}
// The registered Go suite checks each phase. Optional frame export is evidence
// for the production Coverage/API-hook browser proof, not a separate test page.
func TestPBSNamespaceContinuityLifecycle(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, "vm")
frames := map[string]any{}
original := map[string]models.PBSBackup{}
for _, phase := range []string{"observed", "throttled", "timed-out", "recovered", "denied", "restored", "confirmed-empty"} {
switch phase {
case "throttled":
f.tenantStatus.Store(http.StatusTooManyRequests)
f.rootAt.Add(15 * 60)
case "timed-out":
f.tenantStatus.Store(http.StatusRequestTimeout)
case "recovered", "restored":
f.tenantStatus.Store(0)
case "denied":
f.tenantStatus.Store(http.StatusForbidden)
case "confirmed-empty":
f.rootEmpty.Store(true)
f.tenantEmpty.Store(true)
}
f.collect(t, true, nil)
rows := f.live()
want := 2
if phase == "denied" {
want = 1
}
if phase == "confirmed-empty" {
want = 0
}
if len(rows) != want {
t.Fatalf("phase %s live rows=%d want=%d", phase, len(rows), want)
}
if phase == "observed" {
original = rows
}
if (phase == "throttled" || phase == "timed-out") && !reflect.DeepEqual(rows["101"], original["101"]) {
t.Fatal("unread row changed")
}
postures := f.postures(t)
for _, p := range postures {
// Tenant failures leave the root snapshot re-read each poll.
tenantUnread := phase == "throttled" || phase == "timed-out" || phase == "denied"
healthy := phase == "observed" || phase == "recovered" || phase == "restored" ||
(tenantUnread && p.SubjectResourceID == f.ids[0])
if healthy != (p.State == recovery.ProtectionStateProtected) {
t.Fatalf("phase %s posture=%+v", phase, p)
}
}
frames[phase] = map[string]any{"pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}}, "postures": map[string]any{"data": postures, "policy": recovery.DefaultProtectionPosturePolicy.Payload()}, "points": f.points(t, "pbs-main")}
}
if os.Getenv("PULSE_PBS_NAMESPACE_FRAMES") != "1" {
return
}
source, err := os.ReadFile("monitor_backups.go")
if err != nil {
t.Fatal(err)
}
runtimeHashes := map[string]string{"internal/monitoring/monitor_backups.go": fmt.Sprintf("%x", sha256.Sum256(source))}
for _, file := range []string{"backup_guard.go", "../../pkg/pbs/client.go"} {
content, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
runtimeHashes[file] = fmt.Sprintf("%x", sha256.Sum256(content))
}
data, err := json.Marshal(map[string]any{"runtime_sha256": runtimeHashes, "observed_at": time.Now().UTC(), "resource_ids": f.ids, "frames": frames})
if err != nil {
t.Fatal(err)
}
t.Log("PBS_NAMESPACE_FRAMES=" + string(data))
}
func TestPBSNamespaceCarryForwardCannotDisplaceAcceptedRowsAtLimit(t *testing.T) {
f := newPBSNamespaceContinuityFixture(t, "vm")
f.collect(t, true, nil)
root, child := f.live()["100"], f.live()["101"]
previous := []models.PBSBackup{root}
for i := range pbsBackupLiveStateLimit {
row := child
row.ID = fmt.Sprintf("carried-%d", i)
row.VMID = fmt.Sprintf("%d", 200+i)
row.BackupTime = child.BackupTime.Add(time.Duration(i) * time.Second)
previous = append(previous, row)
}
f.m.state.UpdatePBSBackups("pbs-main", previous)
f.tenantStatus.Store(http.StatusServiceUnavailable)
f.collect(t, true, nil)
rows := f.m.state.GetSnapshot().PBSBackups
if len(rows) != pbsBackupLiveStateLimit {
t.Fatalf("live bound=%d want=%d", len(rows), pbsBackupLiveStateLimit)
}
foundAccepted, foundNewest := false, false
for _, row := range rows {
foundAccepted = foundAccepted || row.ID == root.ID
foundNewest = foundNewest || row.ID == fmt.Sprintf("carried-%d", pbsBackupLiveStateLimit-1)
if row.ID == "carried-0" {
t.Error("oldest carry-forward displaced a newer retained file")
}
}
if !foundAccepted || !foundNewest {
t.Error("carry-forward displaced accepted or newest retained inventory")
}
}