Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"reflect"
"strconv"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/operationaltrust"
"github.com/rcourtman/pulse-go-rewrite/internal/recovery"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
)
// These are ordinary inventory reads against loopback fixtures, not guest
// operations, installed backup jobs or a reproduction of a frozen workload.
type pbsBudgetFixture struct {
*pbsPosturePollFixture
phase, reads, lateReads atomic.Int32
at atomic.Int64
boundary string
empty bool
datastores []models.PBSDatastore
blocked chan struct{}
}
func newPBSBudgetFixture(t *testing.T, kind, boundary string) *pbsBudgetFixture {
t.Helper()
count := 3
if boundary == "snapshots" {
count = 10
} else if boundary == "tasks" {
count = 1
}
f := &pbsBudgetFixture{pbsPosturePollFixture: newPBSPosturePollFixture(t, count, kind), boundary: boundary, blocked: make(chan struct{}, 32)}
f.at.Store(f.backupAt)
f.datastores = []models.PBSDatastore{{Name: "archive"}, {Name: "slow"}, {Name: "never"}}
if boundary == "namespace" {
f.datastores = []models.PBSDatastore{{Name: "archive", Namespaces: []models.PBSNamespace{{Path: ""}, {Path: "slow"}, {Path: "never"}}}}
} else if boundary == "snapshots" || boundary == "tasks" {
f.datastores = []models.PBSDatastore{{Name: "archive"}}
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f.reads.Add(1)
stage := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
id := r.URL.Query().Get("backup-id")
scope := ""
if boundary == "namespace" {
scope = r.URL.Query().Get("ns")
} else if strings.Contains(r.URL.Path, "/slow/") {
scope = "slow"
} else if strings.Contains(r.URL.Path, "/never/") {
scope = "never"
}
if f.phase.Load() == 1 {
if scope == "never" {
f.lateReads.Add(1)
}
blocked := scope == "slow" || boundary == "snapshots" && stage == "snapshots" && id != "100" || boundary == "tasks" && stage == "tasks"
if blocked {
f.blocked <- struct{}{}
<-r.Context().Done()
return
}
}
var data any = []any{}
switch stage {
case "groups":
ids := []int{100}
if boundary == "snapshots" {
// More groups than workers: unread queued groups must retain
// their cache without making a request after the deadline.
ids = []int{100, 101, 102, 103, 104, 105, 106, 107, 108, 109}
} else if scope == "slow" {
ids = []int{101}
} else if scope == "never" {
ids = []int{102}
}
groups := []map[string]any{}
for _, vmid := range ids {
if f.empty && f.phase.Load() == 1 && vmid == 100 {
continue
}
groups = append(groups, map[string]any{"backup-type": kind, "backup-id": strconv.Itoa(vmid), "last-backup": f.at.Load(), "backup-count": 1})
}
data = groups
case "snapshots":
row := map[string]any{"backup-type": kind, "backup-id": id, "backup-time": f.at.Load(), "size": 1024,
"files": []string{"index.json.blob"}, "verification": map[string]any{"state": "ok"}, "comment": "guest-" + id}
if boundary == "tasks" && f.phase.Load() == 1 {
row["size"], row["files"] = 0, []string{"guest.conf.blob"}
}
data = []map[string]any{row}
case "tasks":
default:
http.NotFound(w, r)
return
}
_ = json.NewEncoder(w).Encode(map[string]any{"data": data})
}))
t.Cleanup(server.Close)
var err error
f.client, err = pbs.NewClient(pbs.ClientConfig{Host: server.URL, TokenName: "fixture@pbs!reader", TokenValue: "fixture-only"})
if err != nil {
t.Fatal(err)
}
f.m.pbsClients = map[string]*pbs.Client{"pbs-main": f.client}
f.m.setRuntimeContext(context.Background(), nil)
return f
}
func (f *pbsBudgetFixture) collect(t *testing.T, ctx context.Context) {
t.Helper()
f.m.pollPBSBackups(ctx, "pbs-main", f.client, f.datastores, nil)
f.wait(t)
}
func (f *pbsBudgetFixture) frame(t *testing.T) map[string]any {
t.Helper()
return map[string]any{
"pbs": map[string]any{"data": map[string]any{"backups": f.m.state.GetSnapshot().PBSBackups}},
"pve": map[string]any{"data": f.m.state.GetSnapshot().PVEBackups},
"postures": map[string]any{"data": f.postures(t), "policy": recovery.DefaultProtectionPosturePolicy.Payload()},
"points": f.points(t, "pbs-main"),
}
}
func exercisePBSBudgetAcceptedListings(t *testing.T) {
for _, kind := range []string{"vm", "ct"} {
for _, boundary := range []string{"datastore", "namespace", "snapshots", "tasks"} {
for _, cold := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/%s/cold=%t", kind, boundary, cold), func(t *testing.T) {
f := newPBSBudgetFixture(t, kind, boundary)
f.collect(t, context.Background())
oldRaw := f.m.state.GetSnapshot().PBSBackups
oldPoints := f.points(t, "pbs-main")
frames := map[string]any{"observed": f.frame(t)}
f.m.mu.RLock()
cache := make(map[pbsBackupGroupKey]time.Time)
for key, at := range f.m.pbsBackupCacheTime["pbs-main"] {
cache[key] = at
}
f.m.mu.RUnlock()
if cold {
// A replacement monitor has durable history, but neither
// live snapshots nor their in-memory fetch timestamps.
// Orphan timestamps alone are not a reusable snapshot cache.
f.m.state.UpdatePBSBackups("pbs-main", nil)
f.m.mu.Lock()
delete(f.m.pbsBackupCacheTime, "pbs-main")
f.m.mu.Unlock()
for key := range cache {
cache[key] = time.Time{}
}
}
f.at.Add(15 * 60)
f.phase.Store(1)
startReads := f.reads.Load()
owner := f.m.beginPBSBackupPoll("pbs-main", f.client, time.Time{}, time.Now())
if owner == nil {
t.Fatal("collection owner was not admitted")
}
defer f.m.finishPBSBackupPoll("pbs-main", owner)
parent := context.WithValue(owner.ctx, pbsBackupPollContextKey{}, owner)
ctx, cancel := context.WithTimeout(parent, time.Second)
defer cancel()
f.collect(t, ctx)
if ctx.Err() != context.DeadlineExceeded || f.lateReads.Load() != 0 {
t.Fatalf("budget did not expire cleanly: err=%v queued late reads=%d", ctx.Err(), f.lateReads.Load())
}
if boundary == "snapshots" && f.reads.Load()-startReads > 2+pbsBackupSnapshotFetchWorkers {
t.Error("queued snapshot work started after expiry")
}
raw := f.m.state.GetSnapshot().PBSBackups
advanced := false
for _, row := range raw {
if row.VMID == "100" {
advanced = row.BackupTime.Unix() == f.at.Load()
if boundary == "tasks" && (!row.InProgress || row.WriteActivityObserved || row.WriteActive) {
t.Error("unread live tasks invented terminal or successful write evidence")
}
} else {
found := false
for _, old := range oldRaw {
found = found || reflect.DeepEqual(row, old)
}
if cold || !found {
t.Error("unread raw history was invented or freshened")
}
}
}
if !advanced || !cold && len(raw) != len(oldRaw) || cold && len(raw) != 1 {
t.Errorf("accepted snapshot discarded or unread membership changed: advanced=%t rows=%d prior=%d cold=%t", advanced, len(raw), len(oldRaw), cold)
}
for key, old := range cache {
at := f.m.pbsBackupCacheTimeFor("pbs-main", key)
if key.backupID == "100" {
if !at.After(old) || !at.Before(time.Now()) {
t.Error("accepted read time was not published with its snapshot")
}
} else if !at.Equal(old) {
t.Error("unread cached snapshot's read time changed")
}
}
points := f.points(t, "pbs-main")
for _, old := range oldPoints {
found := false
for _, now := range points {
found = found || reflect.DeepEqual(old, now)
}
if !found {
t.Error("partial enumeration changed/deleted previously accepted durable evidence")
}
}
for _, posture := range f.postures(t) {
if len(posture.ProviderStates) != 1 || posture.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryPartial || posture.ProviderStates[0].Permissions != operationaltrust.EvidencePermissionsUnknown || posture.Coverage != recovery.ProtectionCoveragePartial {
t.Errorf("partial collection invented complete history: %+v", posture)
}
// Only guest 100's snapshot was accepted before the budget
// expired; at the tasks boundary it is still being written.
wantProtected := posture.SubjectResourceID == f.ids[0] && boundary != "tasks"
if (posture.State == recovery.ProtectionStateProtected) != wantProtected {
t.Errorf("partial collection protection does not follow the accepted read (protected=%t): %+v", wantProtected, posture)
}
}
frames["partial"] = f.frame(t)
f.phase.Store(2)
f.collect(t, context.Background())
for _, posture := range f.postures(t) {
if posture.ProviderStates[0].HistoryCompleteness != recovery.ProtectionHistoryComplete || posture.State != recovery.ProtectionStateProtected {
t.Errorf("ordinary complete poll failed to recover: %+v", posture)
}
}
frames["recovered"] = f.frame(t)
if !cold && (boundary == "datastore" || boundary == "tasks") {
if path := os.Getenv("PULSE_PBS_BUDGET_FRAMES"); path != "" {
data, err := json.Marshal(frames)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path+"-"+kind+"-"+boundary+".json", data, 0600); err != nil {
t.Fatal(err)
}
t.Logf("PULSE_PBS_FRAME %s-%s %x %s", kind, boundary, sha256.Sum256(data), data)
}
}
t.Logf("kind=%s boundary=%s cold=%t: accepted snapshot advanced; unread raw/cache/durable evidence retained; partial posture and ordinary recovery proved", kind, boundary, cold)
})
}
}
}
}
func TestPBSBudgetAcceptedEmptyDoesNotProveUnreadAbsence(t *testing.T) {
for _, kind := range []string{"vm", "ct"} {
t.Run(kind, func(t *testing.T) {
f := newPBSBudgetFixture(t, kind, "datastore")
f.collect(t, context.Background())
points := f.points(t, "pbs-main")
f.empty = true
f.phase.Store(1)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
f.collect(t, ctx)
if len(f.m.state.GetSnapshot().PBSBackups) != 2 || !reflect.DeepEqual(points, f.points(t, "pbs-main")) {
t.Error("accepted empty scope retained raw rows or partial scope deleted durable evidence")
}
for _, row := range f.m.state.GetSnapshot().PBSBackups {
if row.VMID == "100" {
t.Error("accepted empty listing failed to remove its raw row")
}
}
})
}
}
func exercisePBSBudgetExpiredAlertEvaluation(t *testing.T) {
f := newPBSBudgetFixture(t, "vm", "datastore")
f.at.Store(time.Now().Add(-4 * 24 * time.Hour).Unix())
f.collect(t, context.Background())
am := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(am.Stop)
cfg := am.GetConfig()
cfg.Enabled, cfg.BackupDefaults.Enabled = true, true
cfg.ActivationState = alerts.ActivationActive
cfg.BackupDefaults.WarningDays, cfg.BackupDefaults.CriticalDays = 1, 2
am.UpdateConfig(cfg)
f.m.alertManager = am
f.m.checkBackupAlerts(context.Background())
if self, ages := backupEvaluationAlerts(t, f.m); self != nil || ages != 3 {
t.Fatalf("wanted backup-age controls self=%v ages=%d", self, ages)
}
// Simulate an already-visible evaluation warning, not an actual store
// outage or a new diagnostic request. Collection expiry must not clear it.
am.RaiseSystemAlert(alerts.SystemAlertInput{Type: alerts.BackupEvaluationAlertType,
Level: alerts.AlertLevelWarning, Message: "Fixture standing evaluation warning", Fingerprint: "fixture-standing"})
before := map[string]alerts.Alert{}
for _, alert := range am.GetActiveAlerts() {
before[alert.ID] = alert
}
f.at.Store(time.Now().Add(-time.Hour).Unix())
f.phase.Store(1)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
f.collect(t, ctx)
if ctx.Err() != context.DeadlineExceeded {
t.Fatal("fixture did not spend the collection budget")
}
active := am.GetActiveAlerts()
if len(active) != len(before) {
t.Error("spent collection budget evaluated/cleared standing alerts")
}
for _, alert := range active {
old, ok := before[alert.ID]
if !ok || old.Level != alert.Level || old.Message != alert.Message || !old.StartTime.Equal(alert.StartTime) {
t.Error("spent collection budget replaced a standing alert")
}
}
// A separate ordinary evaluation may use the accepted fresh point. It
// resolves only its age and the evaluation warning, not unread siblings.
f.m.checkBackupAlerts(context.Background())
if self, ages := backupEvaluationAlerts(t, f.m); self != nil || ages != 2 {
t.Errorf("ordinary evaluation self=%v age alerts=%d; want only the two unread workloads", self, ages)
}
}