Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"reflect"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
)
func waitPBSBudget(t *testing.T, ch <-chan struct{}, phase string) {
t.Helper()
select {
case <-ch:
case <-time.After(10 * time.Second):
t.Fatalf("PBS worker did not reach %s", phase)
}
}
// PBS captures its runtime with the client and admission claim, rather than
// reading it again after starting the goroutine, so the inherited context
// itself is what must stay correct.
func TestPBSBackupWorkersInheritRuntimeContext(t *testing.T) {
exercisePBSBackupRuntimeContext(t)
}
func exercisePBSBackupRuntimeContext(t *testing.T) {
t.Run("PBS inherits the canonical runtime", func(t *testing.T) {
type runtimeKey struct{}
value := new(int)
parent, cancel := context.WithTimeout(context.WithValue(context.Background(), runtimeKey{}, value), time.Minute)
defer cancel()
m := &Monitor{}
client := &pbs.Client{}
m.replacePBSClient("reader", client)
m.setRuntimeContext(parent, nil)
owner := m.beginPBSBackupPoll("reader", client, time.Time{}, time.Now())
if owner == nil {
t.Fatal("live runtime did not admit its PBS worker")
}
defer m.finishPBSBackupPoll("reader", owner)
if owner.ctx.Value(runtimeKey{}) != value {
t.Error("PBS worker does not inherit the runtime's context values")
}
deadline, ok := owner.ctx.Deadline()
want, _ := parent.Deadline()
if !ok || !deadline.Equal(want) {
t.Error("PBS worker does not inherit the runtime's deadline")
}
cancel()
if owner.ctx.Err() != context.Canceled {
t.Error("runtime cancellation did not reach its PBS worker")
}
if m.pbsBackupPollers["reader"] != owner {
t.Error("runtime cancellation released the claim before the worker exited")
}
})
t.Run("an ended runtime cannot admit PBS work", func(t *testing.T) {
parent, cancel := context.WithCancel(context.Background())
m := &Monitor{}
client := &pbs.Client{}
m.replacePBSClient("reader", client)
m.setRuntimeContext(parent, nil)
cancel()
if owner := m.beginPBSBackupPoll("reader", client, time.Time{}, time.Now()); owner != nil {
m.finishPBSBackupPoll("reader", owner)
t.Fatal("ended runtime admitted a PBS worker")
}
if len(m.pbsBackupPollers) != 0 || len(m.lastPBSBackupPoll) != 0 {
t.Error("rejected admission left a worker or postponed the next scan")
}
})
}
func exercisePBSBudgetOwnerWithdrawal(t *testing.T) {
for _, transition := range []string{"runtime cancellation", "runtime replacement", "client replacement", "client retirement"} {
t.Run(transition, func(t *testing.T) {
f := newPBSBudgetFixture(t, "vm", "datastore")
f.collect(t, context.Background())
before, raw := f.points(t, "pbs-main"), f.m.state.GetSnapshot().PBSBackups
f.m.replacePBSClient("pbs-main", f.client)
parent, cancel := context.WithCancel(context.Background())
defer cancel()
f.m.setRuntimeContext(parent, nil)
f.m.config = &config.Config{EnableBackupPolling: true, BackupPollingCycles: 1}
cfg := &config.PBSInstance{Name: "pbs-main", MonitorBackups: true, MonitorDatastores: true}
f.at.Add(15 * 60)
f.phase.Store(1)
f.m.pollPBSBackupsAsync("pbs-main", f.client, cfg, f.datastores, nil, nil)
waitPBSBudget(t, f.blocked, "later datastore read")
f.m.mu.RLock()
owner := f.m.pbsBackupPollers["pbs-main"]
f.m.mu.RUnlock()
if owner == nil || owner.ctx.Err() != nil {
t.Fatal("background work lost its live runtime owner")
}
// A second ordinary cycle is not a second worker, even if cadence
// says it is due while the first HTTP request is still in flight.
f.m.pollPBSBackupsAsync("pbs-main", f.client, cfg, f.datastores, nil, nil)
f.m.mu.RLock()
if f.m.pbsBackupPollers["pbs-main"] != owner {
t.Error("ordinary cycle stacked a backup worker")
}
f.m.mu.RUnlock()
switch transition {
case "runtime cancellation":
cancel()
case "runtime replacement":
f.m.setRuntimeContext(context.Background(), nil) // Old parent intentionally remains live.
case "client replacement":
f.m.replacePBSClient("pbs-main", &pbs.Client{})
case "client retirement":
f.m.replacePBSClient("pbs-main", nil)
}
waitPBSBudget(t, owner.done, "withdrawn worker completion")
f.wait(t)
if !reflect.DeepEqual(raw, f.m.state.GetSnapshot().PBSBackups) || !reflect.DeepEqual(before, f.points(t, "pbs-main")) {
t.Error("withdrawn owner published its earlier/late rows")
}
// Dropping an exhausted child deadline must never erase the owner.
ctx := context.WithValue(context.Background(), pbsBackupPollContextKey{}, owner)
if unlock, ok := f.m.beginPBSBackupPublication(context.WithoutCancel(ctx), "pbs-main"); ok {
unlock()
t.Error("late owner was admitted after its binding ended")
}
f.m.mu.RLock()
at := f.m.lastPBSBackupPoll["pbs-main"]
f.m.mu.RUnlock()
if !at.IsZero() {
t.Error("withdrawn worker postponed its replacement's first scan")
}
if transition != "runtime cancellation" {
f.phase.Store(2)
f.m.replacePBSClient("pbs-main", f.client)
f.m.pollPBSBackupsAsync("pbs-main", f.client, cfg, f.datastores, nil, nil)
deadline := time.Now().Add(10 * time.Second)
for {
f.m.mu.RLock()
active := f.m.pbsBackupPollers["pbs-main"]
f.m.mu.RUnlock()
if active == nil {
break
}
if time.Now().After(deadline) {
t.Fatal("replacement ordinary scan did not finish")
}
time.Sleep(time.Millisecond)
}
f.wait(t)
for _, row := range f.m.state.GetSnapshot().PBSBackups {
if row.BackupTime.Unix() != f.at.Load() {
t.Error("ordinary replacement borrowed former client's cached snapshots")
}
}
}
})
}
}
func exercisePBSBudgetOldCompletion(t *testing.T) {
m := &Monitor{}
first, second := &pbs.Client{}, &pbs.Client{}
m.replacePBSClient("reader", first)
m.setRuntimeContext(context.Background(), nil)
old := m.beginPBSBackupPoll("reader", first, time.Time{}, time.Now())
if old == nil {
t.Fatal("first claim was not admitted")
}
m.replacePBSClient("reader", second)
next := time.Now()
current := m.beginPBSBackupPoll("reader", second, time.Time{}, next)
if current == nil {
t.Fatal("new binding did not get an independent first claim")
}
m.finishPBSBackupPoll("reader", old)
if m.pbsBackupPollers["reader"] != current || !m.lastPBSBackupPoll["reader"].Equal(next) {
t.Error("old completion removed the new worker or changed its cadence")
}
ctx, cancel := context.WithDeadline(context.WithValue(current.ctx, pbsBackupPollContextKey{}, current), time.Now().Add(-time.Second))
defer cancel()
unlock, ok := m.beginPBSBackupPublication(context.WithoutCancel(ctx), "reader")
if !ok {
t.Fatal("spent child budget blocked live owner's local publication")
}
unlock()
m.finishPBSBackupPoll("reader", current)
}