Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"sort"
"strings"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
)
func TestPollPMGInstancePopulatesState(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api2/json/access/ticket":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"ticket":"ticket123","CSRFPreventionToken":"csrf123"}}`)
case "/api2/json/version":
if !strings.Contains(r.Header.Get("Cookie"), "PMGAuthCookie=ticket123") {
t.Fatalf("expected auth cookie, got %s", r.Header.Get("Cookie"))
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"version":"8.3.1","release":"1"}}`)
case "/api2/json/config/cluster/status":
if r.URL.Query().Get("list_single_node") != "1" {
t.Fatalf("expected list_single_node query, got %s", r.URL.RawQuery)
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":[{"cid":1,"name":"mail-gateway","type":"master","ip":"10.0.0.1"}]}`)
case "/api2/json/statistics/mail":
// PMG API does not accept timeframe parameter
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":100,"count_in":60,"count_out":40,"spamcount_in":5,"spamcount_out":2,"viruscount_in":1,"viruscount_out":0,"bounces_in":3,"bounces_out":1,"bytes_in":12345,"bytes_out":54321,"glcount":7,"junk_in":4,"avptime":0.5,"rbl_rejects":2,"pregreet_rejects":1}}`)
case "/api2/json/statistics/mailcount":
if r.URL.Query().Get("timespan") != "86400" {
t.Fatalf("expected timespan=86400 (24 hours in seconds), got %s", r.URL.RawQuery)
}
now := time.Now().Unix()
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"data":[{"index":0,"time":%d,"count":100,"count_in":60,"count_out":40,"spamcount_in":5,"spamcount_out":2,"viruscount_in":1,"viruscount_out":0,"bounces_in":3,"bounces_out":1,"rbl_rejects":2,"pregreet_rejects":1,"glcount":7}]}`, now)
case "/api2/json/statistics/spamscores":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":[{"level":"low","count":10,"ratio":0.1}]}`)
case "/api2/json/quarantine/spamstatus":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":5,"avgbytes":0,"mbytes":0}}`)
case "/api2/json/quarantine/virusstatus":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":2,"avgbytes":0,"mbytes":0}}`)
case "/api2/json/nodes/mail-gateway/postfix/qshape":
totals := map[string]string{"active": "5", "deferred": "2", "hold": "0", "incoming": "0"}
total, ok := totals[r.URL.Query().Get("queue")]
if !ok {
t.Fatalf("unexpected qshape queue: %s", r.URL.RawQuery)
}
oldest := "0"
if r.URL.Query().Get("queue") == "deferred" {
oldest = total
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `{"data":[{"domain":"TOTAL","total":"%s","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"%s","1280m+":"0"}]}`, total, oldest)
case "/api2/json/nodes/mail-gateway/backup":
w.Header().Set("Content-Type", "application/json")
timestamp := time.Now().Add(-8 * 24 * time.Hour).Unix()
fmt.Fprintf(w, `{"data":[{"filename":"pmg-backup_2024-01-01.tgz","size":123456,"timestamp":%d}]}`, timestamp)
case "/api2/json/config/domains":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":[{"domain":"example.com","comment":"Primary relay"}]}`)
case "/api2/json/statistics/domains":
if r.URL.Query().Get("starttime") == "" || r.URL.Query().Get("endtime") == "" {
t.Fatalf("expected starttime/endtime query, got %s", r.URL.RawQuery)
}
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":[{"domain":"example.com","count":120,"spamcount":8,"viruscount":1,"bytes":1234567}]}`)
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
defer server.Close()
client, err := pmg.NewClient(pmg.ClientConfig{
Host: server.URL,
User: "api@pmg",
Password: "secret",
VerifySSL: false,
})
if err != nil {
t.Fatalf("unexpected client error: %v", err)
}
cfg := &config.Config{
PMGInstances: []config.PMGInstance{
{
Name: "primary",
Host: server.URL,
User: "api@pmg",
Password: "secret",
MonitorMailStats: true,
MonitorQueues: true,
MonitorQuarantine: true,
MonitorDomainStats: true,
},
},
}
mon, err := New(cfg)
if err != nil {
t.Fatalf("failed to create monitor: %v", err)
}
mon.pollPMGInstance(context.Background(), "primary", client)
snapshot := mon.state.GetSnapshot()
if len(snapshot.PMGInstances) != 1 {
t.Fatalf("expected 1 PMG instance in state, got %d", len(snapshot.PMGInstances))
}
instance := snapshot.PMGInstances[0]
if instance.Status != "online" {
t.Fatalf("expected PMG status online, got %s", instance.Status)
}
if instance.ConnectionHealth != "healthy" {
t.Fatalf("expected connection health healthy, got %s", instance.ConnectionHealth)
}
if instance.MailStats == nil || instance.MailStats.CountTotal != 100 {
t.Fatalf("expected mail stats totals, got %+v", instance.MailStats)
}
if len(instance.MailCount) != 1 {
t.Fatalf("expected 1 mail count point, got %d", len(instance.MailCount))
}
if len(instance.SpamDistribution) != 1 {
t.Fatalf("expected 1 spam distribution bucket, got %d", len(instance.SpamDistribution))
}
if instance.Quarantine == nil || instance.Quarantine.Spam != 5 || instance.Quarantine.Virus != 2 {
t.Fatalf("expected quarantine counts, got %+v", instance.Quarantine)
}
if len(instance.Nodes) != 1 || instance.Nodes[0].QueueStatus == nil {
t.Fatalf("expected node queue status, got %+v", instance.Nodes)
}
if queue := instance.Nodes[0].QueueStatus; queue.Active != 5 || queue.Deferred != 2 || queue.Total != 7 || queue.OldestAge != 640*60 {
t.Fatalf("unexpected queue status: %+v", queue)
}
if health := snapshot.ConnectionHealth["pmg-primary"]; !health {
t.Fatalf("expected connection health tracked as healthy, got %v", health)
}
if failures := mon.authFailures["pmg-primary"]; failures != 0 {
t.Fatalf("expected no auth failures tracked, got %d", failures)
}
if len(snapshot.PMGBackups) != 1 {
t.Fatalf("expected 1 PMG backup in state, got %d", len(snapshot.PMGBackups))
}
pmgBackup := snapshot.PMGBackups[0]
if pmgBackup.Node != "mail-gateway" {
t.Fatalf("expected PMG backup node mail-gateway, got %s", pmgBackup.Node)
}
if len(instance.RelayDomains) != 1 {
t.Fatalf("expected 1 relay domain, got %d", len(instance.RelayDomains))
}
if instance.RelayDomains[0].Domain != "example.com" {
t.Fatalf("expected relay domain example.com, got %q", instance.RelayDomains[0].Domain)
}
if len(instance.DomainStats) != 1 {
t.Fatalf("expected 1 domain stat row, got %d", len(instance.DomainStats))
}
if instance.DomainStats[0].Domain != "example.com" || instance.DomainStats[0].SpamCount != 8 {
t.Fatalf("unexpected domain stats: %+v", instance.DomainStats[0])
}
}
func TestPollPMGInstanceRecordsAuthFailures(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api2/json/version":
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprint(w, "unauthorized")
case "/api2/json/access/ticket":
t.Fatalf("token client should not request auth ticket")
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
defer server.Close()
client, err := pmg.NewClient(pmg.ClientConfig{
Host: server.URL,
User: "apitest@pmg",
TokenName: "apitoken",
TokenValue: "secret",
VerifySSL: false,
})
if err != nil {
t.Fatalf("unexpected error creating token client: %v", err)
}
cfg := &config.Config{
PMGInstances: []config.PMGInstance{
{
Name: "failing",
Host: server.URL,
User: "apitest@pmg",
TokenName: "apitoken",
TokenValue: "secret",
},
},
}
mon, err := New(cfg)
if err != nil {
t.Fatalf("failed to create monitor: %v", err)
}
mon.pollPMGInstance(context.Background(), "failing", client)
snapshot := mon.state.GetSnapshot()
if health := snapshot.ConnectionHealth["pmg-failing"]; health {
t.Fatalf("expected unhealthy connection, got %v", health)
}
if len(snapshot.PMGInstances) != 1 {
t.Fatalf("expected failed PMG instance in state, got %d", len(snapshot.PMGInstances))
}
instance := snapshot.PMGInstances[0]
if instance.Status != "offline" {
t.Fatalf("expected offline status, got %s", instance.Status)
}
if instance.ConnectionHealth != "unhealthy" {
t.Fatalf("expected unhealthy connection status, got %s", instance.ConnectionHealth)
}
if failures := mon.authFailures["pmg-failing"]; failures != 1 {
t.Fatalf("expected one auth failure tracked, got %d", failures)
}
// Regression: pollStatusMap must record the failure. A defer-arg bug
// previously captured pollErr at register-time (always nil), so failed
// polls were recorded as success and the connections aggregator reported
// broken instances as healthy.
status := mon.pollStatusMap["pmg::failing"]
if status == nil {
t.Fatal("expected pollStatusMap entry for pmg::failing, got nil")
}
if !status.LastSuccess.IsZero() {
t.Errorf("expected LastSuccess to remain zero on auth failure, got %v", status.LastSuccess)
}
if status.ConsecutiveFailures == 0 {
t.Error("expected ConsecutiveFailures > 0 after auth failure, got 0")
}
}
// Uses the real poll consumer with a typed, read-only provider boundary. No
// listener, mail, credentials, external services or native appliance is used.
type pmgScopeFixture struct {
calls map[string]int
fail string
onVersion func()
}
func (f *pmgScopeFixture) read(key, data string, out any) error {
f.calls[key]++
if f.fail == key {
return errors.New("fixture collection refused")
}
return json.Unmarshal([]byte(data), out)
}
func (f *pmgScopeFixture) GetVersion(context.Context) (*pmg.VersionInfo, error) {
var v pmg.VersionInfo
err := f.read("version", `{"version":"8.3"}`, &v)
if f.onVersion != nil {
f.onVersion()
}
return &v, err
}
func (f *pmgScopeFixture) GetClusterStatus(_ context.Context, single bool) ([]pmg.ClusterStatusEntry, error) {
if !single {
return nil, errors.New("single-node query omitted")
}
var v []pmg.ClusterStatusEntry
err := f.read("cluster", `[{"name":"one","type":"master"},{"name":"two","type":"slave"}]`, &v)
return v, err
}
func (f *pmgScopeFixture) GetQueueStatus(_ context.Context, node string) (*pmg.QueueStatusEntry, error) {
var v pmg.QueueStatusEntry
err := f.read("queue/"+node, `{"active":20,"deferred":10,"hold":5,"incoming":2,"oldest_age":7200}`, &v)
return &v, err
}
func (f *pmgScopeFixture) ListBackups(_ context.Context, node string) ([]pmg.BackupEntry, error) {
var v []pmg.BackupEntry
err := f.read("backup/"+node, `[{"filename":"fixture.tgz","timestamp":1700000000,"size":123}]`, &v)
return v, err
}
func (f *pmgScopeFixture) GetMailStatistics(_ context.Context, frame string) (*pmg.MailStatistics, error) {
if frame != "" {
return nil, errors.New("unexpected timeframe")
}
var v pmg.MailStatistics
err := f.read("mail", `{"count":123,"avptime":0.5}`, &v)
return &v, err
}
func (f *pmgScopeFixture) GetMailCount(_ context.Context, span int) ([]pmg.MailCountEntry, error) {
if span != 86400 {
return nil, errors.New("wrong collection window")
}
var v []pmg.MailCountEntry
err := f.read("count", `[{"time":1700000000,"count":123}]`, &v)
return v, err
}
func (f *pmgScopeFixture) GetSpamScores(context.Context) ([]pmg.SpamScore, error) {
var v []pmg.SpamScore
err := f.read("scores", `[{"level":"low","count":4}]`, &v)
return v, err
}
func (f *pmgScopeFixture) GetQuarantineStatus(_ context.Context, category string) (*pmg.QuarantineStatus, error) {
var v pmg.QuarantineStatus
err := f.read("quarantine/"+category, `{"count":200}`, &v)
return &v, err
}
func (f *pmgScopeFixture) ListRelayDomains(context.Context) ([]pmg.RelayDomainEntry, error) {
var v []pmg.RelayDomainEntry
err := f.read("domains", `[{"domain":"example.invalid"}]`, &v)
return v, err
}
func (f *pmgScopeFixture) GetDomainStatistics(_ context.Context, start, end int64) ([]pmg.DomainStatisticsEntry, error) {
if end-start != 86400 {
return nil, errors.New("wrong domain window")
}
var v []pmg.DomainStatisticsEntry
err := f.read("domain-stats", `[{"domain":"example.invalid","count":123}]`, &v)
return v, err
}
func pmgScopeMonitor(cfg config.PMGInstance) *Monitor {
cfg.Name = "gateway"
return &Monitor{
config: &config.Config{PMGInstances: []config.PMGInstance{cfg}},
state: models.NewState(), authFailures: map[string]int{}, lastAuthAttempt: map[string]time.Time{},
pollStatusMap: map[string]*pollStatus{}, circuitBreakers: map[string]*circuitBreaker{},
}
}
func allPMGScope() config.PMGInstance {
return config.PMGInstance{MonitoringConfigured: true, MonitorMailStats: true, MonitorQueues: true, MonitorQuarantine: true, MonitorDomainStats: true}
}
func TestPMGCollectionScope(t *testing.T) {
for mask := 0; mask < 16; mask++ {
t.Run(fmt.Sprintf("scope-%04b", mask), func(t *testing.T) {
cfg := config.PMGInstance{MonitoringConfigured: true, MonitorMailStats: mask&1 != 0, MonitorQueues: mask&2 != 0, MonitorQuarantine: mask&4 != 0, MonitorDomainStats: mask&8 != 0}
m := pmgScopeMonitor(cfg)
f := &pmgScopeFixture{calls: map[string]int{}}
m.pollPMGInstance(context.Background(), "gateway", f)
want := map[string]int{"version": 1, "cluster": 1, "backup/one": 1, "backup/two": 1}
if cfg.MonitorMailStats {
want["mail"] = 1
want["count"] = 1
want["scores"] = 1
}
if cfg.MonitorQueues {
want["queue/one"] = 1
want["queue/two"] = 1
}
if cfg.MonitorQuarantine {
want["quarantine/spam"] = 1
want["quarantine/virus"] = 1
}
if cfg.MonitorDomainStats {
want["domains"] = 1
want["domain-stats"] = 1
}
if !reflect.DeepEqual(f.calls, want) {
t.Fatalf("requests=%v want=%v", f.calls, want)
}
v := m.state.GetSnapshot().PMGInstances[0]
if (v.MailStats != nil) != cfg.MonitorMailStats || (len(v.MailCount) > 0) != cfg.MonitorMailStats || (len(v.SpamDistribution) > 0) != cfg.MonitorMailStats {
t.Fatalf("incorrect mail data: %+v", v)
}
if v.MailStats != nil && (v.MailStats.CountTotal != 123 || v.MailStats.AverageProcessTimeMs != 500) {
t.Fatal("selected statistics lost values/units")
}
if (v.Quarantine != nil) != cfg.MonitorQuarantine {
t.Fatalf("incorrect quarantine: %+v", v.Quarantine)
}
for _, n := range v.Nodes {
if (n.QueueStatus != nil) != cfg.MonitorQueues {
t.Fatalf("incorrect queue: %+v", n)
}
if n.QueueStatus != nil && n.QueueStatus.Total != 37 {
t.Fatal("wrong queue total")
}
}
if (len(v.DomainStats) > 0) != cfg.MonitorDomainStats || (len(v.RelayDomains) > 0) != cfg.MonitorDomainStats {
t.Fatal("incorrect domain collection")
}
if v.Status != "online" || v.ConnectionHealth != "healthy" || m.pollStatusMap["pmg::gateway"].ConsecutiveFailures != 0 {
t.Fatal("selected collection not recorded as successful")
}
})
}
}
func TestPMGCollectionLegacyDefault(t *testing.T) {
m := pmgScopeMonitor(config.PMGInstance{})
f := &pmgScopeFixture{calls: map[string]int{}}
m.pollPMGInstance(context.Background(), "gateway", f)
want := map[string]int{"version": 1, "cluster": 1, "backup/one": 1, "backup/two": 1, "mail": 1, "count": 1, "scores": 1}
if !reflect.DeepEqual(f.calls, want) {
t.Fatalf("legacy requests=%v want=%v", f.calls, want)
}
}
func TestPMGCollectionEditsPauseAndResume(t *testing.T) {
m := pmgScopeMonitor(allPMGScope())
f := &pmgScopeFixture{calls: map[string]int{}}
m.pollPMGInstance(context.Background(), "gateway", f)
config.Mu.Lock()
m.config.PMGInstances[0] = config.PMGInstance{Name: "gateway", MonitoringConfigured: true}
config.Mu.Unlock()
f.calls = map[string]int{}
m.pollPMGInstance(context.Background(), "gateway", f)
if len(f.calls) != 4 {
t.Fatalf("all-off still read datasets: %v", f.calls)
}
v := m.state.GetSnapshot().PMGInstances[0]
if v.MailStats != nil || v.Quarantine != nil || v.Nodes[0].QueueStatus != nil || len(v.DomainStats) != 0 {
t.Fatal("old dataset exposed as current after disable")
}
config.Mu.Lock()
m.config.PMGInstances[0].Disabled = true
config.Mu.Unlock()
f.calls = map[string]int{}
m.pollPMGInstance(context.Background(), "gateway", f)
if len(f.calls) != 0 {
t.Fatalf("paused requests=%v", f.calls)
}
if !reflect.DeepEqual(m.state.GetSnapshot().PMGInstances[0], v) {
t.Fatal("pause overwrote last readings")
}
config.Mu.Lock()
m.config.PMGInstances[0].Disabled = false
m.config.PMGInstances[0].MonitorQueues = true
config.Mu.Unlock()
m.pollPMGInstance(context.Background(), "gateway", f)
if f.calls["queue/one"] != 1 || f.calls["mail"] != 0 {
t.Fatalf("resume scope=%v", f.calls)
}
}
func TestPMGCollectionInFlightSnapshot(t *testing.T) {
m := pmgScopeMonitor(allPMGScope())
f := &pmgScopeFixture{calls: map[string]int{}}
reached := make(chan struct{})
resume := make(chan struct{})
done := make(chan struct{})
f.onVersion = func() { close(reached); <-resume }
go func() { defer close(done); m.pollPMGInstance(context.Background(), "gateway", f) }()
<-reached
config.Mu.Lock()
m.config.PMGInstances[0] = config.PMGInstance{Name: "gateway", MonitoringConfigured: true, Disabled: true}
config.Mu.Unlock()
close(resume)
<-done
if f.calls["mail"] != 1 || f.calls["queue/one"] != 1 || f.calls["domain-stats"] != 1 || f.calls["quarantine/spam"] != 1 {
t.Fatal("in-flight snapshot changed during poll")
}
f.onVersion = nil
f.calls = map[string]int{}
m.pollPMGInstance(context.Background(), "gateway", f)
if len(f.calls) != 0 {
t.Fatalf("next paused poll requests=%v", f.calls)
}
}
func TestPMGCollectionFailureEvidence(t *testing.T) {
for _, op := range []string{"version", "cluster", "backup/one", "backup/two", "mail", "count", "scores", "queue/one", "queue/two", "quarantine/spam", "quarantine/virus", "domains", "domain-stats"} {
t.Run(op, func(t *testing.T) {
m := pmgScopeMonitor(allPMGScope())
f := &pmgScopeFixture{calls: map[string]int{}}
m.pollPMGInstance(context.Background(), "gateway", f)
before := m.pollStatusMap["pmg::gateway"].LastSuccess
f.fail = op
m.pollPMGInstance(context.Background(), "gateway", f)
status := m.pollStatusMap["pmg::gateway"]
v := m.state.GetSnapshot().PMGInstances[0]
if op == "version" {
if status.ConsecutiveFailures != 1 || status.LastErrorMessage == "" || !status.LastSuccess.Equal(before) {
t.Fatalf("failed connection recorded as complete: %+v", status)
}
if v.Status != "offline" || v.ConnectionHealth != "unhealthy" {
t.Fatal("failed connection not offline")
}
return
}
// A downstream read failure stays visible on the connection but
// is not a scheduler failure (#2960).
if status.ConsecutiveFailures != 0 || status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory || status.LastSuccess.Before(before) {
t.Fatalf("failed read recorded as complete or as a connection failure: %+v", status)
}
if v.Status != "online" || v.ConnectionHealth != "healthy" || !m.state.GetSnapshot().ConnectionHealth["pmg-gateway"] {
t.Fatal("downstream failure mislabeled as version connection failure")
}
if (op == "quarantine/spam" || op == "quarantine/virus") && v.Quarantine != nil {
t.Fatal("partial quarantine reported as zero/complete")
}
if op == "mail" && v.MailStats != nil {
t.Fatal("failed stats published")
}
if op == "queue/one" && v.Nodes[0].QueueStatus != nil {
t.Fatal("failed queue published")
}
f.fail = ""
m.pollPMGInstance(context.Background(), "gateway", f)
if status.ConsecutiveFailures != 0 || status.LastErrorMessage != "" {
t.Fatal("next complete poll did not recover")
}
})
}
}
func TestPMGCollectionCancelledAndUnknown(t *testing.T) {
m := pmgScopeMonitor(allPMGScope())
f := &pmgScopeFixture{calls: map[string]int{}}
ctx, cancel := context.WithCancel(context.Background())
cancel()
m.pollPMGInstance(ctx, "gateway", f)
m.pollPMGInstance(context.Background(), "unknown", f)
if len(f.calls) != 0 {
t.Fatalf("cancelled/unknown requests=%v", f.calls)
}
}
// pollPMGIssue2960 polls one PMG instance named like the reporter's
// connection ("PMG-01") against a server whose cluster status is served by
// clusterStatus, and records every backup path requested.
func pollPMGIssue2960(t *testing.T, clusterStatus func(w http.ResponseWriter)) (*Monitor, []string) {
t.Helper()
var backupPaths []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.URL.Path == "/api2/json/version":
fmt.Fprint(w, `{"data":{"version":"9.1.4","release":"1"}}`)
case r.URL.Path == "/api2/json/config/cluster/status":
clusterStatus(w)
case strings.HasSuffix(r.URL.Path, "/backup"):
backupPaths = append(backupPaths, r.URL.Path)
if r.URL.Path != "/api2/json/nodes/pmg-01/backup" {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, `{"message":"hostname lookup failed","data":null}`)
return
}
timestamp := time.Now().Add(-24 * time.Hour).Unix()
fmt.Fprintf(w, `{"data":[{"filename":"pmg-backup.tgz","size":2048,"timestamp":%d}]}`, timestamp)
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
t.Cleanup(server.Close)
client, err := pmg.NewClient(pmg.ClientConfig{
Host: server.URL,
TokenName: "pulse@pmg!monitor",
TokenValue: "secret",
VerifySSL: false,
})
if err != nil {
t.Fatalf("unexpected client error: %v", err)
}
mon, err := New(&config.Config{PMGInstances: []config.PMGInstance{{
Name: "PMG-01",
Host: server.URL,
TokenName: "pulse@pmg!monitor",
TokenValue: "secret",
// Explicit scope with every collector off: only cluster status and
// backups are read.
MonitoringConfigured: true,
}}})
if err != nil {
t.Fatalf("failed to create monitor: %v", err)
}
mon.pollPMGInstance(context.Background(), "PMG-01", client)
return mon, backupPaths
}
// Issue #2960: PMG 9 sends the cluster id as a string. The cluster read must
// still succeed, so backups come from the node PMG names, not the connection.
func TestPollPMGInstanceAcceptsStringClusterID(t *testing.T) {
mon, backupPaths := pollPMGIssue2960(t, func(w http.ResponseWriter) {
fmt.Fprint(w, `{"data":[{"cid":"1","name":"pmg-01","type":"master","ip":"10.0.0.5"}]}`)
})
snapshot := mon.state.GetSnapshot()
if len(snapshot.PMGInstances) != 1 || len(snapshot.PMGInstances[0].Nodes) != 1 ||
snapshot.PMGInstances[0].Nodes[0].Name != "pmg-01" {
t.Fatalf("expected the PMG node from cluster status, got %+v", snapshot.PMGInstances)
}
if !reflect.DeepEqual(backupPaths, []string{"/api2/json/nodes/pmg-01/backup"}) {
t.Fatalf("expected backups from node pmg-01 only, got %v", backupPaths)
}
if len(snapshot.PMGBackups) != 1 || snapshot.PMGBackups[0].Node != "pmg-01" {
t.Fatalf("expected one backup on pmg-01, got %+v", snapshot.PMGBackups)
}
}
// Issue #2960: after a failed cluster read the connection's display name is
// not a node, so Pulse must not request backups with it.
func TestPollPMGInstanceDoesNotGuessNodeAfterFailedClusterRead(t *testing.T) {
_, backupPaths := pollPMGIssue2960(t, func(w http.ResponseWriter) {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, `{"message":"cluster status unavailable","data":null}`)
})
if len(backupPaths) != 0 {
t.Fatalf("expected no backup request after a failed cluster read, got %v", backupPaths)
}
}
// A PMG that answers with no cluster entries keeps the connection-name
// fallback some standalone installs rely on.
func TestPollPMGInstanceKeepsNameFallbackForEmptyCluster(t *testing.T) {
_, backupPaths := pollPMGIssue2960(t, func(w http.ResponseWriter) {
fmt.Fprint(w, `{"data":[]}`)
})
if !reflect.DeepEqual(backupPaths, []string{"/api2/json/nodes/PMG-01/backup"}) {
t.Fatalf("expected the connection-name fallback for an empty cluster, got %v", backupPaths)
}
}
// pmgBackupRetentionFixture drives the real PMG poll consumer with a cluster
// listing and per-node backup answers chosen by each test.
type pmgBackupRetentionFixture struct {
*pmgScopeFixture
clusterErr error
nodes []string
backupErr map[string]error
backups map[string][]pmg.BackupEntry
}
func newPMGBackupRetentionFixture() *pmgBackupRetentionFixture {
return &pmgBackupRetentionFixture{
pmgScopeFixture: &pmgScopeFixture{calls: map[string]int{}},
nodes: []string{"one", "two"},
backupErr: map[string]error{},
backups: map[string][]pmg.BackupEntry{
"one": {{Filename: "one.tgz", Timestamp: 1700000000, Size: 10}},
"two": {{Filename: "two.tgz", Timestamp: 1700000100, Size: 20}},
},
}
}
func (f *pmgBackupRetentionFixture) GetClusterStatus(context.Context, bool) ([]pmg.ClusterStatusEntry, error) {
f.calls["cluster"]++
if f.clusterErr != nil {
return nil, f.clusterErr
}
entries := make([]pmg.ClusterStatusEntry, 0, len(f.nodes))
for _, node := range f.nodes {
entries = append(entries, pmg.ClusterStatusEntry{Name: node, Type: "node"})
}
return entries, nil
}
func (f *pmgBackupRetentionFixture) ListBackups(_ context.Context, node string) ([]pmg.BackupEntry, error) {
f.calls["backup/"+node]++
if err := f.backupErr[node]; err != nil {
return nil, err
}
return f.backups[node], nil
}
func pmgBackupFilesByNode(t *testing.T, m *Monitor) map[string][]string {
t.Helper()
out := map[string][]string{}
for _, backup := range m.state.GetSnapshot().PMGBackups {
if backup.Instance != "gateway" {
t.Fatalf("unexpected instance on PMG backup: %+v", backup)
}
out[backup.Node] = append(out[backup.Node], backup.Filename)
}
for node := range out {
sort.Strings(out[node])
}
return out
}
func pollPMGBackupRetention(t *testing.T, m *Monitor, f *pmgBackupRetentionFixture) {
t.Helper()
m.pollPMGInstance(context.Background(), "gateway", f)
}
func assertPMGReadFailed(t *testing.T, m *Monitor) {
t.Helper()
status := m.pollStatusMap["pmg::gateway"]
if status == nil || status.LastErrorMessage == "" || status.LastErrorCategory != pollReadErrorCategory {
t.Fatalf("failed backup enumeration not recorded as a failed read: %+v", status)
}
}
// A failed cluster read cannot enumerate backups, so it must not publish an
// empty list over the backups the last good poll found.
func TestPMGBackupsSurviveFailedClusterRead(t *testing.T) {
m := pmgScopeMonitor(config.PMGInstance{MonitoringConfigured: true})
f := newPMGBackupRetentionFixture()
pollPMGBackupRetention(t, m, f)
want := map[string][]string{"one": {"one.tgz"}, "two": {"two.tgz"}}
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("first poll backups=%v want=%v", got, want)
}
f.clusterErr = errors.New("cluster status unavailable")
f.calls = map[string]int{}
pollPMGBackupRetention(t, m, f)
if f.calls["backup/one"] != 0 || f.calls["backup/two"] != 0 {
t.Fatalf("backups requested without a node list: %v", f.calls)
}
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("failed cluster read replaced backups: got=%v want=%v", got, want)
}
assertPMGReadFailed(t, m)
}
// When every per-node backup read fails, nothing was observed this poll and
// the last-known list stays.
func TestPMGBackupsSurviveEveryNodeBackupReadFailing(t *testing.T) {
m := pmgScopeMonitor(config.PMGInstance{MonitoringConfigured: true})
f := newPMGBackupRetentionFixture()
pollPMGBackupRetention(t, m, f)
f.backupErr["one"] = errors.New("backup list timed out")
f.backupErr["two"] = errors.New("backup list timed out")
pollPMGBackupRetention(t, m, f)
want := map[string][]string{"one": {"one.tgz"}, "two": {"two.tgz"}}
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("failed backup reads replaced backups: got=%v want=%v", got, want)
}
assertPMGReadFailed(t, m)
}
// A failed read on one node keeps that node's last-known backups while the
// nodes that answered publish what they returned.
func TestPMGBackupsKeepFailedNodeOnPartialRead(t *testing.T) {
m := pmgScopeMonitor(config.PMGInstance{MonitoringConfigured: true})
f := newPMGBackupRetentionFixture()
pollPMGBackupRetention(t, m, f)
f.backupErr["one"] = errors.New("backup list timed out")
f.backups["two"] = []pmg.BackupEntry{{Filename: "two-new.tgz", Timestamp: 1700000200, Size: 30}}
pollPMGBackupRetention(t, m, f)
want := map[string][]string{"one": {"one.tgz"}, "two": {"two-new.tgz"}}
if got := pmgBackupFilesByNode(t, m); !reflect.DeepEqual(got, want) {
t.Fatalf("partial read backups=%v want=%v", got, want)
}
assertPMGReadFailed(t, m)
}
// A complete enumeration is authoritative: nodes that answered with no
// backups, and nodes no longer in the cluster, clear their entries.
func TestPMGBackupsClearAfterCompleteEnumeration(t *testing.T) {
m := pmgScopeMonitor(config.PMGInstance{MonitoringConfigured: true})
f := newPMGBackupRetentionFixture()
pollPMGBackupRetention(t, m, f)
f.nodes = []string{"one"}
f.backups["one"] = nil
pollPMGBackupRetention(t, m, f)
if got := pmgBackupFilesByNode(t, m); len(got) != 0 {
t.Fatalf("complete enumeration with no backups left entries: %v", got)
}
if status := m.pollStatusMap["pmg::gateway"]; status == nil || status.ConsecutiveFailures != 0 {
t.Fatalf("complete enumeration recorded as failed: %+v", status)
}
}