Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
)
// A due persisted row must receive current saved quiet-hours policy BEFORE the
// autonomous queue is activated, on initial construction and reconstruction.
// This observes local HTTP acceptance only, not an installed destination.
func TestNewRevalidatesPersistedQuietHoursBeforeDelivery(t *testing.T) {
dir := t.TempDir()
t.Setenv("PULSE_DATA_DIR", dir)
var deliveries atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
deliveries.Add(1)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
quiet := alerts.QuietHours{Enabled: true, Start: "00:00", End: "23:59", Timezone: "UTC",
Days: map[string]bool{"sunday": true, "monday": true, "tuesday": true,
"wednesday": true, "thursday": true, "friday": true, "saturday": true}}
persistence := config.NewConfigPersistence(dir)
saved := alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive,
Schedule: alerts.ScheduleConfig{QuietHours: quiet}}
if err := persistence.SaveAlertConfig(saved); err != nil {
t.Fatal(err)
}
hook := notifications.WebhookConfig{ID: "saved-local-ops", Name: "saved-local-ops", Enabled: true, URL: server.URL, Service: "generic"}
if err := persistence.SaveWebhooks([]notifications.WebhookConfig{hook}); err != nil {
t.Fatal(err)
}
configJSON, err := json.Marshal(hook)
if err != nil {
t.Fatal(err)
}
for startup := 0; startup < 2; startup++ {
seed, err := notifications.NewNotificationQueue(dir)
if err != nil {
t.Fatal(err)
}
due := time.Now().Add(-time.Hour)
id := "saved-quiet-first"
if startup == 1 {
id = "saved-quiet-second"
}
if err := seed.Enqueue(¬ifications.QueuedNotification{ID: id, Type: "webhook", Config: configJSON,
Alerts: []*alerts.Alert{{ID: id, Type: "cpu", Level: alerts.AlertLevelWarning, StartTime: due}},
NextRetryAt: &due}); err != nil {
t.Fatal(err)
}
if err := seed.Stop(); err != nil {
t.Fatal(err)
}
m, err := New(&config.Config{DataPath: dir})
if err != nil {
t.Fatal(err)
}
n := m.GetNotificationManager()
if err := n.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
m.Stop()
t.Fatal(err)
}
q := n.GetQueue()
deadline := time.Now().Add(5 * time.Second)
for {
pending, err := q.GetPending(10)
if err != nil {
m.Stop()
t.Fatal(err)
}
if len(pending) == 0 {
break
}
if time.Now().After(deadline) {
m.Stop()
t.Fatal("persisted quiet notification was not revalidated")
}
time.Sleep(10 * time.Millisecond)
}
stats, err := q.GetQueueStats()
if err != nil || stats["pending"] != startup+1 || stats["sent"] != 0 || stats["dlq"] != 0 || deliveries.Load() != 0 {
m.Stop()
t.Fatalf("QUIET_BOOTSTRAP: saved current schedule did not hold every persisted row: stats=%v deliveries=%d err=%v", stats, deliveries.Load(), err)
}
logs, err := q.GetDeliveryLog(time.Now().Add(-time.Hour), 10)
if err != nil || len(logs) != 0 {
m.Stop()
t.Fatalf("quiet bootstrap manufactured provider attempts: %+v, %v", logs, err)
}
m.Stop()
}
}
// Cover the real monitor constructor, not a fixture which reapplies manager
// setters after restart. Saving here uses persistence directly: this is not
// browser/API save acceptance, a process restart, or destination receipt proof.
func TestNewRestoresSavedNotificationChoices(t *testing.T) {
for _, tc := range []struct {
name, target string
resolve, enabled bool
activation alerts.ActivationState
}{
{"webhook_without_recovery", "webhook", false, true, alerts.ActivationActive},
{"apprise_with_recovery", "apprise", true, true, alerts.ActivationActive},
{"email_disabled", "email", true, false, alerts.ActivationActive},
{"all_pending", "all", false, true, alerts.ActivationPending},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
t.Setenv("PULSE_DATA_DIR", dir)
persistence := config.NewConfigPersistence(dir)
saved := alerts.AlertConfig{Enabled: tc.enabled, ActivationState: tc.activation,
Schedule: alerts.ScheduleConfig{InitialNotify: tc.target, NotifyOnResolve: tc.resolve}}
if err := persistence.SaveAlertConfig(saved); err != nil {
t.Fatal(err)
}
webhooks := []notifications.WebhookConfig{{ID: "saved-ops", Name: "saved-ops", URL: "https://example.invalid/alerts", Enabled: true, Service: "generic", MinimumSeverity: "warning", TagFilter: []string{"ops"}, TagMode: "any"}}
if err := persistence.SaveWebhooks(webhooks); err != nil {
t.Fatal(err)
}
// Synthetic credentials only; no notification is sent by this test.
email := notifications.EmailConfig{
Enabled: tc.enabled, Provider: "custom", SMTPHost: "smtp.example.invalid", SMTPPort: 587,
Username: "fixture-user", Password: "synthetic-email-password", From: "[email protected]",
To: []string{"[email protected]", "[email protected]"}, StartTLS: true, RateLimit: 17,
MinimumSeverity: "critical", TagFilter: []string{"ops"}, TagMode: "all",
}
if err := persistence.SaveEmailConfig(email); err != nil {
t.Fatal(err)
}
apprise := notifications.AppriseConfig{
Enabled: tc.enabled, Mode: notifications.AppriseModeHTTP, CLIPath: "apprise",
Targets: []string{"json://example.invalid/alerts"}, TimeoutSeconds: 23,
ServerURL: "https://apprise.example.invalid", ConfigKey: "fixture-config",
APIKey: "synthetic-apprise-key", APIKeyHeader: "X-Fixture-Key", MinimumSeverity: "warning",
}
if err := persistence.SaveAppriseConfig(apprise); err != nil {
t.Fatal(err)
}
for startup := 0; startup < 2; startup++ {
func() {
m, err := New(&config.Config{DataPath: dir})
if err != nil {
t.Fatal(err)
}
defer m.Stop()
n := m.GetNotificationManager()
if got := n.GetInitialNotifyTarget(); got != tc.target {
t.Errorf("startup %d target=%q, want %q", startup, got, tc.target)
}
if got := n.GetNotifyOnResolve(); got != tc.resolve {
t.Errorf("startup %d resolve=%t, want %t", startup, got, tc.resolve)
}
wantEnabled := tc.enabled && tc.activation == alerts.ActivationActive
if got := n.IsEnabled(); got != wantEnabled {
t.Errorf("startup %d enabled=%t, want %t", startup, got, wantEnabled)
}
if got := n.GetEmailConfig(); !reflect.DeepEqual(got, email) {
t.Errorf("startup %d did not restore saved email configuration", startup)
}
if got := n.GetAppriseConfig(); !reflect.DeepEqual(got, apprise) {
t.Errorf("startup %d did not restore saved Apprise configuration", startup)
}
if got := n.GetWebhooks(); !reflect.DeepEqual(got, webhooks) {
t.Errorf("startup %d did not restore saved webhook configuration", startup)
}
}()
}
})
}
}
// Real PBS breach -> recovery -> new breach, with one callback deliberately
// held at its asynchronous consumer boundary. No clocks or manager maps change.
func TestMonitorResolvedSnapshotKeepsRefiringPBSDelivery(t *testing.T) {
endpoint, receipts := occurrenceEndpoint(t)
n := occurrenceNotifier(t, t.TempDir(), endpoint)
a, pbs := occurrenceManager(t)
m := &Monitor{alertManager: a, notificationMgr: n}
m.wireExternalAlertCallbacks(nil)
entered, release := make(chan struct{}), make(chan struct{})
var releaseOnce sync.Once
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
unsubscribe := a.SubscribeResolvedCallback(func(string) { close(entered); <-release })
t.Cleanup(unsubscribe)
a.CheckPBS(pbs)
first := awaitOccurrenceRows(t, n, 1)[0].Alerts[0]
pbs.CPU = 0
a.CheckPBS(pbs)
select {
case <-entered:
case <-time.After(3 * time.Second):
t.Fatal("resolution callback did not start")
}
pbs.CPU = 95
a.CheckPBS(pbs)
awaitOccurrenceRows(t, n, 2)
active := a.GetActiveAlerts()
if len(active) != 1 || active[0].ID != first.ID || active[0].StartTime.Equal(first.StartTime) {
t.Fatalf("not a new recurring PBS occurrence: %+v", active)
}
current := active[0].Clone()
releaseOnce.Do(func() { close(release) })
remaining := awaitOccurrenceRows(t, n, 1)
if !remaining[0].Alerts[0].StartTime.Equal(current.StartTime) {
t.Fatal("delayed resolution kept the wrong occurrence")
}
assertOccurrenceHTTP(t, n, receipts, current)
}
// No router or post-construction setter may be needed to resume accepted work.
// A tenant must use the installation policy, including revocation, rather than
// a policy file placed in its own persistence directory.
func TestNewRestoresSharedWebhookPolicyBeforeQueueActivation(t *testing.T) {
for _, tc := range []struct {
name string
tenant, allowed bool
configuredURL string
urlLocked bool
}{
{name: "default", allowed: true},
{name: "tenant", tenant: true, allowed: true},
{name: "tenant-cannot-grant", tenant: true},
{name: "saved-url-replaces-stale-config", allowed: true, configuredURL: "https://stale-config.example.test"},
{name: "deployment-url-lock", allowed: true, configuredURL: "https://deployment.example.test", urlLocked: true},
} {
t.Run(tc.name, func(t *testing.T) {
root := t.TempDir()
t.Setenv("PULSE_DATA_DIR", root)
received := make(chan string, 4)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Instance string `json:"instance"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
received <- body.Instance
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
global := config.NewConfigPersistence(root)
settings := config.DefaultSystemSettings()
settings.PublicURL = "https://notification-bootstrap.example.test"
if tc.allowed {
settings.WebhookAllowedPrivateCIDRs = "127.0.0.1/32"
}
if err := global.SaveSystemSettings(*settings); err != nil {
t.Fatal(err)
}
dir := root
if tc.tenant {
dir = filepath.Join(root, "orgs", "queue-policy")
}
persistence := config.NewConfigPersistence(dir)
if tc.tenant {
local := config.DefaultSystemSettings()
local.PublicURL = "https://tenant-policy-must-not-win.example.test"
if !tc.allowed {
local.WebhookAllowedPrivateCIDRs = "127.0.0.1/32"
}
if err := persistence.SaveSystemSettings(*local); err != nil {
t.Fatal(err)
}
}
if err := persistence.SaveAlertConfig(alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive}); err != nil {
t.Fatal(err)
}
hook := notifications.WebhookConfig{ID: "startup-policy", Name: "startup-policy", Enabled: true, URL: server.URL, Service: "generic", Template: `{"instance":"{{.Instance | jsonString}}"}`}
if err := persistence.SaveWebhooks([]notifications.WebhookConfig{hook}); err != nil {
t.Fatal(err)
}
configJSON, err := json.Marshal(hook)
if err != nil {
t.Fatal(err)
}
seed, err := notifications.NewNotificationQueue(dir)
if err != nil {
t.Fatal(err)
}
if err := seed.Enqueue(¬ifications.QueuedNotification{ID: "accepted-before-restart", Type: "webhook", Config: configJSON, Alerts: []*alerts.Alert{{ID: "accepted-policy", Type: "cpu", Level: alerts.AlertLevelWarning, StartTime: time.Now().Add(-time.Minute)}}}); err != nil {
t.Fatal(err)
}
if err := seed.Stop(); err != nil {
t.Fatal(err)
}
m, err := New(&config.Config{DataPath: dir, ConfigPath: dir, PublicURL: tc.configuredURL,
EnvOverrides: map[string]bool{"publicURL": tc.urlLocked}})
if err != nil {
t.Fatal(err)
}
defer m.Stop()
q := m.GetNotificationManager().GetQueue()
want := notifications.DeliveryOutcomeDeadLetter
if tc.allowed {
want = notifications.DeliveryOutcomeSent
}
deadline := time.Now().Add(5 * time.Second)
for {
entries, err := q.GetDeliveryLog(time.Now().Add(-time.Hour), 10)
if err != nil {
t.Fatal(err)
}
// A blocked destination can also queue a delivery-health alert.
// Observe the accepted row, not the total number of outcomes.
reached := false
for _, entry := range entries {
if entry.NotificationID == "accepted-before-restart" && entry.Outcome == want {
reached = true
break
}
}
if reached {
break
}
if time.Now().After(deadline) {
t.Fatalf("BOOTSTRAP_POLICY: accepted row did not reach %s before router construction: %+v", want, entries)
}
time.Sleep(10 * time.Millisecond)
}
if tc.allowed {
select {
case got := <-received:
wantURL := settings.PublicURL
if tc.urlLocked {
wantURL = tc.configuredURL
}
if got != wantURL {
t.Fatalf("bootstrap URL=%q, want effective installation URL %q", got, wantURL)
}
case <-time.After(time.Second):
t.Fatal("accepted row has no actual provider receipt")
}
} else if len(received) != 0 {
t.Fatal("tenant policy granted a destination denied by the installation")
}
})
}
}
func TestNewRejectsUnreadableNotificationPolicyBeforeQueueCreation(t *testing.T) {
for _, bad := range []string{"invalid-json", "invalid-cidr"} {
t.Run(bad, func(t *testing.T) {
dir := t.TempDir()
t.Setenv("PULSE_DATA_DIR", dir)
persistence := config.NewConfigPersistence(dir)
if bad == "invalid-json" {
if err := os.WriteFile(filepath.Join(dir, "system.json"), []byte(`{"broken":`), 0600); err != nil {
t.Fatal(err)
}
} else {
settings := config.DefaultSystemSettings()
settings.WebhookAllowedPrivateCIDRs = "not-a-network"
if err := persistence.SaveSystemSettings(*settings); err != nil {
t.Fatal(err)
}
}
m, err := New(&config.Config{DataPath: dir})
if m != nil {
m.Stop()
t.Fatal("monitor admitted runtime before notification policy was readable")
}
if err == nil || !strings.Contains(err.Error(), "notification") {
t.Fatalf("missing notification policy error: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "notification_queue.db")); !os.IsNotExist(err) {
t.Fatalf("policy failure created queue storage: %v", err)
}
})
}
}
// The real monitor stop/reconstruction path must checkpoint a still-buffered
// warning before replacing the notifier, and bootstrap must leave its deadline
// alone. No provider or artificial queue timestamp is involved.
func TestMonitorReloadKeepsBufferedWarning(t *testing.T) {
dir := t.TempDir()
t.Setenv("PULSE_DATA_DIR", dir)
persistence := config.NewConfigPersistence(dir)
saved := alerts.AlertConfig{Enabled: true, ActivationState: alerts.ActivationActive,
Schedule: alerts.ScheduleConfig{InitialNotify: "webhook", NotifyOnResolve: true,
Grouping: alerts.GroupingConfig{Enabled: true, Window: 600, ByNode: true, ByGuest: true}}}
if err := persistence.SaveAlertConfig(saved); err != nil {
t.Fatal(err)
}
if err := persistence.SaveWebhooks([]notifications.WebhookConfig{{ID: "reload-ops", Name: "Reload ops", URL: "https://example.invalid/not-contacted", Service: "generic", Enabled: true}}); err != nil {
t.Fatal(err)
}
m, err := New(&config.Config{DataPath: dir})
if err != nil {
t.Fatal(err)
}
m.GetNotificationManager().SendAlert(&alerts.Alert{ID: "reload-warning", ResourceID: "reload-host", ResourceName: "Reload host", Type: "cpu", Level: alerts.AlertLevelWarning, StartTime: time.Now().UTC()})
m.Stop()
m, err = New(&config.Config{DataPath: dir})
if err != nil {
t.Fatal(err)
}
defer m.Stop()
q := m.GetNotificationManager().GetQueue()
stats, err := q.GetQueueStats()
if err != nil || stats["pending"] != 1 || stats["sent"] != 0 {
t.Fatalf("reload lost or flushed accepted warning: %v, %v", stats, err)
}
ready, err := q.GetPending(10)
if err != nil || len(ready) != 0 {
t.Fatalf("reload shortened original grouping deadline: %+v, %v", ready, err)
}
entries, err := q.GetDeliveryLog(time.Now().Add(-time.Hour), 10)
if err != nil || len(entries) != 0 {
t.Fatalf("reload manufactured provider attempts: %+v, %v", entries, err)
}
}