Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.5 KB
Raw Blame
package monitoring

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
)

// This is a local HTTP transport receipt, not proof of an installed provider
// delivering to a person's device. Use real monitor callbacks and notification
// rendering rather than invoking the transport directly.
func TestPBSMetricObservationGapNotificationReceipts(t *testing.T) {
	for _, restart := range []bool{false, true} {
		name := "uninterrupted"
		if restart {
			name = "restart"
		}
		t.Run(name, func(t *testing.T) {
			testPBSMetricObservationGapNotificationReceipts(t, restart)
		})
	}
}

func testPBSMetricObservationGapNotificationReceipts(t *testing.T, restart bool) {
	receipts := make(chan []byte, 16)
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		body, err := io.ReadAll(r.Body)
		if err != nil {
			t.Errorf("read webhook: %v", err)
		}
		receipts <- body
		w.WriteHeader(http.StatusOK)
	}))
	t.Cleanup(server.Close)
	// Recreate both managers on the same disk state; this remains an in-process
	// lifecycle test, not installed binary or external provider qualification.
	alertDir, notificationDir := t.TempDir(), t.TempDir()
	var manager *alerts.Manager
	var notifier *notifications.NotificationManager
	start := func() {
		notifier = notifications.NewNotificationManagerWithDataDir("http://pulse.example", notificationDir)
		t.Cleanup(notifier.Stop)
		if err := notifier.UpdateAllowedPrivateCIDRs("127.0.0.1/32,::1/128"); err != nil {
			t.Fatal(err)
		}
		notifier.AddWebhook(notifications.WebhookConfig{ID: "pbs-receipt", Name: "pbs-receipt", URL: server.URL, Enabled: true, Service: "generic"})
		notifier.SetNotifyOnResolve(true)
		notifier.SetGroupingWindow(0)

		manager = alerts.NewManagerWithDataDir(alertDir)
		t.Cleanup(manager.Stop)
		cfg := manager.GetConfig()
		cfg.Enabled = true
		cfg.ActivationState = alerts.ActivationActive
		cfg.Schedule.QuietHours.Enabled = false
		cfg.TimeThresholds["pbs"] = 0
		cfg.PBSDefaults.CPU = &alerts.HysteresisThreshold{Trigger: 80, Clear: 75}
		manager.UpdateConfig(cfg)
		monitor := &Monitor{alertManager: manager, notificationMgr: notifier}
		manager.SetAlertCallback(monitor.handleAlertFired)
		manager.SetResolvedCallback(monitor.handleAlertResolved)
	}
	start()

	receive := func() []byte {
		t.Helper()
		select {
		case body := <-receipts:
			return body
		case <-time.After(12 * time.Second):
			// This asserts delivery, not latency. The autonomous queue polls
			// every five seconds and the resolution callback is asynchronous;
			// a five-second deadline races a legitimate post-restart delivery.
			// Allow the same two-poll bound as the queue's recovery contracts,
			// without forcing its worker or skipping the real HTTP receipt.
			t.Fatal("no HTTP webhook receipt")
			return nil
		}
	}
	quiet := func() {
		t.Helper()
		select {
		case body := <-receipts:
			t.Fatalf("unexpected webhook: %s", body)
		case <-time.After(200 * time.Millisecond):
		}
	}
	firing := func() string {
		t.Helper()
		var payload struct {
			Grouped bool `json:"grouped"`
			Alerts  []struct {
				ID   string `json:"id"`
				Type string `json:"type"`
			} `json:"alerts"`
		}
		if err := json.Unmarshal(receive(), &payload); err != nil {
			t.Fatal(err)
		}
		if !payload.Grouped || len(payload.Alerts) != 1 || payload.Alerts[0].Type != "cpu" {
			t.Fatalf("want one CPU firing, got %+v", payload)
		}
		active := manager.GetActiveAlerts()
		if len(active) != 1 || active[0].ID != payload.Alerts[0].ID {
			t.Fatalf("receipt does not match active alert: %+v", active)
		}
		return payload.Alerts[0].ID
	}

	pbs := models.PBSInstance{ID: "pbs-receipt", Name: "backup", Host: "pbs.example.invalid", Status: "online", ConnectionHealth: "healthy", CPU: 99}
	manager.CheckPBS(pbs)
	id := firing()

	// A failed metrics endpoint supplies zero values but is not recovery.
	pbs.CPU = 0
	pbs.NodeMetricsUnavailable = true
	for range 3 {
		manager.CheckPBS(pbs)
	}
	quiet()
	if active := manager.GetActiveAlerts(); len(active) != 1 || active[0].ID != id {
		t.Fatalf("observation gap lost the active incident: %+v", active)
	}

	if restart {
		original := manager.GetActiveAlerts()[0]
		manager.Stop()
		notifier.Stop()
		start()
		for range 3 {
			manager.CheckPBS(pbs)
		}
		quiet()
		active := manager.GetActiveAlerts()
		if len(active) != 1 || active[0].ID != id || !active[0].StartTime.Equal(original.StartTime) {
			t.Fatalf("restart changed incident identity: %+v", active)
		}
	}

	pbs.NodeMetricsUnavailable = false
	manager.CheckPBS(pbs)
	var recovery struct {
		Event string `json:"event"`
		ID    string `json:"alertIdentifier"`
	}
	if err := json.Unmarshal(receive(), &recovery); err != nil {
		t.Fatal(err)
	}
	if recovery.Event != "resolved" || recovery.ID != id {
		t.Fatalf("wrong recovery receipt: %+v", recovery)
	}
	if active := manager.GetActiveAlerts(); len(active) != 0 {
		t.Fatalf("healthy sample left active alerts: %+v", active)
	}

	if restart {
		manager.Stop()
		notifier.Stop()
		start()
		if active := manager.GetActiveAlerts(); len(active) != 0 {
			t.Fatalf("restart resurrected resolved incident: %+v", active)
		}
		if history := manager.GetAlertHistory(10); len(history) != 1 {
			t.Fatalf("restart lost resolved history: %+v", history)
		}
		quiet()
	}

	// A distinct breach clears the default two-point minimum-delta spam guard.
	pbs.CPU = 95
	manager.CheckPBS(pbs)
	firing()
	quiet()
}