Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.5 KB
Raw Blame
package monitoring

import (
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func trueNASMemoryAlertResource(id, supersededID, hostname string, memory float64) unifiedresources.Resource {
	resource := unifiedresources.Resource{
		ID:      id,
		Type:    unifiedresources.ResourceTypeAgent,
		Name:    hostname,
		Sources: []unifiedresources.DataSource{unifiedresources.SourceTrueNAS},
		TrueNAS: &unifiedresources.TrueNASData{Hostname: hostname},
		Metrics: &unifiedresources.ResourceMetrics{
			Memory: &unifiedresources.MetricValue{
				Value:   memory,
				Percent: memory,
				Unit:    "percent",
				Source:  unifiedresources.SourceTrueNAS,
			},
		},
	}
	if supersededID != "" {
		resource.SupersededCanonicalIDs = []string{supersededID}
	}
	return resource
}

func TestHandleAlertEscalatedPreservesLegacyAppriseAndExactWebhookRouting(t *testing.T) {
	t.Run("legacy Apprise", func(t *testing.T) {
		t.Setenv("PULSE_DATA_DIR", t.TempDir())
		requests := make(chan struct{}, 1)
		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			requests <- struct{}{}
			w.WriteHeader(http.StatusOK)
		}))
		defer server.Close()

		notifMgr := notifications.NewNotificationManager("https://pulse.local")
		defer notifMgr.Stop()
		notifMgr.SetGroupingWindow(0)
		if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
			t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
		}
		notifMgr.SetAppriseConfig(notifications.AppriseConfig{
			Enabled: true, Mode: notifications.AppriseModeHTTP, ServerURL: server.URL,
		})

		manager := alerts.NewManager()
		cfg := manager.GetConfig()
		cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{After: 1, Notify: "apprise"}}
		cfg.Schedule.QuietHours.Enabled = false
		manager.UpdateConfig(cfg)

		(&Monitor{notificationMgr: notifMgr, alertManager: manager}).handleAlertEscalated(nil, activeEscalationFixture(t, manager, "connectivity", alerts.AlertLevelCritical), 1)

		select {
		case <-requests:
		case <-time.After(2 * time.Second):
			t.Fatal("expected legacy Apprise escalation delivery")
		}
	})

	t.Run("exact webhook", func(t *testing.T) {
		t.Setenv("PULSE_DATA_DIR", t.TempDir())
		selectedRequests := make(chan struct{}, 1)
		selectedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			selectedRequests <- struct{}{}
			w.WriteHeader(http.StatusOK)
		}))
		defer selectedServer.Close()
		unselectedRequests := make(chan struct{}, 1)
		unselectedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
			unselectedRequests <- struct{}{}
			w.WriteHeader(http.StatusOK)
		}))
		defer unselectedServer.Close()

		notifMgr := notifications.NewNotificationManager("https://pulse.local")
		defer notifMgr.Stop()
		notifMgr.SetGroupingWindow(0)
		if err := notifMgr.UpdateAllowedPrivateCIDRs("127.0.0.1/32"); err != nil {
			t.Fatalf("UpdateAllowedPrivateCIDRs: %v", err)
		}
		notifMgr.AddWebhook(notifications.WebhookConfig{ID: "ops", Name: "ops", URL: unselectedServer.URL, Enabled: true})
		notifMgr.AddWebhook(notifications.WebhookConfig{ID: "pager", Name: "pager", URL: selectedServer.URL, Enabled: true})

		manager := alerts.NewManager()
		cfg := manager.GetConfig()
		cfg.Schedule.Escalation.Levels = []alerts.EscalationLevel{{
			After: 1, Notify: "webhook", DestinationIDs: []string{"webhook:pager"},
		}}
		cfg.Schedule.QuietHours.Enabled = false
		manager.UpdateConfig(cfg)

		(&Monitor{notificationMgr: notifMgr, alertManager: manager}).handleAlertEscalated(nil, activeEscalationFixture(t, manager, "connectivity", alerts.AlertLevelCritical), 1)

		select {
		case <-selectedRequests:
		case <-time.After(2 * time.Second):
			t.Fatal("expected selected webhook escalation delivery")
		}
		select {
		case <-unselectedRequests:
			t.Fatal("exact escalation selection widened to an unselected webhook")
		case <-time.After(300 * time.Millisecond):
		}
	})
}

func activeMemoryAlertForResource(manager *alerts.Manager, resourceID string) (alerts.Alert, bool) {
	for _, alert := range manager.GetActiveAlerts() {
		if alert.ResourceID == resourceID && alert.Type == "memory" {
			return alert, true
		}
	}
	return alerts.Alert{}, false
}

func waitForAlertNotification(t *testing.T, notifications <-chan alerts.Alert, resourceID string, level alerts.AlertLevel) alerts.Alert {
	t.Helper()
	timeout := time.NewTimer(time.Second)
	defer timeout.Stop()
	for {
		select {
		case notification := <-notifications:
			if notification.ResourceID == resourceID && notification.Level == level {
				return notification
			}
		case <-timeout.C:
			t.Fatalf("timed out waiting for %s notification for %s", level, resourceID)
		}
	}
}

func waitForAlertResolution(t *testing.T, resolutions <-chan string, resourceID string) {
	t.Helper()
	timeout := time.NewTimer(time.Second)
	defer timeout.Stop()
	for {
		select {
		case alertID := <-resolutions:
			if strings.Contains(alertID, resourceID) {
				return
			}
		case <-timeout.C:
			t.Fatalf("timed out waiting for resolution callback for %s", resourceID)
		}
	}
}

func TestSyncUnifiedResourceAlertsPersistsAndEvaluatesTrueNASOverrideSuccession(t *testing.T) {
	const (
		oldID       = "agent-535886018cb53055"
		newID       = "agent-b9ed6d0e20e94eaf"
		secondID    = "agent-4266ee45469c27f1"
		sharedName  = "strawberrynas"
		overridePct = 95.0
	)

	dataDir := t.TempDir()
	persistence := config.NewConfigPersistence(dataDir)
	manager := alerts.NewManagerWithDataDir(dataDir)
	t.Cleanup(manager.Stop)

	alertConfig := manager.GetConfig()
	alertConfig.Enabled = true
	alertConfig.ActivationState = alerts.ActivationActive
	alertConfig.TrueNASDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 85, Clear: 80}
	alertConfig.TimeThresholds["truenas-system"] = 0
	alertConfig.Overrides = map[string]alerts.ThresholdConfig{
		oldID: {
			Memory: &alerts.HysteresisThreshold{Trigger: overridePct, Clear: 90},
		},
	}
	manager.UpdateConfig(alertConfig)
	if err := persistence.SaveAlertConfig(manager.GetConfig()); err != nil {
		t.Fatalf("seed legacy alert config: %v", err)
	}

	monitor := &Monitor{
		alertManager:  manager,
		configPersist: persistence,
		state:         models.NewState(),
	}
	notifications := make(chan alerts.Alert, 8)
	resolutions := make(chan string, 4)
	manager.SetAlertCallback(func(alert *alerts.Alert) {
		notifications <- *alert
	})
	manager.SetResolvedCallback(func(alertID string) {
		resolutions <- alertID
	})

	// Two configured TrueNAS systems can legitimately report the same
	// hostname. The first resource owns the migrated 95% threshold; the
	// second remains on the 85% default. Reordered resource snapshots must
	// never transfer the override between their connection-scoped IDs.
	resources := []unifiedresources.Resource{
		trueNASMemoryAlertResource(secondID, "", sharedName, 90),
		trueNASMemoryAlertResource(newID, oldID, sharedName, 90),
	}
	monitor.syncUnifiedResourceAlertsToState(resources, monitor.mockModeFence.begin())

	inMemory := manager.GetConfig()
	if _, exists := inMemory.Overrides[oldID]; exists {
		t.Fatalf("in-memory override remained under superseded identity %s", oldID)
	}
	if override := inMemory.Overrides[newID]; override.Memory == nil || override.Memory.Trigger != 95 {
		t.Fatalf("in-memory override missing under canonical identity %s: %+v", newID, override)
	}
	if _, exists := activeMemoryAlertForResource(manager, newID); exists {
		t.Fatalf("custom TrueNAS threshold fired at the default threshold for %s", newID)
	}
	defaultAlert, exists := activeMemoryAlertForResource(manager, secondID)
	if !exists || defaultAlert.Threshold != 85 {
		t.Fatalf("second TrueNAS system did not retain its independent default threshold: %+v", defaultAlert)
	}
	waitForAlertNotification(t, notifications, secondID, alerts.AlertLevelWarning)

	reloaded, err := config.NewConfigPersistence(dataDir).LoadAlertConfig()
	if err != nil {
		t.Fatalf("reload migrated alert config: %v", err)
	}
	if _, exists := reloaded.Overrides[oldID]; exists {
		t.Fatalf("persisted override remained under superseded identity %s", oldID)
	}
	if override := reloaded.Overrides[newID]; override.Memory == nil || override.Memory.Trigger != 95 {
		t.Fatalf("reloaded override missing under canonical identity %s: %+v", newID, override)
	}
	if got := reloaded.IdentitySchemaVersion; got != alerts.CurrentAlertIdentitySchemaVersion {
		t.Fatalf("persisted alert identity schema version = %d, want %d", got, alerts.CurrentAlertIdentitySchemaVersion)
	}

	resources = []unifiedresources.Resource{
		trueNASMemoryAlertResource(newID, oldID, sharedName, 96),
		trueNASMemoryAlertResource(secondID, "", sharedName, 70),
	}
	monitor.syncUnifiedResourceAlertsToState(resources, monitor.mockModeFence.begin())
	warning := waitForAlertNotification(t, notifications, newID, alerts.AlertLevelWarning)
	waitForAlertResolution(t, resolutions, secondID)
	if warning.Threshold != overridePct {
		t.Fatalf("warning notification threshold = %.1f, want %.1f", warning.Threshold, overridePct)
	}

	resources[0] = trueNASMemoryAlertResource(newID, oldID, sharedName, 99)
	monitor.syncUnifiedResourceAlertsToState(resources, monitor.mockModeFence.begin())
	critical := waitForAlertNotification(t, notifications, newID, alerts.AlertLevelCritical)
	if critical.Threshold != overridePct {
		t.Fatalf("critical notification threshold = %.1f, want %.1f", critical.Threshold, overridePct)
	}
	if active, exists := activeMemoryAlertForResource(manager, newID); !exists || active.Level != alerts.AlertLevelCritical {
		t.Fatalf("expected critical TrueNAS alert at derived 99%% escalation, got %+v", active)
	}

	resources[0] = trueNASMemoryAlertResource(newID, oldID, sharedName, 89)
	monitor.syncUnifiedResourceAlertsToState(resources, monitor.mockModeFence.begin())
	if _, exists := activeMemoryAlertForResource(manager, newID); exists {
		t.Fatalf("TrueNAS alert did not clear below the persisted 90%% recovery threshold")
	}
	waitForAlertResolution(t, resolutions, newID)

	// Model a process restart from alerts.json. No provider-declared legacy ID
	// is required after migration, and a 90% sample must remain below the
	// persisted 95% override instead of reverting to the 85% default.
	restartedManager := alerts.NewManagerWithDataDir(t.TempDir())
	t.Cleanup(restartedManager.Stop)
	restartedManager.UpdateConfig(*reloaded)
	restartedMonitor := &Monitor{
		alertManager:  restartedManager,
		configPersist: persistence,
		state:         models.NewState(),
	}
	restartedMonitor.syncUnifiedResourceAlertsToState([]unifiedresources.Resource{
		trueNASMemoryAlertResource(newID, "", sharedName, 90),
		trueNASMemoryAlertResource(secondID, "", sharedName, 70),
	}, restartedMonitor.mockModeFence.begin())
	if _, exists := activeMemoryAlertForResource(restartedManager, newID); exists {
		t.Fatalf("restarted manager evaluated %s at the default threshold", newID)
	}
	restartedConfig := restartedManager.GetConfig()
	if override := restartedConfig.Overrides[newID]; override.Memory == nil || override.Memory.Trigger != overridePct || override.Memory.Clear != 90 {
		t.Fatalf("restarted manager lost persisted TrueNAS hysteresis: %+v", override.Memory)
	}
}

func TestMigrateAvailabilityLinkedResources(t *testing.T) {
	t.Parallel()

	guestID := unifiedresources.ProxmoxGuestCanonicalID(unifiedresources.ResourceTypeVM, "delly", 100)
	retiredID := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeVM, unifiedresources.SourceProxmox, "delly:pve1:100")
	resources := []unifiedresources.Resource{
		{
			ID:                     guestID,
			Type:                   unifiedresources.ResourceTypeVM,
			SupersededCanonicalIDs: []string{retiredID},
			Proxmox: &unifiedresources.ProxmoxData{
				Instance: "delly",
				VMID:     100,
				NodeName: "pve2",
			},
		},
		{
			ID:   "agent-1234567890abcdef",
			Type: unifiedresources.ResourceTypeAgent,
		},
	}

	targets := []config.AvailabilityTarget{
		{ID: "t-live", LinkedResourceID: guestID},
		{ID: "t-retired", LinkedResourceID: retiredID},
		{ID: "t-old-node-source", LinkedResourceID: "delly:pve1:100"},
		{ID: "t-unrelated", LinkedResourceID: "agent-1234567890abcdef"},
		{ID: "t-unknown", LinkedResourceID: "delly:pve9:999"},
		{ID: "t-unlinked"},
	}

	migrated, changed := migrateAvailabilityLinkedResources(targets, resources)
	if !changed {
		t.Fatal("expected migration to report changes")
	}

	byID := make(map[string]config.AvailabilityTarget, len(migrated))
	for _, target := range migrated {
		byID[target.ID] = target
	}
	if got := byID["t-live"].LinkedResourceID; got != guestID {
		t.Fatalf("live link rewritten: %q", got)
	}
	if got := byID["t-retired"].LinkedResourceID; got != guestID {
		t.Fatalf("retired canonical link = %q, want %q", got, guestID)
	}
	if got := byID["t-old-node-source"].LinkedResourceID; got != guestID {
		t.Fatalf("old-node source link = %q, want %q", got, guestID)
	}
	if got := byID["t-unrelated"].LinkedResourceID; got != "agent-1234567890abcdef" {
		t.Fatalf("unrelated link rewritten: %q", got)
	}
	if got := byID["t-unknown"].LinkedResourceID; got != "delly:pve9:999" {
		t.Fatalf("unknown guest link rewritten: %q", got)
	}

	// Input slice must stay untouched: the migration returns a copy.
	if targets[1].LinkedResourceID != retiredID {
		t.Fatalf("input slice mutated: %q", targets[1].LinkedResourceID)
	}

	if _, changedAgain := migrateAvailabilityLinkedResources(migrated, resources); changedAgain {
		t.Fatal("second pass must be a no-op")
	}
}

// Two guests claiming the same retired ID (or the same instance+VMID) is
// ambiguous; the link must fail closed rather than re-home to either.
func TestMigrateAvailabilityLinkedResourcesFailsClosedOnAmbiguity(t *testing.T) {
	t.Parallel()

	retiredID := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeVM, unifiedresources.SourceProxmox, "delly:pve1:100")
	resources := []unifiedresources.Resource{
		{
			ID:                     "vm-aaaaaaaaaaaaaaaa",
			Type:                   unifiedresources.ResourceTypeVM,
			SupersededCanonicalIDs: []string{retiredID},
			Proxmox:                &unifiedresources.ProxmoxData{Instance: "delly", VMID: 100},
		},
		{
			ID:                     "vm-bbbbbbbbbbbbbbbb",
			Type:                   unifiedresources.ResourceTypeVM,
			SupersededCanonicalIDs: []string{retiredID},
			Proxmox:                &unifiedresources.ProxmoxData{Instance: "delly", VMID: 100},
		},
	}
	targets := []config.AvailabilityTarget{
		{ID: "t-ambiguous", LinkedResourceID: retiredID},
		{ID: "t-ambiguous-source", LinkedResourceID: "delly:pve1:100"},
	}

	migrated, changed := migrateAvailabilityLinkedResources(targets, resources)
	if changed {
		t.Fatalf("ambiguous claims must not migrate, got %+v", migrated)
	}
}

func TestSyncUnifiedResourceAlertsMigratesDockerContainerOverrideKeys(t *testing.T) {
	dockerApp := func(id, hostID, containerID, name string) unifiedresources.Resource {
		return unifiedresources.Resource{
			ID:   id,
			Type: unifiedresources.ResourceTypeAppContainer,
			Name: name,
			Docker: &unifiedresources.DockerData{
				HostSourceID: hostID,
				ContainerID:  containerID,
			},
		}
	}

	dataDir := t.TempDir()
	persistence := config.NewConfigPersistence(dataDir)
	manager := alerts.NewManagerWithDataDir(dataDir)
	t.Cleanup(manager.Stop)

	alertConfig := manager.GetConfig()
	alertConfig.Enabled = true
	alertConfig.ActivationState = alerts.ActivationActive
	alertConfig.Overrides = map[string]alerts.ThresholdConfig{
		// Container IDs change on every recreate; the sync loop must re-home
		// this live legacy key onto docker:{host}/{name} (#1601).
		"docker:host-1/aaaaaaaaaaaa": {Disabled: true},
		// The v6 UI keyed overrides by the unified hash id; re-homed too.
		"docker:host-1/app-container-0011223344556677": {Disabled: true},
		// Orphan from a past recreate: pruned.
		"docker:host-1/bbbbbbbbbbbb": {Disabled: true},
	}
	manager.UpdateConfig(alertConfig)
	if err := persistence.SaveAlertConfig(manager.GetConfig()); err != nil {
		t.Fatalf("seed legacy alert config: %v", err)
	}

	monitor := &Monitor{
		alertManager:  manager,
		configPersist: persistence,
		state:         models.NewState(),
	}
	monitor.syncUnifiedResourceAlertsToState([]unifiedresources.Resource{
		dockerApp("app-container-1111111111111111", "host-1", "aaaaaaaaaaaa", "media-server"),
		dockerApp("app-container-0011223344556677", "host-1", "cccccccccccc", "proxy"),
	}, monitor.mockModeFence.begin())

	inMemory := manager.GetConfig()
	for _, gone := range []string{
		"docker:host-1/aaaaaaaaaaaa",
		"docker:host-1/app-container-0011223344556677",
		"docker:host-1/bbbbbbbbbbbb",
	} {
		if _, exists := inMemory.Overrides[gone]; exists {
			t.Fatalf("in-memory override remained under retired docker key %s", gone)
		}
	}
	if override := inMemory.Overrides["docker:host-1/media-server"]; !override.Disabled {
		t.Fatalf("legacy container-ID override did not re-home onto the name key: %+v", inMemory.Overrides)
	}
	if override := inMemory.Overrides["docker:host-1/proxy"]; !override.Disabled {
		t.Fatalf("unified-hash override did not re-home onto the name key: %+v", inMemory.Overrides)
	}

	reloaded, err := config.NewConfigPersistence(dataDir).LoadAlertConfig()
	if err != nil {
		t.Fatalf("reload migrated alert config: %v", err)
	}
	if _, exists := reloaded.Overrides["docker:host-1/aaaaaaaaaaaa"]; exists {
		t.Fatalf("persisted override remained under the retired container-ID key")
	}
	if override := reloaded.Overrides["docker:host-1/media-server"]; !override.Disabled {
		t.Fatalf("persisted config missing the re-homed name-keyed override: %+v", reloaded.Overrides)
	}
}