Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.9 KB
Raw Blame
package monitoring

import (
	"encoding/json"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func TestInstallOperatorIntentResolverReconcilesRestoredAlerts(t *testing.T) {
	now := time.Now().UTC()
	start, end := now.Add(-time.Hour), now.Add(time.Hour)
	for name, policy := range map[string]unifiedresources.ResourceOperatorState{
		"muted":       {MonitoringMode: unifiedresources.MonitoringModeMuted},
		"retired":     {LifecycleState: unifiedresources.LifecycleStateRetired},
		"maintenance": {MaintenanceStartAt: &start, MaintenanceEndAt: &end},
	} {
		t.Run(name, func(t *testing.T) {
			dir := t.TempDir()
			fixture := []alerts.Alert{}
			store := unifiedresources.NewMemoryStore()
			registry := unifiedresources.NewRegistry(store)
			for _, id := range []string{"vm:101", "vm:202"} {
				registry.IngestResources([]unifiedresources.Resource{{ID: id, Type: unifiedresources.ResourceTypeVM}})
				fixture = append(fixture, alerts.Alert{
					ID: id + "::metric-threshold:cpu", ResourceID: id, ResourceName: id,
					Type: "cpu", Level: alerts.AlertLevelWarning, StartTime: start, LastSeen: now,
				})
			}
			policy.CanonicalID = "vm:101"
			if err := store.SetResourceOperatorState(policy); err != nil {
				t.Fatal(err)
			}
			payload, err := json.Marshal(fixture)
			if err != nil {
				t.Fatal(err)
			}
			if err := os.MkdirAll(filepath.Join(dir, "alerts"), 0o700); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(dir, "alerts", "active-alerts.json"), payload, 0o600); err != nil {
				t.Fatal(err)
			}
			// Runtime startup restores durable alerts before the resource store
			// and its persisted operator policies are attached to the monitor.
			manager := alerts.NewManagerWithDataDir(dir, alerts.WithDurableAlertStore())
			t.Cleanup(manager.Stop)
			if got := len(manager.GetActiveAlerts()); got != 2 {
				t.Fatalf("restored alerts before policy attachment = %d, want 2", got)
			}
			monitor := &Monitor{alertManager: manager, state: models.NewState()}
			monitor.state.UpdateActiveAlerts(monitor.activeAlertsSnapshot())
			monitor.installOperatorIntentResolver(unifiedresources.NewMonitorAdapter(registry))
			active := manager.GetActiveAlerts()
			if len(active) != 1 || active[0].ResourceID != "vm:202" {
				t.Fatalf("policy attachment retained suppressed alerts or cleared the control: %+v", active)
			}
			if got := monitor.state.GetSnapshot().ActiveAlerts; len(got) != 1 || got[0].ResourceID != "vm:202" {
				t.Fatalf("shared state retained a suppressed restored alert: %+v", got)
			}
			manager.Stop()
			restarted := alerts.NewManagerWithDataDir(dir, alerts.WithDurableAlertStore())
			t.Cleanup(restarted.Stop)
			active = restarted.GetActiveAlerts()
			if len(active) != 1 || active[0].ResourceID != "vm:202" {
				t.Fatalf("suppressed alert returned after another restart: %+v", active)
			}
		})
	}
}

func TestResourcePublicationReconcilesRestoredNativeAlertAliases(t *testing.T) {
	dir := t.TempDir()
	store, err := unifiedresources.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	snapshot := models.StateSnapshot{
		Nodes: []models.Node{{ID: "pve-node", Name: "pve1", Instance: "pve", Status: "online", LinkedAgentID: "host-1"}},
		Hosts: []models.Host{{ID: "host-1", Hostname: "pve1", MachineID: "machine-1", LinkedNodeID: "pve-node", Status: "online", LastSeen: time.Now()}},
	}
	beforeRestart := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store))
	beforeRestart.PopulateFromSnapshot(snapshot)
	canonicalID, found := beforeRestart.ResolveCanonicalResourceID("pve-node")
	if !found || canonicalID == "pve-node" {
		t.Fatalf("expected a distinct merged canonical agent ID, got %q, found=%v", canonicalID, found)
	}
	if err := store.SetResourceOperatorState(unifiedresources.ResourceOperatorState{
		CanonicalID: canonicalID, MonitoringMode: unifiedresources.MonitoringModeMuted,
	}); err != nil {
		t.Fatal(err)
	}
	if err := store.Close(); err != nil {
		t.Fatal(err)
	}
	store, err = unifiedresources.NewSQLiteResourceStore(dir, "default")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })

	now := time.Now().UTC()
	var restored []alerts.Alert
	for _, id := range []string{"pve-node", "agent:host-1", "unaffected"} {
		restored = append(restored, alerts.Alert{
			ID: id + "::connectivity", ResourceID: id, ResourceName: id,
			Type: "connectivity", Level: alerts.AlertLevelWarning, StartTime: now.Add(-time.Hour), LastSeen: now,
		})
	}
	payload, err := json.Marshal(restored)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.MkdirAll(filepath.Join(dir, "alerts"), 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(dir, "alerts", "active-alerts.json"), payload, 0o600); err != nil {
		t.Fatal(err)
	}
	manager := alerts.NewManagerWithDataDir(dir, alerts.WithDurableAlertStore())
	t.Cleanup(manager.Stop)
	if got := len(manager.GetActiveAlerts()); got != 3 {
		t.Fatalf("restored alerts = %d, want 3", got)
	}
	monitor := &Monitor{alertManager: manager, state: models.NewState()}
	// Startup can attach the policy store before any resource observation.
	// The persisted canonical policy cannot yet resolve these native aliases.
	monitor.SetResourceStore(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)))
	if got := len(manager.GetActiveAlerts()); got != 3 {
		t.Fatalf("alerts before resource publication = %d, want 3", got)
	}
	monitor.updateResourceStore(snapshot, monitor.mockModeFence.begin())
	active := manager.GetActiveAlerts()
	if len(active) != 1 || active[0].ResourceID != "unaffected" {
		t.Fatalf("resource publication did not reconcile restored aliases: %+v", active)
	}
	if got := monitor.state.GetSnapshot().ActiveAlerts; len(got) != 1 || got[0].ResourceID != "unaffected" {
		t.Fatalf("shared state retained suppressed native aliases: %+v", got)
	}
}