Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Preserve reviewed maintenance with current host and disk monitoring

pulse-triage[bot]dd4a0a4ad67e
diff --cc internal/monitoring/monitor_mock_alerts_test.go
index a1c37dc3e8,3c5c933339..8110f2ba1d
--- a/internal/monitoring/monitor_mock_alerts_test.go
+++ b/internal/monitoring/monitor_mock_alerts_test.go
@@@ -240,117 -237,126 +240,238 @@@ func TestMockHostAgentPassAfterLeavingM
return
}
}
- t.Fatal("stale mock pass registered fixture hostname pve1 and suppressed the real node's memory alert")
+ t.Fatal("stale mock pass linked a fixture agent to real-pve1 and suppressed its memory alert")
+}
+
+// newMockDockerHostWithExitedContainers returns a reporting Docker host whose
+// containers exited, so two evaluation passes open one alert per container.
+func newMockDockerHostWithExitedContainers(id, hostname string) models.DockerHost {
+ return models.DockerHost{
+ ID: id,
+ Hostname: hostname,
+ DisplayName: hostname,
+ Status: "online",
+ Containers: []models.DockerContainer{
+ {ID: id + "-web", Name: "web", State: "exited", Status: "Exited (137)"},
+ {ID: id + "-db", Name: "db", State: "exited", Status: "Exited (137)"},
+ },
+ }
+}
+
+func dockerAlertIDsForHost(manager *alerts.Manager, hostID string) []string {
+ var ids []string
+ for _, alert := range manager.GetActiveAlerts() {
+ if alert.ResourceID == "docker:"+hostID || strings.HasPrefix(alert.ResourceID, "docker:"+hostID+"/") {
+ ids = append(ids, alert.ID)
+ }
+ }
+ return ids
+}
+
+func TestMockDockerPassAfterLeavingMockModeRaisesNothing(t *testing.T) {
+ manager := newMockHostAlertTestManager(t)
+ monitor := &Monitor{alertManager: manager}
+ mustSetMockEnabled(t, false)
+
+ // A pass whose snapshot predates leaving mock mode reaches the Docker
+ // step after forgetMockFixtureHosts ran.
+ host := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")
+ monitor.evaluateMockDockerHosts([]models.DockerHost{host}, 1)
+ monitor.evaluateMockDockerHosts([]models.DockerHost{host}, 1)
+
+ if ids := dockerAlertIDsForHost(manager, host.ID); len(ids) != 0 {
+ t.Fatalf("stale mock pass opened Docker alerts in live mode: %v", ids)
+ }
+}
+
+func TestLeavingMockModeRemovesFixtureDockerHostAlerts(t *testing.T) {
+ mustSetMockEnabled(t, true)
+ t.Cleanup(func() { mustSetMockEnabled(t, false) })
+ manager := newMockHostAlertTestManager(t)
+ monitor := &Monitor{alertManager: manager}
+
+ host := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")
+ monitor.evaluateMockDockerHosts([]models.DockerHost{host}, 1)
+ monitor.evaluateMockDockerHosts([]models.DockerHost{host}, 1)
+ if ids := dockerAlertIDsForHost(manager, host.ID); len(ids) != len(host.Containers) {
+ t.Fatalf("fixture Docker host opened %v, want one alert per exited container", ids)
+ }
+
+ // The alerts stand in for ones a pass in flight recreated after
+ // SetMockMode(false) cleared the manager.
+ mustSetMockEnabled(t, false)
+ monitor.forgetMockFixtureHosts()
+
+ if ids := dockerAlertIDsForHost(manager, host.ID); len(ids) != 0 {
+ t.Fatalf("fixture Docker host alerts outlived mock mode: %v", ids)
+ }
+}
+
+func TestMockFixturePassFromBeforeAnEstateRebuildIsRejected(t *testing.T) {
+ mustSetMockEnabled(t, true)
+ t.Cleanup(func() { mustSetMockEnabled(t, false) })
+ manager := newMockHostAlertTestManager(t)
+ monitor := &Monitor{alertManager: manager}
+
+ retired := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")
+ current := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:105", "pve1-ct105")
+ monitor.evaluateMockDockerHosts([]models.DockerHost{current}, 2)
+ monitor.evaluateMockDockerHosts([]models.DockerHost{current}, 2)
+
+ // A slow pass whose snapshot predates the rebuild reaches the lock after
+ // the newer passes. Applying it would remove the current host and
+ // re-evaluate the retired one.
+ monitor.evaluateMockDockerHosts([]models.DockerHost{retired}, 1)
+ monitor.evaluateMockDockerHosts([]models.DockerHost{retired}, 1)
+
+ if ids := dockerAlertIDsForHost(manager, current.ID); len(ids) != len(current.Containers) {
+ t.Fatalf("superseded pass disturbed the current host's alerts: %v", ids)
+ }
+ if ids := dockerAlertIDsForHost(manager, retired.ID); len(ids) != 0 {
+ t.Fatalf("superseded pass re-evaluated a retired host: %v", ids)
+ }
+}
+
+func TestMockFixturePassPausedAcrossDisableAndReenableIsRejected(t *testing.T) {
+ mustSetMockEnabled(t, true)
+ t.Cleanup(func() { mustSetMockEnabled(t, false) })
+ manager := newMockHostAlertTestManager(t)
+ monitor := &Monitor{alertManager: manager}
+
+ retired := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")
+ _, staleRevision := mock.CurrentFixtureGraphWithRevision()
+ monitor.evaluateMockDockerHosts([]models.DockerHost{retired}, staleRevision)
+
+ // Mock mode goes off and back on while a pass holding the old snapshot
+ // is paused; it resumes before any pass of the rebuilt estate applies.
+ mustSetMockEnabled(t, false)
+ monitor.forgetMockFixtureHosts()
+ mustSetMockEnabled(t, true)
+ monitor.evaluateMockDockerHosts([]models.DockerHost{retired}, staleRevision)
+ monitor.evaluateMockDockerHosts([]models.DockerHost{retired}, staleRevision)
+
+ if ids := dockerAlertIDsForHost(manager, retired.ID); len(ids) != 0 {
+ t.Fatalf("pass from before the disable re-evaluated the old estate: %v", ids)
+ }
}
+
+ // diskAlertKeys lists the active disk-health and disk-wearout alerts as
+ // "type instance/node device" so assertions name the disk, not the alert ID.
+ func diskAlertKeys(manager *alerts.Manager) map[string]bool {
+ keys := make(map[string]bool)
+ for _, alert := range manager.GetActiveAlerts() {
+ if alert.Type != "disk-health" && alert.Type != "disk-wearout" {
+ continue
+ }
+ devPath, _ := alert.Metadata["disk_path"].(string)
+ keys[alert.Type+" "+alert.Instance+"/"+alert.Node+" "+devPath] = true
+ }
+ return keys
+ }
+
+ func TestCheckMockAlertsEvaluatesPhysicalDisks(t *testing.T) {
+ // Pin the default estate: a PULSE_MOCK_* environment can select one with
+ // no FAILED disk (the public demo's eight nodes have none).
+ previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig()
+ t.Cleanup(func() {
+ mustSetMockEnabled(t, false)
+ mock.SetMockConfig(previousConfig)
+ mustSetMockEnabled(t, previousEnabled)
+ })
+ mustSetMockEnabled(t, false)
+ cfg := mock.DefaultConfig
+ cfg.UpdateInterval = 5 * time.Minute
+ mock.SetMockConfig(cfg)
+ mustSetMockEnabled(t, true)
+ manager := newMockHostAlertTestManager(t)
+
+ monitor := &Monitor{alertManager: manager}
+ monitor.checkMockAlerts()
+
+ // The fixture keeps a FAILED cohort and worn SSDs stable across restarts
+ // so the disk alert lifecycle has evidence to act on. Each must raise the
+ // alert the physical disk poller raises for a live disk, and a disk on a
+ // node the poller would not reach raises nothing.
+ state := mock.CurrentFixtureGraph().State
+ onlineNodes := make(map[string]bool, len(state.Nodes))
+ for _, node := range state.Nodes {
+ onlineNodes[node.Instance+"/"+node.Name] = node.Status == "online"
+ }
+ want := make(map[string]bool)
+ failed, worn := 0, 0
+ for _, disk := range state.PhysicalDisks {
+ if !onlineNodes[disk.Instance+"/"+disk.Node] {
+ continue
+ }
+ if strings.EqualFold(disk.Health, "FAILED") {
+ want["disk-health "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true
+ failed++
+ }
+ if disk.Wearout >= 0 && disk.Wearout < 10 {
+ want["disk-wearout "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true
+ worn++
+ }
+ }
+ if failed == 0 || worn == 0 {
+ t.Fatalf("mock fixture has %d FAILED and %d worn disks on online nodes, want at least one of each", failed, worn)
+ }
+
+ got := diskAlertKeys(manager)
+ for key := range got {
+ if !want[key] {
+ t.Errorf("mock pass raised unexpected disk alert %q", key)
+ }
+ }
+ for key := range want {
+ if !got[key] {
+ t.Errorf("mock pass raised no alert %q", key)
+ }
+ }
+ }
+
+ func TestCheckMockPhysicalDiskAlertsFollowsPollerBoundary(t *testing.T) {
+ manager := newMockHostAlertTestManager(t)
+ monitor := &Monitor{alertManager: manager}
+
+ nodes := []models.Node{
+ {ID: "west-pve1", Name: "pve1", Instance: "west", Status: "online", LinkedAgentID: "host-west-pve1"},
+ {ID: "east-pve1", Name: "pve1", Instance: "east", Status: "offline"},
+ }
+ hosts := []models.Host{{ID: "host-west-pve1", Hostname: "pve1", Status: "online"}}
+ disks := []models.PhysicalDisk{
+ {Instance: "west", Node: "pve1", DevPath: "/dev/sda", Model: "Crucial MX500 2TB", Type: "sata", Health: "FAILED", Wearout: -1},
+ {Instance: "west", Node: "pve1", DevPath: "/dev/nvme0n1", Model: "Samsung 980 PRO 2TB", Type: "nvme", Health: "PASSED", Wearout: 6},
+ // Same node name on another instance: that node is offline, so the
+ // poller would not reach it and its disk must not alert.
+ {Instance: "east", Node: "pve1", DevPath: "/dev/sdb", Model: "WD Red Pro 8TB", Type: "sata", Health: "FAILED", Wearout: -1},
+ }
+
+ monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts)
+ got := diskAlertKeys(manager)
+ if len(got) != 2 || !got["disk-health west/pve1 /dev/sda"] || !got["disk-wearout west/pve1 /dev/nvme0n1"] {
+ t.Fatalf("disk alerts = %v, want only FAILED health on west/pve1 /dev/sda and wearout on west/pve1 /dev/nvme0n1", got)
+ }
+
+ // The linked agent's --disk-exclude forces matched devices healthy, so
+ // excluding a disk resolves the alerts it already raised. Endurance
+ // recovery needs three healthy-looking passes.
+ hosts[0].DiskExclude = []string{"/dev/sda", "nvme0n1"}
+ for i := 0; i < 3; i++ {
+ monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts)
+ }
+ if got := diskAlertKeys(manager); len(got) != 0 {
+ t.Fatalf("excluded devices kept disk alerts %v", got)
+ }
+
+ // A node that leaves the estate takes its disk alerts with it through the
+ // node cleanup the mock pass runs first.
+ hosts[0].DiskExclude = nil
+ monitor.checkMockPhysicalDiskAlerts(disks, nodes, hosts)
+ if got := diskAlertKeys(manager); len(got) != 2 {
+ t.Fatalf("disk alerts after removing the exclusion = %v, want both west/pve1 alerts back", got)
+ }
+ manager.CleanupAlertsForNodes(map[string]bool{"pve2": true})
+ if got := diskAlertKeys(manager); len(got) != 0 {
+ t.Fatalf("disk alerts outlived their departed node: %v", got)
+ }
+ }