Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"bytes"
"context"
"fmt"
"runtime"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestCheckMockDockerHostAlertsUsesHostLifecycleForOfflineFixtures(t *testing.T) {
manager := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(manager.Stop)
config := manager.GetConfig()
config.Enabled = true
config.ActivationState = alerts.ActivationPending
config.TimeThresholds = map[string]int{}
config.DockerDefaults.StateDisableConnectivity = false
manager.UpdateConfig(config)
host := models.DockerHost{
ID: "branch-edge",
Hostname: "branch-edge-01",
DisplayName: "Branch Edge",
Status: "offline",
Containers: []models.DockerContainer{
{ID: "portal-1", Name: "branch-portal", State: "exited", Status: "Exited (137)"},
{ID: "sync-1", Name: "branch-syncthing", State: "exited", Status: "Exited (137)"},
{ID: "vpn-1", Name: "branch-vpn", State: "exited", Status: "Exited (137)"},
},
}
// Prove the fixture would create the child storm if it were incorrectly
// treated as fresh online telemetry.
manager.CheckDockerHost(host)
manager.CheckDockerHost(host)
if got := manager.GetActiveAlerts(); len(got) != len(host.Containers) {
t.Fatalf("fresh container evaluation created %d alerts, want %d", len(got), len(host.Containers))
}
monitor := &Monitor{alertManager: manager}
monitor.checkMockDockerHostAlerts(host)
monitor.checkMockDockerHostAlerts(host)
monitor.checkMockDockerHostAlerts(host)
active := manager.GetActiveAlerts()
if len(active) != 1 {
t.Fatalf("offline mock host produced %d active alerts, want one host incident: %+v", len(active), active)
}
if active[0].Type != "docker-host-offline" || active[0].ResourceID != "docker:branch-edge" {
t.Fatalf("offline mock host alert = %+v, want canonical host connectivity incident", active[0])
}
}
func newMockHostAlertTestManager(t *testing.T) *alerts.Manager {
t.Helper()
manager := alerts.NewManagerWithDataDir(t.TempDir())
t.Cleanup(manager.Stop)
config := manager.GetConfig()
config.Enabled = true
config.ActivationState = alerts.ActivationPending
config.TimeThresholds = map[string]int{}
config.MetricTimeThresholds = nil
config.BackupDefaults.Enabled = false
config.DockerDefaults.StateDisableConnectivity = false
config.AgentDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}
config.NodeDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}
manager.UpdateConfig(config)
return manager
}
func TestCheckMockAlertsEvaluatesHostAgentsBeforeLinkedNodes(t *testing.T) {
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, false) })
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
monitor.checkMockAlerts()
// Agents reading well above the 1% trigger must alert. A NotReady
// Kubernetes node's agent validly reads 0%, and a later fixture read may
// drift slightly from the pass's own snapshot, so the bound leaves margin.
state := mock.CurrentFixtureGraph().State
onlineHosts := make(map[string]models.Host, len(state.Hosts))
for _, host := range state.Hosts {
if !strings.EqualFold(strings.TrimSpace(host.Status), "offline") && host.Memory.HasKnownUsage() && host.Memory.Usage >= 5 {
onlineHosts[host.ID] = host
}
}
if len(onlineHosts) == 0 {
t.Fatal("mock fixture has no online host agents reading above 5% memory")
}
active := manager.GetActiveAlerts()
agentMemoryAlerts := 0
alertByResourceMetric := make(map[string]bool, len(active))
for _, alert := range active {
alertByResourceMetric[alert.ResourceID+"/"+alert.Type] = true
if alert.Type != "memory" || !strings.HasPrefix(alert.ResourceID, "agent:") {
continue
}
if _, ok := onlineHosts[strings.TrimPrefix(alert.ResourceID, "agent:")]; ok {
agentMemoryAlerts++
}
}
if agentMemoryAlerts != len(onlineHosts) {
t.Fatalf("mock pass opened %d agent memory alerts at a 1%% trigger, want one per online agent above 5%% (%d)", agentMemoryAlerts, len(onlineHosts))
}
// A node whose linked agent reported on the same pass must hand its
// memory alert to the agent on the first tick, as node-link
// deduplication does for live agents.
linkedNodes := 0
for _, node := range state.Nodes {
host, ok := onlineHosts[strings.TrimSpace(node.LinkedAgentID)]
if !ok || strings.TrimSpace(host.LinkedNodeID) != node.ID {
continue
}
linkedNodes++
if alertByResourceMetric[node.ID+"/memory"] {
t.Fatalf("node %s raised its own memory alert although linked agent %s reported on the same pass", node.Name, host.ID)
}
}
if linkedNodes == 0 {
t.Fatal("mock fixture has no online node with a linked agent")
}
}
func TestCheckMockHostAlertsUsesHostLifecycleForOfflineFixtures(t *testing.T) {
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
host := models.Host{
ID: "host-linux-9",
Hostname: "branch-gateway",
DisplayName: "Branch Gateway",
Platform: "linux",
Status: "offline",
Memory: models.Memory{Total: 16 << 30, Used: 15 << 30, Usage: 93.75},
}
// Three offline observations confirm the connectivity incident. CheckHost
// would treat the stale fixture as a fresh report: it marks the host
// online and raises the stale 94% memory reading instead.
monitor.checkMockHostAlerts(host, nil)
monitor.checkMockHostAlerts(host, nil)
monitor.checkMockHostAlerts(host, nil)
active := manager.GetActiveAlerts()
if len(active) != 1 || active[0].Type != alerts.HostOfflineAlertType || active[0].ResourceID != "agent:host-linux-9" {
got := make([]string, 0, len(active))
for _, alert := range active {
got = append(got, alert.ResourceID+" "+alert.Type)
}
t.Fatalf("offline mock agent alerts = %v, want only the host-offline incident for agent:host-linux-9", got)
}
host.Status = "online"
monitor.checkMockHostAlerts(host, nil)
for _, alert := range manager.GetActiveAlerts() {
if alert.Type == alerts.HostOfflineAlertType {
t.Fatalf("recovered mock agent kept its offline incident %s", alert.ID)
}
}
}
func TestLeavingMockModeReleasesFixtureAgentNodeLinks(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
mustSetMonitorMockMode(t, monitor, true)
t.Cleanup(func() { mustSetMockEnabled(t, false) })
monitor.checkMockAlerts()
linkedNodeID := ""
for _, host := range mock.CurrentFixtureGraph().State.Hosts {
if host.LinkedNodeID != "" && !strings.EqualFold(host.Status, "offline") {
linkedNodeID = host.LinkedNodeID
break
}
}
if linkedNodeID == "" {
t.Fatal("mock fixture has no online agent linked to a node")
}
mustSetMonitorMockMode(t, monitor, false)
// A node carrying the linked fixture node's ID, with no agent of its own,
// must own its metric alerts once the monitor shows live data again.
node := models.Node{
ID: linkedNodeID,
Name: "pve1",
Instance: "real",
Status: "online",
Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75},
}
manager.CheckNode(node)
for _, alert := range manager.GetActiveAlerts() {
if alert.ResourceID == node.ID && alert.Type == "memory" {
return
}
}
t.Fatalf("node %s raised no memory alert: a fixture agent's node link outlived mock mode", node.ID)
}
func TestMockHostAgentPassAfterLeavingMockModeRegistersNothing(t *testing.T) {
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
// A pass whose snapshot predates leaving mock mode reaches the agent step
// after SetMockMode ended its epoch and forgetMockFixtureHosts ran.
scope := monitor.mockModeFence.begin()
monitor.mockModeFence.advance()
stale := models.Host{
ID: "host-node-mock-cluster-1-pve1",
Hostname: "pve1",
LinkedNodeID: "real-pve1",
Status: "online",
Memory: models.Memory{Total: 32 << 30, Used: 30 << 30, Free: 2 << 30, Usage: 93.75},
}
monitor.evaluateMockHostAgents(scope, []models.Host{stale}, nil, 1)
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("stale mock pass opened %d alerts in live mode", len(active))
}
if len(monitor.mockHostAgents) != 0 {
t.Fatalf("stale mock pass registered fixture agents %v after they were forgotten", monitor.mockHostAgents)
}
node := models.Node{
ID: "real-pve1",
Name: "pve1",
Instance: "real",
Status: "online",
Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75},
}
manager.CheckNode(node)
for _, alert := range manager.GetActiveAlerts() {
if alert.ResourceID == node.ID && alert.Type == "memory" {
return
}
}
t.Fatal("stale mock pass linked a fixture agent to real-pve1 and suppressed its memory alert")
}
// newMockDockerHostWithExitedContainers returns a reporting Docker host whose
// containers exited, so two evaluation passes open one alert per container.
func newMockDockerHostWithExitedContainers(id, hostname string) models.DockerHost {
return models.DockerHost{
ID: id,
Hostname: hostname,
DisplayName: hostname,
Status: "online",
Containers: []models.DockerContainer{
{ID: id + "-web", Name: "web", State: "exited", Status: "Exited (137)"},
{ID: id + "-db", Name: "db", State: "exited", Status: "Exited (137)"},
},
}
}
func dockerAlertIDsForHost(manager *alerts.Manager, hostID string) []string {
var ids []string
for _, alert := range manager.GetActiveAlerts() {
if alert.ResourceID == "docker:"+hostID || strings.HasPrefix(alert.ResourceID, "docker:"+hostID+"/") {
ids = append(ids, alert.ID)
}
}
return ids
}
func TestMockDockerPassAfterLeavingMockModeRaisesNothing(t *testing.T) {
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
mustSetMockEnabled(t, false)
// A pass whose snapshot predates leaving mock mode reaches the Docker
// step after forgetMockFixtureHosts ran.
host := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{host}, 1)
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{host}, 1)
if ids := dockerAlertIDsForHost(manager, host.ID); len(ids) != 0 {
t.Fatalf("stale mock pass opened Docker alerts in live mode: %v", ids)
}
}
func TestLeavingMockModeRemovesFixtureDockerHostAlerts(t *testing.T) {
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, false) })
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
host := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{host}, 1)
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{host}, 1)
if ids := dockerAlertIDsForHost(manager, host.ID); len(ids) != len(host.Containers) {
t.Fatalf("fixture Docker host opened %v, want one alert per exited container", ids)
}
// The alerts stand in for ones a pass in flight recreated after
// SetMockMode(false) cleared the manager.
mustSetMockEnabled(t, false)
monitor.forgetMockFixtureHosts()
if ids := dockerAlertIDsForHost(manager, host.ID); len(ids) != 0 {
t.Fatalf("fixture Docker host alerts outlived mock mode: %v", ids)
}
}
// Upstream mode epochs and the reviewed structural revisions protect different
// boundaries. A current revision cannot make an ended epoch eligible again.
func TestMockFixtureCurrentRevisionCannotReviveAnEndedEpoch(t *testing.T) {
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, false) })
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
stale := monitor.mockModeFence.begin()
monitor.mockModeFence.advance()
_, revision := mock.CurrentFixtureGraphWithRevision()
agent := models.Host{ID: "epoch-agent", Hostname: "epoch-agent", Status: "online"}
docker := newMockDockerHostWithExitedContainers("epoch-docker", "epoch-docker")
monitor.evaluateMockHostAgents(stale, []models.Host{agent}, nil, revision)
for range 2 {
monitor.evaluateMockDockerHosts(stale, []models.DockerHost{docker}, revision)
}
if len(monitor.mockHostAgents) != 0 || len(monitor.mockDockerHosts) != 0 || len(manager.GetActiveAlerts()) != 0 {
t.Fatal("a current fixture revision revived an ended mode epoch")
}
current := monitor.mockModeFence.begin()
monitor.evaluateMockHostAgents(current, []models.Host{agent}, nil, revision)
for range 2 {
monitor.evaluateMockDockerHosts(current, []models.DockerHost{docker}, revision)
}
if len(monitor.mockHostAgents) != 1 || len(monitor.mockDockerHosts) != 1 || len(dockerAlertIDsForHost(manager, docker.ID)) != len(docker.Containers) {
t.Fatal("the current epoch lost normal fixture ownership or Docker evaluation")
}
}
func TestMockFixturePassFromBeforeAnEstateRebuildIsRejected(t *testing.T) {
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, false) })
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
retired := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")
current := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:105", "pve1-ct105")
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{current}, 2)
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{current}, 2)
// A slow pass whose snapshot predates the rebuild reaches the lock after
// the newer passes. Applying it would remove the current host and
// re-evaluate the retired one.
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{retired}, 1)
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{retired}, 1)
if ids := dockerAlertIDsForHost(manager, current.ID); len(ids) != len(current.Containers) {
t.Fatalf("superseded pass disturbed the current host's alerts: %v", ids)
}
if ids := dockerAlertIDsForHost(manager, retired.ID); len(ids) != 0 {
t.Fatalf("superseded pass re-evaluated a retired host: %v", ids)
}
}
func TestMockFixturePassPausedAcrossDisableAndReenableIsRejected(t *testing.T) {
mustSetMockEnabled(t, true)
t.Cleanup(func() { mustSetMockEnabled(t, false) })
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
retired := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")
_, staleRevision := mock.CurrentFixtureGraphWithRevision()
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{retired}, staleRevision)
// Mock mode goes off and back on while a pass holding the old snapshot
// is paused; it resumes before any pass of the rebuilt estate applies.
mustSetMockEnabled(t, false)
monitor.forgetMockFixtureHosts()
mustSetMockEnabled(t, true)
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{retired}, staleRevision)
monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{retired}, staleRevision)
if ids := dockerAlertIDsForHost(manager, retired.ID); len(ids) != 0 {
t.Fatalf("pass from before the disable re-evaluated the old estate: %v", ids)
}
}
// pinDefaultMockEstate enables mock mode on the default estate, whatever
// PULSE_MOCK_* the shell exports, and restores the previous mode and config.
func pinDefaultMockEstate(t *testing.T) {
t.Helper()
previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig()
t.Cleanup(func() {
mustSetMockEnabled(t, false)
mock.SetMockConfig(previousConfig)
mustSetMockEnabled(t, previousEnabled)
})
mustSetMockEnabled(t, false)
cfg := mock.DefaultConfig
cfg.UpdateInterval = 5 * time.Minute
mock.SetMockConfig(cfg)
}
// signalFirstFiring closes the returned channel when manager fires its first
// alert. Lifecycle callbacks run inside the evaluation that fired, so the
// pass that fired is past its fixture read when the channel closes.
func signalFirstFiring(t *testing.T, manager *alerts.Manager) <-chan struct{} {
t.Helper()
fired := make(chan struct{})
var once sync.Once
unsubscribe := manager.SubscribeLifecycleCallback(func(event alerts.LifecycleEvent) {
if event.Type == eventlog.TypeFired {
once.Do(func() { close(fired) })
}
})
t.Cleanup(unsubscribe)
return fired
}
func describeAlerts(active []alerts.Alert) string {
keys := make([]string, 0, len(active))
for _, alert := range active {
keys = append(keys, alert.Type+" "+alert.ResourceID)
}
sort.Strings(keys)
return strings.Join(keys, ", ")
}
// switchMockModeAsync runs SetMockMode on its own goroutine, so a test that is
// holding an evaluation parked can still bound how long the switch takes.
func switchMockModeAsync(t *testing.T, monitor *Monitor, enable bool) <-chan struct{} {
t.Helper()
done := make(chan struct{})
go func() {
defer close(done)
if err := monitor.SetMockMode(enable); err != nil {
t.Errorf("SetMockMode(%v): %v", enable, err)
}
}()
return done
}
// goroutineIn reports whether one goroutine's stack holds every frame.
func goroutineIn(frames ...string) bool {
buf := make([]byte, 4<<20)
n := runtime.Stack(buf, true)
for _, goroutine := range bytes.Split(buf[:n], []byte("\n\n")) {
all := true
for _, frame := range frames {
all = all && bytes.Contains(goroutine, []byte(frame))
}
if all {
return true
}
}
return false
}
// waitForGoroutineIn waits until one goroutine's stack holds every frame. A
// test holding a lock uses it to see the goroutine it started parked on that
// lock at a known point.
func waitForGoroutineIn(t *testing.T, frames ...string) {
t.Helper()
waitForCondition(t, 30*time.Second, func() bool { return goroutineIn(frames...) },
fmt.Sprintf("no goroutine reached %s", strings.Join(frames, " via ")))
}
func TestLeavingMockModeRefusesTheRestOfAnInFlightPass(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
mustSetMonitorMockMode(t, monitor, true)
// Park a real pass at its first storage capacity read: it has read the
// fixture and evaluated guests, agents and nodes; storage, disks, PBS,
// PMG and Docker are still ahead of it.
monitor.metricsHistory.capacityForecastMu.Lock()
parked := true
defer func() {
if parked {
monitor.metricsHistory.capacityForecastMu.Unlock()
}
}()
passDone := make(chan struct{})
go func() {
defer close(passDone)
monitor.checkMockAlerts()
}()
waitForGoroutineIn(t, "monitoring.(*Monitor).storageCapacityTrendFor", "monitoring.(*Monitor).checkMockAlerts")
waitForSignal(t, switchMockModeAsync(t, monitor, false), 30*time.Second, "leaving mock mode waited on a pass parked outside any evaluation")
monitor.metricsHistory.capacityForecastMu.Unlock()
parked = false
waitForSignal(t, passDone, 30*time.Second, "mock pass did not finish after it was released")
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("a mock pass in flight when mock mode ended left %d fixture alerts in live mode: %s",
len(active), describeAlerts(active))
}
}
func TestLeavingMockModeWaitsForAnInFlightEvaluation(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
mustSetMonitorMockMode(t, monitor, true)
// Park a real pass inside its first guest evaluation, in the CPU window
// read the alert manager makes outside its own lock, so nothing but the
// fence holds the switch back.
entered, release := make(chan struct{}), make(chan struct{})
var once sync.Once
manager.SetMetricWindowProvider(func(alerts.MetricWindowRequest) ([]alerts.MetricWindowPoint, error) {
once.Do(func() {
close(entered)
<-release
})
return nil, nil
})
released := false
defer func() {
if !released {
close(release)
}
}()
passDone := make(chan struct{})
go func() {
defer close(passDone)
monitor.checkMockAlerts()
}()
waitForSignal(t, entered, 30*time.Second, "mock pass never evaluated a windowed metric")
// The switch must stop in advance until the evaluation returns. One
// that finishes first cleared alerts the evaluation can still raise.
leftMockMode := switchMockModeAsync(t, monitor, false)
waitForCondition(t, 30*time.Second, func() bool {
select {
case <-leftMockMode:
return true
default:
}
return goroutineIn("monitoring.(*mockModeFence).advance", "monitoring.(*Monitor).SetMockMode")
}, "leaving mock mode neither finished nor waited for the running evaluation")
select {
case <-leftMockMode:
t.Fatal("leaving mock mode cleared alerts while a fixture evaluation was still running")
default:
}
close(release)
released = true
waitForSignal(t, leftMockMode, 30*time.Second, "leaving mock mode did not finish once the evaluation returned")
waitForSignal(t, passDone, 30*time.Second, "mock pass did not finish after it was released")
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("the evaluation running when mock mode ended left %d fixture alerts in live mode: %s",
len(active), describeAlerts(active))
}
}
// blockingFixtureRecords serves one source's fixture records the way the
// router's mock adapter does, reading them on each call, and holds its first
// call until released. Later calls are not held, not even while the first is.
type blockingFixtureRecords struct {
source unifiedresources.DataSource
entered chan struct{}
release chan struct{}
held atomic.Bool
}
func newBlockingFixtureRecords(source unifiedresources.DataSource) *blockingFixtureRecords {
return &blockingFixtureRecords{source: source, entered: make(chan struct{}), release: make(chan struct{})}
}
func (p *blockingFixtureRecords) SupplementalRecords(_ *Monitor, _ string) []unifiedresources.IngestRecord {
records := mock.SupplementalRecords(p.source)
if p.held.CompareAndSwap(false, true) {
close(p.entered)
<-p.release
}
return records
}
func TestLeavingMockModeDiscardsAnInFlightFrontendRefresh(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
provider := newBlockingFixtureRecords(unifiedresources.SourceVMware)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
resourceStore: unifiedresources.NewMonitorAdapter(nil),
supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
unifiedresources.SourceVMware: provider,
},
}
mustSetMonitorMockMode(t, monitor, true)
// Hold a frontend read in its registry rebuild: it has the fixture
// snapshot and the vSphere fixture records, whose datastore, host and
// network incidents its unified alert sync would raise.
released := false
defer func() {
if !released {
close(provider.release)
}
}()
readDone := make(chan struct{})
go func() {
defer close(readDone)
monitor.BuildFrontendState()
}()
waitForSignal(t, provider.entered, 30*time.Second, "frontend read never reached the supplemental records")
waitForSignal(t, switchMockModeAsync(t, monitor, false), 30*time.Second, "leaving mock mode waited on a registry rebuild")
close(provider.release)
released = true
waitForSignal(t, readDone, 30*time.Second, "frontend read did not finish after it was released")
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("a frontend read in flight when mock mode ended left %d fixture alerts in live mode: %s",
len(active), describeAlerts(active))
}
}
// gatedListStore holds the first registry listing made after open is called,
// so a test can publish another generation between a refresh's own publish
// and its read back.
type gatedListStore struct {
*unifiedresources.MonitorAdapter
mu sync.Mutex
armed bool
entered chan struct{}
release chan struct{}
}
func (s *gatedListStore) arm() {
s.mu.Lock()
defer s.mu.Unlock()
s.armed = true
}
func (s *gatedListStore) GetAll() []unifiedresources.Resource {
s.mu.Lock()
hold := s.armed
s.armed = false
s.mu.Unlock()
if hold {
close(s.entered)
<-s.release
}
return s.MonitorAdapter.GetAll()
}
// fixedRecords stands in for a live provider: it serves the same records in
// either mode.
type fixedRecords []unifiedresources.IngestRecord
func (r fixedRecords) SupplementalRecords(_ *Monitor, _ string) []unifiedresources.IngestRecord {
return r
}
func alertTypes(active []alerts.Alert) map[string]int {
types := make(map[string]int)
for _, alert := range active {
types[alert.Type]++
}
return types
}
func TestLiveRefreshRejectsCollectedRecordsFromMockMode(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
mustSetMockEnabled(t, true)
// TrueNAS records stand in for a live provider's data; the vSphere
// fixture source serves nothing once mock mode is off.
liveTrueNAS := fixedRecords(mock.SupplementalRecords(unifiedresources.SourceTrueNAS))
mustSetMockEnabled(t, false)
if len(liveTrueNAS) == 0 {
t.Fatal("mock fixture has no TrueNAS records")
}
provider := newBlockingFixtureRecords(unifiedresources.SourceVMware)
store := &gatedListStore{
MonitorAdapter: unifiedresources.NewMonitorAdapter(nil),
entered: make(chan struct{}),
release: make(chan struct{}),
}
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
resourceStore: store,
supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
unifiedresources.SourceVMware: provider,
unifiedresources.SourceTrueNAS: liveTrueNAS,
},
}
mustSetMonitorMockMode(t, monitor, true)
releasedProvider, releasedList := false, false
defer func() {
if !releasedProvider {
close(provider.release)
}
if !releasedList {
close(store.release)
}
}()
// A mock-mode refresh holds the fixture snapshot and vSphere records and
// has not published them yet.
staleDone := make(chan struct{})
go func() {
defer close(staleDone)
monitor.updateResourceStore(monitor.currentStateWithScope())
}()
waitForSignal(t, provider.entered, 30*time.Second, "mock-mode refresh never reached the supplemental records")
waitForSignal(t, switchMockModeAsync(t, monitor, false), 30*time.Second, "leaving mock mode waited on a registry rebuild")
// A live refresh publishes the live TrueNAS records alone and is held
// before reading the registry back.
store.arm()
liveDone := make(chan struct{})
go func() {
defer close(liveDone)
monitor.updateResourceStore(monitor.currentStateWithScope())
}()
waitForSignal(t, store.entered, 30*time.Second, "live refresh never read the registry back")
for _, resource := range store.MonitorAdapter.GetAll() {
if slices.Contains(resource.Sources, unifiedresources.SourceVMware) {
t.Fatalf("the live refresh published vSphere fixture resource %s", resource.ID)
}
}
// The old collector finishes, but must be refused before publication.
// Collection no longer owns a publication admission spanning the switch.
close(provider.release)
releasedProvider = true
waitForSignal(t, staleDone, 30*time.Second, "mock-mode refresh did not finish after it was released")
close(store.release)
releasedList = true
waitForSignal(t, liveDone, 30*time.Second, "live refresh did not finish after it was released")
for _, resource := range store.MonitorAdapter.GetAll() {
if slices.Contains(resource.Sources, unifiedresources.SourceVMware) {
t.Fatalf("the ended collector republished vSphere fixture resource %s", resource.ID)
}
}
if !monitor.mockModeFence.registryCurrent() {
t.Fatal("the refused collector disturbed the current live publication")
}
// The admitted live pass still evaluates its real source-native records;
// refusal of old collections must not become blanket alert suppression.
types := alertTypes(manager.GetActiveAlerts())
if types["zfs-pool-state"] == 0 {
t.Fatalf("the undisturbed live refresh raised no alert from the live TrueNAS records: %v", types)
}
for _, fixtureType := range []string{"resource-incident", "storage-incident"} {
if types[fixtureType] != 0 {
t.Fatalf("vSphere fixture incidents reached live mode: %v", types)
}
}
}
func TestDockerPruneIgnoresTheRegistryMockModeLeft(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
resourceStore: unifiedresources.NewMonitorAdapter(nil),
}
mustSetMonitorMockMode(t, monitor, true)
monitor.updateResourceStore(monitor.currentStateWithScope())
mustSetMonitorMockMode(t, monitor, false)
// Before any live rebuild replaces the fixture registry, a live Docker
// host reports and then goes offline.
host := models.DockerHost{ID: "live-docker-1", Hostname: "live-docker-1", DisplayName: "live-docker-1", Status: "offline"}
monitor.state.UpsertDockerHost(host)
for range 3 {
manager.HandleDockerHostOffline(host)
}
if types := alertTypes(manager.GetActiveAlerts()); types["docker-host-offline"] != 1 {
t.Fatalf("live Docker host raised no offline alert: %v", types)
}
monitor.SyncAlertState()
if types := alertTypes(manager.GetActiveAlerts()); types["docker-host-offline"] != 1 {
t.Fatalf("Docker pruning removed a live host's alert against the fixture registry: %v", types)
}
}
func TestPreviousStateCarryIgnoresTheRegistryMockModeLeft(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: newMockHostAlertTestManager(t),
metricsHistory: NewMetricsHistory(10, time.Hour),
resourceStore: unifiedresources.NewMonitorAdapter(nil),
}
mustSetMonitorMockMode(t, monitor, true)
monitor.updateResourceStore(monitor.currentStateWithScope())
fixture := mock.CurrentFixtureGraph().State
if len(fixture.Nodes) == 0 || len(fixture.VMs) == 0 {
t.Fatal("mock fixture has no nodes or VMs")
}
instance := fixture.VMs[0].Instance
mustSetMonitorMockMode(t, monitor, false)
// A live instance named like a fixture one polls before any live rebuild
// replaces the fixture registry. Its previous-state carry, which writes
// back into live state, must not see the fixture nodes and guests.
if nodes := monitor.previousNodesForInstance(instance); len(nodes) != 0 {
t.Fatalf("previous-node carry for %q returned %d fixture nodes after leaving mock mode", instance, len(nodes))
}
if guests := monitor.previousGuestContextForInstance(instance); len(guests.vms) != 0 || len(guests.containers) != 0 {
t.Fatalf("previous-guest carry for %q returned %d fixture VMs and %d containers after leaving mock mode",
instance, len(guests.vms), len(guests.containers))
}
}
func TestLeavingMockModeDiscardsAnInFlightBackupEvaluation(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
config := manager.GetConfig()
config.BackupDefaults.Enabled = true
config.BackupDefaults.WarningDays = 1
config.BackupDefaults.CriticalDays = 2
manager.UpdateConfig(config)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
mustSetMonitorMockMode(t, monitor, true)
// Cache the fixture read view the guest lookup will be served from.
monitor.GetUnifiedReadStateOrSnapshot()
// Park a real evaluation at its guest lookup, after it read the fixture
// recovery rollups whose stale backups raise backup-age alerts.
monitor.mockUnifiedViewMu.Lock()
parked := true
defer func() {
if parked {
monitor.mockUnifiedViewMu.Unlock()
}
}()
evalDone := make(chan struct{})
go func() {
defer close(evalDone)
monitor.checkBackupAlerts(context.Background())
}()
waitForGoroutineIn(t, "monitoring.(*Monitor).currentUnifiedStateView", "monitoring.(*Monitor).checkBackupAlerts")
waitForSignal(t, switchMockModeAsync(t, monitor, false), 30*time.Second, "leaving mock mode waited on a backup evaluation parked outside the alert manager")
monitor.mockUnifiedViewMu.Unlock()
parked = false
waitForSignal(t, evalDone, 30*time.Second, "backup evaluation did not finish after it was released")
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("a backup evaluation in flight when mock mode ended left %d fixture alerts in live mode: %.500s",
len(active), describeAlerts(active))
}
}
func TestLeavingMockModeDiscardsAnInFlightConnectionCheck(t *testing.T) {
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
mustSetMonitorMockMode(t, monitor, true)
// A hybrid mock ledger lists real connections beside fixture ones, so the
// same connection can be read degraded in mock mode and again live.
degraded := alerts.ConnectionSnapshot{
ID: "pve-cluster-a",
Name: "cluster-a",
Type: alerts.ConnectionTypePVE,
State: alerts.ConnectionStateUnreachable,
Enabled: true,
}
entered, release := make(chan struct{}), make(chan struct{})
var once sync.Once
monitor.SetConnectionsSnapshotLister(func() []alerts.ConnectionSnapshot {
once.Do(func() {
close(entered)
<-release
})
return []alerts.ConnectionSnapshot{degraded}
})
released := false
defer func() {
if !released {
close(release)
}
}()
checkDone := make(chan struct{})
go func() {
defer close(checkDone)
monitor.checkConnectionAlerts()
}()
waitForSignal(t, entered, 30*time.Second, "connection check never read the ledger")
waitForSignal(t, switchMockModeAsync(t, monitor, false), 30*time.Second, "leaving mock mode waited on a connection check parked in its ledger read")
close(release)
released = true
waitForSignal(t, checkDone, 30*time.Second, "connection check did not finish after it was released")
// The connection-degraded alert confirms on its third consecutive
// observation; the one read in mock mode must not be one of them.
monitor.checkConnectionAlerts()
monitor.checkConnectionAlerts()
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("a connection observation read in mock mode counted toward live confirmation: %s", describeAlerts(active))
}
monitor.checkConnectionAlerts()
if active := manager.GetActiveAlerts(); len(active) != 1 {
t.Fatalf("three live degraded observations raised %d alerts, want 1: %s", len(active), describeAlerts(active))
}
}
// startTenantMonitors starts the default organization's monitor and one
// other tenant's through MultiTenantMonitor, as the server does, each with its
// own alert manager, and waits until both monitoring loops run. An hour
// between ticks leaves each loop the pass it runs on start, so the test drives
// every later pass itself.
func startTenantMonitors(t *testing.T) (defaultMonitor, tenantMonitor *Monitor) {
t.Helper()
mtp, _ := newTestTenantPersistence(t)
mtm := NewMultiTenantMonitor(&config.Config{DataPath: t.TempDir(), PVEPollingInterval: time.Hour}, mtp, nil)
t.Cleanup(mtm.Stop)
mtm.SetMonitorInitializer(func(monitor *Monitor) {
cfg := monitor.alertManager.GetConfig()
cfg.Enabled = true
cfg.TimeThresholds = map[string]int{}
cfg.MetricTimeThresholds = nil
cfg.BackupDefaults.Enabled = false
cfg.AgentDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}
cfg.NodeDefaults.Memory = &alerts.HysteresisThreshold{Trigger: 1, Clear: 0.5}
monitor.alertManager.UpdateConfig(cfg)
})
if err := mtp.SaveOrganization(&models.Organization{ID: "org-b", DisplayName: "Org B"}); err != nil {
t.Fatalf("SaveOrganization(org-b): %v", err)
}
monitors := make([]*Monitor, 0, 2)
for _, orgID := range []string{"default", "org-b"} {
monitor, err := mtm.GetMonitor(orgID)
if err != nil {
t.Fatalf("GetMonitor(%s): %v", orgID, err)
}
// Start sets its runtime context before it joins the monitors a
// switch reaches, so wait for the join itself.
waitForCondition(t, 30*time.Second, func() bool {
mockModeSwitchMu.Lock()
defer mockModeSwitchMu.Unlock()
_, running := mockModeRunningMonitors[monitor]
return running
}, orgID+" monitoring loop did not join mock-mode switches")
monitors = append(monitors, monitor)
}
return monitors[0], monitors[1]
}
func mockHostAgentCount(monitor *Monitor) int {
monitor.mockHostAgentsMu.Lock()
defer monitor.mockHostAgentsMu.Unlock()
return len(monitor.mockHostAgents)
}
func TestLeavingMockModeClearsEveryRunningTenantMonitor(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
mustSetMockEnabled(t, true)
defaultMonitor, tenantMonitor := startTenantMonitors(t)
// Each loop runs a mock pass on start and raises the fixture's alerts in
// its own alert manager.
monitors := map[string]*Monitor{"default": defaultMonitor, "org-b": tenantMonitor}
for name, monitor := range monitors {
waitForCondition(t, 30*time.Second, func() bool {
return len(monitor.alertManager.GetActiveAlerts()) > 0 && mockHostAgentCount(monitor) > 0
}, name+" monitor raised no fixture alerts in mock mode")
}
// The mock-mode API switches through the requesting tenant's monitor.
mustSetMonitorMockMode(t, tenantMonitor, false)
for name, monitor := range monitors {
if active := monitor.alertManager.GetActiveAlerts(); len(active) != 0 {
t.Errorf("%s monitor kept %d fixture alerts after mock mode was left through org-b: %s",
name, len(active), describeAlerts(active))
}
if count := mockHostAgentCount(monitor); count != 0 {
t.Errorf("%s monitor still tracks %d fixture agents in live mode", name, count)
}
}
}
func TestEnteringMockModeClearsEveryRunningTenantMonitor(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
defaultMonitor, tenantMonitor := startTenantMonitors(t)
monitors := map[string]*Monitor{"default": defaultMonitor, "org-b": tenantMonitor}
for name, monitor := range monitors {
monitor.alertManager.CheckNode(models.Node{
ID: name + "-live-node",
Name: "live-node",
Instance: "live",
Status: "online",
Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75},
})
if len(monitor.alertManager.GetActiveAlerts()) == 0 {
t.Fatalf("live node raised no alert in %s", name)
}
}
// The demo-fixture licence sync switches through the default monitor.
mustSetMonitorMockMode(t, defaultMonitor, true)
for name, monitor := range monitors {
for _, alert := range monitor.alertManager.GetActiveAlerts() {
if alert.ResourceID == name+"-live-node" {
t.Errorf("%s kept its live %s alert after mock mode was entered through the default monitor", name, alert.Type)
}
}
}
}
func TestMonitorStartedAfterAMockModeSwitchLeavesTheModeItWasBuiltIn(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
// Built in live mode: New restored this live alert from disk.
manager.CheckNode(models.Node{
ID: "live-node",
Name: "live-node",
Instance: "live",
Status: "online",
Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75},
})
if len(manager.GetActiveAlerts()) == 0 {
t.Fatal("live node raised no alert")
}
// Another monitor switched to mock mode before this one's Start joined.
mustSetMockEnabled(t, true)
started := false
leave := monitor.joinMockModeSwitches(func(mockEnabled bool) { started = mockEnabled })
defer leave()
if !started {
t.Fatal("Start was not handed mock mode")
}
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("a monitor built in live mode started in mock mode with its live alerts: %s", describeAlerts(active))
}
mockModeSwitchMu.Lock()
_, running := mockModeRunningMonitors[monitor]
mockModeSwitchMu.Unlock()
if !running {
t.Fatal("joined monitor is not reached by later switches")
}
}
func TestMonitorStartedAfterAMockModeRoundTripDropsWhatItRaisedMeanwhile(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
newMonitor := func() *Monitor {
return &Monitor{
state: models.NewState(),
alertManager: newMockHostAlertTestManager(t),
metricsHistory: NewMetricsHistory(10, time.Hour),
}
}
switcher, starting := newMonitor(), newMonitor()
// Built in live mode, as New records it, and not running yet.
mockModeSwitchMu.Lock()
starting.mockModeSwitchesSeen = mockModeSwitchCount.Load()
mockModeSwitchMu.Unlock()
mustSetMonitorMockMode(t, switcher, true)
// A read path evaluates fixture data on the monitor before its Start runs.
fixture := mock.CurrentFixtureGraph().State.Nodes[0]
fixture.Status = "online"
fixture.Memory = models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75}
starting.alertManager.CheckNode(fixture)
if len(starting.alertManager.GetActiveAlerts()) == 0 {
t.Fatal("fixture node raised no alert")
}
mustSetMonitorMockMode(t, switcher, false)
leave := starting.joinMockModeSwitches(func(bool) {})
defer leave()
if active := starting.alertManager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("a monitor that missed a mock-mode round trip started in live mode with fixture alerts: %s", describeAlerts(active))
}
}
func TestSetMockModeAlignsAMonitorWhoseModeTheFlagAlreadyHolds(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{
state: models.NewState(),
alertManager: manager,
metricsHistory: NewMetricsHistory(10, time.Hour),
}
// Built in live mode, with a live alert, and not running yet.
manager.CheckNode(models.Node{
ID: "live-node",
Name: "live-node",
Instance: "live",
Status: "online",
Memory: models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75},
})
if len(manager.GetActiveAlerts()) == 0 {
t.Fatal("live node raised no alert")
}
// Another monitor's switch already set the flag.
mustSetMockEnabled(t, true)
mustSetMonitorMockMode(t, monitor, true)
if active := manager.GetActiveAlerts(); len(active) != 0 {
t.Fatalf("SetMockMode(true) left a monitor built in live mode with its live alerts: %s", describeAlerts(active))
}
}
func TestMockModeSwitchLeavesAnUnjoinedMonitorsRuntimeToItsStart(t *testing.T) {
setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
pinDefaultMockEstate(t)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
// Start has set its runtime context but not joined the switches yet.
monitor := &Monitor{
config: &config.Config{},
state: models.NewState(),
alertManager: newMockHostAlertTestManager(t),
metricsHistory: NewMetricsHistory(10, time.Hour),
runtimeCtx: ctx,
}
t.Cleanup(monitor.stopMockMetricsSampler)
mustSetMonitorMockMode(t, monitor, true)
monitor.mu.RLock()
started := monitor.mockMetricsCancel != nil
monitor.mu.RUnlock()
if started {
t.Fatal("a switch started the runtime of a monitor whose Start has not joined; that runtime belongs to its Start")
}
leave := monitor.joinMockModeSwitches(func(mockEnabled bool) {
monitor.startModeRuntime(ctx, nil, mockEnabled)
})
defer leave()
monitor.mu.RLock()
started = monitor.mockMetricsCancel != nil
monitor.mu.RUnlock()
if !started {
t.Fatal("joining in mock mode did not start the mock sampler")
}
}
// diskAlertKeys lists the active disk-health and disk-wearout alerts as
// "type instance/node device" so assertions name the disk, not the alert ID.
func diskAlertKeys(manager *alerts.Manager) map[string]bool {
keys := make(map[string]bool)
for _, alert := range manager.GetActiveAlerts() {
if alert.Type != "disk-health" && alert.Type != "disk-wearout" {
continue
}
devPath, _ := alert.Metadata["disk_path"].(string)
keys[alert.Type+" "+alert.Instance+"/"+alert.Node+" "+devPath] = true
}
return keys
}
func TestCheckMockAlertsEvaluatesPhysicalDisks(t *testing.T) {
// Pin the default estate: a PULSE_MOCK_* environment can select one with
// no FAILED disk (the public demo's eight nodes have none).
previousEnabled, previousConfig := mock.IsMockEnabled(), mock.GetConfig()
t.Cleanup(func() {
mustSetMockEnabled(t, false)
mock.SetMockConfig(previousConfig)
mustSetMockEnabled(t, previousEnabled)
})
mustSetMockEnabled(t, false)
cfg := mock.DefaultConfig
cfg.UpdateInterval = 5 * time.Minute
mock.SetMockConfig(cfg)
mustSetMockEnabled(t, true)
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
monitor.checkMockAlerts()
// The fixture keeps a FAILED cohort and worn SSDs stable across restarts
// so the disk alert lifecycle has evidence to act on. Each must raise the
// alert the physical disk poller raises for a live disk, and a disk on a
// node the poller would not reach raises nothing.
state := mock.CurrentFixtureGraph().State
onlineNodes := make(map[string]bool, len(state.Nodes))
for _, node := range state.Nodes {
onlineNodes[node.Instance+"/"+node.Name] = node.Status == "online"
}
want := make(map[string]bool)
failed, worn := 0, 0
for _, disk := range state.PhysicalDisks {
if !onlineNodes[disk.Instance+"/"+disk.Node] {
continue
}
if strings.EqualFold(disk.Health, "FAILED") {
want["disk-health "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true
failed++
}
if disk.Wearout >= 0 && disk.Wearout < 10 {
want["disk-wearout "+disk.Instance+"/"+disk.Node+" "+disk.DevPath] = true
worn++
}
}
if failed == 0 || worn == 0 {
t.Fatalf("mock fixture has %d FAILED and %d worn disks on online nodes, want at least one of each", failed, worn)
}
got := diskAlertKeys(manager)
for key := range got {
if !want[key] {
t.Errorf("mock pass raised unexpected disk alert %q", key)
}
}
for key := range want {
if !got[key] {
t.Errorf("mock pass raised no alert %q", key)
}
}
}
func TestCheckMockPhysicalDiskAlertsFollowsPollerBoundary(t *testing.T) {
manager := newMockHostAlertTestManager(t)
monitor := &Monitor{alertManager: manager}
nodes := []models.Node{
{ID: "west-pve1", Name: "pve1", Instance: "west", Status: "online", LinkedAgentID: "host-west-pve1"},
{ID: "east-pve1", Name: "pve1", Instance: "east", Status: "offline"},
}
hosts := []models.Host{{ID: "host-west-pve1", Hostname: "pve1", Status: "online"}}
disks := []models.PhysicalDisk{
{Instance: "west", Node: "pve1", DevPath: "/dev/sda", Model: "Crucial MX500 2TB", Type: "sata", Health: "FAILED", Wearout: -1},
{Instance: "west", Node: "pve1", DevPath: "/dev/nvme0n1", Model: "Samsung 980 PRO 2TB", Type: "nvme", Health: "PASSED", Wearout: 6},
// Same node name on another instance: that node is offline, so the
// poller would not reach it and its disk must not alert.
{Instance: "east", Node: "pve1", DevPath: "/dev/sdb", Model: "WD Red Pro 8TB", Type: "sata", Health: "FAILED", Wearout: -1},
}
monitor.checkMockPhysicalDiskAlerts(monitor.mockModeFence.begin(), disks, nodes, hosts)
got := diskAlertKeys(manager)
if len(got) != 2 || !got["disk-health west/pve1 /dev/sda"] || !got["disk-wearout west/pve1 /dev/nvme0n1"] {
t.Fatalf("disk alerts = %v, want only FAILED health on west/pve1 /dev/sda and wearout on west/pve1 /dev/nvme0n1", got)
}
// The linked agent's --disk-exclude forces matched devices healthy, so
// excluding a disk resolves the alerts it already raised. Endurance
// recovery needs three healthy-looking passes.
hosts[0].DiskExclude = []string{"/dev/sda", "nvme0n1"}
for i := 0; i < 3; i++ {
monitor.checkMockPhysicalDiskAlerts(monitor.mockModeFence.begin(), disks, nodes, hosts)
}
if got := diskAlertKeys(manager); len(got) != 0 {
t.Fatalf("excluded devices kept disk alerts %v", got)
}
// A node that leaves the estate takes its disk alerts with it through the
// node cleanup the mock pass runs first.
hosts[0].DiskExclude = nil
monitor.checkMockPhysicalDiskAlerts(monitor.mockModeFence.begin(), disks, nodes, hosts)
if got := diskAlertKeys(manager); len(got) != 2 {
t.Fatalf("disk alerts after removing the exclusion = %v, want both west/pve1 alerts back", got)
}
manager.CleanupAlertsForNodes(map[string]bool{"pve2": true})
if got := diskAlertKeys(manager); len(got) != 0 {
t.Fatalf("disk alerts outlived their departed node: %v", got)
}
}