Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Compose batch-start availability repairs with retained mock fixture cleanup
pulse-triage[bot]bb39f8977199
diff --cc internal/monitoring/monitor.goindex 176e1c6952,36f1a3a536..28744f4953--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@@ -1331,10 -1331,10 +1332,12 @@@ type Monitor structmockMetricsCancel context.CancelFuncmockMetricsWg sync.WaitGroupmockHostAgentsMu sync.Mutex+ mockDockerHosts map[string]models.DockerHost+ mockFixtureRevision uint64mockHostAgents map[string]models.Host // Fixture agents evaluated by the last mock alert passmockModeFence mockModeFence // Keeps mode-dependent alert evaluations inside the epoch they read in+ mockModeAligned bool // Mock mode the alerts and state belong to; guarded by mockModeSwitchMu+ mockModeSwitchesSeen uint64 // mockModeSwitchCount when they were last aligned; guarded by mockModeSwitchMudockerChecker DockerChecker // Optional Docker checker for LXC containersdockerCheckerConfiguredAt time.Time // Last time the Docker checker was configureddockerCheckAllowedVMIDs map[int]struct{} // Optional VMID allowlist gating the LXC Docker socket probe; empty means all guests@@@ -4647,49 -4658,73 +4667,73 @@@ func (m *Monitor) SetMockMode(enable bomockModeSwitchMu.Lock()defer mockModeSwitchMu.Unlock()- current := mock.IsMockEnabled()- if current == enable {- log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state")- return nil+ monitors := mockModeSwitchTargetsLocked(m)+ flip := mock.IsMockEnabled() != enable+ if !flip {+ // The flag already holds the mode, but a monitor that was not running+ // during an earlier switch can still hold the other mode's alerts+ // and state; only those leave it.+ monitors = slices.DeleteFunc(monitors, func(monitor *Monitor) bool {+ return monitor.mockModeAligned == enable+ })+ if len(monitors) == 0 {+ log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state")+ return nil+ }}- // Every evaluation of mode-dependent data that started before the flip- // must finish or be refused before the clear, or it reopens alerts for- // the side the monitor just left (see mockModeFence).- if enable {- m.stopMockMetricsSampler()- if err := mock.SetEnabled(true); err != nil {+ for _, monitor := range monitors {+ monitor.stopMockMetricsSampler()+ }+ if flip {+ if err := mock.SetEnabled(enable); err != nil {return err}- m.mockModeFence.advance()- m.alertManager.ClearActiveAlerts()- m.mu.Lock()- m.resetStateLocked()- m.metricsHistory.Reset()- m.mu.Unlock()- m.StopDiscoveryService()- m.mu.RLock()- ctx := m.runtimeCtx- m.mu.RUnlock()- if ctx != nil {- m.startMockMetricsSampler(ctx)- }- log.Info().Msg("switched monitor to mock mode")+ mockModeSwitchCount.Add(1)+ }+ for _, monitor := range monitors {+ monitor.endMockModeEpoch(enable)+ }+ for _, monitor := range monitors {+ _, running := mockModeRunningMonitors[monitor]+ monitor.resumeAfterMockModeSwitch(enable, running)+ }+ if enable {+ log.Info().Int("monitors", len(monitors)).Msg("switched monitors to mock mode")} else {- m.stopMockMetricsSampler()- if err := mock.SetEnabled(false); err != nil {- return err- }- m.mockModeFence.advance()- m.alertManager.ClearActiveAlerts()+ log.Info().Int("monitors", len(monitors)).Msg("switched monitors to real data mode")+ }+ return nil+ }++ // endMockModeEpoch leaves the mode the monitor's alerts and state belong to+ // once the process flag already holds the new one. Every evaluation of+ // mode-dependent data that started before the flip must finish or be refused+ // before the clear, or it reopens alerts for the side the monitor just left+ // (see mockModeFence). The caller holds mockModeSwitchMu.+ func (m *Monitor) endMockModeEpoch(enable bool) {+ m.mockModeFence.advance()+ m.alertManager.ClearActiveAlerts()+ if !enable {- m.forgetMockHostAgents()+ m.forgetMockFixtureHosts()- m.mu.Lock()- m.resetStateLocked()- m.metricsHistory.Reset()- m.mu.Unlock()- log.Info().Msg("switched monitor to real data mode")}+ m.mu.Lock()+ m.resetStateLocked()+ m.metricsHistory.Reset()+ m.mu.Unlock()+ if enable {+ m.StopDiscoveryService()+ }+ m.mockModeAligned = enable+ m.mockModeSwitchesSeen = mockModeSwitchCount.Load()+ }+ // resumeAfterMockModeSwitch restarts the mode-dependent runtime of a monitor+ // whose Start loop has joined the switches, once every monitor has ended its+ // epoch. A monitor that has not joined yet starts that runtime itself when it+ // does (Start sets its runtime context first), and one whose loop has stopped+ // only broadcasts.+ func (m *Monitor) resumeAfterMockModeSwitch(enable, running bool) {m.mu.RLock()ctx := m.runtimeCtxhub := m.wsHubdiff --cc internal/monitoring/monitor_host_agent_removal_lifecycle_test.goindex b921bf36d2,e9d81cf562..d8a939b217--- a/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go+++ b/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go@@@ -858,29 -859,44 +859,71 @@@ func TestMockHostAgentLeavingFixtureUset.Fatal("node pve9 raised no memory alert after its agent left the mock estate")}+func TestMockDockerHostLeavingFixtureUsesRemovalLifecycle(t *testing.T) {+ mustSetMockEnabled(t, true)+ t.Cleanup(func() { mustSetMockEnabled(t, false) })+ manager := newMockHostAlertTestManager(t)+ monitor := &Monitor{alertManager: manager}++ kept := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")+ // Nested Docker-in-LXC hosts are named after their guest's VMID, which a+ // runtime mock config change can shift.+ departed := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")++ monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept, departed}, 1)+ monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept, departed}, 1)+ if ids := dockerAlertIDsForHost(manager, departed.ID); len(ids) != len(departed.Containers) {+ t.Fatalf("departing host opened %v, want one alert per exited container", ids)+ }++ monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept}, 2)++ if ids := dockerAlertIDsForHost(manager, departed.ID); len(ids) != 0 {+ t.Fatalf("Docker host that left the mock estate kept alerts %v", ids)+ }+ if ids := dockerAlertIDsForHost(manager, kept.ID); len(ids) != len(kept.Containers) {+ t.Fatalf("Docker host still in the mock estate has alerts %v, want one per exited container", ids)+ }+}+++ func TestLeavingMockModeReleasesFixtureAgentNodeLinksOnEveryRunningMonitor(t *testing.T) {+ setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)+ pinDefaultMockEstate(t)+ mustSetMockEnabled(t, true)+ defaultMonitor, tenantMonitor := startTenantMonitors(t)++ // org-b's start pass registers the fixture agents in org-b's own alert+ // manager, each owning the usage alerts of the fixture node it runs on.+ var linked models.Node+ waitForCondition(t, 30*time.Second, func() bool {+ tenantMonitor.mockHostAgentsMu.Lock()+ defer tenantMonitor.mockHostAgentsMu.Unlock()+ for _, host := range tenantMonitor.mockHostAgents {+ if host.Status != "online" || host.LinkedNodeID == "" {+ continue+ }+ for _, node := range mock.CurrentFixtureGraph().State.Nodes {+ if node.ID == host.LinkedNodeID {+ linked = node+ return true+ }+ }+ }+ return false+ }, "org-b registered no online fixture agent linked to a fixture node")++ // The demo-fixture licence sync switches through the default monitor.+ mustSetMonitorMockMode(t, defaultMonitor, false)++ // org-b's fixture agents go through the removal lifecycle too, so the+ // node an agent covered owns its metric alerts again.+ linked.Status = "online"+ linked.Memory = models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75}+ tenantMonitor.alertManager.CheckNode(linked)+ for _, alert := range tenantMonitor.alertManager.GetActiveAlerts() {+ if alert.ResourceID == linked.ID && alert.Type == "memory" {+ return+ }+ }+ t.Fatalf("org-b's node %s raised no memory alert: its fixture agent's node link outlived a switch made through the default monitor", linked.ID)+ }