Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Compose batch-start availability repairs with retained mock fixture cleanup

pulse-triage[bot]bb39f8977199
diff --cc internal/monitoring/monitor.go
index 176e1c6952,36f1a3a536..28744f4953
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@@ -1331,10 -1331,10 +1332,12 @@@ type Monitor struct
mockMetricsCancel context.CancelFunc
mockMetricsWg sync.WaitGroup
mockHostAgentsMu sync.Mutex
+ mockDockerHosts map[string]models.DockerHost
+ mockFixtureRevision uint64
mockHostAgents map[string]models.Host // Fixture agents evaluated by the last mock alert pass
mockModeFence mockModeFence // Keeps mode-dependent alert evaluations inside the epoch they read in
+ mockModeAligned bool // Mock mode the alerts and state belong to; guarded by mockModeSwitchMu
+ mockModeSwitchesSeen uint64 // mockModeSwitchCount when they were last aligned; guarded by mockModeSwitchMu
dockerChecker DockerChecker // Optional Docker checker for LXC containers
dockerCheckerConfiguredAt time.Time // Last time the Docker checker was configured
dockerCheckAllowedVMIDs map[int]struct{} // Optional VMID allowlist gating the LXC Docker socket probe; empty means all guests
@@@ -4647,49 -4658,73 +4667,73 @@@ func (m *Monitor) SetMockMode(enable bo
mockModeSwitchMu.Lock()
defer mockModeSwitchMu.Unlock()
- current := mock.IsMockEnabled()
- if current == enable {
- log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state")
- return nil
+ monitors := mockModeSwitchTargetsLocked(m)
+ flip := mock.IsMockEnabled() != enable
+ if !flip {
+ // The flag already holds the mode, but a monitor that was not running
+ // during an earlier switch can still hold the other mode's alerts
+ // and state; only those leave it.
+ monitors = slices.DeleteFunc(monitors, func(monitor *Monitor) bool {
+ return monitor.mockModeAligned == enable
+ })
+ if len(monitors) == 0 {
+ log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state")
+ return nil
+ }
}
- // Every evaluation of mode-dependent data that started before the flip
- // must finish or be refused before the clear, or it reopens alerts for
- // the side the monitor just left (see mockModeFence).
- if enable {
- m.stopMockMetricsSampler()
- if err := mock.SetEnabled(true); err != nil {
+ for _, monitor := range monitors {
+ monitor.stopMockMetricsSampler()
+ }
+ if flip {
+ if err := mock.SetEnabled(enable); err != nil {
return err
}
- m.mockModeFence.advance()
- m.alertManager.ClearActiveAlerts()
- m.mu.Lock()
- m.resetStateLocked()
- m.metricsHistory.Reset()
- m.mu.Unlock()
- m.StopDiscoveryService()
- m.mu.RLock()
- ctx := m.runtimeCtx
- m.mu.RUnlock()
- if ctx != nil {
- m.startMockMetricsSampler(ctx)
- }
- log.Info().Msg("switched monitor to mock mode")
+ mockModeSwitchCount.Add(1)
+ }
+ for _, monitor := range monitors {
+ monitor.endMockModeEpoch(enable)
+ }
+ for _, monitor := range monitors {
+ _, running := mockModeRunningMonitors[monitor]
+ monitor.resumeAfterMockModeSwitch(enable, running)
+ }
+ if enable {
+ log.Info().Int("monitors", len(monitors)).Msg("switched monitors to mock mode")
} else {
- m.stopMockMetricsSampler()
- if err := mock.SetEnabled(false); err != nil {
- return err
- }
- m.mockModeFence.advance()
- m.alertManager.ClearActiveAlerts()
+ log.Info().Int("monitors", len(monitors)).Msg("switched monitors to real data mode")
+ }
+ return nil
+ }
+
+ // endMockModeEpoch leaves the mode the monitor's alerts and state belong to
+ // once the process flag already holds the new one. Every evaluation of
+ // mode-dependent data that started before the flip must finish or be refused
+ // before the clear, or it reopens alerts for the side the monitor just left
+ // (see mockModeFence). The caller holds mockModeSwitchMu.
+ func (m *Monitor) endMockModeEpoch(enable bool) {
+ m.mockModeFence.advance()
+ m.alertManager.ClearActiveAlerts()
+ if !enable {
- m.forgetMockHostAgents()
+ m.forgetMockFixtureHosts()
- m.mu.Lock()
- m.resetStateLocked()
- m.metricsHistory.Reset()
- m.mu.Unlock()
- log.Info().Msg("switched monitor to real data mode")
}
+ m.mu.Lock()
+ m.resetStateLocked()
+ m.metricsHistory.Reset()
+ m.mu.Unlock()
+ if enable {
+ m.StopDiscoveryService()
+ }
+ m.mockModeAligned = enable
+ m.mockModeSwitchesSeen = mockModeSwitchCount.Load()
+ }
+ // resumeAfterMockModeSwitch restarts the mode-dependent runtime of a monitor
+ // whose Start loop has joined the switches, once every monitor has ended its
+ // epoch. A monitor that has not joined yet starts that runtime itself when it
+ // does (Start sets its runtime context first), and one whose loop has stopped
+ // only broadcasts.
+ func (m *Monitor) resumeAfterMockModeSwitch(enable, running bool) {
m.mu.RLock()
ctx := m.runtimeCtx
hub := m.wsHub
diff --cc internal/monitoring/monitor_host_agent_removal_lifecycle_test.go
index b921bf36d2,e9d81cf562..d8a939b217
--- a/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go
+++ b/internal/monitoring/monitor_host_agent_removal_lifecycle_test.go
@@@ -858,29 -859,44 +859,71 @@@ func TestMockHostAgentLeavingFixtureUse
t.Fatal("node pve9 raised no memory alert after its agent left the mock estate")
}
+func TestMockDockerHostLeavingFixtureUsesRemovalLifecycle(t *testing.T) {
+ mustSetMockEnabled(t, true)
+ t.Cleanup(func() { mustSetMockEnabled(t, false) })
+ manager := newMockHostAlertTestManager(t)
+ monitor := &Monitor{alertManager: manager}
+
+ kept := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")
+ // Nested Docker-in-LXC hosts are named after their guest's VMID, which a
+ // runtime mock config change can shift.
+ departed := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")
+
+ monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept, departed}, 1)
+ monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept, departed}, 1)
+ if ids := dockerAlertIDsForHost(manager, departed.ID); len(ids) != len(departed.Containers) {
+ t.Fatalf("departing host opened %v, want one alert per exited container", ids)
+ }
+
+ monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept}, 2)
+
+ if ids := dockerAlertIDsForHost(manager, departed.ID); len(ids) != 0 {
+ t.Fatalf("Docker host that left the mock estate kept alerts %v", ids)
+ }
+ if ids := dockerAlertIDsForHost(manager, kept.ID); len(ids) != len(kept.Containers) {
+ t.Fatalf("Docker host still in the mock estate has alerts %v, want one per exited container", ids)
+ }
+}
++
+ func TestLeavingMockModeReleasesFixtureAgentNodeLinksOnEveryRunningMonitor(t *testing.T) {
+ setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
+ pinDefaultMockEstate(t)
+ mustSetMockEnabled(t, true)
+ defaultMonitor, tenantMonitor := startTenantMonitors(t)
+
+ // org-b's start pass registers the fixture agents in org-b's own alert
+ // manager, each owning the usage alerts of the fixture node it runs on.
+ var linked models.Node
+ waitForCondition(t, 30*time.Second, func() bool {
+ tenantMonitor.mockHostAgentsMu.Lock()
+ defer tenantMonitor.mockHostAgentsMu.Unlock()
+ for _, host := range tenantMonitor.mockHostAgents {
+ if host.Status != "online" || host.LinkedNodeID == "" {
+ continue
+ }
+ for _, node := range mock.CurrentFixtureGraph().State.Nodes {
+ if node.ID == host.LinkedNodeID {
+ linked = node
+ return true
+ }
+ }
+ }
+ return false
+ }, "org-b registered no online fixture agent linked to a fixture node")
+
+ // The demo-fixture licence sync switches through the default monitor.
+ mustSetMonitorMockMode(t, defaultMonitor, false)
+
+ // org-b's fixture agents go through the removal lifecycle too, so the
+ // node an agent covered owns its metric alerts again.
+ linked.Status = "online"
+ linked.Memory = models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75}
+ tenantMonitor.alertManager.CheckNode(linked)
+ for _, alert := range tenantMonitor.alertManager.GetActiveAlerts() {
+ if alert.ResourceID == linked.ID && alert.Type == "memory" {
+ return
+ }
+ }
+ t.Fatalf("org-b's node %s raised no memory alert: its fixture agent's node link outlived a switch made through the default monitor", linked.ID)
+ }