Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Stop alert evaluation writing live thresholds outside the lock

rcourtman71a470129f27
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index bc0caa053f..6e4f87d46f 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -3143,6 +3143,26 @@ fills every field and fails when a new reference field is not copied.
Regression ownership is `internal/alerts/resolved_lock_discipline_test.go` and
`internal/config/persistence_alert_ownership_test.go`.
+### Alert evaluation owns the thresholds it normalizes
+
+Host, guest and Docker container evaluation copy what they read from
+`m.config` while holding `m.mu`, and only then fill in defaults.
+`CheckHost` and `CheckGuest` clone a per-disk
+override's `Disk` threshold before `ensureHysteresisThreshold` sets a missing
+`Clear` to the trigger minus 5. They used to normalize the live override's
+pointer after unlocking, so a `GetConfig()` clone under the read lock raced
+the write, and the saved override changed to whatever the last poll filled in.
+The live override now keeps the clear it was saved with, and the config API
+returns that value. Docker container evaluation resolves its thresholds from
+cloned `DockerDefaults` plus the override under the same lock, and reads the
+restart-loop and memory-limit settings under the lock. It used to point into
+`m.config.DockerDefaults` with no lock while `UpdateConfig` replaced
+`m.config`. The test-only `getThresholdForMetricFromConfig`, which normalized
+in place, is gone. `internal/alerts/alerts_test.go`
+`TestAlertEvaluationLeavesTheLiveConfigToTheLock` runs host, guest and Docker
+evaluation against `GetConfig()` and `UpdateConfig(GetConfig())` under
+`-race`.
+
### Versioned alert-intent policy
The alerts runtime owns one versioned alert-intent document and its durable
diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go
index f0ad060e0f..dea4577c90 100644
--- a/internal/alerts/alerts_test.go
+++ b/internal/alerts/alerts_test.go
@@ -2756,6 +2756,101 @@ func TestCheckBackupsIgnoresVMIDs(t *testing.T) {
}
}
+// Host and guest evaluation used to fill in the Clear of a disk override's
+// threshold after dropping m.mu, writing through the live config's pointer
+// while GetConfig cloned it under the read lock, and Docker evaluation read
+// m.config.DockerDefaults with no lock while UpdateConfig replaced m.config.
+// Evaluation now copies what it needs under m.mu. Run with -race.
+func TestAlertEvaluationLeavesTheLiveConfigToTheLock(t *testing.T) {
+ originalLogLevel := zerolog.GlobalLevel()
+ zerolog.SetGlobalLevel(zerolog.Disabled)
+ t.Cleanup(func() {
+ zerolog.SetGlobalLevel(originalLogLevel)
+ })
+
+ m := newTestManager(t)
+
+ host := models.Host{
+ ID: "eval-lock",
+ Hostname: "eval-lock",
+ Status: "online",
+ Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/sda1", Usage: 50, Total: 100, Used: 50, Free: 50}},
+ }
+ hostDiskOverrideID, _ := hostDiskResourceID(host, host.Disks[0])
+ vm := models.VM{
+ ID: BuildGuestKey("pve1", "node1", 101),
+ VMID: 101,
+ Name: "eval-lock",
+ Node: "node1",
+ Instance: "pve1",
+ Status: "running",
+ Disks: []models.Disk{{Mountpoint: "/boot", Device: "/dev/vda1", Usage: 50, Total: 100, Used: 50, Free: 50}},
+ }
+ guestDiskOverrideID := "guest-disk:guest:pve1:101/disk:boot-dev-vda1"
+ dockerHost := models.DockerHost{
+ ID: "eval-lock-docker",
+ Hostname: "eval-lock-docker",
+ DisplayName: "eval-lock-docker",
+ Containers: []models.DockerContainer{{
+ ID: "abcdef123456",
+ Name: "web",
+ State: "running",
+ CPUPercent: 10,
+ MemoryPercent: 10,
+ MemoryUsage: 100 << 20,
+ MemoryLimit: 1 << 30,
+ }},
+ }
+
+ // A disk override saved with only a trigger leaves Clear for evaluation
+ // to default.
+ withTriggerOnlyDiskOverrides := func(cfg AlertConfig) AlertConfig {
+ cfg.Enabled = true
+ cfg.ActivationState = ActivationActive
+ cfg.Overrides = map[string]ThresholdConfig{
+ hostDiskOverrideID: {Disk: &HysteresisThreshold{Trigger: 90}},
+ guestDiskOverrideID: {Disk: &HysteresisThreshold{Trigger: 90}},
+ }
+ return cfg
+ }
+ m.UpdateConfig(withTriggerOnlyDiskOverrides(m.GetConfig()))
+
+ m.CheckHost(host)
+ m.CheckGuest(vm, "pve1")
+ live := m.GetConfig()
+ for _, id := range []string{hostDiskOverrideID, guestDiskOverrideID} {
+ if got := live.Overrides[id].Disk.Clear; got != 0 {
+ t.Fatalf("evaluating %s wrote clear %v into the live override", id, got)
+ }
+ }
+
+ var wg sync.WaitGroup
+ wg.Add(3)
+ go func() {
+ defer wg.Done()
+ for i := 0; i < 100; i++ {
+ // Resetting the overrides gives evaluation a Clear to fill again.
+ m.UpdateConfig(withTriggerOnlyDiskOverrides(m.GetConfig()))
+ m.UpdateConfig(m.GetConfig())
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ for i := 0; i < 200; i++ {
+ _ = m.GetConfig()
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ for i := 0; i < 200; i++ {
+ m.CheckHost(host)
+ m.CheckGuest(vm, "pve1")
+ m.CheckDockerHost(dockerHost)
+ }
+ }()
+ wg.Wait()
+}
+
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
m := newTestManager(t)
diff --git a/internal/alerts/docker.go b/internal/alerts/docker.go
index 3eaa0d370a..74071b851d 100644
--- a/internal/alerts/docker.go
+++ b/internal/alerts/docker.go
@@ -352,8 +352,19 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode
resourceType := "app-container"
containerTags := dockerLabelTags(container.Labels)
+ // Resolve the metric thresholds while the lock is held: the Docker defaults
+ // are fields of m.config, which UpdateConfig replaces, and the override
+ // still points into it. applyThresholdOverride clones what it keeps.
m.mu.RLock()
overrideConfig, hasOverride := m.lookupDockerContainerOverrideNoLock(host.ID, container.Name, resourceID)
+ thresholds := ThresholdConfig{
+ CPU: cloneThreshold(&m.config.DockerDefaults.CPU),
+ Memory: cloneThreshold(&m.config.DockerDefaults.Memory),
+ Disk: cloneThreshold(&m.config.DockerDefaults.Disk),
+ }
+ if hasOverride {
+ thresholds = m.applyThresholdOverride(thresholds, overrideConfig)
+ }
m.mu.RUnlock()
if hasOverride && overrideConfig.Disabled {
// Alerts disabled via override; clear any existing alerts and skip evaluation.
@@ -377,16 +388,6 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode
} else {
m.clearDockerContainerStateAlert(resourceID)
- // Use Docker-specific defaults for containers
- thresholds := ThresholdConfig{
- CPU: &m.config.DockerDefaults.CPU,
- Memory: &m.config.DockerDefaults.Memory,
- Disk: &m.config.DockerDefaults.Disk,
- }
- if hasOverride {
- thresholds = m.applyThresholdOverride(thresholds, overrideConfig)
- }
-
if thresholds.CPU != nil {
cpuCapacityPercent := models.DockerContainerCPUCapacityPercent(container, host.CPUs)
cpuMetadata := map[string]interface{}{
@@ -998,6 +999,8 @@ func (m *Manager) checkDockerContainerRestartLoop(host models.DockerHost, contai
alertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID)
now := time.Now()
+ m.mu.Lock()
+
// Get config values with defaults
restartThreshold := m.config.DockerDefaults.RestartCount
if restartThreshold == 0 {
@@ -1008,8 +1011,6 @@ func (m *Manager) checkDockerContainerRestartLoop(host models.DockerHost, contai
timeWindow = 300 // Default: 5 minutes (300 seconds)
}
- m.mu.Lock()
-
record, exists := m.dockerRestartTracking[resourceID]
if !exists {
record = &dockerRestartRecord{
@@ -1164,11 +1165,13 @@ func (m *Manager) checkDockerContainerMemoryLimit(host models.DockerHost, contai
alertID := fmt.Sprintf("docker-container-memory-limit-%s", resourceID)
// Get config values with defaults
+ m.mu.RLock()
warnThreshold := float64(m.config.DockerDefaults.MemoryWarnPct)
+ criticalThreshold := float64(m.config.DockerDefaults.MemoryCriticalPct)
+ m.mu.RUnlock()
if warnThreshold == 0 {
warnThreshold = 90.0 // Default: 90%
}
- criticalThreshold := float64(m.config.DockerDefaults.MemoryCriticalPct)
if criticalThreshold == 0 {
criticalThreshold = 95.0 // Default: 95%
}
diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go
index 25963829ef..edeac41cb3 100644
--- a/internal/alerts/guest.go
+++ b/internal/alerts/guest.go
@@ -328,8 +328,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {
message := fmt.Sprintf("%s disk (%s) at %.1f%%", guestType, label, disk.Usage)
effectiveDiskThreshold := thresholds.Disk
+ // Own the override's Disk before ensureHysteresisThreshold fills
+ // Clear: the returned copy still points into the live config.
m.mu.RLock()
diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource)
+ diskOverride.Disk = cloneThreshold(diskOverride.Disk)
m.mu.RUnlock()
if hasDiskOverride {
if diskOverride.Disabled {
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index 65ae1cbd40..8c3edf1d7a 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -450,9 +450,11 @@ func (m *Manager) CheckHost(host models.Host) {
diskResourceID, diskName := hostDiskResourceID(host, disk)
seenDisks[diskResourceID] = struct{}{}
- // Check for disk-specific override
+ // Check for disk-specific override. The copy's Disk still points into
+ // the live config, so own it before ensureHysteresisThreshold fills Clear.
m.mu.RLock()
diskOverride, hasDiskOverride := m.config.Overrides[diskResourceID]
+ diskOverride.Disk = cloneThreshold(diskOverride.Disk)
m.mu.RUnlock()
// Determine the effective disk threshold
diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.go
index bfe670920d..112840229b 100644
--- a/internal/alerts/metric_runtime.go
+++ b/internal/alerts/metric_runtime.go
@@ -86,16 +86,6 @@ func getThresholdForMetric(config ThresholdConfig, metricType string) *Hysteresi
}
}
-// getThresholdForMetricFromConfig returns the threshold for a specific metric type from a ThresholdConfig
-// ensuring hysteresis is properly set.
-func getThresholdForMetricFromConfig(config ThresholdConfig, metricType string) *HysteresisThreshold {
- th := getThresholdForMetric(config, metricType)
- if th == nil {
- return nil
- }
- return ensureHysteresisThreshold(th)
-}
-
// getTimeThreshold determines the delay to apply for a metric/resource combination.
func (m *Manager) getTimeThreshold(resourceID string, resourceType, metricType string) int {
effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))
diff --git a/internal/alerts/utility_test.go b/internal/alerts/utility_test.go
index a4781633cf..f531aab6b5 100644
--- a/internal/alerts/utility_test.go
+++ b/internal/alerts/utility_test.go
@@ -2469,148 +2469,3 @@ func TestGetThresholdForMetric_EmptyConfig(t *testing.T) {
})
}
}
-
-// TestGetThresholdForMetricFromConfig tests the getThresholdForMetricFromConfig function
-func TestGetThresholdForMetricFromConfig(t *testing.T) {
- // t.Parallel()
-
- tests := []struct {
- name string
- config ThresholdConfig
- metricType string
- wantNil bool
- wantTrigger float64
- wantClear float64
- }{
- {
- name: "empty config returns nil",
- config: ThresholdConfig{},
- metricType: "cpu",
- wantNil: true,
- },
- {
- name: "cpu threshold returned",
- config: ThresholdConfig{
- CPU: &HysteresisThreshold{Trigger: 80, Clear: 70},
- },
- metricType: "cpu",
- wantTrigger: 80,
- wantClear: 70,
- },
- {
- name: "memory threshold returned",
- config: ThresholdConfig{
- Memory: &HysteresisThreshold{Trigger: 85, Clear: 75},
- },
- metricType: "memory",
- wantTrigger: 85,
- wantClear: 75,
- },
- {
- name: "disk threshold returned",
- config: ThresholdConfig{
- Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},
- },
- metricType: "disk",
- wantTrigger: 90,
- wantClear: 85,
- },
- {
- name: "diskRead threshold returned",
- config: ThresholdConfig{
- DiskRead: &HysteresisThreshold{Trigger: 50, Clear: 40},
- },
- metricType: "diskRead",
- wantTrigger: 50,
- wantClear: 40,
- },
- {
- name: "diskWrite threshold returned",
- config: ThresholdConfig{
- DiskWrite: &HysteresisThreshold{Trigger: 55, Clear: 45},
- },
- metricType: "diskWrite",
- wantTrigger: 55,
- wantClear: 45,
- },
- {
- name: "networkIn threshold returned",
- config: ThresholdConfig{
- NetworkIn: &HysteresisThreshold{Trigger: 70, Clear: 60},
- },
- metricType: "networkIn",
- wantTrigger: 70,
- wantClear: 60,
- },
- {
- name: "networkOut threshold returned",
- config: ThresholdConfig{
- NetworkOut: &HysteresisThreshold{Trigger: 75, Clear: 65},
- },
- metricType: "networkOut",
- wantTrigger: 75,
- wantClear: 65,
- },
- {
- name: "temperature threshold returned",
- config: ThresholdConfig{
- Temperature: &HysteresisThreshold{Trigger: 65, Clear: 55},
- },
- metricType: "temperature",
- wantTrigger: 65,
- wantClear: 55,
- },
- {
- name: "usage threshold returned",
- config: ThresholdConfig{
- Usage: &HysteresisThreshold{Trigger: 88, Clear: 78},
- },
- metricType: "usage",
- wantTrigger: 88,
- wantClear: 78,
- },
- {
- name: "unknown metric returns nil",
- config: ThresholdConfig{
- CPU: &HysteresisThreshold{Trigger: 80, Clear: 70},
- },
- metricType: "unknown",
- wantNil: true,
- },
- {
- name: "threshold with zero clear gets default hysteresis",
- config: ThresholdConfig{
- CPU: &HysteresisThreshold{Trigger: 80, Clear: 0},
- },
- metricType: "cpu",
- wantTrigger: 80,
- wantClear: 75, // 80 - 5 default margin
- },
- }
-
- for _, tc := range tests {
- t.Run(tc.name, func(t *testing.T) {
- // t.Parallel()
-
- result := getThresholdForMetricFromConfig(tc.config, tc.metricType)
-
- if tc.wantNil {
- if result != nil {
- t.Errorf("getThresholdForMetricFromConfig() = %v, want nil", result)
- }
- return
- }
-
- if result == nil {
- t.Fatalf("getThresholdForMetricFromConfig() = nil, want non-nil")
- }
-
- if result.Trigger != tc.wantTrigger {
- t.Errorf("Trigger = %v, want %v", result.Trigger, tc.wantTrigger)
- }
- if result.Clear != tc.wantClear {
- t.Errorf("Clear = %v, want %v", result.Clear, tc.wantClear)
- }
- })
- }
-}