Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop alert evaluation writing live thresholds outside the lock
rcourtman71a470129f27
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.mdindex bc0caa053f..6e4f87d46f 100644--- a/docs/release-control/v6/internal/subsystems/alerts.md+++ b/docs/release-control/v6/internal/subsystems/alerts.md@@ -3143,6 +3143,26 @@ fills every field and fails when a new reference field is not copied.Regression ownership is `internal/alerts/resolved_lock_discipline_test.go` and`internal/config/persistence_alert_ownership_test.go`.+### Alert evaluation owns the thresholds it normalizes++Host, guest and Docker container evaluation copy what they read from+`m.config` while holding `m.mu`, and only then fill in defaults.+`CheckHost` and `CheckGuest` clone a per-disk+override's `Disk` threshold before `ensureHysteresisThreshold` sets a missing+`Clear` to the trigger minus 5. They used to normalize the live override's+pointer after unlocking, so a `GetConfig()` clone under the read lock raced+the write, and the saved override changed to whatever the last poll filled in.+The live override now keeps the clear it was saved with, and the config API+returns that value. Docker container evaluation resolves its thresholds from+cloned `DockerDefaults` plus the override under the same lock, and reads the+restart-loop and memory-limit settings under the lock. It used to point into+`m.config.DockerDefaults` with no lock while `UpdateConfig` replaced+`m.config`. The test-only `getThresholdForMetricFromConfig`, which normalized+in place, is gone. `internal/alerts/alerts_test.go`+`TestAlertEvaluationLeavesTheLiveConfigToTheLock` runs host, guest and Docker+evaluation against `GetConfig()` and `UpdateConfig(GetConfig())` under+`-race`.+### Versioned alert-intent policyThe alerts runtime owns one versioned alert-intent document and its durablediff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.goindex f0ad060e0f..dea4577c90 100644--- a/internal/alerts/alerts_test.go+++ b/internal/alerts/alerts_test.go@@ -2756,6 +2756,101 @@ func TestCheckBackupsIgnoresVMIDs(t *testing.T) {}}+// Host and guest evaluation used to fill in the Clear of a disk override's+// threshold after dropping m.mu, writing through the live config's pointer+// while GetConfig cloned it under the read lock, and Docker evaluation read+// m.config.DockerDefaults with no lock while UpdateConfig replaced m.config.+// Evaluation now copies what it needs under m.mu. Run with -race.+func TestAlertEvaluationLeavesTheLiveConfigToTheLock(t *testing.T) {+ originalLogLevel := zerolog.GlobalLevel()+ zerolog.SetGlobalLevel(zerolog.Disabled)+ t.Cleanup(func() {+ zerolog.SetGlobalLevel(originalLogLevel)+ })++ m := newTestManager(t)++ host := models.Host{+ ID: "eval-lock",+ Hostname: "eval-lock",+ Status: "online",+ Disks: []models.Disk{{Mountpoint: "/", Device: "/dev/sda1", Usage: 50, Total: 100, Used: 50, Free: 50}},+ }+ hostDiskOverrideID, _ := hostDiskResourceID(host, host.Disks[0])+ vm := models.VM{+ ID: BuildGuestKey("pve1", "node1", 101),+ VMID: 101,+ Name: "eval-lock",+ Node: "node1",+ Instance: "pve1",+ Status: "running",+ Disks: []models.Disk{{Mountpoint: "/boot", Device: "/dev/vda1", Usage: 50, Total: 100, Used: 50, Free: 50}},+ }+ guestDiskOverrideID := "guest-disk:guest:pve1:101/disk:boot-dev-vda1"+ dockerHost := models.DockerHost{+ ID: "eval-lock-docker",+ Hostname: "eval-lock-docker",+ DisplayName: "eval-lock-docker",+ Containers: []models.DockerContainer{{+ ID: "abcdef123456",+ Name: "web",+ State: "running",+ CPUPercent: 10,+ MemoryPercent: 10,+ MemoryUsage: 100 << 20,+ MemoryLimit: 1 << 30,+ }},+ }++ // A disk override saved with only a trigger leaves Clear for evaluation+ // to default.+ withTriggerOnlyDiskOverrides := func(cfg AlertConfig) AlertConfig {+ cfg.Enabled = true+ cfg.ActivationState = ActivationActive+ cfg.Overrides = map[string]ThresholdConfig{+ hostDiskOverrideID: {Disk: &HysteresisThreshold{Trigger: 90}},+ guestDiskOverrideID: {Disk: &HysteresisThreshold{Trigger: 90}},+ }+ return cfg+ }+ m.UpdateConfig(withTriggerOnlyDiskOverrides(m.GetConfig()))++ m.CheckHost(host)+ m.CheckGuest(vm, "pve1")+ live := m.GetConfig()+ for _, id := range []string{hostDiskOverrideID, guestDiskOverrideID} {+ if got := live.Overrides[id].Disk.Clear; got != 0 {+ t.Fatalf("evaluating %s wrote clear %v into the live override", id, got)+ }+ }++ var wg sync.WaitGroup+ wg.Add(3)+ go func() {+ defer wg.Done()+ for i := 0; i < 100; i++ {+ // Resetting the overrides gives evaluation a Clear to fill again.+ m.UpdateConfig(withTriggerOnlyDiskOverrides(m.GetConfig()))+ m.UpdateConfig(m.GetConfig())+ }+ }()+ go func() {+ defer wg.Done()+ for i := 0; i < 200; i++ {+ _ = m.GetConfig()+ }+ }()+ go func() {+ defer wg.Done()+ for i := 0; i < 200; i++ {+ m.CheckHost(host)+ m.CheckGuest(vm, "pve1")+ m.CheckDockerHost(dockerHost)+ }+ }()+ wg.Wait()+}+func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {m := newTestManager(t)diff --git a/internal/alerts/docker.go b/internal/alerts/docker.goindex 3eaa0d370a..74071b851d 100644--- a/internal/alerts/docker.go+++ b/internal/alerts/docker.go@@ -352,8 +352,19 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container moderesourceType := "app-container"containerTags := dockerLabelTags(container.Labels)+ // Resolve the metric thresholds while the lock is held: the Docker defaults+ // are fields of m.config, which UpdateConfig replaces, and the override+ // still points into it. applyThresholdOverride clones what it keeps.m.mu.RLock()overrideConfig, hasOverride := m.lookupDockerContainerOverrideNoLock(host.ID, container.Name, resourceID)+ thresholds := ThresholdConfig{+ CPU: cloneThreshold(&m.config.DockerDefaults.CPU),+ Memory: cloneThreshold(&m.config.DockerDefaults.Memory),+ Disk: cloneThreshold(&m.config.DockerDefaults.Disk),+ }+ if hasOverride {+ thresholds = m.applyThresholdOverride(thresholds, overrideConfig)+ }m.mu.RUnlock()if hasOverride && overrideConfig.Disabled {// Alerts disabled via override; clear any existing alerts and skip evaluation.@@ -377,16 +388,6 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode} else {m.clearDockerContainerStateAlert(resourceID)- // Use Docker-specific defaults for containers- thresholds := ThresholdConfig{- CPU: &m.config.DockerDefaults.CPU,- Memory: &m.config.DockerDefaults.Memory,- Disk: &m.config.DockerDefaults.Disk,- }- if hasOverride {- thresholds = m.applyThresholdOverride(thresholds, overrideConfig)- }-if thresholds.CPU != nil {cpuCapacityPercent := models.DockerContainerCPUCapacityPercent(container, host.CPUs)cpuMetadata := map[string]interface{}{@@ -998,6 +999,8 @@ func (m *Manager) checkDockerContainerRestartLoop(host models.DockerHost, contaialertID := fmt.Sprintf("docker-container-restart-loop-%s", resourceID)now := time.Now()+ m.mu.Lock()+// Get config values with defaultsrestartThreshold := m.config.DockerDefaults.RestartCountif restartThreshold == 0 {@@ -1008,8 +1011,6 @@ func (m *Manager) checkDockerContainerRestartLoop(host models.DockerHost, contaitimeWindow = 300 // Default: 5 minutes (300 seconds)}- m.mu.Lock()-record, exists := m.dockerRestartTracking[resourceID]if !exists {record = &dockerRestartRecord{@@ -1164,11 +1165,13 @@ func (m *Manager) checkDockerContainerMemoryLimit(host models.DockerHost, contaialertID := fmt.Sprintf("docker-container-memory-limit-%s", resourceID)// Get config values with defaults+ m.mu.RLock()warnThreshold := float64(m.config.DockerDefaults.MemoryWarnPct)+ criticalThreshold := float64(m.config.DockerDefaults.MemoryCriticalPct)+ m.mu.RUnlock()if warnThreshold == 0 {warnThreshold = 90.0 // Default: 90%}- criticalThreshold := float64(m.config.DockerDefaults.MemoryCriticalPct)if criticalThreshold == 0 {criticalThreshold = 95.0 // Default: 95%}diff --git a/internal/alerts/guest.go b/internal/alerts/guest.goindex 25963829ef..edeac41cb3 100644--- a/internal/alerts/guest.go+++ b/internal/alerts/guest.go@@ -328,8 +328,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) {message := fmt.Sprintf("%s disk (%s) at %.1f%%", guestType, label, disk.Usage)effectiveDiskThreshold := thresholds.Disk+ // Own the override's Disk before ensureHysteresisThreshold fills+ // Clear: the returned copy still points into the live config.m.mu.RLock()diskOverride, hasDiskOverride := lookupGuestDiskOverride(m.config.Overrides, guest, guestID, keySource)+ diskOverride.Disk = cloneThreshold(diskOverride.Disk)m.mu.RUnlock()if hasDiskOverride {if diskOverride.Disabled {diff --git a/internal/alerts/host.go b/internal/alerts/host.goindex 65ae1cbd40..8c3edf1d7a 100644--- a/internal/alerts/host.go+++ b/internal/alerts/host.go@@ -450,9 +450,11 @@ func (m *Manager) CheckHost(host models.Host) {diskResourceID, diskName := hostDiskResourceID(host, disk)seenDisks[diskResourceID] = struct{}{}- // Check for disk-specific override+ // Check for disk-specific override. The copy's Disk still points into+ // the live config, so own it before ensureHysteresisThreshold fills Clear.m.mu.RLock()diskOverride, hasDiskOverride := m.config.Overrides[diskResourceID]+ diskOverride.Disk = cloneThreshold(diskOverride.Disk)m.mu.RUnlock()// Determine the effective disk thresholddiff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.goindex bfe670920d..112840229b 100644--- a/internal/alerts/metric_runtime.go+++ b/internal/alerts/metric_runtime.go@@ -86,16 +86,6 @@ func getThresholdForMetric(config ThresholdConfig, metricType string) *Hysteresi}}-// getThresholdForMetricFromConfig returns the threshold for a specific metric type from a ThresholdConfig-// ensuring hysteresis is properly set.-func getThresholdForMetricFromConfig(config ThresholdConfig, metricType string) *HysteresisThreshold {- th := getThresholdForMetric(config, metricType)- if th == nil {- return nil- }- return ensureHysteresisThreshold(th)-}-// getTimeThreshold determines the delay to apply for a metric/resource combination.func (m *Manager) getTimeThreshold(resourceID string, resourceType, metricType string) int {effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, MetricAlertIntentSignal(metricType))diff --git a/internal/alerts/utility_test.go b/internal/alerts/utility_test.goindex a4781633cf..f531aab6b5 100644--- a/internal/alerts/utility_test.go+++ b/internal/alerts/utility_test.go@@ -2469,148 +2469,3 @@ func TestGetThresholdForMetric_EmptyConfig(t *testing.T) {})}}--// TestGetThresholdForMetricFromConfig tests the getThresholdForMetricFromConfig function-func TestGetThresholdForMetricFromConfig(t *testing.T) {- // t.Parallel()-- tests := []struct {- name string- config ThresholdConfig- metricType string- wantNil bool- wantTrigger float64- wantClear float64- }{- {- name: "empty config returns nil",- config: ThresholdConfig{},- metricType: "cpu",- wantNil: true,- },- {- name: "cpu threshold returned",- config: ThresholdConfig{- CPU: &HysteresisThreshold{Trigger: 80, Clear: 70},- },- metricType: "cpu",- wantTrigger: 80,- wantClear: 70,- },- {- name: "memory threshold returned",- config: ThresholdConfig{- Memory: &HysteresisThreshold{Trigger: 85, Clear: 75},- },- metricType: "memory",- wantTrigger: 85,- wantClear: 75,- },- {- name: "disk threshold returned",- config: ThresholdConfig{- Disk: &HysteresisThreshold{Trigger: 90, Clear: 85},- },- metricType: "disk",- wantTrigger: 90,- wantClear: 85,- },- {- name: "diskRead threshold returned",- config: ThresholdConfig{- DiskRead: &HysteresisThreshold{Trigger: 50, Clear: 40},- },- metricType: "diskRead",- wantTrigger: 50,- wantClear: 40,- },- {- name: "diskWrite threshold returned",- config: ThresholdConfig{- DiskWrite: &HysteresisThreshold{Trigger: 55, Clear: 45},- },- metricType: "diskWrite",- wantTrigger: 55,- wantClear: 45,- },- {- name: "networkIn threshold returned",- config: ThresholdConfig{- NetworkIn: &HysteresisThreshold{Trigger: 70, Clear: 60},- },- metricType: "networkIn",- wantTrigger: 70,- wantClear: 60,- },- {- name: "networkOut threshold returned",- config: ThresholdConfig{- NetworkOut: &HysteresisThreshold{Trigger: 75, Clear: 65},- },- metricType: "networkOut",- wantTrigger: 75,- wantClear: 65,- },- {- name: "temperature threshold returned",- config: ThresholdConfig{- Temperature: &HysteresisThreshold{Trigger: 65, Clear: 55},- },- metricType: "temperature",- wantTrigger: 65,- wantClear: 55,- },- {- name: "usage threshold returned",- config: ThresholdConfig{- Usage: &HysteresisThreshold{Trigger: 88, Clear: 78},- },- metricType: "usage",- wantTrigger: 88,- wantClear: 78,- },- {- name: "unknown metric returns nil",- config: ThresholdConfig{- CPU: &HysteresisThreshold{Trigger: 80, Clear: 70},- },- metricType: "unknown",- wantNil: true,- },- {- name: "threshold with zero clear gets default hysteresis",- config: ThresholdConfig{- CPU: &HysteresisThreshold{Trigger: 80, Clear: 0},- },- metricType: "cpu",- wantTrigger: 80,- wantClear: 75, // 80 - 5 default margin- },- }-- for _, tc := range tests {- t.Run(tc.name, func(t *testing.T) {- // t.Parallel()-- result := getThresholdForMetricFromConfig(tc.config, tc.metricType)-- if tc.wantNil {- if result != nil {- t.Errorf("getThresholdForMetricFromConfig() = %v, want nil", result)- }- return- }-- if result == nil {- t.Fatalf("getThresholdForMetricFromConfig() = nil, want non-nil")- }-- if result.Trigger != tc.wantTrigger {- t.Errorf("Trigger = %v, want %v", result.Trigger, tc.wantTrigger)- }- if result.Clear != tc.wantClear {- t.Errorf("Clear = %v, want %v", result.Clear, tc.wantClear)- }- })- }-}