Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Give agent disk, array and sensor alerts their own resource type

rcourtman019963b60e33
diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md
index a306e9a58a..76661a4486 100644
--- a/docs/release-control/v6/internal/subsystems/alerts.md
+++ b/docs/release-control/v6/internal/subsystems/alerts.md
@@ -2475,13 +2475,23 @@ reads that ahead of every fallback. The menu's owner copy comes from
`frontend-modern/src/utils/resourceMonitoringPolicy.ts`, where an explicit
platform outranks the resource-type guess and the agent-removal copy is kept
to the agent's own machine: libvirt VMs and Unraid arrays in resource drawers
-get the neutral source-system copy. Agent disk-usage, SMART disk and Unraid
-array alerts still carry their host's `agent` metadata, so their menu names
-the Pulse agent until those producers emit child-resource metadata. Alerts
-restored from before `platformType` existed gain it at their next full
-evaluation; a metric alert held between its clear and trigger thresholds keeps
-the earlier fallback until then. Menu sentences open with the owner
-label capitalised.
+get the neutral source-system copy. Host-agent alerts
+(`internal/alerts/host.go`) stamp `platformType` `agent`, and only the
+machine's own alerts keep `resourceType` `agent`: filesystem,
+disk-temperature and SMART alerts carry `agent-disk`, RAID and Unraid array
+alerts `agent-storage`, and custom-sensor alerts `agent-sensor`, because the
+menu writes policy to their child resource id, never to the machine. Each
+child type keeps `agent` among its `CanonicalResourceTypeKeys`, and
+configuration re-evaluation treats agent policy as final for host-agent
+alerts (TrueNAS systems and vSphere hosts keep their platform policy), so the
+storage and guest switches cannot resolve a SMART, RAID or sensor alert that
+the next agent report would re-raise. A metric alert held between its clear
+and trigger thresholds still refreshes `resourceType` and `platformType` from
+its producer while its value, message and last-seen time stay at the last
+breach, so a restored alert that is still firing or held picks up the current
+classification at its next evaluation; one that resolves on that evaluation
+keeps the stored classification in its resolved record. Menu sentences open
+with the owner label capitalised.
The retired dashboard recent-alert panel must not be reintroduced as a
parallel alert surface. Alert summary/tone copy belongs to the alert overview
presentation owner, and any future compact alert surface must compose the
diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go
index fcfa75baa5..7e3a76580b 100644
--- a/internal/alerts/canonical_metric.go
+++ b/internal/alerts/canonical_metric.go
@@ -330,6 +330,15 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec
return
}
+ // A hold keeps the last breach's value and message, but what the
+ // alert is about is current: one restored from an older checkpoint,
+ // or raised before its producer named a child type or platform,
+ // must not link to the wrong page until it re-fires or clears.
+ if refreshMetricAlertClassification(existingAlert, alertMetadata) {
+ m.setActiveRecoveryAlert(existingAlert, storageKey)
+ m.saveActiveAlertsAsync("canonical metric classification")
+ }
+
if !triggered && primary == "" {
// Hysteresis latch: below trigger but above recovery — hold
// without refreshing, as the pre-cutover engine did.
@@ -424,3 +433,31 @@ func alertspecsMetricTriggered(spec *alertspecs.MetricThresholdSpec, observed fl
return false
}
}
+
+// metricAlertClassificationKeys name what a metric alert is about and which
+// platform owns it. They describe the resource, not the reading.
+var metricAlertClassificationKeys = []string{"resourceType", alertPlatformTypeKey}
+
+// refreshMetricAlertClassification copies the producer's current
+// classification onto an open alert and reports whether anything changed.
+func refreshMetricAlertClassification(alert *Alert, metadata map[string]interface{}) bool {
+ if alert == nil {
+ return false
+ }
+ changed := false
+ for _, key := range metricAlertClassificationKeys {
+ value, _ := metadata[key].(string)
+ if value == "" {
+ continue
+ }
+ if current, _ := alert.Metadata[key].(string); current == value {
+ continue
+ }
+ if alert.Metadata == nil {
+ alert.Metadata = make(map[string]interface{}, len(metricAlertClassificationKeys))
+ }
+ alert.Metadata[key] = value
+ changed = true
+ }
+ return changed
+}
diff --git a/internal/alerts/config/identity.go b/internal/alerts/config/identity.go
index fdaebaaa16..3908343ca8 100644
--- a/internal/alerts/config/identity.go
+++ b/internal/alerts/config/identity.go
@@ -57,6 +57,13 @@ func CanonicalResourceTypeKeys(resourceType string) []string {
keys = addUnique(keys, "agent-disk")
keys = addUnique(keys, "agent")
keys = addUnique(keys, "storage")
+ case "agent-storage":
+ keys = addUnique(keys, "agent-storage")
+ keys = addUnique(keys, "agent")
+ keys = addUnique(keys, "storage")
+ case "agent-sensor":
+ keys = addUnique(keys, "agent-sensor")
+ keys = addUnique(keys, "agent")
case "pbs":
keys = addUnique(keys, "pbs")
keys = addUnique(keys, "node")
diff --git a/internal/alerts/config_runtime.go b/internal/alerts/config_runtime.go
index 17d0ecf11e..6e803d5fa5 100644
--- a/internal/alerts/config_runtime.go
+++ b/internal/alerts/config_runtime.go
@@ -431,6 +431,16 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
continue
}
threshold = getThresholdForMetric(thresholds, metricType)
+ // Agent policy is the only policy for agent alerts. Falling
+ // through would let the storage or guest switches resolve a
+ // SMART, RAID or sensor alert that CheckHost re-raises on the
+ // next report.
+ if threshold == nil {
+ if isMetricThresholdAlertType(metricType) {
+ alertsToResolve = append(alertsToResolve, alertID)
+ }
+ continue
+ }
}
if alert.Type == "docker-host-offline" ||
diff --git a/internal/alerts/host.go b/internal/alerts/host.go
index cc2969046f..ceb74961dd 100644
--- a/internal/alerts/host.go
+++ b/internal/alerts/host.go
@@ -28,6 +28,27 @@ func hostResourceID(hostID string) string {
return fmt.Sprintf("agent:%s", trimmed)
}
+// Every alert CheckHost raises about something other than the machine itself
+// (a filesystem, disk, array or custom sensor) targets a child resource ID,
+// and the alert card's monitoring policy writes to that ID. These alerts
+// name their own type so the card does not describe retiring the machine.
+// Each type keeps "agent" among its alert policy keys
+// (config.CanonicalResourceTypeKeys), so agent thresholds and switches still
+// govern them.
+const (
+ hostDiskAlertResourceType = "agent-disk" // filesystems, disk temperatures, SMART disks
+ hostStorageAlertResourceType = "agent-storage" // RAID and Unraid arrays
+ hostSensorAlertResourceType = "agent-sensor" // custom sensors
+)
+
+// hostChildAlertMetadata clones a host's base alert metadata for an alert
+// about something the agent reports rather than the machine itself.
+func hostChildAlertMetadata(base map[string]interface{}, resourceType string) map[string]interface{} {
+ metadata := cloneMetadata(base)
+ metadata["resourceType"] = resourceType
+ return metadata
+}
+
func stripHostResourcePrefix(resourceID string) string {
trimmed := strings.TrimSpace(resourceID)
trimmed = strings.TrimPrefix(trimmed, "agent:")
@@ -273,15 +294,16 @@ func (m *Manager) CheckHost(host models.Host) {
instanceName := hostInstanceName(host)
baseMetadata := map[string]interface{}{
- "resourceType": "agent",
- "hostId": host.ID,
- "hostname": host.Hostname,
- "displayName": host.DisplayName,
- "platform": host.Platform,
- "osName": host.OSName,
- "osVersion": host.OSVersion,
- "agentVersion": host.AgentVersion,
- "architecture": host.Architecture,
+ "resourceType": "agent",
+ alertPlatformTypeKey: string(unifiedresources.SourceAgent),
+ "hostId": host.ID,
+ "hostname": host.Hostname,
+ "displayName": host.DisplayName,
+ "platform": host.Platform,
+ "osName": host.OSName,
+ "osVersion": host.OSVersion,
+ "agentVersion": host.AgentVersion,
+ "architecture": host.Architecture,
}
if linkedNodeID := strings.TrimSpace(host.LinkedNodeID); linkedNodeID != "" {
baseMetadata["linkedNodeId"] = linkedNodeID
@@ -359,7 +381,7 @@ func (m *Manager) CheckHost(host models.Host) {
tempResourceID := fmt.Sprintf("%s/disk_temp:%s", hostResourceID(host.ID), sanitizeHostComponent(disk.Device))
tempResourceName := fmt.Sprintf("%s (%s Temp)", hostDisplayName(host), disk.Device)
- diskTempMetadata := cloneMetadata(baseMetadata)
+ diskTempMetadata := hostChildAlertMetadata(baseMetadata, hostDiskAlertResourceType)
diskTempMetadata["metric"] = "diskTemperature"
diskTempMetadata["device"] = disk.Device
diskTempMetadata["temperature"] = disk.Temperature
@@ -446,7 +468,7 @@ func (m *Manager) CheckHost(host models.Host) {
continue
}
- diskMetadata := cloneMetadata(baseMetadata)
+ diskMetadata := hostChildAlertMetadata(baseMetadata, hostDiskAlertResourceType)
diskMetadata["metric"] = "disk"
diskMetadata["mountpoint"] = disk.Mountpoint
diskMetadata["device"] = disk.Device
@@ -517,7 +539,7 @@ func (m *Manager) CheckHost(host models.Host) {
raidName := fmt.Sprintf("%s - %s (%s)", resourceName, array.Device, array.Level)
raidSpecResourceID := fmt.Sprintf("%s/raid:%s", hostResourceID(host.ID), sanitizeRAIDDevice(array.Device))
- raidMetadata := cloneMetadata(baseMetadata)
+ raidMetadata := hostChildAlertMetadata(baseMetadata, hostStorageAlertResourceType)
raidMetadata["metric"] = "raid"
raidMetadata["raidDevice"] = array.Device
raidMetadata["raidLevel"] = array.Level
@@ -671,15 +693,16 @@ func (m *Manager) HandleHostTelemetryExpired(host models.Host) {
m.clearHostUnraidAlerts(host.ID)
} else {
baseMetadata := map[string]interface{}{
- "resourceType": "agent",
- "hostId": host.ID,
- "hostname": host.Hostname,
- "displayName": host.DisplayName,
- "platform": host.Platform,
- "osName": host.OSName,
- "osVersion": host.OSVersion,
- "agentVersion": host.AgentVersion,
- "architecture": host.Architecture,
+ "resourceType": "agent",
+ alertPlatformTypeKey: string(unifiedresources.SourceAgent),
+ "hostId": host.ID,
+ "hostname": host.Hostname,
+ "displayName": host.DisplayName,
+ "platform": host.Platform,
+ "osName": host.OSName,
+ "osVersion": host.OSVersion,
+ "agentVersion": host.AgentVersion,
+ "architecture": host.Architecture,
}
if linkedNodeID := strings.TrimSpace(host.LinkedNodeID); linkedNodeID != "" {
baseMetadata["linkedNodeId"] = linkedNodeID
@@ -793,16 +816,17 @@ func (m *Manager) HandleHostOfflineWithCorrelation(host models.Host, correlation
Message: fmt.Sprintf("Host '%s' is offline", resourceName),
Correlation: correlation,
Metadata: map[string]interface{}{
- "resourceType": "agent",
- "hostId": host.ID,
- "hostname": host.Hostname,
- "displayName": host.DisplayName,
- "platform": host.Platform,
- "osName": host.OSName,
- "osVersion": host.OSVersion,
- "linkedNodeId": strings.TrimSpace(host.LinkedNodeID),
- "linkedVmId": strings.TrimSpace(host.LinkedVMID),
- "linkedContainerId": strings.TrimSpace(host.LinkedContainerID),
+ "resourceType": "agent",
+ alertPlatformTypeKey: string(unifiedresources.SourceAgent),
+ "hostId": host.ID,
+ "hostname": host.Hostname,
+ "displayName": host.DisplayName,
+ "platform": host.Platform,
+ "osName": host.OSName,
+ "osVersion": host.OSVersion,
+ "linkedNodeId": strings.TrimSpace(host.LinkedNodeID),
+ "linkedVmId": strings.TrimSpace(host.LinkedVMID),
+ "linkedContainerId": strings.TrimSpace(host.LinkedContainerID),
},
AddToRecent: true,
AddToHistory: true,
@@ -965,7 +989,7 @@ func (m *Manager) syncHostCustomSensorAlerts(host models.Host, nodeName, instanc
}
}
- metadata := cloneMetadata(baseMetadata)
+ metadata := hostChildAlertMetadata(baseMetadata, hostSensorAlertResourceType)
metadata["metric"] = "customSensor"
metadata["customSensorId"] = metric.ID
metadata["customSensorName"] = metric.Name
@@ -1393,7 +1417,7 @@ func (m *Manager) syncHostSMARTDiskAlert(host models.Host, disk models.HostDiskS
reasonCodes := storageHealthReasonCodes(reasons)
reasonSummaries := storageHealthReasonSummaries(reasons)
- metadata := cloneMetadata(baseMetadata)
+ metadata := hostChildAlertMetadata(baseMetadata, hostDiskAlertResourceType)
metadata["metric"] = alertType
metadata["device"] = disk.Device
metadata["model"] = disk.Model
@@ -1474,7 +1498,7 @@ func (m *Manager) syncHostUnraidStorageAlert(host models.Host, nodeName, instanc
reasonCodes := storageHealthReasonCodes(reasons)
reasonSummaries := storageHealthReasonSummaries(reasons)
- metadata := cloneMetadata(baseMetadata)
+ metadata := hostChildAlertMetadata(baseMetadata, hostStorageAlertResourceType)
metadata["metric"] = "storageTopology"
metadata["storagePlatform"] = "unraid"
metadata["storageTopology"] = "array"
diff --git a/internal/alerts/host_unraid_lifecycle_test.go b/internal/alerts/host_unraid_lifecycle_test.go
index 3388516e35..8a9da9cbc6 100644
--- a/internal/alerts/host_unraid_lifecycle_test.go
+++ b/internal/alerts/host_unraid_lifecycle_test.go
@@ -305,3 +305,242 @@ func TestHostCustomSensorEscalationDelivery(t *testing.T) {
})
}
}
+
+// An agent reports things besides its own machine: filesystems, disks,
+// arrays and custom sensors. Their alerts target a child resource, and the
+// alert card's monitoring policy writes to that child, so they must not
+// carry the machine's "agent" type: the card would offer the machine's
+// retirement copy ("Agent removal remains available from Machines") for a
+// policy that never touches the machine. platformType keeps every one of
+// them linked to the agent's page.
+func TestHostChildAlertsNameTheirOwnResourceType(t *testing.T) {
+ m := newTestManager(t)
+ m.ClearActiveAlerts()
+ m.mu.Lock()
+ m.config.TimeThresholds = map[string]int{}
+ m.config.AgentDefaults.CPU = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.config.AgentDefaults.DiskTemperature = &HysteresisThreshold{Trigger: 50, Clear: 45}
+ m.mu.Unlock()
+
+ pending := int64(2)
+ sensorValue := 23.0
+ linuxHost := models.Host{
+ ID: "child-linux",
+ Hostname: "storage-host",
+ DisplayName: "Storage Host",
+ Platform: "linux",
+ CPUUsage: 95,
+ Disks: []models.Disk{{
+ Mountpoint: "/srv",
+ Device: "/dev/sdc1",
+ Usage: 92,
+ Total: 100,
+ Used: 92,
+ Free: 8,
+ }},
+ Sensors: models.HostSensorSummary{
+ SMART: []models.HostDiskSMART{{
+ Device: "/dev/sda",
+ Model: "IronWolf",
+ Serial: "SERIAL-CHILD-1",
+ Health: "PASSED",
+ Temperature: 55,
+ Attributes: &models.SMARTAttributes{PendingSectors: &pending},
+ }},
+ Custom: []models.HostCustomSensorMetric{{
+ ID: "queue_depth",
+ Name: "Queue depth",
+ Value: &sensorValue,
+ Status: "critical",
+ ObservedAt: time.Now().UTC(),
+ }},
+ },
+ RAID: []models.HostRAIDArray{{
+ Device: "/dev/md0",
+ Level: "raid1",
+ State: "degraded",
+ TotalDevices: 2,
+ ActiveDevices: 1,
+ WorkingDevices: 1,
+ FailedDevices: 1,
+ }},
+ }
+ unraidHost := models.Host{
+ ID: "child-unraid",
+ Hostname: "tower",
+ DisplayName: "Tower",
+ Platform: "unraid",
+ Unraid: &models.HostUnraidStorage{
+ ArrayStarted: true,
+ ArrayState: "STARTED",
+ Disks: []models.HostUnraidDisk{
+ {Name: "disk1", Role: "data", Status: "online", Device: "/dev/sdb"},
+ },
+ },
+ }
+ m.CheckHost(linuxHost)
+ m.CheckHost(unraidHost)
+
+ want := map[string]string{
+ "cpu": "agent",
+ "disk": "agent-disk",
+ "diskTemperature": "agent-disk",
+ "disk-health": "agent-disk",
+ "raid": "agent-storage",
+ "storage-topology": "agent-storage",
+ "custom-sensor": "agent-sensor",
+ }
+ seen := make(map[string]bool, len(want))
+ for _, alert := range m.GetActiveAlerts() {
+ wantType, ok := want[alert.Type]
+ if !ok {
+ continue
+ }
+ seen[alert.Type] = true
+ if got := alert.Metadata["resourceType"]; got != wantType {
+ t.Errorf("%s alert resourceType = %v, want %s", alert.Type, got, wantType)
+ }
+ if got := alert.Metadata["platformType"]; got != "agent" {
+ t.Errorf("%s alert platformType = %v, want agent", alert.Type, got)
+ }
+ hostID, _ := alert.Metadata["hostId"].(string)
+ if wantType != "agent" && alert.ResourceID == hostResourceID(hostID) {
+ t.Errorf("%s alert targets the machine itself (%s), so it is not a child alert", alert.Type, alert.ResourceID)
+ }
+ }
+ for alertType := range want {
+ if !seen[alertType] {
+ t.Errorf("expected an active %s alert", alertType)
+ }
+ }
+}
+
+// Child alerts keep the agent policy path. A configuration save that turns
+// off storage or guest alerts must leave them alone: CheckHost only consults
+// the agent switches, so resolving them here would re-raise and re-notify
+// on the next report.
+func TestHostChildAlertsIgnoreOtherPlatformSwitchesOnConfigSave(t *testing.T) {
+ m := newTestManager(t)
+ m.ClearActiveAlerts()
+ m.mu.Lock()
+ m.config.TimeThresholds = map[string]int{}
+ m.mu.Unlock()
+
+ pending := int64(2)
+ sensorValue := 23.0
+ host := models.Host{
+ ID: "child-policy",
+ Hostname: "storage-host",
+ Platform: "linux",
+ Sensors: models.HostSensorSummary{
+ SMART: []models.HostDiskSMART{{
+ Device: "/dev/sda",
+ Serial: "SERIAL-CHILD-2",
+ Health: "PASSED",
+ Attributes: &models.SMARTAttributes{PendingSectors: &pending},
+ }},
+ Custom: []models.HostCustomSensorMetric{{
+ ID: "queue_depth",
+ Name: "Queue depth",
+ Value: &sensorValue,
+ Status: "critical",
+ ObservedAt: time.Now().UTC(),
+ }},
+ },
+ RAID: []models.HostRAIDArray{{
+ Device: "/dev/md0",
+ Level: "raid1",
+ State: "degraded",
+ TotalDevices: 2,
+ ActiveDevices: 1,
+ WorkingDevices: 1,
+ FailedDevices: 1,
+ }},
+ }
+ m.CheckHost(host)
+ childTypes := []string{"disk-health", "raid", "custom-sensor"}
+ for _, alertType := range childTypes {
+ if !hasAlertType(m.GetActiveAlerts(), alertType) {
+ t.Fatalf("expected an active %s alert before the config save", alertType)
+ }
+ }
+
+ m.mu.Lock()
+ m.config.DisableAllStorage = true
+ m.config.DisableAllGuests = true
+ m.reevaluateActiveAlertsLocked()
+ m.mu.Unlock()
+
+ for _, alertType := range childTypes {
+ if !hasAlertType(m.GetActiveAlerts(), alertType) {
+ t.Errorf("%s alert was resolved by the storage or guest switch", alertType)
+ }
+ }
+
+ m.mu.Lock()
+ m.config.DisableAllAgents = true
+ m.reevaluateActiveAlertsLocked()
+ m.mu.Unlock()
+
+ for _, alertType := range childTypes {
+ if hasAlertType(m.GetActiveAlerts(), alertType) {
+ t.Errorf("%s alert survived disabling agent alerts", alertType)
+ }
+ }
+}
+
+// The Proxmox node sweep removes alerts whose node is not a Proxmox node
+// unless it recognises them as agent alerts, by an agent: resource id or the
+// agent type. Child alerts no longer carry the agent type, so they must keep
+// the agent: id their canonical spec gives them.
+func TestHostChildAlertsSurviveProxmoxNodeCleanup(t *testing.T) {
+ m := newTestManager(t)
+ m.ClearActiveAlerts()
+ pending := int64(2)
+ sensorValue := 23.0
+ host := models.Host{
+ ID: "child-sweep",
+ Hostname: "storage-host",
+ Platform: "linux",
+ Sensors: models.HostSensorSummary{
+ SMART: []models.HostDiskSMART{{
+ Device: "/dev/sda",
+ Serial: "SERIAL-CHILD-3",
+ Health: "PASSED",
+ Attributes: &models.SMARTAttributes{PendingSectors: &pending},
+ }},
+ Custom: []models.HostCustomSensorMetric{{
+ ID: "queue_depth",
+ Name: "Queue depth",
+ Value: &sensorValue,
+ Status: "critical",
+ ObservedAt: time.Now().UTC(),
+ }},
+ },
+ RAID: []models.HostRAIDArray{{
+ Device: "/dev/md0",
+ Level: "raid1",
+ State: "degraded",
+ TotalDevices: 2,
+ ActiveDevices: 1,
+ WorkingDevices: 1,
+ FailedDevices: 1,
+ }},
+ }
+ m.CheckHost(host)
+ childTypes := []string{"disk-health", "raid", "custom-sensor"}
+ for _, alertType := range childTypes {
+ if !hasAlertType(m.GetActiveAlerts(), alertType) {
+ t.Fatalf("expected an active %s alert", alertType)
+ }
+ }
+
+ m.CleanupAlertsForNodes(map[string]bool{"pve1": true})
+
+ for _, alertType := range childTypes {
+ if !hasAlertType(m.GetActiveAlerts(), alertType) {
+ t.Errorf("Proxmox node cleanup removed an agent %s alert", alertType)
+ }
+ }
+}
diff --git a/internal/alerts/migration_characterization_test.go b/internal/alerts/migration_characterization_test.go
index 17a2f075b9..d8814b3fa8 100644
--- a/internal/alerts/migration_characterization_test.go
+++ b/internal/alerts/migration_characterization_test.go
@@ -969,3 +969,93 @@ func TestAlertCharacterizationResolvedCallbackUsesCanonicalIDForCanonicalAliasCl
t.Fatalf("expected resolved callback for %q", alertID)
}
}
+
+// A held threshold alert keeps the value and message of its last breach, but
+// what the alert is about can be newer than the alert: one restored from a
+// checkpoint written before its producer named a child type or stamped
+// platformType would otherwise keep linking to the wrong page and naming the
+// wrong policy owner until it re-fires or clears.
+func TestAlertCharacterizationHeldMetricAlertRefreshesItsClassification(t *testing.T) {
+ type staleAlert struct {
+ resourceType string
+ platformType string
+ }
+ tests := []struct {
+ name string
+ alertID string
+ fire func(*Manager)
+ hold func(*Manager)
+ stale staleAlert
+ wantResourceType string
+ wantPlatform string
+ wantValue float64
+ }{
+ {
+ name: "agent filesystem",
+ alertID: canonicalMetricStateID("agent:held-host/disk:srv", "disk"),
+ fire: func(m *Manager) {
+ m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 92}}})
+ },
+ hold: func(m *Manager) {
+ m.CheckHost(models.Host{ID: "held-host", Hostname: "held", Disks: []models.Disk{{Mountpoint: "/srv", Device: "/dev/sdc1", Usage: 75}}})
+ },
+ stale: staleAlert{resourceType: "agent"},
+ wantResourceType: "agent-disk",
+ wantPlatform: "agent",
+ wantValue: 92,
+ },
+ {
+ name: "vSphere VM",
+ alertID: canonicalMetricStateID("vmware:vc-1:vm:vm-201", "disk"),
+ fire: func(m *Manager) {
+ m.CheckUnifiedResource(&UnifiedResourceInput{ID: "vmware:vc-1:vm:vm-201", Type: "vmware-vm", Name: "app-01", Node: "esxi-01", Instance: "Lab vCenter", Disk: &UnifiedResourceMetric{Percent: 94}})
+ },
+ hold: func(m *Manager) {
+ m.CheckUnifiedResource(&UnifiedResourceInput{ID: "vmware:vc-1:vm:vm-201", Type: "vmware-vm", Name: "app-01", Node: "esxi-01", Instance: "Lab vCenter", Disk: &UnifiedResourceMetric{Percent: 87}})
+ },
+ stale: staleAlert{resourceType: "vSphere VM"},
+ wantResourceType: "vSphere VM",
+ wantPlatform: "vmware-vsphere",
+ wantValue: 94,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ m := newTestManager(t)
+ configureUnifiedEvalManager(t, m, unifiedEvalBaseConfig())
+ m.mu.Lock()
+ m.config.AgentDefaults.Disk = &HysteresisThreshold{Trigger: 80, Clear: 70}
+ m.mu.Unlock()
+
+ tt.fire(m)
+ m.mu.Lock()
+ alert, ok := m.getActiveAlertNoLock(tt.alertID)
+ if !ok {
+ m.mu.Unlock()
+ t.Fatalf("expected active alert %q", tt.alertID)
+ }
+ alert.Metadata["resourceType"] = tt.stale.resourceType
+ delete(alert.Metadata, alertPlatformTypeKey)
+ m.mu.Unlock()
+
+ tt.hold(m)
+
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ held, ok := m.getActiveAlertNoLock(tt.alertID)
+ if !ok {
+ t.Fatal("held alert resolved; the reading should sit between clear and trigger")
+ }
+ if held.Value != tt.wantValue {
+ t.Fatalf("held alert value = %v, want the last breach %v", held.Value, tt.wantValue)
+ }
+ if got := held.Metadata["resourceType"]; got != tt.wantResourceType {
+ t.Fatalf("resourceType = %v, want %s", got, tt.wantResourceType)
+ }
+ if got := held.Metadata[alertPlatformTypeKey]; got != tt.wantPlatform {
+ t.Fatalf("platformType = %v, want %s", got, tt.wantPlatform)
+ }
+ })
+ }
+}