Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
// CheckStorage checks storage against thresholds
func (m *Manager) CheckStorage(storage models.Storage) {
m.CheckStorageWithCapacityTrend(storage, CapacityTrendObservation{})
}
// CheckStorageWithCapacityTrend evaluates static storage policy and optional
// predictive evidence as one canonical capacity lifecycle.
func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend CapacityTrendObservation) {
m.mu.RLock()
if !m.config.Enabled {
m.mu.RUnlock()
return
}
resourceIDs := storageAlertResourceIDs(storage)
if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("storage"); allDisabled {
m.mu.RUnlock()
// Clear any existing storage alerts when all storage alerts are disabled
m.mu.Lock()
for _, resourceID := range resourceIDs {
usageAlertID := canonicalMetricStateID(resourceID, "usage")
if m.clearActiveAlertIfPresentNoLock(usageAlertID) {
log.Info().
Str("alertID", usageAlertID).
Str("storage", storage.Name).
Msg("Cleared usage alert - all storage alerts disabled")
}
offlineAlertID := canonicalConnectivityStateID(resourceID)
if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
log.Info().
Str("alertID", offlineAlertID).
Str("storage", storage.Name).
Msg("Cleared offline alert - all storage alerts disabled")
}
}
m.mu.Unlock()
return
}
thresholds := m.resolveStorageThresholdsNoLock(storage)
m.mu.RUnlock()
if thresholds.Disabled {
m.mu.Lock()
for _, resourceID := range resourceIDs {
// Clear any pending core runs alongside the alerts.
m.core.ApplyDiscrete(reducer.DiscreteSignal{ResourceID: resourceID, Key: canonicalConnectivitySpecID(resourceID), Matched: false, ObservedAt: m.policyNow()}, reducer.DiscreteRule{})
m.core.ApplyMetric(reducer.MetricSignal{ResourceID: resourceID, Key: canonicalMetricSpecID(resourceID, "usage"), Metric: "usage", ObservedAt: m.policyNow()}, reducer.MetricRule{})
}
m.mu.Unlock()
for _, resourceID := range resourceIDs {
m.clearAlert(canonicalMetricStateID(resourceID, "usage"))
m.clearAlert(canonicalConnectivityStateID(resourceID))
}
return
}
m.clearStorageAliasAlerts(storage)
// Check if storage is truly offline/unavailable (not just inactive from other nodes)
// Note: In a cluster, local storage from other nodes shows as inactive which is normal
connectivityStatus := strings.ToLower(strings.TrimSpace(storage.Status))
if thresholds.DisableConnectivity {
m.mu.Lock()
for _, resourceID := range resourceIDs {
m.core.ApplyDiscrete(reducer.DiscreteSignal{ResourceID: resourceID, Key: canonicalConnectivitySpecID(resourceID), Matched: false, ObservedAt: m.policyNow()}, reducer.DiscreteRule{})
}
m.mu.Unlock()
for _, resourceID := range resourceIDs {
m.clearAlert(canonicalConnectivityStateID(resourceID))
}
} else if connectivityStatus == "offline" || connectivityStatus == "unavailable" {
m.checkStorageOffline(storage)
} else if connectivityStatus != "" && connectivityStatus != "unknown" {
// Clear any existing offline alert if storage is back online
m.clearStorageOfflineAlert(storage)
} else {
m.interruptStorageConnectivityRun(storage.ID)
}
// Missing connectivity evidence must not count as a healthy observation.
// Capacity and pool health below remain independently observable.
// Check usage if storage has valid data (even if not currently active on this node)
// In clusters, storage may show as inactive on nodes where it's not currently mounted
// but we still want to alert on high usage
log.Debug().
Str("storage", storage.Name).
Str("id", storage.ID).
Float64("usage", storage.Usage).
Str("status", storage.Status).
Float64("trigger", thresholds.Usage.Trigger).
Float64("clear", thresholds.Usage.Clear).
Msg("Checking storage thresholds")
// Static and forecast decisions both need an observed capacity, not just
// a positive percentage left beside unavailable/contradictory byte counts.
// Share filesystem admission while retaining storage's stricter empty-store
// confirmation: a default zero must not resolve an existing incident.
_, observed := unifiedresources.DiskUsagePercent(models.Disk{Total: storage.Total, Used: storage.Used, Usage: storage.Usage})
confirmedEmpty := storage.Usage == 0 && storage.Total > 0 && storage.Used == 0 && storage.Free == storage.Total
if !liveHysteresisThreshold(thresholds.Usage) {
// Explicit usage-policy disablement is not telemetry recovery. Release
// through the disabled evaluator even when capacity is unavailable or
// nonfinite; connectivity and pool health keep their separate policies.
policyOnly := storage
policyOnly.Usage = 0
m.evaluateStorageCapacity(policyOnly, thresholds, CapacityTrendObservation{})
} else if connectivityStatus != "offline" && connectivityStatus != "unavailable" && observed && (storage.Usage > 0 || confirmedEmpty) {
m.evaluateStorageCapacity(storage, thresholds, trend)
} else {
m.interruptMetricRunIDs(storage.ID, canonicalMetricSpecID(storage.ID, "usage"), canonicalMetricStateID(storage.ID, "usage"))
}
// Check ZFS pool status if this is ZFS storage
if storage.ZFSPool != nil {
m.checkZFSPoolHealth(storage)
} else {
m.clearStorageZFSAlerts(storage)
}
}
// A missing status is not a healthy poll, nor another outage confirmation.
// Preserve a firing occurrence while restarting both consecutive-observation
// runs and any pending intent grace. Capacity remains independently observable.
func (m *Manager) interruptStorageConnectivityRun(resourceID string) {
m.mu.Lock()
for _, state := range m.mirrorStatesNoLock() {
state.InterruptDiscreteRun(resourceID, canonicalConnectivitySpecID(resourceID))
}
intentChanged := m.clearIntentPendingNoLock(canonicalConnectivityStateID(resourceID))
m.mu.Unlock()
if intentChanged {
m.saveActiveAlertsAsync("storage connectivity observation gap")
}
}
// clearStorageZFSAlerts removes ZFS pool/device alerts for a storage that no
// longer carries an attached ZFS pool. checkZFSPoolHealth can only clear its
// own alerts while the pool stays attached, so a storage whose attachment goes
// away (a shared storage that was wrongly matched to a node-local pool, or a
// pool genuinely detached from a storage) would otherwise keep its stale ZFS
// alerts until the multi-day cleanup (#1731).
func (m *Manager) clearStorageZFSAlerts(storage models.Storage) {
storageID := strings.TrimSpace(storage.ID)
if storageID == "" {
return
}
compositePrefix := storageID + "/zfs-pool:"
m.mu.Lock()
defer m.mu.Unlock()
for alertID, alert := range m.activeAlerts {
if alert == nil {
continue
}
switch alert.Type {
case "zfs-pool-state", "zfs-pool-errors", "zfs-device":
default:
continue
}
resourceID := strings.TrimSpace(alert.ResourceID)
if resourceID != storageID && !strings.HasPrefix(resourceID, compositePrefix) {
continue
}
m.clearAlertNoLock(alertID)
}
}
// SyncStorageAlertsForInstance clears storage-related alerts whose underlying
// storage is no longer in the current inventory for an instance. This handles a
// storage disappearing or changing identity between polls; without it the alert
// stays visible until the generic multi-day stale-alert cleanup fires.
//
// Matching is by alert ResourceID (every v6 storage alert sets ResourceID =
// storage.ID, regardless of the composite alert-state key) so it is robust to
// the alert-key format. The sweep is conservative: it only touches alerts the
// instance owns that are recognised as storage-inventory alerts, and a live
// storage keeps its alert because its ResourceID is in the valid set. Ceph pool
// alias IDs are treated as valid so a deduped multi-source pool keeps its alert.
func (m *Manager) SyncStorageAlertsForInstance(instanceName string, storages []models.Storage) {
instanceName = strings.TrimSpace(instanceName)
if instanceName == "" {
return
}
validResourceIDs := make(map[string]struct{}, len(storages)*2)
for _, storage := range storages {
if strings.TrimSpace(storage.Instance) != instanceName {
continue
}
if id := strings.TrimSpace(storage.ID); id != "" {
validResourceIDs[id] = struct{}{}
}
for _, aliasID := range storage.AliasIDs {
if id := strings.TrimSpace(aliasID); id != "" {
validResourceIDs[id] = struct{}{}
}
}
}
m.mu.Lock()
defer m.mu.Unlock()
for alertID, alert := range m.activeAlerts {
if alert == nil {
continue
}
if strings.TrimSpace(alert.Instance) != instanceName {
continue
}
if !isStorageInventoryAlert(alert) {
continue
}
// Ceph pool storage alerts (id "<instance>-ceph-pool-<name>") are raised
// and cleared by the separate Ceph poll path, and Ceph pools are not in
// the `storages` inventory passed here. Never clear them, or they would
// flap (cleared every storage poll, re-raised every Ceph poll).
if strings.Contains(alert.ResourceID, "-ceph-pool-") {
continue
}
resourceID := strings.TrimSpace(alert.ResourceID)
if _, exists := validResourceIDs[resourceID]; exists {
continue
}
// Health assessment alerts (ZFS pool/device) carry a composite
// spec resource ID that extends the base storage ID with
// "/zfs-pool:…". Match those by checking whether the prefix
// before the first "/" is a valid storage ID so they are not
// erroneously cleared every poll cycle.
if idx := strings.Index(resourceID, "/"); idx > 0 {
if _, exists := validResourceIDs[resourceID[:idx]]; exists {
continue
}
}
m.clearAlertNoLock(alertID)
}
}
// isStorageInventoryAlert reports whether an alert belongs to the storage
// inventory, so it should be cleared when its storage leaves the inventory.
// Gated on the storage resourceType metadata that v6 stamps on storage alerts
// plus the ZFS pool/offline alert types; anything else is left untouched.
func isStorageInventoryAlert(alert *Alert) bool {
if alert == nil {
return false
}
if alert.Metadata != nil {
if resourceType, ok := alert.Metadata["resourceType"].(string); ok && strings.EqualFold(strings.TrimSpace(resourceType), "storage") {
return true
}
}
switch alert.Type {
case "storage", "usage", "zfs-pool-state", "zfs-pool-errors", "zfs-device", "offline":
return true
}
return false
}
func (m *Manager) clearStorageAliasAlerts(storage models.Storage) {
for i, resourceID := range storageAlertResourceIDs(storage) {
if i == 0 {
continue
}
m.clearAlert(canonicalMetricStateID(resourceID, "usage"))
m.clearAlert(canonicalConnectivityStateID(resourceID))
}
}
// checkZFSPoolHealth checks ZFS pool for errors and degraded state
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
pool := storage.ZFSPool
if pool == nil {
return
}
poolResourceName := fmt.Sprintf("%s (%s)", storage.Name, pool.Name)
poolAssessment := storagehealth.AssessZFSPool(*pool)
// Check pool state (DEGRADED, FAULTED, etc.)
stateAlertID := fmt.Sprintf("zfs-pool-state-%s", storage.ID)
// Pool and device alerts are raised under "<storage>/zfs-pool:..." ids,
// which name nothing the resource registry can open. storageResourceId
// names the storage the pool belongs to, so the Alerts overview can open
// it on the Storage page.
stateMetadata := map[string]interface{}{
"pool_name": pool.Name,
"pool_state": pool.State,
"storageResourceId": storage.ID,
}
stateReasons := filterStorageHealthReasonsByCodes(poolAssessment.Reasons, zfsPoolAssessmentCodes)
stateResult, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
SpecID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "-state",
Signal: "zfs-pool",
Codes: zfsPoolAssessmentCodes,
Reasons: stateReasons,
AlertID: stateAlertID,
AlertType: "zfs-pool-state",
SpecResourceID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name),
ResourceID: storage.ID,
ResourceName: poolResourceName,
ResourceType: unifiedresources.ResourceTypeStorage,
Node: storage.Node,
Instance: storage.Instance,
Metadata: stateMetadata,
})
if stateResult.Transition != nil && stateResult.Transition.Kind == alertspecs.EvaluationTransitionActivated {
log.Warn().
Str("pool", pool.Name).
Str("state", pool.State).
Str("node", storage.Node).
Msg("ZFS pool is not healthy")
}
// Check for read/write/checksum errors
totalErrors := pool.ReadErrors + pool.WriteErrors + pool.ChecksumErrors
errorsAlertID := fmt.Sprintf("zfs-pool-errors-%s", storage.ID)
errorsSpecResourceID := storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name)
errorsSpecID := errorsSpecResourceID + "-errors"
if totalErrors > 0 {
existingValue, exists := m.activeAlertValue(buildCanonicalStateID(errorsSpecResourceID, errorsSpecID))
if !exists || float64(totalErrors) > existingValue {
errorMetadata := map[string]interface{}{
"pool_name": pool.Name,
"read_errors": pool.ReadErrors,
"write_errors": pool.WriteErrors,
"checksum_errors": pool.ChecksumErrors,
"storageResourceId": storage.ID,
}
errorReasons := filterStorageHealthReasonsByCodes(poolAssessment.Reasons, zfsPoolErrorAssessmentCodes)
result, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
SpecID: errorsSpecID,
Signal: "zfs-pool-errors",
Codes: zfsPoolErrorAssessmentCodes,
Reasons: errorReasons,
AlertID: errorsAlertID,
AlertType: "zfs-pool-errors",
SpecResourceID: errorsSpecResourceID,
ResourceID: storage.ID,
ResourceName: poolResourceName,
ResourceType: unifiedresources.ResourceTypeStorage,
Node: storage.Node,
Instance: storage.Instance,
Metadata: errorMetadata,
MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
return fmt.Sprintf(
"ZFS pool '%s' has errors: %d read, %d write, %d checksum",
pool.Name, pool.ReadErrors, pool.WriteErrors, pool.ChecksumErrors,
), float64(totalErrors), 0
},
})
if result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated {
log.Error().
Str("pool", pool.Name).
Int64("read_errors", pool.ReadErrors).
Int64("write_errors", pool.WriteErrors).
Int64("checksum_errors", pool.ChecksumErrors).
Str("node", storage.Node).
Msg("ZFS pool has I/O errors")
}
}
} else {
m.clearAlert(buildCanonicalStateID(errorsSpecResourceID, errorsSpecID))
}
// Check individual devices for errors
for _, device := range pool.Devices {
alertID := fmt.Sprintf("zfs-device-%s-%s", storage.ID, device.Name)
deviceAssessment := zfsDeviceAssessment(device)
metadata := map[string]interface{}{
"pool_name": pool.Name,
"device_name": device.Name,
"device_state": device.State,
"read_errors": device.ReadErrors,
"write_errors": device.WriteErrors,
"checksum_errors": device.ChecksumErrors,
"storageResourceId": storage.ID,
}
result, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
SpecID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "/device:" + sanitizeHostComponent(device.Name) + "-health",
Signal: "zfs-device",
Codes: zfsDeviceAssessmentCodes,
Reasons: deviceAssessment.Reasons,
AlertID: alertID,
AlertType: "zfs-device",
SpecResourceID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "/device:" + sanitizeHostComponent(device.Name),
ResourceID: storage.ID,
ResourceName: formatZFSDeviceResourceName(storage.Name, pool.Name, device.Name),
ResourceType: unifiedresources.ResourceTypeStorage,
Node: storage.Node,
Instance: storage.Instance,
Metadata: metadata,
MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
return strings.Join(storageHealthReasonSummaries(deviceAssessment.Reasons), "; "), float64(device.ReadErrors + device.WriteErrors + device.ChecksumErrors), 0
},
})
if result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated {
log.Warn().
Str("pool", pool.Name).
Str("device", device.Name).
Str("state", device.State).
Int64("errors", device.ReadErrors+device.WriteErrors+device.ChecksumErrors).
Str("node", storage.Node).
Msg("ZFS device has issues")
}
}
}
func formatZFSDeviceResourceName(storageName, poolName, deviceName string) string {
storageName = strings.TrimSpace(storageName)
if storageName == "" {
storageName = "storage"
}
parts := make([]string, 0, 2)
if poolName = strings.TrimSpace(poolName); poolName != "" {
parts = append(parts, poolName)
}
if deviceName = strings.TrimSpace(deviceName); deviceName != "" {
parts = append(parts, deviceName)
}
if len(parts) == 0 {
return storageName
}
return fmt.Sprintf("%s (%s)", storageName, strings.Join(parts, ", "))
}
// checkStorageOffline creates an alert for offline/unavailable storage
func (m *Manager) checkStorageOffline(storage models.Storage) {
alertID := fmt.Sprintf("storage-offline-%s", storage.ID)
m.mu.Lock()
m.mu.Unlock()
m.mu.RLock()
thresholds := m.resolveStorageThresholdsNoLock(storage)
m.mu.RUnlock()
spec, err := buildCanonicalConnectivitySpec(storage.ID, storage.Name, unifiedresources.ResourceTypeStorage, AlertLevelWarning, 2, thresholds.Disabled || thresholds.DisableConnectivity)
if err != nil {
log.Warn().
Err(err).
Str("storage", storage.Name).
Str("storageID", storage.ID).
Msg("Skipping invalid canonical storage connectivity spec")
return
}
_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
Spec: spec,
Evidence: alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
AlertID: alertID,
AlertType: "offline",
ResourceID: storage.ID,
ResourceName: storage.Name,
Node: storage.Node,
Instance: storage.Instance,
Message: fmt.Sprintf("Storage %s on node %s is unavailable", storage.Name, storage.Node),
Summary: fmt.Sprintf("Unavailable on node %s", storage.Node),
Metadata: map[string]interface{}{
"resourceType": "storage",
"status": storage.Status,
},
RateLimit: true,
DispatchAsync: true,
})
}
// clearStorageOfflineAlert removes offline alert when storage comes back online
func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
m.clearResourceOfflineAlert(storage.ID, storage.Name, storage.Node, "Storage", offlineRecoveryConfirmationsStorage)
}