Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

18.7 KB
Raw Blame
package alerts

import (
	"fmt"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/reducer"
	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

// CheckStorage checks storage against thresholds
func (m *Manager) CheckStorage(storage models.Storage) {
	m.CheckStorageWithCapacityTrend(storage, CapacityTrendObservation{})
}

// CheckStorageWithCapacityTrend evaluates static storage policy and optional
// predictive evidence as one canonical capacity lifecycle.
func (m *Manager) CheckStorageWithCapacityTrend(storage models.Storage, trend CapacityTrendObservation) {
	m.mu.RLock()
	if !m.config.Enabled {
		m.mu.RUnlock()
		return
	}
	resourceIDs := storageAlertResourceIDs(storage)
	if allDisabled, _ := m.alertPolicyTypeSwitchesNoLock("storage"); allDisabled {
		m.mu.RUnlock()
		// Clear any existing storage alerts when all storage alerts are disabled
		m.mu.Lock()
		for _, resourceID := range resourceIDs {
			usageAlertID := canonicalMetricStateID(resourceID, "usage")
			if m.clearActiveAlertIfPresentNoLock(usageAlertID) {
				log.Info().
					Str("alertID", usageAlertID).
					Str("storage", storage.Name).
					Msg("Cleared usage alert - all storage alerts disabled")
			}
			offlineAlertID := canonicalConnectivityStateID(resourceID)
			if m.clearActiveAlertIfPresentNoLock(offlineAlertID) {
				log.Info().
					Str("alertID", offlineAlertID).
					Str("storage", storage.Name).
					Msg("Cleared offline alert - all storage alerts disabled")
			}
		}
		m.mu.Unlock()
		return
	}

	thresholds := m.resolveStorageThresholdsNoLock(storage)
	m.mu.RUnlock()

	if thresholds.Disabled {
		m.mu.Lock()
		for _, resourceID := range resourceIDs {
			// Clear any pending core runs alongside the alerts.
			m.core.ApplyDiscrete(reducer.DiscreteSignal{ResourceID: resourceID, Key: canonicalConnectivitySpecID(resourceID), Matched: false, ObservedAt: m.policyNow()}, reducer.DiscreteRule{})
			m.core.ApplyMetric(reducer.MetricSignal{ResourceID: resourceID, Key: canonicalMetricSpecID(resourceID, "usage"), Metric: "usage", ObservedAt: m.policyNow()}, reducer.MetricRule{})
		}
		m.mu.Unlock()
		for _, resourceID := range resourceIDs {
			m.clearAlert(canonicalMetricStateID(resourceID, "usage"))
			m.clearAlert(canonicalConnectivityStateID(resourceID))
		}
		return
	}

	m.clearStorageAliasAlerts(storage)

	// Check if storage is truly offline/unavailable (not just inactive from other nodes)
	// Note: In a cluster, local storage from other nodes shows as inactive which is normal
	connectivityStatus := strings.ToLower(strings.TrimSpace(storage.Status))
	if thresholds.DisableConnectivity {
		m.mu.Lock()
		for _, resourceID := range resourceIDs {
			m.core.ApplyDiscrete(reducer.DiscreteSignal{ResourceID: resourceID, Key: canonicalConnectivitySpecID(resourceID), Matched: false, ObservedAt: m.policyNow()}, reducer.DiscreteRule{})
		}
		m.mu.Unlock()
		for _, resourceID := range resourceIDs {
			m.clearAlert(canonicalConnectivityStateID(resourceID))
		}
	} else if connectivityStatus == "offline" || connectivityStatus == "unavailable" {
		m.checkStorageOffline(storage)
	} else if connectivityStatus != "" && connectivityStatus != "unknown" {
		// Clear any existing offline alert if storage is back online
		m.clearStorageOfflineAlert(storage)
	} else {
		m.interruptStorageConnectivityRun(storage.ID)
	}
	// Missing connectivity evidence must not count as a healthy observation.
	// Capacity and pool health below remain independently observable.

	// Check usage if storage has valid data (even if not currently active on this node)
	// In clusters, storage may show as inactive on nodes where it's not currently mounted
	// but we still want to alert on high usage
	log.Debug().
		Str("storage", storage.Name).
		Str("id", storage.ID).
		Float64("usage", storage.Usage).
		Str("status", storage.Status).
		Float64("trigger", thresholds.Usage.Trigger).
		Float64("clear", thresholds.Usage.Clear).
		Msg("Checking storage thresholds")

	// Static and forecast decisions both need an observed capacity, not just
	// a positive percentage left beside unavailable/contradictory byte counts.
	// Share filesystem admission while retaining storage's stricter empty-store
	// confirmation: a default zero must not resolve an existing incident.
	_, observed := unifiedresources.DiskUsagePercent(models.Disk{Total: storage.Total, Used: storage.Used, Usage: storage.Usage})
	confirmedEmpty := storage.Usage == 0 && storage.Total > 0 && storage.Used == 0 && storage.Free == storage.Total
	if !liveHysteresisThreshold(thresholds.Usage) {
		// Explicit usage-policy disablement is not telemetry recovery. Release
		// through the disabled evaluator even when capacity is unavailable or
		// nonfinite; connectivity and pool health keep their separate policies.
		policyOnly := storage
		policyOnly.Usage = 0
		m.evaluateStorageCapacity(policyOnly, thresholds, CapacityTrendObservation{})
	} else if connectivityStatus != "offline" && connectivityStatus != "unavailable" && observed && (storage.Usage > 0 || confirmedEmpty) {
		m.evaluateStorageCapacity(storage, thresholds, trend)
	} else {
		m.interruptMetricRunIDs(storage.ID, canonicalMetricSpecID(storage.ID, "usage"), canonicalMetricStateID(storage.ID, "usage"))
	}

	// Check ZFS pool status if this is ZFS storage
	if storage.ZFSPool != nil {
		m.checkZFSPoolHealth(storage)
	} else {
		m.clearStorageZFSAlerts(storage)
	}
}

// A missing status is not a healthy poll, nor another outage confirmation.
// Preserve a firing occurrence while restarting both consecutive-observation
// runs and any pending intent grace. Capacity remains independently observable.
func (m *Manager) interruptStorageConnectivityRun(resourceID string) {
	m.mu.Lock()
	for _, state := range m.mirrorStatesNoLock() {
		state.InterruptDiscreteRun(resourceID, canonicalConnectivitySpecID(resourceID))
	}
	intentChanged := m.clearIntentPendingNoLock(canonicalConnectivityStateID(resourceID))
	m.mu.Unlock()
	if intentChanged {
		m.saveActiveAlertsAsync("storage connectivity observation gap")
	}
}

// clearStorageZFSAlerts removes ZFS pool/device alerts for a storage that no
// longer carries an attached ZFS pool. checkZFSPoolHealth can only clear its
// own alerts while the pool stays attached, so a storage whose attachment goes
// away (a shared storage that was wrongly matched to a node-local pool, or a
// pool genuinely detached from a storage) would otherwise keep its stale ZFS
// alerts until the multi-day cleanup (#1731).
func (m *Manager) clearStorageZFSAlerts(storage models.Storage) {
	storageID := strings.TrimSpace(storage.ID)
	if storageID == "" {
		return
	}
	compositePrefix := storageID + "/zfs-pool:"

	m.mu.Lock()
	defer m.mu.Unlock()
	for alertID, alert := range m.activeAlerts {
		if alert == nil {
			continue
		}
		switch alert.Type {
		case "zfs-pool-state", "zfs-pool-errors", "zfs-device":
		default:
			continue
		}
		resourceID := strings.TrimSpace(alert.ResourceID)
		if resourceID != storageID && !strings.HasPrefix(resourceID, compositePrefix) {
			continue
		}
		m.clearAlertNoLock(alertID)
	}
}

// SyncStorageAlertsForInstance clears storage-related alerts whose underlying
// storage is no longer in the current inventory for an instance. This handles a
// storage disappearing or changing identity between polls; without it the alert
// stays visible until the generic multi-day stale-alert cleanup fires.
//
// Matching is by alert ResourceID (every v6 storage alert sets ResourceID =
// storage.ID, regardless of the composite alert-state key) so it is robust to
// the alert-key format. The sweep is conservative: it only touches alerts the
// instance owns that are recognised as storage-inventory alerts, and a live
// storage keeps its alert because its ResourceID is in the valid set. Ceph pool
// alias IDs are treated as valid so a deduped multi-source pool keeps its alert.
func (m *Manager) SyncStorageAlertsForInstance(instanceName string, storages []models.Storage) {
	instanceName = strings.TrimSpace(instanceName)
	if instanceName == "" {
		return
	}

	validResourceIDs := make(map[string]struct{}, len(storages)*2)
	for _, storage := range storages {
		if strings.TrimSpace(storage.Instance) != instanceName {
			continue
		}
		if id := strings.TrimSpace(storage.ID); id != "" {
			validResourceIDs[id] = struct{}{}
		}
		for _, aliasID := range storage.AliasIDs {
			if id := strings.TrimSpace(aliasID); id != "" {
				validResourceIDs[id] = struct{}{}
			}
		}
	}

	m.mu.Lock()
	defer m.mu.Unlock()

	for alertID, alert := range m.activeAlerts {
		if alert == nil {
			continue
		}
		if strings.TrimSpace(alert.Instance) != instanceName {
			continue
		}
		if !isStorageInventoryAlert(alert) {
			continue
		}
		// Ceph pool storage alerts (id "<instance>-ceph-pool-<name>") are raised
		// and cleared by the separate Ceph poll path, and Ceph pools are not in
		// the `storages` inventory passed here. Never clear them, or they would
		// flap (cleared every storage poll, re-raised every Ceph poll).
		if strings.Contains(alert.ResourceID, "-ceph-pool-") {
			continue
		}
		resourceID := strings.TrimSpace(alert.ResourceID)
		if _, exists := validResourceIDs[resourceID]; exists {
			continue
		}
		// Health assessment alerts (ZFS pool/device) carry a composite
		// spec resource ID that extends the base storage ID with
		// "/zfs-pool:…".  Match those by checking whether the prefix
		// before the first "/" is a valid storage ID so they are not
		// erroneously cleared every poll cycle.
		if idx := strings.Index(resourceID, "/"); idx > 0 {
			if _, exists := validResourceIDs[resourceID[:idx]]; exists {
				continue
			}
		}
		m.clearAlertNoLock(alertID)
	}
}

// isStorageInventoryAlert reports whether an alert belongs to the storage
// inventory, so it should be cleared when its storage leaves the inventory.
// Gated on the storage resourceType metadata that v6 stamps on storage alerts
// plus the ZFS pool/offline alert types; anything else is left untouched.
func isStorageInventoryAlert(alert *Alert) bool {
	if alert == nil {
		return false
	}

	if alert.Metadata != nil {
		if resourceType, ok := alert.Metadata["resourceType"].(string); ok && strings.EqualFold(strings.TrimSpace(resourceType), "storage") {
			return true
		}
	}

	switch alert.Type {
	case "storage", "usage", "zfs-pool-state", "zfs-pool-errors", "zfs-device", "offline":
		return true
	}

	return false
}

func (m *Manager) clearStorageAliasAlerts(storage models.Storage) {
	for i, resourceID := range storageAlertResourceIDs(storage) {
		if i == 0 {
			continue
		}
		m.clearAlert(canonicalMetricStateID(resourceID, "usage"))
		m.clearAlert(canonicalConnectivityStateID(resourceID))
	}
}

// checkZFSPoolHealth checks ZFS pool for errors and degraded state
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
	pool := storage.ZFSPool
	if pool == nil {
		return
	}

	poolResourceName := fmt.Sprintf("%s (%s)", storage.Name, pool.Name)
	poolAssessment := storagehealth.AssessZFSPool(*pool)

	// Check pool state (DEGRADED, FAULTED, etc.)
	stateAlertID := fmt.Sprintf("zfs-pool-state-%s", storage.ID)
	// Pool and device alerts are raised under "<storage>/zfs-pool:..." ids,
	// which name nothing the resource registry can open. storageResourceId
	// names the storage the pool belongs to, so the Alerts overview can open
	// it on the Storage page.
	stateMetadata := map[string]interface{}{
		"pool_name":         pool.Name,
		"pool_state":        pool.State,
		"storageResourceId": storage.ID,
	}
	stateReasons := filterStorageHealthReasonsByCodes(poolAssessment.Reasons, zfsPoolAssessmentCodes)
	stateResult, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
		SpecID:         storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "-state",
		Signal:         "zfs-pool",
		Codes:          zfsPoolAssessmentCodes,
		Reasons:        stateReasons,
		AlertID:        stateAlertID,
		AlertType:      "zfs-pool-state",
		SpecResourceID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name),
		ResourceID:     storage.ID,
		ResourceName:   poolResourceName,
		ResourceType:   unifiedresources.ResourceTypeStorage,
		Node:           storage.Node,
		Instance:       storage.Instance,
		Metadata:       stateMetadata,
	})
	if stateResult.Transition != nil && stateResult.Transition.Kind == alertspecs.EvaluationTransitionActivated {
		log.Warn().
			Str("pool", pool.Name).
			Str("state", pool.State).
			Str("node", storage.Node).
			Msg("ZFS pool is not healthy")
	}

	// Check for read/write/checksum errors
	totalErrors := pool.ReadErrors + pool.WriteErrors + pool.ChecksumErrors
	errorsAlertID := fmt.Sprintf("zfs-pool-errors-%s", storage.ID)
	errorsSpecResourceID := storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name)
	errorsSpecID := errorsSpecResourceID + "-errors"
	if totalErrors > 0 {
		existingValue, exists := m.activeAlertValue(buildCanonicalStateID(errorsSpecResourceID, errorsSpecID))
		if !exists || float64(totalErrors) > existingValue {
			errorMetadata := map[string]interface{}{
				"pool_name":         pool.Name,
				"read_errors":       pool.ReadErrors,
				"write_errors":      pool.WriteErrors,
				"checksum_errors":   pool.ChecksumErrors,
				"storageResourceId": storage.ID,
			}
			errorReasons := filterStorageHealthReasonsByCodes(poolAssessment.Reasons, zfsPoolErrorAssessmentCodes)
			result, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
				SpecID:         errorsSpecID,
				Signal:         "zfs-pool-errors",
				Codes:          zfsPoolErrorAssessmentCodes,
				Reasons:        errorReasons,
				AlertID:        errorsAlertID,
				AlertType:      "zfs-pool-errors",
				SpecResourceID: errorsSpecResourceID,
				ResourceID:     storage.ID,
				ResourceName:   poolResourceName,
				ResourceType:   unifiedresources.ResourceTypeStorage,
				Node:           storage.Node,
				Instance:       storage.Instance,
				Metadata:       errorMetadata,
				MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
					return fmt.Sprintf(
						"ZFS pool '%s' has errors: %d read, %d write, %d checksum",
						pool.Name, pool.ReadErrors, pool.WriteErrors, pool.ChecksumErrors,
					), float64(totalErrors), 0
				},
			})
			if result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated {
				log.Error().
					Str("pool", pool.Name).
					Int64("read_errors", pool.ReadErrors).
					Int64("write_errors", pool.WriteErrors).
					Int64("checksum_errors", pool.ChecksumErrors).
					Str("node", storage.Node).
					Msg("ZFS pool has I/O errors")
			}
		}
	} else {
		m.clearAlert(buildCanonicalStateID(errorsSpecResourceID, errorsSpecID))
	}

	// Check individual devices for errors
	for _, device := range pool.Devices {
		alertID := fmt.Sprintf("zfs-device-%s-%s", storage.ID, device.Name)
		deviceAssessment := zfsDeviceAssessment(device)
		metadata := map[string]interface{}{
			"pool_name":         pool.Name,
			"device_name":       device.Name,
			"device_state":      device.State,
			"read_errors":       device.ReadErrors,
			"write_errors":      device.WriteErrors,
			"checksum_errors":   device.ChecksumErrors,
			"storageResourceId": storage.ID,
		}
		result, _ := m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
			SpecID:         storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "/device:" + sanitizeHostComponent(device.Name) + "-health",
			Signal:         "zfs-device",
			Codes:          zfsDeviceAssessmentCodes,
			Reasons:        deviceAssessment.Reasons,
			AlertID:        alertID,
			AlertType:      "zfs-device",
			SpecResourceID: storage.ID + "/zfs-pool:" + sanitizeHostComponent(pool.Name) + "/device:" + sanitizeHostComponent(device.Name),
			ResourceID:     storage.ID,
			ResourceName:   formatZFSDeviceResourceName(storage.Name, pool.Name, device.Name),
			ResourceType:   unifiedresources.ResourceTypeStorage,
			Node:           storage.Node,
			Instance:       storage.Instance,
			Metadata:       metadata,
			MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
				return strings.Join(storageHealthReasonSummaries(deviceAssessment.Reasons), "; "), float64(device.ReadErrors + device.WriteErrors + device.ChecksumErrors), 0
			},
		})
		if result.Transition != nil && result.Transition.Kind == alertspecs.EvaluationTransitionActivated {
			log.Warn().
				Str("pool", pool.Name).
				Str("device", device.Name).
				Str("state", device.State).
				Int64("errors", device.ReadErrors+device.WriteErrors+device.ChecksumErrors).
				Str("node", storage.Node).
				Msg("ZFS device has issues")
		}
	}
}

func formatZFSDeviceResourceName(storageName, poolName, deviceName string) string {
	storageName = strings.TrimSpace(storageName)
	if storageName == "" {
		storageName = "storage"
	}

	parts := make([]string, 0, 2)
	if poolName = strings.TrimSpace(poolName); poolName != "" {
		parts = append(parts, poolName)
	}
	if deviceName = strings.TrimSpace(deviceName); deviceName != "" {
		parts = append(parts, deviceName)
	}
	if len(parts) == 0 {
		return storageName
	}
	return fmt.Sprintf("%s (%s)", storageName, strings.Join(parts, ", "))
}

// checkStorageOffline creates an alert for offline/unavailable storage
func (m *Manager) checkStorageOffline(storage models.Storage) {
	alertID := fmt.Sprintf("storage-offline-%s", storage.ID)

	m.mu.Lock()
	m.mu.Unlock()

	m.mu.RLock()
	thresholds := m.resolveStorageThresholdsNoLock(storage)
	m.mu.RUnlock()
	spec, err := buildCanonicalConnectivitySpec(storage.ID, storage.Name, unifiedresources.ResourceTypeStorage, AlertLevelWarning, 2, thresholds.Disabled || thresholds.DisableConnectivity)
	if err != nil {
		log.Warn().
			Err(err).
			Str("storage", storage.Name).
			Str("storageID", storage.ID).
			Msg("Skipping invalid canonical storage connectivity spec")
		return
	}

	_, _ = m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{
		Spec:         spec,
		Evidence:     alertspecs.AlertEvidence{ObservedAt: time.Now(), Connectivity: &alertspecs.ConnectivityEvidence{Signal: "status", Connected: false}},
		AlertID:      alertID,
		AlertType:    "offline",
		ResourceID:   storage.ID,
		ResourceName: storage.Name,
		Node:         storage.Node,
		Instance:     storage.Instance,
		Message:      fmt.Sprintf("Storage %s on node %s is unavailable", storage.Name, storage.Node),
		Summary:      fmt.Sprintf("Unavailable on node %s", storage.Node),
		Metadata: map[string]interface{}{
			"resourceType": "storage",
			"status":       storage.Status,
		},
		RateLimit:     true,
		DispatchAsync: true,
	})
}

// clearStorageOfflineAlert removes offline alert when storage comes back online
func (m *Manager) clearStorageOfflineAlert(storage models.Storage) {
	m.clearResourceOfflineAlert(storage.ID, storage.Name, storage.Node, "Storage", offlineRecoveryConfirmationsStorage)
}