Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

12.3 KB
Raw Blame
package alerts

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/henrygd/beszel/internal/entities/system"

	"ganigeorgiev.com/pocketbase/dbx"
	"ganigeorgiev.com/pocketbase/pocketbase/core"
	"ganigeorgiev.com/pocketbase/pocketbase/tools/types"
)

var cpuStateAlerts = map[string]struct {
	index int
	label string
}{
	"CPUIOWait": {2, "CPU I/O Wait"},
	"CPUSteal":  {3, "CPU Steal Time"},
}

func cpuStateAlertValue(name string, breakdown []float64) (float64, bool) {
	state, ok := cpuStateAlerts[name]
	if !ok || len(breakdown) < 5 {
		return 0, false
	}
	var total float64
	for _, value := range breakdown {
		total += value
	}
	if total <= 0 {
		return 0, false
	}
	return breakdown[state.index], true
}

func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error {
	// Systemd alerts are binary state, not numeric thresholds, so they're handled
	// separately. They read their own state from the database and don't use data.
	if err := am.HandleSystemdAlerts(systemRecord); err != nil {
		am.hub.Logger().Error("Error handling systemd alerts", "err", err)
	}

	if data == nil {
		return nil
	}

	alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert)
	if len(alerts) == 0 {
		return nil
	}

	var validAlerts []SystemAlertData
	now := systemRecord.GetDateTime("updated").Time().UTC()
	oldestTime := now

	for _, alertData := range alerts {
		name := alertData.Name
		var val float64
		unit := "%"

		switch name {
		case "CPU":
			val = data.Info.Cpu
		case "Memory":
			val = data.Info.MemPct
		case "Bandwidth":
			val = float64(data.Info.BandwidthBytes) / (1024 * 1024)
			unit = " MB/s"
		case "Disk":
			maxUsedPct := data.Info.DiskPct
			for _, fs := range data.Stats.ExtraFs {
				if fs == nil {
					continue
				}
				usedPct := fs.DiskUsed / fs.DiskTotal * 100
				if usedPct > maxUsedPct {
					maxUsedPct = usedPct
				}
			}
			for _, pool := range data.Stats.ZfsPools {
				if pool != nil && !pool.Raw && pool.Total > 0 {
					usedPct := pool.Used / pool.Total * 100
					if usedPct > maxUsedPct {
						maxUsedPct = usedPct
					}
				}
			}
			val = maxUsedPct
		case "Temperature":
			if data.Info.DashboardTemp < 1 {
				continue
			}
			val = data.Info.DashboardTemp
			unit = "°C"
		case "LoadAvg1":
			val = data.Info.LoadAvg[0]
			unit = ""
		case "LoadAvg5":
			val = data.Info.LoadAvg[1]
			unit = ""
		case "LoadAvg15":
			val = data.Info.LoadAvg[2]
			unit = ""
		case "GPU":
			val = data.Info.GpuPct
		case "Battery":
			if !hasRepresentativeBattery(data.Stats.Battery, data.Stats.Batteries) {
				continue
			}
			val = float64(data.Stats.Battery[0])
		default:
			var ok bool
			if val, ok = cpuStateAlertValue(name, data.Stats.CpuBreakdown); !ok {
				continue
			}
		}

		triggered := alertData.Triggered
		threshold := alertData.Value

		// Battery alert has inverted logic: trigger when value is BELOW threshold
		lowAlert := isLowAlert(name)

		// CONTINUE
		// For normal alerts: IF not triggered and curValue <= threshold, OR triggered and curValue > threshold
		// For low alerts (Battery): IF not triggered and curValue >= threshold, OR triggered and curValue < threshold
		if lowAlert {
			if (!triggered && val >= threshold) || (triggered && val < threshold) {
				continue
			}
		} else {
			if (!triggered && val <= threshold) || (triggered && val > threshold) {
				continue
			}
		}

		min := max(1, alertData.Min)

		alert := SystemAlertData{
			systemRecord: systemRecord,
			alertData:    alertData,
			name:         name,
			unit:         unit,
			val:          val,
			threshold:    threshold,
			triggered:    triggered,
			min:          min,
		}

		// send alert immediately if min is 1 - no need to sum up values.
		if min == 1 {
			if lowAlert {
				alert.triggered = val < threshold
			} else {
				alert.triggered = val > threshold
			}
			go am.sendSystemAlert(alert)
			continue
		}

		alert.time = now.Add(-time.Duration(min) * time.Minute)
		if alert.time.Before(oldestTime) {
			oldestTime = alert.time
		}

		validAlerts = append(validAlerts, alert)
	}

	systemStats := []struct {
		Stats   []byte         `db:"stats"`
		Created types.DateTime `db:"created"`
	}{}

	err := am.hub.DB().
		Select("stats", "created").
		From("system_stats").
		Where(dbx.NewExp(
			"system={:system} AND type='1m' AND created > {:created}",
			dbx.Params{
				"system": systemRecord.Id,
				// subtract some time to give us a bit of buffer
				"created": oldestTime.Add(-time.Second * 90),
			},
		)).
		OrderBy("created").
		All(&systemStats)
	if err != nil || len(systemStats) == 0 {
		return err
	}

	// get oldest record creation time from first record in the slice
	oldestRecordTime := systemStats[0].Created.Time()
	// log.Println("oldestRecordTime", oldestRecordTime.String())

	// Filter validAlerts to keep only those with time newer than oldestRecord
	filteredAlerts := make([]SystemAlertData, 0, len(validAlerts))
	for _, alert := range validAlerts {
		if alert.time.After(oldestRecordTime) {
			filteredAlerts = append(filteredAlerts, alert)
		}
	}
	validAlerts = filteredAlerts

	if len(validAlerts) == 0 {
		// log.Println("no valid alerts found")
		return nil
	}

	var stats SystemAlertStats

	// we can skip the latest systemStats record since it's the current value
	for i := range systemStats {
		stat := systemStats[i]
		// subtract 10 seconds to give a small time buffer
		systemStatsCreation := stat.Created.Time().Add(-time.Second * 10)
		stats = SystemAlertStats{}
		if err := json.Unmarshal(stat.Stats, &stats); err != nil {
			return err
		}
		// log.Println("stats", stats)
		for j := range validAlerts {
			alert := &validAlerts[j]
			// reset alert val on first iteration
			if i == 0 {
				alert.val = 0
			}
			// continue if system_stats is older than alert time range
			if systemStatsCreation.Before(alert.time) {
				continue
			}
			// add to alert value
			switch alert.name {
			case "CPU":
				alert.val += stats.Cpu
			case "Memory":
				alert.val += stats.Mem
			case "Bandwidth":
				alert.val += float64(stats.Bandwidth[0]+stats.Bandwidth[1]) / (1024 * 1024)
			case "Disk":
				if alert.mapSums == nil {
					alert.mapSums = make(map[string]float32, len(stats.ExtraFs)+1)
				}
				// add root disk
				if _, ok := alert.mapSums["root"]; !ok {
					alert.mapSums["root"] = 0.0
				}
				alert.mapSums["root"] += float32(stats.Disk)
				// add extra disks from historical record
				for key, fs := range stats.ExtraFs {
					if fs.DiskTotal > 0 {
						if _, ok := alert.mapSums[key]; !ok {
							alert.mapSums[key] = 0.0
						}
						alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
					}
				}
				// add zfs pool usage from historical record
				for key, pool := range stats.ZfsPools {
					if !pool.Raw && pool.Total > 0 {
						zfsKey := zfsDiskAlertKey(key)
						if _, ok := alert.mapSums[zfsKey]; !ok {
							alert.mapSums[zfsKey] = 0.0
						}
						alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
					}
				}
			case "Temperature":
				if alert.mapSums == nil {
					alert.mapSums = make(map[string]float32, len(stats.Temperatures))
				}
				for key, temp := range stats.Temperatures {
					if _, ok := alert.mapSums[key]; !ok {
						alert.mapSums[key] = float32(0)
					}
					alert.mapSums[key] += temp
				}
			case "LoadAvg1":
				alert.val += stats.LoadAvg[0]
			case "LoadAvg5":
				alert.val += stats.LoadAvg[1]
			case "LoadAvg15":
				alert.val += stats.LoadAvg[2]
			case "GPU":
				if len(stats.GPU) == 0 {
					continue
				}
				maxUsage := 0.0
				for _, gpu := range stats.GPU {
					if gpu.Usage > maxUsage {
						maxUsage = gpu.Usage
					}
				}
				alert.val += maxUsage
			case "Battery":
				if !hasRepresentativeBattery(stats.Battery, stats.Batteries) {
					continue
				}
				alert.val += float64(stats.Battery[0])
			default:
				value, ok := cpuStateAlertValue(alert.name, stats.CpuBreakdown)
				if !ok {
					continue
				}
				alert.val += value
			}
			alert.count++
		}
	}
	// sum up vals for each alert
	for _, alert := range validAlerts {
		if alert.count == 0 {
			continue
		}
		switch alert.name {
		case "Disk":
			maxPct := float32(0)
			for key, value := range alert.mapSums {
				sumPct := float32(value)
				if sumPct > maxPct {
					maxPct = sumPct
					alert.descriptor = diskAlertDescriptor(key)
					if poolKey, ok := strings.CutPrefix(key, "zfs:"); ok {
						if pool := data.Stats.ZfsPools[poolKey]; pool != nil && pool.DisplayName != "" {
							alert.descriptor = diskAlertDescriptor(zfsDiskAlertKey(pool.DisplayName))
						}
					}
				}
			}
			alert.val = float64(maxPct / float32(alert.count))
		case "Temperature":
			maxTemp := float32(0)
			for key, value := range alert.mapSums {
				sumTemp := float32(value) / float32(alert.count)
				if sumTemp > maxTemp {
					maxTemp = sumTemp
					alert.descriptor = fmt.Sprintf("Highest sensor %s", key)
				}
			}
			alert.val = float64(maxTemp)
		default:
			alert.val = alert.val / float64(alert.count)
		}
		minCount := float32(alert.min) / 1.2
		// log.Println("alert", alert.name, "val", alert.val, "threshold", alert.threshold, "triggered", alert.triggered)
		// log.Printf("%s: val %f | count %d | min-count %f | threshold %f\n", alert.name, alert.val, alert.count, minCount, alert.threshold)
		// pass through alert if count is greater than or equal to minCount
		if float32(alert.count) >= minCount {
			// Battery alert has inverted logic: trigger when value is BELOW threshold
			lowAlert := isLowAlert(alert.name)
			if lowAlert {
				if !alert.triggered && alert.val < alert.threshold {
					alert.triggered = true
					go am.sendSystemAlert(alert)
				} else if alert.triggered && alert.val >= alert.threshold {
					alert.triggered = false
					go am.sendSystemAlert(alert)
				}
			} else {
				if !alert.triggered && alert.val > alert.threshold {
					alert.triggered = true
					go am.sendSystemAlert(alert)
				} else if alert.triggered && alert.val <= alert.threshold {
					alert.triggered = false
					go am.sendSystemAlert(alert)
				}
			}
		}
	}
	return nil
}

func zfsDiskAlertKey(poolName string) string {
	return "zfs:" + poolName
}

func diskAlertDescriptor(key string) string {
	if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
		return fmt.Sprintf("Usage of storage pool %s", poolName)
	}
	return fmt.Sprintf("Usage of %s", key)
}

func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
	return legacy != [2]uint8{} || len(batteries) > 0
}

func (am *AlertManager) sendSystemAlert(alert SystemAlertData) {
	// log.Printf("Sending alert %s: val %f | count %d | threshold %f\n", alert.name, alert.val, alert.count, alert.threshold)
	systemName := alert.systemRecord.GetString("name")

	if state, ok := cpuStateAlerts[alert.name]; ok {
		alert.name = state.label
	}
	// change Disk to Disk usage
	if alert.name == "Disk" {
		alert.name += " usage"
	}
	// format LoadAvg5 and LoadAvg15
	if after, ok := strings.CutPrefix(alert.name, "LoadAvg"); ok {
		alert.name = after + "m Load"
	}

	// make title alert name lowercase if not CPU or GPU
	titleAlertName := alert.name
	if titleAlertName != "CPU" && titleAlertName != "GPU" && !strings.HasPrefix(titleAlertName, "CPU") {
		titleAlertName = strings.ToLower(titleAlertName)
	}

	var subject string
	lowAlert := isLowAlert(alert.name)
	if alert.triggered {
		if lowAlert {
			subject = fmt.Sprintf("%s %s below threshold", systemName, titleAlertName)
		} else {
			subject = fmt.Sprintf("%s %s above threshold", systemName, titleAlertName)
		}
	} else {
		if lowAlert {
			subject = fmt.Sprintf("%s %s above threshold", systemName, titleAlertName)
		} else {
			subject = fmt.Sprintf("%s %s below threshold", systemName, titleAlertName)
		}
	}
	minutesLabel := "minute"
	if alert.min > 1 {
		minutesLabel += "s"
	}
	if alert.descriptor == "" {
		alert.descriptor = alert.name
	}
	body := fmt.Sprintf("%s averaged %.2f%s for the previous %v %s.", alert.descriptor, alert.val, alert.unit, alert.min, minutesLabel)

	if err := am.setAlertTriggered(alert.alertData, alert.triggered); err != nil {
		// app.Logger().Error("failed to save alert record", "err", err)
		return
	}
	am.SendAlert(AlertMessageData{
		UserID:   alert.alertData.UserID,
		SystemID: alert.systemRecord.Id,
		Title:    subject,
		Message:  body,
		Link:     am.hub.MakeLink("system", alert.systemRecord.Id),
		LinkText: "View " + systemName,
	})
}

func isLowAlert(name string) bool {
	return name == "Battery"
}