Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

feat(alerts): add TLS certificate expiry alert for network monitors (#2627)

Mike Bogdane984ee12081f
diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.go
index 7a44a22f5..4d21b39be 100644
--- a/internal/alerts/alerts_api.go
+++ b/internal/alerts/alerts_api.go
@@ -29,9 +29,9 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
return e.BadRequestError("Bad data", err)
}
- if reqData.Name == alertNameNetworkMonitorLoss {
- if reqData.Value < 0 || reqData.Value >= 100 {
- return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
+ if isNetworkMonitorAlert(reqData.Name) {
+ if msg := networkMonitorThresholdError(reqData.Name, reqData.Value); msg != "" {
+ return e.BadRequestError(msg, nil)
}
reqData.Min = 0
}
diff --git a/internal/alerts/alerts_cache.go b/internal/alerts/alerts_cache.go
index facc401b3..0dfcd5025 100644
--- a/internal/alerts/alerts_cache.go
+++ b/internal/alerts/alerts_cache.go
@@ -34,7 +34,7 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
a.Triggered = record.GetBool("triggered")
a.Min = uint8(record.GetInt("min"))
a.PendingSince = record.GetDateTime("pending_since").Time()
- if a.Name == alertNameNetworkMonitorLoss {
+ if isNetworkMonitorAlert(a.Name) {
var state networkMonitorAlertState
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
a.MonitorStates = state.Monitors
diff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.go
index f6b1bd4d2..d8283f3c3 100644
--- a/internal/alerts/alerts_history.go
+++ b/internal/alerts/alerts_history.go
@@ -9,7 +9,7 @@ import (
// On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
- if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+ if isNetworkMonitorAlert(e.Record.GetString("name")) {
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
return err
}
@@ -25,7 +25,7 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
// On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
// Network monitor incidents have separate history entries per monitor.
- if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+ if isNetworkMonitorAlert(e.Record.GetString("name")) {
return e.Next()
}
original := e.Record.Original()
diff --git a/internal/alerts/alerts_network_monitor_cert_test.go b/internal/alerts/alerts_network_monitor_cert_test.go
new file mode 100644
index 000000000..154cff894
--- /dev/null
+++ b/internal/alerts/alerts_network_monitor_cert_test.go
@@ -0,0 +1,310 @@
+//go:build testing
+
+package alerts_test
+
+import (
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/henrygd/beszel/internal/alerts"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ beszelTests "github.com/henrygd/beszel/internal/tests"
+ "github.com/pocketbase/dbx"
+ "github.com/pocketbase/pocketbase/core"
+ pbTests "github.com/pocketbase/pocketbase/tests"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// certAlertSetup creates a 30 day certificate alert and an HTTPS monitor without certificate info.
+func certAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, *core.Record) {
+ t.Helper()
+ hub, system, alert := systemdTestSetup(t, false)
+ t.Cleanup(hub.Cleanup)
+ alert.Set("name", "NetworkMonitorCert")
+ alert.Set("value", 30)
+ require.NoError(t, hub.Save(alert))
+ m, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": system.Id, "target": "https://example.com", "protocol": "http", "interval": 60, "enabled": true,
+ })
+ require.NoError(t, err)
+ // Avoid starting a system update worker in tests.
+ _, err = hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
+ require.NoError(t, err)
+ system.Set("status", "up")
+ return hub, system, alert, m
+}
+
+// certExpiringIn returns certificate info for a certificate with the given days
+// left, plus an hour so that the day count is stable during the test.
+func certExpiringIn(days int) *monitor.CertInfo {
+ return &monitor.CertInfo{Expires: time.Now().Add(time.Duration(days)*24*time.Hour + time.Hour).UnixMilli()}
+}
+
+func setMonitorCert(t *testing.T, hub *beszelTests.TestHub, m *core.Record, cert *monitor.CertInfo) {
+ t.Helper()
+ m.Set("certInfo", cert)
+ require.NoError(t, hub.Save(m))
+}
+
+func TestNetworkMonitorCertAlertLifecycle(t *testing.T) {
+ hub, system, alert, m := certAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ count := hub.TestMailer.TotalSend()
+ check := func(active bool, open, sent int) {
+ t.Helper()
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ record, err := hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.Equal(t, active, record.GetBool("triggered"))
+ total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
+ require.NoError(t, err)
+ assert.EqualValues(t, open, total)
+ assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
+ }
+ lastMessage := func() string {
+ return hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text
+ }
+ // Unknown expiry never triggers.
+ check(false, 0, 0)
+ setMonitorCert(t, hub, m, certExpiringIn(40))
+ check(false, 0, 0)
+ // Expiry is checked against the clock, without needing a new result.
+ cert := certExpiringIn(25)
+ setMonitorCert(t, hub, m, cert)
+ check(true, 1, 1)
+ assert.Contains(t, lastMessage(), "https://example.com")
+ assert.Contains(t, lastMessage(), "expires in 25 days ("+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly)+")")
+ assert.Contains(t, lastMessage(), "within the 30-day threshold")
+ // Repeated checks and a hub restart do not notify again.
+ check(true, 1, 1)
+ am = alerts.NewTestAlertManagerWithoutWorker(hub)
+ check(true, 1, 1)
+ // A renewed certificate resolves the incident.
+ setMonitorCert(t, hub, m, certExpiringIn(90))
+ check(false, 0, 2)
+ assert.Contains(t, lastMessage(), "expires in 90 days")
+ assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Subject, "Certificate expiry recovered")
+ assert.Contains(t, lastMessage(), "beyond the 30-day threshold")
+ // A new incident opens when expiry nears again.
+ setMonitorCert(t, hub, m, certExpiringIn(20))
+ check(true, 1, 3)
+ histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
+ require.NoError(t, err)
+ require.Len(t, histories, 2)
+ for _, history := range histories {
+ assert.Equal(t, "NetworkMonitorCert", history.GetString("name"))
+ assert.Equal(t, "https://example.com", history.GetString("monitor_name"))
+ }
+}
+
+func TestNetworkMonitorCertAlertThresholdBoundary(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ offset time.Duration
+ triggered bool
+ }{
+ {name: "a minute beyond", offset: time.Minute, triggered: false},
+ {name: "23 hours beyond", offset: 23 * time.Hour, triggered: false},
+ {name: "a minute within", offset: -time.Minute, triggered: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ hub, system, alert, m := certAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ // Expiry beyond 30 days still displays as 30 days left, but must not trigger a 30 day threshold.
+ setMonitorCert(t, hub, m, &monitor.CertInfo{Expires: time.Now().Add(30*24*time.Hour + tc.offset).UnixMilli()})
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ alert, err := hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.Equal(t, tc.triggered, alert.GetBool("triggered"))
+ })
+ }
+}
+
+func TestNetworkMonitorCertAlertExpired(t *testing.T) {
+ hub, system, _, m := certAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ count := hub.TestMailer.TotalSend()
+ cert := &monitor.CertInfo{Expires: time.Now().Add(-72 * time.Hour).UnixMilli()}
+ setMonitorCert(t, hub, m, cert)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ require.Equal(t, count+1, hub.TestMailer.TotalSend())
+ message := hub.TestMailer.Messages()[count]
+ assert.Contains(t, message.Subject, "Certificate expired")
+ assert.Contains(t, message.Text, "certificate expired on "+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly))
+ histories, err := hub.FindAllRecords("alerts_history")
+ require.NoError(t, err)
+ require.Len(t, histories, 1)
+ assert.Negative(t, histories[0].GetFloat("value"))
+}
+
+func TestNetworkMonitorCertAlertIgnoresMonitorsWithoutCert(t *testing.T) {
+ hub, system, alert, _ := certAlertSetup(t)
+ alert.Set("value", 365)
+ require.NoError(t, hub.Save(alert))
+ _, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": system.Id, "target": "gateway.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
+ })
+ require.NoError(t, err)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ count := hub.TestMailer.TotalSend()
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ alert, err = hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.False(t, alert.GetBool("triggered"))
+}
+
+// The hub stores certificate info with direct SQL, which bypasses record hooks.
+func TestNetworkMonitorCertAlertSQLUpdatedCert(t *testing.T) {
+ hub, system, alert, m := certAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ setMonitorCert(t, hub, m, certExpiringIn(60))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ count := hub.TestMailer.TotalSend()
+ cert := certExpiringIn(10)
+ _, err := hub.DB().Update("network_monitors", dbx.Params{"certInfo": `{"expires":` + strconv.FormatInt(cert.Expires, 10) + `}`}, dbx.HashExp{"id": m.Id}).Execute()
+ require.NoError(t, err)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: {Cert: cert}}))
+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+ alert, err = hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.True(t, alert.GetBool("triggered"))
+}
+
+func TestNetworkMonitorCertAlertSteadyStateNoDatabaseWork(t *testing.T) {
+ hub, system, _, m := certAlertSetup(t)
+ counted := &monitorCountingHub{TestHub: hub}
+ am := alerts.NewTestAlertManagerWithoutWorker(counted)
+ sql := monitorSQLCounter(t, hub)
+ cert := certExpiringIn(10)
+ setMonitorCert(t, hub, m, cert)
+ // The agent resends unchanged certificate info for certificates about to expire.
+ result := monitorResult(0)
+ result.Cert = cert
+ results := map[string]monitor.Result{m.Id: result}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ sql.Store(0)
+ counted.transactions.Store(0)
+ for range 100 {
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ }
+ assert.Zero(t, sql.Load(), "steady state must not issue SQL")
+ assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
+}
+
+func TestNetworkMonitorCertAlertIndependentOfLossAlert(t *testing.T) {
+ hub, system, certAlert, m := certAlertSetup(t)
+ lossAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
+ "name": "NetworkMonitorLoss", "system": system.Id, "user": certAlert.GetString("user"), "value": 5,
+ })
+ require.NoError(t, err)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ setMonitorCert(t, hub, m, certExpiringIn(10))
+ results := map[string]monitor.Result{m.Id: monitorResult(0)}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ results[m.Id] = monitorResult(50)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ // Recovering loss must leave the certificate incident open.
+ results[m.Id] = monitorResult(0)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ for id, triggered := range map[string]bool{certAlert.Id: true, lossAlert.Id: false} {
+ record, err := hub.FindRecordById("alerts", id)
+ require.NoError(t, err)
+ assert.Equal(t, triggered, record.GetBool("triggered"), record.GetString("name"))
+ }
+ open, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})
+ require.NoError(t, err)
+ assert.EqualValues(t, 1, open)
+}
+
+func TestNetworkMonitorCertAlertCleanup(t *testing.T) {
+ for _, scenario := range []string{"disable monitor", "delete monitor", "down"} {
+ t.Run(scenario, func(t *testing.T) {
+ hub, system, alert, m := certAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ setMonitorCert(t, hub, m, certExpiringIn(10))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ count := hub.TestMailer.TotalSend()
+ switch scenario {
+ case "disable monitor":
+ m.Set("enabled", false)
+ require.NoError(t, hub.Save(m))
+ case "delete monitor":
+ require.NoError(t, hub.Delete(m))
+ case "down":
+ _, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
+ require.NoError(t, err)
+ system.Set("status", "down")
+ }
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ record, err := hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
+ require.NoError(t, err)
+ // A disconnected system keeps its incident; removed monitors close it silently.
+ assert.Equal(t, scenario == "down", record.GetBool("triggered"))
+ assert.Equal(t, scenario == "down", open == 1)
+ })
+ }
+}
+
+func TestNetworkMonitorCertAlertAPI(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ value float64
+ status int
+ }{
+ {name: "valid threshold", value: 14, status: 200},
+ {name: "maximum threshold", value: 365, status: 200},
+ {name: "zero threshold", value: 0, status: 400},
+ {name: "fractional threshold", value: 14.5, status: 400},
+ {name: "too large threshold", value: 366, status: 400},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ hub, user := beszelTests.GetHubWithUser(t)
+ defer hub.Cleanup()
+ systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")
+ require.NoError(t, err)
+ token, err := user.NewAuthToken()
+ require.NoError(t, err)
+ for _, direct := range []bool{false, true} {
+ body := map[string]any{"name": "NetworkMonitorCert", "value": tc.value, "min": 10, "systems": []string{systems[0].Id}, "overwrite": true}
+ url := "/api/beszel/user-alerts"
+ if direct {
+ url = "/api/collections/alerts/records"
+ body["system"], body["user"] = systems[0].Id, user.Id
+ existing, err := hub.FindAllRecords("alerts")
+ require.NoError(t, err)
+ for _, record := range existing {
+ require.NoError(t, hub.Delete(record))
+ }
+ }
+ content := `"status":400`
+ if tc.status == 200 {
+ content = `"NetworkMonitorCert"`
+ if !direct {
+ content = `"success":true`
+ }
+ }
+ scenario := beszelTests.ApiScenario{
+ Name: tc.name, Method: "POST", URL: url, Body: jsonReader(body),
+ Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
+ TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
+ }
+ scenario.Test(t)
+ records, err := hub.FindAllRecords("alerts")
+ require.NoError(t, err)
+ if tc.status != 200 {
+ assert.Empty(t, records)
+ continue
+ }
+ require.Len(t, records, 1)
+ assert.Equal(t, tc.value, records[0].GetFloat("value"))
+ assert.Zero(t, records[0].GetInt("min"))
+ }
+ })
+ }
+}
diff --git a/internal/alerts/alerts_network_monitors.go b/internal/alerts/alerts_network_monitors.go
index fb7723eb0..3e6612c0e 100644
--- a/internal/alerts/alerts_network_monitors.go
+++ b/internal/alerts/alerts_network_monitors.go
@@ -6,6 +6,7 @@ import (
"fmt"
"math"
"net"
+ "slices"
"strconv"
"time"
@@ -14,7 +15,12 @@ import (
"github.com/pocketbase/pocketbase/core"
)
-const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
+const (
+ alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
+ alertNameNetworkMonitorCert = "NetworkMonitorCert"
+ // maxCertAlertDays is the largest certificate expiry threshold in days.
+ maxCertAlertDays = 365
+)
// networkMonitorAlertState is this alert type's persisted runtime state.
// Monitor IDs map to their open history entries independently of history retention.
@@ -22,20 +28,48 @@ type networkMonitorAlertState struct {
Monitors map[string]string `json:"monitors"`
}
+// monitorObservation is one monitor's current value for an alert threshold.
+type monitorObservation struct {
+ triggered bool
+ // value is stored in alert history: loss percentage or days left.
+ value float64
+ // certExpires is the certificate expiry in Unix milliseconds (certificate alerts only).
+ certExpires int64
+}
+
+// isNetworkMonitorAlert reports whether an alert tracks incidents per network monitor.
+func isNetworkMonitorAlert(name string) bool {
+ return name == alertNameNetworkMonitorLoss || name == alertNameNetworkMonitorCert
+}
+
+// networkMonitorThresholdError describes why a threshold is invalid, or returns "" if it is valid.
+func networkMonitorThresholdError(name string, value float64) string {
+ switch name {
+ case alertNameNetworkMonitorLoss:
+ if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
+ return "Monitor loss threshold must be at least 0 and below 100"
+ }
+ case alertNameNetworkMonitorCert:
+ if math.IsNaN(value) || value < 1 || value > maxCertAlertDays || value != math.Trunc(value) {
+ return fmt.Sprintf("Certificate expiry threshold must be a whole number of days from 1 to %d", maxCertAlertDays)
+ }
+ }
+ return ""
+}
+
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// Hidden fields are still writable through the record API unless protected.
protectState := func(e *core.RecordRequestEvent) error {
e.Record.Set("state", e.Record.Original().Get("state"))
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
- if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
+ if oldName != "" && (isNetworkMonitorAlert(oldName) || isNetworkMonitorAlert(newName)) &&
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
}
- if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+ if isNetworkMonitorAlert(newName) {
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
- value := e.Record.GetFloat("value")
- if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
- return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
+ if msg := networkMonitorThresholdError(newName, e.Record.GetFloat("value")); msg != "" {
+ return e.BadRequestError(msg, nil)
}
e.Record.Set("min", 0)
}
@@ -62,10 +96,20 @@ func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// using its exact payload (dashboard requests can replace the cached payload).
// Omitted results and disconnected systems never imply recovery.
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
+ // Certificate info is stored by direct SQL without record hooks, so drop the
+ // cached expiry times when a result carries a different expiry. The agent
+ // resends unchanged info hourly for certificates that are about to expire.
+ for id, result := range results {
+ if result.Cert != nil && am.networkMonitors.certChanged(systemRecord.Id, id, result.Cert.Expires) {
+ am.networkMonitors.invalidate(systemRecord.Id)
+ break
+ }
+ }
if systemRecord.GetString("status") != "up" {
return nil
}
- alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
+ alerts := am.alertsCache.GetSystemAlerts(systemRecord.Id)
+ alerts = slices.DeleteFunc(alerts, func(a CachedAlertData) bool { return !isNetworkMonitorAlert(a.Name) })
if len(alerts) == 0 {
return nil
}
@@ -82,7 +126,7 @@ func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, re
}
// networkMonitorTransitionPending does no IO and never mutates cached maps.
-func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
+func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]cachedMonitor, results map[string]monitor.Result, now time.Time) bool {
for _, alert := range alerts {
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
return true
@@ -92,9 +136,23 @@ func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[stri
return true
}
}
+ if alert.Name == alertNameNetworkMonitorCert {
+ // Expiry depends on the clock, so check stored certificates rather than results.
+ cutoff := certExpiryCutoff(alert.Value, now)
+ for id, m := range monitors {
+ if m.certExpires <= 0 {
+ continue
+ }
+ _, active := alert.MonitorStates[id]
+ if (m.certExpires <= cutoff) != active {
+ return true
+ }
+ }
+ continue
+ }
for id, result := range results {
- interval, enabled := monitors[id]
- if !enabled || !monitorResultReady(result, interval, now) {
+ m, enabled := monitors[id]
+ if !enabled || !monitorResultReady(result, m.interval, now) {
continue
}
_, active := alert.MonitorStates[id]
@@ -111,7 +169,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
err := app.RunInTransaction(func(tx core.App) error {
// Read configuration inside the transaction so concurrent threshold changes,
// disabling, and evaluations cannot overwrite each other's incident state.
- alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
+ alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": []any{alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert}})
if err != nil || len(alerts) == 0 {
return err
}
@@ -156,21 +214,27 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
}
}
if system.GetString("status") == "up" {
+ name, threshold := alert.GetString("name"), alert.GetFloat("value")
for _, m := range monitors {
- result, ok := results[m.Id]
- if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
+ var obs monitorObservation
+ var ok bool
+ if name == alertNameNetworkMonitorCert {
+ obs, ok = observeMonitorCert(m, threshold, now)
+ } else {
+ obs, ok = observeMonitorLoss(results[m.Id], m, threshold, now)
+ }
+ if !ok {
continue
}
historyID, active := states[m.Id]
- triggered := result.PacketLoss1h > alert.GetFloat("value")
- if triggered == active {
+ if obs.triggered == active {
continue
}
label := m.GetString("target")
if m.GetString("protocol") == "tcp" {
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
}
- if triggered {
+ if obs.triggered {
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
@@ -178,7 +242,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
history := core.NewRecord(collection)
history.Load(map[string]any{
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
- "name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
+ "name": name, "monitor_name": label, "value": obs.value,
})
if err := tx.Save(history); err != nil {
return err
@@ -191,15 +255,11 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
delete(states, m.Id)
}
changed = true
- state, comparison := "loss", "exceeds"
- if !triggered {
- state, comparison = "recovered", "is at or below"
- }
+ title, message := networkMonitorAlertMessage(name, obs, label, system.GetString("name"), threshold)
messages = append(messages, AlertMessageData{
UserID: alert.GetString("user"), SystemID: systemID,
- Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
- Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
- Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
+ Title: title, Message: message,
+ Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
})
}
}
@@ -226,6 +286,69 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
return nil
}
+// observeMonitorLoss compares a fresh result's one hour loss with the threshold.
+func observeMonitorLoss(result monitor.Result, m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) {
+ if !monitorResultReady(result, m.GetInt("interval"), now) {
+ return monitorObservation{}, false
+ }
+ return monitorObservation{triggered: result.PacketLoss1h > threshold, value: result.PacketLoss1h}, true
+}
+
+// observeMonitorCert compares the stored certificate expiry with a threshold in days.
+// Monitors without certificate info (non-HTTPS or never checked) are skipped.
+func observeMonitorCert(m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) {
+ var cert monitor.CertInfo
+ if err := m.UnmarshalJSONField("certInfo", &cert); err != nil || cert.Expires <= 0 {
+ return monitorObservation{}, false
+ }
+ triggered := cert.Expires <= certExpiryCutoff(threshold, now)
+ return monitorObservation{triggered: triggered, value: float64(certDaysLeft(cert.Expires, now)), certExpires: cert.Expires}, true
+}
+
+// certExpiryCutoff returns the latest expiry, in Unix milliseconds, that is within the given
+// number of days. It uses exact times, so a threshold of 14 does not trigger with 14 days and
+// 23 hours left.
+func certExpiryCutoff(days float64, now time.Time) int64 {
+ return now.Add(time.Duration(days * float64(24*time.Hour))).UnixMilli()
+}
+
+// certDaysLeft returns whole days until expiry for display, matching the days shown in the UI.
+// It is negative once the certificate has expired.
+func certDaysLeft(expires int64, now time.Time) int {
+ return int(math.Floor(float64(expires-now.UnixMilli()) / float64(time.Hour.Milliseconds()*24)))
+}
+
+func networkMonitorAlertMessage(name string, obs monitorObservation, label, systemName string, threshold float64) (title, message string) {
+ if name == alertNameNetworkMonitorCert {
+ days := int(obs.value)
+ date := time.UnixMilli(obs.certExpires).UTC().Format(time.DateOnly)
+ switch {
+ case days < 0:
+ return fmt.Sprintf("Certificate expired on %s: %s", systemName, label),
+ fmt.Sprintf("%s on %s: certificate expired on %s.", label, systemName, date)
+ case obs.triggered:
+ return fmt.Sprintf("Certificate expiring on %s: %s", systemName, label),
+ fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is within the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold))
+ default:
+ return fmt.Sprintf("Certificate expiry recovered on %s: %s", systemName, label),
+ fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is beyond the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold))
+ }
+ }
+ state, comparison := "loss", "exceeds"
+ if !obs.triggered {
+ state, comparison = "recovered", "is at or below"
+ }
+ return fmt.Sprintf("Network monitor %s on %s: %s", state, systemName, label),
+ fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, systemName, obs.value, comparison, threshold)
+}
+
+func pluralDays(days int) string {
+ if days == 1 {
+ return "1 day"
+ }
+ return fmt.Sprintf("%d days", days)
+}
+
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
// Three completed attempts provide a short warm-up, including after an agent
// restart.
diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.go
index ad47d0bd5..39cf3ddc8 100644
--- a/internal/alerts/alerts_system.go
+++ b/internal/alerts/alerts_system.go
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil
}
- alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
+ alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert)
if len(alerts) == 0 {
return nil
}
diff --git a/internal/alerts/network_monitor_cache.go b/internal/alerts/network_monitor_cache.go
index 2ca3718e7..e449c58c7 100644
--- a/internal/alerts/network_monitor_cache.go
+++ b/internal/alerts/network_monitor_cache.go
@@ -1,23 +1,34 @@
package alerts
import (
+ "encoding/json"
"sync"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
+ "github.com/pocketbase/pocketbase/tools/types"
)
-// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
-// needed for the alert fast path. Names and targets are read only on transitions.
-// Returned maps are immutable; configuration changes invalidate the whole entry.
+// cachedMonitor holds the monitor fields needed for the alert fast path.
+type cachedMonitor struct {
+ interval int
+ // certExpires is the stored certificate expiry in Unix milliseconds, or zero if unknown.
+ certExpires int64
+}
+
+// networkMonitorCache keeps just the enabled monitor IDs, probe intervals, and
+// certificate expiry times needed for the alert fast path. Names and targets are
+// read only on transitions. Returned maps are immutable; configuration and
+// certificate changes invalidate the whole entry.
type networkMonitorCache struct {
app core.App
mu sync.RWMutex
- systems map[string]map[string]int
+ systems map[string]map[string]cachedMonitor
}
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
- c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
+ c := &networkMonitorCache{app: app, systems: make(map[string]map[string]cachedMonitor)}
invalidate := func(e *core.RecordEvent) error {
c.invalidate(e.Record.GetString("system"))
return e.Next()
@@ -29,7 +40,8 @@ func newNetworkMonitorCache(app core.App) *networkMonitorCache {
// Realtime metric saves also invoke this hook. They must not evict config.
if old.GetString("system") != e.Record.GetString("system") ||
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
- old.GetInt("interval") != e.Record.GetInt("interval") {
+ old.GetInt("interval") != e.Record.GetInt("interval") ||
+ old.GetString("certInfo") != e.Record.GetString("certInfo") {
c.invalidate(old.GetString("system"))
c.invalidate(e.Record.GetString("system"))
}
@@ -48,7 +60,16 @@ func (c *networkMonitorCache) invalidate(systemID string) {
c.mu.Unlock()
}
-func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
+// certChanged reports whether a cached monitor's certificate expiry differs from
+// expires. Systems that are not cached load fresh data when next needed.
+func (c *networkMonitorCache) certChanged(systemID, monitorID string, expires int64) bool {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ m, ok := c.systems[systemID][monitorID]
+ return ok && m.certExpires != expires
+}
+
+func (c *networkMonitorCache) get(systemID string) (map[string]cachedMonitor, error) {
c.mu.RLock()
monitors, ok := c.systems[systemID]
c.mu.RUnlock()
@@ -63,16 +84,20 @@ func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
// Keep the lock through the load so a concurrent config change cannot be
// invalidated first and then overwritten by the older query result.
var rows []struct {
- ID string `db:"id"`
- Interval int `db:"interval"`
+ ID string `db:"id"`
+ Interval int `db:"interval"`
+ CertInfo types.JSONRaw `db:"certInfo"`
}
- if err := c.app.DB().Select("id", "interval").From("network_monitors").
+ if err := c.app.DB().Select("id", "interval", "certInfo").From("network_monitors").
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
return nil, err
}
- monitors = make(map[string]int, len(rows))
+ monitors = make(map[string]cachedMonitor, len(rows))
for _, row := range rows {
- monitors[row.ID] = row.Interval
+ var cert monitor.CertInfo
+ // Missing or malformed info leaves the expiry unknown.
+ _ = json.Unmarshal(row.CertInfo, &cert)
+ monitors[row.ID] = cachedMonitor{interval: row.Interval, certExpires: cert.Expires}
}
c.systems[systemID] = monitors
return monitors, nil
diff --git a/internal/hub/collections_test.go b/internal/hub/collections_test.go
index 2acd245c0..f39947484 100644
--- a/internal/hub/collections_test.go
+++ b/internal/hub/collections_test.go
@@ -51,7 +51,7 @@ func TestCollectionRulesDefault(t *testing.T) {
assert.Equal(t, isUserMatchesUser+` && @request.body.user:changed = false && @request.body.system:changed = false && system.users.id ?= @request.auth.id`, *alertsCollection.UpdateRule)
assert.Equal(t, isUserMatchesUser, *alertsCollection.DeleteRule)
alertNames := alertsCollection.Fields.GetByName("name").(*core.SelectField).Values
- for _, name := range []string{"CPUIOWait", "CPUSteal"} {
+ for _, name := range []string{"CPUIOWait", "CPUSteal", "NetworkMonitorCert"} {
assert.Contains(t, alertNames, name)
}
for _, name := range []string{"CPUSystem", "CPUUser", "CPUIdle", "CPUOther"} {
diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.go
index a01897aa5..a2875ec67 100644
--- a/internal/hub/systems/system_manager.go
+++ b/internal/hub/systems/system_manager.go
@@ -427,7 +427,7 @@ func deactivateAlerts(app core.App, systemID string, preserveStatusAlert bool) e
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
- alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
+ alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss' && name != 'NetworkMonitorCert'", systemID), "", -1, 0)
if err != nil {
return err
}
diff --git a/internal/migrations/1791926360_network_monitor_cert_alert.go b/internal/migrations/1791926360_network_monitor_cert_alert.go
new file mode 100644
index 000000000..fa3c3e2f1
--- /dev/null
+++ b/internal/migrations/1791926360_network_monitor_cert_alert.go
@@ -0,0 +1,34 @@
+package migrations
+
+import (
+ "slices"
+
+ "github.com/pocketbase/pocketbase/core"
+ m "github.com/pocketbase/pocketbase/migrations"
+)
+
+const networkMonitorCertAlertName = "NetworkMonitorCert"
+
+func init() {
+ m.Register(func(app core.App) error {
+ collection, err := app.FindCollectionByNameOrId("alerts")
+ if err != nil {
+ return err
+ }
+ field := collection.Fields.GetByName("name").(*core.SelectField)
+ if !slices.Contains(field.Values, networkMonitorCertAlertName) {
+ field.Values = append(field.Values, networkMonitorCertAlertName)
+ }
+ return app.Save(collection)
+ }, func(app core.App) error {
+ collection, err := app.FindCollectionByNameOrId("alerts")
+ if err != nil {
+ return err
+ }
+ field := collection.Fields.GetByName("name").(*core.SelectField)
+ field.Values = slices.DeleteFunc(field.Values, func(name string) bool {
+ return name == networkMonitorCertAlertName
+ })
+ return app.Save(collection)
+ })
+}
diff --git a/internal/site/src/components/active-alerts.tsx b/internal/site/src/components/active-alerts.tsx
index e00efd269..c1bdd3584 100644
--- a/internal/site/src/components/active-alerts.tsx
+++ b/internal/site/src/components/active-alerts.tsx
@@ -17,6 +17,13 @@ function AlertTriggeredDesc({ alert }: { alert: AlertRecord }) {
if (alert.name === "NetworkMonitorLoss") {
return <Trans>One or more monitors exceed {alert.value}% loss</Trans>
}
+ if (alert.name === "NetworkMonitorCert") {
+ return (
+ <Trans>
+ One or more certificates expire within <Plural value={alert.value} one="# day" other="# days" />
+ </Trans>
+ )
+ }
if (alert.name === "Status") {
return <Trans>Connection is down</Trans>
}
diff --git a/internal/site/src/components/alerts-history-columns.tsx b/internal/site/src/components/alerts-history-columns.tsx
index 2ae9bd97e..b87bd2a56 100644
--- a/internal/site/src/components/alerts-history-columns.tsx
+++ b/internal/site/src/components/alerts-history-columns.tsx
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro"
-import { Trans } from "@lingui/react/macro"
+import { Plural, Trans } from "@lingui/react/macro"
import type { ColumnDef } from "@tanstack/react-table"
import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
@@ -69,6 +69,13 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [
return <span className="ps-2">{t`Down`}</span>
}
const value = getValue() as number
+ if (name === "NetworkMonitorCert") {
+ return (
+ <span className="tabular-nums ps-2.5">
+ {value < 0 ? <Trans>Expired</Trans> : <Plural value={value} one="# day" other="# days" />}
+ </span>
+ )
+ }
const unit = info?.unit
return (
<span className="tabular-nums ps-2.5">
diff --git a/internal/site/src/components/alerts/alerts-sheet.tsx b/internal/site/src/components/alerts/alerts-sheet.tsx
index 845aa0746..bd0ea8975 100644
--- a/internal/site/src/components/alerts/alerts-sheet.tsx
+++ b/internal/site/src/components/alerts/alerts-sheet.tsx
@@ -350,7 +350,14 @@ export function AlertContent({
{!noThreshold && (
<div className={cn(noDuration && "col-span-full")}>
<p id={`v${name}`} className="text-sm block h-6">
- {alertData.invert ? (
+ {alertKey === "NetworkMonitorCert" ? (
+ <Trans>
+ Expires within{" "}
+ <strong className="text-foreground">
+ <Plural value={value} one="# day" other="# days" />
+ </strong>
+ </Trans>
+ ) : alertData.invert ? (
<Trans>
Average drops below{" "}
<strong className="text-foreground">
@@ -386,6 +393,8 @@ export function AlertContent({
if (!Number.isNaN(val)) {
// keep values on the step grid so the unit can be inferred on reload
if (unit) val = roundToStep(val, inputStep)
+ // certificate thresholds are whole days
+ else if (alertKey === "NetworkMonitorCert") val = Math.round(val)
if (valueMax != null) val = Math.min(val, valueMax)
if (inputMin != null) val = Math.max(val, inputMin)
setValue(val)
diff --git a/internal/site/src/lib/alerts.ts b/internal/site/src/lib/alerts.ts
index 39c6a0f32..f19c0e830 100644
--- a/internal/site/src/lib/alerts.ts
+++ b/internal/site/src/lib/alerts.ts
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro"
-import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
+import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon, ShieldCheckIcon } from "lucide-react"
import type { RecordSubscription } from "pocketbase"
import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
import { $alerts } from "@/lib/stores"
@@ -71,6 +71,16 @@ export const alertInfo: Record<string, AlertInfo> = {
step: 0.1,
start: 5,
},
+ NetworkMonitorCert: {
+ name: () => t`Certificate Expiration`,
+ unit: " days",
+ icon: ShieldCheckIcon,
+ desc: () => t`Triggers when an HTTPS certificate nears expiration`,
+ noDuration: true,
+ min: 1,
+ max: 365,
+ start: 14,
+ },
GPU: {
name: () => t`GPU Usage`,
unit: "%",