Lightweight server monitoring with historical data, docker stats, and alerts.
feat(alerts): add TLS certificate expiry alert for network monitors (#2627)
Mike Bogdane984ee12081f
diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.goindex 7a44a22f5..4d21b39be 100644--- a/internal/alerts/alerts_api.go+++ b/internal/alerts/alerts_api.go@@ -29,9 +29,9 @@ func UpsertUserAlerts(e *core.RequestEvent) error {return e.BadRequestError("Bad data", err)}- if reqData.Name == alertNameNetworkMonitorLoss {- if reqData.Value < 0 || reqData.Value >= 100 {- return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)+ if isNetworkMonitorAlert(reqData.Name) {+ if msg := networkMonitorThresholdError(reqData.Name, reqData.Value); msg != "" {+ return e.BadRequestError(msg, nil)}reqData.Min = 0}diff --git a/internal/alerts/alerts_cache.go b/internal/alerts/alerts_cache.goindex facc401b3..0dfcd5025 100644--- a/internal/alerts/alerts_cache.go+++ b/internal/alerts/alerts_cache.go@@ -34,7 +34,7 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {a.Triggered = record.GetBool("triggered")a.Min = uint8(record.GetInt("min"))a.PendingSince = record.GetDateTime("pending_since").Time()- if a.Name == alertNameNetworkMonitorLoss {+ if isNetworkMonitorAlert(a.Name) {var state networkMonitorAlertStatea.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nila.MonitorStates = state.Monitorsdiff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.goindex f6b1bd4d2..d8283f3c3 100644--- a/internal/alerts/alerts_history.go+++ b/internal/alerts/alerts_history.go@@ -9,7 +9,7 @@ import (// On triggered alert record delete, set matching alert history record to resolvedfunc resolveHistoryOnAlertDelete(e *core.RecordEvent) error {- if e.Record.GetString("name") == alertNameNetworkMonitorLoss {+ if isNetworkMonitorAlert(e.Record.GetString("name")) {if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {return err}@@ -25,7 +25,7 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {// On alert record update, update alert history recordfunc updateHistoryOnAlertUpdate(e *core.RecordEvent) error {// Network monitor incidents have separate history entries per monitor.- if e.Record.GetString("name") == alertNameNetworkMonitorLoss {+ if isNetworkMonitorAlert(e.Record.GetString("name")) {return e.Next()}original := e.Record.Original()diff --git a/internal/alerts/alerts_network_monitor_cert_test.go b/internal/alerts/alerts_network_monitor_cert_test.gonew file mode 100644index 000000000..154cff894--- /dev/null+++ b/internal/alerts/alerts_network_monitor_cert_test.go@@ -0,0 +1,310 @@+//go:build testing++package alerts_test++import (+ "strconv"+ "testing"+ "time"++ "github.com/henrygd/beszel/internal/alerts"+ "github.com/henrygd/beszel/internal/entities/monitor"+ beszelTests "github.com/henrygd/beszel/internal/tests"+ "github.com/pocketbase/dbx"+ "github.com/pocketbase/pocketbase/core"+ pbTests "github.com/pocketbase/pocketbase/tests"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// certAlertSetup creates a 30 day certificate alert and an HTTPS monitor without certificate info.+func certAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, *core.Record) {+ t.Helper()+ hub, system, alert := systemdTestSetup(t, false)+ t.Cleanup(hub.Cleanup)+ alert.Set("name", "NetworkMonitorCert")+ alert.Set("value", 30)+ require.NoError(t, hub.Save(alert))+ m, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": system.Id, "target": "https://example.com", "protocol": "http", "interval": 60, "enabled": true,+ })+ require.NoError(t, err)+ // Avoid starting a system update worker in tests.+ _, err = hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()+ require.NoError(t, err)+ system.Set("status", "up")+ return hub, system, alert, m+}++// certExpiringIn returns certificate info for a certificate with the given days+// left, plus an hour so that the day count is stable during the test.+func certExpiringIn(days int) *monitor.CertInfo {+ return &monitor.CertInfo{Expires: time.Now().Add(time.Duration(days)*24*time.Hour + time.Hour).UnixMilli()}+}++func setMonitorCert(t *testing.T, hub *beszelTests.TestHub, m *core.Record, cert *monitor.CertInfo) {+ t.Helper()+ m.Set("certInfo", cert)+ require.NoError(t, hub.Save(m))+}++func TestNetworkMonitorCertAlertLifecycle(t *testing.T) {+ hub, system, alert, m := certAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ count := hub.TestMailer.TotalSend()+ check := func(active bool, open, sent int) {+ t.Helper()+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ record, err := hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.Equal(t, active, record.GetBool("triggered"))+ total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})+ require.NoError(t, err)+ assert.EqualValues(t, open, total)+ assert.Equal(t, count+sent, hub.TestMailer.TotalSend())+ }+ lastMessage := func() string {+ return hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text+ }+ // Unknown expiry never triggers.+ check(false, 0, 0)+ setMonitorCert(t, hub, m, certExpiringIn(40))+ check(false, 0, 0)+ // Expiry is checked against the clock, without needing a new result.+ cert := certExpiringIn(25)+ setMonitorCert(t, hub, m, cert)+ check(true, 1, 1)+ assert.Contains(t, lastMessage(), "https://example.com")+ assert.Contains(t, lastMessage(), "expires in 25 days ("+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly)+")")+ assert.Contains(t, lastMessage(), "within the 30-day threshold")+ // Repeated checks and a hub restart do not notify again.+ check(true, 1, 1)+ am = alerts.NewTestAlertManagerWithoutWorker(hub)+ check(true, 1, 1)+ // A renewed certificate resolves the incident.+ setMonitorCert(t, hub, m, certExpiringIn(90))+ check(false, 0, 2)+ assert.Contains(t, lastMessage(), "expires in 90 days")+ assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Subject, "Certificate expiry recovered")+ assert.Contains(t, lastMessage(), "beyond the 30-day threshold")+ // A new incident opens when expiry nears again.+ setMonitorCert(t, hub, m, certExpiringIn(20))+ check(true, 1, 3)+ histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})+ require.NoError(t, err)+ require.Len(t, histories, 2)+ for _, history := range histories {+ assert.Equal(t, "NetworkMonitorCert", history.GetString("name"))+ assert.Equal(t, "https://example.com", history.GetString("monitor_name"))+ }+}++func TestNetworkMonitorCertAlertThresholdBoundary(t *testing.T) {+ for _, tc := range []struct {+ name string+ offset time.Duration+ triggered bool+ }{+ {name: "a minute beyond", offset: time.Minute, triggered: false},+ {name: "23 hours beyond", offset: 23 * time.Hour, triggered: false},+ {name: "a minute within", offset: -time.Minute, triggered: true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ hub, system, alert, m := certAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ // Expiry beyond 30 days still displays as 30 days left, but must not trigger a 30 day threshold.+ setMonitorCert(t, hub, m, &monitor.CertInfo{Expires: time.Now().Add(30*24*time.Hour + tc.offset).UnixMilli()})+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ alert, err := hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.Equal(t, tc.triggered, alert.GetBool("triggered"))+ })+ }+}++func TestNetworkMonitorCertAlertExpired(t *testing.T) {+ hub, system, _, m := certAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ count := hub.TestMailer.TotalSend()+ cert := &monitor.CertInfo{Expires: time.Now().Add(-72 * time.Hour).UnixMilli()}+ setMonitorCert(t, hub, m, cert)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ require.Equal(t, count+1, hub.TestMailer.TotalSend())+ message := hub.TestMailer.Messages()[count]+ assert.Contains(t, message.Subject, "Certificate expired")+ assert.Contains(t, message.Text, "certificate expired on "+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly))+ histories, err := hub.FindAllRecords("alerts_history")+ require.NoError(t, err)+ require.Len(t, histories, 1)+ assert.Negative(t, histories[0].GetFloat("value"))+}++func TestNetworkMonitorCertAlertIgnoresMonitorsWithoutCert(t *testing.T) {+ hub, system, alert, _ := certAlertSetup(t)+ alert.Set("value", 365)+ require.NoError(t, hub.Save(alert))+ _, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": system.Id, "target": "gateway.example.com", "protocol": "icmp", "interval": 60, "enabled": true,+ })+ require.NoError(t, err)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ count := hub.TestMailer.TotalSend()+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ alert, err = hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.False(t, alert.GetBool("triggered"))+}++// The hub stores certificate info with direct SQL, which bypasses record hooks.+func TestNetworkMonitorCertAlertSQLUpdatedCert(t *testing.T) {+ hub, system, alert, m := certAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ setMonitorCert(t, hub, m, certExpiringIn(60))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ count := hub.TestMailer.TotalSend()+ cert := certExpiringIn(10)+ _, err := hub.DB().Update("network_monitors", dbx.Params{"certInfo": `{"expires":` + strconv.FormatInt(cert.Expires, 10) + `}`}, dbx.HashExp{"id": m.Id}).Execute()+ require.NoError(t, err)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: {Cert: cert}}))+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())+ alert, err = hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.True(t, alert.GetBool("triggered"))+}++func TestNetworkMonitorCertAlertSteadyStateNoDatabaseWork(t *testing.T) {+ hub, system, _, m := certAlertSetup(t)+ counted := &monitorCountingHub{TestHub: hub}+ am := alerts.NewTestAlertManagerWithoutWorker(counted)+ sql := monitorSQLCounter(t, hub)+ cert := certExpiringIn(10)+ setMonitorCert(t, hub, m, cert)+ // The agent resends unchanged certificate info for certificates about to expire.+ result := monitorResult(0)+ result.Cert = cert+ results := map[string]monitor.Result{m.Id: result}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ sql.Store(0)+ counted.transactions.Store(0)+ for range 100 {+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ }+ assert.Zero(t, sql.Load(), "steady state must not issue SQL")+ assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")+}++func TestNetworkMonitorCertAlertIndependentOfLossAlert(t *testing.T) {+ hub, system, certAlert, m := certAlertSetup(t)+ lossAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{+ "name": "NetworkMonitorLoss", "system": system.Id, "user": certAlert.GetString("user"), "value": 5,+ })+ require.NoError(t, err)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ setMonitorCert(t, hub, m, certExpiringIn(10))+ results := map[string]monitor.Result{m.Id: monitorResult(0)}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ results[m.Id] = monitorResult(50)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ // Recovering loss must leave the certificate incident open.+ results[m.Id] = monitorResult(0)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ for id, triggered := range map[string]bool{certAlert.Id: true, lossAlert.Id: false} {+ record, err := hub.FindRecordById("alerts", id)+ require.NoError(t, err)+ assert.Equal(t, triggered, record.GetBool("triggered"), record.GetString("name"))+ }+ open, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})+ require.NoError(t, err)+ assert.EqualValues(t, 1, open)+}++func TestNetworkMonitorCertAlertCleanup(t *testing.T) {+ for _, scenario := range []string{"disable monitor", "delete monitor", "down"} {+ t.Run(scenario, func(t *testing.T) {+ hub, system, alert, m := certAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ setMonitorCert(t, hub, m, certExpiringIn(10))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ count := hub.TestMailer.TotalSend()+ switch scenario {+ case "disable monitor":+ m.Set("enabled", false)+ require.NoError(t, hub.Save(m))+ case "delete monitor":+ require.NoError(t, hub.Delete(m))+ case "down":+ _, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()+ require.NoError(t, err)+ system.Set("status", "down")+ }+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ record, err := hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})+ require.NoError(t, err)+ // A disconnected system keeps its incident; removed monitors close it silently.+ assert.Equal(t, scenario == "down", record.GetBool("triggered"))+ assert.Equal(t, scenario == "down", open == 1)+ })+ }+}++func TestNetworkMonitorCertAlertAPI(t *testing.T) {+ for _, tc := range []struct {+ name string+ value float64+ status int+ }{+ {name: "valid threshold", value: 14, status: 200},+ {name: "maximum threshold", value: 365, status: 200},+ {name: "zero threshold", value: 0, status: 400},+ {name: "fractional threshold", value: 14.5, status: 400},+ {name: "too large threshold", value: 366, status: 400},+ } {+ t.Run(tc.name, func(t *testing.T) {+ hub, user := beszelTests.GetHubWithUser(t)+ defer hub.Cleanup()+ systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")+ require.NoError(t, err)+ token, err := user.NewAuthToken()+ require.NoError(t, err)+ for _, direct := range []bool{false, true} {+ body := map[string]any{"name": "NetworkMonitorCert", "value": tc.value, "min": 10, "systems": []string{systems[0].Id}, "overwrite": true}+ url := "/api/beszel/user-alerts"+ if direct {+ url = "/api/collections/alerts/records"+ body["system"], body["user"] = systems[0].Id, user.Id+ existing, err := hub.FindAllRecords("alerts")+ require.NoError(t, err)+ for _, record := range existing {+ require.NoError(t, hub.Delete(record))+ }+ }+ content := `"status":400`+ if tc.status == 200 {+ content = `"NetworkMonitorCert"`+ if !direct {+ content = `"success":true`+ }+ }+ scenario := beszelTests.ApiScenario{+ Name: tc.name, Method: "POST", URL: url, Body: jsonReader(body),+ Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},+ TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },+ }+ scenario.Test(t)+ records, err := hub.FindAllRecords("alerts")+ require.NoError(t, err)+ if tc.status != 200 {+ assert.Empty(t, records)+ continue+ }+ require.Len(t, records, 1)+ assert.Equal(t, tc.value, records[0].GetFloat("value"))+ assert.Zero(t, records[0].GetInt("min"))+ }+ })+ }+}diff --git a/internal/alerts/alerts_network_monitors.go b/internal/alerts/alerts_network_monitors.goindex fb7723eb0..3e6612c0e 100644--- a/internal/alerts/alerts_network_monitors.go+++ b/internal/alerts/alerts_network_monitors.go@@ -6,6 +6,7 @@ import ("fmt""math""net"+ "slices""strconv""time"@@ -14,7 +15,12 @@ import ("github.com/pocketbase/pocketbase/core")-const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"+const (+ alertNameNetworkMonitorLoss = "NetworkMonitorLoss"+ alertNameNetworkMonitorCert = "NetworkMonitorCert"+ // maxCertAlertDays is the largest certificate expiry threshold in days.+ maxCertAlertDays = 365+)// networkMonitorAlertState is this alert type's persisted runtime state.// Monitor IDs map to their open history entries independently of history retention.@@ -22,20 +28,48 @@ type networkMonitorAlertState struct {Monitors map[string]string `json:"monitors"`}+// monitorObservation is one monitor's current value for an alert threshold.+type monitorObservation struct {+ triggered bool+ // value is stored in alert history: loss percentage or days left.+ value float64+ // certExpires is the certificate expiry in Unix milliseconds (certificate alerts only).+ certExpires int64+}++// isNetworkMonitorAlert reports whether an alert tracks incidents per network monitor.+func isNetworkMonitorAlert(name string) bool {+ return name == alertNameNetworkMonitorLoss || name == alertNameNetworkMonitorCert+}++// networkMonitorThresholdError describes why a threshold is invalid, or returns "" if it is valid.+func networkMonitorThresholdError(name string, value float64) string {+ switch name {+ case alertNameNetworkMonitorLoss:+ if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {+ return "Monitor loss threshold must be at least 0 and below 100"+ }+ case alertNameNetworkMonitorCert:+ if math.IsNaN(value) || value < 1 || value > maxCertAlertDays || value != math.Trunc(value) {+ return fmt.Sprintf("Certificate expiry threshold must be a whole number of days from 1 to %d", maxCertAlertDays)+ }+ }+ return ""+}+func (am *AlertManager) bindNetworkMonitorAlertEvents() {// Hidden fields are still writable through the record API unless protected.protectState := func(e *core.RecordRequestEvent) error {e.Record.Set("state", e.Record.Original().Get("state"))oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")- if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&+ if oldName != "" && (isNetworkMonitorAlert(oldName) || isNetworkMonitorAlert(newName)) &&(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)}- if e.Record.GetString("name") == alertNameNetworkMonitorLoss {+ if isNetworkMonitorAlert(newName) {e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))- value := e.Record.GetFloat("value")- if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {- return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)+ if msg := networkMonitorThresholdError(newName, e.Record.GetFloat("value")); msg != "" {+ return e.BadRequestError(msg, nil)}e.Record.Set("min", 0)}@@ -62,10 +96,20 @@ func (am *AlertManager) bindNetworkMonitorAlertEvents() {// using its exact payload (dashboard requests can replace the cached payload).// Omitted results and disconnected systems never imply recovery.func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {+ // Certificate info is stored by direct SQL without record hooks, so drop the+ // cached expiry times when a result carries a different expiry. The agent+ // resends unchanged info hourly for certificates that are about to expire.+ for id, result := range results {+ if result.Cert != nil && am.networkMonitors.certChanged(systemRecord.Id, id, result.Cert.Expires) {+ am.networkMonitors.invalidate(systemRecord.Id)+ break+ }+ }if systemRecord.GetString("status") != "up" {return nil}- alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)+ alerts := am.alertsCache.GetSystemAlerts(systemRecord.Id)+ alerts = slices.DeleteFunc(alerts, func(a CachedAlertData) bool { return !isNetworkMonitorAlert(a.Name) })if len(alerts) == 0 {return nil}@@ -82,7 +126,7 @@ func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, re}// networkMonitorTransitionPending does no IO and never mutates cached maps.-func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {+func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]cachedMonitor, results map[string]monitor.Result, now time.Time) bool {for _, alert := range alerts {if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {return true@@ -92,9 +136,23 @@ func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[strireturn true}}+ if alert.Name == alertNameNetworkMonitorCert {+ // Expiry depends on the clock, so check stored certificates rather than results.+ cutoff := certExpiryCutoff(alert.Value, now)+ for id, m := range monitors {+ if m.certExpires <= 0 {+ continue+ }+ _, active := alert.MonitorStates[id]+ if (m.certExpires <= cutoff) != active {+ return true+ }+ }+ continue+ }for id, result := range results {- interval, enabled := monitors[id]- if !enabled || !monitorResultReady(result, interval, now) {+ m, enabled := monitors[id]+ if !enabled || !monitorResultReady(result, m.interval, now) {continue}_, active := alert.MonitorStates[id]@@ -111,7 +169,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID strierr := app.RunInTransaction(func(tx core.App) error {// Read configuration inside the transaction so concurrent threshold changes,// disabling, and evaluations cannot overwrite each other's incident state.- alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})+ alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": []any{alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert}})if err != nil || len(alerts) == 0 {return err}@@ -156,21 +214,27 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri}}if system.GetString("status") == "up" {+ name, threshold := alert.GetString("name"), alert.GetFloat("value")for _, m := range monitors {- result, ok := results[m.Id]- if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {+ var obs monitorObservation+ var ok bool+ if name == alertNameNetworkMonitorCert {+ obs, ok = observeMonitorCert(m, threshold, now)+ } else {+ obs, ok = observeMonitorLoss(results[m.Id], m, threshold, now)+ }+ if !ok {continue}historyID, active := states[m.Id]- triggered := result.PacketLoss1h > alert.GetFloat("value")- if triggered == active {+ if obs.triggered == active {continue}label := m.GetString("target")if m.GetString("protocol") == "tcp" {label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))}- if triggered {+ if obs.triggered {collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")if err != nil {return err@@ -178,7 +242,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID strihistory := core.NewRecord(collection)history.Load(map[string]any{"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,- "name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,+ "name": name, "monitor_name": label, "value": obs.value,})if err := tx.Save(history); err != nil {return err@@ -191,15 +255,11 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stridelete(states, m.Id)}changed = true- state, comparison := "loss", "exceeds"- if !triggered {- state, comparison = "recovered", "is at or below"- }+ title, message := networkMonitorAlertMessage(name, obs, label, system.GetString("name"), threshold)messages = append(messages, AlertMessageData{UserID: alert.GetString("user"), SystemID: systemID,- Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),- Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),- Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),+ Title: title, Message: message,+ Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),})}}@@ -226,6 +286,69 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID strireturn nil}+// observeMonitorLoss compares a fresh result's one hour loss with the threshold.+func observeMonitorLoss(result monitor.Result, m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) {+ if !monitorResultReady(result, m.GetInt("interval"), now) {+ return monitorObservation{}, false+ }+ return monitorObservation{triggered: result.PacketLoss1h > threshold, value: result.PacketLoss1h}, true+}++// observeMonitorCert compares the stored certificate expiry with a threshold in days.+// Monitors without certificate info (non-HTTPS or never checked) are skipped.+func observeMonitorCert(m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) {+ var cert monitor.CertInfo+ if err := m.UnmarshalJSONField("certInfo", &cert); err != nil || cert.Expires <= 0 {+ return monitorObservation{}, false+ }+ triggered := cert.Expires <= certExpiryCutoff(threshold, now)+ return monitorObservation{triggered: triggered, value: float64(certDaysLeft(cert.Expires, now)), certExpires: cert.Expires}, true+}++// certExpiryCutoff returns the latest expiry, in Unix milliseconds, that is within the given+// number of days. It uses exact times, so a threshold of 14 does not trigger with 14 days and+// 23 hours left.+func certExpiryCutoff(days float64, now time.Time) int64 {+ return now.Add(time.Duration(days * float64(24*time.Hour))).UnixMilli()+}++// certDaysLeft returns whole days until expiry for display, matching the days shown in the UI.+// It is negative once the certificate has expired.+func certDaysLeft(expires int64, now time.Time) int {+ return int(math.Floor(float64(expires-now.UnixMilli()) / float64(time.Hour.Milliseconds()*24)))+}++func networkMonitorAlertMessage(name string, obs monitorObservation, label, systemName string, threshold float64) (title, message string) {+ if name == alertNameNetworkMonitorCert {+ days := int(obs.value)+ date := time.UnixMilli(obs.certExpires).UTC().Format(time.DateOnly)+ switch {+ case days < 0:+ return fmt.Sprintf("Certificate expired on %s: %s", systemName, label),+ fmt.Sprintf("%s on %s: certificate expired on %s.", label, systemName, date)+ case obs.triggered:+ return fmt.Sprintf("Certificate expiring on %s: %s", systemName, label),+ fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is within the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold))+ default:+ return fmt.Sprintf("Certificate expiry recovered on %s: %s", systemName, label),+ fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is beyond the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold))+ }+ }+ state, comparison := "loss", "exceeds"+ if !obs.triggered {+ state, comparison = "recovered", "is at or below"+ }+ return fmt.Sprintf("Network monitor %s on %s: %s", state, systemName, label),+ fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, systemName, obs.value, comparison, threshold)+}++func pluralDays(days int) string {+ if days == 1 {+ return "1 day"+ }+ return fmt.Sprintf("%d days", days)+}+func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {// Three completed attempts provide a short warm-up, including after an agent// restart.diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.goindex ad47d0bd5..39cf3ddc8 100644--- a/internal/alerts/alerts_system.go+++ b/internal/alerts/alerts_system.go@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *systreturn nil}- alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)+ alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert)if len(alerts) == 0 {return nil}diff --git a/internal/alerts/network_monitor_cache.go b/internal/alerts/network_monitor_cache.goindex 2ca3718e7..e449c58c7 100644--- a/internal/alerts/network_monitor_cache.go+++ b/internal/alerts/network_monitor_cache.go@@ -1,23 +1,34 @@package alertsimport (+ "encoding/json""sync"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/pocketbase/dbx""github.com/pocketbase/pocketbase/core"+ "github.com/pocketbase/pocketbase/tools/types")-// networkMonitorCache keeps just the enabled monitor IDs and probe intervals-// needed for the alert fast path. Names and targets are read only on transitions.-// Returned maps are immutable; configuration changes invalidate the whole entry.+// cachedMonitor holds the monitor fields needed for the alert fast path.+type cachedMonitor struct {+ interval int+ // certExpires is the stored certificate expiry in Unix milliseconds, or zero if unknown.+ certExpires int64+}++// networkMonitorCache keeps just the enabled monitor IDs, probe intervals, and+// certificate expiry times needed for the alert fast path. Names and targets are+// read only on transitions. Returned maps are immutable; configuration and+// certificate changes invalidate the whole entry.type networkMonitorCache struct {app core.Appmu sync.RWMutex- systems map[string]map[string]int+ systems map[string]map[string]cachedMonitor}func newNetworkMonitorCache(app core.App) *networkMonitorCache {- c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}+ c := &networkMonitorCache{app: app, systems: make(map[string]map[string]cachedMonitor)}invalidate := func(e *core.RecordEvent) error {c.invalidate(e.Record.GetString("system"))return e.Next()@@ -29,7 +40,8 @@ func newNetworkMonitorCache(app core.App) *networkMonitorCache {// Realtime metric saves also invoke this hook. They must not evict config.if old.GetString("system") != e.Record.GetString("system") ||old.GetBool("enabled") != e.Record.GetBool("enabled") ||- old.GetInt("interval") != e.Record.GetInt("interval") {+ old.GetInt("interval") != e.Record.GetInt("interval") ||+ old.GetString("certInfo") != e.Record.GetString("certInfo") {c.invalidate(old.GetString("system"))c.invalidate(e.Record.GetString("system"))}@@ -48,7 +60,16 @@ func (c *networkMonitorCache) invalidate(systemID string) {c.mu.Unlock()}-func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {+// certChanged reports whether a cached monitor's certificate expiry differs from+// expires. Systems that are not cached load fresh data when next needed.+func (c *networkMonitorCache) certChanged(systemID, monitorID string, expires int64) bool {+ c.mu.RLock()+ defer c.mu.RUnlock()+ m, ok := c.systems[systemID][monitorID]+ return ok && m.certExpires != expires+}++func (c *networkMonitorCache) get(systemID string) (map[string]cachedMonitor, error) {c.mu.RLock()monitors, ok := c.systems[systemID]c.mu.RUnlock()@@ -63,16 +84,20 @@ func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {// Keep the lock through the load so a concurrent config change cannot be// invalidated first and then overwritten by the older query result.var rows []struct {- ID string `db:"id"`- Interval int `db:"interval"`+ ID string `db:"id"`+ Interval int `db:"interval"`+ CertInfo types.JSONRaw `db:"certInfo"`}- if err := c.app.DB().Select("id", "interval").From("network_monitors").+ if err := c.app.DB().Select("id", "interval", "certInfo").From("network_monitors").Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {return nil, err}- monitors = make(map[string]int, len(rows))+ monitors = make(map[string]cachedMonitor, len(rows))for _, row := range rows {- monitors[row.ID] = row.Interval+ var cert monitor.CertInfo+ // Missing or malformed info leaves the expiry unknown.+ _ = json.Unmarshal(row.CertInfo, &cert)+ monitors[row.ID] = cachedMonitor{interval: row.Interval, certExpires: cert.Expires}}c.systems[systemID] = monitorsreturn monitors, nildiff --git a/internal/hub/collections_test.go b/internal/hub/collections_test.goindex 2acd245c0..f39947484 100644--- a/internal/hub/collections_test.go+++ b/internal/hub/collections_test.go@@ -51,7 +51,7 @@ func TestCollectionRulesDefault(t *testing.T) {assert.Equal(t, isUserMatchesUser+` && @request.body.user:changed = false && @request.body.system:changed = false && system.users.id ?= @request.auth.id`, *alertsCollection.UpdateRule)assert.Equal(t, isUserMatchesUser, *alertsCollection.DeleteRule)alertNames := alertsCollection.Fields.GetByName("name").(*core.SelectField).Values- for _, name := range []string{"CPUIOWait", "CPUSteal"} {+ for _, name := range []string{"CPUIOWait", "CPUSteal", "NetworkMonitorCert"} {assert.Contains(t, alertNames, name)}for _, name := range []string{"CPUSystem", "CPUUser", "CPUIdle", "CPUOther"} {diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.goindex a01897aa5..a2875ec67 100644--- a/internal/hub/systems/system_manager.go+++ b/internal/hub/systems/system_manager.go@@ -427,7 +427,7 @@ func deactivateAlerts(app core.App, systemID string, preserveStatusAlert bool) e// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()- alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)+ alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss' && name != 'NetworkMonitorCert'", systemID), "", -1, 0)if err != nil {return err}diff --git a/internal/migrations/1791926360_network_monitor_cert_alert.go b/internal/migrations/1791926360_network_monitor_cert_alert.gonew file mode 100644index 000000000..fa3c3e2f1--- /dev/null+++ b/internal/migrations/1791926360_network_monitor_cert_alert.go@@ -0,0 +1,34 @@+package migrations++import (+ "slices"++ "github.com/pocketbase/pocketbase/core"+ m "github.com/pocketbase/pocketbase/migrations"+)++const networkMonitorCertAlertName = "NetworkMonitorCert"++func init() {+ m.Register(func(app core.App) error {+ collection, err := app.FindCollectionByNameOrId("alerts")+ if err != nil {+ return err+ }+ field := collection.Fields.GetByName("name").(*core.SelectField)+ if !slices.Contains(field.Values, networkMonitorCertAlertName) {+ field.Values = append(field.Values, networkMonitorCertAlertName)+ }+ return app.Save(collection)+ }, func(app core.App) error {+ collection, err := app.FindCollectionByNameOrId("alerts")+ if err != nil {+ return err+ }+ field := collection.Fields.GetByName("name").(*core.SelectField)+ field.Values = slices.DeleteFunc(field.Values, func(name string) bool {+ return name == networkMonitorCertAlertName+ })+ return app.Save(collection)+ })+}diff --git a/internal/site/src/components/active-alerts.tsx b/internal/site/src/components/active-alerts.tsxindex e00efd269..c1bdd3584 100644--- a/internal/site/src/components/active-alerts.tsx+++ b/internal/site/src/components/active-alerts.tsx@@ -17,6 +17,13 @@ function AlertTriggeredDesc({ alert }: { alert: AlertRecord }) {if (alert.name === "NetworkMonitorLoss") {return <Trans>One or more monitors exceed {alert.value}% loss</Trans>}+ if (alert.name === "NetworkMonitorCert") {+ return (+ <Trans>+ One or more certificates expire within <Plural value={alert.value} one="# day" other="# days" />+ </Trans>+ )+ }if (alert.name === "Status") {return <Trans>Connection is down</Trans>}diff --git a/internal/site/src/components/alerts-history-columns.tsx b/internal/site/src/components/alerts-history-columns.tsxindex 2ae9bd97e..b87bd2a56 100644--- a/internal/site/src/components/alerts-history-columns.tsx+++ b/internal/site/src/components/alerts-history-columns.tsx@@ -1,5 +1,5 @@import { t } from "@lingui/core/macro"-import { Trans } from "@lingui/react/macro"+import { Plural, Trans } from "@lingui/react/macro"import type { ColumnDef } from "@tanstack/react-table"import { Badge } from "@/components/ui/badge"import { Button } from "@/components/ui/button"@@ -69,6 +69,13 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [return <span className="ps-2">{t`Down`}</span>}const value = getValue() as number+ if (name === "NetworkMonitorCert") {+ return (+ <span className="tabular-nums ps-2.5">+ {value < 0 ? <Trans>Expired</Trans> : <Plural value={value} one="# day" other="# days" />}+ </span>+ )+ }const unit = info?.unitreturn (<span className="tabular-nums ps-2.5">diff --git a/internal/site/src/components/alerts/alerts-sheet.tsx b/internal/site/src/components/alerts/alerts-sheet.tsxindex 845aa0746..bd0ea8975 100644--- a/internal/site/src/components/alerts/alerts-sheet.tsx+++ b/internal/site/src/components/alerts/alerts-sheet.tsx@@ -350,7 +350,14 @@ export function AlertContent({{!noThreshold && (<div className={cn(noDuration && "col-span-full")}><p id={`v${name}`} className="text-sm block h-6">- {alertData.invert ? (+ {alertKey === "NetworkMonitorCert" ? (+ <Trans>+ Expires within{" "}+ <strong className="text-foreground">+ <Plural value={value} one="# day" other="# days" />+ </strong>+ </Trans>+ ) : alertData.invert ? (<Trans>Average drops below{" "}<strong className="text-foreground">@@ -386,6 +393,8 @@ export function AlertContent({if (!Number.isNaN(val)) {// keep values on the step grid so the unit can be inferred on reloadif (unit) val = roundToStep(val, inputStep)+ // certificate thresholds are whole days+ else if (alertKey === "NetworkMonitorCert") val = Math.round(val)if (valueMax != null) val = Math.min(val, valueMax)if (inputMin != null) val = Math.max(val, inputMin)setValue(val)diff --git a/internal/site/src/lib/alerts.ts b/internal/site/src/lib/alerts.tsindex 39c6a0f32..f19c0e830 100644--- a/internal/site/src/lib/alerts.ts+++ b/internal/site/src/lib/alerts.ts@@ -1,5 +1,5 @@import { t } from "@lingui/core/macro"-import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"+import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon, ShieldCheckIcon } from "lucide-react"import type { RecordSubscription } from "pocketbase"import { EthernetIcon, GpuIcon } from "@/components/ui/icons"import { $alerts } from "@/lib/stores"@@ -71,6 +71,16 @@ export const alertInfo: Record<string, AlertInfo> = {step: 0.1,start: 5,},+ NetworkMonitorCert: {+ name: () => t`Certificate Expiration`,+ unit: " days",+ icon: ShieldCheckIcon,+ desc: () => t`Triggers when an HTTPS certificate nears expiration`,+ noDuration: true,+ min: 1,+ max: 365,+ start: 14,+ },GPU: {name: () => t`GPU Usage`,unit: "%",