Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

feat: network monitoring from agents (#2266, #1911)

hankbb1b39928e31
diff --git a/.dockerignore b/.dockerignore
index d0ebc4d7d..9ded2435f 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,6 +1,6 @@
# Node.js dependencies
-node_modules
-internalsite/node_modules
+node_modules/
+**/node_modules/
# Go build artifacts and binaries
build
diff --git a/agent/agent.go b/agent/agent.go
index 86abb8a3f..454c565e5 100644
--- a/agent/agent.go
+++ b/agent/agent.go
@@ -48,6 +48,7 @@ type Agent struct {
keys []gossh.PublicKey // SSH public keys
smartManager *SmartManager // Manages SMART data
systemdManager *systemdManager // Manages systemd services
+ monitorManager *MonitorManager // Manages network monitors
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
}
@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
// initialize handler registry
agent.handlerRegistry = NewHandlerRegistry()
+ // initialize monitor manager
+ agent.monitorManager = newMonitorManager()
+
agent.storagePoolManager = newStoragePoolManager()
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
}
}
+ if a.monitorManager != nil {
+ data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
+ slog.Debug("Monitors", "data", data.Monitors)
+ }
+
// skip updating systemd services if cache time is not the default 60sec interval
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
totalCount := uint16(a.systemdManager.getServiceStatsCount())
diff --git a/agent/connection_manager.go b/agent/connection_manager.go
index 28ea98aec..0f29bd24a 100644
--- a/agent/connection_manager.go
+++ b/agent/connection_manager.go
@@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
// }
func (c *ConnectionManager) stop() error {
_ = c.agent.StopServer()
+ c.agent.monitorManager.Stop()
c.closeWebSocket()
return health.CleanUp()
}
diff --git a/agent/gpu_test.go b/agent/gpu_test.go
index 1ac6e0187..889f6c26b 100644
--- a/agent/gpu_test.go
+++ b/agent/gpu_test.go
@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {
}
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
- // Save original PATH
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
tests := []struct {
@@ -1213,7 +1212,7 @@ echo "[]"`
{
name: "no gpu tools available",
setupCommands: func(_ string) error {
- t.Setenv("PATH", "")
+ // The subtest already restricts PATH to its empty temporary directory.
return nil
},
wantErr: true,
diff --git a/agent/handlers.go b/agent/handlers.go
index 9da87e542..67b69a73d 100644
--- a/agent/handlers.go
+++ b/agent/handlers.go
@@ -7,6 +7,7 @@ import (
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart"
"log/slog"
@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
+ registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
return registry
@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
return hctx.SendResponse(details, hctx.RequestID)
}
+
+////////////////////////////////////////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////////
+
+// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
+type SyncNetworkMonitorsHandler struct{}
+
+func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
+ var req monitor.SyncRequest
+ if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
+ return err
+ }
+ resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
+ if err != nil {
+ return err
+ }
+ return hctx.SendResponse(resp, hctx.RequestID)
+}
diff --git a/agent/network_monitor.go b/agent/network_monitor.go
new file mode 100644
index 000000000..d5257675a
--- /dev/null
+++ b/agent/network_monitor.go
@@ -0,0 +1,176 @@
+package agent
+
+import (
+ "errors"
+ "fmt"
+ "net/http"
+ "sync"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// MonitorManager manages network monitor configurations and task lifetimes.
+type MonitorManager struct {
+ mu sync.RWMutex
+ monitors map[string]*monitorTask // keyed by monitor ID
+ probe monitorProbe
+ resumeGuard monitorResumeGuard
+}
+
+func newMonitorManager() *MonitorManager {
+ return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
+}
+
+func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
+ return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
+}
+
+// SyncMonitors replaces all monitor tasks with the given configs.
+func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
+ pm.mu.Lock()
+ defer pm.mu.Unlock()
+
+ // Build set of new keys
+ newKeys := make(map[string]monitor.Config, len(configs))
+ for _, cfg := range configs {
+ if cfg.ID == "" {
+ continue
+ }
+ newKeys[cfg.ID] = cfg
+ }
+
+ // Stop removed monitors
+ for key, task := range pm.monitors {
+ if _, exists := newKeys[key]; !exists {
+ task.cancel()
+ delete(pm.monitors, key)
+ }
+ }
+
+ // Start new monitors and restart tasks whose config changed.
+ for key, cfg := range newKeys {
+ task, exists := pm.monitors[key]
+ if exists && task.config == cfg {
+ continue
+ }
+ if exists {
+ task.cancel()
+ }
+ task = newMonitorTaskFromExisting(cfg, task)
+ task.resumeGuard = &pm.resumeGuard
+ pm.resumeGuard.start()
+ pm.monitors[key] = task
+ pm.startMonitor(task)
+ }
+ if len(pm.monitors) == 0 {
+ pm.resumeGuard.shutdown()
+ }
+}
+
+// HandleSyncRequest applies a full or incremental monitor sync request.
+func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
+ switch req.Action {
+ case monitor.SyncActionReplace:
+ pm.SyncMonitors(req.Configs)
+ return monitor.SyncResponse{}, nil
+ case monitor.SyncActionUpsert:
+ result, err := pm.UpsertMonitor(req.Config, req.RunNow)
+ if err != nil {
+ return monitor.SyncResponse{}, err
+ }
+ if result == nil {
+ return monitor.SyncResponse{}, nil
+ }
+ return monitor.SyncResponse{Result: *result}, nil
+ case monitor.SyncActionDelete:
+ if req.Config.ID == "" {
+ return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
+ }
+ pm.DeleteMonitor(req.Config.ID)
+ return monitor.SyncResponse{}, nil
+ default:
+ return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
+ }
+}
+
+// UpsertMonitor creates or replaces a single monitor task.
+func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
+ if config.ID == "" {
+ return nil, errors.New("missing monitor ID")
+ }
+
+ pm.mu.Lock()
+ task, exists := pm.monitors[config.ID]
+ if exists && task.config == config {
+ pm.mu.Unlock()
+ if !runNow {
+ return nil, nil
+ }
+ return task.runProbe(pm.probe), nil
+ }
+ if exists {
+ task.cancel()
+ }
+ task = newMonitorTaskFromExisting(config, task)
+ task.resumeGuard = &pm.resumeGuard
+ pm.resumeGuard.start()
+ pm.monitors[config.ID] = task
+ pm.mu.Unlock()
+
+ if runNow {
+ result := task.runProbe(pm.probe)
+ pm.startMonitor(task)
+ return result, nil
+ }
+ pm.startMonitor(task)
+ return nil, nil
+}
+
+// DeleteMonitor stops and removes a single monitor task.
+func (pm *MonitorManager) DeleteMonitor(id string) {
+ if id == "" {
+ return
+ }
+ pm.mu.Lock()
+ defer pm.mu.Unlock()
+ if task, exists := pm.monitors[id]; exists {
+ task.cancel()
+ delete(pm.monitors, id)
+ }
+ if len(pm.monitors) == 0 {
+ pm.resumeGuard.shutdown()
+ }
+}
+
+// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
+func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
+ pm.mu.RLock()
+ defer pm.mu.RUnlock()
+
+ results := make(map[string]monitor.Result, len(pm.monitors))
+ now := time.Now()
+ duration := time.Duration(durationMs) * time.Millisecond
+
+ for _, task := range pm.monitors {
+ result, ok := task.history.result(duration, now)
+
+ if !ok {
+ continue
+ }
+ results[task.config.ID] = result
+ }
+
+ return results
+}
+
+// Stop stops all monitor tasks.
+func (pm *MonitorManager) Stop() {
+ pm.mu.Lock()
+ defer pm.mu.Unlock()
+ for key, task := range pm.monitors {
+ task.cancel()
+ delete(pm.monitors, key)
+ }
+ pm.resumeGuard.shutdown()
+}
diff --git a/agent/network_monitor_history.go b/agent/network_monitor_history.go
new file mode 100644
index 000000000..9db1639ab
--- /dev/null
+++ b/agent/network_monitor_history.go
@@ -0,0 +1,274 @@
+package agent
+
+import (
+ "math"
+ "sync"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// Monitors run at user-defined intervals (e.g., every 10s).
+// To keep memory usage low and constant, data is stored in two layers:
+// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
+// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
+// wall-clock minute. Samples collected within the same minute are aggregated
+// (sum, min, max, count) into a single bucket.
+//
+// Short-term requests (<= 61s) use raw samples.
+// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
+// of individual data points.
+
+const (
+ // monitorRawRetention is the duration to keep individual samples
+ monitorRawRetention = 61 * time.Second
+ // monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
+ monitorMinuteBucketLen int32 = 61
+)
+
+// monitorHistory owns retention and aggregation, independently of probe execution.
+type monitorHistory struct {
+ mu sync.Mutex
+ sampleCount int64
+ samples []monitorSample
+ buckets [monitorMinuteBucketLen]monitorBucket
+}
+
+func newMonitorHistory() *monitorHistory {
+ // Start small for typical intervals; append grows the buffer for faster probes.
+ return &monitorHistory{samples: make([]monitorSample, 0, 4)}
+}
+
+func (h *monitorHistory) clone() *monitorHistory {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ cloned := newMonitorHistory()
+ cloned.samples = append(cloned.samples, h.samples...)
+ cloned.buckets = h.buckets
+ cloned.sampleCount = h.sampleCount
+ return cloned
+}
+
+func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ return h.resultLocked(duration, now)
+}
+
+func (h *monitorHistory) record(sample monitorSample) monitor.Result {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ h.addSampleLocked(sample)
+ result, _ := h.resultLocked(time.Minute, sample.timestamp)
+ return result
+}
+
+// monitorSample stores one monitor attempt and its collection time.
+type monitorSample struct {
+ responseUs int64 // -1 means loss
+ timestamp time.Time
+}
+
+// monitorBucket stores one minute of aggregated monitor data.
+type monitorBucket struct {
+ minute int32
+ filled bool
+ stats monitorAggregate
+}
+
+// monitorAggregate accumulates successful response stats and total sample counts.
+type monitorAggregate struct {
+ sumUs int64
+ minUs int64
+ maxUs int64
+ totalCount int64
+ successCount int64
+}
+
+// newMonitorAggregate initializes an aggregate with an unset minimum value.
+func newMonitorAggregate() monitorAggregate {
+ return monitorAggregate{minUs: math.MaxInt64}
+}
+
+// addResponse folds a single monitor sample into the aggregate.
+func (agg *monitorAggregate) addResponse(responseUs int64) {
+ agg.totalCount++
+ if responseUs < 0 {
+ return
+ }
+ agg.successCount++
+ agg.sumUs += responseUs
+ if responseUs < agg.minUs {
+ agg.minUs = responseUs
+ }
+ if responseUs > agg.maxUs {
+ agg.maxUs = responseUs
+ }
+}
+
+// addAggregate merges another aggregate into this one.
+func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
+ if other.totalCount == 0 {
+ return
+ }
+ agg.totalCount += other.totalCount
+ agg.successCount += other.successCount
+ agg.sumUs += other.sumUs
+ if other.successCount == 0 {
+ return
+ }
+ if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
+ agg.minUs = other.minUs
+ }
+ if other.maxUs > agg.maxUs {
+ agg.maxUs = other.maxUs
+ }
+}
+
+// hasData reports whether the aggregate contains any samples.
+func (agg monitorAggregate) hasData() bool {
+ return agg.totalCount > 0
+}
+
+// result converts the aggregate into the monitor result format.
+func (agg monitorAggregate) result() monitor.Result {
+ avg := agg.avgResponse()
+ result := monitor.Result{
+ AvgResponse: avg,
+ MinResponse: agg.minUs,
+ MaxResponse: agg.maxUs,
+ PacketLoss: agg.lossPercentage(),
+ TotalCount: agg.totalCount,
+ SuccessCount: agg.successCount,
+ ResponseSum: agg.sumUs,
+ }
+ if agg.successCount == 0 {
+ result.MinResponse, result.MaxResponse = 0, 0
+ }
+ return result
+}
+
+// avgResponse returns the rounded average of successful samples.
+func (agg monitorAggregate) avgResponse() int64 {
+ if agg.successCount == 0 {
+ return 0
+ }
+ return agg.sumUs / agg.successCount
+
+}
+
+// lossPercentage returns the rounded failure rate for the aggregate.
+func (agg monitorAggregate) lossPercentage() float64 {
+ if agg.totalCount == 0 {
+ return 0
+ }
+ return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
+}
+
+// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
+func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
+ agg := h.aggregateLocked(duration, now)
+ if !agg.hasData() {
+ // short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
+ // between monitor samples since monitors run at longer, user-defined intervals; fall back to
+ // the most recent sample so realtime requests still report current status.
+ agg = h.latestSampleAggregateLocked()
+ }
+ hourAgg := h.aggregateLocked(time.Hour, now)
+ if !agg.hasData() {
+ return monitor.Result{}, false
+ }
+
+ result := agg.result()
+ if len(h.samples) > 0 {
+ result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
+ }
+
+ result.AvgResponse1h = hourAgg.avgResponse()
+ result.MinResponse1h = hourAgg.minUs
+ result.MaxResponse1h = hourAgg.maxUs
+ result.PacketLoss1h = hourAgg.lossPercentage()
+ result.SampleCount = h.sampleCount
+
+ if hourAgg.successCount == 0 {
+ result.MinResponse1h, result.MaxResponse1h = 0, 0
+ }
+ return result, true
+}
+
+// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
+func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
+ agg := newMonitorAggregate()
+ if len(h.samples) == 0 {
+ return agg
+ }
+ agg.addResponse(h.samples[len(h.samples)-1].responseUs)
+ return agg
+}
+
+// aggregateLocked collects monitor data for the requested time window.
+func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
+ cutoff := now.Add(-duration)
+ // Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
+ if duration <= monitorRawRetention {
+ return aggregateSamplesSince(h.samples, cutoff)
+ }
+ return aggregateBucketsSince(h.buckets[:], cutoff, now)
+}
+
+// aggregateSamplesSince aggregates raw samples newer than the cutoff.
+func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
+ agg := newMonitorAggregate()
+ for _, sample := range samples {
+ if sample.timestamp.Before(cutoff) {
+ continue
+ }
+ agg.addResponse(sample.responseUs)
+ }
+ return agg
+}
+
+// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
+func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
+ agg := newMonitorAggregate()
+ startMinute := int32(cutoff.Unix() / 60)
+ endMinute := int32(now.Unix() / 60)
+ for _, bucket := range buckets {
+ if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
+ continue
+ }
+ agg.addAggregate(bucket.stats)
+ }
+ return agg
+}
+
+// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
+func (h *monitorHistory) addSampleLocked(sample monitorSample) {
+ h.sampleCount++
+ cutoff := sample.timestamp.Add(-monitorRawRetention)
+ start := 0
+ for i := range h.samples {
+ if !h.samples[i].timestamp.Before(cutoff) {
+ start = i
+ break
+ }
+ if i == len(h.samples)-1 {
+ start = len(h.samples)
+ }
+ }
+ if start > 0 {
+ size := copy(h.samples, h.samples[start:])
+ h.samples = h.samples[:size]
+ }
+ h.samples = append(h.samples, sample)
+
+ minute := int32(sample.timestamp.Unix() / 60)
+ // Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
+ bucket := &h.buckets[minute%monitorMinuteBucketLen]
+ if !bucket.filled || bucket.minute != minute {
+ bucket.minute = minute
+ bucket.filled = true
+ bucket.stats = newMonitorAggregate()
+ }
+ bucket.stats.addResponse(sample.responseUs)
+}
diff --git a/agent/network_monitor_history_test.go b/agent/network_monitor_history_test.go
new file mode 100644
index 000000000..12a944abf
--- /dev/null
+++ b/agent/network_monitor_history_test.go
@@ -0,0 +1,154 @@
+package agent
+
+import (
+ "testing"
+ "time"
+
+ "github.com/fxamacker/cbor/v2"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMonitorHistoryWindowCounts(t *testing.T) {
+ history := newMonitorHistory()
+ now := time.Now()
+ // This older success counts toward lifetime warm-up, but not this window.
+ history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
+ history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
+ history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
+ history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
+ result, ok := history.result(time.Minute, now)
+ require.True(t, ok)
+ assert.EqualValues(t, 4, result.SampleCount)
+ assert.EqualValues(t, 3, result.TotalCount)
+ assert.EqualValues(t, 2, result.SuccessCount)
+ assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
+ assert.EqualValues(t, 15, result.AvgResponse)
+ assert.Equal(t, 33.33, result.PacketLoss)
+
+ encoded, err := cbor.Marshal(result)
+ require.NoError(t, err)
+ var decoded monitor.Result
+ require.NoError(t, cbor.Unmarshal(encoded, &decoded))
+ assert.Equal(t, result, decoded)
+ stats := monitor.Stats{}.FromResult(decoded)
+ assert.Equal(t, result.TotalCount, stats.TotalCount)
+ assert.Equal(t, result.SuccessCount, stats.SuccessCount)
+ assert.Equal(t, result.ResponseSum, stats.ResponseSum)
+
+ // Reads do not consume samples. A short window's latest-sample fallback
+ // carries the count for that single failure, not the minute or lifetime count.
+ repeated, _ := history.result(time.Minute, now)
+ assert.Equal(t, result, repeated)
+ fallback, ok := history.result(time.Second, now)
+ require.True(t, ok)
+ assert.EqualValues(t, 1, fallback.TotalCount)
+ assert.Zero(t, fallback.SuccessCount)
+ assert.Zero(t, fallback.ResponseSum)
+ assert.Equal(t, 100.0, fallback.PacketLoss)
+ assert.EqualValues(t, 4, fallback.SampleCount)
+}
+
+func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
+ now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
+ history := newMonitorHistory()
+
+ history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
+ history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
+ history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
+
+ agg := history.aggregateLocked(time.Minute, now)
+ require.True(t, agg.hasData())
+ assert.Equal(t, int64(2), agg.totalCount)
+ assert.Equal(t, int64(1), agg.successCount)
+ result := agg.result()
+ assert.Equal(t, int64(20), result.AvgResponse)
+ assert.Equal(t, int64(20), result.MinResponse)
+ assert.Equal(t, int64(20), result.MaxResponse)
+ assert.Equal(t, 50.0, result.PacketLoss)
+}
+
+func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
+ now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
+ history := newMonitorHistory()
+
+ history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
+ history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
+ history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
+ history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
+ history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
+
+ agg := history.aggregateLocked(10*time.Minute, now)
+ require.True(t, agg.hasData())
+ assert.Equal(t, int64(4), agg.totalCount)
+ assert.Equal(t, int64(3), agg.successCount)
+ result := agg.result()
+ assert.Equal(t, int64(30), result.AvgResponse)
+ assert.Equal(t, int64(20), result.MinResponse)
+ assert.Equal(t, int64(40), result.MaxResponse)
+ assert.Equal(t, 25.0, result.PacketLoss)
+}
+
+func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
+ now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
+ history := newMonitorHistory()
+
+ history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
+ history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
+
+ require.Len(t, history.samples, 1)
+ assert.Equal(t, int64(20), history.samples[0].responseUs)
+
+ agg := history.aggregateLocked(10*time.Minute, now)
+ require.True(t, agg.hasData())
+ assert.Equal(t, int64(2), agg.totalCount)
+ assert.Equal(t, int64(2), agg.successCount)
+ result := agg.result()
+ assert.Equal(t, int64(15), result.AvgResponse)
+ assert.Equal(t, int64(10), result.MinResponse)
+ assert.Equal(t, int64(20), result.MaxResponse)
+ assert.Equal(t, 0.0, result.PacketLoss)
+}
+
+func TestMonitorHistoryProbeTimestamp(t *testing.T) {
+ history := newMonitorHistory()
+ start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
+ _, ok := history.result(time.Minute, start)
+ require.False(t, ok)
+ first := history.record(monitorSample{responseUs: 20, timestamp: start})
+ assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
+ for minute := 0; minute < 5; minute++ {
+ now := start.Add(time.Duration(minute)*time.Minute + time.Second)
+ // Realtime reads must not consume freshness for the persistence request.
+ for _, window := range []time.Duration{time.Second, time.Minute} {
+ result, ok := history.result(window, now)
+ require.True(t, ok)
+ assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
+ assert.Equal(t, int64(20), result.AvgResponse)
+ }
+ }
+ next := start.Add(5 * time.Minute)
+ failed := history.record(monitorSample{responseUs: -1, timestamp: next})
+ assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
+ assert.Equal(t, float64(100), failed.PacketLoss)
+ repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
+ require.True(t, ok)
+ assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
+ assert.Equal(t, float64(100), repeated.PacketLoss)
+}
+
+func TestMonitorHistorySampleCount(t *testing.T) {
+ history := newMonitorHistory()
+ now := time.Now()
+ // Both failed and successful probes count, including older samples so
+ // monitors with hourly intervals can finish warming up.
+ history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
+ for i, response := range []int64{10, -1, 20} {
+ result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
+ assert.EqualValues(t, i+2, result.SampleCount)
+ }
+ result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
+ require.True(t, ok)
+ assert.EqualValues(t, 4, result.SampleCount)
+}
diff --git a/agent/network_monitor_ping.go b/agent/network_monitor_ping.go
new file mode 100644
index 000000000..632afd668
--- /dev/null
+++ b/agent/network_monitor_ping.go
@@ -0,0 +1,312 @@
+package agent
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "errors"
+ "fmt"
+ "math"
+ "net"
+ "os"
+ "os/exec"
+ "regexp"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "golang.org/x/net/icmp"
+ "golang.org/x/net/ipv4"
+ "golang.org/x/net/ipv6"
+
+ "log/slog"
+)
+
+// Match the numeric RTT independently of the localized label used by Windows.
+var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
+
+var icmpSequence atomic.Uint32
+
+type icmpPacketConn interface {
+ Close() error
+}
+
+// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
+// all native methods fail, the choice is cached so subsequent monitors skip
+// the trial-and-error overhead.
+type icmpMethod uint8
+
+const (
+ icmpUntried icmpMethod = iota // haven't tried yet
+ icmpRaw // privileged raw socket
+ icmpDatagram // unprivileged datagram socket
+ icmpExecFallback // shell out to system ping command
+)
+
+// icmpFamily holds the network parameters and cached detection result for one address family.
+type icmpFamily struct {
+ rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
+ dgramNetwork string // e.g. "udp4" or "udp6"
+ listenAddr string // "0.0.0.0" or "::"
+ echoType icmp.Type // outgoing echo request type
+ replyType icmp.Type // expected echo reply type
+ proto int // IANA protocol number for parsing replies
+ isIPv6 bool
+ mode icmpMethod // cached detection result (guarded by icmpModeMu)
+}
+
+var (
+ icmpV4 = icmpFamily{
+ rawNetwork: "ip4:icmp",
+ dgramNetwork: "udp4",
+ listenAddr: "0.0.0.0",
+ echoType: ipv4.ICMPTypeEcho,
+ replyType: ipv4.ICMPTypeEchoReply,
+ proto: 1,
+ }
+ icmpV6 = icmpFamily{
+ rawNetwork: "ip6:ipv6-icmp",
+ dgramNetwork: "udp6",
+ listenAddr: "::",
+ echoType: ipv6.ICMPTypeEchoRequest,
+ replyType: ipv6.ICMPTypeEchoReply,
+ proto: 58,
+ isIPv6: true,
+ }
+ icmpModeMu sync.Mutex
+ icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
+ return icmp.ListenPacket(network, listenAddr)
+ }
+)
+
+// monitorICMP sends an ICMP echo request and measures round-trip response.
+// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
+// unprivileged datagram, or exec fallback) is detected once per address
+// family and cached for subsequent monitors.
+// Returns response in microseconds, or -1 and an error on failure.
+func monitorICMP(ctx context.Context, target string) (int64, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ family, ip, err := resolveICMPTarget(ctx, target)
+ if err != nil {
+ return -1, err
+ }
+
+ icmpModeMu.Lock()
+ if family.mode == icmpUntried {
+ family.mode = detectICMPMode(family, icmpListen)
+ }
+ mode := family.mode
+ icmpModeMu.Unlock()
+
+ switch mode {
+ case icmpRaw:
+ return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
+ case icmpDatagram:
+ return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
+ case icmpExecFallback:
+ return monitorICMPExec(ctx, ip.String(), family.isIPv6)
+ default:
+ return -1, errors.New("unsupported ICMP mode")
+ }
+}
+
+// resolveICMPTarget resolves a target hostname or IP to determine the address
+// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
+func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
+ if ip := net.ParseIP(target); ip != nil {
+ if ip.To4() != nil {
+ return &icmpV4, ip.To4(), nil
+ }
+ return &icmpV6, ip, nil
+ }
+
+ ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
+ if err != nil || len(ips) == 0 {
+ return nil, nil, err
+ }
+ for _, ip := range ips {
+ if v4 := ip.To4(); v4 != nil {
+ return &icmpV4, v4, nil
+ }
+ }
+ return &icmpV6, ips[0], nil
+}
+
+func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
+ label := "IPv4"
+ if family.isIPv6 {
+ label = "IPv6"
+ }
+
+ conn, err := listen(family.rawNetwork, family.listenAddr)
+ slog.Debug("ICMP raw socket test", "family", label, "err", err)
+ if err == nil {
+ conn.Close()
+ return icmpRaw
+ }
+
+ conn, err = listen(family.dgramNetwork, family.listenAddr)
+ slog.Debug("ICMP datagram socket test", "family", label, "err", err)
+ if err == nil {
+ conn.Close()
+ return icmpDatagram
+ }
+
+ return icmpExecFallback
+}
+
+// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
+func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
+ conn, err := icmp.ListenPacket(network, family.listenAddr)
+ if err != nil {
+ return -1, err
+ }
+ defer conn.Close()
+
+ return monitorICMPPacket(ctx, conn, family, dst)
+}
+
+func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
+ if err := ctx.Err(); err != nil {
+ return -1, err
+ }
+ // Closing the socket interrupts both reads and writes on cancellation.
+ stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
+ defer stop()
+
+ // Prepare correlation data before starting the round-trip timer. The token
+ // also distinguishes delayed replies after the 16-bit sequence wraps.
+ token := make([]byte, 16)
+ if _, err := rand.Read(token); err != nil {
+ return -1, err
+ }
+ echo := &icmp.Echo{
+ ID: os.Getpid() & 0xffff,
+ Seq: int(icmpSequence.Add(1) & 0xffff),
+ Data: token,
+ }
+ // Linux ping sockets replace the Echo ID with their bound port. Darwin
+ // datagram sockets and raw sockets preserve the supplied ID.
+ if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
+ echo.ID = local.Port
+ }
+ targetIP := icmpAddrIP(dst)
+ msg := &icmp.Message{
+ Type: family.echoType,
+ Code: 0,
+ Body: echo,
+ }
+ msgBytes, err := msg.Marshal(nil)
+ if err != nil {
+ return -1, err
+ }
+
+ // Set deadline before sending
+ if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
+ return -1, err
+ }
+
+ buf := make([]byte, 1500)
+ start := time.Now()
+ if _, err := conn.WriteTo(msgBytes, dst); err != nil {
+ return -1, err
+ }
+
+ // Read reply
+ for {
+ n, peer, err := conn.ReadFrom(buf)
+ received := time.Now()
+ if err != nil {
+ return -1, err
+ }
+ if !targetIP.Equal(icmpAddrIP(peer)) {
+ continue
+ }
+
+ reply, err := icmp.ParseMessage(family.proto, buf[:n])
+ if err != nil || reply.Type != family.replyType || reply.Code != 0 {
+ continue
+ }
+
+ body, ok := reply.Body.(*icmp.Echo)
+ if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
+ return received.Sub(start).Microseconds(), nil
+ }
+ // Keep waiting for our reply without extending the original deadline.
+ }
+}
+
+func icmpAddrIP(addr net.Addr) net.IP {
+ switch addr := addr.(type) {
+ case *net.IPAddr:
+ return addr.IP
+ case *net.UDPAddr:
+ return addr.IP
+ default:
+ return nil
+ }
+}
+
+// pingCommand selects the executable and arguments for the supported agent platforms.
+// The context deadline enforces the timeout: -W has incompatible meanings across
+// Linux, BSD IPv4 ping, and macOS ping6.
+func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
+ family := "-4"
+ if isIPv6 {
+ family = "-6"
+ }
+ switch goos {
+ case "windows":
+ return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
+ case "linux":
+ return "ping", []string{family, "-n", "-c", "1", target}, nil
+ case "darwin", "freebsd", "openbsd":
+ command := "ping"
+ if isIPv6 {
+ command = "ping6"
+ }
+ return command, []string{"-n", "-c", "1", target}, nil
+ default:
+ return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
+ }
+}
+
+// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
+func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
+ if err != nil {
+ return -1, err
+ }
+ cmd := exec.CommandContext(ctx, name, args...)
+ // Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
+ cmd.Env = append(os.Environ(), "LC_ALL=C")
+ output, err := cmd.Output()
+ if ctx.Err() != nil {
+ return -1, ctx.Err()
+ }
+ if err != nil {
+ return -1, fmt.Errorf("%s failed: %w", name, err)
+ }
+ return parsePingResponse(output)
+}
+
+// parsePingResponse returns the reported RTT, never subprocess execution time.
+// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
+func parsePingResponse(output []byte) (int64, error) {
+ matches := pingTimeRegex.FindSubmatch(output)
+ if len(matches) < 2 {
+ return -1, errors.New("ping output contains no round-trip time")
+ }
+ ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
+ if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
+ return -1, errors.New("invalid round-trip time in ping output")
+ }
+ return int64(math.Round(ms * 1000)), nil
+}
diff --git a/agent/network_monitor_ping_test.go b/agent/network_monitor_ping_test.go
new file mode 100644
index 000000000..89ff92e66
--- /dev/null
+++ b/agent/network_monitor_ping_test.go
@@ -0,0 +1,433 @@
+//go:build testing
+
+package agent
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/net/icmp"
+)
+
+type testICMPPacketConn struct{}
+
+func (testICMPPacketConn) Close() error { return nil }
+
+type blockingICMPConn struct {
+ net.PacketConn
+ reading chan struct{}
+}
+
+func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
+ return len(p), nil
+}
+
+func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
+ close(c.reading)
+ return c.PacketConn.ReadFrom(p)
+}
+
+func TestMonitorICMPPacketCancellation(t *testing.T) {
+ conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
+ require.NoError(t, err)
+ defer conn.Close()
+ blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ _, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
+ done <- err
+ }()
+ select {
+ case <-blocking.reading:
+ case <-time.After(time.Second):
+ t.Fatal("probe did not begin reading")
+ }
+ cancel()
+ select {
+ case err := <-done:
+ require.Error(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("cancellation did not interrupt the socket read")
+ }
+}
+
+func TestMonitorICMPExecCancellation(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a POSIX shell stub for ping")
+ }
+ dir := t.TempDir()
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
+ ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ _, err := monitorICMPExec(ctx, "127.0.0.1", false)
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ case <-time.After(time.Second):
+ t.Fatal("cancellation did not terminate ping")
+ }
+}
+
+func TestPingCommand(t *testing.T) {
+ for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
+ for _, ipv6 := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
+ target, family := "192.0.2.1", "-4"
+ if ipv6 {
+ target, family = "2001:db8::1", "-6"
+ }
+ name, args, err := pingCommand(goos, target, ipv6)
+ require.NoError(t, err)
+ wantName := "ping"
+ wantArgs := []string{"-n", "-c", "1", target}
+ switch goos {
+ case "windows":
+ wantArgs = []string{family, "-n", "1", "-w", "3000", target}
+ case "linux":
+ wantArgs = append([]string{family}, wantArgs...)
+ default:
+ if ipv6 {
+ wantName = "ping6"
+ }
+ }
+ assert.Equal(t, wantName, name)
+ assert.Equal(t, wantArgs, args)
+ })
+ }
+ }
+ _, _, err := pingCommand("unsupported", "192.0.2.1", false)
+ require.Error(t, err)
+}
+
+func TestParsePingResponse(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ output string
+ wantUs int64
+ }{
+ {"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
+ {"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
+ {"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
+ {"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
+ {"windows submillisecond", "Reply from ::1: time<1ms", 1000},
+ {"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
+ {"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
+ {"rounding", "time=0.1236 ms", 124},
+ {"empty", "", -1},
+ {"timeout", "Request timed out.", -1},
+ {"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
+ {"malformed", "time=oops ms", -1},
+ {"negative", "time=-1 ms", -1},
+ {"overflow", "time=999999999999999999999 ms", -1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ responseUs, err := parsePingResponse([]byte(tc.output))
+ if tc.wantUs < 0 {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ assert.Equal(t, tc.wantUs, responseUs)
+ })
+ }
+}
+
+func TestMonitorICMPExecOutput(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("test uses a POSIX shell stub for ping")
+ }
+ for _, tc := range []struct {
+ name string
+ output string
+ exit int
+ wantUs int64
+ }{
+ {"success", "time=1.234 ms", 0, 1234},
+ {"missing RTT", "unrecognized output", 0, -1},
+ {"failed command with RTT", "time=1.234 ms", 1, -1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ dir := t.TempDir()
+ // Also verify an inherited locale cannot override the C locale.
+ script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
+ t.Setenv("LC_ALL", "de_DE.UTF-8")
+ responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
+ if tc.wantUs < 0 {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ assert.Equal(t, tc.wantUs, responseUs)
+ })
+ }
+}
+
+type icmpTestReply struct {
+ data []byte
+ peer net.Addr
+}
+
+type scriptedICMPConn struct {
+ net.PacketConn
+ local net.Addr
+ onWrite func([]byte, net.Addr)
+ replies []icmpTestReply
+ reads int
+ deadlineSets int
+}
+
+func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
+
+func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
+ c.deadlineSets++
+ return nil
+}
+
+func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
+ c.onWrite(data, dst)
+ return len(data), nil
+}
+
+func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
+ c.reads++
+ if len(c.replies) == 0 {
+ return 0, nil, os.ErrDeadlineExceeded
+ }
+ reply := c.replies[0]
+ c.replies = c.replies[1:]
+ return copy(buf, reply.data), reply.peer, nil
+}
+
+func TestMonitorICMPReplyCorrelation(t *testing.T) {
+ for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
+ for _, datagram := range []bool{false, true} {
+ network := family.rawNetwork
+ ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
+ if family.isIPv6 {
+ ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
+ }
+ var dst net.Addr = &net.IPAddr{IP: ip}
+ var wrongPeer net.Addr = &net.IPAddr{IP: other}
+ if datagram {
+ network = family.dgramNetwork
+ dst = &net.UDPAddr{IP: ip}
+ wrongPeer = &net.UDPAddr{IP: other}
+ }
+ for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
+ for _, eventuallyMatches := range []bool{false, true} {
+ ending := "timeout"
+ if eventuallyMatches {
+ ending = "success"
+ }
+ t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
+ conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
+ if datagram {
+ conn.local = &net.UDPAddr{Port: 12345}
+ if runtime.GOOS == "linux" {
+ // Deliberately differ from the process ID.
+ conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
+ }
+ }
+ conn.onWrite = func(data []byte, target net.Addr) {
+ require.Equal(t, dst, target)
+ request, err := icmp.ParseMessage(family.proto, data)
+ require.NoError(t, err)
+ echo := request.Body.(*icmp.Echo)
+ expectedID := os.Getpid() & 0xffff
+ if datagram && runtime.GOOS == "linux" {
+ expectedID = conn.local.(*net.UDPAddr).Port
+ }
+ require.Equal(t, expectedID, echo.ID)
+ reply := &icmp.Message{Type: family.replyType, Body: echo}
+ valid, err := reply.Marshal(nil)
+ require.NoError(t, err)
+ peer := dst
+ switch mismatch {
+ case "source":
+ peer = wrongPeer
+ case "id":
+ echo.ID ^= 1
+ case "sequence":
+ echo.Seq ^= 1
+ case "payload":
+ echo.Data[0] ^= 1
+ case "type":
+ reply.Type = family.echoType
+ case "code":
+ reply.Code = 1
+ }
+ invalid, err := reply.Marshal(nil)
+ require.NoError(t, err)
+ if mismatch == "malformed" {
+ invalid = invalid[:2]
+ }
+ conn.replies = []icmpTestReply{{invalid, peer}}
+ if eventuallyMatches {
+ conn.replies = append(conn.replies, icmpTestReply{valid, dst})
+ }
+ }
+ elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
+ if eventuallyMatches {
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, elapsed, int64(0))
+ } else {
+ require.ErrorIs(t, err, os.ErrDeadlineExceeded)
+ assert.Equal(t, int64(-1), elapsed)
+ }
+ assert.Equal(t, 2, conn.reads)
+ assert.Equal(t, 1, conn.deadlineSets)
+ })
+ }
+ }
+ }
+ }
+}
+
+func TestMonitorICMPLoopback(t *testing.T) {
+ for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
+ for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
+ t.Run(network, func(t *testing.T) {
+ conn, err := icmp.ListenPacket(network, family.listenAddr)
+ if err != nil {
+ t.Skipf("ICMP socket unavailable: %v", err)
+ }
+ defer conn.Close()
+ ip := net.ParseIP("127.0.0.1")
+ if family.isIPv6 {
+ ip = net.ParseIP("::1")
+ }
+ var dst net.Addr = &net.IPAddr{IP: ip}
+ if network == family.dgramNetwork {
+ dst = &net.UDPAddr{IP: ip}
+ }
+ elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, elapsed, int64(0))
+ })
+ }
+ }
+}
+
+func TestDetectICMPMode(t *testing.T) {
+ tests := []struct {
+ name string
+ family *icmpFamily
+ rawErr error
+ udpErr error
+ want icmpMethod
+ wantNetworks []string
+ }{
+ {
+ name: "IPv4 prefers raw socket when available",
+ family: &icmpV4,
+ want: icmpRaw,
+ wantNetworks: []string{"ip4:icmp"},
+ },
+ {
+ name: "IPv4 uses datagram when raw unavailable",
+ family: &icmpV4,
+ rawErr: errors.New("operation not permitted"),
+ want: icmpDatagram,
+ wantNetworks: []string{"ip4:icmp", "udp4"},
+ },
+ {
+ name: "IPv4 falls back to exec when both unavailable",
+ family: &icmpV4,
+ rawErr: errors.New("operation not permitted"),
+ udpErr: errors.New("protocol not supported"),
+ want: icmpExecFallback,
+ wantNetworks: []string{"ip4:icmp", "udp4"},
+ },
+ {
+ name: "IPv6 prefers raw socket when available",
+ family: &icmpV6,
+ want: icmpRaw,
+ wantNetworks: []string{"ip6:ipv6-icmp"},
+ },
+ {
+ name: "IPv6 uses datagram when raw unavailable",
+ family: &icmpV6,
+ rawErr: errors.New("operation not permitted"),
+ want: icmpDatagram,
+ wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
+ },
+ {
+ name: "IPv6 falls back to exec when both unavailable",
+ family: &icmpV6,
+ rawErr: errors.New("operation not permitted"),
+ udpErr: errors.New("protocol not supported"),
+ want: icmpExecFallback,
+ wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ calls := make([]string, 0, 2)
+ listen := func(network, listenAddr string) (icmpPacketConn, error) {
+ require.Equal(t, tt.family.listenAddr, listenAddr)
+ calls = append(calls, network)
+ switch network {
+ case tt.family.rawNetwork:
+ if tt.rawErr != nil {
+ return nil, tt.rawErr
+ }
+ case tt.family.dgramNetwork:
+ if tt.udpErr != nil {
+ return nil, tt.udpErr
+ }
+ default:
+ t.Fatalf("unexpected network %q", network)
+ }
+ return testICMPPacketConn{}, nil
+ }
+
+ assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
+ assert.Equal(t, tt.wantNetworks, calls)
+ })
+ }
+}
+
+func TestResolveICMPTarget(t *testing.T) {
+ t.Run("IPv4 literal", func(t *testing.T) {
+ family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
+ require.NoError(t, err)
+ require.NotNil(t, family)
+ assert.False(t, family.isIPv6)
+ assert.Equal(t, "127.0.0.1", ip.String())
+ })
+
+ t.Run("IPv6 literal", func(t *testing.T) {
+ family, ip, err := resolveICMPTarget(context.Background(), "::1")
+ require.NoError(t, err)
+ require.NotNil(t, family)
+ assert.True(t, family.isIPv6)
+ assert.Equal(t, "::1", ip.String())
+ })
+
+ t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
+ family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
+ require.NoError(t, err)
+ require.NotNil(t, family)
+ assert.False(t, family.isIPv6)
+ assert.Equal(t, "127.0.0.1", ip.String())
+ })
+}
diff --git a/agent/network_monitor_probe.go b/agent/network_monitor_probe.go
new file mode 100644
index 000000000..187a6bf37
--- /dev/null
+++ b/agent/network_monitor_probe.go
@@ -0,0 +1,105 @@
+package agent
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "net/http"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// monitorProbe performs one check. Errors are recorded as loss by the task runner.
+// Implementations must honor cancellation and bound their execution time.
+type monitorProbe func(context.Context, monitor.Config) (int64, error)
+
+func networkMonitorProbe(client *http.Client) monitorProbe {
+ return func(ctx context.Context, config monitor.Config) (int64, error) {
+ switch config.Protocol {
+ case "icmp":
+ return monitorICMP(ctx, config.Target)
+ case "tcp":
+ return monitorTCP(ctx, config.Target, config.Port)
+ case "http":
+ return monitorHTTP(ctx, client, config.Target)
+ case "dns":
+ return monitorDNS(ctx, config.Target)
+ default:
+ return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
+ }
+ }
+}
+
+// monitorTCP measures connection establishment time, including address fallback
+// but excluding DNS resolution.
+// Returns -1 and an error on failure.
+func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ // Resolve DNS first, outside the timing window but within the probe deadline.
+ ips, err := net.DefaultResolver.LookupHost(ctx, target)
+ if err != nil {
+ return -1, err
+ }
+ if len(ips) == 0 {
+ return -1, errors.New("no addresses resolved for TCP monitor")
+ }
+ portString := fmt.Sprintf("%d", port)
+ deadline, _ := ctx.Deadline()
+
+ // Share the remaining probe budget across addresses so an unresponsive
+ // first address cannot consume all the time available for alternatives.
+ start := time.Now()
+ for i, ip := range ips {
+ if err := ctx.Err(); err != nil {
+ return -1, err
+ }
+ dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
+ var conn net.Conn
+ conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
+ if err != nil {
+ continue
+ }
+ responseUs := time.Since(start).Microseconds()
+ conn.Close()
+ return responseUs, nil
+ }
+ return -1, err
+}
+
+// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
+func monitorDNS(ctx context.Context, target string) (int64, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+
+ start := time.Now()
+ ips, err := net.DefaultResolver.LookupHost(ctx, target)
+ if err != nil || len(ips) == 0 {
+ return -1, err
+ }
+ return time.Since(start).Microseconds(), nil
+}
+
+// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
+func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
+ if client == nil {
+ client = http.DefaultClient
+ }
+ start := time.Now()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return -1, err
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return -1, err
+ }
+ resp.Body.Close()
+ if resp.StatusCode >= 400 {
+ return -1, fmt.Errorf("HTTP error: %s", resp.Status)
+ }
+ return time.Since(start).Microseconds(), nil
+}
diff --git a/agent/network_monitor_resume.go b/agent/network_monitor_resume.go
new file mode 100644
index 000000000..495266232
--- /dev/null
+++ b/agent/network_monitor_resume.go
@@ -0,0 +1,88 @@
+package agent
+
+import (
+ "sync"
+ "time"
+)
+
+const (
+ monitorResumeHeartbeat = 10 * time.Second
+ // Allow scheduling jitter without mistaking an ordinary tick for resume.
+ monitorResumeGap = 2 * monitorResumeHeartbeat
+ monitorResumePause = 10 * time.Second
+)
+
+// monitorResumeGuard detects likely suspend/resume using wall time. A long
+// process stall or forward clock adjustment can also trigger the bounded pause.
+// One heartbeat is shared by all configured monitors.
+type monitorResumeGuard struct {
+ mu sync.Mutex
+ stop chan struct{}
+ lastTick time.Time
+ pauseUntil time.Time
+ generation uint32
+}
+
+func (g *monitorResumeGuard) start() {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if g.stop != nil {
+ return
+ }
+ stop := make(chan struct{})
+ g.stop = stop
+ g.lastTick = time.Now().Round(0)
+ g.pauseUntil = time.Time{}
+ go func() {
+ ticker := time.NewTicker(monitorResumeHeartbeat)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-stop:
+ return
+ case <-ticker.C:
+ g.mu.Lock()
+ if g.stop == stop {
+ g.observe(time.Now())
+ }
+ g.mu.Unlock()
+ }
+ }
+ }()
+}
+
+func (g *monitorResumeGuard) shutdown() {
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if g.stop != nil {
+ close(g.stop)
+ g.stop = nil
+ g.generation++
+ }
+}
+
+// observe requires mu. Strip the monotonic component because it can stop during
+// suspend. Read the current time rather than the ticker's queued timestamp.
+func (g *monitorResumeGuard) observe(now time.Time) {
+ now = now.Round(0)
+ if now.Sub(g.lastTick) > monitorResumeGap {
+ g.pauseUntil = now.Add(monitorResumePause)
+ g.generation++
+ }
+ g.lastTick = now
+}
+
+// snapshot also observes time so a probe waking before the heartbeat detects
+// resume itself. A changed generation invalidates probes spanning suspend.
+func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
+ if g == nil {
+ return 0, true
+ }
+ g.mu.Lock()
+ defer g.mu.Unlock()
+ if g.stop == nil {
+ return g.generation, true
+ }
+ g.observe(time.Now())
+ return g.generation, !g.lastTick.Before(g.pauseUntil)
+}
diff --git a/agent/network_monitor_resume_test.go b/agent/network_monitor_resume_test.go
new file mode 100644
index 000000000..f1a637c4d
--- /dev/null
+++ b/agent/network_monitor_resume_test.go
@@ -0,0 +1,121 @@
+//go:build testing
+
+package agent
+
+import (
+ "context"
+ "errors"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func simulateMonitorSleep(g *monitorResumeGuard) {
+ g.mu.Lock()
+ g.lastTick = time.Now().Add(-time.Hour).Round(0)
+ g.mu.Unlock()
+}
+
+func TestMonitorResumePause(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var g monitorResumeGuard
+ g.start()
+ defer g.shutdown()
+ generation, allowed := g.snapshot()
+ require.True(t, allowed)
+ // Heartbeats alone must keep the guard current between infrequent probes.
+ time.Sleep(time.Minute)
+ synctest.Wait()
+ steadyGeneration, allowed := g.snapshot()
+ require.True(t, allowed)
+ require.Equal(t, generation, steadyGeneration)
+ // The probe, rather than the heartbeat, must detect this gap.
+ simulateMonitorSleep(&g)
+ next, allowed := g.snapshot()
+ assert.False(t, allowed)
+ assert.NotEqual(t, generation, next)
+ time.Sleep(9 * time.Second)
+ _, allowed = g.snapshot()
+ assert.False(t, allowed)
+ time.Sleep(time.Second)
+ end, allowed := g.snapshot()
+ assert.True(t, allowed)
+ assert.Equal(t, next, end)
+ })
+}
+
+func TestMonitorResumeGuardLifecycle(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
+ defer pm.Stop()
+ assert.Nil(t, pm.resumeGuard.stop)
+ pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
+ stop := pm.resumeGuard.stop
+ require.NotNil(t, stop)
+ pm.DeleteMonitor("a")
+ assert.Equal(t, stop, pm.resumeGuard.stop)
+ pm.DeleteMonitor("b")
+ assert.Nil(t, pm.resumeGuard.stop)
+ select {
+ case <-stop:
+ default:
+ t.Fatal("heartbeat was not stopped")
+ }
+ time.Sleep(time.Hour)
+ _, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
+ require.NoError(t, err)
+ _, allowed := pm.resumeGuard.snapshot()
+ assert.True(t, allowed, "idle time must not trigger a resume pause")
+ pm.SyncMonitors(nil)
+ assert.Nil(t, pm.resumeGuard.stop)
+ })
+}
+
+func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var g monitorResumeGuard
+ g.start()
+ defer g.shutdown()
+ task := newMonitorTask(monitor.Config{ID: "test"})
+ defer task.cancel()
+ task.resumeGuard = &g
+ result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
+ simulateMonitorSleep(&g)
+ return 0, errors.New("network not ready")
+ })
+ assert.Nil(t, result)
+ assert.Empty(t, task.history.samples)
+ // Explicit requests may still run during the pause and record real failures.
+ result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
+ return 0, errors.New("unreachable")
+ })
+ require.NotNil(t, result)
+ assert.Equal(t, 100.0, result.PacketLoss)
+ })
+}
+
+func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var calls atomic.Int32
+ pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
+ calls.Add(1)
+ return 1, nil
+ })
+ defer pm.Stop()
+ pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
+ simulateMonitorSleep(&pm.resumeGuard)
+ pm.resumeGuard.snapshot()
+ time.Sleep(9 * time.Second)
+ synctest.Wait()
+ assert.Zero(t, calls.Load())
+ assert.Empty(t, pm.GetResults(1000))
+ time.Sleep(2 * time.Second)
+ synctest.Wait()
+ assert.Positive(t, calls.Load())
+ })
+}
diff --git a/agent/network_monitor_schedule.go b/agent/network_monitor_schedule.go
new file mode 100644
index 000000000..fa53c36c4
--- /dev/null
+++ b/agent/network_monitor_schedule.go
@@ -0,0 +1,60 @@
+package agent
+
+import (
+ "context"
+ "log/slog"
+ "math/rand"
+ "time"
+)
+
+func (pm *MonitorManager) startMonitor(task *monitorTask) {
+ interval := time.Duration(task.config.Interval) * time.Second
+ if interval < time.Second {
+ interval = 30 * time.Second
+ }
+ delay := getStagger(interval.Milliseconds())
+ slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
+ go runMonitorSchedule(task.ctx, interval, delay, func() {
+ if _, allowed := task.resumeGuard.snapshot(); allowed {
+ task.runProbe(pm.probe)
+ }
+ })
+}
+
+// runMonitorSchedule owns only timing. Checks run serially, and slow checks
+// naturally drop missed ticks rather than building an execution backlog.
+func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
+ timer := time.NewTimer(delay)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ return
+ case <-timer.C:
+ }
+ if ctx.Err() != nil {
+ return
+ }
+ run()
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ if ctx.Err() != nil {
+ return
+ }
+ run()
+ }
+ }
+}
+
+// getStagger returns an initial delay between half an interval and one interval.
+func getStagger(intervalMilli int64) time.Duration {
+ delay := rand.Intn(int(intervalMilli))
+ if delay < int(intervalMilli)/2 {
+ delay += int(intervalMilli) / 2
+ }
+ return time.Duration(delay) * time.Millisecond
+}
diff --git a/agent/network_monitor_schedule_test.go b/agent/network_monitor_schedule_test.go
new file mode 100644
index 000000000..a66a59abe
--- /dev/null
+++ b/agent/network_monitor_schedule_test.go
@@ -0,0 +1,167 @@
+//go:build testing
+
+package agent
+
+import (
+ "context"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMonitorScheduleTiming(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ var calls atomic.Int32
+ go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
+ synctest.Wait()
+ time.Sleep(4 * time.Second)
+ synctest.Wait()
+ assert.Equal(t, 0, int(calls.Load()))
+ time.Sleep(time.Second)
+ synctest.Wait()
+ assert.Equal(t, 1, int(calls.Load()))
+ time.Sleep(10 * time.Second)
+ synctest.Wait()
+ assert.Equal(t, 2, int(calls.Load()))
+ cancel()
+ synctest.Wait()
+ time.Sleep(time.Minute)
+ synctest.Wait()
+ assert.Equal(t, 2, int(calls.Load()))
+ })
+}
+
+func TestMonitorScheduleSlowProbe(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ var calls atomic.Int32
+ release := make(chan struct{})
+ go runMonitorSchedule(ctx, time.Second, 0, func() {
+ calls.Add(1)
+ select {
+ case <-release:
+ case <-ctx.Done():
+ }
+ })
+ synctest.Wait()
+ assert.Equal(t, 1, int(calls.Load()))
+ time.Sleep(time.Minute)
+ synctest.Wait()
+ assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
+ close(release)
+ synctest.Wait()
+ assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
+ time.Sleep(time.Second)
+ synctest.Wait()
+ assert.Equal(t, 2, int(calls.Load()))
+ cancel()
+ synctest.Wait()
+ })
+}
+
+func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var calls atomic.Int32
+ release := make(chan struct{})
+ cfg := monitor.Config{ID: "test", Interval: 10}
+ pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
+ assert.Equal(t, cfg, config)
+ calls.Add(1)
+ <-release
+ return 42, nil
+ })
+ defer pm.Stop()
+ task := newMonitorTask(cfg)
+ pm.monitors[cfg.ID] = task
+ go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
+ synctest.Wait()
+ results := make(chan *monitor.Result, 2)
+ for range 2 {
+ go func() {
+ result, _ := pm.UpsertMonitor(cfg, true)
+ results <- result
+ }()
+ }
+ synctest.Wait()
+ assert.Equal(t, 1, int(calls.Load()))
+ assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
+ close(release)
+ synctest.Wait()
+ first, second := <-results, <-results
+ require.NotNil(t, first)
+ require.NotNil(t, second)
+ assert.Equal(t, int64(42), first.AvgResponse)
+ assert.Equal(t, first, second)
+ assert.NotSame(t, first, second, "callers must not share mutable result pointers")
+ assert.Len(t, task.history.samples, 1)
+ // A later explicit request must still perform a fresh probe.
+ _, err := pm.UpsertMonitor(cfg, true)
+ require.NoError(t, err)
+ assert.Equal(t, 2, int(calls.Load()))
+ assert.Len(t, task.history.samples, 2)
+ })
+}
+
+func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ cfg := monitor.Config{ID: "test", Interval: 10}
+ pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
+ if config.Interval == 10 {
+ <-ctx.Done()
+ return 0, ctx.Err()
+ }
+ return 30, nil
+ })
+ defer pm.Stop()
+ task := newMonitorTask(cfg)
+ task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
+ pm.monitors[cfg.ID] = task
+ results := make(chan *monitor.Result, 2)
+ for range 2 {
+ go func() {
+ result, _ := pm.UpsertMonitor(cfg, true)
+ results <- result
+ }()
+ }
+ synctest.Wait()
+ updated := cfg
+ updated.Interval = 20
+ result, err := pm.UpsertMonitor(updated, true)
+ require.NoError(t, err)
+ require.NotNil(t, result)
+ assert.Equal(t, int64(20), result.AvgResponse)
+ assert.Zero(t, result.PacketLoss)
+ synctest.Wait()
+ assert.Nil(t, <-results)
+ assert.Nil(t, <-results)
+ assert.Len(t, task.history.samples, 1)
+ assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
+ })
+}
+
+func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ <-ctx.Done()
+ return 0, ctx.Err()
+ })
+ defer pm.Stop()
+ start := time.Now()
+ result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
+ require.NoError(t, err)
+ require.NotNil(t, result)
+ assert.Equal(t, 3*time.Second, time.Since(start))
+ assert.Equal(t, 100.0, result.PacketLoss)
+ assert.NoError(t, pm.monitors["test"].ctx.Err())
+ })
+}
diff --git a/agent/network_monitor_task.go b/agent/network_monitor_task.go
new file mode 100644
index 000000000..2c3469ab2
--- /dev/null
+++ b/agent/network_monitor_task.go
@@ -0,0 +1,116 @@
+package agent
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+const monitorFailureLogInterval = 5 * time.Minute
+
+// monitorTask coordinates a probe and its history for one immutable configuration.
+type monitorTask struct {
+ config monitor.Config
+ ctx context.Context
+ cancel context.CancelFunc
+ history *monitorHistory
+ resumeGuard *monitorResumeGuard
+ runMu sync.Mutex
+ inflight *monitorRun
+ lastFailureLog int64 // Unix nanoseconds
+}
+
+type monitorRun struct {
+ done chan struct{}
+ result *monitor.Result // published by closing done; never mutated afterwards
+}
+
+func newMonitorTask(config monitor.Config) *monitorTask {
+ ctx, cancel := context.WithCancel(context.Background())
+ task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
+ // Serialize cancellation with publication, so canceled probes cannot enter
+ // history copied into a replacement task.
+ task.cancel = func() {
+ task.runMu.Lock()
+ cancel()
+ task.runMu.Unlock()
+ }
+ return task
+}
+
+func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
+ task := newMonitorTask(config)
+ if existing != nil {
+ task.history = existing.history.clone()
+ }
+ return task
+}
+
+// runProbe shares an in-flight check between scheduled and immediate requests.
+// Every completed check contributes exactly one sample, regardless of how many
+// callers were waiting for it. No task or history lock is held during network I/O.
+func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
+ task.runMu.Lock()
+ if task.ctx.Err() != nil {
+ task.runMu.Unlock()
+ return nil
+ }
+ if run := task.inflight; run != nil {
+ task.runMu.Unlock()
+ select {
+ case <-task.ctx.Done():
+ return nil
+ case <-run.done:
+ if task.ctx.Err() != nil {
+ return nil
+ }
+ return copyMonitorResult(run.result)
+ }
+ }
+ run := &monitorRun{done: make(chan struct{})}
+ task.inflight = run
+ task.runMu.Unlock()
+
+ generation, _ := task.resumeGuard.snapshot()
+ responseUs, err := probe(task.ctx, task.config)
+ var logFailure bool
+ task.runMu.Lock()
+ currentGeneration, _ := task.resumeGuard.snapshot()
+ if task.ctx.Err() == nil && generation == currentGeneration {
+ now := time.Now()
+ if err != nil {
+ responseUs = -1
+ logAt := now.UnixNano()
+ if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
+ logFailure = true
+ task.lastFailureLog = logAt
+ }
+ } else {
+ task.lastFailureLog = 0
+ }
+ result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
+ run.result = &result
+ }
+
+ task.inflight = nil
+ close(run.done)
+ task.runMu.Unlock()
+ if logFailure {
+ slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
+ }
+ if task.ctx.Err() != nil {
+ return nil
+ }
+ return copyMonitorResult(run.result)
+}
+
+func copyMonitorResult(result *monitor.Result) *monitor.Result {
+ if result == nil {
+ return nil
+ }
+ copy := *result
+ return &copy
+}
diff --git a/agent/network_monitor_task_test.go b/agent/network_monitor_task_test.go
new file mode 100644
index 000000000..427f71d5a
--- /dev/null
+++ b/agent/network_monitor_task_test.go
@@ -0,0 +1,79 @@
+//go:build testing
+
+package agent
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "log/slog"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestMonitorFailureLogCooldown(t *testing.T) {
+ var logs bytes.Buffer
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ synctest.Test(t, func(t *testing.T) {
+ task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
+ defer task.cancel()
+ failure := errors.New("connection refused")
+ probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
+ var samples int64
+ check := func(wantLog bool) {
+ t.Helper()
+ logs.Reset()
+ result := task.runProbe(probe)
+ require.NotNil(t, result)
+ samples++
+ assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
+ if !wantLog {
+ assert.Empty(t, logs.String())
+ } else {
+ assert.Contains(t, logs.String(), `msg="monitor failed"`)
+ assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
+ }
+ }
+
+ check(true)
+ check(false)
+ time.Sleep(5*time.Minute - time.Nanosecond)
+ check(false)
+ time.Sleep(time.Nanosecond)
+ check(true)
+ check(false)
+ time.Sleep(5 * time.Minute)
+ check(true)
+ check(false)
+
+ // Recovery clears the cooldown.
+ failure = nil
+ check(false)
+ failure = errors.New("connection refused again")
+ check(true)
+
+ // Another monitor has its own cooldown.
+ other := newMonitorTask(task.config)
+ defer other.cancel()
+ logs.Reset()
+ require.NotNil(t, other.runProbe(probe))
+ assert.Contains(t, logs.String(), `msg="monitor failed"`)
+
+ // A canceled probe must not publish a failure or emit a warning.
+ logs.Reset()
+ result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
+ other.cancel()
+ return -1, context.Canceled
+ })
+ assert.Nil(t, result)
+ assert.Empty(t, logs.String())
+ })
+}
diff --git a/agent/network_monitor_test.go b/agent/network_monitor_test.go
new file mode 100644
index 000000000..b24ab043e
--- /dev/null
+++ b/agent/network_monitor_test.go
@@ -0,0 +1,524 @@
+package agent
+
+import (
+ "context"
+ "encoding/binary"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "golang.org/x/net/dns/dnsmessage"
+)
+
+func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
+ now := time.Now().UTC()
+ task := newMonitorTask(monitor.Config{ID: "monitor-1"})
+ task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
+ task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
+ task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
+ task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
+ task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
+
+ pm := newMonitorManager()
+ pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
+
+ results := pm.GetResults(uint16(time.Minute / time.Millisecond))
+ result, ok := results["monitor-1"]
+ require.True(t, ok)
+ assert.Equal(t, int64(30), result.AvgResponse)
+ assert.Equal(t, int64(25), result.AvgResponse1h)
+ assert.Equal(t, int64(30), result.MinResponse)
+ assert.Equal(t, int64(10), result.MinResponse1h)
+ assert.Equal(t, int64(30), result.MaxResponse)
+ assert.Equal(t, int64(40), result.MaxResponse1h)
+ assert.Equal(t, 50.0, result.PacketLoss)
+ assert.Equal(t, 20.0, result.PacketLoss1h)
+}
+
+func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
+ now := time.Now().UTC()
+ task := newMonitorTask(monitor.Config{ID: "monitor-1"})
+ task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
+ task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
+
+ pm := newMonitorManager()
+ pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
+
+ results := pm.GetResults(uint16(time.Minute / time.Millisecond))
+ result, ok := results["monitor-1"]
+ require.True(t, ok)
+ assert.Equal(t, int64(0), result.AvgResponse)
+ assert.Equal(t, int64(0), result.AvgResponse1h)
+ assert.Equal(t, int64(0), result.MinResponse)
+ assert.Equal(t, int64(0), result.MinResponse1h)
+ assert.Equal(t, int64(0), result.MaxResponse)
+ assert.Equal(t, int64(0), result.MaxResponse1h)
+ assert.Equal(t, 100.0, result.PacketLoss)
+ assert.Equal(t, 100.0, result.PacketLoss1h)
+}
+
+func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
+ cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
+ assert.Equal(t, "monitor-1", cfg.ID)
+}
+
+func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
+ validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
+ invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
+
+ pm := newMonitorManager()
+ pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
+ defer pm.Stop()
+
+ _, validExists := pm.monitors[validCfg.ID]
+ _, invalidExists := pm.monitors[invalidCfg.ID]
+ assert.True(t, validExists)
+ assert.False(t, invalidExists)
+}
+
+func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
+ keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
+ removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
+
+ keptTask := newMonitorTask(keepCfg)
+ removedTask := newMonitorTask(removeCfg)
+ pm := newMonitorManager()
+ pm.monitors = map[string]*monitorTask{
+ keepCfg.ID: keptTask,
+ removeCfg.ID: removedTask,
+ }
+
+ pm.SyncMonitors([]monitor.Config{keepCfg})
+
+ assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
+ _, exists := pm.monitors[removeCfg.ID]
+ assert.False(t, exists)
+
+ select {
+ case <-removedTask.ctx.Done():
+ default:
+ t.Fatal("expected removed monitor task to be cancelled")
+ }
+
+ select {
+ case <-keptTask.ctx.Done():
+ t.Fatal("expected existing monitor task to remain active")
+ default:
+ }
+}
+
+func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
+ originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
+ updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
+ originalTask := newMonitorTask(originalCfg)
+ pm := newMonitorManager()
+ pm.monitors = map[string]*monitorTask{
+ originalCfg.ID: originalTask,
+ }
+
+ pm.SyncMonitors([]monitor.Config{updatedCfg})
+ defer pm.Stop()
+
+ restartedTask := pm.monitors[updatedCfg.ID]
+ assert.NotSame(t, originalTask, restartedTask)
+ assert.Equal(t, updatedCfg, restartedTask.config)
+
+ select {
+ case <-originalTask.ctx.Done():
+ default:
+ t.Fatal("expected changed monitor task to be cancelled")
+ }
+}
+
+func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ pm := &MonitorManager{
+ monitors: make(map[string]*monitorTask),
+ probe: networkMonitorProbe(server.Client()),
+ }
+
+ resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
+ Action: monitor.SyncActionUpsert,
+ Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
+ RunNow: true,
+ })
+ defer pm.Stop()
+
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
+ assert.Equal(t, 0.0, resp.Result.PacketLoss)
+ assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
+
+ task := pm.monitors["monitor-1"]
+ require.NotNil(t, task)
+ task.history.mu.Lock()
+ defer task.history.mu.Unlock()
+ require.Len(t, task.history.samples, 1)
+}
+
+func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
+ originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
+ updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
+ now := time.Now().UTC()
+
+ existingTask := newMonitorTask(originalCfg)
+ existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
+ existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
+
+ pm := newMonitorManager()
+ pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
+
+ result, err := pm.UpsertMonitor(updatedCfg, false)
+ defer pm.Stop()
+
+ require.NoError(t, err)
+ assert.Nil(t, result)
+
+ updatedTask := pm.monitors[updatedCfg.ID]
+ require.NotNil(t, updatedTask)
+ assert.NotSame(t, existingTask, updatedTask)
+ assert.Equal(t, updatedCfg, updatedTask.config)
+
+ updatedTask.history.mu.Lock()
+ defer updatedTask.history.mu.Unlock()
+ require.Len(t, updatedTask.history.samples, 1)
+ assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
+
+ agg := updatedTask.history.aggregateLocked(time.Hour, now)
+ require.True(t, agg.hasData())
+ assert.Equal(t, int64(2), agg.totalCount)
+ assert.Equal(t, int64(2), agg.successCount)
+ assert.Equal(t, int64(18), agg.avgResponse())
+
+ select {
+ case <-existingTask.ctx.Done():
+ default:
+ t.Fatal("expected original monitor task to be cancelled")
+ }
+}
+
+func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
+ config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
+ task := newMonitorTask(config)
+ pm := newMonitorManager()
+ pm.monitors = map[string]*monitorTask{config.ID: task}
+
+ _, err := pm.HandleSyncRequest(monitor.SyncRequest{
+ Action: monitor.SyncActionDelete,
+ Config: monitor.Config{ID: config.ID},
+ })
+
+ require.NoError(t, err)
+ _, exists := pm.monitors[config.ID]
+ assert.False(t, exists)
+
+ select {
+ case <-task.ctx.Done():
+ default:
+ t.Fatal("expected deleted monitor task to be cancelled")
+ }
+}
+
+func TestMonitorManagerGetRandomDelay(t *testing.T) {
+ for i := 1000; i < 360_000; i += 1000 {
+ delay := getStagger(int64(i))
+ assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
+ assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
+ }
+}
+
+func TestMonitorHTTP(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, responseUs, int64(0))
+ })
+
+ t.Run("server error", func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "boom", http.StatusInternalServerError)
+ }))
+ defer server.Close()
+
+ responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
+ assert.Equal(t, int64(-1), responseUs)
+ require.Error(t, err)
+ })
+}
+
+func TestMonitorTCP(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+ defer listener.Close()
+
+ accepted := make(chan struct{})
+ go func() {
+ defer close(accepted)
+ conn, err := listener.Accept()
+ if err == nil {
+ _ = conn.Close()
+ }
+ }()
+
+ port := uint16(listener.Addr().(*net.TCPAddr).Port)
+ responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, responseUs, int64(0))
+ <-accepted
+ })
+
+ t.Run("connection failure", func(t *testing.T) {
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
+ require.NoError(t, err)
+
+ port := uint16(listener.Addr().(*net.TCPAddr).Port)
+ require.NoError(t, listener.Close())
+
+ responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
+ assert.Equal(t, int64(-1), responseUs)
+ require.Error(t, err)
+ })
+}
+
+func TestMonitorTCPAddressFallback(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ ips []string
+ loss bool
+ }{
+ {"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
+ {"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
+ {"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ listener, err := net.Listen("tcp4", "127.0.0.1:0")
+ require.NoError(t, err)
+ defer listener.Close()
+ original := net.DefaultResolver
+ net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
+ defer func() { net.DefaultResolver = original }()
+
+ // Verify the resolver preserves the intended order, so success cannot
+ // accidentally bypass the failed first address in the regression case.
+ ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
+ require.NoError(t, err)
+ require.Equal(t, tc.ips, ips)
+ responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
+ if tc.loss {
+ require.Error(t, err)
+ assert.Equal(t, int64(-1), responseUs)
+ } else {
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, responseUs, int64(0))
+ }
+ })
+ }
+}
+
+// tcpMonitorTestResolver supplies multiple A records without external DNS.
+func tcpMonitorTestResolver(ips []string) *net.Resolver {
+ return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+ client, server := net.Pipe()
+ go func() {
+ defer server.Close()
+ // net.Resolver uses TCP framing when its connection is not a PacketConn.
+ var size uint16
+ if err := binary.Read(server, binary.BigEndian, &size); err != nil {
+ return
+ }
+ packet := make([]byte, size)
+ if _, err := io.ReadFull(server, packet); err != nil {
+ return
+ }
+ var msg dnsmessage.Message
+ if err := msg.Unpack(packet); err != nil {
+ return
+ }
+ msg.Header.Response = true
+ msg.Header.RecursionAvailable = true
+ for _, question := range msg.Questions {
+ if question.Type != dnsmessage.TypeA {
+ continue
+ }
+ for _, ip := range ips {
+ msg.Answers = append(msg.Answers, dnsmessage.Resource{
+ Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
+ Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
+ })
+ }
+ }
+ packet, err := msg.Pack()
+ if err != nil {
+ return
+ }
+ response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
+ _, _ = server.Write(append(response, packet...))
+ }()
+ return client, nil
+ }}
+}
+
+func TestMonitorDNS(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ responseUs, err := monitorDNS(context.Background(), "localhost")
+ require.NoError(t, err)
+ assert.GreaterOrEqual(t, responseUs, int64(0))
+ })
+
+ t.Run("lookup failure", func(t *testing.T) {
+ responseUs, err := monitorDNS(context.Background(), "")
+ assert.Equal(t, int64(-1), responseUs)
+ require.Error(t, err)
+ })
+}
+
+func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
+ for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
+ t.Run(action, func(t *testing.T) {
+ started := make(chan struct{})
+ canceled := make(chan struct{})
+ release := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ close(started)
+ select {
+ case <-r.Context().Done():
+ close(canceled)
+ case <-release:
+ }
+ }))
+ defer server.Close()
+ defer close(release)
+ pm := newMonitorManager()
+ defer pm.Stop()
+ cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
+ task := newMonitorTask(cfg)
+ // Seed history to ensure a canceled RunNow does not return an old result.
+ task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
+ pm.monitors[cfg.ID] = task
+ done := make(chan *monitor.Result, 1)
+ go func() {
+ result, _ := pm.UpsertMonitor(cfg, true)
+ done <- result
+ }()
+ select {
+ case <-started:
+ case <-time.After(time.Second):
+ t.Fatal("probe did not start")
+ }
+ updated := cfg
+ updated.Interval--
+ switch action {
+ case "stop":
+ pm.Stop()
+ case "delete":
+ pm.DeleteMonitor(cfg.ID)
+ case "upsert":
+ _, err := pm.UpsertMonitor(updated, false)
+ require.NoError(t, err)
+ case "sync replace":
+ pm.SyncMonitors([]monitor.Config{updated})
+ case "sync remove":
+ pm.SyncMonitors(nil)
+ }
+ select {
+ case <-canceled:
+ case <-time.After(time.Second):
+ t.Fatal("active HTTP request was not canceled")
+ }
+ select {
+ case result := <-done:
+ assert.Nil(t, result)
+ case <-time.After(time.Second):
+ t.Fatal("RunNow did not return after cancellation")
+ }
+ task.history.mu.Lock()
+ assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
+ task.history.mu.Unlock()
+ })
+ }
+}
+
+func TestMonitorResolutionCancellation(t *testing.T) {
+ for _, protocol := range []string{"tcp", "dns", "icmp"} {
+ t.Run(protocol, func(t *testing.T) {
+ started := make(chan struct{}, 1)
+ original := net.DefaultResolver
+ net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+ select {
+ case started <- struct{}{}:
+ default:
+ }
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }}
+ defer func() { net.DefaultResolver = original }()
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ var err error
+ switch protocol {
+ case "tcp":
+ _, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
+ case "dns":
+ _, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
+ case "icmp":
+ _, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
+ }
+ done <- err
+ }()
+ select {
+ case <-started:
+ case <-time.After(time.Second):
+ t.Fatal("lookup did not start")
+ }
+ cancel()
+ select {
+ case err := <-done:
+ require.Error(t, err)
+ case <-time.After(time.Second):
+ t.Fatal("lookup did not cancel")
+ }
+ })
+ }
+}
+
+func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
+ release := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ select {
+ case <-r.Context().Done():
+ case <-release:
+ }
+ }))
+ defer server.Close()
+ defer close(release)
+ pm := newMonitorManager()
+ pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
+ task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
+ defer task.cancel()
+
+ result := task.runProbe(pm.probe)
+ require.NotNil(t, result)
+ assert.Equal(t, 100.0, result.PacketLoss)
+ assert.Equal(t, 100.0, result.PacketLoss1h)
+ require.Len(t, task.history.samples, 1)
+ assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
+ assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
+}
diff --git a/beszel.go b/beszel.go
index 57d8f4e1d..93f19b471 100644
--- a/beszel.go
+++ b/beszel.go
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
const (
// Version is the current version of the application.
- Version = "0.19.0"
+ Version = "0.20.0"
// AppName is the name of the application.
AppName = "beszel"
)
@@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
var MinVersionZfsData = semver.MustParse("0.18.9")
+
+// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
+var MinVersionNetworkMonitors = semver.MustParse("0.20.0")
diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go
index 8857addfc..589ec8363 100644
--- a/internal/alerts/alerts.go
+++ b/internal/alerts/alerts.go
@@ -20,10 +20,11 @@ type hubLike interface {
}
type AlertManager struct {
- hub hubLike
- stopOnce sync.Once
- pendingAlerts sync.Map
- alertsCache *AlertsCache
+ hub hubLike
+ stopOnce sync.Once
+ pendingAlerts sync.Map
+ alertsCache *AlertsCache
+ networkMonitors *networkMonitorCache
}
type AlertMessageData struct {
@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{
// NewAlertManager creates a new AlertManager instance.
func NewAlertManager(app hubLike) *AlertManager {
am := &AlertManager{
- hub: app,
- alertsCache: NewAlertsCache(app),
+ hub: app,
+ alertsCache: NewAlertsCache(app),
+ networkMonitors: newNetworkMonitorCache(app),
}
am.bindEvents()
return am
@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {
// Bind events to the alerts collection lifecycle
func (am *AlertManager) bindEvents() {
+ am.bindNetworkMonitorAlertEvents()
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.go
index 6e98c3bcb..7a44a22f5 100644
--- a/internal/alerts/alerts_api.go
+++ b/internal/alerts/alerts_api.go
@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
return e.BadRequestError("Bad data", err)
}
+ if reqData.Name == alertNameNetworkMonitorLoss {
+ if reqData.Value < 0 || reqData.Value >= 100 {
+ return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
+ }
+ reqData.Min = 0
+ }
+
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
if err != nil {
return err
diff --git a/internal/alerts/alerts_cache.go b/internal/alerts/alerts_cache.go
index 0fce5c641..58550ff02 100644
--- a/internal/alerts/alerts_cache.go
+++ b/internal/alerts/alerts_cache.go
@@ -1,6 +1,7 @@
package alerts
import (
+ "sync"
"time"
"github.com/pocketbase/dbx"
@@ -18,6 +19,9 @@ type CachedAlertData struct {
Triggered bool
Min uint8
PendingSince time.Time
+ // Immutable after publication; decoded only when the alert record changes.
+ MonitorStates map[string]string
+ MonitorStatesValid bool
// Created types.DateTime
}
@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
a.Triggered = record.GetBool("triggered")
a.Min = uint8(record.GetInt("min"))
a.PendingSince = record.GetDateTime("pending_since").Time()
+ if a.Name == alertNameNetworkMonitorLoss {
+ var state networkMonitorAlertState
+ a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
+ a.MonitorStates = state.Monitors
+ }
// a.Created = record.GetDateTime("created")
}
// AlertsCache provides an in-memory cache for system alerts.
type AlertsCache struct {
+ // Serialize lazy loads with updates so a late load cannot replace newer state.
+ loadMu sync.Mutex
app core.App
store *store.Store[string, *store.Store[string, CachedAlertData]]
populated bool
@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
func (c *AlertsCache) PopulateFromDB(force bool) error {
+ c.loadMu.Lock()
+ defer c.loadMu.Unlock()
if !force && c.populated {
return nil
}
@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
}
c.store.RemoveAll()
for _, record := range records {
- c.Update(record)
+ c.update(record)
}
c.populated = true
return nil
@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
// Update adds or updates an alert record in the cache.
func (c *AlertsCache) Update(record *core.Record) {
+ c.loadMu.Lock()
+ defer c.loadMu.Unlock()
+ c.update(record)
+}
+
+func (c *AlertsCache) update(record *core.Record) {
systemID := record.GetString("system")
if systemID == "" {
return
@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {
// Delete removes an alert record from the cache.
func (c *AlertsCache) Delete(record *core.Record) {
+ c.loadMu.Lock()
+ defer c.loadMu.Unlock()
systemID := record.GetString("system")
if systemID == "" {
return
@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
systemStore, ok := c.store.GetOk(systemID)
if !ok {
- // Populate cache for this system
- records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
- if err != nil {
- return nil
+ c.loadMu.Lock()
+ defer c.loadMu.Unlock()
+ systemStore, ok = c.store.GetOk(systemID)
+ if !ok {
+ // Populate cache for this system
+ records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
+ if err != nil {
+ return nil
+ }
+ systemStore = store.New(map[string]CachedAlertData{})
+ for _, record := range records {
+ var ca CachedAlertData
+ ca.PopulateFromRecord(record)
+ systemStore.Set(record.Id, ca)
+ }
+ c.store.Set(systemID, systemStore)
}
- systemStore = store.New(map[string]CachedAlertData{})
- for _, record := range records {
- var ca CachedAlertData
- ca.PopulateFromRecord(record)
- systemStore.Set(record.Id, ca)
- }
- c.store.Set(systemID, systemStore)
}
all := systemStore.GetAll()
alerts := make([]CachedAlertData, 0, len(all))
diff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.go
index 9654456fd..c29475f6f 100644
--- a/internal/alerts/alerts_history.go
+++ b/internal/alerts/alerts_history.go
@@ -9,6 +9,12 @@ import (
// On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
+ if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+ if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
+ return err
+ }
+ return e.Next()
+ }
if !e.Record.GetBool("triggered") {
return e.Next()
}
@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
// On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
+ // Network monitor incidents have separate history entries per monitor.
+ if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+ return e.Next()
+ }
original := e.Record.Original()
new := e.Record
diff --git a/internal/alerts/alerts_network_monitors.go b/internal/alerts/alerts_network_monitors.go
new file mode 100644
index 000000000..6b69c3ccf
--- /dev/null
+++ b/internal/alerts/alerts_network_monitors.go
@@ -0,0 +1,269 @@
+package alerts
+
+import (
+ "database/sql"
+ "errors"
+ "fmt"
+ "math"
+ "net"
+ "strconv"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/pocketbase/dbx"
+ "github.com/pocketbase/pocketbase/core"
+)
+
+const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
+
+// networkMonitorAlertState is this alert type's persisted runtime state.
+// Monitor IDs map to their open history entries independently of history retention.
+type networkMonitorAlertState struct {
+ Monitors map[string]string `json:"monitors"`
+}
+
+func (am *AlertManager) bindNetworkMonitorAlertEvents() {
+ // Hidden fields are still writable through the record API unless protected.
+ protectState := func(e *core.RecordRequestEvent) error {
+ e.Record.Set("state", e.Record.Original().Get("state"))
+ oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
+ if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
+ (oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
+ return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
+ }
+ if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
+ if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
+ return e.ForbiddenError("You do not have access to this system", nil)
+ }
+ e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
+ value := e.Record.GetFloat("value")
+ if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
+ return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
+ }
+ e.Record.Set("min", 0)
+ }
+ return e.Next()
+ }
+ am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
+ am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
+ cleanup := func(e *core.RecordEvent) error {
+ if err := e.Next(); err != nil {
+ return err
+ }
+ return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
+ }
+ am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
+ am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+ if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
+ return e.Next()
+ }
+ return cleanup(e)
+ })
+}
+
+// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
+// using its exact payload (dashboard requests can replace the cached payload).
+// Omitted results and disconnected systems never imply recovery.
+func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
+ if systemRecord.GetString("status") != "up" {
+ return nil
+ }
+ alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
+ if len(alerts) == 0 {
+ return nil
+ }
+ monitors, err := am.networkMonitors.get(systemRecord.Id)
+ if err != nil {
+ return err
+ }
+ if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
+ return nil
+ }
+ // The cache only predicts a transition. Reload and recheck under the DB
+ // transaction before persisting, including current system/monitor status.
+ return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
+}
+
+// networkMonitorTransitionPending does no IO and never mutates cached maps.
+func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
+ for _, alert := range alerts {
+ if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
+ return true
+ }
+ for id := range alert.MonitorStates {
+ if _, enabled := monitors[id]; !enabled {
+ return true
+ }
+ }
+ for id, result := range results {
+ interval, enabled := monitors[id]
+ if !enabled || !monitorResultReady(result, interval, now) {
+ continue
+ }
+ _, active := alert.MonitorStates[id]
+ if (result.PacketLoss1h > alert.Value) != active {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
+ var messages []AlertMessageData
+ err := app.RunInTransaction(func(tx core.App) error {
+ // Read configuration inside the transaction so concurrent threshold changes,
+ // disabling, and evaluations cannot overwrite each other's incident state.
+ alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
+ if err != nil || len(alerts) == 0 {
+ return err
+ }
+ system, err := tx.FindRecordById("systems", systemID)
+ if errors.Is(err, sql.ErrNoRows) {
+ // System deletion cascades to its alerts.
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
+ if err != nil {
+ return err
+ }
+ enabled := make(map[string]*core.Record, len(monitors))
+ for _, m := range monitors {
+ enabled[m.Id] = m
+ }
+ now := time.Now()
+ for _, alert := range alerts {
+ var state networkMonitorAlertState
+ if err := alert.UnmarshalJSONField("state", &state); err != nil {
+ return err
+ }
+ states := state.Monitors
+ if states == nil {
+ states = map[string]string{}
+ }
+ changed := false
+ // Removing or disabling a monitor closes its incident silently.
+ for id, historyID := range states {
+ if _, ok := enabled[id]; !ok {
+ if err := resolveMonitorIncident(tx, historyID, now); err != nil {
+ return err
+ }
+ delete(states, id)
+ changed = true
+ }
+ }
+ if system.GetString("status") == "up" {
+ for _, m := range monitors {
+ result, ok := results[m.Id]
+ if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
+ continue
+ }
+ historyID, active := states[m.Id]
+ triggered := result.PacketLoss1h > alert.GetFloat("value")
+ if triggered == active {
+ continue
+ }
+ label := m.GetString("target")
+ if m.GetString("protocol") == "tcp" {
+ label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
+ }
+ if triggered {
+ collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
+ if err != nil {
+ return err
+ }
+ history := core.NewRecord(collection)
+ history.Load(map[string]any{
+ "alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
+ "name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
+ })
+ if err := tx.Save(history); err != nil {
+ return err
+ }
+ states[m.Id] = history.Id
+ } else {
+ if err := resolveMonitorIncident(tx, historyID, now); err != nil {
+ return err
+ }
+ delete(states, m.Id)
+ }
+ changed = true
+ state, comparison := "loss", "exceeds"
+ if !triggered {
+ state, comparison = "recovered", "is at or below"
+ }
+ messages = append(messages, AlertMessageData{
+ UserID: alert.GetString("user"), SystemID: systemID,
+ Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
+ Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
+ Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
+ })
+ }
+ }
+ if changed || alert.GetBool("triggered") != (len(states) > 0) {
+ alert.Set("state", networkMonitorAlertState{Monitors: states})
+ alert.Set("triggered", len(states) > 0)
+ if err := tx.Save(alert); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ // Match other alert types: persist transitions before delivery, and respect
+ // the user's existing notification destinations and quiet hours.
+ for _, message := range messages {
+ if err := am.SendAlert(message); err != nil {
+ app.Logger().Error("Failed to send network monitor alert", "err", err)
+ }
+ }
+ return nil
+}
+
+func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
+ // Three completed attempts provide a short warm-up, including after an agent
+ // restart.
+ if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
+ return false
+ }
+ // Never interpret an empty one-hour window as zero loss.
+ maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
+ age := now.Sub(time.UnixMilli(result.LastProbeAt))
+ return age >= -time.Minute && age <= maxAge
+}
+
+func resolveMonitorIncident(app core.App, id string, now time.Time) error {
+ record, err := app.FindRecordById("alerts_history", id)
+ if errors.Is(err, sql.ErrNoRows) {
+ // History can be purged independently.
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if !record.GetDateTime("resolved").IsZero() {
+ return nil
+ }
+ record.Set("resolved", now.UTC())
+ return app.Save(record)
+}
+
+func resolveNetworkMonitorHistory(app core.App, alertID string) error {
+ records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
+ if err != nil {
+ return err
+ }
+ for _, record := range records {
+ record.Set("resolved", time.Now().UTC())
+ if err := app.Save(record); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/internal/alerts/alerts_network_monitors_test.go b/internal/alerts/alerts_network_monitors_test.go
new file mode 100644
index 000000000..aa02cf430
--- /dev/null
+++ b/internal/alerts/alerts_network_monitors_test.go
@@ -0,0 +1,513 @@
+//go:build testing
+
+package alerts_test
+
+import (
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/henrygd/beszel/internal/alerts"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ beszelTests "github.com/henrygd/beszel/internal/tests"
+ "github.com/pocketbase/dbx"
+ "github.com/pocketbase/pocketbase/core"
+ pbTests "github.com/pocketbase/pocketbase/tests"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
+ t.Helper()
+ hub, system, alert := systemdTestSetup(t, false)
+ t.Cleanup(hub.Cleanup)
+ alert.Set("name", "NetworkMonitorLoss")
+ alert.Set("value", 5)
+ require.NoError(t, hub.Save(alert))
+ var monitors []*core.Record
+ for _, name := range []string{"gateway", "website"} {
+ record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
+ })
+ require.NoError(t, err)
+ monitors = append(monitors, record)
+ }
+ // Avoid starting a system update worker in tests.
+ _, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
+ require.NoError(t, err)
+ system.Set("status", "up")
+ return hub, system, alert, monitors
+}
+
+func monitorResult(loss float64) monitor.Result {
+ return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
+}
+
+func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ count := hub.TestMailer.TotalSend()
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
+ check := func(active bool, open, sent int) {
+ t.Helper()
+ record, err := hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.Equal(t, active, record.GetBool("triggered"))
+ total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
+ require.NoError(t, err)
+ assert.EqualValues(t, open, total)
+ assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
+ }
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ check(true, 1, 1)
+ message := hub.TestMailer.Messages()[count]
+ assert.Contains(t, message.Text, "gateway.example.com")
+ assert.Contains(t, message.Text, "10.00%")
+ assert.Contains(t, message.Text, "5.00%")
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ check(true, 1, 1)
+ // Persisted monitor state prevents duplicate notifications after a hub restart.
+ am = alerts.NewTestAlertManagerWithoutWorker(hub)
+ results[monitors[1].Id] = monitorResult(20)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ check(true, 2, 2)
+ results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ check(true, 1, 3)
+ results[monitors[1].Id] = monitorResult(0)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ check(false, 0, 4)
+ histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
+ require.NoError(t, err)
+ require.Len(t, histories, 2)
+ for _, history := range histories {
+ assert.NotEmpty(t, history.GetString("monitor_name"))
+ }
+}
+
+func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
+ for _, tc := range []struct {
+ protocol, target, label string
+ port int
+ }{
+ {"icmp", "gateway.example.com", "gateway.example.com", 0},
+ {"http", "https://example.com/health", "https://example.com/health", 0},
+ {"tcp", "example.com", "example.com:8443", 8443},
+ {"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
+ } {
+ t.Run(tc.label, func(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ m := monitors[0]
+ m.Set("protocol", tc.protocol)
+ m.Set("target", tc.target)
+ m.Set("port", tc.port)
+ require.NoError(t, hub.Save(m))
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
+ histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
+ require.NoError(t, err)
+ require.Len(t, histories, 1)
+ assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
+ assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
+ })
+ }
+}
+
+func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
+ for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
+ t.Run(scenario, func(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ apply := func(loss float64) {
+ result := monitorResult(loss)
+ results := map[string]monitor.Result{monitors[0].Id: result}
+ switch scenario {
+ case "missing":
+ results = nil
+ case "stale":
+ result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
+ results[monitors[0].Id] = result
+ case "expired hourly window":
+ monitors[0].Set("interval", 3600)
+ require.NoError(t, hub.Save(monitors[0]))
+ result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
+ results[monitors[0].Id] = result
+ case "future":
+ result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
+ results[monitors[0].Id] = result
+ case "warmup":
+ result.SampleCount = 2
+ results[monitors[0].Id] = result
+ case "no probes":
+ result.SampleCount = 0
+ results[monitors[0].Id] = result
+ case "down":
+ _, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
+ require.NoError(t, err)
+ case "paused":
+ record, err := hub.FindRecordById("systems", system.Id)
+ require.NoError(t, err)
+ record.Set("status", "paused")
+ require.NoError(t, hub.Save(record))
+ }
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ }
+ count := hub.TestMailer.TotalSend()
+ apply(100)
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ _, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
+ require.NoError(t, err)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
+ apply(0)
+ assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
+ record, err := hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.True(t, record.GetBool("triggered"))
+ })
+ }
+}
+
+func TestNetworkMonitorAlertCleanup(t *testing.T) {
+ for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
+ t.Run(scenario, func(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ count := hub.TestMailer.TotalSend()
+ switch scenario {
+ case "disable monitor":
+ monitors[0].Set("enabled", false)
+ require.NoError(t, hub.Save(monitors[0]))
+ case "delete monitor":
+ require.NoError(t, hub.Delete(monitors[0]))
+ case "disable alert":
+ require.NoError(t, hub.Delete(alert))
+ case "delete system":
+ require.NoError(t, hub.Delete(system))
+ case "purge history":
+ history, err := hub.FindAllRecords("alerts_history")
+ require.NoError(t, err)
+ for _, record := range history {
+ require.NoError(t, hub.Delete(record))
+ }
+ }
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
+ require.NoError(t, err)
+ if scenario == "disable monitor" || scenario == "delete monitor" {
+ assert.EqualValues(t, 1, open)
+ record, err := hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.True(t, record.GetBool("triggered"))
+ require.NoError(t, hub.Delete(monitors[1]))
+ record, err = hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.False(t, record.GetBool("triggered"))
+ } else {
+ assert.Zero(t, open)
+ }
+ })
+ }
+}
+
+func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ user, err := beszelTests.CreateUser(hub, "[email protected]", "password")
+ require.NoError(t, err)
+ other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
+ require.NoError(t, err)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ other, err = hub.FindRecordById("alerts", other.Id)
+ require.NoError(t, err)
+ assert.False(t, other.GetBool("triggered"))
+ // Editing the threshold re-evaluates on the next batch, without losing state.
+ alert, err = hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ alert.Set("value", 15)
+ require.NoError(t, hub.Save(alert))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ alert, err = hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ assert.False(t, alert.GetBool("triggered"))
+}
+
+func TestNetworkMonitorAlertAPI(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ value float64
+ direct, denied, patch bool
+ status int
+ }{
+ {name: "zero threshold", value: 0, status: 200},
+ {name: "fractional threshold", value: 5.5, status: 200},
+ {name: "negative threshold", value: -1, status: 400},
+ {name: "unreachable threshold", value: 100, status: 400},
+ {name: "bulk inaccessible system", value: 5, denied: true, status: 200},
+ {name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
+ {name: "direct invalid threshold", value: -1, direct: true, status: 400},
+ {name: "direct private state", value: 5, direct: true, status: 200},
+ {name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ hub, user := beszelTests.GetHubWithUser(t)
+ defer hub.Cleanup()
+ owner := user.Id
+ if tc.denied {
+ other, err := beszelTests.CreateUser(hub, "[email protected]", "password")
+ require.NoError(t, err)
+ owner = other.Id
+ }
+ systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
+ require.NoError(t, err)
+ token, err := user.NewAuthToken()
+ require.NoError(t, err)
+ body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
+ url, method := "/api/beszel/user-alerts", "POST"
+ if tc.direct {
+ url = "/api/collections/alerts/records"
+ body["system"], body["user"] = systems[0].Id, user.Id
+ body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
+ }
+ if tc.patch {
+ alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
+ require.NoError(t, err)
+ url += "/" + alert.Id
+ method = "PATCH"
+ body["triggered"] = false
+ }
+ content := `"success":true`
+ if tc.direct {
+ content = `"name":"NetworkMonitorLoss"`
+ }
+ if tc.status == 400 {
+ content = `"status":400`
+ }
+ if tc.status == 403 {
+ content = `"status":403`
+ }
+ scenario := beszelTests.ApiScenario{
+ Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
+ Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
+ TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
+ }
+ scenario.Test(t)
+ records, err := hub.FindAllRecords("alerts")
+ require.NoError(t, err)
+ if tc.status != 200 || tc.denied {
+ assert.Empty(t, records)
+ return
+ }
+ require.Len(t, records, 1)
+ assert.Equal(t, tc.value, records[0].GetFloat("value"))
+ assert.Zero(t, records[0].GetInt("min"))
+ state := struct {
+ Monitors map[string]string `json:"monitors"`
+ }{}
+ require.NoError(t, records[0].UnmarshalJSONField("state", &state))
+ states := state.Monitors
+ if tc.patch {
+ assert.Equal(t, map[string]string{"real": "history"}, states)
+ assert.True(t, records[0].GetBool("triggered"))
+ } else {
+ assert.Empty(t, states)
+ assert.False(t, records[0].GetBool("triggered"))
+ }
+ })
+ }
+}
+
+type monitorCountingHub struct {
+ *beszelTests.TestHub
+ transactions atomic.Int64
+ beforeTransaction func()
+}
+
+func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
+ h.transactions.Add(1)
+ if h.beforeTransaction != nil {
+ h.beforeTransaction()
+ }
+ return h.App.RunInTransaction(fn)
+}
+
+// Count actual SQL on both DB connections, including queries through record APIs.
+func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
+ t.Helper()
+ count := &atomic.Int64{}
+ for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
+ db := builder.(*dbx.DB)
+ old := db.LogFunc
+ db.LogFunc = func(string, ...any) { count.Add(1) }
+ t.Cleanup(func() { db.LogFunc = old })
+ }
+ return count
+}
+
+func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ counted := &monitorCountingHub{TestHub: hub}
+ am := alerts.NewTestAlertManagerWithoutWorker(counted)
+ sql := monitorSQLCounter(t, hub)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+ evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
+ noWork := func() {
+ t.Helper()
+ sql.Store(0)
+ counted.transactions.Store(0)
+ for range 100 {
+ evaluate()
+ }
+ assert.Zero(t, sql.Load(), "steady state must not issue SQL")
+ assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
+ }
+ // One-time lazy loads are permitted, including on hub restart.
+ evaluate()
+ assert.Positive(t, sql.Load())
+ noWork()
+ // Realtime metric saves invoke record hooks but must not invalidate config.
+ fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
+ require.NoError(t, err)
+ monitors[0] = fresh
+ monitors[0].Set("loss1h", 0)
+ monitors[0].Set("res", 100)
+ require.NoError(t, hub.Save(monitors[0]))
+ noWork()
+ results[monitors[0].Id] = monitorResult(10)
+ evaluate()
+ assert.Positive(t, sql.Load(), "transitions must still be persisted")
+ assert.EqualValues(t, 1, counted.transactions.Load())
+ noWork()
+ // Missing and stale observations must not enter the transaction either.
+ results = nil
+ noWork()
+ results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
+ noWork()
+ results[monitors[0].Id] = monitorResult(0)
+ evaluate()
+ noWork()
+ require.NoError(t, hub.Delete(alert))
+ noWork()
+}
+
+func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ count := hub.TestMailer.TotalSend()
+ // Widening the interval makes this observation fresh. A stale interval cache
+ // would miss the failure indefinitely, even though results keep arriving.
+ result := monitorResult(10)
+ result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
+ results[monitors[0].Id] = result
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ monitors[0].Set("interval", 120)
+ require.NoError(t, hub.Save(monitors[0]))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+ // Disable, then re-enable the same ID: its new failure must be detected.
+ monitors[0].Set("enabled", false)
+ require.NoError(t, hub.Save(monitors[0]))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ monitors[0].Set("enabled", true)
+ require.NoError(t, hub.Save(monitors[0]))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count+2, hub.TestMailer.TotalSend())
+ // A new monitor must also become eligible without restarting the hub.
+ created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
+ })
+ require.NoError(t, err)
+ results[created.Id] = monitorResult(10)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count+3, hub.TestMailer.TotalSend())
+ // Threshold changes refresh cached config and preserve the active incidents.
+ alert, err = hub.FindRecordById("alerts", alert.Id)
+ require.NoError(t, err)
+ alert.Set("value", 15)
+ require.NoError(t, hub.Save(alert))
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count+5, hub.TestMailer.TotalSend())
+}
+
+func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
+ for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
+ t.Run(change, func(t *testing.T) {
+ hub, system, alert, monitors := networkAlertSetup(t)
+ counted := &monitorCountingHub{TestHub: hub}
+ am := alerts.NewTestAlertManagerWithoutWorker(counted)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ count := hub.TestMailer.TotalSend()
+ // Change the DB after the cache predicts a transition, before its transaction.
+ counted.beforeTransaction = func() {
+ counted.beforeTransaction = nil
+ switch change {
+ case "threshold":
+ alert.Set("value", 20)
+ require.NoError(t, hub.Save(alert))
+ case "disable alert":
+ require.NoError(t, hub.Delete(alert))
+ case "disable monitor":
+ monitors[0].Set("enabled", false)
+ require.NoError(t, hub.Save(monitors[0]))
+ case "down":
+ _, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
+ require.NoError(t, err)
+ }
+ }
+ results[monitors[0].Id] = monitorResult(10)
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.EqualValues(t, 1, counted.transactions.Load())
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ histories, err := hub.CountRecords("alerts_history")
+ require.NoError(t, err)
+ assert.Zero(t, histories)
+ })
+ }
+}
+
+func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
+ hub, system, _, monitors := networkAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
+ count := hub.TestMailer.TotalSend()
+ var wg sync.WaitGroup
+ errs := make(chan error, 8)
+ for range 8 {
+ wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
+ }
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ require.NoError(t, err)
+ }
+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+}
+
+func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
+ hub, system, _, monitors := networkAlertSetup(t)
+ am := alerts.NewTestAlertManagerWithoutWorker(hub)
+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ _, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+ require.NoError(t, err)
+ count := hub.TestMailer.TotalSend()
+ results[monitors[0].Id] = monitorResult(10)
+ require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count, hub.TestMailer.TotalSend())
+ histories, err := hub.CountRecords("alerts_history")
+ require.NoError(t, err)
+ assert.Zero(t, histories)
+ _, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
+ require.NoError(t, err)
+ // A failed transition must not be published to the cache and mask the retry.
+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())
+}
diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.go
index ef0a61d9b..ad47d0bd5 100644
--- a/internal/alerts/alerts_system.go
+++ b/internal/alerts/alerts_system.go
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil
}
- alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)
+ alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
if len(alerts) == 0 {
return nil
}
diff --git a/internal/alerts/alerts_test_helpers.go b/internal/alerts/alerts_test_helpers.go
index df796b361..89f067c2a 100644
--- a/internal/alerts/alerts_test_helpers.go
+++ b/internal/alerts/alerts_test_helpers.go
@@ -11,8 +11,9 @@ import (
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
return &AlertManager{
- hub: app,
- alertsCache: NewAlertsCache(app),
+ hub: app,
+ alertsCache: NewAlertsCache(app),
+ networkMonitors: newNetworkMonitorCache(app),
}
}
diff --git a/internal/alerts/network_monitor_cache.go b/internal/alerts/network_monitor_cache.go
new file mode 100644
index 000000000..2ca3718e7
--- /dev/null
+++ b/internal/alerts/network_monitor_cache.go
@@ -0,0 +1,79 @@
+package alerts
+
+import (
+ "sync"
+
+ "github.com/pocketbase/dbx"
+ "github.com/pocketbase/pocketbase/core"
+)
+
+// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
+// needed for the alert fast path. Names and targets are read only on transitions.
+// Returned maps are immutable; configuration changes invalidate the whole entry.
+type networkMonitorCache struct {
+ app core.App
+ mu sync.RWMutex
+ systems map[string]map[string]int
+}
+
+func newNetworkMonitorCache(app core.App) *networkMonitorCache {
+ c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
+ invalidate := func(e *core.RecordEvent) error {
+ c.invalidate(e.Record.GetString("system"))
+ return e.Next()
+ }
+ app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
+ app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
+ app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+ old := e.Record.Original()
+ // Realtime metric saves also invoke this hook. They must not evict config.
+ if old.GetString("system") != e.Record.GetString("system") ||
+ old.GetBool("enabled") != e.Record.GetBool("enabled") ||
+ old.GetInt("interval") != e.Record.GetInt("interval") {
+ c.invalidate(old.GetString("system"))
+ c.invalidate(e.Record.GetString("system"))
+ }
+ return e.Next()
+ })
+ app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
+ c.invalidate(e.Record.Id)
+ return e.Next()
+ })
+ return c
+}
+
+func (c *networkMonitorCache) invalidate(systemID string) {
+ c.mu.Lock()
+ delete(c.systems, systemID)
+ c.mu.Unlock()
+}
+
+func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
+ c.mu.RLock()
+ monitors, ok := c.systems[systemID]
+ c.mu.RUnlock()
+ if ok {
+ return monitors, nil
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if monitors, ok := c.systems[systemID]; ok {
+ return monitors, nil
+ }
+ // Keep the lock through the load so a concurrent config change cannot be
+ // invalidated first and then overwritten by the older query result.
+ var rows []struct {
+ ID string `db:"id"`
+ Interval int `db:"interval"`
+ }
+ if err := c.app.DB().Select("id", "interval").From("network_monitors").
+ Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
+ return nil, err
+ }
+ monitors = make(map[string]int, len(rows))
+ for _, row := range rows {
+ monitors[row.ID] = row.Interval
+ }
+ c.systems[systemID] = monitors
+ return monitors, nil
+}
diff --git a/internal/common/common-ws.go b/internal/common/common-ws.go
index 3f69ba889..b9e8aea0c 100644
--- a/internal/common/common-ws.go
+++ b/internal/common/common-ws.go
@@ -24,6 +24,8 @@ const (
GetSystemdInfo
// Request ZFS detail data from agent
GetZfsData
+ // Sync network monitor configuration to agent
+ SyncNetworkMonitors
// Add new actions here...
)
diff --git a/internal/entities/monitor/monitor.go b/internal/entities/monitor/monitor.go
new file mode 100644
index 000000000..46995ce81
--- /dev/null
+++ b/internal/entities/monitor/monitor.go
@@ -0,0 +1,102 @@
+package monitor
+
+import "time"
+
+// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).
+// Hub requests that run a probe must allow this time in addition to transport overhead.
+const MaxProbeTimeout = 10 * time.Second
+
+type SyncAction uint8
+
+const (
+ // SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.
+ SyncActionReplace SyncAction = iota
+ // SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.
+ SyncActionUpsert
+ // SyncActionDelete indicates an incremental sync where the provided config should be removed.
+ SyncActionDelete
+)
+
+// Config defines a network monitor task sent from hub to agent.
+type Config struct {
+ // ID is the stable network_monitors record ID generated by the hub.
+ ID string `cbor:"0,keyasint"`
+ Target string `cbor:"1,keyasint"`
+ Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"
+ Port uint16 `cbor:"3,keyasint,omitempty"`
+ Interval uint16 `cbor:"4,keyasint"` // seconds
+}
+
+// SyncRequest defines an incremental or full monitor sync request sent to the agent.
+type SyncRequest struct {
+ Action SyncAction `cbor:"0,keyasint"`
+ Config Config `cbor:"1,keyasint,omitempty"`
+ Configs []Config `cbor:"2,keyasint,omitempty"`
+ RunNow bool `cbor:"3,keyasint,omitempty"`
+}
+
+// SyncResponse returns the immediate result for an upsert when requested.
+type SyncResponse struct {
+ Result Result `cbor:"0,keyasint,omitempty"`
+}
+
+// Result holds aggregated monitor results for a single target.
+//
+// 0: avg response in microseconds
+//
+// 1: 1h average response in microseconds
+//
+// 2: min response in microseconds
+//
+// 3: 1h min response in microseconds
+//
+// 4: max response in microseconds
+//
+// 5: 1h max response in microseconds
+//
+// 6: packet loss percentage (0-100)
+//
+// 7: 1h packet loss percentage (0-100)
+type Result struct {
+ AvgResponse int64 `cbor:"0,keyasint,omitempty"`
+ AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`
+ MinResponse int64 `cbor:"2,keyasint,omitempty"`
+ MinResponse1h int64 `cbor:"3,keyasint,omitempty"`
+ MaxResponse int64 `cbor:"4,keyasint,omitempty"`
+ MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`
+ PacketLoss float64 `cbor:"6,keyasint,omitempty"`
+ PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`
+ // LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.
+ LastProbeAt int64 `cbor:"8,keyasint"`
+ // SampleCount includes all completed probes since this monitor started.
+ // Used for alert warm-up even when the interval is longer than 20 minutes.
+ SampleCount int64 `cbor:"9,keyasint,omitempty"`
+ // Counts and sum cover the current response window (or latest-sample
+ // fallback), not the hourly window or lifetime SampleCount.
+ TotalCount int64 `cbor:"10,keyasint"`
+ SuccessCount int64 `cbor:"11,keyasint"`
+ ResponseSum int64 `cbor:"12,keyasint"`
+}
+
+// Stats holds response times in microseconds and packet loss percentage (0-100).
+type Stats struct {
+ ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.
+ ResMin float64 `json:"res_min" db:"res_min"`
+ ResMax float64 `json:"res_max" db:"res_max"`
+ Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.
+ TotalCount int64 `json:"-" db:"total_count"`
+ SuccessCount int64 `json:"-" db:"success_count"`
+ ResponseSum int64 `json:"-" db:"res_sum"`
+}
+
+func (s Stats) FromResult(result Result) Stats {
+ return Stats{
+ ResAvg: float64(result.AvgResponse),
+ ResMin: float64(result.MinResponse),
+ ResMax: float64(result.MaxResponse),
+ Loss: result.PacketLoss,
+ TotalCount: result.TotalCount,
+ SuccessCount: result.SuccessCount,
+ ResponseSum: result.ResponseSum,
+ }
+}
diff --git a/internal/entities/system/system.go b/internal/entities/system/system.go
index fd3c42e45..c6e8d0ea3 100644
--- a/internal/entities/system/system.go
+++ b/internal/entities/system/system.go
@@ -7,6 +7,7 @@ import (
"time"
"github.com/henrygd/beszel/internal/entities/container"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/systemd"
)
@@ -210,5 +211,6 @@ type CombinedData struct {
Details *Details `cbor:"4,keyasint,omitempty"`
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
// that omitted systemd data (for example, a short-cache dashboard request).
- SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
+ SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
+ Monitors map[string]monitor.Result `cbor:"6,keyasint"`
}
diff --git a/internal/hub/collections.go b/internal/hub/collections.go
index 667f4bf8e..3a0c3e85f 100644
--- a/internal/hub/collections.go
+++ b/internal/hub/collections.go
@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {
return err
}
- if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{
+ if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{
list: &systemScopedReadRule,
}); err != nil {
return err
@@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error {
return err
}
+ if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{
+ list: &systemScopedReadRule,
+ view: &systemScopedReadRule,
+ create: &systemScopedWriteRule,
+ update: &systemScopedWriteRule,
+ delete: &systemScopedWriteRule,
+ }); err != nil {
+ return err
+ }
+
if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{
list: &systemScopedReadRule,
view: &systemScopedReadRule,
diff --git a/internal/hub/hub.go b/internal/hub/hub.go
index 30548cf2d..3f5f241b7 100644
--- a/internal/hub/hub.go
+++ b/internal/hub/hub.go
@@ -109,6 +109,8 @@ func (h *Hub) StartHub() error {
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
+ bindNetworkMonitorsEvents(h)
+
pb, ok := h.App.(*pocketbase.PocketBase)
if !ok {
return errors.New("not a pocketbase app")
diff --git a/internal/hub/network_monitors.go b/internal/hub/network_monitors.go
new file mode 100644
index 000000000..25ad1b652
--- /dev/null
+++ b/internal/hub/network_monitors.go
@@ -0,0 +1,158 @@
+package hub
+
+import (
+ "strconv"
+ "time"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/henrygd/beszel/internal/hub/systems"
+ "github.com/pocketbase/pocketbase/core"
+ "github.com/pocketbase/pocketbase/tools/types"
+)
+
+// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.
+func generateMonitorID(systemId string, config monitor.Config) string {
+ args := []string{systemId, config.Target, config.Protocol}
+ // only use port for TCP monitors, since for other protocols it's not relevant as standalone value
+ if config.Protocol == "tcp" {
+ args = append(args, strconv.FormatUint(uint64(config.Port), 10))
+ }
+ return systems.MakeStableHashId(args...)
+}
+
+// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.
+func bindNetworkMonitorsEvents(hub *Hub) {
+ // on create, make sure the id is set to a stable hash
+ hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+ systemID := e.Record.GetString("system")
+ config := monitorConfigFromRecord(e.Record)
+ id := generateMonitorID(systemID, *config)
+ e.Record.Set("id", id)
+ return e.Next()
+ })
+
+ // sync monitor to agent on creation and persist the first result immediately when available
+ hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+ err := e.Next()
+ if err != nil {
+ return err
+ }
+ if !e.Record.GetBool("enabled") {
+ return nil
+ }
+ // If connected, run the monitor immediately. Paused systems may be absent
+ // from the manager; their monitors will sync when they reconnect.
+ system, err := hub.sm.GetSystem(e.Record.GetString("system"))
+ if err == nil && system.Status == "up" {
+ go hub.upsertNetworkMonitor(e.Record, true)
+ }
+ return nil
+ })
+
+ // On API update requests, if the monitor config changed in a way that requires a new ID, create a new
+ // record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.
+ hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {
+ systemID := e.Record.GetString("system")
+ // only tcp uses port - set other protocols port to zero
+ if e.Record.GetString("protocol") != "tcp" {
+ e.Record.Set("port", 0)
+ }
+ ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))
+ if ID != e.Record.Id {
+ newRecord := copyMonitorToNewRecord(e.Record, ID)
+ if err := e.App.Save(newRecord); err != nil {
+ return err
+ }
+ if err := e.App.Delete(e.Record); err != nil {
+ return err
+ }
+ return nil
+ }
+ err := e.Next()
+ if err != nil {
+ return err
+ }
+ if e.Record.GetBool("enabled") {
+ // if the monitor is enabled, sync the updated config to the agent now
+ runNow := !e.Record.Original().GetBool("enabled")
+ err = hub.upsertNetworkMonitor(e.Record, runNow)
+ } else {
+ // if the monitor is paused, remove it from the agent
+ err = hub.deleteNetworkMonitor(e.Record)
+ }
+ if err != nil {
+ hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)
+ }
+ return nil
+ })
+
+ // sync monitor to agent on delete
+ hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
+ if err := hub.deleteNetworkMonitor(e.Record); err != nil {
+ hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)
+ }
+ return e.Next()
+ })
+}
+
+// monitorConfigFromRecord builds a monitor config from a network_monitors record.
+func monitorConfigFromRecord(record *core.Record) *monitor.Config {
+ return &monitor.Config{
+ ID: record.Id,
+ Target: record.GetString("target"),
+ Protocol: record.GetString("protocol"),
+ Port: uint16(record.GetInt("port")),
+ Interval: uint16(record.GetInt("interval")),
+ }
+}
+
+// setMonitorResultFields stores the latest monitor result values on the record.
+func setMonitorResultFields(record *core.Record, result monitor.Result) {
+ nowString := time.Now().UTC().Format(types.DefaultDateLayout)
+ record.Set("res", result.AvgResponse)
+ record.Set("resAvg1h", result.AvgResponse1h)
+ record.Set("resMin1h", result.MinResponse1h)
+ record.Set("resMax1h", result.MaxResponse1h)
+ record.Set("loss1h", result.PacketLoss1h)
+ record.Set("updated", nowString)
+}
+
+// copyMonitorToNewRecord creates a new record with the same field values as the old one.
+// This is used when the monitor config changes in a way that requires a new ID, so we need
+// to create a new record with the new ID and delete the old one.
+func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {
+ collection := oldRecord.Collection()
+ newRecord := core.NewRecord(collection)
+ newRecord.Id = newID
+ fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}
+ for _, field := range fields {
+ newRecord.Set(field, oldRecord.Get(field))
+ }
+ return newRecord
+}
+
+// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow
+// is true, it will also trigger an immediate monitor run and update the record with the result.
+func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {
+ systemID := record.GetString("system")
+ system, err := h.sm.GetSystem(systemID)
+ if err != nil {
+ return err
+ }
+ result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)
+ if err != nil || result == nil {
+ return err
+ }
+ setMonitorResultFields(record, *result)
+ return h.App.SaveNoValidate(record)
+}
+
+// deleteNetworkMonitor removes the record's monitor from the target system.
+func (h *Hub) deleteNetworkMonitor(record *core.Record) error {
+ systemID := record.GetString("system")
+ system, err := h.sm.GetSystem(systemID)
+ if err != nil {
+ return err
+ }
+ return system.DeleteNetworkMonitor(record.Id)
+}
diff --git a/internal/hub/network_monitors_test.go b/internal/hub/network_monitors_test.go
new file mode 100644
index 000000000..f206ff84a
--- /dev/null
+++ b/internal/hub/network_monitors_test.go
@@ -0,0 +1,225 @@
+package hub
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/pocketbase/pocketbase/apis"
+ "github.com/pocketbase/pocketbase/core"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {
+ for _, batch := range []bool{false, true} {
+ name := "single"
+ if batch {
+ name = "batch"
+ }
+ t.Run(name, func(t *testing.T) {
+ hub, testApp, err := createTestHub(t)
+ require.NoError(t, err)
+ defer cleanupTestHub(hub, testApp)
+ bindNetworkMonitorsEvents(hub)
+
+ user, err := createTestUser(hub)
+ require.NoError(t, err)
+ system, err := createTestRecord(hub, "systems", map[string]any{
+ "name": "Paused", "host": "localhost", "port": "45876",
+ "status": "paused", "users": []string{user.Id},
+ })
+ require.NoError(t, err)
+ // Paused systems are not loaded into the manager at startup.
+ _, err = hub.sm.GetSystem(system.Id)
+ require.Error(t, err)
+
+ payload := func(target string) map[string]any {
+ return map[string]any{
+ "system": system.Id, "target": target, "protocol": "icmp",
+ "interval": 60, "enabled": true,
+ }
+ }
+ url := "/api/collections/network_monitors/records"
+ var body any = payload("1.1.1.1")
+ count := 1
+ if batch {
+ body = map[string]any{"requests": []map[string]any{
+ {"method": "POST", "url": url, "body": payload("1.1.1.1")},
+ {"method": "POST", "url": url, "body": payload("8.8.8.8")},
+ }}
+ url = "/api/batch"
+ count = 2
+ }
+ data, err := json.Marshal(body)
+ require.NoError(t, err)
+ token, err := user.NewAuthToken()
+ require.NoError(t, err)
+ router, err := apis.NewRouter(hub)
+ require.NoError(t, err)
+ handler, err := router.BuildMux()
+ require.NoError(t, err)
+ request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))
+ request.Header.Set("Content-Type", "application/json")
+ request.Header.Set("Authorization", token)
+ response := httptest.NewRecorder()
+ handler.ServeHTTP(response, request)
+ assert.Equal(t, http.StatusOK, response.Code, response.Body.String())
+
+ records, err := hub.FindAllRecords("network_monitors")
+ require.NoError(t, err)
+ require.Len(t, records, count)
+ for _, record := range records {
+ assert.Equal(t, system.Id, record.GetString("system"))
+ assert.True(t, record.GetBool("enabled"))
+ }
+ })
+ }
+}
+
+func TestGenerateMonitorID(t *testing.T) {
+ tests := []struct {
+ name string
+ systemID string
+ config monitor.Config
+ expected string
+ }{
+ {
+ name: "HTTP monitor on example.com",
+ systemID: "sys123",
+ config: monitor.Config{
+ Protocol: "http",
+ Target: "example.com",
+ Port: 0,
+ Interval: 60,
+ },
+ expected: "a20a5827",
+ },
+ {
+ name: "HTTP monitor on example.com with different port",
+ systemID: "sys123",
+ config: monitor.Config{
+ Protocol: "http",
+ Target: "example.com",
+ Port: 8080,
+ Interval: 60,
+ },
+ expected: "a20a5827",
+ },
+ {
+ name: "HTTP monitor on example.com with different system ID",
+ systemID: "sys1234",
+ config: monitor.Config{
+ Protocol: "http",
+ Target: "example.com",
+ Port: 80,
+ Interval: 60,
+ },
+ expected: "ab602ae7",
+ },
+ {
+ name: "Same monitor, different interval",
+ systemID: "sys1234",
+ config: monitor.Config{
+ Protocol: "http",
+ Target: "example.com",
+ Port: 80,
+ Interval: 120,
+ },
+ expected: "ab602ae7",
+ },
+ {
+ name: "ICMP monitor on 1.1.1.1",
+ systemID: "sys456",
+ config: monitor.Config{
+ Protocol: "icmp",
+ Target: "1.1.1.1",
+ Port: 0,
+ Interval: 10,
+ },
+ expected: "6d13a4a4",
+ }, {
+ name: "ICMP monitor on 1.1.1.1 with different system ID",
+ systemID: "sys4567",
+ config: monitor.Config{
+ Protocol: "icmp",
+ Target: "1.1.1.1",
+ Port: 0,
+ Interval: 10,
+ },
+ expected: "ddd6c81",
+ },
+ {
+ name: "TCP monitor on example.com with port 443",
+ systemID: "sys789",
+ config: monitor.Config{
+ Protocol: "tcp",
+ Target: "example.com",
+ Port: 443,
+ Interval: 30,
+ },
+ expected: "677b991",
+ },
+ {
+ name: "TCP monitor on example.com with port 8443",
+ systemID: "sys789",
+ config: monitor.Config{
+ Protocol: "tcp",
+ Target: "example.com",
+ Port: 8443,
+ Interval: 30,
+ },
+ expected: "84167969",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := generateMonitorID(tt.systemID, tt.config)
+ assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)
+ })
+ }
+}
+
+func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {
+ hub, testApp, err := createTestHub(t)
+ require.NoError(t, err)
+ defer cleanupTestHub(hub, testApp)
+
+ collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")
+ require.NoError(t, err)
+ assert.Nil(t, collection.Fields.GetByName("name"))
+
+ oldRecord := core.NewRecord(collection)
+ oldRecord.Load(map[string]any{
+ "system": "sys123",
+ "target": "https://example.com",
+ "protocol": "http",
+ "port": 443,
+ "interval": 60,
+ "enabled": true,
+ "res": 1200,
+ "resAvg1h": 1300,
+ "resMin1h": 900,
+ "resMax1h": 1600,
+ "loss1h": 5,
+ "updated": "2026-04-29 12:00:00.000Z",
+ })
+
+ newRecord := copyMonitorToNewRecord(oldRecord, "next12345")
+
+ assert.Equal(t, "next12345", newRecord.Id)
+ assert.Equal(t, "https://example.com", newRecord.GetString("target"))
+ assert.Equal(t, "http", newRecord.GetString("protocol"))
+ assert.Equal(t, 443, newRecord.GetInt("port"))
+ assert.True(t, newRecord.GetBool("enabled"))
+ assert.Zero(t, newRecord.GetFloat("res"))
+ assert.Zero(t, newRecord.GetFloat("resAvg1h"))
+ assert.Zero(t, newRecord.GetFloat("resMin1h"))
+ assert.Zero(t, newRecord.GetFloat("resMax1h"))
+ assert.Zero(t, newRecord.GetFloat("loss1h"))
+ assert.Equal(t, "", newRecord.GetString("updated"))
+}
diff --git a/internal/hub/systems/network_monitor_stats_test.go b/internal/hub/systems/network_monitor_stats_test.go
new file mode 100644
index 000000000..38bc8cb04
--- /dev/null
+++ b/internal/hub/systems/network_monitor_stats_test.go
@@ -0,0 +1,224 @@
+//go:build testing
+
+package systems
+
+import (
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/fxamacker/cbor/v2"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/henrygd/beszel/internal/entities/system"
+ "github.com/pocketbase/pocketbase/core"
+ "github.com/pocketbase/pocketbase/tools/subscriptions"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNetworkMonitorProbePruning(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ monitors map[string]monitor.Result
+ fail bool
+ want map[string]int64
+ }{
+ {"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
+ {"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
+ {"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
+ {"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
+ // Preserve the distinction between nil and empty across the agent transport.
+ encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
+ require.NoError(t, err)
+ var data system.CombinedData
+ require.NoError(t, cbor.Unmarshal(encoded, &data))
+ if tc.fail {
+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+ require.NoError(t, err)
+ }
+ _, err = sys.createRecords(&data)
+ if tc.fail {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+ assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
+ })
+ }
+}
+
+func TestNetworkMonitorStatsFreshness(t *testing.T) {
+ for _, realtime := range []bool{false, true} {
+ name := "sql"
+ if realtime {
+ name = "realtime"
+ }
+ t.Run(name, func(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ if realtime {
+ client := subscriptions.NewDefaultClient()
+ client.Subscribe("network_monitors/*")
+ app.SubscriptionsBroker().Register(client)
+ t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
+ }
+ col, err := app.FindCachedCollectionByNameOrId("network_monitors")
+ require.NoError(t, err)
+ for _, id := range []string{"monitor1", "monitor2"} {
+ record := core.NewRecord(col)
+ record.Id = id
+ record.Set("system", sys.Id)
+ require.NoError(t, app.SaveNoValidate(record))
+ }
+ data := &system.CombinedData{Monitors: map[string]monitor.Result{
+ "monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
+ "monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
+ }}
+ count := func(want int64) {
+ t.Helper()
+ got, err := app.CountRecords("network_monitor_stats")
+ require.NoError(t, err)
+ assert.Equal(t, want, got)
+ }
+ save := func() {
+ t.Helper()
+ _, err := sys.createRecords(data)
+ require.NoError(t, err)
+ }
+ save()
+ count(2)
+ stored, err := app.FindAllRecords("network_monitor_stats")
+ require.NoError(t, err)
+ for _, record := range stored {
+ result := data.Monitors[record.GetString("monitor")]
+ assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
+ assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
+ assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
+ }
+ // A resume can overlap the scheduled update with the same probe.
+ errs := make(chan error, 4)
+ for range 4 {
+ go func() {
+ _, err := sys.createRecords(data)
+ errs <- err
+ }()
+ }
+ for range 4 {
+ require.NoError(t, <-errs)
+ }
+ count(2)
+
+ // A rolling hourly value can change without a new probe.
+ result := data.Monitors["monitor1"]
+ result.AvgResponse1h = 42
+ data.Monitors["monitor1"] = result
+ save()
+ count(2)
+ record, err := app.FindRecordById("network_monitors", "monitor1")
+ require.NoError(t, err)
+ assert.Equal(t, 42, record.GetInt("resAvg1h"))
+
+ // Identical response values and failed probes still count as new measurements.
+ for id, result := range data.Monitors {
+ result.LastProbeAt = 301000
+ data.Monitors[id] = result
+ }
+ save()
+ count(4)
+
+ // A failed individual insert must remain retryable, even if others commit.
+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
+ require.NoError(t, err)
+ for id, result := range data.Monitors {
+ result.LastProbeAt = 601000
+ data.Monitors[id] = result
+ }
+ save()
+ count(5)
+ assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
+ assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
+ _, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
+ require.NoError(t, err)
+ save()
+ count(6)
+
+ // Failure after inserting stats rolls back the whole transaction and its markers.
+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+ require.NoError(t, err)
+ result = data.Monitors["monitor1"]
+ result.LastProbeAt = 901000
+ data.Monitors["monitor1"] = result
+ _, err = sys.createRecords(data)
+ require.Error(t, err)
+ count(6)
+ assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
+ _, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
+ require.NoError(t, err)
+ save()
+ count(7)
+
+ // Clock rollback is a new probe identity, not a reason to stall writes.
+ result.LastProbeAt = 500
+ data.Monitors["monitor1"] = result
+ save()
+ count(8)
+
+ // Recreated systems intentionally accept the first result without restoring state.
+ sys = &System{Id: sys.Id, manager: sys.manager}
+ save()
+ count(10)
+ })
+ }
+}
+
+// Observes the committed DB through the hub, not the transaction's app.
+type monitorAlertHub struct {
+ stubHub
+ handle func(*core.Record, map[string]monitor.Result) error
+}
+
+func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
+ return h.handle(record, results)
+}
+
+func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
+ for _, realtime := range []bool{false, true} {
+ t.Run(fmt.Sprint(realtime), func(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ if realtime {
+ client := subscriptions.NewDefaultClient()
+ client.Subscribe("network_monitors/*")
+ app.SubscriptionsBroker().Register(client)
+ }
+ collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
+ require.NoError(t, err)
+ record := core.NewRecord(collection)
+ record.Set("system", sys.Id)
+ require.NoError(t, app.SaveNoValidate(record))
+ called := 0
+ result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
+ sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
+ called++
+ assert.Equal(t, sys.Id, systemRecord.Id)
+ assert.Equal(t, result, results[record.Id])
+ saved, err := app.FindRecordById("network_monitors", record.Id)
+ require.NoError(t, err)
+ assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
+ return nil
+ }}
+ data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
+ _, err = sys.createRecords(data)
+ require.NoError(t, err)
+ assert.Equal(t, 1, called)
+ // A transaction that fails after writing monitor stats must not notify.
+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
+ require.NoError(t, err)
+ _, err = sys.createRecords(data)
+ require.Error(t, err)
+ assert.Equal(t, 1, called)
+ })
+ }
+}
diff --git a/internal/hub/systems/network_monitor_sync_test.go b/internal/hub/systems/network_monitor_sync_test.go
new file mode 100644
index 000000000..f9c7666ed
--- /dev/null
+++ b/internal/hub/systems/network_monitor_sync_test.go
@@ -0,0 +1,144 @@
+//go:build testing
+
+package systems
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/blang/semver"
+ "github.com/fxamacker/cbor/v2"
+ "github.com/henrygd/beszel/internal/common"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/henrygd/beszel/internal/hub/ws"
+ "github.com/lxzan/gws"
+ "github.com/pocketbase/pocketbase/core"
+ "github.com/stretchr/testify/require"
+)
+
+type monitorSyncClient struct {
+ gws.BuiltinEventHandler
+ requests chan common.HubRequest[monitor.SyncRequest]
+}
+
+func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
+ defer message.Close()
+ var req common.HubRequest[monitor.SyncRequest]
+ if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
+ return
+ }
+ c.requests <- req
+ data, _ := cbor.Marshal(monitor.SyncResponse{})
+ response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, Data: data})
+ _ = conn.WriteMessage(gws.OpcodeBinary, response)
+}
+
+// Avoid the production delayed disconnect notification; these tests explicitly
+// remove each connection from the manager before reconnecting.
+type monitorSyncServer struct{ ws.Handler }
+
+func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
+
+func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
+ for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
+ t.Run(version, func(t *testing.T) {
+ // No transport: attempting to send any request would fail.
+ sys := &System{agentVersion: semver.MustParse(version)}
+ require.NoError(t, sys.SyncNetworkMonitors(nil))
+ result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
+ require.NoError(t, err)
+ require.Nil(t, result)
+ require.NoError(t, sys.DeleteNetworkMonitor("test"))
+ })
+ }
+}
+
+func TestNetworkMonitorReconnectSync(t *testing.T) {
+ for _, change := range []string{"delete", "disable"} {
+ t.Run(change, func(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ record, err := app.FindRecordById("systems", sys.Id)
+ require.NoError(t, err)
+ // Suppress unrelated system-stat requests while exercising reconnects.
+ record.Set("status", paused)
+ require.NoError(t, app.SaveNoValidate(record))
+ collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
+ require.NoError(t, err)
+ probe := core.NewRecord(collection)
+ probe.Load(map[string]any{
+ "system": sys.Id, "target": "localhost", "protocol": "tcp",
+ "port": 80, "interval": 60, "enabled": true,
+ })
+ require.NoError(t, app.SaveNoValidate(probe))
+
+ sm := NewSystemManager(stubHub{app})
+ t.Cleanup(func() {
+ sm.cancel()
+ _ = sm.RemoveSystem(sys.Id)
+ sm.smartFetchMap.StopCleaner()
+ sm.zfsFetchMap.StopCleaner()
+ })
+ version := semver.MustParse("0.20.0")
+ connections := make(chan *ws.WsConn, 1)
+ upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ wsConn := ws.NewWsConnection(conn, version)
+ conn.Session().Store("wsConn", wsConn)
+ connections <- wsConn
+ conn.ReadLoop()
+ }))
+ t.Cleanup(server.Close)
+ client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
+ connect := func() monitor.SyncRequest {
+ t.Helper()
+ conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = conn.NetConn().Close() })
+ go conn.ReadLoop()
+ select {
+ case wsConn := <-connections:
+ require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
+ case <-time.After(3 * time.Second):
+ t.Fatal("websocket connection was not established")
+ }
+ select {
+ case req := <-client.requests:
+ require.Equal(t, common.SyncNetworkMonitors, req.Action)
+ require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
+ return req.Data
+ case <-time.After(3 * time.Second):
+ t.Fatal("reconnected agent did not receive a monitor replacement")
+ return monitor.SyncRequest{}
+ }
+ }
+
+ initial := connect()
+ require.Len(t, initial.Configs, 1)
+ require.Equal(t, probe.Id, initial.Configs[0].ID)
+ require.NoError(t, sm.RemoveSystem(sys.Id))
+ if change == "delete" {
+ require.NoError(t, app.Delete(probe))
+ } else {
+ probe.Set("enabled", false)
+ require.NoError(t, app.SaveNoValidate(probe))
+ }
+ require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
+ })
+ }
+}
+
+func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
+ sys, app := newTestSystemWithHub(t)
+ _, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
+ require.NoError(t, err)
+ _, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
+ require.Error(t, err, "a failed query must not be treated as an empty monitor set")
+}
diff --git a/internal/hub/systems/network_monitors.go b/internal/hub/systems/network_monitors.go
new file mode 100644
index 000000000..0fb44ed47
--- /dev/null
+++ b/internal/hub/systems/network_monitors.go
@@ -0,0 +1,58 @@
+package systems
+
+import (
+ "context"
+ "time"
+
+ "github.com/henrygd/beszel"
+ "github.com/henrygd/beszel/internal/common"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+)
+
+// SyncNetworkMonitors sends monitor configurations to the agent.
+func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
+ _, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
+ return err
+}
+
+// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
+func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
+ resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
+ Action: monitor.SyncActionUpsert,
+ Config: config,
+ RunNow: runNow,
+ })
+ if err != nil {
+ return nil, err
+ }
+ if resp.Result == (monitor.Result{}) {
+ return nil, nil
+ }
+ result := resp.Result
+ return &result, nil
+}
+
+// DeleteNetworkMonitor removes a single monitor task from the agent.
+func (sys *System) DeleteNetworkMonitor(id string) error {
+ _, err := sys.syncNetworkMonitors(monitor.SyncRequest{
+ Action: monitor.SyncActionDelete,
+ Config: monitor.Config{ID: id},
+ })
+ return err
+}
+
+func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
+ if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
+ return monitor.SyncResponse{}, nil
+ }
+ timeout := 5 * time.Second
+ if req.Action == monitor.SyncActionUpsert && req.RunNow {
+ // Allow the probe to finish, including a timeout result, while preserving
+ // the normal request budget for transport and response handling.
+ timeout += monitor.MaxProbeTimeout
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+ var result monitor.SyncResponse
+ return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
+}
diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.go
index 2ba22169e..6b0ca713f 100644
--- a/internal/hub/systems/system.go
+++ b/internal/hub/systems/system.go
@@ -9,6 +9,7 @@ import (
"math/rand"
"net"
"strings"
+ "sync"
"sync/atomic"
"time"
@@ -18,6 +19,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/container"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/systemd"
@@ -30,6 +32,8 @@ import (
"github.com/lxzan/gws"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
+ "github.com/pocketbase/pocketbase/tools/security"
+ "github.com/pocketbase/pocketbase/tools/types"
"golang.org/x/crypto/ssh"
)
@@ -52,6 +56,10 @@ type System struct {
smartInterval time.Duration // Interval for periodic SMART data updates
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
+ // Serialize persistence from scheduled updates and resumes through commit.
+ recordsMu sync.Mutex
+ // Protected by recordsMu; realtime reads don't consume probes.
+ lastSavedMonitorProbe map[string]int64
}
func (sm *SystemManager) NewSystem(systemId string) *System {
@@ -211,11 +219,15 @@ func (sys *System) handlePaused() {
// createRecords updates the system record and adds system_stats and container_stats records
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
+ sys.recordsMu.Lock()
+ defer sys.recordsMu.Unlock()
+
systemRecord, err := sys.getRecord(sys.manager.hub)
if err != nil {
return nil, err
}
hub := sys.manager.hub
+ savedMonitorProbes := make(map[string]int64)
err = hub.RunInTransaction(func(txApp core.App) error {
// add system_stats record
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
@@ -266,6 +278,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
}
}
+ if data.Monitors != nil {
+ if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
+ return err
+ }
+ }
+
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
return err
}
@@ -287,6 +305,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
return nil
})
+ // Publish only successful inserts after the entire transaction commits.
+ if err == nil && len(savedMonitorProbes) > 0 {
+ if sys.lastSavedMonitorProbe == nil {
+ sys.lastSavedMonitorProbe = savedMonitorProbes
+ } else {
+ for id, timestamp := range savedMonitorProbes {
+ sys.lastSavedMonitorProbe[id] = timestamp
+ }
+ }
+ }
+ // A non-nil report includes cached results for all remaining monitors.
+ if err == nil && data.Monitors != nil {
+ for id := range sys.lastSavedMonitorProbe {
+ if _, exists := data.Monitors[id]; !exists {
+ delete(sys.lastSavedMonitorProbe, id)
+ }
+ }
+ }
+ if err == nil {
+ if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
+ hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
+ }
+ }
return systemRecord, err
}
@@ -337,7 +378,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
}
suffix := fmt.Sprintf("%d", i)
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
- params["id"+suffix] = makeStableHashId(systemId, service.Name)
+ params["id"+suffix] = MakeStableHashId(systemId, service.Name)
params["name"+suffix] = service.Name
params["state"+suffix] = service.State
params["sub"+suffix] = service.Sub
@@ -363,6 +404,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
return err
}
+func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
+ if len(monitorResults) == 0 {
+ return nil
+ }
+ var err error
+ systemId := sys.Id
+ const monitorCollectionName = "network_monitors"
+
+ // If realtime updates are active, we save via PocketBase records to trigger realtime events.
+ // Otherwise we can do a more efficient direct update via SQL
+ realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
+ return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
+ })
+
+ now := time.Now().UTC()
+ nowMilli := now.UnixMilli()
+ nowString := now.Format(types.DefaultDateLayout)
+ var db dbx.Builder
+ var updateQuery *dbx.Query
+ if !realtimeActive {
+ db = app.DB()
+ monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
+ setClauses := make([]string, len(monitorFields))
+ for i, f := range monitorFields {
+ setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
+ }
+ queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
+ updateQuery = db.NewQuery(queryString)
+ }
+
+ // update network_monitors records
+ for id, result := range monitorResults {
+ monitorData := map[string]any{
+ "id": id,
+ "res": result.AvgResponse,
+ "resAvg1h": result.AvgResponse1h,
+ "resMin1h": result.MinResponse1h,
+ "resMax1h": result.MaxResponse1h,
+ "loss1h": result.PacketLoss1h,
+ "updated": nowString,
+ }
+ switch realtimeActive {
+ case true:
+ var record *core.Record
+ record, err = app.FindRecordById(monitorCollectionName, id)
+ if err == nil {
+ record.Load(monitorData)
+ err = app.SaveNoValidate(record)
+ }
+ default:
+ _, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
+ }
+ if err != nil {
+ app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
+ }
+ }
+
+ // handle stats collection — one record per monitor
+ const statsCollectionName = "network_monitor_stats"
+
+ var statsCollection *core.Collection
+ if realtimeActive {
+ statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
+ }
+
+ for monitorId, result := range monitorResults {
+ // Compare identity, not ordering, so agent clock changes don't stall writes.
+ if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
+ continue
+ }
+ statsRecordData := map[string]any{
+ "system": systemId,
+ "monitor": monitorId,
+ "type": "1m",
+ "created": nowMilli,
+ "res_min": result.MinResponse,
+ "res_max": result.MaxResponse,
+ "total_count": result.TotalCount,
+ "success_count": result.SuccessCount,
+ "res_sum": result.ResponseSum,
+ }
+ switch realtimeActive {
+ case true:
+ record := core.NewRecord(statsCollection)
+ record.Load(statsRecordData)
+ err = app.SaveNoValidate(record)
+ default:
+ statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
+ _, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
+ }
+ if err != nil {
+ app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
+ } else {
+ savedProbes[monitorId] = result.LastProbeAt
+ }
+ }
+
+ return nil
+}
+
// createContainerRecords creates container records
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
if len(data) == 0 {
@@ -622,7 +763,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
return &result, err
}
-func makeStableHashId(strings ...string) string {
+func MakeStableHashId(strings ...string) string {
hash := fnv.New32a()
for _, str := range strings {
hash.Write([]byte(str))
diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.go
index c1a4cde46..db279609c 100644
--- a/internal/hub/systems/system_manager.go
+++ b/internal/hub/systems/system_manager.go
@@ -9,6 +9,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap"
@@ -17,6 +18,7 @@ import (
"github.com/henrygd/beszel"
"github.com/blang/semver"
+ "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/store"
"golang.org/x/crypto/ssh"
@@ -62,6 +64,7 @@ type hubLike interface {
core.App
GetSSHKey(dataDir string) (ssh.Signer, error)
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
+ HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
HandleStatusAlerts(status string, systemRecord *core.Record) error
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
CancelPendingStatusAlerts(systemID string)
@@ -350,6 +353,20 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
if err := sm.AddRecord(systemRecord, system); err != nil {
return err
}
+
+ // Sync network monitors to the newly connected agent
+ go func() {
+ configs, err := sm.GetMonitorConfigsForSystem(systemId)
+ if err != nil {
+ sm.hub.Logger().Warn("failed to load monitors for agent", "system", systemId, "err", err)
+ return
+ }
+ // An empty set must also replace any probes retained across a disconnect.
+ if err := system.SyncNetworkMonitors(configs); err != nil {
+ sm.hub.Logger().Warn("failed to sync monitors to agent", "system", systemId, "err", err)
+ }
+ }()
+
return nil
}
@@ -362,6 +379,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
}
}
+// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
+func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
+ var configs []monitor.Config
+ err := sm.hub.DB().
+ NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
+ Bind(dbx.Params{"system": systemID}).
+ All(&configs)
+ return configs, err
+}
+
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
// agent reconnect retries ZFS discovery immediately after configuration changes.
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
@@ -397,11 +424,12 @@ func (sm *SystemManager) createSSHClientConfig() error {
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
// This is called when a system is paused or goes offline to prevent continued alerts.
+// Monitor incidents remain open: a missing observation does not establish recovery.
func deactivateAlerts(app core.App, systemID string) error {
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
- alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0)
+ alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
if err != nil {
return err
}
diff --git a/internal/hub/systems/system_realtime.go b/internal/hub/systems/system_realtime.go
index 0fb6b6f9a..f290eda6e 100644
--- a/internal/hub/systems/system_realtime.go
+++ b/internal/hub/systems/system_realtime.go
@@ -6,6 +6,8 @@ import (
"time"
"github.com/henrygd/beszel/internal/common"
+ "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/utils"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/apis"
@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {
if err != nil {
return
}
- bytes, err := json.Marshal(data)
+ bytes, err := marshalRealtimeData(data)
if err == nil {
notify(sm.hub, system, fetch.subscription, bytes)
}
@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
}
}
+// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
+// the per-monitor results into the derived metric fields the frontend charts expect.
+func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
+ if len(data.Monitors) == 0 {
+ return json.Marshal(data)
+ }
+ monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
+ for id, result := range data.Monitors {
+ monitorStats[id] = monitor.Stats{}.FromResult(result)
+ }
+ return json.Marshal(struct {
+ *system.CombinedData
+ Monitors map[string]monitor.Stats `json:"Monitors"`
+ }{data, monitorStats})
+}
+
// notify broadcasts realtime data to all clients subscribed to a specific subscription.
// Custom topics bypass collection rules, so check current access for every
// recipient, including clients whose authentication or membership was revoked.
diff --git a/internal/hub/systems/system_smart.go b/internal/hub/systems/system_smart.go
index 68b750fe4..9a6db8de4 100644
--- a/internal/hub/systems/system_smart.go
+++ b/internal/hub/systems/system_smart.go
@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
currentIDs := make(map[string]struct{}, len(smartData))
for deviceKey := range smartData {
- currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{}
+ currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
}
err = hub.RunInTransaction(func(txApp core.App) error {
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
}
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
- recordID := makeStableHashId(sys.Id, deviceKey)
+ recordID := MakeStableHashId(sys.Id, deviceKey)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
diff --git a/internal/hub/systems/system_smart_test.go b/internal/hub/systems/system_smart_test.go
index aa98633db..460a38f3e 100644
--- a/internal/hub/systems/system_smart_test.go
+++ b/internal/hub/systems/system_smart_test.go
@@ -7,6 +7,7 @@ import (
"testing"
"time"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap"
@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
return nil
}
-func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
+func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
+func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
return nil
}
@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
}, false))
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
- recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA"))
+ recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
require.NoError(t, err)
assert.EqualValues(t, 42, recordA.GetInt("temp"))
}
diff --git a/internal/hub/systems/system_systemd_test.go b/internal/hub/systems/system_systemd_test.go
index c2d890d5a..d5b103e0b 100644
--- a/internal/hub/systems/system_systemd_test.go
+++ b/internal/hub/systems/system_systemd_test.go
@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName := "nginx.service"
// Call multiple times and ensure same result
- id1 := makeStableHashId(systemId, serviceName)
- id2 := makeStableHashId(systemId, serviceName)
- id3 := makeStableHashId(systemId, serviceName)
+ id1 := MakeStableHashId(systemId, serviceName)
+ id2 := MakeStableHashId(systemId, serviceName)
+ id3 := MakeStableHashId(systemId, serviceName)
assert.Equal(t, id1, id2)
assert.Equal(t, id2, id3)
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName1 := "nginx.service"
serviceName2 := "apache.service"
- id1 := makeStableHashId(systemId1, serviceName1)
- id2 := makeStableHashId(systemId2, serviceName1)
- id3 := makeStableHashId(systemId1, serviceName2)
- id4 := makeStableHashId(systemId2, serviceName2)
+ id1 := MakeStableHashId(systemId1, serviceName1)
+ id2 := MakeStableHashId(systemId2, serviceName1)
+ id3 := MakeStableHashId(systemId1, serviceName2)
+ id4 := MakeStableHashId(systemId2, serviceName2)
// All IDs should be different
assert.NotEqual(t, id1, id2)
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
}
for _, tc := range testCases {
- id := makeStableHashId(tc.systemId, tc.serviceName)
+ id := MakeStableHashId(tc.systemId, tc.serviceName)
// FNV-32 produces 8 hex characters
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
}
})
t.Run("hexadecimal output", func(t *testing.T) {
- id := makeStableHashId("test-system", "test-service")
+ id := MakeStableHashId("test-system", "test-service")
assert.NotEmpty(t, id)
// Should only contain hexadecimal characters
diff --git a/internal/hub/systems/system_zfs.go b/internal/hub/systems/system_zfs.go
index 30f575024..934657117 100644
--- a/internal/hub/systems/system_zfs.go
+++ b/internal/hub/systems/system_zfs.go
@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
}
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
- recordID := makeStableHashId(sys.Id, pool.Name)
+ recordID := MakeStableHashId(sys.Id, pool.Name)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
if pool == nil {
continue
}
- recordID := makeStableHashId(sys.Id, name)
+ recordID := MakeStableHashId(sys.Id, name)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
if !errors.Is(err, sql.ErrNoRows) {
diff --git a/internal/hub/systems/system_zfs_test.go b/internal/hub/systems/system_zfs_test.go
index b61d687a4..b035dd80c 100644
--- a/internal/hub/systems/system_zfs_test.go
+++ b/internal/hub/systems/system_zfs_test.go
@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
}}
require.NoError(t, sys.saveZfsPools(initial))
- failedID := makeStableHashId(sys.Id, failedKey)
+ failedID := MakeStableHashId(sys.Id, failedKey)
before, err := app.FindRecordById("zfs_pools", failedID)
require.NoError(t, err)
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
}}
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
- fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey))
+ fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
require.NoError(t, err)
assert.EqualValues(t, 20, fresh.GetInt("alloc"))
cached, err := app.FindRecordById("zfs_pools", failedID)
@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
}))
- record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
+ record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err)
firstUpdated := record.GetDateTime("updated")
assert.Equal(t, "ONLINE", record.GetString("health"))
@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
func TestZfsRawCapacityPersistence(t *testing.T) {
sys, app := newTestSystemWithHub(t)
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
- record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs"))
+ record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
require.NoError(t, err)
require.True(t, record.GetBool("raw"))
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
key: {DisplayName: "tank", Health: "ONLINE"},
"tank": {Health: "ONLINE"},
}))
- id := makeStableHashId(sys.Id, key)
+ id := MakeStableHashId(sys.Id, key)
record, err := app.FindRecordById("zfs_pools", id)
require.NoError(t, err)
assert.Equal(t, "tank", record.GetString("display_name"))
@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
record, err = app.FindRecordById("zfs_pools", id)
require.NoError(t, err)
assert.Equal(t, "detail name", record.GetString("display_name"))
- _, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank"))
+ _, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err)
}
diff --git a/internal/hub/utils/utils.go b/internal/hub/utils/utils.go
index 438388684..0cfc1fa9d 100644
--- a/internal/hub/utils/utils.go
+++ b/internal/hub/utils/utils.go
@@ -1,7 +1,11 @@
// Package utils provides utility functions for the hub.
package utils
-import "os"
+import (
+ "os"
+
+ "github.com/pocketbase/pocketbase/core"
+)
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
func GetEnv(key string) (value string, exists bool) {
@@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) {
}
return os.LookupEnv(key)
}
+
+// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.
+func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {
+ broker := app.SubscriptionsBroker()
+ if broker.TotalClients() == 0 {
+ return false
+ }
+ for _, client := range broker.Clients() {
+ subs := client.Subscriptions(collectionName)
+ if len(subs) > 0 {
+ if validateFn == nil {
+ return true
+ }
+ for k := range subs {
+ filter := subs[k].Query["filter"]
+ if validateFn(filter) {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
diff --git a/internal/migrations/0_collections_snapshot_0_19_0.go b/internal/migrations/0_collections_snapshot_0_20_0_dev_5.go
similarity index 81%
rename from internal/migrations/0_collections_snapshot_0_19_0.go
rename to internal/migrations/0_collections_snapshot_0_20_0_dev_5.go
index 73b89d2b5..c11985a56 100644
--- a/internal/migrations/0_collections_snapshot_0_19_0.go
+++ b/internal/migrations/0_collections_snapshot_0_20_0_dev_5.go
@@ -83,7 +83,8 @@ func init() {
"ContainerHealth",
"SystemdFailed",
"CPUIOWait",
- "CPUSteal"
+ "CPUSteal",
+ "NetworkMonitorLoss"
]
},
{
@@ -119,6 +120,16 @@ func init() {
"system": false,
"type": "bool"
},
+ {
+ "hidden": true,
+ "id": "json4000656575",
+ "maxSize": 0,
+ "name": "state",
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "json"
+ },
{
"hidden": true,
"id": "date1302749137",
@@ -152,6 +163,7 @@ func init() {
}
],
"indexes": [
+ "CREATE INDEX idx_alerts_system_name ON alerts (system, name)",
"CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)"
],
"system": false
@@ -234,6 +246,20 @@ func init() {
"system": false,
"type": "text"
},
+ {
+ "autogeneratePattern": "",
+ "hidden": false,
+ "id": "text3888135399",
+ "max": 0,
+ "min": 0,
+ "name": "monitor_name",
+ "pattern": "",
+ "presentable": false,
+ "primaryKey": false,
+ "required": false,
+ "system": false,
+ "type": "text"
+ },
{
"hidden": false,
"id": "number494360628",
@@ -1721,6 +1747,7 @@ func init() {
"fields": [
{
"autogeneratePattern": "[a-z0-9]{15}",
+ "help": "",
"hidden": false,
"id": "text3208210256",
"max": 15,
@@ -1736,6 +1763,7 @@ func init() {
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
+ "help": "",
"hidden": false,
"id": "relation1204987316",
"maxSelect": 1,
@@ -1748,6 +1776,7 @@ func init() {
},
{
"autogeneratePattern": "",
+ "help": "",
"hidden": false,
"id": "text7739291048",
"max": 0,
@@ -1762,6 +1791,7 @@ func init() {
},
{
"autogeneratePattern": "",
+ "help": "",
"hidden": false,
"id": "text5528164482",
"max": 0,
@@ -1775,6 +1805,7 @@ func init() {
"type": "text"
},
{
+ "help": "",
"hidden": false,
"id": "number8862034195",
"max": null,
@@ -1787,6 +1818,7 @@ func init() {
"type": "number"
},
{
+ "help": "",
"hidden": false,
"id": "number4418907321",
"max": null,
@@ -1799,6 +1831,7 @@ func init() {
"type": "number"
},
{
+ "help": "",
"hidden": false,
"id": "number2904183765",
"max": null,
@@ -1811,6 +1844,7 @@ func init() {
"type": "number"
},
{
+ "help": "",
"hidden": false,
"id": "json4466109723",
"maxSize": 0,
@@ -1821,6 +1855,7 @@ func init() {
"type": "json"
},
{
+ "help": "",
"hidden": false,
"id": "json9012873456",
"maxSize": 0,
@@ -1831,6 +1866,7 @@ func init() {
"type": "json"
},
{
+ "help": "",
"hidden": false,
"id": "json7182045639",
"maxSize": 0,
@@ -1841,6 +1877,7 @@ func init() {
"type": "json"
},
{
+ "help": "",
"hidden": false,
"id": "date9274163058",
"max": "",
@@ -1860,19 +1897,401 @@ func init() {
"presentable": false,
"system": false,
"type": "autodate"
+ },
+ {
+ "autogeneratePattern": "",
+ "help": "",
+ "hidden": false,
+ "id": "text3578368839",
+ "max": 0,
+ "min": 0,
+ "name": "display_name",
+ "pattern": "",
+ "presentable": false,
+ "primaryKey": false,
+ "required": false,
+ "system": false,
+ "type": "text"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "bool447994709",
+ "name": "raw",
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "bool"
}
],
"id": "pbc_8441057391",
"indexes": [
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
],
- "listRule": null,
+ "listRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id",
"name": "zfs_pools",
"system": false,
"type": "base",
"updateRule": null,
+ "viewRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id"
+ },
+ {
+ "createRule": null,
+ "deleteRule": null,
+ "fields": [
+ {
+ "autogeneratePattern": "[a-z0-9]{10}",
+ "help": "",
+ "hidden": false,
+ "id": "text3208210256",
+ "max": 10,
+ "min": 6,
+ "name": "id",
+ "pattern": "^[a-z0-9]+$",
+ "presentable": false,
+ "primaryKey": true,
+ "required": true,
+ "system": true,
+ "type": "text"
+ },
+ {
+ "cascadeDelete": true,
+ "collectionId": "2hz5ncl8tizk5nx",
+ "help": "",
+ "hidden": false,
+ "id": "nm_system",
+ "maxSelect": 1,
+ "minSelect": 0,
+ "name": "system",
+ "presentable": false,
+ "required": true,
+ "system": false,
+ "type": "relation"
+ },
+ {
+ "autogeneratePattern": "",
+ "help": "",
+ "hidden": false,
+ "id": "nm_target",
+ "max": 500,
+ "min": 1,
+ "name": "target",
+ "pattern": "",
+ "presentable": false,
+ "primaryKey": false,
+ "required": true,
+ "system": false,
+ "type": "text"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "nm_protocol",
+ "maxSelect": 1,
+ "name": "protocol",
+ "presentable": false,
+ "required": true,
+ "system": false,
+ "type": "select",
+ "values": [
+ "icmp",
+ "tcp",
+ "http",
+ "dns"
+ ]
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "nm_port",
+ "max": 65535,
+ "min": 0,
+ "name": "port",
+ "onlyInt": true,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "nm_interval",
+ "max": 3600,
+ "min": 1,
+ "name": "interval",
+ "onlyInt": true,
+ "presentable": false,
+ "required": true,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "number926446584",
+ "max": null,
+ "min": null,
+ "name": "res",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "number1006954605",
+ "max": null,
+ "min": null,
+ "name": "resAvg1h",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "number4267669802",
+ "max": null,
+ "min": null,
+ "name": "resMin1h",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "number591433223",
+ "max": null,
+ "min": null,
+ "name": "resMax1h",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "number3726709001",
+ "max": null,
+ "min": null,
+ "name": "loss1h",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "nm_enabled",
+ "name": "enabled",
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "bool"
+ },
+ {
+ "hidden": false,
+ "id": "autodate2990389176",
+ "name": "created",
+ "onCreate": true,
+ "onUpdate": false,
+ "presentable": false,
+ "system": false,
+ "type": "autodate"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "date3332085495",
+ "max": "",
+ "min": "",
+ "name": "updated",
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "date"
+ }
+ ],
+ "id": "nm_monitors_001",
+ "indexes": [
+ "CREATE INDEX ` + "`" + `idx_nm_system_enabled` + "`" + ` ON ` + "`" + `network_monitors` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `enabled` + "`" + `)"
+ ],
+ "listRule": null,
+ "name": "network_monitors",
+ "system": false,
+ "type": "base",
+ "updateRule": null,
"viewRule": null
- }
+ },
+ {
+ "createRule": null,
+ "deleteRule": null,
+ "fields": [
+ {
+ "autogeneratePattern": "[a-z0-9]{10}",
+ "help": "",
+ "hidden": false,
+ "id": "text3208210256",
+ "max": 10,
+ "min": 10,
+ "name": "id",
+ "pattern": "^[a-z0-9]+$",
+ "presentable": false,
+ "primaryKey": true,
+ "required": true,
+ "system": true,
+ "type": "text"
+ },
+ {
+ "cascadeDelete": true,
+ "collectionId": "2hz5ncl8tizk5nx",
+ "help": "",
+ "hidden": false,
+ "id": "nms_system",
+ "maxSelect": 1,
+ "minSelect": 0,
+ "name": "system",
+ "presentable": false,
+ "required": true,
+ "system": false,
+ "type": "relation"
+ },
+ {
+ "cascadeDelete": true,
+ "collectionId": "nm_monitors_001",
+ "help": "",
+ "hidden": false,
+ "id": "nms_monitor",
+ "maxSelect": 1,
+ "minSelect": 0,
+ "name": "monitor",
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "relation"
+ },
+ {
+ "help": "Number of probe attempts",
+ "hidden": false,
+ "id": "nms_total_count",
+ "max": null,
+ "min": 0,
+ "name": "total_count",
+ "onlyInt": true,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "Number of successful probe attempts",
+ "hidden": false,
+ "id": "nms_success_count",
+ "max": null,
+ "min": 0,
+ "name": "success_count",
+ "onlyInt": true,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "Sum of successful response times in microseconds",
+ "hidden": false,
+ "id": "nms_res_sum",
+ "max": null,
+ "min": 0,
+ "name": "res_sum",
+ "onlyInt": true,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "Response time in microseconds",
+ "hidden": false,
+ "id": "nms_res_min",
+ "max": null,
+ "min": 0,
+ "name": "res_min",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "Response time in microseconds",
+ "hidden": false,
+ "id": "nms_res_max",
+ "max": null,
+ "min": 0,
+ "name": "res_max",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "nms_type",
+ "maxSelect": 1,
+ "name": "type",
+ "presentable": false,
+ "required": true,
+ "system": false,
+ "type": "select",
+ "values": [
+ "1m",
+ "10m",
+ "20m",
+ "120m",
+ "480m"
+ ]
+ },
+ {
+ "help": "",
+ "hidden": false,
+ "id": "number2990389176",
+ "max": null,
+ "min": null,
+ "name": "created",
+ "onlyInt": false,
+ "presentable": false,
+ "required": false,
+ "system": false,
+ "type": "number"
+ }
+ ],
+ "id": "nm_stats_001",
+ "indexes": [
+ "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_system_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
+ "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_monitor_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `monitor` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
+ "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)"
+ ],
+ "listRule": null,
+ "name": "network_monitor_stats",
+ "system": false,
+ "type": "base",
+ "updateRule": null,
+ "viewRule": null
+ }
]`
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
diff --git a/internal/records/monitor_averaging_test.go b/internal/records/monitor_averaging_test.go
new file mode 100644
index 000000000..da00330fa
--- /dev/null
+++ b/internal/records/monitor_averaging_test.go
@@ -0,0 +1,226 @@
+//go:build testing
+
+package records_test
+
+import (
+ "testing"
+ "time"
+
+ monitorEntity "github.com/henrygd/beszel/internal/entities/monitor"
+ "github.com/henrygd/beszel/internal/records"
+ "github.com/henrygd/beszel/internal/tests"
+
+ "github.com/pocketbase/dbx"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAverageMonitorStats(t *testing.T) {
+ hub, err := tests.NewTestHub(t.TempDir())
+ require.NoError(t, err)
+ defer hub.Cleanup()
+
+ collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats")
+ require.NoError(t, err)
+ assert.Nil(t, collection.Fields.GetByName("res_avg"))
+ assert.Nil(t, collection.Fields.GetByName("loss"))
+
+ rm := records.NewRecordManager(hub)
+ user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
+ require.NoError(t, err)
+ sys, err := tests.CreateRecord(hub, "systems", map[string]any{
+ "name": "monitor-avg-system",
+ "host": "localhost",
+ "port": "45876",
+ "status": "up",
+ "users": []string{user.Id},
+ })
+ require.NoError(t, err)
+ monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": sys.Id,
+ "name": "cloudflare",
+ "target": "1.1.1.1",
+ "protocol": "icmp",
+ "interval": 30,
+ "enabled": true,
+ })
+ require.NoError(t, err)
+
+ created := time.Now().UnixMilli()
+ // Unequal probe counts must weight both latency and loss.
+ recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
+ "system": sys.Id,
+ "monitor": monitor.Id,
+ "type": "1m",
+ "created": created,
+ "res_min": 5,
+ "res_max": 20,
+ "total_count": 6, "success_count": 6, "res_sum": 60,
+ })
+ require.NoError(t, err)
+ recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
+ "system": sys.Id,
+ "monitor": monitor.Id,
+ "type": "1m",
+ "created": created,
+ "res_min": 10,
+ "res_max": 60,
+ "total_count": 1, "success_count": 1, "res_sum": 22,
+ })
+ require.NoError(t, err)
+
+ result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
+ require.NoError(t, err)
+ assert.Equal(t, 2, count)
+ assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result)
+
+ for _, tc := range []struct {
+ name, monitor, recordType string
+ after int64
+ }{
+ {"other monitor", "missing", "1m", created - 1},
+ {"other type", monitor.Id, "10m", created - 1},
+ {"exclusive cutoff", monitor.Id, "1m", created},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after)
+ require.NoError(t, err)
+ assert.Zero(t, count)
+ assert.Equal(t, monitorEntity.Stats{}, stats)
+ })
+ }
+
+ // A failure-only bucket counts toward loss but must not lower latency.
+ recordB.Set("res_min", 0)
+ recordB.Set("res_max", 0)
+ recordB.Set("success_count", 0)
+ recordB.Set("res_sum", 0)
+ require.NoError(t, hub.Save(recordB))
+ result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
+ require.NoError(t, err)
+ assert.Equal(t, 2, count)
+ assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result)
+ // Sparse monitor records must propagate through every rollup level.
+ rm.CreateLongerRecords()
+ for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
+ rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
+ require.NoError(t, err)
+ require.Len(t, rollups, 1, recordType)
+ assert.Equal(t, 5.0, rollups[0].GetFloat("res_min"))
+ assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
+ assert.Equal(t, 7, rollups[0].GetInt("total_count"))
+ assert.Equal(t, 6, rollups[0].GetInt("success_count"))
+ assert.Equal(t, 60, rollups[0].GetInt("res_sum"))
+ // A sibling with a different number of probes must retain its actual
+ // weight when the next tier combines their underlying counts.
+ _, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
+ "system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created,
+ "res_min": 100, "res_max": 100,
+ "total_count": 3, "success_count": 1, "res_sum": 100,
+ })
+ require.NoError(t, err)
+ merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1)
+ require.NoError(t, err)
+ assert.Equal(t, 2, count)
+ assert.Equal(t, monitorEntity.Stats{
+ ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30,
+ TotalCount: 10, SuccessCount: 7, ResponseSum: 160,
+ }, merged)
+ }
+ // All failures produce zero latency, while a genuine zero-microsecond
+ // success remains a valid minimum (it must not be filtered out as a sentinel).
+ recordA.Set("success_count", 0)
+ recordA.Set("res_sum", 0)
+ require.NoError(t, hub.Save(recordA))
+ result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
+ require.NoError(t, err)
+ assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result)
+ recordA.Set("success_count", 1)
+ recordA.Set("res_min", 0)
+ recordA.Set("res_max", 0)
+ require.NoError(t, hub.Save(recordA))
+ result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
+ require.NoError(t, err)
+ assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result)
+}
+
+func TestSparseMonitorRollups(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ interval int
+ samples int
+ disabled bool
+ }{
+ {"five minute interval", 300, 2, false},
+ {"ten minute interval", 600, 1, false},
+ {"fifteen minute interval", 900, 1, false},
+ {"empty window", 900, 0, false},
+ {"disabled with pending history", 300, 2, true},
+ {"disabled empty window", 900, 0, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ hub, err := tests.NewTestHub(t.TempDir())
+ require.NoError(t, err)
+ defer hub.Cleanup()
+ user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
+ require.NoError(t, err)
+ sys, err := tests.CreateRecord(hub, "systems", map[string]any{
+ "name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up",
+ "users": []string{user.Id},
+ })
+ require.NoError(t, err)
+ monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
+ "interval": tc.interval, "enabled": true,
+ })
+ require.NoError(t, err)
+ now := time.Now()
+ for i := range tc.samples {
+ _, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
+ "system": sys.Id, "monitor": monitor.Id, "type": "1m",
+ "created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(),
+ "res_min": 8, "res_max": 20,
+ "total_count": 4, "success_count": 3, "res_sum": 36,
+ })
+ require.NoError(t, err)
+ }
+ // Other collections must still reject fewer than nine minute records.
+ for _, collection := range []string{"system_stats", "container_stats"} {
+ stats := `{"cpu":10}`
+ if collection == "container_stats" {
+ stats = `[{"name":"test","cpu":10}]`
+ }
+ for range 8 {
+ _, err := tests.CreateRecord(hub, collection, map[string]any{
+ "system": sys.Id, "type": "1m", "stats": stats,
+ })
+ require.NoError(t, err)
+ }
+ }
+ if tc.disabled {
+ monitor.Set("enabled", false)
+ require.NoError(t, hub.Save(monitor))
+ }
+ records.NewRecordManager(hub).CreateLongerRecords()
+ for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
+ rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
+ require.NoError(t, err)
+ if tc.samples == 0 {
+ assert.Empty(t, rollups, recordType)
+ } else {
+ require.Len(t, rollups, 1, recordType)
+ assert.Equal(t, 8.0, rollups[0].GetFloat("res_min"))
+ assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
+ assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count"))
+ assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count"))
+ assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum"))
+ }
+ for _, collection := range []string{"system_stats", "container_stats"} {
+ count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType})
+ require.NoError(t, err)
+ assert.Zero(t, count, "%s %s", collection, recordType)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/records/records.go b/internal/records/records.go
index 4ef1b1914..7c4bf281c 100644
--- a/internal/records/records.go
+++ b/internal/records/records.go
@@ -3,15 +3,16 @@ package records
import (
"encoding/json"
- "log/slog"
"math"
"time"
"github.com/henrygd/beszel/internal/entities/container"
+ "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
+ "github.com/pocketbase/pocketbase/tools/types"
)
type RecordManager struct {
@@ -39,7 +40,7 @@ type StatsRecord struct {
// Create longer records by averaging shorter records
func (rm *RecordManager) CreateLongerRecords() {
- // start := time.Now()
+ now := time.Now().UTC()
longerRecordData := []LongerRecordData{
{
shorterType: "1m",
@@ -69,23 +70,28 @@ func (rm *RecordManager) CreateLongerRecords() {
}
// wrap the operations in a transaction
// Pocketbase cron does not handle errors, log them here.
- rm.app.RunInTransaction(func(txApp core.App) error {
+ err := rm.app.RunInTransaction(func(txApp core.App) error {
var err error
+
collections := [2]*core.Collection{}
collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats")
if err != nil {
- slog.Error("Error finding cached collection using system stats:", "err", err)
return err
}
collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats")
if err != nil {
- slog.Error("Error finding cached collection using container stats:", "err", err)
+ return err
+ }
+ monitorStatsColl, err := txApp.FindCachedCollectionByNameOrId("network_monitor_stats")
+ if err != nil {
return err
}
var systems RecordIds
db := txApp.DB()
- db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems)
+ if err := db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems); err != nil {
+ return err
+ }
// loop through all active systems, time periods, and collections
for _, system := range systems {
@@ -94,44 +100,52 @@ func (rm *RecordManager) CreateLongerRecords() {
recordData := longerRecordData[i]
// log.Println("processing longer record type", recordData.longerType)
// add one minute padding for longer records because they are created slightly later than the job start time
- longerRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration + time.Minute)
+ longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
// shorter records are created independently of longer records, so we shouldn't need to add padding
- shorterRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration)
- // loop through both collections
+ shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
for _, collection := range collections {
// check creation time of last longer record if not 10m, since 10m is created every run
if recordData.longerType != "10m" {
- count, err := txApp.CountRecords(
- collection.Id,
- dbx.NewExp(
- "system = {:system} AND type = {:type} AND created > {:created}",
- dbx.Params{"type": recordData.longerType, "system": system.Id, "created": longerRecordPeriod},
- ),
- )
+ count, err := txApp.CountRecords(collection.Id, dbx.NewExp(
+ "system = {:system} AND type = {:type} AND created > {:created}",
+ dbx.Params{
+ "type": recordData.longerType,
+ "system": system.Id,
+ "created": longerRecordPeriod.Format(types.DefaultDateLayout),
+ },
+ ))
+ if err != nil {
+ return err
+ }
// continue if longer record exists
- if err != nil || count > 0 {
+ if count > 0 {
continue
}
}
// get shorter records from the past x minutes
var recordIds RecordIds
- err := txApp.DB().
+ params := dbx.Params{
+ "type": recordData.shorterType,
+ "system": system.Id,
+ "created": shorterRecordPeriod.Format(types.DefaultDateLayout),
+ }
+
+ err := db.
Select("id").
From(collection.Name).
- AndWhere(dbx.NewExp(
+ Where(dbx.NewExp(
"system={:system} AND type={:type} AND created > {:created}",
- dbx.Params{
- "type": recordData.shorterType,
- "system": system.Id,
- "created": shorterRecordPeriod,
- },
+ params,
)).
OrderBy("created").
All(&recordIds)
+ if err != nil {
+ return err
+ }
// continue if not enough shorter records
- if err != nil || len(recordIds) < recordData.minShorterRecords {
+ if len(recordIds) < recordData.minShorterRecords {
continue
}
// average the shorter records and create longer record
@@ -142,20 +156,88 @@ func (rm *RecordManager) CreateLongerRecords() {
case "system_stats":
longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds))
case "container_stats":
-
longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds))
}
if err := txApp.SaveNoValidate(longerRecord); err != nil {
- slog.Error("failed to save longer record", "err", err)
+ txApp.Logger().Error("failed to save longer record", "err", err)
+ }
+ }
+ }
+ }
+
+ // network_monitor_stats is aggregated per monitor (not per system)
+ var monitors []struct {
+ Id string `db:"id"`
+ System string `db:"system"`
+ }
+ // Disabled monitors still have history that must advance through retention tiers.
+ if err := db.NewQuery("SELECT id, system FROM network_monitors").All(&monitors); err != nil {
+ return err
+ }
+
+ for _, monitorRec := range monitors {
+ for i := range longerRecordData {
+ recordData := longerRecordData[i]
+ longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
+ shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
+
+ if recordData.longerType != "10m" {
+ count, err := txApp.CountRecords(monitorStatsColl.Id, dbx.NewExp(
+ "monitor={:monitor} AND type={:type} AND created>{:created}",
+ dbx.Params{
+ "monitor": monitorRec.Id,
+ "type": recordData.longerType,
+ "created": longerRecordPeriod.UnixMilli(),
+ },
+ ))
+ if err != nil {
+ return err
+ }
+ if count > 0 {
+ continue
}
}
+
+ stats, count, err := rm.AverageMonitorStats(db, monitorRec.Id, recordData.shorterType, shorterRecordPeriod.UnixMilli())
+ if err != nil {
+ txApp.Logger().Error("failed to average monitor stats", "monitor", monitorRec.Id, "err", err)
+ continue
+ }
+ // Monitor intervals can exceed the aggregation window, so average
+ // any available records at every level and skip only empty windows.
+ if count == 0 {
+ continue
+ }
+
+ longerRecord := core.NewRecord(monitorStatsColl)
+ longerRecord.Set("system", monitorRec.System)
+ longerRecord.Set("monitor", monitorRec.Id)
+ longerRecord.Set("type", recordData.longerType)
+ longerRecord.Set("created", now.UnixMilli())
+ longerRecord.Set("res_min", stats.ResMin)
+ longerRecord.Set("res_max", stats.ResMax)
+ longerRecord.Set("total_count", stats.TotalCount)
+ longerRecord.Set("success_count", stats.SuccessCount)
+ longerRecord.Set("res_sum", stats.ResponseSum)
+ if err := txApp.SaveNoValidate(longerRecord); err != nil {
+ txApp.Logger().Error("failed to save monitor longer record", "err", err)
+ }
}
}
return nil
})
+ if err != nil {
+ rm.app.Logger().Error("failed to create longer records", "err", err)
+ }
+}
- // log.Println("finished creating longer records", "time (ms)", time.Since(start).Milliseconds())
+func getCreatedTimeField(collectionName string, period time.Time) any {
+ // network_monitor_stats stores created as unix timestamp in ms, not as a date string
+ if collectionName == "network_monitor_stats" {
+ return period.UnixMilli()
+ }
+ return period.Format(types.DefaultDateLayout)
}
// Calculate the average stats of a list of system_stats records without reflect
@@ -596,6 +678,36 @@ func AverageContainerStatsSlice(records [][]container.Stats) []container.Stats {
return result
}
+// AverageMonitorStats merges probe counts and response sums, preserving their
+// weights through every retention tier. Failed probes do not contribute latency.
+func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordType string, createdAfter int64) (monitor.Stats, int, error) {
+ var result struct {
+ monitor.Stats
+ Count int `db:"count"`
+ }
+ err := db.Select(
+ "COUNT(*) AS count",
+ "COALESCE(SUM(total_count), 0) AS total_count",
+ "COALESCE(SUM(success_count), 0) AS success_count",
+ "COALESCE(SUM(res_sum), 0) AS res_sum",
+ "COALESCE(MIN(CASE WHEN success_count > 0 THEN res_min END), 0) AS res_min",
+ "COALESCE(MAX(CASE WHEN success_count > 0 THEN res_max END), 0) AS res_max",
+ ).From("network_monitor_stats").Where(dbx.NewExp(
+ "monitor={:monitor} AND type={:type} AND created>{:created}",
+ dbx.Params{"monitor": monitorID, "type": recordType, "created": createdAfter},
+ )).One(&result)
+ if err != nil {
+ return monitor.Stats{}, 0, err
+ }
+ if result.SuccessCount > 0 {
+ result.ResAvg = twoDecimals(float64(result.ResponseSum) / float64(result.SuccessCount))
+ }
+ if result.TotalCount > 0 {
+ result.Loss = twoDecimals(float64(result.TotalCount-result.SuccessCount) * 100 / float64(result.TotalCount))
+ }
+ return result.Stats, result.Count, nil
+}
+
/* Round float to two decimals */
func twoDecimals(value float64) float64 {
return math.Round(value*100) / 100
diff --git a/internal/records/records_deletion.go b/internal/records/records_deletion.go
index 6632ded98..94bd0aaea 100644
--- a/internal/records/records_deletion.go
+++ b/internal/records/records_deletion.go
@@ -3,7 +3,6 @@ package records
import (
"fmt"
"log/slog"
- "strings"
"time"
"github.com/pocketbase/dbx"
@@ -60,7 +59,7 @@ func deleteOldAlertsHistory(app core.App, countToKeep, countBeforeDeletion int)
// Deletes system_stats records older than what is displayed in the UI
func deleteOldSystemStats(app core.App) error {
// Collections to process
- collections := [2]string{"system_stats", "container_stats"}
+ collections := [3]string{"system_stats", "container_stats", "network_monitor_stats"}
// Record types and their retention periods
type RecordDeletionData struct {
@@ -76,24 +75,17 @@ func deleteOldSystemStats(app core.App) error {
}
now := time.Now().UTC()
+ db := app.DB()
for _, collection := range collections {
- // Build the WHERE clause
- var conditionParts []string
- var params dbx.Params = make(map[string]any)
- for i := range recordData {
- rd := recordData[i]
- // Create parameterized condition for this record type
- dateParam := fmt.Sprintf("date%d", i)
- conditionParts = append(conditionParts, fmt.Sprintf("(type = '%s' AND created < {:%s})", rd.recordType, dateParam))
- params[dateParam] = now.Add(-rd.retention)
- }
- // Combine conditions with OR
- conditionStr := strings.Join(conditionParts, " OR ")
- // Construct and execute the full raw query
- rawQuery := fmt.Sprintf("DELETE FROM %s WHERE %s", collection, conditionStr)
- if _, err := app.DB().NewQuery(rawQuery).Bind(params).Execute(); err != nil {
- return fmt.Errorf("failed to delete from %s: %v", collection, err)
+ query := db.Delete(collection, dbx.NewExp("type={:type} AND created<{:created}"))
+ for _, rd := range recordData {
+ if _, err := query.Bind(dbx.Params{
+ "type": rd.recordType,
+ "created": getCreatedTimeField(collection, now.Add(-rd.retention)),
+ }).Execute(); err != nil {
+ return fmt.Errorf("failed to delete from %s: %v", collection, err)
+ }
}
}
return nil
diff --git a/internal/records/records_rollup_test.go b/internal/records/records_rollup_test.go
new file mode 100644
index 000000000..16f0076af
--- /dev/null
+++ b/internal/records/records_rollup_test.go
@@ -0,0 +1,86 @@
+//go:build testing
+
+package records_test
+
+import (
+ "testing"
+ "time"
+
+ "github.com/henrygd/beszel/internal/records"
+ "github.com/henrygd/beszel/internal/tests"
+ "github.com/pocketbase/dbx"
+ "github.com/pocketbase/pocketbase/tools/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLongerRecordsPreventDuplicates(t *testing.T) {
+ for _, collection := range []string{"system_stats", "container_stats", "network_monitor_stats"} {
+ for _, tier := range []struct {
+ shorter, longer string
+ count int
+ }{
+ {"10m", "20m", 2},
+ {"20m", "120m", 6},
+ {"120m", "480m", 4},
+ } {
+ t.Run(collection+"/"+tier.longer, func(t *testing.T) {
+ hub, err := tests.NewTestHub(t.TempDir())
+ require.NoError(t, err)
+ defer hub.Cleanup()
+
+ user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
+ require.NoError(t, err)
+ sys, err := tests.CreateRecord(hub, "systems", map[string]any{
+ "name": "rollup-system", "host": "localhost", "port": "45876",
+ "status": "up", "users": []string{user.Id},
+ })
+ require.NoError(t, err)
+
+ created := time.Now().UTC().Add(-time.Minute)
+ data := map[string]any{
+ "system": sys.Id, "type": tier.shorter,
+ "created": created.Format(types.DefaultDateLayout),
+ }
+ filter := dbx.HashExp{"system": sys.Id, "type": tier.longer}
+ switch collection {
+ case "system_stats":
+ data["stats"] = `{"cpu":10}`
+ case "container_stats":
+ data["stats"] = `[{"name":"test","cpu":10}]`
+ case "network_monitor_stats":
+ monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
+ "system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
+ "interval": 30, "enabled": true,
+ })
+ require.NoError(t, err)
+ data["monitor"] = monitor.Id
+ data["created"] = created.UnixMilli()
+ data["total_count"] = 1
+ data["success_count"] = 1
+ data["res_sum"] = 10
+ data["res_min"] = 10
+ data["res_max"] = 10
+ filter["monitor"] = monitor.Id
+ }
+ for range tier.count {
+ _, err := tests.CreateRecord(hub, collection, data)
+ require.NoError(t, err)
+ }
+
+ rm := records.NewRecordManager(hub)
+ rm.CreateLongerRecords()
+ first, err := hub.FindAllRecords(collection, filter)
+ require.NoError(t, err)
+ require.Len(t, first, 1)
+
+ // The shorter records remain eligible, but the existing longer
+ // record must prevent another rollup on a subsequent invocation.
+ rm.CreateLongerRecords()
+ second, err := hub.FindAllRecords(collection, filter)
+ require.NoError(t, err)
+ require.Len(t, second, 1)
+ require.Equal(t, first[0].Id, second[0].Id)
+ })
+ }
+ }
+}
diff --git a/internal/site/package.json b/internal/site/package.json
index a69c345cf..b30e14619 100644
--- a/internal/site/package.json
+++ b/internal/site/package.json
@@ -1,7 +1,7 @@
{
"name": "beszel",
"private": true,
- "version": "0.19.0",
+ "version": "0.20.0",
"type": "module",
"scripts": {
"dev": "vite --host",
@@ -74,4 +74,4 @@
"optionalDependencies": {
"@esbuild/linux-arm64": "^0.21.5"
}
-}
+}
\ No newline at end of file
diff --git a/internal/site/src/components/active-alerts.tsx b/internal/site/src/components/active-alerts.tsx
index 2c5019d90..1e71762ff 100644
--- a/internal/site/src/components/active-alerts.tsx
+++ b/internal/site/src/components/active-alerts.tsx
@@ -61,6 +61,8 @@ export const ActiveAlerts = () => {
<AlertDescription>
{info.triggeredDesc ? (
info.triggeredDesc()
+ ) : alert.name === "NetworkMonitorLoss" ? (
+ <Trans>One or more monitors exceed {alert.value}% loss</Trans>
) : alert.name === "Status" ? (
<Trans>Connection is down</Trans>
) : info.invert ? (
diff --git a/internal/site/src/components/alerts-history-columns.tsx b/internal/site/src/components/alerts-history-columns.tsx
index 887ae0b20..2ae9bd97e 100644
--- a/internal/site/src/components/alerts-history-columns.tsx
+++ b/internal/site/src/components/alerts-history-columns.tsx
@@ -30,7 +30,8 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [
accessorFn: (record) => {
const name = record.name
const info = alertInfo[name]
- return info?.name().replace("cpu", "CPU") || name
+ const label = info?.name().replace("cpu", "CPU") || name
+ return record.monitor_name ? `${label}: ${record.monitor_name}` : label
},
header: ({ column }) => (
<Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
diff --git a/internal/site/src/components/alerts/alerts-sheet.tsx b/internal/site/src/components/alerts/alerts-sheet.tsx
index e5efeeb2a..94e4e3b30 100644
--- a/internal/site/src/components/alerts/alerts-sheet.tsx
+++ b/internal/site/src/components/alerts/alerts-sheet.tsx
@@ -239,13 +239,13 @@ export function AlertContent({
/** Alerts that fire on first observation have no duration to configure */
const noDuration = alertData.noDuration === true
/** Binary alerts have no threshold to configure */
- const noThreshold = !!singleDescription || noDuration
+ const noThreshold = !!singleDescription || alertData.noThreshold === true
/** Whether enabling the alert reveals anything to configure */
const hasControls = !(noThreshold && noDuration)
const [checked, setChecked] = useState(global ? false : !!alert)
const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10))
- const [value, setValue] = useState(alert?.value || (noThreshold ? 0 : (alertData.start ?? 80)))
+ const [value, setValue] = useState(alert?.value ?? (noThreshold ? 0 : (alertData.start ?? 80)))
const Icon = alertData.icon
@@ -319,7 +319,7 @@ export function AlertContent({
<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground">
<Suspense fallback={<div className="h-10" />}>
{!noThreshold && (
- <div>
+ <div className={cn(noDuration && "col-span-full")}>
<p id={`v${name}`} className="text-sm block h-6">
{alertData.invert ? (
<Trans>
diff --git a/internal/site/src/components/charts/area-chart.tsx b/internal/site/src/components/charts/area-chart.tsx
index 052de9c7b..183600293 100644
--- a/internal/site/src/components/charts/area-chart.tsx
+++ b/internal/site/src/components/charts/area-chart.tsx
@@ -66,7 +66,7 @@ export default function AreaChartDefault({
}) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
- const sourceData = customData ?? chartData.systemStats
+ const sourceData = customData ?? chartData.systemStats ?? []
const [displayData, setDisplayData] = useState(sourceData)
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
@@ -111,6 +111,8 @@ export default function AreaChartDefault({
})
}, [areasKey, displayMaxToggled])
+ const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
+
return useMemo(() => {
if (displayData.length === 0) {
return null
@@ -146,7 +148,7 @@ export default function AreaChartDefault({
axisLine={false}
/>
)}
- {xAxis(chartData)}
+ {XAxis}
<ChartTooltip
animationEasing="ease-out"
animationDuration={150}
@@ -167,5 +169,5 @@ export default function AreaChartDefault({
</AreaChart>
</ChartContainer>
)
- }, [displayData, yAxisWidth, filter, Areas])
+ }, [displayData, yAxisWidth, filter, Areas, XAxis])
}
diff --git a/internal/site/src/components/charts/chart-time-select.tsx b/internal/site/src/components/charts/chart-time-select.tsx
index ab3473aa2..7ff7f9810 100644
--- a/internal/site/src/components/charts/chart-time-select.tsx
+++ b/internal/site/src/components/charts/chart-time-select.tsx
@@ -9,14 +9,21 @@ import { memo } from "react"
export default memo(function ChartTimeSelect({
className,
agentVersion,
+ chartTimeStore = $chartTime,
+ allowRealtime = true,
}: {
className?: string
agentVersion: SemVer
+ chartTimeStore?: typeof $chartTime
+ allowRealtime?: boolean
}) {
- const chartTime = useStore($chartTime)
+ const chartTime = useStore(chartTimeStore)
// remove chart times that are not supported by the system agent version
- const availableChartTimes = Object.entries(chartTimeData).filter(([_, { minVersion }]) => {
+ const availableChartTimes = Object.entries(chartTimeData).filter(([value, { minVersion }]) => {
+ if (value === "1m" && !allowRealtime) {
+ return false
+ }
if (!minVersion) {
return true
}
@@ -24,7 +31,7 @@ export default memo(function ChartTimeSelect({
})
return (
- <Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => $chartTime.set(value)}>
+ <Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => chartTimeStore.set(value)}>
<SelectTrigger className={cn(className, "relative ps-10 pe-5")}>
<HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
<SelectValue />
diff --git a/internal/site/src/components/charts/line-chart.tsx b/internal/site/src/components/charts/line-chart.tsx
index f60a341b8..97927526b 100644
--- a/internal/site/src/components/charts/line-chart.tsx
+++ b/internal/site/src/components/charts/line-chart.tsx
@@ -22,6 +22,10 @@ export type DataPoint<T = SystemStatsRecord> = {
order?: number
strokeOpacity?: number
activeDot?: boolean
+ dot?: boolean
+ /** Which Y axis this series plots against. Defaults to "left". */
+ yAxisId?: "left" | "right"
+ strokeDasharray?: string
}
export default function LineChartDefault({
@@ -30,9 +34,12 @@ export default function LineChartDefault({
max,
maxToggled,
tickFormatter,
+ tickFormatter2,
contentFormatter,
dataPoints,
domain,
+ domain2,
+ max2,
legend,
itemSorter,
showTotal = false,
@@ -41,18 +48,24 @@ export default function LineChartDefault({
filter,
truncate = false,
chartProps,
+ connectNulls,
}: {
chartData: ChartData
// biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData)
customData?: any[]
max?: number
+ max2?: number
maxToggled?: boolean
tickFormatter: (value: number, index: number) => string
+ /** Tick formatter for the right ("right"-yAxisId) axis, when any dataPoint uses it. */
+ tickFormatter2?: (value: number, index: number) => string
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
contentFormatter: (item: any, key: string) => ReactNode
// biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic types
dataPoints?: DataPoint<any>[]
domain?: AxisDomain
+ /** Domain for the right axis, when any dataPoint uses it. */
+ domain2?: AxisDomain
legend?: boolean
showTotal?: boolean
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
@@ -62,10 +75,15 @@ export default function LineChartDefault({
filter?: string
truncate?: boolean
chartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin">
+ connectNulls?: boolean
}) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
+ const hasRightAxis = !!dataPoints?.some((dp) => dp.yAxisId === "right")
+ // fixed width for the secondary axis rather than measured, since its labels (e.g. loss %) are short
+ // and predictable, and this avoids depending on a second async width-measurement pass to settle
+ const rightAxisWidth = 38
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
- const sourceData = customData ?? chartData.systemStats
+ const sourceData = customData ?? chartData.systemStats ?? []
const [displayData, setDisplayData] = useState(sourceData)
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
@@ -83,7 +101,9 @@ export default function LineChartDefault({
}, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData])
// Use a stable key derived from data point identities and visual properties
- const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity ?? ""}`).join("\0")
+ const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity}${d.dot}${d.yAxisId}${d.strokeDasharray}`).join("\0")
+
+ const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
const Lines = useMemo(() => {
return dataPoints?.map((dataPoint, i) => {
@@ -94,17 +114,20 @@ export default function LineChartDefault({
return (
<Line
key={dataPoint.label}
+ yAxisId={dataPoint.yAxisId ?? "left"}
dataKey={dataPoint.dataKey}
name={dataPoint.label}
type="monotoneX"
- dot={false}
+ dot={dataPoint.dot || false}
strokeWidth={1.5}
stroke={color}
strokeOpacity={dataPoint.strokeOpacity}
+ strokeDasharray={dataPoint.strokeDasharray}
isAnimationActive={false}
// stackId={dataPoint.stackId}
order={dataPoint.order || i}
activeDot={dataPoint.activeDot ?? true}
+ connectNulls={connectNulls}
/>
)
})
@@ -135,6 +158,7 @@ export default function LineChartDefault({
<CartesianGrid vertical={false} />
{!hideYAxis && (
<YAxis
+ yAxisId="left"
direction="ltr"
orientation={chartData.orientation}
className="tracking-tighter"
@@ -145,7 +169,20 @@ export default function LineChartDefault({
axisLine={false}
/>
)}
- {xAxis(chartData)}
+ {!hideYAxis && hasRightAxis && (
+ <YAxis
+ yAxisId="right"
+ direction="ltr"
+ orientation={chartData.orientation === "left" ? "right" : "left"}
+ className="tracking-tighter"
+ width={rightAxisWidth}
+ domain={domain2 ?? [0, max2 ?? "auto"]}
+ tickFormatter={tickFormatter2 ?? tickFormatter}
+ tickLine={false}
+ axisLine={false}
+ />
+ )}
+ {XAxis}
<ChartTooltip
animationEasing="ease-out"
animationDuration={150}
@@ -166,5 +203,5 @@ export default function LineChartDefault({
</LineChart>
</ChartContainer>
)
- }, [displayData, yAxisWidth, filter, Lines])
+ }, [displayData, yAxisWidth, hasRightAxis, filter, Lines, XAxis])
}
diff --git a/internal/site/src/components/command-palette.tsx b/internal/site/src/components/command-palette.tsx
index a0c73818a..71080a114 100644
--- a/internal/site/src/components/command-palette.tsx
+++ b/internal/site/src/components/command-palette.tsx
@@ -11,6 +11,7 @@ import {
HardDriveIcon,
LogsIcon,
MailIcon,
+ NetworkIcon,
Server,
ServerIcon,
SettingsIcon,
@@ -122,6 +123,20 @@ export default memo(function CommandPalette({ open, setOpen }: { open: boolean;
<Trans>Page</Trans>
</CommandShortcut>
</CommandItem>
+ <CommandItem
+ onSelect={() => {
+ navigate(getPagePath($router, "monitors"))
+ setOpen(false)
+ }}
+ >
+ <NetworkIcon className="me-2 size-4" />
+ <span>
+ <Trans>Network Monitors</Trans>
+ </span>
+ <CommandShortcut>
+ <Trans>Page</Trans>
+ </CommandShortcut>
+ </CommandItem>
<CommandItem
onSelect={() => {
navigate(getPagePath($router, "settings", { name: "general" }))
diff --git a/internal/site/src/components/containers-table/containers-table-columns.tsx b/internal/site/src/components/containers-table/containers-table-columns.tsx
index 89ec6eea6..d3dfbd6a8 100644
--- a/internal/site/src/components/containers-table/containers-table-columns.tsx
+++ b/internal/site/src/components/containers-table/containers-table-columns.tsx
@@ -16,7 +16,7 @@ import {
import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons"
import { Badge } from "../ui/badge"
import { t } from "@lingui/core/macro"
-import { $allSystemsById, $longestSystemNameLen } from "@/lib/stores"
+import { $allSystemsById, $longestSystemName } from "@/lib/stores"
import { useStore } from "@nanostores/react"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
@@ -64,10 +64,13 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById)
- const longestName = useStore($longestSystemNameLen)
+ const longestName = useStore($longestSystemName)
return (
- <div className="ms-1 max-w-40 truncate" style={{ width: `${longestName / 1.05}ch` }}>
- {allSystems[getValue() as string]?.name ?? ""}
+ <div className="ms-1 relative w-fit max-w-40">
+ <span className="invisible block whitespace-nowrap" aria-hidden="true">
+ {longestName}
+ </span>
+ <span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
</div>
)
},
diff --git a/internal/site/src/components/navbar.tsx b/internal/site/src/components/navbar.tsx
index 956338f16..8f967b2ae 100644
--- a/internal/site/src/components/navbar.tsx
+++ b/internal/site/src/components/navbar.tsx
@@ -8,6 +8,7 @@ import {
LogOutIcon,
LogsIcon,
MenuIcon,
+ NetworkIcon,
PlusIcon,
SearchIcon,
ServerIcon,
@@ -108,6 +109,13 @@ export default function Navbar() {
<HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
<span>S.M.A.R.T.</span>
</DropdownMenuItem>
+ <DropdownMenuItem
+ onClick={() => navigate(getPagePath($router, "monitors"))}
+ className="flex items-center"
+ >
+ <NetworkIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
+ <Trans>Network Monitors</Trans>
+ </DropdownMenuItem>
<DropdownMenuItem
onClick={() => navigate(getPagePath($router, "settings", { name: "general" }))}
className="flex items-center"
@@ -179,6 +187,21 @@ export default function Navbar() {
</TooltipTrigger>
<TooltipContent>S.M.A.R.T.</TooltipContent>
</Tooltip>
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <Link
+ href={getPagePath($router, "monitors")}
+ className={cn("hidden md:grid", buttonVariants({ variant: "ghost", size: "icon" }))}
+ aria-label="Network Monitors"
+ onMouseEnter={() => import("@/components/routes/monitors")}
+ >
+ <NetworkIcon className="h-[1.2rem] w-[1.2rem]" strokeWidth={1.5} />
+ </Link>
+ </TooltipTrigger>
+ <TooltipContent>
+ <Trans>Network Monitors</Trans>
+ </TooltipContent>
+ </Tooltip>
<ModeToggle />
<Tooltip>
<TooltipTrigger asChild>
diff --git a/internal/site/src/components/network-monitors-table/monitor-dialog.tsx b/internal/site/src/components/network-monitors-table/monitor-dialog.tsx
new file mode 100644
index 000000000..dcbb27ae3
--- /dev/null
+++ b/internal/site/src/components/network-monitors-table/monitor-dialog.tsx
@@ -0,0 +1,778 @@
+import { useCallback, useEffect, useRef, useState } from "react"
+import { Trans, useLingui } from "@lingui/react/macro"
+import { useStore } from "@nanostores/react"
+import { pb } from "@/lib/api"
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogFooter,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog"
+import {
+ DropdownMenu,
+ DropdownMenuCheckboxItem,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu"
+import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet"
+import { Button } from "@/components/ui/button"
+import { Input } from "@/components/ui/input"
+import { Label } from "@/components/ui/label"
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
+import { Textarea } from "@/components/ui/textarea"
+import { ChevronDownIcon, ListIcon, SearchIcon, ServerIcon } from "lucide-react"
+import { useToast } from "@/components/ui/use-toast"
+import { $systems } from "@/lib/stores"
+import { cn, supportsNetworkMonitors } from "@/lib/utils"
+import type { NetworkMonitorRecord } from "@/types"
+import * as v from "valibot"
+
+type MonitorProtocol = "icmp" | "tcp" | "http" | "dns"
+
+type MonitorValues = {
+ system: string
+ target: string
+ protocol: MonitorProtocol
+ port: number
+ interval: string
+}
+
+type NormalizedMonitorValues = Omit<MonitorValues, "system" | "interval"> & {
+ interval: number
+}
+
+type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval">
+
+const defaultInterval = 30
+
+const MonitorProtocolSchema = v.picklist(["icmp", "tcp", "http", "dns"])
+
+const MonitorIntervalSchema = v.pipe(v.string(), v.toNumber(), v.minValue(1), v.maxValue(3600))
+
+// Both the single-monitor form and the bulk importer flow through this schema so
+// defaults and HTTP target normalization stay in one place.
+const NormalizedMonitorValuesSchema = v.pipe(
+ v.object({
+ target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
+ protocol: MonitorProtocolSchema,
+ port: v.number(),
+ interval: MonitorIntervalSchema,
+ }),
+ v.transform((input): NormalizedMonitorValues => {
+ let { protocol, port } = input
+ let httpTarget = input.target
+ if (protocol === "icmp" || protocol === "http" || protocol === "dns") {
+ if (protocol === "http") {
+ httpTarget = normalizeHttpTarget(input.target, port)
+ }
+ port = 0
+ } else if (protocol === "tcp" && !port) {
+ port = 443
+ }
+ return {
+ // HTTP monitors may be entered as bare hostnames, so normalize them to a
+ // scheme-bearing URL before the payload is sent to PocketBase.
+ target: protocol === "http" ? httpTarget : input.target,
+ protocol,
+ port,
+ interval: input.interval,
+ }
+ }),
+ v.forward(
+ v.check((input) => {
+ if (input.protocol === "icmp" || input.protocol === "http" || input.protocol === "dns") {
+ return input.port === 0
+ }
+
+ return Number.isInteger(input.port) && input.port >= 1 && input.port <= 65535
+ }, "Port must be between 1 and 65535"),
+ ["port"]
+ )
+)
+
+// Bulk parsing only trims raw CSV fields. Inference, defaults, and protocol-
+// specific validation still go through the shared normalization schema above.
+const BulkMonitorSchema = v.object({
+ target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
+ protocol: v.optional(v.pipe(v.string(), v.trim())),
+ port: v.optional(v.pipe(v.string(), v.trim())),
+ interval: v.optional(v.pipe(v.string(), v.trim())),
+})
+
+function normalizeHttpTarget(target: string, port = 0) {
+ const useExplicitPort = port > 0 && port !== 80 && port !== 443
+ const hasOriginOnlyTarget = /^https?:\/\/[^/?#]+$/i.test(target)
+ if (!/^https?:\/\//i.test(target)) {
+ const scheme = port === 80 ? "http" : "https"
+ return `${scheme}://${target}${useExplicitPort ? `:${port}` : ""}`
+ }
+
+ let parsedUrl: URL
+ try {
+ parsedUrl = new URL(target)
+ } catch {
+ return target
+ }
+
+ if (!parsedUrl.port && useExplicitPort) {
+ parsedUrl.port = `${port}`
+ }
+
+ // avoid converting "http://localhost:8090" to "http://localhost:8090/" - keep the original formatting if the URL is just an origin
+ if (hasOriginOnlyTarget && parsedUrl.pathname === "/" && !parsedUrl.search && !parsedUrl.hash) {
+ return parsedUrl.origin
+ }
+
+ return parsedUrl.toString()
+}
+
+function trimTrailingEmptyFields(fields: string[]) {
+ let lastValueIndex = fields.length - 1
+ while (lastValueIndex > 0 && !fields[lastValueIndex]) {
+ lastValueIndex--
+ }
+ return fields.slice(0, lastValueIndex + 1)
+}
+
+function buildMonitorPayload(values: MonitorValues, enabled = true) {
+ const normalizedValues = v.safeParse(NormalizedMonitorValuesSchema, values)
+ if (!normalizedValues.success) {
+ throw new Error(normalizedValues.issues[0]?.message || "Invalid monitor")
+ }
+
+ const payload = {
+ system: values.system,
+ enabled,
+ ...normalizedValues.output,
+ }
+
+ return payload
+}
+
+type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port">
+function getMonitorIdentityKey({ system, target, protocol, port }: MonitorIdentity) {
+ return `${system}${target}${protocol}${port}`
+}
+
+function parseBulkMonitorLine(line: string, lineNumber: number, system: string) {
+ const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = ""] = line.split(",")
+ const parsed = v.safeParse(BulkMonitorSchema, {
+ target: rawTarget,
+ protocol: rawProtocol,
+ port: rawPort,
+ interval: rawInterval,
+ })
+ if (!parsed.success) {
+ throw new Error(`Line ${lineNumber}: ${parsed.issues[0]?.message || "invalid monitor entry"}`)
+ }
+ const protocol = (parsed.output.protocol?.toLowerCase() ||
+ (/^https?:\/\//i.test(parsed.output.target) ? "http" : "icmp")) as MonitorProtocol
+
+ return buildMonitorPayload({
+ system,
+ target: parsed.output.target,
+ protocol,
+ port: parsed.output.port ? Number(parsed.output.port) : 0,
+ interval: parsed.output.interval || `${defaultInterval}`,
+ })
+}
+
+export function formatBulkMonitorLine(monitor: BulkMonitorLineSource) {
+ const port = monitor.protocol !== "tcp" || monitor.port === 443 ? "" : `${monitor.port}`
+ const interval = monitor.interval === defaultInterval ? "" : `${monitor.interval}`
+ return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval]).join(",")
+}
+
+function SystemMultiSelect({
+ id,
+ selectedSystemIds,
+ onChange,
+ disabled,
+ className,
+}: {
+ id: string
+ selectedSystemIds: Set<string>
+ onChange: (ids: Set<string>) => void
+ disabled?: boolean
+ className?: string
+}) {
+ const systems = useStore($systems)
+ const { t } = useLingui()
+ const [search, setSearch] = useState("")
+ const searchRef = useRef<HTMLInputElement>(null)
+ const focusSearchOnMount = useCallback((node: HTMLInputElement | null) => {
+ searchRef.current = node
+ if (!node) return
+ // Focus after the menu has completed its own initial focus handling.
+ const frame = requestAnimationFrame(() => node.focus())
+ return () => cancelAnimationFrame(frame)
+ }, [])
+ const contentRef = useRef<HTMLDivElement>(null)
+ const query = search.trim().toLocaleLowerCase()
+ const filteredSystems = systems.filter(
+ (system) => supportsNetworkMonitors(system) && system.name.toLocaleLowerCase().includes(query)
+ )
+ const allSelected = filteredSystems.every((system) => selectedSystemIds.has(system.id))
+ const anySelected = filteredSystems.some((system) => selectedSystemIds.has(system.id))
+
+ const selectFiltered = (selected: boolean) => {
+ const next = new Set(selectedSystemIds)
+ for (const system of filteredSystems) {
+ if (selected) next.add(system.id)
+ else next.delete(system.id)
+ }
+ onChange(next)
+ }
+ return (
+ <DropdownMenu onOpenChange={() => setSearch("")}>
+ <DropdownMenuTrigger asChild>
+ <Button
+ id={id}
+ disabled={disabled}
+ type="button"
+ variant="outline"
+ className={cn("relative w-full min-w-0 ps-10 pe-10 justify-start font-normal text-start", className)}
+ >
+ <ServerIcon className="size-3.5 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
+ <span className="truncate">
+ {selectedSystemIds.size === 0
+ ? t`Select systems`
+ : selectedSystemIds.size === 1
+ ? systems.find((s) => selectedSystemIds.has(s.id))?.name
+ : t`${selectedSystemIds.size} systems selected`}
+ </span>
+ <ChevronDownIcon className="size-4 absolute end-4 top-1/2 -translate-y-1/2 opacity-50" />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent
+ ref={contentRef}
+ onKeyDown={(event) => {
+ if (event.key === "Tab") {
+ event.preventDefault()
+ searchRef.current?.focus()
+ }
+ }}
+ align="start"
+ className="w-[var(--radix-dropdown-menu-trigger-width)] max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] flex flex-col overflow-hidden"
+ >
+ <div className="shrink-0 border-b mb-1">
+ <div className="flex items-center gap-2 px-2.5">
+ <SearchIcon aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" />
+ <Input
+ ref={focusSearchOnMount}
+ value={search}
+ onChange={(event) => setSearch(event.target.value)}
+ placeholder={t`Search systems`}
+ aria-label={t`Search systems`}
+ className="h-10 min-w-0 rounded-none border-0 bg-transparent px-0 shadow-none focus-visible:ring-0 focus-visible:ring-offset-0"
+ onKeyDown={(event) => {
+ if (event.key === "Escape") return
+ // Keep menu typeahead and form submission from consuming search input.
+ event.stopPropagation()
+ if (event.key === "Enter") event.preventDefault()
+ if (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Tab") {
+ event.preventDefault()
+ const items = contentRef.current?.querySelectorAll<HTMLElement>(
+ '[role^="menuitem"]:not([data-disabled])'
+ )
+ const index = event.key === "ArrowUp" || event.shiftKey ? (items?.length ?? 1) - 1 : 0
+ items?.[index]?.focus()
+ }
+ }}
+ />
+ </div>
+ <div className="flex flex-wrap items-center justify-between gap-x-3 gap-y-1 px-1 pb-1">
+ <div className="flex items-center">
+ <DropdownMenuItem
+ className="px-1.5 py-1 text-xs text-muted-foreground"
+ disabled={!filteredSystems.length || allSelected}
+ onSelect={(event) => {
+ event.preventDefault()
+ selectFiltered(true)
+ }}
+ >
+ {query ? <Trans>Select matches</Trans> : <Trans>Select all</Trans>}
+ </DropdownMenuItem>
+ <span aria-hidden="true" className="text-xs text-muted-foreground/50">
+ ·
+ </span>
+ <DropdownMenuItem
+ className="px-1.5 py-1 text-xs text-muted-foreground"
+ disabled={!anySelected}
+ onSelect={(event) => {
+ event.preventDefault()
+ selectFiltered(false)
+ }}
+ >
+ {query ? <Trans>Clear matches</Trans> : <Trans>Clear all</Trans>}
+ </DropdownMenuItem>
+ </div>
+ <span className="px-1.5 text-xs tabular-nums text-muted-foreground">
+ {t`${selectedSystemIds.size} selected`}
+ </span>
+ </div>
+ </div>
+ <div className="min-h-0 overflow-y-auto">
+ {filteredSystems.length === 0 && (
+ <output className="block px-2.5 py-3 text-sm text-muted-foreground">
+ <Trans>No systems found.</Trans>
+ </output>
+ )}
+ {filteredSystems.map((sys) => (
+ <DropdownMenuCheckboxItem
+ key={sys.id}
+ checked={selectedSystemIds.has(sys.id)}
+ onSelect={(event) => event.preventDefault()}
+ onCheckedChange={(checked) => {
+ const next = new Set(selectedSystemIds)
+ if (checked) next.add(sys.id)
+ else next.delete(sys.id)
+ onChange(next)
+ }}
+ className="group min-w-0 gap-2.5 py-2 ps-2.5"
+ indicatorClassName="static size-4 shrink-0 rounded border border-input group-data-[state=checked]:border-primary group-data-[state=checked]:bg-primary group-data-[state=checked]:text-primary-foreground [&_svg]:size-3"
+ >
+ <span className="truncate">{sys.name}</span>
+ </DropdownMenuCheckboxItem>
+ ))}
+ </div>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ )
+}
+
+export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; monitors: NetworkMonitorRecord[] }) {
+ const [open, setOpen] = useState(false)
+ const [bulkOpen, setBulkOpen] = useState(false)
+ const [bulkInput, setBulkInput] = useState("")
+ const [bulkLoading, setBulkLoading] = useState(false)
+ const [bulkSelectedSystemIds, setBulkSelectedSystemIds] = useState<Set<string>>(new Set())
+ const bulkFormRef = useRef<HTMLFormElement>(null)
+ const { toast } = useToast()
+ const { t } = useLingui()
+
+ const resetBulkForm = () => {
+ setBulkInput("")
+ }
+
+ const openBulkAdd = (selectedSystemIds?: Set<string>) => {
+ if (!systemId && selectedSystemIds) {
+ setBulkSelectedSystemIds(new Set(selectedSystemIds))
+ }
+ setOpen(false)
+ setBulkOpen(true)
+ }
+
+ const openAdd = () => {
+ setBulkOpen(false)
+ setOpen(true)
+ }
+
+ async function handleBulkSubmit(e: React.FormEvent) {
+ e.preventDefault()
+ setBulkLoading(true)
+ let closedForSubmit = false
+
+ try {
+ const targetSystems = systemId ? [systemId] : Array.from(bulkSelectedSystemIds)
+ if (!targetSystems.length) {
+ throw new Error("Select at least one system.")
+ }
+ const rawLines = bulkInput.split(/\r?\n/).filter((line) => line.trim())
+ if (!rawLines.length) {
+ throw new Error("Enter at least one monitor.")
+ }
+
+ let totalCreated = 0
+ closedForSubmit = true
+
+ for (const system of targetSystems) {
+ const payloads = rawLines.map((line, index) => parseBulkMonitorLine(line, index + 1, system))
+ const existingMonitorKeys = new Set(
+ monitors.filter((monitor) => monitor.system === system).map((monitor) => getMonitorIdentityKey(monitor))
+ )
+ const newPayloads: typeof payloads = []
+
+ for (const payload of payloads) {
+ const monitorKey = getMonitorIdentityKey(payload)
+ if (existingMonitorKeys.has(monitorKey)) {
+ continue
+ }
+ existingMonitorKeys.add(monitorKey)
+ newPayloads.push(payload)
+ }
+
+ if (!newPayloads.length) continue
+
+ let batch = pb.createBatch()
+ let inBatch = 0
+ for (const payload of newPayloads) {
+ batch.collection("network_monitors").create(payload)
+ inBatch++
+ if (inBatch > 20) {
+ await batch.send()
+ batch = pb.createBatch()
+ inBatch = 0
+ }
+ }
+ if (inBatch) {
+ await batch.send()
+ }
+ totalCreated += newPayloads.length
+ }
+
+ if (!totalCreated) {
+ throw new Error("No new monitors. All entries already exist.")
+ }
+
+ resetBulkForm()
+ toast({ title: t`Monitors created`, description: `${totalCreated} monitor(s) added.` })
+ } catch (err: unknown) {
+ if (closedForSubmit) {
+ setBulkOpen(true)
+ }
+ toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
+ } finally {
+ setBulkLoading(false)
+ }
+ }
+
+ return (
+ <>
+ <div className="flex gap-0 rounded-lg">
+ <Button variant="outline" onClick={openAdd} className="rounded-e-none grow">
+ {/* <PlusIcon className="size-4 me-1" /> */}
+ <Trans>Add {{ foo: t`Monitor` }}</Trans>
+ </Button>
+ <div className="w-px h-full bg-muted"></div>
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <Button variant="outline" className="px-2 rounded-s-none border-s-0" aria-label={t`More monitor actions`}>
+ <ChevronDownIcon className="size-4" />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="end">
+ <DropdownMenuItem onClick={() => openBulkAdd()}>
+ <ListIcon className="size-4 me-2" />
+ <Trans>Bulk Add</Trans>
+ </DropdownMenuItem>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ </div>
+ <Dialog
+ open={open}
+ onOpenChange={(nextOpen) => {
+ setOpen(nextOpen)
+ }}
+ >
+ <MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} onOpenBulkAdd={openBulkAdd} />
+ </Dialog>
+
+ <Sheet
+ open={bulkOpen}
+ onOpenChange={(nextOpen) => {
+ setBulkOpen(nextOpen)
+ if (!nextOpen) {
+ resetBulkForm()
+ }
+ }}
+ >
+ <SheetContent className="w-full sm:max-w-xl gap-0">
+ <SheetHeader className="border-b">
+ <SheetTitle>
+ <Trans>Bulk Add {{ foo: t`Network Monitors` }}</Trans>
+ </SheetTitle>
+ <SheetDescription>target[,protocol[,port[,interval]]]</SheetDescription>
+ </SheetHeader>
+ <form ref={bulkFormRef} onSubmit={handleBulkSubmit} className="flex h-full flex-col overflow-hidden">
+ <div className="flex-1 flex flex-col space-y-4 overflow-auto p-4">
+ {!systemId && (
+ <div className="grid gap-2">
+ <Label htmlFor="bulk-monitor-systems" className="sr-only">
+ <Trans>Systems</Trans>
+ </Label>
+ <SystemMultiSelect
+ id="bulk-monitor-systems"
+ selectedSystemIds={bulkSelectedSystemIds}
+ onChange={setBulkSelectedSystemIds}
+ disabled={bulkLoading}
+ className="bg-card"
+ />
+ </div>
+ )}
+ <div className="grow flex flex-col gap-2">
+ <Label htmlFor="bulk-monitors" className="sr-only">
+ Entries
+ </Label>
+ <Textarea
+ id="bulk-monitors"
+ value={bulkInput}
+ onChange={(e) => setBulkInput(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {
+ e.preventDefault()
+ bulkFormRef.current?.requestSubmit()
+ }
+ }}
+ className="font-mono grow text-sm bg-card"
+ placeholder={["1.1.1.1", "example.com,tcp", "https://example.com,http,,60"].join("\n")}
+ required
+ />
+ <p className="text-xs text-muted-foreground">target[,protocol[,port[,interval]]]</p>
+ </div>
+ </div>
+ <SheetFooter className="border-t">
+ <Button type="submit" disabled={bulkLoading || (!systemId && !bulkSelectedSystemIds.size)}>
+ <Trans>Add {{ foo: t`Network Monitors` }}</Trans>
+ </Button>
+ </SheetFooter>
+ </form>
+ </SheetContent>
+ </Sheet>
+ </>
+ )
+}
+
+export function EditMonitorDialog({
+ open,
+ setOpen,
+ systemId,
+ monitor,
+}: {
+ open: boolean
+ setOpen: (open: boolean) => void
+ systemId?: string
+ monitor?: NetworkMonitorRecord
+}) {
+ const hasOpened = useRef(false)
+ if (!monitor && !hasOpened.current) {
+ return null
+ }
+ hasOpened.current = true
+ return (
+ <Dialog open={open} onOpenChange={setOpen}>
+ <MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} monitor={monitor} />
+ </Dialog>
+ )
+}
+
+function MonitorDialogContent({
+ open,
+ setOpen,
+ systemId,
+ monitor,
+ onOpenBulkAdd,
+}: {
+ open: boolean
+ setOpen: (open: boolean) => void
+ systemId?: string
+ monitor?: NetworkMonitorRecord
+ onOpenBulkAdd?: (selectedSystemIds: Set<string>) => void
+}) {
+ const [protocol, setProtocol] = useState<MonitorProtocol>(monitor?.protocol ?? "icmp")
+ const [target, setTarget] = useState(monitor?.target ?? "")
+ const [port, setPort] = useState(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
+ const [monitorInterval, setMonitorInterval] = useState(String(monitor?.interval ?? defaultInterval))
+ const [loading, setLoading] = useState(false)
+ const [selectedSystemId, setSelectedSystemId] = useState(monitor?.system ?? "")
+ const [selectedSystemIds, setSelectedSystemIds] = useState<Set<string>>(new Set())
+ const systems = useStore($systems)
+ const { toast } = useToast()
+ const { t } = useLingui()
+ const isEditing = !!monitor
+
+ // When the dialog is opened, initialize form fields with monitor values (if editing) or defaults (if adding).
+ useEffect(() => {
+ if (!open) {
+ return
+ }
+
+ setProtocol(monitor?.protocol ?? "icmp")
+ setTarget(monitor?.target ?? "")
+ setPort(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
+ setMonitorInterval(String(monitor?.interval ?? defaultInterval))
+ setSelectedSystemId(monitor?.system ?? "")
+ setSelectedSystemIds(new Set())
+ setLoading(false)
+ }, [open, monitor])
+
+ async function handleSubmit(e: React.FormEvent) {
+ e.preventDefault()
+ setLoading(true)
+
+ const targetSystems = systemId ? [systemId] : monitor ? [selectedSystemId] : Array.from(selectedSystemIds)
+ const remainingSystemIds = new Set(targetSystems)
+ try {
+ if (!targetSystems.length || !targetSystems[0]) throw new Error("Select at least one system.")
+ const payload = buildMonitorPayload(
+ {
+ system: targetSystems[0],
+ target,
+ protocol,
+ port: protocol === "tcp" ? Number(port) : 0,
+ interval: monitorInterval,
+ },
+ monitor ? monitor.enabled : true
+ )
+ if (monitor) {
+ await pb.collection("network_monitors").update(monitor.id, payload)
+ } else {
+ for (const system of targetSystems) {
+ await pb.collection("network_monitors").create({ ...payload, system })
+ remainingSystemIds.delete(system)
+ }
+ }
+ setOpen(false)
+ } catch (err: unknown) {
+ if (!monitor && !systemId) {
+ // Retain only unfinished systems so retrying cannot duplicate successful creates.
+ setSelectedSystemIds(remainingSystemIds)
+ }
+ toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
+ } finally {
+ setLoading(false)
+ }
+ }
+
+ return (
+ <DialogContent className="max-w-md">
+ <DialogHeader>
+ <DialogTitle>
+ {isEditing ? (
+ <Trans>Edit {{ foo: t`Network Monitor` }}</Trans>
+ ) : (
+ <Trans>Add {{ foo: t`Network Monitor` }}</Trans>
+ )}
+ </DialogTitle>
+ <DialogDescription>
+ <Trans>Configure response monitoring from this agent.</Trans>
+ </DialogDescription>
+ </DialogHeader>
+ <form onSubmit={handleSubmit} className="grid gap-4 tabular-nums">
+ {!systemId && !isEditing && (
+ <div className="grid gap-2">
+ <Label htmlFor="monitor-systems">
+ <Trans>Systems</Trans>
+ </Label>
+ <SystemMultiSelect
+ id="monitor-systems"
+ selectedSystemIds={selectedSystemIds}
+ onChange={setSelectedSystemIds}
+ disabled={loading}
+ />
+ </div>
+ )}
+ {!systemId && isEditing && (
+ <div className="grid gap-2">
+ <Label>
+ <Trans>System</Trans>
+ </Label>
+ <Select value={selectedSystemId} onValueChange={setSelectedSystemId} required>
+ <SelectTrigger>
+ <SelectValue placeholder={t`Select a system`} />
+ </SelectTrigger>
+ <SelectContent>
+ {systems
+ .filter((sys) => sys.id === monitor?.system || supportsNetworkMonitors(sys))
+ .map((sys) => (
+ <SelectItem key={sys.id} value={sys.id}>
+ {sys.name}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+ )}
+ <div className="grid gap-2">
+ <Label>
+ <Trans>Target</Trans>
+ </Label>
+ <Input
+ value={target}
+ onChange={(e) => setTarget(e.target.value)}
+ placeholder={protocol === "http" ? "http://localhost:8090" : "1.1.1.1"}
+ required
+ />
+ </div>
+ <div className="grid gap-2">
+ <Label>
+ <Trans>Protocol</Trans>
+ </Label>
+
+ <Select value={protocol} onValueChange={(value) => setProtocol(value as MonitorProtocol)}>
+ <SelectTrigger>
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="icmp">ICMP</SelectItem>
+ <SelectItem value="tcp">TCP</SelectItem>
+ <SelectItem value="http">HTTP</SelectItem>
+ <SelectItem value="dns">DNS</SelectItem>
+ </SelectContent>
+ </Select>
+ </div>
+ {protocol === "tcp" && (
+ <div className="grid gap-2">
+ <Label>
+ <Trans>Port</Trans>
+ </Label>
+ <Input
+ type="number"
+ value={port}
+ onChange={(e) => setPort(e.target.value)}
+ placeholder="443"
+ min={1}
+ max={65535}
+ />
+ </div>
+ )}
+ <div className="grid gap-2">
+ <Label>
+ <Trans>Interval (seconds)</Trans>
+ </Label>
+ <Input
+ type="number"
+ value={monitorInterval}
+ onChange={(e) => setMonitorInterval(e.target.value)}
+ min={1}
+ max={3600}
+ required
+ />
+ </div>
+ <DialogFooter>
+ {!isEditing && onOpenBulkAdd && (
+ <Button
+ type="button"
+ variant="outline"
+ onClick={() => onOpenBulkAdd(selectedSystemIds)}
+ disabled={loading}
+ className="me-auto"
+ >
+ <ListIcon className="size-4 me-2" />
+ <Trans>Bulk Add</Trans>
+ </Button>
+ )}
+ <Button
+ type="submit"
+ disabled={loading || (!systemId && (isEditing ? !selectedSystemId : !selectedSystemIds.size))}
+ >
+ {loading ? (
+ isEditing ? (
+ <Trans>Saving...</Trans>
+ ) : (
+ <Trans>Creating...</Trans>
+ )
+ ) : isEditing ? (
+ <Trans>Save {{ foo: t`Monitor` }}</Trans>
+ ) : (
+ <Trans>Add {{ foo: t`Monitor` }}</Trans>
+ )}
+ </Button>
+ </DialogFooter>
+ </form>
+ </DialogContent>
+ )
+}
diff --git a/internal/site/src/components/network-monitors-table/network-monitors-columns.tsx b/internal/site/src/components/network-monitors-table/network-monitors-columns.tsx
new file mode 100644
index 000000000..921b30f0b
--- /dev/null
+++ b/internal/site/src/components/network-monitors-table/network-monitors-columns.tsx
@@ -0,0 +1,411 @@
+import type { CellContext, Column, ColumnDef } from "@tanstack/react-table"
+import { Button } from "@/components/ui/button"
+import { cn, copyToClipboard, decimalString, formatMicroseconds, hourWithSeconds } from "@/lib/utils"
+import {
+ GlobeIcon,
+ TimerIcon,
+ WifiOffIcon,
+ Trash2Icon,
+ ArrowLeftRightIcon,
+ MoreHorizontalIcon,
+ ServerIcon,
+ ClockIcon,
+ RefreshCwIcon,
+ PenBoxIcon,
+ PauseCircleIcon,
+ PlayCircleIcon,
+ CopyIcon,
+ CopyPlusIcon,
+} from "lucide-react"
+import { t } from "@lingui/core/macro"
+import type { NetworkMonitorRecord, SystemRecord } from "@/types"
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuSeparator,
+ DropdownMenuSub,
+ DropdownMenuSubContent,
+ DropdownMenuSubTrigger,
+ DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu"
+import { Trans } from "@lingui/react/macro"
+import { $allSystemsById, $longestSystemName } from "@/lib/stores"
+import { useStore } from "@nanostores/react"
+import { SystemStatus } from "@/lib/enums"
+import { Checkbox } from "@/components/ui/checkbox"
+import { useMemo } from "react"
+import { formatBulkMonitorLine } from "@/components/network-monitors-table/monitor-dialog"
+import { Badge } from "../ui/badge"
+import { getMonitorTarget } from "@/lib/network-monitor-utils"
+import { pb } from "@/lib/api"
+
+const protocolColors: Record<string, string> = {
+ icmp: "bg-blue-500/15! text-blue-600 dark:text-blue-400",
+ tcp: "bg-purple-500/15! text-purple-600 dark:text-purple-400",
+ http: "bg-green-500/15! text-green-700 dark:text-green-400",
+ dns: "bg-amber-500/15! text-amber-600 dark:text-amber-400",
+}
+
+const SYSTEM_STATUS_COLORS = {
+ [SystemStatus.Up]: "bg-green-500",
+ [SystemStatus.Down]: "bg-red-500",
+ [SystemStatus.Paused]: "bg-primary/40",
+ [SystemStatus.Pending]: "bg-yellow-500",
+} as const
+
+/**
+ * A monitor is considered muted if it's disabled or if its associated system is not up.
+ */
+const isMuted = (record: NetworkMonitorRecord, systemRecord: SystemRecord | undefined) =>
+ !record.enabled || systemRecord?.status !== SystemStatus.Up
+
+export function getMonitorColumns(
+ longestTarget = "",
+ {
+ onEdit,
+ onDelete,
+ onSetEnabled,
+ }: {
+ onEdit?: (monitor: NetworkMonitorRecord) => void
+ onDelete?: (monitors: NetworkMonitorRecord[]) => void | Promise<void>
+ onSetEnabled?: (monitors: NetworkMonitorRecord[], enabled: boolean) => void | Promise<void>
+ } = {}
+): ColumnDef<NetworkMonitorRecord>[] {
+ return [
+ {
+ id: "select",
+ header: ({ table }) => (
+ <Checkbox
+ className="ms-2"
+ checked={table.getIsAllRowsSelected() || (table.getIsSomeRowsSelected() && "indeterminate")}
+ onClick={(event) => event.stopPropagation()}
+ onCheckedChange={(value) => table.toggleAllRowsSelected(!!value)}
+ aria-label={t`Select all`}
+ />
+ ),
+ cell: ({ row }) => (
+ <Checkbox
+ checked={row.getIsSelected()}
+ onClick={(event) => event.stopPropagation()}
+ onCheckedChange={(value) => row.toggleSelected(!!value)}
+ aria-label={t`Select row`}
+ />
+ ),
+ enableSorting: false,
+ enableHiding: false,
+ size: 44,
+ },
+ {
+ id: "system",
+ accessorFn: (record) => record.system,
+ sortingFn: (a, b) => {
+ const allSystems = $allSystemsById.get()
+ const systemNameA = allSystems[a.original.system]?.name ?? ""
+ const systemNameB = allSystems[b.original.system]?.name ?? ""
+ const primary = systemNameA.localeCompare(systemNameB)
+ if (primary !== 0) {
+ return primary
+ }
+ return a.original.target.localeCompare(b.original.target)
+ },
+ header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
+ cell: ({ getValue }) => {
+ const system = useStore($allSystemsById)[getValue() as string] as SystemRecord | undefined
+ const longestSystemName = useStore($longestSystemName)
+ const name = system?.name
+ const status = system?.status as SystemStatus // undefined val is fine but makes lsp mad
+
+ return useMemo(
+ () => (
+ <div className="ms-1.5 max-w-44 flex gap-2 items-center tabular-nums">
+ <span className={cn("shrink-0 size-2 rounded-full", SYSTEM_STATUS_COLORS[status])} />
+ <div className="relative w-fit min-w-0 max-w-full">
+ <span className="invisible block whitespace-nowrap" aria-hidden="true">
+ {longestSystemName}
+ </span>
+ <span className="absolute inset-0 truncate">{name}</span>
+ </div>
+ </div>
+ ),
+ [status, name]
+ )
+ },
+ },
+ {
+ id: "target",
+ sortingFn: (a, b) => a.original.target.localeCompare(b.original.target),
+ accessorFn: (record) => getMonitorTarget(record),
+ header: ({ column }) => <HeaderButton column={column} name={t`Target`} Icon={GlobeIcon} />,
+ cell: ({ row, getValue }) => {
+ const monitor = row.original
+ const { status } = useStore($allSystemsById)[monitor.system] || {}
+
+ let color = "bg-green-500"
+ if (!monitor.enabled || status === SystemStatus.Paused) {
+ color = "bg-primary/40"
+ } else if (status === SystemStatus.Down || status === SystemStatus.Pending) {
+ color = "bg-yellow-500"
+ }
+ return (
+ <div className="ms-1.5 max-w-64 flex gap-2 items-center tabular-nums">
+ <span className={cn("shrink-0 size-2 rounded-full", color)} />
+ <div className="relative w-fit min-w-0 max-w-full">
+ <span className="invisible block overflow-hidden whitespace-nowrap" aria-hidden="true">
+ {longestTarget}
+ </span>
+ <span className="absolute inset-0 truncate">{getValue() as string}</span>
+ </div>
+ </div>
+ )
+ },
+ },
+ {
+ id: "protocol",
+ accessorFn: (record) => record.protocol,
+ header: ({ column }) => <HeaderButton column={column} name={t`Protocol`} Icon={ArrowLeftRightIcon} />,
+ cell: ({ getValue }) => {
+ const protocol = getValue() as string
+ return <Badge className={cn("uppercase", protocolColors[protocol])}>{protocol}</Badge>
+ },
+ },
+ {
+ id: "interval",
+ accessorFn: (record) => record.interval,
+ invertSorting: true,
+ header: ({ column }) => <HeaderButton column={column} name={t`Interval`} Icon={RefreshCwIcon} />,
+ cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{getValue() as number}s</span>,
+ },
+ {
+ id: "res",
+ accessorFn: (record) => record.res,
+ invertSorting: true,
+ header: ({ column }) => <HeaderButton column={column} name={t`Response`} Icon={TimerIcon} />,
+ cell: responseTimeCell,
+ },
+ {
+ id: "res1h",
+ accessorFn: (record) => record.resAvg1h,
+ invertSorting: true,
+ header: ({ column }) => <HeaderButton column={column} name={t`Avg 1h`} Icon={TimerIcon} />,
+ cell: responseTimeCell,
+ },
+ {
+ id: "max1h",
+ accessorFn: (record) => record.resMax1h,
+ invertSorting: true,
+ header: ({ column }) => <HeaderButton column={column} name={t`Max 1h`} Icon={TimerIcon} />,
+ cell: responseTimeCell,
+ },
+ {
+ id: "min1h",
+ accessorFn: (record) => record.resMin1h,
+ invertSorting: true,
+ header: ({ column }) => <HeaderButton column={column} name={t`Min 1h`} Icon={TimerIcon} />,
+ cell: responseTimeCell,
+ },
+ {
+ id: "loss",
+ accessorFn: (record) => record.loss1h,
+ invertSorting: true,
+ header: ({ column }) => <HeaderButton column={column} name={t`Loss 1h`} Icon={WifiOffIcon} />,
+ cell: ({ row }) => {
+ const { loss1h, res, system } = row.original
+ const systemRecord = useStore($allSystemsById)[system]
+
+ if (loss1h === undefined || (!res && !loss1h)) {
+ return <span className="ms-1.5 text-muted-foreground">-</span>
+ }
+
+ const muted = isMuted(row.original, systemRecord)
+ let color = "bg-green-500"
+ if (muted) {
+ color = "bg-muted-foreground/50"
+ } else if (loss1h) {
+ color = loss1h > 20 ? "bg-red-500" : "bg-yellow-500"
+ }
+ return (
+ <span className="ms-1.5 tabular-nums flex gap-2 items-center">
+ <span className={cn("shrink-0 size-2 rounded-full", color)} />
+ {loss1h === 100 ? loss1h : decimalString(loss1h, loss1h >= 10 ? 1 : 2)}%
+ </span>
+ )
+ },
+ },
+ {
+ id: "updated",
+ invertSorting: true,
+ accessorFn: (record) => record.updated,
+ header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
+ cell: ({ getValue }) => {
+ const timestamp = getValue() as number
+ if (!timestamp) {
+ return <span className="ms-1.5 text-muted-foreground">-</span>
+ }
+ return <span className="ms-1.5 tabular-nums">{hourWithSeconds(timestamp)}</span>
+ },
+ },
+ {
+ id: "actions",
+ enableSorting: false,
+ enableHiding: false,
+ header: () => null,
+ size: 40,
+ cell: ({ row, table }) => {
+ const selectedRows = table.getSelectedRowModel().rows
+ const actionRows =
+ row.getIsSelected() && selectedRows.length > 1
+ ? selectedRows.map((selectedRow) => selectedRow.original)
+ : [row.original]
+ const isBulkAction = actionRows.length > 1
+ const shouldPause = actionRows.some((monitor) => monitor.enabled)
+ const bulkCopyContent = actionRows.map((monitor) => formatBulkMonitorLine(monitor)).join("\n")
+ const allSystems = useStore($allSystemsById)
+ const otherSystems = useMemo(
+ () => Object.values(allSystems).filter((s) => !isBulkAction && s.id !== row.original.system),
+ [allSystems, isBulkAction]
+ )
+ return (
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <Button variant="ghost" size="icon" className="size-10">
+ <span className="sr-only">
+ <Trans>Open menu</Trans>
+ </span>
+ <MoreHorizontalIcon className="w-5" />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
+ {!isBulkAction && (
+ <DropdownMenuItem
+ onClick={() => {
+ onEdit?.(row.original)
+ }}
+ >
+ <PenBoxIcon className="me-2.5 size-4" />
+ <Trans>Edit</Trans>
+ </DropdownMenuItem>
+ )}
+ <DropdownMenuItem
+ onClick={() => {
+ onSetEnabled?.(actionRows, !shouldPause)
+ }}
+ >
+ {shouldPause ? (
+ <>
+ <PauseCircleIcon className="me-2.5 size-4" />
+ <Trans>Pause</Trans>
+ </>
+ ) : (
+ <>
+ <PlayCircleIcon className="me-2.5 size-4" />
+ <Trans>Resume</Trans>
+ </>
+ )}
+ </DropdownMenuItem>
+ <DropdownMenuItem
+ onClick={() => {
+ copyToClipboard(bulkCopyContent)
+ }}
+ >
+ <CopyIcon className="me-2.5 size-4" />
+ <Trans>Bulk copy</Trans>
+ </DropdownMenuItem>
+ {!isBulkAction && otherSystems.length > 0 && (
+ <DropdownMenuSub>
+ <DropdownMenuSubTrigger>
+ <CopyPlusIcon className="me-2.5 size-4" />
+ <Trans>Copy to system</Trans>
+ </DropdownMenuSubTrigger>
+ <DropdownMenuSubContent className="max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] overflow-y-auto">
+ {otherSystems.map((sys) => (
+ <DropdownMenuItem
+ key={sys.id}
+ onClick={() => {
+ const { id: _id, system: _system, ...rest } = row.original
+ pb.collection("network_monitors")
+ .create({ ...rest, system: sys.id })
+ .catch(() => {})
+ }}
+ >
+ {sys.name}
+ </DropdownMenuItem>
+ ))}
+ </DropdownMenuSubContent>
+ </DropdownMenuSub>
+ )}
+ <DropdownMenuSeparator />
+ <DropdownMenuItem
+ onClick={() => {
+ onDelete?.(actionRows)
+ }}
+ >
+ <Trash2Icon className="me-2.5 size-4" />
+ <Trans>Delete</Trans>
+ </DropdownMenuItem>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ )
+ },
+ },
+ ]
+}
+
+const responseTimeThresholds = {
+ http: { warning: 800_000, critical: 3_000_000 },
+ tcp: { warning: 500_000, critical: 2_000_000 },
+ icmp: { warning: 100_000, critical: 500_000 },
+ dns: { warning: 150_000, critical: 800_000 },
+}
+
+function responseTimeCell(cell: CellContext<NetworkMonitorRecord, unknown>) {
+ const monitor = cell.row.original
+ const systemRecord = useStore($allSystemsById)[monitor.system]
+ const responseTime = cell.getValue() as number | undefined
+
+ if (!responseTime) {
+ return <span className="ms-1.5 text-muted-foreground">-</span>
+ }
+
+ const muted = isMuted(monitor, systemRecord)
+ let color = "bg-green-500"
+ if (muted) {
+ color = "bg-muted-foreground/50"
+ } else if (responseTime > responseTimeThresholds[monitor.protocol].warning) {
+ color = "bg-yellow-500"
+ }
+ if (!muted && responseTime > responseTimeThresholds[monitor.protocol].critical) {
+ color = "bg-red-500"
+ }
+ return (
+ <span className="ms-1.5 tabular-nums flex gap-2 items-center">
+ <span className={cn("shrink-0 size-2 rounded-full", color)} />
+ {formatMicroseconds(responseTime)}
+ </span>
+ )
+}
+
+function HeaderButton({
+ column,
+ name,
+ Icon,
+}: {
+ column: Column<NetworkMonitorRecord>
+ name: string
+ Icon: React.ElementType
+}) {
+ const isSorted = column.getIsSorted()
+ return (
+ <Button
+ className={cn(
+ "h-9 px-3 flex items-center gap-2 duration-50",
+ isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
+ )}
+ variant="ghost"
+ onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
+ >
+ {Icon && <Icon className="size-4" />}
+ {name}
+ </Button>
+ )
+}
diff --git a/internal/site/src/components/network-monitors-table/network-monitors-table.tsx b/internal/site/src/components/network-monitors-table/network-monitors-table.tsx
new file mode 100644
index 000000000..9e42d803a
--- /dev/null
+++ b/internal/site/src/components/network-monitors-table/network-monitors-table.tsx
@@ -0,0 +1,575 @@
+import { getMonitorTarget } from "@/lib/network-monitor-utils"
+import { t } from "@lingui/core/macro"
+import { Trans } from "@lingui/react/macro"
+import {
+ type ColumnFiltersState,
+ flexRender,
+ getCoreRowModel,
+ getFilteredRowModel,
+ getSortedRowModel,
+ type Row,
+ type RowSelectionState,
+ type SortingState,
+ type Table as TableType,
+ useReactTable,
+ type VisibilityState,
+} from "@tanstack/react-table"
+import { useVirtualizer, type VirtualItem } from "@tanstack/react-virtual"
+import {
+ AlertDialog,
+ AlertDialogAction,
+ AlertDialogCancel,
+ AlertDialogContent,
+ AlertDialogDescription,
+ AlertDialogFooter,
+ AlertDialogHeader,
+ AlertDialogTitle,
+} from "@/components/ui/alert-dialog"
+import { Button, buttonVariants } from "@/components/ui/button"
+import { memo, useCallback, useMemo, useRef, useState } from "react"
+import { getMonitorColumns } from "@/components/network-monitors-table/network-monitors-columns"
+import { Card, CardHeader, CardTitle } from "@/components/ui/card"
+import { Input } from "@/components/ui/input"
+import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
+import { useToast } from "@/components/ui/use-toast"
+import { isReadOnlyUser } from "@/lib/api"
+import { pb } from "@/lib/api"
+import { $allSystemsById, $direction, $userSettings } from "@/lib/stores"
+import {
+ cn,
+ isVisuallyLonger,
+ matchesFilterGroups,
+ parseFilterGroups,
+ parseSemVer,
+ useBrowserStorage,
+} from "@/lib/utils"
+import type { ChartData, NetworkMonitorRecord } from "@/types"
+import { AddMonitorDialog, EditMonitorDialog } from "./monitor-dialog"
+import { ArrowLeftRightIcon, EthernetPortIcon, LoaderCircleIcon, ServerIcon, XIcon } from "lucide-react"
+import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
+import ChartTimeSelect from "@/components/charts/chart-time-select"
+import { LossChart, AvgMinMaxResponseChart } from "@/components/routes/system/charts/monitors-charts"
+import { useNetworkMonitorStats } from "@/lib/use-network-monitors"
+import { useStore } from "@nanostores/react"
+import { atom } from "nanostores"
+import { Separator } from "../ui/separator"
+import { $router, Link } from "../router"
+import { getPagePath } from "@nanostores/router"
+
+export default function NetworkMonitorsTableNew({
+ systemId,
+ monitors,
+ isLoading,
+}: {
+ systemId?: string
+ monitors: NetworkMonitorRecord[]
+ isLoading: boolean
+}) {
+ const [sorting, setSorting] = useBrowserStorage<SortingState>(
+ `sort-np-target-${systemId ? 1 : 0}`,
+ [{ id: systemId ? "target" : "system", desc: false }],
+ sessionStorage
+ )
+ const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
+ const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
+ const [rowSelection, setRowSelection] = useState<RowSelectionState>({})
+ const [globalFilter, setGlobalFilter] = useState("")
+ const [deleteOpen, setDeleteOpen] = useState(false)
+ const [pendingDeleteIds, setPendingDeleteIds] = useState<string[]>([])
+ const [editingMonitor, setEditingMonitor] = useState<NetworkMonitorRecord>()
+
+ const { toast } = useToast()
+ const canManageMonitors = !isReadOnlyUser()
+
+ const longestTarget = useMemo(() => {
+ let longestTarget = ""
+ for (const p of monitors) {
+ if (isVisuallyLonger(getMonitorTarget(p), longestTarget)) {
+ longestTarget = getMonitorTarget(p)
+ }
+ }
+ return longestTarget
+ }, [monitors])
+
+ const runMonitorBatch = useCallback(
+ async (ids: string[], enqueue: (batch: ReturnType<typeof pb.createBatch>, id: string) => void) => {
+ let batch = pb.createBatch()
+ let inBatch = 0
+ for (const id of ids) {
+ enqueue(batch, id)
+ if (++inBatch >= 20) {
+ await batch.send()
+ batch = pb.createBatch()
+ inBatch = 0
+ }
+ }
+ if (inBatch) {
+ await batch.send()
+ }
+ },
+ []
+ )
+
+ const handleDeleteRequest = useCallback(
+ async (monitorsToDelete: NetworkMonitorRecord[]) => {
+ if (!monitorsToDelete.length) {
+ return
+ }
+
+ const ids = monitorsToDelete.map((monitor) => monitor.id)
+ if (ids.length === 1) {
+ try {
+ await pb.collection("network_monitors").delete(ids[0])
+ } catch (err: unknown) {
+ toast({
+ variant: "destructive",
+ title: t`Error`,
+ description: (err as Error)?.message || t`Failed to delete monitors.`,
+ })
+ }
+ return
+ }
+
+ setPendingDeleteIds(ids)
+ setDeleteOpen(true)
+ },
+ [toast]
+ )
+
+ const handleBulkDelete = async () => {
+ setDeleteOpen(false)
+ if (!pendingDeleteIds.length) {
+ return
+ }
+
+ try {
+ await runMonitorBatch(pendingDeleteIds, (batch, id) => batch.collection("network_monitors").delete(id))
+ setPendingDeleteIds([])
+ setRowSelection({})
+ } catch (err: unknown) {
+ toast({
+ variant: "destructive",
+ title: t`Error`,
+ description: (err as Error)?.message || t`Failed to delete monitors.`,
+ })
+ }
+ }
+
+ const handleSetEnabled = useCallback(
+ async (monitorsToUpdate: NetworkMonitorRecord[], enabled: boolean) => {
+ if (!monitorsToUpdate.length) {
+ return
+ }
+
+ const pendingUpdates = monitorsToUpdate.filter((monitor) => monitor.enabled !== enabled)
+ if (!pendingUpdates.length) {
+ return
+ }
+
+ try {
+ if (pendingUpdates.length === 1) {
+ await pb.collection("network_monitors").update(pendingUpdates[0].id, { enabled })
+ return
+ }
+ await runMonitorBatch(
+ pendingUpdates.map((monitor) => monitor.id),
+ (batch, id) => batch.collection("network_monitors").update(id, { enabled })
+ )
+ if (monitorsToUpdate.length > 1) {
+ setRowSelection({})
+ }
+ } catch (err: unknown) {
+ toast({
+ variant: "destructive",
+ title: t`Error`,
+ description: (err as Error)?.message || t`Failed to update monitors.`,
+ })
+ }
+ },
+ [runMonitorBatch, toast]
+ )
+
+ const columns = useMemo(() => {
+ let columns = getMonitorColumns(longestTarget, {
+ onEdit: setEditingMonitor,
+ onDelete: handleDeleteRequest,
+ onSetEnabled: handleSetEnabled,
+ })
+ columns = systemId ? columns.filter((col) => col.id !== "system") : columns
+ columns = canManageMonitors ? columns : columns.filter((col) => col.id !== "actions")
+ return columns
+ }, [canManageMonitors, handleDeleteRequest, handleSetEnabled, systemId, longestTarget])
+
+ const table = useReactTable({
+ data: monitors,
+ columns,
+ getRowId: (row) => row.id,
+ getCoreRowModel: getCoreRowModel(),
+ getSortedRowModel: getSortedRowModel(),
+ getFilteredRowModel: getFilteredRowModel(),
+ onSortingChange: setSorting,
+ onColumnFiltersChange: setColumnFilters,
+ onColumnVisibilityChange: setColumnVisibility,
+ onRowSelectionChange: setRowSelection,
+ defaultColumn: {
+ sortUndefined: "last",
+ size: 900,
+ minSize: 0,
+ },
+ state: {
+ sorting,
+ columnFilters,
+ columnVisibility,
+ rowSelection,
+ globalFilter,
+ },
+ onGlobalFilterChange: setGlobalFilter,
+ globalFilterFn: (row, _columnId, filterValue) => {
+ const value = (filterValue as string).trim()
+ if (!value) return true
+ const monitor = row.original
+ const systemName = $allSystemsById.get()[monitor.system]?.name ?? ""
+ const searchString = `${getMonitorTarget(monitor)}${monitor.protocol}${systemName}`.toLocaleLowerCase()
+ return matchesFilterGroups(searchString, parseFilterGroups(value))
+ },
+ })
+
+ const rows = table.getRowModel().rows
+ const visibleColumns = table.getVisibleLeafColumns()
+
+ return (
+ <Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
+ <CardHeader className="p-0 mb-3 sm:mb-4">
+ <div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
+ <div className="px-2 sm:px-1">
+ <CardTitle className="mb-2">
+ <Trans>Network Monitors</Trans>
+ </CardTitle>
+ <div className="text-sm text-muted-foreground flex items-center flex-wrap">
+ <Trans>Response time monitoring from agents.</Trans>
+ </div>
+ </div>
+ <div className="md:ms-auto flex items-center gap-2">
+ {monitors.length > 0 && (
+ <div className="relative">
+ <Input
+ placeholder={t`Filter...`}
+ title={t`Use commas to match any of multiple terms, e.g. "system1, system2"`}
+ value={globalFilter}
+ onChange={(e) => setGlobalFilter(e.target.value)}
+ className="ms-auto px-4 w-full max-w-full md:w-50"
+ />
+ {globalFilter && (
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon"
+ aria-label={t`Clear`}
+ className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
+ onClick={() => setGlobalFilter("")}
+ >
+ <XIcon className="h-4 w-4" />
+ </Button>
+ )}
+ </div>
+ )}
+ {canManageMonitors ? <AddMonitorDialog systemId={systemId} monitors={monitors} /> : null}
+ {canManageMonitors ? (
+ <EditMonitorDialog
+ systemId={systemId}
+ monitor={editingMonitor}
+ open={!!editingMonitor}
+ setOpen={(open) => {
+ if (!open) {
+ setEditingMonitor(undefined)
+ }
+ }}
+ />
+ ) : null}
+ <AlertDialog
+ open={deleteOpen}
+ onOpenChange={(open) => {
+ setDeleteOpen(open)
+ if (!open) {
+ setPendingDeleteIds([])
+ }
+ }}
+ >
+ <AlertDialogContent>
+ <AlertDialogHeader>
+ <AlertDialogTitle>
+ <Trans>Are you sure?</Trans>
+ </AlertDialogTitle>
+ <AlertDialogDescription>
+ <Trans>This will permanently delete all selected records from the database.</Trans>
+ </AlertDialogDescription>
+ </AlertDialogHeader>
+ <AlertDialogFooter>
+ <AlertDialogCancel>
+ <Trans>Cancel</Trans>
+ </AlertDialogCancel>
+ <AlertDialogAction
+ className={cn(buttonVariants({ variant: "destructive" }))}
+ onClick={handleBulkDelete}
+ >
+ <Trans>Continue</Trans>
+ </AlertDialogAction>
+ </AlertDialogFooter>
+ </AlertDialogContent>
+ </AlertDialog>
+ </div>
+ </div>
+ </CardHeader>
+ <div className="rounded-md">
+ <NetworkMonitorsTable
+ table={table}
+ rows={rows}
+ colLength={visibleColumns.length}
+ rowSelection={rowSelection}
+ isLoading={isLoading}
+ />
+ </div>
+ </Card>
+ )
+}
+
+const NetworkMonitorsTable = memo(function NetworkMonitorTable({
+ table,
+ rows,
+ colLength,
+ rowSelection,
+ isLoading,
+}: {
+ table: TableType<NetworkMonitorRecord>
+ rows: Row<NetworkMonitorRecord>[]
+ colLength: number
+ rowSelection: RowSelectionState
+ isLoading: boolean
+}) {
+ const scrollRef = useRef<HTMLDivElement>(null)
+ const [sheetOpen, setSheetOpen] = useState(false)
+ const [activeMonitorId, setActiveMonitorId] = useState<string | null>(null)
+ const activeMonitor = activeMonitorId
+ ? table.options.data.find((monitor) => monitor.id === activeMonitorId)
+ : undefined
+ const openSheet = useCallback((monitor: NetworkMonitorRecord) => {
+ setActiveMonitorId(monitor.id)
+ setSheetOpen(true)
+ }, [])
+
+ const virtualizer = useVirtualizer<HTMLDivElement, HTMLTableRowElement>({
+ count: rows.length,
+ estimateSize: () => 54,
+ getScrollElement: () => scrollRef.current,
+ overscan: 5,
+ })
+ const virtualRows = virtualizer.getVirtualItems()
+
+ const paddingTop = Math.max(0, virtualRows[0]?.start ?? 0 - virtualizer.options.scrollMargin)
+ const paddingBottom = Math.max(0, virtualizer.getTotalSize() - (virtualRows[virtualRows.length - 1]?.end ?? 0))
+
+ return (
+ <div
+ className={cn(
+ "h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto border rounded-md",
+ (!rows.length || rows.length > 2) && "min-h-50"
+ )}
+ ref={scrollRef}
+ >
+ <div style={{ height: `${virtualizer.getTotalSize() + 48}px`, paddingTop, paddingBottom }}>
+ <table className="text-sm w-full h-full text-nowrap">
+ <NetworkMonitorTableHead table={table} />
+ <TableBody>
+ {rows.length ? (
+ virtualRows.map((virtualRow) => {
+ const row = rows[virtualRow.index]
+ return (
+ <NetworkMonitorTableRow
+ key={row.id}
+ row={row}
+ virtualRow={virtualRow}
+ isSelected={row.getIsSelected()}
+ rowSelection={rowSelection}
+ openSheet={openSheet}
+ />
+ )
+ })
+ ) : (
+ <TableRow>
+ <TableCell colSpan={colLength} className="h-37 text-center pointer-events-none">
+ {isLoading ? (
+ <LoaderCircleIcon className="animate-spin size-10 opacity-60 mx-auto" />
+ ) : (
+ <Trans>No results.</Trans>
+ )}
+ </TableCell>
+ </TableRow>
+ )}
+ </TableBody>
+ </table>
+ </div>
+ <NetworkMonitorSheet
+ open={sheetOpen}
+ onOpenChange={(nextOpen) => {
+ setSheetOpen(nextOpen)
+ }}
+ monitor={activeMonitor}
+ />
+ </div>
+ )
+})
+
+function NetworkMonitorTableHead({ table }: { table: TableType<NetworkMonitorRecord> }) {
+ return (
+ <TableHeader className="sticky top-0 z-50 w-full border-b-2">
+ {table.getHeaderGroups().map((headerGroup) => (
+ <tr key={headerGroup.id}>
+ {headerGroup.headers.map((header) => {
+ return (
+ <TableHead className="px-2" key={header.id}>
+ {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
+ </TableHead>
+ )
+ })}
+ </tr>
+ ))}
+ </TableHeader>
+ )
+}
+
+const NetworkMonitorTableRow = memo(function NetworkMonitorTableRow({
+ row,
+ virtualRow,
+ isSelected,
+ rowSelection: _rowSelection,
+ openSheet,
+}: {
+ row: Row<NetworkMonitorRecord>
+ virtualRow: VirtualItem
+ isSelected: boolean
+ // Menus depend on the entire selection, including changes to other rows.
+ rowSelection: RowSelectionState
+ openSheet: (monitor: NetworkMonitorRecord) => void
+}) {
+ return (
+ <TableRow
+ data-state={isSelected && "selected"}
+ className="cursor-pointer transition-opacity"
+ onClick={() => openSheet(row.original)}
+ >
+ {row.getVisibleCells().map((cell) => (
+ <TableCell
+ key={cell.id}
+ className="py-0"
+ style={{
+ width: `${cell.column.getSize()}px`,
+ height: virtualRow.size,
+ }}
+ >
+ {flexRender(cell.column.columnDef.cell, cell.getContext())}
+ </TableCell>
+ ))}
+ </TableRow>
+ )
+})
+
+function NetworkMonitorSheet({
+ open,
+ onOpenChange,
+ monitor,
+}: {
+ open: boolean
+ onOpenChange: (open: boolean) => void
+ monitor?: NetworkMonitorRecord
+}) {
+ if (!monitor) {
+ return null
+ }
+
+ return <NetworkMonitorSheetContent key={monitor.system} open={open} onOpenChange={onOpenChange} monitor={monitor} />
+}
+
+function NetworkMonitorSheetContent({
+ open,
+ onOpenChange,
+ monitor,
+}: {
+ open: boolean
+ onOpenChange: (open: boolean) => void
+ monitor: NetworkMonitorRecord
+}) {
+ // Keep monitor exploration independent of the system charts' time range.
+ const [chartTimeStore] = useState(() => {
+ const defaultTime = $userSettings.get().chartTime
+ return atom(defaultTime === "1m" ? "1h" : defaultTime)
+ })
+ const chartTime = useStore(chartTimeStore)
+ const direction = useStore($direction)
+ const system = useStore($allSystemsById)[monitor.system]
+
+ const monitorStats = useNetworkMonitorStats({
+ systemId: monitor.system,
+ monitorId: monitor.id,
+ chartTime,
+ enabled: open,
+ })
+
+ const chartData = useMemo<ChartData>(
+ () => ({
+ agentVersion: parseSemVer(system?.info?.v),
+ orientation: direction === "rtl" ? "right" : "left",
+ chartTime,
+ }),
+ [system?.info?.v, direction, chartTime]
+ )
+ const hasMonitorStats = monitorStats.some((record) => record.stats?.[monitor.id] != null)
+ const monitorLabel = getMonitorTarget(monitor)
+
+ return (
+ <Sheet open={open} onOpenChange={onOpenChange}>
+ <SheetContent className="w-full sm:max-w-220 overflow-auto p-4 sm:p-6">
+ <SheetHeader className="mb-0 border-b p-0 pb-4">
+ <SheetTitle>{monitorLabel}</SheetTitle>
+ <SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
+ <ServerIcon className="size-3.5 text-muted-foreground" />
+ <Link className="hover:underline" href={getPagePath($router, "system", { id: system?.id ?? "" })}>
+ {system?.name ?? ""}
+ </Link>
+ <Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
+ <ArrowLeftRightIcon className="size-3.5 text-muted-foreground" />
+ {monitor.protocol.toUpperCase()}
+ {monitor.protocol === "tcp" && monitor.port > 0 && (
+ <>
+ <Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
+ <EthernetPortIcon className="size-3.5 text-muted-foreground" />
+ <span>{monitor.port}</span>
+ </>
+ )}
+ </SheetDescription>
+ </SheetHeader>
+ <div className="grid gap-4">
+ <ChartTimeSelect
+ className="bg-card"
+ agentVersion={chartData.agentVersion}
+ chartTimeStore={chartTimeStore}
+ allowRealtime={false}
+ />
+ <AvgMinMaxResponseChart
+ monitorStats={monitorStats}
+ monitor={monitor}
+ chartData={chartData}
+ empty={!hasMonitorStats}
+ />
+ <LossChart
+ monitorStats={monitorStats}
+ grid={false}
+ monitors={[monitor]}
+ chartData={chartData}
+ empty={!hasMonitorStats}
+ showFilter={false}
+ />
+ </div>
+ </SheetContent>
+ </Sheet>
+ )
+}
diff --git a/internal/site/src/components/router.tsx b/internal/site/src/components/router.tsx
index fb5259729..5c54f6137 100644
--- a/internal/site/src/components/router.tsx
+++ b/internal/site/src/components/router.tsx
@@ -4,6 +4,7 @@ const routes = {
home: "/",
containers: "/containers",
smart: "/smart",
+ monitors: "/monitors",
system: `/system/:id`,
settings: `/settings/:name?`,
forgot_password: `/forgot-password`,
diff --git a/internal/site/src/components/routes/monitors.tsx b/internal/site/src/components/routes/monitors.tsx
new file mode 100644
index 000000000..51bee7790
--- /dev/null
+++ b/internal/site/src/components/routes/monitors.tsx
@@ -0,0 +1,33 @@
+import { useLingui } from "@lingui/react/macro"
+import { memo, useEffect } from "react"
+import NetworkMonitorsTableNew from "@/components/network-monitors-table/network-monitors-table"
+import { ActiveAlerts } from "@/components/active-alerts"
+import { FooterRepoLink } from "@/components/footer-repo-link"
+import { useNetworkMonitors } from "@/lib/use-network-monitors"
+import { $allSystemsById } from "@/lib/stores"
+import { supportsNetworkMonitors } from "@/lib/utils"
+import { useStore } from "@nanostores/react"
+
+export default memo(() => {
+ const { t } = useLingui()
+ const { monitors, isLoading } = useNetworkMonitors({})
+ const systems = useStore($allSystemsById)
+ const visibleMonitors = monitors.filter((monitor) => {
+ const system = systems[monitor.system]
+ return !system || supportsNetworkMonitors(system)
+ })
+
+ useEffect(() => {
+ document.title = `${t`Network Monitors`} / Beszel`
+ }, [t])
+
+ return (
+ <>
+ <div className="grid gap-4">
+ <ActiveAlerts />
+ <NetworkMonitorsTableNew monitors={visibleMonitors} isLoading={isLoading} />
+ </div>
+ <FooterRepoLink />
+ </>
+ )
+})
diff --git a/internal/site/src/components/routes/settings/alerts-history-data-table.tsx b/internal/site/src/components/routes/settings/alerts-history-data-table.tsx
index 84758ba2d..763daa063 100644
--- a/internal/site/src/components/routes/settings/alerts-history-data-table.tsx
+++ b/internal/site/src/components/routes/settings/alerts-history-data-table.tsx
@@ -78,7 +78,7 @@ export default function AlertsHistoryDataTable() {
let unsubscribe: (() => void) | undefined
const pbOptions = {
expand: "system",
- fields: "id,name,value,state,created,resolved,expand.system.name",
+ fields: "id,name,monitor_name,value,state,created,resolved,expand.system.name",
}
// Initial load
pb.collection<AlertsHistoryRecord>("alerts_history")
@@ -199,7 +199,7 @@ export default function AlertsHistoryDataTable() {
if (!selectedRows.length) return
const cells: Record<string, (record: AlertsHistoryRecord) => string> = {
system: (record) => record.expand?.system?.name || record.system,
- name: (record) => alertInfo[record.name]?.name() || record.name,
+ name: (record) => [alertInfo[record.name]?.name() || record.name, record.monitor_name].filter(Boolean).join(": "),
value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""),
state: (record) => (record.resolved ? t`Resolved` : t`Active`),
created: (record) => formatShortDate(record.created),
diff --git a/internal/site/src/components/routes/system.tsx b/internal/site/src/components/routes/system.tsx
index f43ce9562..1163e81b3 100644
--- a/internal/site/src/components/routes/system.tsx
+++ b/internal/site/src/components/routes/system.tsx
@@ -1,6 +1,6 @@
import { memo, useState } from "react"
import { Trans } from "@lingui/react/macro"
-import { compareSemVer, parseSemVer } from "@/lib/utils"
+import { compareSemVer, parseSemVer, supportsNetworkMonitors } from "@/lib/utils"
import type { GPUData } from "@/types"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import InfoBar from "./system/info-bar"
@@ -12,9 +12,15 @@ import { ZfsCharts } from "./system/charts/storage-pool-charts"
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts"
-import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"
+import {
+ LazyContainersTable,
+ LazyNetworkMonitorsTable,
+ LazySmartTable,
+ LazySystemdTable,
+ LazyZfsTable,
+} from "./system/lazy-tables"
import { LoadAverageChart } from "./system/charts/load-average-chart"
-import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
+import { ContainerIcon, CpuIcon, HardDriveIcon, NetworkIcon, TerminalSquareIcon } from "lucide-react"
import { GpuIcon } from "../ui/icons"
import SystemdTable from "../systemd-table/systemd-table"
import ContainersTable from "../containers-table/containers-table"
@@ -65,9 +71,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
const hasSystemd = system.info.sv
const hasGpu = hasGpuData || hasGpuPowerData
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
+ const hasNetworkMonitors = supportsNetworkMonitors(system)
// keep tabsRef in sync for keyboard navigation
- const tabs = ["core", "disk"]
+ const tabs = ["core", "network", "disk"]
if (hasGpu) tabs.push("gpu")
if (hasContainers) tabs.push("containers")
if (hasSystemd) tabs.push("services")
@@ -153,6 +160,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
{hasSystemd && <LazySystemdTable systemId={system.id} />}
+
+ {hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
</>
)
}
@@ -165,6 +174,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<CpuIcon className="size-3.5" />
<Trans context="Core system metrics">Core</Trans>
</TabsTrigger>
+ <TabsTrigger value="network" className="w-full flex items-center gap-1.5">
+ <NetworkIcon className="size-3.5" />
+ <Trans>Network</Trans>
+ </TabsTrigger>
<TabsTrigger value="disk" className="w-full flex items-center gap-1.5">
<HardDriveIcon className="size-3.5" />
<Trans>Disk</Trans>
@@ -192,9 +205,9 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}>
<div className="grid xl:grid-cols-2 gap-4">
<CpuChart {...coreProps} />
- <MemoryChart {...coreProps} />
<LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} />
- <BandwidthChart {...coreProps} systemStats={systemStats} />
+ <MemoryChart {...coreProps} />
+ <SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} />
<TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} />
<FanChart {...coreProps} />
<BatteryChart system={system} {...coreProps} />
@@ -203,6 +216,17 @@ export default memo(function SystemDetail({ id }: { id: string }) {
</div>
</TabsContent>
+ <TabsContent value="network" forceMount className={activeTab === "network" ? "contents" : "hidden"}>
+ {mountedTabs.has("network") && (
+ <>
+ <div className="grid xl:grid-cols-2 gap-4">
+ <BandwidthChart {...coreProps} systemStats={systemStats} />
+ </div>
+ {hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
+ </>
+ )}
+ </TabsContent>
+
<TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}>
{mountedTabs.has("disk") && (
<>
diff --git a/internal/site/src/components/routes/system/chart-card.tsx b/internal/site/src/components/routes/system/chart-card.tsx
index 1ad93facf..c3e12447e 100644
--- a/internal/site/src/components/routes/system/chart-card.tsx
+++ b/internal/site/src/components/routes/system/chart-card.tsx
@@ -44,6 +44,7 @@ export function FilterBar({ store = $containerFilter }: { store?: typeof $contai
<>
<Input
placeholder={t`Filter...`}
+ title={t`Use commas to match any of multiple terms, e.g. "system1, system2"`}
className="ps-4 pe-8 w-full sm:w-44"
onChange={handleChange}
value={inputValue}
diff --git a/internal/site/src/components/routes/system/chart-data.ts b/internal/site/src/components/routes/system/chart-data.ts
index 8769caafe..8535a054e 100644
--- a/internal/site/src/components/routes/system/chart-data.ts
+++ b/internal/site/src/components/routes/system/chart-data.ts
@@ -1,7 +1,13 @@
-import { timeTicks } from "d3-time"
import { getPbTimestamp, pb } from "@/lib/api"
import { chartTimeData } from "@/lib/utils"
-import type { ChartData, ChartTimes, ContainerStatsRecord, SystemStatsRecord } from "@/types"
+import type {
+ ChartData,
+ ChartDataContainer,
+ ChartTimes,
+ ContainerStatsRecord,
+ NetworkMonitorStatsRecord,
+ SystemStatsRecord,
+} from "@/types"
type ChartTimeData = {
time: number
@@ -17,31 +23,10 @@ export const cache = new Map<
ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"]
>()
-// create ticks and domain for charts
-export function getTimeData(chartTime: ChartTimes, lastCreated: number) {
- const cached = cache.get("td") as ChartTimeData | undefined
- if (cached && cached.chartTime === chartTime) {
- if (!lastCreated || cached.time >= lastCreated) {
- return cached.data
- }
- }
-
- // const buffer = chartTime === "1m" ? 400 : 20_000
- const now = new Date(Date.now())
- const startTime = chartTimeData[chartTime].getOffset(now)
- const ticks = timeTicks(startTime, now, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
- const data = {
- ticks,
- domain: [chartTimeData[chartTime].getOffset(now).getTime(), now.getTime()],
- }
- cache.set("td", { time: now.getTime(), data, chartTime })
- return data
-}
-
/** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place.
* Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */
export function appendData<T extends { created: string | number | null }>(
- prev: T[],
+ prev: T[] = [],
newRecords: T[],
expectedInterval: number,
maxLen?: number
@@ -66,17 +51,18 @@ export function appendData<T extends { created: string | number | null }>(
return result
}
-export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord>(
+export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord | NetworkMonitorStatsRecord>(
collection: string,
systemId: string,
- chartTime: ChartTimes
+ chartTime: ChartTimes,
+ cachedStats?: { created: string | number | null }[],
+ createdIsNumber?: boolean
): Promise<T[]> {
- const cachedStats = cache.get(`${systemId}_${chartTime}_${collection}`) as T[] | undefined
const lastCached = cachedStats?.at(-1)?.created as number
return await pb.collection<T>(collection).getFullList({
filter: pb.filter("system={:id} && created > {:created} && type={:type}", {
id: systemId,
- created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined),
+ created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, createdIsNumber),
type: chartTimeData[chartTime].type,
}),
fields: "created,stats",
@@ -84,11 +70,11 @@ export async function getStats<T extends SystemStatsRecord | ContainerStatsRecor
})
}
-export function makeContainerData(containers: ContainerStatsRecord[]): ChartData["containerData"] {
- const result = [] as ChartData["containerData"]
+export function makeContainerData(containers: ContainerStatsRecord[]): ChartDataContainer[] {
+ const result = [] as ChartDataContainer[]
for (const { created, stats } of containers) {
if (!created) {
- result.push({ created: null } as ChartData["containerData"][0])
+ result.push({ created: null } as ChartDataContainer)
continue
}
result.push(makeContainerPoint(new Date(created).getTime(), stats))
@@ -97,11 +83,8 @@ export function makeContainerData(containers: ContainerStatsRecord[]): ChartData
}
/** Transform a single realtime container stats message into a ChartDataContainer point. */
-export function makeContainerPoint(
- created: number,
- stats: ContainerStatsRecord["stats"]
-): ChartData["containerData"][0] {
- const point: ChartData["containerData"][0] = { created } as ChartData["containerData"][0]
+export function makeContainerPoint(created: number, stats: ContainerStatsRecord["stats"]): ChartDataContainer {
+ const point: ChartDataContainer = { created } as ChartDataContainer
for (const container of stats) {
;(point as Record<string, unknown>)[container.n] = container
}
diff --git a/internal/site/src/components/routes/system/charts/monitors-charts.tsx b/internal/site/src/components/routes/system/charts/monitors-charts.tsx
new file mode 100644
index 000000000..d5ac790d4
--- /dev/null
+++ b/internal/site/src/components/routes/system/charts/monitors-charts.tsx
@@ -0,0 +1,212 @@
+import { getMonitorTarget } from "@/lib/network-monitor-utils"
+import LineChartDefault from "@/components/charts/line-chart"
+import type { DataPoint } from "@/components/charts/line-chart"
+import { decimalString, formatMicroseconds, matchesFilterGroups, parseFilterGroups, toFixedFloat } from "@/lib/utils"
+import { $monitorFilter } from "@/lib/stores"
+import { useLingui } from "@lingui/react/macro"
+import { ChartCard, FilterBar } from "../chart-card"
+import type { ChartData, MonitorStats, NetworkMonitorRecord, NetworkMonitorStatsRecord } from "@/types"
+import { useMemo } from "react"
+import { useStore } from "@nanostores/react"
+
+type MonitorChartProps = {
+ monitorStats: NetworkMonitorStatsRecord[]
+ grid?: boolean
+ monitors: NetworkMonitorRecord[]
+ chartData: ChartData
+ empty: boolean
+ showFilter?: boolean
+ /** Prepended to the chart title, e.g. a target/system name (rendered as "{titlePrefix} — Response"). */
+ titlePrefix?: string
+}
+
+type MonitorChartBaseProps = MonitorChartProps & {
+ metric: keyof MonitorStats
+ title: string
+ description: string
+ tickFormatter: (value: number) => string
+ contentFormatter: ({ value }: { value: number | string }) => string | number
+ domain?: [number | "auto", number | "auto"]
+}
+
+function MonitorChart({
+ monitorStats,
+ grid,
+ monitors,
+ chartData,
+ empty,
+ metric,
+ title,
+ description,
+ tickFormatter,
+ contentFormatter,
+ domain,
+ showFilter = monitors.length > 1,
+}: MonitorChartBaseProps) {
+ const storedFilter = useStore($monitorFilter)
+ const filter = showFilter ? storedFilter : ""
+
+ const { dataPoints, visibleKeys } = useMemo(() => {
+ const sortedMonitors = [...monitors].sort((a, b) => b.resAvg1h - a.resAvg1h)
+ const count = sortedMonitors.length
+ const points: DataPoint<NetworkMonitorStatsRecord>[] = []
+ const visibleIDs: string[] = []
+ const filterGroups = parseFilterGroups(filter)
+ const dot = chartData.chartTime === "1m"
+ for (let i = 0; i < count; i++) {
+ const p = sortedMonitors[i]
+ const label = getMonitorTarget(p)
+ const labelLower = label.toLowerCase()
+ const filtered = filterGroups.length > 0 && !matchesFilterGroups(labelLower, filterGroups)
+ if (filtered) {
+ continue
+ }
+ visibleIDs.push(p.id)
+ points.push({
+ order: i,
+ label,
+ dataKey: (record: NetworkMonitorStatsRecord) => record.stats?.[p.id]?.[metric] ?? null,
+ dot,
+ color: count <= 5 ? i + 1 : `hsl(${(i * 360) / count}, var(--chart-saturation), var(--chart-lightness))`,
+ })
+ }
+ return { dataPoints: points, visibleKeys: visibleIDs }
+ }, [monitors, filter, metric, chartData.chartTime])
+
+ const filteredMonitorStats = useMemo(() => {
+ if (!visibleKeys.length) return monitorStats
+ return monitorStats.filter((record) => visibleKeys.some((id) => record.stats?.[id] != null))
+ }, [monitorStats, visibleKeys])
+
+ const legend = dataPoints.length < 10 && showFilter
+
+ return (
+ <ChartCard
+ legend={legend || !showFilter}
+ cornerEl={showFilter ? <FilterBar store={$monitorFilter} /> : undefined}
+ empty={empty}
+ title={title}
+ description={description}
+ grid={grid}
+ >
+ <LineChartDefault
+ truncate
+ chartData={chartData}
+ customData={filteredMonitorStats}
+ dataPoints={dataPoints}
+ domain={domain ?? ["auto", "auto"]}
+ connectNulls
+ tickFormatter={tickFormatter}
+ contentFormatter={contentFormatter}
+ legend={legend}
+ filter={filter}
+ />
+ </ChartCard>
+ )
+}
+
+interface AvgMinMaxResponseChartProps {
+ monitorStats: NetworkMonitorStatsRecord[]
+ monitor: NetworkMonitorRecord | null
+ chartData: ChartData
+ empty: boolean
+}
+
+export function AvgMinMaxResponseChart({ monitorStats, monitor, chartData, empty }: AvgMinMaxResponseChartProps) {
+ const { t } = useLingui()
+
+ const { chartTime } = chartData
+ const hasLongInterval = (monitor?.interval ?? 61) > 60
+
+ // only one monitor is relevant for this chart
+ const dataPoints: DataPoint<NetworkMonitorStatsRecord>[] = useMemo(() => {
+ const dataFn = (metric: keyof MonitorStats) => (record: NetworkMonitorStatsRecord) =>
+ record.stats?.[monitor?.id ?? ""]?.[metric] ?? "-"
+ const avgPoint = {
+ label: "Avg",
+ dataKey: dataFn("res_avg"),
+ color: 1,
+ order: 0,
+ }
+ if (chartTime === "1m" || (hasLongInterval && chartTime === "1h")) {
+ // avg, min, max are all the same for 1m interval, so just show avg
+ return [avgPoint]
+ }
+ return [
+ {
+ label: "Max",
+ dataKey: dataFn("res_max"),
+ color: 3,
+ order: 0,
+ },
+ avgPoint,
+ {
+ label: "Min",
+ dataKey: dataFn("res_min"),
+ color: 2,
+ order: 2,
+ },
+ ]
+ }, [chartTime, hasLongInterval, monitor?.id])
+
+ const data = useMemo(() => {
+ if (!monitor) return []
+ return monitorStats.filter((record) => record.stats && monitor.id in record.stats)
+ }, [monitor, monitorStats])
+
+ const legend = dataPoints.length > 1
+
+ return (
+ <ChartCard
+ legend={true}
+ empty={empty}
+ title={t`Response`}
+ description={t`Average, minimum, and maximum response time`}
+ grid={false}
+ >
+ <LineChartDefault
+ truncate
+ chartData={chartData}
+ customData={data}
+ dataPoints={dataPoints}
+ domain={["auto", "auto"]}
+ connectNulls
+ legend={legend}
+ tickFormatter={(value) => formatMicroseconds(value, false)}
+ contentFormatter={({ value }) => {
+ if (typeof value !== "number") {
+ return value
+ }
+ return formatMicroseconds(value)
+ }}
+ />
+ </ChartCard>
+ )
+}
+
+export function LossChart({ monitorStats, grid, monitors, chartData, empty, titlePrefix }: MonitorChartProps) {
+ const { t } = useLingui()
+ const lossTitle = t`Loss`
+ const title = titlePrefix ? `${titlePrefix} — ${lossTitle}` : lossTitle
+
+ return (
+ <MonitorChart
+ monitorStats={monitorStats}
+ grid={grid}
+ monitors={monitors}
+ chartData={chartData}
+ empty={empty}
+ metric="loss"
+ title={title}
+ description={t`Packet loss (%)`}
+ domain={[0, 100]}
+ tickFormatter={(value) => `${toFixedFloat(value, value >= 10 ? 0 : 1)}%`}
+ contentFormatter={({ value }) => {
+ if (typeof value !== "number") {
+ return value
+ }
+ return `${decimalString(value, 2)}%`
+ }}
+ />
+ )
+}
diff --git a/internal/site/src/components/routes/system/lazy-tables.tsx b/internal/site/src/components/routes/system/lazy-tables.tsx
index b2e0a9fca..60b9eb5c5 100644
--- a/internal/site/src/components/routes/system/lazy-tables.tsx
+++ b/internal/site/src/components/routes/system/lazy-tables.tsx
@@ -1,6 +1,7 @@
import { lazy } from "react"
import { useIntersectionObserver } from "@/lib/use-intersection-observer"
import { cn } from "@/lib/utils"
+import { useNetworkMonitors } from "@/lib/use-network-monitors"
const ContainersTable = lazy(() => import("../../containers-table/containers-table"))
@@ -45,3 +46,19 @@ export function LazySystemdTable({ systemId }: { systemId: string }) {
</div>
)
}
+
+const NetworkMonitorsTable = lazy(() => import("../../network-monitors-table/network-monitors-table"))
+
+export function LazyNetworkMonitorsTable({ systemId }: { systemId: string }) {
+ const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
+ return (
+ <div ref={ref} className={cn(isIntersecting && "contents")}>
+ {isIntersecting && <SystemNetworkMonitorsTable systemId={systemId} />}
+ </div>
+ )
+}
+
+function SystemNetworkMonitorsTable({ systemId }: { systemId: string }) {
+ const { monitors, isLoading } = useNetworkMonitors({ systemId })
+ return <NetworkMonitorsTable systemId={systemId} monitors={monitors} isLoading={isLoading} />
+}
diff --git a/internal/site/src/components/routes/system/smart-table.tsx b/internal/site/src/components/routes/system/smart-table.tsx
index 2efeb7910..9bc047d3f 100644
--- a/internal/site/src/components/routes/system/smart-table.tsx
+++ b/internal/site/src/components/routes/system/smart-table.tsx
@@ -43,7 +43,6 @@ import {
toFixedFloat,
formatTemperature,
cn,
- getVisualStringWidth,
secondsToString,
hourWithSeconds,
formatShortDate,
@@ -117,9 +116,9 @@ function formatDataUnits(units: number): string {
const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated"
export const createColumns = (
- longestName: number,
- longestModel: number,
- longestDevice: number
+ longestName: string,
+ longestModel: string,
+ longestDevice: string
): ColumnDef<SmartDeviceRecord>[] => [
{
id: "system",
@@ -134,8 +133,11 @@ export const createColumns = (
cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById)
return (
- <div className="ms-1.5 max-w-40 block truncate" style={{ width: `${longestName / 1.05}ch` }}>
- {allSystems[getValue() as string]?.name ?? ""}
+ <div className="ms-1.5 relative w-fit max-w-44">
+ <span className="invisible block whitespace-nowrap" aria-hidden="true">
+ {longestName}
+ </span>
+ <span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
</div>
)
},
@@ -145,12 +147,11 @@ export const createColumns = (
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />,
cell: ({ getValue }) => (
- <div
- className="font-medium max-w-40 truncate ms-1"
- title={getValue() as string}
- style={{ width: `${longestDevice / 1.05}ch` }}
- >
- {getValue() as string}
+ <div className="font-medium ms-1 relative w-fit max-w-44" title={getValue() as string}>
+ <span className="invisible block whitespace-nowrap" aria-hidden="true">
+ {longestDevice}
+ </span>
+ <span className="absolute inset-0 truncate">{getValue() as string}</span>
</div>
),
},
@@ -161,12 +162,11 @@ export const createColumns = (
<HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} />
),
cell: ({ getValue }) => (
- <div
- className="max-w-48 truncate ms-1"
- title={getValue() as string}
- style={{ width: `${longestModel / 1.05}ch` }}
- >
- {getValue() as string}
+ <div className="ms-1 relative w-fit max-w-44" title={getValue() as string}>
+ <span className="invisible block whitespace-nowrap" aria-hidden="true">
+ {longestModel}
+ </span>
+ <span className="absolute inset-0 truncate">{getValue() as string}</span>
</div>
),
},
@@ -320,7 +320,7 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
// Calculate the right width for the columns based on the longest strings among the displayed devices
const { longestName, longestModel, longestDevice } = useMemo(() => {
- const result = { longestName: 0, longestModel: 0, longestDevice: 0 }
+ const result = { longestName: "", longestModel: "", longestDevice: "" }
if (!smartDevices || Object.keys(allSystems).length === 0) {
return result
}
@@ -329,10 +329,16 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
if (!systemId && !seenSystems.has(device.system)) {
seenSystems.add(device.system)
const name = allSystems[device.system]?.name ?? ""
- result.longestName = Math.max(result.longestName, getVisualStringWidth(name))
+ if (name.length > result.longestName.length) {
+ result.longestName = name
+ }
+ }
+ if ((device.model ?? "").length > result.longestModel.length) {
+ result.longestModel = device.model ?? ""
+ }
+ if ((device.name ?? "").length > result.longestDevice.length) {
+ result.longestDevice = device.name ?? ""
}
- result.longestModel = Math.max(result.longestModel, getVisualStringWidth(device.model ?? ""))
- result.longestDevice = Math.max(result.longestDevice, getVisualStringWidth(device.name ?? ""))
}
return result
}, [smartDevices, systemId, allSystems])
diff --git a/internal/site/src/components/routes/system/use-system-data.ts b/internal/site/src/components/routes/system/use-system-data.ts
index 0a5dc15f1..bd2f2af48 100644
--- a/internal/site/src/components/routes/system/use-system-data.ts
+++ b/internal/site/src/components/routes/system/use-system-data.ts
@@ -26,7 +26,7 @@ import type {
SystemStatsRecord,
} from "@/types"
import { $router, navigate } from "../../router"
-import { appendData, cache, getStats, getTimeData, makeContainerData, makeContainerPoint } from "./chart-data"
+import { appendData, cache, getStats, makeContainerData, makeContainerPoint } from "./chart-data"
export type SystemData = ReturnType<typeof useSystemData>
@@ -185,16 +185,11 @@ export function useSystemData(id: string) {
const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v])
const chartData: ChartData = useMemo(() => {
- const lastCreated = Math.max(
- (systemStats.at(-1)?.created as number) ?? 0,
- (containerData.at(-1)?.created as number) ?? 0
- )
return {
systemStats,
containerData,
chartTime,
orientation: direction === "rtl" ? "right" : "left",
- ...getTimeData(chartTime, lastCreated),
agentVersion,
}
}, [systemStats, containerData, direction])
@@ -234,8 +229,8 @@ export function useSystemData(id: string) {
}
Promise.allSettled([
- getStats<SystemStatsRecord>("system_stats", systemId, chartTime),
- getStats<ContainerStatsRecord>("container_stats", systemId, chartTime),
+ getStats<SystemStatsRecord>("system_stats", systemId, chartTime, cachedSystemStats),
+ getStats<ContainerStatsRecord>("container_stats", systemId, chartTime, cachedContainerData),
]).then(([systemStats, containerStats]) => {
// Ignore responses for a previous system or chart time
if (requestId !== statsRequestId.current) {
@@ -327,7 +322,7 @@ export function useSystemData(id: string) {
// derived values
const isLongerChart = !["1m", "1h"].includes(chartTime)
const showMax = maxValues && isLongerChart
- const dataEmpty = !chartLoading && chartData.systemStats.length === 0
+ const dataEmpty = !chartLoading && chartData.systemStats?.length === 0
const lastGpus = systemStats.at(-1)?.stats?.g
const isPodman = details?.podman ?? system.info?.p ?? false
diff --git a/internal/site/src/components/systems-table/systems-table-columns.tsx b/internal/site/src/components/systems-table/systems-table-columns.tsx
index 40d6c7d52..1db27673d 100644
--- a/internal/site/src/components/systems-table/systems-table-columns.tsx
+++ b/internal/site/src/components/systems-table/systems-table-columns.tsx
@@ -26,7 +26,7 @@ import { memo, useMemo, useRef, useState } from "react"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
import { isReadOnlyUser, pb } from "@/lib/api"
import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums"
-import { $longestSystemNameLen, $userSettings } from "@/lib/stores"
+import { $longestSystemName, $userSettings } from "@/lib/stores"
import {
cn,
copyToClipboard,
@@ -135,7 +135,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
Icon: ServerIcon,
cell: (info) => {
const { name, id } = info.row.original
- const longestName = useStore($longestSystemNameLen)
+ const longestName = useStore($longestSystemName)
const linkUrl = getPagePath($router, "system", { id })
return (
@@ -145,8 +145,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
<Link
href={linkUrl}
tabIndex={-1}
- className="truncate z-10 relative"
- style={{ width: `${longestName / 1.05}ch` }}
+ className="relative w-fit max-w-48 z-10"
onMouseEnter={(e) => {
// set title on hover if text is truncated to show full name
const a = e.currentTarget
@@ -157,7 +156,10 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
}
}}
>
- {name}
+ <span className="invisible block" aria-hidden="true">
+ {longestName}
+ </span>
+ <span className="absolute inset-0 truncate">{name}</span>
</Link>
</span>
<Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link>
diff --git a/internal/site/src/components/ui/chart.tsx b/internal/site/src/components/ui/chart.tsx
index 5fc4e2fee..35c606ce9 100644
--- a/internal/site/src/components/ui/chart.tsx
+++ b/internal/site/src/components/ui/chart.tsx
@@ -1,11 +1,11 @@
-import type { JSX } from "react"
import { useLingui } from "@lingui/react/macro"
import * as React from "react"
import * as RechartsPrimitive from "recharts"
import { chartTimeData, cn } from "@/lib/utils"
-import type { ChartData } from "@/types"
+import type { ChartTimes } from "@/types"
import { Separator } from "./separator"
-import { AxisDomain } from "recharts/types/util/types"
+import type { AxisDomain } from "recharts/types/util/types"
+import { timeTicks } from "d3-time"
// Format: { THEME_NAME: CSS_SELECTOR }
const THEMES = { light: "", dark: ".dark" } as const
@@ -101,7 +101,7 @@ const ChartTooltipContent = React.forwardRef<
labelKey?: string
unit?: string
filter?: string
- contentFormatter?: (item: any, key: string) => React.ReactNode | string
+ contentFormatter?: (item: unknown, key: string) => React.ReactNode | string
truncate?: boolean
showTotal?: boolean
totalLabel?: React.ReactNode
@@ -175,7 +175,13 @@ const ChartTooltipContent = React.forwardRef<
}
const totalKey = "__total__"
- const totalItem: any = {
+ const totalItem: {
+ value: number
+ name: string
+ dataKey: string
+ color: string | undefined
+ payload?: unknown
+ } = {
value: totalValue,
name: totalName,
dataKey: totalKey,
@@ -222,6 +228,11 @@ const ChartTooltipContent = React.forwardRef<
return null
}
+ payload = payload.filter((item) => item.value != null)
+ if (!payload.length) {
+ return null
+ }
+
// const nestLabel = payload.length === 1 && indicator !== 'dot'
const nestLabel = false
@@ -400,26 +411,57 @@ function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key:
return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config]
}
-let cachedAxis: JSX.Element
-const xAxis = ({ domain, ticks, chartTime }: ChartData) => {
- if (cachedAxis && domain[0] === cachedAxis.props.domain[0]) {
- return cachedAxis
+interface XAxisData {
+ el: React.ReactElement
+ domain: [number, number]
+}
+
+const xAxisCache = new Map<ChartTimes, XAxisData>()
+
+function createXAxisData(chartTime: ChartTimes): XAxisData {
+ // console.log("Creating XAxis for", chartTime, new Date())
+ const axisEndTime = Date.now() + 500
+ const axisEndDate = new Date(axisEndTime)
+ const startTime = chartTimeData[chartTime].getOffset(axisEndDate)
+ const ticks = timeTicks(startTime, axisEndDate, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
+ const domain: [number, number] = [startTime.getTime(), axisEndTime]
+
+ return {
+ domain,
+ el: (
+ <RechartsPrimitive.XAxis
+ dataKey="created"
+ domain={domain}
+ ticks={ticks}
+ allowDataOverflow
+ type="number"
+ scale="time"
+ minTickGap={12}
+ tickMargin={8}
+ axisLine={false}
+ tickFormatter={chartTimeData[chartTime].format}
+ />
+ ),
}
- cachedAxis = (
- <RechartsPrimitive.XAxis
- dataKey="created"
- domain={domain}
- ticks={ticks}
- allowDataOverflow
- type="number"
- scale="time"
- minTickGap={12}
- tickMargin={8}
- axisLine={false}
- tickFormatter={chartTimeData[chartTime].format}
- />
- )
- return cachedAxis
+}
+
+function xAxis(chartTime: ChartTimes, lastCreated: number) {
+ if (!lastCreated) {
+ return null
+ }
+ const cachedAxis = xAxisCache.get(chartTime)
+
+ const expectedInterval = chartTimeData[chartTime].expectedInterval
+ const conservativeEndTime = Date.now() - expectedInterval / 2
+ const axisEndTime = Math.max(lastCreated, conservativeEndTime)
+
+ if (cachedAxis && axisEndTime < cachedAxis.domain[1]) {
+ return cachedAxis.el
+ }
+
+ const axisData = createXAxisData(chartTime)
+ xAxisCache.set(chartTime, axisData)
+ return axisData.el
}
export {
diff --git a/internal/site/src/components/ui/dropdown-menu.tsx b/internal/site/src/components/ui/dropdown-menu.tsx
index b49693aa0..7312bd757 100644
--- a/internal/site/src/components/ui/dropdown-menu.tsx
+++ b/internal/site/src/components/ui/dropdown-menu.tsx
@@ -90,8 +90,10 @@ DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName
const DropdownMenuCheckboxItem = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
- React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem>
->(({ className, children, checked, ...props }, ref) => (
+ React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem> & {
+ indicatorClassName?: string
+ }
+>(({ className, children, checked, indicatorClassName, ...props }, ref) => (
<DropdownMenuPrimitive.CheckboxItem
ref={ref}
className={cn(
@@ -101,7 +103,7 @@ const DropdownMenuCheckboxItem = React.forwardRef<
checked={checked}
{...props}
>
- <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+ <span className={cn("absolute left-2 flex h-3.5 w-3.5 items-center justify-center", indicatorClassName)}>
<DropdownMenuPrimitive.ItemIndicator>
<Check className="h-4 w-4" />
</DropdownMenuPrimitive.ItemIndicator>
diff --git a/internal/site/src/components/ui/table.tsx b/internal/site/src/components/ui/table.tsx
index a22206c18..377565a36 100644
--- a/internal/site/src/components/ui/table.tsx
+++ b/internal/site/src/components/ui/table.tsx
@@ -41,7 +41,7 @@ const TableRow = React.forwardRef<HTMLTableRowElement, React.HTMLAttributes<HTML
<tr
ref={ref}
className={cn(
- "border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted!",
+ "border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted/40!",
className
)}
{...props}
diff --git a/internal/site/src/lib/alerts.ts b/internal/site/src/lib/alerts.ts
index b0329af01..b5f52a7dc 100644
--- a/internal/site/src/lib/alerts.ts
+++ b/internal/site/src/lib/alerts.ts
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro"
-import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
+import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
import type { RecordSubscription } from "pocketbase"
import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
import { $alerts } from "@/lib/stores"
@@ -9,210 +9,223 @@ import { ThermometerIcon, BatteryMediumIcon, HourglassIcon } from "@/components/
/** Alert info for each alert type */
export const alertInfo: Record<string, AlertInfo> = {
- Status: {
- name: () => t`Status`,
- unit: "",
- icon: ServerIcon,
- desc: () => t`Triggers when status switches between up and down`,
- /** "for x minutes" is appended to desc when only one value */
- singleDesc: () => `${t`System`} ${t`Down`}`,
- },
- CPU: {
- name: () => t`CPU Usage`,
- unit: "%",
- icon: CpuIcon,
- desc: () => t`Triggers when CPU usage exceeds a threshold`,
- },
- CPUIOWait: {
- name: () => t`CPU I/O Wait`,
- unit: "%",
- icon: CpuIcon,
- desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
- },
- CPUSteal: {
- name: () => t`CPU Steal Time`,
- unit: "%",
- icon: CpuIcon,
- desc: () => t`Triggers when CPU steal time exceeds a threshold`,
- },
- Memory: {
- name: () => t`Memory Usage`,
- unit: "%",
- icon: MemoryStickIcon,
- desc: () => t`Triggers when memory usage exceeds a threshold`,
- },
- Disk: {
- name: () => t`Disk Usage`,
- unit: "%",
- icon: HardDriveIcon,
- desc: () => t`Triggers when usage of any disk exceeds a threshold`,
- },
- Bandwidth: {
- name: () => t`Bandwidth`,
- unit: " MB/s",
- icon: EthernetIcon,
- desc: () => t`Triggers when combined up/down exceeds a threshold`,
- max: 250,
- },
- GPU: {
- name: () => t`GPU Usage`,
- unit: "%",
- icon: GpuIcon,
- desc: () => t`Triggers when GPU usage exceeds a threshold`,
- },
- Temperature: {
- name: () => t`Temperature`,
- unit: "°C",
- icon: ThermometerIcon,
- desc: () => t`Triggers when any sensor exceeds a threshold`,
- },
- LoadAvg1: {
- name: () => t`Load Average 1m`,
- unit: "",
- icon: HourglassIcon,
- max: 100,
- min: 0.1,
- start: 10,
- step: 0.1,
- desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
- },
- LoadAvg5: {
- name: () => t`Load Average 5m`,
- unit: "",
- icon: HourglassIcon,
- max: 100,
- min: 0.1,
- start: 10,
- step: 0.1,
- desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
- },
- LoadAvg15: {
- name: () => t`Load Average 15m`,
- unit: "",
- icon: HourglassIcon,
- min: 0.1,
- max: 100,
- start: 10,
- step: 0.1,
- desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
- },
- Battery: {
- name: () => t`Battery`,
- unit: "%",
- icon: BatteryMediumIcon,
- desc: () => t`Triggers when battery charge drops below a threshold`,
- start: 20,
- invert: true,
- },
- ContainerHealth: {
- name: () => t`Container Health`,
- unit: "",
- icon: ContainerIcon,
- desc: () => t`Triggers when a container's health check reports unhealthy`,
- note: () =>
- t`Notifications may include recent container log excerpts.`,
- triggeredDesc: () => t`One or more containers are unhealthy`,
- singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
+ Status: {
+ name: () => t`Status`,
+ unit: "",
+ icon: ServerIcon,
+ desc: () => t`Triggers when status switches between up and down`,
+ /** "for x minutes" is appended to desc when only one value */
+ singleDesc: () => `${t`System`} ${t`Down`}`,
+ },
+ CPU: {
+ name: () => t`CPU Usage`,
+ unit: "%",
+ icon: CpuIcon,
+ desc: () => t`Triggers when CPU usage exceeds a threshold`,
+ },
+ CPUIOWait: {
+ name: () => t`CPU I/O Wait`,
+ unit: "%",
+ icon: CpuIcon,
+ desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
+ },
+ CPUSteal: {
+ name: () => t`CPU Steal Time`,
+ unit: "%",
+ icon: CpuIcon,
+ desc: () => t`Triggers when CPU steal time exceeds a threshold`,
+ },
+ Memory: {
+ name: () => t`Memory Usage`,
+ unit: "%",
+ icon: MemoryStickIcon,
+ desc: () => t`Triggers when memory usage exceeds a threshold`,
+ },
+ Disk: {
+ name: () => t`Disk Usage`,
+ unit: "%",
+ icon: HardDriveIcon,
+ desc: () => t`Triggers when usage of any disk exceeds a threshold`,
+ },
+ Bandwidth: {
+ name: () => t`Bandwidth`,
+ unit: " MB/s",
+ icon: EthernetIcon,
+ desc: () => t`Triggers when combined up/down exceeds a threshold`,
+ max: 250,
+ },
+ NetworkMonitorLoss: {
+ name: () => t`Network Monitor Loss`,
+ unit: "%",
+ icon: NetworkIcon,
+ desc: () => t`Triggers when one hour loss exceeds a threshold`,
+ // note: () => t`Uses available history after three probes.`,
+ noDuration: true,
+ min: 0,
+ max: 99.9,
+ step: 0.1,
+ start: 5,
+ },
+ GPU: {
+ name: () => t`GPU Usage`,
+ unit: "%",
+ icon: GpuIcon,
+ desc: () => t`Triggers when GPU usage exceeds a threshold`,
+ },
+ Temperature: {
+ name: () => t`Temperature`,
+ unit: "°C",
+ icon: ThermometerIcon,
+ desc: () => t`Triggers when any sensor exceeds a threshold`,
+ },
+ LoadAvg1: {
+ name: () => t`Load Average 1m`,
+ unit: "",
+ icon: HourglassIcon,
+ max: 100,
+ min: 0.1,
+ start: 10,
+ step: 0.1,
+ desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
+ },
+ LoadAvg5: {
+ name: () => t`Load Average 5m`,
+ unit: "",
+ icon: HourglassIcon,
+ max: 100,
+ min: 0.1,
+ start: 10,
+ step: 0.1,
+ desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
+ },
+ LoadAvg15: {
+ name: () => t`Load Average 15m`,
+ unit: "",
+ icon: HourglassIcon,
+ min: 0.1,
+ max: 100,
+ start: 10,
+ step: 0.1,
+ desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
+ },
+ Battery: {
+ name: () => t`Battery`,
+ unit: "%",
+ icon: BatteryMediumIcon,
+ desc: () => t`Triggers when battery charge drops below a threshold`,
+ start: 20,
+ invert: true,
+ },
+ ContainerHealth: {
+ name: () => t`Container Health`,
+ unit: "",
+ icon: ContainerIcon,
+ desc: () => t`Triggers when a container's health check reports unhealthy`,
+ note: () =>
+ t`Notifications may include recent container log excerpts.`,
+ triggeredDesc: () => t`One or more containers are unhealthy`,
+ singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
- },
- SystemdFailed: {
- name: () => t`Failed Services`,
- unit: "",
- icon: ServerCrashIcon,
- desc: () => t`Triggers when any systemd service enters the failed state`,
- triggeredDesc: () => t`One or more services are in a failed state`,
- /** Fires on first observation - the agent only polls systemd every 10 minutes */
- noDuration: true,
- },
+ },
+ SystemdFailed: {
+ name: () => t`Failed Services`,
+ unit: "",
+ icon: ServerCrashIcon,
+ desc: () => t`Triggers when any systemd service enters the failed state`,
+ triggeredDesc: () => t`One or more services are in a failed state`,
+ /** Fires on first observation - the agent only polls systemd every 10 minutes */
+ noDuration: true,
+ noThreshold: true,
+ },
} as const
/** Helper to manage user alerts */
export const alertManager = (() => {
- const collection = pb.collection<AlertRecord>("alerts")
- let unsub: () => void
+ const collection = pb.collection<AlertRecord>("alerts")
+ let unsub: () => void
- /** Fields to fetch from alerts collection */
- const fields = "id,name,system,value,min,triggered"
+ /** Fields to fetch from alerts collection */
+ const fields = "id,name,system,value,min,triggered"
- /** Fetch alerts from collection */
- async function fetchAlerts(): Promise<AlertRecord[]> {
- return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
- }
+ /** Fetch alerts from collection */
+ async function fetchAlerts(): Promise<AlertRecord[]> {
+ return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
+ }
- /** Format alerts into a map of system id to alert name to alert record */
- function add(alerts: AlertRecord[]) {
- for (const alert of alerts) {
- const systemId = alert.system
- const systemAlerts = $alerts.get()[systemId] ?? new Map()
- const newAlerts = new Map(systemAlerts)
- newAlerts.set(alert.name, alert)
- $alerts.setKey(systemId, newAlerts)
- }
- }
+ /** Format alerts into a map of system id to alert name to alert record */
+ function add(alerts: AlertRecord[]) {
+ for (const alert of alerts) {
+ const systemId = alert.system
+ const systemAlerts = $alerts.get()[systemId] ?? new Map()
+ const newAlerts = new Map(systemAlerts)
+ newAlerts.set(alert.name, alert)
+ $alerts.setKey(systemId, newAlerts)
+ }
+ }
- function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
- for (const alert of alerts) {
- const systemId = alert.system
- const systemAlerts = $alerts.get()[systemId]
- const newAlerts = new Map(systemAlerts)
- newAlerts.delete(alert.name)
- $alerts.setKey(systemId, newAlerts)
- }
- }
+ function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
+ for (const alert of alerts) {
+ const systemId = alert.system
+ const systemAlerts = $alerts.get()[systemId]
+ const newAlerts = new Map(systemAlerts)
+ newAlerts.delete(alert.name)
+ $alerts.setKey(systemId, newAlerts)
+ }
+ }
- const actionFns = {
- create: add,
- update: add,
- delete: remove,
- }
+ const actionFns = {
+ create: add,
+ update: add,
+ delete: remove,
+ }
- // batch alert updates to prevent unnecessary re-renders when adding many alerts at once
- const batchUpdate = (() => {
- const batch = new Map<string, RecordSubscription<AlertRecord>>()
- let timeout: ReturnType<typeof setTimeout>
+ // batch alert updates to prevent unnecessary re-renders when adding many alerts at once
+ const batchUpdate = (() => {
+ const batch = new Map<string, RecordSubscription<AlertRecord>>()
+ let timeout: ReturnType<typeof setTimeout>
- return (data: RecordSubscription<AlertRecord>) => {
- const { record } = data
- batch.set(`${record.system}${record.name}`, data)
- clearTimeout(timeout)
- timeout = setTimeout(() => {
- const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
- for (const { action, record } of batch.values()) {
- groups[action]?.push(record)
- }
- for (const key in groups) {
- if (groups[key].length) {
- actionFns[key as keyof typeof actionFns]?.(groups[key])
- }
- }
- batch.clear()
- }, 50)
- }
- })()
+ return (data: RecordSubscription<AlertRecord>) => {
+ const { record } = data
+ batch.set(`${record.system}${record.name}`, data)
+ clearTimeout(timeout)
+ timeout = setTimeout(() => {
+ const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
+ for (const { action, record } of batch.values()) {
+ groups[action]?.push(record)
+ }
+ for (const key in groups) {
+ if (groups[key].length) {
+ actionFns[key as keyof typeof actionFns]?.(groups[key])
+ }
+ }
+ batch.clear()
+ }, 50)
+ }
+ })()
- async function subscribe() {
- unsub = await collection.subscribe("*", batchUpdate, { fields })
- }
+ async function subscribe() {
+ unsub = await collection.subscribe("*", batchUpdate, { fields })
+ }
- function unsubscribe() {
- unsub?.()
- }
+ function unsubscribe() {
+ unsub?.()
+ }
- async function refresh() {
- const records = await fetchAlerts()
- add(records)
- }
+ async function refresh() {
+ const records = await fetchAlerts()
+ add(records)
+ }
- return {
- /** Add alerts to store */
- add,
- /** Remove alerts from store */
- remove,
- /** Subscribe to alerts */
- subscribe,
- /** Unsubscribe from alerts */
- unsubscribe,
- /** Refresh alerts with latest data from hub */
- refresh,
- }
+ return {
+ /** Add alerts to store */
+ add,
+ /** Remove alerts from store */
+ remove,
+ /** Subscribe to alerts */
+ subscribe,
+ /** Unsubscribe from alerts */
+ unsubscribe,
+ /** Refresh alerts with latest data from hub */
+ refresh,
+ }
})()
diff --git a/internal/site/src/lib/api.ts b/internal/site/src/lib/api.ts
index cfa44d98e..fdcf582e0 100644
--- a/internal/site/src/lib/api.ts
+++ b/internal/site/src/lib/api.ts
@@ -106,8 +106,11 @@ export async function updateUserSettings() {
}
}
-export function getPbTimestamp(timeString: ChartTimes, d?: Date) {
+export function getPbTimestamp(timeString: ChartTimes, d?: Date, createdIsNumber?: boolean) {
d ||= chartTimeData[timeString].getOffset(new Date())
+ if (createdIsNumber) {
+ return d.getTime()
+ }
const year = d.getUTCFullYear()
const month = String(d.getUTCMonth() + 1).padStart(2, "0")
const day = String(d.getUTCDate()).padStart(2, "0")
diff --git a/internal/site/src/lib/network-monitor-utils.ts b/internal/site/src/lib/network-monitor-utils.ts
new file mode 100644
index 000000000..9ab1303d7
--- /dev/null
+++ b/internal/site/src/lib/network-monitor-utils.ts
@@ -0,0 +1,17 @@
+import type { MonitorStats, NetworkMonitorRecord, RawMonitorStatsRecord } from "@/types"
+
+/** Derive chart metrics from the counts and response sum stored at every retention tier. */
+export function getMonitorStats(record: RawMonitorStatsRecord): MonitorStats {
+ return {
+ res_avg: record.success_count > 0 ? record.res_sum / record.success_count : 0,
+ res_min: record.res_min,
+ res_max: record.res_max,
+ loss: record.total_count > 0 ? ((record.total_count - record.success_count) / record.total_count) * 100 : 0,
+ }
+}
+
+export function getMonitorTarget(monitor: Pick<NetworkMonitorRecord, "target" | "protocol" | "port">) {
+ if (monitor.protocol !== "tcp") return monitor.target
+ const host = monitor.target.includes(":") && !monitor.target.startsWith("[") ? `[${monitor.target}]` : monitor.target
+ return `${host}:${monitor.port}`
+}
diff --git a/internal/site/src/lib/stores.ts b/internal/site/src/lib/stores.ts
index 0778c1a3d..49184b8fd 100644
--- a/internal/site/src/lib/stores.ts
+++ b/internal/site/src/lib/stores.ts
@@ -64,6 +64,9 @@ export const $containerFilter = atom("")
/** Temperature chart filter */
export const $temperatureFilter = atom("")
+/** Filter for network monitor charts (compare page and per-system monitor charts) */
+export const $monitorFilter = atom("")
+
/** Fan-speed chart filter */
export const $fanFilter = atom("")
@@ -73,7 +76,5 @@ export const $copyContent = atom("")
/** Direction for localization */
export const $direction = atom<"ltr" | "rtl">("ltr")
-/** Longest system name length. Used to set table column width. I know this
- * is stupid but the table is virtualized and I know this will work.
- */
-export const $longestSystemNameLen = atom(8)
+/** Longest system name string. Used to reserve width in virtualized tables. */
+export const $longestSystemName = atom("")
diff --git a/internal/site/src/lib/systemsManager.ts b/internal/site/src/lib/systemsManager.ts
index f3c87fbfc..287d58eef 100644
--- a/internal/site/src/lib/systemsManager.ts
+++ b/internal/site/src/lib/systemsManager.ts
@@ -5,20 +5,17 @@ import {
$allSystemsById,
$allSystemsByName,
$downSystems,
- $longestSystemNameLen,
+ $longestSystemName,
$pausedSystems,
$upSystems,
} from "@/lib/stores"
-import { getVisualStringWidth, updateFavicon } from "@/lib/utils"
+import { isVisuallyLonger, updateFavicon } from "@/lib/utils"
import type { SystemRecord } from "@/types"
import { SystemStatus } from "./enums"
const COLLECTION = pb.collection<SystemRecord>("systems")
const FIELDS_DEFAULT = "id,name,host,port,info,status"
-/** Maximum system name length for display purposes */
-const MAX_SYSTEM_NAME_LENGTH = 22
-
let initialized = false
// biome-ignore lint/suspicious/noConfusingVoidType: typescript rocks
let unsub: (() => void) | undefined | void
@@ -44,7 +41,7 @@ export function init() {
}
if (!newSystem) {
- onSystemsChanged(newSystems, undefined)
+ onSystemsChanged(newSystems, newSystem, oldSystem)
return
}
@@ -68,20 +65,28 @@ export function init() {
}
// run things that need to be done when systems change
- onSystemsChanged(newSystems, newSystem)
+ onSystemsChanged(newSystems, newSystem, oldSystem)
})
}
-/** Update the longest system name length and favicon based on system status */
-function onSystemsChanged(_: Record<string, SystemRecord>, changedSystem: SystemRecord | undefined) {
+/** Update the longest system name string and favicon based on system status */
+function onSystemsChanged(systems: Record<string, SystemRecord>, newSystem?: SystemRecord, oldSystem?: SystemRecord) {
const downSystemsStore = $downSystems.get()
const downSystems = Object.values(downSystemsStore)
- // Update longest system name length
- const longestName = $longestSystemNameLen.get()
- const nameLen = Math.min(MAX_SYSTEM_NAME_LENGTH, getVisualStringWidth(changedSystem?.name || ""))
- if (nameLen > longestName) {
- $longestSystemNameLen.set(nameLen)
+ // if the old system's old name was the longest, we need to find the new longest name
+ // otherwise, if the changed system's new name is longer than the current longest, update it
+ const longestName = $longestSystemName.get()
+ if (oldSystem?.name === longestName && oldSystem.name !== newSystem?.name) {
+ let newLongest = ""
+ for (const id in systems) {
+ if (isVisuallyLonger(systems[id].name, newLongest)) {
+ newLongest = systems[id].name
+ }
+ }
+ $longestSystemName.set(newLongest)
+ } else if (newSystem && newSystem.name !== longestName && isVisuallyLonger(newSystem.name, longestName)) {
+ $longestSystemName.set(newSystem.name)
}
updateFavicon(downSystems.length)
diff --git a/internal/site/src/lib/use-network-monitors.ts b/internal/site/src/lib/use-network-monitors.ts
new file mode 100644
index 000000000..8b8e5014c
--- /dev/null
+++ b/internal/site/src/lib/use-network-monitors.ts
@@ -0,0 +1,351 @@
+import { chartTimeData } from "@/lib/utils"
+import { getMonitorStats } from "@/lib/network-monitor-utils"
+import type {
+ ChartTimes,
+ MonitorStats,
+ NetworkMonitorRecord,
+ NetworkMonitorStatsRecord,
+ RawMonitorStatsRecord,
+} from "@/types"
+import { useEffect, useRef, useState } from "react"
+import { appendData } from "@/components/routes/system/chart-data"
+import { pb, getPbTimestamp } from "@/lib/api"
+import { toast } from "@/components/ui/use-toast"
+import type { RecordListOptions, RecordSubscription } from "pocketbase"
+
+const cache = new Map<string, NetworkMonitorStatsRecord[]>()
+
+function getCacheValue(monitorId: string, chartTime: ChartTimes | "rt") {
+ return cache.get(`${monitorId}:${chartTime}`) || []
+}
+
+function appendCacheValue(
+ monitorId: string,
+ chartTime: ChartTimes | "rt",
+ newStats: NetworkMonitorStatsRecord[],
+ maxPoints = 100
+) {
+ const cache_key = `${monitorId}:${chartTime}`
+ const existingStats = getCacheValue(monitorId, chartTime)
+ if (existingStats) {
+ const { expectedInterval } = chartTimeData[chartTime]
+ const updatedStats = appendData(existingStats, newStats, expectedInterval, maxPoints)
+ cache.set(cache_key, updatedStats)
+ return updatedStats
+ } else {
+ cache.set(cache_key, newStats)
+ return newStats
+ }
+}
+
+/** Merge an array of per-monitor raw records into the map-keyed format expected by chart components. */
+export function mergeMonitorStats(rawRecords: RawMonitorStatsRecord[]): NetworkMonitorStatsRecord[] {
+ const byTimestamp = new Map<number, Record<string, MonitorStats>>()
+ for (const rec of rawRecords) {
+ let statsMap = byTimestamp.get(rec.created)
+ if (!statsMap) {
+ statsMap = {}
+ byTimestamp.set(rec.created, statsMap)
+ }
+ statsMap[rec.monitor] = getMonitorStats(rec)
+ }
+ return Array.from(byTimestamp.entries())
+ .sort(([a], [b]) => a - b)
+ .map(([created, stats]) => ({ created, stats }))
+}
+
+/** Fetch stats for one monitor and time range, returning merged chart records. */
+async function fetchMonitorStats(
+ monitorId: string,
+ chartTime: ChartTimes,
+ cached?: NetworkMonitorStatsRecord[]
+): Promise<NetworkMonitorStatsRecord[]> {
+ const lastCached = cached?.at(-1)?.created as number | undefined
+ const rawRecords = await pb.collection<RawMonitorStatsRecord>("network_monitor_stats").getFullList({
+ filter: pb.filter("monitor={:id} && created>{:created} && type={:type}", {
+ id: monitorId,
+ created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, true),
+ type: chartTimeData[chartTime].type,
+ }),
+ fields: "monitor,res_min,res_max,total_count,success_count,res_sum,created",
+ sort: "created",
+ })
+ return mergeMonitorStats(rawRecords)
+}
+
+const NETWORK_MONITOR_FIELDS =
+ "id,system,target,protocol,port,interval,res,resMin1h,resMax1h,resAvg1h,loss1h,enabled,updated"
+
+interface UseNetworkMonitorsProps {
+ systemId?: string
+}
+
+export function useNetworkMonitors(props: UseNetworkMonitorsProps) {
+ const { systemId } = props
+
+ const [monitors, setMonitors] = useState<NetworkMonitorRecord[]>([])
+ const [isLoading, setIsLoading] = useState(true)
+ const pendingMonitorEvents = useRef(new Map<string, RecordSubscription<NetworkMonitorRecord>>())
+ const monitorBatchTimeout = useRef<ReturnType<typeof setTimeout> | null>(null)
+
+ // initial load
+ useEffect(() => {
+ let cancelled = false
+ setIsLoading(true)
+ setMonitors([])
+ fetchMonitors(systemId).then((monitors) => {
+ if (cancelled) return
+ setMonitors(monitors)
+ setIsLoading(false)
+ })
+ return () => {
+ cancelled = true
+ }
+ }, [systemId])
+
+ // subscribe to updates
+ useEffect(() => {
+ let unsubscribe: (() => void) | undefined
+
+ function flushPendingMonitorEvents() {
+ monitorBatchTimeout.current = null
+ if (!pendingMonitorEvents.current.size) {
+ return
+ }
+ const events = pendingMonitorEvents.current
+ pendingMonitorEvents.current = new Map()
+ setMonitors((currentMonitors) => {
+ return applyMonitorEvents(currentMonitors ?? [], events.values(), systemId)
+ })
+ }
+
+ const pbOptions: RecordListOptions = { fields: NETWORK_MONITOR_FIELDS }
+ if (systemId) {
+ pbOptions.filter = pb.filter("system = {:system}", { system: systemId })
+ }
+
+ ;(async () => {
+ try {
+ unsubscribe = await pb.collection<NetworkMonitorRecord>("network_monitors").subscribe(
+ "*",
+ (event) => {
+ pendingMonitorEvents.current.set(event.record.id, event)
+ if (!monitorBatchTimeout.current) {
+ monitorBatchTimeout.current = setTimeout(flushPendingMonitorEvents, 50)
+ }
+ },
+ pbOptions
+ )
+ } catch (error) {
+ console.error("Failed to subscribe to monitors", error)
+ }
+ })()
+
+ return () => {
+ if (monitorBatchTimeout.current !== null) {
+ clearTimeout(monitorBatchTimeout.current)
+ monitorBatchTimeout.current = null
+ }
+ pendingMonitorEvents.current.clear()
+ unsubscribe?.()
+ }
+ }, [systemId])
+
+ return { monitors, isLoading }
+}
+
+interface UseNetworkMonitorStatsProps {
+ systemId: string
+ monitorId: string
+ chartTime: ChartTimes
+ enabled?: boolean
+}
+
+export function useNetworkMonitorStats(props: UseNetworkMonitorStatsProps) {
+ const { systemId, monitorId, chartTime, enabled = true } = props
+ const [monitorStats, setMonitorStats] = useState<NetworkMonitorStatsRecord[]>([])
+ // pending raw events to be merged (keyed by monitor+created)
+ const pendingRaw = useRef(new Map<string, RawMonitorStatsRecord>())
+ const mergeBatchTimeout = useRef<ReturnType<typeof setTimeout> | null>(null)
+
+ useEffect(() => {
+ setMonitorStats(getCacheValue(monitorId, chartTime === "1m" ? "rt" : chartTime))
+ }, [monitorId, chartTime])
+
+ // Fetch only the selected monitor's missing history.
+ useEffect(() => {
+ if (!enabled || chartTime === "1m") {
+ return
+ }
+
+ let cancelled = false
+ const { expectedInterval } = chartTimeData[chartTime]
+ const cachedMonitorStats = getCacheValue(monitorId, chartTime)
+
+ if (cachedMonitorStats.length) {
+ setMonitorStats(cachedMonitorStats)
+ const lastCreated = cachedMonitorStats.at(-1)?.created
+ if (lastCreated && Date.now() - lastCreated < expectedInterval * 0.9) {
+ return
+ }
+ }
+
+ fetchMonitorStats(monitorId, chartTime, cachedMonitorStats)
+ .then((newMonitorStats) => {
+ if (cancelled) return
+ setMonitorStats(appendCacheValue(monitorId, chartTime, newMonitorStats))
+ })
+ .catch((error) => {
+ if (!cancelled) console.error("Failed to fetch monitor stats:", error)
+ })
+ return () => {
+ cancelled = true
+ }
+ }, [monitorId, chartTime, enabled])
+
+ // subscribe to new per-monitor stats records; batch them into merged chart records
+ useEffect(() => {
+ if (!enabled || chartTime === "1m") {
+ return
+ }
+ let cancelled = false
+ let unsubscribe: (() => void) | undefined
+ const pbOptions = {
+ fields: "monitor,res_min,res_max,total_count,success_count,res_sum,created,type",
+ filter: pb.filter("monitor={:monitor} && type={:type}", {
+ monitor: monitorId,
+ type: chartTimeData[chartTime].type,
+ }),
+ }
+
+ function flushPending() {
+ mergeBatchTimeout.current = null
+ const pending = pendingRaw.current
+ pendingRaw.current = new Map()
+ const merged = mergeMonitorStats(Array.from(pending.values()))
+ if (merged.length > 0) {
+ const newStats = appendCacheValue(monitorId, chartTime, merged)
+ setMonitorStats(newStats)
+ }
+ }
+
+ ;(async () => {
+ try {
+ unsubscribe = await pb.collection<RawMonitorStatsRecord>("network_monitor_stats").subscribe(
+ "*",
+ (event) => {
+ if (cancelled || event.action !== "create") {
+ return
+ }
+ const rec = event.record
+ pendingRaw.current.set(`${rec.monitor}:${rec.created}`, rec)
+ if (!mergeBatchTimeout.current) {
+ mergeBatchTimeout.current = setTimeout(flushPending, 200)
+ }
+ },
+ pbOptions
+ )
+ if (cancelled) unsubscribe()
+ } catch (error) {
+ console.error("Failed to subscribe to monitor stats:", error)
+ }
+ })()
+
+ return () => {
+ cancelled = true
+ if (mergeBatchTimeout.current) {
+ clearTimeout(mergeBatchTimeout.current)
+ mergeBatchTimeout.current = null
+ }
+ pendingRaw.current.clear()
+ unsubscribe?.()
+ }
+ }, [monitorId, chartTime, enabled])
+
+ // subscribe to realtime metrics if chart time is 1m
+ useEffect(() => {
+ if (!enabled || chartTime !== "1m") {
+ return
+ }
+ let cancelled = false
+ let unsubscribe: (() => void) | undefined
+ pb.realtime
+ .subscribe(
+ `rt_metrics`,
+ (data: { Monitors: NetworkMonitorStatsRecord["stats"] }) => {
+ const monitorStats = data.Monitors?.[monitorId]
+ if (cancelled || !monitorStats) return
+ const stats = { created: Date.now(), stats: { [monitorId]: monitorStats } }
+ const newStats = appendCacheValue(monitorId, "rt", [stats], 120)
+ setMonitorStats(newStats)
+ },
+ { query: { system: systemId } }
+ )
+ .then((us) => {
+ unsubscribe = us
+ if (cancelled) unsubscribe()
+ })
+ return () => {
+ cancelled = true
+ unsubscribe?.()
+ }
+ }, [chartTime, systemId, monitorId, enabled])
+
+ return monitorStats
+}
+
+async function fetchMonitors(system?: string) {
+ try {
+ return await pb.collection<NetworkMonitorRecord>("network_monitors").getFullList({
+ fields: NETWORK_MONITOR_FIELDS,
+ filter: system ? pb.filter("system={:system}", { system }) : undefined,
+ })
+ } catch (error) {
+ toast({
+ title: "Error",
+ description: (error as Error)?.message,
+ variant: "destructive",
+ })
+ return []
+ }
+}
+
+function applyMonitorEvents(
+ monitors: NetworkMonitorRecord[],
+ events: Iterable<RecordSubscription<NetworkMonitorRecord>>,
+ systemId?: string
+) {
+ const monitorById = new Map(monitors.map((monitor) => [monitor.id, monitor]))
+ const createdMonitors: NetworkMonitorRecord[] = []
+
+ for (const { action, record } of events) {
+ const matchesSystemScope = !systemId || record.system === systemId
+
+ if (action === "delete" || !matchesSystemScope) {
+ monitorById.delete(record.id)
+ continue
+ }
+
+ if (!monitorById.has(record.id)) {
+ createdMonitors.push(record)
+ }
+
+ monitorById.set(record.id, record)
+ }
+
+ const nextMonitors: NetworkMonitorRecord[] = []
+ for (let index = createdMonitors.length - 1; index >= 0; index -= 1) {
+ nextMonitors.push(createdMonitors[index])
+ }
+
+ for (const monitor of monitors) {
+ const nextMonitor = monitorById.get(monitor.id)
+ if (!nextMonitor) {
+ continue
+ }
+ nextMonitors.push(nextMonitor)
+ monitorById.delete(monitor.id)
+ }
+
+ return nextMonitors
+}
diff --git a/internal/site/src/lib/utils.ts b/internal/site/src/lib/utils.ts
index d7900bcc5..0b06f63be 100644
--- a/internal/site/src/lib/utils.ts
+++ b/internal/site/src/lib/utils.ts
@@ -72,7 +72,7 @@ export const formatShortDate = (timestamp: string) => {
return shortDateFormatter.format(new Date(timestamp))
}
-export const hourWithSeconds = (timestamp: string) => {
+export const hourWithSeconds = (timestamp: string | number) => {
return hourWithSecondsFormatter.format(new Date(timestamp))
}
@@ -111,17 +111,18 @@ export const updateFavicon = (() => {
</linearGradient>
</defs>
<path fill="url(#gradient)" d="M35 70H0V0h35q4.4 0 8.2 1.7a21.4 21.4 0 0 1 6.6 4.5q2.9 2.8 4.5 6.6Q56 16.7 56 21a15.4 15.4 0 0 1-.3 3.2 17.6 17.6 0 0 1-.2.8 19.4 19.4 0 0 1-1.5 4 17 17 0 0 1-2.4 3.4 13.5 13.5 0 0 1-2.6 2.3 12.5 12.5 0 0 1-.4.3q1.7 1 3 2.5Q53 39.1 54 41a18.3 18.3 0 0 1 1.5 4 17.4 17.4 0 0 1 .5 3 15.3 15.3 0 0 1 0 1q0 4.4-1.7 8.2a21.4 21.4 0 0 1-4.5 6.6q-2.8 2.9-6.6 4.6Q39.4 70 35 70ZM14 14v14h21a7 7 0 0 0 2.3-.3 6.6 6.6 0 0 0 .4-.2Q39 27 40 26a6.9 6.9 0 0 0 1.5-2.2q.5-1.3.5-2.8a7 7 0 0 0-.4-2.3 6.6 6.6 0 0 0-.1-.4Q40.9 17 40 16a7 7 0 0 0-2.3-1.4 6.9 6.9 0 0 0-2.5-.6 7.9 7.9 0 0 0-.2 0H14Zm0 28v14h21a7 7 0 0 0 2.3-.4 6.6 6.6 0 0 0 .4-.1Q39 54.9 40 54a7 7 0 0 0 1.5-2.2 6.9 6.9 0 0 0 .5-2.6 7.9 7.9 0 0 0 0-.2 7 7 0 0 0-.4-2.3 6.6 6.6 0 0 0-.1-.4Q40.9 45 40 44a7 7 0 0 0-2.3-1.5 6.9 6.9 0 0 0-2.5-.6 7.9 7.9 0 0 0-.2 0H14Z"/>
- ${downCount > 0 &&
- `
+ ${
+ downCount > 0 &&
+ `
<circle cx="40" cy="50" r="22" fill="#f00"/>
<text x="40" y="60" font-size="34" text-anchor="middle" fill="#fff" font-family="Arial" font-weight="bold">${downCount}</text>
`
- }
+ }
</svg>
`
const blob = new Blob([svg], { type: "image/svg+xml" })
const url = URL.createObjectURL(blob)
- ; (document.querySelector("link[rel='icon']") as HTMLLinkElement).href = url
+ ;(document.querySelector("link[rel='icon']") as HTMLLinkElement).href = url
}
})()
@@ -198,6 +199,26 @@ export function decimalString(num: number, digits = 2) {
return formatter.format(num)
}
+export function formatMicroseconds(microseconds: number, showDigits = true): string {
+ if (!Number.isFinite(microseconds)) {
+ return "-"
+ }
+
+ if (microseconds < 1000) {
+ return `${microseconds}μs`
+ }
+
+ if (microseconds < 1_000_000) {
+ const milliseconds = microseconds / 1000
+ const digits = milliseconds >= 10 ? 1 : 2
+ return `${decimalString(milliseconds, showDigits ? digits : 0)}ms`
+ }
+
+ const seconds = microseconds / 1_000_000
+ const digits = seconds >= 10 ? 1 : 2
+ return `${decimalString(seconds, showDigits ? digits : 0)}s`
+}
+
/** Get value from local or session storage */
function getStorageValue(key: string, defaultValue: unknown, storageInterface: Storage = localStorage) {
const saved = storageInterface?.getItem(key)
@@ -365,12 +386,12 @@ export function formatDuration(
.join(" ")
}
-/** Parse semver string into major, minor, and patch numbers
+/** Parse semver string into major, minor, and patch numbers
* @example
* const semVer = "1.2.3"
* const { major, minor, patch } = parseSemVer(semVer)
* console.log(major, minor, patch) // 1, 2, 3
-*/
+ */
export const parseSemVer = (semVer = ""): SemVer => {
// if (semVer.startsWith("v")) {
// semVer = semVer.slice(1)
@@ -393,6 +414,12 @@ export function compareSemVer(a: SemVer, b: SemVer) {
return a.patch - b.patch
}
+const MIN_NETWORK_MONITOR_AGENT_VERSION = parseSemVer("0.20.0")
+
+export function supportsNetworkMonitors(system: Pick<SystemRecord, "info">) {
+ return compareSemVer(parseSemVer(system.info?.v), MIN_NETWORK_MONITOR_AGENT_VERSION) >= 0
+}
+
// biome-ignore lint/suspicious/noExplicitAny: any is used to allow any function to be passed in
export function debounce<T extends (...args: any[]) => any>(func: T, wait: number): (...args: Parameters<T>) => void {
let timeout: ReturnType<typeof setTimeout>
@@ -422,10 +449,22 @@ export function runOnce<T extends (...args: any[]) => any>(fn: T): T {
}) as T
}
-/** Get the visual width of a string, accounting for full-width characters */
-export function getVisualStringWidth(str: string): number {
+const visualWidthCache = new Map<string, number>()
+
+/** Get the visual width of a string, accounting for full-width and narrow punctuation characters.
+ * Don't use for monospaced fonts, use .length instead
+ */
+function getVisualStringWidth(str: string): number {
+ const cached = visualWidthCache.get(str)
+ if (cached !== undefined) {
+ return cached
+ }
let width = 0
for (const char of str) {
+ if (char === ".") {
+ width += 0.7
+ continue
+ }
const code = char.codePointAt(0) || 0
// Hangul Jamo and Syllables are often slightly thinner than Hanzi/Kanji
if ((code >= 0x1100 && code <= 0x115f) || (code >= 0xac00 && code <= 0xd7af)) {
@@ -443,16 +482,41 @@ export function getVisualStringWidth(str: string): number {
code > 0xffff // Emojis and other supplementary plane characters
width += isFullWidth ? 2 : 1
}
+ visualWidthCache.set(str, width)
return width
}
+/** Compare the visual width of two strings imprecisely */
+export function isVisuallyLonger(str1: string, str2: string): boolean {
+ return getVisualStringWidth(str1) > getVisualStringWidth(str2)
+}
+
+/** Parses a filter string into OR'd groups of AND'd terms: "a b, c" -> [["a","b"], ["c"]] */
+export function parseFilterGroups(value: string): string[][] {
+ return value
+ .toLowerCase()
+ .split(",")
+ .map((group) => group.trim().split(" ").filter((term) => term.length > 0))
+ .filter((terms) => terms.length > 0)
+}
+
+/** True if every term in at least one OR'd group is found in searchString. */
+export function matchesFilterGroups(searchString: string, groups: string[][]): boolean {
+ return groups.some((terms) => terms.every((term) => searchString.includes(term)))
+}
+
/** Format seconds to hours, minutes, or seconds */
export function secondsToString(seconds: number, unit: "hour" | "minute" | "day"): string {
const count = Math.floor(seconds / (unit === "hour" ? 3600 : unit === "minute" ? 60 : 86400))
const countString = count.toLocaleString()
switch (unit) {
case "minute":
- return plural(count, { one: `${countString} minute`, few: `${countString} minutes`, many: `${countString} minutes`, other: `${countString} minutes` })
+ return plural(count, {
+ one: `${countString} minute`,
+ few: `${countString} minutes`,
+ many: `${countString} minutes`,
+ other: `${countString} minutes`,
+ })
case "hour":
return plural(count, { one: `${countString} hour`, other: `${countString} hours` })
case "day":
@@ -469,4 +533,4 @@ export function secondsToUptimeString(seconds: number): string {
} else {
return secondsToString(seconds, "day")
}
-}
\ No newline at end of file
+}
diff --git a/internal/site/src/main.tsx b/internal/site/src/main.tsx
index a4483501b..92f8da1c2 100644
--- a/internal/site/src/main.tsx
+++ b/internal/site/src/main.tsx
@@ -30,6 +30,7 @@ const LoginPage = lazy(() => import("@/components/login/login.tsx"))
const Home = lazy(() => import("@/components/routes/home.tsx"))
const Containers = lazy(() => import("@/components/routes/containers.tsx"))
const Smart = lazy(() => import("@/components/routes/smart.tsx"))
+const Monitors = lazy(() => import("@/components/routes/monitors.tsx"))
const SystemDetail = lazy(() => import("@/components/routes/system.tsx"))
const CopyToClipboardDialog = lazy(() => import("@/components/copy-to-clipboard.tsx"))
@@ -79,6 +80,8 @@ const App = memo(() => {
return <Containers />
} else if (page.route === "smart") {
return <Smart />
+ } else if (page.route === "monitors") {
+ return <Monitors />
} else if (page.route === "settings") {
return <Settings />
}
diff --git a/internal/site/src/types.d.ts b/internal/site/src/types.d.ts
index f9a7ef7c2..291c84340 100644
--- a/internal/site/src/types.d.ts
+++ b/internal/site/src/types.d.ts
@@ -307,6 +307,7 @@ export interface AlertRecord extends RecordModel {
}
export interface AlertsHistoryRecord extends RecordModel {
+ monitor_name?: string
alert: string
user: string
system: string
@@ -393,11 +394,9 @@ export interface SemVer {
export interface ChartData {
agentVersion: SemVer
- systemStats: SystemStatsRecord[]
- containerData: ChartDataContainer[]
+ systemStats?: SystemStatsRecord[]
+ containerData?: ChartDataContainer[]
orientation: "right" | "left"
- ticks: number[]
- domain: number[]
chartTime: ChartTimes
}
@@ -414,6 +413,8 @@ export interface AlertInfo {
singleDesc?: () => string
/** Hides the duration slider for alerts that fire on first observation */
noDuration?: boolean
+ /** Hides the threshold control for binary alerts */
+ noThreshold?: boolean
/** Description shown instead of numeric threshold and duration values */
triggeredDesc?: () => string
/** Additional information that remains visible while the alert is enabled */
@@ -632,3 +633,52 @@ export interface UpdateInfo {
v: string // new version
url: string // url to new version
}
+
+export interface NetworkMonitorRecord {
+ id: string
+ system: string
+ target: string
+ protocol: "icmp" | "tcp" | "http" | "dns"
+ port: number
+ res: number
+ resMin1h: number
+ resMax1h: number
+ resAvg1h: number
+ loss: number
+ loss1h: number
+ interval: number
+ enabled: boolean
+ updated: string
+}
+
+/** Response times in microseconds and packet loss percentage (0-100). */
+export interface MonitorStats {
+ res_avg: number
+ res_min: number
+ res_max: number
+ loss: number
+}
+
+/** Raw per-monitor record stored in the DB. */
+export interface RawMonitorStatsRecord {
+ res_min: number
+ res_max: number
+ total_count: number
+ success_count: number
+ res_sum: number
+ id?: string
+ type?: string
+ monitor: string
+ created: number // unix timestamp (ms)
+}
+
+/**
+ * Merged stats record keyed by monitor ID, used by chart components.
+ * Constructed from multiple RawMonitorStatsRecord entries sharing the same timestamp.
+ */
+export interface NetworkMonitorStatsRecord {
+ id?: string
+ type?: string
+ stats: Record<string, MonitorStats>
+ created: number // unix timestamp (ms) for Recharts xAxis
+}
diff --git a/internal/site/tests/network-monitor-utils.test.ts b/internal/site/tests/network-monitor-utils.test.ts
new file mode 100644
index 000000000..80aeec4fa
--- /dev/null
+++ b/internal/site/tests/network-monitor-utils.test.ts
@@ -0,0 +1,36 @@
+import { describe, expect, test } from "bun:test"
+import { getMonitorStats } from "../src/lib/network-monitor-utils"
+
+describe("monitor stats derived from stored counts", () => {
+ test("retains probe weights and response precision", () => {
+ const stats = getMonitorStats({
+ monitor: "monitor1",
+ created: 1000,
+ res_min: 5,
+ res_max: 20,
+ total_count: 7,
+ success_count: 6,
+ res_sum: 61,
+ })
+ expect(stats.res_avg).toBeCloseTo(10.1666667)
+ expect(stats.loss).toBeCloseTo(14.2857143)
+ expect(stats.res_min).toBe(5)
+ expect(stats.res_max).toBe(20)
+ })
+
+ test.each([
+ { total_count: 3, success_count: 0, loss: 100 },
+ { total_count: 0, success_count: 0, loss: 0 },
+ { total_count: 1, success_count: 1, loss: 0 },
+ ])("handles zero sums with $total_count attempts and $success_count successes", ({ loss, ...counts }) => {
+ const stats = getMonitorStats({
+ monitor: "monitor1",
+ created: 1000,
+ res_min: 0,
+ res_max: 0,
+ res_sum: 0,
+ ...counts,
+ })
+ expect(stats).toEqual({ res_avg: 0, res_min: 0, res_max: 0, loss })
+ })
+})