Lightweight server monitoring with historical data, docker stats, and alerts.
feat: network monitoring from agents (#2266, #1911)
hankbb1b39928e31
diff --git a/.dockerignore b/.dockerignoreindex d0ebc4d7d..9ded2435f 100644--- a/.dockerignore+++ b/.dockerignore@@ -1,6 +1,6 @@# Node.js dependencies-node_modules-internalsite/node_modules+node_modules/+**/node_modules/# Go build artifacts and binariesbuilddiff --git a/agent/agent.go b/agent/agent.goindex 86abb8a3f..454c565e5 100644--- a/agent/agent.go+++ b/agent/agent.go@@ -48,6 +48,7 @@ type Agent struct {keys []gossh.PublicKey // SSH public keyssmartManager *SmartManager // Manages SMART datasystemdManager *systemdManager // Manages systemd services+ monitorManager *MonitorManager // Manages network monitorsstoragePoolManager *StoragePoolManager // Manages storage pool and dataset data}@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {// initialize handler registryagent.handlerRegistry = NewHandlerRegistry()+ // initialize monitor manager+ agent.monitorManager = newMonitorManager()+agent.storagePoolManager = newStoragePoolManager()// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD}}+ if a.monitorManager != nil {+ data.Monitors = a.monitorManager.GetResults(cacheTimeMs)+ slog.Debug("Monitors", "data", data.Monitors)+ }+// skip updating systemd services if cache time is not the default 60sec intervalif a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {totalCount := uint16(a.systemdManager.getServiceStatsCount())diff --git a/agent/connection_manager.go b/agent/connection_manager.goindex 28ea98aec..0f29bd24a 100644--- a/agent/connection_manager.go+++ b/agent/connection_manager.go@@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {// }func (c *ConnectionManager) stop() error {_ = c.agent.StopServer()+ c.agent.monitorManager.Stop()c.closeWebSocket()return health.CleanUp()}diff --git a/agent/gpu_test.go b/agent/gpu_test.goindex 1ac6e0187..889f6c26b 100644--- a/agent/gpu_test.go+++ b/agent/gpu_test.go@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {}func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {- // Save original PATHhasAmdSysfs := (&GPUManager{}).hasAmdSysfs()tests := []struct {@@ -1213,7 +1212,7 @@ echo "[]"`{name: "no gpu tools available",setupCommands: func(_ string) error {- t.Setenv("PATH", "")+ // The subtest already restricts PATH to its empty temporary directory.return nil},wantErr: true,diff --git a/agent/handlers.go b/agent/handlers.goindex 9da87e542..67b69a73d 100644--- a/agent/handlers.go+++ b/agent/handlers.go@@ -7,6 +7,7 @@ import ("github.com/fxamacker/cbor/v2""github.com/henrygd/beszel/internal/common"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/henrygd/beszel/internal/entities/smart""log/slog"@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})registry.Register(common.GetSmartData, &GetSmartDataHandler{})registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})+ registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})registry.Register(common.GetZfsData, &GetZfsDataHandler{})return registry@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {return hctx.SendResponse(details, hctx.RequestID)}++////////////////////////////////////////////////////////////////////////////+////////////////////////////////////////////////////////////////////////////++// SyncNetworkMonitorsHandler handles monitor configuration sync from hub+type SyncNetworkMonitorsHandler struct{}++func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {+ var req monitor.SyncRequest+ if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {+ return err+ }+ resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)+ if err != nil {+ return err+ }+ return hctx.SendResponse(resp, hctx.RequestID)+}diff --git a/agent/network_monitor.go b/agent/network_monitor.gonew file mode 100644index 000000000..d5257675a--- /dev/null+++ b/agent/network_monitor.go@@ -0,0 +1,176 @@+package agent++import (+ "errors"+ "fmt"+ "net/http"+ "sync"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+)++// MonitorManager manages network monitor configurations and task lifetimes.+type MonitorManager struct {+ mu sync.RWMutex+ monitors map[string]*monitorTask // keyed by monitor ID+ probe monitorProbe+ resumeGuard monitorResumeGuard+}++func newMonitorManager() *MonitorManager {+ return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))+}++func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {+ return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}+}++// SyncMonitors replaces all monitor tasks with the given configs.+func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {+ pm.mu.Lock()+ defer pm.mu.Unlock()++ // Build set of new keys+ newKeys := make(map[string]monitor.Config, len(configs))+ for _, cfg := range configs {+ if cfg.ID == "" {+ continue+ }+ newKeys[cfg.ID] = cfg+ }++ // Stop removed monitors+ for key, task := range pm.monitors {+ if _, exists := newKeys[key]; !exists {+ task.cancel()+ delete(pm.monitors, key)+ }+ }++ // Start new monitors and restart tasks whose config changed.+ for key, cfg := range newKeys {+ task, exists := pm.monitors[key]+ if exists && task.config == cfg {+ continue+ }+ if exists {+ task.cancel()+ }+ task = newMonitorTaskFromExisting(cfg, task)+ task.resumeGuard = &pm.resumeGuard+ pm.resumeGuard.start()+ pm.monitors[key] = task+ pm.startMonitor(task)+ }+ if len(pm.monitors) == 0 {+ pm.resumeGuard.shutdown()+ }+}++// HandleSyncRequest applies a full or incremental monitor sync request.+func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {+ switch req.Action {+ case monitor.SyncActionReplace:+ pm.SyncMonitors(req.Configs)+ return monitor.SyncResponse{}, nil+ case monitor.SyncActionUpsert:+ result, err := pm.UpsertMonitor(req.Config, req.RunNow)+ if err != nil {+ return monitor.SyncResponse{}, err+ }+ if result == nil {+ return monitor.SyncResponse{}, nil+ }+ return monitor.SyncResponse{Result: *result}, nil+ case monitor.SyncActionDelete:+ if req.Config.ID == "" {+ return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")+ }+ pm.DeleteMonitor(req.Config.ID)+ return monitor.SyncResponse{}, nil+ default:+ return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)+ }+}++// UpsertMonitor creates or replaces a single monitor task.+func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {+ if config.ID == "" {+ return nil, errors.New("missing monitor ID")+ }++ pm.mu.Lock()+ task, exists := pm.monitors[config.ID]+ if exists && task.config == config {+ pm.mu.Unlock()+ if !runNow {+ return nil, nil+ }+ return task.runProbe(pm.probe), nil+ }+ if exists {+ task.cancel()+ }+ task = newMonitorTaskFromExisting(config, task)+ task.resumeGuard = &pm.resumeGuard+ pm.resumeGuard.start()+ pm.monitors[config.ID] = task+ pm.mu.Unlock()++ if runNow {+ result := task.runProbe(pm.probe)+ pm.startMonitor(task)+ return result, nil+ }+ pm.startMonitor(task)+ return nil, nil+}++// DeleteMonitor stops and removes a single monitor task.+func (pm *MonitorManager) DeleteMonitor(id string) {+ if id == "" {+ return+ }+ pm.mu.Lock()+ defer pm.mu.Unlock()+ if task, exists := pm.monitors[id]; exists {+ task.cancel()+ delete(pm.monitors, id)+ }+ if len(pm.monitors) == 0 {+ pm.resumeGuard.shutdown()+ }+}++// GetResults returns aggregated results for all monitors over the last supplied duration in ms.+func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {+ pm.mu.RLock()+ defer pm.mu.RUnlock()++ results := make(map[string]monitor.Result, len(pm.monitors))+ now := time.Now()+ duration := time.Duration(durationMs) * time.Millisecond++ for _, task := range pm.monitors {+ result, ok := task.history.result(duration, now)++ if !ok {+ continue+ }+ results[task.config.ID] = result+ }++ return results+}++// Stop stops all monitor tasks.+func (pm *MonitorManager) Stop() {+ pm.mu.Lock()+ defer pm.mu.Unlock()+ for key, task := range pm.monitors {+ task.cancel()+ delete(pm.monitors, key)+ }+ pm.resumeGuard.shutdown()+}diff --git a/agent/network_monitor_history.go b/agent/network_monitor_history.gonew file mode 100644index 000000000..9db1639ab--- /dev/null+++ b/agent/network_monitor_history.go@@ -0,0 +1,274 @@+package agent++import (+ "math"+ "sync"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+)++// Monitors run at user-defined intervals (e.g., every 10s).+// To keep memory usage low and constant, data is stored in two layers:+// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).+// 2. Minute buckets: A ring buffer of 61 buckets, each representing one+// wall-clock minute. Samples collected within the same minute are aggregated+// (sum, min, max, count) into a single bucket.+//+// Short-term requests (<= 61s) use raw samples.+// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands+// of individual data points.++const (+ // monitorRawRetention is the duration to keep individual samples+ monitorRawRetention = 61 * time.Second+ // monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)+ monitorMinuteBucketLen int32 = 61+)++// monitorHistory owns retention and aggregation, independently of probe execution.+type monitorHistory struct {+ mu sync.Mutex+ sampleCount int64+ samples []monitorSample+ buckets [monitorMinuteBucketLen]monitorBucket+}++func newMonitorHistory() *monitorHistory {+ // Start small for typical intervals; append grows the buffer for faster probes.+ return &monitorHistory{samples: make([]monitorSample, 0, 4)}+}++func (h *monitorHistory) clone() *monitorHistory {+ h.mu.Lock()+ defer h.mu.Unlock()+ cloned := newMonitorHistory()+ cloned.samples = append(cloned.samples, h.samples...)+ cloned.buckets = h.buckets+ cloned.sampleCount = h.sampleCount+ return cloned+}++func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {+ h.mu.Lock()+ defer h.mu.Unlock()+ return h.resultLocked(duration, now)+}++func (h *monitorHistory) record(sample monitorSample) monitor.Result {+ h.mu.Lock()+ defer h.mu.Unlock()+ h.addSampleLocked(sample)+ result, _ := h.resultLocked(time.Minute, sample.timestamp)+ return result+}++// monitorSample stores one monitor attempt and its collection time.+type monitorSample struct {+ responseUs int64 // -1 means loss+ timestamp time.Time+}++// monitorBucket stores one minute of aggregated monitor data.+type monitorBucket struct {+ minute int32+ filled bool+ stats monitorAggregate+}++// monitorAggregate accumulates successful response stats and total sample counts.+type monitorAggregate struct {+ sumUs int64+ minUs int64+ maxUs int64+ totalCount int64+ successCount int64+}++// newMonitorAggregate initializes an aggregate with an unset minimum value.+func newMonitorAggregate() monitorAggregate {+ return monitorAggregate{minUs: math.MaxInt64}+}++// addResponse folds a single monitor sample into the aggregate.+func (agg *monitorAggregate) addResponse(responseUs int64) {+ agg.totalCount+++ if responseUs < 0 {+ return+ }+ agg.successCount+++ agg.sumUs += responseUs+ if responseUs < agg.minUs {+ agg.minUs = responseUs+ }+ if responseUs > agg.maxUs {+ agg.maxUs = responseUs+ }+}++// addAggregate merges another aggregate into this one.+func (agg *monitorAggregate) addAggregate(other monitorAggregate) {+ if other.totalCount == 0 {+ return+ }+ agg.totalCount += other.totalCount+ agg.successCount += other.successCount+ agg.sumUs += other.sumUs+ if other.successCount == 0 {+ return+ }+ if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {+ agg.minUs = other.minUs+ }+ if other.maxUs > agg.maxUs {+ agg.maxUs = other.maxUs+ }+}++// hasData reports whether the aggregate contains any samples.+func (agg monitorAggregate) hasData() bool {+ return agg.totalCount > 0+}++// result converts the aggregate into the monitor result format.+func (agg monitorAggregate) result() monitor.Result {+ avg := agg.avgResponse()+ result := monitor.Result{+ AvgResponse: avg,+ MinResponse: agg.minUs,+ MaxResponse: agg.maxUs,+ PacketLoss: agg.lossPercentage(),+ TotalCount: agg.totalCount,+ SuccessCount: agg.successCount,+ ResponseSum: agg.sumUs,+ }+ if agg.successCount == 0 {+ result.MinResponse, result.MaxResponse = 0, 0+ }+ return result+}++// avgResponse returns the rounded average of successful samples.+func (agg monitorAggregate) avgResponse() int64 {+ if agg.successCount == 0 {+ return 0+ }+ return agg.sumUs / agg.successCount++}++// lossPercentage returns the rounded failure rate for the aggregate.+func (agg monitorAggregate) lossPercentage() float64 {+ if agg.totalCount == 0 {+ return 0+ }+ return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100+}++// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.+func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {+ agg := h.aggregateLocked(duration, now)+ if !agg.hasData() {+ // short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall+ // between monitor samples since monitors run at longer, user-defined intervals; fall back to+ // the most recent sample so realtime requests still report current status.+ agg = h.latestSampleAggregateLocked()+ }+ hourAgg := h.aggregateLocked(time.Hour, now)+ if !agg.hasData() {+ return monitor.Result{}, false+ }++ result := agg.result()+ if len(h.samples) > 0 {+ result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()+ }++ result.AvgResponse1h = hourAgg.avgResponse()+ result.MinResponse1h = hourAgg.minUs+ result.MaxResponse1h = hourAgg.maxUs+ result.PacketLoss1h = hourAgg.lossPercentage()+ result.SampleCount = h.sampleCount++ if hourAgg.successCount == 0 {+ result.MinResponse1h, result.MaxResponse1h = 0, 0+ }+ return result, true+}++// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.+func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {+ agg := newMonitorAggregate()+ if len(h.samples) == 0 {+ return agg+ }+ agg.addResponse(h.samples[len(h.samples)-1].responseUs)+ return agg+}++// aggregateLocked collects monitor data for the requested time window.+func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {+ cutoff := now.Add(-duration)+ // Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.+ if duration <= monitorRawRetention {+ return aggregateSamplesSince(h.samples, cutoff)+ }+ return aggregateBucketsSince(h.buckets[:], cutoff, now)+}++// aggregateSamplesSince aggregates raw samples newer than the cutoff.+func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {+ agg := newMonitorAggregate()+ for _, sample := range samples {+ if sample.timestamp.Before(cutoff) {+ continue+ }+ agg.addResponse(sample.responseUs)+ }+ return agg+}++// aggregateBucketsSince aggregates minute buckets overlapping the requested window.+func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {+ agg := newMonitorAggregate()+ startMinute := int32(cutoff.Unix() / 60)+ endMinute := int32(now.Unix() / 60)+ for _, bucket := range buckets {+ if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {+ continue+ }+ agg.addAggregate(bucket.stats)+ }+ return agg+}++// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.+func (h *monitorHistory) addSampleLocked(sample monitorSample) {+ h.sampleCount+++ cutoff := sample.timestamp.Add(-monitorRawRetention)+ start := 0+ for i := range h.samples {+ if !h.samples[i].timestamp.Before(cutoff) {+ start = i+ break+ }+ if i == len(h.samples)-1 {+ start = len(h.samples)+ }+ }+ if start > 0 {+ size := copy(h.samples, h.samples[start:])+ h.samples = h.samples[:size]+ }+ h.samples = append(h.samples, sample)++ minute := int32(sample.timestamp.Unix() / 60)+ // Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.+ bucket := &h.buckets[minute%monitorMinuteBucketLen]+ if !bucket.filled || bucket.minute != minute {+ bucket.minute = minute+ bucket.filled = true+ bucket.stats = newMonitorAggregate()+ }+ bucket.stats.addResponse(sample.responseUs)+}diff --git a/agent/network_monitor_history_test.go b/agent/network_monitor_history_test.gonew file mode 100644index 000000000..12a944abf--- /dev/null+++ b/agent/network_monitor_history_test.go@@ -0,0 +1,154 @@+package agent++import (+ "testing"+ "time"++ "github.com/fxamacker/cbor/v2"+ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestMonitorHistoryWindowCounts(t *testing.T) {+ history := newMonitorHistory()+ now := time.Now()+ // This older success counts toward lifetime warm-up, but not this window.+ history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})+ history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})+ history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})+ history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})+ result, ok := history.result(time.Minute, now)+ require.True(t, ok)+ assert.EqualValues(t, 4, result.SampleCount)+ assert.EqualValues(t, 3, result.TotalCount)+ assert.EqualValues(t, 2, result.SuccessCount)+ assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")+ assert.EqualValues(t, 15, result.AvgResponse)+ assert.Equal(t, 33.33, result.PacketLoss)++ encoded, err := cbor.Marshal(result)+ require.NoError(t, err)+ var decoded monitor.Result+ require.NoError(t, cbor.Unmarshal(encoded, &decoded))+ assert.Equal(t, result, decoded)+ stats := monitor.Stats{}.FromResult(decoded)+ assert.Equal(t, result.TotalCount, stats.TotalCount)+ assert.Equal(t, result.SuccessCount, stats.SuccessCount)+ assert.Equal(t, result.ResponseSum, stats.ResponseSum)++ // Reads do not consume samples. A short window's latest-sample fallback+ // carries the count for that single failure, not the minute or lifetime count.+ repeated, _ := history.result(time.Minute, now)+ assert.Equal(t, result, repeated)+ fallback, ok := history.result(time.Second, now)+ require.True(t, ok)+ assert.EqualValues(t, 1, fallback.TotalCount)+ assert.Zero(t, fallback.SuccessCount)+ assert.Zero(t, fallback.ResponseSum)+ assert.Equal(t, 100.0, fallback.PacketLoss)+ assert.EqualValues(t, 4, fallback.SampleCount)+}++func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {+ now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)+ history := newMonitorHistory()++ history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})+ history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})+ history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})++ agg := history.aggregateLocked(time.Minute, now)+ require.True(t, agg.hasData())+ assert.Equal(t, int64(2), agg.totalCount)+ assert.Equal(t, int64(1), agg.successCount)+ result := agg.result()+ assert.Equal(t, int64(20), result.AvgResponse)+ assert.Equal(t, int64(20), result.MinResponse)+ assert.Equal(t, int64(20), result.MaxResponse)+ assert.Equal(t, 50.0, result.PacketLoss)+}++func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {+ now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)+ history := newMonitorHistory()++ history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})+ history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})+ history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})+ history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})+ history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})++ agg := history.aggregateLocked(10*time.Minute, now)+ require.True(t, agg.hasData())+ assert.Equal(t, int64(4), agg.totalCount)+ assert.Equal(t, int64(3), agg.successCount)+ result := agg.result()+ assert.Equal(t, int64(30), result.AvgResponse)+ assert.Equal(t, int64(20), result.MinResponse)+ assert.Equal(t, int64(40), result.MaxResponse)+ assert.Equal(t, 25.0, result.PacketLoss)+}++func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {+ now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)+ history := newMonitorHistory()++ history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})+ history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})++ require.Len(t, history.samples, 1)+ assert.Equal(t, int64(20), history.samples[0].responseUs)++ agg := history.aggregateLocked(10*time.Minute, now)+ require.True(t, agg.hasData())+ assert.Equal(t, int64(2), agg.totalCount)+ assert.Equal(t, int64(2), agg.successCount)+ result := agg.result()+ assert.Equal(t, int64(15), result.AvgResponse)+ assert.Equal(t, int64(10), result.MinResponse)+ assert.Equal(t, int64(20), result.MaxResponse)+ assert.Equal(t, 0.0, result.PacketLoss)+}++func TestMonitorHistoryProbeTimestamp(t *testing.T) {+ history := newMonitorHistory()+ start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)+ _, ok := history.result(time.Minute, start)+ require.False(t, ok)+ first := history.record(monitorSample{responseUs: 20, timestamp: start})+ assert.Equal(t, start.UnixMilli(), first.LastProbeAt)+ for minute := 0; minute < 5; minute++ {+ now := start.Add(time.Duration(minute)*time.Minute + time.Second)+ // Realtime reads must not consume freshness for the persistence request.+ for _, window := range []time.Duration{time.Second, time.Minute} {+ result, ok := history.result(window, now)+ require.True(t, ok)+ assert.Equal(t, first.LastProbeAt, result.LastProbeAt)+ assert.Equal(t, int64(20), result.AvgResponse)+ }+ }+ next := start.Add(5 * time.Minute)+ failed := history.record(monitorSample{responseUs: -1, timestamp: next})+ assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)+ assert.Equal(t, float64(100), failed.PacketLoss)+ repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))+ require.True(t, ok)+ assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)+ assert.Equal(t, float64(100), repeated.PacketLoss)+}++func TestMonitorHistorySampleCount(t *testing.T) {+ history := newMonitorHistory()+ now := time.Now()+ // Both failed and successful probes count, including older samples so+ // monitors with hourly intervals can finish warming up.+ history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})+ for i, response := range []int64{10, -1, 20} {+ result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})+ assert.EqualValues(t, i+2, result.SampleCount)+ }+ result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))+ require.True(t, ok)+ assert.EqualValues(t, 4, result.SampleCount)+}diff --git a/agent/network_monitor_ping.go b/agent/network_monitor_ping.gonew file mode 100644index 000000000..632afd668--- /dev/null+++ b/agent/network_monitor_ping.go@@ -0,0 +1,312 @@+package agent++import (+ "bytes"+ "context"+ "crypto/rand"+ "errors"+ "fmt"+ "math"+ "net"+ "os"+ "os/exec"+ "regexp"+ "runtime"+ "strconv"+ "strings"+ "sync"+ "sync/atomic"+ "time"++ "golang.org/x/net/icmp"+ "golang.org/x/net/ipv4"+ "golang.org/x/net/ipv6"++ "log/slog"+)++// Match the numeric RTT independently of the localized label used by Windows.+var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)++var icmpSequence atomic.Uint32++type icmpPacketConn interface {+ Close() error+}++// icmpMethod tracks which ICMP approach to use. Once a method succeeds or+// all native methods fail, the choice is cached so subsequent monitors skip+// the trial-and-error overhead.+type icmpMethod uint8++const (+ icmpUntried icmpMethod = iota // haven't tried yet+ icmpRaw // privileged raw socket+ icmpDatagram // unprivileged datagram socket+ icmpExecFallback // shell out to system ping command+)++// icmpFamily holds the network parameters and cached detection result for one address family.+type icmpFamily struct {+ rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"+ dgramNetwork string // e.g. "udp4" or "udp6"+ listenAddr string // "0.0.0.0" or "::"+ echoType icmp.Type // outgoing echo request type+ replyType icmp.Type // expected echo reply type+ proto int // IANA protocol number for parsing replies+ isIPv6 bool+ mode icmpMethod // cached detection result (guarded by icmpModeMu)+}++var (+ icmpV4 = icmpFamily{+ rawNetwork: "ip4:icmp",+ dgramNetwork: "udp4",+ listenAddr: "0.0.0.0",+ echoType: ipv4.ICMPTypeEcho,+ replyType: ipv4.ICMPTypeEchoReply,+ proto: 1,+ }+ icmpV6 = icmpFamily{+ rawNetwork: "ip6:ipv6-icmp",+ dgramNetwork: "udp6",+ listenAddr: "::",+ echoType: ipv6.ICMPTypeEchoRequest,+ replyType: ipv6.ICMPTypeEchoReply,+ proto: 58,+ isIPv6: true,+ }+ icmpModeMu sync.Mutex+ icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {+ return icmp.ListenPacket(network, listenAddr)+ }+)++// monitorICMP sends an ICMP echo request and measures round-trip response.+// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,+// unprivileged datagram, or exec fallback) is detected once per address+// family and cached for subsequent monitors.+// Returns response in microseconds, or -1 and an error on failure.+func monitorICMP(ctx context.Context, target string) (int64, error) {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()++ family, ip, err := resolveICMPTarget(ctx, target)+ if err != nil {+ return -1, err+ }++ icmpModeMu.Lock()+ if family.mode == icmpUntried {+ family.mode = detectICMPMode(family, icmpListen)+ }+ mode := family.mode+ icmpModeMu.Unlock()++ switch mode {+ case icmpRaw:+ return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})+ case icmpDatagram:+ return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})+ case icmpExecFallback:+ return monitorICMPExec(ctx, ip.String(), family.isIPv6)+ default:+ return -1, errors.New("unsupported ICMP mode")+ }+}++// resolveICMPTarget resolves a target hostname or IP to determine the address+// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.+func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {+ if ip := net.ParseIP(target); ip != nil {+ if ip.To4() != nil {+ return &icmpV4, ip.To4(), nil+ }+ return &icmpV6, ip, nil+ }++ ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)+ if err != nil || len(ips) == 0 {+ return nil, nil, err+ }+ for _, ip := range ips {+ if v4 := ip.To4(); v4 != nil {+ return &icmpV4, v4, nil+ }+ }+ return &icmpV6, ips[0], nil+}++func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {+ label := "IPv4"+ if family.isIPv6 {+ label = "IPv6"+ }++ conn, err := listen(family.rawNetwork, family.listenAddr)+ slog.Debug("ICMP raw socket test", "family", label, "err", err)+ if err == nil {+ conn.Close()+ return icmpRaw+ }++ conn, err = listen(family.dgramNetwork, family.listenAddr)+ slog.Debug("ICMP datagram socket test", "family", label, "err", err)+ if err == nil {+ conn.Close()+ return icmpDatagram+ }++ return icmpExecFallback+}++// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.+func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {+ conn, err := icmp.ListenPacket(network, family.listenAddr)+ if err != nil {+ return -1, err+ }+ defer conn.Close()++ return monitorICMPPacket(ctx, conn, family, dst)+}++func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {+ if err := ctx.Err(); err != nil {+ return -1, err+ }+ // Closing the socket interrupts both reads and writes on cancellation.+ stop := context.AfterFunc(ctx, func() { _ = conn.Close() })+ defer stop()++ // Prepare correlation data before starting the round-trip timer. The token+ // also distinguishes delayed replies after the 16-bit sequence wraps.+ token := make([]byte, 16)+ if _, err := rand.Read(token); err != nil {+ return -1, err+ }+ echo := &icmp.Echo{+ ID: os.Getpid() & 0xffff,+ Seq: int(icmpSequence.Add(1) & 0xffff),+ Data: token,+ }+ // Linux ping sockets replace the Echo ID with their bound port. Darwin+ // datagram sockets and raw sockets preserve the supplied ID.+ if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {+ echo.ID = local.Port+ }+ targetIP := icmpAddrIP(dst)+ msg := &icmp.Message{+ Type: family.echoType,+ Code: 0,+ Body: echo,+ }+ msgBytes, err := msg.Marshal(nil)+ if err != nil {+ return -1, err+ }++ // Set deadline before sending+ if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {+ return -1, err+ }++ buf := make([]byte, 1500)+ start := time.Now()+ if _, err := conn.WriteTo(msgBytes, dst); err != nil {+ return -1, err+ }++ // Read reply+ for {+ n, peer, err := conn.ReadFrom(buf)+ received := time.Now()+ if err != nil {+ return -1, err+ }+ if !targetIP.Equal(icmpAddrIP(peer)) {+ continue+ }++ reply, err := icmp.ParseMessage(family.proto, buf[:n])+ if err != nil || reply.Type != family.replyType || reply.Code != 0 {+ continue+ }++ body, ok := reply.Body.(*icmp.Echo)+ if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {+ return received.Sub(start).Microseconds(), nil+ }+ // Keep waiting for our reply without extending the original deadline.+ }+}++func icmpAddrIP(addr net.Addr) net.IP {+ switch addr := addr.(type) {+ case *net.IPAddr:+ return addr.IP+ case *net.UDPAddr:+ return addr.IP+ default:+ return nil+ }+}++// pingCommand selects the executable and arguments for the supported agent platforms.+// The context deadline enforces the timeout: -W has incompatible meanings across+// Linux, BSD IPv4 ping, and macOS ping6.+func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {+ family := "-4"+ if isIPv6 {+ family = "-6"+ }+ switch goos {+ case "windows":+ return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil+ case "linux":+ return "ping", []string{family, "-n", "-c", "1", target}, nil+ case "darwin", "freebsd", "openbsd":+ command := "ping"+ if isIPv6 {+ command = "ping6"+ }+ return command, []string{"-n", "-c", "1", target}, nil+ default:+ return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)+ }+}++// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.+func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()+ name, args, err := pingCommand(runtime.GOOS, target, isIPv6)+ if err != nil {+ return -1, err+ }+ cmd := exec.CommandContext(ctx, name, args...)+ // Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.+ cmd.Env = append(os.Environ(), "LC_ALL=C")+ output, err := cmd.Output()+ if ctx.Err() != nil {+ return -1, ctx.Err()+ }+ if err != nil {+ return -1, fmt.Errorf("%s failed: %w", name, err)+ }+ return parsePingResponse(output)+}++// parsePingResponse returns the reported RTT, never subprocess execution time.+// For a bounded value such as Windows' time<1ms, retain the reported upper bound.+func parsePingResponse(output []byte) (int64, error) {+ matches := pingTimeRegex.FindSubmatch(output)+ if len(matches) < 2 {+ return -1, errors.New("ping output contains no round-trip time")+ }+ ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)+ if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {+ return -1, errors.New("invalid round-trip time in ping output")+ }+ return int64(math.Round(ms * 1000)), nil+}diff --git a/agent/network_monitor_ping_test.go b/agent/network_monitor_ping_test.gonew file mode 100644index 000000000..89ff92e66--- /dev/null+++ b/agent/network_monitor_ping_test.go@@ -0,0 +1,433 @@+//go:build testing++package agent++import (+ "context"+ "errors"+ "fmt"+ "net"+ "os"+ "path/filepath"+ "runtime"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "golang.org/x/net/icmp"+)++type testICMPPacketConn struct{}++func (testICMPPacketConn) Close() error { return nil }++type blockingICMPConn struct {+ net.PacketConn+ reading chan struct{}+}++func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {+ return len(p), nil+}++func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {+ close(c.reading)+ return c.PacketConn.ReadFrom(p)+}++func TestMonitorICMPPacketCancellation(t *testing.T) {+ conn, err := net.ListenPacket("udp4", "127.0.0.1:0")+ require.NoError(t, err)+ defer conn.Close()+ blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ done := make(chan error, 1)+ go func() {+ _, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())+ done <- err+ }()+ select {+ case <-blocking.reading:+ case <-time.After(time.Second):+ t.Fatal("probe did not begin reading")+ }+ cancel()+ select {+ case err := <-done:+ require.Error(t, err)+ case <-time.After(time.Second):+ t.Fatal("cancellation did not interrupt the socket read")+ }+}++func TestMonitorICMPExecCancellation(t *testing.T) {+ if runtime.GOOS == "windows" {+ t.Skip("test uses a POSIX shell stub for ping")+ }+ dir := t.TempDir()+ require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))+ ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)+ defer cancel()+ done := make(chan error, 1)+ go func() {+ _, err := monitorICMPExec(ctx, "127.0.0.1", false)+ done <- err+ }()+ select {+ case err := <-done:+ require.ErrorIs(t, err, context.DeadlineExceeded)+ case <-time.After(time.Second):+ t.Fatal("cancellation did not terminate ping")+ }+}++func TestPingCommand(t *testing.T) {+ for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {+ for _, ipv6 := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {+ target, family := "192.0.2.1", "-4"+ if ipv6 {+ target, family = "2001:db8::1", "-6"+ }+ name, args, err := pingCommand(goos, target, ipv6)+ require.NoError(t, err)+ wantName := "ping"+ wantArgs := []string{"-n", "-c", "1", target}+ switch goos {+ case "windows":+ wantArgs = []string{family, "-n", "1", "-w", "3000", target}+ case "linux":+ wantArgs = append([]string{family}, wantArgs...)+ default:+ if ipv6 {+ wantName = "ping6"+ }+ }+ assert.Equal(t, wantName, name)+ assert.Equal(t, wantArgs, args)+ })+ }+ }+ _, _, err := pingCommand("unsupported", "192.0.2.1", false)+ require.Error(t, err)+}++func TestParsePingResponse(t *testing.T) {+ for _, tc := range []struct {+ name string+ output string+ wantUs int64+ }{+ {"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},+ {"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},+ {"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},+ {"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},+ {"windows submillisecond", "Reply from ::1: time<1ms", 1000},+ {"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},+ {"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},+ {"rounding", "time=0.1236 ms", 124},+ {"empty", "", -1},+ {"timeout", "Request timed out.", -1},+ {"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},+ {"malformed", "time=oops ms", -1},+ {"negative", "time=-1 ms", -1},+ {"overflow", "time=999999999999999999999 ms", -1},+ } {+ t.Run(tc.name, func(t *testing.T) {+ responseUs, err := parsePingResponse([]byte(tc.output))+ if tc.wantUs < 0 {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ }+ assert.Equal(t, tc.wantUs, responseUs)+ })+ }+}++func TestMonitorICMPExecOutput(t *testing.T) {+ if runtime.GOOS == "windows" {+ t.Skip("test uses a POSIX shell stub for ping")+ }+ for _, tc := range []struct {+ name string+ output string+ exit int+ wantUs int64+ }{+ {"success", "time=1.234 ms", 0, 1234},+ {"missing RTT", "unrecognized output", 0, -1},+ {"failed command with RTT", "time=1.234 ms", 1, -1},+ } {+ t.Run(tc.name, func(t *testing.T) {+ dir := t.TempDir()+ // Also verify an inherited locale cannot override the C locale.+ script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)+ require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))+ t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))+ t.Setenv("LC_ALL", "de_DE.UTF-8")+ responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)+ if tc.wantUs < 0 {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ }+ assert.Equal(t, tc.wantUs, responseUs)+ })+ }+}++type icmpTestReply struct {+ data []byte+ peer net.Addr+}++type scriptedICMPConn struct {+ net.PacketConn+ local net.Addr+ onWrite func([]byte, net.Addr)+ replies []icmpTestReply+ reads int+ deadlineSets int+}++func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }++func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {+ c.deadlineSets+++ return nil+}++func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {+ c.onWrite(data, dst)+ return len(data), nil+}++func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {+ c.reads+++ if len(c.replies) == 0 {+ return 0, nil, os.ErrDeadlineExceeded+ }+ reply := c.replies[0]+ c.replies = c.replies[1:]+ return copy(buf, reply.data), reply.peer, nil+}++func TestMonitorICMPReplyCorrelation(t *testing.T) {+ for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {+ for _, datagram := range []bool{false, true} {+ network := family.rawNetwork+ ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")+ if family.isIPv6 {+ ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")+ }+ var dst net.Addr = &net.IPAddr{IP: ip}+ var wrongPeer net.Addr = &net.IPAddr{IP: other}+ if datagram {+ network = family.dgramNetwork+ dst = &net.UDPAddr{IP: ip}+ wrongPeer = &net.UDPAddr{IP: other}+ }+ for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {+ for _, eventuallyMatches := range []bool{false, true} {+ ending := "timeout"+ if eventuallyMatches {+ ending = "success"+ }+ t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {+ conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}+ if datagram {+ conn.local = &net.UDPAddr{Port: 12345}+ if runtime.GOOS == "linux" {+ // Deliberately differ from the process ID.+ conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}+ }+ }+ conn.onWrite = func(data []byte, target net.Addr) {+ require.Equal(t, dst, target)+ request, err := icmp.ParseMessage(family.proto, data)+ require.NoError(t, err)+ echo := request.Body.(*icmp.Echo)+ expectedID := os.Getpid() & 0xffff+ if datagram && runtime.GOOS == "linux" {+ expectedID = conn.local.(*net.UDPAddr).Port+ }+ require.Equal(t, expectedID, echo.ID)+ reply := &icmp.Message{Type: family.replyType, Body: echo}+ valid, err := reply.Marshal(nil)+ require.NoError(t, err)+ peer := dst+ switch mismatch {+ case "source":+ peer = wrongPeer+ case "id":+ echo.ID ^= 1+ case "sequence":+ echo.Seq ^= 1+ case "payload":+ echo.Data[0] ^= 1+ case "type":+ reply.Type = family.echoType+ case "code":+ reply.Code = 1+ }+ invalid, err := reply.Marshal(nil)+ require.NoError(t, err)+ if mismatch == "malformed" {+ invalid = invalid[:2]+ }+ conn.replies = []icmpTestReply{{invalid, peer}}+ if eventuallyMatches {+ conn.replies = append(conn.replies, icmpTestReply{valid, dst})+ }+ }+ elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)+ if eventuallyMatches {+ require.NoError(t, err)+ assert.GreaterOrEqual(t, elapsed, int64(0))+ } else {+ require.ErrorIs(t, err, os.ErrDeadlineExceeded)+ assert.Equal(t, int64(-1), elapsed)+ }+ assert.Equal(t, 2, conn.reads)+ assert.Equal(t, 1, conn.deadlineSets)+ })+ }+ }+ }+ }+}++func TestMonitorICMPLoopback(t *testing.T) {+ for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {+ for _, network := range []string{family.rawNetwork, family.dgramNetwork} {+ t.Run(network, func(t *testing.T) {+ conn, err := icmp.ListenPacket(network, family.listenAddr)+ if err != nil {+ t.Skipf("ICMP socket unavailable: %v", err)+ }+ defer conn.Close()+ ip := net.ParseIP("127.0.0.1")+ if family.isIPv6 {+ ip = net.ParseIP("::1")+ }+ var dst net.Addr = &net.IPAddr{IP: ip}+ if network == family.dgramNetwork {+ dst = &net.UDPAddr{IP: ip}+ }+ elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)+ require.NoError(t, err)+ assert.GreaterOrEqual(t, elapsed, int64(0))+ })+ }+ }+}++func TestDetectICMPMode(t *testing.T) {+ tests := []struct {+ name string+ family *icmpFamily+ rawErr error+ udpErr error+ want icmpMethod+ wantNetworks []string+ }{+ {+ name: "IPv4 prefers raw socket when available",+ family: &icmpV4,+ want: icmpRaw,+ wantNetworks: []string{"ip4:icmp"},+ },+ {+ name: "IPv4 uses datagram when raw unavailable",+ family: &icmpV4,+ rawErr: errors.New("operation not permitted"),+ want: icmpDatagram,+ wantNetworks: []string{"ip4:icmp", "udp4"},+ },+ {+ name: "IPv4 falls back to exec when both unavailable",+ family: &icmpV4,+ rawErr: errors.New("operation not permitted"),+ udpErr: errors.New("protocol not supported"),+ want: icmpExecFallback,+ wantNetworks: []string{"ip4:icmp", "udp4"},+ },+ {+ name: "IPv6 prefers raw socket when available",+ family: &icmpV6,+ want: icmpRaw,+ wantNetworks: []string{"ip6:ipv6-icmp"},+ },+ {+ name: "IPv6 uses datagram when raw unavailable",+ family: &icmpV6,+ rawErr: errors.New("operation not permitted"),+ want: icmpDatagram,+ wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},+ },+ {+ name: "IPv6 falls back to exec when both unavailable",+ family: &icmpV6,+ rawErr: errors.New("operation not permitted"),+ udpErr: errors.New("protocol not supported"),+ want: icmpExecFallback,+ wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ calls := make([]string, 0, 2)+ listen := func(network, listenAddr string) (icmpPacketConn, error) {+ require.Equal(t, tt.family.listenAddr, listenAddr)+ calls = append(calls, network)+ switch network {+ case tt.family.rawNetwork:+ if tt.rawErr != nil {+ return nil, tt.rawErr+ }+ case tt.family.dgramNetwork:+ if tt.udpErr != nil {+ return nil, tt.udpErr+ }+ default:+ t.Fatalf("unexpected network %q", network)+ }+ return testICMPPacketConn{}, nil+ }++ assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))+ assert.Equal(t, tt.wantNetworks, calls)+ })+ }+}++func TestResolveICMPTarget(t *testing.T) {+ t.Run("IPv4 literal", func(t *testing.T) {+ family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")+ require.NoError(t, err)+ require.NotNil(t, family)+ assert.False(t, family.isIPv6)+ assert.Equal(t, "127.0.0.1", ip.String())+ })++ t.Run("IPv6 literal", func(t *testing.T) {+ family, ip, err := resolveICMPTarget(context.Background(), "::1")+ require.NoError(t, err)+ require.NotNil(t, family)+ assert.True(t, family.isIPv6)+ assert.Equal(t, "::1", ip.String())+ })++ t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {+ family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")+ require.NoError(t, err)+ require.NotNil(t, family)+ assert.False(t, family.isIPv6)+ assert.Equal(t, "127.0.0.1", ip.String())+ })+}diff --git a/agent/network_monitor_probe.go b/agent/network_monitor_probe.gonew file mode 100644index 000000000..187a6bf37--- /dev/null+++ b/agent/network_monitor_probe.go@@ -0,0 +1,105 @@+package agent++import (+ "context"+ "errors"+ "fmt"+ "net"+ "net/http"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+)++// monitorProbe performs one check. Errors are recorded as loss by the task runner.+// Implementations must honor cancellation and bound their execution time.+type monitorProbe func(context.Context, monitor.Config) (int64, error)++func networkMonitorProbe(client *http.Client) monitorProbe {+ return func(ctx context.Context, config monitor.Config) (int64, error) {+ switch config.Protocol {+ case "icmp":+ return monitorICMP(ctx, config.Target)+ case "tcp":+ return monitorTCP(ctx, config.Target, config.Port)+ case "http":+ return monitorHTTP(ctx, client, config.Target)+ case "dns":+ return monitorDNS(ctx, config.Target)+ default:+ return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)+ }+ }+}++// monitorTCP measures connection establishment time, including address fallback+// but excluding DNS resolution.+// Returns -1 and an error on failure.+func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()++ // Resolve DNS first, outside the timing window but within the probe deadline.+ ips, err := net.DefaultResolver.LookupHost(ctx, target)+ if err != nil {+ return -1, err+ }+ if len(ips) == 0 {+ return -1, errors.New("no addresses resolved for TCP monitor")+ }+ portString := fmt.Sprintf("%d", port)+ deadline, _ := ctx.Deadline()++ // Share the remaining probe budget across addresses so an unresponsive+ // first address cannot consume all the time available for alternatives.+ start := time.Now()+ for i, ip := range ips {+ if err := ctx.Err(); err != nil {+ return -1, err+ }+ dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}+ var conn net.Conn+ conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))+ if err != nil {+ continue+ }+ responseUs := time.Since(start).Microseconds()+ conn.Close()+ return responseUs, nil+ }+ return -1, err+}++// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.+func monitorDNS(ctx context.Context, target string) (int64, error) {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()++ start := time.Now()+ ips, err := net.DefaultResolver.LookupHost(ctx, target)+ if err != nil || len(ips) == 0 {+ return -1, err+ }+ return time.Since(start).Microseconds(), nil+}++// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.+func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {+ if client == nil {+ client = http.DefaultClient+ }+ start := time.Now()+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)+ if err != nil {+ return -1, err+ }+ resp, err := client.Do(req)+ if err != nil {+ return -1, err+ }+ resp.Body.Close()+ if resp.StatusCode >= 400 {+ return -1, fmt.Errorf("HTTP error: %s", resp.Status)+ }+ return time.Since(start).Microseconds(), nil+}diff --git a/agent/network_monitor_resume.go b/agent/network_monitor_resume.gonew file mode 100644index 000000000..495266232--- /dev/null+++ b/agent/network_monitor_resume.go@@ -0,0 +1,88 @@+package agent++import (+ "sync"+ "time"+)++const (+ monitorResumeHeartbeat = 10 * time.Second+ // Allow scheduling jitter without mistaking an ordinary tick for resume.+ monitorResumeGap = 2 * monitorResumeHeartbeat+ monitorResumePause = 10 * time.Second+)++// monitorResumeGuard detects likely suspend/resume using wall time. A long+// process stall or forward clock adjustment can also trigger the bounded pause.+// One heartbeat is shared by all configured monitors.+type monitorResumeGuard struct {+ mu sync.Mutex+ stop chan struct{}+ lastTick time.Time+ pauseUntil time.Time+ generation uint32+}++func (g *monitorResumeGuard) start() {+ g.mu.Lock()+ defer g.mu.Unlock()+ if g.stop != nil {+ return+ }+ stop := make(chan struct{})+ g.stop = stop+ g.lastTick = time.Now().Round(0)+ g.pauseUntil = time.Time{}+ go func() {+ ticker := time.NewTicker(monitorResumeHeartbeat)+ defer ticker.Stop()+ for {+ select {+ case <-stop:+ return+ case <-ticker.C:+ g.mu.Lock()+ if g.stop == stop {+ g.observe(time.Now())+ }+ g.mu.Unlock()+ }+ }+ }()+}++func (g *monitorResumeGuard) shutdown() {+ g.mu.Lock()+ defer g.mu.Unlock()+ if g.stop != nil {+ close(g.stop)+ g.stop = nil+ g.generation+++ }+}++// observe requires mu. Strip the monotonic component because it can stop during+// suspend. Read the current time rather than the ticker's queued timestamp.+func (g *monitorResumeGuard) observe(now time.Time) {+ now = now.Round(0)+ if now.Sub(g.lastTick) > monitorResumeGap {+ g.pauseUntil = now.Add(monitorResumePause)+ g.generation+++ }+ g.lastTick = now+}++// snapshot also observes time so a probe waking before the heartbeat detects+// resume itself. A changed generation invalidates probes spanning suspend.+func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {+ if g == nil {+ return 0, true+ }+ g.mu.Lock()+ defer g.mu.Unlock()+ if g.stop == nil {+ return g.generation, true+ }+ g.observe(time.Now())+ return g.generation, !g.lastTick.Before(g.pauseUntil)+}diff --git a/agent/network_monitor_resume_test.go b/agent/network_monitor_resume_test.gonew file mode 100644index 000000000..f1a637c4d--- /dev/null+++ b/agent/network_monitor_resume_test.go@@ -0,0 +1,121 @@+//go:build testing++package agent++import (+ "context"+ "errors"+ "sync/atomic"+ "testing"+ "testing/synctest"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func simulateMonitorSleep(g *monitorResumeGuard) {+ g.mu.Lock()+ g.lastTick = time.Now().Add(-time.Hour).Round(0)+ g.mu.Unlock()+}++func TestMonitorResumePause(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ var g monitorResumeGuard+ g.start()+ defer g.shutdown()+ generation, allowed := g.snapshot()+ require.True(t, allowed)+ // Heartbeats alone must keep the guard current between infrequent probes.+ time.Sleep(time.Minute)+ synctest.Wait()+ steadyGeneration, allowed := g.snapshot()+ require.True(t, allowed)+ require.Equal(t, generation, steadyGeneration)+ // The probe, rather than the heartbeat, must detect this gap.+ simulateMonitorSleep(&g)+ next, allowed := g.snapshot()+ assert.False(t, allowed)+ assert.NotEqual(t, generation, next)+ time.Sleep(9 * time.Second)+ _, allowed = g.snapshot()+ assert.False(t, allowed)+ time.Sleep(time.Second)+ end, allowed := g.snapshot()+ assert.True(t, allowed)+ assert.Equal(t, next, end)+ })+}++func TestMonitorResumeGuardLifecycle(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })+ defer pm.Stop()+ assert.Nil(t, pm.resumeGuard.stop)+ pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})+ stop := pm.resumeGuard.stop+ require.NotNil(t, stop)+ pm.DeleteMonitor("a")+ assert.Equal(t, stop, pm.resumeGuard.stop)+ pm.DeleteMonitor("b")+ assert.Nil(t, pm.resumeGuard.stop)+ select {+ case <-stop:+ default:+ t.Fatal("heartbeat was not stopped")+ }+ time.Sleep(time.Hour)+ _, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)+ require.NoError(t, err)+ _, allowed := pm.resumeGuard.snapshot()+ assert.True(t, allowed, "idle time must not trigger a resume pause")+ pm.SyncMonitors(nil)+ assert.Nil(t, pm.resumeGuard.stop)+ })+}++func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ var g monitorResumeGuard+ g.start()+ defer g.shutdown()+ task := newMonitorTask(monitor.Config{ID: "test"})+ defer task.cancel()+ task.resumeGuard = &g+ result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {+ simulateMonitorSleep(&g)+ return 0, errors.New("network not ready")+ })+ assert.Nil(t, result)+ assert.Empty(t, task.history.samples)+ // Explicit requests may still run during the pause and record real failures.+ result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {+ return 0, errors.New("unreachable")+ })+ require.NotNil(t, result)+ assert.Equal(t, 100.0, result.PacketLoss)+ })+}++func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ var calls atomic.Int32+ pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {+ calls.Add(1)+ return 1, nil+ })+ defer pm.Stop()+ pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})+ simulateMonitorSleep(&pm.resumeGuard)+ pm.resumeGuard.snapshot()+ time.Sleep(9 * time.Second)+ synctest.Wait()+ assert.Zero(t, calls.Load())+ assert.Empty(t, pm.GetResults(1000))+ time.Sleep(2 * time.Second)+ synctest.Wait()+ assert.Positive(t, calls.Load())+ })+}diff --git a/agent/network_monitor_schedule.go b/agent/network_monitor_schedule.gonew file mode 100644index 000000000..fa53c36c4--- /dev/null+++ b/agent/network_monitor_schedule.go@@ -0,0 +1,60 @@+package agent++import (+ "context"+ "log/slog"+ "math/rand"+ "time"+)++func (pm *MonitorManager) startMonitor(task *monitorTask) {+ interval := time.Duration(task.config.Interval) * time.Second+ if interval < time.Second {+ interval = 30 * time.Second+ }+ delay := getStagger(interval.Milliseconds())+ slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)+ go runMonitorSchedule(task.ctx, interval, delay, func() {+ if _, allowed := task.resumeGuard.snapshot(); allowed {+ task.runProbe(pm.probe)+ }+ })+}++// runMonitorSchedule owns only timing. Checks run serially, and slow checks+// naturally drop missed ticks rather than building an execution backlog.+func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {+ timer := time.NewTimer(delay)+ defer timer.Stop()+ select {+ case <-ctx.Done():+ return+ case <-timer.C:+ }+ if ctx.Err() != nil {+ return+ }+ run()+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ if ctx.Err() != nil {+ return+ }+ run()+ }+ }+}++// getStagger returns an initial delay between half an interval and one interval.+func getStagger(intervalMilli int64) time.Duration {+ delay := rand.Intn(int(intervalMilli))+ if delay < int(intervalMilli)/2 {+ delay += int(intervalMilli) / 2+ }+ return time.Duration(delay) * time.Millisecond+}diff --git a/agent/network_monitor_schedule_test.go b/agent/network_monitor_schedule_test.gonew file mode 100644index 000000000..a66a59abe--- /dev/null+++ b/agent/network_monitor_schedule_test.go@@ -0,0 +1,167 @@+//go:build testing++package agent++import (+ "context"+ "sync/atomic"+ "testing"+ "testing/synctest"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestMonitorScheduleTiming(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ var calls atomic.Int32+ go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })+ synctest.Wait()+ time.Sleep(4 * time.Second)+ synctest.Wait()+ assert.Equal(t, 0, int(calls.Load()))+ time.Sleep(time.Second)+ synctest.Wait()+ assert.Equal(t, 1, int(calls.Load()))+ time.Sleep(10 * time.Second)+ synctest.Wait()+ assert.Equal(t, 2, int(calls.Load()))+ cancel()+ synctest.Wait()+ time.Sleep(time.Minute)+ synctest.Wait()+ assert.Equal(t, 2, int(calls.Load()))+ })+}++func TestMonitorScheduleSlowProbe(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ var calls atomic.Int32+ release := make(chan struct{})+ go runMonitorSchedule(ctx, time.Second, 0, func() {+ calls.Add(1)+ select {+ case <-release:+ case <-ctx.Done():+ }+ })+ synctest.Wait()+ assert.Equal(t, 1, int(calls.Load()))+ time.Sleep(time.Minute)+ synctest.Wait()+ assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")+ close(release)+ synctest.Wait()+ assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")+ time.Sleep(time.Second)+ synctest.Wait()+ assert.Equal(t, 2, int(calls.Load()))+ cancel()+ synctest.Wait()+ })+}++func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ var calls atomic.Int32+ release := make(chan struct{})+ cfg := monitor.Config{ID: "test", Interval: 10}+ pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {+ assert.Equal(t, cfg, config)+ calls.Add(1)+ <-release+ return 42, nil+ })+ defer pm.Stop()+ task := newMonitorTask(cfg)+ pm.monitors[cfg.ID] = task+ go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })+ synctest.Wait()+ results := make(chan *monitor.Result, 2)+ for range 2 {+ go func() {+ result, _ := pm.UpsertMonitor(cfg, true)+ results <- result+ }()+ }+ synctest.Wait()+ assert.Equal(t, 1, int(calls.Load()))+ assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")+ close(release)+ synctest.Wait()+ first, second := <-results, <-results+ require.NotNil(t, first)+ require.NotNil(t, second)+ assert.Equal(t, int64(42), first.AvgResponse)+ assert.Equal(t, first, second)+ assert.NotSame(t, first, second, "callers must not share mutable result pointers")+ assert.Len(t, task.history.samples, 1)+ // A later explicit request must still perform a fresh probe.+ _, err := pm.UpsertMonitor(cfg, true)+ require.NoError(t, err)+ assert.Equal(t, 2, int(calls.Load()))+ assert.Len(t, task.history.samples, 2)+ })+}++func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ cfg := monitor.Config{ID: "test", Interval: 10}+ pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {+ if config.Interval == 10 {+ <-ctx.Done()+ return 0, ctx.Err()+ }+ return 30, nil+ })+ defer pm.Stop()+ task := newMonitorTask(cfg)+ task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})+ pm.monitors[cfg.ID] = task+ results := make(chan *monitor.Result, 2)+ for range 2 {+ go func() {+ result, _ := pm.UpsertMonitor(cfg, true)+ results <- result+ }()+ }+ synctest.Wait()+ updated := cfg+ updated.Interval = 20+ result, err := pm.UpsertMonitor(updated, true)+ require.NoError(t, err)+ require.NotNil(t, result)+ assert.Equal(t, int64(20), result.AvgResponse)+ assert.Zero(t, result.PacketLoss)+ synctest.Wait()+ assert.Nil(t, <-results)+ assert.Nil(t, <-results)+ assert.Len(t, task.history.samples, 1)+ assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)+ })+}++func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {+ synctest.Test(t, func(t *testing.T) {+ pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)+ defer cancel()+ <-ctx.Done()+ return 0, ctx.Err()+ })+ defer pm.Stop()+ start := time.Now()+ result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)+ require.NoError(t, err)+ require.NotNil(t, result)+ assert.Equal(t, 3*time.Second, time.Since(start))+ assert.Equal(t, 100.0, result.PacketLoss)+ assert.NoError(t, pm.monitors["test"].ctx.Err())+ })+}diff --git a/agent/network_monitor_task.go b/agent/network_monitor_task.gonew file mode 100644index 000000000..2c3469ab2--- /dev/null+++ b/agent/network_monitor_task.go@@ -0,0 +1,116 @@+package agent++import (+ "context"+ "log/slog"+ "sync"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+)++const monitorFailureLogInterval = 5 * time.Minute++// monitorTask coordinates a probe and its history for one immutable configuration.+type monitorTask struct {+ config monitor.Config+ ctx context.Context+ cancel context.CancelFunc+ history *monitorHistory+ resumeGuard *monitorResumeGuard+ runMu sync.Mutex+ inflight *monitorRun+ lastFailureLog int64 // Unix nanoseconds+}++type monitorRun struct {+ done chan struct{}+ result *monitor.Result // published by closing done; never mutated afterwards+}++func newMonitorTask(config monitor.Config) *monitorTask {+ ctx, cancel := context.WithCancel(context.Background())+ task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}+ // Serialize cancellation with publication, so canceled probes cannot enter+ // history copied into a replacement task.+ task.cancel = func() {+ task.runMu.Lock()+ cancel()+ task.runMu.Unlock()+ }+ return task+}++func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {+ task := newMonitorTask(config)+ if existing != nil {+ task.history = existing.history.clone()+ }+ return task+}++// runProbe shares an in-flight check between scheduled and immediate requests.+// Every completed check contributes exactly one sample, regardless of how many+// callers were waiting for it. No task or history lock is held during network I/O.+func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {+ task.runMu.Lock()+ if task.ctx.Err() != nil {+ task.runMu.Unlock()+ return nil+ }+ if run := task.inflight; run != nil {+ task.runMu.Unlock()+ select {+ case <-task.ctx.Done():+ return nil+ case <-run.done:+ if task.ctx.Err() != nil {+ return nil+ }+ return copyMonitorResult(run.result)+ }+ }+ run := &monitorRun{done: make(chan struct{})}+ task.inflight = run+ task.runMu.Unlock()++ generation, _ := task.resumeGuard.snapshot()+ responseUs, err := probe(task.ctx, task.config)+ var logFailure bool+ task.runMu.Lock()+ currentGeneration, _ := task.resumeGuard.snapshot()+ if task.ctx.Err() == nil && generation == currentGeneration {+ now := time.Now()+ if err != nil {+ responseUs = -1+ logAt := now.UnixNano()+ if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {+ logFailure = true+ task.lastFailureLog = logAt+ }+ } else {+ task.lastFailureLog = 0+ }+ result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})+ run.result = &result+ }++ task.inflight = nil+ close(run.done)+ task.runMu.Unlock()+ if logFailure {+ slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)+ }+ if task.ctx.Err() != nil {+ return nil+ }+ return copyMonitorResult(run.result)+}++func copyMonitorResult(result *monitor.Result) *monitor.Result {+ if result == nil {+ return nil+ }+ copy := *result+ return ©+}diff --git a/agent/network_monitor_task_test.go b/agent/network_monitor_task_test.gonew file mode 100644index 000000000..427f71d5a--- /dev/null+++ b/agent/network_monitor_task_test.go@@ -0,0 +1,79 @@+//go:build testing++package agent++import (+ "bytes"+ "context"+ "errors"+ "log/slog"+ "testing"+ "testing/synctest"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestMonitorFailureLogCooldown(t *testing.T) {+ var logs bytes.Buffer+ previous := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))+ t.Cleanup(func() { slog.SetDefault(previous) })++ synctest.Test(t, func(t *testing.T) {+ task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})+ defer task.cancel()+ failure := errors.New("connection refused")+ probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }+ var samples int64+ check := func(wantLog bool) {+ t.Helper()+ logs.Reset()+ result := task.runProbe(probe)+ require.NotNil(t, result)+ samples+++ assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")+ if !wantLog {+ assert.Empty(t, logs.String())+ } else {+ assert.Contains(t, logs.String(), `msg="monitor failed"`)+ assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))+ }+ }++ check(true)+ check(false)+ time.Sleep(5*time.Minute - time.Nanosecond)+ check(false)+ time.Sleep(time.Nanosecond)+ check(true)+ check(false)+ time.Sleep(5 * time.Minute)+ check(true)+ check(false)++ // Recovery clears the cooldown.+ failure = nil+ check(false)+ failure = errors.New("connection refused again")+ check(true)++ // Another monitor has its own cooldown.+ other := newMonitorTask(task.config)+ defer other.cancel()+ logs.Reset()+ require.NotNil(t, other.runProbe(probe))+ assert.Contains(t, logs.String(), `msg="monitor failed"`)++ // A canceled probe must not publish a failure or emit a warning.+ logs.Reset()+ result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {+ other.cancel()+ return -1, context.Canceled+ })+ assert.Nil(t, result)+ assert.Empty(t, logs.String())+ })+}diff --git a/agent/network_monitor_test.go b/agent/network_monitor_test.gonew file mode 100644index 000000000..b24ab043e--- /dev/null+++ b/agent/network_monitor_test.go@@ -0,0 +1,524 @@+package agent++import (+ "context"+ "encoding/binary"+ "io"+ "net"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "golang.org/x/net/dns/dnsmessage"+)++func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {+ now := time.Now().UTC()+ task := newMonitorTask(monitor.Config{ID: "monitor-1"})+ task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})+ task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})+ task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})+ task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})+ task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})++ pm := newMonitorManager()+ pm.monitors = map[string]*monitorTask{"icmp:example.com": task}++ results := pm.GetResults(uint16(time.Minute / time.Millisecond))+ result, ok := results["monitor-1"]+ require.True(t, ok)+ assert.Equal(t, int64(30), result.AvgResponse)+ assert.Equal(t, int64(25), result.AvgResponse1h)+ assert.Equal(t, int64(30), result.MinResponse)+ assert.Equal(t, int64(10), result.MinResponse1h)+ assert.Equal(t, int64(30), result.MaxResponse)+ assert.Equal(t, int64(40), result.MaxResponse1h)+ assert.Equal(t, 50.0, result.PacketLoss)+ assert.Equal(t, 20.0, result.PacketLoss1h)+}++func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {+ now := time.Now().UTC()+ task := newMonitorTask(monitor.Config{ID: "monitor-1"})+ task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})+ task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})++ pm := newMonitorManager()+ pm.monitors = map[string]*monitorTask{"icmp:example.com": task}++ results := pm.GetResults(uint16(time.Minute / time.Millisecond))+ result, ok := results["monitor-1"]+ require.True(t, ok)+ assert.Equal(t, int64(0), result.AvgResponse)+ assert.Equal(t, int64(0), result.AvgResponse1h)+ assert.Equal(t, int64(0), result.MinResponse)+ assert.Equal(t, int64(0), result.MinResponse1h)+ assert.Equal(t, int64(0), result.MaxResponse)+ assert.Equal(t, int64(0), result.MaxResponse1h)+ assert.Equal(t, 100.0, result.PacketLoss)+ assert.Equal(t, 100.0, result.PacketLoss1h)+}++func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {+ cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}+ assert.Equal(t, "monitor-1", cfg.ID)+}++func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {+ validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}+ invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}++ pm := newMonitorManager()+ pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})+ defer pm.Stop()++ _, validExists := pm.monitors[validCfg.ID]+ _, invalidExists := pm.monitors[invalidCfg.ID]+ assert.True(t, validExists)+ assert.False(t, invalidExists)+}++func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {+ keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}+ removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}++ keptTask := newMonitorTask(keepCfg)+ removedTask := newMonitorTask(removeCfg)+ pm := newMonitorManager()+ pm.monitors = map[string]*monitorTask{+ keepCfg.ID: keptTask,+ removeCfg.ID: removedTask,+ }++ pm.SyncMonitors([]monitor.Config{keepCfg})++ assert.Same(t, keptTask, pm.monitors[keepCfg.ID])+ _, exists := pm.monitors[removeCfg.ID]+ assert.False(t, exists)++ select {+ case <-removedTask.ctx.Done():+ default:+ t.Fatal("expected removed monitor task to be cancelled")+ }++ select {+ case <-keptTask.ctx.Done():+ t.Fatal("expected existing monitor task to remain active")+ default:+ }+}++func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {+ originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}+ updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}+ originalTask := newMonitorTask(originalCfg)+ pm := newMonitorManager()+ pm.monitors = map[string]*monitorTask{+ originalCfg.ID: originalTask,+ }++ pm.SyncMonitors([]monitor.Config{updatedCfg})+ defer pm.Stop()++ restartedTask := pm.monitors[updatedCfg.ID]+ assert.NotSame(t, originalTask, restartedTask)+ assert.Equal(t, updatedCfg, restartedTask.config)++ select {+ case <-originalTask.ctx.Done():+ default:+ t.Fatal("expected changed monitor task to be cancelled")+ }+}++func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ pm := &MonitorManager{+ monitors: make(map[string]*monitorTask),+ probe: networkMonitorProbe(server.Client()),+ }++ resp, err := pm.HandleSyncRequest(monitor.SyncRequest{+ Action: monitor.SyncActionUpsert,+ Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},+ RunNow: true,+ })+ defer pm.Stop()++ require.NoError(t, err)+ assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))+ assert.Equal(t, 0.0, resp.Result.PacketLoss)+ assert.Equal(t, 0.0, resp.Result.PacketLoss1h)++ task := pm.monitors["monitor-1"]+ require.NotNil(t, task)+ task.history.mu.Lock()+ defer task.history.mu.Unlock()+ require.Len(t, task.history.samples, 1)+}++func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {+ originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}+ updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}+ now := time.Now().UTC()++ existingTask := newMonitorTask(originalCfg)+ existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})+ existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})++ pm := newMonitorManager()+ pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}++ result, err := pm.UpsertMonitor(updatedCfg, false)+ defer pm.Stop()++ require.NoError(t, err)+ assert.Nil(t, result)++ updatedTask := pm.monitors[updatedCfg.ID]+ require.NotNil(t, updatedTask)+ assert.NotSame(t, existingTask, updatedTask)+ assert.Equal(t, updatedCfg, updatedTask.config)++ updatedTask.history.mu.Lock()+ defer updatedTask.history.mu.Unlock()+ require.Len(t, updatedTask.history.samples, 1)+ assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)++ agg := updatedTask.history.aggregateLocked(time.Hour, now)+ require.True(t, agg.hasData())+ assert.Equal(t, int64(2), agg.totalCount)+ assert.Equal(t, int64(2), agg.successCount)+ assert.Equal(t, int64(18), agg.avgResponse())++ select {+ case <-existingTask.ctx.Done():+ default:+ t.Fatal("expected original monitor task to be cancelled")+ }+}++func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {+ config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}+ task := newMonitorTask(config)+ pm := newMonitorManager()+ pm.monitors = map[string]*monitorTask{config.ID: task}++ _, err := pm.HandleSyncRequest(monitor.SyncRequest{+ Action: monitor.SyncActionDelete,+ Config: monitor.Config{ID: config.ID},+ })++ require.NoError(t, err)+ _, exists := pm.monitors[config.ID]+ assert.False(t, exists)++ select {+ case <-task.ctx.Done():+ default:+ t.Fatal("expected deleted monitor task to be cancelled")+ }+}++func TestMonitorManagerGetRandomDelay(t *testing.T) {+ for i := 1000; i < 360_000; i += 1000 {+ delay := getStagger(int64(i))+ assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)+ assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)+ }+}++func TestMonitorHTTP(t *testing.T) {+ t.Run("success", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)+ require.NoError(t, err)+ assert.GreaterOrEqual(t, responseUs, int64(0))+ })++ t.Run("server error", func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "boom", http.StatusInternalServerError)+ }))+ defer server.Close()++ responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)+ assert.Equal(t, int64(-1), responseUs)+ require.Error(t, err)+ })+}++func TestMonitorTCP(t *testing.T) {+ t.Run("success", func(t *testing.T) {+ listener, err := net.Listen("tcp", "127.0.0.1:0")+ require.NoError(t, err)+ defer listener.Close()++ accepted := make(chan struct{})+ go func() {+ defer close(accepted)+ conn, err := listener.Accept()+ if err == nil {+ _ = conn.Close()+ }+ }()++ port := uint16(listener.Addr().(*net.TCPAddr).Port)+ responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)+ require.NoError(t, err)+ assert.GreaterOrEqual(t, responseUs, int64(0))+ <-accepted+ })++ t.Run("connection failure", func(t *testing.T) {+ listener, err := net.Listen("tcp", "127.0.0.1:0")+ require.NoError(t, err)++ port := uint16(listener.Addr().(*net.TCPAddr).Port)+ require.NoError(t, listener.Close())++ responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)+ assert.Equal(t, int64(-1), responseUs)+ require.Error(t, err)+ })+}++func TestMonitorTCPAddressFallback(t *testing.T) {+ for _, tc := range []struct {+ name string+ ips []string+ loss bool+ }{+ {"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},+ {"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},+ {"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ listener, err := net.Listen("tcp4", "127.0.0.1:0")+ require.NoError(t, err)+ defer listener.Close()+ original := net.DefaultResolver+ net.DefaultResolver = tcpMonitorTestResolver(tc.ips)+ defer func() { net.DefaultResolver = original }()++ // Verify the resolver preserves the intended order, so success cannot+ // accidentally bypass the failed first address in the regression case.+ ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")+ require.NoError(t, err)+ require.Equal(t, tc.ips, ips)+ responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))+ if tc.loss {+ require.Error(t, err)+ assert.Equal(t, int64(-1), responseUs)+ } else {+ require.NoError(t, err)+ assert.GreaterOrEqual(t, responseUs, int64(0))+ }+ })+ }+}++// tcpMonitorTestResolver supplies multiple A records without external DNS.+func tcpMonitorTestResolver(ips []string) *net.Resolver {+ return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {+ client, server := net.Pipe()+ go func() {+ defer server.Close()+ // net.Resolver uses TCP framing when its connection is not a PacketConn.+ var size uint16+ if err := binary.Read(server, binary.BigEndian, &size); err != nil {+ return+ }+ packet := make([]byte, size)+ if _, err := io.ReadFull(server, packet); err != nil {+ return+ }+ var msg dnsmessage.Message+ if err := msg.Unpack(packet); err != nil {+ return+ }+ msg.Header.Response = true+ msg.Header.RecursionAvailable = true+ for _, question := range msg.Questions {+ if question.Type != dnsmessage.TypeA {+ continue+ }+ for _, ip := range ips {+ msg.Answers = append(msg.Answers, dnsmessage.Resource{+ Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},+ Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},+ })+ }+ }+ packet, err := msg.Pack()+ if err != nil {+ return+ }+ response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))+ _, _ = server.Write(append(response, packet...))+ }()+ return client, nil+ }}+}++func TestMonitorDNS(t *testing.T) {+ t.Run("success", func(t *testing.T) {+ responseUs, err := monitorDNS(context.Background(), "localhost")+ require.NoError(t, err)+ assert.GreaterOrEqual(t, responseUs, int64(0))+ })++ t.Run("lookup failure", func(t *testing.T) {+ responseUs, err := monitorDNS(context.Background(), "")+ assert.Equal(t, int64(-1), responseUs)+ require.Error(t, err)+ })+}++func TestMonitorManagerCancelsActiveProbe(t *testing.T) {+ for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {+ t.Run(action, func(t *testing.T) {+ started := make(chan struct{})+ canceled := make(chan struct{})+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ close(started)+ select {+ case <-r.Context().Done():+ close(canceled)+ case <-release:+ }+ }))+ defer server.Close()+ defer close(release)+ pm := newMonitorManager()+ defer pm.Stop()+ cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}+ task := newMonitorTask(cfg)+ // Seed history to ensure a canceled RunNow does not return an old result.+ task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})+ pm.monitors[cfg.ID] = task+ done := make(chan *monitor.Result, 1)+ go func() {+ result, _ := pm.UpsertMonitor(cfg, true)+ done <- result+ }()+ select {+ case <-started:+ case <-time.After(time.Second):+ t.Fatal("probe did not start")+ }+ updated := cfg+ updated.Interval--+ switch action {+ case "stop":+ pm.Stop()+ case "delete":+ pm.DeleteMonitor(cfg.ID)+ case "upsert":+ _, err := pm.UpsertMonitor(updated, false)+ require.NoError(t, err)+ case "sync replace":+ pm.SyncMonitors([]monitor.Config{updated})+ case "sync remove":+ pm.SyncMonitors(nil)+ }+ select {+ case <-canceled:+ case <-time.After(time.Second):+ t.Fatal("active HTTP request was not canceled")+ }+ select {+ case result := <-done:+ assert.Nil(t, result)+ case <-time.After(time.Second):+ t.Fatal("RunNow did not return after cancellation")+ }+ task.history.mu.Lock()+ assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")+ task.history.mu.Unlock()+ })+ }+}++func TestMonitorResolutionCancellation(t *testing.T) {+ for _, protocol := range []string{"tcp", "dns", "icmp"} {+ t.Run(protocol, func(t *testing.T) {+ started := make(chan struct{}, 1)+ original := net.DefaultResolver+ net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {+ select {+ case started <- struct{}{}:+ default:+ }+ <-ctx.Done()+ return nil, ctx.Err()+ }}+ defer func() { net.DefaultResolver = original }()+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ done := make(chan error, 1)+ go func() {+ var err error+ switch protocol {+ case "tcp":+ _, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)+ case "dns":+ _, err = monitorDNS(ctx, "monitor-cancellation.invalid.")+ case "icmp":+ _, err = monitorICMP(ctx, "monitor-cancellation.invalid.")+ }+ done <- err+ }()+ select {+ case <-started:+ case <-time.After(time.Second):+ t.Fatal("lookup did not start")+ }+ cancel()+ select {+ case err := <-done:+ require.Error(t, err)+ case <-time.After(time.Second):+ t.Fatal("lookup did not cancel")+ }+ })+ }+}++func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ select {+ case <-r.Context().Done():+ case <-release:+ }+ }))+ defer server.Close()+ defer close(release)+ pm := newMonitorManager()+ pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})+ task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})+ defer task.cancel()++ result := task.runProbe(pm.probe)+ require.NotNil(t, result)+ assert.Equal(t, 100.0, result.PacketLoss)+ assert.Equal(t, 100.0, result.PacketLoss1h)+ require.Len(t, task.history.samples, 1)+ assert.Equal(t, int64(-1), task.history.samples[0].responseUs)+ assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")+}diff --git a/beszel.go b/beszel.goindex 57d8f4e1d..93f19b471 100644--- a/beszel.go+++ b/beszel.go@@ -6,7 +6,7 @@ import "github.com/blang/semver"const (// Version is the current version of the application.- Version = "0.19.0"+ Version = "0.20.0"// AppName is the name of the application.AppName = "beszel")@@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.var MinVersionZfsData = semver.MustParse("0.18.9")++// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.+var MinVersionNetworkMonitors = semver.MustParse("0.20.0")diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.goindex 8857addfc..589ec8363 100644--- a/internal/alerts/alerts.go+++ b/internal/alerts/alerts.go@@ -20,10 +20,11 @@ type hubLike interface {}type AlertManager struct {- hub hubLike- stopOnce sync.Once- pendingAlerts sync.Map- alertsCache *AlertsCache+ hub hubLike+ stopOnce sync.Once+ pendingAlerts sync.Map+ alertsCache *AlertsCache+ networkMonitors *networkMonitorCache}type AlertMessageData struct {@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{// NewAlertManager creates a new AlertManager instance.func NewAlertManager(app hubLike) *AlertManager {am := &AlertManager{- hub: app,- alertsCache: NewAlertsCache(app),+ hub: app,+ alertsCache: NewAlertsCache(app),+ networkMonitors: newNetworkMonitorCache(app),}am.bindEvents()return am@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {// Bind events to the alerts collection lifecyclefunc (am *AlertManager) bindEvents() {+ am.bindNetworkMonitorAlertEvents()am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.goindex 6e98c3bcb..7a44a22f5 100644--- a/internal/alerts/alerts_api.go+++ b/internal/alerts/alerts_api.go@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {return e.BadRequestError("Bad data", err)}+ if reqData.Name == alertNameNetworkMonitorLoss {+ if reqData.Value < 0 || reqData.Value >= 100 {+ return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)+ }+ reqData.Min = 0+ }+alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")if err != nil {return errdiff --git a/internal/alerts/alerts_cache.go b/internal/alerts/alerts_cache.goindex 0fce5c641..58550ff02 100644--- a/internal/alerts/alerts_cache.go+++ b/internal/alerts/alerts_cache.go@@ -1,6 +1,7 @@package alertsimport (+ "sync""time""github.com/pocketbase/dbx"@@ -18,6 +19,9 @@ type CachedAlertData struct {Triggered boolMin uint8PendingSince time.Time+ // Immutable after publication; decoded only when the alert record changes.+ MonitorStates map[string]string+ MonitorStatesValid bool// Created types.DateTime}@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {a.Triggered = record.GetBool("triggered")a.Min = uint8(record.GetInt("min"))a.PendingSince = record.GetDateTime("pending_since").Time()+ if a.Name == alertNameNetworkMonitorLoss {+ var state networkMonitorAlertState+ a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil+ a.MonitorStates = state.Monitors+ }// a.Created = record.GetDateTime("created")}// AlertsCache provides an in-memory cache for system alerts.type AlertsCache struct {+ // Serialize lazy loads with updates so a late load cannot replace newer state.+ loadMu sync.Mutexapp core.Appstore *store.Store[string, *store.Store[string, CachedAlertData]]populated bool@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {// PopulateFromDB clears current entries and loads all alerts from the database into the cache.func (c *AlertsCache) PopulateFromDB(force bool) error {+ c.loadMu.Lock()+ defer c.loadMu.Unlock()if !force && c.populated {return nil}@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {}c.store.RemoveAll()for _, record := range records {- c.Update(record)+ c.update(record)}c.populated = truereturn nil@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {// Update adds or updates an alert record in the cache.func (c *AlertsCache) Update(record *core.Record) {+ c.loadMu.Lock()+ defer c.loadMu.Unlock()+ c.update(record)+}++func (c *AlertsCache) update(record *core.Record) {systemID := record.GetString("system")if systemID == "" {return@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {// Delete removes an alert record from the cache.func (c *AlertsCache) Delete(record *core.Record) {+ c.loadMu.Lock()+ defer c.loadMu.Unlock()systemID := record.GetString("system")if systemID == "" {return@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {systemStore, ok := c.store.GetOk(systemID)if !ok {- // Populate cache for this system- records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))- if err != nil {- return nil+ c.loadMu.Lock()+ defer c.loadMu.Unlock()+ systemStore, ok = c.store.GetOk(systemID)+ if !ok {+ // Populate cache for this system+ records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))+ if err != nil {+ return nil+ }+ systemStore = store.New(map[string]CachedAlertData{})+ for _, record := range records {+ var ca CachedAlertData+ ca.PopulateFromRecord(record)+ systemStore.Set(record.Id, ca)+ }+ c.store.Set(systemID, systemStore)}- systemStore = store.New(map[string]CachedAlertData{})- for _, record := range records {- var ca CachedAlertData- ca.PopulateFromRecord(record)- systemStore.Set(record.Id, ca)- }- c.store.Set(systemID, systemStore)}all := systemStore.GetAll()alerts := make([]CachedAlertData, 0, len(all))diff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.goindex 9654456fd..c29475f6f 100644--- a/internal/alerts/alerts_history.go+++ b/internal/alerts/alerts_history.go@@ -9,6 +9,12 @@ import (// On triggered alert record delete, set matching alert history record to resolvedfunc resolveHistoryOnAlertDelete(e *core.RecordEvent) error {+ if e.Record.GetString("name") == alertNameNetworkMonitorLoss {+ if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {+ return err+ }+ return e.Next()+ }if !e.Record.GetBool("triggered") {return e.Next()}@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {// On alert record update, update alert history recordfunc updateHistoryOnAlertUpdate(e *core.RecordEvent) error {+ // Network monitor incidents have separate history entries per monitor.+ if e.Record.GetString("name") == alertNameNetworkMonitorLoss {+ return e.Next()+ }original := e.Record.Original()new := e.Recorddiff --git a/internal/alerts/alerts_network_monitors.go b/internal/alerts/alerts_network_monitors.gonew file mode 100644index 000000000..6b69c3ccf--- /dev/null+++ b/internal/alerts/alerts_network_monitors.go@@ -0,0 +1,269 @@+package alerts++import (+ "database/sql"+ "errors"+ "fmt"+ "math"+ "net"+ "strconv"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/pocketbase/dbx"+ "github.com/pocketbase/pocketbase/core"+)++const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"++// networkMonitorAlertState is this alert type's persisted runtime state.+// Monitor IDs map to their open history entries independently of history retention.+type networkMonitorAlertState struct {+ Monitors map[string]string `json:"monitors"`+}++func (am *AlertManager) bindNetworkMonitorAlertEvents() {+ // Hidden fields are still writable through the record API unless protected.+ protectState := func(e *core.RecordRequestEvent) error {+ e.Record.Set("state", e.Record.Original().Get("state"))+ oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")+ if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&+ (oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {+ return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)+ }+ if e.Record.GetString("name") == alertNameNetworkMonitorLoss {+ if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {+ return e.ForbiddenError("You do not have access to this system", nil)+ }+ e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))+ value := e.Record.GetFloat("value")+ if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {+ return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)+ }+ e.Record.Set("min", 0)+ }+ return e.Next()+ }+ am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)+ am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)+ cleanup := func(e *core.RecordEvent) error {+ if err := e.Next(); err != nil {+ return err+ }+ return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)+ }+ am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)+ am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {+ if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {+ return e.Next()+ }+ return cleanup(e)+ })+}++// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,+// using its exact payload (dashboard requests can replace the cached payload).+// Omitted results and disconnected systems never imply recovery.+func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {+ if systemRecord.GetString("status") != "up" {+ return nil+ }+ alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)+ if len(alerts) == 0 {+ return nil+ }+ monitors, err := am.networkMonitors.get(systemRecord.Id)+ if err != nil {+ return err+ }+ if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {+ return nil+ }+ // The cache only predicts a transition. Reload and recheck under the DB+ // transaction before persisting, including current system/monitor status.+ return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)+}++// networkMonitorTransitionPending does no IO and never mutates cached maps.+func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {+ for _, alert := range alerts {+ if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {+ return true+ }+ for id := range alert.MonitorStates {+ if _, enabled := monitors[id]; !enabled {+ return true+ }+ }+ for id, result := range results {+ interval, enabled := monitors[id]+ if !enabled || !monitorResultReady(result, interval, now) {+ continue+ }+ _, active := alert.MonitorStates[id]+ if (result.PacketLoss1h > alert.Value) != active {+ return true+ }+ }+ }+ return false+}++func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {+ var messages []AlertMessageData+ err := app.RunInTransaction(func(tx core.App) error {+ // Read configuration inside the transaction so concurrent threshold changes,+ // disabling, and evaluations cannot overwrite each other's incident state.+ alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})+ if err != nil || len(alerts) == 0 {+ return err+ }+ system, err := tx.FindRecordById("systems", systemID)+ if errors.Is(err, sql.ErrNoRows) {+ // System deletion cascades to its alerts.+ return nil+ }+ if err != nil {+ return err+ }+ monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})+ if err != nil {+ return err+ }+ enabled := make(map[string]*core.Record, len(monitors))+ for _, m := range monitors {+ enabled[m.Id] = m+ }+ now := time.Now()+ for _, alert := range alerts {+ var state networkMonitorAlertState+ if err := alert.UnmarshalJSONField("state", &state); err != nil {+ return err+ }+ states := state.Monitors+ if states == nil {+ states = map[string]string{}+ }+ changed := false+ // Removing or disabling a monitor closes its incident silently.+ for id, historyID := range states {+ if _, ok := enabled[id]; !ok {+ if err := resolveMonitorIncident(tx, historyID, now); err != nil {+ return err+ }+ delete(states, id)+ changed = true+ }+ }+ if system.GetString("status") == "up" {+ for _, m := range monitors {+ result, ok := results[m.Id]+ if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {+ continue+ }+ historyID, active := states[m.Id]+ triggered := result.PacketLoss1h > alert.GetFloat("value")+ if triggered == active {+ continue+ }+ label := m.GetString("target")+ if m.GetString("protocol") == "tcp" {+ label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))+ }+ if triggered {+ collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")+ if err != nil {+ return err+ }+ history := core.NewRecord(collection)+ history.Load(map[string]any{+ "alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,+ "name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,+ })+ if err := tx.Save(history); err != nil {+ return err+ }+ states[m.Id] = history.Id+ } else {+ if err := resolveMonitorIncident(tx, historyID, now); err != nil {+ return err+ }+ delete(states, m.Id)+ }+ changed = true+ state, comparison := "loss", "exceeds"+ if !triggered {+ state, comparison = "recovered", "is at or below"+ }+ messages = append(messages, AlertMessageData{+ UserID: alert.GetString("user"), SystemID: systemID,+ Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),+ Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),+ Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),+ })+ }+ }+ if changed || alert.GetBool("triggered") != (len(states) > 0) {+ alert.Set("state", networkMonitorAlertState{Monitors: states})+ alert.Set("triggered", len(states) > 0)+ if err := tx.Save(alert); err != nil {+ return err+ }+ }+ }+ return nil+ })+ if err != nil {+ return err+ }+ // Match other alert types: persist transitions before delivery, and respect+ // the user's existing notification destinations and quiet hours.+ for _, message := range messages {+ if err := am.SendAlert(message); err != nil {+ app.Logger().Error("Failed to send network monitor alert", "err", err)+ }+ }+ return nil+}++func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {+ // Three completed attempts provide a short warm-up, including after an agent+ // restart.+ if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {+ return false+ }+ // Never interpret an empty one-hour window as zero loss.+ maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))+ age := now.Sub(time.UnixMilli(result.LastProbeAt))+ return age >= -time.Minute && age <= maxAge+}++func resolveMonitorIncident(app core.App, id string, now time.Time) error {+ record, err := app.FindRecordById("alerts_history", id)+ if errors.Is(err, sql.ErrNoRows) {+ // History can be purged independently.+ return nil+ }+ if err != nil {+ return err+ }+ if !record.GetDateTime("resolved").IsZero() {+ return nil+ }+ record.Set("resolved", now.UTC())+ return app.Save(record)+}++func resolveNetworkMonitorHistory(app core.App, alertID string) error {+ records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})+ if err != nil {+ return err+ }+ for _, record := range records {+ record.Set("resolved", time.Now().UTC())+ if err := app.Save(record); err != nil {+ return err+ }+ }+ return nil+}diff --git a/internal/alerts/alerts_network_monitors_test.go b/internal/alerts/alerts_network_monitors_test.gonew file mode 100644index 000000000..aa02cf430--- /dev/null+++ b/internal/alerts/alerts_network_monitors_test.go@@ -0,0 +1,513 @@+//go:build testing++package alerts_test++import (+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/henrygd/beszel/internal/alerts"+ "github.com/henrygd/beszel/internal/entities/monitor"+ beszelTests "github.com/henrygd/beszel/internal/tests"+ "github.com/pocketbase/dbx"+ "github.com/pocketbase/pocketbase/core"+ pbTests "github.com/pocketbase/pocketbase/tests"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {+ t.Helper()+ hub, system, alert := systemdTestSetup(t, false)+ t.Cleanup(hub.Cleanup)+ alert.Set("name", "NetworkMonitorLoss")+ alert.Set("value", 5)+ require.NoError(t, hub.Save(alert))+ var monitors []*core.Record+ for _, name := range []string{"gateway", "website"} {+ record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,+ })+ require.NoError(t, err)+ monitors = append(monitors, record)+ }+ // Avoid starting a system update worker in tests.+ _, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()+ require.NoError(t, err)+ system.Set("status", "up")+ return hub, system, alert, monitors+}++func monitorResult(loss float64) monitor.Result {+ return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}+}++func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ count := hub.TestMailer.TotalSend()+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}+ check := func(active bool, open, sent int) {+ t.Helper()+ record, err := hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.Equal(t, active, record.GetBool("triggered"))+ total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})+ require.NoError(t, err)+ assert.EqualValues(t, open, total)+ assert.Equal(t, count+sent, hub.TestMailer.TotalSend())+ }+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ check(true, 1, 1)+ message := hub.TestMailer.Messages()[count]+ assert.Contains(t, message.Text, "gateway.example.com")+ assert.Contains(t, message.Text, "10.00%")+ assert.Contains(t, message.Text, "5.00%")+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ check(true, 1, 1)+ // Persisted monitor state prevents duplicate notifications after a hub restart.+ am = alerts.NewTestAlertManagerWithoutWorker(hub)+ results[monitors[1].Id] = monitorResult(20)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ check(true, 2, 2)+ results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ check(true, 1, 3)+ results[monitors[1].Id] = monitorResult(0)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ check(false, 0, 4)+ histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})+ require.NoError(t, err)+ require.Len(t, histories, 2)+ for _, history := range histories {+ assert.NotEmpty(t, history.GetString("monitor_name"))+ }+}++func TestNetworkMonitorAlertTargetLabel(t *testing.T) {+ for _, tc := range []struct {+ protocol, target, label string+ port int+ }{+ {"icmp", "gateway.example.com", "gateway.example.com", 0},+ {"http", "https://example.com/health", "https://example.com/health", 0},+ {"tcp", "example.com", "example.com:8443", 8443},+ {"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},+ } {+ t.Run(tc.label, func(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ m := monitors[0]+ m.Set("protocol", tc.protocol)+ m.Set("target", tc.target)+ m.Set("port", tc.port)+ require.NoError(t, hub.Save(m))+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))+ histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})+ require.NoError(t, err)+ require.Len(t, histories, 1)+ assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))+ assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)+ })+ }+}++func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {+ for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {+ t.Run(scenario, func(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ apply := func(loss float64) {+ result := monitorResult(loss)+ results := map[string]monitor.Result{monitors[0].Id: result}+ switch scenario {+ case "missing":+ results = nil+ case "stale":+ result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()+ results[monitors[0].Id] = result+ case "expired hourly window":+ monitors[0].Set("interval", 3600)+ require.NoError(t, hub.Save(monitors[0]))+ result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()+ results[monitors[0].Id] = result+ case "future":+ result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()+ results[monitors[0].Id] = result+ case "warmup":+ result.SampleCount = 2+ results[monitors[0].Id] = result+ case "no probes":+ result.SampleCount = 0+ results[monitors[0].Id] = result+ case "down":+ _, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()+ require.NoError(t, err)+ case "paused":+ record, err := hub.FindRecordById("systems", system.Id)+ require.NoError(t, err)+ record.Set("status", "paused")+ require.NoError(t, hub.Save(record))+ }+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ }+ count := hub.TestMailer.TotalSend()+ apply(100)+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ _, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()+ require.NoError(t, err)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))+ apply(0)+ assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")+ record, err := hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.True(t, record.GetBool("triggered"))+ })+ }+}++func TestNetworkMonitorAlertCleanup(t *testing.T) {+ for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {+ t.Run(scenario, func(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ count := hub.TestMailer.TotalSend()+ switch scenario {+ case "disable monitor":+ monitors[0].Set("enabled", false)+ require.NoError(t, hub.Save(monitors[0]))+ case "delete monitor":+ require.NoError(t, hub.Delete(monitors[0]))+ case "disable alert":+ require.NoError(t, hub.Delete(alert))+ case "delete system":+ require.NoError(t, hub.Delete(system))+ case "purge history":+ history, err := hub.FindAllRecords("alerts_history")+ require.NoError(t, err)+ for _, record := range history {+ require.NoError(t, hub.Delete(record))+ }+ }+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})+ require.NoError(t, err)+ if scenario == "disable monitor" || scenario == "delete monitor" {+ assert.EqualValues(t, 1, open)+ record, err := hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.True(t, record.GetBool("triggered"))+ require.NoError(t, hub.Delete(monitors[1]))+ record, err = hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.False(t, record.GetBool("triggered"))+ } else {+ assert.Zero(t, open)+ }+ })+ }+}++func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ user, err := beszelTests.CreateUser(hub, "[email protected]", "password")+ require.NoError(t, err)+ other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})+ require.NoError(t, err)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ other, err = hub.FindRecordById("alerts", other.Id)+ require.NoError(t, err)+ assert.False(t, other.GetBool("triggered"))+ // Editing the threshold re-evaluates on the next batch, without losing state.+ alert, err = hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ alert.Set("value", 15)+ require.NoError(t, hub.Save(alert))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ alert, err = hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ assert.False(t, alert.GetBool("triggered"))+}++func TestNetworkMonitorAlertAPI(t *testing.T) {+ for _, tc := range []struct {+ name string+ value float64+ direct, denied, patch bool+ status int+ }{+ {name: "zero threshold", value: 0, status: 200},+ {name: "fractional threshold", value: 5.5, status: 200},+ {name: "negative threshold", value: -1, status: 400},+ {name: "unreachable threshold", value: 100, status: 400},+ {name: "bulk inaccessible system", value: 5, denied: true, status: 200},+ {name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},+ {name: "direct invalid threshold", value: -1, direct: true, status: 400},+ {name: "direct private state", value: 5, direct: true, status: 200},+ {name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},+ } {+ t.Run(tc.name, func(t *testing.T) {+ hub, user := beszelTests.GetHubWithUser(t)+ defer hub.Cleanup()+ owner := user.Id+ if tc.denied {+ other, err := beszelTests.CreateUser(hub, "[email protected]", "password")+ require.NoError(t, err)+ owner = other.Id+ }+ systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")+ require.NoError(t, err)+ token, err := user.NewAuthToken()+ require.NoError(t, err)+ body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}+ url, method := "/api/beszel/user-alerts", "POST"+ if tc.direct {+ url = "/api/collections/alerts/records"+ body["system"], body["user"] = systems[0].Id, user.Id+ body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true+ }+ if tc.patch {+ alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})+ require.NoError(t, err)+ url += "/" + alert.Id+ method = "PATCH"+ body["triggered"] = false+ }+ content := `"success":true`+ if tc.direct {+ content = `"name":"NetworkMonitorLoss"`+ }+ if tc.status == 400 {+ content = `"status":400`+ }+ if tc.status == 403 {+ content = `"status":403`+ }+ scenario := beszelTests.ApiScenario{+ Name: tc.name, Method: method, URL: url, Body: jsonReader(body),+ Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},+ TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },+ }+ scenario.Test(t)+ records, err := hub.FindAllRecords("alerts")+ require.NoError(t, err)+ if tc.status != 200 || tc.denied {+ assert.Empty(t, records)+ return+ }+ require.Len(t, records, 1)+ assert.Equal(t, tc.value, records[0].GetFloat("value"))+ assert.Zero(t, records[0].GetInt("min"))+ state := struct {+ Monitors map[string]string `json:"monitors"`+ }{}+ require.NoError(t, records[0].UnmarshalJSONField("state", &state))+ states := state.Monitors+ if tc.patch {+ assert.Equal(t, map[string]string{"real": "history"}, states)+ assert.True(t, records[0].GetBool("triggered"))+ } else {+ assert.Empty(t, states)+ assert.False(t, records[0].GetBool("triggered"))+ }+ })+ }+}++type monitorCountingHub struct {+ *beszelTests.TestHub+ transactions atomic.Int64+ beforeTransaction func()+}++func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {+ h.transactions.Add(1)+ if h.beforeTransaction != nil {+ h.beforeTransaction()+ }+ return h.App.RunInTransaction(fn)+}++// Count actual SQL on both DB connections, including queries through record APIs.+func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {+ t.Helper()+ count := &atomic.Int64{}+ for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {+ db := builder.(*dbx.DB)+ old := db.LogFunc+ db.LogFunc = func(string, ...any) { count.Add(1) }+ t.Cleanup(func() { db.LogFunc = old })+ }+ return count+}++func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ counted := &monitorCountingHub{TestHub: hub}+ am := alerts.NewTestAlertManagerWithoutWorker(counted)+ sql := monitorSQLCounter(t, hub)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}+ evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }+ noWork := func() {+ t.Helper()+ sql.Store(0)+ counted.transactions.Store(0)+ for range 100 {+ evaluate()+ }+ assert.Zero(t, sql.Load(), "steady state must not issue SQL")+ assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")+ }+ // One-time lazy loads are permitted, including on hub restart.+ evaluate()+ assert.Positive(t, sql.Load())+ noWork()+ // Realtime metric saves invoke record hooks but must not invalidate config.+ fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)+ require.NoError(t, err)+ monitors[0] = fresh+ monitors[0].Set("loss1h", 0)+ monitors[0].Set("res", 100)+ require.NoError(t, hub.Save(monitors[0]))+ noWork()+ results[monitors[0].Id] = monitorResult(10)+ evaluate()+ assert.Positive(t, sql.Load(), "transitions must still be persisted")+ assert.EqualValues(t, 1, counted.transactions.Load())+ noWork()+ // Missing and stale observations must not enter the transaction either.+ results = nil+ noWork()+ results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}+ noWork()+ results[monitors[0].Id] = monitorResult(0)+ evaluate()+ noWork()+ require.NoError(t, hub.Delete(alert))+ noWork()+}++func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ count := hub.TestMailer.TotalSend()+ // Widening the interval makes this observation fresh. A stale interval cache+ // would miss the failure indefinitely, even though results keep arriving.+ result := monitorResult(10)+ result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()+ results[monitors[0].Id] = result+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ monitors[0].Set("interval", 120)+ require.NoError(t, hub.Save(monitors[0]))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())+ // Disable, then re-enable the same ID: its new failure must be detected.+ monitors[0].Set("enabled", false)+ require.NoError(t, hub.Save(monitors[0]))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ monitors[0].Set("enabled", true)+ require.NoError(t, hub.Save(monitors[0]))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count+2, hub.TestMailer.TotalSend())+ // A new monitor must also become eligible without restarting the hub.+ created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,+ })+ require.NoError(t, err)+ results[created.Id] = monitorResult(10)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count+3, hub.TestMailer.TotalSend())+ // Threshold changes refresh cached config and preserve the active incidents.+ alert, err = hub.FindRecordById("alerts", alert.Id)+ require.NoError(t, err)+ alert.Set("value", 15)+ require.NoError(t, hub.Save(alert))+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count+5, hub.TestMailer.TotalSend())+}++func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {+ for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {+ t.Run(change, func(t *testing.T) {+ hub, system, alert, monitors := networkAlertSetup(t)+ counted := &monitorCountingHub{TestHub: hub}+ am := alerts.NewTestAlertManagerWithoutWorker(counted)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ count := hub.TestMailer.TotalSend()+ // Change the DB after the cache predicts a transition, before its transaction.+ counted.beforeTransaction = func() {+ counted.beforeTransaction = nil+ switch change {+ case "threshold":+ alert.Set("value", 20)+ require.NoError(t, hub.Save(alert))+ case "disable alert":+ require.NoError(t, hub.Delete(alert))+ case "disable monitor":+ monitors[0].Set("enabled", false)+ require.NoError(t, hub.Save(monitors[0]))+ case "down":+ _, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()+ require.NoError(t, err)+ }+ }+ results[monitors[0].Id] = monitorResult(10)+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.EqualValues(t, 1, counted.transactions.Load())+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ histories, err := hub.CountRecords("alerts_history")+ require.NoError(t, err)+ assert.Zero(t, histories)+ })+ }+}++func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {+ hub, system, _, monitors := networkAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}+ count := hub.TestMailer.TotalSend()+ var wg sync.WaitGroup+ errs := make(chan error, 8)+ for range 8 {+ wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })+ }+ wg.Wait()+ close(errs)+ for err := range errs {+ require.NoError(t, err)+ }+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())+}++func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {+ hub, system, _, monitors := networkAlertSetup(t)+ am := alerts.NewTestAlertManagerWithoutWorker(hub)+ results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ _, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()+ require.NoError(t, err)+ count := hub.TestMailer.TotalSend()+ results[monitors[0].Id] = monitorResult(10)+ require.Error(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count, hub.TestMailer.TotalSend())+ histories, err := hub.CountRecords("alerts_history")+ require.NoError(t, err)+ assert.Zero(t, histories)+ _, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()+ require.NoError(t, err)+ // A failed transition must not be published to the cache and mask the retry.+ require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))+ assert.Equal(t, count+1, hub.TestMailer.TotalSend())+}diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.goindex ef0a61d9b..ad47d0bd5 100644--- a/internal/alerts/alerts_system.go+++ b/internal/alerts/alerts_system.go@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *systreturn nil}- alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)+ alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)if len(alerts) == 0 {return nil}diff --git a/internal/alerts/alerts_test_helpers.go b/internal/alerts/alerts_test_helpers.goindex df796b361..89f067c2a 100644--- a/internal/alerts/alerts_test_helpers.go+++ b/internal/alerts/alerts_test_helpers.go@@ -11,8 +11,9 @@ import (func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {return &AlertManager{- hub: app,- alertsCache: NewAlertsCache(app),+ hub: app,+ alertsCache: NewAlertsCache(app),+ networkMonitors: newNetworkMonitorCache(app),}}diff --git a/internal/alerts/network_monitor_cache.go b/internal/alerts/network_monitor_cache.gonew file mode 100644index 000000000..2ca3718e7--- /dev/null+++ b/internal/alerts/network_monitor_cache.go@@ -0,0 +1,79 @@+package alerts++import (+ "sync"++ "github.com/pocketbase/dbx"+ "github.com/pocketbase/pocketbase/core"+)++// networkMonitorCache keeps just the enabled monitor IDs and probe intervals+// needed for the alert fast path. Names and targets are read only on transitions.+// Returned maps are immutable; configuration changes invalidate the whole entry.+type networkMonitorCache struct {+ app core.App+ mu sync.RWMutex+ systems map[string]map[string]int+}++func newNetworkMonitorCache(app core.App) *networkMonitorCache {+ c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}+ invalidate := func(e *core.RecordEvent) error {+ c.invalidate(e.Record.GetString("system"))+ return e.Next()+ }+ app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)+ app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)+ app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {+ old := e.Record.Original()+ // Realtime metric saves also invoke this hook. They must not evict config.+ if old.GetString("system") != e.Record.GetString("system") ||+ old.GetBool("enabled") != e.Record.GetBool("enabled") ||+ old.GetInt("interval") != e.Record.GetInt("interval") {+ c.invalidate(old.GetString("system"))+ c.invalidate(e.Record.GetString("system"))+ }+ return e.Next()+ })+ app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {+ c.invalidate(e.Record.Id)+ return e.Next()+ })+ return c+}++func (c *networkMonitorCache) invalidate(systemID string) {+ c.mu.Lock()+ delete(c.systems, systemID)+ c.mu.Unlock()+}++func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {+ c.mu.RLock()+ monitors, ok := c.systems[systemID]+ c.mu.RUnlock()+ if ok {+ return monitors, nil+ }+ c.mu.Lock()+ defer c.mu.Unlock()+ if monitors, ok := c.systems[systemID]; ok {+ return monitors, nil+ }+ // Keep the lock through the load so a concurrent config change cannot be+ // invalidated first and then overwritten by the older query result.+ var rows []struct {+ ID string `db:"id"`+ Interval int `db:"interval"`+ }+ if err := c.app.DB().Select("id", "interval").From("network_monitors").+ Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {+ return nil, err+ }+ monitors = make(map[string]int, len(rows))+ for _, row := range rows {+ monitors[row.ID] = row.Interval+ }+ c.systems[systemID] = monitors+ return monitors, nil+}diff --git a/internal/common/common-ws.go b/internal/common/common-ws.goindex 3f69ba889..b9e8aea0c 100644--- a/internal/common/common-ws.go+++ b/internal/common/common-ws.go@@ -24,6 +24,8 @@ const (GetSystemdInfo// Request ZFS detail data from agentGetZfsData+ // Sync network monitor configuration to agent+ SyncNetworkMonitors// Add new actions here...)diff --git a/internal/entities/monitor/monitor.go b/internal/entities/monitor/monitor.gonew file mode 100644index 000000000..46995ce81--- /dev/null+++ b/internal/entities/monitor/monitor.go@@ -0,0 +1,102 @@+package monitor++import "time"++// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).+// Hub requests that run a probe must allow this time in addition to transport overhead.+const MaxProbeTimeout = 10 * time.Second++type SyncAction uint8++const (+ // SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.+ SyncActionReplace SyncAction = iota+ // SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.+ SyncActionUpsert+ // SyncActionDelete indicates an incremental sync where the provided config should be removed.+ SyncActionDelete+)++// Config defines a network monitor task sent from hub to agent.+type Config struct {+ // ID is the stable network_monitors record ID generated by the hub.+ ID string `cbor:"0,keyasint"`+ Target string `cbor:"1,keyasint"`+ Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"+ Port uint16 `cbor:"3,keyasint,omitempty"`+ Interval uint16 `cbor:"4,keyasint"` // seconds+}++// SyncRequest defines an incremental or full monitor sync request sent to the agent.+type SyncRequest struct {+ Action SyncAction `cbor:"0,keyasint"`+ Config Config `cbor:"1,keyasint,omitempty"`+ Configs []Config `cbor:"2,keyasint,omitempty"`+ RunNow bool `cbor:"3,keyasint,omitempty"`+}++// SyncResponse returns the immediate result for an upsert when requested.+type SyncResponse struct {+ Result Result `cbor:"0,keyasint,omitempty"`+}++// Result holds aggregated monitor results for a single target.+//+// 0: avg response in microseconds+//+// 1: 1h average response in microseconds+//+// 2: min response in microseconds+//+// 3: 1h min response in microseconds+//+// 4: max response in microseconds+//+// 5: 1h max response in microseconds+//+// 6: packet loss percentage (0-100)+//+// 7: 1h packet loss percentage (0-100)+type Result struct {+ AvgResponse int64 `cbor:"0,keyasint,omitempty"`+ AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`+ MinResponse int64 `cbor:"2,keyasint,omitempty"`+ MinResponse1h int64 `cbor:"3,keyasint,omitempty"`+ MaxResponse int64 `cbor:"4,keyasint,omitempty"`+ MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`+ PacketLoss float64 `cbor:"6,keyasint,omitempty"`+ PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`+ // LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.+ LastProbeAt int64 `cbor:"8,keyasint"`+ // SampleCount includes all completed probes since this monitor started.+ // Used for alert warm-up even when the interval is longer than 20 minutes.+ SampleCount int64 `cbor:"9,keyasint,omitempty"`+ // Counts and sum cover the current response window (or latest-sample+ // fallback), not the hourly window or lifetime SampleCount.+ TotalCount int64 `cbor:"10,keyasint"`+ SuccessCount int64 `cbor:"11,keyasint"`+ ResponseSum int64 `cbor:"12,keyasint"`+}++// Stats holds response times in microseconds and packet loss percentage (0-100).+type Stats struct {+ ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.+ ResMin float64 `json:"res_min" db:"res_min"`+ ResMax float64 `json:"res_max" db:"res_max"`+ Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.+ TotalCount int64 `json:"-" db:"total_count"`+ SuccessCount int64 `json:"-" db:"success_count"`+ ResponseSum int64 `json:"-" db:"res_sum"`+}++func (s Stats) FromResult(result Result) Stats {+ return Stats{+ ResAvg: float64(result.AvgResponse),+ ResMin: float64(result.MinResponse),+ ResMax: float64(result.MaxResponse),+ Loss: result.PacketLoss,+ TotalCount: result.TotalCount,+ SuccessCount: result.SuccessCount,+ ResponseSum: result.ResponseSum,+ }+}diff --git a/internal/entities/system/system.go b/internal/entities/system/system.goindex fd3c42e45..c6e8d0ea3 100644--- a/internal/entities/system/system.go+++ b/internal/entities/system/system.go@@ -7,6 +7,7 @@ import ("time""github.com/henrygd/beszel/internal/entities/container"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/henrygd/beszel/internal/entities/systemd")@@ -210,5 +211,6 @@ type CombinedData struct {Details *Details `cbor:"4,keyasint,omitempty"`// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response// that omitted systemd data (for example, a short-cache dashboard request).- SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`+ SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`+ Monitors map[string]monitor.Result `cbor:"6,keyasint"`}diff --git a/internal/hub/collections.go b/internal/hub/collections.goindex 667f4bf8e..3a0c3e85f 100644--- a/internal/hub/collections.go+++ b/internal/hub/collections.go@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {return err}- if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{+ if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{list: &systemScopedReadRule,}); err != nil {return err@@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error {return err}+ if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{+ list: &systemScopedReadRule,+ view: &systemScopedReadRule,+ create: &systemScopedWriteRule,+ update: &systemScopedWriteRule,+ delete: &systemScopedWriteRule,+ }); err != nil {+ return err+ }+if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{list: &systemScopedReadRule,view: &systemScopedReadRule,diff --git a/internal/hub/hub.go b/internal/hub/hub.goindex 30548cf2d..3f5f241b7 100644--- a/internal/hub/hub.go+++ b/internal/hub/hub.go@@ -109,6 +109,8 @@ func (h *Hub) StartHub() error {h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)+ bindNetworkMonitorsEvents(h)+pb, ok := h.App.(*pocketbase.PocketBase)if !ok {return errors.New("not a pocketbase app")diff --git a/internal/hub/network_monitors.go b/internal/hub/network_monitors.gonew file mode 100644index 000000000..25ad1b652--- /dev/null+++ b/internal/hub/network_monitors.go@@ -0,0 +1,158 @@+package hub++import (+ "strconv"+ "time"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/henrygd/beszel/internal/hub/systems"+ "github.com/pocketbase/pocketbase/core"+ "github.com/pocketbase/pocketbase/tools/types"+)++// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.+func generateMonitorID(systemId string, config monitor.Config) string {+ args := []string{systemId, config.Target, config.Protocol}+ // only use port for TCP monitors, since for other protocols it's not relevant as standalone value+ if config.Protocol == "tcp" {+ args = append(args, strconv.FormatUint(uint64(config.Port), 10))+ }+ return systems.MakeStableHashId(args...)+}++// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.+func bindNetworkMonitorsEvents(hub *Hub) {+ // on create, make sure the id is set to a stable hash+ hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {+ systemID := e.Record.GetString("system")+ config := monitorConfigFromRecord(e.Record)+ id := generateMonitorID(systemID, *config)+ e.Record.Set("id", id)+ return e.Next()+ })++ // sync monitor to agent on creation and persist the first result immediately when available+ hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {+ err := e.Next()+ if err != nil {+ return err+ }+ if !e.Record.GetBool("enabled") {+ return nil+ }+ // If connected, run the monitor immediately. Paused systems may be absent+ // from the manager; their monitors will sync when they reconnect.+ system, err := hub.sm.GetSystem(e.Record.GetString("system"))+ if err == nil && system.Status == "up" {+ go hub.upsertNetworkMonitor(e.Record, true)+ }+ return nil+ })++ // On API update requests, if the monitor config changed in a way that requires a new ID, create a new+ // record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.+ hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {+ systemID := e.Record.GetString("system")+ // only tcp uses port - set other protocols port to zero+ if e.Record.GetString("protocol") != "tcp" {+ e.Record.Set("port", 0)+ }+ ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))+ if ID != e.Record.Id {+ newRecord := copyMonitorToNewRecord(e.Record, ID)+ if err := e.App.Save(newRecord); err != nil {+ return err+ }+ if err := e.App.Delete(e.Record); err != nil {+ return err+ }+ return nil+ }+ err := e.Next()+ if err != nil {+ return err+ }+ if e.Record.GetBool("enabled") {+ // if the monitor is enabled, sync the updated config to the agent now+ runNow := !e.Record.Original().GetBool("enabled")+ err = hub.upsertNetworkMonitor(e.Record, runNow)+ } else {+ // if the monitor is paused, remove it from the agent+ err = hub.deleteNetworkMonitor(e.Record)+ }+ if err != nil {+ hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)+ }+ return nil+ })++ // sync monitor to agent on delete+ hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {+ if err := hub.deleteNetworkMonitor(e.Record); err != nil {+ hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)+ }+ return e.Next()+ })+}++// monitorConfigFromRecord builds a monitor config from a network_monitors record.+func monitorConfigFromRecord(record *core.Record) *monitor.Config {+ return &monitor.Config{+ ID: record.Id,+ Target: record.GetString("target"),+ Protocol: record.GetString("protocol"),+ Port: uint16(record.GetInt("port")),+ Interval: uint16(record.GetInt("interval")),+ }+}++// setMonitorResultFields stores the latest monitor result values on the record.+func setMonitorResultFields(record *core.Record, result monitor.Result) {+ nowString := time.Now().UTC().Format(types.DefaultDateLayout)+ record.Set("res", result.AvgResponse)+ record.Set("resAvg1h", result.AvgResponse1h)+ record.Set("resMin1h", result.MinResponse1h)+ record.Set("resMax1h", result.MaxResponse1h)+ record.Set("loss1h", result.PacketLoss1h)+ record.Set("updated", nowString)+}++// copyMonitorToNewRecord creates a new record with the same field values as the old one.+// This is used when the monitor config changes in a way that requires a new ID, so we need+// to create a new record with the new ID and delete the old one.+func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {+ collection := oldRecord.Collection()+ newRecord := core.NewRecord(collection)+ newRecord.Id = newID+ fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}+ for _, field := range fields {+ newRecord.Set(field, oldRecord.Get(field))+ }+ return newRecord+}++// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow+// is true, it will also trigger an immediate monitor run and update the record with the result.+func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {+ systemID := record.GetString("system")+ system, err := h.sm.GetSystem(systemID)+ if err != nil {+ return err+ }+ result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)+ if err != nil || result == nil {+ return err+ }+ setMonitorResultFields(record, *result)+ return h.App.SaveNoValidate(record)+}++// deleteNetworkMonitor removes the record's monitor from the target system.+func (h *Hub) deleteNetworkMonitor(record *core.Record) error {+ systemID := record.GetString("system")+ system, err := h.sm.GetSystem(systemID)+ if err != nil {+ return err+ }+ return system.DeleteNetworkMonitor(record.Id)+}diff --git a/internal/hub/network_monitors_test.go b/internal/hub/network_monitors_test.gonew file mode 100644index 000000000..f206ff84a--- /dev/null+++ b/internal/hub/network_monitors_test.go@@ -0,0 +1,225 @@+package hub++import (+ "bytes"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/pocketbase/pocketbase/apis"+ "github.com/pocketbase/pocketbase/core"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {+ for _, batch := range []bool{false, true} {+ name := "single"+ if batch {+ name = "batch"+ }+ t.Run(name, func(t *testing.T) {+ hub, testApp, err := createTestHub(t)+ require.NoError(t, err)+ defer cleanupTestHub(hub, testApp)+ bindNetworkMonitorsEvents(hub)++ user, err := createTestUser(hub)+ require.NoError(t, err)+ system, err := createTestRecord(hub, "systems", map[string]any{+ "name": "Paused", "host": "localhost", "port": "45876",+ "status": "paused", "users": []string{user.Id},+ })+ require.NoError(t, err)+ // Paused systems are not loaded into the manager at startup.+ _, err = hub.sm.GetSystem(system.Id)+ require.Error(t, err)++ payload := func(target string) map[string]any {+ return map[string]any{+ "system": system.Id, "target": target, "protocol": "icmp",+ "interval": 60, "enabled": true,+ }+ }+ url := "/api/collections/network_monitors/records"+ var body any = payload("1.1.1.1")+ count := 1+ if batch {+ body = map[string]any{"requests": []map[string]any{+ {"method": "POST", "url": url, "body": payload("1.1.1.1")},+ {"method": "POST", "url": url, "body": payload("8.8.8.8")},+ }}+ url = "/api/batch"+ count = 2+ }+ data, err := json.Marshal(body)+ require.NoError(t, err)+ token, err := user.NewAuthToken()+ require.NoError(t, err)+ router, err := apis.NewRouter(hub)+ require.NoError(t, err)+ handler, err := router.BuildMux()+ require.NoError(t, err)+ request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))+ request.Header.Set("Content-Type", "application/json")+ request.Header.Set("Authorization", token)+ response := httptest.NewRecorder()+ handler.ServeHTTP(response, request)+ assert.Equal(t, http.StatusOK, response.Code, response.Body.String())++ records, err := hub.FindAllRecords("network_monitors")+ require.NoError(t, err)+ require.Len(t, records, count)+ for _, record := range records {+ assert.Equal(t, system.Id, record.GetString("system"))+ assert.True(t, record.GetBool("enabled"))+ }+ })+ }+}++func TestGenerateMonitorID(t *testing.T) {+ tests := []struct {+ name string+ systemID string+ config monitor.Config+ expected string+ }{+ {+ name: "HTTP monitor on example.com",+ systemID: "sys123",+ config: monitor.Config{+ Protocol: "http",+ Target: "example.com",+ Port: 0,+ Interval: 60,+ },+ expected: "a20a5827",+ },+ {+ name: "HTTP monitor on example.com with different port",+ systemID: "sys123",+ config: monitor.Config{+ Protocol: "http",+ Target: "example.com",+ Port: 8080,+ Interval: 60,+ },+ expected: "a20a5827",+ },+ {+ name: "HTTP monitor on example.com with different system ID",+ systemID: "sys1234",+ config: monitor.Config{+ Protocol: "http",+ Target: "example.com",+ Port: 80,+ Interval: 60,+ },+ expected: "ab602ae7",+ },+ {+ name: "Same monitor, different interval",+ systemID: "sys1234",+ config: monitor.Config{+ Protocol: "http",+ Target: "example.com",+ Port: 80,+ Interval: 120,+ },+ expected: "ab602ae7",+ },+ {+ name: "ICMP monitor on 1.1.1.1",+ systemID: "sys456",+ config: monitor.Config{+ Protocol: "icmp",+ Target: "1.1.1.1",+ Port: 0,+ Interval: 10,+ },+ expected: "6d13a4a4",+ }, {+ name: "ICMP monitor on 1.1.1.1 with different system ID",+ systemID: "sys4567",+ config: monitor.Config{+ Protocol: "icmp",+ Target: "1.1.1.1",+ Port: 0,+ Interval: 10,+ },+ expected: "ddd6c81",+ },+ {+ name: "TCP monitor on example.com with port 443",+ systemID: "sys789",+ config: monitor.Config{+ Protocol: "tcp",+ Target: "example.com",+ Port: 443,+ Interval: 30,+ },+ expected: "677b991",+ },+ {+ name: "TCP monitor on example.com with port 8443",+ systemID: "sys789",+ config: monitor.Config{+ Protocol: "tcp",+ Target: "example.com",+ Port: 8443,+ Interval: 30,+ },+ expected: "84167969",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := generateMonitorID(tt.systemID, tt.config)+ assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)+ })+ }+}++func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {+ hub, testApp, err := createTestHub(t)+ require.NoError(t, err)+ defer cleanupTestHub(hub, testApp)++ collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")+ require.NoError(t, err)+ assert.Nil(t, collection.Fields.GetByName("name"))++ oldRecord := core.NewRecord(collection)+ oldRecord.Load(map[string]any{+ "system": "sys123",+ "target": "https://example.com",+ "protocol": "http",+ "port": 443,+ "interval": 60,+ "enabled": true,+ "res": 1200,+ "resAvg1h": 1300,+ "resMin1h": 900,+ "resMax1h": 1600,+ "loss1h": 5,+ "updated": "2026-04-29 12:00:00.000Z",+ })++ newRecord := copyMonitorToNewRecord(oldRecord, "next12345")++ assert.Equal(t, "next12345", newRecord.Id)+ assert.Equal(t, "https://example.com", newRecord.GetString("target"))+ assert.Equal(t, "http", newRecord.GetString("protocol"))+ assert.Equal(t, 443, newRecord.GetInt("port"))+ assert.True(t, newRecord.GetBool("enabled"))+ assert.Zero(t, newRecord.GetFloat("res"))+ assert.Zero(t, newRecord.GetFloat("resAvg1h"))+ assert.Zero(t, newRecord.GetFloat("resMin1h"))+ assert.Zero(t, newRecord.GetFloat("resMax1h"))+ assert.Zero(t, newRecord.GetFloat("loss1h"))+ assert.Equal(t, "", newRecord.GetString("updated"))+}diff --git a/internal/hub/systems/network_monitor_stats_test.go b/internal/hub/systems/network_monitor_stats_test.gonew file mode 100644index 000000000..38bc8cb04--- /dev/null+++ b/internal/hub/systems/network_monitor_stats_test.go@@ -0,0 +1,224 @@+//go:build testing++package systems++import (+ "fmt"+ "testing"+ "time"++ "github.com/fxamacker/cbor/v2"+ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/henrygd/beszel/internal/entities/system"+ "github.com/pocketbase/pocketbase/core"+ "github.com/pocketbase/pocketbase/tools/subscriptions"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestNetworkMonitorProbePruning(t *testing.T) {+ for _, tc := range []struct {+ name string+ monitors map[string]monitor.Result+ fail bool+ want map[string]int64+ }{+ {"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},+ {"empty report", map[string]monitor.Result{}, false, map[string]int64{}},+ {"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},+ {"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ sys, app := newTestSystemWithHub(t)+ sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}+ // Preserve the distinction between nil and empty across the agent transport.+ encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})+ require.NoError(t, err)+ var data system.CombinedData+ require.NoError(t, cbor.Unmarshal(encoded, &data))+ if tc.fail {+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()+ require.NoError(t, err)+ }+ _, err = sys.createRecords(&data)+ if tc.fail {+ require.Error(t, err)+ } else {+ require.NoError(t, err)+ }+ assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)+ })+ }+}++func TestNetworkMonitorStatsFreshness(t *testing.T) {+ for _, realtime := range []bool{false, true} {+ name := "sql"+ if realtime {+ name = "realtime"+ }+ t.Run(name, func(t *testing.T) {+ sys, app := newTestSystemWithHub(t)+ if realtime {+ client := subscriptions.NewDefaultClient()+ client.Subscribe("network_monitors/*")+ app.SubscriptionsBroker().Register(client)+ t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })+ }+ col, err := app.FindCachedCollectionByNameOrId("network_monitors")+ require.NoError(t, err)+ for _, id := range []string{"monitor1", "monitor2"} {+ record := core.NewRecord(col)+ record.Id = id+ record.Set("system", sys.Id)+ require.NoError(t, app.SaveNoValidate(record))+ }+ data := &system.CombinedData{Monitors: map[string]monitor.Result{+ "monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},+ "monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},+ }}+ count := func(want int64) {+ t.Helper()+ got, err := app.CountRecords("network_monitor_stats")+ require.NoError(t, err)+ assert.Equal(t, want, got)+ }+ save := func() {+ t.Helper()+ _, err := sys.createRecords(data)+ require.NoError(t, err)+ }+ save()+ count(2)+ stored, err := app.FindAllRecords("network_monitor_stats")+ require.NoError(t, err)+ for _, record := range stored {+ result := data.Monitors[record.GetString("monitor")]+ assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))+ assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))+ assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))+ }+ // A resume can overlap the scheduled update with the same probe.+ errs := make(chan error, 4)+ for range 4 {+ go func() {+ _, err := sys.createRecords(data)+ errs <- err+ }()+ }+ for range 4 {+ require.NoError(t, <-errs)+ }+ count(2)++ // A rolling hourly value can change without a new probe.+ result := data.Monitors["monitor1"]+ result.AvgResponse1h = 42+ data.Monitors["monitor1"] = result+ save()+ count(2)+ record, err := app.FindRecordById("network_monitors", "monitor1")+ require.NoError(t, err)+ assert.Equal(t, 42, record.GetInt("resAvg1h"))++ // Identical response values and failed probes still count as new measurements.+ for id, result := range data.Monitors {+ result.LastProbeAt = 301000+ data.Monitors[id] = result+ }+ save()+ count(4)++ // A failed individual insert must remain retryable, even if others commit.+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()+ require.NoError(t, err)+ for id, result := range data.Monitors {+ result.LastProbeAt = 601000+ data.Monitors[id] = result+ }+ save()+ count(5)+ assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])+ assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])+ _, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()+ require.NoError(t, err)+ save()+ count(6)++ // Failure after inserting stats rolls back the whole transaction and its markers.+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()+ require.NoError(t, err)+ result = data.Monitors["monitor1"]+ result.LastProbeAt = 901000+ data.Monitors["monitor1"] = result+ _, err = sys.createRecords(data)+ require.Error(t, err)+ count(6)+ assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])+ _, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()+ require.NoError(t, err)+ save()+ count(7)++ // Clock rollback is a new probe identity, not a reason to stall writes.+ result.LastProbeAt = 500+ data.Monitors["monitor1"] = result+ save()+ count(8)++ // Recreated systems intentionally accept the first result without restoring state.+ sys = &System{Id: sys.Id, manager: sys.manager}+ save()+ count(10)+ })+ }+}++// Observes the committed DB through the hub, not the transaction's app.+type monitorAlertHub struct {+ stubHub+ handle func(*core.Record, map[string]monitor.Result) error+}++func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {+ return h.handle(record, results)+}++func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {+ for _, realtime := range []bool{false, true} {+ t.Run(fmt.Sprint(realtime), func(t *testing.T) {+ sys, app := newTestSystemWithHub(t)+ if realtime {+ client := subscriptions.NewDefaultClient()+ client.Subscribe("network_monitors/*")+ app.SubscriptionsBroker().Register(client)+ }+ collection, err := app.FindCachedCollectionByNameOrId("network_monitors")+ require.NoError(t, err)+ record := core.NewRecord(collection)+ record.Set("system", sys.Id)+ require.NoError(t, app.SaveNoValidate(record))+ called := 0+ result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}+ sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {+ called+++ assert.Equal(t, sys.Id, systemRecord.Id)+ assert.Equal(t, result, results[record.Id])+ saved, err := app.FindRecordById("network_monitors", record.Id)+ require.NoError(t, err)+ assert.Equal(t, 10.0, saved.GetFloat("loss1h"))+ return nil+ }}+ data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}+ _, err = sys.createRecords(data)+ require.NoError(t, err)+ assert.Equal(t, 1, called)+ // A transaction that fails after writing monitor stats must not notify.+ _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()+ require.NoError(t, err)+ _, err = sys.createRecords(data)+ require.Error(t, err)+ assert.Equal(t, 1, called)+ })+ }+}diff --git a/internal/hub/systems/network_monitor_sync_test.go b/internal/hub/systems/network_monitor_sync_test.gonew file mode 100644index 000000000..f9c7666ed--- /dev/null+++ b/internal/hub/systems/network_monitor_sync_test.go@@ -0,0 +1,144 @@+//go:build testing++package systems++import (+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/blang/semver"+ "github.com/fxamacker/cbor/v2"+ "github.com/henrygd/beszel/internal/common"+ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/henrygd/beszel/internal/hub/ws"+ "github.com/lxzan/gws"+ "github.com/pocketbase/pocketbase/core"+ "github.com/stretchr/testify/require"+)++type monitorSyncClient struct {+ gws.BuiltinEventHandler+ requests chan common.HubRequest[monitor.SyncRequest]+}++func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {+ defer message.Close()+ var req common.HubRequest[monitor.SyncRequest]+ if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {+ return+ }+ c.requests <- req+ data, _ := cbor.Marshal(monitor.SyncResponse{})+ response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, Data: data})+ _ = conn.WriteMessage(gws.OpcodeBinary, response)+}++// Avoid the production delayed disconnect notification; these tests explicitly+// remove each connection from the manager before reconnecting.+type monitorSyncServer struct{ ws.Handler }++func (*monitorSyncServer) OnClose(*gws.Conn, error) {}++func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {+ for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {+ t.Run(version, func(t *testing.T) {+ // No transport: attempting to send any request would fail.+ sys := &System{agentVersion: semver.MustParse(version)}+ require.NoError(t, sys.SyncNetworkMonitors(nil))+ result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)+ require.NoError(t, err)+ require.Nil(t, result)+ require.NoError(t, sys.DeleteNetworkMonitor("test"))+ })+ }+}++func TestNetworkMonitorReconnectSync(t *testing.T) {+ for _, change := range []string{"delete", "disable"} {+ t.Run(change, func(t *testing.T) {+ sys, app := newTestSystemWithHub(t)+ record, err := app.FindRecordById("systems", sys.Id)+ require.NoError(t, err)+ // Suppress unrelated system-stat requests while exercising reconnects.+ record.Set("status", paused)+ require.NoError(t, app.SaveNoValidate(record))+ collection, err := app.FindCachedCollectionByNameOrId("network_monitors")+ require.NoError(t, err)+ probe := core.NewRecord(collection)+ probe.Load(map[string]any{+ "system": sys.Id, "target": "localhost", "protocol": "tcp",+ "port": 80, "interval": 60, "enabled": true,+ })+ require.NoError(t, app.SaveNoValidate(probe))++ sm := NewSystemManager(stubHub{app})+ t.Cleanup(func() {+ sm.cancel()+ _ = sm.RemoveSystem(sys.Id)+ sm.smartFetchMap.StopCleaner()+ sm.zfsFetchMap.StopCleaner()+ })+ version := semver.MustParse("0.20.0")+ connections := make(chan *ws.WsConn, 1)+ upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ conn, err := upgrader.Upgrade(w, r)+ if err != nil {+ t.Error(err)+ return+ }+ wsConn := ws.NewWsConnection(conn, version)+ conn.Session().Store("wsConn", wsConn)+ connections <- wsConn+ conn.ReadLoop()+ }))+ t.Cleanup(server.Close)+ client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}+ connect := func() monitor.SyncRequest {+ t.Helper()+ conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})+ require.NoError(t, err)+ t.Cleanup(func() { _ = conn.NetConn().Close() })+ go conn.ReadLoop()+ select {+ case wsConn := <-connections:+ require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))+ case <-time.After(3 * time.Second):+ t.Fatal("websocket connection was not established")+ }+ select {+ case req := <-client.requests:+ require.Equal(t, common.SyncNetworkMonitors, req.Action)+ require.Equal(t, monitor.SyncActionReplace, req.Data.Action)+ return req.Data+ case <-time.After(3 * time.Second):+ t.Fatal("reconnected agent did not receive a monitor replacement")+ return monitor.SyncRequest{}+ }+ }++ initial := connect()+ require.Len(t, initial.Configs, 1)+ require.Equal(t, probe.Id, initial.Configs[0].ID)+ require.NoError(t, sm.RemoveSystem(sys.Id))+ if change == "delete" {+ require.NoError(t, app.Delete(probe))+ } else {+ probe.Set("enabled", false)+ require.NoError(t, app.SaveNoValidate(probe))+ }+ require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")+ })+ }+}++func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {+ sys, app := newTestSystemWithHub(t)+ _, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()+ require.NoError(t, err)+ _, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)+ require.Error(t, err, "a failed query must not be treated as an empty monitor set")+}diff --git a/internal/hub/systems/network_monitors.go b/internal/hub/systems/network_monitors.gonew file mode 100644index 000000000..0fb44ed47--- /dev/null+++ b/internal/hub/systems/network_monitors.go@@ -0,0 +1,58 @@+package systems++import (+ "context"+ "time"++ "github.com/henrygd/beszel"+ "github.com/henrygd/beszel/internal/common"+ "github.com/henrygd/beszel/internal/entities/monitor"+)++// SyncNetworkMonitors sends monitor configurations to the agent.+func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {+ _, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})+ return err+}++// UpsertNetworkMonitor sends a single monitor configuration change to the agent.+func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {+ resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{+ Action: monitor.SyncActionUpsert,+ Config: config,+ RunNow: runNow,+ })+ if err != nil {+ return nil, err+ }+ if resp.Result == (monitor.Result{}) {+ return nil, nil+ }+ result := resp.Result+ return &result, nil+}++// DeleteNetworkMonitor removes a single monitor task from the agent.+func (sys *System) DeleteNetworkMonitor(id string) error {+ _, err := sys.syncNetworkMonitors(monitor.SyncRequest{+ Action: monitor.SyncActionDelete,+ Config: monitor.Config{ID: id},+ })+ return err+}++func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {+ if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {+ return monitor.SyncResponse{}, nil+ }+ timeout := 5 * time.Second+ if req.Action == monitor.SyncActionUpsert && req.RunNow {+ // Allow the probe to finish, including a timeout result, while preserving+ // the normal request budget for transport and response handling.+ timeout += monitor.MaxProbeTimeout+ }+ ctx, cancel := context.WithTimeout(context.Background(), timeout)+ defer cancel()+ var result monitor.SyncResponse+ return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)+}diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.goindex 2ba22169e..6b0ca713f 100644--- a/internal/hub/systems/system.go+++ b/internal/hub/systems/system.go@@ -9,6 +9,7 @@ import ("math/rand""net""strings"+ "sync""sync/atomic""time"@@ -18,6 +19,7 @@ import ("github.com/henrygd/beszel/internal/hub/ws""github.com/henrygd/beszel/internal/entities/container"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/henrygd/beszel/internal/entities/smart""github.com/henrygd/beszel/internal/entities/system""github.com/henrygd/beszel/internal/entities/systemd"@@ -30,6 +32,8 @@ import ("github.com/lxzan/gws""github.com/pocketbase/dbx""github.com/pocketbase/pocketbase/core"+ "github.com/pocketbase/pocketbase/tools/security"+ "github.com/pocketbase/pocketbase/tools/types""golang.org/x/crypto/ssh")@@ -52,6 +56,10 @@ type System struct {smartInterval time.Duration // Interval for periodic SMART data updateszfsFetching atomic.Bool // True if ZFS pools are currently being fetchedzfsInterval time.Duration // Interval for periodic ZFS detail data updates+ // Serialize persistence from scheduled updates and resumes through commit.+ recordsMu sync.Mutex+ // Protected by recordsMu; realtime reads don't consume probes.+ lastSavedMonitorProbe map[string]int64}func (sm *SystemManager) NewSystem(systemId string) *System {@@ -211,11 +219,15 @@ func (sys *System) handlePaused() {// createRecords updates the system record and adds system_stats and container_stats recordsfunc (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {+ sys.recordsMu.Lock()+ defer sys.recordsMu.Unlock()+systemRecord, err := sys.getRecord(sys.manager.hub)if err != nil {return nil, err}hub := sys.manager.hub+ savedMonitorProbes := make(map[string]int64)err = hub.RunInTransaction(func(txApp core.App) error {// add system_stats recordsystemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")@@ -266,6 +278,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error}}+ if data.Monitors != nil {+ if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {+ return err+ }+ }+if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {return err}@@ -287,6 +305,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, errorreturn nil})+ // Publish only successful inserts after the entire transaction commits.+ if err == nil && len(savedMonitorProbes) > 0 {+ if sys.lastSavedMonitorProbe == nil {+ sys.lastSavedMonitorProbe = savedMonitorProbes+ } else {+ for id, timestamp := range savedMonitorProbes {+ sys.lastSavedMonitorProbe[id] = timestamp+ }+ }+ }+ // A non-nil report includes cached results for all remaining monitors.+ if err == nil && data.Monitors != nil {+ for id := range sys.lastSavedMonitorProbe {+ if _, exists := data.Monitors[id]; !exists {+ delete(sys.lastSavedMonitorProbe, id)+ }+ }+ }+ if err == nil {+ if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {+ hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)+ }+ }return systemRecord, err}@@ -337,7 +378,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s}suffix := fmt.Sprintf("%d", i)valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))- params["id"+suffix] = makeStableHashId(systemId, service.Name)+ params["id"+suffix] = MakeStableHashId(systemId, service.Name)params["name"+suffix] = service.Nameparams["state"+suffix] = service.Stateparams["sub"+suffix] = service.Sub@@ -363,6 +404,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId sreturn err}+func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {+ if len(monitorResults) == 0 {+ return nil+ }+ var err error+ systemId := sys.Id+ const monitorCollectionName = "network_monitors"++ // If realtime updates are active, we save via PocketBase records to trigger realtime events.+ // Otherwise we can do a more efficient direct update via SQL+ realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {+ return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)+ })++ now := time.Now().UTC()+ nowMilli := now.UnixMilli()+ nowString := now.Format(types.DefaultDateLayout)+ var db dbx.Builder+ var updateQuery *dbx.Query+ if !realtimeActive {+ db = app.DB()+ monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}+ setClauses := make([]string, len(monitorFields))+ for i, f := range monitorFields {+ setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)+ }+ queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))+ updateQuery = db.NewQuery(queryString)+ }++ // update network_monitors records+ for id, result := range monitorResults {+ monitorData := map[string]any{+ "id": id,+ "res": result.AvgResponse,+ "resAvg1h": result.AvgResponse1h,+ "resMin1h": result.MinResponse1h,+ "resMax1h": result.MaxResponse1h,+ "loss1h": result.PacketLoss1h,+ "updated": nowString,+ }+ switch realtimeActive {+ case true:+ var record *core.Record+ record, err = app.FindRecordById(monitorCollectionName, id)+ if err == nil {+ record.Load(monitorData)+ err = app.SaveNoValidate(record)+ }+ default:+ _, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()+ }+ if err != nil {+ app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)+ }+ }++ // handle stats collection — one record per monitor+ const statsCollectionName = "network_monitor_stats"++ var statsCollection *core.Collection+ if realtimeActive {+ statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)+ }++ for monitorId, result := range monitorResults {+ // Compare identity, not ordering, so agent clock changes don't stall writes.+ if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {+ continue+ }+ statsRecordData := map[string]any{+ "system": systemId,+ "monitor": monitorId,+ "type": "1m",+ "created": nowMilli,+ "res_min": result.MinResponse,+ "res_max": result.MaxResponse,+ "total_count": result.TotalCount,+ "success_count": result.SuccessCount,+ "res_sum": result.ResponseSum,+ }+ switch realtimeActive {+ case true:+ record := core.NewRecord(statsCollection)+ record.Load(statsRecordData)+ err = app.SaveNoValidate(record)+ default:+ statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)+ _, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()+ }+ if err != nil {+ app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)+ } else {+ savedProbes[monitorId] = result.LastProbeAt+ }+ }++ return nil+}+// createContainerRecords creates container recordsfunc createContainerRecords(app core.App, data []*container.Stats, systemId string) error {if len(data) == 0 {@@ -622,7 +763,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {return &result, err}-func makeStableHashId(strings ...string) string {+func MakeStableHashId(strings ...string) string {hash := fnv.New32a()for _, str := range strings {hash.Write([]byte(str))diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.goindex c1a4cde46..db279609c 100644--- a/internal/hub/systems/system_manager.go+++ b/internal/hub/systems/system_manager.go@@ -9,6 +9,7 @@ import ("github.com/henrygd/beszel/internal/hub/ws"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/henrygd/beszel/internal/entities/system""github.com/henrygd/beszel/internal/hub/expirymap"@@ -17,6 +18,7 @@ import ("github.com/henrygd/beszel""github.com/blang/semver"+ "github.com/pocketbase/dbx""github.com/pocketbase/pocketbase/core""github.com/pocketbase/pocketbase/tools/store""golang.org/x/crypto/ssh"@@ -62,6 +64,7 @@ type hubLike interface {core.AppGetSSHKey(dataDir string) (ssh.Signer, error)HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error+ HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) errorHandleStatusAlerts(status string, systemRecord *core.Record) errorHandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) errorCancelPendingStatusAlerts(systemID string)@@ -350,6 +353,20 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semverif err := sm.AddRecord(systemRecord, system); err != nil {return err}++ // Sync network monitors to the newly connected agent+ go func() {+ configs, err := sm.GetMonitorConfigsForSystem(systemId)+ if err != nil {+ sm.hub.Logger().Warn("failed to load monitors for agent", "system", systemId, "err", err)+ return+ }+ // An empty set must also replace any probes retained across a disconnect.+ if err := system.SyncNetworkMonitors(configs); err != nil {+ sm.hub.Logger().Warn("failed to sync monitors to agent", "system", systemId, "err", err)+ }+ }()+return nil}@@ -362,6 +379,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {}}+// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.+func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {+ var configs []monitor.Config+ err := sm.hub.DB().+ NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").+ Bind(dbx.Params{"system": systemID}).+ All(&configs)+ return configs, err+}+// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh// agent reconnect retries ZFS discovery immediately after configuration changes.func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {@@ -397,11 +424,12 @@ func (sm *SystemManager) createSSHClientConfig() error {// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.// This is called when a system is paused or goes offline to prevent continued alerts.+// Monitor incidents remain open: a missing observation does not establish recovery.func deactivateAlerts(app core.App, systemID string) error {// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()- alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0)+ alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)if err != nil {return err}diff --git a/internal/hub/systems/system_realtime.go b/internal/hub/systems/system_realtime.goindex 0fb6b6f9a..f290eda6e 100644--- a/internal/hub/systems/system_realtime.go+++ b/internal/hub/systems/system_realtime.go@@ -6,6 +6,8 @@ import ("time""github.com/henrygd/beszel/internal/common"+ "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/henrygd/beszel/internal/entities/system""github.com/henrygd/beszel/internal/hub/utils""github.com/pocketbase/dbx""github.com/pocketbase/pocketbase/apis"@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {if err != nil {return}- bytes, err := json.Marshal(data)+ bytes, err := marshalRealtimeData(data)if err == nil {notify(sm.hub, system, fetch.subscription, bytes)}@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {}}+// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting+// the per-monitor results into the derived metric fields the frontend charts expect.+func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {+ if len(data.Monitors) == 0 {+ return json.Marshal(data)+ }+ monitorStats := make(map[string]monitor.Stats, len(data.Monitors))+ for id, result := range data.Monitors {+ monitorStats[id] = monitor.Stats{}.FromResult(result)+ }+ return json.Marshal(struct {+ *system.CombinedData+ Monitors map[string]monitor.Stats `json:"Monitors"`+ }{data, monitorStats})+}+// notify broadcasts realtime data to all clients subscribed to a specific subscription.// Custom topics bypass collection rules, so check current access for every// recipient, including clients whose authentication or membership was revoked.diff --git a/internal/hub/systems/system_smart.go b/internal/hub/systems/system_smart.goindex 68b750fe4..9a6db8de4 100644--- a/internal/hub/systems/system_smart.go+++ b/internal/hub/systems/system_smart.go@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, complecurrentIDs := make(map[string]struct{}, len(smartData))for deviceKey := range smartData {- currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{}+ currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}}err = hub.RunInTransaction(func(txApp core.App) error {@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple}func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {- recordID := makeStableHashId(sys.Id, deviceKey)+ recordID := MakeStableHashId(sys.Id, deviceKey)record, err := app.FindRecordById(collection, recordID)if err != nil {diff --git a/internal/hub/systems/system_smart_test.go b/internal/hub/systems/system_smart_test.goindex aa98633db..460a38f3e 100644--- a/internal/hub/systems/system_smart_test.go+++ b/internal/hub/systems/system_smart_test.go@@ -7,6 +7,7 @@ import ("testing""time"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/henrygd/beszel/internal/entities/smart"esystem "github.com/henrygd/beszel/internal/entities/system""github.com/henrygd/beszel/internal/hub/expirymap"@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {return nil}-func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }+func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }+func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {return nil}@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {}, false))assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)- recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA"))+ recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))require.NoError(t, err)assert.EqualValues(t, 42, recordA.GetInt("temp"))}diff --git a/internal/hub/systems/system_systemd_test.go b/internal/hub/systems/system_systemd_test.goindex c2d890d5a..d5b103e0b 100644--- a/internal/hub/systems/system_systemd_test.go+++ b/internal/hub/systems/system_systemd_test.go@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {serviceName := "nginx.service"// Call multiple times and ensure same result- id1 := makeStableHashId(systemId, serviceName)- id2 := makeStableHashId(systemId, serviceName)- id3 := makeStableHashId(systemId, serviceName)+ id1 := MakeStableHashId(systemId, serviceName)+ id2 := MakeStableHashId(systemId, serviceName)+ id3 := MakeStableHashId(systemId, serviceName)assert.Equal(t, id1, id2)assert.Equal(t, id2, id3)@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {serviceName1 := "nginx.service"serviceName2 := "apache.service"- id1 := makeStableHashId(systemId1, serviceName1)- id2 := makeStableHashId(systemId2, serviceName1)- id3 := makeStableHashId(systemId1, serviceName2)- id4 := makeStableHashId(systemId2, serviceName2)+ id1 := MakeStableHashId(systemId1, serviceName1)+ id2 := MakeStableHashId(systemId2, serviceName1)+ id3 := MakeStableHashId(systemId1, serviceName2)+ id4 := MakeStableHashId(systemId2, serviceName2)// All IDs should be differentassert.NotEqual(t, id1, id2)@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {}for _, tc := range testCases {- id := makeStableHashId(tc.systemId, tc.serviceName)+ id := MakeStableHashId(tc.systemId, tc.serviceName)// FNV-32 produces 8 hex charactersassert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)}})t.Run("hexadecimal output", func(t *testing.T) {- id := makeStableHashId("test-system", "test-service")+ id := MakeStableHashId("test-system", "test-service")assert.NotEmpty(t, id)// Should only contain hexadecimal charactersdiff --git a/internal/hub/systems/system_zfs.go b/internal/hub/systems/system_zfs.goindex 30f575024..934657117 100644--- a/internal/hub/systems/system_zfs.go+++ b/internal/hub/systems/system_zfs.go@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {}func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {- recordID := makeStableHashId(sys.Id, pool.Name)+ recordID := MakeStableHashId(sys.Id, pool.Name)record, err := app.FindRecordById(collection, recordID)if err != nil {@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsPif pool == nil {continue}- recordID := makeStableHashId(sys.Id, name)+ recordID := MakeStableHashId(sys.Id, name)record, err := app.FindRecordById(collection, recordID)if err != nil {if !errors.Is(err, sql.ErrNoRows) {diff --git a/internal/hub/systems/system_zfs_test.go b/internal/hub/systems/system_zfs_test.goindex b61d687a4..b035dd80c 100644--- a/internal/hub/systems/system_zfs_test.go+++ b/internal/hub/systems/system_zfs_test.go@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},}}require.NoError(t, sys.saveZfsPools(initial))- failedID := makeStableHashId(sys.Id, failedKey)+ failedID := MakeStableHashId(sys.Id, failedKey)before, err := app.FindRecordById("zfs_pools", failedID)require.NoError(t, err)partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},}}assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)- fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey))+ fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))require.NoError(t, err)assert.EqualValues(t, 20, fresh.GetInt("alloc"))cached, err := app.FindRecordById("zfs_pools", failedID)@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"tank": {Total: 100, Used: 25, Health: "ONLINE"},}))- record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))+ record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))require.NoError(t, err)firstUpdated := record.GetDateTime("updated")assert.Equal(t, "ONLINE", record.GetString("health"))@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {func TestZfsRawCapacityPersistence(t *testing.T) {sys, app := newTestSystemWithHub(t)require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))- record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs"))+ record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))require.NoError(t, err)require.True(t, record.GetBool("raw"))require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {key: {DisplayName: "tank", Health: "ONLINE"},"tank": {Health: "ONLINE"},}))- id := makeStableHashId(sys.Id, key)+ id := MakeStableHashId(sys.Id, key)record, err := app.FindRecordById("zfs_pools", id)require.NoError(t, err)assert.Equal(t, "tank", record.GetString("display_name"))@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {record, err = app.FindRecordById("zfs_pools", id)require.NoError(t, err)assert.Equal(t, "detail name", record.GetString("display_name"))- _, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank"))+ _, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))require.NoError(t, err)}diff --git a/internal/hub/utils/utils.go b/internal/hub/utils/utils.goindex 438388684..0cfc1fa9d 100644--- a/internal/hub/utils/utils.go+++ b/internal/hub/utils/utils.go@@ -1,7 +1,11 @@// Package utils provides utility functions for the hub.package utils-import "os"+import (+ "os"++ "github.com/pocketbase/pocketbase/core"+)// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.func GetEnv(key string) (value string, exists bool) {@@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) {}return os.LookupEnv(key)}++// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.+func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {+ broker := app.SubscriptionsBroker()+ if broker.TotalClients() == 0 {+ return false+ }+ for _, client := range broker.Clients() {+ subs := client.Subscriptions(collectionName)+ if len(subs) > 0 {+ if validateFn == nil {+ return true+ }+ for k := range subs {+ filter := subs[k].Query["filter"]+ if validateFn(filter) {+ return true+ }+ }+ }+ }+ return false+}diff --git a/internal/migrations/0_collections_snapshot_0_19_0.go b/internal/migrations/0_collections_snapshot_0_20_0_dev_5.gosimilarity index 81%rename from internal/migrations/0_collections_snapshot_0_19_0.gorename to internal/migrations/0_collections_snapshot_0_20_0_dev_5.goindex 73b89d2b5..c11985a56 100644--- a/internal/migrations/0_collections_snapshot_0_19_0.go+++ b/internal/migrations/0_collections_snapshot_0_20_0_dev_5.go@@ -83,7 +83,8 @@ func init() {"ContainerHealth","SystemdFailed","CPUIOWait",- "CPUSteal"+ "CPUSteal",+ "NetworkMonitorLoss"]},{@@ -119,6 +120,16 @@ func init() {"system": false,"type": "bool"},+ {+ "hidden": true,+ "id": "json4000656575",+ "maxSize": 0,+ "name": "state",+ "presentable": false,+ "required": false,+ "system": false,+ "type": "json"+ },{"hidden": true,"id": "date1302749137",@@ -152,6 +163,7 @@ func init() {}],"indexes": [+ "CREATE INDEX idx_alerts_system_name ON alerts (system, name)","CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)"],"system": false@@ -234,6 +246,20 @@ func init() {"system": false,"type": "text"},+ {+ "autogeneratePattern": "",+ "hidden": false,+ "id": "text3888135399",+ "max": 0,+ "min": 0,+ "name": "monitor_name",+ "pattern": "",+ "presentable": false,+ "primaryKey": false,+ "required": false,+ "system": false,+ "type": "text"+ },{"hidden": false,"id": "number494360628",@@ -1721,6 +1747,7 @@ func init() {"fields": [{"autogeneratePattern": "[a-z0-9]{15}",+ "help": "","hidden": false,"id": "text3208210256","max": 15,@@ -1736,6 +1763,7 @@ func init() {{"cascadeDelete": true,"collectionId": "2hz5ncl8tizk5nx",+ "help": "","hidden": false,"id": "relation1204987316","maxSelect": 1,@@ -1748,6 +1776,7 @@ func init() {},{"autogeneratePattern": "",+ "help": "","hidden": false,"id": "text7739291048","max": 0,@@ -1762,6 +1791,7 @@ func init() {},{"autogeneratePattern": "",+ "help": "","hidden": false,"id": "text5528164482","max": 0,@@ -1775,6 +1805,7 @@ func init() {"type": "text"},{+ "help": "","hidden": false,"id": "number8862034195","max": null,@@ -1787,6 +1818,7 @@ func init() {"type": "number"},{+ "help": "","hidden": false,"id": "number4418907321","max": null,@@ -1799,6 +1831,7 @@ func init() {"type": "number"},{+ "help": "","hidden": false,"id": "number2904183765","max": null,@@ -1811,6 +1844,7 @@ func init() {"type": "number"},{+ "help": "","hidden": false,"id": "json4466109723","maxSize": 0,@@ -1821,6 +1855,7 @@ func init() {"type": "json"},{+ "help": "","hidden": false,"id": "json9012873456","maxSize": 0,@@ -1831,6 +1866,7 @@ func init() {"type": "json"},{+ "help": "","hidden": false,"id": "json7182045639","maxSize": 0,@@ -1841,6 +1877,7 @@ func init() {"type": "json"},{+ "help": "","hidden": false,"id": "date9274163058","max": "",@@ -1860,19 +1897,401 @@ func init() {"presentable": false,"system": false,"type": "autodate"+ },+ {+ "autogeneratePattern": "",+ "help": "",+ "hidden": false,+ "id": "text3578368839",+ "max": 0,+ "min": 0,+ "name": "display_name",+ "pattern": "",+ "presentable": false,+ "primaryKey": false,+ "required": false,+ "system": false,+ "type": "text"+ },+ {+ "help": "",+ "hidden": false,+ "id": "bool447994709",+ "name": "raw",+ "presentable": false,+ "required": false,+ "system": false,+ "type": "bool"}],"id": "pbc_8441057391","indexes": ["CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"],- "listRule": null,+ "listRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id","name": "zfs_pools","system": false,"type": "base","updateRule": null,+ "viewRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id"+ },+ {+ "createRule": null,+ "deleteRule": null,+ "fields": [+ {+ "autogeneratePattern": "[a-z0-9]{10}",+ "help": "",+ "hidden": false,+ "id": "text3208210256",+ "max": 10,+ "min": 6,+ "name": "id",+ "pattern": "^[a-z0-9]+$",+ "presentable": false,+ "primaryKey": true,+ "required": true,+ "system": true,+ "type": "text"+ },+ {+ "cascadeDelete": true,+ "collectionId": "2hz5ncl8tizk5nx",+ "help": "",+ "hidden": false,+ "id": "nm_system",+ "maxSelect": 1,+ "minSelect": 0,+ "name": "system",+ "presentable": false,+ "required": true,+ "system": false,+ "type": "relation"+ },+ {+ "autogeneratePattern": "",+ "help": "",+ "hidden": false,+ "id": "nm_target",+ "max": 500,+ "min": 1,+ "name": "target",+ "pattern": "",+ "presentable": false,+ "primaryKey": false,+ "required": true,+ "system": false,+ "type": "text"+ },+ {+ "help": "",+ "hidden": false,+ "id": "nm_protocol",+ "maxSelect": 1,+ "name": "protocol",+ "presentable": false,+ "required": true,+ "system": false,+ "type": "select",+ "values": [+ "icmp",+ "tcp",+ "http",+ "dns"+ ]+ },+ {+ "help": "",+ "hidden": false,+ "id": "nm_port",+ "max": 65535,+ "min": 0,+ "name": "port",+ "onlyInt": true,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "nm_interval",+ "max": 3600,+ "min": 1,+ "name": "interval",+ "onlyInt": true,+ "presentable": false,+ "required": true,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "number926446584",+ "max": null,+ "min": null,+ "name": "res",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "number1006954605",+ "max": null,+ "min": null,+ "name": "resAvg1h",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "number4267669802",+ "max": null,+ "min": null,+ "name": "resMin1h",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "number591433223",+ "max": null,+ "min": null,+ "name": "resMax1h",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "number3726709001",+ "max": null,+ "min": null,+ "name": "loss1h",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "nm_enabled",+ "name": "enabled",+ "presentable": false,+ "required": false,+ "system": false,+ "type": "bool"+ },+ {+ "hidden": false,+ "id": "autodate2990389176",+ "name": "created",+ "onCreate": true,+ "onUpdate": false,+ "presentable": false,+ "system": false,+ "type": "autodate"+ },+ {+ "help": "",+ "hidden": false,+ "id": "date3332085495",+ "max": "",+ "min": "",+ "name": "updated",+ "presentable": false,+ "required": false,+ "system": false,+ "type": "date"+ }+ ],+ "id": "nm_monitors_001",+ "indexes": [+ "CREATE INDEX ` + "`" + `idx_nm_system_enabled` + "`" + ` ON ` + "`" + `network_monitors` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `enabled` + "`" + `)"+ ],+ "listRule": null,+ "name": "network_monitors",+ "system": false,+ "type": "base",+ "updateRule": null,"viewRule": null- }+ },+ {+ "createRule": null,+ "deleteRule": null,+ "fields": [+ {+ "autogeneratePattern": "[a-z0-9]{10}",+ "help": "",+ "hidden": false,+ "id": "text3208210256",+ "max": 10,+ "min": 10,+ "name": "id",+ "pattern": "^[a-z0-9]+$",+ "presentable": false,+ "primaryKey": true,+ "required": true,+ "system": true,+ "type": "text"+ },+ {+ "cascadeDelete": true,+ "collectionId": "2hz5ncl8tizk5nx",+ "help": "",+ "hidden": false,+ "id": "nms_system",+ "maxSelect": 1,+ "minSelect": 0,+ "name": "system",+ "presentable": false,+ "required": true,+ "system": false,+ "type": "relation"+ },+ {+ "cascadeDelete": true,+ "collectionId": "nm_monitors_001",+ "help": "",+ "hidden": false,+ "id": "nms_monitor",+ "maxSelect": 1,+ "minSelect": 0,+ "name": "monitor",+ "presentable": false,+ "required": false,+ "system": false,+ "type": "relation"+ },+ {+ "help": "Number of probe attempts",+ "hidden": false,+ "id": "nms_total_count",+ "max": null,+ "min": 0,+ "name": "total_count",+ "onlyInt": true,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "Number of successful probe attempts",+ "hidden": false,+ "id": "nms_success_count",+ "max": null,+ "min": 0,+ "name": "success_count",+ "onlyInt": true,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "Sum of successful response times in microseconds",+ "hidden": false,+ "id": "nms_res_sum",+ "max": null,+ "min": 0,+ "name": "res_sum",+ "onlyInt": true,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "Response time in microseconds",+ "hidden": false,+ "id": "nms_res_min",+ "max": null,+ "min": 0,+ "name": "res_min",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "Response time in microseconds",+ "hidden": false,+ "id": "nms_res_max",+ "max": null,+ "min": 0,+ "name": "res_max",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ },+ {+ "help": "",+ "hidden": false,+ "id": "nms_type",+ "maxSelect": 1,+ "name": "type",+ "presentable": false,+ "required": true,+ "system": false,+ "type": "select",+ "values": [+ "1m",+ "10m",+ "20m",+ "120m",+ "480m"+ ]+ },+ {+ "help": "",+ "hidden": false,+ "id": "number2990389176",+ "max": null,+ "min": null,+ "name": "created",+ "onlyInt": false,+ "presentable": false,+ "required": false,+ "system": false,+ "type": "number"+ }+ ],+ "id": "nm_stats_001",+ "indexes": [+ "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_system_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",+ "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_monitor_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `monitor` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",+ "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)"+ ],+ "listRule": null,+ "name": "network_monitor_stats",+ "system": false,+ "type": "base",+ "updateRule": null,+ "viewRule": null+ }]`err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)diff --git a/internal/records/monitor_averaging_test.go b/internal/records/monitor_averaging_test.gonew file mode 100644index 000000000..da00330fa--- /dev/null+++ b/internal/records/monitor_averaging_test.go@@ -0,0 +1,226 @@+//go:build testing++package records_test++import (+ "testing"+ "time"++ monitorEntity "github.com/henrygd/beszel/internal/entities/monitor"+ "github.com/henrygd/beszel/internal/records"+ "github.com/henrygd/beszel/internal/tests"++ "github.com/pocketbase/dbx"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestAverageMonitorStats(t *testing.T) {+ hub, err := tests.NewTestHub(t.TempDir())+ require.NoError(t, err)+ defer hub.Cleanup()++ collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats")+ require.NoError(t, err)+ assert.Nil(t, collection.Fields.GetByName("res_avg"))+ assert.Nil(t, collection.Fields.GetByName("loss"))++ rm := records.NewRecordManager(hub)+ user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")+ require.NoError(t, err)+ sys, err := tests.CreateRecord(hub, "systems", map[string]any{+ "name": "monitor-avg-system",+ "host": "localhost",+ "port": "45876",+ "status": "up",+ "users": []string{user.Id},+ })+ require.NoError(t, err)+ monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": sys.Id,+ "name": "cloudflare",+ "target": "1.1.1.1",+ "protocol": "icmp",+ "interval": 30,+ "enabled": true,+ })+ require.NoError(t, err)++ created := time.Now().UnixMilli()+ // Unequal probe counts must weight both latency and loss.+ recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{+ "system": sys.Id,+ "monitor": monitor.Id,+ "type": "1m",+ "created": created,+ "res_min": 5,+ "res_max": 20,+ "total_count": 6, "success_count": 6, "res_sum": 60,+ })+ require.NoError(t, err)+ recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{+ "system": sys.Id,+ "monitor": monitor.Id,+ "type": "1m",+ "created": created,+ "res_min": 10,+ "res_max": 60,+ "total_count": 1, "success_count": 1, "res_sum": 22,+ })+ require.NoError(t, err)++ result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)+ require.NoError(t, err)+ assert.Equal(t, 2, count)+ assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result)++ for _, tc := range []struct {+ name, monitor, recordType string+ after int64+ }{+ {"other monitor", "missing", "1m", created - 1},+ {"other type", monitor.Id, "10m", created - 1},+ {"exclusive cutoff", monitor.Id, "1m", created},+ } {+ t.Run(tc.name, func(t *testing.T) {+ stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after)+ require.NoError(t, err)+ assert.Zero(t, count)+ assert.Equal(t, monitorEntity.Stats{}, stats)+ })+ }++ // A failure-only bucket counts toward loss but must not lower latency.+ recordB.Set("res_min", 0)+ recordB.Set("res_max", 0)+ recordB.Set("success_count", 0)+ recordB.Set("res_sum", 0)+ require.NoError(t, hub.Save(recordB))+ result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)+ require.NoError(t, err)+ assert.Equal(t, 2, count)+ assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result)+ // Sparse monitor records must propagate through every rollup level.+ rm.CreateLongerRecords()+ for _, recordType := range []string{"10m", "20m", "120m", "480m"} {+ rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})+ require.NoError(t, err)+ require.Len(t, rollups, 1, recordType)+ assert.Equal(t, 5.0, rollups[0].GetFloat("res_min"))+ assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))+ assert.Equal(t, 7, rollups[0].GetInt("total_count"))+ assert.Equal(t, 6, rollups[0].GetInt("success_count"))+ assert.Equal(t, 60, rollups[0].GetInt("res_sum"))+ // A sibling with a different number of probes must retain its actual+ // weight when the next tier combines their underlying counts.+ _, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{+ "system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created,+ "res_min": 100, "res_max": 100,+ "total_count": 3, "success_count": 1, "res_sum": 100,+ })+ require.NoError(t, err)+ merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1)+ require.NoError(t, err)+ assert.Equal(t, 2, count)+ assert.Equal(t, monitorEntity.Stats{+ ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30,+ TotalCount: 10, SuccessCount: 7, ResponseSum: 160,+ }, merged)+ }+ // All failures produce zero latency, while a genuine zero-microsecond+ // success remains a valid minimum (it must not be filtered out as a sentinel).+ recordA.Set("success_count", 0)+ recordA.Set("res_sum", 0)+ require.NoError(t, hub.Save(recordA))+ result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)+ require.NoError(t, err)+ assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result)+ recordA.Set("success_count", 1)+ recordA.Set("res_min", 0)+ recordA.Set("res_max", 0)+ require.NoError(t, hub.Save(recordA))+ result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)+ require.NoError(t, err)+ assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result)+}++func TestSparseMonitorRollups(t *testing.T) {+ for _, tc := range []struct {+ name string+ interval int+ samples int+ disabled bool+ }{+ {"five minute interval", 300, 2, false},+ {"ten minute interval", 600, 1, false},+ {"fifteen minute interval", 900, 1, false},+ {"empty window", 900, 0, false},+ {"disabled with pending history", 300, 2, true},+ {"disabled empty window", 900, 0, true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ hub, err := tests.NewTestHub(t.TempDir())+ require.NoError(t, err)+ defer hub.Cleanup()+ user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")+ require.NoError(t, err)+ sys, err := tests.CreateRecord(hub, "systems", map[string]any{+ "name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up",+ "users": []string{user.Id},+ })+ require.NoError(t, err)+ monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",+ "interval": tc.interval, "enabled": true,+ })+ require.NoError(t, err)+ now := time.Now()+ for i := range tc.samples {+ _, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{+ "system": sys.Id, "monitor": monitor.Id, "type": "1m",+ "created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(),+ "res_min": 8, "res_max": 20,+ "total_count": 4, "success_count": 3, "res_sum": 36,+ })+ require.NoError(t, err)+ }+ // Other collections must still reject fewer than nine minute records.+ for _, collection := range []string{"system_stats", "container_stats"} {+ stats := `{"cpu":10}`+ if collection == "container_stats" {+ stats = `[{"name":"test","cpu":10}]`+ }+ for range 8 {+ _, err := tests.CreateRecord(hub, collection, map[string]any{+ "system": sys.Id, "type": "1m", "stats": stats,+ })+ require.NoError(t, err)+ }+ }+ if tc.disabled {+ monitor.Set("enabled", false)+ require.NoError(t, hub.Save(monitor))+ }+ records.NewRecordManager(hub).CreateLongerRecords()+ for _, recordType := range []string{"10m", "20m", "120m", "480m"} {+ rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})+ require.NoError(t, err)+ if tc.samples == 0 {+ assert.Empty(t, rollups, recordType)+ } else {+ require.Len(t, rollups, 1, recordType)+ assert.Equal(t, 8.0, rollups[0].GetFloat("res_min"))+ assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))+ assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count"))+ assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count"))+ assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum"))+ }+ for _, collection := range []string{"system_stats", "container_stats"} {+ count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType})+ require.NoError(t, err)+ assert.Zero(t, count, "%s %s", collection, recordType)+ }+ }+ })+ }+}diff --git a/internal/records/records.go b/internal/records/records.goindex 4ef1b1914..7c4bf281c 100644--- a/internal/records/records.go+++ b/internal/records/records.go@@ -3,15 +3,16 @@ package recordsimport ("encoding/json"- "log/slog""math""time""github.com/henrygd/beszel/internal/entities/container"+ "github.com/henrygd/beszel/internal/entities/monitor""github.com/henrygd/beszel/internal/entities/system""github.com/pocketbase/dbx""github.com/pocketbase/pocketbase/core"+ "github.com/pocketbase/pocketbase/tools/types")type RecordManager struct {@@ -39,7 +40,7 @@ type StatsRecord struct {// Create longer records by averaging shorter recordsfunc (rm *RecordManager) CreateLongerRecords() {- // start := time.Now()+ now := time.Now().UTC()longerRecordData := []LongerRecordData{{shorterType: "1m",@@ -69,23 +70,28 @@ func (rm *RecordManager) CreateLongerRecords() {}// wrap the operations in a transaction// Pocketbase cron does not handle errors, log them here.- rm.app.RunInTransaction(func(txApp core.App) error {+ err := rm.app.RunInTransaction(func(txApp core.App) error {var err error+collections := [2]*core.Collection{}collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats")if err != nil {- slog.Error("Error finding cached collection using system stats:", "err", err)return err}collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats")if err != nil {- slog.Error("Error finding cached collection using container stats:", "err", err)+ return err+ }+ monitorStatsColl, err := txApp.FindCachedCollectionByNameOrId("network_monitor_stats")+ if err != nil {return err}var systems RecordIdsdb := txApp.DB()- db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems)+ if err := db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems); err != nil {+ return err+ }// loop through all active systems, time periods, and collectionsfor _, system := range systems {@@ -94,44 +100,52 @@ func (rm *RecordManager) CreateLongerRecords() {recordData := longerRecordData[i]// log.Println("processing longer record type", recordData.longerType)// add one minute padding for longer records because they are created slightly later than the job start time- longerRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration + time.Minute)+ longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)// shorter records are created independently of longer records, so we shouldn't need to add padding- shorterRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration)- // loop through both collections+ shorterRecordPeriod := now.Add(recordData.longerTimeDuration)for _, collection := range collections {// check creation time of last longer record if not 10m, since 10m is created every runif recordData.longerType != "10m" {- count, err := txApp.CountRecords(- collection.Id,- dbx.NewExp(- "system = {:system} AND type = {:type} AND created > {:created}",- dbx.Params{"type": recordData.longerType, "system": system.Id, "created": longerRecordPeriod},- ),- )+ count, err := txApp.CountRecords(collection.Id, dbx.NewExp(+ "system = {:system} AND type = {:type} AND created > {:created}",+ dbx.Params{+ "type": recordData.longerType,+ "system": system.Id,+ "created": longerRecordPeriod.Format(types.DefaultDateLayout),+ },+ ))+ if err != nil {+ return err+ }// continue if longer record exists- if err != nil || count > 0 {+ if count > 0 {continue}}// get shorter records from the past x minutesvar recordIds RecordIds- err := txApp.DB().+ params := dbx.Params{+ "type": recordData.shorterType,+ "system": system.Id,+ "created": shorterRecordPeriod.Format(types.DefaultDateLayout),+ }++ err := db.Select("id").From(collection.Name).- AndWhere(dbx.NewExp(+ Where(dbx.NewExp("system={:system} AND type={:type} AND created > {:created}",- dbx.Params{- "type": recordData.shorterType,- "system": system.Id,- "created": shorterRecordPeriod,- },+ params,)).OrderBy("created").All(&recordIds)+ if err != nil {+ return err+ }// continue if not enough shorter records- if err != nil || len(recordIds) < recordData.minShorterRecords {+ if len(recordIds) < recordData.minShorterRecords {continue}// average the shorter records and create longer record@@ -142,20 +156,88 @@ func (rm *RecordManager) CreateLongerRecords() {case "system_stats":longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds))case "container_stats":-longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds))}if err := txApp.SaveNoValidate(longerRecord); err != nil {- slog.Error("failed to save longer record", "err", err)+ txApp.Logger().Error("failed to save longer record", "err", err)+ }+ }+ }+ }++ // network_monitor_stats is aggregated per monitor (not per system)+ var monitors []struct {+ Id string `db:"id"`+ System string `db:"system"`+ }+ // Disabled monitors still have history that must advance through retention tiers.+ if err := db.NewQuery("SELECT id, system FROM network_monitors").All(&monitors); err != nil {+ return err+ }++ for _, monitorRec := range monitors {+ for i := range longerRecordData {+ recordData := longerRecordData[i]+ longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)+ shorterRecordPeriod := now.Add(recordData.longerTimeDuration)++ if recordData.longerType != "10m" {+ count, err := txApp.CountRecords(monitorStatsColl.Id, dbx.NewExp(+ "monitor={:monitor} AND type={:type} AND created>{:created}",+ dbx.Params{+ "monitor": monitorRec.Id,+ "type": recordData.longerType,+ "created": longerRecordPeriod.UnixMilli(),+ },+ ))+ if err != nil {+ return err+ }+ if count > 0 {+ continue}}++ stats, count, err := rm.AverageMonitorStats(db, monitorRec.Id, recordData.shorterType, shorterRecordPeriod.UnixMilli())+ if err != nil {+ txApp.Logger().Error("failed to average monitor stats", "monitor", monitorRec.Id, "err", err)+ continue+ }+ // Monitor intervals can exceed the aggregation window, so average+ // any available records at every level and skip only empty windows.+ if count == 0 {+ continue+ }++ longerRecord := core.NewRecord(monitorStatsColl)+ longerRecord.Set("system", monitorRec.System)+ longerRecord.Set("monitor", monitorRec.Id)+ longerRecord.Set("type", recordData.longerType)+ longerRecord.Set("created", now.UnixMilli())+ longerRecord.Set("res_min", stats.ResMin)+ longerRecord.Set("res_max", stats.ResMax)+ longerRecord.Set("total_count", stats.TotalCount)+ longerRecord.Set("success_count", stats.SuccessCount)+ longerRecord.Set("res_sum", stats.ResponseSum)+ if err := txApp.SaveNoValidate(longerRecord); err != nil {+ txApp.Logger().Error("failed to save monitor longer record", "err", err)+ }}}return nil})+ if err != nil {+ rm.app.Logger().Error("failed to create longer records", "err", err)+ }+}- // log.Println("finished creating longer records", "time (ms)", time.Since(start).Milliseconds())+func getCreatedTimeField(collectionName string, period time.Time) any {+ // network_monitor_stats stores created as unix timestamp in ms, not as a date string+ if collectionName == "network_monitor_stats" {+ return period.UnixMilli()+ }+ return period.Format(types.DefaultDateLayout)}// Calculate the average stats of a list of system_stats records without reflect@@ -596,6 +678,36 @@ func AverageContainerStatsSlice(records [][]container.Stats) []container.Stats {return result}+// AverageMonitorStats merges probe counts and response sums, preserving their+// weights through every retention tier. Failed probes do not contribute latency.+func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordType string, createdAfter int64) (monitor.Stats, int, error) {+ var result struct {+ monitor.Stats+ Count int `db:"count"`+ }+ err := db.Select(+ "COUNT(*) AS count",+ "COALESCE(SUM(total_count), 0) AS total_count",+ "COALESCE(SUM(success_count), 0) AS success_count",+ "COALESCE(SUM(res_sum), 0) AS res_sum",+ "COALESCE(MIN(CASE WHEN success_count > 0 THEN res_min END), 0) AS res_min",+ "COALESCE(MAX(CASE WHEN success_count > 0 THEN res_max END), 0) AS res_max",+ ).From("network_monitor_stats").Where(dbx.NewExp(+ "monitor={:monitor} AND type={:type} AND created>{:created}",+ dbx.Params{"monitor": monitorID, "type": recordType, "created": createdAfter},+ )).One(&result)+ if err != nil {+ return monitor.Stats{}, 0, err+ }+ if result.SuccessCount > 0 {+ result.ResAvg = twoDecimals(float64(result.ResponseSum) / float64(result.SuccessCount))+ }+ if result.TotalCount > 0 {+ result.Loss = twoDecimals(float64(result.TotalCount-result.SuccessCount) * 100 / float64(result.TotalCount))+ }+ return result.Stats, result.Count, nil+}+/* Round float to two decimals */func twoDecimals(value float64) float64 {return math.Round(value*100) / 100diff --git a/internal/records/records_deletion.go b/internal/records/records_deletion.goindex 6632ded98..94bd0aaea 100644--- a/internal/records/records_deletion.go+++ b/internal/records/records_deletion.go@@ -3,7 +3,6 @@ package recordsimport ("fmt""log/slog"- "strings""time""github.com/pocketbase/dbx"@@ -60,7 +59,7 @@ func deleteOldAlertsHistory(app core.App, countToKeep, countBeforeDeletion int)// Deletes system_stats records older than what is displayed in the UIfunc deleteOldSystemStats(app core.App) error {// Collections to process- collections := [2]string{"system_stats", "container_stats"}+ collections := [3]string{"system_stats", "container_stats", "network_monitor_stats"}// Record types and their retention periodstype RecordDeletionData struct {@@ -76,24 +75,17 @@ func deleteOldSystemStats(app core.App) error {}now := time.Now().UTC()+ db := app.DB()for _, collection := range collections {- // Build the WHERE clause- var conditionParts []string- var params dbx.Params = make(map[string]any)- for i := range recordData {- rd := recordData[i]- // Create parameterized condition for this record type- dateParam := fmt.Sprintf("date%d", i)- conditionParts = append(conditionParts, fmt.Sprintf("(type = '%s' AND created < {:%s})", rd.recordType, dateParam))- params[dateParam] = now.Add(-rd.retention)- }- // Combine conditions with OR- conditionStr := strings.Join(conditionParts, " OR ")- // Construct and execute the full raw query- rawQuery := fmt.Sprintf("DELETE FROM %s WHERE %s", collection, conditionStr)- if _, err := app.DB().NewQuery(rawQuery).Bind(params).Execute(); err != nil {- return fmt.Errorf("failed to delete from %s: %v", collection, err)+ query := db.Delete(collection, dbx.NewExp("type={:type} AND created<{:created}"))+ for _, rd := range recordData {+ if _, err := query.Bind(dbx.Params{+ "type": rd.recordType,+ "created": getCreatedTimeField(collection, now.Add(-rd.retention)),+ }).Execute(); err != nil {+ return fmt.Errorf("failed to delete from %s: %v", collection, err)+ }}}return nildiff --git a/internal/records/records_rollup_test.go b/internal/records/records_rollup_test.gonew file mode 100644index 000000000..16f0076af--- /dev/null+++ b/internal/records/records_rollup_test.go@@ -0,0 +1,86 @@+//go:build testing++package records_test++import (+ "testing"+ "time"++ "github.com/henrygd/beszel/internal/records"+ "github.com/henrygd/beszel/internal/tests"+ "github.com/pocketbase/dbx"+ "github.com/pocketbase/pocketbase/tools/types"+ "github.com/stretchr/testify/require"+)++func TestLongerRecordsPreventDuplicates(t *testing.T) {+ for _, collection := range []string{"system_stats", "container_stats", "network_monitor_stats"} {+ for _, tier := range []struct {+ shorter, longer string+ count int+ }{+ {"10m", "20m", 2},+ {"20m", "120m", 6},+ {"120m", "480m", 4},+ } {+ t.Run(collection+"/"+tier.longer, func(t *testing.T) {+ hub, err := tests.NewTestHub(t.TempDir())+ require.NoError(t, err)+ defer hub.Cleanup()++ user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")+ require.NoError(t, err)+ sys, err := tests.CreateRecord(hub, "systems", map[string]any{+ "name": "rollup-system", "host": "localhost", "port": "45876",+ "status": "up", "users": []string{user.Id},+ })+ require.NoError(t, err)++ created := time.Now().UTC().Add(-time.Minute)+ data := map[string]any{+ "system": sys.Id, "type": tier.shorter,+ "created": created.Format(types.DefaultDateLayout),+ }+ filter := dbx.HashExp{"system": sys.Id, "type": tier.longer}+ switch collection {+ case "system_stats":+ data["stats"] = `{"cpu":10}`+ case "container_stats":+ data["stats"] = `[{"name":"test","cpu":10}]`+ case "network_monitor_stats":+ monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{+ "system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",+ "interval": 30, "enabled": true,+ })+ require.NoError(t, err)+ data["monitor"] = monitor.Id+ data["created"] = created.UnixMilli()+ data["total_count"] = 1+ data["success_count"] = 1+ data["res_sum"] = 10+ data["res_min"] = 10+ data["res_max"] = 10+ filter["monitor"] = monitor.Id+ }+ for range tier.count {+ _, err := tests.CreateRecord(hub, collection, data)+ require.NoError(t, err)+ }++ rm := records.NewRecordManager(hub)+ rm.CreateLongerRecords()+ first, err := hub.FindAllRecords(collection, filter)+ require.NoError(t, err)+ require.Len(t, first, 1)++ // The shorter records remain eligible, but the existing longer+ // record must prevent another rollup on a subsequent invocation.+ rm.CreateLongerRecords()+ second, err := hub.FindAllRecords(collection, filter)+ require.NoError(t, err)+ require.Len(t, second, 1)+ require.Equal(t, first[0].Id, second[0].Id)+ })+ }+ }+}diff --git a/internal/site/package.json b/internal/site/package.jsonindex a69c345cf..b30e14619 100644--- a/internal/site/package.json+++ b/internal/site/package.json@@ -1,7 +1,7 @@{"name": "beszel","private": true,- "version": "0.19.0",+ "version": "0.20.0","type": "module","scripts": {"dev": "vite --host",@@ -74,4 +74,4 @@"optionalDependencies": {"@esbuild/linux-arm64": "^0.21.5"}-}+}\ No newline at end of filediff --git a/internal/site/src/components/active-alerts.tsx b/internal/site/src/components/active-alerts.tsxindex 2c5019d90..1e71762ff 100644--- a/internal/site/src/components/active-alerts.tsx+++ b/internal/site/src/components/active-alerts.tsx@@ -61,6 +61,8 @@ export const ActiveAlerts = () => {<AlertDescription>{info.triggeredDesc ? (info.triggeredDesc()+ ) : alert.name === "NetworkMonitorLoss" ? (+ <Trans>One or more monitors exceed {alert.value}% loss</Trans>) : alert.name === "Status" ? (<Trans>Connection is down</Trans>) : info.invert ? (diff --git a/internal/site/src/components/alerts-history-columns.tsx b/internal/site/src/components/alerts-history-columns.tsxindex 887ae0b20..2ae9bd97e 100644--- a/internal/site/src/components/alerts-history-columns.tsx+++ b/internal/site/src/components/alerts-history-columns.tsx@@ -30,7 +30,8 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [accessorFn: (record) => {const name = record.nameconst info = alertInfo[name]- return info?.name().replace("cpu", "CPU") || name+ const label = info?.name().replace("cpu", "CPU") || name+ return record.monitor_name ? `${label}: ${record.monitor_name}` : label},header: ({ column }) => (<Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>diff --git a/internal/site/src/components/alerts/alerts-sheet.tsx b/internal/site/src/components/alerts/alerts-sheet.tsxindex e5efeeb2a..94e4e3b30 100644--- a/internal/site/src/components/alerts/alerts-sheet.tsx+++ b/internal/site/src/components/alerts/alerts-sheet.tsx@@ -239,13 +239,13 @@ export function AlertContent({/** Alerts that fire on first observation have no duration to configure */const noDuration = alertData.noDuration === true/** Binary alerts have no threshold to configure */- const noThreshold = !!singleDescription || noDuration+ const noThreshold = !!singleDescription || alertData.noThreshold === true/** Whether enabling the alert reveals anything to configure */const hasControls = !(noThreshold && noDuration)const [checked, setChecked] = useState(global ? false : !!alert)const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10))- const [value, setValue] = useState(alert?.value || (noThreshold ? 0 : (alertData.start ?? 80)))+ const [value, setValue] = useState(alert?.value ?? (noThreshold ? 0 : (alertData.start ?? 80)))const Icon = alertData.icon@@ -319,7 +319,7 @@ export function AlertContent({<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground"><Suspense fallback={<div className="h-10" />}>{!noThreshold && (- <div>+ <div className={cn(noDuration && "col-span-full")}><p id={`v${name}`} className="text-sm block h-6">{alertData.invert ? (<Trans>diff --git a/internal/site/src/components/charts/area-chart.tsx b/internal/site/src/components/charts/area-chart.tsxindex 052de9c7b..183600293 100644--- a/internal/site/src/components/charts/area-chart.tsx+++ b/internal/site/src/components/charts/area-chart.tsx@@ -66,7 +66,7 @@ export default function AreaChartDefault({}) {const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })- const sourceData = customData ?? chartData.systemStats+ const sourceData = customData ?? chartData.systemStats ?? []const [displayData, setDisplayData] = useState(sourceData)const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)@@ -111,6 +111,8 @@ export default function AreaChartDefault({})}, [areasKey, displayMaxToggled])+ const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)+return useMemo(() => {if (displayData.length === 0) {return null@@ -146,7 +148,7 @@ export default function AreaChartDefault({axisLine={false}/>)}- {xAxis(chartData)}+ {XAxis}<ChartTooltipanimationEasing="ease-out"animationDuration={150}@@ -167,5 +169,5 @@ export default function AreaChartDefault({</AreaChart></ChartContainer>)- }, [displayData, yAxisWidth, filter, Areas])+ }, [displayData, yAxisWidth, filter, Areas, XAxis])}diff --git a/internal/site/src/components/charts/chart-time-select.tsx b/internal/site/src/components/charts/chart-time-select.tsxindex ab3473aa2..7ff7f9810 100644--- a/internal/site/src/components/charts/chart-time-select.tsx+++ b/internal/site/src/components/charts/chart-time-select.tsx@@ -9,14 +9,21 @@ import { memo } from "react"export default memo(function ChartTimeSelect({className,agentVersion,+ chartTimeStore = $chartTime,+ allowRealtime = true,}: {className?: stringagentVersion: SemVer+ chartTimeStore?: typeof $chartTime+ allowRealtime?: boolean}) {- const chartTime = useStore($chartTime)+ const chartTime = useStore(chartTimeStore)// remove chart times that are not supported by the system agent version- const availableChartTimes = Object.entries(chartTimeData).filter(([_, { minVersion }]) => {+ const availableChartTimes = Object.entries(chartTimeData).filter(([value, { minVersion }]) => {+ if (value === "1m" && !allowRealtime) {+ return false+ }if (!minVersion) {return true}@@ -24,7 +31,7 @@ export default memo(function ChartTimeSelect({})return (- <Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => $chartTime.set(value)}>+ <Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => chartTimeStore.set(value)}><SelectTrigger className={cn(className, "relative ps-10 pe-5")}><HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" /><SelectValue />diff --git a/internal/site/src/components/charts/line-chart.tsx b/internal/site/src/components/charts/line-chart.tsxindex f60a341b8..97927526b 100644--- a/internal/site/src/components/charts/line-chart.tsx+++ b/internal/site/src/components/charts/line-chart.tsx@@ -22,6 +22,10 @@ export type DataPoint<T = SystemStatsRecord> = {order?: numberstrokeOpacity?: numberactiveDot?: boolean+ dot?: boolean+ /** Which Y axis this series plots against. Defaults to "left". */+ yAxisId?: "left" | "right"+ strokeDasharray?: string}export default function LineChartDefault({@@ -30,9 +34,12 @@ export default function LineChartDefault({max,maxToggled,tickFormatter,+ tickFormatter2,contentFormatter,dataPoints,domain,+ domain2,+ max2,legend,itemSorter,showTotal = false,@@ -41,18 +48,24 @@ export default function LineChartDefault({filter,truncate = false,chartProps,+ connectNulls,}: {chartData: ChartData// biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData)customData?: any[]max?: number+ max2?: numbermaxToggled?: booleantickFormatter: (value: number, index: number) => string+ /** Tick formatter for the right ("right"-yAxisId) axis, when any dataPoint uses it. */+ tickFormatter2?: (value: number, index: number) => string// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interopcontentFormatter: (item: any, key: string) => ReactNode// biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic typesdataPoints?: DataPoint<any>[]domain?: AxisDomain+ /** Domain for the right axis, when any dataPoint uses it. */+ domain2?: AxisDomainlegend?: booleanshowTotal?: boolean// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop@@ -62,10 +75,15 @@ export default function LineChartDefault({filter?: stringtruncate?: booleanchartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin">+ connectNulls?: boolean}) {const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()+ const hasRightAxis = !!dataPoints?.some((dp) => dp.yAxisId === "right")+ // fixed width for the secondary axis rather than measured, since its labels (e.g. loss %) are short+ // and predictable, and this avoids depending on a second async width-measurement pass to settle+ const rightAxisWidth = 38const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })- const sourceData = customData ?? chartData.systemStats+ const sourceData = customData ?? chartData.systemStats ?? []const [displayData, setDisplayData] = useState(sourceData)const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)@@ -83,7 +101,9 @@ export default function LineChartDefault({}, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData])// Use a stable key derived from data point identities and visual properties- const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity ?? ""}`).join("\0")+ const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity}${d.dot}${d.yAxisId}${d.strokeDasharray}`).join("\0")++ const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)const Lines = useMemo(() => {return dataPoints?.map((dataPoint, i) => {@@ -94,17 +114,20 @@ export default function LineChartDefault({return (<Linekey={dataPoint.label}+ yAxisId={dataPoint.yAxisId ?? "left"}dataKey={dataPoint.dataKey}name={dataPoint.label}type="monotoneX"- dot={false}+ dot={dataPoint.dot || false}strokeWidth={1.5}stroke={color}strokeOpacity={dataPoint.strokeOpacity}+ strokeDasharray={dataPoint.strokeDasharray}isAnimationActive={false}// stackId={dataPoint.stackId}order={dataPoint.order || i}activeDot={dataPoint.activeDot ?? true}+ connectNulls={connectNulls}/>)})@@ -135,6 +158,7 @@ export default function LineChartDefault({<CartesianGrid vertical={false} />{!hideYAxis && (<YAxis+ yAxisId="left"direction="ltr"orientation={chartData.orientation}className="tracking-tighter"@@ -145,7 +169,20 @@ export default function LineChartDefault({axisLine={false}/>)}- {xAxis(chartData)}+ {!hideYAxis && hasRightAxis && (+ <YAxis+ yAxisId="right"+ direction="ltr"+ orientation={chartData.orientation === "left" ? "right" : "left"}+ className="tracking-tighter"+ width={rightAxisWidth}+ domain={domain2 ?? [0, max2 ?? "auto"]}+ tickFormatter={tickFormatter2 ?? tickFormatter}+ tickLine={false}+ axisLine={false}+ />+ )}+ {XAxis}<ChartTooltipanimationEasing="ease-out"animationDuration={150}@@ -166,5 +203,5 @@ export default function LineChartDefault({</LineChart></ChartContainer>)- }, [displayData, yAxisWidth, filter, Lines])+ }, [displayData, yAxisWidth, hasRightAxis, filter, Lines, XAxis])}diff --git a/internal/site/src/components/command-palette.tsx b/internal/site/src/components/command-palette.tsxindex a0c73818a..71080a114 100644--- a/internal/site/src/components/command-palette.tsx+++ b/internal/site/src/components/command-palette.tsx@@ -11,6 +11,7 @@ import {HardDriveIcon,LogsIcon,MailIcon,+ NetworkIcon,Server,ServerIcon,SettingsIcon,@@ -122,6 +123,20 @@ export default memo(function CommandPalette({ open, setOpen }: { open: boolean;<Trans>Page</Trans></CommandShortcut></CommandItem>+ <CommandItem+ onSelect={() => {+ navigate(getPagePath($router, "monitors"))+ setOpen(false)+ }}+ >+ <NetworkIcon className="me-2 size-4" />+ <span>+ <Trans>Network Monitors</Trans>+ </span>+ <CommandShortcut>+ <Trans>Page</Trans>+ </CommandShortcut>+ </CommandItem><CommandItemonSelect={() => {navigate(getPagePath($router, "settings", { name: "general" }))diff --git a/internal/site/src/components/containers-table/containers-table-columns.tsx b/internal/site/src/components/containers-table/containers-table-columns.tsxindex 89ec6eea6..d3dfbd6a8 100644--- a/internal/site/src/components/containers-table/containers-table-columns.tsx+++ b/internal/site/src/components/containers-table/containers-table-columns.tsx@@ -16,7 +16,7 @@ import {import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons"import { Badge } from "../ui/badge"import { t } from "@lingui/core/macro"-import { $allSystemsById, $longestSystemNameLen } from "@/lib/stores"+import { $allSystemsById, $longestSystemName } from "@/lib/stores"import { useStore } from "@nanostores/react"import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"@@ -64,10 +64,13 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,cell: ({ getValue }) => {const allSystems = useStore($allSystemsById)- const longestName = useStore($longestSystemNameLen)+ const longestName = useStore($longestSystemName)return (- <div className="ms-1 max-w-40 truncate" style={{ width: `${longestName / 1.05}ch` }}>- {allSystems[getValue() as string]?.name ?? ""}+ <div className="ms-1 relative w-fit max-w-40">+ <span className="invisible block whitespace-nowrap" aria-hidden="true">+ {longestName}+ </span>+ <span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span></div>)},diff --git a/internal/site/src/components/navbar.tsx b/internal/site/src/components/navbar.tsxindex 956338f16..8f967b2ae 100644--- a/internal/site/src/components/navbar.tsx+++ b/internal/site/src/components/navbar.tsx@@ -8,6 +8,7 @@ import {LogOutIcon,LogsIcon,MenuIcon,+ NetworkIcon,PlusIcon,SearchIcon,ServerIcon,@@ -108,6 +109,13 @@ export default function Navbar() {<HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} /><span>S.M.A.R.T.</span></DropdownMenuItem>+ <DropdownMenuItem+ onClick={() => navigate(getPagePath($router, "monitors"))}+ className="flex items-center"+ >+ <NetworkIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />+ <Trans>Network Monitors</Trans>+ </DropdownMenuItem><DropdownMenuItemonClick={() => navigate(getPagePath($router, "settings", { name: "general" }))}className="flex items-center"@@ -179,6 +187,21 @@ export default function Navbar() {</TooltipTrigger><TooltipContent>S.M.A.R.T.</TooltipContent></Tooltip>+ <Tooltip>+ <TooltipTrigger asChild>+ <Link+ href={getPagePath($router, "monitors")}+ className={cn("hidden md:grid", buttonVariants({ variant: "ghost", size: "icon" }))}+ aria-label="Network Monitors"+ onMouseEnter={() => import("@/components/routes/monitors")}+ >+ <NetworkIcon className="h-[1.2rem] w-[1.2rem]" strokeWidth={1.5} />+ </Link>+ </TooltipTrigger>+ <TooltipContent>+ <Trans>Network Monitors</Trans>+ </TooltipContent>+ </Tooltip><ModeToggle /><Tooltip><TooltipTrigger asChild>diff --git a/internal/site/src/components/network-monitors-table/monitor-dialog.tsx b/internal/site/src/components/network-monitors-table/monitor-dialog.tsxnew file mode 100644index 000000000..dcbb27ae3--- /dev/null+++ b/internal/site/src/components/network-monitors-table/monitor-dialog.tsx@@ -0,0 +1,778 @@+import { useCallback, useEffect, useRef, useState } from "react"+import { Trans, useLingui } from "@lingui/react/macro"+import { useStore } from "@nanostores/react"+import { pb } from "@/lib/api"+import {+ Dialog,+ DialogContent,+ DialogDescription,+ DialogFooter,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog"+import {+ DropdownMenu,+ DropdownMenuCheckboxItem,+ DropdownMenuContent,+ DropdownMenuItem,+ DropdownMenuTrigger,+} from "@/components/ui/dropdown-menu"+import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet"+import { Button } from "@/components/ui/button"+import { Input } from "@/components/ui/input"+import { Label } from "@/components/ui/label"+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"+import { Textarea } from "@/components/ui/textarea"+import { ChevronDownIcon, ListIcon, SearchIcon, ServerIcon } from "lucide-react"+import { useToast } from "@/components/ui/use-toast"+import { $systems } from "@/lib/stores"+import { cn, supportsNetworkMonitors } from "@/lib/utils"+import type { NetworkMonitorRecord } from "@/types"+import * as v from "valibot"++type MonitorProtocol = "icmp" | "tcp" | "http" | "dns"++type MonitorValues = {+ system: string+ target: string+ protocol: MonitorProtocol+ port: number+ interval: string+}++type NormalizedMonitorValues = Omit<MonitorValues, "system" | "interval"> & {+ interval: number+}++type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval">++const defaultInterval = 30++const MonitorProtocolSchema = v.picklist(["icmp", "tcp", "http", "dns"])++const MonitorIntervalSchema = v.pipe(v.string(), v.toNumber(), v.minValue(1), v.maxValue(3600))++// Both the single-monitor form and the bulk importer flow through this schema so+// defaults and HTTP target normalization stay in one place.+const NormalizedMonitorValuesSchema = v.pipe(+ v.object({+ target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),+ protocol: MonitorProtocolSchema,+ port: v.number(),+ interval: MonitorIntervalSchema,+ }),+ v.transform((input): NormalizedMonitorValues => {+ let { protocol, port } = input+ let httpTarget = input.target+ if (protocol === "icmp" || protocol === "http" || protocol === "dns") {+ if (protocol === "http") {+ httpTarget = normalizeHttpTarget(input.target, port)+ }+ port = 0+ } else if (protocol === "tcp" && !port) {+ port = 443+ }+ return {+ // HTTP monitors may be entered as bare hostnames, so normalize them to a+ // scheme-bearing URL before the payload is sent to PocketBase.+ target: protocol === "http" ? httpTarget : input.target,+ protocol,+ port,+ interval: input.interval,+ }+ }),+ v.forward(+ v.check((input) => {+ if (input.protocol === "icmp" || input.protocol === "http" || input.protocol === "dns") {+ return input.port === 0+ }++ return Number.isInteger(input.port) && input.port >= 1 && input.port <= 65535+ }, "Port must be between 1 and 65535"),+ ["port"]+ )+)++// Bulk parsing only trims raw CSV fields. Inference, defaults, and protocol-+// specific validation still go through the shared normalization schema above.+const BulkMonitorSchema = v.object({+ target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),+ protocol: v.optional(v.pipe(v.string(), v.trim())),+ port: v.optional(v.pipe(v.string(), v.trim())),+ interval: v.optional(v.pipe(v.string(), v.trim())),+})++function normalizeHttpTarget(target: string, port = 0) {+ const useExplicitPort = port > 0 && port !== 80 && port !== 443+ const hasOriginOnlyTarget = /^https?:\/\/[^/?#]+$/i.test(target)+ if (!/^https?:\/\//i.test(target)) {+ const scheme = port === 80 ? "http" : "https"+ return `${scheme}://${target}${useExplicitPort ? `:${port}` : ""}`+ }++ let parsedUrl: URL+ try {+ parsedUrl = new URL(target)+ } catch {+ return target+ }++ if (!parsedUrl.port && useExplicitPort) {+ parsedUrl.port = `${port}`+ }++ // avoid converting "http://localhost:8090" to "http://localhost:8090/" - keep the original formatting if the URL is just an origin+ if (hasOriginOnlyTarget && parsedUrl.pathname === "/" && !parsedUrl.search && !parsedUrl.hash) {+ return parsedUrl.origin+ }++ return parsedUrl.toString()+}++function trimTrailingEmptyFields(fields: string[]) {+ let lastValueIndex = fields.length - 1+ while (lastValueIndex > 0 && !fields[lastValueIndex]) {+ lastValueIndex--+ }+ return fields.slice(0, lastValueIndex + 1)+}++function buildMonitorPayload(values: MonitorValues, enabled = true) {+ const normalizedValues = v.safeParse(NormalizedMonitorValuesSchema, values)+ if (!normalizedValues.success) {+ throw new Error(normalizedValues.issues[0]?.message || "Invalid monitor")+ }++ const payload = {+ system: values.system,+ enabled,+ ...normalizedValues.output,+ }++ return payload+}++type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port">+function getMonitorIdentityKey({ system, target, protocol, port }: MonitorIdentity) {+ return `${system}${target}${protocol}${port}`+}++function parseBulkMonitorLine(line: string, lineNumber: number, system: string) {+ const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = ""] = line.split(",")+ const parsed = v.safeParse(BulkMonitorSchema, {+ target: rawTarget,+ protocol: rawProtocol,+ port: rawPort,+ interval: rawInterval,+ })+ if (!parsed.success) {+ throw new Error(`Line ${lineNumber}: ${parsed.issues[0]?.message || "invalid monitor entry"}`)+ }+ const protocol = (parsed.output.protocol?.toLowerCase() ||+ (/^https?:\/\//i.test(parsed.output.target) ? "http" : "icmp")) as MonitorProtocol++ return buildMonitorPayload({+ system,+ target: parsed.output.target,+ protocol,+ port: parsed.output.port ? Number(parsed.output.port) : 0,+ interval: parsed.output.interval || `${defaultInterval}`,+ })+}++export function formatBulkMonitorLine(monitor: BulkMonitorLineSource) {+ const port = monitor.protocol !== "tcp" || monitor.port === 443 ? "" : `${monitor.port}`+ const interval = monitor.interval === defaultInterval ? "" : `${monitor.interval}`+ return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval]).join(",")+}++function SystemMultiSelect({+ id,+ selectedSystemIds,+ onChange,+ disabled,+ className,+}: {+ id: string+ selectedSystemIds: Set<string>+ onChange: (ids: Set<string>) => void+ disabled?: boolean+ className?: string+}) {+ const systems = useStore($systems)+ const { t } = useLingui()+ const [search, setSearch] = useState("")+ const searchRef = useRef<HTMLInputElement>(null)+ const focusSearchOnMount = useCallback((node: HTMLInputElement | null) => {+ searchRef.current = node+ if (!node) return+ // Focus after the menu has completed its own initial focus handling.+ const frame = requestAnimationFrame(() => node.focus())+ return () => cancelAnimationFrame(frame)+ }, [])+ const contentRef = useRef<HTMLDivElement>(null)+ const query = search.trim().toLocaleLowerCase()+ const filteredSystems = systems.filter(+ (system) => supportsNetworkMonitors(system) && system.name.toLocaleLowerCase().includes(query)+ )+ const allSelected = filteredSystems.every((system) => selectedSystemIds.has(system.id))+ const anySelected = filteredSystems.some((system) => selectedSystemIds.has(system.id))++ const selectFiltered = (selected: boolean) => {+ const next = new Set(selectedSystemIds)+ for (const system of filteredSystems) {+ if (selected) next.add(system.id)+ else next.delete(system.id)+ }+ onChange(next)+ }+ return (+ <DropdownMenu onOpenChange={() => setSearch("")}>+ <DropdownMenuTrigger asChild>+ <Button+ id={id}+ disabled={disabled}+ type="button"+ variant="outline"+ className={cn("relative w-full min-w-0 ps-10 pe-10 justify-start font-normal text-start", className)}+ >+ <ServerIcon className="size-3.5 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />+ <span className="truncate">+ {selectedSystemIds.size === 0+ ? t`Select systems`+ : selectedSystemIds.size === 1+ ? systems.find((s) => selectedSystemIds.has(s.id))?.name+ : t`${selectedSystemIds.size} systems selected`}+ </span>+ <ChevronDownIcon className="size-4 absolute end-4 top-1/2 -translate-y-1/2 opacity-50" />+ </Button>+ </DropdownMenuTrigger>+ <DropdownMenuContent+ ref={contentRef}+ onKeyDown={(event) => {+ if (event.key === "Tab") {+ event.preventDefault()+ searchRef.current?.focus()+ }+ }}+ align="start"+ className="w-[var(--radix-dropdown-menu-trigger-width)] max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] flex flex-col overflow-hidden"+ >+ <div className="shrink-0 border-b mb-1">+ <div className="flex items-center gap-2 px-2.5">+ <SearchIcon aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" />+ <Input+ ref={focusSearchOnMount}+ value={search}+ onChange={(event) => setSearch(event.target.value)}+ placeholder={t`Search systems`}+ aria-label={t`Search systems`}+ className="h-10 min-w-0 rounded-none border-0 bg-transparent px-0 shadow-none focus-visible:ring-0 focus-visible:ring-offset-0"+ onKeyDown={(event) => {+ if (event.key === "Escape") return+ // Keep menu typeahead and form submission from consuming search input.+ event.stopPropagation()+ if (event.key === "Enter") event.preventDefault()+ if (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Tab") {+ event.preventDefault()+ const items = contentRef.current?.querySelectorAll<HTMLElement>(+ '[role^="menuitem"]:not([data-disabled])'+ )+ const index = event.key === "ArrowUp" || event.shiftKey ? (items?.length ?? 1) - 1 : 0+ items?.[index]?.focus()+ }+ }}+ />+ </div>+ <div className="flex flex-wrap items-center justify-between gap-x-3 gap-y-1 px-1 pb-1">+ <div className="flex items-center">+ <DropdownMenuItem+ className="px-1.5 py-1 text-xs text-muted-foreground"+ disabled={!filteredSystems.length || allSelected}+ onSelect={(event) => {+ event.preventDefault()+ selectFiltered(true)+ }}+ >+ {query ? <Trans>Select matches</Trans> : <Trans>Select all</Trans>}+ </DropdownMenuItem>+ <span aria-hidden="true" className="text-xs text-muted-foreground/50">+ ·+ </span>+ <DropdownMenuItem+ className="px-1.5 py-1 text-xs text-muted-foreground"+ disabled={!anySelected}+ onSelect={(event) => {+ event.preventDefault()+ selectFiltered(false)+ }}+ >+ {query ? <Trans>Clear matches</Trans> : <Trans>Clear all</Trans>}+ </DropdownMenuItem>+ </div>+ <span className="px-1.5 text-xs tabular-nums text-muted-foreground">+ {t`${selectedSystemIds.size} selected`}+ </span>+ </div>+ </div>+ <div className="min-h-0 overflow-y-auto">+ {filteredSystems.length === 0 && (+ <output className="block px-2.5 py-3 text-sm text-muted-foreground">+ <Trans>No systems found.</Trans>+ </output>+ )}+ {filteredSystems.map((sys) => (+ <DropdownMenuCheckboxItem+ key={sys.id}+ checked={selectedSystemIds.has(sys.id)}+ onSelect={(event) => event.preventDefault()}+ onCheckedChange={(checked) => {+ const next = new Set(selectedSystemIds)+ if (checked) next.add(sys.id)+ else next.delete(sys.id)+ onChange(next)+ }}+ className="group min-w-0 gap-2.5 py-2 ps-2.5"+ indicatorClassName="static size-4 shrink-0 rounded border border-input group-data-[state=checked]:border-primary group-data-[state=checked]:bg-primary group-data-[state=checked]:text-primary-foreground [&_svg]:size-3"+ >+ <span className="truncate">{sys.name}</span>+ </DropdownMenuCheckboxItem>+ ))}+ </div>+ </DropdownMenuContent>+ </DropdownMenu>+ )+}++export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; monitors: NetworkMonitorRecord[] }) {+ const [open, setOpen] = useState(false)+ const [bulkOpen, setBulkOpen] = useState(false)+ const [bulkInput, setBulkInput] = useState("")+ const [bulkLoading, setBulkLoading] = useState(false)+ const [bulkSelectedSystemIds, setBulkSelectedSystemIds] = useState<Set<string>>(new Set())+ const bulkFormRef = useRef<HTMLFormElement>(null)+ const { toast } = useToast()+ const { t } = useLingui()++ const resetBulkForm = () => {+ setBulkInput("")+ }++ const openBulkAdd = (selectedSystemIds?: Set<string>) => {+ if (!systemId && selectedSystemIds) {+ setBulkSelectedSystemIds(new Set(selectedSystemIds))+ }+ setOpen(false)+ setBulkOpen(true)+ }++ const openAdd = () => {+ setBulkOpen(false)+ setOpen(true)+ }++ async function handleBulkSubmit(e: React.FormEvent) {+ e.preventDefault()+ setBulkLoading(true)+ let closedForSubmit = false++ try {+ const targetSystems = systemId ? [systemId] : Array.from(bulkSelectedSystemIds)+ if (!targetSystems.length) {+ throw new Error("Select at least one system.")+ }+ const rawLines = bulkInput.split(/\r?\n/).filter((line) => line.trim())+ if (!rawLines.length) {+ throw new Error("Enter at least one monitor.")+ }++ let totalCreated = 0+ closedForSubmit = true++ for (const system of targetSystems) {+ const payloads = rawLines.map((line, index) => parseBulkMonitorLine(line, index + 1, system))+ const existingMonitorKeys = new Set(+ monitors.filter((monitor) => monitor.system === system).map((monitor) => getMonitorIdentityKey(monitor))+ )+ const newPayloads: typeof payloads = []++ for (const payload of payloads) {+ const monitorKey = getMonitorIdentityKey(payload)+ if (existingMonitorKeys.has(monitorKey)) {+ continue+ }+ existingMonitorKeys.add(monitorKey)+ newPayloads.push(payload)+ }++ if (!newPayloads.length) continue++ let batch = pb.createBatch()+ let inBatch = 0+ for (const payload of newPayloads) {+ batch.collection("network_monitors").create(payload)+ inBatch+++ if (inBatch > 20) {+ await batch.send()+ batch = pb.createBatch()+ inBatch = 0+ }+ }+ if (inBatch) {+ await batch.send()+ }+ totalCreated += newPayloads.length+ }++ if (!totalCreated) {+ throw new Error("No new monitors. All entries already exist.")+ }++ resetBulkForm()+ toast({ title: t`Monitors created`, description: `${totalCreated} monitor(s) added.` })+ } catch (err: unknown) {+ if (closedForSubmit) {+ setBulkOpen(true)+ }+ toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })+ } finally {+ setBulkLoading(false)+ }+ }++ return (+ <>+ <div className="flex gap-0 rounded-lg">+ <Button variant="outline" onClick={openAdd} className="rounded-e-none grow">+ {/* <PlusIcon className="size-4 me-1" /> */}+ <Trans>Add {{ foo: t`Monitor` }}</Trans>+ </Button>+ <div className="w-px h-full bg-muted"></div>+ <DropdownMenu>+ <DropdownMenuTrigger asChild>+ <Button variant="outline" className="px-2 rounded-s-none border-s-0" aria-label={t`More monitor actions`}>+ <ChevronDownIcon className="size-4" />+ </Button>+ </DropdownMenuTrigger>+ <DropdownMenuContent align="end">+ <DropdownMenuItem onClick={() => openBulkAdd()}>+ <ListIcon className="size-4 me-2" />+ <Trans>Bulk Add</Trans>+ </DropdownMenuItem>+ </DropdownMenuContent>+ </DropdownMenu>+ </div>+ <Dialog+ open={open}+ onOpenChange={(nextOpen) => {+ setOpen(nextOpen)+ }}+ >+ <MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} onOpenBulkAdd={openBulkAdd} />+ </Dialog>++ <Sheet+ open={bulkOpen}+ onOpenChange={(nextOpen) => {+ setBulkOpen(nextOpen)+ if (!nextOpen) {+ resetBulkForm()+ }+ }}+ >+ <SheetContent className="w-full sm:max-w-xl gap-0">+ <SheetHeader className="border-b">+ <SheetTitle>+ <Trans>Bulk Add {{ foo: t`Network Monitors` }}</Trans>+ </SheetTitle>+ <SheetDescription>target[,protocol[,port[,interval]]]</SheetDescription>+ </SheetHeader>+ <form ref={bulkFormRef} onSubmit={handleBulkSubmit} className="flex h-full flex-col overflow-hidden">+ <div className="flex-1 flex flex-col space-y-4 overflow-auto p-4">+ {!systemId && (+ <div className="grid gap-2">+ <Label htmlFor="bulk-monitor-systems" className="sr-only">+ <Trans>Systems</Trans>+ </Label>+ <SystemMultiSelect+ id="bulk-monitor-systems"+ selectedSystemIds={bulkSelectedSystemIds}+ onChange={setBulkSelectedSystemIds}+ disabled={bulkLoading}+ className="bg-card"+ />+ </div>+ )}+ <div className="grow flex flex-col gap-2">+ <Label htmlFor="bulk-monitors" className="sr-only">+ Entries+ </Label>+ <Textarea+ id="bulk-monitors"+ value={bulkInput}+ onChange={(e) => setBulkInput(e.target.value)}+ onKeyDown={(e) => {+ if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {+ e.preventDefault()+ bulkFormRef.current?.requestSubmit()+ }+ }}+ className="font-mono grow text-sm bg-card"+ placeholder={["1.1.1.1", "example.com,tcp", "https://example.com,http,,60"].join("\n")}+ required+ />+ <p className="text-xs text-muted-foreground">target[,protocol[,port[,interval]]]</p>+ </div>+ </div>+ <SheetFooter className="border-t">+ <Button type="submit" disabled={bulkLoading || (!systemId && !bulkSelectedSystemIds.size)}>+ <Trans>Add {{ foo: t`Network Monitors` }}</Trans>+ </Button>+ </SheetFooter>+ </form>+ </SheetContent>+ </Sheet>+ </>+ )+}++export function EditMonitorDialog({+ open,+ setOpen,+ systemId,+ monitor,+}: {+ open: boolean+ setOpen: (open: boolean) => void+ systemId?: string+ monitor?: NetworkMonitorRecord+}) {+ const hasOpened = useRef(false)+ if (!monitor && !hasOpened.current) {+ return null+ }+ hasOpened.current = true+ return (+ <Dialog open={open} onOpenChange={setOpen}>+ <MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} monitor={monitor} />+ </Dialog>+ )+}++function MonitorDialogContent({+ open,+ setOpen,+ systemId,+ monitor,+ onOpenBulkAdd,+}: {+ open: boolean+ setOpen: (open: boolean) => void+ systemId?: string+ monitor?: NetworkMonitorRecord+ onOpenBulkAdd?: (selectedSystemIds: Set<string>) => void+}) {+ const [protocol, setProtocol] = useState<MonitorProtocol>(monitor?.protocol ?? "icmp")+ const [target, setTarget] = useState(monitor?.target ?? "")+ const [port, setPort] = useState(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")+ const [monitorInterval, setMonitorInterval] = useState(String(monitor?.interval ?? defaultInterval))+ const [loading, setLoading] = useState(false)+ const [selectedSystemId, setSelectedSystemId] = useState(monitor?.system ?? "")+ const [selectedSystemIds, setSelectedSystemIds] = useState<Set<string>>(new Set())+ const systems = useStore($systems)+ const { toast } = useToast()+ const { t } = useLingui()+ const isEditing = !!monitor++ // When the dialog is opened, initialize form fields with monitor values (if editing) or defaults (if adding).+ useEffect(() => {+ if (!open) {+ return+ }++ setProtocol(monitor?.protocol ?? "icmp")+ setTarget(monitor?.target ?? "")+ setPort(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")+ setMonitorInterval(String(monitor?.interval ?? defaultInterval))+ setSelectedSystemId(monitor?.system ?? "")+ setSelectedSystemIds(new Set())+ setLoading(false)+ }, [open, monitor])++ async function handleSubmit(e: React.FormEvent) {+ e.preventDefault()+ setLoading(true)++ const targetSystems = systemId ? [systemId] : monitor ? [selectedSystemId] : Array.from(selectedSystemIds)+ const remainingSystemIds = new Set(targetSystems)+ try {+ if (!targetSystems.length || !targetSystems[0]) throw new Error("Select at least one system.")+ const payload = buildMonitorPayload(+ {+ system: targetSystems[0],+ target,+ protocol,+ port: protocol === "tcp" ? Number(port) : 0,+ interval: monitorInterval,+ },+ monitor ? monitor.enabled : true+ )+ if (monitor) {+ await pb.collection("network_monitors").update(monitor.id, payload)+ } else {+ for (const system of targetSystems) {+ await pb.collection("network_monitors").create({ ...payload, system })+ remainingSystemIds.delete(system)+ }+ }+ setOpen(false)+ } catch (err: unknown) {+ if (!monitor && !systemId) {+ // Retain only unfinished systems so retrying cannot duplicate successful creates.+ setSelectedSystemIds(remainingSystemIds)+ }+ toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })+ } finally {+ setLoading(false)+ }+ }++ return (+ <DialogContent className="max-w-md">+ <DialogHeader>+ <DialogTitle>+ {isEditing ? (+ <Trans>Edit {{ foo: t`Network Monitor` }}</Trans>+ ) : (+ <Trans>Add {{ foo: t`Network Monitor` }}</Trans>+ )}+ </DialogTitle>+ <DialogDescription>+ <Trans>Configure response monitoring from this agent.</Trans>+ </DialogDescription>+ </DialogHeader>+ <form onSubmit={handleSubmit} className="grid gap-4 tabular-nums">+ {!systemId && !isEditing && (+ <div className="grid gap-2">+ <Label htmlFor="monitor-systems">+ <Trans>Systems</Trans>+ </Label>+ <SystemMultiSelect+ id="monitor-systems"+ selectedSystemIds={selectedSystemIds}+ onChange={setSelectedSystemIds}+ disabled={loading}+ />+ </div>+ )}+ {!systemId && isEditing && (+ <div className="grid gap-2">+ <Label>+ <Trans>System</Trans>+ </Label>+ <Select value={selectedSystemId} onValueChange={setSelectedSystemId} required>+ <SelectTrigger>+ <SelectValue placeholder={t`Select a system`} />+ </SelectTrigger>+ <SelectContent>+ {systems+ .filter((sys) => sys.id === monitor?.system || supportsNetworkMonitors(sys))+ .map((sys) => (+ <SelectItem key={sys.id} value={sys.id}>+ {sys.name}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ </div>+ )}+ <div className="grid gap-2">+ <Label>+ <Trans>Target</Trans>+ </Label>+ <Input+ value={target}+ onChange={(e) => setTarget(e.target.value)}+ placeholder={protocol === "http" ? "http://localhost:8090" : "1.1.1.1"}+ required+ />+ </div>+ <div className="grid gap-2">+ <Label>+ <Trans>Protocol</Trans>+ </Label>++ <Select value={protocol} onValueChange={(value) => setProtocol(value as MonitorProtocol)}>+ <SelectTrigger>+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="icmp">ICMP</SelectItem>+ <SelectItem value="tcp">TCP</SelectItem>+ <SelectItem value="http">HTTP</SelectItem>+ <SelectItem value="dns">DNS</SelectItem>+ </SelectContent>+ </Select>+ </div>+ {protocol === "tcp" && (+ <div className="grid gap-2">+ <Label>+ <Trans>Port</Trans>+ </Label>+ <Input+ type="number"+ value={port}+ onChange={(e) => setPort(e.target.value)}+ placeholder="443"+ min={1}+ max={65535}+ />+ </div>+ )}+ <div className="grid gap-2">+ <Label>+ <Trans>Interval (seconds)</Trans>+ </Label>+ <Input+ type="number"+ value={monitorInterval}+ onChange={(e) => setMonitorInterval(e.target.value)}+ min={1}+ max={3600}+ required+ />+ </div>+ <DialogFooter>+ {!isEditing && onOpenBulkAdd && (+ <Button+ type="button"+ variant="outline"+ onClick={() => onOpenBulkAdd(selectedSystemIds)}+ disabled={loading}+ className="me-auto"+ >+ <ListIcon className="size-4 me-2" />+ <Trans>Bulk Add</Trans>+ </Button>+ )}+ <Button+ type="submit"+ disabled={loading || (!systemId && (isEditing ? !selectedSystemId : !selectedSystemIds.size))}+ >+ {loading ? (+ isEditing ? (+ <Trans>Saving...</Trans>+ ) : (+ <Trans>Creating...</Trans>+ )+ ) : isEditing ? (+ <Trans>Save {{ foo: t`Monitor` }}</Trans>+ ) : (+ <Trans>Add {{ foo: t`Monitor` }}</Trans>+ )}+ </Button>+ </DialogFooter>+ </form>+ </DialogContent>+ )+}diff --git a/internal/site/src/components/network-monitors-table/network-monitors-columns.tsx b/internal/site/src/components/network-monitors-table/network-monitors-columns.tsxnew file mode 100644index 000000000..921b30f0b--- /dev/null+++ b/internal/site/src/components/network-monitors-table/network-monitors-columns.tsx@@ -0,0 +1,411 @@+import type { CellContext, Column, ColumnDef } from "@tanstack/react-table"+import { Button } from "@/components/ui/button"+import { cn, copyToClipboard, decimalString, formatMicroseconds, hourWithSeconds } from "@/lib/utils"+import {+ GlobeIcon,+ TimerIcon,+ WifiOffIcon,+ Trash2Icon,+ ArrowLeftRightIcon,+ MoreHorizontalIcon,+ ServerIcon,+ ClockIcon,+ RefreshCwIcon,+ PenBoxIcon,+ PauseCircleIcon,+ PlayCircleIcon,+ CopyIcon,+ CopyPlusIcon,+} from "lucide-react"+import { t } from "@lingui/core/macro"+import type { NetworkMonitorRecord, SystemRecord } from "@/types"+import {+ DropdownMenu,+ DropdownMenuContent,+ DropdownMenuItem,+ DropdownMenuSeparator,+ DropdownMenuSub,+ DropdownMenuSubContent,+ DropdownMenuSubTrigger,+ DropdownMenuTrigger,+} from "@/components/ui/dropdown-menu"+import { Trans } from "@lingui/react/macro"+import { $allSystemsById, $longestSystemName } from "@/lib/stores"+import { useStore } from "@nanostores/react"+import { SystemStatus } from "@/lib/enums"+import { Checkbox } from "@/components/ui/checkbox"+import { useMemo } from "react"+import { formatBulkMonitorLine } from "@/components/network-monitors-table/monitor-dialog"+import { Badge } from "../ui/badge"+import { getMonitorTarget } from "@/lib/network-monitor-utils"+import { pb } from "@/lib/api"++const protocolColors: Record<string, string> = {+ icmp: "bg-blue-500/15! text-blue-600 dark:text-blue-400",+ tcp: "bg-purple-500/15! text-purple-600 dark:text-purple-400",+ http: "bg-green-500/15! text-green-700 dark:text-green-400",+ dns: "bg-amber-500/15! text-amber-600 dark:text-amber-400",+}++const SYSTEM_STATUS_COLORS = {+ [SystemStatus.Up]: "bg-green-500",+ [SystemStatus.Down]: "bg-red-500",+ [SystemStatus.Paused]: "bg-primary/40",+ [SystemStatus.Pending]: "bg-yellow-500",+} as const++/**+ * A monitor is considered muted if it's disabled or if its associated system is not up.+ */+const isMuted = (record: NetworkMonitorRecord, systemRecord: SystemRecord | undefined) =>+ !record.enabled || systemRecord?.status !== SystemStatus.Up++export function getMonitorColumns(+ longestTarget = "",+ {+ onEdit,+ onDelete,+ onSetEnabled,+ }: {+ onEdit?: (monitor: NetworkMonitorRecord) => void+ onDelete?: (monitors: NetworkMonitorRecord[]) => void | Promise<void>+ onSetEnabled?: (monitors: NetworkMonitorRecord[], enabled: boolean) => void | Promise<void>+ } = {}+): ColumnDef<NetworkMonitorRecord>[] {+ return [+ {+ id: "select",+ header: ({ table }) => (+ <Checkbox+ className="ms-2"+ checked={table.getIsAllRowsSelected() || (table.getIsSomeRowsSelected() && "indeterminate")}+ onClick={(event) => event.stopPropagation()}+ onCheckedChange={(value) => table.toggleAllRowsSelected(!!value)}+ aria-label={t`Select all`}+ />+ ),+ cell: ({ row }) => (+ <Checkbox+ checked={row.getIsSelected()}+ onClick={(event) => event.stopPropagation()}+ onCheckedChange={(value) => row.toggleSelected(!!value)}+ aria-label={t`Select row`}+ />+ ),+ enableSorting: false,+ enableHiding: false,+ size: 44,+ },+ {+ id: "system",+ accessorFn: (record) => record.system,+ sortingFn: (a, b) => {+ const allSystems = $allSystemsById.get()+ const systemNameA = allSystems[a.original.system]?.name ?? ""+ const systemNameB = allSystems[b.original.system]?.name ?? ""+ const primary = systemNameA.localeCompare(systemNameB)+ if (primary !== 0) {+ return primary+ }+ return a.original.target.localeCompare(b.original.target)+ },+ header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,+ cell: ({ getValue }) => {+ const system = useStore($allSystemsById)[getValue() as string] as SystemRecord | undefined+ const longestSystemName = useStore($longestSystemName)+ const name = system?.name+ const status = system?.status as SystemStatus // undefined val is fine but makes lsp mad++ return useMemo(+ () => (+ <div className="ms-1.5 max-w-44 flex gap-2 items-center tabular-nums">+ <span className={cn("shrink-0 size-2 rounded-full", SYSTEM_STATUS_COLORS[status])} />+ <div className="relative w-fit min-w-0 max-w-full">+ <span className="invisible block whitespace-nowrap" aria-hidden="true">+ {longestSystemName}+ </span>+ <span className="absolute inset-0 truncate">{name}</span>+ </div>+ </div>+ ),+ [status, name]+ )+ },+ },+ {+ id: "target",+ sortingFn: (a, b) => a.original.target.localeCompare(b.original.target),+ accessorFn: (record) => getMonitorTarget(record),+ header: ({ column }) => <HeaderButton column={column} name={t`Target`} Icon={GlobeIcon} />,+ cell: ({ row, getValue }) => {+ const monitor = row.original+ const { status } = useStore($allSystemsById)[monitor.system] || {}++ let color = "bg-green-500"+ if (!monitor.enabled || status === SystemStatus.Paused) {+ color = "bg-primary/40"+ } else if (status === SystemStatus.Down || status === SystemStatus.Pending) {+ color = "bg-yellow-500"+ }+ return (+ <div className="ms-1.5 max-w-64 flex gap-2 items-center tabular-nums">+ <span className={cn("shrink-0 size-2 rounded-full", color)} />+ <div className="relative w-fit min-w-0 max-w-full">+ <span className="invisible block overflow-hidden whitespace-nowrap" aria-hidden="true">+ {longestTarget}+ </span>+ <span className="absolute inset-0 truncate">{getValue() as string}</span>+ </div>+ </div>+ )+ },+ },+ {+ id: "protocol",+ accessorFn: (record) => record.protocol,+ header: ({ column }) => <HeaderButton column={column} name={t`Protocol`} Icon={ArrowLeftRightIcon} />,+ cell: ({ getValue }) => {+ const protocol = getValue() as string+ return <Badge className={cn("uppercase", protocolColors[protocol])}>{protocol}</Badge>+ },+ },+ {+ id: "interval",+ accessorFn: (record) => record.interval,+ invertSorting: true,+ header: ({ column }) => <HeaderButton column={column} name={t`Interval`} Icon={RefreshCwIcon} />,+ cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{getValue() as number}s</span>,+ },+ {+ id: "res",+ accessorFn: (record) => record.res,+ invertSorting: true,+ header: ({ column }) => <HeaderButton column={column} name={t`Response`} Icon={TimerIcon} />,+ cell: responseTimeCell,+ },+ {+ id: "res1h",+ accessorFn: (record) => record.resAvg1h,+ invertSorting: true,+ header: ({ column }) => <HeaderButton column={column} name={t`Avg 1h`} Icon={TimerIcon} />,+ cell: responseTimeCell,+ },+ {+ id: "max1h",+ accessorFn: (record) => record.resMax1h,+ invertSorting: true,+ header: ({ column }) => <HeaderButton column={column} name={t`Max 1h`} Icon={TimerIcon} />,+ cell: responseTimeCell,+ },+ {+ id: "min1h",+ accessorFn: (record) => record.resMin1h,+ invertSorting: true,+ header: ({ column }) => <HeaderButton column={column} name={t`Min 1h`} Icon={TimerIcon} />,+ cell: responseTimeCell,+ },+ {+ id: "loss",+ accessorFn: (record) => record.loss1h,+ invertSorting: true,+ header: ({ column }) => <HeaderButton column={column} name={t`Loss 1h`} Icon={WifiOffIcon} />,+ cell: ({ row }) => {+ const { loss1h, res, system } = row.original+ const systemRecord = useStore($allSystemsById)[system]++ if (loss1h === undefined || (!res && !loss1h)) {+ return <span className="ms-1.5 text-muted-foreground">-</span>+ }++ const muted = isMuted(row.original, systemRecord)+ let color = "bg-green-500"+ if (muted) {+ color = "bg-muted-foreground/50"+ } else if (loss1h) {+ color = loss1h > 20 ? "bg-red-500" : "bg-yellow-500"+ }+ return (+ <span className="ms-1.5 tabular-nums flex gap-2 items-center">+ <span className={cn("shrink-0 size-2 rounded-full", color)} />+ {loss1h === 100 ? loss1h : decimalString(loss1h, loss1h >= 10 ? 1 : 2)}%+ </span>+ )+ },+ },+ {+ id: "updated",+ invertSorting: true,+ accessorFn: (record) => record.updated,+ header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,+ cell: ({ getValue }) => {+ const timestamp = getValue() as number+ if (!timestamp) {+ return <span className="ms-1.5 text-muted-foreground">-</span>+ }+ return <span className="ms-1.5 tabular-nums">{hourWithSeconds(timestamp)}</span>+ },+ },+ {+ id: "actions",+ enableSorting: false,+ enableHiding: false,+ header: () => null,+ size: 40,+ cell: ({ row, table }) => {+ const selectedRows = table.getSelectedRowModel().rows+ const actionRows =+ row.getIsSelected() && selectedRows.length > 1+ ? selectedRows.map((selectedRow) => selectedRow.original)+ : [row.original]+ const isBulkAction = actionRows.length > 1+ const shouldPause = actionRows.some((monitor) => monitor.enabled)+ const bulkCopyContent = actionRows.map((monitor) => formatBulkMonitorLine(monitor)).join("\n")+ const allSystems = useStore($allSystemsById)+ const otherSystems = useMemo(+ () => Object.values(allSystems).filter((s) => !isBulkAction && s.id !== row.original.system),+ [allSystems, isBulkAction]+ )+ return (+ <DropdownMenu>+ <DropdownMenuTrigger asChild>+ <Button variant="ghost" size="icon" className="size-10">+ <span className="sr-only">+ <Trans>Open menu</Trans>+ </span>+ <MoreHorizontalIcon className="w-5" />+ </Button>+ </DropdownMenuTrigger>+ <DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>+ {!isBulkAction && (+ <DropdownMenuItem+ onClick={() => {+ onEdit?.(row.original)+ }}+ >+ <PenBoxIcon className="me-2.5 size-4" />+ <Trans>Edit</Trans>+ </DropdownMenuItem>+ )}+ <DropdownMenuItem+ onClick={() => {+ onSetEnabled?.(actionRows, !shouldPause)+ }}+ >+ {shouldPause ? (+ <>+ <PauseCircleIcon className="me-2.5 size-4" />+ <Trans>Pause</Trans>+ </>+ ) : (+ <>+ <PlayCircleIcon className="me-2.5 size-4" />+ <Trans>Resume</Trans>+ </>+ )}+ </DropdownMenuItem>+ <DropdownMenuItem+ onClick={() => {+ copyToClipboard(bulkCopyContent)+ }}+ >+ <CopyIcon className="me-2.5 size-4" />+ <Trans>Bulk copy</Trans>+ </DropdownMenuItem>+ {!isBulkAction && otherSystems.length > 0 && (+ <DropdownMenuSub>+ <DropdownMenuSubTrigger>+ <CopyPlusIcon className="me-2.5 size-4" />+ <Trans>Copy to system</Trans>+ </DropdownMenuSubTrigger>+ <DropdownMenuSubContent className="max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] overflow-y-auto">+ {otherSystems.map((sys) => (+ <DropdownMenuItem+ key={sys.id}+ onClick={() => {+ const { id: _id, system: _system, ...rest } = row.original+ pb.collection("network_monitors")+ .create({ ...rest, system: sys.id })+ .catch(() => {})+ }}+ >+ {sys.name}+ </DropdownMenuItem>+ ))}+ </DropdownMenuSubContent>+ </DropdownMenuSub>+ )}+ <DropdownMenuSeparator />+ <DropdownMenuItem+ onClick={() => {+ onDelete?.(actionRows)+ }}+ >+ <Trash2Icon className="me-2.5 size-4" />+ <Trans>Delete</Trans>+ </DropdownMenuItem>+ </DropdownMenuContent>+ </DropdownMenu>+ )+ },+ },+ ]+}++const responseTimeThresholds = {+ http: { warning: 800_000, critical: 3_000_000 },+ tcp: { warning: 500_000, critical: 2_000_000 },+ icmp: { warning: 100_000, critical: 500_000 },+ dns: { warning: 150_000, critical: 800_000 },+}++function responseTimeCell(cell: CellContext<NetworkMonitorRecord, unknown>) {+ const monitor = cell.row.original+ const systemRecord = useStore($allSystemsById)[monitor.system]+ const responseTime = cell.getValue() as number | undefined++ if (!responseTime) {+ return <span className="ms-1.5 text-muted-foreground">-</span>+ }++ const muted = isMuted(monitor, systemRecord)+ let color = "bg-green-500"+ if (muted) {+ color = "bg-muted-foreground/50"+ } else if (responseTime > responseTimeThresholds[monitor.protocol].warning) {+ color = "bg-yellow-500"+ }+ if (!muted && responseTime > responseTimeThresholds[monitor.protocol].critical) {+ color = "bg-red-500"+ }+ return (+ <span className="ms-1.5 tabular-nums flex gap-2 items-center">+ <span className={cn("shrink-0 size-2 rounded-full", color)} />+ {formatMicroseconds(responseTime)}+ </span>+ )+}++function HeaderButton({+ column,+ name,+ Icon,+}: {+ column: Column<NetworkMonitorRecord>+ name: string+ Icon: React.ElementType+}) {+ const isSorted = column.getIsSorted()+ return (+ <Button+ className={cn(+ "h-9 px-3 flex items-center gap-2 duration-50",+ isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"+ )}+ variant="ghost"+ onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}+ >+ {Icon && <Icon className="size-4" />}+ {name}+ </Button>+ )+}diff --git a/internal/site/src/components/network-monitors-table/network-monitors-table.tsx b/internal/site/src/components/network-monitors-table/network-monitors-table.tsxnew file mode 100644index 000000000..9e42d803a--- /dev/null+++ b/internal/site/src/components/network-monitors-table/network-monitors-table.tsx@@ -0,0 +1,575 @@+import { getMonitorTarget } from "@/lib/network-monitor-utils"+import { t } from "@lingui/core/macro"+import { Trans } from "@lingui/react/macro"+import {+ type ColumnFiltersState,+ flexRender,+ getCoreRowModel,+ getFilteredRowModel,+ getSortedRowModel,+ type Row,+ type RowSelectionState,+ type SortingState,+ type Table as TableType,+ useReactTable,+ type VisibilityState,+} from "@tanstack/react-table"+import { useVirtualizer, type VirtualItem } from "@tanstack/react-virtual"+import {+ AlertDialog,+ AlertDialogAction,+ AlertDialogCancel,+ AlertDialogContent,+ AlertDialogDescription,+ AlertDialogFooter,+ AlertDialogHeader,+ AlertDialogTitle,+} from "@/components/ui/alert-dialog"+import { Button, buttonVariants } from "@/components/ui/button"+import { memo, useCallback, useMemo, useRef, useState } from "react"+import { getMonitorColumns } from "@/components/network-monitors-table/network-monitors-columns"+import { Card, CardHeader, CardTitle } from "@/components/ui/card"+import { Input } from "@/components/ui/input"+import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"+import { useToast } from "@/components/ui/use-toast"+import { isReadOnlyUser } from "@/lib/api"+import { pb } from "@/lib/api"+import { $allSystemsById, $direction, $userSettings } from "@/lib/stores"+import {+ cn,+ isVisuallyLonger,+ matchesFilterGroups,+ parseFilterGroups,+ parseSemVer,+ useBrowserStorage,+} from "@/lib/utils"+import type { ChartData, NetworkMonitorRecord } from "@/types"+import { AddMonitorDialog, EditMonitorDialog } from "./monitor-dialog"+import { ArrowLeftRightIcon, EthernetPortIcon, LoaderCircleIcon, ServerIcon, XIcon } from "lucide-react"+import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"+import ChartTimeSelect from "@/components/charts/chart-time-select"+import { LossChart, AvgMinMaxResponseChart } from "@/components/routes/system/charts/monitors-charts"+import { useNetworkMonitorStats } from "@/lib/use-network-monitors"+import { useStore } from "@nanostores/react"+import { atom } from "nanostores"+import { Separator } from "../ui/separator"+import { $router, Link } from "../router"+import { getPagePath } from "@nanostores/router"++export default function NetworkMonitorsTableNew({+ systemId,+ monitors,+ isLoading,+}: {+ systemId?: string+ monitors: NetworkMonitorRecord[]+ isLoading: boolean+}) {+ const [sorting, setSorting] = useBrowserStorage<SortingState>(+ `sort-np-target-${systemId ? 1 : 0}`,+ [{ id: systemId ? "target" : "system", desc: false }],+ sessionStorage+ )+ const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])+ const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})+ const [rowSelection, setRowSelection] = useState<RowSelectionState>({})+ const [globalFilter, setGlobalFilter] = useState("")+ const [deleteOpen, setDeleteOpen] = useState(false)+ const [pendingDeleteIds, setPendingDeleteIds] = useState<string[]>([])+ const [editingMonitor, setEditingMonitor] = useState<NetworkMonitorRecord>()++ const { toast } = useToast()+ const canManageMonitors = !isReadOnlyUser()++ const longestTarget = useMemo(() => {+ let longestTarget = ""+ for (const p of monitors) {+ if (isVisuallyLonger(getMonitorTarget(p), longestTarget)) {+ longestTarget = getMonitorTarget(p)+ }+ }+ return longestTarget+ }, [monitors])++ const runMonitorBatch = useCallback(+ async (ids: string[], enqueue: (batch: ReturnType<typeof pb.createBatch>, id: string) => void) => {+ let batch = pb.createBatch()+ let inBatch = 0+ for (const id of ids) {+ enqueue(batch, id)+ if (++inBatch >= 20) {+ await batch.send()+ batch = pb.createBatch()+ inBatch = 0+ }+ }+ if (inBatch) {+ await batch.send()+ }+ },+ []+ )++ const handleDeleteRequest = useCallback(+ async (monitorsToDelete: NetworkMonitorRecord[]) => {+ if (!monitorsToDelete.length) {+ return+ }++ const ids = monitorsToDelete.map((monitor) => monitor.id)+ if (ids.length === 1) {+ try {+ await pb.collection("network_monitors").delete(ids[0])+ } catch (err: unknown) {+ toast({+ variant: "destructive",+ title: t`Error`,+ description: (err as Error)?.message || t`Failed to delete monitors.`,+ })+ }+ return+ }++ setPendingDeleteIds(ids)+ setDeleteOpen(true)+ },+ [toast]+ )++ const handleBulkDelete = async () => {+ setDeleteOpen(false)+ if (!pendingDeleteIds.length) {+ return+ }++ try {+ await runMonitorBatch(pendingDeleteIds, (batch, id) => batch.collection("network_monitors").delete(id))+ setPendingDeleteIds([])+ setRowSelection({})+ } catch (err: unknown) {+ toast({+ variant: "destructive",+ title: t`Error`,+ description: (err as Error)?.message || t`Failed to delete monitors.`,+ })+ }+ }++ const handleSetEnabled = useCallback(+ async (monitorsToUpdate: NetworkMonitorRecord[], enabled: boolean) => {+ if (!monitorsToUpdate.length) {+ return+ }++ const pendingUpdates = monitorsToUpdate.filter((monitor) => monitor.enabled !== enabled)+ if (!pendingUpdates.length) {+ return+ }++ try {+ if (pendingUpdates.length === 1) {+ await pb.collection("network_monitors").update(pendingUpdates[0].id, { enabled })+ return+ }+ await runMonitorBatch(+ pendingUpdates.map((monitor) => monitor.id),+ (batch, id) => batch.collection("network_monitors").update(id, { enabled })+ )+ if (monitorsToUpdate.length > 1) {+ setRowSelection({})+ }+ } catch (err: unknown) {+ toast({+ variant: "destructive",+ title: t`Error`,+ description: (err as Error)?.message || t`Failed to update monitors.`,+ })+ }+ },+ [runMonitorBatch, toast]+ )++ const columns = useMemo(() => {+ let columns = getMonitorColumns(longestTarget, {+ onEdit: setEditingMonitor,+ onDelete: handleDeleteRequest,+ onSetEnabled: handleSetEnabled,+ })+ columns = systemId ? columns.filter((col) => col.id !== "system") : columns+ columns = canManageMonitors ? columns : columns.filter((col) => col.id !== "actions")+ return columns+ }, [canManageMonitors, handleDeleteRequest, handleSetEnabled, systemId, longestTarget])++ const table = useReactTable({+ data: monitors,+ columns,+ getRowId: (row) => row.id,+ getCoreRowModel: getCoreRowModel(),+ getSortedRowModel: getSortedRowModel(),+ getFilteredRowModel: getFilteredRowModel(),+ onSortingChange: setSorting,+ onColumnFiltersChange: setColumnFilters,+ onColumnVisibilityChange: setColumnVisibility,+ onRowSelectionChange: setRowSelection,+ defaultColumn: {+ sortUndefined: "last",+ size: 900,+ minSize: 0,+ },+ state: {+ sorting,+ columnFilters,+ columnVisibility,+ rowSelection,+ globalFilter,+ },+ onGlobalFilterChange: setGlobalFilter,+ globalFilterFn: (row, _columnId, filterValue) => {+ const value = (filterValue as string).trim()+ if (!value) return true+ const monitor = row.original+ const systemName = $allSystemsById.get()[monitor.system]?.name ?? ""+ const searchString = `${getMonitorTarget(monitor)}${monitor.protocol}${systemName}`.toLocaleLowerCase()+ return matchesFilterGroups(searchString, parseFilterGroups(value))+ },+ })++ const rows = table.getRowModel().rows+ const visibleColumns = table.getVisibleLeafColumns()++ return (+ <Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">+ <CardHeader className="p-0 mb-3 sm:mb-4">+ <div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">+ <div className="px-2 sm:px-1">+ <CardTitle className="mb-2">+ <Trans>Network Monitors</Trans>+ </CardTitle>+ <div className="text-sm text-muted-foreground flex items-center flex-wrap">+ <Trans>Response time monitoring from agents.</Trans>+ </div>+ </div>+ <div className="md:ms-auto flex items-center gap-2">+ {monitors.length > 0 && (+ <div className="relative">+ <Input+ placeholder={t`Filter...`}+ title={t`Use commas to match any of multiple terms, e.g. "system1, system2"`}+ value={globalFilter}+ onChange={(e) => setGlobalFilter(e.target.value)}+ className="ms-auto px-4 w-full max-w-full md:w-50"+ />+ {globalFilter && (+ <Button+ type="button"+ variant="ghost"+ size="icon"+ aria-label={t`Clear`}+ className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"+ onClick={() => setGlobalFilter("")}+ >+ <XIcon className="h-4 w-4" />+ </Button>+ )}+ </div>+ )}+ {canManageMonitors ? <AddMonitorDialog systemId={systemId} monitors={monitors} /> : null}+ {canManageMonitors ? (+ <EditMonitorDialog+ systemId={systemId}+ monitor={editingMonitor}+ open={!!editingMonitor}+ setOpen={(open) => {+ if (!open) {+ setEditingMonitor(undefined)+ }+ }}+ />+ ) : null}+ <AlertDialog+ open={deleteOpen}+ onOpenChange={(open) => {+ setDeleteOpen(open)+ if (!open) {+ setPendingDeleteIds([])+ }+ }}+ >+ <AlertDialogContent>+ <AlertDialogHeader>+ <AlertDialogTitle>+ <Trans>Are you sure?</Trans>+ </AlertDialogTitle>+ <AlertDialogDescription>+ <Trans>This will permanently delete all selected records from the database.</Trans>+ </AlertDialogDescription>+ </AlertDialogHeader>+ <AlertDialogFooter>+ <AlertDialogCancel>+ <Trans>Cancel</Trans>+ </AlertDialogCancel>+ <AlertDialogAction+ className={cn(buttonVariants({ variant: "destructive" }))}+ onClick={handleBulkDelete}+ >+ <Trans>Continue</Trans>+ </AlertDialogAction>+ </AlertDialogFooter>+ </AlertDialogContent>+ </AlertDialog>+ </div>+ </div>+ </CardHeader>+ <div className="rounded-md">+ <NetworkMonitorsTable+ table={table}+ rows={rows}+ colLength={visibleColumns.length}+ rowSelection={rowSelection}+ isLoading={isLoading}+ />+ </div>+ </Card>+ )+}++const NetworkMonitorsTable = memo(function NetworkMonitorTable({+ table,+ rows,+ colLength,+ rowSelection,+ isLoading,+}: {+ table: TableType<NetworkMonitorRecord>+ rows: Row<NetworkMonitorRecord>[]+ colLength: number+ rowSelection: RowSelectionState+ isLoading: boolean+}) {+ const scrollRef = useRef<HTMLDivElement>(null)+ const [sheetOpen, setSheetOpen] = useState(false)+ const [activeMonitorId, setActiveMonitorId] = useState<string | null>(null)+ const activeMonitor = activeMonitorId+ ? table.options.data.find((monitor) => monitor.id === activeMonitorId)+ : undefined+ const openSheet = useCallback((monitor: NetworkMonitorRecord) => {+ setActiveMonitorId(monitor.id)+ setSheetOpen(true)+ }, [])++ const virtualizer = useVirtualizer<HTMLDivElement, HTMLTableRowElement>({+ count: rows.length,+ estimateSize: () => 54,+ getScrollElement: () => scrollRef.current,+ overscan: 5,+ })+ const virtualRows = virtualizer.getVirtualItems()++ const paddingTop = Math.max(0, virtualRows[0]?.start ?? 0 - virtualizer.options.scrollMargin)+ const paddingBottom = Math.max(0, virtualizer.getTotalSize() - (virtualRows[virtualRows.length - 1]?.end ?? 0))++ return (+ <div+ className={cn(+ "h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto border rounded-md",+ (!rows.length || rows.length > 2) && "min-h-50"+ )}+ ref={scrollRef}+ >+ <div style={{ height: `${virtualizer.getTotalSize() + 48}px`, paddingTop, paddingBottom }}>+ <table className="text-sm w-full h-full text-nowrap">+ <NetworkMonitorTableHead table={table} />+ <TableBody>+ {rows.length ? (+ virtualRows.map((virtualRow) => {+ const row = rows[virtualRow.index]+ return (+ <NetworkMonitorTableRow+ key={row.id}+ row={row}+ virtualRow={virtualRow}+ isSelected={row.getIsSelected()}+ rowSelection={rowSelection}+ openSheet={openSheet}+ />+ )+ })+ ) : (+ <TableRow>+ <TableCell colSpan={colLength} className="h-37 text-center pointer-events-none">+ {isLoading ? (+ <LoaderCircleIcon className="animate-spin size-10 opacity-60 mx-auto" />+ ) : (+ <Trans>No results.</Trans>+ )}+ </TableCell>+ </TableRow>+ )}+ </TableBody>+ </table>+ </div>+ <NetworkMonitorSheet+ open={sheetOpen}+ onOpenChange={(nextOpen) => {+ setSheetOpen(nextOpen)+ }}+ monitor={activeMonitor}+ />+ </div>+ )+})++function NetworkMonitorTableHead({ table }: { table: TableType<NetworkMonitorRecord> }) {+ return (+ <TableHeader className="sticky top-0 z-50 w-full border-b-2">+ {table.getHeaderGroups().map((headerGroup) => (+ <tr key={headerGroup.id}>+ {headerGroup.headers.map((header) => {+ return (+ <TableHead className="px-2" key={header.id}>+ {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}+ </TableHead>+ )+ })}+ </tr>+ ))}+ </TableHeader>+ )+}++const NetworkMonitorTableRow = memo(function NetworkMonitorTableRow({+ row,+ virtualRow,+ isSelected,+ rowSelection: _rowSelection,+ openSheet,+}: {+ row: Row<NetworkMonitorRecord>+ virtualRow: VirtualItem+ isSelected: boolean+ // Menus depend on the entire selection, including changes to other rows.+ rowSelection: RowSelectionState+ openSheet: (monitor: NetworkMonitorRecord) => void+}) {+ return (+ <TableRow+ data-state={isSelected && "selected"}+ className="cursor-pointer transition-opacity"+ onClick={() => openSheet(row.original)}+ >+ {row.getVisibleCells().map((cell) => (+ <TableCell+ key={cell.id}+ className="py-0"+ style={{+ width: `${cell.column.getSize()}px`,+ height: virtualRow.size,+ }}+ >+ {flexRender(cell.column.columnDef.cell, cell.getContext())}+ </TableCell>+ ))}+ </TableRow>+ )+})++function NetworkMonitorSheet({+ open,+ onOpenChange,+ monitor,+}: {+ open: boolean+ onOpenChange: (open: boolean) => void+ monitor?: NetworkMonitorRecord+}) {+ if (!monitor) {+ return null+ }++ return <NetworkMonitorSheetContent key={monitor.system} open={open} onOpenChange={onOpenChange} monitor={monitor} />+}++function NetworkMonitorSheetContent({+ open,+ onOpenChange,+ monitor,+}: {+ open: boolean+ onOpenChange: (open: boolean) => void+ monitor: NetworkMonitorRecord+}) {+ // Keep monitor exploration independent of the system charts' time range.+ const [chartTimeStore] = useState(() => {+ const defaultTime = $userSettings.get().chartTime+ return atom(defaultTime === "1m" ? "1h" : defaultTime)+ })+ const chartTime = useStore(chartTimeStore)+ const direction = useStore($direction)+ const system = useStore($allSystemsById)[monitor.system]++ const monitorStats = useNetworkMonitorStats({+ systemId: monitor.system,+ monitorId: monitor.id,+ chartTime,+ enabled: open,+ })++ const chartData = useMemo<ChartData>(+ () => ({+ agentVersion: parseSemVer(system?.info?.v),+ orientation: direction === "rtl" ? "right" : "left",+ chartTime,+ }),+ [system?.info?.v, direction, chartTime]+ )+ const hasMonitorStats = monitorStats.some((record) => record.stats?.[monitor.id] != null)+ const monitorLabel = getMonitorTarget(monitor)++ return (+ <Sheet open={open} onOpenChange={onOpenChange}>+ <SheetContent className="w-full sm:max-w-220 overflow-auto p-4 sm:p-6">+ <SheetHeader className="mb-0 border-b p-0 pb-4">+ <SheetTitle>{monitorLabel}</SheetTitle>+ <SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">+ <ServerIcon className="size-3.5 text-muted-foreground" />+ <Link className="hover:underline" href={getPagePath($router, "system", { id: system?.id ?? "" })}>+ {system?.name ?? ""}+ </Link>+ <Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />+ <ArrowLeftRightIcon className="size-3.5 text-muted-foreground" />+ {monitor.protocol.toUpperCase()}+ {monitor.protocol === "tcp" && monitor.port > 0 && (+ <>+ <Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />+ <EthernetPortIcon className="size-3.5 text-muted-foreground" />+ <span>{monitor.port}</span>+ </>+ )}+ </SheetDescription>+ </SheetHeader>+ <div className="grid gap-4">+ <ChartTimeSelect+ className="bg-card"+ agentVersion={chartData.agentVersion}+ chartTimeStore={chartTimeStore}+ allowRealtime={false}+ />+ <AvgMinMaxResponseChart+ monitorStats={monitorStats}+ monitor={monitor}+ chartData={chartData}+ empty={!hasMonitorStats}+ />+ <LossChart+ monitorStats={monitorStats}+ grid={false}+ monitors={[monitor]}+ chartData={chartData}+ empty={!hasMonitorStats}+ showFilter={false}+ />+ </div>+ </SheetContent>+ </Sheet>+ )+}diff --git a/internal/site/src/components/router.tsx b/internal/site/src/components/router.tsxindex fb5259729..5c54f6137 100644--- a/internal/site/src/components/router.tsx+++ b/internal/site/src/components/router.tsx@@ -4,6 +4,7 @@ const routes = {home: "/",containers: "/containers",smart: "/smart",+ monitors: "/monitors",system: `/system/:id`,settings: `/settings/:name?`,forgot_password: `/forgot-password`,diff --git a/internal/site/src/components/routes/monitors.tsx b/internal/site/src/components/routes/monitors.tsxnew file mode 100644index 000000000..51bee7790--- /dev/null+++ b/internal/site/src/components/routes/monitors.tsx@@ -0,0 +1,33 @@+import { useLingui } from "@lingui/react/macro"+import { memo, useEffect } from "react"+import NetworkMonitorsTableNew from "@/components/network-monitors-table/network-monitors-table"+import { ActiveAlerts } from "@/components/active-alerts"+import { FooterRepoLink } from "@/components/footer-repo-link"+import { useNetworkMonitors } from "@/lib/use-network-monitors"+import { $allSystemsById } from "@/lib/stores"+import { supportsNetworkMonitors } from "@/lib/utils"+import { useStore } from "@nanostores/react"++export default memo(() => {+ const { t } = useLingui()+ const { monitors, isLoading } = useNetworkMonitors({})+ const systems = useStore($allSystemsById)+ const visibleMonitors = monitors.filter((monitor) => {+ const system = systems[monitor.system]+ return !system || supportsNetworkMonitors(system)+ })++ useEffect(() => {+ document.title = `${t`Network Monitors`} / Beszel`+ }, [t])++ return (+ <>+ <div className="grid gap-4">+ <ActiveAlerts />+ <NetworkMonitorsTableNew monitors={visibleMonitors} isLoading={isLoading} />+ </div>+ <FooterRepoLink />+ </>+ )+})diff --git a/internal/site/src/components/routes/settings/alerts-history-data-table.tsx b/internal/site/src/components/routes/settings/alerts-history-data-table.tsxindex 84758ba2d..763daa063 100644--- a/internal/site/src/components/routes/settings/alerts-history-data-table.tsx+++ b/internal/site/src/components/routes/settings/alerts-history-data-table.tsx@@ -78,7 +78,7 @@ export default function AlertsHistoryDataTable() {let unsubscribe: (() => void) | undefinedconst pbOptions = {expand: "system",- fields: "id,name,value,state,created,resolved,expand.system.name",+ fields: "id,name,monitor_name,value,state,created,resolved,expand.system.name",}// Initial loadpb.collection<AlertsHistoryRecord>("alerts_history")@@ -199,7 +199,7 @@ export default function AlertsHistoryDataTable() {if (!selectedRows.length) returnconst cells: Record<string, (record: AlertsHistoryRecord) => string> = {system: (record) => record.expand?.system?.name || record.system,- name: (record) => alertInfo[record.name]?.name() || record.name,+ name: (record) => [alertInfo[record.name]?.name() || record.name, record.monitor_name].filter(Boolean).join(": "),value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""),state: (record) => (record.resolved ? t`Resolved` : t`Active`),created: (record) => formatShortDate(record.created),diff --git a/internal/site/src/components/routes/system.tsx b/internal/site/src/components/routes/system.tsxindex f43ce9562..1163e81b3 100644--- a/internal/site/src/components/routes/system.tsx+++ b/internal/site/src/components/routes/system.tsx@@ -1,6 +1,6 @@import { memo, useState } from "react"import { Trans } from "@lingui/react/macro"-import { compareSemVer, parseSemVer } from "@/lib/utils"+import { compareSemVer, parseSemVer, supportsNetworkMonitors } from "@/lib/utils"import type { GPUData } from "@/types"import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"import InfoBar from "./system/info-bar"@@ -12,9 +12,15 @@ import { ZfsCharts } from "./system/charts/storage-pool-charts"import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts"-import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"+import {+ LazyContainersTable,+ LazyNetworkMonitorsTable,+ LazySmartTable,+ LazySystemdTable,+ LazyZfsTable,+} from "./system/lazy-tables"import { LoadAverageChart } from "./system/charts/load-average-chart"-import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"+import { ContainerIcon, CpuIcon, HardDriveIcon, NetworkIcon, TerminalSquareIcon } from "lucide-react"import { GpuIcon } from "../ui/icons"import SystemdTable from "../systemd-table/systemd-table"import ContainersTable from "../containers-table/containers-table"@@ -65,9 +71,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {const hasSystemd = system.info.svconst hasGpu = hasGpuData || hasGpuPowerDataconst hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0+ const hasNetworkMonitors = supportsNetworkMonitors(system)// keep tabsRef in sync for keyboard navigation- const tabs = ["core", "disk"]+ const tabs = ["core", "network", "disk"]if (hasGpu) tabs.push("gpu")if (hasContainers) tabs.push("containers")if (hasSystemd) tabs.push("services")@@ -153,6 +160,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {{hasContainersTable && <LazyContainersTable systemId={system.id} />}{hasSystemd && <LazySystemdTable systemId={system.id} />}++ {hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}</>)}@@ -165,6 +174,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {<CpuIcon className="size-3.5" /><Trans context="Core system metrics">Core</Trans></TabsTrigger>+ <TabsTrigger value="network" className="w-full flex items-center gap-1.5">+ <NetworkIcon className="size-3.5" />+ <Trans>Network</Trans>+ </TabsTrigger><TabsTrigger value="disk" className="w-full flex items-center gap-1.5"><HardDriveIcon className="size-3.5" /><Trans>Disk</Trans>@@ -192,9 +205,9 @@ export default memo(function SystemDetail({ id }: { id: string }) {<TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}><div className="grid xl:grid-cols-2 gap-4"><CpuChart {...coreProps} />- <MemoryChart {...coreProps} /><LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} />- <BandwidthChart {...coreProps} systemStats={systemStats} />+ <MemoryChart {...coreProps} />+ <SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} /><TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} /><FanChart {...coreProps} /><BatteryChart system={system} {...coreProps} />@@ -203,6 +216,17 @@ export default memo(function SystemDetail({ id }: { id: string }) {</div></TabsContent>+ <TabsContent value="network" forceMount className={activeTab === "network" ? "contents" : "hidden"}>+ {mountedTabs.has("network") && (+ <>+ <div className="grid xl:grid-cols-2 gap-4">+ <BandwidthChart {...coreProps} systemStats={systemStats} />+ </div>+ {hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}+ </>+ )}+ </TabsContent>+<TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}>{mountedTabs.has("disk") && (<>diff --git a/internal/site/src/components/routes/system/chart-card.tsx b/internal/site/src/components/routes/system/chart-card.tsxindex 1ad93facf..c3e12447e 100644--- a/internal/site/src/components/routes/system/chart-card.tsx+++ b/internal/site/src/components/routes/system/chart-card.tsx@@ -44,6 +44,7 @@ export function FilterBar({ store = $containerFilter }: { store?: typeof $contai<><Inputplaceholder={t`Filter...`}+ title={t`Use commas to match any of multiple terms, e.g. "system1, system2"`}className="ps-4 pe-8 w-full sm:w-44"onChange={handleChange}value={inputValue}diff --git a/internal/site/src/components/routes/system/chart-data.ts b/internal/site/src/components/routes/system/chart-data.tsindex 8769caafe..8535a054e 100644--- a/internal/site/src/components/routes/system/chart-data.ts+++ b/internal/site/src/components/routes/system/chart-data.ts@@ -1,7 +1,13 @@-import { timeTicks } from "d3-time"import { getPbTimestamp, pb } from "@/lib/api"import { chartTimeData } from "@/lib/utils"-import type { ChartData, ChartTimes, ContainerStatsRecord, SystemStatsRecord } from "@/types"+import type {+ ChartData,+ ChartDataContainer,+ ChartTimes,+ ContainerStatsRecord,+ NetworkMonitorStatsRecord,+ SystemStatsRecord,+} from "@/types"type ChartTimeData = {time: number@@ -17,31 +23,10 @@ export const cache = new Map<ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"]>()-// create ticks and domain for charts-export function getTimeData(chartTime: ChartTimes, lastCreated: number) {- const cached = cache.get("td") as ChartTimeData | undefined- if (cached && cached.chartTime === chartTime) {- if (!lastCreated || cached.time >= lastCreated) {- return cached.data- }- }-- // const buffer = chartTime === "1m" ? 400 : 20_000- const now = new Date(Date.now())- const startTime = chartTimeData[chartTime].getOffset(now)- const ticks = timeTicks(startTime, now, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())- const data = {- ticks,- domain: [chartTimeData[chartTime].getOffset(now).getTime(), now.getTime()],- }- cache.set("td", { time: now.getTime(), data, chartTime })- return data-}-/** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place.* Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */export function appendData<T extends { created: string | number | null }>(- prev: T[],+ prev: T[] = [],newRecords: T[],expectedInterval: number,maxLen?: number@@ -66,17 +51,18 @@ export function appendData<T extends { created: string | number | null }>(return result}-export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord>(+export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord | NetworkMonitorStatsRecord>(collection: string,systemId: string,- chartTime: ChartTimes+ chartTime: ChartTimes,+ cachedStats?: { created: string | number | null }[],+ createdIsNumber?: boolean): Promise<T[]> {- const cachedStats = cache.get(`${systemId}_${chartTime}_${collection}`) as T[] | undefinedconst lastCached = cachedStats?.at(-1)?.created as numberreturn await pb.collection<T>(collection).getFullList({filter: pb.filter("system={:id} && created > {:created} && type={:type}", {id: systemId,- created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined),+ created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, createdIsNumber),type: chartTimeData[chartTime].type,}),fields: "created,stats",@@ -84,11 +70,11 @@ export async function getStats<T extends SystemStatsRecord | ContainerStatsRecor})}-export function makeContainerData(containers: ContainerStatsRecord[]): ChartData["containerData"] {- const result = [] as ChartData["containerData"]+export function makeContainerData(containers: ContainerStatsRecord[]): ChartDataContainer[] {+ const result = [] as ChartDataContainer[]for (const { created, stats } of containers) {if (!created) {- result.push({ created: null } as ChartData["containerData"][0])+ result.push({ created: null } as ChartDataContainer)continue}result.push(makeContainerPoint(new Date(created).getTime(), stats))@@ -97,11 +83,8 @@ export function makeContainerData(containers: ContainerStatsRecord[]): ChartData}/** Transform a single realtime container stats message into a ChartDataContainer point. */-export function makeContainerPoint(- created: number,- stats: ContainerStatsRecord["stats"]-): ChartData["containerData"][0] {- const point: ChartData["containerData"][0] = { created } as ChartData["containerData"][0]+export function makeContainerPoint(created: number, stats: ContainerStatsRecord["stats"]): ChartDataContainer {+ const point: ChartDataContainer = { created } as ChartDataContainerfor (const container of stats) {;(point as Record<string, unknown>)[container.n] = container}diff --git a/internal/site/src/components/routes/system/charts/monitors-charts.tsx b/internal/site/src/components/routes/system/charts/monitors-charts.tsxnew file mode 100644index 000000000..d5ac790d4--- /dev/null+++ b/internal/site/src/components/routes/system/charts/monitors-charts.tsx@@ -0,0 +1,212 @@+import { getMonitorTarget } from "@/lib/network-monitor-utils"+import LineChartDefault from "@/components/charts/line-chart"+import type { DataPoint } from "@/components/charts/line-chart"+import { decimalString, formatMicroseconds, matchesFilterGroups, parseFilterGroups, toFixedFloat } from "@/lib/utils"+import { $monitorFilter } from "@/lib/stores"+import { useLingui } from "@lingui/react/macro"+import { ChartCard, FilterBar } from "../chart-card"+import type { ChartData, MonitorStats, NetworkMonitorRecord, NetworkMonitorStatsRecord } from "@/types"+import { useMemo } from "react"+import { useStore } from "@nanostores/react"++type MonitorChartProps = {+ monitorStats: NetworkMonitorStatsRecord[]+ grid?: boolean+ monitors: NetworkMonitorRecord[]+ chartData: ChartData+ empty: boolean+ showFilter?: boolean+ /** Prepended to the chart title, e.g. a target/system name (rendered as "{titlePrefix} — Response"). */+ titlePrefix?: string+}++type MonitorChartBaseProps = MonitorChartProps & {+ metric: keyof MonitorStats+ title: string+ description: string+ tickFormatter: (value: number) => string+ contentFormatter: ({ value }: { value: number | string }) => string | number+ domain?: [number | "auto", number | "auto"]+}++function MonitorChart({+ monitorStats,+ grid,+ monitors,+ chartData,+ empty,+ metric,+ title,+ description,+ tickFormatter,+ contentFormatter,+ domain,+ showFilter = monitors.length > 1,+}: MonitorChartBaseProps) {+ const storedFilter = useStore($monitorFilter)+ const filter = showFilter ? storedFilter : ""++ const { dataPoints, visibleKeys } = useMemo(() => {+ const sortedMonitors = [...monitors].sort((a, b) => b.resAvg1h - a.resAvg1h)+ const count = sortedMonitors.length+ const points: DataPoint<NetworkMonitorStatsRecord>[] = []+ const visibleIDs: string[] = []+ const filterGroups = parseFilterGroups(filter)+ const dot = chartData.chartTime === "1m"+ for (let i = 0; i < count; i++) {+ const p = sortedMonitors[i]+ const label = getMonitorTarget(p)+ const labelLower = label.toLowerCase()+ const filtered = filterGroups.length > 0 && !matchesFilterGroups(labelLower, filterGroups)+ if (filtered) {+ continue+ }+ visibleIDs.push(p.id)+ points.push({+ order: i,+ label,+ dataKey: (record: NetworkMonitorStatsRecord) => record.stats?.[p.id]?.[metric] ?? null,+ dot,+ color: count <= 5 ? i + 1 : `hsl(${(i * 360) / count}, var(--chart-saturation), var(--chart-lightness))`,+ })+ }+ return { dataPoints: points, visibleKeys: visibleIDs }+ }, [monitors, filter, metric, chartData.chartTime])++ const filteredMonitorStats = useMemo(() => {+ if (!visibleKeys.length) return monitorStats+ return monitorStats.filter((record) => visibleKeys.some((id) => record.stats?.[id] != null))+ }, [monitorStats, visibleKeys])++ const legend = dataPoints.length < 10 && showFilter++ return (+ <ChartCard+ legend={legend || !showFilter}+ cornerEl={showFilter ? <FilterBar store={$monitorFilter} /> : undefined}+ empty={empty}+ title={title}+ description={description}+ grid={grid}+ >+ <LineChartDefault+ truncate+ chartData={chartData}+ customData={filteredMonitorStats}+ dataPoints={dataPoints}+ domain={domain ?? ["auto", "auto"]}+ connectNulls+ tickFormatter={tickFormatter}+ contentFormatter={contentFormatter}+ legend={legend}+ filter={filter}+ />+ </ChartCard>+ )+}++interface AvgMinMaxResponseChartProps {+ monitorStats: NetworkMonitorStatsRecord[]+ monitor: NetworkMonitorRecord | null+ chartData: ChartData+ empty: boolean+}++export function AvgMinMaxResponseChart({ monitorStats, monitor, chartData, empty }: AvgMinMaxResponseChartProps) {+ const { t } = useLingui()++ const { chartTime } = chartData+ const hasLongInterval = (monitor?.interval ?? 61) > 60++ // only one monitor is relevant for this chart+ const dataPoints: DataPoint<NetworkMonitorStatsRecord>[] = useMemo(() => {+ const dataFn = (metric: keyof MonitorStats) => (record: NetworkMonitorStatsRecord) =>+ record.stats?.[monitor?.id ?? ""]?.[metric] ?? "-"+ const avgPoint = {+ label: "Avg",+ dataKey: dataFn("res_avg"),+ color: 1,+ order: 0,+ }+ if (chartTime === "1m" || (hasLongInterval && chartTime === "1h")) {+ // avg, min, max are all the same for 1m interval, so just show avg+ return [avgPoint]+ }+ return [+ {+ label: "Max",+ dataKey: dataFn("res_max"),+ color: 3,+ order: 0,+ },+ avgPoint,+ {+ label: "Min",+ dataKey: dataFn("res_min"),+ color: 2,+ order: 2,+ },+ ]+ }, [chartTime, hasLongInterval, monitor?.id])++ const data = useMemo(() => {+ if (!monitor) return []+ return monitorStats.filter((record) => record.stats && monitor.id in record.stats)+ }, [monitor, monitorStats])++ const legend = dataPoints.length > 1++ return (+ <ChartCard+ legend={true}+ empty={empty}+ title={t`Response`}+ description={t`Average, minimum, and maximum response time`}+ grid={false}+ >+ <LineChartDefault+ truncate+ chartData={chartData}+ customData={data}+ dataPoints={dataPoints}+ domain={["auto", "auto"]}+ connectNulls+ legend={legend}+ tickFormatter={(value) => formatMicroseconds(value, false)}+ contentFormatter={({ value }) => {+ if (typeof value !== "number") {+ return value+ }+ return formatMicroseconds(value)+ }}+ />+ </ChartCard>+ )+}++export function LossChart({ monitorStats, grid, monitors, chartData, empty, titlePrefix }: MonitorChartProps) {+ const { t } = useLingui()+ const lossTitle = t`Loss`+ const title = titlePrefix ? `${titlePrefix} — ${lossTitle}` : lossTitle++ return (+ <MonitorChart+ monitorStats={monitorStats}+ grid={grid}+ monitors={monitors}+ chartData={chartData}+ empty={empty}+ metric="loss"+ title={title}+ description={t`Packet loss (%)`}+ domain={[0, 100]}+ tickFormatter={(value) => `${toFixedFloat(value, value >= 10 ? 0 : 1)}%`}+ contentFormatter={({ value }) => {+ if (typeof value !== "number") {+ return value+ }+ return `${decimalString(value, 2)}%`+ }}+ />+ )+}diff --git a/internal/site/src/components/routes/system/lazy-tables.tsx b/internal/site/src/components/routes/system/lazy-tables.tsxindex b2e0a9fca..60b9eb5c5 100644--- a/internal/site/src/components/routes/system/lazy-tables.tsx+++ b/internal/site/src/components/routes/system/lazy-tables.tsx@@ -1,6 +1,7 @@import { lazy } from "react"import { useIntersectionObserver } from "@/lib/use-intersection-observer"import { cn } from "@/lib/utils"+import { useNetworkMonitors } from "@/lib/use-network-monitors"const ContainersTable = lazy(() => import("../../containers-table/containers-table"))@@ -45,3 +46,19 @@ export function LazySystemdTable({ systemId }: { systemId: string }) {</div>)}++const NetworkMonitorsTable = lazy(() => import("../../network-monitors-table/network-monitors-table"))++export function LazyNetworkMonitorsTable({ systemId }: { systemId: string }) {+ const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })+ return (+ <div ref={ref} className={cn(isIntersecting && "contents")}>+ {isIntersecting && <SystemNetworkMonitorsTable systemId={systemId} />}+ </div>+ )+}++function SystemNetworkMonitorsTable({ systemId }: { systemId: string }) {+ const { monitors, isLoading } = useNetworkMonitors({ systemId })+ return <NetworkMonitorsTable systemId={systemId} monitors={monitors} isLoading={isLoading} />+}diff --git a/internal/site/src/components/routes/system/smart-table.tsx b/internal/site/src/components/routes/system/smart-table.tsxindex 2efeb7910..9bc047d3f 100644--- a/internal/site/src/components/routes/system/smart-table.tsx+++ b/internal/site/src/components/routes/system/smart-table.tsx@@ -43,7 +43,6 @@ import {toFixedFloat,formatTemperature,cn,- getVisualStringWidth,secondsToString,hourWithSeconds,formatShortDate,@@ -117,9 +116,9 @@ function formatDataUnits(units: number): string {const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated"export const createColumns = (- longestName: number,- longestModel: number,- longestDevice: number+ longestName: string,+ longestModel: string,+ longestDevice: string): ColumnDef<SmartDeviceRecord>[] => [{id: "system",@@ -134,8 +133,11 @@ export const createColumns = (cell: ({ getValue }) => {const allSystems = useStore($allSystemsById)return (- <div className="ms-1.5 max-w-40 block truncate" style={{ width: `${longestName / 1.05}ch` }}>- {allSystems[getValue() as string]?.name ?? ""}+ <div className="ms-1.5 relative w-fit max-w-44">+ <span className="invisible block whitespace-nowrap" aria-hidden="true">+ {longestName}+ </span>+ <span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span></div>)},@@ -145,12 +147,11 @@ export const createColumns = (sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />,cell: ({ getValue }) => (- <div- className="font-medium max-w-40 truncate ms-1"- title={getValue() as string}- style={{ width: `${longestDevice / 1.05}ch` }}- >- {getValue() as string}+ <div className="font-medium ms-1 relative w-fit max-w-44" title={getValue() as string}>+ <span className="invisible block whitespace-nowrap" aria-hidden="true">+ {longestDevice}+ </span>+ <span className="absolute inset-0 truncate">{getValue() as string}</span></div>),},@@ -161,12 +162,11 @@ export const createColumns = (<HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} />),cell: ({ getValue }) => (- <div- className="max-w-48 truncate ms-1"- title={getValue() as string}- style={{ width: `${longestModel / 1.05}ch` }}- >- {getValue() as string}+ <div className="ms-1 relative w-fit max-w-44" title={getValue() as string}>+ <span className="invisible block whitespace-nowrap" aria-hidden="true">+ {longestModel}+ </span>+ <span className="absolute inset-0 truncate">{getValue() as string}</span></div>),},@@ -320,7 +320,7 @@ export default function DisksTable({ systemId }: { systemId?: string }) {// Calculate the right width for the columns based on the longest strings among the displayed devicesconst { longestName, longestModel, longestDevice } = useMemo(() => {- const result = { longestName: 0, longestModel: 0, longestDevice: 0 }+ const result = { longestName: "", longestModel: "", longestDevice: "" }if (!smartDevices || Object.keys(allSystems).length === 0) {return result}@@ -329,10 +329,16 @@ export default function DisksTable({ systemId }: { systemId?: string }) {if (!systemId && !seenSystems.has(device.system)) {seenSystems.add(device.system)const name = allSystems[device.system]?.name ?? ""- result.longestName = Math.max(result.longestName, getVisualStringWidth(name))+ if (name.length > result.longestName.length) {+ result.longestName = name+ }+ }+ if ((device.model ?? "").length > result.longestModel.length) {+ result.longestModel = device.model ?? ""+ }+ if ((device.name ?? "").length > result.longestDevice.length) {+ result.longestDevice = device.name ?? ""}- result.longestModel = Math.max(result.longestModel, getVisualStringWidth(device.model ?? ""))- result.longestDevice = Math.max(result.longestDevice, getVisualStringWidth(device.name ?? ""))}return result}, [smartDevices, systemId, allSystems])diff --git a/internal/site/src/components/routes/system/use-system-data.ts b/internal/site/src/components/routes/system/use-system-data.tsindex 0a5dc15f1..bd2f2af48 100644--- a/internal/site/src/components/routes/system/use-system-data.ts+++ b/internal/site/src/components/routes/system/use-system-data.ts@@ -26,7 +26,7 @@ import type {SystemStatsRecord,} from "@/types"import { $router, navigate } from "../../router"-import { appendData, cache, getStats, getTimeData, makeContainerData, makeContainerPoint } from "./chart-data"+import { appendData, cache, getStats, makeContainerData, makeContainerPoint } from "./chart-data"export type SystemData = ReturnType<typeof useSystemData>@@ -185,16 +185,11 @@ export function useSystemData(id: string) {const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v])const chartData: ChartData = useMemo(() => {- const lastCreated = Math.max(- (systemStats.at(-1)?.created as number) ?? 0,- (containerData.at(-1)?.created as number) ?? 0- )return {systemStats,containerData,chartTime,orientation: direction === "rtl" ? "right" : "left",- ...getTimeData(chartTime, lastCreated),agentVersion,}}, [systemStats, containerData, direction])@@ -234,8 +229,8 @@ export function useSystemData(id: string) {}Promise.allSettled([- getStats<SystemStatsRecord>("system_stats", systemId, chartTime),- getStats<ContainerStatsRecord>("container_stats", systemId, chartTime),+ getStats<SystemStatsRecord>("system_stats", systemId, chartTime, cachedSystemStats),+ getStats<ContainerStatsRecord>("container_stats", systemId, chartTime, cachedContainerData),]).then(([systemStats, containerStats]) => {// Ignore responses for a previous system or chart timeif (requestId !== statsRequestId.current) {@@ -327,7 +322,7 @@ export function useSystemData(id: string) {// derived valuesconst isLongerChart = !["1m", "1h"].includes(chartTime)const showMax = maxValues && isLongerChart- const dataEmpty = !chartLoading && chartData.systemStats.length === 0+ const dataEmpty = !chartLoading && chartData.systemStats?.length === 0const lastGpus = systemStats.at(-1)?.stats?.gconst isPodman = details?.podman ?? system.info?.p ?? falsediff --git a/internal/site/src/components/systems-table/systems-table-columns.tsx b/internal/site/src/components/systems-table/systems-table-columns.tsxindex 40d6c7d52..1db27673d 100644--- a/internal/site/src/components/systems-table/systems-table-columns.tsx+++ b/internal/site/src/components/systems-table/systems-table-columns.tsx@@ -26,7 +26,7 @@ import { memo, useMemo, useRef, useState } from "react"import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"import { isReadOnlyUser, pb } from "@/lib/api"import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums"-import { $longestSystemNameLen, $userSettings } from "@/lib/stores"+import { $longestSystemName, $userSettings } from "@/lib/stores"import {cn,copyToClipboard,@@ -135,7 +135,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<SysteIcon: ServerIcon,cell: (info) => {const { name, id } = info.row.original- const longestName = useStore($longestSystemNameLen)+ const longestName = useStore($longestSystemName)const linkUrl = getPagePath($router, "system", { id })return (@@ -145,8 +145,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste<Linkhref={linkUrl}tabIndex={-1}- className="truncate z-10 relative"- style={{ width: `${longestName / 1.05}ch` }}+ className="relative w-fit max-w-48 z-10"onMouseEnter={(e) => {// set title on hover if text is truncated to show full nameconst a = e.currentTarget@@ -157,7 +156,10 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste}}}>- {name}+ <span className="invisible block" aria-hidden="true">+ {longestName}+ </span>+ <span className="absolute inset-0 truncate">{name}</span></Link></span><Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link>diff --git a/internal/site/src/components/ui/chart.tsx b/internal/site/src/components/ui/chart.tsxindex 5fc4e2fee..35c606ce9 100644--- a/internal/site/src/components/ui/chart.tsx+++ b/internal/site/src/components/ui/chart.tsx@@ -1,11 +1,11 @@-import type { JSX } from "react"import { useLingui } from "@lingui/react/macro"import * as React from "react"import * as RechartsPrimitive from "recharts"import { chartTimeData, cn } from "@/lib/utils"-import type { ChartData } from "@/types"+import type { ChartTimes } from "@/types"import { Separator } from "./separator"-import { AxisDomain } from "recharts/types/util/types"+import type { AxisDomain } from "recharts/types/util/types"+import { timeTicks } from "d3-time"// Format: { THEME_NAME: CSS_SELECTOR }const THEMES = { light: "", dark: ".dark" } as const@@ -101,7 +101,7 @@ const ChartTooltipContent = React.forwardRef<labelKey?: stringunit?: stringfilter?: string- contentFormatter?: (item: any, key: string) => React.ReactNode | string+ contentFormatter?: (item: unknown, key: string) => React.ReactNode | stringtruncate?: booleanshowTotal?: booleantotalLabel?: React.ReactNode@@ -175,7 +175,13 @@ const ChartTooltipContent = React.forwardRef<}const totalKey = "__total__"- const totalItem: any = {+ const totalItem: {+ value: number+ name: string+ dataKey: string+ color: string | undefined+ payload?: unknown+ } = {value: totalValue,name: totalName,dataKey: totalKey,@@ -222,6 +228,11 @@ const ChartTooltipContent = React.forwardRef<return null}+ payload = payload.filter((item) => item.value != null)+ if (!payload.length) {+ return null+ }+// const nestLabel = payload.length === 1 && indicator !== 'dot'const nestLabel = false@@ -400,26 +411,57 @@ function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key:return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config]}-let cachedAxis: JSX.Element-const xAxis = ({ domain, ticks, chartTime }: ChartData) => {- if (cachedAxis && domain[0] === cachedAxis.props.domain[0]) {- return cachedAxis+interface XAxisData {+ el: React.ReactElement+ domain: [number, number]+}++const xAxisCache = new Map<ChartTimes, XAxisData>()++function createXAxisData(chartTime: ChartTimes): XAxisData {+ // console.log("Creating XAxis for", chartTime, new Date())+ const axisEndTime = Date.now() + 500+ const axisEndDate = new Date(axisEndTime)+ const startTime = chartTimeData[chartTime].getOffset(axisEndDate)+ const ticks = timeTicks(startTime, axisEndDate, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())+ const domain: [number, number] = [startTime.getTime(), axisEndTime]++ return {+ domain,+ el: (+ <RechartsPrimitive.XAxis+ dataKey="created"+ domain={domain}+ ticks={ticks}+ allowDataOverflow+ type="number"+ scale="time"+ minTickGap={12}+ tickMargin={8}+ axisLine={false}+ tickFormatter={chartTimeData[chartTime].format}+ />+ ),}- cachedAxis = (- <RechartsPrimitive.XAxis- dataKey="created"- domain={domain}- ticks={ticks}- allowDataOverflow- type="number"- scale="time"- minTickGap={12}- tickMargin={8}- axisLine={false}- tickFormatter={chartTimeData[chartTime].format}- />- )- return cachedAxis+}++function xAxis(chartTime: ChartTimes, lastCreated: number) {+ if (!lastCreated) {+ return null+ }+ const cachedAxis = xAxisCache.get(chartTime)++ const expectedInterval = chartTimeData[chartTime].expectedInterval+ const conservativeEndTime = Date.now() - expectedInterval / 2+ const axisEndTime = Math.max(lastCreated, conservativeEndTime)++ if (cachedAxis && axisEndTime < cachedAxis.domain[1]) {+ return cachedAxis.el+ }++ const axisData = createXAxisData(chartTime)+ xAxisCache.set(chartTime, axisData)+ return axisData.el}export {diff --git a/internal/site/src/components/ui/dropdown-menu.tsx b/internal/site/src/components/ui/dropdown-menu.tsxindex b49693aa0..7312bd757 100644--- a/internal/site/src/components/ui/dropdown-menu.tsx+++ b/internal/site/src/components/ui/dropdown-menu.tsx@@ -90,8 +90,10 @@ DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayNameconst DropdownMenuCheckboxItem = React.forwardRef<React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,- React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem>->(({ className, children, checked, ...props }, ref) => (+ React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem> & {+ indicatorClassName?: string+ }+>(({ className, children, checked, indicatorClassName, ...props }, ref) => (<DropdownMenuPrimitive.CheckboxItemref={ref}className={cn(@@ -101,7 +103,7 @@ const DropdownMenuCheckboxItem = React.forwardRef<checked={checked}{...props}>- <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">+ <span className={cn("absolute left-2 flex h-3.5 w-3.5 items-center justify-center", indicatorClassName)}><DropdownMenuPrimitive.ItemIndicator><Check className="h-4 w-4" /></DropdownMenuPrimitive.ItemIndicator>diff --git a/internal/site/src/components/ui/table.tsx b/internal/site/src/components/ui/table.tsxindex a22206c18..377565a36 100644--- a/internal/site/src/components/ui/table.tsx+++ b/internal/site/src/components/ui/table.tsx@@ -41,7 +41,7 @@ const TableRow = React.forwardRef<HTMLTableRowElement, React.HTMLAttributes<HTML<trref={ref}className={cn(- "border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted!",+ "border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted/40!",className)}{...props}diff --git a/internal/site/src/lib/alerts.ts b/internal/site/src/lib/alerts.tsindex b0329af01..b5f52a7dc 100644--- a/internal/site/src/lib/alerts.ts+++ b/internal/site/src/lib/alerts.ts@@ -1,5 +1,5 @@import { t } from "@lingui/core/macro"-import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, ServerCrashIcon, ServerIcon } from "lucide-react"+import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"import type { RecordSubscription } from "pocketbase"import { EthernetIcon, GpuIcon } from "@/components/ui/icons"import { $alerts } from "@/lib/stores"@@ -9,210 +9,223 @@ import { ThermometerIcon, BatteryMediumIcon, HourglassIcon } from "@/components//** Alert info for each alert type */export const alertInfo: Record<string, AlertInfo> = {- Status: {- name: () => t`Status`,- unit: "",- icon: ServerIcon,- desc: () => t`Triggers when status switches between up and down`,- /** "for x minutes" is appended to desc when only one value */- singleDesc: () => `${t`System`} ${t`Down`}`,- },- CPU: {- name: () => t`CPU Usage`,- unit: "%",- icon: CpuIcon,- desc: () => t`Triggers when CPU usage exceeds a threshold`,- },- CPUIOWait: {- name: () => t`CPU I/O Wait`,- unit: "%",- icon: CpuIcon,- desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,- },- CPUSteal: {- name: () => t`CPU Steal Time`,- unit: "%",- icon: CpuIcon,- desc: () => t`Triggers when CPU steal time exceeds a threshold`,- },- Memory: {- name: () => t`Memory Usage`,- unit: "%",- icon: MemoryStickIcon,- desc: () => t`Triggers when memory usage exceeds a threshold`,- },- Disk: {- name: () => t`Disk Usage`,- unit: "%",- icon: HardDriveIcon,- desc: () => t`Triggers when usage of any disk exceeds a threshold`,- },- Bandwidth: {- name: () => t`Bandwidth`,- unit: " MB/s",- icon: EthernetIcon,- desc: () => t`Triggers when combined up/down exceeds a threshold`,- max: 250,- },- GPU: {- name: () => t`GPU Usage`,- unit: "%",- icon: GpuIcon,- desc: () => t`Triggers when GPU usage exceeds a threshold`,- },- Temperature: {- name: () => t`Temperature`,- unit: "°C",- icon: ThermometerIcon,- desc: () => t`Triggers when any sensor exceeds a threshold`,- },- LoadAvg1: {- name: () => t`Load Average 1m`,- unit: "",- icon: HourglassIcon,- max: 100,- min: 0.1,- start: 10,- step: 0.1,- desc: () => t`Triggers when 1 minute load average exceeds a threshold`,- },- LoadAvg5: {- name: () => t`Load Average 5m`,- unit: "",- icon: HourglassIcon,- max: 100,- min: 0.1,- start: 10,- step: 0.1,- desc: () => t`Triggers when 5 minute load average exceeds a threshold`,- },- LoadAvg15: {- name: () => t`Load Average 15m`,- unit: "",- icon: HourglassIcon,- min: 0.1,- max: 100,- start: 10,- step: 0.1,- desc: () => t`Triggers when 15 minute load average exceeds a threshold`,- },- Battery: {- name: () => t`Battery`,- unit: "%",- icon: BatteryMediumIcon,- desc: () => t`Triggers when battery charge drops below a threshold`,- start: 20,- invert: true,- },- ContainerHealth: {- name: () => t`Container Health`,- unit: "",- icon: ContainerIcon,- desc: () => t`Triggers when a container's health check reports unhealthy`,- note: () =>- t`Notifications may include recent container log excerpts.`,- triggeredDesc: () => t`One or more containers are unhealthy`,- singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,+ Status: {+ name: () => t`Status`,+ unit: "",+ icon: ServerIcon,+ desc: () => t`Triggers when status switches between up and down`,+ /** "for x minutes" is appended to desc when only one value */+ singleDesc: () => `${t`System`} ${t`Down`}`,+ },+ CPU: {+ name: () => t`CPU Usage`,+ unit: "%",+ icon: CpuIcon,+ desc: () => t`Triggers when CPU usage exceeds a threshold`,+ },+ CPUIOWait: {+ name: () => t`CPU I/O Wait`,+ unit: "%",+ icon: CpuIcon,+ desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,+ },+ CPUSteal: {+ name: () => t`CPU Steal Time`,+ unit: "%",+ icon: CpuIcon,+ desc: () => t`Triggers when CPU steal time exceeds a threshold`,+ },+ Memory: {+ name: () => t`Memory Usage`,+ unit: "%",+ icon: MemoryStickIcon,+ desc: () => t`Triggers when memory usage exceeds a threshold`,+ },+ Disk: {+ name: () => t`Disk Usage`,+ unit: "%",+ icon: HardDriveIcon,+ desc: () => t`Triggers when usage of any disk exceeds a threshold`,+ },+ Bandwidth: {+ name: () => t`Bandwidth`,+ unit: " MB/s",+ icon: EthernetIcon,+ desc: () => t`Triggers when combined up/down exceeds a threshold`,+ max: 250,+ },+ NetworkMonitorLoss: {+ name: () => t`Network Monitor Loss`,+ unit: "%",+ icon: NetworkIcon,+ desc: () => t`Triggers when one hour loss exceeds a threshold`,+ // note: () => t`Uses available history after three probes.`,+ noDuration: true,+ min: 0,+ max: 99.9,+ step: 0.1,+ start: 5,+ },+ GPU: {+ name: () => t`GPU Usage`,+ unit: "%",+ icon: GpuIcon,+ desc: () => t`Triggers when GPU usage exceeds a threshold`,+ },+ Temperature: {+ name: () => t`Temperature`,+ unit: "°C",+ icon: ThermometerIcon,+ desc: () => t`Triggers when any sensor exceeds a threshold`,+ },+ LoadAvg1: {+ name: () => t`Load Average 1m`,+ unit: "",+ icon: HourglassIcon,+ max: 100,+ min: 0.1,+ start: 10,+ step: 0.1,+ desc: () => t`Triggers when 1 minute load average exceeds a threshold`,+ },+ LoadAvg5: {+ name: () => t`Load Average 5m`,+ unit: "",+ icon: HourglassIcon,+ max: 100,+ min: 0.1,+ start: 10,+ step: 0.1,+ desc: () => t`Triggers when 5 minute load average exceeds a threshold`,+ },+ LoadAvg15: {+ name: () => t`Load Average 15m`,+ unit: "",+ icon: HourglassIcon,+ min: 0.1,+ max: 100,+ start: 10,+ step: 0.1,+ desc: () => t`Triggers when 15 minute load average exceeds a threshold`,+ },+ Battery: {+ name: () => t`Battery`,+ unit: "%",+ icon: BatteryMediumIcon,+ desc: () => t`Triggers when battery charge drops below a threshold`,+ start: 20,+ invert: true,+ },+ ContainerHealth: {+ name: () => t`Container Health`,+ unit: "",+ icon: ContainerIcon,+ desc: () => t`Triggers when a container's health check reports unhealthy`,+ note: () =>+ t`Notifications may include recent container log excerpts.`,+ triggeredDesc: () => t`One or more containers are unhealthy`,+ singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,- },- SystemdFailed: {- name: () => t`Failed Services`,- unit: "",- icon: ServerCrashIcon,- desc: () => t`Triggers when any systemd service enters the failed state`,- triggeredDesc: () => t`One or more services are in a failed state`,- /** Fires on first observation - the agent only polls systemd every 10 minutes */- noDuration: true,- },+ },+ SystemdFailed: {+ name: () => t`Failed Services`,+ unit: "",+ icon: ServerCrashIcon,+ desc: () => t`Triggers when any systemd service enters the failed state`,+ triggeredDesc: () => t`One or more services are in a failed state`,+ /** Fires on first observation - the agent only polls systemd every 10 minutes */+ noDuration: true,+ noThreshold: true,+ },} as const/** Helper to manage user alerts */export const alertManager = (() => {- const collection = pb.collection<AlertRecord>("alerts")- let unsub: () => void+ const collection = pb.collection<AlertRecord>("alerts")+ let unsub: () => void- /** Fields to fetch from alerts collection */- const fields = "id,name,system,value,min,triggered"+ /** Fields to fetch from alerts collection */+ const fields = "id,name,system,value,min,triggered"- /** Fetch alerts from collection */- async function fetchAlerts(): Promise<AlertRecord[]> {- return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })- }+ /** Fetch alerts from collection */+ async function fetchAlerts(): Promise<AlertRecord[]> {+ return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })+ }- /** Format alerts into a map of system id to alert name to alert record */- function add(alerts: AlertRecord[]) {- for (const alert of alerts) {- const systemId = alert.system- const systemAlerts = $alerts.get()[systemId] ?? new Map()- const newAlerts = new Map(systemAlerts)- newAlerts.set(alert.name, alert)- $alerts.setKey(systemId, newAlerts)- }- }+ /** Format alerts into a map of system id to alert name to alert record */+ function add(alerts: AlertRecord[]) {+ for (const alert of alerts) {+ const systemId = alert.system+ const systemAlerts = $alerts.get()[systemId] ?? new Map()+ const newAlerts = new Map(systemAlerts)+ newAlerts.set(alert.name, alert)+ $alerts.setKey(systemId, newAlerts)+ }+ }- function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {- for (const alert of alerts) {- const systemId = alert.system- const systemAlerts = $alerts.get()[systemId]- const newAlerts = new Map(systemAlerts)- newAlerts.delete(alert.name)- $alerts.setKey(systemId, newAlerts)- }- }+ function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {+ for (const alert of alerts) {+ const systemId = alert.system+ const systemAlerts = $alerts.get()[systemId]+ const newAlerts = new Map(systemAlerts)+ newAlerts.delete(alert.name)+ $alerts.setKey(systemId, newAlerts)+ }+ }- const actionFns = {- create: add,- update: add,- delete: remove,- }+ const actionFns = {+ create: add,+ update: add,+ delete: remove,+ }- // batch alert updates to prevent unnecessary re-renders when adding many alerts at once- const batchUpdate = (() => {- const batch = new Map<string, RecordSubscription<AlertRecord>>()- let timeout: ReturnType<typeof setTimeout>+ // batch alert updates to prevent unnecessary re-renders when adding many alerts at once+ const batchUpdate = (() => {+ const batch = new Map<string, RecordSubscription<AlertRecord>>()+ let timeout: ReturnType<typeof setTimeout>- return (data: RecordSubscription<AlertRecord>) => {- const { record } = data- batch.set(`${record.system}${record.name}`, data)- clearTimeout(timeout)- timeout = setTimeout(() => {- const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>- for (const { action, record } of batch.values()) {- groups[action]?.push(record)- }- for (const key in groups) {- if (groups[key].length) {- actionFns[key as keyof typeof actionFns]?.(groups[key])- }- }- batch.clear()- }, 50)- }- })()+ return (data: RecordSubscription<AlertRecord>) => {+ const { record } = data+ batch.set(`${record.system}${record.name}`, data)+ clearTimeout(timeout)+ timeout = setTimeout(() => {+ const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>+ for (const { action, record } of batch.values()) {+ groups[action]?.push(record)+ }+ for (const key in groups) {+ if (groups[key].length) {+ actionFns[key as keyof typeof actionFns]?.(groups[key])+ }+ }+ batch.clear()+ }, 50)+ }+ })()- async function subscribe() {- unsub = await collection.subscribe("*", batchUpdate, { fields })- }+ async function subscribe() {+ unsub = await collection.subscribe("*", batchUpdate, { fields })+ }- function unsubscribe() {- unsub?.()- }+ function unsubscribe() {+ unsub?.()+ }- async function refresh() {- const records = await fetchAlerts()- add(records)- }+ async function refresh() {+ const records = await fetchAlerts()+ add(records)+ }- return {- /** Add alerts to store */- add,- /** Remove alerts from store */- remove,- /** Subscribe to alerts */- subscribe,- /** Unsubscribe from alerts */- unsubscribe,- /** Refresh alerts with latest data from hub */- refresh,- }+ return {+ /** Add alerts to store */+ add,+ /** Remove alerts from store */+ remove,+ /** Subscribe to alerts */+ subscribe,+ /** Unsubscribe from alerts */+ unsubscribe,+ /** Refresh alerts with latest data from hub */+ refresh,+ }})()diff --git a/internal/site/src/lib/api.ts b/internal/site/src/lib/api.tsindex cfa44d98e..fdcf582e0 100644--- a/internal/site/src/lib/api.ts+++ b/internal/site/src/lib/api.ts@@ -106,8 +106,11 @@ export async function updateUserSettings() {}}-export function getPbTimestamp(timeString: ChartTimes, d?: Date) {+export function getPbTimestamp(timeString: ChartTimes, d?: Date, createdIsNumber?: boolean) {d ||= chartTimeData[timeString].getOffset(new Date())+ if (createdIsNumber) {+ return d.getTime()+ }const year = d.getUTCFullYear()const month = String(d.getUTCMonth() + 1).padStart(2, "0")const day = String(d.getUTCDate()).padStart(2, "0")diff --git a/internal/site/src/lib/network-monitor-utils.ts b/internal/site/src/lib/network-monitor-utils.tsnew file mode 100644index 000000000..9ab1303d7--- /dev/null+++ b/internal/site/src/lib/network-monitor-utils.ts@@ -0,0 +1,17 @@+import type { MonitorStats, NetworkMonitorRecord, RawMonitorStatsRecord } from "@/types"++/** Derive chart metrics from the counts and response sum stored at every retention tier. */+export function getMonitorStats(record: RawMonitorStatsRecord): MonitorStats {+ return {+ res_avg: record.success_count > 0 ? record.res_sum / record.success_count : 0,+ res_min: record.res_min,+ res_max: record.res_max,+ loss: record.total_count > 0 ? ((record.total_count - record.success_count) / record.total_count) * 100 : 0,+ }+}++export function getMonitorTarget(monitor: Pick<NetworkMonitorRecord, "target" | "protocol" | "port">) {+ if (monitor.protocol !== "tcp") return monitor.target+ const host = monitor.target.includes(":") && !monitor.target.startsWith("[") ? `[${monitor.target}]` : monitor.target+ return `${host}:${monitor.port}`+}diff --git a/internal/site/src/lib/stores.ts b/internal/site/src/lib/stores.tsindex 0778c1a3d..49184b8fd 100644--- a/internal/site/src/lib/stores.ts+++ b/internal/site/src/lib/stores.ts@@ -64,6 +64,9 @@ export const $containerFilter = atom("")/** Temperature chart filter */export const $temperatureFilter = atom("")+/** Filter for network monitor charts (compare page and per-system monitor charts) */+export const $monitorFilter = atom("")+/** Fan-speed chart filter */export const $fanFilter = atom("")@@ -73,7 +76,5 @@ export const $copyContent = atom("")/** Direction for localization */export const $direction = atom<"ltr" | "rtl">("ltr")-/** Longest system name length. Used to set table column width. I know this- * is stupid but the table is virtualized and I know this will work.- */-export const $longestSystemNameLen = atom(8)+/** Longest system name string. Used to reserve width in virtualized tables. */+export const $longestSystemName = atom("")diff --git a/internal/site/src/lib/systemsManager.ts b/internal/site/src/lib/systemsManager.tsindex f3c87fbfc..287d58eef 100644--- a/internal/site/src/lib/systemsManager.ts+++ b/internal/site/src/lib/systemsManager.ts@@ -5,20 +5,17 @@ import {$allSystemsById,$allSystemsByName,$downSystems,- $longestSystemNameLen,+ $longestSystemName,$pausedSystems,$upSystems,} from "@/lib/stores"-import { getVisualStringWidth, updateFavicon } from "@/lib/utils"+import { isVisuallyLonger, updateFavicon } from "@/lib/utils"import type { SystemRecord } from "@/types"import { SystemStatus } from "./enums"const COLLECTION = pb.collection<SystemRecord>("systems")const FIELDS_DEFAULT = "id,name,host,port,info,status"-/** Maximum system name length for display purposes */-const MAX_SYSTEM_NAME_LENGTH = 22-let initialized = false// biome-ignore lint/suspicious/noConfusingVoidType: typescript rockslet unsub: (() => void) | undefined | void@@ -44,7 +41,7 @@ export function init() {}if (!newSystem) {- onSystemsChanged(newSystems, undefined)+ onSystemsChanged(newSystems, newSystem, oldSystem)return}@@ -68,20 +65,28 @@ export function init() {}// run things that need to be done when systems change- onSystemsChanged(newSystems, newSystem)+ onSystemsChanged(newSystems, newSystem, oldSystem)})}-/** Update the longest system name length and favicon based on system status */-function onSystemsChanged(_: Record<string, SystemRecord>, changedSystem: SystemRecord | undefined) {+/** Update the longest system name string and favicon based on system status */+function onSystemsChanged(systems: Record<string, SystemRecord>, newSystem?: SystemRecord, oldSystem?: SystemRecord) {const downSystemsStore = $downSystems.get()const downSystems = Object.values(downSystemsStore)- // Update longest system name length- const longestName = $longestSystemNameLen.get()- const nameLen = Math.min(MAX_SYSTEM_NAME_LENGTH, getVisualStringWidth(changedSystem?.name || ""))- if (nameLen > longestName) {- $longestSystemNameLen.set(nameLen)+ // if the old system's old name was the longest, we need to find the new longest name+ // otherwise, if the changed system's new name is longer than the current longest, update it+ const longestName = $longestSystemName.get()+ if (oldSystem?.name === longestName && oldSystem.name !== newSystem?.name) {+ let newLongest = ""+ for (const id in systems) {+ if (isVisuallyLonger(systems[id].name, newLongest)) {+ newLongest = systems[id].name+ }+ }+ $longestSystemName.set(newLongest)+ } else if (newSystem && newSystem.name !== longestName && isVisuallyLonger(newSystem.name, longestName)) {+ $longestSystemName.set(newSystem.name)}updateFavicon(downSystems.length)diff --git a/internal/site/src/lib/use-network-monitors.ts b/internal/site/src/lib/use-network-monitors.tsnew file mode 100644index 000000000..8b8e5014c--- /dev/null+++ b/internal/site/src/lib/use-network-monitors.ts@@ -0,0 +1,351 @@+import { chartTimeData } from "@/lib/utils"+import { getMonitorStats } from "@/lib/network-monitor-utils"+import type {+ ChartTimes,+ MonitorStats,+ NetworkMonitorRecord,+ NetworkMonitorStatsRecord,+ RawMonitorStatsRecord,+} from "@/types"+import { useEffect, useRef, useState } from "react"+import { appendData } from "@/components/routes/system/chart-data"+import { pb, getPbTimestamp } from "@/lib/api"+import { toast } from "@/components/ui/use-toast"+import type { RecordListOptions, RecordSubscription } from "pocketbase"++const cache = new Map<string, NetworkMonitorStatsRecord[]>()++function getCacheValue(monitorId: string, chartTime: ChartTimes | "rt") {+ return cache.get(`${monitorId}:${chartTime}`) || []+}++function appendCacheValue(+ monitorId: string,+ chartTime: ChartTimes | "rt",+ newStats: NetworkMonitorStatsRecord[],+ maxPoints = 100+) {+ const cache_key = `${monitorId}:${chartTime}`+ const existingStats = getCacheValue(monitorId, chartTime)+ if (existingStats) {+ const { expectedInterval } = chartTimeData[chartTime]+ const updatedStats = appendData(existingStats, newStats, expectedInterval, maxPoints)+ cache.set(cache_key, updatedStats)+ return updatedStats+ } else {+ cache.set(cache_key, newStats)+ return newStats+ }+}++/** Merge an array of per-monitor raw records into the map-keyed format expected by chart components. */+export function mergeMonitorStats(rawRecords: RawMonitorStatsRecord[]): NetworkMonitorStatsRecord[] {+ const byTimestamp = new Map<number, Record<string, MonitorStats>>()+ for (const rec of rawRecords) {+ let statsMap = byTimestamp.get(rec.created)+ if (!statsMap) {+ statsMap = {}+ byTimestamp.set(rec.created, statsMap)+ }+ statsMap[rec.monitor] = getMonitorStats(rec)+ }+ return Array.from(byTimestamp.entries())+ .sort(([a], [b]) => a - b)+ .map(([created, stats]) => ({ created, stats }))+}++/** Fetch stats for one monitor and time range, returning merged chart records. */+async function fetchMonitorStats(+ monitorId: string,+ chartTime: ChartTimes,+ cached?: NetworkMonitorStatsRecord[]+): Promise<NetworkMonitorStatsRecord[]> {+ const lastCached = cached?.at(-1)?.created as number | undefined+ const rawRecords = await pb.collection<RawMonitorStatsRecord>("network_monitor_stats").getFullList({+ filter: pb.filter("monitor={:id} && created>{:created} && type={:type}", {+ id: monitorId,+ created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, true),+ type: chartTimeData[chartTime].type,+ }),+ fields: "monitor,res_min,res_max,total_count,success_count,res_sum,created",+ sort: "created",+ })+ return mergeMonitorStats(rawRecords)+}++const NETWORK_MONITOR_FIELDS =+ "id,system,target,protocol,port,interval,res,resMin1h,resMax1h,resAvg1h,loss1h,enabled,updated"++interface UseNetworkMonitorsProps {+ systemId?: string+}++export function useNetworkMonitors(props: UseNetworkMonitorsProps) {+ const { systemId } = props++ const [monitors, setMonitors] = useState<NetworkMonitorRecord[]>([])+ const [isLoading, setIsLoading] = useState(true)+ const pendingMonitorEvents = useRef(new Map<string, RecordSubscription<NetworkMonitorRecord>>())+ const monitorBatchTimeout = useRef<ReturnType<typeof setTimeout> | null>(null)++ // initial load+ useEffect(() => {+ let cancelled = false+ setIsLoading(true)+ setMonitors([])+ fetchMonitors(systemId).then((monitors) => {+ if (cancelled) return+ setMonitors(monitors)+ setIsLoading(false)+ })+ return () => {+ cancelled = true+ }+ }, [systemId])++ // subscribe to updates+ useEffect(() => {+ let unsubscribe: (() => void) | undefined++ function flushPendingMonitorEvents() {+ monitorBatchTimeout.current = null+ if (!pendingMonitorEvents.current.size) {+ return+ }+ const events = pendingMonitorEvents.current+ pendingMonitorEvents.current = new Map()+ setMonitors((currentMonitors) => {+ return applyMonitorEvents(currentMonitors ?? [], events.values(), systemId)+ })+ }++ const pbOptions: RecordListOptions = { fields: NETWORK_MONITOR_FIELDS }+ if (systemId) {+ pbOptions.filter = pb.filter("system = {:system}", { system: systemId })+ }++ ;(async () => {+ try {+ unsubscribe = await pb.collection<NetworkMonitorRecord>("network_monitors").subscribe(+ "*",+ (event) => {+ pendingMonitorEvents.current.set(event.record.id, event)+ if (!monitorBatchTimeout.current) {+ monitorBatchTimeout.current = setTimeout(flushPendingMonitorEvents, 50)+ }+ },+ pbOptions+ )+ } catch (error) {+ console.error("Failed to subscribe to monitors", error)+ }+ })()++ return () => {+ if (monitorBatchTimeout.current !== null) {+ clearTimeout(monitorBatchTimeout.current)+ monitorBatchTimeout.current = null+ }+ pendingMonitorEvents.current.clear()+ unsubscribe?.()+ }+ }, [systemId])++ return { monitors, isLoading }+}++interface UseNetworkMonitorStatsProps {+ systemId: string+ monitorId: string+ chartTime: ChartTimes+ enabled?: boolean+}++export function useNetworkMonitorStats(props: UseNetworkMonitorStatsProps) {+ const { systemId, monitorId, chartTime, enabled = true } = props+ const [monitorStats, setMonitorStats] = useState<NetworkMonitorStatsRecord[]>([])+ // pending raw events to be merged (keyed by monitor+created)+ const pendingRaw = useRef(new Map<string, RawMonitorStatsRecord>())+ const mergeBatchTimeout = useRef<ReturnType<typeof setTimeout> | null>(null)++ useEffect(() => {+ setMonitorStats(getCacheValue(monitorId, chartTime === "1m" ? "rt" : chartTime))+ }, [monitorId, chartTime])++ // Fetch only the selected monitor's missing history.+ useEffect(() => {+ if (!enabled || chartTime === "1m") {+ return+ }++ let cancelled = false+ const { expectedInterval } = chartTimeData[chartTime]+ const cachedMonitorStats = getCacheValue(monitorId, chartTime)++ if (cachedMonitorStats.length) {+ setMonitorStats(cachedMonitorStats)+ const lastCreated = cachedMonitorStats.at(-1)?.created+ if (lastCreated && Date.now() - lastCreated < expectedInterval * 0.9) {+ return+ }+ }++ fetchMonitorStats(monitorId, chartTime, cachedMonitorStats)+ .then((newMonitorStats) => {+ if (cancelled) return+ setMonitorStats(appendCacheValue(monitorId, chartTime, newMonitorStats))+ })+ .catch((error) => {+ if (!cancelled) console.error("Failed to fetch monitor stats:", error)+ })+ return () => {+ cancelled = true+ }+ }, [monitorId, chartTime, enabled])++ // subscribe to new per-monitor stats records; batch them into merged chart records+ useEffect(() => {+ if (!enabled || chartTime === "1m") {+ return+ }+ let cancelled = false+ let unsubscribe: (() => void) | undefined+ const pbOptions = {+ fields: "monitor,res_min,res_max,total_count,success_count,res_sum,created,type",+ filter: pb.filter("monitor={:monitor} && type={:type}", {+ monitor: monitorId,+ type: chartTimeData[chartTime].type,+ }),+ }++ function flushPending() {+ mergeBatchTimeout.current = null+ const pending = pendingRaw.current+ pendingRaw.current = new Map()+ const merged = mergeMonitorStats(Array.from(pending.values()))+ if (merged.length > 0) {+ const newStats = appendCacheValue(monitorId, chartTime, merged)+ setMonitorStats(newStats)+ }+ }++ ;(async () => {+ try {+ unsubscribe = await pb.collection<RawMonitorStatsRecord>("network_monitor_stats").subscribe(+ "*",+ (event) => {+ if (cancelled || event.action !== "create") {+ return+ }+ const rec = event.record+ pendingRaw.current.set(`${rec.monitor}:${rec.created}`, rec)+ if (!mergeBatchTimeout.current) {+ mergeBatchTimeout.current = setTimeout(flushPending, 200)+ }+ },+ pbOptions+ )+ if (cancelled) unsubscribe()+ } catch (error) {+ console.error("Failed to subscribe to monitor stats:", error)+ }+ })()++ return () => {+ cancelled = true+ if (mergeBatchTimeout.current) {+ clearTimeout(mergeBatchTimeout.current)+ mergeBatchTimeout.current = null+ }+ pendingRaw.current.clear()+ unsubscribe?.()+ }+ }, [monitorId, chartTime, enabled])++ // subscribe to realtime metrics if chart time is 1m+ useEffect(() => {+ if (!enabled || chartTime !== "1m") {+ return+ }+ let cancelled = false+ let unsubscribe: (() => void) | undefined+ pb.realtime+ .subscribe(+ `rt_metrics`,+ (data: { Monitors: NetworkMonitorStatsRecord["stats"] }) => {+ const monitorStats = data.Monitors?.[monitorId]+ if (cancelled || !monitorStats) return+ const stats = { created: Date.now(), stats: { [monitorId]: monitorStats } }+ const newStats = appendCacheValue(monitorId, "rt", [stats], 120)+ setMonitorStats(newStats)+ },+ { query: { system: systemId } }+ )+ .then((us) => {+ unsubscribe = us+ if (cancelled) unsubscribe()+ })+ return () => {+ cancelled = true+ unsubscribe?.()+ }+ }, [chartTime, systemId, monitorId, enabled])++ return monitorStats+}++async function fetchMonitors(system?: string) {+ try {+ return await pb.collection<NetworkMonitorRecord>("network_monitors").getFullList({+ fields: NETWORK_MONITOR_FIELDS,+ filter: system ? pb.filter("system={:system}", { system }) : undefined,+ })+ } catch (error) {+ toast({+ title: "Error",+ description: (error as Error)?.message,+ variant: "destructive",+ })+ return []+ }+}++function applyMonitorEvents(+ monitors: NetworkMonitorRecord[],+ events: Iterable<RecordSubscription<NetworkMonitorRecord>>,+ systemId?: string+) {+ const monitorById = new Map(monitors.map((monitor) => [monitor.id, monitor]))+ const createdMonitors: NetworkMonitorRecord[] = []++ for (const { action, record } of events) {+ const matchesSystemScope = !systemId || record.system === systemId++ if (action === "delete" || !matchesSystemScope) {+ monitorById.delete(record.id)+ continue+ }++ if (!monitorById.has(record.id)) {+ createdMonitors.push(record)+ }++ monitorById.set(record.id, record)+ }++ const nextMonitors: NetworkMonitorRecord[] = []+ for (let index = createdMonitors.length - 1; index >= 0; index -= 1) {+ nextMonitors.push(createdMonitors[index])+ }++ for (const monitor of monitors) {+ const nextMonitor = monitorById.get(monitor.id)+ if (!nextMonitor) {+ continue+ }+ nextMonitors.push(nextMonitor)+ monitorById.delete(monitor.id)+ }++ return nextMonitors+}diff --git a/internal/site/src/lib/utils.ts b/internal/site/src/lib/utils.tsindex d7900bcc5..0b06f63be 100644--- a/internal/site/src/lib/utils.ts+++ b/internal/site/src/lib/utils.ts@@ -72,7 +72,7 @@ export const formatShortDate = (timestamp: string) => {return shortDateFormatter.format(new Date(timestamp))}-export const hourWithSeconds = (timestamp: string) => {+export const hourWithSeconds = (timestamp: string | number) => {return hourWithSecondsFormatter.format(new Date(timestamp))}@@ -111,17 +111,18 @@ export const updateFavicon = (() => {</linearGradient></defs><path fill="url(#gradient)" d="M35 70H0V0h35q4.4 0 8.2 1.7a21.4 21.4 0 0 1 6.6 4.5q2.9 2.8 4.5 6.6Q56 16.7 56 21a15.4 15.4 0 0 1-.3 3.2 17.6 17.6 0 0 1-.2.8 19.4 19.4 0 0 1-1.5 4 17 17 0 0 1-2.4 3.4 13.5 13.5 0 0 1-2.6 2.3 12.5 12.5 0 0 1-.4.3q1.7 1 3 2.5Q53 39.1 54 41a18.3 18.3 0 0 1 1.5 4 17.4 17.4 0 0 1 .5 3 15.3 15.3 0 0 1 0 1q0 4.4-1.7 8.2a21.4 21.4 0 0 1-4.5 6.6q-2.8 2.9-6.6 4.6Q39.4 70 35 70ZM14 14v14h21a7 7 0 0 0 2.3-.3 6.6 6.6 0 0 0 .4-.2Q39 27 40 26a6.9 6.9 0 0 0 1.5-2.2q.5-1.3.5-2.8a7 7 0 0 0-.4-2.3 6.6 6.6 0 0 0-.1-.4Q40.9 17 40 16a7 7 0 0 0-2.3-1.4 6.9 6.9 0 0 0-2.5-.6 7.9 7.9 0 0 0-.2 0H14Zm0 28v14h21a7 7 0 0 0 2.3-.4 6.6 6.6 0 0 0 .4-.1Q39 54.9 40 54a7 7 0 0 0 1.5-2.2 6.9 6.9 0 0 0 .5-2.6 7.9 7.9 0 0 0 0-.2 7 7 0 0 0-.4-2.3 6.6 6.6 0 0 0-.1-.4Q40.9 45 40 44a7 7 0 0 0-2.3-1.5 6.9 6.9 0 0 0-2.5-.6 7.9 7.9 0 0 0-.2 0H14Z"/>- ${downCount > 0 &&- `+ ${+ downCount > 0 &&+ `<circle cx="40" cy="50" r="22" fill="#f00"/><text x="40" y="60" font-size="34" text-anchor="middle" fill="#fff" font-family="Arial" font-weight="bold">${downCount}</text>`- }+ }</svg>`const blob = new Blob([svg], { type: "image/svg+xml" })const url = URL.createObjectURL(blob)- ; (document.querySelector("link[rel='icon']") as HTMLLinkElement).href = url+ ;(document.querySelector("link[rel='icon']") as HTMLLinkElement).href = url}})()@@ -198,6 +199,26 @@ export function decimalString(num: number, digits = 2) {return formatter.format(num)}+export function formatMicroseconds(microseconds: number, showDigits = true): string {+ if (!Number.isFinite(microseconds)) {+ return "-"+ }++ if (microseconds < 1000) {+ return `${microseconds}μs`+ }++ if (microseconds < 1_000_000) {+ const milliseconds = microseconds / 1000+ const digits = milliseconds >= 10 ? 1 : 2+ return `${decimalString(milliseconds, showDigits ? digits : 0)}ms`+ }++ const seconds = microseconds / 1_000_000+ const digits = seconds >= 10 ? 1 : 2+ return `${decimalString(seconds, showDigits ? digits : 0)}s`+}+/** Get value from local or session storage */function getStorageValue(key: string, defaultValue: unknown, storageInterface: Storage = localStorage) {const saved = storageInterface?.getItem(key)@@ -365,12 +386,12 @@ export function formatDuration(.join(" ")}-/** Parse semver string into major, minor, and patch numbers+/** Parse semver string into major, minor, and patch numbers* @example* const semVer = "1.2.3"* const { major, minor, patch } = parseSemVer(semVer)* console.log(major, minor, patch) // 1, 2, 3-*/+ */export const parseSemVer = (semVer = ""): SemVer => {// if (semVer.startsWith("v")) {// semVer = semVer.slice(1)@@ -393,6 +414,12 @@ export function compareSemVer(a: SemVer, b: SemVer) {return a.patch - b.patch}+const MIN_NETWORK_MONITOR_AGENT_VERSION = parseSemVer("0.20.0")++export function supportsNetworkMonitors(system: Pick<SystemRecord, "info">) {+ return compareSemVer(parseSemVer(system.info?.v), MIN_NETWORK_MONITOR_AGENT_VERSION) >= 0+}+// biome-ignore lint/suspicious/noExplicitAny: any is used to allow any function to be passed inexport function debounce<T extends (...args: any[]) => any>(func: T, wait: number): (...args: Parameters<T>) => void {let timeout: ReturnType<typeof setTimeout>@@ -422,10 +449,22 @@ export function runOnce<T extends (...args: any[]) => any>(fn: T): T {}) as T}-/** Get the visual width of a string, accounting for full-width characters */-export function getVisualStringWidth(str: string): number {+const visualWidthCache = new Map<string, number>()++/** Get the visual width of a string, accounting for full-width and narrow punctuation characters.+ * Don't use for monospaced fonts, use .length instead+ */+function getVisualStringWidth(str: string): number {+ const cached = visualWidthCache.get(str)+ if (cached !== undefined) {+ return cached+ }let width = 0for (const char of str) {+ if (char === ".") {+ width += 0.7+ continue+ }const code = char.codePointAt(0) || 0// Hangul Jamo and Syllables are often slightly thinner than Hanzi/Kanjiif ((code >= 0x1100 && code <= 0x115f) || (code >= 0xac00 && code <= 0xd7af)) {@@ -443,16 +482,41 @@ export function getVisualStringWidth(str: string): number {code > 0xffff // Emojis and other supplementary plane characterswidth += isFullWidth ? 2 : 1}+ visualWidthCache.set(str, width)return width}+/** Compare the visual width of two strings imprecisely */+export function isVisuallyLonger(str1: string, str2: string): boolean {+ return getVisualStringWidth(str1) > getVisualStringWidth(str2)+}++/** Parses a filter string into OR'd groups of AND'd terms: "a b, c" -> [["a","b"], ["c"]] */+export function parseFilterGroups(value: string): string[][] {+ return value+ .toLowerCase()+ .split(",")+ .map((group) => group.trim().split(" ").filter((term) => term.length > 0))+ .filter((terms) => terms.length > 0)+}++/** True if every term in at least one OR'd group is found in searchString. */+export function matchesFilterGroups(searchString: string, groups: string[][]): boolean {+ return groups.some((terms) => terms.every((term) => searchString.includes(term)))+}+/** Format seconds to hours, minutes, or seconds */export function secondsToString(seconds: number, unit: "hour" | "minute" | "day"): string {const count = Math.floor(seconds / (unit === "hour" ? 3600 : unit === "minute" ? 60 : 86400))const countString = count.toLocaleString()switch (unit) {case "minute":- return plural(count, { one: `${countString} minute`, few: `${countString} minutes`, many: `${countString} minutes`, other: `${countString} minutes` })+ return plural(count, {+ one: `${countString} minute`,+ few: `${countString} minutes`,+ many: `${countString} minutes`,+ other: `${countString} minutes`,+ })case "hour":return plural(count, { one: `${countString} hour`, other: `${countString} hours` })case "day":@@ -469,4 +533,4 @@ export function secondsToUptimeString(seconds: number): string {} else {return secondsToString(seconds, "day")}-}\ No newline at end of file+}diff --git a/internal/site/src/main.tsx b/internal/site/src/main.tsxindex a4483501b..92f8da1c2 100644--- a/internal/site/src/main.tsx+++ b/internal/site/src/main.tsx@@ -30,6 +30,7 @@ const LoginPage = lazy(() => import("@/components/login/login.tsx"))const Home = lazy(() => import("@/components/routes/home.tsx"))const Containers = lazy(() => import("@/components/routes/containers.tsx"))const Smart = lazy(() => import("@/components/routes/smart.tsx"))+const Monitors = lazy(() => import("@/components/routes/monitors.tsx"))const SystemDetail = lazy(() => import("@/components/routes/system.tsx"))const CopyToClipboardDialog = lazy(() => import("@/components/copy-to-clipboard.tsx"))@@ -79,6 +80,8 @@ const App = memo(() => {return <Containers />} else if (page.route === "smart") {return <Smart />+ } else if (page.route === "monitors") {+ return <Monitors />} else if (page.route === "settings") {return <Settings />}diff --git a/internal/site/src/types.d.ts b/internal/site/src/types.d.tsindex f9a7ef7c2..291c84340 100644--- a/internal/site/src/types.d.ts+++ b/internal/site/src/types.d.ts@@ -307,6 +307,7 @@ export interface AlertRecord extends RecordModel {}export interface AlertsHistoryRecord extends RecordModel {+ monitor_name?: stringalert: stringuser: stringsystem: string@@ -393,11 +394,9 @@ export interface SemVer {export interface ChartData {agentVersion: SemVer- systemStats: SystemStatsRecord[]- containerData: ChartDataContainer[]+ systemStats?: SystemStatsRecord[]+ containerData?: ChartDataContainer[]orientation: "right" | "left"- ticks: number[]- domain: number[]chartTime: ChartTimes}@@ -414,6 +413,8 @@ export interface AlertInfo {singleDesc?: () => string/** Hides the duration slider for alerts that fire on first observation */noDuration?: boolean+ /** Hides the threshold control for binary alerts */+ noThreshold?: boolean/** Description shown instead of numeric threshold and duration values */triggeredDesc?: () => string/** Additional information that remains visible while the alert is enabled */@@ -632,3 +633,52 @@ export interface UpdateInfo {v: string // new versionurl: string // url to new version}++export interface NetworkMonitorRecord {+ id: string+ system: string+ target: string+ protocol: "icmp" | "tcp" | "http" | "dns"+ port: number+ res: number+ resMin1h: number+ resMax1h: number+ resAvg1h: number+ loss: number+ loss1h: number+ interval: number+ enabled: boolean+ updated: string+}++/** Response times in microseconds and packet loss percentage (0-100). */+export interface MonitorStats {+ res_avg: number+ res_min: number+ res_max: number+ loss: number+}++/** Raw per-monitor record stored in the DB. */+export interface RawMonitorStatsRecord {+ res_min: number+ res_max: number+ total_count: number+ success_count: number+ res_sum: number+ id?: string+ type?: string+ monitor: string+ created: number // unix timestamp (ms)+}++/**+ * Merged stats record keyed by monitor ID, used by chart components.+ * Constructed from multiple RawMonitorStatsRecord entries sharing the same timestamp.+ */+export interface NetworkMonitorStatsRecord {+ id?: string+ type?: string+ stats: Record<string, MonitorStats>+ created: number // unix timestamp (ms) for Recharts xAxis+}diff --git a/internal/site/tests/network-monitor-utils.test.ts b/internal/site/tests/network-monitor-utils.test.tsnew file mode 100644index 000000000..80aeec4fa--- /dev/null+++ b/internal/site/tests/network-monitor-utils.test.ts@@ -0,0 +1,36 @@+import { describe, expect, test } from "bun:test"+import { getMonitorStats } from "../src/lib/network-monitor-utils"++describe("monitor stats derived from stored counts", () => {+ test("retains probe weights and response precision", () => {+ const stats = getMonitorStats({+ monitor: "monitor1",+ created: 1000,+ res_min: 5,+ res_max: 20,+ total_count: 7,+ success_count: 6,+ res_sum: 61,+ })+ expect(stats.res_avg).toBeCloseTo(10.1666667)+ expect(stats.loss).toBeCloseTo(14.2857143)+ expect(stats.res_min).toBe(5)+ expect(stats.res_max).toBe(20)+ })++ test.each([+ { total_count: 3, success_count: 0, loss: 100 },+ { total_count: 0, success_count: 0, loss: 0 },+ { total_count: 1, success_count: 1, loss: 0 },+ ])("handles zero sums with $total_count attempts and $success_count successes", ({ loss, ...counts }) => {+ const stats = getMonitorStats({+ monitor: "monitor1",+ created: 1000,+ res_min: 0,+ res_max: 0,+ res_sum: 0,+ ...counts,+ })+ expect(stats).toEqual({ res_avg: 0, res_min: 0, res_max: 0, loss })+ })+})