Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

6.1 KB
Raw Blame
package agent

import (
	"crypto/tls"
	"errors"
	"fmt"
	"log/slog"
	"net/http"
	"sync"
	"sync/atomic"
	"time"

	"github.com/henrygd/beszel/internal/entities/monitor"
)

// monitorIdleTimeout is three default hub update intervals. The hub collects
// monitor results only with stats, which it stops requesting while the system is
// paused or the agent is disconnected, so scheduled probes stop after this long.
const monitorIdleTimeout = 3 * time.Minute

// MonitorManager manages network monitor configurations and task lifetimes.
type MonitorManager struct {
	mu          sync.RWMutex
	monitors    map[string]*monitorTask // keyed by monitor ID
	probe       monitorProbe
	certCheck   certChecker
	resumeGuard monitorResumeGuard
	agent       *Agent // Reference to the parent agent; nil means the hub is never idle
	idle        atomic.Bool
}

func newMonitorManager(agent *Agent) *MonitorManager {
	client := &http.Client{Timeout: monitor.MaxProbeTimeout}
	insecureTransport := http.DefaultTransport.(*http.Transport).Clone()
	insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	insecureClient := &http.Client{Timeout: monitor.MaxProbeTimeout, Transport: insecureTransport}
	pm := newMonitorManagerWithProbe(networkMonitorProbe(client, insecureClient))
	pm.agent = agent
	return pm
}

func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
	return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe, certCheck: checkCert}
}

// SyncMonitors replaces all monitor tasks with the given configs.
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
	pm.mu.Lock()
	defer pm.mu.Unlock()

	// Build set of new keys
	newKeys := make(map[string]monitor.Config, len(configs))
	for _, cfg := range configs {
		if cfg.ID == "" {
			continue
		}
		newKeys[cfg.ID] = cfg
	}

	// Stop removed monitors
	for key, task := range pm.monitors {
		if _, exists := newKeys[key]; !exists {
			task.cancel()
			delete(pm.monitors, key)
		}
	}

	// Start new monitors and restart tasks whose config changed.
	for key, cfg := range newKeys {
		task, exists := pm.monitors[key]
		if exists && task.config == cfg {
			continue
		}
		if exists {
			task.cancel()
		}
		task = newMonitorTaskFromExisting(cfg, task)
		task.resumeGuard = &pm.resumeGuard
		pm.resumeGuard.start()
		pm.monitors[key] = task
		pm.startMonitor(task)
	}
	if len(pm.monitors) == 0 {
		pm.resumeGuard.shutdown()
	}
}

// HandleSyncRequest applies a full or incremental monitor sync request.
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
	switch req.Action {
	case monitor.SyncActionReplace:
		pm.SyncMonitors(req.Configs)
		return monitor.SyncResponse{}, nil
	case monitor.SyncActionUpsert:
		result, err := pm.UpsertMonitor(req.Config, req.RunNow)
		if err != nil {
			return monitor.SyncResponse{}, err
		}
		if result == nil {
			return monitor.SyncResponse{}, nil
		}
		return monitor.SyncResponse{Result: *result}, nil
	case monitor.SyncActionDelete:
		if req.Config.ID == "" {
			return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
		}
		pm.DeleteMonitor(req.Config.ID)
		return monitor.SyncResponse{}, nil
	default:
		return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
	}
}

// UpsertMonitor creates or replaces a single monitor task.
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
	if config.ID == "" {
		return nil, errors.New("missing monitor ID")
	}

	pm.mu.Lock()
	task, exists := pm.monitors[config.ID]
	if exists && task.config == config {
		pm.mu.Unlock()
		if !runNow {
			return nil, nil
		}
		return pm.runNow(task), nil
	}
	if exists {
		task.cancel()
	}
	task = newMonitorTaskFromExisting(config, task)
	task.resumeGuard = &pm.resumeGuard
	pm.resumeGuard.start()
	pm.monitors[config.ID] = task
	pm.mu.Unlock()

	if runNow {
		result := pm.runNow(task)
		pm.startMonitor(task)
		return result, nil
	}
	pm.startMonitor(task)
	return nil, nil
}

// runNow runs a probe and any due certificate check concurrently, so the
// response fits within the hub's single probe timeout budget.
func (pm *MonitorManager) runNow(task *monitorTask) *monitor.Result {
	var wg sync.WaitGroup
	wg.Go(func() { task.refreshCert(pm.certCheck) })
	result := task.runProbe(pm.probe)
	wg.Wait()
	if result != nil {
		result.Cert = task.certInfo()
	}
	return result
}

// DeleteMonitor stops and removes a single monitor task.
func (pm *MonitorManager) DeleteMonitor(id string) {
	if id == "" {
		return
	}
	pm.mu.Lock()
	defer pm.mu.Unlock()
	if task, exists := pm.monitors[id]; exists {
		task.cancel()
		delete(pm.monitors, id)
	}
	if len(pm.monitors) == 0 {
		pm.resumeGuard.shutdown()
	}
}

// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
	pm.mu.RLock()
	defer pm.mu.RUnlock()

	results := make(map[string]monitor.Result, len(pm.monitors))
	now := time.Now()
	duration := time.Duration(durationMs) * time.Millisecond

	for _, task := range pm.monitors {
		result, ok := task.history.result(duration, now)

		if !ok {
			continue
		}
		// Only the default interval updates monitor records on the hub, so
		// realtime requests must not consume the unsent certificate.
		if durationMs == defaultDataCacheTimeMs {
			result.Cert = task.takeUnsentCert()
		}
		results[task.config.ID] = result
	}

	return results
}

// hubIdle reports whether scheduled probes should be skipped because the hub
// hasn't requested stats recently. Immediate probes requested by the hub still run.
func (pm *MonitorManager) hubIdle() bool {
	idle := pm.agent != nil && pm.agent.hubIdleFor(monitorIdleTimeout)
	if pm.idle.Swap(idle) != idle {
		if idle {
			slog.Debug("pausing network monitors until the hub requests stats")
		} else {
			slog.Debug("resuming network monitors")
		}
	}
	return idle
}

// Stop stops all monitor tasks.
func (pm *MonitorManager) Stop() {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	for key, task := range pm.monitors {
		task.cancel()
		delete(pm.monitors, key)
	}
	pm.resumeGuard.shutdown()
}