Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

11.9 KB
Raw Blame
//go:build testing

package records_test

import (
	"testing"
	"time"

	monitorEntity "github.com/henrygd/beszel/internal/entities/monitor"
	"github.com/henrygd/beszel/internal/records"
	"github.com/henrygd/beszel/internal/tests"

	"ganigeorgiev.com/pocketbase/dbx"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestAverageMonitorStats(t *testing.T) {
	hub, err := tests.NewTestHub(t.TempDir())
	require.NoError(t, err)
	defer hub.Cleanup()

	collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats")
	require.NoError(t, err)
	assert.Nil(t, collection.Fields.GetByName("res_avg"))
	assert.Nil(t, collection.Fields.GetByName("loss"))

	rm := records.NewRecordManager(hub)
	user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
	require.NoError(t, err)
	sys, err := tests.CreateRecord(hub, "systems", map[string]any{
		"name":   "monitor-avg-system",
		"host":   "localhost",
		"port":   "45876",
		"status": "up",
		"users":  []string{user.Id},
	})
	require.NoError(t, err)
	monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
		"system":   sys.Id,
		"name":     "cloudflare",
		"target":   "1.1.1.1",
		"protocol": "icmp",
		"interval": 30,
		"enabled":  true,
	})
	require.NoError(t, err)

	created := time.Now().UnixMilli()
	// Unequal probe counts must weight both latency and loss.
	recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
		"system":      sys.Id,
		"monitor":     monitor.Id,
		"type":        "1m",
		"created":     created,
		"res_min":     5,
		"res_max":     20,
		"total_count": 6, "success_count": 6, "res_sum": 60,
	})
	require.NoError(t, err)
	recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
		"system":      sys.Id,
		"monitor":     monitor.Id,
		"type":        "1m",
		"created":     created,
		"res_min":     10,
		"res_max":     60,
		"total_count": 1, "success_count": 1, "res_sum": 22,
	})
	require.NoError(t, err)

	result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
	require.NoError(t, err)
	assert.Equal(t, 2, count)
	assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result)

	for _, tc := range []struct {
		name, monitor, recordType string
		after                     int64
	}{
		{"other monitor", "missing", "1m", created - 1},
		{"other type", monitor.Id, "10m", created - 1},
		{"exclusive cutoff", monitor.Id, "1m", created},
	} {
		t.Run(tc.name, func(t *testing.T) {
			stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after)
			require.NoError(t, err)
			assert.Zero(t, count)
			assert.Equal(t, monitorEntity.Stats{}, stats)
		})
	}

	// A failure-only bucket counts toward loss but must not lower latency.
	recordB.Set("res_min", 0)
	recordB.Set("res_max", 0)
	recordB.Set("success_count", 0)
	recordB.Set("res_sum", 0)
	require.NoError(t, hub.Save(recordB))
	result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
	require.NoError(t, err)
	assert.Equal(t, 2, count)
	assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result)
	// Sparse monitor records must propagate through every rollup level.
	rm.CreateLongerRecords()
	for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
		rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
		require.NoError(t, err)
		require.Len(t, rollups, 1, recordType)
		assert.Equal(t, 5.0, rollups[0].GetFloat("res_min"))
		assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
		assert.Equal(t, 7, rollups[0].GetInt("total_count"))
		assert.Equal(t, 6, rollups[0].GetInt("success_count"))
		assert.Equal(t, 60, rollups[0].GetInt("res_sum"))
		// A sibling with a different number of probes must retain its actual
		// weight when the next tier combines their underlying counts.
		_, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
			"system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created,
			"res_min": 100, "res_max": 100,
			"total_count": 3, "success_count": 1, "res_sum": 100,
		})
		require.NoError(t, err)
		merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1)
		require.NoError(t, err)
		assert.Equal(t, 2, count)
		assert.Equal(t, monitorEntity.Stats{
			ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30,
			TotalCount: 10, SuccessCount: 7, ResponseSum: 160,
		}, merged)
	}
	// All failures produce zero latency, while a genuine zero-microsecond
	// success remains a valid minimum (it must not be filtered out as a sentinel).
	recordA.Set("success_count", 0)
	recordA.Set("res_sum", 0)
	require.NoError(t, hub.Save(recordA))
	result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
	require.NoError(t, err)
	assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result)
	recordA.Set("success_count", 1)
	recordA.Set("res_min", 0)
	recordA.Set("res_max", 0)
	require.NoError(t, hub.Save(recordA))
	result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
	require.NoError(t, err)
	assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result)
}

func TestSparseMonitorRollups(t *testing.T) {
	for _, tc := range []struct {
		name     string
		interval int
		samples  int
		disabled bool
	}{
		{"five minute interval", 300, 2, false},
		{"ten minute interval", 600, 1, false},
		{"fifteen minute interval", 900, 1, false},
		{"empty window", 900, 0, false},
		{"disabled with pending history", 300, 2, true},
		{"disabled empty window", 900, 0, true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			hub, err := tests.NewTestHub(t.TempDir())
			require.NoError(t, err)
			defer hub.Cleanup()
			user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
			require.NoError(t, err)
			sys, err := tests.CreateRecord(hub, "systems", map[string]any{
				"name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up",
				"users": []string{user.Id},
			})
			require.NoError(t, err)
			monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
				"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
				"interval": tc.interval, "enabled": true,
			})
			require.NoError(t, err)
			now := time.Now()
			for i := range tc.samples {
				_, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
					"system": sys.Id, "monitor": monitor.Id, "type": "1m",
					"created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(),
					"res_min": 8, "res_max": 20,
					"total_count": 4, "success_count": 3, "res_sum": 36,
				})
				require.NoError(t, err)
			}
			// Other collections must still reject fewer than nine minute records.
			for _, collection := range []string{"system_stats", "container_stats"} {
				stats := `{"cpu":10}`
				if collection == "container_stats" {
					stats = `[{"name":"test","cpu":10}]`
				}
				for range 8 {
					_, err := tests.CreateRecord(hub, collection, map[string]any{
						"system": sys.Id, "type": "1m", "stats": stats,
					})
					require.NoError(t, err)
				}
			}
			if tc.disabled {
				monitor.Set("enabled", false)
				require.NoError(t, hub.Save(monitor))
			}
			records.NewRecordManager(hub).CreateLongerRecords()
			for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
				rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
				require.NoError(t, err)
				if tc.samples == 0 {
					assert.Empty(t, rollups, recordType)
				} else {
					require.Len(t, rollups, 1, recordType)
					assert.Equal(t, 8.0, rollups[0].GetFloat("res_min"))
					assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
					assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count"))
					assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count"))
					assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum"))
				}
				for _, collection := range []string{"system_stats", "container_stats"} {
					count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType})
					require.NoError(t, err)
					assert.Zero(t, count, "%s %s", collection, recordType)
				}
			}
		})
	}
}

// TestMonitorRollupExcludesPreviousInterval verifies that a monitor rollup created one full
// period ago is not merged into the next longer record, which would double count its probes.
func TestMonitorRollupExcludesPreviousInterval(t *testing.T) {
	hub, err := tests.NewTestHub(t.TempDir())
	require.NoError(t, err)
	defer hub.Cleanup()
	user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
	require.NoError(t, err)
	sys, err := tests.CreateRecord(hub, "systems", map[string]any{
		"name": "m", "host": "localhost", "port": "45876", "status": "up", "users": []string{user.Id},
	})
	require.NoError(t, err)
	monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
		"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp", "interval": 60, "enabled": true,
	})
	require.NoError(t, err)
	now := time.Now()
	seed := func(typ string, created time.Time, total int) {
		_, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
			"system": sys.Id, "monitor": monitor.Id, "type": typ, "created": created.UnixMilli(),
			"res_min": 8, "res_max": 20, "total_count": total, "success_count": total, "res_sum": total,
		})
		require.NoError(t, err)
	}
	seed("1m", now.Add(-time.Minute), 4)                           // becomes this run's 10m record
	seed("10m", now.Add(-10*time.Minute+50*time.Millisecond), 4)   // previous run
	seed("10m", now.Add(-20*time.Minute+50*time.Millisecond), 100) // run before that: previous 20m interval
	records.NewRecordManager(hub).CreateLongerRecords()
	rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": "20m"})
	require.NoError(t, err)
	require.Len(t, rollups, 1)
	require.Equal(t, 8, rollups[0].GetInt("total_count"))
}

// TestMonitorRollupOffsetPhase covers 20m monitor records whose phase is offset 10 minutes from the 120m rollup
// (e.g. after the hub missed one run). The oldest one was created 110 minutes
// ago, just after the previous 120m rollup, and has not been rolled up yet.
func TestMonitorRollupOffsetPhase(t *testing.T) {
	hub, err := tests.NewTestHub(t.TempDir())
	require.NoError(t, err)
	defer hub.Cleanup()
	user, err := tests.CreateUser(hub, "[email protected]", "testtesttest")
	require.NoError(t, err)
	sys, err := tests.CreateRecord(hub, "systems", map[string]any{
		"name": "m", "host": "localhost", "port": "45876", "status": "up", "users": []string{user.Id},
	})
	require.NoError(t, err)
	monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
		"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp", "interval": 60, "enabled": true,
	})
	require.NoError(t, err)
	now := time.Now()
	seed := func(typ string, created time.Time, total int) {
		_, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
			"system": sys.Id, "monitor": monitor.Id, "type": typ, "created": created.UnixMilli(),
			"res_min": 8, "res_max": 20, "total_count": total, "success_count": total, "res_sum": total,
		})
		require.NoError(t, err)
	}
	seed("120m", now.Add(-120*time.Minute-50*time.Millisecond), 1000) // previous 120m rollup
	for i := range 6 {
		// created at -110m, -90m, ... -10m, by runs that started 50ms earlier in their second than this one
		seed("20m", now.Add(-time.Duration(10+20*i)*time.Minute-50*time.Millisecond), 4)
	}
	records.NewRecordManager(hub).CreateLongerRecords()
	rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": "120m"})
	require.NoError(t, err)
	require.Len(t, rollups, 2)
	got := 0
	for _, r := range rollups {
		if c := r.GetInt("total_count"); c != 1000 {
			got = c
		}
	}
	require.Equal(t, 24, got)
}