Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

10.5 KB
Raw Blame
//go:build testing && linux

package agent

import (
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"sync"
	"testing"

	"github.com/henrygd/beszel/internal/entities/system"
	"github.com/shirou/gopsutil/v4/mem"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func setupMemoryCgroup(t *testing.T, v2 bool) string {
	t.Helper()
	swapCpuContainerSeams(t)
	if v2 {
		writeCpuFixture(t, procSelfCgroup, "0::/system.slice/agent.service\n")
		writeCpuFixture(t, cgroupMountinfo, "")
		return cgroupRoot
	}
	dir := filepath.Join(filepath.Dir(cgroupRoot), "memory")
	writeCpuFixture(t, procSelfCgroup, "2:memory:/system.slice/agent.service\n")
	writeCpuFixture(t, cgroupMountinfo,
		"30 25 0:26 / "+dir+" rw - cgroup cgroup rw,memory\n")
	return dir
}

func writeMemoryFixture(t *testing.T, dir string, v2 bool, usage, limit, stat string) {
	t.Helper()
	usageName, limitName := "memory.usage_in_bytes", "memory.limit_in_bytes"
	if v2 {
		usageName, limitName = "memory.current", "memory.max"
	}
	writeCpuFixture(t, filepath.Join(dir, usageName), usage)
	writeCpuFixture(t, filepath.Join(dir, limitName), limit)
	writeCpuFixture(t, filepath.Join(dir, "memory.stat"), stat)
}

func TestContainerMemoryMetrics(t *testing.T) {
	for _, v2 := range []bool{true, false} {
		t.Run(strconv.FormatBool(v2), func(t *testing.T) {
			dir := setupMemoryCgroup(t, v2)
			key, shmemKey := "total_cache", "total_shmem"
			if v2 {
				key, shmemKey = "file", "shmem"
			}
			stat := func(cache, shmem string) string {
				return key + " " + cache + "\n" + shmemKey + " " + shmem + "\n"
			}
			for _, tt := range []struct {
				name, usage, limit, stat string
				host                     uint64
				want                     memoryMetrics
				ok                       bool
			}{
				{"file cache excluded", "600", "1000", stat("200", "0"), 2000, memoryMetrics{1000, 400, 200}, true},
				{"shared memory retained", "600", "1000", stat("200", "50"), 2000, memoryMetrics{1000, 450, 150}, true},
				{"shared memory larger than cache", "600", "1000", stat("200", "300"), 2000, memoryMetrics{1000, 600, 0}, true},
				{"zero usage", "0", "1000", stat("0", "0"), 2000, memoryMetrics{1000, 0, 0}, true},
				{"cache larger than charge", "100", "1000", stat("200", "0"), 2000, memoryMetrics{1000, 0, 100}, true},
				{"host caps limit", "600", "3000", stat("200", "0"), 2000, memoryMetrics{2000, 400, 200}, true},
				{"finite without host", "600", "1000", stat("200", "0"), 0, memoryMetrics{1000, 400, 200}, true},
				{"missing cache key", "600", "1000", shmemKey + " 0\n", 2000, memoryMetrics{}, false},
				{"missing shared memory key", "600", "1000", key + " 200\n", 2000, memoryMetrics{}, false},
				{"invalid cache", "600", "1000", stat("-1", "0"), 2000, memoryMetrics{}, false},
				{"invalid shared memory", "600", "1000", stat("200", "-1"), 2000, memoryMetrics{}, false},
				{"invalid usage", "bad", "1000", stat("200", "0"), 2000, memoryMetrics{}, false},
				{"invalid limit", "600", "bad", stat("200", "0"), 2000, memoryMetrics{}, false},
				{"zero limit", "600", "0", stat("200", "0"), 2000, memoryMetrics{}, false},
			} {
				t.Run(tt.name, func(t *testing.T) {
					writeMemoryFixture(t, dir, v2, tt.usage, tt.limit, tt.stat)
					metrics, ok := containerMemoryMetrics(tt.host, true)
					assert.Equal(t, tt.ok, ok)
					assert.Equal(t, tt.want, metrics)
				})
			}
			limit := "9223372036854771712"
			if v2 {
				limit = "max"
			}
			writeMemoryFixture(t, dir, v2, "600", limit, stat("200", "0"))
			writeMemoryFixture(t, filepath.Join(dir, "system.slice/agent.service"), v2, "1", "100", stat("0", "0"))
			metrics, ok := containerMemoryMetrics(2000, true)
			require.True(t, ok)
			assert.Equal(t, memoryMetrics{2000, 400, 200}, metrics)
			_, ok = containerMemoryMetrics(0, true)
			assert.False(t, ok)
			if !v2 {
				writeMemoryFixture(t, dir, false, "600", "2147479552", stat("200", "0"))
				metrics, ok = containerMemoryMetrics(2000, true)
				require.True(t, ok)
				assert.EqualValues(t, 2000, metrics.Total)
			}
		})
	}
}

func TestContainerMemoryScopeAndEnablement(t *testing.T) {
	dir := setupMemoryCgroup(t, true)
	// A discovered non-default mount must win over the default root.
	mount := filepath.Join(t.TempDir(), "unified")
	writeCpuFixture(t, cgroupMountinfo,
		"30 25 0:26 /guest "+mount+" rw - cgroup2 cgroup2 rw\n")
	writeMemoryFixture(t, dir, true, "1", "1000", "file 0\nshmem 0\n")
	writeMemoryFixture(t, mount, true, "600", "1000", "file 200\nshmem 0\n")
	writeMemoryFixture(t, filepath.Join(mount, "system.slice/agent.service"), true, "10", "100", "file 0\nshmem 0\n")
	_, ok := containerMemoryMetrics(2000, false)
	assert.False(t, ok) // Docker/host agents keep the existing path by default.
	markLxc(t)
	inLxc = sync.OnceValue(detectLxc) // Simulate a fresh process in LXC.
	_, ok = containerMemoryMetrics(2000, false)
	assert.False(t, ok) // lxcfs already serves container-scoped /proc/meminfo.
	metrics, ok := containerMemoryMetrics(2000, true)
	require.True(t, ok)
	assert.Equal(t, memoryMetrics{1000, 400, 200}, metrics)
	require.NoError(t, os.Remove(filepath.Join(mount, "memory.stat")))
	_, ok = containerMemoryMetrics(2000, true)
	assert.False(t, ok)
}

func TestCanSubtractZfsArc(t *testing.T) {
	setupMemoryCgroup(t, true)
	inLxc = sync.OnceValue(detectLxc)
	assert.True(t, canSubtractZfsArc())
	markLxc(t)
	inLxc = sync.OnceValue(detectLxc)
	assert.False(t, canSubtractZfsArc()) // host ARC must not leak into LXC accounting
}

func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
	dir := setupMemoryCgroup(t, true)
	const gib = uint64(1 << 30)
	writeMemoryFixture(t, dir, true, strconv.FormatUint(6*gib, 10), strconv.FormatUint(10*gib, 10),
		"file "+strconv.FormatUint(2*gib, 10)+"\nshmem 0\n")
	original := memoryVirtualMemory
	t.Cleanup(func() { memoryVirtualMemory = original })
	memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
		return &mem.VirtualMemoryStat{Total: 20 * gib, Used: 12 * gib, Free: 4 * gib,
			Cached: 4 * gib, SwapTotal: 2 * gib, SwapFree: gib}, nil
	}
	a := &Agent{forceUseCgroup: true, memCalc: "htop", zfs: true}
	var stats system.Stats
	a.updateMemoryStats(&stats)
	assert.Equal(t, float64(10), stats.Mem)
	assert.Equal(t, float64(4), stats.MemUsed)
	assert.Equal(t, float64(2), stats.MemBuffCache)
	assert.Equal(t, float64(40), stats.MemPct)
	assert.Zero(t, stats.MemZfsArc)
	assert.Equal(t, float64(2), stats.Swap)
	assert.Equal(t, float64(1), stats.SwapUsed)
	assert.Equal(t, [4]uint64{10 * gib, 4 * gib, 2 * gib, 0}, stats.MemBytes)
	assert.Equal(t, [2]uint64{2 * gib, gib}, stats.SwapBytes)
	assert.Equal(t, 10*gib, a.systemDetails.MemoryTotal)
	assert.True(t, a.detailsDirty)

	a.detailsDirty = false
	a.updateMemoryStats(&system.Stats{})
	assert.False(t, a.detailsDirty)
	writeCpuFixture(t, filepath.Join(dir, "memory.max"), strconv.FormatUint(8*gib, 10))
	a.updateMemoryStats(&system.Stats{})
	assert.Equal(t, 8*gib, a.systemDetails.MemoryTotal)
	assert.True(t, a.detailsDirty)

	// A missing required field falls back to the original htop calculation.
	writeCpuFixture(t, filepath.Join(dir, "memory.stat"), "file 123\n")
	a.zfs = false
	stats = system.Stats{}
	a.updateMemoryStats(&stats)
	assert.Equal(t, float64(20), stats.Mem)
	assert.Equal(t, float64(12), stats.MemUsed)
	assert.Equal(t, float64(4), stats.MemBuffCache)
	assert.Equal(t, float64(60), stats.MemPct)
	assert.Equal(t, [4]uint64{20 * gib, 12 * gib, 4 * gib, 0}, stats.MemBytes)
	assert.Equal(t, 20*gib, a.systemDetails.MemoryTotal)

	// Finite cgroup limits still work if /proc/meminfo cannot be read.
	memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) { return nil, errors.New("unreadable") }
	writeCpuFixture(t, filepath.Join(dir, "memory.stat"), "file 0\nshmem 0\n")
	stats = system.Stats{}
	a.updateMemoryStats(&stats)
	assert.Equal(t, float64(8), stats.Mem)
	assert.Equal(t, float64(6), stats.MemUsed)
	assert.Equal(t, float64(75), stats.MemPct)
	assert.Equal(t, [4]uint64{8 * gib, 6 * gib, 0, 0}, stats.MemBytes)
	assert.Zero(t, stats.Swap)
	assert.Zero(t, stats.SwapBytes)
}

func TestMemoryDetailsInitiallyUseCgroupLimit(t *testing.T) {
	dir := setupMemoryCgroup(t, true)
	writeMemoryFixture(t, dir, true, "600", "1000", "file 200\nshmem 0\n")
	original := memoryVirtualMemory
	t.Cleanup(func() { memoryVirtualMemory = original })
	memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
		return &mem.VirtualMemoryStat{Total: 2000}, nil
	}
	a := &Agent{forceUseCgroup: true}
	a.refreshSystemDetails()
	assert.EqualValues(t, 1000, a.systemDetails.MemoryTotal)
}

// Captured from this container: active file cache made the working-set
// calculation report ~80 GiB despite only ~17 GiB of anonymous RSS.
func TestMemoryStatsExcludeActiveFileCache(t *testing.T) {
	dir := setupMemoryCgroup(t, false)
	const usage = uint64(135881601024)
	const cache = uint64(116823166976)
	const shmem = uint64(77180928)
	writeMemoryFixture(t, dir, false, strconv.FormatUint(usage, 10), "137438953472",
		"total_cache 116823166976\ntotal_shmem 77180928\n"+
			"total_inactive_file 49902608384\ntotal_active_file 67684671488\n")
	original := memoryVirtualMemory
	t.Cleanup(func() { memoryVirtualMemory = original })
	memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
		return &mem.VirtualMemoryStat{Total: 1580621720 * 1024}, nil
	}
	metrics, ok := containerMemoryMetrics(1580621720*1024, true)
	require.True(t, ok)
	assert.Equal(t, usage-(cache-shmem), metrics.Used)
	assert.Equal(t, cache-shmem, metrics.BuffCache)
	assert.Equal(t, usage, metrics.Used+metrics.BuffCache)
	var stats system.Stats
	a := &Agent{forceUseCgroup: true}
	a.updateMemoryStats(&stats)
	assert.Equal(t, float64(17.82), stats.MemUsed)
	assert.Equal(t, float64(108.73), stats.MemBuffCache)
	assert.Equal(t, float64(13.92), stats.MemPct)
	assert.Equal(t, [4]uint64{137438953472, usage - (cache - shmem), cache - shmem, 0}, stats.MemBytes)
}

// Low memory systems need byte precision because the GB fields round to 10 MiB.
func TestMemoryStatsLowMemoryPrecision(t *testing.T) {
	dir := setupMemoryCgroup(t, true)
	const mib = uint64(1 << 20)
	writeMemoryFixture(t, dir, true, strconv.FormatUint(5*mib, 10), strconv.FormatUint(64*mib, 10),
		"file "+strconv.FormatUint(2*mib, 10)+"\nshmem 0\n")
	original := memoryVirtualMemory
	t.Cleanup(func() { memoryVirtualMemory = original })
	memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
		return &mem.VirtualMemoryStat{Total: 128 * mib, SwapTotal: 32 * mib, SwapFree: 31 * mib}, nil
	}
	var stats system.Stats
	a := &Agent{forceUseCgroup: true}
	a.updateMemoryStats(&stats)
	assert.Equal(t, [4]uint64{64 * mib, 3 * mib, 2 * mib, 0}, stats.MemBytes)
	assert.Equal(t, [2]uint64{32 * mib, mib}, stats.SwapBytes)
	assert.Equal(t, float64(0), stats.MemUsed)
}