Skip to content

Lightweight server monitoring with historical data, docker stats, and alerts.

21.4 KB
Raw Blame
//go:build testing

package agent

import (
	"errors"
	"fmt"
	"os/exec"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/henrygd/beszel/agent/btrfs"
	"github.com/henrygd/beszel/agent/zfs"
	"github.com/henrygd/beszel/internal/entities/system"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestOptionalPoolSource(t *testing.T) {
	failure := errors.New("timeout")
	for _, err := range []error{nil, zfs.ErrNoZfs, fmt.Errorf("zpool: %w", exec.ErrNotFound), errors.ErrUnsupported, failure} {
		_, got := optionalPoolSource(func() ([]zfs.PoolStat, error) { return nil, err })()
		if err == failure {
			assert.ErrorIs(t, got, failure)
		} else {
			assert.NoError(t, got)
		}
	}
}

type poolTestBackend struct {
	name  string
	err   error
	alloc uint64
	read  uint64
	empty bool
}

func (state *poolTestBackend) backend() *poolBackend {
	name := "zfs"
	if strings.HasPrefix(state.name, "b:") {
		name = "btrfs"
	}
	return &poolBackend{
		name: name,
		poolStatsFn: func() ([]zfs.PoolStat, error) {
			if state.empty {
				return nil, state.err
			}
			return []zfs.PoolStat{{Name: state.name, Size: 100, Alloc: state.alloc}}, state.err
		},
		kernelStatsFn: func() ([]zfs.PoolKernelStat, error) {
			return []zfs.PoolKernelStat{{Name: state.name, NRead: state.read}}, state.err
		},
		poolStatusesFn: func() ([]zfs.PoolStatus, error) { return nil, nil },
		datasetsFn:     func() ([]zfs.Dataset, error) { return nil, nil },
	}
}

func TestIndependentPoolBackendCaches(t *testing.T) {
	for _, failed := range []int{0, 1} {
		t.Run([]string{"zfs", "btrfs"}[failed], func(t *testing.T) {
			states := []*poolTestBackend{{name: "tank", alloc: 10}, {name: "b:uuid", alloc: 10}}
			managers := []*poolBackend{states[0].backend(), states[1].backend()}
			zm := &StoragePoolManager{backends: managers, detailInterval: time.Hour}
			var stats system.Stats
			zm.Update(&stats)
			require.Len(t, stats.ZfsPools, 2)
			require.True(t, zm.GetDetail(true).Complete)
			baseline := poolKernelSample{at: time.Now().Add(-time.Second)}
			for i, m := range managers {
				m.lastPoolStats = time.Time{}
				m.kernelSamples[states[i].name] = baseline
				states[i].alloc = 20
				states[i].read = 100
			}
			states[failed].err = errors.New("collection failed")
			zm.Update(&stats)
			healthy := 1 - failed
			assert.Equal(t, uint64(10), managers[failed].poolData[0].Alloc)
			assert.Equal(t, uint64(20), managers[healthy].poolData[0].Alloc)
			assert.Equal(t, baseline, managers[failed].kernelSamples[states[failed].name])
			assert.Zero(t, stats.ZfsPools[states[failed].name].ReadBytes)
			assert.Positive(t, stats.ZfsPools[states[healthy].name].ReadBytes)
			partial := zm.GetDetail(true)
			assert.False(t, partial.Complete)
			assert.False(t, partial.CanRefreshPool(states[failed].name))
			assert.True(t, partial.CanRefreshPool(states[healthy].name))
			assert.Equal(t, uint64(10), partial.Pools[failed].Alloc)
			assert.Equal(t, uint64(20), partial.Pools[healthy].Alloc)
			assert.False(t, zm.GetDetail(false).Complete, "a failed forced refresh must not become complete from cache")

			// Successful empty inventory removes only the healthy backend's pool.
			states[healthy].empty = true
			managers[healthy].lastPoolStats = time.Time{}
			zm.Update(&stats)
			require.Len(t, stats.ZfsPools, 1)
			assert.Contains(t, stats.ZfsPools, states[failed].name)
			partial = zm.GetDetail(true)
			require.Len(t, partial.Pools, 1)
			assert.True(t, partial.CanRefreshPool(states[healthy].name))

			// Recovery uses the retained I/O baseline, then normal removal works.
			states[failed].err = nil
			managers[failed].lastPoolStats = time.Time{}
			zm.Update(&stats)
			assert.Positive(t, stats.ZfsPools[states[failed].name].ReadBytes)
			assert.True(t, zm.GetDetail(true).Complete)
			states[failed].empty = true
			managers[failed].lastPoolStats = time.Time{}
			zm.Update(&stats)
			assert.Empty(t, stats.ZfsPools)
			assert.Empty(t, zm.GetDetail(true).Pools)
		})
	}
}

func TestIndependentBackendsWithoutCache(t *testing.T) {
	z := &poolTestBackend{name: "tank", err: errors.New("ZFS failure")}
	b := &poolTestBackend{name: "b:uuid", alloc: 20}
	zm := &StoragePoolManager{backends: []*poolBackend{z.backend(), b.backend()}, detailInterval: time.Hour}
	var stats system.Stats
	zm.Update(&stats)
	require.Len(t, stats.ZfsPools, 1)
	assert.Contains(t, stats.ZfsPools, "b:uuid")
	detail := zm.GetDetail(true)
	require.Len(t, detail.Pools, 1)
	assert.False(t, detail.Complete)
	assert.Equal(t, []string{"btrfs"}, detail.CompleteBackends)
}

func TestConcurrentBackendDetailsAndMetrics(t *testing.T) {
	zm := &StoragePoolManager{backends: []*poolBackend{(&poolTestBackend{name: "tank"}).backend(), (&poolTestBackend{name: "b:uuid"}).backend()}, detailInterval: time.Hour}
	var wg sync.WaitGroup
	for i := 0; i < 3; i++ {
		wg.Add(1)
		go func(metrics bool) {
			defer wg.Done()
			for j := 0; j < 10; j++ {
				if metrics {
					zm.Update(&system.Stats{})
				} else {
					zm.GetDetail(true)
				}
			}
		}(i == 0)
	}
	wg.Wait()
}

func TestStoragePoolBackendOrder(t *testing.T) {
	z := (&poolTestBackend{name: "tank"}).backend()
	b := (&poolTestBackend{name: "b:uuid"}).backend()
	b.datasetsFn = nil // Btrfs does not expose datasets.
	z.datasetsFn = func() ([]zfs.Dataset, error) {
		return []zfs.Dataset{{Name: "tank/data", Mountpoint: "/tank", Used: 10}}, nil
	}
	m := &StoragePoolManager{backends: []*poolBackend{b, z}, detailInterval: time.Hour}
	var stats system.Stats
	m.Update(&stats)
	require.Len(t, stats.ZfsPools, 2)
	detail := m.GetDetail(true)
	require.True(t, detail.Complete)
	assert.Equal(t, []string{"btrfs", "zfs"}, detail.CompleteBackends)
	assert.Empty(t, detail.Pools[0].Datasets)
	assert.Len(t, detail.Pools[1].Datasets, 1)
	assert.Equal(t, uint64(10), m.DatasetUsage()["/tank"].used)

	b.poolData[0].MountID = "uuid"
	b.poolData[0].IODevice = "sda"
	calls := 0
	m.markDuplicateCharts(&stats, map[string]*system.FsStats{
		"sda": {Mountpoint: "/", DiskTotal: 100},
	}, func(string) string { calls++; return "uuid" })
	assert.Equal(t, 1, calls, "resolve each filesystem once across all backends")
	assert.True(t, stats.ZfsPools["b:uuid"].HideUsage)
	assert.True(t, stats.ZfsPools["b:uuid"].HideIO)
}

func TestUpdatePopulatesZfsPools(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		return []zfs.PoolStat{{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "DEGRADED"}}, nil
	}
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
		return []zfs.Dataset{
			{Name: "tank/apps", Used: 5000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
			{Name: "tank/backup", Used: 6000000000000, Avail: 11999000000000, Mountpoint: "/tank/backup"},
			// Small zvol (Proxmox VM EFI disk): must not round to zero.
			{Name: "rpool/vm-100-disk-2", Used: 4194304, Avail: 0, Mountpoint: "-"},
		}, nil
	}
	var kernelCalls int
	zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
		kernelCalls++
		return []zfs.PoolKernelStat{{
			Name: "tank", Health: "ONLINE",
			NRead: uint64(kernelCalls-1) * 1250, NWrite: uint64(kernelCalls-1) * 5120,
		}}, nil
	}

	var stats system.Stats
	// The first kernel sample establishes the cumulative-counter baseline.
	zm.Update(&stats)
	zm.backends[0].kernelSamples["tank"] = poolKernelSample{at: time.Now().Add(-time.Second)}
	zm.Update(&stats)
	require.NotNil(t, stats.ZfsPools)
	require.Contains(t, stats.ZfsPools, "tank")
	assert.InDelta(t, 22350.8105, stats.ZfsPools["tank"].Total, 0.0001) // Size in GiB
	assert.InDelta(t, 11175.8709, stats.ZfsPools["tank"].Used, 0.0001)  // Alloc in GiB
	assert.Equal(t, "ONLINE", stats.ZfsPools["tank"].Health)
	assert.InDelta(t, 1250, stats.ZfsPools["tank"].ReadBytes, 5)
	assert.InDelta(t, 5120, stats.ZfsPools["tank"].WriteBytes, 5)

}

// TestUpdateKernelStatsMissing verifies pools without a kernel sample report zero
// I/O instead of erroring.
func TestUpdateKernelStatsMissing(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		return []zfs.PoolStat{{Name: "tank", Size: 1, Alloc: 1, Health: "ONLINE"}}, nil
	}
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
	zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
		return nil, zfs.ErrNoZfs
	}

	var stats system.Stats
	zm.Update(&stats)
	require.NotNil(t, stats.ZfsPools)
	assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
	assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
}

func TestUpdateKernelCounterReset(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		return []zfs.PoolStat{{Name: "tank", Health: "ONLINE"}}, nil
	}
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
	zm.backends[0].kernelSamples = map[string]poolKernelSample{
		"tank": {nread: 100, nwrite: 200, at: time.Now().Add(-time.Second)},
	}
	zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
		return []zfs.PoolKernelStat{{Name: "tank", Health: "ONLINE", NRead: 10, NWrite: 20}}, nil
	}

	var stats system.Stats
	zm.Update(&stats)
	assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
	assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
}

func TestUpdateNoZfs(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	calls := 0
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		calls++
		return nil, zfs.ErrNoZfs
	}

	var stats system.Stats
	zm.Update(&stats)
	zm.Update(&stats)
	assert.Nil(t, stats.ZfsPools)
	assert.Equal(t, 1, calls, "failed pool discovery should be cached until the next refresh interval")
}

func TestUpdateEmptyPools(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	calls := 0
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		calls++
		return nil, nil
	}

	var stats system.Stats
	zm.Update(&stats)
	zm.Update(&stats)
	assert.Nil(t, stats.ZfsPools)
	assert.Equal(t, 1, calls, "an empty pool inventory should be cached until the next refresh interval")
}

func TestDatasetUsage(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	calls := 0
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
		calls++
		return []zfs.Dataset{
			{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
			{Name: "tank/apps", Used: 1000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
			{Name: "rpool", Used: 900000000000, Avail: 300000000000, Mountpoint: "-"}, // zvol/unmounted: excluded
		}, nil
	}

	usage := zm.DatasetUsage()
	require.Len(t, usage, 2)
	assert.Equal(t, zfsDatasetUsage{used: 12000000000000, avail: 11999000000000}, usage["/tank"])
	assert.Equal(t, zfsDatasetUsage{used: 1000000000000, avail: 11999000000000}, usage["/tank/apps"])
	assert.Equal(t, 1, calls)

	// Second call within the refresh window must not re-run the collector.
	zm.DatasetUsage()
	assert.Equal(t, 1, calls)
}

func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
		return []zfs.Dataset{{Name: "tank", Used: 1, Avail: 1, Mountpoint: "/tank"}}, nil
	}
	assert.Len(t, zm.DatasetUsage(), 1)

	// Force refresh window expiry, then a failing collector.
	zm.backends[0].lastUsageRefresh = time.Now().Add(-10 * time.Minute)
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
		return nil, zfs.ErrNoZfs
	}
	usage := zm.DatasetUsage()
	assert.Len(t, usage, 1, "previous usage should be retained on error")
}

func TestDatasetUsageClearsAbsentBackend(t *testing.T) {
	b := newZfsBackend()
	b.datasetUsage = map[string]zfsDatasetUsage{"/tank": {used: 1, avail: 1}}
	b.datasetsFn = optionalPoolSource(func() ([]zfs.Dataset, error) {
		return nil, zfs.ErrNoZfs
	})

	datasets, err := b.datasets()
	require.NoError(t, err, "an absent backend must not produce an error to log")
	assert.Empty(t, datasets)
	b.refreshDatasetUsage()
	assert.Empty(t, b.datasetUsage)
	assert.False(t, b.lastUsageRefresh.IsZero())
}

func TestGetDetailForceRefresh(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	poolCalls := 0
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		poolCalls++
		return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
	}
	zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }

	first := zm.GetDetail(false)
	assert.True(t, first.Complete)
	require.Len(t, first.Pools, 1)
	assert.Equal(t, uint64(1), first.Pools[0].Alloc)

	cached := zm.GetDetail(false)
	require.Len(t, cached.Pools, 1)
	assert.Equal(t, uint64(1), cached.Pools[0].Alloc)
	assert.Equal(t, 1, poolCalls)

	refreshed := zm.GetDetail(true)
	assert.True(t, refreshed.Complete)
	require.Len(t, refreshed.Pools, 1)
	assert.Equal(t, uint64(2), refreshed.Pools[0].Alloc)
	assert.Equal(t, 2, poolCalls)
}

func TestGetDetailSuccessfulEmptyInventoryClearsCache(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		return []zfs.PoolStat{{Name: "tank"}}, nil
	}
	zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }

	require.Len(t, zm.GetDetail(false).Pools, 1)
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, nil }
	empty := zm.GetDetail(true)
	assert.True(t, empty.Complete)
	assert.Empty(t, empty.Pools)
}

func TestGetDetailFailureReturnsIncompleteCachedInventory(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
		return []zfs.PoolStat{{Name: "tank"}}, nil
	}
	zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) {
		return []zfs.PoolStatus{{Name: "tank", Status: "test advisory", Vdevs: []zfs.VdevStatus{{Name: "mirror-0"}}}}, nil
	}
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
		return []zfs.Dataset{{Name: "tank/data"}}, nil
	}
	first := zm.GetDetail(false)
	require.True(t, first.Complete)
	assert.Equal(t, "test advisory", first.Pools[0].Status)
	require.Len(t, first.Pools[0].Vdevs, 1)
	require.Len(t, first.Pools[0].Datasets, 1)

	zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, zfs.ErrNoZfs }
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, zfs.ErrNoZfs }
	partial := zm.GetDetail(true)
	require.True(t, partial.Complete)
	assert.Equal(t, "test advisory", partial.Pools[0].Status)
	require.Len(t, partial.Pools[0].Vdevs, 1)
	require.Len(t, partial.Pools[0].Datasets, 1)

	zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, zfs.ErrNoZfs }
	lastSuccessfulRefresh := zm.backends[0].lastDetailRefresh
	failed := zm.GetDetail(true)
	assert.False(t, failed.Complete)
	require.Len(t, failed.Pools, 1)
	assert.Equal(t, "tank", failed.Pools[0].Name)
	assert.Equal(t, lastSuccessfulRefresh, zm.backends[0].lastDetailRefresh)
}

func TestZfsMountpoints(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
	zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
		return []zfs.Dataset{
			{Name: "tank", Mountpoint: "/tank"},
			{Name: "rpool/ROOT/pve-1", Mountpoint: "/"},
		}, nil
	}
	mountpoints := zm.ZfsMountpoints()
	assert.Len(t, mountpoints, 2)
	assert.True(t, mountpoints["/tank"])
	assert.True(t, mountpoints["/"])
}

func TestBtrfsRawCapacityPropagates(t *testing.T) {
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolStatsFn: func() ([]zfs.PoolStat, error) {
		return []zfs.PoolStat{btrfsPoolStats(btrfs.Filesystem{UUID: "raw", Name: "raw", Size: 200, Alloc: 100, Raw: true})}, nil
	},
		poolStatusesFn: func() ([]zfs.PoolStatus, error) { return nil, nil },
		datasetsFn:     func() ([]zfs.Dataset, error) { return nil, nil }}}}
	var stats system.Stats
	zm.Update(&stats)
	require.True(t, stats.ZfsPools["b:raw"].Raw)
	detail := zm.GetDetail(true)
	require.True(t, detail.Complete)
	require.Len(t, detail.Pools, 1)
	assert.True(t, detail.Pools[0].Raw)
}

func TestMarkDuplicatePoolCharts(t *testing.T) {
	for _, tc := range []struct {
		name, poolID, device string
		raw                  bool
		diskTotal            float64
		wantUsage, wantIO    bool
	}{
		{"single device root", "fs1", "dm-0", false, 100, true, true},
		{"multi device", "fs1", "", false, 100, true, false},
		{"different IO device", "fs1", "nvme0n1", false, 100, true, false},
		{"different filesystem", "fs2", "dm-0", false, 100, false, false},
		{"unknown identity", "", "dm-0", false, 100, false, false},
		{"raw usage", "fs1", "dm-0", true, 100, false, true},
		{"failed disk collection", "fs1", "dm-0", false, 0, false, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolData: []zfs.PoolStat{{Name: "arbitrary label", MountID: tc.poolID, IODevice: tc.device, Raw: tc.raw}}}}}
			stats := &system.Stats{ZfsPools: map[string]*system.ZfsPool{"arbitrary label": {}}}
			fs := map[string]*system.FsStats{"dm-0": {Root: true, Mountpoint: "/", DiskTotal: tc.diskTotal}}
			zm.markDuplicateCharts(stats, fs, func(string) string { return "fs1" })
			assert.Equal(t, tc.wantUsage, stats.ZfsPools["arbitrary label"].HideUsage)
			assert.Equal(t, tc.wantIO, stats.ZfsPools["arbitrary label"].HideIO)
			// Bind mounts and custom extra-filesystem names have the same identity.
			fs["dm-0"].Root = false
			fs["dm-0"].Mountpoint = "/extra-filesystems/storage"
			fs["dm-0"].Name = "custom name"
			stats.ZfsPools["arbitrary label"] = &system.ZfsPool{}
			zm.markDuplicateCharts(stats, fs, func(string) string { return "fs1" })
			assert.Equal(t, tc.wantUsage, stats.ZfsPools["arbitrary label"].HideUsage)
			assert.Equal(t, tc.wantIO, stats.ZfsPools["arbitrary label"].HideIO)
		})
	}
}

func TestBtrfsPoolIdentities(t *testing.T) {
	old := btrfsFilesystems
	t.Cleanup(func() { btrfsFilesystems = old })
	label := "tank"
	btrfsFilesystems = func() ([]btrfs.Filesystem, error) {
		return []btrfs.Filesystem{
			{UUID: "11111111-1111-4111-8111-111111111111", Name: label, Size: 100, Health: "ONLINE", NRead: 100, Devices: []btrfs.Device{{Name: "first"}}},
			{UUID: "22222222-2222-4222-8222-222222222222", Name: "tank", Size: 200, Health: "DEGRADED", NRead: 200, Devices: []btrfs.Device{{Name: "second"}}},
		}, nil
	}
	zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolStatsFn: func() ([]zfs.PoolStat, error) { return []zfs.PoolStat{{Name: "tank", Size: 300}}, nil },
		kernelStatsFn: func() ([]zfs.PoolKernelStat, error) { return []zfs.PoolKernelStat{{Name: "tank", NRead: 300}}, nil },
		poolStatusesFn: func() ([]zfs.PoolStatus, error) {
			return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "zfs-device"}}}}, nil
		},

		datasetsFn: func() ([]zfs.Dataset, error) { return []zfs.Dataset{{Name: "tank/data"}}, nil }}, newBtrfsBackend()}}
	first := "b:11111111-1111-4111-8111-111111111111"
	second := "b:22222222-2222-4222-8222-222222222222"
	var stats system.Stats
	zm.Update(&stats)
	require.Len(t, stats.ZfsPools, 3)
	assert.Contains(t, stats.ZfsPools, "tank")
	assert.Equal(t, "ONLINE", stats.ZfsPools[first].Health)
	assert.Equal(t, "DEGRADED", stats.ZfsPools[second].Health)
	assert.Equal(t, uint64(100), zm.backends[1].kernelSamples[first].nread)
	assert.Equal(t, uint64(200), zm.backends[1].kernelSamples[second].nread)
	detail := zm.GetDetail(true)
	require.Len(t, detail.Pools, 3)
	assert.Equal(t, "zfs-device", detail.Pools[0].Vdevs[0].Name)
	assert.Len(t, detail.Pools[0].Datasets, 1)
	assert.Equal(t, "first", detail.Pools[1].Vdevs[0].Name)
	assert.Empty(t, detail.Pools[1].Datasets)
	assert.Equal(t, "second", detail.Pools[2].Vdevs[0].Name)
	label = "renamed"
	zm.backends[0].lastPoolStats = time.Time{}
	zm.backends[1].lastPoolStats = time.Time{}
	zm.Update(&stats)
	require.Len(t, stats.ZfsPools, 3)
	assert.Equal(t, "renamed", stats.ZfsPools[first].DisplayName)
	assert.Equal(t, first, zm.GetDetail(true).Pools[1].Name)
	assert.Equal(t, "renamed", zm.GetDetail(true).Pools[1].DisplayName)
}

func TestStaleUtilityCachesRefreshInBackground(t *testing.T) {
	release := make(chan struct{})
	b := &poolBackend{name: "zfs"}
	b.poolStatsFn = func() ([]zfs.PoolStat, error) {
		<-release
		return []zfs.PoolStat{{Name: "new"}}, nil
	}
	b.datasetsFn = func() ([]zfs.Dataset, error) {
		<-release
		return []zfs.Dataset{{Name: "new", Mountpoint: "/new"}}, nil
	}
	b.poolData = []zfs.PoolStat{{Name: "old"}}
	b.lastPoolStats = time.Now().Add(-2 * poolStatsRefreshInterval)
	b.datasetUsage = map[string]zfsDatasetUsage{"/old": {}}
	b.lastUsageRefresh = time.Now().Add(-2 * datasetUsageRefreshInterval)

	// A hung utility must not block collection; cached data is served meanwhile.
	for range 2 {
		assert.Equal(t, "old", b.poolStats()[0].Name)
		assert.Contains(t, b.refreshDatasetUsage(), "/old")
	}

	close(release)
	require.Eventually(t, func() bool {
		return b.poolStats()[0].Name == "new" && b.refreshDatasetUsage()["/new"] == zfsDatasetUsage{}
	}, time.Second, time.Millisecond)
}