Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

20.5 KB
Raw Blame
package hostmetrics

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	"github.com/rs/zerolog/log"
)

const (
	maxZpoolCommandOutputSize = 1 << 20 // 1 MiB
	maxZFSCommandOutputSize   = 4 << 20 // 4 MiB
	maxZFSDatasetsPerPool     = 128
	maxZFSDatasetNameBytes    = 512
	maxZFSMountpointBytes     = 1024
)

var errZpoolCommandOutputTooLarge = errors.New("zpool command output exceeded limit")

// zpoolStats represents capacity data reported by `zpool list`.
type zpoolStats struct {
	Size  uint64
	Alloc uint64
	Free  uint64
}

// zfsDatasetUsage preserves per-dataset usage so we can reconcile pools later.
type zfsDatasetUsage struct {
	Pool       string
	Dataset    string
	Mountpoint string
	Total      uint64
	Used       uint64
	Free       uint64
}

var queryZpoolStats = fetchZpoolStats
var queryZFSDatasets = fetchZFSDatasets
var zpoolLookPath = exec.LookPath
var zpoolStat = os.Stat
var zpoolCommandRunner = runZpoolCommand
var zfsLookPath = exec.LookPath
var zfsStat = os.Stat
var zfsCommandRunner = runZFSCommand

type limitedBuffer struct {
	buf      bytes.Buffer
	maxBytes int
	exceeded bool
}

func (b *limitedBuffer) Write(p []byte) (int, error) {
	remaining := b.maxBytes - b.buf.Len()
	if remaining <= 0 {
		b.exceeded = true
		return 0, errZpoolCommandOutputTooLarge
	}
	if len(p) > remaining {
		b.exceeded = true
		written, _ := b.buf.Write(p[:remaining])
		return written, errZpoolCommandOutputTooLarge
	}
	return b.buf.Write(p)
}

func (b *limitedBuffer) Bytes() []byte {
	return b.buf.Bytes()
}

func summarizeZFSPools(ctx context.Context, datasets []zfsDatasetUsage) []agentshost.Disk {
	if len(datasets) == 0 {
		log.Debug().Msg("zfs: no datasets to summarize")
		return nil
	}

	pools := uniqueZFSPools(datasets)
	if len(pools) == 0 {
		log.Debug().Msg("zfs: no unique pools found from datasets")
		return nil
	}
	log.Debug().Int("datasetCount", len(datasets)).Strs("pools", pools).Msg("zfs: summarizing pools")

	bestDatasets := bestZFSPoolDatasets(datasets)
	mountpoints := bestZFSMountpoints(datasets)
	for pool, ds := range bestDatasets {
		log.Debug().Str("pool", pool).Str("dataset", ds.Dataset).Str("mount", ds.Mountpoint).Uint64("total", ds.Total).Uint64("used", ds.Used).Msg("zfs: best dataset for pool")
	}

	stats, err := queryZpoolStats(ctx, pools)
	if err == nil && len(stats) > 0 {
		log.Debug().Int("zpoolStatsCount", len(stats)).Msg("zfs: using zpool stats")
		return disksFromZpoolStats(pools, stats, mountpoints, bestDatasets)
	}

	if err != nil {
		log.Debug().Err(err).Msg("zfs: zpool stats unavailable, using fallback")
	} else {
		log.Debug().Msg("zfs: zpool stats query returned no data, using fallback")
	}
	return fallbackZFSDisks(bestDatasets, mountpoints)
}

// newZFSDisk constructs a Disk entry for a ZFS pool with computed usage.
func newZFSDisk(pool, mountpoint string, total, used, free uint64) agentshost.Disk {
	return agentshost.Disk{
		Device:     pool,
		Mountpoint: mountpoint,
		Filesystem: "zfs",
		Type:       "zfs",
		TotalBytes: int64(total),
		UsedBytes:  int64(used),
		FreeBytes:  int64(free),
		Usage:      clampPercent(calculatePercent(total, used)),
	}
}

func disksFromZpoolStats(
	pools []string,
	stats map[string]zpoolStats,
	mountpoints map[string]string,
	bestDatasets map[string]zfsDatasetUsage,
) []agentshost.Disk {
	disks := make([]agentshost.Disk, 0, len(pools))

	for _, pool := range pools {
		stat, ok := stats[pool]
		mp := mountpoints[pool]
		if mp == "" {
			mp = fmt.Sprintf("zpool:%s", pool)
		}

		ds := bestDatasets[pool]
		log.Debug().Str("pool", pool).Bool("hasZpoolStats", ok).Uint64("zpoolSize", stat.Size).Uint64("zpoolAlloc", stat.Alloc).Uint64("zpoolFree", stat.Free).Uint64("dsTotal", ds.Total).Uint64("dsUsed", ds.Used).Str("mount", mp).Msg("zfs: processing pool")

		if ok && stat.Size > 0 {
			// Compute pool-level usable capacity by combining zpool stats with
			// dataset stats. ZFS statfs on a dataset returns per-dataset Used
			// (missing zvols and other datasets), but its Free reflects real
			// pool-available space. We use the ratio ds.Free/stat.Free to
			// convert the raw zpool Size to usable capacity. This handles
			// RAIDZ (parity overhead), mirrors, and simple pools uniformly,
			// and Used = Total - Free captures all pool consumers including
			// zvols. (issues #1052, mirror-vdev fix)
			totalBytes := stat.Size
			freeBytes := stat.Free
			if ds.Free > 0 && stat.Free > 0 && stat.Free >= ds.Free {
				// Convert raw pool total to usable capacity using the
				// raw-to-usable ratio derived from free space.
				// For mirrors the ratio is ~1 (no overhead).
				// For RAIDZ the ratio is (N-P)/N (parity overhead).
				totalBytes = uint64(float64(stat.Size) * (float64(ds.Free) / float64(stat.Free)))
				freeBytes = ds.Free
				log.Debug().Str("pool", pool).Uint64("usableTotal", totalBytes).Uint64("usableFree", freeBytes).Uint64("zpoolSize", stat.Size).Uint64("zpoolFree", stat.Free).Uint64("dsFree", ds.Free).Msg("zfs: computed usable capacity from free-space ratio")
			} else {
				log.Debug().Str("pool", pool).Uint64("zpoolSize", stat.Size).Uint64("zpoolFree", stat.Free).Uint64("dsFree", ds.Free).Msg("zfs: using raw zpool stats (no usable dataset free)")
			}
			usedBytes := totalBytes - freeBytes
			if freeBytes > totalBytes {
				usedBytes = 0
			}

			log.Debug().Str("pool", pool).Int64("totalBytes", int64(totalBytes)).Int64("usedBytes", int64(usedBytes)).Int64("freeBytes", int64(freeBytes)).Float64("usage", clampPercent(calculatePercent(totalBytes, usedBytes))).Msg("zfs: emitting disk entry")
			disks = append(disks, newZFSDisk(pool, mp, totalBytes, usedBytes, freeBytes))
			continue
		}

		if ds.Total > 0 {
			log.Debug().Str("pool", pool).Int64("totalBytes", int64(ds.Total)).Int64("usedBytes", int64(ds.Used)).Float64("usage", clampPercent(calculatePercent(ds.Total, ds.Used))).Msg("zfs: emitting disk entry from dataset only (no zpool stats)")
			disks = append(disks, newZFSDisk(pool, mp, ds.Total, ds.Used, ds.Free))
		} else {
			log.Debug().Str("pool", pool).Msg("zfs: skipping pool with no zpool stats and zero dataset total")
		}
	}

	return disks
}

func fallbackZFSDisks(bestDatasets map[string]zfsDatasetUsage, mountpoints map[string]string) []agentshost.Disk {
	log.Debug().Int("poolCount", len(bestDatasets)).Msg("zfs: fallback disk generation")
	if len(bestDatasets) == 0 {
		return nil
	}

	pools := make([]string, 0, len(bestDatasets))
	for pool := range bestDatasets {
		pools = append(pools, pool)
	}
	sort.Strings(pools)

	disks := make([]agentshost.Disk, 0, len(pools))
	for _, pool := range pools {
		ds := bestDatasets[pool]
		if ds.Total == 0 {
			continue
		}

		mp := mountpoints[pool]
		if mp == "" {
			mp = fmt.Sprintf("zpool:%s", pool)
		}

		disks = append(disks, newZFSDisk(pool, mp, ds.Total, ds.Used, ds.Free))
	}

	return disks
}

func enrichZFSPoolDisksWithDatasets(
	ctx context.Context,
	disks []agentshost.Disk,
	mounted []zfsDatasetUsage,
) {
	if len(disks) == 0 {
		return
	}
	pools := make([]string, 0, len(disks))
	for _, disk := range disks {
		if pool, ok := normalizeZFSPoolName(disk.Device); ok {
			pools = append(pools, pool)
		}
	}

	datasets, err := queryZFSDatasets(ctx, pools)
	if err != nil {
		log.Debug().Err(err).Msg("zfs: dataset inventory unavailable, using mounted datasets")
		datasets = mountedZFSDatasets(mounted)
	}
	for index := range disks {
		disks[index].ZFSDatasets = append(
			[]agentshost.ZFSDataset(nil),
			datasets[disks[index].Device]...,
		)
	}
}

func mountedZFSDatasets(mounted []zfsDatasetUsage) map[string][]agentshost.ZFSDataset {
	result := make(map[string][]agentshost.ZFSDataset)
	seen := make(map[string]struct{})
	for _, dataset := range mounted {
		pool, ok := normalizeZFSPoolName(dataset.Pool)
		name := strings.TrimSpace(dataset.Dataset)
		mountpoint := strings.TrimSpace(dataset.Mountpoint)
		if !ok || name == pool || !strings.HasPrefix(name, pool+"/") ||
			len(name) > maxZFSDatasetNameBytes || len(mountpoint) > maxZFSMountpointBytes ||
			len(result[pool]) >= maxZFSDatasetsPerPool {
			continue
		}
		if _, duplicate := seen[name]; duplicate {
			continue
		}
		seen[name] = struct{}{}
		result[pool] = append(result[pool], agentshost.ZFSDataset{
			Name:           name,
			Type:           "filesystem",
			Mountpoint:     mountpoint,
			UsedBytes:      int64(dataset.Used),
			AvailableBytes: int64(dataset.Free),
		})
	}
	for pool := range result {
		sort.Slice(result[pool], func(i, j int) bool {
			return result[pool][i].Name < result[pool][j].Name
		})
	}
	return result
}

// commonZpoolPaths lists common locations for the zpool binary.
// TrueNAS SCALE, FreeBSD, and various Linux distributions may install
// zpool in different locations that might not be in the agent's PATH.
// This helps fix issue #718 where TrueNAS reports inflated storage.
var commonZpoolPaths = []string{
	"/usr/sbin/zpool",       // TrueNAS SCALE, Debian, Ubuntu
	"/sbin/zpool",           // FreeBSD, older Linux
	"/usr/local/sbin/zpool", // FreeBSD ports, custom builds
	"/usr/local/bin/zpool",  // Custom installations
	"/opt/zfs/bin/zpool",    // Some enterprise Linux
	"/usr/bin/zpool",        // Some distributions
}

var commonZFSPaths = []string{
	"/usr/sbin/zfs",
	"/sbin/zfs",
	"/usr/local/sbin/zfs",
	"/usr/local/bin/zfs",
	"/opt/zfs/bin/zfs",
	"/usr/bin/zfs",
}

// findZpool returns the path to the zpool binary by preferring known absolute
// locations first, then falling back to PATH lookup.
func findZpool() (string, error) {
	// Prefer common absolute paths so execution does not depend on PATH order.
	// This is especially important for TrueNAS SCALE where the agent
	// might run with a restricted PATH that doesn't include /usr/sbin
	for _, path := range commonZpoolPaths {
		if _, err := zpoolStat(path); err == nil {
			log.Debug().Str("path", path).Msg("zfs: found zpool at hardcoded path")
			return path, nil
		}
	}

	// Fall back to PATH lookup for non-standard installations.
	path, err := zpoolLookPath("zpool")
	if err != nil {
		log.Debug().Msg("zfs: zpool binary not found in PATH or common locations")
		return "", fmt.Errorf("zpool binary not found in PATH or common locations")
	}
	path = filepath.Clean(path)
	if !filepath.IsAbs(path) {
		return "", fmt.Errorf("zpool path is not absolute: %q", path)
	}
	if _, err := zpoolStat(path); err != nil {
		return "", fmt.Errorf("zpool path unavailable: %w", err)
	}

	log.Debug().Str("path", path).Msg("zfs: found zpool via PATH")
	return path, nil
}

func findZFS() (string, error) {
	for _, path := range commonZFSPaths {
		if _, err := zfsStat(path); err == nil {
			return path, nil
		}
	}
	path, err := zfsLookPath("zfs")
	if err != nil {
		return "", fmt.Errorf("zfs binary not found in PATH or common locations")
	}
	path = filepath.Clean(path)
	if !filepath.IsAbs(path) {
		return "", fmt.Errorf("zfs path is not absolute: %q", path)
	}
	if _, err := zfsStat(path); err != nil {
		return "", fmt.Errorf("zfs path unavailable: %w", err)
	}
	return path, nil
}

func fetchZFSDatasets(
	ctx context.Context,
	pools []string,
) (map[string][]agentshost.ZFSDataset, error) {
	pools = filterValidZFSPoolNames(pools)
	if len(pools) == 0 {
		return nil, nil
	}
	path, err := findZFS()
	if err != nil {
		return nil, err
	}
	args := []string{
		"list", "-Hp", "-t", "filesystem,volume",
		"-o", "name,type,used,available,referenced,mountpoint",
	}
	args = append(args, pools...)
	output, stderr, err := zfsCommandRunner(ctx, path, args...)
	if err != nil {
		if errors.Is(err, errZpoolCommandOutputTooLarge) {
			return nil, fmt.Errorf("zfs list output exceeded %d bytes", maxZFSCommandOutputSize)
		}
		return nil, fmt.Errorf("zfs list failed (stderr %d bytes): %w", len(stderr), err)
	}
	return parseZFSList(output, pools)
}

func runZFSCommand(ctx context.Context, name string, args ...string) ([]byte, []byte, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	stdout := limitedBuffer{maxBytes: maxZFSCommandOutputSize}
	stderr := limitedBuffer{maxBytes: maxZFSCommandOutputSize}
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if stdout.exceeded || stderr.exceeded {
		return stdout.Bytes(), stderr.Bytes(), errZpoolCommandOutputTooLarge
	}
	return stdout.Bytes(), stderr.Bytes(), err
}

func parseZFSList(
	output []byte,
	allowedPools []string,
) (map[string][]agentshost.ZFSDataset, error) {
	allowed := make(map[string]struct{}, len(allowedPools))
	for _, pool := range filterValidZFSPoolNames(allowedPools) {
		allowed[pool] = struct{}{}
	}
	result := make(map[string][]agentshost.ZFSDataset)
	scanner := bufio.NewScanner(bytes.NewReader(output))
	scanner.Buffer(make([]byte, 64*1024), maxZFSCommandOutputSize)
	for scanner.Scan() {
		fields := strings.Split(scanner.Text(), "\t")
		if len(fields) != 6 {
			continue
		}
		name := strings.TrimSpace(fields[0])
		pool := zfsPoolFromDevice(name)
		if _, ok := allowed[pool]; !ok || name == pool || !strings.HasPrefix(name, pool+"/") ||
			len(name) > maxZFSDatasetNameBytes || len(result[pool]) >= maxZFSDatasetsPerPool {
			continue
		}
		datasetType := strings.ToLower(strings.TrimSpace(fields[1]))
		if datasetType != "filesystem" && datasetType != "volume" {
			continue
		}
		used, usedErr := strconv.ParseInt(fields[2], 10, 64)
		available, availableErr := strconv.ParseInt(fields[3], 10, 64)
		referenced, referencedErr := strconv.ParseInt(fields[4], 10, 64)
		if usedErr != nil || availableErr != nil || referencedErr != nil ||
			used < 0 || available < 0 || referenced < 0 {
			continue
		}
		mountpoint := strings.TrimSpace(fields[5])
		if mountpoint == "-" || mountpoint == "none" || mountpoint == "legacy" {
			mountpoint = ""
		}
		if len(mountpoint) > maxZFSMountpointBytes {
			continue
		}
		result[pool] = append(result[pool], agentshost.ZFSDataset{
			Name:            name,
			Type:            datasetType,
			Mountpoint:      mountpoint,
			UsedBytes:       used,
			AvailableBytes:  available,
			ReferencedBytes: referenced,
		})
	}
	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("scan zfs list output: %w", err)
	}
	for pool := range result {
		sort.Slice(result[pool], func(i, j int) bool {
			return result[pool][i].Name < result[pool][j].Name
		})
	}
	return result, nil
}

func fetchZpoolStats(ctx context.Context, pools []string) (map[string]zpoolStats, error) {
	if len(pools) == 0 {
		return nil, nil
	}
	pools = filterValidZFSPoolNames(pools)
	if len(pools) == 0 {
		return nil, fmt.Errorf("no valid zfs pool names to query")
	}

	path, err := findZpool()
	if err != nil {
		return nil, fmt.Errorf("zfs: locate zpool binary: %w", err)
	}

	args := []string{"list", "-Hp", "-o", "name,size,allocated,free"}
	args = append(args, pools...)

	log.Debug().Str("path", path).Strs("args", args).Msg("zfs: executing zpool list")
	output, stderr, err := zpoolCommandRunner(ctx, path, args...)
	if err != nil {
		if errors.Is(err, errZpoolCommandOutputTooLarge) {
			return nil, fmt.Errorf("zpool list output exceeded %d bytes", maxZpoolCommandOutputSize)
		}
		log.Debug().Err(err).Str("path", path).Strs("args", args).Int("stderrBytes", len(stderr)).Msg("zfs: zpool list failed")
		return nil, err
	}
	log.Debug().Int("outputBytes", len(output)).Msg("zfs: zpool list succeeded")

	stats, err := parseZpoolList(output)
	if err != nil {
		return nil, fmt.Errorf("zfs: parse zpool list output: %w", err)
	}
	return stats, nil
}

func runZpoolCommand(ctx context.Context, name string, args ...string) ([]byte, []byte, error) {
	cmd := exec.CommandContext(ctx, name, args...)
	stdout := limitedBuffer{maxBytes: maxZpoolCommandOutputSize}
	stderr := limitedBuffer{maxBytes: maxZpoolCommandOutputSize}
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	if stdout.exceeded || stderr.exceeded {
		return stdout.Bytes(), stderr.Bytes(), errZpoolCommandOutputTooLarge
	}
	return stdout.Bytes(), stderr.Bytes(), err
}

func parseZpoolList(output []byte) (map[string]zpoolStats, error) {
	stats := make(map[string]zpoolStats)
	scanner := bufio.NewScanner(bytes.NewReader(output))
	var firstParseErr error
	invalidLines := 0
	lineNumber := 0
	for scanner.Scan() {
		lineNumber++
		line := strings.TrimSpace(scanner.Text())
		if line == "" {
			continue
		}

		fields := strings.Split(line, "\t")
		if len(fields) < 4 {
			invalidLines++
			if firstParseErr == nil {
				firstParseErr = fmt.Errorf("line %d: expected at least 4 tab-separated fields", lineNumber)
			}
			continue
		}
		pool, ok := normalizeZFSPoolName(fields[0])
		if !ok {
			continue
		}

		size, err := strconv.ParseUint(fields[1], 10, 64)
		if err != nil {
			invalidLines++
			if firstParseErr == nil {
				firstParseErr = fmt.Errorf("line %d: parse size %q: %w", lineNumber, fields[1], err)
			}
			continue
		}
		alloc, err := strconv.ParseUint(fields[2], 10, 64)
		if err != nil {
			invalidLines++
			if firstParseErr == nil {
				firstParseErr = fmt.Errorf("line %d: parse allocated %q: %w", lineNumber, fields[2], err)
			}
			continue
		}
		free, err := strconv.ParseUint(fields[3], 10, 64)
		if err != nil {
			invalidLines++
			if firstParseErr == nil {
				firstParseErr = fmt.Errorf("line %d: parse free %q: %w", lineNumber, fields[3], err)
			}
			continue
		}

		stats[pool] = zpoolStats{
			Size:  size,
			Alloc: alloc,
			Free:  free,
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, fmt.Errorf("scan zpool list output: %w", err)
	}
	if len(stats) == 0 {
		if firstParseErr != nil {
			return nil, fmt.Errorf("zpool list returned no usable data (%d invalid lines): %w", invalidLines, firstParseErr)
		}
		return nil, errors.New("zpool list returned no usable data")
	}
	if invalidLines > 0 {
		log.Debug().
			Int("invalidLines", invalidLines).
			Int("validPools", len(stats)).
			Msg("zfs: skipped malformed zpool list rows")
	}
	return stats, nil
}

func uniqueZFSPools(datasets []zfsDatasetUsage) []string {
	set := make(map[string]struct{}, len(datasets))
	for _, ds := range datasets {
		pool, ok := normalizeZFSPoolName(ds.Pool)
		if !ok {
			continue
		}
		set[pool] = struct{}{}
	}
	if len(set) == 0 {
		return nil
	}

	pools := make([]string, 0, len(set))
	for pool := range set {
		pools = append(pools, pool)
	}
	sort.Strings(pools)
	return pools
}

func filterValidZFSPoolNames(pools []string) []string {
	filtered := make([]string, 0, len(pools))
	seen := make(map[string]struct{}, len(pools))
	for _, pool := range pools {
		normalized, ok := normalizeZFSPoolName(pool)
		if !ok {
			continue
		}
		if _, exists := seen[normalized]; exists {
			continue
		}
		seen[normalized] = struct{}{}
		filtered = append(filtered, normalized)
	}
	return filtered
}

func normalizeZFSPoolName(pool string) (string, bool) {
	pool = strings.TrimSpace(pool)
	if pool == "" || len(pool) > 255 {
		return "", false
	}
	if strings.HasPrefix(pool, "-") {
		return "", false
	}

	for _, r := range pool {
		switch {
		case r >= 'a' && r <= 'z':
		case r >= 'A' && r <= 'Z':
		case r >= '0' && r <= '9':
		case r == '_' || r == '-' || r == '.' || r == ':':
		default:
			return "", false
		}
	}

	return pool, true
}

func bestZFSMountpoints(datasets []zfsDatasetUsage) map[string]string {
	mounts := make(map[string]string, len(datasets))
	scores := make(map[string]int, len(datasets))

	for _, ds := range datasets {
		if ds.Pool == "" || ds.Mountpoint == "" {
			continue
		}

		score := zfsMountpointScore(ds)
		if current, ok := scores[ds.Pool]; ok && score >= current {
			continue
		}
		scores[ds.Pool] = score
		mounts[ds.Pool] = ds.Mountpoint
	}

	return mounts
}

func zfsMountpointScore(ds zfsDatasetUsage) int {
	// A dataset actually mounted at "/" is the best representative of the pool,
	// even if it is a child dataset (e.g. rpool/ROOT/pve-1), so it must outrank a
	// pool-root dataset mounted elsewhere (e.g. rpool at /mnt/storage).
	if strings.TrimSpace(ds.Mountpoint) == "/" {
		return 0
	}
	if ds.Dataset != "" && !strings.Contains(ds.Dataset, "/") {
		return 1
	}
	path := strings.Trim(ds.Mountpoint, "/")
	if path == "" {
		return 0
	}
	return 2 + strings.Count(path, "/")
}

func zfsPoolFromDevice(device string) string {
	device = strings.TrimSpace(device)
	if device == "" {
		return ""
	}
	if idx := strings.Index(device, "/"); idx >= 0 {
		return device[:idx]
	}
	return device
}

func calculatePercent(total, used uint64) float64 {
	if total == 0 {
		return 0
	}
	return (float64(used) / float64(total)) * 100
}

func clampPercent(value float64) float64 {
	switch {
	case value < 0:
		return 0
	case value > 100:
		return 100
	default:
		return value
	}
}

func bestZFSPoolDatasets(datasets []zfsDatasetUsage) map[string]zfsDatasetUsage {
	best := make(map[string]zfsDatasetUsage)
	for _, ds := range datasets {
		if ds.Pool == "" {
			continue
		}
		if current, ok := best[ds.Pool]; !ok || ds.Total > current.Total {
			best[ds.Pool] = ds
		}
	}
	return best
}