Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

18.4 KB
Raw Blame
package hostagent

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)

// Pre-compiled regexes for performance (avoid recompilation on each call)
var (
	mdDeviceRe    = regexp.MustCompile(`^\s*(md\d+)\s*:`)
	mdArrayPathRe = regexp.MustCompile(`^/dev/md\d+$`)
	mdCountRe     = regexp.MustCompile(`\[(\d+)/(\d+)\]\s+\[([U_]+)\]`)
	mdTokenRe     = regexp.MustCompile(`^([A-Za-z0-9_.!/-]+)\[(\d+)\](.*)$`)
	progressRe    = regexp.MustCompile(`=\s*([0-9]+(?:\.[0-9]+)?)%`)
	slotRe        = regexp.MustCompile(`^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s+(/dev/.+)$`)
	speedRe       = regexp.MustCompile(`speed=(\S+)`)
	// /proc/mdstat progress line action.
	// Example: "[==>..................]  check = 12.6% (...) finish=... speed=..."
	mdstatOperationRe = regexp.MustCompile(`(?i)\b(recovery|resync|check|reshape)\b\s*=`)
	mdadmLookPath     = exec.LookPath
	mdadmStat         = os.Stat
	openFileForRead   = func(name string) (io.ReadCloser, error) { return os.Open(name) }
	readFile          = func(name string) ([]byte, error) {
		file, err := openFileForRead(name)
		if err != nil {
			return nil, err
		}
		defer file.Close()

		// Enforce the size cap while reading to avoid allocating arbitrarily
		// large buffers before validation.
		return io.ReadAll(io.LimitReader(file, maxMDStatBytes+1))
	}
	resolveMdadmBinary    = resolveMdadmPath
	mdadmRunCommandOutput = func(ctx context.Context, name string, args ...string) ([]byte, error) {
		cmd := exec.CommandContext(ctx, name, args...)
		return cmd.Output()
	}
)

const (
	mdstatPath     = "/proc/mdstat"
	maxMDStatBytes = 1 << 20 // 1 MiB hard limit for /proc/mdstat reads.
)

// commonMdadmPaths lists common mdadm locations to avoid relying on PATH order.
var commonMdadmPaths = []string{
	"/usr/sbin/mdadm",
	"/sbin/mdadm",
	"/usr/local/sbin/mdadm",
	"/usr/bin/mdadm",
	"/bin/mdadm",
}

// CollectArrays discovers and collects status for all mdadm RAID arrays on the system.
// Returns an empty slice when no arrays are found. If mdadm is unavailable,
// /proc/mdstat still provides a baseline topology report.
func CollectRAIDArrays(ctx context.Context) ([]agentshost.RAIDArray, error) {
	mdstatArrays, err := listMDStatArrays(ctx)
	if err != nil {
		return nil, fmt.Errorf("list array devices: %w", err)
	}

	if len(mdstatArrays) == 0 {
		return nil, nil
	}

	// /proc/mdstat is the canonical discovery source. mdadm enriches that
	// baseline when available, but a detail probe failure must not hide a
	// kernel-reported array from Pulse.
	if !isMdadmAvailable(ctx) {
		return mdstatArrays, nil
	}

	// Collect detailed info for each array
	arrays := make([]agentshost.RAIDArray, 0, len(mdstatArrays))
	for _, fallback := range mdstatArrays {
		array, err := collectArrayDetail(ctx, fallback.Device)
		if err != nil {
			arrays = append(arrays, fallback)
			continue
		}
		arrays = append(arrays, mergeMDStatFallback(array, fallback))
	}

	return arrays, nil
}

// isMdadmAvailable checks if mdadm binary is accessible
func isMdadmAvailable(ctx context.Context) bool {
	mdadmPath, err := resolveMdadmBinary()
	if err != nil {
		return false
	}

	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
	defer cancel()

	_, err = mdadmRunCommandOutput(ctx, mdadmPath, "--version")
	return err == nil
}

// listArrayDevices scans /proc/mdstat to find all md devices
func listArrayDevices(_ context.Context) ([]string, error) {
	arrays, err := listMDStatArrays(context.Background())
	if err != nil {
		return nil, err
	}

	devices := make([]string, 0, len(arrays))
	for _, array := range arrays {
		devices = append(devices, array.Device)
	}

	return devices, nil
}

// listMDStatArrays scans /proc/mdstat and returns a baseline array report for
// every md device the kernel currently exposes.
func listMDStatArrays(_ context.Context) ([]agentshost.RAIDArray, error) {
	output, err := readProcMDStat()
	if err != nil {
		return nil, err
	}
	return parseMDStatArrays(string(output)), nil
}

func parseMDStatArrays(output string) []agentshost.RAIDArray {
	lines := strings.Split(output, "\n")
	arrays := make([]agentshost.RAIDArray, 0)

	for i := 0; i < len(lines); {
		line := strings.TrimSpace(lines[i])
		matches := mdDeviceRe.FindStringSubmatch(line)
		if len(matches) <= 1 {
			i++
			continue
		}

		section := []string{line}
		i++
		for i < len(lines) {
			next := strings.TrimSpace(lines[i])
			if next == "" || strings.HasPrefix(next, "Personalities") || strings.HasPrefix(next, "unused devices") {
				break
			}
			if mdDeviceRe.MatchString(next) {
				break
			}
			section = append(section, next)
			i++
		}

		arrays = append(arrays, parseMDStatArraySection(matches[1], section))
	}

	return arrays
}

func parseMDStatArraySection(deviceName string, section []string) agentshost.RAIDArray {
	array := agentshost.RAIDArray{
		Device:  "/dev/" + deviceName,
		Devices: []agentshost.RAIDDevice{},
	}
	if len(section) == 0 {
		return array
	}

	header := section[0]
	_, fieldsRaw, ok := strings.Cut(header, ":")
	if !ok {
		return array
	}

	fields := strings.Fields(fieldsRaw)
	if len(fields) > 0 {
		array.State = strings.ToLower(fields[0])
	}

	levelIndex := -1
	for i, field := range fields {
		if isMDStatLevelToken(field) {
			levelIndex = i
			array.Level = strings.ToLower(field)
			break
		}
	}

	deviceStart := 1
	if levelIndex >= 0 {
		deviceStart = levelIndex + 1
	}
	for _, field := range fields[deviceStart:] {
		device, ok := parseMDStatDeviceToken(field, array.State)
		if ok {
			array.Devices = append(array.Devices, device)
		}
	}

	for _, line := range section[1:] {
		if matches := mdCountRe.FindStringSubmatch(line); len(matches) > 3 {
			total, _ := strconv.Atoi(matches[1])
			active, _ := strconv.Atoi(matches[2])
			array.TotalDevices = total
			array.RequiredDevices = total
			array.ActiveDevices = active
		}

		if array.Operation == "" {
			if matches := mdstatOperationRe.FindStringSubmatch(line); len(matches) > 1 {
				array.Operation = strings.ToLower(matches[1])
			}
		}
		if array.RebuildSpeed == "" {
			if matches := speedRe.FindStringSubmatch(line); len(matches) > 1 {
				array.RebuildSpeed = matches[1]
			}
		}
		if array.RebuildPercent == 0 {
			if matches := progressRe.FindStringSubmatch(line); len(matches) > 1 {
				array.RebuildPercent, _ = strconv.ParseFloat(matches[1], 64)
			}
		}
	}

	var workingDevices, failedDevices, spareDevices int
	for _, device := range array.Devices {
		state := strings.ToLower(device.State)
		switch {
		case strings.Contains(state, "faulty"):
			failedDevices++
		case strings.Contains(state, "spare"):
			spareDevices++
			workingDevices++
		default:
			workingDevices++
		}
	}
	if array.ActiveDevices == 0 && array.RequiredDevices == 0 {
		array.ActiveDevices = workingDevices - spareDevices
	}
	if array.TotalDevices == 0 {
		array.TotalDevices = array.ActiveDevices
	}
	if mdStatUsesSparseRoleBitmap(array, failedDevices) {
		// QNAP's system RAID1 arrays expose a fixed-width bay/role bitmap such
		// as [24/2] [UU______________________], while their active members use
		// roles 24 and 25. Those unused bitmap positions are not failed disks.
		array.TotalDevices = array.ActiveDevices
		array.RequiredDevices = array.ActiveDevices
	}
	if array.TotalDevices > array.ActiveDevices && !strings.Contains(array.State, "degraded") {
		array.State = appendMDStatState(array.State, "degraded")
	}
	if failedDevices == 0 && array.TotalDevices > array.ActiveDevices {
		failedDevices = array.TotalDevices - array.ActiveDevices
	}
	array.WorkingDevices = workingDevices
	array.FailedDevices = failedDevices
	array.SpareDevices = spareDevices

	return array
}

func mdStatUsesSparseRoleBitmap(array agentshost.RAIDArray, failedDevices int) bool {
	if !strings.EqualFold(array.State, "active") ||
		!strings.EqualFold(array.Level, "raid1") ||
		failedDevices != 0 ||
		array.TotalDevices <= array.ActiveDevices ||
		array.ActiveDevices == 0 {
		return false
	}

	activeMembers := 0
	for _, device := range array.Devices {
		state := strings.ToLower(device.State)
		if strings.Contains(state, "faulty") || strings.Contains(state, "spare") {
			continue
		}
		activeMembers++
		if device.Slot < array.TotalDevices {
			return false
		}
	}

	return activeMembers == array.ActiveDevices
}

func parseMDStatDeviceToken(token string, arrayState string) (agentshost.RAIDDevice, bool) {
	token = strings.Trim(token, ",")
	matches := mdTokenRe.FindStringSubmatch(token)
	if len(matches) < 4 {
		return agentshost.RAIDDevice{}, false
	}

	slot, err := strconv.Atoi(matches[2])
	if err != nil {
		return agentshost.RAIDDevice{}, false
	}

	state := "active sync"
	flags := strings.ToUpper(matches[3])
	switch {
	case strings.Contains(flags, "F"):
		state = "faulty"
	case strings.Contains(flags, "S"):
		state = "spare"
	case strings.EqualFold(arrayState, "inactive"):
		state = "inactive"
	}

	device := strings.TrimSpace(matches[1])
	if !strings.HasPrefix(device, "/dev/") {
		device = "/dev/" + device
	}

	return agentshost.RAIDDevice{
		Device: device,
		State:  state,
		Slot:   slot,
	}, true
}

func isMDStatLevelToken(token string) bool {
	token = strings.ToLower(strings.TrimSpace(token))
	return strings.HasPrefix(token, "raid") || token == "linear" || token == "multipath"
}

func appendMDStatState(state, suffix string) string {
	state = strings.TrimSpace(state)
	if state == "" {
		return suffix
	}
	return state + ", " + suffix
}

func mergeMDStatFallback(array, fallback agentshost.RAIDArray) agentshost.RAIDArray {
	// A detail report with a configured requirement can legitimately have no
	// active members. Do not turn that observed zero into kernel-fallback
	// members from a different probe in the same collection cycle.
	detailRequirementKnown := array.RequiredDevices > 0
	if strings.TrimSpace(array.Device) == "" {
		array.Device = fallback.Device
	}
	if strings.TrimSpace(array.Level) == "" {
		array.Level = fallback.Level
	}
	if strings.TrimSpace(array.State) == "" {
		array.State = fallback.State
	}
	if array.RequiredDevices == 0 {
		array.RequiredDevices = fallback.RequiredDevices
	}
	if array.TotalDevices == 0 {
		array.TotalDevices = fallback.TotalDevices
	}
	if array.ActiveDevices == 0 && !detailRequirementKnown {
		array.ActiveDevices = fallback.ActiveDevices
	}
	if array.WorkingDevices == 0 {
		array.WorkingDevices = fallback.WorkingDevices
	}
	if array.FailedDevices == 0 {
		array.FailedDevices = fallback.FailedDevices
	}
	if array.SpareDevices == 0 {
		array.SpareDevices = fallback.SpareDevices
	}
	if len(array.Devices) == 0 {
		array.Devices = fallback.Devices
	}
	if array.RebuildPercent == 0 {
		array.RebuildPercent = fallback.RebuildPercent
	}
	if strings.TrimSpace(array.RebuildSpeed) == "" {
		array.RebuildSpeed = fallback.RebuildSpeed
	}
	if strings.TrimSpace(array.Operation) == "" {
		array.Operation = fallback.Operation
	}
	return array
}

// collectArrayDetail runs mdadm --detail on a specific device and parses the output
func collectArrayDetail(ctx context.Context, device string) (host.RAIDArray, error) {
	if !mdArrayPathRe.MatchString(device) {
		return host.RAIDArray{}, fmt.Errorf("invalid md device path %q", device)
	}

	mdadmPath, err := resolveMdadmBinary()
	if err != nil {
		return host.RAIDArray{}, fmt.Errorf("resolve mdadm binary: %w", err)
	}

	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	output, err := mdadmRunCommandOutput(ctx, mdadmPath, "--detail", device)
	if err != nil {
		return agentshost.RAIDArray{}, fmt.Errorf("mdadm --detail %s: %w", device, err)
	}

	return parseMdadmDetail(device, string(output))
}

// parseMdadmDetail parses the output of mdadm --detail
func parseMdadmDetail(device, output string) (agentshost.RAIDArray, error) {
	array := agentshost.RAIDArray{
		Device:  device,
		Devices: []agentshost.RAIDDevice{},
	}

	lines := strings.Split(output, "\n")
	inDeviceSection := false

	for _, line := range lines {
		line = strings.TrimSpace(line)

		// Skip empty lines
		if line == "" {
			continue
		}

		// Check if we're entering the device list section
		if strings.Contains(line, "Number") && strings.Contains(line, "Major") && strings.Contains(line, "Minor") {
			inDeviceSection = true
			continue
		}

		// Parse device entries
		if inDeviceSection {
			matches := slotRe.FindStringSubmatch(line)
			if len(matches) >= 7 {
				slot, err := strconv.Atoi(matches[1])
				if err != nil {
					return host.RAIDArray{}, fmt.Errorf("parse slot for %s from %q: %w", device, matches[1], err)
				}
				state := strings.TrimSpace(matches[5])
				devicePath := strings.TrimSpace(matches[6])

				array.Devices = append(array.Devices, agentshost.RAIDDevice{
					Device: devicePath,
					State:  state,
					Slot:   slot,
				})
				continue
			}

			// Handle spare/faulty devices (different format)
			if strings.Contains(line, "spare") || strings.Contains(line, "faulty") {
				parts := strings.Fields(line)
				if len(parts) >= 2 {
					state := "spare"
					if strings.Contains(line, "faulty") {
						state = "faulty"
					}
					devicePath := parts[len(parts)-1]

					array.Devices = append(array.Devices, agentshost.RAIDDevice{
						Device: devicePath,
						State:  state,
						Slot:   -1,
					})
				}
			}
			continue
		}

		// Parse key-value pairs
		if strings.Contains(line, ":") {
			// SplitN with n=2 always returns 2 elements when ":" exists
			parts := strings.SplitN(line, ":", 2)
			key := strings.TrimSpace(parts[0])
			value := strings.TrimSpace(parts[1])

			switch key {
			case "Name":
				array.Name = value
			case "Raid Level":
				array.Level = strings.ToLower(value)
			case "State":
				array.State = strings.ToLower(value)
			case "Raid Devices":
				requiredDevices, err := parseIntField(device, key, value)
				if err != nil {
					return host.RAIDArray{}, err
				}
				array.RequiredDevices = requiredDevices
			case "Total Devices":
				totalDevices, err := parseIntField(device, key, value)
				if err != nil {
					return host.RAIDArray{}, err
				}
				array.TotalDevices = totalDevices
			case "Active Devices":
				activeDevices, err := parseIntField(device, key, value)
				if err != nil {
					return host.RAIDArray{}, err
				}
				array.ActiveDevices = activeDevices
			case "Working Devices":
				workingDevices, err := parseIntField(device, key, value)
				if err != nil {
					return host.RAIDArray{}, err
				}
				array.WorkingDevices = workingDevices
			case "Failed Devices":
				failedDevices, err := parseIntField(device, key, value)
				if err != nil {
					return host.RAIDArray{}, err
				}
				array.FailedDevices = failedDevices
			case "Spare Devices":
				spareDevices, err := parseIntField(device, key, value)
				if err != nil {
					return host.RAIDArray{}, err
				}
				array.SpareDevices = spareDevices
			case "UUID":
				array.UUID = value
			case "Rebuild Status", "Reshape Status":
				array.RebuildPercent = parsePercentValue(value)
			}
		}
	}

	// /proc/mdstat is the authoritative source for distinguishing a true
	// rebuild from routine maintenance such as a scrub on kernels that do not
	// surface scrub state in mdadm --detail.
	operation, speed := getMdstatProgress(device)
	if operation != "" {
		array.Operation = operation
	}
	if speed != "" {
		array.RebuildSpeed = speed
	}

	return array, nil
}

// parsePercentValue parses a percentage string like "50% complete" and returns the numeric value.
func parsePercentValue(value string) float64 {
	if strings.Contains(value, "%") {
		percentStr := strings.TrimSpace(strings.Split(value, "%")[0])
		result, _ := strconv.ParseFloat(percentStr, 64)
		return result
	}
	return 0
}

// parseIntField parses an integer field from mdadm --detail output.
func parseIntField(device, key, value string) (int, error) {
	n, err := strconv.Atoi(strings.TrimSpace(value))
	if err != nil {
		return 0, fmt.Errorf("parse %s for %s from %q: %w", key, device, value, err)
	}
	if n < 0 {
		return 0, fmt.Errorf("parse %s for %s: device count must not be negative", key, device)
	}
	return n, nil
}

// getMdstatProgress extracts the in-progress md operation and speed from
// /proc/mdstat for the named array. It returns empty strings when the array is
// idle, absent, or the progress line is unavailable.
func getMdstatProgress(device string) (operation, speed string) {
	// Remove /dev/ prefix for /proc/mdstat lookup
	deviceName := strings.TrimPrefix(device, "/dev/")

	output, err := readProcMDStat()
	if err != nil {
		return "", ""
	}

	// Example progress line shapes:
	//   [==>..................]  recovery = 12.6% (...) finish=... speed=33440K/sec
	//   [=>...................]  resync = 5.0% (...) finish=... speed=...
	//   [==>..................]  check = 12.6% (...) finish=... speed=...
	lines := strings.Split(string(output), "\n")
	inSection := false

	for _, line := range lines {
		if strings.HasPrefix(strings.TrimSpace(line), deviceName) {
			inSection = true
			continue
		}

		if !inSection {
			continue
		}

		if strings.TrimSpace(line) == "" || (strings.HasPrefix(strings.TrimSpace(line), "md") && strings.Contains(line, ":")) {
			break
		}

		if operation == "" {
			if matches := mdstatOperationRe.FindStringSubmatch(line); len(matches) > 1 {
				operation = strings.ToLower(matches[1])
			}
		}
		if speed == "" {
			if matches := speedRe.FindStringSubmatch(line); len(matches) > 1 {
				speed = matches[1]
			}
		}
		if operation != "" && speed != "" {
			break
		}
	}

	return operation, speed
}

// readProcMDStat reads /proc/mdstat directly (without shelling out)
// and bounds input size to avoid unbounded memory growth.
var readProcMDStat = func() ([]byte, error) {
	output, err := readFile(mdstatPath)
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", mdstatPath, err)
	}
	if len(output) > maxMDStatBytes {
		return nil, fmt.Errorf("%s exceeds max size (%d bytes)", mdstatPath, maxMDStatBytes)
	}
	return output, nil
}

// resolveMdadmPath resolves a trusted mdadm executable path.
// It prefers known absolute paths before falling back to PATH lookup.
func resolveMdadmPath() (string, error) {
	for _, candidate := range commonMdadmPaths {
		candidate = filepath.Clean(candidate)
		if !filepath.IsAbs(candidate) {
			continue
		}
		if _, err := mdadmStat(candidate); err == nil {
			return candidate, nil
		}
	}

	path, err := mdadmLookPath("mdadm")
	if err != nil {
		return "", fmt.Errorf("mdadm binary not found in PATH or common locations")
	}
	path = filepath.Clean(path)
	if !filepath.IsAbs(path) {
		return "", fmt.Errorf("mdadm path is not absolute: %q", path)
	}
	if _, err := mdadmStat(path); err != nil {
		return "", fmt.Errorf("mdadm path unavailable: %w", err)
	}

	return path, nil
}