Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

34.9 KB
Raw Blame
package monitoring

import (
	"context"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
)

type vmBuildState struct {
	memTotal                uint64
	memUsed                 uint64
	memorySource            string
	guestRaw                VMMemoryRaw
	diskReadBytes           int64
	diskWriteBytes          int64
	networkInBytes          int64
	networkOutBytes         int64
	counterPresence         models.IOCounterPresence
	counterObservedAt       time.Time
	counterObservationTimes models.IOCounterObservationTimes
	counterUptime           uint64
	diskTotal               uint64
	diskUsed                uint64
	diskFree                uint64
	diskUsage               float64
	diskFromAgent           bool
	diskStatusReason        string
	diskObservation         models.GuestDiskObservation
	guestAgentStatus        string
	guestAgentExpected      bool
	guestAgentEvidence      models.GuestAgentEvidence
	individualDisks         []models.Disk
	ipAddresses             []string
	networkInterfaces       []models.GuestNetworkInterface
	osName                  string
	osVersion               string
	agentVersion            string
	detailedStatus          *proxmox.VMStatus
	onBoot                  *bool
}

func (m *Monitor) applyVMStatusDetails(
	ctx context.Context,
	instanceName string,
	res proxmox.ClusterResource,
	client PVEClientInterface,
	guestID string,
	vmIDToHostAgent map[string]models.Host,
	status *proxmox.VMStatus,
	recentGuestAgentEvidence bool,
	state *vmBuildState,
) {
	if status == nil || state == nil {
		return
	}

	if res.Lock != "" {
		copy := *status
		copy.Lock = res.Lock
		status = &copy
	}
	res.Lock = status.Lock
	state.detailedStatus = status
	state.guestAgentStatus, state.guestAgentExpected = vmGuestAgentRuntimeState(status, recentGuestAgentEvidence)
	// status/current reports agent=1 whenever the VM config enables the guest
	// agent, not when the in-guest service answers. Fallback eligibility is
	// renewed only by an actual reply (filesystems below, metadata at the end).
	// Filesystem usage is cross-platform. Read it before optional Linux memory
	// and metadata commands can start a shared uncertainty pause. Every command
	// still verifies the same operation lock and is admitted only once.
	if status.Agent.IsAvailable() {
		m.applyVMGuestAgentFSInfo(ctx, instanceName, res, client, state)
	} else {
		// Agent disabled - show allocated disk size
		if state.diskTotal > 0 {
			state.diskUsage = -1 // Show as allocated size
		}
		if status.Agent.IsEnabled() {
			state.diskStatusReason = "agent-not-running"
		} else {
			state.diskStatusReason = "agent-disabled"
		}
		log.Debug().
			Str("instance", instanceName).
			Str("vm", res.Name).
			Int("vmid", res.VMID).
			Int("agent", status.Agent.Value).
			Bool("agentEnabled", status.Agent.IsEnabled()).
			Msg("VM guest agent is not currently queryable")
	}

	// Learn or refresh the OS before the optional Linux-only memory fallback.
	// Otherwise an uncertain /proc/meminfo reply on a cold/expired Windows
	// guest can fence OS discovery on every poll, so the Windows exclusion
	// never becomes usable. Filesystems remain first; metadata and memory
	// retain the same per-command lock, completion and shared-channel guard.
	guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)
	if metadataDeferred {
		state.guestAgentStatus = "deferred"
	}
	if len(guestIPs) > 0 {
		state.ipAddresses = guestIPs
	}
	if len(guestIfaces) > 0 {
		state.networkInterfaces = guestIfaces
	}
	if guestOSName != "" {
		state.osName = guestOSName
	}
	if guestOSVersion != "" {
		state.osVersion = guestOSVersion
	}
	if guestAgentVersion != "" {
		state.agentVersion = guestAgentVersion
	}

	var memoryDeferred bool
	state.memTotal, state.memUsed, state.memorySource, memoryDeferred = m.resolveGuestStatusMemory(
		ctx,
		client,
		instanceName,
		res.Name,
		res.Node,
		res.VMID,
		guestID,
		status,
		vmIDToHostAgent,
		state.memTotal,
		state.memorySource,
		&state.guestRaw,
	)

	if memoryDeferred {
		state.guestAgentStatus = "deferred"
	}

	mergeVMRuntimeCounters(state, status)
}

func (m *Monitor) applyVMGuestAgentFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface, state *vmBuildState) {
	var fsDisks []models.Disk
	state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, fsDisks, state.diskFromAgent, state.diskStatusReason = m.updateVMDisksFromGuestAgentFSInfo(
		ctx, instanceName, res, client, state.diskTotal, state.diskUsed, state.diskUsage,
	)
	if len(fsDisks) > 0 {
		state.individualDisks = fsDisks
	}
	if state.diskFromAgent {
		state.diskObservation.ObservedAt = time.Now()
		renewGuestAgentEvidence(&state.guestAgentEvidence, state.diskObservation.ObservedAt, state.diskObservation.ObservedAt)
	}
	if guestAgentDiskDeferred(state.diskStatusReason) {
		state.guestAgentStatus = "deferred"
	}
}

func mergeVMRuntimeCounters(state *vmBuildState, status *proxmox.VMStatus) {
	if state == nil || status == nil {
		return
	}

	// A detailed status response is newer and more authoritative than the
	// cluster listing, but only for fields it actually contains.
	statusPresence := status.IOCounters.Effective()
	if statusPresence.DiskRead {
		state.diskReadBytes = int64(status.DiskRead)
		state.counterPresence.DiskRead = true
		state.counterObservationTimes.DiskRead = status.ObservedAt
	}
	if statusPresence.DiskWrite {
		state.diskWriteBytes = int64(status.DiskWrite)
		state.counterPresence.DiskWrite = true
		state.counterObservationTimes.DiskWrite = status.ObservedAt
	}
	if statusPresence.NetworkIn {
		state.networkInBytes = int64(status.NetIn)
		state.counterPresence.NetworkIn = true
		state.counterObservationTimes.NetworkIn = status.ObservedAt
	}
	if statusPresence.NetworkOut {
		state.networkOutBytes = int64(status.NetOut)
		state.counterPresence.NetworkOut = true
		state.counterObservationTimes.NetworkOut = status.ObservedAt
	}
	if !status.ObservedAt.IsZero() {
		state.counterObservedAt = status.ObservedAt
	}
	if status.Uptime > 0 {
		state.counterUptime = status.Uptime
	}
}

func vmGuestAgentRuntimeState(status *proxmox.VMStatus, recentGuestAgentEvidence bool) (string, bool) {
	if status == nil {
		return "", false
	}
	if status.Lock != "" {
		return "deferred", status.Agent.IsEnabled()
	}
	if status.Agent.IsAvailable() {
		return "available", true
	}
	if status.Agent.IsEnabled() {
		if recentGuestAgentEvidence {
			return "expected-unreachable", true
		}
		return "not-running", false
	}
	return "disabled", false
}

func (m *Monitor) buildVMFromClusterResource(
	ctx context.Context,
	instanceName string,
	res proxmox.ClusterResource,
	client PVEClientInterface,
	guestID string,
	vmIDToHostAgent map[string]models.Host,
	prevVM *models.VM,
) (models.VM, VMMemoryRaw, string, []string, time.Time, bool) {
	// Skip templates if configured
	if res.Template == 1 {
		return models.VM{}, VMMemoryRaw{}, "", nil, time.Time{}, false
	}

	if guestID == "" {
		guestID = makeGuestID(instanceName, res.Node, res.VMID)
	}
	prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID)
	prePollTime := time.Now()
	recentGuestAgentEvidence := hasRecentGuestAgentEvidence(prevVM, prePollTime) ||
		m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, prePollTime)

	state := vmBuildState{
		guestAgentEvidence:      retainGuestAgentEvidence(prevVM, prePollTime),
		diskObservation:         models.GuestDiskObservation{Source: "guest-agent"},
		memTotal:                res.MaxMem,
		memUsed:                 res.Mem,
		memorySource:            "cluster-resources",
		guestRaw:                VMMemoryRaw{ListingMem: res.Mem, ListingMaxMem: res.MaxMem},
		diskReadBytes:           int64(res.DiskRead),
		diskWriteBytes:          int64(res.DiskWrite),
		networkInBytes:          int64(res.NetIn),
		networkOutBytes:         int64(res.NetOut),
		counterPresence:         pveCounterPresence(res.IOCounters),
		counterObservedAt:       observedAtOr(res.ObservedAt, prePollTime),
		counterObservationTimes: counterObservationTimes(observedAtOr(res.ObservedAt, prePollTime)),
		counterUptime:           res.Uptime,
		diskTotal:               res.MaxDisk,
		diskUsed:                res.Disk,
	}
	state.diskFree = state.diskTotal - state.diskUsed
	state.diskUsage = safePercentage(float64(state.diskUsed), float64(state.diskTotal))

	// If VM shows 0 disk usage but has allocated disk, it's likely guest agent issue
	// Set to -1 to indicate "unknown" rather than showing misleading 0%
	if res.Type == "qemu" && state.diskUsed == 0 && state.diskTotal > 0 && res.Status == "running" {
		state.diskUsage = -1
	}
	if res.Status != "running" {
		// A subsequent start must obtain its own guest memory observation rather
		// than inherit a success or failure from before this stopped poll.
		m.clearGuestMemoryCache(instanceName, res.Node, res.VMID)
		if state.diskTotal > 0 {
			state.diskUsage = -1
			state.diskStatusReason = "vm-stopped"
		}
	}

	// For running VMs, always try to get filesystem info from guest agent
	// The cluster/resources endpoint often returns 0 or incorrect values for disk usage
	// We should prefer guest agent data when available for accurate metrics
	if res.Status == "running" && res.Type == "qemu" {
		// First check if agent is enabled by getting VM status
		status, err := client.GetVMStatus(ctx, res.Node, res.VMID)
		if err != nil {
			if state.diskTotal > 0 {
				state.diskStatusReason = "no-status"
			}
			log.Debug().
				Err(err).
				Str("instance", instanceName).
				Str("vm", res.Name).
				Int("vmid", res.VMID).
				Msg("Could not get VM status to check guest agent availability")
		} else if status != nil {
			m.applyVMStatusDetails(ctx, instanceName, res, client, guestID, vmIDToHostAgent, status, recentGuestAgentEvidence, &state)
		} else {
			if state.diskTotal > 0 {
				state.diskStatusReason = "no-status"
			}
			// No vmStatus available - keep cluster/resources data
			log.Debug().
				Str("instance", instanceName).
				Str("vm", res.Name).
				Int("vmid", res.VMID).
				Msg("Could not get VM status, using cluster/resources disk data")
		}

		if state.detailedStatus != nil && state.detailedStatus.Lock != "" {
			res.Lock = state.detailedStatus.Lock
		}
		now := time.Now()
		guestAgentAvailable := shouldQueryGuestAgent(state.detailedStatus, prevVM, now) ||
			m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, now)
		if guestAgentAvailable && res.Lock == "" && state.detailedStatus == nil {
			m.applyVMGuestAgentFSInfo(ctx, instanceName, res, client, &state)
			guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(
				ctx,
				client,
				instanceName,
				res.Node,
				res.Name,
				res.VMID,
				nil,
				true,
			)
			if metadataDeferred {
				state.guestAgentStatus = "deferred"
			}
			if len(guestIPs) > 0 {
				state.ipAddresses = guestIPs
			}
			if len(guestIfaces) > 0 {
				state.networkInterfaces = guestIfaces
			}
			if guestOSName != "" {
				state.osName = guestOSName
			}
			if guestOSVersion != "" {
				state.osVersion = guestOSVersion
			}
			if guestAgentVersion != "" {
				state.agentVersion = guestAgentVersion
			}

			state.guestAgentExpected = true
			// Cached strings are display continuity, not a new successful read.
			metadataAt := m.guestMetadataEvidenceTime(instanceName, res.Node, res.VMID)
			metadataObserved := !metadataAt.Before(prePollTime) && !metadataAt.After(time.Now())
			if state.guestAgentStatus != "deferred" && (state.diskFromAgent || metadataObserved) {
				state.guestAgentStatus = "available"
			}
		}

		if res.Lock != "" {
			state.guestAgentStatus = "deferred"
			state.diskStatusReason = "vm-locked"
			state.diskFromAgent = false
			state.diskUsage = -1
			if guestMemorySourceReliability(state.memorySource) == guestMemoryReliabilityLow {
				state.memorySource = "unavailable"
				state.memUsed = 0
			}
			if prevVM != nil {
				state.ipAddresses = cloneStringSlice(prevVM.IPAddresses)
				state.networkInterfaces = cloneGuestNetworkInterfaces(prevVM.NetworkInterfaces)
				state.osName, state.osVersion, state.agentVersion = prevVM.OSName, prevVM.OSVersion, prevVM.AgentVersion
			}
		}

		var preferred bool
		state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason, preferred = preferLinkedHostAgentDiskInventory(
			guestID,
			vmIDToHostAgent,
			state.diskTotal,
			state.diskUsed,
			state.diskFree,
			state.diskUsage,
			state.individualDisks,
			state.diskStatusReason,
		)
		if preferred {
			state.diskObservation = models.GuestDiskObservation{Source: "agent", ObservedAt: vmIDToHostAgent[guestID].LastSeen}
			log.Debug().
				Str("instance", instanceName).
				Str("vm", res.Name).
				Int("vmid", res.VMID).
				Bool("guestAgentDiskAvailable", state.diskFromAgent).
				Float64("usage", state.diskUsage).
				Msg("QEMU disk: preferring linked Pulse host agent disk inventory")
		}
	}

	sampleTime := time.Now()
	if res.Status == "running" {
		renewGuestAgentEvidence(&state.guestAgentEvidence, m.guestMetadataEvidenceTime(instanceName, res.Node, res.VMID), sampleTime)
	} else {
		// A stopped guest supplies no current fallback eligibility evidence.
		state.guestAgentEvidence = models.GuestAgentEvidence{Explicit: true}
	}
	state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason = stabilizeGuestLowTrustDisk(
		prevVM,
		res.Status,
		state.diskTotal,
		state.diskUsed,
		state.diskFree,
		state.diskUsage,
		state.individualDisks,
		state.diskStatusReason,
		state.diskFromAgent,
		sampleTime,
	)
	if strings.HasPrefix(state.diskStatusReason, "prev-") {
		state.diskObservation.ObservedAt = guestDiskObservationTime(prevVM)
	}

	if res.Status != "running" {
		state.memorySource = "powered-off"
		state.memUsed = 0
		state.onBoot = m.fetchVMOnBoot(ctx, client, res.Node, res.VMID)
	}

	var snapshotNotes []string
	if prevSnapshot != nil && (prevSnapshot.Raw.StatusMaxMem > 0 || prevSnapshot.Raw.ListingMaxMem > 0) {
		currentCapacity := state.guestRaw.StatusMaxMem
		if currentCapacity == 0 {
			currentCapacity = state.guestRaw.ListingMaxMem
		}
		if guestMemoryConfiguredCapacity(prevSnapshot) != currentCapacity {
			prevSnapshot = nil // A resize cannot retain the earlier guest tuple.
		}
	}
	state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(
		prevSnapshot,
		res.Status,
		state.memorySource,
		state.memTotal,
		state.memUsed,
		sampleTime,
		guestAgentSignalsHealthy(
			state.detailedStatus,
			state.diskFromAgent,
			state.ipAddresses,
			state.networkInterfaces,
			state.osName,
			state.osVersion,
			state.agentVersion,
		),
	)
	if state.memorySource == "previous-snapshot" && prevSnapshot != nil {
		// Retention keeps the original guest denominator as well as its usage.
		// The selector has already checked capacity continuity and source age.
		state.memTotal = uint64(prevSnapshot.Memory.Total)
	}
	memFree := uint64(0)
	if state.memTotal >= state.memUsed {
		memFree = state.memTotal - state.memUsed
	}
	currentMetrics := IOMetrics{
		DiskRead:     state.diskReadBytes,
		DiskWrite:    state.diskWriteBytes,
		NetworkIn:    state.networkInBytes,
		NetworkOut:   state.networkOutBytes,
		Timestamp:    state.counterObservedAt,
		Presence:     state.counterPresence,
		ObservedAt:   state.counterObservationTimes,
		SourceUptime: state.counterUptime,
	}
	diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(
		makeGuestRateKey(instanceName, "qemu", res.VMID),
		currentMetrics,
	)
	diskReadValue, diskWriteValue, networkInValue, networkOutValue, rateValidity := guestRateValues(
		diskReadRate,
		diskWriteRate,
		netInRate,
		netOutRate,
	)

	memory := models.UnavailableMemory(clampToInt64(state.memTotal))
	if CanonicalMemorySource(state.memorySource) != "unavailable" {
		memory = models.Memory{
			Total: int64(state.memTotal),
			Used:  int64(state.memUsed),
			Free:  int64(memFree),
			Usage: safePercentage(float64(state.memUsed), float64(state.memTotal)),
		}
	}
	if memory.Free < 0 {
		memory.Free = 0
	}
	if memory.Used > memory.Total {
		memory.Used = memory.Total
	}
	// Free above is total-used, i.e. available. When the guest reported its
	// truly-free pages (meminfo), split the reclaimable cache back out.
	var trulyFree uint64
	switch CanonicalMemorySource(state.memorySource) {
	case "available-field", "derived-free-buffers-cached", "derived-total-minus-used":
		trulyFree = state.guestRaw.MemInfoFree
	case "guest-agent-meminfo", "guest-agent-meminfo-derived":
		trulyFree = state.guestRaw.GuestAgentMemFree
	}
	splitReclaimableMemory(&memory, trulyFree)
	if state.memorySource == "previous-snapshot" && prevSnapshot != nil && prevSnapshot.Memory.Observation.State != "" {
		memory = prevSnapshot.Memory
	}
	if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {
		memory.Balloon = int64(state.detailedStatus.Balloon)
	}
	if state.guestAgentStatus == "deferred" && (CanonicalMemorySource(state.memorySource) == "unavailable" || state.memorySource == "previous-snapshot") {
		if retained, ok := m.deferredVMGuestMemory(instanceName, res.Node, res.VMID, state.memTotal, prevSnapshot, sampleTime); ok {
			memory = retained
			state.memorySource = "previous-snapshot"
			snapshotNotes = append(snapshotNotes, "preserved-original-guest-memory-during-deferral")
		}
	}
	providerObservedAt := observedAtOr(res.ObservedAt, sampleTime)
	if state.detailedStatus != nil && CanonicalMemorySource(state.memorySource) != "cluster-resources" {
		providerObservedAt = observedAtOr(state.detailedStatus.ObservedAt, sampleTime)
	}
	memory.Observation = m.guestMemoryObservation(instanceName, "qemu", res.Node, res.VMID, memory, guestMemoryObservationContext{
		source: state.memorySource, providerObservedAt: providerObservedAt,
		deferred: state.guestAgentStatus == "deferred", previous: prevSnapshot,
		linkedAgent: vmIDToHostAgent[guestID],
	}, sampleTime)

	vm := models.VM{
		ID:       guestID,
		VMID:     res.VMID,
		Name:     res.Name,
		Node:     res.Node,
		Pool:     strings.TrimSpace(res.Pool),
		Instance: instanceName,
		Status:   res.Status,
		Type:     "qemu",
		CPU:      safeFloat(res.CPU),
		CPUs:     res.MaxCPU,
		Memory:   memory,
		Disk: models.Disk{
			Total: int64(state.diskTotal),
			Used:  int64(state.diskUsed),
			Free:  int64(state.diskFree),
			Usage: state.diskUsage,
		},
		Disks:              state.individualDisks,
		DiskStatusReason:   state.diskStatusReason,
		DiskObservation:    state.diskObservation,
		GuestAgentStatus:   state.guestAgentStatus,
		Lock:               res.Lock,
		GuestAgentExpected: state.guestAgentExpected,
		GuestAgentEvidence: state.guestAgentEvidence,
		IPAddresses:        state.ipAddresses,
		OSName:             state.osName,
		OSVersion:          state.osVersion,
		AgentVersion:       state.agentVersion,
		NetworkInterfaces:  state.networkInterfaces,
		NetworkIn:          networkInValue,
		NetworkOut:         networkOutValue,
		DiskRead:           diskReadValue,
		DiskWrite:          diskWriteValue,
		IORateValidity:     rateValidity,
		Uptime:             int64(res.Uptime),
		Template:           res.Template == 1,
		OnBoot:             state.onBoot,
		LastSeen:           sampleTime,
	}

	// Parse tags
	if res.Tags != "" {
		vm.Tags = strings.Split(res.Tags, ";")

		// Log if Pulse-specific tags are detected
		for _, tag := range vm.Tags {
			switch tag {
			case "pulse-no-alerts", "pulse-monitor-only", "pulse-relaxed":
				log.Info().
					Str("vm", vm.Name).
					Str("node", vm.Node).
					Str("tag", tag).
					Msg("Pulse control tag detected on VM")
			}
		}
	}
	if res.Status != "running" {
		vm.NetworkIn = 0
		vm.NetworkOut = 0
		vm.DiskRead = 0
		vm.DiskWrite = 0
		vm.IORateValidity = models.IORateValidity{
			Explicit:   true,
			DiskRead:   true,
			DiskWrite:  true,
			NetworkIn:  true,
			NetworkOut: true,
		}
	}

	return vm, state.guestRaw, state.memorySource, snapshotNotes, sampleTime, true
}

// fetchVMOnBoot retrieves the onboot (autostart) setting for a stopped VM by
// fetching its config. Returns nil only when the config is unavailable or the
// value is unrecognised. When onboot is absent from a valid config, returns
// false because Proxmox's default is "do not start on boot". Uses a type assertion because PVEClientInterface does not include
// GetVMConfig (it is only on the concrete client, matching the pattern in
// guest_config.go).
func (m *Monitor) fetchVMOnBoot(ctx context.Context, client PVEClientInterface, node string, vmid int) *bool {
	type vmConfigClient interface {
		GetVMConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error)
	}
	vmClient, ok := client.(vmConfigClient)
	if !ok {
		return nil
	}
	configCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	configData, err := vmClient.GetVMConfig(configCtx, node, vmid)
	if err != nil || len(configData) == 0 {
		return nil
	}
	return parseProxmoxOnBoot(configData)
}

type vmFSInfoSummary struct {
	totalBytes      uint64
	usedBytes       uint64
	individualDisks []models.Disk
	skippedFS       []string
	includedFS      []string
	invalidBytes    bool
}

func (m *Monitor) fetchVMFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface) ([]proxmox.VMFileSystem, string, bool) {
	if res.Lock != "" {
		return nil, "vm-locked", false
	}
	// Use retry logic for guest agent calls to handle transient timeouts (refs #630)
	fsInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentFSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
		return client.GetVMFSInfo(ctx, res.Node, res.VMID)
	})
	var fsInfo []proxmox.VMFileSystem
	if err == nil {
		if fs, ok := fsInfoRaw.([]proxmox.VMFileSystem); ok {
			fsInfo = fs
		}
	}
	if err != nil {
		if reason := proxmox.GuestAgentDeferredReason(err); reason != "" {
			return nil, reason, false
		}
		reason := classifyGuestAgentDiskStatusError(err)
		logVMFilesystemUnavailable(ctx, instanceName, res, reason)
		return nil, classifyGuestAgentDiskStatusError(err), false
	}
	if len(fsInfo) == 0 {
		logVMFilesystemUnavailable(ctx, instanceName, res, "no-filesystems")
		return nil, "no-filesystems", false
	}
	return fsInfo, "", true
}

// Keep ordinary process logging when no context logger is installed. Tests and
// callers with an explicit logger can own their output without global mutation.
func guestFilesystemLogger(ctx context.Context) *zerolog.Logger {
	logger := log.Ctx(ctx)
	if logger.GetLevel() == zerolog.Disabled {
		return &log.Logger
	}
	return logger
}

// Missing optional readings do not establish a need to activate/restart QGA,
// change backup policy, or reconfigure a shared Proxmox role. Keep this guidance
// independent of provider error text and the guest's operating system.
func logVMFilesystemUnavailable(ctx context.Context, instanceName string, res proxmox.ClusterResource, reason string) {
	message := "Guest filesystem query failed; check disk usage inside the guest. Keep existing guest-agent and backup settings."
	switch reason {
	case "agent-not-running":
		message = "Proxmox reports the guest agent is not running; check disk usage inside the guest. Keep existing guest-agent and backup settings."
	case "permission-denied":
		message = "Guest filesystem query was not authorised; check the existing credential's access to this VM. Do not broaden shared roles or change guest-agent and backup settings to diagnose missing readings."
	case "agent-timeout":
		message = "Guest filesystem query did not complete; check disk usage inside the guest. Keep existing guest-agent and backup settings."
	case "no-filesystems":
		message = "Guest agent returned no filesystem readings; check disk usage inside the guest. Keep existing guest-agent and backup settings."
	}
	guestFilesystemLogger(ctx).Info().
		Str("instance", instanceName).
		Str("vm", res.Name).
		Int("vmid", res.VMID).
		Str("reason", reason).
		Msg(message)
}

func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterResource, fsInfo []proxmox.VMFileSystem) vmFSInfoSummary {
	summary := vmFSInfoSummary{}

	// Track seen filesystems to deduplicate mounts that share the same device:
	// 1. btrfs/zfs mount multiple subvolumes from a shared pool
	// 2. Kubernetes bind-mounts volumes at both pod and plugin paths
	// In both cases, only count the device's capacity once.
	// Mounts are sampled separately, so aliases of one volume can disagree on
	// usage. Count its capacity once and retain the highest observed usage;
	// taking the first mount makes the aggregate depend on response order.
	seenFilesystems := make(map[string]uint64)

	// Log all filesystems received for debugging
	log.Debug().
		Str("instance", instanceName).
		Str("vm", res.Name).
		Int("vmid", res.VMID).
		Int("filesystem_count", len(fsInfo)).
		Msg("Processing filesystems from guest agent")

	for _, fs := range fsInfo {
		// Other PVEClientInterface implementations can supply records without
		// the wire decoder. Never subtract/cast contradictory or unrepresentable
		// counters into the signed disk model, even in an individual mount row.
		if fs.UsedBytes > fs.TotalBytes || fs.TotalBytes > math.MaxInt64 {
			summary.invalidBytes = true
			summary.skippedFS = append(summary.skippedFS, fs.Mountpoint+"(invalid-byte-counts)")
			continue
		}
		// Skip special filesystems and mounts
		shouldSkip, reasons := fsfilters.ShouldSkipFilesystem(fs.Type, fs.Mountpoint, fs.TotalBytes, fs.UsedBytes)
		if shouldSkip {
			// Check if any reason is read-only for detailed logging
			for _, r := range reasons {
				if strings.HasPrefix(r, "read-only-") {
					log.Debug().
						Str("instance", instanceName).
						Str("vm", res.Name).
						Int("vmid", res.VMID).
						Str("mountpoint", fs.Mountpoint).
						Str("type", fs.Type).
						Float64("total_gb", float64(fs.TotalBytes)/1073741824).
						Float64("used_gb", float64(fs.UsedBytes)/1073741824).
						Msg("Skipping read-only filesystem from disk aggregation")
					break
				}
			}
			summary.skippedFS = append(summary.skippedFS, fmt.Sprintf("%s(%s,%s)",
				fs.Mountpoint, fs.Type, strings.Join(reasons, ",")))
			continue
		}

		// Only count real filesystems with valid data
		// Some filesystems report 0 bytes (like unformatted or system partitions)
		if fs.TotalBytes > 0 {
			// Deduplication: the same device can appear multiple times when:
			// 1. btrfs/zfs mount multiple subvolumes from a shared pool
			// 2. Kubernetes bind-mounts volumes at both pod and plugin paths
			// In both cases, only count the device's capacity once.
			fsTypeLower := strings.ToLower(fs.Type)
			countThisFS := true
			var dedupeKey string
			if volumeKey := windowsVMFilesystemCapacityKey(fs); volumeKey != "" {
				// QGA's disk.dev is the backing physical drive on Windows,
				// not the volume. Equal-sized partitions must remain separate.
				// A GUID can still identify one volume mounted at several paths.
				dedupeKey = fmt.Sprintf("windows:%s:%d", volumeKey, fs.TotalBytes)
				_, seen := seenFilesystems[dedupeKey]
				countThisFS = !seen
			} else if fs.Disk != "" {
				// Same non-Windows device at multiple mount paths → count once
				dedupeKey = fmt.Sprintf("%s:%d", fs.Disk, fs.TotalBytes)
				if _, seen := seenFilesystems[dedupeKey]; seen {
					countThisFS = false
					log.Debug().
						Str("instance", instanceName).
						Str("vm", res.Name).
						Int("vmid", res.VMID).
						Str("mountpoint", fs.Mountpoint).
						Str("type", fs.Type).
						Str("device", fs.Disk).
						Uint64("total", fs.TotalBytes).
						Str("dedupe_key", dedupeKey).
						Msg("Skipping duplicate device mount in total calculation")
				}
			} else if fsTypeLower == "btrfs" || fsTypeLower == "zfs" ||
				strings.HasPrefix(fsTypeLower, "zfs") {
				// No device info — fall back to type+size dedup for COW filesystems
				dedupeKey = fmt.Sprintf("%s::%d", fsTypeLower, fs.TotalBytes)
				if _, seen := seenFilesystems[dedupeKey]; seen {
					countThisFS = false
					log.Debug().
						Str("instance", instanceName).
						Str("vm", res.Name).
						Int("vmid", res.VMID).
						Str("mountpoint", fs.Mountpoint).
						Str("type", fs.Type).
						Uint64("total", fs.TotalBytes).
						Str("dedupe_key", dedupeKey).
						Msg("Skipping duplicate btrfs/zfs subvolume in total calculation")
				}
			}

			if dedupeKey != "" {
				previousUsed := seenFilesystems[dedupeKey]
				if !countThisFS && fs.UsedBytes > previousUsed {
					// Each used count is bounded by its already-counted total,
					// so this delta cannot exceed the signed aggregate limit.
					summary.usedBytes += fs.UsedBytes - previousUsed
				}
				if countThisFS || fs.UsedBytes > previousUsed {
					seenFilesystems[dedupeKey] = fs.UsedBytes
				}
			}

			if countThisFS {
				if fs.TotalBytes > math.MaxInt64-summary.totalBytes {
					// A partial aggregate would look like complete guest usage.
					// Leave it unavailable rather than wrap or silently drop a disk.
					return vmFSInfoSummary{invalidBytes: true}
				}
				summary.totalBytes += fs.TotalBytes
				summary.usedBytes += fs.UsedBytes
			}
			summary.includedFS = append(summary.includedFS, fmt.Sprintf("%s(%s,%.1fGB)",
				fs.Mountpoint, fs.Type, float64(fs.TotalBytes)/1073741824))

			// Add to individual disks array (always include for display)
			summary.individualDisks = append(summary.individualDisks, models.Disk{
				Total:      int64(fs.TotalBytes),
				Used:       int64(fs.UsedBytes),
				Free:       int64(fs.TotalBytes - fs.UsedBytes),
				Usage:      safePercentage(float64(fs.UsedBytes), float64(fs.TotalBytes)),
				Mountpoint: fs.Mountpoint,
				Type:       fs.Type,
				Device:     fs.Disk,
			})

			log.Debug().
				Str("instance", instanceName).
				Str("vm", res.Name).
				Int("vmid", res.VMID).
				Str("mountpoint", fs.Mountpoint).
				Str("type", fs.Type).
				Uint64("total", fs.TotalBytes).
				Uint64("used", fs.UsedBytes).
				Float64("total_gb", float64(fs.TotalBytes)/1073741824).
				Float64("used_gb", float64(fs.UsedBytes)/1073741824).
				Bool("counted_in_total", countThisFS).
				Msg("Including filesystem in disk usage calculation")
		} else if fs.TotalBytes == 0 && len(fs.Mountpoint) > 0 {
			summary.skippedFS = append(summary.skippedFS, fmt.Sprintf("%s(%s,0GB)", fs.Mountpoint, fs.Type))
			log.Debug().
				Str("instance", instanceName).
				Str("vm", res.Name).
				Int("vmid", res.VMID).
				Str("mountpoint", fs.Mountpoint).
				Str("type", fs.Type).
				Msg("Skipping filesystem with zero total bytes")
		}
	}

	if len(summary.skippedFS) > 0 {
		log.Debug().
			Str("instance", instanceName).
			Str("vm", res.Name).
			Strs("skipped", summary.skippedFS).
			Msg("Skipped special filesystems")
	}

	if len(summary.includedFS) > 0 {
		log.Debug().
			Str("instance", instanceName).
			Str("vm", res.Name).
			Int("vmid", res.VMID).
			Strs("included", summary.includedFS).
			Msg("Filesystems included in disk calculation")
	}

	return summary
}

func (m *Monitor) updateVMDisksFromGuestAgentFSInfo(
	ctx context.Context,
	instanceName string,
	res proxmox.ClusterResource,
	client PVEClientInterface,
	diskTotal uint64,
	diskUsed uint64,
	diskUsage float64,
) (uint64, uint64, uint64, float64, []models.Disk, bool, string) {
	log.Debug().
		Str("instance", instanceName).
		Str("vm", res.Name).
		Int("vmid", res.VMID).
		Msg("Guest agent enabled, querying filesystem info for accurate disk usage")

	fsInfo, diskStatusReason, ok := m.fetchVMFSInfo(ctx, instanceName, res, client)
	if !ok {
		// No trustworthy usage data: cluster/resources reports 0 used for
		// QEMU, so passing it through renders a confident "0%" in the UI.
		// Use the -1 sentinel (same as the agent-disabled path) so the
		// frontend shows unavailable + diskStatusReason instead; the
		// stabilizer may still replace this with the previous good sample.
		if diskTotal > 0 {
			diskUsage = -1
		}
		return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, diskStatusReason
	}

	log.Debug().
		Str("instance", instanceName).
		Str("vm", res.Name).
		Int("filesystems", len(fsInfo)).
		Msg("Got filesystem info from guest agent")

	summary := m.summarizeVMFSInfo(instanceName, res, fsInfo)

	// Valid peer rows are not complete guest usage if any reading was rejected.
	// The wire client reports that distinction as an error; alternate clients
	// can supply invalid counters directly, so enforce it at this boundary too.
	if summary.totalBytes > 0 && !summary.invalidBytes {
		// Sanity check: if the reported disk is way larger than allocated disk,
		// we might be getting host disk info somehow
		allocatedDiskGB := float64(res.MaxDisk) / 1073741824
		reportedDiskGB := float64(summary.totalBytes) / 1073741824

		// If reported disk is more than 2x the allocated disk, log for debugging.
		// This is expected for pass-through disks and can also indicate we're getting host disk or network shares.
		if allocatedDiskGB > 0 && reportedDiskGB > allocatedDiskGB*2 {
			log.Debug().
				Str("instance", instanceName).
				Str("vm", res.Name).
				Int("vmid", res.VMID).
				Float64("allocated_gb", allocatedDiskGB).
				Float64("reported_gb", reportedDiskGB).
				Float64("ratio", reportedDiskGB/allocatedDiskGB).
				Strs("filesystems", summary.includedFS).
				Msg("VM reports disk usage significantly larger than allocated disk - possible issue with filesystem detection")
		}

		diskTotal = summary.totalBytes
		diskUsed = summary.usedBytes
		diskUsage = safePercentage(float64(summary.usedBytes), float64(summary.totalBytes))

		log.Debug().
			Str("instance", instanceName).
			Str("vm", res.Name).
			Int("vmid", res.VMID).
			Uint64("totalBytes", summary.totalBytes).
			Uint64("usedBytes", summary.usedBytes).
			Float64("total_gb", float64(summary.totalBytes)/1073741824).
			Float64("used_gb", float64(summary.usedBytes)/1073741824).
			Float64("allocated_gb", allocatedDiskGB).
			Float64("usage", diskUsage).
			Uint64("old_disk", res.Disk).
			Uint64("old_maxdisk", res.MaxDisk).
			Msg("Using guest agent data for accurate disk usage (replacing cluster/resources data)")
		return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, summary.individualDisks, true, ""
	}

	// No usable aggregate - show allocated disk size instead. Invalid counters
	// are a reading failure, not evidence that every filesystem is special.
	if diskTotal > 0 {
		diskUsage = -1 // Show as allocated size
	}
	log.Info().
		Str("instance", instanceName).
		Str("vm", res.Name).
		Int("filesystems_found", len(fsInfo)).
		Bool("invalid_byte_counts", summary.invalidBytes).
		Msg("Guest agent provided filesystem info but no usable disk aggregate was available")

	if summary.invalidBytes {
		return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "agent-error"
	}
	return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "special-filesystems-only"
}

func guestAgentDiskDeferred(reason string) bool {
	switch strings.TrimPrefix(reason, "prev-") {
	case "vm-locked", "lock-unverified", "agent-busy", "agent-cooldown", "agent-capacity", "agent-timeout", "agent-response-incomplete", "agent-redirect", "agent-transport-unverified", "agent-completion-unverified":
		return true
	}
	return false
}