Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"fmt"
"math"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/fsfilters"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
type vmBuildState struct {
memTotal uint64
memUsed uint64
memorySource string
guestRaw VMMemoryRaw
diskReadBytes int64
diskWriteBytes int64
networkInBytes int64
networkOutBytes int64
counterPresence models.IOCounterPresence
counterObservedAt time.Time
counterObservationTimes models.IOCounterObservationTimes
counterUptime uint64
diskTotal uint64
diskUsed uint64
diskFree uint64
diskUsage float64
diskFromAgent bool
diskStatusReason string
diskObservation models.GuestDiskObservation
guestAgentStatus string
guestAgentExpected bool
guestAgentEvidence models.GuestAgentEvidence
individualDisks []models.Disk
ipAddresses []string
networkInterfaces []models.GuestNetworkInterface
osName string
osVersion string
agentVersion string
detailedStatus *proxmox.VMStatus
onBoot *bool
}
func (m *Monitor) applyVMStatusDetails(
ctx context.Context,
instanceName string,
res proxmox.ClusterResource,
client PVEClientInterface,
guestID string,
vmIDToHostAgent map[string]models.Host,
status *proxmox.VMStatus,
recentGuestAgentEvidence bool,
state *vmBuildState,
) {
if status == nil || state == nil {
return
}
if res.Lock != "" {
copy := *status
copy.Lock = res.Lock
status = ©
}
res.Lock = status.Lock
state.detailedStatus = status
state.guestAgentStatus, state.guestAgentExpected = vmGuestAgentRuntimeState(status, recentGuestAgentEvidence)
// status/current reports agent=1 whenever the VM config enables the guest
// agent, not when the in-guest service answers. Fallback eligibility is
// renewed only by an actual reply (filesystems below, metadata at the end).
// Filesystem usage is cross-platform. Read it before optional Linux memory
// and metadata commands can start a shared uncertainty pause. Every command
// still verifies the same operation lock and is admitted only once.
if status.Agent.IsAvailable() {
m.applyVMGuestAgentFSInfo(ctx, instanceName, res, client, state)
} else {
// Agent disabled - show allocated disk size
if state.diskTotal > 0 {
state.diskUsage = -1 // Show as allocated size
}
if status.Agent.IsEnabled() {
state.diskStatusReason = "agent-not-running"
} else {
state.diskStatusReason = "agent-disabled"
}
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Int("agent", status.Agent.Value).
Bool("agentEnabled", status.Agent.IsEnabled()).
Msg("VM guest agent is not currently queryable")
}
// Learn or refresh the OS before the optional Linux-only memory fallback.
// Otherwise an uncertain /proc/meminfo reply on a cold/expired Windows
// guest can fence OS discovery on every poll, so the Windows exclusion
// never becomes usable. Filesystems remain first; metadata and memory
// retain the same per-command lock, completion and shared-channel guard.
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(ctx, client, instanceName, res.Node, res.Name, res.VMID, status, false)
if metadataDeferred {
state.guestAgentStatus = "deferred"
}
if len(guestIPs) > 0 {
state.ipAddresses = guestIPs
}
if len(guestIfaces) > 0 {
state.networkInterfaces = guestIfaces
}
if guestOSName != "" {
state.osName = guestOSName
}
if guestOSVersion != "" {
state.osVersion = guestOSVersion
}
if guestAgentVersion != "" {
state.agentVersion = guestAgentVersion
}
var memoryDeferred bool
state.memTotal, state.memUsed, state.memorySource, memoryDeferred = m.resolveGuestStatusMemory(
ctx,
client,
instanceName,
res.Name,
res.Node,
res.VMID,
guestID,
status,
vmIDToHostAgent,
state.memTotal,
state.memorySource,
&state.guestRaw,
)
if memoryDeferred {
state.guestAgentStatus = "deferred"
}
mergeVMRuntimeCounters(state, status)
}
func (m *Monitor) applyVMGuestAgentFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface, state *vmBuildState) {
var fsDisks []models.Disk
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, fsDisks, state.diskFromAgent, state.diskStatusReason = m.updateVMDisksFromGuestAgentFSInfo(
ctx, instanceName, res, client, state.diskTotal, state.diskUsed, state.diskUsage,
)
if len(fsDisks) > 0 {
state.individualDisks = fsDisks
}
if state.diskFromAgent {
state.diskObservation.ObservedAt = time.Now()
renewGuestAgentEvidence(&state.guestAgentEvidence, state.diskObservation.ObservedAt, state.diskObservation.ObservedAt)
}
if guestAgentDiskDeferred(state.diskStatusReason) {
state.guestAgentStatus = "deferred"
}
}
func mergeVMRuntimeCounters(state *vmBuildState, status *proxmox.VMStatus) {
if state == nil || status == nil {
return
}
// A detailed status response is newer and more authoritative than the
// cluster listing, but only for fields it actually contains.
statusPresence := status.IOCounters.Effective()
if statusPresence.DiskRead {
state.diskReadBytes = int64(status.DiskRead)
state.counterPresence.DiskRead = true
state.counterObservationTimes.DiskRead = status.ObservedAt
}
if statusPresence.DiskWrite {
state.diskWriteBytes = int64(status.DiskWrite)
state.counterPresence.DiskWrite = true
state.counterObservationTimes.DiskWrite = status.ObservedAt
}
if statusPresence.NetworkIn {
state.networkInBytes = int64(status.NetIn)
state.counterPresence.NetworkIn = true
state.counterObservationTimes.NetworkIn = status.ObservedAt
}
if statusPresence.NetworkOut {
state.networkOutBytes = int64(status.NetOut)
state.counterPresence.NetworkOut = true
state.counterObservationTimes.NetworkOut = status.ObservedAt
}
if !status.ObservedAt.IsZero() {
state.counterObservedAt = status.ObservedAt
}
if status.Uptime > 0 {
state.counterUptime = status.Uptime
}
}
func vmGuestAgentRuntimeState(status *proxmox.VMStatus, recentGuestAgentEvidence bool) (string, bool) {
if status == nil {
return "", false
}
if status.Lock != "" {
return "deferred", status.Agent.IsEnabled()
}
if status.Agent.IsAvailable() {
return "available", true
}
if status.Agent.IsEnabled() {
if recentGuestAgentEvidence {
return "expected-unreachable", true
}
return "not-running", false
}
return "disabled", false
}
func (m *Monitor) buildVMFromClusterResource(
ctx context.Context,
instanceName string,
res proxmox.ClusterResource,
client PVEClientInterface,
guestID string,
vmIDToHostAgent map[string]models.Host,
prevVM *models.VM,
) (models.VM, VMMemoryRaw, string, []string, time.Time, bool) {
// Skip templates if configured
if res.Template == 1 {
return models.VM{}, VMMemoryRaw{}, "", nil, time.Time{}, false
}
if guestID == "" {
guestID = makeGuestID(instanceName, res.Node, res.VMID)
}
prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID)
prePollTime := time.Now()
recentGuestAgentEvidence := hasRecentGuestAgentEvidence(prevVM, prePollTime) ||
m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, prePollTime)
state := vmBuildState{
guestAgentEvidence: retainGuestAgentEvidence(prevVM, prePollTime),
diskObservation: models.GuestDiskObservation{Source: "guest-agent"},
memTotal: res.MaxMem,
memUsed: res.Mem,
memorySource: "cluster-resources",
guestRaw: VMMemoryRaw{ListingMem: res.Mem, ListingMaxMem: res.MaxMem},
diskReadBytes: int64(res.DiskRead),
diskWriteBytes: int64(res.DiskWrite),
networkInBytes: int64(res.NetIn),
networkOutBytes: int64(res.NetOut),
counterPresence: pveCounterPresence(res.IOCounters),
counterObservedAt: observedAtOr(res.ObservedAt, prePollTime),
counterObservationTimes: counterObservationTimes(observedAtOr(res.ObservedAt, prePollTime)),
counterUptime: res.Uptime,
diskTotal: res.MaxDisk,
diskUsed: res.Disk,
}
state.diskFree = state.diskTotal - state.diskUsed
state.diskUsage = safePercentage(float64(state.diskUsed), float64(state.diskTotal))
// If VM shows 0 disk usage but has allocated disk, it's likely guest agent issue
// Set to -1 to indicate "unknown" rather than showing misleading 0%
if res.Type == "qemu" && state.diskUsed == 0 && state.diskTotal > 0 && res.Status == "running" {
state.diskUsage = -1
}
if res.Status != "running" {
// A subsequent start must obtain its own guest memory observation rather
// than inherit a success or failure from before this stopped poll.
m.clearGuestMemoryCache(instanceName, res.Node, res.VMID)
if state.diskTotal > 0 {
state.diskUsage = -1
state.diskStatusReason = "vm-stopped"
}
}
// For running VMs, always try to get filesystem info from guest agent
// The cluster/resources endpoint often returns 0 or incorrect values for disk usage
// We should prefer guest agent data when available for accurate metrics
if res.Status == "running" && res.Type == "qemu" {
// First check if agent is enabled by getting VM status
status, err := client.GetVMStatus(ctx, res.Node, res.VMID)
if err != nil {
if state.diskTotal > 0 {
state.diskStatusReason = "no-status"
}
log.Debug().
Err(err).
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Could not get VM status to check guest agent availability")
} else if status != nil {
m.applyVMStatusDetails(ctx, instanceName, res, client, guestID, vmIDToHostAgent, status, recentGuestAgentEvidence, &state)
} else {
if state.diskTotal > 0 {
state.diskStatusReason = "no-status"
}
// No vmStatus available - keep cluster/resources data
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Could not get VM status, using cluster/resources disk data")
}
if state.detailedStatus != nil && state.detailedStatus.Lock != "" {
res.Lock = state.detailedStatus.Lock
}
now := time.Now()
guestAgentAvailable := shouldQueryGuestAgent(state.detailedStatus, prevVM, now) ||
m.hasRecentGuestMetadataEvidence(instanceName, res.Node, res.VMID, now)
if guestAgentAvailable && res.Lock == "" && state.detailedStatus == nil {
m.applyVMGuestAgentFSInfo(ctx, instanceName, res, client, &state)
guestIPs, guestIfaces, guestOSName, guestOSVersion, guestAgentVersion, metadataDeferred := m.fetchGuestAgentMetadata(
ctx,
client,
instanceName,
res.Node,
res.Name,
res.VMID,
nil,
true,
)
if metadataDeferred {
state.guestAgentStatus = "deferred"
}
if len(guestIPs) > 0 {
state.ipAddresses = guestIPs
}
if len(guestIfaces) > 0 {
state.networkInterfaces = guestIfaces
}
if guestOSName != "" {
state.osName = guestOSName
}
if guestOSVersion != "" {
state.osVersion = guestOSVersion
}
if guestAgentVersion != "" {
state.agentVersion = guestAgentVersion
}
state.guestAgentExpected = true
// Cached strings are display continuity, not a new successful read.
metadataAt := m.guestMetadataEvidenceTime(instanceName, res.Node, res.VMID)
metadataObserved := !metadataAt.Before(prePollTime) && !metadataAt.After(time.Now())
if state.guestAgentStatus != "deferred" && (state.diskFromAgent || metadataObserved) {
state.guestAgentStatus = "available"
}
}
if res.Lock != "" {
state.guestAgentStatus = "deferred"
state.diskStatusReason = "vm-locked"
state.diskFromAgent = false
state.diskUsage = -1
if guestMemorySourceReliability(state.memorySource) == guestMemoryReliabilityLow {
state.memorySource = "unavailable"
state.memUsed = 0
}
if prevVM != nil {
state.ipAddresses = cloneStringSlice(prevVM.IPAddresses)
state.networkInterfaces = cloneGuestNetworkInterfaces(prevVM.NetworkInterfaces)
state.osName, state.osVersion, state.agentVersion = prevVM.OSName, prevVM.OSVersion, prevVM.AgentVersion
}
}
var preferred bool
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason, preferred = preferLinkedHostAgentDiskInventory(
guestID,
vmIDToHostAgent,
state.diskTotal,
state.diskUsed,
state.diskFree,
state.diskUsage,
state.individualDisks,
state.diskStatusReason,
)
if preferred {
state.diskObservation = models.GuestDiskObservation{Source: "agent", ObservedAt: vmIDToHostAgent[guestID].LastSeen}
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Bool("guestAgentDiskAvailable", state.diskFromAgent).
Float64("usage", state.diskUsage).
Msg("QEMU disk: preferring linked Pulse host agent disk inventory")
}
}
sampleTime := time.Now()
if res.Status == "running" {
renewGuestAgentEvidence(&state.guestAgentEvidence, m.guestMetadataEvidenceTime(instanceName, res.Node, res.VMID), sampleTime)
} else {
// A stopped guest supplies no current fallback eligibility evidence.
state.guestAgentEvidence = models.GuestAgentEvidence{Explicit: true}
}
state.diskTotal, state.diskUsed, state.diskFree, state.diskUsage, state.individualDisks, state.diskStatusReason = stabilizeGuestLowTrustDisk(
prevVM,
res.Status,
state.diskTotal,
state.diskUsed,
state.diskFree,
state.diskUsage,
state.individualDisks,
state.diskStatusReason,
state.diskFromAgent,
sampleTime,
)
if strings.HasPrefix(state.diskStatusReason, "prev-") {
state.diskObservation.ObservedAt = guestDiskObservationTime(prevVM)
}
if res.Status != "running" {
state.memorySource = "powered-off"
state.memUsed = 0
state.onBoot = m.fetchVMOnBoot(ctx, client, res.Node, res.VMID)
}
var snapshotNotes []string
if prevSnapshot != nil && (prevSnapshot.Raw.StatusMaxMem > 0 || prevSnapshot.Raw.ListingMaxMem > 0) {
currentCapacity := state.guestRaw.StatusMaxMem
if currentCapacity == 0 {
currentCapacity = state.guestRaw.ListingMaxMem
}
if guestMemoryConfiguredCapacity(prevSnapshot) != currentCapacity {
prevSnapshot = nil // A resize cannot retain the earlier guest tuple.
}
}
state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(
prevSnapshot,
res.Status,
state.memorySource,
state.memTotal,
state.memUsed,
sampleTime,
guestAgentSignalsHealthy(
state.detailedStatus,
state.diskFromAgent,
state.ipAddresses,
state.networkInterfaces,
state.osName,
state.osVersion,
state.agentVersion,
),
)
if state.memorySource == "previous-snapshot" && prevSnapshot != nil {
// Retention keeps the original guest denominator as well as its usage.
// The selector has already checked capacity continuity and source age.
state.memTotal = uint64(prevSnapshot.Memory.Total)
}
memFree := uint64(0)
if state.memTotal >= state.memUsed {
memFree = state.memTotal - state.memUsed
}
currentMetrics := IOMetrics{
DiskRead: state.diskReadBytes,
DiskWrite: state.diskWriteBytes,
NetworkIn: state.networkInBytes,
NetworkOut: state.networkOutBytes,
Timestamp: state.counterObservedAt,
Presence: state.counterPresence,
ObservedAt: state.counterObservationTimes,
SourceUptime: state.counterUptime,
}
diskReadRate, diskWriteRate, netInRate, netOutRate := m.rateTracker.CalculateRates(
makeGuestRateKey(instanceName, "qemu", res.VMID),
currentMetrics,
)
diskReadValue, diskWriteValue, networkInValue, networkOutValue, rateValidity := guestRateValues(
diskReadRate,
diskWriteRate,
netInRate,
netOutRate,
)
memory := models.UnavailableMemory(clampToInt64(state.memTotal))
if CanonicalMemorySource(state.memorySource) != "unavailable" {
memory = models.Memory{
Total: int64(state.memTotal),
Used: int64(state.memUsed),
Free: int64(memFree),
Usage: safePercentage(float64(state.memUsed), float64(state.memTotal)),
}
}
if memory.Free < 0 {
memory.Free = 0
}
if memory.Used > memory.Total {
memory.Used = memory.Total
}
// Free above is total-used, i.e. available. When the guest reported its
// truly-free pages (meminfo), split the reclaimable cache back out.
var trulyFree uint64
switch CanonicalMemorySource(state.memorySource) {
case "available-field", "derived-free-buffers-cached", "derived-total-minus-used":
trulyFree = state.guestRaw.MemInfoFree
case "guest-agent-meminfo", "guest-agent-meminfo-derived":
trulyFree = state.guestRaw.GuestAgentMemFree
}
splitReclaimableMemory(&memory, trulyFree)
if state.memorySource == "previous-snapshot" && prevSnapshot != nil && prevSnapshot.Memory.Observation.State != "" {
memory = prevSnapshot.Memory
}
if state.detailedStatus != nil && state.detailedStatus.Balloon > 0 {
memory.Balloon = int64(state.detailedStatus.Balloon)
}
if state.guestAgentStatus == "deferred" && (CanonicalMemorySource(state.memorySource) == "unavailable" || state.memorySource == "previous-snapshot") {
if retained, ok := m.deferredVMGuestMemory(instanceName, res.Node, res.VMID, state.memTotal, prevSnapshot, sampleTime); ok {
memory = retained
state.memorySource = "previous-snapshot"
snapshotNotes = append(snapshotNotes, "preserved-original-guest-memory-during-deferral")
}
}
providerObservedAt := observedAtOr(res.ObservedAt, sampleTime)
if state.detailedStatus != nil && CanonicalMemorySource(state.memorySource) != "cluster-resources" {
providerObservedAt = observedAtOr(state.detailedStatus.ObservedAt, sampleTime)
}
memory.Observation = m.guestMemoryObservation(instanceName, "qemu", res.Node, res.VMID, memory, guestMemoryObservationContext{
source: state.memorySource, providerObservedAt: providerObservedAt,
deferred: state.guestAgentStatus == "deferred", previous: prevSnapshot,
linkedAgent: vmIDToHostAgent[guestID],
}, sampleTime)
vm := models.VM{
ID: guestID,
VMID: res.VMID,
Name: res.Name,
Node: res.Node,
Pool: strings.TrimSpace(res.Pool),
Instance: instanceName,
Status: res.Status,
Type: "qemu",
CPU: safeFloat(res.CPU),
CPUs: res.MaxCPU,
Memory: memory,
Disk: models.Disk{
Total: int64(state.diskTotal),
Used: int64(state.diskUsed),
Free: int64(state.diskFree),
Usage: state.diskUsage,
},
Disks: state.individualDisks,
DiskStatusReason: state.diskStatusReason,
DiskObservation: state.diskObservation,
GuestAgentStatus: state.guestAgentStatus,
Lock: res.Lock,
GuestAgentExpected: state.guestAgentExpected,
GuestAgentEvidence: state.guestAgentEvidence,
IPAddresses: state.ipAddresses,
OSName: state.osName,
OSVersion: state.osVersion,
AgentVersion: state.agentVersion,
NetworkInterfaces: state.networkInterfaces,
NetworkIn: networkInValue,
NetworkOut: networkOutValue,
DiskRead: diskReadValue,
DiskWrite: diskWriteValue,
IORateValidity: rateValidity,
Uptime: int64(res.Uptime),
Template: res.Template == 1,
OnBoot: state.onBoot,
LastSeen: sampleTime,
}
// Parse tags
if res.Tags != "" {
vm.Tags = strings.Split(res.Tags, ";")
// Log if Pulse-specific tags are detected
for _, tag := range vm.Tags {
switch tag {
case "pulse-no-alerts", "pulse-monitor-only", "pulse-relaxed":
log.Info().
Str("vm", vm.Name).
Str("node", vm.Node).
Str("tag", tag).
Msg("Pulse control tag detected on VM")
}
}
}
if res.Status != "running" {
vm.NetworkIn = 0
vm.NetworkOut = 0
vm.DiskRead = 0
vm.DiskWrite = 0
vm.IORateValidity = models.IORateValidity{
Explicit: true,
DiskRead: true,
DiskWrite: true,
NetworkIn: true,
NetworkOut: true,
}
}
return vm, state.guestRaw, state.memorySource, snapshotNotes, sampleTime, true
}
// fetchVMOnBoot retrieves the onboot (autostart) setting for a stopped VM by
// fetching its config. Returns nil only when the config is unavailable or the
// value is unrecognised. When onboot is absent from a valid config, returns
// false because Proxmox's default is "do not start on boot". Uses a type assertion because PVEClientInterface does not include
// GetVMConfig (it is only on the concrete client, matching the pattern in
// guest_config.go).
func (m *Monitor) fetchVMOnBoot(ctx context.Context, client PVEClientInterface, node string, vmid int) *bool {
type vmConfigClient interface {
GetVMConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error)
}
vmClient, ok := client.(vmConfigClient)
if !ok {
return nil
}
configCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
configData, err := vmClient.GetVMConfig(configCtx, node, vmid)
if err != nil || len(configData) == 0 {
return nil
}
return parseProxmoxOnBoot(configData)
}
type vmFSInfoSummary struct {
totalBytes uint64
usedBytes uint64
individualDisks []models.Disk
skippedFS []string
includedFS []string
invalidBytes bool
}
func (m *Monitor) fetchVMFSInfo(ctx context.Context, instanceName string, res proxmox.ClusterResource, client PVEClientInterface) ([]proxmox.VMFileSystem, string, bool) {
if res.Lock != "" {
return nil, "vm-locked", false
}
// Use retry logic for guest agent calls to handle transient timeouts (refs #630)
fsInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentFSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
return client.GetVMFSInfo(ctx, res.Node, res.VMID)
})
var fsInfo []proxmox.VMFileSystem
if err == nil {
if fs, ok := fsInfoRaw.([]proxmox.VMFileSystem); ok {
fsInfo = fs
}
}
if err != nil {
if reason := proxmox.GuestAgentDeferredReason(err); reason != "" {
return nil, reason, false
}
reason := classifyGuestAgentDiskStatusError(err)
logVMFilesystemUnavailable(ctx, instanceName, res, reason)
return nil, classifyGuestAgentDiskStatusError(err), false
}
if len(fsInfo) == 0 {
logVMFilesystemUnavailable(ctx, instanceName, res, "no-filesystems")
return nil, "no-filesystems", false
}
return fsInfo, "", true
}
// Keep ordinary process logging when no context logger is installed. Tests and
// callers with an explicit logger can own their output without global mutation.
func guestFilesystemLogger(ctx context.Context) *zerolog.Logger {
logger := log.Ctx(ctx)
if logger.GetLevel() == zerolog.Disabled {
return &log.Logger
}
return logger
}
// Missing optional readings do not establish a need to activate/restart QGA,
// change backup policy, or reconfigure a shared Proxmox role. Keep this guidance
// independent of provider error text and the guest's operating system.
func logVMFilesystemUnavailable(ctx context.Context, instanceName string, res proxmox.ClusterResource, reason string) {
message := "Guest filesystem query failed; check disk usage inside the guest. Keep existing guest-agent and backup settings."
switch reason {
case "agent-not-running":
message = "Proxmox reports the guest agent is not running; check disk usage inside the guest. Keep existing guest-agent and backup settings."
case "permission-denied":
message = "Guest filesystem query was not authorised; check the existing credential's access to this VM. Do not broaden shared roles or change guest-agent and backup settings to diagnose missing readings."
case "agent-timeout":
message = "Guest filesystem query did not complete; check disk usage inside the guest. Keep existing guest-agent and backup settings."
case "no-filesystems":
message = "Guest agent returned no filesystem readings; check disk usage inside the guest. Keep existing guest-agent and backup settings."
}
guestFilesystemLogger(ctx).Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("reason", reason).
Msg(message)
}
func (m *Monitor) summarizeVMFSInfo(instanceName string, res proxmox.ClusterResource, fsInfo []proxmox.VMFileSystem) vmFSInfoSummary {
summary := vmFSInfoSummary{}
// Track seen filesystems to deduplicate mounts that share the same device:
// 1. btrfs/zfs mount multiple subvolumes from a shared pool
// 2. Kubernetes bind-mounts volumes at both pod and plugin paths
// In both cases, only count the device's capacity once.
// Mounts are sampled separately, so aliases of one volume can disagree on
// usage. Count its capacity once and retain the highest observed usage;
// taking the first mount makes the aggregate depend on response order.
seenFilesystems := make(map[string]uint64)
// Log all filesystems received for debugging
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Int("filesystem_count", len(fsInfo)).
Msg("Processing filesystems from guest agent")
for _, fs := range fsInfo {
// Other PVEClientInterface implementations can supply records without
// the wire decoder. Never subtract/cast contradictory or unrepresentable
// counters into the signed disk model, even in an individual mount row.
if fs.UsedBytes > fs.TotalBytes || fs.TotalBytes > math.MaxInt64 {
summary.invalidBytes = true
summary.skippedFS = append(summary.skippedFS, fs.Mountpoint+"(invalid-byte-counts)")
continue
}
// Skip special filesystems and mounts
shouldSkip, reasons := fsfilters.ShouldSkipFilesystem(fs.Type, fs.Mountpoint, fs.TotalBytes, fs.UsedBytes)
if shouldSkip {
// Check if any reason is read-only for detailed logging
for _, r := range reasons {
if strings.HasPrefix(r, "read-only-") {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Float64("total_gb", float64(fs.TotalBytes)/1073741824).
Float64("used_gb", float64(fs.UsedBytes)/1073741824).
Msg("Skipping read-only filesystem from disk aggregation")
break
}
}
summary.skippedFS = append(summary.skippedFS, fmt.Sprintf("%s(%s,%s)",
fs.Mountpoint, fs.Type, strings.Join(reasons, ",")))
continue
}
// Only count real filesystems with valid data
// Some filesystems report 0 bytes (like unformatted or system partitions)
if fs.TotalBytes > 0 {
// Deduplication: the same device can appear multiple times when:
// 1. btrfs/zfs mount multiple subvolumes from a shared pool
// 2. Kubernetes bind-mounts volumes at both pod and plugin paths
// In both cases, only count the device's capacity once.
fsTypeLower := strings.ToLower(fs.Type)
countThisFS := true
var dedupeKey string
if volumeKey := windowsVMFilesystemCapacityKey(fs); volumeKey != "" {
// QGA's disk.dev is the backing physical drive on Windows,
// not the volume. Equal-sized partitions must remain separate.
// A GUID can still identify one volume mounted at several paths.
dedupeKey = fmt.Sprintf("windows:%s:%d", volumeKey, fs.TotalBytes)
_, seen := seenFilesystems[dedupeKey]
countThisFS = !seen
} else if fs.Disk != "" {
// Same non-Windows device at multiple mount paths → count once
dedupeKey = fmt.Sprintf("%s:%d", fs.Disk, fs.TotalBytes)
if _, seen := seenFilesystems[dedupeKey]; seen {
countThisFS = false
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Str("device", fs.Disk).
Uint64("total", fs.TotalBytes).
Str("dedupe_key", dedupeKey).
Msg("Skipping duplicate device mount in total calculation")
}
} else if fsTypeLower == "btrfs" || fsTypeLower == "zfs" ||
strings.HasPrefix(fsTypeLower, "zfs") {
// No device info — fall back to type+size dedup for COW filesystems
dedupeKey = fmt.Sprintf("%s::%d", fsTypeLower, fs.TotalBytes)
if _, seen := seenFilesystems[dedupeKey]; seen {
countThisFS = false
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Uint64("total", fs.TotalBytes).
Str("dedupe_key", dedupeKey).
Msg("Skipping duplicate btrfs/zfs subvolume in total calculation")
}
}
if dedupeKey != "" {
previousUsed := seenFilesystems[dedupeKey]
if !countThisFS && fs.UsedBytes > previousUsed {
// Each used count is bounded by its already-counted total,
// so this delta cannot exceed the signed aggregate limit.
summary.usedBytes += fs.UsedBytes - previousUsed
}
if countThisFS || fs.UsedBytes > previousUsed {
seenFilesystems[dedupeKey] = fs.UsedBytes
}
}
if countThisFS {
if fs.TotalBytes > math.MaxInt64-summary.totalBytes {
// A partial aggregate would look like complete guest usage.
// Leave it unavailable rather than wrap or silently drop a disk.
return vmFSInfoSummary{invalidBytes: true}
}
summary.totalBytes += fs.TotalBytes
summary.usedBytes += fs.UsedBytes
}
summary.includedFS = append(summary.includedFS, fmt.Sprintf("%s(%s,%.1fGB)",
fs.Mountpoint, fs.Type, float64(fs.TotalBytes)/1073741824))
// Add to individual disks array (always include for display)
summary.individualDisks = append(summary.individualDisks, models.Disk{
Total: int64(fs.TotalBytes),
Used: int64(fs.UsedBytes),
Free: int64(fs.TotalBytes - fs.UsedBytes),
Usage: safePercentage(float64(fs.UsedBytes), float64(fs.TotalBytes)),
Mountpoint: fs.Mountpoint,
Type: fs.Type,
Device: fs.Disk,
})
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Uint64("total", fs.TotalBytes).
Uint64("used", fs.UsedBytes).
Float64("total_gb", float64(fs.TotalBytes)/1073741824).
Float64("used_gb", float64(fs.UsedBytes)/1073741824).
Bool("counted_in_total", countThisFS).
Msg("Including filesystem in disk usage calculation")
} else if fs.TotalBytes == 0 && len(fs.Mountpoint) > 0 {
summary.skippedFS = append(summary.skippedFS, fmt.Sprintf("%s(%s,0GB)", fs.Mountpoint, fs.Type))
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Str("mountpoint", fs.Mountpoint).
Str("type", fs.Type).
Msg("Skipping filesystem with zero total bytes")
}
}
if len(summary.skippedFS) > 0 {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Strs("skipped", summary.skippedFS).
Msg("Skipped special filesystems")
}
if len(summary.includedFS) > 0 {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Strs("included", summary.includedFS).
Msg("Filesystems included in disk calculation")
}
return summary
}
func (m *Monitor) updateVMDisksFromGuestAgentFSInfo(
ctx context.Context,
instanceName string,
res proxmox.ClusterResource,
client PVEClientInterface,
diskTotal uint64,
diskUsed uint64,
diskUsage float64,
) (uint64, uint64, uint64, float64, []models.Disk, bool, string) {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Msg("Guest agent enabled, querying filesystem info for accurate disk usage")
fsInfo, diskStatusReason, ok := m.fetchVMFSInfo(ctx, instanceName, res, client)
if !ok {
// No trustworthy usage data: cluster/resources reports 0 used for
// QEMU, so passing it through renders a confident "0%" in the UI.
// Use the -1 sentinel (same as the agent-disabled path) so the
// frontend shows unavailable + diskStatusReason instead; the
// stabilizer may still replace this with the previous good sample.
if diskTotal > 0 {
diskUsage = -1
}
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, diskStatusReason
}
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("filesystems", len(fsInfo)).
Msg("Got filesystem info from guest agent")
summary := m.summarizeVMFSInfo(instanceName, res, fsInfo)
// Valid peer rows are not complete guest usage if any reading was rejected.
// The wire client reports that distinction as an error; alternate clients
// can supply invalid counters directly, so enforce it at this boundary too.
if summary.totalBytes > 0 && !summary.invalidBytes {
// Sanity check: if the reported disk is way larger than allocated disk,
// we might be getting host disk info somehow
allocatedDiskGB := float64(res.MaxDisk) / 1073741824
reportedDiskGB := float64(summary.totalBytes) / 1073741824
// If reported disk is more than 2x the allocated disk, log for debugging.
// This is expected for pass-through disks and can also indicate we're getting host disk or network shares.
if allocatedDiskGB > 0 && reportedDiskGB > allocatedDiskGB*2 {
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Float64("allocated_gb", allocatedDiskGB).
Float64("reported_gb", reportedDiskGB).
Float64("ratio", reportedDiskGB/allocatedDiskGB).
Strs("filesystems", summary.includedFS).
Msg("VM reports disk usage significantly larger than allocated disk - possible issue with filesystem detection")
}
diskTotal = summary.totalBytes
diskUsed = summary.usedBytes
diskUsage = safePercentage(float64(summary.usedBytes), float64(summary.totalBytes))
log.Debug().
Str("instance", instanceName).
Str("vm", res.Name).
Int("vmid", res.VMID).
Uint64("totalBytes", summary.totalBytes).
Uint64("usedBytes", summary.usedBytes).
Float64("total_gb", float64(summary.totalBytes)/1073741824).
Float64("used_gb", float64(summary.usedBytes)/1073741824).
Float64("allocated_gb", allocatedDiskGB).
Float64("usage", diskUsage).
Uint64("old_disk", res.Disk).
Uint64("old_maxdisk", res.MaxDisk).
Msg("Using guest agent data for accurate disk usage (replacing cluster/resources data)")
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, summary.individualDisks, true, ""
}
// No usable aggregate - show allocated disk size instead. Invalid counters
// are a reading failure, not evidence that every filesystem is special.
if diskTotal > 0 {
diskUsage = -1 // Show as allocated size
}
log.Info().
Str("instance", instanceName).
Str("vm", res.Name).
Int("filesystems_found", len(fsInfo)).
Bool("invalid_byte_counts", summary.invalidBytes).
Msg("Guest agent provided filesystem info but no usable disk aggregate was available")
if summary.invalidBytes {
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "agent-error"
}
return diskTotal, diskUsed, diskTotal - diskUsed, diskUsage, nil, false, "special-filesystems-only"
}
func guestAgentDiskDeferred(reason string) bool {
switch strings.TrimPrefix(reason, "prev-") {
case "vm-locked", "lock-unverified", "agent-busy", "agent-cooldown", "agent-capacity", "agent-timeout", "agent-response-incomplete", "agent-redirect", "agent-transport-unverified", "agent-completion-unverified":
return true
}
return false
}