Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"errors"
"math"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
"github.com/rs/zerolog/log"
)
const guestStatusMemoryMismatchTolerance uint64 = 128 * 1024 * 1024 // 128 MiB
// deriveGuestMemInfoAvailable normalizes guest meminfo-based availability
// selection so the efficient VM builder and the node-by-node VM polling path
// do not drift apart on degraded Proxmox guest-agent payloads.
func deriveGuestMemInfoAvailable(memInfo *proxmox.VMMemInfo, guestRaw *VMMemoryRaw) (uint64, string) {
if memInfo == nil {
return 0, ""
}
if guestRaw != nil {
guestRaw.MemInfoUsed = memInfo.Used
guestRaw.MemInfoFree = memInfo.Free
guestRaw.MemInfoTotal = memInfo.Total
guestRaw.MemInfoAvailable = memInfo.Available
guestRaw.MemInfoBuffers = memInfo.Buffers
guestRaw.MemInfoCached = memInfo.Cached
guestRaw.MemInfoShared = memInfo.Shared
}
componentAvailable := memInfo.Free
if memInfo.Buffers > 0 {
if math.MaxUint64-componentAvailable < memInfo.Buffers {
componentAvailable = math.MaxUint64
} else {
componentAvailable += memInfo.Buffers
}
}
if memInfo.Cached > 0 {
if math.MaxUint64-componentAvailable < memInfo.Cached {
componentAvailable = math.MaxUint64
} else {
componentAvailable += memInfo.Cached
}
}
componentsConflict := memInfo.Total > 0 && componentAvailable > memInfo.Total
availableFromUsed := uint64(0)
if memInfo.Total > 0 && memInfo.Used > 0 && memInfo.Total >= memInfo.Used {
availableFromUsed = memInfo.Total - memInfo.Used
if guestRaw != nil {
guestRaw.MemInfoTotalMinusUsed = availableFromUsed
}
}
missingCacheMetrics := memInfo.Available == 0 &&
memInfo.Buffers == 0 &&
memInfo.Cached == 0
switch {
case memInfo.HasAvailable() && (memInfo.Total == 0 || memInfo.Available <= memInfo.Total):
return memInfo.Available, "available-field"
case memInfo.Free > 0 || memInfo.Buffers > 0 || memInfo.Cached > 0:
if availableFromUsed > 0 && missingCacheMetrics {
const vmTotalMinusUsedGapTolerance uint64 = 4 * 1024 * 1024
if availableFromUsed > componentAvailable {
gap := availableFromUsed - componentAvailable
if componentAvailable == 0 || gap >= vmTotalMinusUsedGapTolerance {
return availableFromUsed, "derived-total-minus-used"
}
}
}
if missingCacheMetrics {
return 0, ""
}
if componentsConflict {
return 0, ""
}
return componentAvailable, "derived-free-buffers-cached"
default:
if availableFromUsed > 0 && missingCacheMetrics {
const vmTotalMinusUsedGapTolerance uint64 = 4 * 1024 * 1024
if availableFromUsed > componentAvailable {
gap := availableFromUsed - componentAvailable
if componentAvailable == 0 || gap >= vmTotalMinusUsedGapTolerance {
return availableFromUsed, "derived-total-minus-used"
}
}
}
return 0, ""
}
}
func saturatingAddUint64(lhs, rhs uint64) uint64 {
if math.MaxUint64-lhs < rhs {
return math.MaxUint64
}
return lhs + rhs
}
func guestStatusFreeMem(status *proxmox.VMStatus) uint64 {
if status == nil {
return 0
}
if status.FreeMem > 0 {
return status.FreeMem
}
if status.BalloonInfo != nil {
return status.BalloonInfo.FreeMem
}
return 0
}
func effectiveGuestFreeMemTotal(memTotal uint64, status *proxmox.VMStatus) uint64 {
if status == nil {
return memTotal
}
freeMem := guestStatusFreeMem(status)
if status.BalloonInfo != nil {
if status.BalloonInfo.TotalMem > 0 && freeMem <= status.BalloonInfo.TotalMem {
if memTotal > 0 && status.BalloonInfo.TotalMem > memTotal {
return memTotal
}
return status.BalloonInfo.TotalMem
}
if status.BalloonInfo.Actual > 0 && freeMem <= status.BalloonInfo.Actual {
if memTotal > 0 && status.BalloonInfo.Actual > memTotal {
return memTotal
}
return status.BalloonInfo.Actual
}
}
if status.Balloon > 0 && status.Balloon <= memTotal && freeMem <= status.Balloon {
return status.Balloon
}
return memTotal
}
func selectGuestLowTrustUsedMemory(memTotal uint64, status *proxmox.VMStatus) (uint64, string) {
if status == nil {
return 0, ""
}
freeMemTotal := effectiveGuestFreeMemTotal(memTotal, status)
freeMem := guestStatusFreeMem(status)
hasFreeFallback := freeMem > 0 && freeMemTotal >= freeMem
freeDerivedUsed := uint64(0)
if hasFreeFallback {
freeDerivedUsed = freeMemTotal - freeMem
}
if status.Mem > 0 {
if hasFreeFallback && freeDerivedUsed < status.Mem {
statusMemPlusFree := saturatingAddUint64(status.Mem, freeMem)
if status.Mem >= freeMemTotal && freeDerivedUsed < freeMemTotal {
return freeDerivedUsed, "status-freemem"
}
if statusMemPlusFree > freeMemTotal+guestStatusMemoryMismatchTolerance {
return freeDerivedUsed, "status-freemem"
}
}
return status.Mem, "status-mem"
}
if hasFreeFallback {
return freeDerivedUsed, "status-freemem"
}
return 0, ""
}
func shouldPreferGuestAgentMemAvailable(status *proxmox.VMStatus, memTotal uint64) bool {
if status == nil || status.Lock != "" || !status.Agent.IsAvailable() || status.Mem == 0 {
return false
}
if guestStatusFreeMem(status) > 0 {
return false
}
effectiveTotal := memTotal
if status.MaxMem > 0 {
effectiveTotal = status.MaxMem
}
if effectiveTotal == 0 {
return false
}
if status.Mem >= effectiveTotal {
return true
}
return effectiveTotal-status.Mem <= guestStatusMemoryMismatchTolerance
}
func guestMemoryFallbackReason(source string) string {
return MemorySourceFallbackReason(source)
}
// The final result is admission deferral, not a failed or successful sample.
func (m *Monitor) tryGuestAgentMemAvailable(
ctx context.Context,
client PVEClientInterface,
instanceName string,
guestName string,
node string,
vmid int,
memTotal uint64,
guestRaw *VMMemoryRaw,
) (uint64, uint64, string, bool, bool) {
availability, agentErr := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)
if agentErr != nil || availability.Source == "" {
return 0, 0, "", false, errors.Is(agentErr, proxmox.ErrGuestAgentDeferred)
}
// MemAvailable belongs to this guest's MemTotal, not the configured
// maximum. Ballooning and kernel-reserved pages can make them different.
// Older availability-only clients retain their supplied capacity fallback.
configuredTotal := memTotal
if availability.Total > 0 {
memTotal = availability.Total
}
agentAvailable := availability.EffectiveAvailable
if guestRaw != nil {
guestRaw.GuestAgentMemTotal = availability.Total
guestRaw.GuestAgentMemAvailable = agentAvailable
guestRaw.GuestAgentMemFree = availability.Free
guestRaw.GuestAgentMemBuffers = availability.Buffers
guestRaw.GuestAgentMemCached = availability.Cached
guestRaw.GuestAgentSReclaimable = availability.SReclaimable
guestRaw.GuestAgentShmem = availability.Shmem
guestRaw.GuestAgentDerived = availability.Source == "meminfo-derived"
}
if memTotal == 0 || memTotal > math.MaxInt64 || agentAvailable > memTotal ||
(configuredTotal > 0 && memTotal > configuredTotal) {
return 0, 0, "", false, false
}
source := "guest-agent-meminfo"
if availability.Source == "meminfo-derived" {
source = "guest-agent-meminfo-derived"
}
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", agentAvailable).
Str("source", source).
Msg("QEMU memory: using guest agent /proc/meminfo fallback (excludes reclaimable cache)")
return memTotal, agentAvailable, source, true, false
}
// Return shared guest-command deferral even when independent memory is usable.
func (m *Monitor) resolveGuestStatusMemory(
ctx context.Context,
client PVEClientInterface,
instanceName string,
guestName string,
node string,
vmid int,
guestID string,
status *proxmox.VMStatus,
vmIDToHostAgent map[string]models.Host,
memTotal uint64,
memorySource string,
guestRaw *VMMemoryRaw,
) (uint64, uint64, string, bool) {
if status == nil {
return memTotal, 0, memorySource, false
}
if guestRaw != nil {
guestRaw.StatusMaxMem = status.MaxMem
guestRaw.StatusMem = status.Mem
guestRaw.StatusFreeMem = guestStatusFreeMem(status)
guestRaw.Balloon = status.Balloon
guestRaw.BalloonMin = status.BalloonMin
guestRaw.Agent = status.Agent.Value
if status.BalloonInfo != nil {
guestRaw.BalloonInfoFreeMem = status.BalloonInfo.FreeMem
guestRaw.BalloonInfoTotalMem = status.BalloonInfo.TotalMem
guestRaw.BalloonInfoActual = status.BalloonInfo.Actual
}
}
if status.MaxMem > 0 {
memTotal = status.MaxMem
}
memAvailable := uint64(0)
hasMemAvailable := false
derivedTotalMinusUsedAvailable := uint64(0)
derivedTotalMinusUsedTotal := uint64(0)
selectedUsed := uint64(0)
hasSelectedUsed := false
if status.MemInfo != nil {
memInfoTotal := memTotal
if status.MemInfo.Total > 0 {
memInfoTotal = status.MemInfo.Total
}
memAvailable, memorySource = deriveGuestMemInfoAvailable(status.MemInfo, guestRaw)
hasMemAvailable = memorySource != ""
if memInfoTotal == 0 || memInfoTotal > math.MaxInt64 || memAvailable > memInfoTotal ||
(memTotal > 0 && memInfoTotal > memTotal) {
memAvailable = 0
hasMemAvailable = false
memorySource = ""
}
if memorySource == "derived-total-minus-used" {
derivedTotalMinusUsedAvailable = memAvailable
derivedTotalMinusUsedTotal = memInfoTotal
memAvailable = 0
hasMemAvailable = false
memorySource = ""
} else if hasMemAvailable {
memTotal = memInfoTotal
}
}
triedGuestAgentMemAvailable := false
guestReadDeferred := false
if !hasMemAvailable && shouldPreferGuestAgentMemAvailable(status, memTotal) {
triedGuestAgentMemAvailable = true
agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
guestReadDeferred = guestReadDeferred || deferred
if ok {
memTotal = agentTotal
memAvailable = agentAvailable
hasMemAvailable = true
memorySource = agentSource
}
}
// No RRD fallback here: PVE guest rrddata carries only cache-inclusive
// mem/maxmem columns — the cache-aware memused/memavailable columns exist
// only in node RRD, so a guest RRD lookup can never yield memory evidence
// (#1634).
if !hasMemAvailable && !hasSelectedUsed && status.Lock == "" && status.Agent.IsAvailable() && !triedGuestAgentMemAvailable {
agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
guestReadDeferred = guestReadDeferred || deferred
if ok {
memTotal = agentTotal
memAvailable = agentAvailable
hasMemAvailable = true
memorySource = agentSource
}
}
if !hasMemAvailable && !hasSelectedUsed && derivedTotalMinusUsedAvailable > 0 {
memTotal = derivedTotalMinusUsedTotal
memAvailable = derivedTotalMinusUsedAvailable
hasMemAvailable = true
memorySource = "derived-total-minus-used"
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Uint64("availableFromUsed", derivedTotalMinusUsedAvailable).
Msg("QEMU memory: deriving guest available from total-used gap after preferred fallbacks")
}
if !hasMemAvailable && !hasSelectedUsed {
if agentHost, ok := vmIDToHostAgent[guestID]; ok &&
agentHost.Memory.HasKnownUsage() &&
agentHost.Memory.Total > 0 &&
(memTotal == 0 || uint64(agentHost.Memory.Total) <= memTotal) {
memTotal = uint64(agentHost.Memory.Total)
agentAvailable := uint64(agentHost.Memory.Total - agentHost.Memory.Used)
memAvailable = agentAvailable
hasMemAvailable = true
selectedUsed = uint64(agentHost.Memory.Used)
hasSelectedUsed = true
memorySource = "agent"
if guestRaw != nil {
guestRaw.HostAgentTotal = uint64(agentHost.Memory.Total)
guestRaw.HostAgentUsed = uint64(agentHost.Memory.Used)
}
log.Debug().
Str("vm", guestName).
Str("node", node).
Int("vmid", vmid).
Uint64("total", memTotal).
Uint64("available", memAvailable).
Int64("agentTotal", agentHost.Memory.Total).
Int64("agentUsed", agentHost.Memory.Used).
Msg("QEMU memory: using linked Pulse host agent memory (excludes page cache)")
}
}
memUsed := uint64(0)
switch {
case hasSelectedUsed:
memUsed = selectedUsed
case hasMemAvailable:
memUsed = memTotal - memAvailable
default:
if status.MemInfo != nil {
memorySource = "unavailable"
} else {
memUsed, memorySource = selectGuestLowTrustUsedMemory(memTotal, status)
if memorySource == "status-freemem" {
// That fallback already derives used from the balloon capacity.
// Keep its denominator with it, without upgrading its cache trust.
memTotal = effectiveGuestFreeMemTotal(memTotal, status)
}
if memorySource == "" {
memorySource = "unavailable"
}
}
}
if memUsed > memTotal {
memUsed = memTotal
}
return memTotal, memUsed, memorySource, guestReadDeferred
}
// preferLinkedAgentLXCMemory replaces a low-trust Proxmox LXC memory reading
// with the linked Pulse agent's own sample when one is available. The agent runs
// inside the container, so its sample reflects the workload's real footprint,
// while cluster/resources is a cache-inclusive cgroup fallback that can badly
// under- or over-report shared-memory workloads (#2148, mirroring #1962).
//
// An agent inside a container without lxcfs sees the host's /proc/meminfo, so a
// sample is only trusted when its total matches the container's configured
// limit. Preferred provider sources are never overridden.
func preferLinkedAgentLXCMemory(
status string,
memTotal uint64,
memUsed uint64,
memorySource string,
guestRaw *VMMemoryRaw,
agentHost models.Host,
hasAgent bool,
) (uint64, uint64, string) {
if !hasAgent || status != "running" || memTotal == 0 {
return memTotal, memUsed, memorySource
}
switch CanonicalMemorySource(memorySource) {
case "cluster-resources", "unavailable":
default:
return memTotal, memUsed, memorySource
}
if !agentHost.Memory.HasKnownUsage() || agentHost.Memory.Total <= 0 || agentHost.Memory.Used < 0 {
return memTotal, memUsed, memorySource
}
agentTotal := uint64(agentHost.Memory.Total)
agentUsed := uint64(agentHost.Memory.Used)
if agentTotal != memTotal || agentUsed > memTotal {
return memTotal, memUsed, memorySource
}
if guestRaw != nil {
guestRaw.HostAgentTotal = agentTotal
guestRaw.HostAgentUsed = agentUsed
}
log.Debug().
Uint64("total", memTotal).
Uint64("used", agentUsed).
Str("previousSource", memorySource).
Msg("LXC memory: using linked Pulse agent memory over cluster-resources fallback")
return memTotal, agentUsed, "agent"
}