Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.4 KB
Raw Blame
package monitoring

import (
	"context"
	"errors"
	"math"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
)

const guestStatusMemoryMismatchTolerance uint64 = 128 * 1024 * 1024 // 128 MiB

// deriveGuestMemInfoAvailable normalizes guest meminfo-based availability
// selection so the efficient VM builder and the node-by-node VM polling path
// do not drift apart on degraded Proxmox guest-agent payloads.
func deriveGuestMemInfoAvailable(memInfo *proxmox.VMMemInfo, guestRaw *VMMemoryRaw) (uint64, string) {
	if memInfo == nil {
		return 0, ""
	}

	if guestRaw != nil {
		guestRaw.MemInfoUsed = memInfo.Used
		guestRaw.MemInfoFree = memInfo.Free
		guestRaw.MemInfoTotal = memInfo.Total
		guestRaw.MemInfoAvailable = memInfo.Available
		guestRaw.MemInfoBuffers = memInfo.Buffers
		guestRaw.MemInfoCached = memInfo.Cached
		guestRaw.MemInfoShared = memInfo.Shared
	}

	componentAvailable := memInfo.Free
	if memInfo.Buffers > 0 {
		if math.MaxUint64-componentAvailable < memInfo.Buffers {
			componentAvailable = math.MaxUint64
		} else {
			componentAvailable += memInfo.Buffers
		}
	}
	if memInfo.Cached > 0 {
		if math.MaxUint64-componentAvailable < memInfo.Cached {
			componentAvailable = math.MaxUint64
		} else {
			componentAvailable += memInfo.Cached
		}
	}
	componentsConflict := memInfo.Total > 0 && componentAvailable > memInfo.Total

	availableFromUsed := uint64(0)
	if memInfo.Total > 0 && memInfo.Used > 0 && memInfo.Total >= memInfo.Used {
		availableFromUsed = memInfo.Total - memInfo.Used
		if guestRaw != nil {
			guestRaw.MemInfoTotalMinusUsed = availableFromUsed
		}
	}

	missingCacheMetrics := memInfo.Available == 0 &&
		memInfo.Buffers == 0 &&
		memInfo.Cached == 0

	switch {
	case memInfo.HasAvailable() && (memInfo.Total == 0 || memInfo.Available <= memInfo.Total):
		return memInfo.Available, "available-field"
	case memInfo.Free > 0 || memInfo.Buffers > 0 || memInfo.Cached > 0:
		if availableFromUsed > 0 && missingCacheMetrics {
			const vmTotalMinusUsedGapTolerance uint64 = 4 * 1024 * 1024
			if availableFromUsed > componentAvailable {
				gap := availableFromUsed - componentAvailable
				if componentAvailable == 0 || gap >= vmTotalMinusUsedGapTolerance {
					return availableFromUsed, "derived-total-minus-used"
				}
			}
		}
		if missingCacheMetrics {
			return 0, ""
		}
		if componentsConflict {
			return 0, ""
		}
		return componentAvailable, "derived-free-buffers-cached"
	default:
		if availableFromUsed > 0 && missingCacheMetrics {
			const vmTotalMinusUsedGapTolerance uint64 = 4 * 1024 * 1024
			if availableFromUsed > componentAvailable {
				gap := availableFromUsed - componentAvailable
				if componentAvailable == 0 || gap >= vmTotalMinusUsedGapTolerance {
					return availableFromUsed, "derived-total-minus-used"
				}
			}
		}
		return 0, ""
	}
}

func saturatingAddUint64(lhs, rhs uint64) uint64 {
	if math.MaxUint64-lhs < rhs {
		return math.MaxUint64
	}
	return lhs + rhs
}

func guestStatusFreeMem(status *proxmox.VMStatus) uint64 {
	if status == nil {
		return 0
	}
	if status.FreeMem > 0 {
		return status.FreeMem
	}
	if status.BalloonInfo != nil {
		return status.BalloonInfo.FreeMem
	}
	return 0
}

func effectiveGuestFreeMemTotal(memTotal uint64, status *proxmox.VMStatus) uint64 {
	if status == nil {
		return memTotal
	}

	freeMem := guestStatusFreeMem(status)
	if status.BalloonInfo != nil {
		if status.BalloonInfo.TotalMem > 0 && freeMem <= status.BalloonInfo.TotalMem {
			if memTotal > 0 && status.BalloonInfo.TotalMem > memTotal {
				return memTotal
			}
			return status.BalloonInfo.TotalMem
		}
		if status.BalloonInfo.Actual > 0 && freeMem <= status.BalloonInfo.Actual {
			if memTotal > 0 && status.BalloonInfo.Actual > memTotal {
				return memTotal
			}
			return status.BalloonInfo.Actual
		}
	}

	if status.Balloon > 0 && status.Balloon <= memTotal && freeMem <= status.Balloon {
		return status.Balloon
	}
	return memTotal
}

func selectGuestLowTrustUsedMemory(memTotal uint64, status *proxmox.VMStatus) (uint64, string) {
	if status == nil {
		return 0, ""
	}

	freeMemTotal := effectiveGuestFreeMemTotal(memTotal, status)
	freeMem := guestStatusFreeMem(status)
	hasFreeFallback := freeMem > 0 && freeMemTotal >= freeMem
	freeDerivedUsed := uint64(0)
	if hasFreeFallback {
		freeDerivedUsed = freeMemTotal - freeMem
	}

	if status.Mem > 0 {
		if hasFreeFallback && freeDerivedUsed < status.Mem {
			statusMemPlusFree := saturatingAddUint64(status.Mem, freeMem)
			if status.Mem >= freeMemTotal && freeDerivedUsed < freeMemTotal {
				return freeDerivedUsed, "status-freemem"
			}
			if statusMemPlusFree > freeMemTotal+guestStatusMemoryMismatchTolerance {
				return freeDerivedUsed, "status-freemem"
			}
		}
		return status.Mem, "status-mem"
	}

	if hasFreeFallback {
		return freeDerivedUsed, "status-freemem"
	}

	return 0, ""
}

func shouldPreferGuestAgentMemAvailable(status *proxmox.VMStatus, memTotal uint64) bool {
	if status == nil || status.Lock != "" || !status.Agent.IsAvailable() || status.Mem == 0 {
		return false
	}
	if guestStatusFreeMem(status) > 0 {
		return false
	}

	effectiveTotal := memTotal
	if status.MaxMem > 0 {
		effectiveTotal = status.MaxMem
	}
	if effectiveTotal == 0 {
		return false
	}
	if status.Mem >= effectiveTotal {
		return true
	}

	return effectiveTotal-status.Mem <= guestStatusMemoryMismatchTolerance
}

func guestMemoryFallbackReason(source string) string {
	return MemorySourceFallbackReason(source)
}

// The final result is admission deferral, not a failed or successful sample.
func (m *Monitor) tryGuestAgentMemAvailable(
	ctx context.Context,
	client PVEClientInterface,
	instanceName string,
	guestName string,
	node string,
	vmid int,
	memTotal uint64,
	guestRaw *VMMemoryRaw,
) (uint64, uint64, string, bool, bool) {
	availability, agentErr := m.getVMAgentMemoryAvailability(ctx, client, instanceName, node, vmid)
	if agentErr != nil || availability.Source == "" {
		return 0, 0, "", false, errors.Is(agentErr, proxmox.ErrGuestAgentDeferred)
	}
	// MemAvailable belongs to this guest's MemTotal, not the configured
	// maximum. Ballooning and kernel-reserved pages can make them different.
	// Older availability-only clients retain their supplied capacity fallback.
	configuredTotal := memTotal
	if availability.Total > 0 {
		memTotal = availability.Total
	}
	agentAvailable := availability.EffectiveAvailable
	if guestRaw != nil {
		guestRaw.GuestAgentMemTotal = availability.Total
		guestRaw.GuestAgentMemAvailable = agentAvailable
		guestRaw.GuestAgentMemFree = availability.Free
		guestRaw.GuestAgentMemBuffers = availability.Buffers
		guestRaw.GuestAgentMemCached = availability.Cached
		guestRaw.GuestAgentSReclaimable = availability.SReclaimable
		guestRaw.GuestAgentShmem = availability.Shmem
		guestRaw.GuestAgentDerived = availability.Source == "meminfo-derived"
	}
	if memTotal == 0 || memTotal > math.MaxInt64 || agentAvailable > memTotal ||
		(configuredTotal > 0 && memTotal > configuredTotal) {
		return 0, 0, "", false, false
	}
	source := "guest-agent-meminfo"
	if availability.Source == "meminfo-derived" {
		source = "guest-agent-meminfo-derived"
	}
	log.Debug().
		Str("vm", guestName).
		Str("node", node).
		Int("vmid", vmid).
		Uint64("total", memTotal).
		Uint64("available", agentAvailable).
		Str("source", source).
		Msg("QEMU memory: using guest agent /proc/meminfo fallback (excludes reclaimable cache)")
	return memTotal, agentAvailable, source, true, false
}

// Return shared guest-command deferral even when independent memory is usable.
func (m *Monitor) resolveGuestStatusMemory(
	ctx context.Context,
	client PVEClientInterface,
	instanceName string,
	guestName string,
	node string,
	vmid int,
	guestID string,
	status *proxmox.VMStatus,
	vmIDToHostAgent map[string]models.Host,
	memTotal uint64,
	memorySource string,
	guestRaw *VMMemoryRaw,
) (uint64, uint64, string, bool) {
	if status == nil {
		return memTotal, 0, memorySource, false
	}

	if guestRaw != nil {
		guestRaw.StatusMaxMem = status.MaxMem
		guestRaw.StatusMem = status.Mem
		guestRaw.StatusFreeMem = guestStatusFreeMem(status)
		guestRaw.Balloon = status.Balloon
		guestRaw.BalloonMin = status.BalloonMin
		guestRaw.Agent = status.Agent.Value
		if status.BalloonInfo != nil {
			guestRaw.BalloonInfoFreeMem = status.BalloonInfo.FreeMem
			guestRaw.BalloonInfoTotalMem = status.BalloonInfo.TotalMem
			guestRaw.BalloonInfoActual = status.BalloonInfo.Actual
		}
	}
	if status.MaxMem > 0 {
		memTotal = status.MaxMem
	}

	memAvailable := uint64(0)
	hasMemAvailable := false
	derivedTotalMinusUsedAvailable := uint64(0)
	derivedTotalMinusUsedTotal := uint64(0)
	selectedUsed := uint64(0)
	hasSelectedUsed := false
	if status.MemInfo != nil {
		memInfoTotal := memTotal
		if status.MemInfo.Total > 0 {
			memInfoTotal = status.MemInfo.Total
		}
		memAvailable, memorySource = deriveGuestMemInfoAvailable(status.MemInfo, guestRaw)
		hasMemAvailable = memorySource != ""
		if memInfoTotal == 0 || memInfoTotal > math.MaxInt64 || memAvailable > memInfoTotal ||
			(memTotal > 0 && memInfoTotal > memTotal) {
			memAvailable = 0
			hasMemAvailable = false
			memorySource = ""
		}
		if memorySource == "derived-total-minus-used" {
			derivedTotalMinusUsedAvailable = memAvailable
			derivedTotalMinusUsedTotal = memInfoTotal
			memAvailable = 0
			hasMemAvailable = false
			memorySource = ""
		} else if hasMemAvailable {
			memTotal = memInfoTotal
		}
	}

	triedGuestAgentMemAvailable := false
	guestReadDeferred := false
	if !hasMemAvailable && shouldPreferGuestAgentMemAvailable(status, memTotal) {
		triedGuestAgentMemAvailable = true
		agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
		guestReadDeferred = guestReadDeferred || deferred
		if ok {
			memTotal = agentTotal
			memAvailable = agentAvailable
			hasMemAvailable = true
			memorySource = agentSource
		}
	}

	// No RRD fallback here: PVE guest rrddata carries only cache-inclusive
	// mem/maxmem columns — the cache-aware memused/memavailable columns exist
	// only in node RRD, so a guest RRD lookup can never yield memory evidence
	// (#1634).

	if !hasMemAvailable && !hasSelectedUsed && status.Lock == "" && status.Agent.IsAvailable() && !triedGuestAgentMemAvailable {
		agentTotal, agentAvailable, agentSource, ok, deferred := m.tryGuestAgentMemAvailable(ctx, client, instanceName, guestName, node, vmid, memTotal, guestRaw)
		guestReadDeferred = guestReadDeferred || deferred
		if ok {
			memTotal = agentTotal
			memAvailable = agentAvailable
			hasMemAvailable = true
			memorySource = agentSource
		}
	}

	if !hasMemAvailable && !hasSelectedUsed && derivedTotalMinusUsedAvailable > 0 {
		memTotal = derivedTotalMinusUsedTotal
		memAvailable = derivedTotalMinusUsedAvailable
		hasMemAvailable = true
		memorySource = "derived-total-minus-used"
		log.Debug().
			Str("vm", guestName).
			Str("node", node).
			Int("vmid", vmid).
			Uint64("total", memTotal).
			Uint64("available", memAvailable).
			Uint64("availableFromUsed", derivedTotalMinusUsedAvailable).
			Msg("QEMU memory: deriving guest available from total-used gap after preferred fallbacks")
	}

	if !hasMemAvailable && !hasSelectedUsed {
		if agentHost, ok := vmIDToHostAgent[guestID]; ok &&
			agentHost.Memory.HasKnownUsage() &&
			agentHost.Memory.Total > 0 &&
			(memTotal == 0 || uint64(agentHost.Memory.Total) <= memTotal) {
			memTotal = uint64(agentHost.Memory.Total)
			agentAvailable := uint64(agentHost.Memory.Total - agentHost.Memory.Used)
			memAvailable = agentAvailable
			hasMemAvailable = true
			selectedUsed = uint64(agentHost.Memory.Used)
			hasSelectedUsed = true
			memorySource = "agent"
			if guestRaw != nil {
				guestRaw.HostAgentTotal = uint64(agentHost.Memory.Total)
				guestRaw.HostAgentUsed = uint64(agentHost.Memory.Used)
			}
			log.Debug().
				Str("vm", guestName).
				Str("node", node).
				Int("vmid", vmid).
				Uint64("total", memTotal).
				Uint64("available", memAvailable).
				Int64("agentTotal", agentHost.Memory.Total).
				Int64("agentUsed", agentHost.Memory.Used).
				Msg("QEMU memory: using linked Pulse host agent memory (excludes page cache)")
		}
	}

	memUsed := uint64(0)
	switch {
	case hasSelectedUsed:
		memUsed = selectedUsed
	case hasMemAvailable:
		memUsed = memTotal - memAvailable
	default:
		if status.MemInfo != nil {
			memorySource = "unavailable"
		} else {
			memUsed, memorySource = selectGuestLowTrustUsedMemory(memTotal, status)
			if memorySource == "status-freemem" {
				// That fallback already derives used from the balloon capacity.
				// Keep its denominator with it, without upgrading its cache trust.
				memTotal = effectiveGuestFreeMemTotal(memTotal, status)
			}
			if memorySource == "" {
				memorySource = "unavailable"
			}
		}
	}
	if memUsed > memTotal {
		memUsed = memTotal
	}

	return memTotal, memUsed, memorySource, guestReadDeferred
}

// preferLinkedAgentLXCMemory replaces a low-trust Proxmox LXC memory reading
// with the linked Pulse agent's own sample when one is available. The agent runs
// inside the container, so its sample reflects the workload's real footprint,
// while cluster/resources is a cache-inclusive cgroup fallback that can badly
// under- or over-report shared-memory workloads (#2148, mirroring #1962).
//
// An agent inside a container without lxcfs sees the host's /proc/meminfo, so a
// sample is only trusted when its total matches the container's configured
// limit. Preferred provider sources are never overridden.
func preferLinkedAgentLXCMemory(
	status string,
	memTotal uint64,
	memUsed uint64,
	memorySource string,
	guestRaw *VMMemoryRaw,
	agentHost models.Host,
	hasAgent bool,
) (uint64, uint64, string) {
	if !hasAgent || status != "running" || memTotal == 0 {
		return memTotal, memUsed, memorySource
	}
	switch CanonicalMemorySource(memorySource) {
	case "cluster-resources", "unavailable":
	default:
		return memTotal, memUsed, memorySource
	}
	if !agentHost.Memory.HasKnownUsage() || agentHost.Memory.Total <= 0 || agentHost.Memory.Used < 0 {
		return memTotal, memUsed, memorySource
	}
	agentTotal := uint64(agentHost.Memory.Total)
	agentUsed := uint64(agentHost.Memory.Used)
	if agentTotal != memTotal || agentUsed > memTotal {
		return memTotal, memUsed, memorySource
	}
	if guestRaw != nil {
		guestRaw.HostAgentTotal = agentTotal
		guestRaw.HostAgentUsed = agentUsed
	}
	log.Debug().
		Uint64("total", memTotal).
		Uint64("used", agentUsed).
		Str("previousSource", memorySource).
		Msg("LXC memory: using linked Pulse agent memory over cluster-resources fallback")
	return memTotal, agentUsed, "agent"
}