Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

12.7 KB
Raw Blame
package monitoring

import (
	"fmt"
	"sort"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
)

// NodeMemoryRaw captures the raw memory fields returned by Proxmox for a node.
type NodeMemoryRaw struct {
	Total               uint64 `json:"total"`
	Used                uint64 `json:"used"`
	Free                uint64 `json:"free"`
	Available           uint64 `json:"available"`
	Avail               uint64 `json:"avail"`
	Buffers             uint64 `json:"buffers"`
	Cached              uint64 `json:"cached"`
	Shared              uint64 `json:"shared"`
	EffectiveAvailable  uint64 `json:"effectiveAvailable"`
	RRDAvailable        uint64 `json:"rrdAvailable,omitempty"`
	RRDUsed             uint64 `json:"rrdUsed,omitempty"`
	RRDTotal            uint64 `json:"rrdTotal,omitempty"`
	TotalMinusUsed      uint64 `json:"totalMinusUsed,omitempty"`
	FallbackTotal       uint64 `json:"fallbackTotal,omitempty"`
	FallbackUsed        uint64 `json:"fallbackUsed,omitempty"`
	FallbackFree        uint64 `json:"fallbackFree,omitempty"`
	FallbackCalculated  bool   `json:"fallbackCalculated,omitempty"`
	ProxmoxMemorySource string `json:"proxmoxMemorySource,omitempty"`
}

// NodeMemorySnapshot records the final node memory calculation alongside the raw data.
type NodeMemorySnapshot struct {
	Instance       string        `json:"instance"`
	Node           string        `json:"node"`
	RetrievedAt    time.Time     `json:"retrievedAt"`
	MemorySource   string        `json:"memorySource"`
	FallbackReason string        `json:"fallbackReason,omitempty"`
	Memory         models.Memory `json:"memory"`
	Raw            NodeMemoryRaw `json:"raw"`
}

// VMMemoryRaw captures both the listing and detailed status memory fields for a VM/CT.
type VMMemoryRaw struct {
	ListingMem             uint64 `json:"listingMem"`
	ListingMaxMem          uint64 `json:"listingMaxmem"`
	StatusMem              uint64 `json:"statusMem,omitempty"`
	StatusFreeMem          uint64 `json:"statusFreemem,omitempty"`
	StatusMaxMem           uint64 `json:"statusMaxmem,omitempty"`
	Balloon                uint64 `json:"balloon,omitempty"`
	BalloonMin             uint64 `json:"balloonMin,omitempty"`
	BalloonInfoFreeMem     uint64 `json:"balloonInfoFreeMem,omitempty"`
	BalloonInfoTotalMem    uint64 `json:"balloonInfoTotalMem,omitempty"`
	BalloonInfoActual      uint64 `json:"balloonInfoActual,omitempty"`
	MemInfoUsed            uint64 `json:"meminfoUsed,omitempty"`
	MemInfoFree            uint64 `json:"meminfoFree,omitempty"`
	MemInfoTotal           uint64 `json:"meminfoTotal,omitempty"`
	MemInfoAvailable       uint64 `json:"meminfoAvailable,omitempty"`
	MemInfoBuffers         uint64 `json:"meminfoBuffers,omitempty"`
	MemInfoCached          uint64 `json:"meminfoCached,omitempty"`
	MemInfoShared          uint64 `json:"meminfoShared,omitempty"`
	MemInfoTotalMinusUsed  uint64 `json:"meminfoTotalMinusUsed,omitempty"`
	GuestAgentMemTotal     uint64 `json:"guestAgentMemTotal,omitempty"`
	GuestAgentMemAvailable uint64 `json:"guestAgentMemAvailable,omitempty"`
	GuestAgentMemFree      uint64 `json:"guestAgentMemFree,omitempty"`
	GuestAgentMemBuffers   uint64 `json:"guestAgentMemBuffers,omitempty"`
	GuestAgentMemCached    uint64 `json:"guestAgentMemCached,omitempty"`
	GuestAgentSReclaimable uint64 `json:"guestAgentSReclaimable,omitempty"`
	GuestAgentShmem        uint64 `json:"guestAgentShmem,omitempty"`
	GuestAgentDerived      bool   `json:"guestAgentDerived,omitempty"`
	HostAgentTotal         uint64 `json:"hostAgentTotal,omitempty"`
	HostAgentUsed          uint64 `json:"hostAgentUsed,omitempty"`
	Agent                  int    `json:"agent,omitempty"`
	DerivedFromBall        bool   `json:"derivedFromBalloon,omitempty"`
}

// GuestMemorySnapshot records the memory calculation for a guest (VM/LXC).
type GuestMemorySnapshot struct {
	GuestType      string        `json:"guestType"`
	Instance       string        `json:"instance"`
	Node           string        `json:"node"`
	Name           string        `json:"name"`
	VMID           int           `json:"vmid"`
	Status         string        `json:"status"`
	RetrievedAt    time.Time     `json:"retrievedAt"`
	MemorySource   string        `json:"memorySource"`
	FallbackReason string        `json:"fallbackReason,omitempty"`
	Memory         models.Memory `json:"memory"`
	Raw            VMMemoryRaw   `json:"raw"`
	Notes          []string      `json:"notes"`
}

// DiagnosticSnapshotSet aggregates the latest node and guest snapshots.
type DiagnosticSnapshotSet struct {
	Nodes  []NodeMemorySnapshot  `json:"nodes"`
	Guests []GuestMemorySnapshot `json:"guests"`
}

func makeNodeSnapshotKey(instance, node string) string {
	return fmt.Sprintf("%s|%s", instance, node)
}

func makeGuestSnapshotKey(instance, guestType, node string, vmid int) string {
	return fmt.Sprintf("%s|%s|%s|%d", instance, guestType, node, vmid)
}

func normalizeNodeMemorySnapshot(snapshot NodeMemorySnapshot) NodeMemorySnapshot {
	snapshot.MemorySource = CanonicalMemorySource(snapshot.MemorySource)
	if snapshot.FallbackReason == "" {
		snapshot.FallbackReason = MemorySourceFallbackReason(snapshot.MemorySource)
	}
	return snapshot
}

func normalizeGuestMemorySnapshot(snapshot GuestMemorySnapshot) GuestMemorySnapshot {
	snapshot.MemorySource = CanonicalMemorySource(snapshot.MemorySource)
	if snapshot.FallbackReason == "" {
		snapshot.FallbackReason = MemorySourceFallbackReason(snapshot.MemorySource)
	}
	if snapshot.Notes == nil {
		snapshot.Notes = []string{}
	}
	return snapshot
}

func (m *Monitor) logNodeMemorySource(instance, node string, snapshot NodeMemorySnapshot) {
	if m == nil {
		return
	}

	key := makeNodeSnapshotKey(instance, node)

	source := snapshot.MemorySource
	var prevSource string
	m.diagMu.RLock()
	if existing, ok := m.nodeSnapshots[key]; ok {
		prevSource = existing.MemorySource
	}
	m.diagMu.RUnlock()

	if prevSource == snapshot.MemorySource {
		return
	}

	var evt *zerolog.Event
	if MemorySourceIsFallback(source) {
		evt = log.Warn()
	} else {
		evt = log.Debug()
	}

	evt = evt.
		Str("instance", instance).
		Str("node", node).
		Str("memorySource", source).
		Str("proxmoxSource", snapshot.Raw.ProxmoxMemorySource)

	if snapshot.FallbackReason != "" {
		evt = evt.Str("fallbackReason", snapshot.FallbackReason)
	}
	if snapshot.Raw.Available > 0 {
		evt = evt.Uint64("rawAvailable", snapshot.Raw.Available)
	}
	if snapshot.Raw.Buffers > 0 {
		evt = evt.Uint64("rawBuffers", snapshot.Raw.Buffers)
	}
	if snapshot.Raw.Cached > 0 {
		evt = evt.Uint64("rawCached", snapshot.Raw.Cached)
	}
	if snapshot.Raw.TotalMinusUsed > 0 {
		evt = evt.Uint64("rawTotalMinusUsed", snapshot.Raw.TotalMinusUsed)
	}
	if snapshot.Raw.RRDAvailable > 0 {
		evt = evt.Uint64("rrdAvailable", snapshot.Raw.RRDAvailable)
	}
	if snapshot.Raw.RRDUsed > 0 {
		evt = evt.Uint64("rrdUsed", snapshot.Raw.RRDUsed)
	}
	if snapshot.Raw.RRDTotal > 0 {
		evt = evt.Uint64("rrdTotal", snapshot.Raw.RRDTotal)
	}
	if snapshot.Memory.Total > 0 {
		evt = evt.Int64("total", snapshot.Memory.Total)
	}
	if snapshot.Memory.Used > 0 {
		evt = evt.Int64("used", snapshot.Memory.Used)
	}
	if snapshot.Memory.Free > 0 {
		evt = evt.Int64("free", snapshot.Memory.Free)
	}
	if snapshot.Memory.Usage > 0 {
		evt = evt.Float64("usage", snapshot.Memory.Usage)
	}

	evt.Msg("node memory source updated")
}

func (m *Monitor) logGuestMemorySource(instance, guestType, node string, vmid int, snapshot GuestMemorySnapshot) {
	if m == nil {
		return
	}

	key := makeGuestSnapshotKey(instance, guestType, node, vmid)

	var prevSource string
	m.diagMu.RLock()
	if existing, ok := m.guestSnapshots[key]; ok {
		prevSource = existing.MemorySource
	}
	m.diagMu.RUnlock()

	if prevSource == snapshot.MemorySource {
		return
	}

	source := snapshot.MemorySource
	var evt *zerolog.Event
	if MemorySourceIsFallback(source) {
		evt = log.Warn()
	} else {
		evt = log.Debug()
	}

	evt = evt.
		Str("instance", instance).
		Str("guestType", guestType).
		Str("node", node).
		Int("vmid", vmid).
		Str("name", snapshot.Name).
		Str("status", snapshot.Status).
		Str("memorySource", source)

	if snapshot.FallbackReason != "" {
		evt = evt.Str("fallbackReason", snapshot.FallbackReason)
	}
	if snapshot.Raw.ListingMem > 0 {
		evt = evt.Uint64("listingMem", snapshot.Raw.ListingMem)
	}
	if snapshot.Raw.ListingMaxMem > 0 {
		evt = evt.Uint64("listingMaxMem", snapshot.Raw.ListingMaxMem)
	}
	if snapshot.Raw.StatusMem > 0 {
		evt = evt.Uint64("statusMem", snapshot.Raw.StatusMem)
	}
	if snapshot.Raw.StatusFreeMem > 0 {
		evt = evt.Uint64("statusFreeMem", snapshot.Raw.StatusFreeMem)
	}
	if snapshot.Raw.StatusMaxMem > 0 {
		evt = evt.Uint64("statusMaxMem", snapshot.Raw.StatusMaxMem)
	}
	if snapshot.Raw.BalloonInfoFreeMem > 0 {
		evt = evt.Uint64("balloonInfoFreeMem", snapshot.Raw.BalloonInfoFreeMem)
	}
	if snapshot.Raw.BalloonInfoTotalMem > 0 {
		evt = evt.Uint64("balloonInfoTotalMem", snapshot.Raw.BalloonInfoTotalMem)
	}
	if snapshot.Raw.MemInfoAvailable > 0 {
		evt = evt.Uint64("memInfoAvailable", snapshot.Raw.MemInfoAvailable)
	}
	if snapshot.Raw.GuestAgentMemAvailable > 0 {
		evt = evt.Uint64("guestAgentMemAvailable", snapshot.Raw.GuestAgentMemAvailable)
	}
	if snapshot.Raw.GuestAgentDerived {
		evt = evt.Bool("guestAgentDerived", true)
	}
	if snapshot.Raw.MemInfoBuffers > 0 {
		evt = evt.Uint64("memInfoBuffers", snapshot.Raw.MemInfoBuffers)
	}
	if snapshot.Raw.MemInfoCached > 0 {
		evt = evt.Uint64("memInfoCached", snapshot.Raw.MemInfoCached)
	}
	if snapshot.Raw.MemInfoTotalMinusUsed > 0 {
		evt = evt.Uint64("memInfoTotalMinusUsed", snapshot.Raw.MemInfoTotalMinusUsed)
	}
	if snapshot.Raw.HostAgentTotal > 0 {
		evt = evt.Uint64("hostAgentTotal", snapshot.Raw.HostAgentTotal)
	}
	if snapshot.Raw.HostAgentUsed > 0 {
		evt = evt.Uint64("hostAgentUsed", snapshot.Raw.HostAgentUsed)
	}
	if snapshot.Memory.Total > 0 {
		evt = evt.Int64("total", snapshot.Memory.Total)
	}
	if snapshot.Memory.Used > 0 {
		evt = evt.Int64("used", snapshot.Memory.Used)
	}
	if snapshot.Memory.Free > 0 {
		evt = evt.Int64("free", snapshot.Memory.Free)
	}
	if snapshot.Memory.Usage > 0 {
		evt = evt.Float64("usage", snapshot.Memory.Usage)
	}

	evt.Msg("guest memory source updated")
}

func (m *Monitor) recordNodeSnapshot(instance, node string, snapshot NodeMemorySnapshot) {
	if m == nil {
		return
	}

	snapshot = normalizeNodeMemorySnapshot(snapshot)
	snapshot.Instance = instance
	snapshot.Node = node
	if snapshot.RetrievedAt.IsZero() {
		snapshot.RetrievedAt = time.Now()
	}

	m.logNodeMemorySource(instance, node, snapshot)

	m.diagMu.Lock()
	defer m.diagMu.Unlock()
	if m.nodeSnapshots == nil {
		m.nodeSnapshots = make(map[string]NodeMemorySnapshot)
	}
	m.nodeSnapshots[makeNodeSnapshotKey(instance, node)] = snapshot
}

func (m *Monitor) recordGuestSnapshot(instance, guestType, node string, vmid int, snapshot GuestMemorySnapshot) {
	if m == nil {
		return
	}

	snapshot = normalizeGuestMemorySnapshot(snapshot)
	snapshot.Instance = instance
	snapshot.GuestType = guestType
	snapshot.Node = node
	snapshot.VMID = vmid
	if snapshot.RetrievedAt.IsZero() {
		snapshot.RetrievedAt = time.Now()
	}

	m.logGuestMemorySource(instance, guestType, node, vmid, snapshot)

	m.diagMu.Lock()
	defer m.diagMu.Unlock()
	if m.guestSnapshots == nil {
		m.guestSnapshots = make(map[string]GuestMemorySnapshot)
	}
	m.guestSnapshots[makeGuestSnapshotKey(instance, guestType, node, vmid)] = snapshot
}

// GetDiagnosticSnapshots returns copies of the latest node and guest memory snapshots.
func (m *Monitor) GetDiagnosticSnapshots() DiagnosticSnapshotSet {
	result := DiagnosticSnapshotSet{
		Nodes:  []NodeMemorySnapshot{},
		Guests: []GuestMemorySnapshot{},
	}

	if m == nil {
		return result
	}

	m.diagMu.RLock()
	defer m.diagMu.RUnlock()

	if len(m.nodeSnapshots) > 0 {
		result.Nodes = make([]NodeMemorySnapshot, 0, len(m.nodeSnapshots))
		for _, snapshot := range m.nodeSnapshots {
			result.Nodes = append(result.Nodes, normalizeNodeMemorySnapshot(snapshot))
		}
		sort.Slice(result.Nodes, func(i, j int) bool {
			if result.Nodes[i].Instance == result.Nodes[j].Instance {
				return result.Nodes[i].Node < result.Nodes[j].Node
			}
			return result.Nodes[i].Instance < result.Nodes[j].Instance
		})
	}

	if len(m.guestSnapshots) > 0 {
		result.Guests = make([]GuestMemorySnapshot, 0, len(m.guestSnapshots))
		for _, snapshot := range m.guestSnapshots {
			result.Guests = append(result.Guests, normalizeGuestMemorySnapshot(snapshot))
		}
		sort.Slice(result.Guests, func(i, j int) bool {
			if result.Guests[i].Instance == result.Guests[j].Instance {
				if result.Guests[i].Node == result.Guests[j].Node {
					if result.Guests[i].GuestType == result.Guests[j].GuestType {
						return result.Guests[i].VMID < result.Guests[j].VMID
					}
					return result.Guests[i].GuestType < result.Guests[j].GuestType
				}
				return result.Guests[i].Node < result.Guests[j].Node
			}
			return result.Guests[i].Instance < result.Guests[j].Instance
		})
	}

	return result
}