Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

14.2 KB
Raw Blame
package monitoring

import (
	"context"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
	"github.com/rs/zerolog/log"
)

const (
	// Deep guest detail is deliberately bounded below the whole PVE poll
	// deadline. The authoritative cluster/resources generation can contain
	// hundreds of guests; reserving the tail keeps connection-health
	// publication from inheriting an exhausted context.
	pveGuestEnrichmentMaxDuration = 60 * time.Second
	pvePollTailReserve            = 15 * time.Second
)

func pveGuestEnrichmentContext(ctx context.Context) (context.Context, context.CancelFunc) {
	budget := pveGuestEnrichmentMaxDuration
	if deadline, ok := ctx.Deadline(); ok {
		available := time.Until(deadline) - pvePollTailReserve
		if available <= 0 {
			boundedCtx, cancel := context.WithCancel(ctx)
			cancel()
			return boundedCtx, func() {}
		}
		if available < budget {
			budget = available
		}
	}
	return context.WithTimeout(ctx, budget)
}

// pollVMsAndContainersEfficient uses the cluster/resources endpoint to get all VMs and containers in one call
// This works on both clustered and standalone nodes for efficient polling
// When the instance is part of a cluster, the cluster name is used for guest IDs to prevent duplicates
// when multiple cluster nodes are configured as separate PVE instances.
func (m *Monitor) pollVMsAndContainersEfficient(ctx context.Context, instanceName string, clusterName string, isCluster bool, client PVEClientInterface, nodeEffectiveStatus map[string]string) bool {
	log.Debug().
		Str("instance", instanceName).
		Str("clusterName", clusterName).
		Bool("isCluster", isCluster).
		Msg("Polling VMs and containers using efficient cluster/resources endpoint")

	// Get all resources in a single API call
	cycleStart := time.Now()
	resources, err := client.GetClusterResources(ctx, "vm")
	if err != nil {
		log.Debug().Err(err).Str("instance", instanceName).Msg("cluster/resources not available, falling back to traditional polling")
		return false
	}
	stagePVEGuestEffect(ctx, func() { m.updatePVEBackupTemplateSubjectsFromClusterResources(instanceName, resources) })
	enrichmentCtx, cancelEnrichment := pveGuestEnrichmentContext(ctx)
	defer cancelEnrichment()

	// Capture previous guest state once per poll cycle so fallback and grace-period
	// behavior is based on a consistent pre-poll snapshot.
	prevGuests := m.previousGuestContextForInstance(instanceName)

	allVMs, allContainers := m.collectGuestsFromClusterResources(
		enrichmentCtx,
		instanceName,
		resources,
		client,
		prevGuests.containerOCIByVMID,
		prevGuests.containersByID,
		prevGuests.vmsByID,
		prevGuests.hostAgentsByVMID,
	)

	allVMs, allContainers = m.preserveGuestsForGracePeriod(instanceName, resources, prevGuests.vms, prevGuests.containers, nodeEffectiveStatus, allVMs, allContainers)

	// Check Docker presence for containers that need it (new, restarted, started)
	allContainers = m.CheckContainersForDocker(enrichmentCtx, allContainers)
	m.CollectProxmoxGuestDockerInventory(enrichmentCtx, allContainers)

	// Publish the complete guest generation only after both VM and container
	// collection/enrichment has finished. Empty authoritative results still
	// remove genuinely deleted guests.
	stagePVEGuestEffect(ctx, func() { m.recordGuestMetrics(allVMs, allContainers, cycleStart) })
	m.publishPVEGuests(ctx, instanceName, allVMs, allContainers)

	log.Debug().
		Str("instance", instanceName).
		Int("vms", len(allVMs)).
		Int("containers", len(allContainers)).
		Msg("VMs and containers polled efficiently with cluster/resources")

	return true
}

func (m *Monitor) collectGuestsFromClusterResources(
	ctx context.Context,
	instanceName string,
	resources []proxmox.ClusterResource,
	client PVEClientInterface,
	prevContainerIsOCI map[int]bool,
	prevContainerByID map[string]models.Container,
	prevVMByID map[string]models.VM,
	vmIDToHostAgent map[string]models.Host,
) ([]models.VM, []models.Container) {
	allVMs := make([]models.VM, 0, len(resources))
	allContainers := make([]models.Container, 0, len(resources))
	vmResources := make([]indexedClusterResource, 0, len(resources))
	containerResources := make([]indexedClusterResource, 0, len(resources))

	for _, res := range resources {
		// Generate canonical guest ID: instance:node:vmid
		guestID := makeGuestID(instanceName, res.Node, res.VMID)

		// Debug log the resource type
		log.Debug().
			Str("instance", instanceName).
			Str("name", res.Name).
			Int("vmid", res.VMID).
			Str("type", res.Type).
			Msg("Processing cluster resource")

		switch res.Type {
		case "qemu":
			vmResources = append(vmResources, indexedClusterResource{
				order:    len(vmResources),
				resource: res,
				guestID:  guestID,
			})
		case "lxc":
			containerResources = append(containerResources, indexedClusterResource{
				order:    len(containerResources),
				resource: res,
				guestID:  guestID,
			})
		}
	}

	// Let VM and LXC detail compete fairly for the shared enrichment slots.
	// Running one type to completion first can consume the entire bounded budget
	// on a large mixed cluster and permanently starve the other type.
	var collectionWG sync.WaitGroup
	if len(containerResources) > 0 {
		collectionWG.Add(1)
		go func() {
			defer collectionWG.Done()
			allContainers = m.collectClusterContainerResources(ctx, instanceName, containerResources, client, prevContainerIsOCI, prevContainerByID, vmIDToHostAgent)
		}()
	}
	if len(vmResources) > 0 {
		collectionWG.Add(1)
		go func() {
			defer collectionWG.Done()
			allVMs = m.collectClusterVMResources(ctx, instanceName, vmResources, client, prevVMByID, vmIDToHostAgent)
		}()
	}
	collectionWG.Wait()

	return allVMs, allContainers
}

type indexedClusterResource struct {
	order    int
	resource proxmox.ClusterResource
	guestID  string
}

type clusterVMResourceResult struct {
	order int
	vm    models.VM
	ok    bool
}

func rotatingResourceOffset(total int) int {
	if total <= 1 {
		return 0
	}
	return int(uint64(time.Now().UnixNano()) % uint64(total))
}

func preserveVMOptionalEnrichment(current models.VM, previous *models.VM) models.VM {
	if previous == nil {
		return current
	}
	if len(current.IPAddresses) == 0 {
		current.IPAddresses = cloneStringSlice(previous.IPAddresses)
	}
	if len(current.NetworkInterfaces) == 0 {
		current.NetworkInterfaces = cloneGuestNetworkInterfaces(previous.NetworkInterfaces)
	}
	if current.OSName == "" {
		current.OSName = previous.OSName
	}
	if current.OSVersion == "" {
		current.OSVersion = previous.OSVersion
	}
	if current.AgentVersion == "" {
		current.AgentVersion = previous.AgentVersion
	}
	if len(current.Disks) == 0 {
		current.Disks = cloneGuestDisks(previous.Disks)
	}
	if current.OnBoot == nil && previous.OnBoot != nil {
		onBoot := *previous.OnBoot
		current.OnBoot = &onBoot
	}
	if current.Lock == "" {
		current.Lock = previous.Lock
	}
	return current
}

func preserveContainerOptionalEnrichment(current models.Container, previous *models.Container) models.Container {
	if previous == nil {
		return current
	}
	if len(current.IPAddresses) == 0 {
		current.IPAddresses = cloneStringSlice(previous.IPAddresses)
	}
	if len(current.NetworkInterfaces) == 0 {
		current.NetworkInterfaces = cloneGuestNetworkInterfaces(previous.NetworkInterfaces)
	}
	if current.OSName == "" {
		current.OSName = previous.OSName
	}
	if current.OSTemplate == "" {
		current.OSTemplate = previous.OSTemplate
	}
	if len(current.Disks) == 0 {
		current.Disks = cloneGuestDisks(previous.Disks)
	}
	if current.OnBoot == nil && previous.OnBoot != nil {
		onBoot := *previous.OnBoot
		current.OnBoot = &onBoot
	}
	if current.DockerCheckedAt.IsZero() {
		current.HasDocker = previous.HasDocker
		current.DockerCheckedAt = previous.DockerCheckedAt
	}
	if current.Lock == "" {
		current.Lock = previous.Lock
	}
	return current
}

func (m *Monitor) collectClusterContainerResources(
	ctx context.Context,
	instanceName string,
	resources []indexedClusterResource,
	client PVEClientInterface,
	prevContainerIsOCI map[int]bool,
	prevContainerByID map[string]models.Container,
	vmIDToHostAgent map[string]models.Host,
) []models.Container {
	orderedContainers := make([]models.Container, len(resources))
	orderedOK := make([]bool, len(resources))
	offset := rotatingResourceOffset(len(resources))

	for i := range resources {
		entry := resources[(offset+i)%len(resources)]
		var container models.Container
		var ok bool
		ran := m.runGuestAgentVMWork(ctx, func(workCtx context.Context) {
			container, ok = m.handleClusterContainerResource(workCtx, instanceName, entry.resource, entry.guestID, client, prevContainerIsOCI, vmIDToHostAgent)
		})
		if !ran {
			// The enrichment budget is exhausted, but cluster/resources is still
			// authoritative inventory. A canceled context makes remote detail
			// calls fail immediately while the builder retains the base row.
			container, ok = m.handleClusterContainerResource(ctx, instanceName, entry.resource, entry.guestID, client, prevContainerIsOCI, vmIDToHostAgent)
		}
		if ctx.Err() != nil {
			previous := prevContainerByID[entry.guestID]
			container = preserveContainerOptionalEnrichment(container, &previous)
		}
		if ok {
			orderedContainers[entry.order] = container
			orderedOK[entry.order] = true
		}
	}

	containers := make([]models.Container, 0, len(resources))
	for i, ok := range orderedOK {
		if ok {
			containers = append(containers, orderedContainers[i])
		}
	}
	return containers
}

func (m *Monitor) collectClusterVMResources(
	ctx context.Context,
	instanceName string,
	resources []indexedClusterResource,
	client PVEClientInterface,
	prevVMByID map[string]models.VM,
	vmIDToHostAgent map[string]models.Host,
) []models.VM {
	resultCh := make(chan clusterVMResourceResult, len(resources))
	jobCh := make(chan indexedClusterResource, len(resources))
	var wg sync.WaitGroup

	workerCount := m.efficientQEMUWorkerCount(len(resources))
	for i := 0; i < workerCount; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for entry := range jobCh {
				var result clusterVMResourceResult
				ran := m.runGuestAgentVMWork(ctx, func(workCtx context.Context) {
					var prevVM *models.VM
					if prev, ok := prevVMByID[entry.guestID]; ok {
						prevVM = &prev
					}
					vm, ok := m.handleClusterVMResource(workCtx, instanceName, entry.resource, entry.guestID, client, prevVM, vmIDToHostAgent)
					result = clusterVMResourceResult{
						order: entry.order,
						vm:    vm,
						ok:    ok,
					}
				})
				if !ran {
					// Do not turn an exhausted detail budget into missing inventory.
					// Re-run only the builder with the canceled context so it emits the
					// current cluster/resources row without remote enrichment.
					var prevVM *models.VM
					if prev, ok := prevVMByID[entry.guestID]; ok {
						prevVM = &prev
					}
					vm, ok := m.handleClusterVMResource(ctx, instanceName, entry.resource, entry.guestID, client, prevVM, vmIDToHostAgent)
					result = clusterVMResourceResult{order: entry.order, vm: vm, ok: ok}
				}
				if ctx.Err() != nil && result.ok {
					previous, exists := prevVMByID[entry.guestID]
					if exists {
						result.vm = preserveVMOptionalEnrichment(result.vm, &previous)
					}
				}
				resultCh <- result
			}
		}()
	}

	offset := rotatingResourceOffset(len(resources))
	for i := range resources {
		jobCh <- resources[(offset+i)%len(resources)]
	}
	close(jobCh)

	go func() {
		wg.Wait()
		close(resultCh)
	}()

	orderedVMs := make([]models.VM, len(resources))
	orderedOK := make([]bool, len(resources))
	for result := range resultCh {
		if result.order < 0 || result.order >= len(resources) || !result.ok {
			continue
		}
		orderedVMs[result.order] = result.vm
		orderedOK[result.order] = true
	}

	vms := make([]models.VM, 0, len(resources))
	for i, ok := range orderedOK {
		if ok {
			vms = append(vms, orderedVMs[i])
		}
	}
	return vms
}

func (m *Monitor) handleClusterVMResource(
	ctx context.Context,
	instanceName string,
	res proxmox.ClusterResource,
	guestID string,
	client PVEClientInterface,
	prevVM *models.VM,
	vmIDToHostAgent map[string]models.Host,
) (models.VM, bool) {
	vm, guestRaw, memorySource, snapshotNotes, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent, prevVM)
	if !ok {
		return models.VM{}, false
	}

	stagePVEGuestEffect(ctx, func() {
		m.recordGuestSnapshot(instanceName, vm.Type, res.Node, res.VMID, GuestMemorySnapshot{
			Name:           vm.Name,
			Status:         vm.Status,
			RetrievedAt:    sampleTime,
			MemorySource:   memorySource,
			FallbackReason: guestMemoryFallbackReason(memorySource),
			Memory:         vm.Memory,
			Raw:            guestRaw,
			Notes:          snapshotNotes,
		})

		m.checkGuestAlertsForVM(instanceName, vm)
	})

	return vm, true
}

func (m *Monitor) handleClusterContainerResource(
	ctx context.Context,
	instanceName string,
	res proxmox.ClusterResource,
	guestID string,
	client PVEClientInterface,
	prevContainerIsOCI map[int]bool,
	vmIDToHostAgent map[string]models.Host,
) (models.Container, bool) {
	container, guestRaw, memorySource, sampleTime, ok := m.buildContainerFromClusterResource(ctx, instanceName, res, client, prevContainerIsOCI, vmIDToHostAgent)
	if !ok {
		return models.Container{}, false
	}

	stagePVEGuestEffect(ctx, func() {
		m.recordGuestSnapshot(instanceName, container.Type, res.Node, res.VMID, GuestMemorySnapshot{
			Name:           container.Name,
			Status:         container.Status,
			RetrievedAt:    sampleTime,
			MemorySource:   memorySource,
			FallbackReason: guestMemoryFallbackReason(memorySource),
			Memory:         container.Memory,
			Raw:            guestRaw,
		})

		m.alertManager.CheckGuest(container, instanceName)
	})

	return container, true
}

func (m *Monitor) checkGuestAlertsForVM(instanceName string, vm models.VM) {
	// For non-running VMs, zero out resource usage metrics to prevent false alerts.
	// Proxmox may report stale or residual metrics for stopped VMs.
	alertVM := vm
	if alertVM.Status != "running" {
		log.Debug().
			Str("vm", alertVM.Name).
			Str("status", alertVM.Status).
			Float64("originalCpu", alertVM.CPU).
			Float64("originalMemUsage", alertVM.Memory.Usage).
			Msg("Non-running VM detected - zeroing metrics")

		// Zero out all usage metrics for stopped/paused/suspended VMs
		alertVM.CPU = 0
		alertVM.Memory.Usage = 0
		alertVM.Disk.Usage = 0
		alertVM.NetworkIn = 0
		alertVM.NetworkOut = 0
		alertVM.DiskRead = 0
		alertVM.DiskWrite = 0
	}

	m.alertManager.CheckGuest(alertVM, instanceName)
}