Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

7.5 KB
Raw Blame
package monitoring

import (
	"crypto/sha256"
	"encoding/hex"
	"math"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)

const (
	maxAgentLibvirtDomains         = 128
	maxAgentLibvirtDomainNameBytes = 256
)

// agentLibvirtSampleTime returns when a libvirt inventory's domain counters
// were sampled. The agent stamps CollectedAt on its own clock just before it
// reads the domain stats, nearer that sample than the report's
// end-of-collection Timestamp. As with agentCounterSampleTime, only a
// sequenced report is ordered on that clock; legacy reports and inventories
// without CollectedAt keep receipt time.
func agentLibvirtSampleTime(sequenceID string, inventory *agentshost.LibvirtInventory) counterSampleTime {
	if inventory == nil {
		return counterSampleTime{}
	}
	return agentCounterSampleTime(sequenceID, inventory.CollectedAt)
}

// normalizeAgentLibvirtInventory admits a host report's libvirt inventory.
// observedAt is Pulse receipt time for freshness; sampled times the CPU and
// I/O counter intervals.
func (m *Monitor) normalizeAgentLibvirtInventory(
	report *agentshost.LibvirtInventory,
	hostID string,
	previous *models.Host,
	observedAt time.Time,
	sampled counterSampleTime,
) *models.HostLibvirtInventory {
	if report == nil {
		if previous != nil && previous.Libvirt != nil {
			preserved := *previous.Libvirt
			preserved.Domains = append([]models.HostLibvirtDomain(nil), previous.Libvirt.Domains...)
			return &preserved
		}
		return nil
	}

	previousByID := make(map[string]models.HostLibvirtDomain)
	if previous != nil && previous.Libvirt != nil {
		for _, domain := range previous.Libvirt.Domains {
			previousByID[domain.ID] = domain
		}
	}

	result := &models.HostLibvirtInventory{
		Domains:     make([]models.HostLibvirtDomain, 0, min(len(report.Domains), maxAgentLibvirtDomains)),
		CollectedAt: observedAt.UTC(),
	}
	seen := make(map[string]struct{}, len(report.Domains))
	for _, incoming := range report.Domains {
		if len(result.Domains) == maxAgentLibvirtDomains {
			break
		}
		name := strings.TrimSpace(incoming.Name)
		if !validAgentLibvirtDomainName(name) {
			continue
		}
		dedupeKey := strings.ToLower(name)
		if _, exists := seen[dedupeKey]; exists {
			continue
		}
		seen[dedupeKey] = struct{}{}

		domain := models.HostLibvirtDomain{
			ID:                 agentLibvirtDomainID(name),
			Name:               name,
			State:              normalizeAgentLibvirtState(incoming.State),
			VCPUs:              max(0, min(incoming.VCPUs, 4096)),
			CPUTimeNanoseconds: incoming.CPUTimeNanoseconds,
			MemoryCurrentBytes: max(int64(0), incoming.MemoryCurrentBytes),
			MemoryMaximumBytes: max(int64(0), incoming.MemoryMaximumBytes),
			NetworkRXBytes:     incoming.NetworkRXBytes,
			NetworkTXBytes:     incoming.NetworkTXBytes,
			DiskReadBytes:      incoming.DiskReadBytes,
			DiskWriteBytes:     incoming.DiskWriteBytes,
		}
		if domain.MemoryMaximumBytes > 0 && domain.MemoryCurrentBytes > domain.MemoryMaximumBytes {
			domain.MemoryCurrentBytes = domain.MemoryMaximumBytes
		}

		if m != nil && m.rateTracker != nil {
			seriesID := "libvirt:" + strings.TrimSpace(hostID) + ":" + domain.ID
			// CPU time is sampled with the domain's I/O counters, so it is
			// measured over the same interval on the same clock. Every sample
			// moves its baseline; only a domain present in the previous report
			// whose CPU time did not reset yields a usage.
			cpuRate := m.rateTracker.calculateSampledCounterRate(
				seriesID+":cpu",
				domain.CPUTimeNanoseconds,
				observedAt,
				sampled,
			)
			if prior, ok := previousByID[domain.ID]; ok && domain.CPUTimeNanoseconds >= prior.CPUTimeNanoseconds {
				domain.CPUUsagePercent, domain.CPUUsageValid = libvirtCPUUsagePercent(cpuRate, domain.VCPUs)
			}

			diskRead, diskWrite, _, netIn, netOut := m.rateTracker.calculateSampledRates(
				seriesID,
				IOMetrics{
					DiskRead:   uint64ToInt64(domain.DiskReadBytes),
					DiskWrite:  uint64ToInt64(domain.DiskWriteBytes),
					NetworkIn:  uint64ToInt64(domain.NetworkRXBytes),
					NetworkOut: uint64ToInt64(domain.NetworkTXBytes),
					Timestamp:  observedAt,
				},
				sampled,
			)
			if diskRead >= 0 && diskWrite >= 0 && netIn >= 0 && netOut >= 0 {
				domain.DiskReadRate = diskRead
				domain.DiskWriteRate = diskWrite
				domain.NetworkInRate = netIn
				domain.NetworkOutRate = netOut
				domain.IORatesValid = true
			}
		}

		result.Domains = append(result.Domains, domain)
	}
	sort.Slice(result.Domains, func(i, j int) bool {
		return strings.ToLower(result.Domains[i].Name) < strings.ToLower(result.Domains[j].Name)
	})
	return result
}

func validAgentLibvirtDomainName(name string) bool {
	if name == "" || len(name) > maxAgentLibvirtDomainNameBytes || !utf8.ValidString(name) {
		return false
	}
	for _, r := range name {
		if r < 0x20 || r == 0x7f {
			return false
		}
	}
	return true
}

func normalizeAgentLibvirtState(value string) string {
	switch strings.ToLower(strings.TrimSpace(value)) {
	case "running", "blocked", "paused", "shutting-down", "shutoff", "crashed", "suspended":
		return strings.ToLower(strings.TrimSpace(value))
	default:
		return "unknown"
	}
}

func agentLibvirtDomainID(name string) string {
	sum := sha256.Sum256([]byte(name))
	return "domain-" + hex.EncodeToString(sum[:8])
}

// libvirtCPUUsagePercent converts a domain's CPU-time rate, in nanoseconds per
// second summed across its vCPUs, to a percentage of its vCPU capacity. A
// negative rate means no comparable interval.
func libvirtCPUUsagePercent(cpuNanosecondsPerSecond float64, vcpus int) (float64, bool) {
	if cpuNanosecondsPerSecond < 0 {
		return 0, false
	}
	if vcpus <= 0 {
		vcpus = 1
	}
	percent := (cpuNanosecondsPerSecond / float64(time.Second) / float64(vcpus)) * 100
	if math.IsNaN(percent) || math.IsInf(percent, 0) {
		return 0, false
	}
	return math.Max(0, math.Min(100, percent)), true
}

func uint64ToInt64(value uint64) int64 {
	if value > math.MaxInt64 {
		return math.MaxInt64
	}
	return int64(value)
}

func libvirtDomainMetricID(hostID, domainID string) string {
	return strings.TrimSpace(hostID) + ":libvirt:" + strings.TrimSpace(domainID)
}

func (m *Monitor) writeLibvirtDomainMetrics(host models.Host, observedAt time.Time) {
	if m == nil || host.Libvirt == nil {
		return
	}
	for _, domain := range host.Libvirt.Domains {
		metricID := libvirtDomainMetricID(host.ID, domain.ID)
		if domain.CPUUsageValid {
			if m.metricsHistory != nil {
				m.metricsHistory.AddGuestMetric(metricID, "cpu", domain.CPUUsagePercent, observedAt)
			}
			if m.metricsStore != nil {
				m.metricsStore.Write("vm", metricID, "cpu", domain.CPUUsagePercent, observedAt)
			}
		}
		if domain.MemoryMaximumBytes > 0 {
			memoryPercent := math.Max(0, math.Min(
				100,
				(float64(domain.MemoryCurrentBytes)/float64(domain.MemoryMaximumBytes))*100,
			))
			if m.metricsHistory != nil {
				m.metricsHistory.AddGuestMetric(metricID, "memory", memoryPercent, observedAt)
			}
			if m.metricsStore != nil {
				m.metricsStore.Write("vm", metricID, "memory", memoryPercent, observedAt)
			}
		}
		if !domain.IORatesValid {
			continue
		}
		for metricType, value := range map[string]float64{
			"netin":     domain.NetworkInRate,
			"netout":    domain.NetworkOutRate,
			"diskread":  domain.DiskReadRate,
			"diskwrite": domain.DiskWriteRate,
		} {
			if m.metricsHistory != nil {
				m.metricsHistory.AddGuestMetric(metricID, metricType, value, observedAt)
			}
			if m.metricsStore != nil {
				m.metricsStore.Write("vm", metricID, metricType, value, observedAt)
			}
		}
	}
}