Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

16.0 KB
Raw Blame
package monitoring

import (
	"fmt"
	"math"
	"reflect"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)

const (
	// One vCPU-second per second across four vCPUs: 25% of capacity.
	libvirtClockCPUNanosPerSecond = 1_000_000_000
	libvirtClockVCPUs             = 4
	libvirtClockNetInPerSecond    = 500_000
	libvirtClockNetOutPerSecond   = 125_000
	libvirtClockReadPerSecond     = 2_000_000
	libvirtClockWritePerSecond    = 1_000_000
	// The agent stamps CollectedAt before it reads domain stats and the report
	// Timestamp after every collector, so CollectedAt sits earlier.
	libvirtClockCollectedLead = 7 * time.Second
)

var libvirtClockTrueRates = map[string]float64{
	"cpu":       100.0 / libvirtClockVCPUs,
	"netin":     libvirtClockNetInPerSecond,
	"netout":    libvirtClockNetOutPerSecond,
	"diskread":  libvirtClockReadPerSecond,
	"diskwrite": libvirtClockWritePerSecond,
}

func TestApplyHostReportLibvirtCPUCounterRetainsUnsignedRange(t *testing.T) {
	m := newCounterClockHostMonitor(t)
	agent := newCounterClockAgent(time.Now().Add(-time.Hour))
	for slot := 1; slot <= 3; slot++ {
		domain := agent.libvirtDomain(slot, 0)
		// A busy, long-running domain can accumulate more than MaxInt64
		// nanoseconds across its vCPUs. Clamping both samples loses the delta.
		domain.CPUTimeNanoseconds = uint64(math.MaxInt64) - 15_000_000_000 + uint64(slot)*30_000_000_000
		report := agent.libvirtHostReport("stream-a", uint64(slot), slot, agent.collectedAt(slot), agent.libvirtCollectedAt(slot), domain)
		got := libvirtClockDomain(t, "unsigned CPU counter", applyCounterClockHost(t, m, report))
		if slot > 1 {
			wantLibvirtTrueRates(t, "unsigned CPU counter", got)
		}
	}
}

// libvirtCollectedAt is when the fake agent read slot's domain stats.
func (a counterClockAgent) libvirtCollectedAt(slot int) time.Time {
	return a.collectedAt(slot).Add(-libvirtClockCollectedLead)
}

// libvirtCounter is a domain counter at slot. A domain restarted at
// restartSlot counts again from zero; restartSlot 0 means it never restarted.
func (a counterClockAgent) libvirtCounter(perSecond uint64, slot, restartSlot int) uint64 {
	if restartSlot == 0 {
		return a.counter(perSecond, slot)
	}
	return perSecond * uint64(slot-restartSlot) * uint64(a.interval/time.Second)
}

func (a counterClockAgent) libvirtDomain(slot, restartSlot int) agentshost.LibvirtDomain {
	return agentshost.LibvirtDomain{
		Name:               "app",
		State:              "running",
		VCPUs:              libvirtClockVCPUs,
		CPUTimeNanoseconds: a.libvirtCounter(libvirtClockCPUNanosPerSecond, slot, restartSlot),
		MemoryCurrentBytes: 2 << 30,
		MemoryMaximumBytes: 4 << 30,
		NetworkRXBytes:     a.libvirtCounter(libvirtClockNetInPerSecond, slot, restartSlot),
		NetworkTXBytes:     a.libvirtCounter(libvirtClockNetOutPerSecond, slot, restartSlot),
		DiskReadBytes:      a.libvirtCounter(libvirtClockReadPerSecond, slot, restartSlot),
		DiskWriteBytes:     a.libvirtCounter(libvirtClockWritePerSecond, slot, restartSlot),
	}
}

// libvirtHostReport is hostReport carrying a readable libvirt inventory
// collected at collectedAt (zero: an inventory without CollectedAt), holding
// the "app" domain unless it is omitted.
func (a counterClockAgent) libvirtHostReport(stream string, seq uint64, slot int, authoredAt, collectedAt time.Time, domains ...agentshost.LibvirtDomain) agentshost.Report {
	report := a.hostReport(stream, seq, slot, authoredAt)
	report.Libvirt = &agentshost.LibvirtInventory{
		Domains:     append([]agentshost.LibvirtDomain{}, domains...),
		CollectedAt: collectedAt,
	}
	return report
}

func libvirtClockDomain(t *testing.T, label string, host models.Host) models.HostLibvirtDomain {
	t.Helper()
	if host.Libvirt == nil || len(host.Libvirt.Domains) != 1 {
		t.Fatalf("%s libvirt inventory = %+v, want the app domain", label, host.Libvirt)
	}
	return host.Libvirt.Domains[0]
}

func wantLibvirtTrueRates(t *testing.T, label string, domain models.HostLibvirtDomain) {
	t.Helper()
	if !domain.CPUUsageValid || !domain.IORatesValid {
		t.Fatalf("%s reported no rate (cpu valid %v, io valid %v), want the domain's real rates", label, domain.CPUUsageValid, domain.IORatesValid)
	}
	wantTrueRate(t, label+" CPU usage", domain.CPUUsagePercent, libvirtClockTrueRates["cpu"])
	wantTrueRate(t, label+" NetworkInRate", domain.NetworkInRate, libvirtClockNetInPerSecond)
	wantTrueRate(t, label+" NetworkOutRate", domain.NetworkOutRate, libvirtClockNetOutPerSecond)
	wantTrueRate(t, label+" DiskReadRate", domain.DiskReadRate, libvirtClockReadPerSecond)
	wantTrueRate(t, label+" DiskWriteRate", domain.DiskWriteRate, libvirtClockWritePerSecond)
}

func wantLibvirtUnknown(t *testing.T, label string, domain models.HostLibvirtDomain) {
	t.Helper()
	if domain.CPUUsageValid || domain.IORatesValid {
		t.Fatalf("%s reported rates without a comparable interval: cpu %.1f (valid %v), in %.1f (valid %v)",
			label, domain.CPUUsagePercent, domain.CPUUsageValid, domain.NetworkInRate, domain.IORatesValid)
	}
}

// libvirtClockHistory returns every CPU and I/O history point recorded for
// the app domain, by metric.
func libvirtClockHistory(m *Monitor, hostID string) map[string][]float64 {
	metricID := libvirtDomainMetricID(hostID, agentLibvirtDomainID("app"))
	series := make(map[string][]float64)
	for metric := range libvirtClockTrueRates {
		for _, point := range m.metricsHistory.GetGuestMetrics(metricID, metric, 24*time.Hour) {
			series[metric] = append(series[metric], point.Value)
		}
	}
	return series
}

func wantLibvirtHistoryTrueRates(t *testing.T, m *Monitor, hostID string) {
	t.Helper()
	series := libvirtClockHistory(m, hostID)
	for metric, want := range libvirtClockTrueRates {
		if len(series[metric]) == 0 {
			t.Fatalf("libvirt %s history is empty", metric)
		}
		for i, got := range series[metric] {
			wantTrueRate(t, fmt.Sprintf("libvirt %s history point %d", metric, i), got, want)
		}
	}
}

// A drained FIFO delivers host reports collected 30s apart within
// milliseconds. The libvirt domain's CPU and I/O counters inside each report
// must be measured over the 30s between their CollectedAt stamps, not over the
// receipt gap, and not over the report Timestamp, which the agent stamps after
// a variable-length tail of other collectors.
func TestApplyHostReportDrainedFIFOLibvirtRatesUseCollectedAt(t *testing.T) {
	m := newCounterClockHostMonitor(t)
	agent := newCounterClockAgent(time.Now().Add(-48 * time.Hour))
	authoredAt := func(slot int) time.Time {
		return agent.collectedAt(slot).Add(time.Duration(slot%3) * 4 * time.Second)
	}
	report := func(slot int) agentshost.Report {
		return agent.libvirtHostReport("stream-a", uint64(slot), slot, authoredAt(slot), agent.libvirtCollectedAt(slot), agent.libvirtDomain(slot, 0))
	}

	first := applyCounterClockHost(t, m, report(1))
	wantLibvirtUnknown(t, "first sample", libvirtClockDomain(t, "first sample", first))
	for slot := 2; slot <= 11; slot++ {
		host := applyCounterClockHost(t, m, report(slot))
		wantLibvirtTrueRates(t, fmt.Sprintf("drained slot %d", slot), libvirtClockDomain(t, "drained", host))
	}
	wantLibvirtHistoryTrueRates(t, m, first.ID)

	// Both domain series were received just now; cleanup keyed on their
	// two-day-old sample clock would drop the baselines.
	m.cleanupRateTracker(time.Now())
	next := applyCounterClockHost(t, m, report(12))
	wantLibvirtTrueRates(t, "report after rate-series cleanup", libvirtClockDomain(t, "after cleanup", next))
}

// Each libvirt clock transition either has a comparable CollectedAt interval
// or reports no CPU or I/O rate and writes no rate history for that sample.
func TestApplyHostReportLibvirtClockTransitionsReportUnknownWithoutComparableInterval(t *testing.T) {
	m := newCounterClockHostMonitor(t)
	agent := newCounterClockAgent(time.Now().Add(-6 * time.Hour))
	steppedBack := func(slot int) time.Time { return agent.libvirtCollectedAt(slot).Add(-time.Hour) }

	// A legacy report sets receipt-time baselines even though it carries
	// CollectedAt: without a stream it has no ordered source clock.
	legacy := agent.libvirtHostReport("", 0, 1, agent.collectedAt(1), agent.libvirtCollectedAt(1), agent.libvirtDomain(1, 0))
	hostID := applyCounterClockHost(t, m, legacy).ID

	steps := []struct {
		name        string
		seq         uint64
		slot        int
		authoredAt  time.Time
		collectedAt time.Time
		want        string
	}{
		{"upgrade to a sequenced stream after receipt-time baselines", 1, 2, agent.collectedAt(2), agent.libvirtCollectedAt(2), "unknown"},
		{"second CollectedAt sample", 2, 3, agent.collectedAt(3), agent.libvirtCollectedAt(3), "true"},
		{"report without a Timestamp keeps the inventory's own clock", 3, 4, time.Time{}, agent.libvirtCollectedAt(4), "true"},
		{"inventory without CollectedAt", 4, 5, agent.collectedAt(5), time.Time{}, "unknown"},
		{"CollectedAt returns", 5, 6, agent.collectedAt(6), agent.libvirtCollectedAt(6), "unknown"},
		{"CollectedAt interval again", 6, 7, agent.collectedAt(7), agent.libvirtCollectedAt(7), "true"},
		{"clock steps back an hour", 7, 8, agent.collectedAt(8).Add(-time.Hour), steppedBack(8), "unknown"},
		{"first sample on the stepped clock", 8, 9, agent.collectedAt(9).Add(-time.Hour), steppedBack(9), "unknown"},
		{"interval on the stepped clock", 9, 10, agent.collectedAt(10).Add(-time.Hour), steppedBack(10), "true"},
		{"equal CollectedAt with the next sequence", 10, 11, agent.collectedAt(11).Add(-time.Hour), steppedBack(10), "unknown"},
		{"first sample after the repeated CollectedAt", 11, 12, agent.collectedAt(12).Add(-time.Hour), steppedBack(12), "unknown"},
		{"interval after the repeated CollectedAt", 12, 13, agent.collectedAt(13).Add(-time.Hour), steppedBack(13), "true"},
	}
	for _, step := range steps {
		before := libvirtClockHistory(m, hostID)
		host := applyCounterClockHost(t, m, agent.libvirtHostReport("stream-a", step.seq, step.slot, step.authoredAt, step.collectedAt, agent.libvirtDomain(step.slot, 0)))
		domain := libvirtClockDomain(t, step.name, host)
		switch step.want {
		case "unknown":
			wantLibvirtUnknown(t, step.name, domain)
			if after := libvirtClockHistory(m, hostID); !reflect.DeepEqual(after, before) {
				t.Fatalf("%s wrote libvirt rate history without a comparable interval:\nbefore %v\nafter  %v", step.name, before, after)
			}
		case "true":
			wantLibvirtTrueRates(t, step.name, domain)
		}
	}
	wantLibvirtHistoryTrueRates(t, m, hostID)
}

// An agent restart keeps the machine clock, so the new stream's first libvirt
// sample is measured from the old stream's last, and a delayed old-stream
// report changes neither the domain nor its history.
func TestApplyHostReportLibvirtStreamChangeKeepsMachineClock(t *testing.T) {
	m := newCounterClockHostMonitor(t)
	agent := newCounterClockAgent(time.Now().Add(-time.Hour))
	report := func(stream string, seq uint64, slot int) agentshost.Report {
		return agent.libvirtHostReport(stream, seq, slot, agent.collectedAt(slot), agent.libvirtCollectedAt(slot), agent.libvirtDomain(slot, 0))
	}
	for slot := 1; slot <= 3; slot++ {
		applyCounterClockHost(t, m, report("stream-a", uint64(slot), slot))
	}

	restarted := applyCounterClockHost(t, m, report("stream-b", 1, 4))
	wantLibvirtTrueRates(t, "first report of the new stream", libvirtClockDomain(t, "restarted", restarted))

	before := libvirtClockHistory(m, restarted.ID)
	delayed, err := m.ApplyHostReport(report("stream-a", 4, 3), nil)
	if err != nil {
		t.Fatalf("ApplyHostReport delayed old-stream report: %v", err)
	}
	if delayed.UptimeSeconds != 4 {
		t.Fatalf("retired-stream report replaced state: host holds slot %d, want 4", delayed.UptimeSeconds)
	}
	wantLibvirtTrueRates(t, "stored domain returned for the retired stream", libvirtClockDomain(t, "delayed", delayed))
	if after := libvirtClockHistory(m, restarted.ID); !reflect.DeepEqual(after, before) {
		t.Fatalf("retired-stream report wrote libvirt history:\nbefore %v\nafter  %v", before, after)
	}

	next := applyCounterClockHost(t, m, report("stream-b", 2, 5))
	wantLibvirtTrueRates(t, "second report of the new stream", libvirtClockDomain(t, "next", next))
}

// Omissions keep their existing meaning: a readable inventory without the
// domain retires it, so CPU usage restarts on its return, while its
// cumulative I/O counters measure the whole span since their last sample; an
// inventory the agent failed to read preserves the last one, and the next
// read measures CPU from that last read rather than from the host report in
// between; a domain whose counters restart has no CPU usage for that sample.
func TestApplyHostReportLibvirtOmittedDomainInventoryAndRestart(t *testing.T) {
	m := newCounterClockHostMonitor(t)
	agent := newCounterClockAgent(time.Now().Add(-time.Hour))
	report := func(seq uint64, slot int, domains ...agentshost.LibvirtDomain) agentshost.Report {
		return agent.libvirtHostReport("stream-a", seq, slot, agent.collectedAt(slot), agent.libvirtCollectedAt(slot), domains...)
	}
	for slot := 1; slot <= 2; slot++ {
		applyCounterClockHost(t, m, report(uint64(slot), slot, agent.libvirtDomain(slot, 0)))
	}

	omitted := applyCounterClockHost(t, m, report(3, 3))
	if omitted.Libvirt == nil || len(omitted.Libvirt.Domains) != 0 {
		t.Fatalf("readable inventory without the domain = %+v, want no domains", omitted.Libvirt)
	}

	returned := libvirtClockDomain(t, "returned", applyCounterClockHost(t, m, report(4, 4, agent.libvirtDomain(4, 0))))
	if returned.CPUUsageValid {
		t.Fatalf("domain absent from the previous report reported CPU usage %.1f", returned.CPUUsagePercent)
	}
	if !returned.IORatesValid {
		t.Fatal("returned domain lost its I/O rates")
	}
	wantTrueRate(t, "returned NetworkInRate", returned.NetworkInRate, libvirtClockNetInPerSecond)
	wantTrueRate(t, "returned DiskWriteRate", returned.DiskWriteRate, libvirtClockWritePerSecond)

	wantLibvirtTrueRates(t, "domain after its return", libvirtClockDomain(t, "slot 5", applyCounterClockHost(t, m, report(5, 5, agent.libvirtDomain(5, 0)))))

	unread := agent.hostReport("stream-a", 6, 6, agent.collectedAt(6))
	wantLibvirtTrueRates(t, "inventory preserved through a failed read", libvirtClockDomain(t, "slot 6", applyCounterClockHost(t, m, unread)))
	wantLibvirtTrueRates(t, "read after the failed one", libvirtClockDomain(t, "slot 7", applyCounterClockHost(t, m, report(7, 7, agent.libvirtDomain(7, 0)))))

	restarted := libvirtClockDomain(t, "restarted", applyCounterClockHost(t, m, report(8, 8, agent.libvirtDomain(8, 8))))
	if restarted.CPUUsageValid {
		t.Fatalf("restarted domain reported CPU usage %.1f across its counter reset", restarted.CPUUsagePercent)
	}
	if !restarted.IORatesValid || restarted.NetworkInRate != 0 || restarted.DiskReadRate != 0 {
		t.Fatalf("restarted domain I/O = %+v, want the counter-reset rebase of zero", restarted)
	}
	wantLibvirtTrueRates(t, "first interval after the restart", libvirtClockDomain(t, "slot 9", applyCounterClockHost(t, m, report(9, 9, agent.libvirtDomain(9, 8)))))
}

// Legacy reports have no ordered source clock, so their libvirt counters keep
// receipt time even when the inventory carries CollectedAt.
func TestApplyHostReportLegacyLibvirtRatesStayOnReceiptTime(t *testing.T) {
	m := newCounterClockHostMonitor(t)
	agent := newCounterClockAgent(time.Now().Add(-time.Hour))
	report := func(slot int) agentshost.Report {
		return agent.libvirtHostReport("", 0, slot, agent.collectedAt(slot), agent.libvirtCollectedAt(slot), agent.libvirtDomain(slot, 0))
	}
	// Start the clock before the first report so wall spans both receipt
	// stamps; starting after it let a slow first apply exceed the second's.
	sentAt := time.Now()
	applyCounterClockHost(t, m, report(1))
	domain := libvirtClockDomain(t, "legacy", applyCounterClockHost(t, m, report(2)))
	wall := time.Since(sentAt).Seconds()
	if !domain.CPUUsageValid || !domain.IORatesValid {
		t.Fatalf("legacy domain lost its receipt-time rates: %+v", domain)
	}
	delta := float64(agent.libvirtCounter(libvirtClockNetInPerSecond, 2, 0) - agent.libvirtCounter(libvirtClockNetInPerSecond, 1, 0))
	if domain.NetworkInRate < delta/wall {
		t.Fatalf("legacy NetworkInRate = %.1f, want the receipt-interval rate of at least %.1f (CollectedAt would give %.1f)",
			domain.NetworkInRate, delta/wall, delta/agent.interval.Seconds())
	}
}