Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

12.4 KB
Raw Blame
package monitoring

import (
	"context"
	"encoding/json"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)

// Use the served JSON rather than the proposed Go type so this control also
// compiles against the parent that loses observation provenance entirely.
func assertGuestMemoryObservation(t *testing.T, memory models.Memory, state, source string, at time.Time) {
	t.Helper()
	data, err := json.Marshal(memory)
	if err != nil {
		t.Fatal(err)
	}
	var wire struct {
		Observation struct {
			State      string     `json:"state"`
			Source     string     `json:"source"`
			ObservedAt *time.Time `json:"observedAt"`
		} `json:"observation"`
	}
	if err := json.Unmarshal(data, &wire); err != nil {
		t.Fatal(err)
	}
	got := wire.Observation
	if got.State != state || got.Source != source {
		t.Errorf("memory observation = %s, want %s / %s", data, state, source)
	}
	if at.IsZero() {
		if got.ObservedAt != nil {
			t.Errorf("unobserved memory acquired a timestamp: %s", data)
		}
	} else if got.ObservedAt == nil || !got.ObservedAt.Equal(at) {
		t.Errorf("memory observation time = %v, want original %v", got.ObservedAt, at)
	}
}

func testGuestMemoryObservationLifecycle(t *testing.T) {
	const gib = uint64(1024 * 1024 * 1024)
	identity := makeGuestID("provenance", "node", 105)
	registry := unifiedresources.NewRegistry(nil)
	store := unifiedresources.NewMonitorAdapter(registry)
	m := &Monitor{guestMetadataLimiter: make(map[string]time.Time), rateTracker: NewRateTracker(), metricsHistory: NewMetricsHistory(32, time.Hour), resourceStore: store}
	client := &vmMemoryTrustStubClient{stubPVEClient: &stubPVEClient{}, vmAgentMemAvailable: 5 * gib,
		vmStatus: &proxmox.VMStatus{Status: "running", MaxMem: 8 * gib, Mem: 8 * gib, Agent: proxmox.VMAgentField{Value: 1}}}
	res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * gib, Mem: 8 * gib}
	var previous *models.VM
	build := func() models.VM {
		t.Helper()
		// The cycle starts before collection, as in pollVMsAndContainersEfficient.
		cycleStart := time.Now()
		vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "provenance", res, client, identity, nil, previous)
		if !ok {
			t.Fatal("guest disappeared")
		}
		m.recordGuestSnapshot("provenance", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
		m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
		store.PopulateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
		views := store.VMs()
		if len(views) != 1 {
			t.Fatal("guest did not reach unified view")
		}
		next := previousVMFromView(views[0])
		previous = &next
		return vm
	}
	assertProjection := func(vm models.VM, state, source string, at time.Time) {
		t.Helper()
		assertGuestMemoryObservation(t, vm.Memory, state, source, at)
		legacy := vm.ToFrontend()
		if legacy.Memory == nil {
			t.Fatal("legacy projection lost memory")
		}
		assertGuestMemoryObservation(t, *legacy.Memory, state, source, at)
		// REST and WebSocket state use this production conversion, including
		// registry cloning and the nested Proxmox payload the drawer consumes.
		front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())
		data, err := json.Marshal(front.Resources)
		if err != nil {
			t.Fatal(err)
		}
		var wire []struct {
			Memory  *models.ResourceMetricFrontend `json:"memory"`
			Proxmox struct {
				Memory models.Memory `json:"memory"`
			} `json:"proxmox"`
		}
		if err := json.Unmarshal(data, &wire); err != nil || len(wire) != 1 {
			t.Fatalf("broadcast guest memory unavailable: %s / %v", data, err)
		}
		assertGuestMemoryObservation(t, wire[0].Proxmox.Memory, state, source, at)
		if vm.Memory.HasKnownUsage() {
			if wire[0].Memory == nil {
				t.Fatal("selected memory metric disappeared")
			}
			assertGuestMemoryObservation(t, models.Memory{Observation: wire[0].Memory.Observation}, state, source, at)
		} else if wire[0].Memory != nil {
			t.Fatal("unavailable memory became a numeric metric")
		}
		if wire[0].Proxmox.Memory.Used != vm.Memory.Used || wire[0].Proxmox.Memory.UsageUnavailable != vm.Memory.UsageUnavailable {
			t.Fatal("provenance changed numeric/unknown memory projection")
		}
	}
	initial := build()
	key := guestMemoryCacheKey("provenance", "node", 105)
	original := m.vmAgentMemCache[key].fetchedAt
	if original.IsZero() || initial.Memory.Used != int64(3*gib) {
		t.Fatal("successful QGA observation missing")
	}
	assertProjection(initial, "current", "guest-agent-meminfo", original)
	cached := build()
	assertProjection(cached, "current", "guest-agent-meminfo", original)
	if client.vmAgentMemCalls != 1 {
		t.Fatal("normal cache caused a new guest read")
	}
	res.Lock, client.vmStatus.Lock = "backup", "backup"
	for poll := 0; poll < 4; poll++ {
		locked := build()
		assertProjection(locked, "last-known", "guest-agent-meminfo", original)
		if locked.Memory.Used != initial.Memory.Used || m.vmAgentMemCache[key].fetchedAt != original || client.vmAgentMemCalls != 1 {
			t.Fatal("deferral changed evidence, its age, or guest read admission")
		}
	}
	if got := len(m.metricsHistory.GetGuestMetrics(identity, "memory", time.Hour)); got != 2 {
		t.Errorf("deferred memory became fresh History points: %d, want two normal polls", got)
	}
	entry := m.vmAgentMemCache[key]
	expiredAt := time.Now().Add(-vmAgentMemCleanupMaxAge - time.Second)
	entry.fetchedAt = expiredAt
	m.vmAgentMemCache[key] = entry
	// Cache expiry alone cannot expire the independently annotated snapshot.
	// Its original observation is still within the short carry-forward window,
	// even though the cache no longer qualifies for deferred QGA retention.
	retained := build()
	assertProjection(retained, "last-known", "guest-agent-meminfo", original)
	if retained.Memory.Used != initial.Memory.Used || m.vmAgentMemCache[key].fetchedAt != expiredAt || client.vmAgentMemCalls != 1 {
		t.Fatal("cache expiry lost the recent original snapshot or renewed QGA evidence")
	}
	// Model elapsed time consistently in both original evidence owners. Keep
	// the diagnostic receipt and served LastSeen fresh: poll timestamps cannot
	// renew the reading. No retention limit or production clock is changed.
	snapshot := m.previousGuestSnapshot("provenance", "qemu", "node", 105)
	if snapshot == nil {
		t.Fatal("retained diagnostic snapshot missing")
	}
	snapshot.Memory.Observation.ObservedAt = expiredAt
	m.recordGuestSnapshot("provenance", "qemu", "node", 105, *snapshot)
	previous.Memory.Observation.ObservedAt = expiredAt
	for poll := 0; poll < 4; poll++ {
		expired := build()
		assertProjection(expired, "unavailable", "unavailable", time.Time{})
		if expired.Memory.HasKnownUsage() || m.vmAgentMemCache[key].fetchedAt != expiredAt || client.vmAgentMemCalls != 1 {
			t.Fatal("expired evidence became usage, renewed its age or queued a guest read")
		}
	}
	for _, metric := range []string{"memory", "memoryused"} {
		if got := len(m.metricsHistory.GetGuestMetrics(identity, metric, time.Hour)); got != 2 {
			t.Errorf("cache/snapshot expiry renewed %s History: %d, want two normal polls", metric, got)
		}
	}
	res.Lock, client.vmStatus.Lock = "", ""
	client.vmAgentMemAvailable = 4 * gib
	resumed := build()
	resumedAt := m.vmAgentMemCache[key].fetchedAt
	assertProjection(resumed, "current", "guest-agent-meminfo", resumedAt)
	if !resumedAt.After(original) || resumed.Memory.Used != int64(4*gib) || client.vmAgentMemCalls != 2 {
		t.Fatal("resumption did not replace original guest observation")
	}
	// Backup/disk deferral must not disqualify an independently current PVE
	// memory reading. Its timestamp belongs to that source, not this poll.
	statusAt := time.Now().Add(-time.Second)
	res.Lock, client.vmStatus.Lock = "backup", "backup"
	client.vmStatus.ObservedAt = statusAt
	client.vmStatus.MemInfo = &proxmox.VMMemInfo{Total: 8 * gib, Available: 6 * gib}
	live := build()
	assertProjection(live, "current", "available-field", statusAt)
	if live.Memory.Used != int64(2*gib) || live.GuestAgentStatus != "deferred" || client.vmAgentMemCalls != 2 {
		t.Fatal("independent PVE memory was replaced by deferred QGA evidence")
	}
	res.Status = "stopped"
	stopped := build()
	assertProjection(stopped, "unavailable", "powered-off", time.Time{})
	if stopped.Memory.Used != 0 {
		t.Fatal("powered-off numeric compatibility changed")
	}
}

func testGuestMemoryObservationKeepsOriginsSeparate(t *testing.T) {
	now := time.Now()
	at := now.Add(-2 * time.Minute)
	observed := models.Memory{Total: 100, Used: 25, Free: 75, Usage: 25}
	previous := GuestMemorySnapshot{Instance: "site", GuestType: "qemu", Node: "node", VMID: 105, Status: "running", RetrievedAt: now,
		MemorySource: "previous-snapshot", Memory: observed}
	previous.Memory.Observation = models.MemoryObservation{State: "last-known", Source: "available-field", ObservedAt: at}
	m := &Monitor{vmAgentMemCache: map[string]agentMemCacheEntry{guestMemoryCacheKey("site", "node", 105): {
		fetchedAt: now.Add(-time.Second), info: proxmox.LinuxMemoryAvailability{Source: "meminfo-available", EffectiveAvailable: 75}}}}
	for _, tc := range []struct {
		name   string
		change func(*GuestMemorySnapshot)
		source string
		at     time.Time
	}{
		{name: "identical numeric QGA cache cannot replace PVE provenance", source: "available-field", at: at},
		{name: "poll timestamp does not renew legacy retained evidence", source: "previous-snapshot", change: func(p *GuestMemorySnapshot) { p.Memory.Observation = models.MemoryObservation{} }},
		{name: "foreign guest snapshot cannot lend evidence", source: "previous-snapshot", change: func(p *GuestMemorySnapshot) { p.VMID++ }},
		{name: "foreign site snapshot cannot lend evidence", source: "previous-snapshot", change: func(p *GuestMemorySnapshot) { p.Instance = "other" }},
		{name: "future original time remains unknown", source: "available-field", change: func(p *GuestMemorySnapshot) { p.Memory.Observation.ObservedAt = now.Add(time.Second) }},
	} {
		t.Run(tc.name, func(t *testing.T) {
			prior := previous
			if tc.change != nil {
				tc.change(&prior)
			}
			memory := observed
			memory.Observation = m.guestMemoryObservation("site", "qemu", "node", 105, memory, guestMemoryObservationContext{source: "previous-snapshot", previous: &prior}, now)
			assertGuestMemoryObservation(t, memory, "last-known", tc.source, tc.at)
		})
	}
}

func testGuestMemoryObservationPreservesIndependentLinkedAgent(t *testing.T) {
	at := time.Now().Add(-30 * time.Second)
	resource := &unifiedresources.Resource{Type: unifiedresources.ResourceTypeVM, LastSeen: time.Now(),
		Agent:        &unifiedresources.AgentData{Memory: &unifiedresources.AgentMemoryMeta{Total: 100, Used: 40, Free: 60}},
		SourceStatus: map[unifiedresources.DataSource]unifiedresources.SourceStatus{unifiedresources.SourceAgent: {Status: "online", LastSeen: at}}}
	view := unifiedresources.NewVMView(resource)
	agentMemory, ok := view.LinkedAgentMemory()
	if !ok {
		t.Fatal("source-owned linked memory missing")
	}
	for _, guestType := range []string{"qemu", "lxc"} {
		t.Run(guestType, func(t *testing.T) {
			m := &Monitor{rateTracker: NewRateTracker(), guestMetadataLimiter: make(map[string]time.Time)}
			client := &vmMemoryTrustStubClient{stubPVEClient: &stubPVEClient{}, vmStatus: &proxmox.VMStatus{Status: "running", Lock: "backup", MaxMem: 100, Mem: 100}}
			res := proxmox.ClusterResource{Type: guestType, Node: "node", VMID: 105, Status: "running", Lock: "backup", MaxMem: 100, Mem: 100, ObservedAt: time.Now()}
			identity := makeGuestID("site", "node", 105)
			linked := map[string]models.Host{identity: {Memory: agentMemory, Status: "online"}}
			var memory models.Memory
			if guestType == "qemu" {
				vm, _, source, _, _, ok := m.buildVMFromClusterResource(context.Background(), "site", res, client, identity, linked, nil)
				if !ok || source != "agent" {
					t.Fatalf("linked VM agent was not selected: %s", source)
				}
				memory = vm.Memory
			} else {
				ct, _, source, _, ok := m.buildContainerFromClusterResource(context.Background(), "site", res, client, nil, linked)
				if !ok || source != "agent" {
					t.Fatalf("linked LXC agent was not selected: %s", source)
				}
				memory = ct.Memory
			}
			assertGuestMemoryObservation(t, memory, "current", "agent", at)
			if memory.Used != 40 || client.vmAgentMemCalls != 0 {
				t.Fatal("independent memory changed or caused a guest-agent read")
			}
		})
	}
}