Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Stop sibling guest poll tests depending on host speed

rcourtman013f14ade9f6
diff --git a/internal/monitoring/guest_agent_backup_integration_test.go b/internal/monitoring/guest_agent_backup_integration_test.go
index 9d37a96c32..70e8e3b970 100644
--- a/internal/monitoring/guest_agent_backup_integration_test.go
+++ b/internal/monitoring/guest_agent_backup_integration_test.go
@@ -86,7 +86,7 @@ func testGuestAgentBackupMonitoringLifecycleWithUnverifiedConfig(t *testing.T, u
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -96,12 +96,14 @@ func testGuestAgentBackupMonitoringLifecycleWithUnverifiedConfig(t *testing.T, u
identity := "fixture:node:105"
registry := unifiedresources.NewRegistry(nil)
build := func() (models.VM, string) {
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "fixture", res, client, identity, nil, previous)
if !ok {
t.Fatal("VM disappeared")
}
m.recordGuestSnapshot("fixture", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
// Production obtains previous guest context from the unified read view,
// not the builder's raw model (the context need not contain memory).
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
@@ -220,7 +222,7 @@ func TestGuestAgentDeferralPreservesMetadataAndMemoryCache(t *testing.T) {
http.Error(w, "QEMU command timed out", 500)
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/monitoring/guest_agent_evidence_age_test.go b/internal/monitoring/guest_agent_evidence_age_test.go
index 3f27ff890b..2b41d35c27 100644
--- a/internal/monitoring/guest_agent_evidence_age_test.go
+++ b/internal/monitoring/guest_agent_evidence_age_test.go
@@ -197,7 +197,7 @@ func testGuestAgentRetainedIdentityCannotRenewAdmission(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -211,6 +211,8 @@ func testGuestAgentRetainedIdentityCannotRenewAdmission(t *testing.T) {
registry := unifiedresources.NewRegistry(nil)
build := func() models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
ctx := context.Background()
if phase.Load() == -1 {
canceled, cancel := context.WithCancel(ctx)
@@ -227,7 +229,7 @@ func testGuestAgentRetainedIdentityCannotRenewAdmission(t *testing.T) {
t.Fatal("guest disappeared")
}
vm := vms[0]
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
prev = previousVMFromView(registry.VMs()[0])
if vm.ID != id || vm.CPU != .2 {
diff --git a/internal/monitoring/guest_agent_optional_read_integration_test.go b/internal/monitoring/guest_agent_optional_read_integration_test.go
index 5804ef02c6..991c650a82 100644
--- a/internal/monitoring/guest_agent_optional_read_integration_test.go
+++ b/internal/monitoring/guest_agent_optional_read_integration_test.go
@@ -104,7 +104,7 @@ func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {
}))
defer server.Close()
newClient := func() PVEClientInterface {
- cfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}
+ cfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard}
if clientKind == "cluster" {
return proxmox.NewClusterClient("optional", cfg, []string{server.URL}, nil)
}
@@ -134,12 +134,14 @@ func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {
}
build := func() models.VM {
scope := m.mockModeFence.begin()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "optional", res, client, id, nil, previous)
if !ok {
t.Fatal("guest disappeared")
}
m.recordGuestSnapshot("optional", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
view := registry.VMs()[0]
canonicalID, resolved := registry.ResolveReferenceID(id)
diff --git a/internal/monitoring/guest_agent_transport_integration_test.go b/internal/monitoring/guest_agent_transport_integration_test.go
index b0b1296dbf..6ae60c8084 100644
--- a/internal/monitoring/guest_agent_transport_integration_test.go
+++ b/internal/monitoring/guest_agent_transport_integration_test.go
@@ -90,7 +90,7 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -99,12 +99,14 @@ func testGuestAgentTransportDeferralKeepsLastKnownHistory(t *testing.T, kinds ..
identity := "transport:node:105"
var previous *models.VM
build := func() (models.VM, string) {
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "transport", res, client, identity, nil, previous)
if !ok {
t.Fatal("VM disappeared")
}
m.recordGuestSnapshot("transport", "qemu", "node", 105, GuestMemorySnapshot{Name: vm.Name, Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
previous = &vm
return vm, source
}
diff --git a/internal/monitoring/guest_alert_observation_integration_test.go b/internal/monitoring/guest_alert_observation_integration_test.go
index ec6c293914..012a8a8b5a 100644
--- a/internal/monitoring/guest_alert_observation_integration_test.go
+++ b/internal/monitoring/guest_alert_observation_integration_test.go
@@ -61,7 +61,7 @@ func TestGuestBackupDeferralAlertObservationLifecycle(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -81,12 +81,14 @@ func TestGuestBackupDeferralAlertObservationLifecycle(t *testing.T) {
var previous *models.VM
build := func() models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "alert-observation", res, client, identity, nil, previous)
if !ok {
t.Fatal("VM disappeared")
}
m.recordGuestSnapshot("alert-observation", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
next := previousVMFromView(registry.VMs()[0])
previous = &next
diff --git a/internal/monitoring/guest_disk_expiry_test.go b/internal/monitoring/guest_disk_expiry_test.go
index 53c860aa51..94e984410f 100644
--- a/internal/monitoring/guest_disk_expiry_test.go
+++ b/internal/monitoring/guest_disk_expiry_test.go
@@ -173,7 +173,7 @@ func testGuestDiskOrdinaryDeferralExpiryAndRecovery(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -199,6 +199,8 @@ func testGuestDiskOrdinaryDeferralExpiryAndRecovery(t *testing.T) {
}
build := func(ctx context.Context) models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
previous := m.previousGuestContextForInstance("disk-expiry").vmsByID[id]
var vm models.VM
if path == "node" {
@@ -216,7 +218,7 @@ func testGuestDiskOrdinaryDeferralExpiryAndRecovery(t *testing.T) {
m.checkGuestAlertsForVM("disk-expiry", vm)
}
publish(vm)
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
view := m.currentModeReadState().VMs()[0]
if view.DiskStatusReason() != vm.DiskStatusReason || view.DiskObservation() != vm.DiskObservation {
t.Fatal("source disk observation changed at canonical read boundary")
diff --git a/internal/monitoring/guest_filesystem_alias_usage_test.go b/internal/monitoring/guest_filesystem_alias_usage_test.go
index d6bc03d16d..6c88369fde 100644
--- a/internal/monitoring/guest_filesystem_alias_usage_test.go
+++ b/internal/monitoring/guest_filesystem_alias_usage_test.go
@@ -116,7 +116,7 @@ func TestVMFilesystemAliasUsagePollingAndHistory(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -141,6 +141,8 @@ func TestVMFilesystemAliasUsagePollingAndHistory(t *testing.T) {
time.Sleep(1100 * time.Millisecond)
}
phase.Store(int32(step))
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
previous := m.previousGuestContextForInstance("aliases").vmsByID[id]
var vm models.VM
if collector == "node" {
@@ -171,7 +173,7 @@ func TestVMFilesystemAliasUsagePollingAndHistory(t *testing.T) {
}
m.state.UpdateVMs([]models.VM{vm})
m.updateResourceStore(m.currentStateWithScope())
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
view := m.currentModeReadState().VMs()[0]
if view.DiskTotal() != 1000 || view.DiskUsed() != wantUsed || view.DiskPercent() != vm.Disk.Usage || !reflect.DeepEqual(previousVMFromView(view).Disks, vm.Disks) {
t.Fatal("canonical state lost aggregate or individual mount evidence")
diff --git a/internal/monitoring/guest_filesystem_completeness_test.go b/internal/monitoring/guest_filesystem_completeness_test.go
index e9170728a9..a357630748 100644
--- a/internal/monitoring/guest_filesystem_completeness_test.go
+++ b/internal/monitoring/guest_filesystem_completeness_test.go
@@ -94,7 +94,7 @@ func TestVMFilesystemIncompletePollingPreservesHistoryAndBreach(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -243,7 +243,7 @@ func TestVMFilesystemIncompleteColdPollRemainsUnavailable(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/monitoring/guest_filesystem_identity_test.go b/internal/monitoring/guest_filesystem_identity_test.go
index e477a00a40..308989b803 100644
--- a/internal/monitoring/guest_filesystem_identity_test.go
+++ b/internal/monitoring/guest_filesystem_identity_test.go
@@ -107,7 +107,7 @@ func TestVMFilesystemWindowsVolumePollingHistory(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -129,11 +129,13 @@ func TestVMFilesystemWindowsVolumePollingHistory(t *testing.T) {
}
poll.Store(int32(phase))
scope := m.mockModeFence.begin()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, _, _, _, _, ok := m.buildVMFromClusterResource(context.Background(), "volumes", res, client, id, nil, previous)
if !ok || vm.Disk.Total != want.total || vm.Disk.Used != want.used || vm.Disk.Free != want.total-want.used || vm.Disk.Usage != want.usage || len(vm.Disks) != want.rows || vm.DiskStatusReason != "" {
t.Fatalf("poll %d did not publish complete current volume usage: %+v", phase, vm)
}
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
m.updateResourceStore(models.StateSnapshot{VMs: []models.VM{vm}, LastUpdate: vm.LastSeen}, scope)
views := m.GetUnifiedReadState().VMs()
if len(views) != 1 || views[0].DiskTotal() != want.total || views[0].DiskUsed() != want.used || len(views[0].Disks()) != want.rows {
diff --git a/internal/monitoring/guest_history_observation_test.go b/internal/monitoring/guest_history_observation_test.go
index 18cff8ad74..fbb4beb7b7 100644
--- a/internal/monitoring/guest_history_observation_test.go
+++ b/internal/monitoring/guest_history_observation_test.go
@@ -19,6 +19,13 @@ import (
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
+// guestPollFixtureHangGuard is the proxmox client deadline for the loopback
+// guest poll fixtures. They answer every request at once, so the deadline only
+// stops a hung test. A tight one makes the result depend on the machine: one
+// late reply defers the poll (lock-unverified) or fences the guest after an
+// agent-timeout, and the History and command counts the tests assert change.
+const guestPollFixtureHangGuard = 15 * time.Minute
+
func guestHistoryObservationMonitor(t *testing.T) *Monitor {
t.Helper()
wasMock := mock.IsMockEnabled()
@@ -93,8 +100,10 @@ func TestGuestHistoryObservationContinuity(t *testing.T) {
ct := models.Container{ID: "history:node:106", Status: "running", LastSeen: now, CPU: .2, Memory: vm.Memory, Disk: vm.Disk}
record := func() {
t.Helper()
+ // The cycle starts before the guests are observed, as in production.
+ cycleStart := time.Now()
vm.LastSeen, ct.LastSeen = time.Now(), time.Now()
- m.recordGuestMetrics([]models.VM{vm}, []models.Container{ct}, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, []models.Container{ct}, cycleStart)
}
record()
type series struct{ kind, id, metric string }
@@ -186,8 +195,10 @@ func TestGuestHistoryPreservesIndependentCurrentMemory(t *testing.T) {
// producer observations, not a change to numeric/display availability,
// are what let independently current memory bypass that legacy heuristic.
m := guestHistoryObservationMonitor(t)
+ // The cycle starts before the guest is observed, as in production.
+ cycleStart := time.Now()
vm := models.VM{ID: "legacy-deferred", Status: "running", LastSeen: time.Now(), GuestAgentStatus: "deferred", Memory: models.Memory{Total: 100, Used: 25, Free: 75, Usage: 25}}
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
for _, metric := range []string{"memory", "memoryused"} {
if got := m.metricsHistory.GetGuestMetrics(vm.ID, metric, time.Hour); len(got) != 0 {
t.Errorf("unannotated deferral lost its historical guard: %s", metric)
@@ -236,7 +247,7 @@ func TestGuestHistoryPollFailureAndRecovery(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -249,12 +260,14 @@ func TestGuestHistoryPollFailureAndRecovery(t *testing.T) {
var previous *models.VM
build := func() models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "poll-history", res, client, id, nil, previous)
if !ok {
t.Fatal("guest disappeared")
}
m.recordGuestSnapshot("poll-history", "qemu", res.Node, res.VMID, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
next := previousVMFromView(registry.VMs()[0])
previous = &next
@@ -278,11 +291,13 @@ func TestGuestHistoryPollFailureAndRecovery(t *testing.T) {
// positive number is not filesystem evidence, even without prev- prefix.
other := res
other.VMID = 106
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
newVM, _, _, _, _, ok := m.buildVMFromClusterResource(context.Background(), "poll-history", other, client, makeGuestID("poll-history", "node", 106), nil, nil)
if !ok || newVM.DiskStatusReason != "no-status" || newVM.Disk.Usage <= 0 {
t.Fatalf("fixture did not retain the unverified positive allocation: %+v", newVM)
}
- m.recordGuestMetrics([]models.VM{newVM}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{newVM}, nil, cycleStart)
if got := m.metricsHistory.GetGuestMetrics(newVM.ID, "disk", time.Hour); len(got) != 0 {
t.Error("unverified cluster disk allocation became a filesystem History sample")
}
diff --git a/internal/monitoring/guest_memory_capacity_test.go b/internal/monitoring/guest_memory_capacity_test.go
index 11b1b57888..c9b6e564f3 100644
--- a/internal/monitoring/guest_memory_capacity_test.go
+++ b/internal/monitoring/guest_memory_capacity_test.go
@@ -163,7 +163,7 @@ func testGuestMemorySampleCapacityPolling(t *testing.T) {
}
}))
defer server.Close()
- pcfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}
+ pcfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard}
var client PVEClientInterface
if clientKind == "direct" {
var err error
@@ -189,6 +189,8 @@ func testGuestMemorySampleCapacityPolling(t *testing.T) {
previous := map[string]models.VM{}
build := func() models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: configured.Load(), Mem: configured.Load(), MaxDisk: gib, CPU: .1}
var vms []models.VM
if collector == "cluster-resources" {
@@ -200,7 +202,7 @@ func testGuestMemorySampleCapacityPolling(t *testing.T) {
t.Fatalf("collector lost VM: %v", vms)
}
vm := vms[0]
- m.recordGuestMetrics(vms, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics(vms, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: vms})
view := registry.VMs()[0]
if vm.Memory.HasKnownUsage() && (view.MemoryTotal() != vm.Memory.Total || view.MemoryUsed() != vm.Memory.Used || view.MemoryPercent() != vm.Memory.Usage) {
diff --git a/internal/monitoring/guest_memory_expiry_test.go b/internal/monitoring/guest_memory_expiry_test.go
index e0115623f7..6e95a50df7 100644
--- a/internal/monitoring/guest_memory_expiry_test.go
+++ b/internal/monitoring/guest_memory_expiry_test.go
@@ -136,7 +136,7 @@ func testGuestMemoryCarryForwardOrdinaryDeferralAndRecovery(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -158,12 +158,14 @@ func testGuestMemoryCarryForwardOrdinaryDeferralAndRecovery(t *testing.T) {
m.guestMetadataCache = map[string]guestMetadataCacheEntry{guestMetadataCacheKey("expiry", "node", 105): {fetchedAt: time.Now(), agentVersion: "fixture"}}
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * gib, Mem: 8 * gib, MaxDisk: gib, CPU: 0.2}
build := func() (models.VM, string) {
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "expiry", res, client, identity, nil, nil)
if !ok {
t.Fatal("guest disappeared")
}
m.recordGuestSnapshot("expiry", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
return vm, source
}
vm, source := build()
diff --git a/internal/monitoring/guest_memory_observation_test.go b/internal/monitoring/guest_memory_observation_test.go
index f1b09836ce..95981c40d2 100644
--- a/internal/monitoring/guest_memory_observation_test.go
+++ b/internal/monitoring/guest_memory_observation_test.go
@@ -54,12 +54,14 @@ func testGuestMemoryObservationLifecycle(t *testing.T) {
var previous *models.VM
build := func() models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "provenance", res, client, identity, nil, previous)
if !ok {
t.Fatal("guest disappeared")
}
m.recordGuestSnapshot("provenance", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
store.PopulateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
views := store.VMs()
if len(views) != 1 {
diff --git a/internal/monitoring/guest_memory_status_test.go b/internal/monitoring/guest_memory_status_test.go
index 8959736ce5..caa1e16019 100644
--- a/internal/monitoring/guest_memory_status_test.go
+++ b/internal/monitoring/guest_memory_status_test.go
@@ -91,7 +91,7 @@ func testGuestStatusMemoryAvailabilityPolling(t *testing.T) {
}
}))
defer server.Close()
- pcfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}
+ pcfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard}
var client PVEClientInterface
if kind == "direct" {
var err error
@@ -115,6 +115,8 @@ func testGuestStatusMemoryAvailabilityPolling(t *testing.T) {
previous := map[string]models.VM{}
build := func() models.VM {
t.Helper()
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 20 * gib, Mem: 20 * gib, CPU: .1}
var vms []models.VM
if collector == "cluster-resources" {
@@ -126,7 +128,7 @@ func testGuestStatusMemoryAvailabilityPolling(t *testing.T) {
t.Fatalf("collector lost VM: %v", vms)
}
vm := vms[0]
- m.recordGuestMetrics(vms, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics(vms, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: vms})
views := registry.VMs()
if len(views) != 1 {
diff --git a/internal/monitoring/guest_metadata_observation_test.go b/internal/monitoring/guest_metadata_observation_test.go
index c19a1bcf6c..862a450d32 100644
--- a/internal/monitoring/guest_metadata_observation_test.go
+++ b/internal/monitoring/guest_metadata_observation_test.go
@@ -200,7 +200,7 @@ func TestGuestMetadataObservationHTTPPollLifecycle(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
diff --git a/internal/monitoring/guest_metadata_suppression_test.go b/internal/monitoring/guest_metadata_suppression_test.go
index 701dc5e2b7..6dd0a69dd1 100644
--- a/internal/monitoring/guest_metadata_suppression_test.go
+++ b/internal/monitoring/guest_metadata_suppression_test.go
@@ -202,7 +202,7 @@ func TestGuestMetadataOSInfoSuppressionHTTPPollLifecycle(t *testing.T) {
}
}))
defer server.Close()
- client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second})
+ client, err := proxmox.NewClient(proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard})
if err != nil {
t.Fatal(err)
}
@@ -257,6 +257,8 @@ func TestGuestMetadataOSInfoSuppressionHTTPPollLifecycle(t *testing.T) {
}
locked.Store(false)
m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ // The cycle starts before collection, as in pollVMsAndContainersEfficient.
+ cycleStart := time.Now()
resumed := build()
if resumed.GuestAgentStatus != "available" || resumed.AgentVersion != "2.0" || resumed.Disk.Used != int64(300*mib) || len(resumed.Disks) != 1 || len(resumed.NetworkInterfaces) != 1 || resumed.NetworkInterfaces[0].Name != "eth1" {
t.Errorf("normal poll did not resume useful disk/network/version without OS-info: status=%s version=%s disks=%d", resumed.GuestAgentStatus, resumed.AgentVersion, len(resumed.Disks))
@@ -267,7 +269,7 @@ func TestGuestMetadataOSInfoSuppressionHTTPPollLifecycle(t *testing.T) {
if !m.guestMetadataCache[key].osInfoSkip || !m.hasRecentGuestMetadataEvidence("suppression", "node", 105, time.Now()) {
t.Error("resumed metadata lost suppression or useful current evidence")
}
- m.recordGuestMetrics([]models.VM{resumed}, nil, time.Now().Add(-time.Second))
+ m.recordGuestMetrics([]models.VM{resumed}, nil, cycleStart)
if got := len(m.metricsHistory.GetGuestMetrics(resumed.ID, "disk", time.Hour)); got != 1 {
t.Errorf("fresh disk History points=%d, want one", got)
}
diff --git a/internal/monitoring/monitor_pve_guest_cpu_authority_test.go b/internal/monitoring/monitor_pve_guest_cpu_authority_test.go
index 5bcfd84f05..c7493cc8e3 100644
--- a/internal/monitoring/monitor_pve_guest_cpu_authority_test.go
+++ b/internal/monitoring/monitor_pve_guest_cpu_authority_test.go
@@ -26,7 +26,8 @@ func TestRecordGuestMetricsKeepsLiveAndHistoricalProxmoxCPUUnitsAligned(t *testi
t.Cleanup(func() {
_ = store.Close()
})
- if err := store.WaitForMaintenance(5 * time.Second); err != nil {
+ // Wait on the maintenance barrier itself; a timer only fails a busy machine.
+ if err := store.WaitForMaintenance(0); err != nil {
t.Fatalf("wait for metrics maintenance: %v", err)
}
@@ -71,7 +72,8 @@ func TestRecordGuestMetricsKeepsLiveAndHistoricalProxmoxCPUUnitsAligned(t *testi
}
now := time.Now().UTC()
- storePoints, err := store.Query(tc.resourceType, tc.resourceID, "cpu", now.Add(-time.Minute), now.Add(time.Minute), 0)
+ // A stalled run must not age the point out; an hour stays in the raw tier.
+ storePoints, err := store.Query(tc.resourceType, tc.resourceID, "cpu", now.Add(-time.Hour), now.Add(time.Minute), 0)
if err != nil {
t.Fatalf("query persistent CPU history: %v", err)
}