Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Stop the Windows polling contract depending on host speed
rcourtman1d244a7dd387
diff --git a/internal/monitoring/guest_history_observation_test.go b/internal/monitoring/guest_history_observation_test.goindex 3f6395eb68..18cff8ad74 100644--- a/internal/monitoring/guest_history_observation_test.go+++ b/internal/monitoring/guest_history_observation_test.go@@ -32,7 +32,9 @@ func guestHistoryObservationMonitor(t *testing.T) *Monitor {t.Fatal(err)}t.Cleanup(func() { _ = store.Close() })- if err := store.WaitForMaintenance(5 * time.Second); err != nil {+ // Wait on the maintenance barrier itself. Startup maintenance finishes when+ // it finishes, and a timer here only fails the test on a busy machine.+ if err := store.WaitForMaintenance(0); err != nil {t.Fatal(err)}return &Monitor{metricsHistory: NewMetricsHistory(128, 24*time.Hour), metricsStore: store}@@ -42,7 +44,10 @@ func guestHistoryStoredPoints(t *testing.T, m *Monitor, kind, id, metric string)t.Helper()m.metricsStore.Flush()now := time.Now()- points, err := m.metricsStore.Query(kind, id, metric, now.Add(-time.Minute), now.Add(time.Minute), 0)+ // Every point was written by the calling test. Reach back far enough that a+ // stalled run cannot age its earliest points out of the window, yet stay+ // inside the raw tier (ranges up to two hours).+ points, err := m.metricsStore.Query(kind, id, metric, now.Add(-time.Hour), now.Add(time.Minute), 0)if err != nil {t.Fatal(err)}diff --git a/internal/monitoring/guest_windows_polling_test.go b/internal/monitoring/guest_windows_polling_test.goindex 958e6d24d3..67783ef203 100644--- a/internal/monitoring/guest_windows_polling_test.go+++ b/internal/monitoring/guest_windows_polling_test.go@@ -19,6 +19,22 @@ import ("github.com/rcourtman/pulse-go-rewrite/pkg/proxmox")+const (+ // Bar one deliberately late reply, the fixture answers every request at+ // once, so the deadlines it sets are hang guards, not latency budgets. A+ // tight one makes the contract depend on the machine running it: the+ // guest-agent guard defers a poll whose lock check fails (lock-unverified)+ // and fences the guest for a minute after a command times out or its body+ // is cut short (agent-timeout, agent-response-incomplete, then+ // agent-cooldown). That is right for a real guest and wrong for this+ // contract. The proxmox transport still limits dialing and the wait for+ // response headers to ten seconds, and a cluster client probes a new+ // endpoint within three; the fixture cannot set either.+ windowsPollingHangGuard = 15 * time.Minute+ // A reply later than the one-second client budget this fixture used to set.+ windowsPollingSlowReply = 1200 * time.Millisecond+)+// Synthetic, ordinary QGA-only polling after a successful Windows OS reply.// The deliberately uncertain Linux read is an adverse fixture, not a replay of// issue #2619 or a claim about the response/cause on its native installation.@@ -28,7 +44,10 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {t.Run(kind+"/"+path, func(t *testing.T) {const mib = uint64(1024 * 1024)var poll, fileReads, fsReads, configReads atomic.Int32- var locked atomic.Bool+ var locked, slowLockCheck atomic.Bool+ // The four subtests share one client budget, so one late reply+ // holds it; more would only lengthen the run.+ lateLockCheck := kind == "client" && path == "cluster"server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {t.Error("lost configured authentication")@@ -40,6 +59,11 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {fmt.Fprint(w, `{"data":[]}`)case strings.HasSuffix(r.URL.Path, "/config"):configReads.Add(1)+ // One lock check later than the old client budget, in an+ // otherwise ordinary poll: lateness alone must not defer it.+ if lateLockCheck && poll.Load() == 2 && slowLockCheck.CompareAndSwap(false, true) {+ time.Sleep(windowsPollingSlowReply)+ }if locked.Load() {fmt.Fprint(w, `{"data":{"lock":"backup"}}`)} else {@@ -72,7 +96,7 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {}}))defer server.Close()- cfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}+ cfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: windowsPollingHangGuard}var client PVEClientInterfaceif kind == "cluster-client" {client = proxmox.NewClusterClient("windows", cfg, []string{server.URL}, nil)@@ -84,6 +108,8 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {}}m := guestHistoryObservationMonitor(t)+ m.guestAgentFSInfoTimeout, m.guestAgentNetworkTimeout = windowsPollingHangGuard, windowsPollingHangGuard+ m.guestAgentOSInfoTimeout, m.guestAgentVersionTimeout = windowsPollingHangGuard, windowsPollingHangGuardm.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())t.Cleanup(m.alertManager.Stop)m.config, m.state, m.rateTracker = &config.Config{}, models.NewState(), NewRateTracker()@@ -107,6 +133,12 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {id := makeGuestID("windows", "node", 105)res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, MaxDisk: 1000 * mib}build := func() models.VM {+ // The poll loop takes its cycle start before building, so the+ // guest is always observed in the cycle it records. A window+ // measured back from the record call would drop the sample+ // whenever the state update and resource-store refresh in+ // between take over a second.+ cycleStart := time.Now()previous := m.previousGuestContextForInstance("windows").vmsByID[id]var vm models.VMif path == "node" {@@ -124,7 +156,7 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {}m.state.UpdateVMs([]models.VM{vm})m.updateResourceStore(m.currentStateWithScope())- m.recordGuestMetrics([]models.VM{vm}, nil, time.Now().Add(-time.Second))+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)view := m.currentModeReadState().VMs()[0]if view.DiskUsed() != vm.Disk.Used || view.DiskStatusReason() != vm.DiskStatusReason || view.OSName() != vm.OSName {t.Error("accepted Windows poll changed at canonical read boundary")@@ -158,6 +190,9 @@ func testGuestWindowsMeminfoPolling(t *testing.T) {t.Errorf("poll %d lost independent Windows readings: disk=%d reason=%q guest=%q memory=%d/%s cpu=%v", i, last.Disk.Used, last.DiskStatusReason, last.GuestAgentStatus, last.Memory.Used, last.Memory.Observation.State, last.CPU)}}+ if lateLockCheck && !slowLockCheck.Load() {+ t.Error("the late lock verification never reached the fixture")+ }if fileReads.Load() != 0 || fsReads.Load() != 3 || !reflect.DeepEqual(m.vmAgentMemCache[memoryKey], originalMemory) || !reflect.DeepEqual(m.guestMetadataCache[metadataKey], originalMetadata) {t.Errorf("known Windows sent Linux work or renewed evidence: file=%d fs=%d", fileReads.Load(), fsReads.Load())}