Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Expose Windows guest bootstrap starvation after restart

pulse-triage[bot]affddfd22824
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go
index 3e959f0eea..29dae0cd86 100644
--- a/internal/monitoring/canonical_guardrails_test.go
+++ b/internal/monitoring/canonical_guardrails_test.go
@@ -3251,6 +3251,11 @@ func TestGuestWindowsMeminfoPollingContract(t *testing.T) {
testGuestWindowsMeminfoPolling(t)
}
+func TestGuestWindowsRestartPollingContract(t *testing.T) {
+ t.Run("cold-and-expired", testGuestWindowsBootstrapPolling)
+ t.Run("process-restart", testGuestWindowsBootstrapProcessRestart)
+}
+
func TestGuestMemoryObservationContract(t *testing.T) {
t.Run("poll-to-served-observation", testGuestMemoryObservationLifecycle)
t.Run("identity-and-origin-boundaries", testGuestMemoryObservationKeepsOriginsSeparate)
diff --git a/internal/monitoring/guest_windows_bootstrap_test.go b/internal/monitoring/guest_windows_bootstrap_test.go
new file mode 100644
index 0000000000..2e5b7eea81
--- /dev/null
+++ b/internal/monitoring/guest_windows_bootstrap_test.go
@@ -0,0 +1,377 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+const bootstrapMiB = uint64(1024 * 1024)
+const bootstrapWindowsOS = "Microsoft Windows Server 2025"
+
+// Deliberately make an inappropriate Linux read uncertain. The repair must
+// avoid sending it, not reinterpret a 500 as safe completion or clear its fence.
+type windowsBootstrapFixture struct {
+ t *testing.T
+ poll atomic.Int32
+ mu sync.Mutex
+ commands []string
+ osName string
+ uncertain string
+ locked bool
+ linuxAvailable uint64
+ fsEntered chan struct{}
+ fsRelease chan struct{}
+ fsOnce sync.Once
+}
+
+func newWindowsBootstrapFixture(t *testing.T) (*windowsBootstrapFixture, *httptest.Server) {
+ t.Helper()
+ f := &windowsBootstrapFixture{t: t, osName: bootstrapWindowsOS, linuxAvailable: 6 * bootstrapMiB}
+ f.poll.Store(1)
+ s := httptest.NewServer(http.HandlerFunc(f.serve))
+ t.Cleanup(s.Close)
+ return f, s
+}
+
+func (f *windowsBootstrapFixture) trace() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]string(nil), f.commands...)
+}
+
+func (f *windowsBootstrapFixture) resetTrace() {
+ f.mu.Lock()
+ f.commands = nil
+ f.mu.Unlock()
+}
+
+func (f *windowsBootstrapFixture) serve(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
+ f.t.Error("bootstrap lost configured fixture authentication")
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ f.mu.Lock()
+ osName, uncertain, locked, available := f.osName, f.uncertain, f.locked, f.linuxAvailable
+ f.mu.Unlock()
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[]}`)
+ return
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ if locked {
+ fmt.Fprint(w, `{"data":{"lock":"backup","ostype":"win11"}}`)
+ } else {
+ // A configured OS hint is not the classification used by the poller.
+ fmt.Fprint(w, `{"data":{"ostype":"win11"}}`)
+ }
+ return
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d}}`, 8*bootstrapMiB, uint64(2+f.poll.Load())*bootstrapMiB)
+ return
+ }
+ parts := strings.Split(r.URL.Path, "/")
+ cmd := parts[len(parts)-1]
+ f.mu.Lock()
+ f.commands = append(f.commands, cmd)
+ f.mu.Unlock()
+ if cmd == "get-fsinfo" && f.fsEntered != nil {
+ f.fsOnce.Do(func() { close(f.fsEntered) })
+ <-f.fsRelease
+ }
+ if cmd == uncertain || cmd == "file-read" && osName == bootstrapWindowsOS {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, "fixture command completion is unknown")
+ return
+ }
+ switch cmd {
+ case "get-fsinfo":
+ cUsed, dUsed := uint64(400)*bootstrapMiB, uint64(100)*bootstrapMiB
+ if f.poll.Load() == 2 {
+ cUsed, dUsed = 0, 0
+ }
+ fmt.Fprintf(w, `{"data":{"result":[{"mountpoint":"C:\\","type":"ntfs","total-bytes":%d,"used-bytes":%d},{"mountpoint":"D:\\","type":"ntfs","total-bytes":%d,"used-bytes":%d}]}}`, 1500*bootstrapMiB, cUsed, 500*bootstrapMiB, dUsed)
+ case "network-get-interfaces":
+ fmt.Fprint(w, `{"data":{"result":[{"name":"Ethernet","ip-addresses":[{"ip-address":"192.0.2.10","ip-address-type":"ipv4","prefix":24}]}]}}`)
+ case "get-osinfo":
+ if osName == "" {
+ fmt.Fprint(w, `{"data":{}}`)
+ } else {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]string{"name": osName}})
+ }
+ case "info":
+ fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)
+ case "file-read":
+ if r.URL.Query().Get("file") != "/proc/meminfo" {
+ f.t.Error("bootstrap requested an unexpected guest file")
+ }
+ content := fmt.Sprintf("MemTotal: 8192 kB\nMemAvailable: %d kB\nMemFree: 2048 kB\n", available/1024)
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]interface{}{"content": content, "truncated": false}})
+ default:
+ f.t.Errorf("unexpected bootstrap request %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+}
+
+func windowsBootstrapClient(t *testing.T, host, kind string) PVEClientInterface {
+ t.Helper()
+ cfg := proxmox.ClientConfig{Host: host, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: windowsPollingHangGuard}
+ if kind == "cluster-client" {
+ return proxmox.NewClusterClient("bootstrap", cfg, []string{host}, nil)
+ }
+ c, err := proxmox.NewClient(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+func windowsBootstrapMonitor(t *testing.T) *Monitor {
+ t.Helper()
+ m := guestHistoryObservationMonitor(t)
+ m.guestAgentFSInfoTimeout, m.guestAgentNetworkTimeout = windowsPollingHangGuard, windowsPollingHangGuard
+ m.guestAgentOSInfoTimeout, m.guestAgentVersionTimeout = windowsPollingHangGuard, windowsPollingHangGuard
+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
+ t.Cleanup(m.alertManager.Stop)
+ m.config, m.state, m.rateTracker = &config.Config{}, models.NewState(), NewRateTracker()
+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
+ return m
+}
+
+func windowsBootstrapPoll(t *testing.T, m *Monitor, client PVEClientInterface, collector, instance, node string, vmid, poll int) models.VM {
+ t.Helper()
+ id := makeGuestID(instance, node, vmid)
+ cycleStart := time.Now()
+ previous := m.previousGuestContextForInstance(instance).vmsByID[id]
+ res := proxmox.ClusterResource{Type: "qemu", Node: node, Name: "guest", VMID: vmid, Status: "running", MaxMem: 8 * bootstrapMiB, MaxDisk: 2000 * bootstrapMiB, CPU: float64(poll) / 10}
+ var vm models.VM
+ if collector == "node" {
+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), instance, node, []proxmox.VM{{VMID: vmid, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, client, map[string]models.VM{id: previous}, nil)
+ if len(vms) != 1 {
+ t.Errorf("node bootstrap lost guest: %d records", len(vms))
+ return models.VM{}
+ }
+ vm = vms[0]
+ } else {
+ var ok bool
+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), instance, res, client, id, nil, &previous)
+ if !ok {
+ t.Error("cluster bootstrap lost guest")
+ return models.VM{}
+ }
+ }
+ m.state.UpdateVMs([]models.VM{vm})
+ m.updateResourceStore(m.currentStateWithScope())
+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
+ view := m.currentModeReadState().VMs()[0]
+ if view.ID() != id || view.DiskUsed() != vm.Disk.Used || view.DiskStatusReason() != vm.DiskStatusReason || view.OSName() != vm.OSName {
+ t.Error("bootstrap changed at the canonical read boundary")
+ }
+ front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())
+ wire, err := json.Marshal(front.Resources)
+ if err != nil || strings.Contains(string(wire), "windowsGuest") || strings.Contains(string(wire), "osInfoObservedAt") {
+ t.Error("private OS admission evidence leaked into bootstrap JSON")
+ }
+ var served []struct{ Proxmox unifiedresources.ProxmoxData }
+ if err := json.Unmarshal(wire, &served); err != nil || len(served) != 1 || served[0].Proxmox.OSName != vm.OSName || served[0].Proxmox.GuestAgentStatus != vm.GuestAgentStatus || served[0].Proxmox.DiskStatusReason != vm.DiskStatusReason {
+ t.Error("served bootstrap reading lost OS or status")
+ }
+ return vm
+}
+
+func checkWindowsBootstrapReading(t *testing.T, vm models.VM, poll int) {
+ t.Helper()
+ wantDisk := int64(500 * bootstrapMiB)
+ if poll == 2 {
+ wantDisk = 0
+ }
+ if vm.OSName != bootstrapWindowsOS || vm.Disk.Total != int64(2000*bootstrapMiB) || vm.Disk.Used != wantDisk || len(vm.Disks) != 2 || vm.GuestAgentStatus != "available" || vm.DiskStatusReason != "" || vm.Memory.Used != int64(2+poll)*int64(bootstrapMiB) || vm.Memory.Observation.Source != "status-mem" || vm.Memory.Observation.State != "current" || vm.CPU != float64(poll)/10 {
+ t.Errorf("bootstrap poll %d: OS=%q disk=%d/%d rows=%d guest=%q reason=%q memory=%d/%s/%s CPU=%v", poll, vm.OSName, vm.Disk.Used, vm.Disk.Total, len(vm.Disks), vm.GuestAgentStatus, vm.DiskStatusReason, vm.Memory.Used, vm.Memory.Observation.Source, vm.Memory.Observation.State, vm.CPU)
+ }
+}
+
+func checkBootstrapHistory(t *testing.T, m *Monitor, id, metric string, want []float64) {
+ t.Helper()
+ var stored, memory []float64
+ for _, p := range guestHistoryStoredPoints(t, m, "vm", id, metric) {
+ stored = append(stored, p.Value)
+ }
+ for _, p := range m.metricsHistory.GetGuestMetrics(id, metric, time.Hour) {
+ memory = append(memory, p.Value)
+ }
+ if !reflect.DeepEqual(stored, want) || !reflect.DeepEqual(memory, want) {
+ t.Errorf("bootstrap %s History stored=%v memory=%v, want %v", metric, stored, memory, want)
+ }
+}
+
+func testGuestWindowsBootstrapPolling(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, collector := range []string{"cluster", "node"} {
+ for _, state := range []string{"cold", "expired"} {
+ t.Run(kind+"/"+collector+"/"+state, func(t *testing.T) {
+ f, s := newWindowsBootstrapFixture(t)
+ client := windowsBootstrapClient(t, s.URL, kind)
+ m := windowsBootstrapMonitor(t)
+ key := guestMetadataCacheKey("bootstrap", "node", 105)
+ if len(m.guestMetadataCache) != 0 || len(m.vmAgentMemCache) != 0 {
+ t.Fatal("fresh monitor was not cold")
+ }
+ if state == "expired" {
+ // Only this expiry control learns OS ahead of the poll. Its
+ // actual receipt is then aged; no Windows strings are seeded,
+ // and no command uncertainty fence is shortened.
+ _, _, name, _, _, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "bootstrap", "node", "guest", 105, &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, false)
+ if name != bootstrapWindowsOS || deferred {
+ t.Fatal("expiry control did not obtain its real OS receipt")
+ }
+ aged := time.Now().Add(-2 * vmAgentMemCleanupMaxAge)
+ entry := m.guestMetadataCache[key]
+ entry.fetchedAt, entry.osInfoObservedAt = aged, aged
+ m.guestMetadataCache[key] = entry
+ m.guestMetadataLimiter[key] = aged
+ f.resetTrace()
+ }
+ before := time.Now()
+ first := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)
+ checkWindowsBootstrapReading(t, first, 1)
+ osReceipt := m.guestMetadataCache[key]
+ if !osReceipt.windowsGuest || osReceipt.osInfoObservedAt.Before(before) || osReceipt.osInfoObservedAt.After(time.Now()) {
+ t.Error("ordinary first poll did not establish fresh actual OS evidence")
+ }
+ // Separate second-resolution History keys. This is not a QGA
+ // cooldown wait; the successful path creates no such cooldown.
+ time.Sleep(1100 * time.Millisecond)
+ f.poll.Store(2)
+ second := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 2)
+ checkWindowsBootstrapReading(t, second, 2)
+ wantTrace := []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info", "get-fsinfo"}
+ if got := f.trace(); !reflect.DeepEqual(got, wantTrace) {
+ t.Errorf("ordinary %s bootstrap wire trace=%v, want %v", state, got, wantTrace)
+ }
+ if !reflect.DeepEqual(m.guestMetadataCache[key], osReceipt) || len(m.vmAgentMemCache) != 0 {
+ t.Error("second poll renewed OS evidence or invented a Linux memory observation")
+ }
+ id := makeGuestID("bootstrap", "node", 105)
+ checkBootstrapHistory(t, m, id, "disk", []float64{25, 0})
+ checkBootstrapHistory(t, m, id, "memory", []float64{37.5, 50})
+ checkBootstrapHistory(t, m, id, "cpu", []float64{10, 20})
+ })
+ }
+ }
+ }
+}
+
+const bootstrapChildEnv = "PULSE_TEST_WINDOWS_BOOTSTRAP_CHILD"
+
+type windowsBootstrapChild struct {
+ Host, Kind, Collector, Output string
+ Poll int
+}
+
+// Executed only by the parent contract using the same compiled test binary.
+// Each invocation starts a new OS process, client, monitor and empty caches.
+// This is source-level restart acceptance against loopback, not native PVE/Pro.
+func TestGuestWindowsBootstrapProcessWorker(t *testing.T) {
+ raw := os.Getenv(bootstrapChildEnv)
+ if raw == "" {
+ return
+ }
+ var child windowsBootstrapChild
+ if err := json.Unmarshal([]byte(raw), &child); err != nil {
+ t.Fatal(err)
+ }
+ m := windowsBootstrapMonitor(t)
+ if len(m.guestMetadataCache) != 0 || len(m.vmAgentMemCache) != 0 {
+ t.Fatal("new process inherited guest caches")
+ }
+ before := time.Now()
+ vm := windowsBootstrapPoll(t, m, windowsBootstrapClient(t, child.Host, child.Kind), child.Collector, "bootstrap", "node", 105, child.Poll)
+ checkWindowsBootstrapReading(t, vm, child.Poll)
+ entry := m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)]
+ if !entry.windowsGuest || entry.osInfoObservedAt.Before(before) || len(m.vmAgentMemCache) != 0 {
+ t.Error("new process did not learn OS before optional Linux memory")
+ }
+ id := makeGuestID("bootstrap", "node", 105)
+ wantDisk := float64(25)
+ if child.Poll == 2 {
+ wantDisk = 0
+ }
+ checkBootstrapHistory(t, m, id, "disk", []float64{wantDisk})
+ checkBootstrapHistory(t, m, id, "memory", []float64{float64(2+child.Poll) / 8 * 100})
+ checkBootstrapHistory(t, m, id, "cpu", []float64{float64(child.Poll) * 10})
+ data, err := json.Marshal(struct {
+ PID int
+ VM models.VM
+ }{os.Getpid(), vm})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(child.Output, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func testGuestWindowsBootstrapProcessRestart(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, collector := range []string{"cluster", "node"} {
+ t.Run(kind+"/"+collector, func(t *testing.T) {
+ f, s := newWindowsBootstrapFixture(t)
+ var firstPID int
+ for poll := 1; poll <= 2; poll++ {
+ f.poll.Store(int32(poll))
+ f.resetTrace()
+ out := filepath.Join(t.TempDir(), "child.json")
+ cfg, err := json.Marshal(windowsBootstrapChild{s.URL, kind, collector, out, poll})
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), windowsPollingHangGuard)
+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestGuestWindowsBootstrapProcessWorker$", "-test.count=1")
+ cmd.Env = append(os.Environ(), bootstrapChildEnv+"="+string(cfg))
+ output, err := cmd.CombinedOutput()
+ cancel()
+ if err != nil {
+ t.Errorf("fresh process %d failed: %v\n%s", poll, err, output)
+ }
+ var result struct {
+ PID int
+ VM models.VM
+ }
+ data, err := os.ReadFile(out)
+ if err != nil || json.Unmarshal(data, &result) != nil {
+ t.Fatal("fresh-process result missing or invalid")
+ }
+ checkWindowsBootstrapReading(t, result.VM, poll)
+ if result.PID == os.Getpid() || poll == 2 && result.PID == firstPID {
+ t.Fatal("restart reused the old process")
+ }
+ firstPID = result.PID
+ wantTrace := []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}
+ if got := f.trace(); !reflect.DeepEqual(got, wantTrace) {
+ t.Errorf("fresh process %d wire trace=%v, want %v", poll, got, wantTrace)
+ }
+ t.Logf("process %d PID=%d OS=%q disk=%d memory-source=%s", poll, result.PID, result.VM.OSName, result.VM.Disk.Used, result.VM.Memory.Observation.Source)
+ }
+ })
+ }
+ }
+}