Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Expose Windows guest bootstrap starvation after restart
pulse-triage[bot]affddfd22824
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.goindex 3e959f0eea..29dae0cd86 100644--- a/internal/monitoring/canonical_guardrails_test.go+++ b/internal/monitoring/canonical_guardrails_test.go@@ -3251,6 +3251,11 @@ func TestGuestWindowsMeminfoPollingContract(t *testing.T) {testGuestWindowsMeminfoPolling(t)}+func TestGuestWindowsRestartPollingContract(t *testing.T) {+ t.Run("cold-and-expired", testGuestWindowsBootstrapPolling)+ t.Run("process-restart", testGuestWindowsBootstrapProcessRestart)+}+func TestGuestMemoryObservationContract(t *testing.T) {t.Run("poll-to-served-observation", testGuestMemoryObservationLifecycle)t.Run("identity-and-origin-boundaries", testGuestMemoryObservationKeepsOriginsSeparate)diff --git a/internal/monitoring/guest_windows_bootstrap_test.go b/internal/monitoring/guest_windows_bootstrap_test.gonew file mode 100644index 0000000000..2e5b7eea81--- /dev/null+++ b/internal/monitoring/guest_windows_bootstrap_test.go@@ -0,0 +1,377 @@+package monitoring++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "os/exec"+ "path/filepath"+ "reflect"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++const bootstrapMiB = uint64(1024 * 1024)+const bootstrapWindowsOS = "Microsoft Windows Server 2025"++// Deliberately make an inappropriate Linux read uncertain. The repair must+// avoid sending it, not reinterpret a 500 as safe completion or clear its fence.+type windowsBootstrapFixture struct {+ t *testing.T+ poll atomic.Int32+ mu sync.Mutex+ commands []string+ osName string+ uncertain string+ locked bool+ linuxAvailable uint64+ fsEntered chan struct{}+ fsRelease chan struct{}+ fsOnce sync.Once+}++func newWindowsBootstrapFixture(t *testing.T) (*windowsBootstrapFixture, *httptest.Server) {+ t.Helper()+ f := &windowsBootstrapFixture{t: t, osName: bootstrapWindowsOS, linuxAvailable: 6 * bootstrapMiB}+ f.poll.Store(1)+ s := httptest.NewServer(http.HandlerFunc(f.serve))+ t.Cleanup(s.Close)+ return f, s+}++func (f *windowsBootstrapFixture) trace() []string {+ f.mu.Lock()+ defer f.mu.Unlock()+ return append([]string(nil), f.commands...)+}++func (f *windowsBootstrapFixture) resetTrace() {+ f.mu.Lock()+ f.commands = nil+ f.mu.Unlock()+}++func (f *windowsBootstrapFixture) serve(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {+ f.t.Error("bootstrap lost configured fixture authentication")+ w.WriteHeader(http.StatusForbidden)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ f.mu.Lock()+ osName, uncertain, locked, available := f.osName, f.uncertain, f.locked, f.linuxAvailable+ f.mu.Unlock()+ switch {+ case strings.HasSuffix(r.URL.Path, "/nodes"):+ fmt.Fprint(w, `{"data":[]}`)+ return+ case strings.HasSuffix(r.URL.Path, "/config"):+ if locked {+ fmt.Fprint(w, `{"data":{"lock":"backup","ostype":"win11"}}`)+ } else {+ // A configured OS hint is not the classification used by the poller.+ fmt.Fprint(w, `{"data":{"ostype":"win11"}}`)+ }+ return+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d}}`, 8*bootstrapMiB, uint64(2+f.poll.Load())*bootstrapMiB)+ return+ }+ parts := strings.Split(r.URL.Path, "/")+ cmd := parts[len(parts)-1]+ f.mu.Lock()+ f.commands = append(f.commands, cmd)+ f.mu.Unlock()+ if cmd == "get-fsinfo" && f.fsEntered != nil {+ f.fsOnce.Do(func() { close(f.fsEntered) })+ <-f.fsRelease+ }+ if cmd == uncertain || cmd == "file-read" && osName == bootstrapWindowsOS {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, "fixture command completion is unknown")+ return+ }+ switch cmd {+ case "get-fsinfo":+ cUsed, dUsed := uint64(400)*bootstrapMiB, uint64(100)*bootstrapMiB+ if f.poll.Load() == 2 {+ cUsed, dUsed = 0, 0+ }+ fmt.Fprintf(w, `{"data":{"result":[{"mountpoint":"C:\\","type":"ntfs","total-bytes":%d,"used-bytes":%d},{"mountpoint":"D:\\","type":"ntfs","total-bytes":%d,"used-bytes":%d}]}}`, 1500*bootstrapMiB, cUsed, 500*bootstrapMiB, dUsed)+ case "network-get-interfaces":+ fmt.Fprint(w, `{"data":{"result":[{"name":"Ethernet","ip-addresses":[{"ip-address":"192.0.2.10","ip-address-type":"ipv4","prefix":24}]}]}}`)+ case "get-osinfo":+ if osName == "" {+ fmt.Fprint(w, `{"data":{}}`)+ } else {+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]string{"name": osName}})+ }+ case "info":+ fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)+ case "file-read":+ if r.URL.Query().Get("file") != "/proc/meminfo" {+ f.t.Error("bootstrap requested an unexpected guest file")+ }+ content := fmt.Sprintf("MemTotal: 8192 kB\nMemAvailable: %d kB\nMemFree: 2048 kB\n", available/1024)+ _ = json.NewEncoder(w).Encode(map[string]interface{}{"data": map[string]interface{}{"content": content, "truncated": false}})+ default:+ f.t.Errorf("unexpected bootstrap request %s", r.URL.Path)+ http.NotFound(w, r)+ }+}++func windowsBootstrapClient(t *testing.T, host, kind string) PVEClientInterface {+ t.Helper()+ cfg := proxmox.ClientConfig{Host: host, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: windowsPollingHangGuard}+ if kind == "cluster-client" {+ return proxmox.NewClusterClient("bootstrap", cfg, []string{host}, nil)+ }+ c, err := proxmox.NewClient(cfg)+ if err != nil {+ t.Fatal(err)+ }+ return c+}++func windowsBootstrapMonitor(t *testing.T) *Monitor {+ t.Helper()+ m := guestHistoryObservationMonitor(t)+ m.guestAgentFSInfoTimeout, m.guestAgentNetworkTimeout = windowsPollingHangGuard, windowsPollingHangGuard+ m.guestAgentOSInfoTimeout, m.guestAgentVersionTimeout = windowsPollingHangGuard, windowsPollingHangGuard+ m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())+ t.Cleanup(m.alertManager.Stop)+ m.config, m.state, m.rateTracker = &config.Config{}, models.NewState(), NewRateTracker()+ m.resourceStore = unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))+ return m+}++func windowsBootstrapPoll(t *testing.T, m *Monitor, client PVEClientInterface, collector, instance, node string, vmid, poll int) models.VM {+ t.Helper()+ id := makeGuestID(instance, node, vmid)+ cycleStart := time.Now()+ previous := m.previousGuestContextForInstance(instance).vmsByID[id]+ res := proxmox.ClusterResource{Type: "qemu", Node: node, Name: "guest", VMID: vmid, Status: "running", MaxMem: 8 * bootstrapMiB, MaxDisk: 2000 * bootstrapMiB, CPU: float64(poll) / 10}+ var vm models.VM+ if collector == "node" {+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), instance, node, []proxmox.VM{{VMID: vmid, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, client, map[string]models.VM{id: previous}, nil)+ if len(vms) != 1 {+ t.Errorf("node bootstrap lost guest: %d records", len(vms))+ return models.VM{}+ }+ vm = vms[0]+ } else {+ var ok bool+ vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), instance, res, client, id, nil, &previous)+ if !ok {+ t.Error("cluster bootstrap lost guest")+ return models.VM{}+ }+ }+ m.state.UpdateVMs([]models.VM{vm})+ m.updateResourceStore(m.currentStateWithScope())+ m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)+ view := m.currentModeReadState().VMs()[0]+ if view.ID() != id || view.DiskUsed() != vm.Disk.Used || view.DiskStatusReason() != vm.DiskStatusReason || view.OSName() != vm.OSName {+ t.Error("bootstrap changed at the canonical read boundary")+ }+ front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())+ wire, err := json.Marshal(front.Resources)+ if err != nil || strings.Contains(string(wire), "windowsGuest") || strings.Contains(string(wire), "osInfoObservedAt") {+ t.Error("private OS admission evidence leaked into bootstrap JSON")+ }+ var served []struct{ Proxmox unifiedresources.ProxmoxData }+ if err := json.Unmarshal(wire, &served); err != nil || len(served) != 1 || served[0].Proxmox.OSName != vm.OSName || served[0].Proxmox.GuestAgentStatus != vm.GuestAgentStatus || served[0].Proxmox.DiskStatusReason != vm.DiskStatusReason {+ t.Error("served bootstrap reading lost OS or status")+ }+ return vm+}++func checkWindowsBootstrapReading(t *testing.T, vm models.VM, poll int) {+ t.Helper()+ wantDisk := int64(500 * bootstrapMiB)+ if poll == 2 {+ wantDisk = 0+ }+ if vm.OSName != bootstrapWindowsOS || vm.Disk.Total != int64(2000*bootstrapMiB) || vm.Disk.Used != wantDisk || len(vm.Disks) != 2 || vm.GuestAgentStatus != "available" || vm.DiskStatusReason != "" || vm.Memory.Used != int64(2+poll)*int64(bootstrapMiB) || vm.Memory.Observation.Source != "status-mem" || vm.Memory.Observation.State != "current" || vm.CPU != float64(poll)/10 {+ t.Errorf("bootstrap poll %d: OS=%q disk=%d/%d rows=%d guest=%q reason=%q memory=%d/%s/%s CPU=%v", poll, vm.OSName, vm.Disk.Used, vm.Disk.Total, len(vm.Disks), vm.GuestAgentStatus, vm.DiskStatusReason, vm.Memory.Used, vm.Memory.Observation.Source, vm.Memory.Observation.State, vm.CPU)+ }+}++func checkBootstrapHistory(t *testing.T, m *Monitor, id, metric string, want []float64) {+ t.Helper()+ var stored, memory []float64+ for _, p := range guestHistoryStoredPoints(t, m, "vm", id, metric) {+ stored = append(stored, p.Value)+ }+ for _, p := range m.metricsHistory.GetGuestMetrics(id, metric, time.Hour) {+ memory = append(memory, p.Value)+ }+ if !reflect.DeepEqual(stored, want) || !reflect.DeepEqual(memory, want) {+ t.Errorf("bootstrap %s History stored=%v memory=%v, want %v", metric, stored, memory, want)+ }+}++func testGuestWindowsBootstrapPolling(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, collector := range []string{"cluster", "node"} {+ for _, state := range []string{"cold", "expired"} {+ t.Run(kind+"/"+collector+"/"+state, func(t *testing.T) {+ f, s := newWindowsBootstrapFixture(t)+ client := windowsBootstrapClient(t, s.URL, kind)+ m := windowsBootstrapMonitor(t)+ key := guestMetadataCacheKey("bootstrap", "node", 105)+ if len(m.guestMetadataCache) != 0 || len(m.vmAgentMemCache) != 0 {+ t.Fatal("fresh monitor was not cold")+ }+ if state == "expired" {+ // Only this expiry control learns OS ahead of the poll. Its+ // actual receipt is then aged; no Windows strings are seeded,+ // and no command uncertainty fence is shortened.+ _, _, name, _, _, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "bootstrap", "node", "guest", 105, &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}, false)+ if name != bootstrapWindowsOS || deferred {+ t.Fatal("expiry control did not obtain its real OS receipt")+ }+ aged := time.Now().Add(-2 * vmAgentMemCleanupMaxAge)+ entry := m.guestMetadataCache[key]+ entry.fetchedAt, entry.osInfoObservedAt = aged, aged+ m.guestMetadataCache[key] = entry+ m.guestMetadataLimiter[key] = aged+ f.resetTrace()+ }+ before := time.Now()+ first := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 1)+ checkWindowsBootstrapReading(t, first, 1)+ osReceipt := m.guestMetadataCache[key]+ if !osReceipt.windowsGuest || osReceipt.osInfoObservedAt.Before(before) || osReceipt.osInfoObservedAt.After(time.Now()) {+ t.Error("ordinary first poll did not establish fresh actual OS evidence")+ }+ // Separate second-resolution History keys. This is not a QGA+ // cooldown wait; the successful path creates no such cooldown.+ time.Sleep(1100 * time.Millisecond)+ f.poll.Store(2)+ second := windowsBootstrapPoll(t, m, client, collector, "bootstrap", "node", 105, 2)+ checkWindowsBootstrapReading(t, second, 2)+ wantTrace := []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info", "get-fsinfo"}+ if got := f.trace(); !reflect.DeepEqual(got, wantTrace) {+ t.Errorf("ordinary %s bootstrap wire trace=%v, want %v", state, got, wantTrace)+ }+ if !reflect.DeepEqual(m.guestMetadataCache[key], osReceipt) || len(m.vmAgentMemCache) != 0 {+ t.Error("second poll renewed OS evidence or invented a Linux memory observation")+ }+ id := makeGuestID("bootstrap", "node", 105)+ checkBootstrapHistory(t, m, id, "disk", []float64{25, 0})+ checkBootstrapHistory(t, m, id, "memory", []float64{37.5, 50})+ checkBootstrapHistory(t, m, id, "cpu", []float64{10, 20})+ })+ }+ }+ }+}++const bootstrapChildEnv = "PULSE_TEST_WINDOWS_BOOTSTRAP_CHILD"++type windowsBootstrapChild struct {+ Host, Kind, Collector, Output string+ Poll int+}++// Executed only by the parent contract using the same compiled test binary.+// Each invocation starts a new OS process, client, monitor and empty caches.+// This is source-level restart acceptance against loopback, not native PVE/Pro.+func TestGuestWindowsBootstrapProcessWorker(t *testing.T) {+ raw := os.Getenv(bootstrapChildEnv)+ if raw == "" {+ return+ }+ var child windowsBootstrapChild+ if err := json.Unmarshal([]byte(raw), &child); err != nil {+ t.Fatal(err)+ }+ m := windowsBootstrapMonitor(t)+ if len(m.guestMetadataCache) != 0 || len(m.vmAgentMemCache) != 0 {+ t.Fatal("new process inherited guest caches")+ }+ before := time.Now()+ vm := windowsBootstrapPoll(t, m, windowsBootstrapClient(t, child.Host, child.Kind), child.Collector, "bootstrap", "node", 105, child.Poll)+ checkWindowsBootstrapReading(t, vm, child.Poll)+ entry := m.guestMetadataCache[guestMetadataCacheKey("bootstrap", "node", 105)]+ if !entry.windowsGuest || entry.osInfoObservedAt.Before(before) || len(m.vmAgentMemCache) != 0 {+ t.Error("new process did not learn OS before optional Linux memory")+ }+ id := makeGuestID("bootstrap", "node", 105)+ wantDisk := float64(25)+ if child.Poll == 2 {+ wantDisk = 0+ }+ checkBootstrapHistory(t, m, id, "disk", []float64{wantDisk})+ checkBootstrapHistory(t, m, id, "memory", []float64{float64(2+child.Poll) / 8 * 100})+ checkBootstrapHistory(t, m, id, "cpu", []float64{float64(child.Poll) * 10})+ data, err := json.Marshal(struct {+ PID int+ VM models.VM+ }{os.Getpid(), vm})+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(child.Output, data, 0600); err != nil {+ t.Fatal(err)+ }+}++func testGuestWindowsBootstrapProcessRestart(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, collector := range []string{"cluster", "node"} {+ t.Run(kind+"/"+collector, func(t *testing.T) {+ f, s := newWindowsBootstrapFixture(t)+ var firstPID int+ for poll := 1; poll <= 2; poll++ {+ f.poll.Store(int32(poll))+ f.resetTrace()+ out := filepath.Join(t.TempDir(), "child.json")+ cfg, err := json.Marshal(windowsBootstrapChild{s.URL, kind, collector, out, poll})+ if err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), windowsPollingHangGuard)+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestGuestWindowsBootstrapProcessWorker$", "-test.count=1")+ cmd.Env = append(os.Environ(), bootstrapChildEnv+"="+string(cfg))+ output, err := cmd.CombinedOutput()+ cancel()+ if err != nil {+ t.Errorf("fresh process %d failed: %v\n%s", poll, err, output)+ }+ var result struct {+ PID int+ VM models.VM+ }+ data, err := os.ReadFile(out)+ if err != nil || json.Unmarshal(data, &result) != nil {+ t.Fatal("fresh-process result missing or invalid")+ }+ checkWindowsBootstrapReading(t, result.VM, poll)+ if result.PID == os.Getpid() || poll == 2 && result.PID == firstPID {+ t.Fatal("restart reused the old process")+ }+ firstPID = result.PID+ wantTrace := []string{"get-fsinfo", "network-get-interfaces", "get-osinfo", "info"}+ if got := f.trace(); !reflect.DeepEqual(got, wantTrace) {+ t.Errorf("fresh process %d wire trace=%v, want %v", poll, got, wantTrace)+ }+ t.Logf("process %d PID=%d OS=%q disk=%d memory-source=%s", poll, result.PID, result.VM.OSName, result.VM.Disk.Used, result.VM.Memory.Observation.Source)+ }+ })+ }+ }+}