Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
const (
// Bar one deliberately late reply, the fixture answers every request at
// once, so the deadlines it sets are hang guards, not latency budgets. A
// tight one makes the contract depend on the machine running it: the
// guest-agent guard defers a poll whose lock check fails (lock-unverified)
// and fences the guest for a minute after a command times out or its body
// is cut short (agent-timeout, agent-response-incomplete, then
// agent-cooldown). That is right for a real guest and wrong for this
// contract. The proxmox transport still limits dialing and the wait for
// response headers to ten seconds, and a cluster client probes a new
// endpoint within three; the fixture cannot set either.
windowsPollingHangGuard = 15 * time.Minute
// A reply later than the one-second client budget this fixture used to set.
windowsPollingSlowReply = 1200 * time.Millisecond
)
// Synthetic, ordinary QGA-only polling after a successful Windows OS reply.
// The deliberately uncertain Linux read is an adverse fixture, not a replay of
// issue #2619 or a claim about the response/cause on its native installation.
func testGuestWindowsMeminfoPolling(t *testing.T) {
for _, kind := range []string{"client", "cluster-client"} {
for _, path := range []string{"cluster", "node"} {
t.Run(kind+"/"+path, func(t *testing.T) {
const mib = uint64(1024 * 1024)
var poll, fileReads, fsReads, configReads atomic.Int32
var locked, slowLockCheck atomic.Bool
// The four subtests share one client budget, so one late reply
// holds it; more would only lengthen the run.
lateLockCheck := kind == "client" && path == "cluster"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
t.Error("lost configured authentication")
w.WriteHeader(http.StatusForbidden)
return
}
switch {
case strings.HasSuffix(r.URL.Path, "/nodes"):
fmt.Fprint(w, `{"data":[]}`)
case strings.HasSuffix(r.URL.Path, "/config"):
configReads.Add(1)
// One lock check later than the old client budget, in an
// otherwise ordinary poll: lateness alone must not defer it.
if lateLockCheck && poll.Load() == 2 && slowLockCheck.CompareAndSwap(false, true) {
time.Sleep(windowsPollingSlowReply)
}
if locked.Load() {
fmt.Fprint(w, `{"data":{"lock":"backup"}}`)
} else {
fmt.Fprint(w, `{"data":{}}`)
}
case strings.HasSuffix(r.URL.Path, "/status/current"):
fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d}}`, 8*mib, uint64(2+poll.Load())*mib)
case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):
fsReads.Add(1)
used := uint64(200+100*poll.Load()) * mib
if poll.Load() == 3 {
used = 0
}
fmt.Fprintf(w, `{"data":{"result":[{"mountpoint":"C:\\","type":"ntfs","total-bytes":%d,"used-bytes":%d}]}}`, 1000*mib, used)
case strings.HasSuffix(r.URL.Path, "/agent/file-read"):
fileReads.Add(1)
if r.URL.Query().Get("file") != "/proc/meminfo" {
t.Error("unexpected guest file")
}
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, "optional Linux file-read failed")
case strings.HasSuffix(r.URL.Path, "/agent/network-get-interfaces"):
fmt.Fprint(w, `{"data":{"result":[{"name":"Ethernet","ip-addresses":[{"ip-address":"192.0.2.10","ip-address-type":"ipv4","prefix":24}]}]}}`)
case strings.HasSuffix(r.URL.Path, "/agent/get-osinfo"):
fmt.Fprint(w, `{"data":{"name":"Microsoft Windows Server 2022","id":"mswindows"}}`)
case strings.HasSuffix(r.URL.Path, "/agent/info"):
fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
cfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: windowsPollingHangGuard}
var client PVEClientInterface
if kind == "cluster-client" {
client = proxmox.NewClusterClient("windows", cfg, []string{server.URL}, nil)
} else {
var err error
client, err = proxmox.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
}
m := guestHistoryObservationMonitor(t)
m.guestAgentFSInfoTimeout, m.guestAgentNetworkTimeout = windowsPollingHangGuard, windowsPollingHangGuard
m.guestAgentOSInfoTimeout, m.guestAgentVersionTimeout = windowsPollingHangGuard, windowsPollingHangGuard
m.alertManager = alerts.NewManagerWithDataDir(t.TempDir(), alerts.WithoutPersistedAlertRestore())
t.Cleanup(m.alertManager.Stop)
m.config, m.state, m.rateTracker = &config.Config{}, models.NewState(), NewRateTracker()
m.guestMetadataLimiter = make(map[string]time.Time)
registry := unifiedresources.NewRegistry(nil)
m.resourceStore = unifiedresources.NewMonitorAdapter(registry)
status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
// Learn from the real, unlocked OS endpoint, not seeded display
// strings or a configured Proxmox OS hint. No Pulse guest agent.
_, _, name, _, _, deferred := m.fetchGuestAgentMetadata(context.Background(), client, "windows", "node", "guest", 105, status, false)
if name != "Microsoft Windows Server 2022" || deferred || fileReads.Load() != 0 {
t.Fatal("normal OS observation absent")
}
metadataKey := guestMetadataCacheKey("windows", "node", 105)
originalMetadata := m.guestMetadataCache[metadataKey]
memoryKey := guestMemoryCacheKey("windows", "node", 105)
// The existing Windows failure backoff has expired: it cannot
// hide the exact parent's inappropriate read in this control.
originalMemory := agentMemCacheEntry{negative: true, fetchedAt: time.Now().Add(-2 * vmAgentMemNegativeTTL)}
m.vmAgentMemCache = map[string]agentMemCacheEntry{memoryKey: originalMemory}
id := makeGuestID("windows", "node", 105)
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, MaxDisk: 1000 * mib}
build := func() models.VM {
// The poll loop takes its cycle start before building, so the
// guest is always observed in the cycle it records. A window
// measured back from the record call would drop the sample
// whenever the state update and resource-store refresh in
// between take over a second.
cycleStart := time.Now()
previous := m.previousGuestContextForInstance("windows").vmsByID[id]
var vm models.VM
if path == "node" {
vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), "windows", "node", []proxmox.VM{{VMID: 105, Name: res.Name, Status: res.Status, MaxMem: res.MaxMem, MaxDisk: res.MaxDisk, CPU: res.CPU}}, client, map[string]models.VM{id: previous}, nil)
if len(vms) != 1 {
t.Fatal("node collector lost the guest")
}
vm = vms[0]
} else {
var ok bool
vm, _, _, _, _, ok = m.buildVMFromClusterResource(context.Background(), "windows", res, client, id, nil, &previous)
if !ok {
t.Fatal("cluster builder lost the guest")
}
}
m.state.UpdateVMs([]models.VM{vm})
m.updateResourceStore(m.currentStateWithScope())
m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
view := m.currentModeReadState().VMs()[0]
if view.DiskUsed() != vm.Disk.Used || view.DiskStatusReason() != vm.DiskStatusReason || view.OSName() != vm.OSName {
t.Error("accepted Windows poll changed at canonical read boundary")
}
front := m.buildBroadcastFrontendStateFromSnapshot(models.StateSnapshot{VMs: []models.VM{vm}}, m.mockModeFence.begin())
wire, err := json.Marshal(front.Resources)
if err != nil || strings.Contains(string(wire), "windowsGuest") || strings.Contains(string(wire), "osInfoObservedAt") {
t.Fatal("private OS admission evidence leaked into JSON")
}
var served []struct{ Proxmox unifiedresources.ProxmoxData }
if err := json.Unmarshal(wire, &served); err != nil || len(served) != 1 || served[0].Proxmox.GuestAgentStatus != vm.GuestAgentStatus || served[0].Proxmox.DiskStatusReason != vm.DiskStatusReason {
t.Fatal("served Windows reading lost its status")
}
return vm
}
var last models.VM
for i := int32(1); i <= 3; i++ {
if i > 1 {
// Persistent History uses second-resolution keys; separate
// ordinary observations rather than accepting overwritten points.
time.Sleep(1100 * time.Millisecond)
}
poll.Store(i)
res.CPU = float64(i) / 10
last = build()
wantUsed := int64(200+100*i) * int64(mib)
if i == 3 {
wantUsed = 0
}
if last.ID != id || last.Disk.Used != wantUsed || last.DiskStatusReason != "" || last.GuestAgentStatus != "available" || last.CPU != res.CPU || last.Memory.Used != int64(2+i)*int64(mib) || last.Memory.Observation.State != "current" {
t.Errorf("poll %d lost independent Windows readings: disk=%d reason=%q guest=%q memory=%d/%s cpu=%v", i, last.Disk.Used, last.DiskStatusReason, last.GuestAgentStatus, last.Memory.Used, last.Memory.Observation.State, last.CPU)
}
}
if lateLockCheck && !slowLockCheck.Load() {
t.Error("the late lock verification never reached the fixture")
}
if fileReads.Load() != 0 || fsReads.Load() != 3 || !reflect.DeepEqual(m.vmAgentMemCache[memoryKey], originalMemory) || !reflect.DeepEqual(m.guestMetadataCache[metadataKey], originalMetadata) {
t.Errorf("known Windows sent Linux work or renewed evidence: file=%d fs=%d", fileReads.Load(), fsReads.Load())
}
for metric, want := range map[string][]float64{"disk": {30, 40, 0}, "cpu": {10, 20, 30}, "memory": {37.5, 50, 62.5}} {
points := guestHistoryStoredPoints(t, m, "vm", id, metric)
var values []float64
for _, point := range points {
values = append(values, point.Value)
}
if !reflect.DeepEqual(values, want) || len(m.metricsHistory.GetGuestMetrics(id, metric, time.Hour)) != 3 {
t.Errorf("Windows %s History = %v, want %v", metric, values, want)
}
}
// OS evidence never bypasses a current operation lock.
beforeConfigs := configReads.Load()
locked.Store(true)
blocked := build()
if blocked.GuestAgentStatus != "deferred" || blocked.DiskStatusReason != "prev-vm-locked" || fsReads.Load() != 3 || fileReads.Load() != 0 || configReads.Load() <= beforeConfigs || blocked.Disk.Used != last.Disk.Used {
t.Error("Windows classification relaxed backup-lock admission")
}
locked.Store(false)
if resumed := build(); resumed.GuestAgentStatus != "available" || resumed.DiskStatusReason != "" || resumed.Disk.Used != 0 || fsReads.Load() != 4 || fileReads.Load() != 0 {
t.Error("normal unlocked Windows disk polling did not resume")
}
})
}
}
}