Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
// These are deliberately synthetic protocol outcomes, not captured responses
// from discussion #2538. Filesystem success is independent of optional Linux
// memory/metadata support, but an uncertain command still stops every later read.
func testGuestAgentOptionalReadOrdering(t *testing.T, withoutStatus bool) {
for _, clientKind := range []string{"client", "cluster"} {
for _, tc := range []struct {
name, failure, os, fs string
terminal bool
}{
{"windows-unsupported-memory", "file-read", "Microsoft Windows 11", "ntfs", true},
{"linux-unsupported-memory", "file-read", "Linux", "ext4", true},
{"android-unsupported-os", "get-osinfo", "Android", "ext4", true},
{"uncertain-memory", "file-read", "Linux", "ext4", false},
{"unknown-os-uncertain-memory", "file-read", "", "ext4", false},
{"uncertain-network", "network-get-interfaces", "Android", "ext4", false},
{"uncertain-os", "get-osinfo", "Android", "ext4", false},
{"uncertain-version", "info", "Microsoft Windows 11", "ntfs", false},
} {
if withoutStatus && tc.failure != "get-osinfo" {
continue // No-status enrichment does not issue a memory fallback.
}
t.Run(clientKind+"/"+tc.name, func(t *testing.T) {
const mib = uint64(1024 * 1024)
var poll atomic.Int32
var mu sync.Mutex
var commands []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
t.Error("lost configured authentication")
w.WriteHeader(http.StatusForbidden)
return
}
parts := strings.Split(r.URL.Path, "/")
switch {
case strings.HasSuffix(r.URL.Path, "/nodes"):
fmt.Fprint(w, `{"data":[]}`)
case strings.HasSuffix(r.URL.Path, "/config"):
fmt.Fprint(w, `{"data":{}}`)
case strings.HasSuffix(r.URL.Path, "/status/current"):
if withoutStatus {
w.WriteHeader(http.StatusForbidden)
return
}
fmt.Fprintf(w, `{"data":{"status":"running","agent":1,"maxmem":%d,"mem":%d}}`, 8*mib, 8*mib)
case strings.Contains(r.URL.Path, "/agent/"):
command := parts[len(parts)-1]
id, _ := strconv.Atoi(parts[len(parts)-3])
mu.Lock()
commands = append(commands, fmt.Sprintf("%d/%s", id, command))
mu.Unlock()
if id == 105 && command == tc.failure {
w.WriteHeader(http.StatusInternalServerError)
if tc.terminal {
fmt.Fprint(w, "unsupported command: guest-"+command)
} else {
// A complete but unexplained 500, even mentioning a
// missing Linux file, is NOT proof of completion.
fmt.Fprint(w, "optional guest read failed: /proc/meminfo unavailable")
}
return
}
switch command {
case "get-fsinfo":
json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"result": []any{map[string]any{"mountpoint": map[bool]string{true: `C:\`, false: "/"}[tc.fs == "ntfs"], "type": tc.fs, "total-bytes": 1000 * mib, "used-bytes": uint64(200+poll.Load()*100) * mib}}}})
case "file-read":
if r.URL.Query().Get("file") != "/proc/meminfo" {
t.Error("unexpected guest file")
}
json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": "MemTotal: 8192 kB\nMemAvailable: 5120 kB\n"}})
case "network-get-interfaces":
fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","hardware-address":"02:00:00:00:00:01","ip-addresses":[{"ip-address":"192.0.2.10","ip-address-type":"ipv4","prefix":24}]}]}}`)
case "get-osinfo":
json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"name": tc.os, "version": "fixture"}})
case "info":
fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)
default:
t.Error("unexpected guest command", command)
http.NotFound(w, r)
}
default:
http.NotFound(w, r)
}
}))
defer server.Close()
newClient := func() PVEClientInterface {
cfg := proxmox.ClientConfig{Host: server.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: guestPollFixtureHangGuard}
if clientKind == "cluster" {
return proxmox.NewClusterClient("optional", cfg, []string{server.URL}, nil)
}
client, err := proxmox.NewClient(cfg)
if err != nil {
t.Fatal(err)
}
return client
}
client := newClient()
m := guestHistoryObservationMonitor(t)
m.rateTracker = NewRateTracker()
m.guestMetadataLimiter = make(map[string]time.Time)
registry := unifiedresources.NewRegistry(nil)
m.resourceStore = unifiedresources.NewMonitorAdapter(registry)
res := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib, CPU: .1}
id := makeGuestID("optional", "node", 105)
var previous *models.VM
if withoutStatus {
previous = &models.VM{ID: id, Type: "qemu", GuestAgentStatus: "available", Status: "running", LastSeen: time.Now()}
// Prior useful identity admits the bounded no-status path.
// Expired metadata is retained on uncertainty, not renewed by
// the new request. It must not suppress the optional control.
m.guestMetadataCache = map[string]guestMetadataCacheEntry{
guestMetadataCacheKey("optional", "node", 105): {agentVersion: "prior", fetchedAt: time.Now().Add(-2 * guestMetadataCacheTTL)},
}
}
build := func() models.VM {
scope := m.mockModeFence.begin()
// The cycle starts before collection, as in pollVMsAndContainersEfficient.
cycleStart := time.Now()
vm, raw, source, notes, at, ok := m.buildVMFromClusterResource(context.Background(), "optional", res, client, id, nil, previous)
if !ok {
t.Fatal("guest disappeared")
}
m.recordGuestSnapshot("optional", "qemu", "node", 105, GuestMemorySnapshot{Status: vm.Status, RetrievedAt: at, MemorySource: source, Memory: vm.Memory, Raw: raw, Notes: notes})
m.recordGuestMetrics([]models.VM{vm}, nil, cycleStart)
registry.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
view := registry.VMs()[0]
canonicalID, resolved := registry.ResolveReferenceID(id)
// Canonical metrics can retain a numeric carrier on an
// unavailable read. Only a current or explicitly last-known
// disk has byte-value parity; the native reason owns whether
// that carrier can be presented/recorded as an observation.
knownDisk := vm.DiskStatusReason == "" || strings.HasPrefix(vm.DiskStatusReason, "prev-")
if !resolved || view.ID() != canonicalID || (knownDisk && view.DiskUsed() != vm.Disk.Used) || view.DiskStatusReason() != vm.DiskStatusReason {
t.Fatalf("filesystem observation changed at the read boundary: id=%q resolved=%q ok=%v used=%d/%d reason=%q/%q", view.ID(), canonicalID, resolved, view.DiskUsed(), vm.Disk.Used, view.DiskStatusReason(), vm.DiskStatusReason)
}
// Publish the completed poll at the normal ingest boundary.
// Read-only hydration may deliberately reuse a generation for
// two seconds; it is not a replacement for poll publication.
snapshot := models.StateSnapshot{VMs: []models.VM{vm}, LastUpdate: vm.LastSeen}
m.updateResourceStore(snapshot, scope)
live := m.GetUnifiedReadState().VMs()[0]
if live.ID() != canonicalID || (knownDisk && live.DiskUsed() != vm.Disk.Used) || live.DiskStatusReason() != vm.DiskStatusReason {
t.Fatal("accepted poll did not replace the production read generation")
}
encoded, err := json.Marshal(m.buildBroadcastFrontendStateFromSnapshot(snapshot, scope).Resources)
if err != nil || !strings.Contains(string(encoded), canonicalID) {
t.Fatal("guest identity missing from JSON read projection")
}
var served []models.ResourceFrontend
if err := json.Unmarshal(encoded, &served); err != nil || len(served) != 1 {
t.Fatal("guest missing from production JSON")
}
var facet unifiedresources.ProxmoxData
if err := json.Unmarshal(served[0].Proxmox, &facet); err != nil || facet.DiskStatusReason != vm.DiskStatusReason || facet.GuestAgentStatus != vm.GuestAgentStatus || facet.GuestAgentExpected != vm.GuestAgentExpected {
t.Fatalf("guest admission state changed in production JSON: reason=%q/%q state=%q/%q expected=%v/%v", facet.DiskStatusReason, vm.DiskStatusReason, facet.GuestAgentStatus, vm.GuestAgentStatus, facet.GuestAgentExpected, vm.GuestAgentExpected)
}
next := previousVMFromView(view)
previous = &next
return vm
}
poll.Store(1)
initial := build()
if initial.Disk.Used != int64(300*mib) || initial.Disk.Usage != 30 || initial.DiskStatusReason != "" {
t.Fatalf("optional %s starved independent filesystem success: used=%d usage=%v reason=%q", tc.failure, initial.Disk.Used, initial.Disk.Usage, initial.DiskStatusReason)
}
if !tc.terminal && initial.GuestAgentStatus != "deferred" {
t.Fatal("successful filesystem concealed later command uncertainty")
}
if tc.terminal && initial.GuestAgentStatus == "deferred" {
t.Fatal("completed unsupported command became a shared pause")
}
mu.Lock()
firstCommands := append([]string(nil), commands...)
mu.Unlock()
if firstCommands[0] != "105/get-fsinfo" {
t.Fatalf("first command = %q, want filesystem before optional reads", firstCommands[0])
}
memoryKey := guestMemoryCacheKey("optional", "node", 105)
originalMemory := m.vmAgentMemCache[memoryKey]
originalMetadata := m.guestMetadataCache[guestMetadataCacheKey("optional", "node", 105)]
originalDisk := guestHistoryStoredPoints(t, m, "vm", id, "disk")
if len(originalDisk) != 1 || originalDisk[0].Value != 30 {
t.Fatal("initial filesystem did not reach persistent History")
}
for i := int32(2); i <= 3; i++ {
// Distinct persisted timestamps ensure a falsely renewed value
// cannot hide behind a same-second storage key.
time.Sleep(1100 * time.Millisecond)
poll.Store(i)
client = newClient() // A new diagnostic/cluster client cannot escape admission.
vm := build()
if tc.terminal {
if vm.Disk.Usage != float64(20+i*10) || vm.DiskStatusReason != "" || vm.GuestAgentStatus == "deferred" {
t.Fatal("completed unsupported reply prevented repeated fresh filesystem polls")
}
} else {
// Filesystem success owns its original age, independently
// of optional metadata support and the latest row refresh.
if vm.Disk.Used != initial.Disk.Used || vm.DiskStatusReason != "prev-agent-cooldown" || vm.DiskObservation != initial.DiskObservation {
t.Fatalf("uncertainty lost or renewed eligible retained disk: %+v", vm.Disk)
}
if vm.GuestAgentStatus != "deferred" {
t.Fatal("unavailable/retained disk concealed active guest deferral")
}
if !reflect.DeepEqual(m.vmAgentMemCache[memoryKey], originalMemory) || (originalMemory.info.Source != "" && !vm.Memory.Observation.ObservedAt.Equal(initial.Memory.Observation.ObservedAt)) {
t.Fatal("cooldown renewed original memory evidence")
}
if !reflect.DeepEqual(m.guestMetadataCache[guestMetadataCacheKey("optional", "node", 105)], originalMetadata) {
t.Fatal("cooldown renewed original metadata evidence")
}
}
}
if !tc.terminal {
mu.Lock()
unchanged := reflect.DeepEqual(commands, firstCommands)
mu.Unlock()
if !unchanged || !reflect.DeepEqual(guestHistoryStoredPoints(t, m, "vm", id, "disk"), originalDisk) {
t.Fatal("cooldown dispatched another command or renewed persistent disk History")
}
// A cold collector on the same endpoint shares the live process's
// guard. This is NOT a new Pulse process or native restart proof.
cold := &Monitor{rateTracker: NewRateTracker(), guestMetadataLimiter: make(map[string]time.Time)}
vm, _, _, _, _, _ := cold.buildVMFromClusterResource(context.Background(), "optional", res, newClient(), id, nil, nil)
if !withoutStatus && (vm.Disk.Usage != -1 || vm.DiskStatusReason != "agent-cooldown" || vm.GuestAgentStatus != "deferred") {
t.Fatal("cold collector bypassed active cooldown or fabricated a disk")
}
mu.Lock()
unchanged = reflect.DeepEqual(commands, firstCommands)
mu.Unlock()
if !unchanged {
t.Fatal("cold collector sent a command during active cooldown")
}
}
wantDiskPoints := 1
if tc.terminal {
wantDiskPoints = 3
}
for _, points := range [][]MetricPoint{m.metricsHistory.GetGuestMetrics(id, "disk", time.Hour), m.GetGuestMetricsForChart(id, "vm", id, time.Hour)["disk"]} {
if len(points) != wantDiskPoints {
t.Fatalf("disk History/chart has %d points, want %d", len(points), wantDiskPoints)
}
}
if len(guestHistoryStoredPoints(t, m, "vm", id, "disk")) != wantDiskPoints || len(m.metricsHistory.GetGuestMetrics(id, "cpu", time.Hour)) != 3 {
t.Fatal("persistent disk or independent CPU history disagreed with polling")
}
})
}
}
}