Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep active guest command-safety state through metadata cleanup
pulse-triage[bot]96dc74860a58
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex b8355cfbee..6865c10571 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -81,6 +81,16 @@ unchanged, including their separate approval and entitlement enforcement.### QGA query-eligibility provenance stays internal+Metadata-cache activity is monitor-owned housekeeping, not an agent report,+heartbeat or successful QGA observation. Cleanup retains active OS-command+safety state without refreshing guest evidence; idle activity expires and+explicit unavailable-agent invalidation or PVE instance retirement removes it.+The existing enrollment, credentials, machine identity, reporting and command+protocols remain unchanged. `TestGuestMetadataIdleCleanupContract` and the+activity invalidation/concurrent-cleanup controls in+`internal/monitoring/monitor_host_agents_test.go` verify this shared boundary;+they are not installed agent or native guest-recovery proof.+The shared VM model's `GuestAgentEvidence` is monitor-owned source-timecoordination, not agent input or a public field. It prevents retained guestidentity from renewing fallback QGA eligibility. Agent enrollment, token anddiff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 9b645d110e..27edae546b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -683,6 +683,26 @@ metadata is not published, old identity is not made fresh, and the shared pauseand configured retry backoff remain in force. A network or OS-info deferraldoes not establish another completed OS-info failure.+Metadata-cache cleanup judges inactivity separately from the last accepted+useful observation. A guest that is still being polled keeps its OS-info failure+count and unsupported-command suppression even when network/version replies+are empty, fail, or are deferred. Cache access is process-local bookkeeping:+it cannot renew useful metadata or OS evidence, make retained Windows identity+current, satisfy fallback eligibility, change backoff, or admit a guest command.+The janitor never evicts an in-flight fetch. Idle cache/access entries still+expire after ten minutes; legacy entries use their original useful timestamp.+Explicit unavailable-agent invalidation and instance retirement remove access+bookkeeping while retaining outstanding fetch ownership.++`TestGuestMetadataCleanupSuppressionContract` exercises both ordinary collectors+with single/cluster HTTP clients, cold/retained caches, unsupported and repeated+completed failures, a janitor pass between polls, and network/version recovery+without repeating the suppressed OS request. The idle/invalidation/concurrent+cleanup contracts in `monitor_host_agents_test.go` verify bounded idle eviction,+in-flight ownership, cross-instance isolation and unchanged observation ages.+These are synthetic source controls, not native guest/backup recovery or proof+of the cause or resolution of #2439, #2619 or #2822.+`guest_metadata_suppression_test.go` covers cold and retained caches, thresholdaccumulation, supported reset and earlier deferrals. Its real HTTP fixturedrives cluster and node poll entry points through a lock appearing before thediff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.goindex 00c05ee685..0530e81759 100644--- a/internal/monitoring/guest_metadata.go+++ b/internal/monitoring/guest_metadata.go@@ -139,6 +139,18 @@ func (m *Monitor) releaseGuestMetadataFetch(key string) {m.guestMetadataLimiterMu.Unlock()}+// Activity keeps an actively monitored guest's command-safety state alive even+// when it supplies no useful metadata. It cannot renew any observation or make+// an otherwise ineligible guest command queryable.+func (m *Monitor) recordGuestMetadataAccess(key string, now time.Time) {+ m.guestMetadataLimiterMu.Lock()+ if m.guestMetadataLastAccess == nil {+ m.guestMetadataLastAccess = make(map[string]time.Time)+ }+ m.guestMetadataLastAccess[key] = now+ m.guestMetadataLimiterMu.Unlock()+}+func (m *Monitor) scheduleNextGuestMetadataFetch(key string, now time.Time) {if m == nil {return@@ -262,6 +274,7 @@ func (m *Monitor) retryGuestAgentCall(ctx context.Context, timeout time.Durationfunc (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus, allowWithoutStatus bool) ([]string, []models.GuestNetworkInterface, string, string, string, bool) {key := guestMetadataCacheKey(instanceName, nodeName, vmid)now := time.Now()+ m.recordGuestMetadataAccess(key, now)m.guestMetadataMu.RLock()cached, ok := m.guestMetadataCache[key]@@ -527,6 +540,7 @@ func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid in// into a later available poll, or discard an outstanding fetch's ownership.m.guestMetadataLimiterMu.Lock()delete(m.guestMetadataLimiter, key)+ delete(m.guestMetadataLastAccess, key)m.guestMetadataLimiterMu.Unlock()}diff --git a/internal/monitoring/guest_metadata_cleanup_regression_test.go b/internal/monitoring/guest_metadata_cleanup_regression_test.gonew file mode 100644index 0000000000..daf93d2457--- /dev/null+++ b/internal/monitoring/guest_metadata_cleanup_regression_test.go@@ -0,0 +1,136 @@+package monitoring++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// A janitor must not erase the command-safety history of a guest that is still+// being polled. Empty replies are not useful observations. All endpoints here+// are synthetic; no guest, backup or uncertainty cooldown is manipulated.+func TestGuestMetadataCleanupSuppressionContract(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, route := range []string{"cluster", "node"} {+ for _, seeded := range []bool{false, true} {+ for _, mode := range []string{"unsupported", "repeated-failure"} {+ t.Run(fmt.Sprintf("%s/%s/cached-%t/%s", kind, route, seeded, mode), func(t *testing.T) {+ const instance = "cleanup-suppression"+ const mib = uint64(1024 * 1024)+ var osCalls, networkCalls, versionCalls, fsCalls atomic.Int32+ var healthy atomic.Bool+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch {+ case strings.HasSuffix(r.URL.Path, "/config"):+ fmt.Fprint(w, `{"data":{}}`)+ case strings.HasSuffix(r.URL.Path, "/status/current"):+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":8388608,"meminfo":{"total":8388608,"available":5242880}}}`)+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):+ fsCalls.Add(1)+ fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1048576000,"used-bytes":314572800}]}}`)+ case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):+ networkCalls.Add(1)+ if healthy.Load() {+ fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","hardware-address":"02:00:00:00:00:03","ip-addresses":[{"ip-address":"192.0.2.33","ip-address-type":"ipv4","prefix":24}]}]}}`)+ } else {+ fmt.Fprint(w, `{"data":{"result":[]}}`)+ }+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):+ osCalls.Add(1)+ if mode == "unsupported" {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, `{"data":null,"message":"VM 105 qmp command 'guest-get-osinfo' failed - guest agent command failed: Failed to open file '/etc/os-release': No such file or directory"}`)+ } else {+ w.WriteHeader(http.StatusBadRequest)+ fmt.Fprint(w, `{"data":null,"message":"OS info request failed"}`)+ }+ case strings.HasSuffix(r.URL.Path, "/info"):+ versionCalls.Add(1)+ if healthy.Load() {+ fmt.Fprint(w, `{"data":{"result":{"version":"2.0"}}}`)+ } else {+ fmt.Fprint(w, `{"data":{"result":{}}}`)+ }+ default:+ t.Errorf("unexpected cleanup-fixture request %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ client := windowsBootstrapClient(t, server.URL, kind)+ m := windowsBootstrapMonitor(t)+ m.guestMetadataLimiter = make(map[string]time.Time)+ key := guestMetadataCacheKey(instance, "node", 105)+ original := guestMetadataCacheEntry{}+ if seeded {+ original = metadataObservationFixture(time.Now().Add(-3 * guestMetadataCacheTTL))+ m.guestMetadataCache = map[string]guestMetadataCacheEntry{key: original}+ }+ resource := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib}+ build := func() models.VM {+ if route == "cluster" {+ vm, _, _, _, _, ok := m.buildVMFromClusterResource(context.Background(), instance, resource, client, makeGuestID(instance, "node", 105), nil, nil)+ if !ok {+ t.Fatal("VM disappeared")+ }+ return vm+ }+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), instance, "node", []proxmox.VM{{VMID: 105, Name: "guest", Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib}}, client, nil, nil)+ if len(vms) != 1 {+ t.Fatal("node VM disappeared")+ }+ return vms[0]+ }+ failures := 1+ if mode == "repeated-failure" {+ failures = guestAgentOSInfoFailureThreshold+ }+ for round := 1; round <= failures+1; round++ {+ // Only advance test-owned metadata scheduling. Do not shorten a QGA+ // cooldown or replay any uncertain command.+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ vm := build()+ before := m.guestMetadataCache[key]+ m.cleanupGuestMetadataCache(time.Now())+ entry, exists := m.guestMetadataCache[key]+ expectedCount := guestAgentOSInfoFailureThreshold+ if mode == "repeated-failure" && round < failures {+ expectedCount = round+ }+ if !exists || entry.osInfoFailureCount != expectedCount || entry.osInfoSkip != (expectedCount == guestAgentOSInfoFailureThreshold) {+ t.Errorf("active safety history erased at round %d: exists=%t failures=%d skip=%t", round, exists, entry.osInfoFailureCount, entry.osInfoSkip)+ }+ if !reflect.DeepEqual(entry, before) || !entry.fetchedAt.Equal(original.fetchedAt) || !entry.osInfoObservedAt.Equal(original.osInfoObservedAt) || m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {+ t.Error("cleanup changed retained metadata or renewed stale observation evidence")+ }+ if vm.Disk.Used != int64(300*mib) || vm.DiskStatusReason != "" || vm.OSName != original.osName || vm.AgentVersion != original.agentVersion {+ t.Error("safety retention lost independent filesystems or changed last-known identity")+ }+ }+ healthy.Store(true)+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)+ resumed := build()+ m.cleanupGuestMetadataCache(time.Now())+ wantPolls := int32(failures + 2)+ if osCalls.Load() != int32(failures) || networkCalls.Load() != wantPolls || versionCalls.Load() != wantPolls || fsCalls.Load() != wantPolls {+ t.Errorf("command counts os/network/version/fs=%d/%d/%d/%d, want %d/%d/%d/%d", osCalls.Load(), networkCalls.Load(), versionCalls.Load(), fsCalls.Load(), failures, wantPolls, wantPolls, wantPolls)+ }+ if resumed.AgentVersion != "2.0" || len(resumed.NetworkInterfaces) != 1 || resumed.NetworkInterfaces[0].Name != "eth0" || !m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) || !m.guestMetadataCache[key].osInfoSkip {+ t.Error("ordinary network/version recovery did not preserve unsupported OS safety")+ }+ })+ }+ }+ }+ }+}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 17da6faf26..19fcc18a65 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1405,7 +1405,8 @@ type Monitor struct {guestMetadataCache map[string]guestMetadataCacheEntryguestMetadataLimiterMu sync.MutexguestMetadataLimiter map[string]time.Time- guestMetadataInFlight map[string]bool // Guarded by guestMetadataLimiterMu; released only on fetch completion.+ guestMetadataInFlight map[string]bool // Guarded by guestMetadataLimiterMu; released only on fetch completion.+ guestMetadataLastAccess map[string]time.Time // Guarded by guestMetadataLimiterMu; cache activity, never guest evidence.guestMetadataSlots chan struct{}guestMetadataMinRefresh time.DurationguestMetadataRefreshJitter time.Duration@@ -2021,6 +2022,7 @@ func New(cfg *config.Config) (*Monitor, error) {dockerCommandIndex: make(map[string]string),guestMetadataCache: make(map[string]guestMetadataCacheEntry),guestMetadataLimiter: make(map[string]time.Time),+ guestMetadataLastAccess: make(map[string]time.Time),guestMetadataMinRefresh: minRefresh,guestMetadataRefreshJitter: jitter,guestMetadataRetryBackoff: retryBackoff,diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.goindex 7a391fc4a7..f5fb3485f8 100644--- a/internal/monitoring/monitor_agents.go+++ b/internal/monitoring/monitor_agents.go@@ -4985,14 +4985,25 @@ func (m *Monitor) cleanupRemovedHostAgents(now time.Time) {}// cleanupGuestMetadataCache removes stale guest metadata cache and limiter entries.-// Entries older than 2x the cache TTL (10 minutes) are removed to prevent unbounded growth-// when VMs are deleted or moved.+// Idle entries older than 2x the cache TTL (10 minutes) are removed to prevent+// unbounded growth when VMs are deleted or moved. A successful observation's age+// is not cache activity: empty/failed metadata must not erase an active guest's+// OS-info failure counter or known-unsupported command suppression.func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {const maxAge = 2 * guestMetadataCacheTTL // 10 minutes+ // Serialize the activity check with touches and reservations. No caller holds+ // the metadata lock while acquiring the limiter lock; this order also keeps+ // in-flight ownership intact throughout a sweep.+ m.guestMetadataLimiterMu.Lock()+ defer m.guestMetadataLimiterMu.Unlock()m.guestMetadataMu.Lock()for key, entry := range m.guestMetadataCache {- if now.Sub(entry.fetchedAt) > maxAge {+ activityAt := m.guestMetadataLastAccess[key]+ if entry.fetchedAt.After(activityAt) {+ activityAt = entry.fetchedAt // Also supports legacy/seeded cache entries.+ }+ if !m.guestMetadataInFlight[key] && now.Sub(activityAt) > maxAge {delete(m.guestMetadataCache, key)log.Debug().Str("key", key).@@ -5002,11 +5013,14 @@ func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {}m.guestMetadataMu.Unlock()- m.guestMetadataLimiterMu.Lock()- defer m.guestMetadataLimiterMu.Unlock()+ for key, accessedAt := range m.guestMetadataLastAccess {+ if !m.guestMetadataInFlight[key] && now.Sub(accessedAt) > maxAge {+ delete(m.guestMetadataLastAccess, key)+ }+ }for key, nextAllowed := range m.guestMetadataLimiter {// Keep near-term limiter state; remove long-idle keys.- if now.Sub(nextAllowed) > maxAge {+ if !m.guestMetadataInFlight[key] && now.Sub(nextAllowed) > maxAge {delete(m.guestMetadataLimiter, key)log.Debug().Str("key", key).diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.goindex 34a0ee81d9..e7915ad17a 100644--- a/internal/monitoring/monitor_cluster_dedupe.go+++ b/internal/monitoring/monitor_cluster_dedupe.go@@ -152,6 +152,11 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {delete(m.guestMetadataLimiter, key)}}+ for key := range m.guestMetadataLastAccess {+ if strings.HasPrefix(key, instanceName+"|") {+ delete(m.guestMetadataLastAccess, key)+ }+ }m.guestMetadataLimiterMu.Unlock()if m.state != nil {diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.goindex b43522ba4a..44b4b4ee48 100644--- a/internal/monitoring/monitor_host_agents_test.go+++ b/internal/monitoring/monitor_host_agents_test.go@@ -7,6 +7,7 @@ import ("fmt""net""os"+ "reflect""slices""strings""sync/atomic"@@ -8319,3 +8320,139 @@ func TestMonitorPBSInstancesSnapshotCarriesReportedNodeName(t *testing.T) {t.Fatalf("PBSInstancesSnapshot()[0].NodeName = %q, want the reported node name %q", instances[0].NodeName, "pbs01")}}++func TestGuestMetadataIdleCleanupContract(t *testing.T) {+ now := time.Now()+ stale := now.Add(-3 * guestMetadataCacheTTL)+ suppressed := guestMetadataCacheEntry{osInfoFailureCount: guestAgentOSInfoFailureThreshold, osInfoSkip: true}+ retained := suppressed+ retained.osName, retained.windowsGuest = "Microsoft Windows", true+ retained.fetchedAt, retained.osInfoObservedAt = stale, stale+ m := &Monitor{+ guestMetadataCache: map[string]guestMetadataCacheEntry{+ "active-empty": suppressed,+ "active-retained": retained,+ "inactive": suppressed,+ "legacy-fresh": metadataObservationFixture(now),+ "legacy-stale": metadataObservationFixture(stale),+ "inflight": suppressed,+ },+ guestMetadataLastAccess: map[string]time.Time{"active-empty": now, "active-retained": now, "inactive": stale, "only-active": now, "only-idle": stale, "inflight": stale},+ guestMetadataLimiter: map[string]time.Time{"active-empty": now.Add(time.Minute), "inactive": stale, "inflight": stale},+ guestMetadataInFlight: map[string]bool{"inflight": true},+ }+ m.cleanupGuestMetadataCache(now)+ for _, key := range []string{"active-empty", "active-retained", "legacy-fresh", "inflight"} {+ if _, exists := m.guestMetadataCache[key]; !exists {+ t.Errorf("live cache %s pruned", key)+ }+ }+ for _, key := range []string{"inactive", "legacy-stale"} {+ if _, exists := m.guestMetadataCache[key]; exists {+ t.Errorf("idle cache %s retained", key)+ }+ }+ for _, key := range []string{"inactive", "only-idle"} {+ if _, exists := m.guestMetadataLastAccess[key]; exists {+ t.Errorf("idle bookkeeping %s retained", key)+ }+ }+ if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || !m.guestMetadataInFlight["inflight"] {+ t.Error("janitor changed current activity, backoff or in-flight ownership")+ }+ empty := m.guestMetadataCache["active-empty"]+ old := m.guestMetadataCache["active-retained"]+ if !reflect.DeepEqual(empty, suppressed) || !reflect.DeepEqual(old, retained) {+ t.Error("retention changed safety or observation state")+ }+ // Evidence consumers use actual observation age, never the activity clock.+ key := guestMetadataCacheKey("activity", "node", 105)+ m.guestMetadataCache[key] = retained+ m.guestMetadataLastAccess[key] = now+ m.cleanupGuestMetadataCache(now)+ if m.hasRecentGuestMetadataEvidence("activity", "node", 105, now) || m.guestWindowsMeminfoUnsupported("activity", "node", 105, now) || !m.guestMetadataEvidenceTime("activity", "node", 105).Equal(stale) {+ t.Error("activity invented a useful/Windows OS observation")+ }+ delete(m.guestMetadataInFlight, "inflight")+ m.cleanupGuestMetadataCache(now)+ if _, exists := m.guestMetadataCache["inflight"]; exists {+ t.Error("completed idle owner retained cache")+ }+ if _, exists := m.guestMetadataLastAccess["inflight"]; exists {+ t.Error("completed idle owner retained bookkeeping")+ }+ if _, exists := m.guestMetadataLimiter["inflight"]; exists {+ t.Error("completed idle owner retained obsolete deadline")+ }+}++func TestGuestMetadataActivityInvalidationContract(t *testing.T) {+ key := guestMetadataCacheKey("activity", "node", 105)+ sibling := guestMetadataCacheKey("activity-elsewhere", "node", 105)+ m := &Monitor{+ guestMetadataCache: map[string]guestMetadataCacheEntry{key: {osInfoSkip: true}, sibling: {osInfoSkip: true}},+ guestMetadataLimiter: map[string]time.Time{key: time.Now().Add(time.Hour), sibling: time.Now().Add(time.Hour)},+ guestMetadataLastAccess: map[string]time.Time{key: time.Now(), sibling: time.Now()},+ guestMetadataInFlight: map[string]bool{key: true},+ }+ m.clearGuestMetadataCache("activity", "node", 105)+ if _, exists := m.guestMetadataCache[key]; exists {+ t.Error("explicit invalidation retained metadata")+ }+ if _, exists := m.guestMetadataLimiter[key]; exists {+ t.Error("explicit invalidation retained deadline")+ }+ if _, exists := m.guestMetadataLastAccess[key]; exists {+ t.Error("explicit invalidation retained activity")+ }+ if !m.guestMetadataInFlight[key] || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {+ t.Error("invalidation discarded ownership or another instance")+ }+ m.guestMetadataLastAccess[key] = time.Now()+ m.guestMetadataLastAccess[guestMetadataCacheKey("activity", "node", 106)] = time.Now()+ m.retirePVEInstanceRuntime("activity")+ for remaining := range m.guestMetadataLastAccess {+ if strings.HasPrefix(remaining, "activity|") {+ t.Error("retired instance retained activity bookkeeping")+ }+ }+ if m.guestMetadataLastAccess[sibling].IsZero() || !m.guestMetadataInFlight[key] {+ t.Error("retirement crossed instance scope or discarded active ownership")+ }+}++func TestGuestMetadataActivityConcurrentCleanupContract(t *testing.T) {+ const instance = "activity"+ key := guestMetadataCacheKey(instance, "node", 105)+ old := metadataObservationFixture(time.Now().Add(-3 * guestMetadataCacheTTL))+ old.osInfoFailureCount, old.osInfoSkip = guestAgentOSInfoFailureThreshold, true+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: old}}+ m.recordGuestMetadataAccess(key, time.Now())+ // An active backup blocks commands, not cache use. Touches and the janitor+ // must coordinate without extending the old useful/OS observation timestamps.+ status := &proxmox.VMStatus{Lock: "backup"}+ done := make(chan struct{})+ go func() {+ defer close(done)+ for i := 0; i < 200; i++ {+ _, _, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), nil, instance, "node", "guest", 105, status, false)+ if name != old.osName || version != old.osVersion || agent != old.agentVersion || deferred {+ t.Error("locked activity changed retained identity or invented a command deferral")+ }+ }+ }()+ for i := 0; i < 200; i++ {+ m.cleanupGuestMetadataCache(time.Now())+ }+ select {+ case <-done:+ case <-time.After(5 * time.Second):+ t.Fatal("cache activity and cleanup did not finish")+ }+ m.guestMetadataMu.RLock()+ retained := m.guestMetadataCache[key]+ m.guestMetadataMu.RUnlock()+ if retained.osName != old.osName || retained.osInfoSkip != old.osInfoSkip || retained.osInfoFailureCount != old.osInfoFailureCount || !retained.fetchedAt.Equal(old.fetchedAt) || !retained.osInfoObservedAt.Equal(old.osInfoObservedAt) || m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {+ t.Error("concurrent cleanup lost suppression or renewed stale useful evidence")+ }+}