Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep active guest command-safety state through metadata cleanup

pulse-triage[bot]96dc74860a58
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index b8355cfbee..6865c10571 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -81,6 +81,16 @@ unchanged, including their separate approval and entitlement enforcement.
### QGA query-eligibility provenance stays internal
+Metadata-cache activity is monitor-owned housekeeping, not an agent report,
+heartbeat or successful QGA observation. Cleanup retains active OS-command
+safety state without refreshing guest evidence; idle activity expires and
+explicit unavailable-agent invalidation or PVE instance retirement removes it.
+The existing enrollment, credentials, machine identity, reporting and command
+protocols remain unchanged. `TestGuestMetadataIdleCleanupContract` and the
+activity invalidation/concurrent-cleanup controls in
+`internal/monitoring/monitor_host_agents_test.go` verify this shared boundary;
+they are not installed agent or native guest-recovery proof.
+
The shared VM model's `GuestAgentEvidence` is monitor-owned source-time
coordination, not agent input or a public field. It prevents retained guest
identity from renewing fallback QGA eligibility. Agent enrollment, token and
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 9b645d110e..27edae546b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -683,6 +683,26 @@ metadata is not published, old identity is not made fresh, and the shared pause
and configured retry backoff remain in force. A network or OS-info deferral
does not establish another completed OS-info failure.
+Metadata-cache cleanup judges inactivity separately from the last accepted
+useful observation. A guest that is still being polled keeps its OS-info failure
+count and unsupported-command suppression even when network/version replies
+are empty, fail, or are deferred. Cache access is process-local bookkeeping:
+it cannot renew useful metadata or OS evidence, make retained Windows identity
+current, satisfy fallback eligibility, change backoff, or admit a guest command.
+The janitor never evicts an in-flight fetch. Idle cache/access entries still
+expire after ten minutes; legacy entries use their original useful timestamp.
+Explicit unavailable-agent invalidation and instance retirement remove access
+bookkeeping while retaining outstanding fetch ownership.
+
+`TestGuestMetadataCleanupSuppressionContract` exercises both ordinary collectors
+with single/cluster HTTP clients, cold/retained caches, unsupported and repeated
+completed failures, a janitor pass between polls, and network/version recovery
+without repeating the suppressed OS request. The idle/invalidation/concurrent
+cleanup contracts in `monitor_host_agents_test.go` verify bounded idle eviction,
+in-flight ownership, cross-instance isolation and unchanged observation ages.
+These are synthetic source controls, not native guest/backup recovery or proof
+of the cause or resolution of #2439, #2619 or #2822.
+
`guest_metadata_suppression_test.go` covers cold and retained caches, threshold
accumulation, supported reset and earlier deferrals. Its real HTTP fixture
drives cluster and node poll entry points through a lock appearing before the
diff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.go
index 00c05ee685..0530e81759 100644
--- a/internal/monitoring/guest_metadata.go
+++ b/internal/monitoring/guest_metadata.go
@@ -139,6 +139,18 @@ func (m *Monitor) releaseGuestMetadataFetch(key string) {
m.guestMetadataLimiterMu.Unlock()
}
+// Activity keeps an actively monitored guest's command-safety state alive even
+// when it supplies no useful metadata. It cannot renew any observation or make
+// an otherwise ineligible guest command queryable.
+func (m *Monitor) recordGuestMetadataAccess(key string, now time.Time) {
+ m.guestMetadataLimiterMu.Lock()
+ if m.guestMetadataLastAccess == nil {
+ m.guestMetadataLastAccess = make(map[string]time.Time)
+ }
+ m.guestMetadataLastAccess[key] = now
+ m.guestMetadataLimiterMu.Unlock()
+}
+
func (m *Monitor) scheduleNextGuestMetadataFetch(key string, now time.Time) {
if m == nil {
return
@@ -262,6 +274,7 @@ func (m *Monitor) retryGuestAgentCall(ctx context.Context, timeout time.Duration
func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus, allowWithoutStatus bool) ([]string, []models.GuestNetworkInterface, string, string, string, bool) {
key := guestMetadataCacheKey(instanceName, nodeName, vmid)
now := time.Now()
+ m.recordGuestMetadataAccess(key, now)
m.guestMetadataMu.RLock()
cached, ok := m.guestMetadataCache[key]
@@ -527,6 +540,7 @@ func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid in
// into a later available poll, or discard an outstanding fetch's ownership.
m.guestMetadataLimiterMu.Lock()
delete(m.guestMetadataLimiter, key)
+ delete(m.guestMetadataLastAccess, key)
m.guestMetadataLimiterMu.Unlock()
}
diff --git a/internal/monitoring/guest_metadata_cleanup_regression_test.go b/internal/monitoring/guest_metadata_cleanup_regression_test.go
new file mode 100644
index 0000000000..daf93d2457
--- /dev/null
+++ b/internal/monitoring/guest_metadata_cleanup_regression_test.go
@@ -0,0 +1,136 @@
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// A janitor must not erase the command-safety history of a guest that is still
+// being polled. Empty replies are not useful observations. All endpoints here
+// are synthetic; no guest, backup or uncertainty cooldown is manipulated.
+func TestGuestMetadataCleanupSuppressionContract(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, route := range []string{"cluster", "node"} {
+ for _, seeded := range []bool{false, true} {
+ for _, mode := range []string{"unsupported", "repeated-failure"} {
+ t.Run(fmt.Sprintf("%s/%s/cached-%t/%s", kind, route, seeded, mode), func(t *testing.T) {
+ const instance = "cleanup-suppression"
+ const mib = uint64(1024 * 1024)
+ var osCalls, networkCalls, versionCalls, fsCalls atomic.Int32
+ var healthy atomic.Bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":8388608,"meminfo":{"total":8388608,"available":5242880}}}`)
+ case strings.HasSuffix(r.URL.Path, "/get-fsinfo"):
+ fsCalls.Add(1)
+ fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1048576000,"used-bytes":314572800}]}}`)
+ case strings.HasSuffix(r.URL.Path, "/network-get-interfaces"):
+ networkCalls.Add(1)
+ if healthy.Load() {
+ fmt.Fprint(w, `{"data":{"result":[{"name":"eth0","hardware-address":"02:00:00:00:00:03","ip-addresses":[{"ip-address":"192.0.2.33","ip-address-type":"ipv4","prefix":24}]}]}}`)
+ } else {
+ fmt.Fprint(w, `{"data":{"result":[]}}`)
+ }
+ case strings.HasSuffix(r.URL.Path, "/get-osinfo"):
+ osCalls.Add(1)
+ if mode == "unsupported" {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, `{"data":null,"message":"VM 105 qmp command 'guest-get-osinfo' failed - guest agent command failed: Failed to open file '/etc/os-release': No such file or directory"}`)
+ } else {
+ w.WriteHeader(http.StatusBadRequest)
+ fmt.Fprint(w, `{"data":null,"message":"OS info request failed"}`)
+ }
+ case strings.HasSuffix(r.URL.Path, "/info"):
+ versionCalls.Add(1)
+ if healthy.Load() {
+ fmt.Fprint(w, `{"data":{"result":{"version":"2.0"}}}`)
+ } else {
+ fmt.Fprint(w, `{"data":{"result":{}}}`)
+ }
+ default:
+ t.Errorf("unexpected cleanup-fixture request %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ client := windowsBootstrapClient(t, server.URL, kind)
+ m := windowsBootstrapMonitor(t)
+ m.guestMetadataLimiter = make(map[string]time.Time)
+ key := guestMetadataCacheKey(instance, "node", 105)
+ original := guestMetadataCacheEntry{}
+ if seeded {
+ original = metadataObservationFixture(time.Now().Add(-3 * guestMetadataCacheTTL))
+ m.guestMetadataCache = map[string]guestMetadataCacheEntry{key: original}
+ }
+ resource := proxmox.ClusterResource{Type: "qemu", Node: "node", Name: "guest", VMID: 105, Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib}
+ build := func() models.VM {
+ if route == "cluster" {
+ vm, _, _, _, _, ok := m.buildVMFromClusterResource(context.Background(), instance, resource, client, makeGuestID(instance, "node", 105), nil, nil)
+ if !ok {
+ t.Fatal("VM disappeared")
+ }
+ return vm
+ }
+ vms, _ := m.pollNodeVMsWithClusterResourceBuilder(context.Background(), instance, "node", []proxmox.VM{{VMID: 105, Name: "guest", Status: "running", MaxMem: 8 * mib, Mem: 8 * mib, MaxDisk: 1000 * mib}}, client, nil, nil)
+ if len(vms) != 1 {
+ t.Fatal("node VM disappeared")
+ }
+ return vms[0]
+ }
+ failures := 1
+ if mode == "repeated-failure" {
+ failures = guestAgentOSInfoFailureThreshold
+ }
+ for round := 1; round <= failures+1; round++ {
+ // Only advance test-owned metadata scheduling. Do not shorten a QGA
+ // cooldown or replay any uncertain command.
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ vm := build()
+ before := m.guestMetadataCache[key]
+ m.cleanupGuestMetadataCache(time.Now())
+ entry, exists := m.guestMetadataCache[key]
+ expectedCount := guestAgentOSInfoFailureThreshold
+ if mode == "repeated-failure" && round < failures {
+ expectedCount = round
+ }
+ if !exists || entry.osInfoFailureCount != expectedCount || entry.osInfoSkip != (expectedCount == guestAgentOSInfoFailureThreshold) {
+ t.Errorf("active safety history erased at round %d: exists=%t failures=%d skip=%t", round, exists, entry.osInfoFailureCount, entry.osInfoSkip)
+ }
+ if !reflect.DeepEqual(entry, before) || !entry.fetchedAt.Equal(original.fetchedAt) || !entry.osInfoObservedAt.Equal(original.osInfoObservedAt) || m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {
+ t.Error("cleanup changed retained metadata or renewed stale observation evidence")
+ }
+ if vm.Disk.Used != int64(300*mib) || vm.DiskStatusReason != "" || vm.OSName != original.osName || vm.AgentVersion != original.agentVersion {
+ t.Error("safety retention lost independent filesystems or changed last-known identity")
+ }
+ }
+ healthy.Store(true)
+ m.guestMetadataLimiter[key] = time.Now().Add(-time.Second)
+ resumed := build()
+ m.cleanupGuestMetadataCache(time.Now())
+ wantPolls := int32(failures + 2)
+ if osCalls.Load() != int32(failures) || networkCalls.Load() != wantPolls || versionCalls.Load() != wantPolls || fsCalls.Load() != wantPolls {
+ t.Errorf("command counts os/network/version/fs=%d/%d/%d/%d, want %d/%d/%d/%d", osCalls.Load(), networkCalls.Load(), versionCalls.Load(), fsCalls.Load(), failures, wantPolls, wantPolls, wantPolls)
+ }
+ if resumed.AgentVersion != "2.0" || len(resumed.NetworkInterfaces) != 1 || resumed.NetworkInterfaces[0].Name != "eth0" || !m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) || !m.guestMetadataCache[key].osInfoSkip {
+ t.Error("ordinary network/version recovery did not preserve unsupported OS safety")
+ }
+ })
+ }
+ }
+ }
+ }
+}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 17da6faf26..19fcc18a65 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1405,7 +1405,8 @@ type Monitor struct {
guestMetadataCache map[string]guestMetadataCacheEntry
guestMetadataLimiterMu sync.Mutex
guestMetadataLimiter map[string]time.Time
- guestMetadataInFlight map[string]bool // Guarded by guestMetadataLimiterMu; released only on fetch completion.
+ guestMetadataInFlight map[string]bool // Guarded by guestMetadataLimiterMu; released only on fetch completion.
+ guestMetadataLastAccess map[string]time.Time // Guarded by guestMetadataLimiterMu; cache activity, never guest evidence.
guestMetadataSlots chan struct{}
guestMetadataMinRefresh time.Duration
guestMetadataRefreshJitter time.Duration
@@ -2021,6 +2022,7 @@ func New(cfg *config.Config) (*Monitor, error) {
dockerCommandIndex: make(map[string]string),
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
guestMetadataLimiter: make(map[string]time.Time),
+ guestMetadataLastAccess: make(map[string]time.Time),
guestMetadataMinRefresh: minRefresh,
guestMetadataRefreshJitter: jitter,
guestMetadataRetryBackoff: retryBackoff,
diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go
index 7a391fc4a7..f5fb3485f8 100644
--- a/internal/monitoring/monitor_agents.go
+++ b/internal/monitoring/monitor_agents.go
@@ -4985,14 +4985,25 @@ func (m *Monitor) cleanupRemovedHostAgents(now time.Time) {
}
// cleanupGuestMetadataCache removes stale guest metadata cache and limiter entries.
-// Entries older than 2x the cache TTL (10 minutes) are removed to prevent unbounded growth
-// when VMs are deleted or moved.
+// Idle entries older than 2x the cache TTL (10 minutes) are removed to prevent
+// unbounded growth when VMs are deleted or moved. A successful observation's age
+// is not cache activity: empty/failed metadata must not erase an active guest's
+// OS-info failure counter or known-unsupported command suppression.
func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {
const maxAge = 2 * guestMetadataCacheTTL // 10 minutes
+ // Serialize the activity check with touches and reservations. No caller holds
+ // the metadata lock while acquiring the limiter lock; this order also keeps
+ // in-flight ownership intact throughout a sweep.
+ m.guestMetadataLimiterMu.Lock()
+ defer m.guestMetadataLimiterMu.Unlock()
m.guestMetadataMu.Lock()
for key, entry := range m.guestMetadataCache {
- if now.Sub(entry.fetchedAt) > maxAge {
+ activityAt := m.guestMetadataLastAccess[key]
+ if entry.fetchedAt.After(activityAt) {
+ activityAt = entry.fetchedAt // Also supports legacy/seeded cache entries.
+ }
+ if !m.guestMetadataInFlight[key] && now.Sub(activityAt) > maxAge {
delete(m.guestMetadataCache, key)
log.Debug().
Str("key", key).
@@ -5002,11 +5013,14 @@ func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {
}
m.guestMetadataMu.Unlock()
- m.guestMetadataLimiterMu.Lock()
- defer m.guestMetadataLimiterMu.Unlock()
+ for key, accessedAt := range m.guestMetadataLastAccess {
+ if !m.guestMetadataInFlight[key] && now.Sub(accessedAt) > maxAge {
+ delete(m.guestMetadataLastAccess, key)
+ }
+ }
for key, nextAllowed := range m.guestMetadataLimiter {
// Keep near-term limiter state; remove long-idle keys.
- if now.Sub(nextAllowed) > maxAge {
+ if !m.guestMetadataInFlight[key] && now.Sub(nextAllowed) > maxAge {
delete(m.guestMetadataLimiter, key)
log.Debug().
Str("key", key).
diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.go
index 34a0ee81d9..e7915ad17a 100644
--- a/internal/monitoring/monitor_cluster_dedupe.go
+++ b/internal/monitoring/monitor_cluster_dedupe.go
@@ -152,6 +152,11 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {
delete(m.guestMetadataLimiter, key)
}
}
+ for key := range m.guestMetadataLastAccess {
+ if strings.HasPrefix(key, instanceName+"|") {
+ delete(m.guestMetadataLastAccess, key)
+ }
+ }
m.guestMetadataLimiterMu.Unlock()
if m.state != nil {
diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go
index b43522ba4a..44b4b4ee48 100644
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@ -7,6 +7,7 @@ import (
"fmt"
"net"
"os"
+ "reflect"
"slices"
"strings"
"sync/atomic"
@@ -8319,3 +8320,139 @@ func TestMonitorPBSInstancesSnapshotCarriesReportedNodeName(t *testing.T) {
t.Fatalf("PBSInstancesSnapshot()[0].NodeName = %q, want the reported node name %q", instances[0].NodeName, "pbs01")
}
}
+
+func TestGuestMetadataIdleCleanupContract(t *testing.T) {
+ now := time.Now()
+ stale := now.Add(-3 * guestMetadataCacheTTL)
+ suppressed := guestMetadataCacheEntry{osInfoFailureCount: guestAgentOSInfoFailureThreshold, osInfoSkip: true}
+ retained := suppressed
+ retained.osName, retained.windowsGuest = "Microsoft Windows", true
+ retained.fetchedAt, retained.osInfoObservedAt = stale, stale
+ m := &Monitor{
+ guestMetadataCache: map[string]guestMetadataCacheEntry{
+ "active-empty": suppressed,
+ "active-retained": retained,
+ "inactive": suppressed,
+ "legacy-fresh": metadataObservationFixture(now),
+ "legacy-stale": metadataObservationFixture(stale),
+ "inflight": suppressed,
+ },
+ guestMetadataLastAccess: map[string]time.Time{"active-empty": now, "active-retained": now, "inactive": stale, "only-active": now, "only-idle": stale, "inflight": stale},
+ guestMetadataLimiter: map[string]time.Time{"active-empty": now.Add(time.Minute), "inactive": stale, "inflight": stale},
+ guestMetadataInFlight: map[string]bool{"inflight": true},
+ }
+ m.cleanupGuestMetadataCache(now)
+ for _, key := range []string{"active-empty", "active-retained", "legacy-fresh", "inflight"} {
+ if _, exists := m.guestMetadataCache[key]; !exists {
+ t.Errorf("live cache %s pruned", key)
+ }
+ }
+ for _, key := range []string{"inactive", "legacy-stale"} {
+ if _, exists := m.guestMetadataCache[key]; exists {
+ t.Errorf("idle cache %s retained", key)
+ }
+ }
+ for _, key := range []string{"inactive", "only-idle"} {
+ if _, exists := m.guestMetadataLastAccess[key]; exists {
+ t.Errorf("idle bookkeeping %s retained", key)
+ }
+ }
+ if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || !m.guestMetadataInFlight["inflight"] {
+ t.Error("janitor changed current activity, backoff or in-flight ownership")
+ }
+ empty := m.guestMetadataCache["active-empty"]
+ old := m.guestMetadataCache["active-retained"]
+ if !reflect.DeepEqual(empty, suppressed) || !reflect.DeepEqual(old, retained) {
+ t.Error("retention changed safety or observation state")
+ }
+ // Evidence consumers use actual observation age, never the activity clock.
+ key := guestMetadataCacheKey("activity", "node", 105)
+ m.guestMetadataCache[key] = retained
+ m.guestMetadataLastAccess[key] = now
+ m.cleanupGuestMetadataCache(now)
+ if m.hasRecentGuestMetadataEvidence("activity", "node", 105, now) || m.guestWindowsMeminfoUnsupported("activity", "node", 105, now) || !m.guestMetadataEvidenceTime("activity", "node", 105).Equal(stale) {
+ t.Error("activity invented a useful/Windows OS observation")
+ }
+ delete(m.guestMetadataInFlight, "inflight")
+ m.cleanupGuestMetadataCache(now)
+ if _, exists := m.guestMetadataCache["inflight"]; exists {
+ t.Error("completed idle owner retained cache")
+ }
+ if _, exists := m.guestMetadataLastAccess["inflight"]; exists {
+ t.Error("completed idle owner retained bookkeeping")
+ }
+ if _, exists := m.guestMetadataLimiter["inflight"]; exists {
+ t.Error("completed idle owner retained obsolete deadline")
+ }
+}
+
+func TestGuestMetadataActivityInvalidationContract(t *testing.T) {
+ key := guestMetadataCacheKey("activity", "node", 105)
+ sibling := guestMetadataCacheKey("activity-elsewhere", "node", 105)
+ m := &Monitor{
+ guestMetadataCache: map[string]guestMetadataCacheEntry{key: {osInfoSkip: true}, sibling: {osInfoSkip: true}},
+ guestMetadataLimiter: map[string]time.Time{key: time.Now().Add(time.Hour), sibling: time.Now().Add(time.Hour)},
+ guestMetadataLastAccess: map[string]time.Time{key: time.Now(), sibling: time.Now()},
+ guestMetadataInFlight: map[string]bool{key: true},
+ }
+ m.clearGuestMetadataCache("activity", "node", 105)
+ if _, exists := m.guestMetadataCache[key]; exists {
+ t.Error("explicit invalidation retained metadata")
+ }
+ if _, exists := m.guestMetadataLimiter[key]; exists {
+ t.Error("explicit invalidation retained deadline")
+ }
+ if _, exists := m.guestMetadataLastAccess[key]; exists {
+ t.Error("explicit invalidation retained activity")
+ }
+ if !m.guestMetadataInFlight[key] || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {
+ t.Error("invalidation discarded ownership or another instance")
+ }
+ m.guestMetadataLastAccess[key] = time.Now()
+ m.guestMetadataLastAccess[guestMetadataCacheKey("activity", "node", 106)] = time.Now()
+ m.retirePVEInstanceRuntime("activity")
+ for remaining := range m.guestMetadataLastAccess {
+ if strings.HasPrefix(remaining, "activity|") {
+ t.Error("retired instance retained activity bookkeeping")
+ }
+ }
+ if m.guestMetadataLastAccess[sibling].IsZero() || !m.guestMetadataInFlight[key] {
+ t.Error("retirement crossed instance scope or discarded active ownership")
+ }
+}
+
+func TestGuestMetadataActivityConcurrentCleanupContract(t *testing.T) {
+ const instance = "activity"
+ key := guestMetadataCacheKey(instance, "node", 105)
+ old := metadataObservationFixture(time.Now().Add(-3 * guestMetadataCacheTTL))
+ old.osInfoFailureCount, old.osInfoSkip = guestAgentOSInfoFailureThreshold, true
+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: old}}
+ m.recordGuestMetadataAccess(key, time.Now())
+ // An active backup blocks commands, not cache use. Touches and the janitor
+ // must coordinate without extending the old useful/OS observation timestamps.
+ status := &proxmox.VMStatus{Lock: "backup"}
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ for i := 0; i < 200; i++ {
+ _, _, name, version, agent, deferred := m.fetchGuestAgentMetadata(context.Background(), nil, instance, "node", "guest", 105, status, false)
+ if name != old.osName || version != old.osVersion || agent != old.agentVersion || deferred {
+ t.Error("locked activity changed retained identity or invented a command deferral")
+ }
+ }
+ }()
+ for i := 0; i < 200; i++ {
+ m.cleanupGuestMetadataCache(time.Now())
+ }
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("cache activity and cleanup did not finish")
+ }
+ m.guestMetadataMu.RLock()
+ retained := m.guestMetadataCache[key]
+ m.guestMetadataMu.RUnlock()
+ if retained.osName != old.osName || retained.osInfoSkip != old.osInfoSkip || retained.osInfoFailureCount != old.osInfoFailureCount || !retained.fetchedAt.Equal(old.fetchedAt) || !retained.osInfoObservedAt.Equal(old.osInfoObservedAt) || m.hasRecentGuestMetadataEvidence(instance, "node", 105, time.Now()) {
+ t.Error("concurrent cleanup lost suppression or renewed stale useful evidence")
+ }
+}