Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Discard guest metadata from invalidated fetch lifecycles
pulse-triage[bot]2488be4e17fc
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex 6865c10571..a635beac44 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -91,6 +91,17 @@ activity invalidation/concurrent-cleanup controls in`internal/monitoring/monitor_host_agents_test.go` verify this shared boundary;they are not installed agent or native guest-recovery proof.+Explicit metadata invalidation and PVE instance retirement cancel an active+enrichment context without releasing its owner early. Late replies cannot+repopulate the old lifecycle's cache, OS-command safety state or retry timing;+new polling and diagnostics still honour the shared consumed-command uncertainty+fence. This remains an internal monitor lifecycle, with no change to agent+identity, credentials or the command protocol.+`internal/monitoring/guest_metadata_lifecycle_test.go` verifies cold/retained+late replies, both invalidation routes, direct/cluster HTTP cancellation,+independent keys and ordinary new-lifecycle recovery. It is not installed+removal, native guest recovery or evidence that a cancelled guest command stopped.+The shared VM model's `GuestAgentEvidence` is monitor-owned source-timecoordination, not agent input or a public field. It prevents retained guestidentity from renewing fallback QGA eligibility. Agent enrollment, token anddiff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 27edae546b..500b1e0351 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -971,6 +971,28 @@ cache invalidation on agent unavailability also discards its obsolete schedulingdeadline, so a later available poll can begin a fresh metadata lifecycle; itnever discards an outstanding fetch owner or the shared Proxmox command fence.+Invalidation and PVE instance retirement also cancel that owner's context and+mark its lifecycle obsolete until the fetch actually returns. A late network,+OS or version reply cannot recreate identity, OS-command suppression, useful/OS+evidence or refresh/backoff deadlines after either operation. Publication and+invalidation share one lock order, and fallback metadata is re-read within the+reserved lifecycle rather than copied from the pre-admission cache snapshot.+Slot waiting is cancelled without releasing another fetch's slot. Cancellation+is not proof that a consumed QGA command stopped: the existing shared transport+uncertainty fence still applies to new polling and independent diagnostics.+There are no persistent generation tombstones; the invalidated owner is removed+only on completion. Other instance/VM keys and ordinary new-lifecycle discovery+remain independent.++`guest_metadata_lifecycle_test.go` covers late synthetic replies at every+enrichment stage, cold/retained caches, explicit invalidation/retirement, late+OS-suppression/version-deferral publication, slot waiting and independent+new-lifecycle recovery. Direct and cluster HTTP clients cover cancellation of+consumed incomplete bodies, no further commands, preserved diagnostic/caller+cooldown and independent VM discovery, without shortening or replaying that+fence. These are source lifecycle/safety controls, not native thaw, restart or+every-volume/History recovery evidence.+`guest_metadata_reservation_test.go` exercises real HTTP enrichment with coldand retained caches, active and expired scheduling holds, early unverified-lockand cancelled-slot exits, independent VMs and ordinary resumption. The existingdiff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.goindex 0530e81759..d17133f990 100644--- a/internal/monitoring/guest_metadata.go+++ b/internal/monitoring/guest_metadata.go@@ -50,6 +50,14 @@ type guestMetadataCacheEntry struct {osInfoSkip bool // Skip OS info calls after repeated failures (refs #692)}+// A cancelled fetch still owns its key until it returns. Cancellation cannot+// prove that a guest command stopped, and its late reply cannot repopulate a+// deliberately invalidated lifecycle. Guarded by guestMetadataLimiterMu.+type guestMetadataFetch struct {+ invalidated bool+ cancel context.CancelFunc+}+func guestMetadataCacheHasUsefulData(entry guestMetadataCacheEntry) bool {return len(entry.ipAddresses) > 0 ||len(entry.networkInterfaces) > 0 ||@@ -112,7 +120,7 @@ func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {// A time-based hold limits retry rate, but is not completion evidence.// Keep ownership until the fetch returns, even if its hold has expired.- if m.guestMetadataInFlight[key] {+ if m.guestMetadataInFlight[key] != nil {return false}if next, ok := m.guestMetadataLimiter[key]; ok && now.Before(next) {@@ -126,10 +134,10 @@ func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {m.guestMetadataLimiter = make(map[string]time.Time)}if m.guestMetadataInFlight == nil {- m.guestMetadataInFlight = make(map[string]bool)+ m.guestMetadataInFlight = make(map[string]*guestMetadataFetch)}m.guestMetadataLimiter[key] = now.Add(hold)- m.guestMetadataInFlight[key] = true+ m.guestMetadataInFlight[key] = &guestMetadataFetch{}return true}@@ -139,6 +147,41 @@ func (m *Monitor) releaseGuestMetadataFetch(key string) {m.guestMetadataLimiterMu.Unlock()}+func (m *Monitor) invalidateGuestMetadataFetchLocked(key string) {+ if fetch := m.guestMetadataInFlight[key]; fetch != nil {+ fetch.invalidated = true+ if fetch.cancel != nil {+ fetch.cancel()+ }+ }+}++func (m *Monitor) guestMetadataFetchInvalidated(key string) bool {+ m.guestMetadataLimiterMu.Lock()+ defer m.guestMetadataLimiterMu.Unlock()+ return m.guestMetadataInFlight[key] != nil && m.guestMetadataInFlight[key].invalidated+}++// Serialize publication with explicit invalidation, using the same lock order+// as the janitor. Both complete metadata and partial OS-command safety state+// belong to the lifecycle that reserved this fetch, not a subsequent one.+func (m *Monitor) updateGuestMetadataFetch(key string, update func(*guestMetadataCacheEntry)) bool {+ m.guestMetadataLimiterMu.Lock()+ defer m.guestMetadataLimiterMu.Unlock()+ if fetch := m.guestMetadataInFlight[key]; fetch == nil || fetch.invalidated {+ return false+ }+ m.guestMetadataMu.Lock()+ defer m.guestMetadataMu.Unlock()+ if m.guestMetadataCache == nil {+ m.guestMetadataCache = make(map[string]guestMetadataCacheEntry)+ }+ entry := m.guestMetadataCache[key]+ update(&entry)+ m.guestMetadataCache[key] = entry+ return true+}+// Activity keeps an actively monitored guest's command-safety state alive even// when it supplies no useful metadata. It cannot renew any observation or make// an otherwise ineligible guest command queryable.@@ -168,6 +211,10 @@ func (m *Monitor) scheduleNextGuestMetadataFetch(key string, now time.Time) {m.rngMu.Unlock()}m.guestMetadataLimiterMu.Lock()+ if fetch := m.guestMetadataInFlight[key]; fetch != nil && fetch.invalidated {+ m.guestMetadataLimiterMu.Unlock()+ return+ }m.guestMetadataLimiter[key] = now.Add(interval)m.guestMetadataLimiterMu.Unlock()}@@ -178,6 +225,10 @@ func (m *Monitor) scheduleGuestMetadataFetchForEntry(key string, now time.Time,}if guestMetadataCacheEntryTTL(entry) == guestMetadataEmptyTTL {m.guestMetadataLimiterMu.Lock()+ if fetch := m.guestMetadataInFlight[key]; fetch != nil && fetch.invalidated {+ m.guestMetadataLimiterMu.Unlock()+ return+ }m.guestMetadataLimiter[key] = now.Add(guestMetadataEmptyTTL)m.guestMetadataLimiterMu.Unlock()return@@ -194,6 +245,10 @@ func (m *Monitor) deferGuestMetadataRetry(key string, now time.Time) {backoff = config.DefaultGuestMetadataRetryBackoff}m.guestMetadataLimiterMu.Lock()+ if fetch := m.guestMetadataInFlight[key]; fetch != nil && fetch.invalidated {+ m.guestMetadataLimiterMu.Unlock()+ return+ }m.guestMetadataLimiter[key] = now.Add(backoff)m.guestMetadataLimiterMu.Unlock()}@@ -271,6 +326,7 @@ func (m *Monitor) retryGuestAgentCall(ctx context.Context, timeout time.Duration// The final result reports command deferral separately from retained metadata.// A previously successful filesystem must not conceal a later shared pause.+// Invalidation also defers this fetch: no old-lifecycle metadata is usable.func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus, allowWithoutStatus bool) ([]string, []models.GuestNetworkInterface, string, string, string, bool) {key := guestMetadataCacheKey(instanceName, nodeName, vmid)now := time.Now()@@ -311,6 +367,24 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI}defer m.releaseGuestMetadataFetch(key)+ // Attach cancellation and re-read the fallback in the reservation's lifecycle.+ // Invalidation may have happened between the initial cache read and admission;+ // neither that old snapshot nor an obsolete context can survive this check.+ fetchCtx, cancelFetch := context.WithCancel(ctx)+ defer cancelFetch()+ m.guestMetadataLimiterMu.Lock()+ fetch := m.guestMetadataInFlight[key]+ if fetch.invalidated {+ m.guestMetadataLimiterMu.Unlock()+ return nil, nil, "", "", "", true+ }+ fetch.cancel = cancelFetch+ m.guestMetadataMu.RLock()+ cached = m.guestMetadataCache[key]+ m.guestMetadataMu.RUnlock()+ m.guestMetadataLimiterMu.Unlock()+ ctx = fetchCtx+// Start with cached values as fallback in case new calls failipAddresses := cloneStringSlice(cached.ipAddresses)networkIfaces := cloneGuestNetworkInterfaces(cached.networkInterfaces)@@ -321,16 +395,25 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientIif reserved {if !m.acquireGuestMetadataSlot(ctx) {+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }m.deferGuestMetadataRetry(key, time.Now())return ipAddresses, networkIfaces, osName, osVersion, agentVersion, false}defer m.releaseGuestMetadataSlot()}+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }// Network interfaces with configurable timeout and retry (refs #592)interfaces, err := m.retryGuestAgentCall(ctx, m.guestAgentNetworkTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {return client.GetVMNetworkInterfaces(ctx, nodeName, vmid)})+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }if errors.Is(err, proxmox.ErrGuestAgentDeferred) {m.deferGuestMetadataRetry(key, time.Now())return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true@@ -374,6 +457,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientIagentInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentOSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {return client.GetVMAgentInfo(ctx, nodeName, vmid)})+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }if errors.Is(err, proxmox.ErrGuestAgentDeferred) {m.deferGuestMetadataRetry(key, time.Now())return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true@@ -449,23 +535,26 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI}// Agent version with configurable timeout and retry (refs #592)+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }versionRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentVersionTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {return client.GetVMAgentVersion(ctx, nodeName, vmid)})+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }if errors.Is(err, proxmox.ErrGuestAgentDeferred) {// A later shared pause does not undo a completed OS-info outcome.// Retain its safety suppression without publishing partial metadata or// renewing the age of last-known guest-agent evidence.- m.guestMetadataMu.Lock()- if m.guestMetadataCache == nil {- m.guestMetadataCache = make(map[string]guestMetadataCacheEntry)+ if !m.updateGuestMetadataFetch(key, func(entry *guestMetadataCacheEntry) {+ entry.osInfoFailureCount = osInfoFailureCount+ entry.osInfoSkip = osInfoSkip+ entry.windowsGuest, entry.osInfoObservedAt = windowsGuest, osInfoObservedAt+ }) {+ return nil, nil, "", "", "", true}- entry := m.guestMetadataCache[key]- entry.osInfoFailureCount = osInfoFailureCount- entry.osInfoSkip = osInfoSkip- entry.windowsGuest, entry.osInfoObservedAt = windowsGuest, osInfoObservedAt- m.guestMetadataCache[key] = entry- m.guestMetadataMu.Unlock()m.deferGuestMetadataRetry(key, time.Now())return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true}@@ -503,12 +592,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientIosInfoSkip: osInfoSkip,}- m.guestMetadataMu.Lock()- if m.guestMetadataCache == nil {- m.guestMetadataCache = make(map[string]guestMetadataCacheEntry)+ if !m.updateGuestMetadataFetch(key, func(current *guestMetadataCacheEntry) { *current = entry }) {+ return nil, nil, "", "", "", true}- m.guestMetadataCache[key] = entry- m.guestMetadataMu.Unlock()if reserved {if metadataObserved {m.scheduleGuestMetadataFetchForEntry(key, time.Now(), entry)@@ -516,6 +602,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientIm.deferGuestMetadataRetry(key, time.Now())}}+ if m.guestMetadataFetchInvalidated(key) {+ return nil, nil, "", "", "", true+ }return ipAddresses, networkIfaces, osName, osVersion, agentVersion, false}@@ -530,6 +619,11 @@ func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid in}key := guestMetadataCacheKey(instanceName, nodeName, vmid)+ // Keep the old owner until completion, but cancel its context and exclude all+ // subsequent publication/scheduling. Never release the shared command fence.+ m.guestMetadataLimiterMu.Lock()+ defer m.guestMetadataLimiterMu.Unlock()+ m.invalidateGuestMetadataFetchLocked(key)m.guestMetadataMu.Lock()if m.guestMetadataCache != nil {delete(m.guestMetadataCache, key)@@ -538,10 +632,8 @@ func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid in// Deliberate invalidation when the agent is unavailable starts a new// metadata lifecycle. Do not carry its obsolete refresh/backoff deadline// into a later available poll, or discard an outstanding fetch's ownership.- m.guestMetadataLimiterMu.Lock()delete(m.guestMetadataLimiter, key)delete(m.guestMetadataLastAccess, key)- m.guestMetadataLimiterMu.Unlock()}func cloneStringSlice(src []string) []string {diff --git a/internal/monitoring/guest_metadata_lifecycle_test.go b/internal/monitoring/guest_metadata_lifecycle_test.goindex b346670a45..721c8ac9b9 100644--- a/internal/monitoring/guest_metadata_lifecycle_test.go+++ b/internal/monitoring/guest_metadata_lifecycle_test.go@@ -4,7 +4,10 @@ import ("context""errors""fmt"+ "net/http"+ "net/http/httptest""reflect"+ "strings""sync""testing""time"@@ -24,6 +27,165 @@ type metadataLifecycleClient struct {resume chan struct{}}+func TestGuestMetadataLifecycleCancellationKeepsHTTPFence(t *testing.T) {+ for _, kind := range []string{"client", "cluster-client"} {+ for _, disposition := range []string{"invalidate", "retire"} {+ for _, phase := range []string{"network-get-interfaces", "get-osinfo", "info"} {+ t.Run(kind+"/"+disposition+"/"+phase, func(t *testing.T) {+ entered, resume := make(chan struct{}, 1), make(chan struct{})+ var unblock sync.Once+ var mu sync.Mutex+ var trace []string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/nodes") {+ fmt.Fprint(w, `{"data":[]}`)+ return+ }+ if strings.Contains(r.URL.Path, "/qemu/105/agent/") {+ mu.Lock()+ trace = append(trace, r.URL.Path)+ mu.Unlock()+ if strings.HasSuffix(r.URL.Path, "/"+phase) {+ // Headers arrive, but the consumed command's body+ // remains incomplete even after its client cancels.+ w.WriteHeader(http.StatusOK)+ w.(http.Flusher).Flush()+ entered <- struct{}{}+ <-resume+ return+ }+ }+ metadataReservationPayload(w, r)+ }))+ defer server.Close()+ defer unblock.Do(func() { close(resume) })+ client := windowsBootstrapClient(t, server.URL, kind)+ const instance = "http-lifecycle"+ key := guestMetadataCacheKey(instance, "node", 105)+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))}, guestMetadataSlots: make(chan struct{}, 1)}+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}+ finished := make(chan struct{})+ var oldName string+ var paused bool+ go func() {+ defer close(finished)+ _, _, oldName, _, _, paused = m.fetchGuestAgentMetadata(context.Background(), client, instance, "node", "guest", 105, status, false)+ }()+ defer func() {+ unblock.Do(func() { close(resume) })+ select {+ case <-finished:+ case <-time.After(5 * time.Second):+ t.Error("cancelled HTTP fetch did not finish")+ }+ }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ t.Fatal("synthetic HTTP body did not reach selected phase")+ }+ if disposition == "invalidate" {+ m.clearGuestMetadataCache(instance, "node", 105)+ } else {+ m.retirePVEInstanceRuntime(instance)+ }+ select {+ case <-finished:+ case <-time.After(5 * time.Second):+ t.Fatal("obsolete HTTP fetch waited for a consumed command instead of cancelling")+ }+ if !paused || oldName != "" || len(m.guestMetadataSlots) != 0 || !m.guestMetadataLimiter[key].IsZero() {+ t.Error("cancelled fetch published identity/backoff or leaked its slot")+ }+ if _, exists := m.guestMetadataCache[key]; exists {+ t.Error("cancelled fetch resurrected cache state")+ }+ assertMetadataReservationReleased(t, m)+ mu.Lock()+ before := append([]string(nil), trace...)+ mu.Unlock()+ // A new diagnostic client and the new metadata lifecycle must+ // honour the original transport's uncertain-completion fence.+ diagnostic := windowsBootstrapClient(t, server.URL, "client")+ if _, err := diagnostic.GetVMAgentInfo(context.Background(), "node", 105); proxmox.GuestAgentDeferredReason(err) != "agent-cooldown" {+ t.Errorf("cancellation discarded shared uncertainty: %v", err)+ }+ _, _, _, _, _, stillPaused := m.fetchGuestAgentMetadata(context.Background(), client, instance, "node", "guest", 105, status, false)+ if !stillPaused {+ t.Error("new lifecycle bypassed uncertain command completion")+ }+ mu.Lock()+ unchanged := reflect.DeepEqual(trace, before)+ mu.Unlock()+ wantCommands := 1+ if phase == "get-osinfo" {+ wantCommands = 2+ } else if phase == "info" {+ wantCommands = 3+ }+ if !unchanged || len(before) != wantCommands {+ t.Error("obsolete/cooldown fetch sent an additional or repeated command")+ }+ // Independent VM discovery remains useful, without shortening+ // or replaying the cancelled VM's uncertainty fence.+ _, _, name, _, agent, otherPaused := m.fetchGuestAgentMetadata(context.Background(), client, instance, "node", "other", 106, status, false)+ if otherPaused || name != "FixtureOS" || agent != "2.0" {+ t.Error("cancelled lifecycle blocked an independent VM")+ }+ unblock.Do(func() { close(resume) })+ })+ }+ }+ }+}++func TestGuestMetadataLifecycleCancelsSlotWait(t *testing.T) {+ key := guestMetadataCacheKey("slot-lifecycle", "node", 105)+ m := &Monitor{guestMetadataSlots: make(chan struct{}, 1)}+ m.guestMetadataSlots <- struct{}{} // A different fetch owns this slot.+ client := &metadataSuppressionClient{os: map[string]interface{}{"name": "CurrentOS"}}+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}+ finished := make(chan bool, 1)+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ go func() {+ _, _, _, _, _, paused := m.fetchGuestAgentMetadata(ctx, client, "slot-lifecycle", "node", "guest", 105, status, false)+ finished <- paused+ }()+ deadline := time.Now().Add(5 * time.Second)+ for {+ m.guestMetadataLimiterMu.Lock()+ fetch := m.guestMetadataInFlight[key]+ waiting := fetch != nil && fetch.cancel != nil+ m.guestMetadataLimiterMu.Unlock()+ if waiting {+ break+ }+ if time.Now().After(deadline) {+ t.Fatal("metadata fetch did not reserve its slot-wait lifecycle")+ }+ time.Sleep(time.Millisecond)+ }+ m.clearGuestMetadataCache("slot-lifecycle", "node", 105)+ select {+ case paused := <-finished:+ if !paused {+ t.Error("invalidated slot waiter did not defer")+ }+ case <-time.After(5 * time.Second):+ t.Fatal("invalidation did not cancel slot waiting")+ }+ if len(m.guestMetadataSlots) != 1 || client.networkCalls+client.osCalls+client.versionCalls != 0 || len(m.guestMetadataCache) != 0 || !m.guestMetadataLimiter[key].IsZero() {+ t.Error("invalidated slot waiter consumed another owner's slot, sent a command or published state")+ }+ assertMetadataReservationReleased(t, m)+ <-m.guestMetadataSlots+ _, _, name, _, _, paused := m.fetchGuestAgentMetadata(context.Background(), client, "slot-lifecycle", "node", "guest", 105, status, false)+ if paused || name != "CurrentOS" || len(m.guestMetadataSlots) != 0 {+ t.Error("ordinary discovery did not resume after slot-wait invalidation")+ }+}+func (c *metadataLifecycleClient) command(ctx context.Context, name string) {c.mu.Lock()c.commands = append(c.commands, name)diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 19fcc18a65..3dd13e6d1e 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -1405,8 +1405,8 @@ type Monitor struct {guestMetadataCache map[string]guestMetadataCacheEntryguestMetadataLimiterMu sync.MutexguestMetadataLimiter map[string]time.Time- guestMetadataInFlight map[string]bool // Guarded by guestMetadataLimiterMu; released only on fetch completion.- guestMetadataLastAccess map[string]time.Time // Guarded by guestMetadataLimiterMu; cache activity, never guest evidence.+ guestMetadataInFlight map[string]*guestMetadataFetch // Guarded by guestMetadataLimiterMu; retained until fetch completion, even after invalidation.+ guestMetadataLastAccess map[string]time.Time // Guarded by guestMetadataLimiterMu; cache activity, never guest evidence.guestMetadataSlots chan struct{}guestMetadataMinRefresh time.DurationguestMetadataRefreshJitter time.Durationdiff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.goindex f5fb3485f8..2451bba09a 100644--- a/internal/monitoring/monitor_agents.go+++ b/internal/monitoring/monitor_agents.go@@ -5003,7 +5003,7 @@ func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {if entry.fetchedAt.After(activityAt) {activityAt = entry.fetchedAt // Also supports legacy/seeded cache entries.}- if !m.guestMetadataInFlight[key] && now.Sub(activityAt) > maxAge {+ if m.guestMetadataInFlight[key] == nil && now.Sub(activityAt) > maxAge {delete(m.guestMetadataCache, key)log.Debug().Str("key", key).@@ -5014,13 +5014,13 @@ func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {m.guestMetadataMu.Unlock()for key, accessedAt := range m.guestMetadataLastAccess {- if !m.guestMetadataInFlight[key] && now.Sub(accessedAt) > maxAge {+ if m.guestMetadataInFlight[key] == nil && now.Sub(accessedAt) > maxAge {delete(m.guestMetadataLastAccess, key)}}for key, nextAllowed := range m.guestMetadataLimiter {// Keep near-term limiter state; remove long-idle keys.- if !m.guestMetadataInFlight[key] && now.Sub(nextAllowed) > maxAge {+ if m.guestMetadataInFlight[key] == nil && now.Sub(nextAllowed) > maxAge {delete(m.guestMetadataLimiter, key)log.Debug().Str("key", key).diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.goindex e7915ad17a..798572ce60 100644--- a/internal/monitoring/monitor_cluster_dedupe.go+++ b/internal/monitoring/monitor_cluster_dedupe.go@@ -138,6 +138,12 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {}m.diagMu.Unlock()+ m.guestMetadataLimiterMu.Lock()+ for key := range m.guestMetadataInFlight {+ if strings.HasPrefix(key, instanceName+"|") {+ m.invalidateGuestMetadataFetchLocked(key)+ }+ }m.guestMetadataMu.Lock()for key := range m.guestMetadataCache {if strings.HasPrefix(key, instanceName+"|") {@@ -146,7 +152,6 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {}m.guestMetadataMu.Unlock()- m.guestMetadataLimiterMu.Lock()for key := range m.guestMetadataLimiter {if strings.HasPrefix(key, instanceName+"|") {delete(m.guestMetadataLimiter, key)diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.goindex 44b4b4ee48..8c5b891047 100644--- a/internal/monitoring/monitor_host_agents_test.go+++ b/internal/monitoring/monitor_host_agents_test.go@@ -8339,7 +8339,7 @@ func TestGuestMetadataIdleCleanupContract(t *testing.T) {},guestMetadataLastAccess: map[string]time.Time{"active-empty": now, "active-retained": now, "inactive": stale, "only-active": now, "only-idle": stale, "inflight": stale},guestMetadataLimiter: map[string]time.Time{"active-empty": now.Add(time.Minute), "inactive": stale, "inflight": stale},- guestMetadataInFlight: map[string]bool{"inflight": true},+ guestMetadataInFlight: map[string]*guestMetadataFetch{"inflight": {}},}m.cleanupGuestMetadataCache(now)for _, key := range []string{"active-empty", "active-retained", "legacy-fresh", "inflight"} {@@ -8357,7 +8357,7 @@ func TestGuestMetadataIdleCleanupContract(t *testing.T) {t.Errorf("idle bookkeeping %s retained", key)}}- if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || !m.guestMetadataInFlight["inflight"] {+ if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || m.guestMetadataInFlight["inflight"] == nil {t.Error("janitor changed current activity, backoff or in-flight ownership")}empty := m.guestMetadataCache["active-empty"]@@ -8393,7 +8393,7 @@ func TestGuestMetadataActivityInvalidationContract(t *testing.T) {guestMetadataCache: map[string]guestMetadataCacheEntry{key: {osInfoSkip: true}, sibling: {osInfoSkip: true}},guestMetadataLimiter: map[string]time.Time{key: time.Now().Add(time.Hour), sibling: time.Now().Add(time.Hour)},guestMetadataLastAccess: map[string]time.Time{key: time.Now(), sibling: time.Now()},- guestMetadataInFlight: map[string]bool{key: true},+ guestMetadataInFlight: map[string]*guestMetadataFetch{key: {}},}m.clearGuestMetadataCache("activity", "node", 105)if _, exists := m.guestMetadataCache[key]; exists {@@ -8405,7 +8405,7 @@ func TestGuestMetadataActivityInvalidationContract(t *testing.T) {if _, exists := m.guestMetadataLastAccess[key]; exists {t.Error("explicit invalidation retained activity")}- if !m.guestMetadataInFlight[key] || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {+ if m.guestMetadataInFlight[key] == nil || !m.guestMetadataInFlight[key].invalidated || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {t.Error("invalidation discarded ownership or another instance")}m.guestMetadataLastAccess[key] = time.Now()@@ -8416,7 +8416,7 @@ func TestGuestMetadataActivityInvalidationContract(t *testing.T) {t.Error("retired instance retained activity bookkeeping")}}- if m.guestMetadataLastAccess[sibling].IsZero() || !m.guestMetadataInFlight[key] {+ if m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataInFlight[key] == nil || !m.guestMetadataInFlight[key].invalidated {t.Error("retirement crossed instance scope or discarded active ownership")}}