Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Discard guest metadata from invalidated fetch lifecycles

pulse-triage[bot]2488be4e17fc
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 6865c10571..a635beac44 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -91,6 +91,17 @@ activity invalidation/concurrent-cleanup controls in
`internal/monitoring/monitor_host_agents_test.go` verify this shared boundary;
they are not installed agent or native guest-recovery proof.
+Explicit metadata invalidation and PVE instance retirement cancel an active
+enrichment context without releasing its owner early. Late replies cannot
+repopulate the old lifecycle's cache, OS-command safety state or retry timing;
+new polling and diagnostics still honour the shared consumed-command uncertainty
+fence. This remains an internal monitor lifecycle, with no change to agent
+identity, credentials or the command protocol.
+`internal/monitoring/guest_metadata_lifecycle_test.go` verifies cold/retained
+late replies, both invalidation routes, direct/cluster HTTP cancellation,
+independent keys and ordinary new-lifecycle recovery. It is not installed
+removal, native guest recovery or evidence that a cancelled guest command stopped.
+
The shared VM model's `GuestAgentEvidence` is monitor-owned source-time
coordination, not agent input or a public field. It prevents retained guest
identity from renewing fallback QGA eligibility. Agent enrollment, token and
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 27edae546b..500b1e0351 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -971,6 +971,28 @@ cache invalidation on agent unavailability also discards its obsolete scheduling
deadline, so a later available poll can begin a fresh metadata lifecycle; it
never discards an outstanding fetch owner or the shared Proxmox command fence.
+Invalidation and PVE instance retirement also cancel that owner's context and
+mark its lifecycle obsolete until the fetch actually returns. A late network,
+OS or version reply cannot recreate identity, OS-command suppression, useful/OS
+evidence or refresh/backoff deadlines after either operation. Publication and
+invalidation share one lock order, and fallback metadata is re-read within the
+reserved lifecycle rather than copied from the pre-admission cache snapshot.
+Slot waiting is cancelled without releasing another fetch's slot. Cancellation
+is not proof that a consumed QGA command stopped: the existing shared transport
+uncertainty fence still applies to new polling and independent diagnostics.
+There are no persistent generation tombstones; the invalidated owner is removed
+only on completion. Other instance/VM keys and ordinary new-lifecycle discovery
+remain independent.
+
+`guest_metadata_lifecycle_test.go` covers late synthetic replies at every
+enrichment stage, cold/retained caches, explicit invalidation/retirement, late
+OS-suppression/version-deferral publication, slot waiting and independent
+new-lifecycle recovery. Direct and cluster HTTP clients cover cancellation of
+consumed incomplete bodies, no further commands, preserved diagnostic/caller
+cooldown and independent VM discovery, without shortening or replaying that
+fence. These are source lifecycle/safety controls, not native thaw, restart or
+every-volume/History recovery evidence.
+
`guest_metadata_reservation_test.go` exercises real HTTP enrichment with cold
and retained caches, active and expired scheduling holds, early unverified-lock
and cancelled-slot exits, independent VMs and ordinary resumption. The existing
diff --git a/internal/monitoring/guest_metadata.go b/internal/monitoring/guest_metadata.go
index 0530e81759..d17133f990 100644
--- a/internal/monitoring/guest_metadata.go
+++ b/internal/monitoring/guest_metadata.go
@@ -50,6 +50,14 @@ type guestMetadataCacheEntry struct {
osInfoSkip bool // Skip OS info calls after repeated failures (refs #692)
}
+// A cancelled fetch still owns its key until it returns. Cancellation cannot
+// prove that a guest command stopped, and its late reply cannot repopulate a
+// deliberately invalidated lifecycle. Guarded by guestMetadataLimiterMu.
+type guestMetadataFetch struct {
+ invalidated bool
+ cancel context.CancelFunc
+}
+
func guestMetadataCacheHasUsefulData(entry guestMetadataCacheEntry) bool {
return len(entry.ipAddresses) > 0 ||
len(entry.networkInterfaces) > 0 ||
@@ -112,7 +120,7 @@ func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {
// A time-based hold limits retry rate, but is not completion evidence.
// Keep ownership until the fetch returns, even if its hold has expired.
- if m.guestMetadataInFlight[key] {
+ if m.guestMetadataInFlight[key] != nil {
return false
}
if next, ok := m.guestMetadataLimiter[key]; ok && now.Before(next) {
@@ -126,10 +134,10 @@ func (m *Monitor) tryReserveGuestMetadataFetch(key string, now time.Time) bool {
m.guestMetadataLimiter = make(map[string]time.Time)
}
if m.guestMetadataInFlight == nil {
- m.guestMetadataInFlight = make(map[string]bool)
+ m.guestMetadataInFlight = make(map[string]*guestMetadataFetch)
}
m.guestMetadataLimiter[key] = now.Add(hold)
- m.guestMetadataInFlight[key] = true
+ m.guestMetadataInFlight[key] = &guestMetadataFetch{}
return true
}
@@ -139,6 +147,41 @@ func (m *Monitor) releaseGuestMetadataFetch(key string) {
m.guestMetadataLimiterMu.Unlock()
}
+func (m *Monitor) invalidateGuestMetadataFetchLocked(key string) {
+ if fetch := m.guestMetadataInFlight[key]; fetch != nil {
+ fetch.invalidated = true
+ if fetch.cancel != nil {
+ fetch.cancel()
+ }
+ }
+}
+
+func (m *Monitor) guestMetadataFetchInvalidated(key string) bool {
+ m.guestMetadataLimiterMu.Lock()
+ defer m.guestMetadataLimiterMu.Unlock()
+ return m.guestMetadataInFlight[key] != nil && m.guestMetadataInFlight[key].invalidated
+}
+
+// Serialize publication with explicit invalidation, using the same lock order
+// as the janitor. Both complete metadata and partial OS-command safety state
+// belong to the lifecycle that reserved this fetch, not a subsequent one.
+func (m *Monitor) updateGuestMetadataFetch(key string, update func(*guestMetadataCacheEntry)) bool {
+ m.guestMetadataLimiterMu.Lock()
+ defer m.guestMetadataLimiterMu.Unlock()
+ if fetch := m.guestMetadataInFlight[key]; fetch == nil || fetch.invalidated {
+ return false
+ }
+ m.guestMetadataMu.Lock()
+ defer m.guestMetadataMu.Unlock()
+ if m.guestMetadataCache == nil {
+ m.guestMetadataCache = make(map[string]guestMetadataCacheEntry)
+ }
+ entry := m.guestMetadataCache[key]
+ update(&entry)
+ m.guestMetadataCache[key] = entry
+ return true
+}
+
// Activity keeps an actively monitored guest's command-safety state alive even
// when it supplies no useful metadata. It cannot renew any observation or make
// an otherwise ineligible guest command queryable.
@@ -168,6 +211,10 @@ func (m *Monitor) scheduleNextGuestMetadataFetch(key string, now time.Time) {
m.rngMu.Unlock()
}
m.guestMetadataLimiterMu.Lock()
+ if fetch := m.guestMetadataInFlight[key]; fetch != nil && fetch.invalidated {
+ m.guestMetadataLimiterMu.Unlock()
+ return
+ }
m.guestMetadataLimiter[key] = now.Add(interval)
m.guestMetadataLimiterMu.Unlock()
}
@@ -178,6 +225,10 @@ func (m *Monitor) scheduleGuestMetadataFetchForEntry(key string, now time.Time,
}
if guestMetadataCacheEntryTTL(entry) == guestMetadataEmptyTTL {
m.guestMetadataLimiterMu.Lock()
+ if fetch := m.guestMetadataInFlight[key]; fetch != nil && fetch.invalidated {
+ m.guestMetadataLimiterMu.Unlock()
+ return
+ }
m.guestMetadataLimiter[key] = now.Add(guestMetadataEmptyTTL)
m.guestMetadataLimiterMu.Unlock()
return
@@ -194,6 +245,10 @@ func (m *Monitor) deferGuestMetadataRetry(key string, now time.Time) {
backoff = config.DefaultGuestMetadataRetryBackoff
}
m.guestMetadataLimiterMu.Lock()
+ if fetch := m.guestMetadataInFlight[key]; fetch != nil && fetch.invalidated {
+ m.guestMetadataLimiterMu.Unlock()
+ return
+ }
m.guestMetadataLimiter[key] = now.Add(backoff)
m.guestMetadataLimiterMu.Unlock()
}
@@ -271,6 +326,7 @@ func (m *Monitor) retryGuestAgentCall(ctx context.Context, timeout time.Duration
// The final result reports command deferral separately from retained metadata.
// A previously successful filesystem must not conceal a later shared pause.
+// Invalidation also defers this fetch: no old-lifecycle metadata is usable.
func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientInterface, instanceName, nodeName, vmName string, vmid int, vmStatus *proxmox.VMStatus, allowWithoutStatus bool) ([]string, []models.GuestNetworkInterface, string, string, string, bool) {
key := guestMetadataCacheKey(instanceName, nodeName, vmid)
now := time.Now()
@@ -311,6 +367,24 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
}
defer m.releaseGuestMetadataFetch(key)
+ // Attach cancellation and re-read the fallback in the reservation's lifecycle.
+ // Invalidation may have happened between the initial cache read and admission;
+ // neither that old snapshot nor an obsolete context can survive this check.
+ fetchCtx, cancelFetch := context.WithCancel(ctx)
+ defer cancelFetch()
+ m.guestMetadataLimiterMu.Lock()
+ fetch := m.guestMetadataInFlight[key]
+ if fetch.invalidated {
+ m.guestMetadataLimiterMu.Unlock()
+ return nil, nil, "", "", "", true
+ }
+ fetch.cancel = cancelFetch
+ m.guestMetadataMu.RLock()
+ cached = m.guestMetadataCache[key]
+ m.guestMetadataMu.RUnlock()
+ m.guestMetadataLimiterMu.Unlock()
+ ctx = fetchCtx
+
// Start with cached values as fallback in case new calls fail
ipAddresses := cloneStringSlice(cached.ipAddresses)
networkIfaces := cloneGuestNetworkInterfaces(cached.networkInterfaces)
@@ -321,16 +395,25 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
if reserved {
if !m.acquireGuestMetadataSlot(ctx) {
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
m.deferGuestMetadataRetry(key, time.Now())
return ipAddresses, networkIfaces, osName, osVersion, agentVersion, false
}
defer m.releaseGuestMetadataSlot()
}
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
// Network interfaces with configurable timeout and retry (refs #592)
interfaces, err := m.retryGuestAgentCall(ctx, m.guestAgentNetworkTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
return client.GetVMNetworkInterfaces(ctx, nodeName, vmid)
})
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
if errors.Is(err, proxmox.ErrGuestAgentDeferred) {
m.deferGuestMetadataRetry(key, time.Now())
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true
@@ -374,6 +457,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
agentInfoRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentOSInfoTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
return client.GetVMAgentInfo(ctx, nodeName, vmid)
})
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
if errors.Is(err, proxmox.ErrGuestAgentDeferred) {
m.deferGuestMetadataRetry(key, time.Now())
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true
@@ -449,23 +535,26 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
}
// Agent version with configurable timeout and retry (refs #592)
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
versionRaw, err := m.retryGuestAgentCall(ctx, m.guestAgentVersionTimeout, m.guestAgentRetries, func(ctx context.Context) (interface{}, error) {
return client.GetVMAgentVersion(ctx, nodeName, vmid)
})
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
if errors.Is(err, proxmox.ErrGuestAgentDeferred) {
// A later shared pause does not undo a completed OS-info outcome.
// Retain its safety suppression without publishing partial metadata or
// renewing the age of last-known guest-agent evidence.
- m.guestMetadataMu.Lock()
- if m.guestMetadataCache == nil {
- m.guestMetadataCache = make(map[string]guestMetadataCacheEntry)
+ if !m.updateGuestMetadataFetch(key, func(entry *guestMetadataCacheEntry) {
+ entry.osInfoFailureCount = osInfoFailureCount
+ entry.osInfoSkip = osInfoSkip
+ entry.windowsGuest, entry.osInfoObservedAt = windowsGuest, osInfoObservedAt
+ }) {
+ return nil, nil, "", "", "", true
}
- entry := m.guestMetadataCache[key]
- entry.osInfoFailureCount = osInfoFailureCount
- entry.osInfoSkip = osInfoSkip
- entry.windowsGuest, entry.osInfoObservedAt = windowsGuest, osInfoObservedAt
- m.guestMetadataCache[key] = entry
- m.guestMetadataMu.Unlock()
m.deferGuestMetadataRetry(key, time.Now())
return cloneStringSlice(cached.ipAddresses), cloneGuestNetworkInterfaces(cached.networkInterfaces), cached.osName, cached.osVersion, cached.agentVersion, true
}
@@ -503,12 +592,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
osInfoSkip: osInfoSkip,
}
- m.guestMetadataMu.Lock()
- if m.guestMetadataCache == nil {
- m.guestMetadataCache = make(map[string]guestMetadataCacheEntry)
+ if !m.updateGuestMetadataFetch(key, func(current *guestMetadataCacheEntry) { *current = entry }) {
+ return nil, nil, "", "", "", true
}
- m.guestMetadataCache[key] = entry
- m.guestMetadataMu.Unlock()
if reserved {
if metadataObserved {
m.scheduleGuestMetadataFetchForEntry(key, time.Now(), entry)
@@ -516,6 +602,9 @@ func (m *Monitor) fetchGuestAgentMetadata(ctx context.Context, client PVEClientI
m.deferGuestMetadataRetry(key, time.Now())
}
}
+ if m.guestMetadataFetchInvalidated(key) {
+ return nil, nil, "", "", "", true
+ }
return ipAddresses, networkIfaces, osName, osVersion, agentVersion, false
}
@@ -530,6 +619,11 @@ func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid in
}
key := guestMetadataCacheKey(instanceName, nodeName, vmid)
+ // Keep the old owner until completion, but cancel its context and exclude all
+ // subsequent publication/scheduling. Never release the shared command fence.
+ m.guestMetadataLimiterMu.Lock()
+ defer m.guestMetadataLimiterMu.Unlock()
+ m.invalidateGuestMetadataFetchLocked(key)
m.guestMetadataMu.Lock()
if m.guestMetadataCache != nil {
delete(m.guestMetadataCache, key)
@@ -538,10 +632,8 @@ func (m *Monitor) clearGuestMetadataCache(instanceName, nodeName string, vmid in
// Deliberate invalidation when the agent is unavailable starts a new
// metadata lifecycle. Do not carry its obsolete refresh/backoff deadline
// into a later available poll, or discard an outstanding fetch's ownership.
- m.guestMetadataLimiterMu.Lock()
delete(m.guestMetadataLimiter, key)
delete(m.guestMetadataLastAccess, key)
- m.guestMetadataLimiterMu.Unlock()
}
func cloneStringSlice(src []string) []string {
diff --git a/internal/monitoring/guest_metadata_lifecycle_test.go b/internal/monitoring/guest_metadata_lifecycle_test.go
index b346670a45..721c8ac9b9 100644
--- a/internal/monitoring/guest_metadata_lifecycle_test.go
+++ b/internal/monitoring/guest_metadata_lifecycle_test.go
@@ -4,7 +4,10 @@ import (
"context"
"errors"
"fmt"
+ "net/http"
+ "net/http/httptest"
"reflect"
+ "strings"
"sync"
"testing"
"time"
@@ -24,6 +27,165 @@ type metadataLifecycleClient struct {
resume chan struct{}
}
+func TestGuestMetadataLifecycleCancellationKeepsHTTPFence(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, disposition := range []string{"invalidate", "retire"} {
+ for _, phase := range []string{"network-get-interfaces", "get-osinfo", "info"} {
+ t.Run(kind+"/"+disposition+"/"+phase, func(t *testing.T) {
+ entered, resume := make(chan struct{}, 1), make(chan struct{})
+ var unblock sync.Once
+ var mu sync.Mutex
+ var trace []string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/nodes") {
+ fmt.Fprint(w, `{"data":[]}`)
+ return
+ }
+ if strings.Contains(r.URL.Path, "/qemu/105/agent/") {
+ mu.Lock()
+ trace = append(trace, r.URL.Path)
+ mu.Unlock()
+ if strings.HasSuffix(r.URL.Path, "/"+phase) {
+ // Headers arrive, but the consumed command's body
+ // remains incomplete even after its client cancels.
+ w.WriteHeader(http.StatusOK)
+ w.(http.Flusher).Flush()
+ entered <- struct{}{}
+ <-resume
+ return
+ }
+ }
+ metadataReservationPayload(w, r)
+ }))
+ defer server.Close()
+ defer unblock.Do(func() { close(resume) })
+ client := windowsBootstrapClient(t, server.URL, kind)
+ const instance = "http-lifecycle"
+ key := guestMetadataCacheKey(instance, "node", 105)
+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{key: metadataObservationFixture(time.Now().Add(-2 * guestMetadataCacheTTL))}, guestMetadataSlots: make(chan struct{}, 1)}
+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
+ finished := make(chan struct{})
+ var oldName string
+ var paused bool
+ go func() {
+ defer close(finished)
+ _, _, oldName, _, _, paused = m.fetchGuestAgentMetadata(context.Background(), client, instance, "node", "guest", 105, status, false)
+ }()
+ defer func() {
+ unblock.Do(func() { close(resume) })
+ select {
+ case <-finished:
+ case <-time.After(5 * time.Second):
+ t.Error("cancelled HTTP fetch did not finish")
+ }
+ }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("synthetic HTTP body did not reach selected phase")
+ }
+ if disposition == "invalidate" {
+ m.clearGuestMetadataCache(instance, "node", 105)
+ } else {
+ m.retirePVEInstanceRuntime(instance)
+ }
+ select {
+ case <-finished:
+ case <-time.After(5 * time.Second):
+ t.Fatal("obsolete HTTP fetch waited for a consumed command instead of cancelling")
+ }
+ if !paused || oldName != "" || len(m.guestMetadataSlots) != 0 || !m.guestMetadataLimiter[key].IsZero() {
+ t.Error("cancelled fetch published identity/backoff or leaked its slot")
+ }
+ if _, exists := m.guestMetadataCache[key]; exists {
+ t.Error("cancelled fetch resurrected cache state")
+ }
+ assertMetadataReservationReleased(t, m)
+ mu.Lock()
+ before := append([]string(nil), trace...)
+ mu.Unlock()
+ // A new diagnostic client and the new metadata lifecycle must
+ // honour the original transport's uncertain-completion fence.
+ diagnostic := windowsBootstrapClient(t, server.URL, "client")
+ if _, err := diagnostic.GetVMAgentInfo(context.Background(), "node", 105); proxmox.GuestAgentDeferredReason(err) != "agent-cooldown" {
+ t.Errorf("cancellation discarded shared uncertainty: %v", err)
+ }
+ _, _, _, _, _, stillPaused := m.fetchGuestAgentMetadata(context.Background(), client, instance, "node", "guest", 105, status, false)
+ if !stillPaused {
+ t.Error("new lifecycle bypassed uncertain command completion")
+ }
+ mu.Lock()
+ unchanged := reflect.DeepEqual(trace, before)
+ mu.Unlock()
+ wantCommands := 1
+ if phase == "get-osinfo" {
+ wantCommands = 2
+ } else if phase == "info" {
+ wantCommands = 3
+ }
+ if !unchanged || len(before) != wantCommands {
+ t.Error("obsolete/cooldown fetch sent an additional or repeated command")
+ }
+ // Independent VM discovery remains useful, without shortening
+ // or replaying the cancelled VM's uncertainty fence.
+ _, _, name, _, agent, otherPaused := m.fetchGuestAgentMetadata(context.Background(), client, instance, "node", "other", 106, status, false)
+ if otherPaused || name != "FixtureOS" || agent != "2.0" {
+ t.Error("cancelled lifecycle blocked an independent VM")
+ }
+ unblock.Do(func() { close(resume) })
+ })
+ }
+ }
+ }
+}
+
+func TestGuestMetadataLifecycleCancelsSlotWait(t *testing.T) {
+ key := guestMetadataCacheKey("slot-lifecycle", "node", 105)
+ m := &Monitor{guestMetadataSlots: make(chan struct{}, 1)}
+ m.guestMetadataSlots <- struct{}{} // A different fetch owns this slot.
+ client := &metadataSuppressionClient{os: map[string]interface{}{"name": "CurrentOS"}}
+ status := &proxmox.VMStatus{Agent: proxmox.VMAgentField{Value: 1}}
+ finished := make(chan bool, 1)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go func() {
+ _, _, _, _, _, paused := m.fetchGuestAgentMetadata(ctx, client, "slot-lifecycle", "node", "guest", 105, status, false)
+ finished <- paused
+ }()
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ m.guestMetadataLimiterMu.Lock()
+ fetch := m.guestMetadataInFlight[key]
+ waiting := fetch != nil && fetch.cancel != nil
+ m.guestMetadataLimiterMu.Unlock()
+ if waiting {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("metadata fetch did not reserve its slot-wait lifecycle")
+ }
+ time.Sleep(time.Millisecond)
+ }
+ m.clearGuestMetadataCache("slot-lifecycle", "node", 105)
+ select {
+ case paused := <-finished:
+ if !paused {
+ t.Error("invalidated slot waiter did not defer")
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("invalidation did not cancel slot waiting")
+ }
+ if len(m.guestMetadataSlots) != 1 || client.networkCalls+client.osCalls+client.versionCalls != 0 || len(m.guestMetadataCache) != 0 || !m.guestMetadataLimiter[key].IsZero() {
+ t.Error("invalidated slot waiter consumed another owner's slot, sent a command or published state")
+ }
+ assertMetadataReservationReleased(t, m)
+ <-m.guestMetadataSlots
+ _, _, name, _, _, paused := m.fetchGuestAgentMetadata(context.Background(), client, "slot-lifecycle", "node", "guest", 105, status, false)
+ if paused || name != "CurrentOS" || len(m.guestMetadataSlots) != 0 {
+ t.Error("ordinary discovery did not resume after slot-wait invalidation")
+ }
+}
+
func (c *metadataLifecycleClient) command(ctx context.Context, name string) {
c.mu.Lock()
c.commands = append(c.commands, name)
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 19fcc18a65..3dd13e6d1e 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1405,8 +1405,8 @@ type Monitor struct {
guestMetadataCache map[string]guestMetadataCacheEntry
guestMetadataLimiterMu sync.Mutex
guestMetadataLimiter map[string]time.Time
- guestMetadataInFlight map[string]bool // Guarded by guestMetadataLimiterMu; released only on fetch completion.
- guestMetadataLastAccess map[string]time.Time // Guarded by guestMetadataLimiterMu; cache activity, never guest evidence.
+ guestMetadataInFlight map[string]*guestMetadataFetch // Guarded by guestMetadataLimiterMu; retained until fetch completion, even after invalidation.
+ guestMetadataLastAccess map[string]time.Time // Guarded by guestMetadataLimiterMu; cache activity, never guest evidence.
guestMetadataSlots chan struct{}
guestMetadataMinRefresh time.Duration
guestMetadataRefreshJitter time.Duration
diff --git a/internal/monitoring/monitor_agents.go b/internal/monitoring/monitor_agents.go
index f5fb3485f8..2451bba09a 100644
--- a/internal/monitoring/monitor_agents.go
+++ b/internal/monitoring/monitor_agents.go
@@ -5003,7 +5003,7 @@ func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {
if entry.fetchedAt.After(activityAt) {
activityAt = entry.fetchedAt // Also supports legacy/seeded cache entries.
}
- if !m.guestMetadataInFlight[key] && now.Sub(activityAt) > maxAge {
+ if m.guestMetadataInFlight[key] == nil && now.Sub(activityAt) > maxAge {
delete(m.guestMetadataCache, key)
log.Debug().
Str("key", key).
@@ -5014,13 +5014,13 @@ func (m *Monitor) cleanupGuestMetadataCache(now time.Time) {
m.guestMetadataMu.Unlock()
for key, accessedAt := range m.guestMetadataLastAccess {
- if !m.guestMetadataInFlight[key] && now.Sub(accessedAt) > maxAge {
+ if m.guestMetadataInFlight[key] == nil && now.Sub(accessedAt) > maxAge {
delete(m.guestMetadataLastAccess, key)
}
}
for key, nextAllowed := range m.guestMetadataLimiter {
// Keep near-term limiter state; remove long-idle keys.
- if !m.guestMetadataInFlight[key] && now.Sub(nextAllowed) > maxAge {
+ if m.guestMetadataInFlight[key] == nil && now.Sub(nextAllowed) > maxAge {
delete(m.guestMetadataLimiter, key)
log.Debug().
Str("key", key).
diff --git a/internal/monitoring/monitor_cluster_dedupe.go b/internal/monitoring/monitor_cluster_dedupe.go
index e7915ad17a..798572ce60 100644
--- a/internal/monitoring/monitor_cluster_dedupe.go
+++ b/internal/monitoring/monitor_cluster_dedupe.go
@@ -138,6 +138,12 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {
}
m.diagMu.Unlock()
+ m.guestMetadataLimiterMu.Lock()
+ for key := range m.guestMetadataInFlight {
+ if strings.HasPrefix(key, instanceName+"|") {
+ m.invalidateGuestMetadataFetchLocked(key)
+ }
+ }
m.guestMetadataMu.Lock()
for key := range m.guestMetadataCache {
if strings.HasPrefix(key, instanceName+"|") {
@@ -146,7 +152,6 @@ func (m *Monitor) retirePVEInstanceRuntime(instanceName string) {
}
m.guestMetadataMu.Unlock()
- m.guestMetadataLimiterMu.Lock()
for key := range m.guestMetadataLimiter {
if strings.HasPrefix(key, instanceName+"|") {
delete(m.guestMetadataLimiter, key)
diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go
index 44b4b4ee48..8c5b891047 100644
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@ -8339,7 +8339,7 @@ func TestGuestMetadataIdleCleanupContract(t *testing.T) {
},
guestMetadataLastAccess: map[string]time.Time{"active-empty": now, "active-retained": now, "inactive": stale, "only-active": now, "only-idle": stale, "inflight": stale},
guestMetadataLimiter: map[string]time.Time{"active-empty": now.Add(time.Minute), "inactive": stale, "inflight": stale},
- guestMetadataInFlight: map[string]bool{"inflight": true},
+ guestMetadataInFlight: map[string]*guestMetadataFetch{"inflight": {}},
}
m.cleanupGuestMetadataCache(now)
for _, key := range []string{"active-empty", "active-retained", "legacy-fresh", "inflight"} {
@@ -8357,7 +8357,7 @@ func TestGuestMetadataIdleCleanupContract(t *testing.T) {
t.Errorf("idle bookkeeping %s retained", key)
}
}
- if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || !m.guestMetadataInFlight["inflight"] {
+ if !m.guestMetadataLastAccess["only-active"].Equal(now) || !m.guestMetadataLimiter["active-empty"].Equal(now.Add(time.Minute)) || !m.guestMetadataLimiter["inflight"].Equal(stale) || m.guestMetadataInFlight["inflight"] == nil {
t.Error("janitor changed current activity, backoff or in-flight ownership")
}
empty := m.guestMetadataCache["active-empty"]
@@ -8393,7 +8393,7 @@ func TestGuestMetadataActivityInvalidationContract(t *testing.T) {
guestMetadataCache: map[string]guestMetadataCacheEntry{key: {osInfoSkip: true}, sibling: {osInfoSkip: true}},
guestMetadataLimiter: map[string]time.Time{key: time.Now().Add(time.Hour), sibling: time.Now().Add(time.Hour)},
guestMetadataLastAccess: map[string]time.Time{key: time.Now(), sibling: time.Now()},
- guestMetadataInFlight: map[string]bool{key: true},
+ guestMetadataInFlight: map[string]*guestMetadataFetch{key: {}},
}
m.clearGuestMetadataCache("activity", "node", 105)
if _, exists := m.guestMetadataCache[key]; exists {
@@ -8405,7 +8405,7 @@ func TestGuestMetadataActivityInvalidationContract(t *testing.T) {
if _, exists := m.guestMetadataLastAccess[key]; exists {
t.Error("explicit invalidation retained activity")
}
- if !m.guestMetadataInFlight[key] || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {
+ if m.guestMetadataInFlight[key] == nil || !m.guestMetadataInFlight[key].invalidated || !m.guestMetadataCache[sibling].osInfoSkip || m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataLimiter[sibling].IsZero() {
t.Error("invalidation discarded ownership or another instance")
}
m.guestMetadataLastAccess[key] = time.Now()
@@ -8416,7 +8416,7 @@ func TestGuestMetadataActivityInvalidationContract(t *testing.T) {
t.Error("retired instance retained activity bookkeeping")
}
}
- if m.guestMetadataLastAccess[sibling].IsZero() || !m.guestMetadataInFlight[key] {
+ if m.guestMetadataLastAccess[sibling].IsZero() || m.guestMetadataInFlight[key] == nil || !m.guestMetadataInFlight[key].invalidated {
t.Error("retirement crossed instance scope or discarded active ownership")
}
}