Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Back off optional Proxmox permission-denied reads

pulse-triage[bot]5095c651dcdd
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index b0c8d1a2a2..9a689abfe4 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -2172,6 +2172,13 @@ update, profile rollout, command reachability, or fleet-control authority.
own alert and names that agent as display text, but the link itself stays
the agent lifecycle's, and the field cannot create, change or authorize it.
27. `internal/monitoring/monitor.go` shared with `monitoring`: monitor construction owns both monitoring runtime initialization and fail-closed agent lifecycle journal hydration before report admission.
+ The private `pendingUpdatesCache.deniedAt` is only the scheduled PVE APT
+ retry clock. It cannot change agent enrolment, report admission, tombstones,
+ links or independent host memory. Runtime guest-poll withdrawal keeps that
+ denial and the instance/node/VM-scoped memory denial without renewing them;
+ the ordinary independent host snapshot remains unchanged.
+ `TestGuestPollOwnershipRuntimeDoesNotWithdrawIndependentAgentEvidence` pins
+ this separation alongside existing successful guest-memory evidence.
Both live-mode startup and explicit discovery startup bind the discovery
service to this monitor's organization. Discovery start, progress, found
server, update and completion events stay within that organization; a
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 712f770762..dc0b253639 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -8117,10 +8117,30 @@ cache entry exists. An offline node is `not_checked` with `node_offline`.
Failures never refresh the successful timestamp or convert permission and
reachability errors into zero. Cached successful observations remain checked
-until their normal refresh boundary, and a failed refresh is retried without
-discarding the cache. `node_pending_updates_evidence_test.go` and the cluster
-client pending-update tests pin zero, permission, reachability, stale-cache,
-and offline behavior.
+until their normal refresh boundary. An observed HTTP 403 backs off the optional
+APT check for 30 minutes per existing node identity. Its private denial clock
+never replaces the last successful count or timestamp: no prior success stays
+`unavailable`, and prior success (including a measured zero) stays `stale` on
+every backoff poll. Expiry permits an ordinary reprobe; a successful empty list
+is a fresh checked zero and clears the denial. Offline nodes issue no check.
+
+Guest Linux `/proc/meminfo` likewise retains the existing five-minute negative
+interval for an observed HTTP 403, even when known-Linux metadata would normally
+allow a 30-second retry. Successful memory reads still cache for 60 seconds;
+non-403 failures keep their ordinary OS-specific cadence. Denial is not a memory
+sample or a new successful origin. The same instance/node/VM key, read ownership,
+lifecycle invalidation, cancellation and uncertainty fences remain effective.
+
+Both paths use `proxmox.APIErrorStatus`, including wrapped client/cluster errors,
+not a quoted status in a provider body. A known non-403 HTTP error cannot be
+labelled permission-denied from its text. No token permission, guest operation,
+public field, rendering policy or observation-age window changes.
+`optional_permission_denial_regression_test.go` and
+`optional_permission_denial_test.go` pin real synthetic direct/cluster HTTP
+refusals, prior/absent/zero evidence, no renewal, independent target isolation,
+expiry/recovery, repeat denial, quoted-status controls and late lifecycle replies.
+`node_pending_updates_evidence_test.go` and the existing memory, Windows,
+cancellation and optional-read suites continue to cover their independent paths.
### Summary-only container collection remains explicit through ingestion
diff --git a/internal/monitoring/guest_memory_agent.go b/internal/monitoring/guest_memory_agent.go
index e07959d37e..e32fbb8ba8 100644
--- a/internal/monitoring/guest_memory_agent.go
+++ b/internal/monitoring/guest_memory_agent.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "net/http"
"strings"
"time"
@@ -20,10 +21,11 @@ const (
)
type agentMemCacheEntry struct {
- available uint64
- info proxmox.LinuxMemoryAvailability
- negative bool
- fetchedAt time.Time
+ available uint64
+ info proxmox.LinuxMemoryAvailability
+ negative bool
+ permissionDenied bool // Only an observed HTTP 403 overrides known-Linux retry cadence.
+ fetchedAt time.Time
}
type guestAgentMemAvailableClient interface {
@@ -110,7 +112,10 @@ func (m *Monitor) getVMAgentMemoryAvailability(ctx context.Context, client PVECl
if entry, ok := m.vmAgentMemCache[cacheKey]; ok {
ttl := vmAgentMemCacheTTL
if entry.negative {
- ttl = m.vmAgentMemNegativeCacheTTL(instanceName, node, vmid)
+ ttl = vmAgentMemNegativeTTL
+ if !entry.permissionDenied {
+ ttl = m.vmAgentMemNegativeCacheTTL(instanceName, node, vmid)
+ }
}
if now.Sub(entry.fetchedAt) < ttl {
m.rrdCacheMu.Unlock()
@@ -179,7 +184,12 @@ func (m *Monitor) getVMAgentMemoryAvailability(ctx context.Context, client PVECl
m.vmAgentMemCache = make(map[string]agentMemCacheEntry)
}
if err != nil || info.Source == "" {
- m.vmAgentMemCache[cacheKey] = agentMemCacheEntry{negative: true, fetchedAt: now}
+ status, known := proxmox.APIErrorStatus(err)
+ m.vmAgentMemCache[cacheKey] = agentMemCacheEntry{
+ negative: true,
+ permissionDenied: known && status == http.StatusForbidden,
+ fetchedAt: now,
+ }
if err == nil {
err = fmt.Errorf("guest agent meminfo fallback unavailable")
}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 38dc7bac80..7afe0f68ed 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -1556,6 +1556,7 @@ type rrdMemCacheEntry struct {
type pendingUpdatesCache struct {
count int
checkedAt time.Time
+ deniedAt time.Time // Retry clock only; never a successful observation.
}
// TTL for pending updates cache (30 minutes - balance between freshness and API load)
diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go
index e0fe1f1eca..bd56f1c556 100644
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@ -131,7 +131,11 @@ func TestGuestPollOwnershipRuntimeDoesNotWithdrawIndependentAgentEvidence(t *tes
m.guestMetadataCache = map[string]guestMetadataCacheEntry{key: metadata}
memKey := guestMemoryCacheKey("site-a", "node", 105)
memory := agentMemCacheEntry{info: memoryLifecycleSample(4096), fetchedAt: time.Now().Add(-time.Minute)}
- m.vmAgentMemCache = map[string]agentMemCacheEntry{memKey: memory}
+ deniedKey := guestMemoryCacheKey("site-a", "node", 106)
+ denied := agentMemCacheEntry{negative: true, permissionDenied: true, fetchedAt: time.Now().Add(-time.Minute)}
+ apt := pendingUpdatesCache{count: 7, checkedAt: time.Now().Add(-time.Hour), deniedAt: time.Now().Add(-time.Minute)}
+ m.vmAgentMemCache = map[string]agentMemCacheEntry{memKey: memory, deniedKey: denied}
+ m.nodePendingUpdatesCache = map[string]pendingUpdatesCache{"site-a-node": apt}
ctx, owner := m.beginPVEGuestPoll(context.Background(), "site-a", c)
if owner == nil {
t.Fatal("registered cycle not admitted")
@@ -143,7 +147,8 @@ func TestGuestPollOwnershipRuntimeDoesNotWithdrawIndependentAgentEvidence(t *tes
if owner.ctx.Err() == nil || len(m.GetDiagnosticSnapshots().Guests) != 0 || !reflect.DeepEqual(m.state.GetSnapshot().Hosts, hosts) {
t.Fatal("runtime withdrawal admitted obsolete guest evidence or withdrew independent agent memory")
}
- if !reflect.DeepEqual(m.guestMetadataCache[key], metadata) || m.vmAgentMemCache[memKey] != memory {
+ if !reflect.DeepEqual(m.guestMetadataCache[key], metadata) || m.vmAgentMemCache[memKey] != memory ||
+ m.vmAgentMemCache[deniedKey] != denied || m.nodePendingUpdatesCache["site-a-node"] != apt {
t.Fatal("runtime withdrawal erased OS-command safety or renewed accepted cache age")
}
}
diff --git a/internal/monitoring/monitor_polling_node_helpers.go b/internal/monitoring/monitor_polling_node_helpers.go
index 1e76953aa8..ba35f553c2 100644
--- a/internal/monitoring/monitor_polling_node_helpers.go
+++ b/internal/monitoring/monitor_polling_node_helpers.go
@@ -5,6 +5,7 @@ import (
stderrors "errors"
"fmt"
"net"
+ "net/http"
"net/url"
"strings"
"time"
@@ -439,75 +440,72 @@ func (m *Monitor) applyNodePendingUpdates(ctx context.Context, instanceName stri
return
}
- // Poll pending apt updates (less frequently - every 30 minutes)
- // Only for online nodes to avoid wasting API calls on offline nodes
- if effectiveStatus == "online" {
- now := time.Now()
- m.mu.RLock()
- if m.nodePendingUpdatesCache == nil {
- m.mu.RUnlock()
+ // Successful observations and explicit access denials have separate clocks.
+ // Reusing a denial must not turn an old count (including zero) current.
+ now := time.Now()
+ m.mu.RLock()
+ cached := m.nodePendingUpdatesCache[nodeID]
+ m.mu.RUnlock()
+
+ if !cached.deniedAt.IsZero() && !cached.deniedAt.After(now) && now.Sub(cached.deniedAt) < pendingUpdatesCacheTTL {
+ modelNode.PendingUpdatesReason = "permission_denied"
+ } else if !cached.checkedAt.IsZero() && now.Sub(cached.checkedAt) < pendingUpdatesCacheTTL {
+ modelNode.PendingUpdates = cached.count
+ modelNode.PendingUpdatesCheckedAt = cached.checkedAt
+ modelNode.PendingUpdatesStatus = "checked"
+ return
+ } else {
+ pendingPkgs, err := client.GetNodePendingUpdates(ctx, node.Node)
+ if err == nil {
+ modelNode.PendingUpdates = len(pendingPkgs)
+ modelNode.PendingUpdatesCheckedAt = now
+ modelNode.PendingUpdatesStatus = "checked"
m.mu.Lock()
if m.nodePendingUpdatesCache == nil {
m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
}
+ m.nodePendingUpdatesCache[nodeID] = pendingUpdatesCache{count: len(pendingPkgs), checkedAt: now}
m.mu.Unlock()
- m.mu.RLock()
+ return
}
- cached, hasCached := m.nodePendingUpdatesCache[nodeID]
- m.mu.RUnlock()
-
- if !hasCached || now.Sub(cached.checkedAt) >= pendingUpdatesCacheTTL {
- // Time to check for updates
- pendingPkgs, err := client.GetNodePendingUpdates(ctx, node.Node)
- if err != nil {
- // Preserve the last successful value, but mark it stale rather than
- // presenting a permission or reachability failure as a confirmed zero.
- log.Debug().
- Err(err).
- Str("node", node.Node).
- Str("instance", instanceName).
- Msg("Could not check pending apt updates with configured Proxmox credentials")
- modelNode.PendingUpdatesReason = pendingUpdatesFailureReason(err)
- if hasCached {
- modelNode.PendingUpdates = cached.count
- modelNode.PendingUpdatesCheckedAt = cached.checkedAt
- modelNode.PendingUpdatesStatus = "stale"
- } else {
- modelNode.PendingUpdatesStatus = "unavailable"
- }
- } else {
- updateCount := len(pendingPkgs)
- modelNode.PendingUpdates = updateCount
- modelNode.PendingUpdatesCheckedAt = now
- modelNode.PendingUpdatesStatus = "checked"
-
- // Cache the result
- m.mu.Lock()
- m.nodePendingUpdatesCache[nodeID] = pendingUpdatesCache{
- count: updateCount,
- checkedAt: now,
- }
- m.mu.Unlock()
-
- log.Debug().
- Str("node", node.Node).
- Str("instance", instanceName).
- Int("pendingUpdates", updateCount).
- Msg("Checked pending apt updates")
+ log.Debug().Err(err).Str("node", node.Node).Str("instance", instanceName).
+ Msg("Could not check pending apt updates with configured Proxmox credentials")
+ modelNode.PendingUpdatesReason = pendingUpdatesFailureReason(err)
+ if status, known := proxmox.APIErrorStatus(err); known && status == http.StatusForbidden {
+ // A quoted status in an error body is not access-denial evidence.
+ // Keep ordinary transient failures on their existing retry path.
+ cached.deniedAt = now
+ m.mu.Lock()
+ if m.nodePendingUpdatesCache == nil {
+ m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
}
- } else {
- // Use cached value
- modelNode.PendingUpdates = cached.count
- modelNode.PendingUpdatesCheckedAt = cached.checkedAt
- modelNode.PendingUpdatesStatus = "checked"
+ m.nodePendingUpdatesCache[nodeID] = cached
+ m.mu.Unlock()
}
}
+
+ modelNode.PendingUpdates = cached.count
+ modelNode.PendingUpdatesCheckedAt = cached.checkedAt
+ modelNode.PendingUpdatesStatus = "unavailable"
+ if !cached.checkedAt.IsZero() {
+ modelNode.PendingUpdatesStatus = "stale"
+ }
}
func pendingUpdatesFailureReason(err error) string {
if err == nil {
return ""
}
+ if status, known := proxmox.APIErrorStatus(err); known {
+ switch {
+ case status == http.StatusForbidden:
+ return "permission_denied"
+ case status >= http.StatusInternalServerError:
+ return "source_unavailable"
+ default:
+ return "check_failed"
+ }
+ }
message := strings.ToLower(err.Error())
switch {
case strings.Contains(message, "403"), strings.Contains(message, "permission"):
diff --git a/internal/monitoring/optional_permission_denial_regression_test.go b/internal/monitoring/optional_permission_denial_regression_test.go
new file mode 100644
index 0000000000..be263ae7df
--- /dev/null
+++ b/internal/monitoring/optional_permission_denial_regression_test.go
@@ -0,0 +1,169 @@
+package monitoring
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Synthetic, completed HTTP refusals only; no real guest or token is used.
+type optionalDenialFixture struct {
+ status atomic.Int32
+ aptCalls atomic.Int32
+ memoryCalls atomic.Int32
+}
+
+func optionalDenialClient(t *testing.T, kind string, status int) (*optionalDenialFixture, PVEClientInterface) {
+ t.Helper()
+ f := &optionalDenialFixture{}
+ f.status.Store(int32(status))
+ s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
+ t.Error("configured authentication changed")
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case strings.HasSuffix(r.URL.Path, "/nodes"):
+ fmt.Fprint(w, `{"data":[]}`)
+ case strings.HasSuffix(r.URL.Path, "/config"):
+ // Honour the production guest-operation lock check.
+ fmt.Fprint(w, `{"data":{}}`)
+ case strings.HasSuffix(r.URL.Path, "/status/current"):
+ fmt.Fprint(w, `{"data":{"status":"running","agent":1,"maxmem":8388608,"mem":8388608}}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/get-fsinfo"):
+ fmt.Fprint(w, `{"data":{"result":[{"mountpoint":"/","type":"ext4","total-bytes":1048576000,"used-bytes":314572800}]}}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/network-get-interfaces"):
+ fmt.Fprint(w, `{"data":{"result":[]}}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/get-osinfo"):
+ fmt.Fprint(w, `{"data":{"name":"Debian GNU/Linux","version":"fixture"}}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/info"):
+ fmt.Fprint(w, `{"data":{"result":{"version":"fixture"}}}`)
+ case strings.HasSuffix(r.URL.Path, "/apt/update"):
+ f.aptCalls.Add(1)
+ if code := int(f.status.Load()); code != http.StatusOK {
+ w.WriteHeader(code)
+ fmt.Fprint(w, "upstream API error 403: permission denied")
+ return
+ }
+ fmt.Fprint(w, `{"data":[]}`)
+ case strings.HasSuffix(r.URL.Path, "/agent/file-read"):
+ f.memoryCalls.Add(1)
+ if r.URL.Query().Get("file") != "/proc/meminfo" {
+ t.Error("unexpected guest file")
+ }
+ if code := int(f.status.Load()); code != http.StatusOK {
+ w.WriteHeader(code)
+ fmt.Fprint(w, "upstream API error 403: permission denied")
+ return
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"content": "MemTotal: 8192 kB\nMemAvailable: 6144 kB\nMemFree: 2048 kB\n", "truncated": false}})
+ default:
+ t.Errorf("unexpected request %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(s.Close)
+ return f, windowsBootstrapClient(t, s.URL, kind)
+}
+
+func TestOptionalAPTDenialKeepsBackoffAndEvidence(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ for _, prior := range []string{"none", "zero", "seven"} {
+ t.Run(kind+"/"+prior, func(t *testing.T) {
+ f, client := optionalDenialClient(t, kind, http.StatusForbidden)
+ m := &Monitor{}
+ original := time.Time{}
+ count, state := 0, "unavailable"
+ if prior != "none" {
+ original = time.Now().Add(-pendingUpdatesCacheTTL - time.Minute)
+ if prior == "seven" {
+ count = 7
+ }
+ state = "stale"
+ m.nodePendingUpdatesCache = map[string]pendingUpdatesCache{"lab-node": {count: count, checkedAt: original}}
+ }
+ for poll := 0; poll < 3; poll++ {
+ node := &models.Node{}
+ m.applyNodePendingUpdates(context.Background(), "lab", client, proxmox.Node{Node: "node"}, "lab-node", "online", node)
+ if node.PendingUpdatesStatus != state || node.PendingUpdatesReason != "permission_denied" || node.PendingUpdates != count || !node.PendingUpdatesCheckedAt.Equal(original) {
+ t.Errorf("poll %d: denial fabricated/renewed evidence: %+v", poll, node)
+ }
+ }
+ if got := f.aptCalls.Load(); got != 1 {
+ t.Errorf("three polls sent %d APT requests, want 1", got)
+ }
+ })
+ }
+ }
+}
+
+func TestOptionalMemoryDenialUsesLongBackoff(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ t.Run(kind, func(t *testing.T) {
+ f, client := optionalDenialClient(t, kind, http.StatusForbidden)
+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{guestMetadataCacheKey("lab", "node", 105): {osName: "Debian GNU/Linux"}}}
+ key := guestMemoryCacheKey("lab", "node", 105)
+ if info, err := m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105); err == nil || info != (proxmox.LinuxMemoryAvailability{}) {
+ t.Fatal("denied file-read became measured memory")
+ }
+ entry := m.vmAgentMemCache[key]
+ entry.fetchedAt = time.Now().Add(-time.Minute)
+ m.vmAgentMemCache[key] = entry // simulate elapsed polling time, not a new observation
+ for i := 0; i < 2; i++ {
+ if info, err := m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105); err == nil || info != (proxmox.LinuxMemoryAvailability{}) {
+ t.Error("cached denial became measured memory")
+ }
+ }
+ if got := f.memoryCalls.Load(); got != 1 {
+ t.Errorf("denial retried after 60 seconds: calls=%d, want 1", got)
+ }
+ if after := m.vmAgentMemCache[key]; !after.negative || !after.fetchedAt.Equal(entry.fetchedAt) {
+ t.Error("backoff poll renewed the failed observation clock")
+ }
+ })
+ }
+}
+
+func TestOptionalDenialCadenceControls(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ t.Run(kind+"/APT502quoting403", func(t *testing.T) {
+ f, client := optionalDenialClient(t, kind, http.StatusBadGateway)
+ m := &Monitor{}
+ for i := 0; i < 3; i++ {
+ m.applyNodePendingUpdates(context.Background(), "lab", client, proxmox.Node{Node: "node"}, "lab-node", "online", &models.Node{})
+ }
+ want := int32(3)
+ if kind == "cluster-client" {
+ want = 9 // The existing cluster transport retries each 502 three times.
+ }
+ if got := f.aptCalls.Load(); got != want {
+ t.Errorf("non-403 backoff changed: calls=%d, want %d", got, want)
+ }
+ })
+ t.Run(kind+"/memory400quoting403", func(t *testing.T) {
+ f, client := optionalDenialClient(t, kind, http.StatusBadRequest)
+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{guestMetadataCacheKey("lab", "node", 105): {osName: "Debian GNU/Linux"}}}
+ key := guestMemoryCacheKey("lab", "node", 105)
+ _, _ = m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105)
+ entry := m.vmAgentMemCache[key]
+ entry.fetchedAt = time.Now().Add(-time.Minute)
+ m.vmAgentMemCache[key] = entry
+ f.status.Store(http.StatusOK)
+ info, err := m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105)
+ if err != nil || info.EffectiveAvailable != 6144*1024 || f.memoryCalls.Load() != 2 {
+ t.Errorf("non-403 recovery changed: %+v / %v / calls %d", info, err, f.memoryCalls.Load())
+ }
+ })
+ }
+}
diff --git a/internal/monitoring/optional_permission_denial_test.go b/internal/monitoring/optional_permission_denial_test.go
new file mode 100644
index 0000000000..e402ef8f25
--- /dev/null
+++ b/internal/monitoring/optional_permission_denial_test.go
@@ -0,0 +1,143 @@
+package monitoring
+
+import (
+ "context"
+ "net/http"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+func TestOptionalAPTDenialExpiryRecoveryAndIsolation(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ t.Run(kind, func(t *testing.T) {
+ f, client := optionalDenialClient(t, kind, http.StatusForbidden)
+ original := time.Now().Add(-2 * pendingUpdatesCacheTTL)
+ m := &Monitor{nodePendingUpdatesCache: map[string]pendingUpdatesCache{"lab-node": {count: 7, checkedAt: original}}}
+ poll := func(instance, name, id, status string) models.Node {
+ n := models.Node{}
+ m.applyNodePendingUpdates(context.Background(), instance, client, proxmox.Node{Node: name}, id, status, &n)
+ return n
+ }
+ first := poll("lab", "node", "lab-node", "online")
+ denied := m.nodePendingUpdatesCache["lab-node"]
+ if first.PendingUpdatesStatus != "stale" || denied.deniedAt.IsZero() || !denied.checkedAt.Equal(original) || denied.count != 7 {
+ t.Fatalf("denial lost original evidence: %+v / %+v", first, denied)
+ }
+ f.status.Store(http.StatusOK)
+ poll("lab", "node", "lab-node", "online")
+ offline := poll("lab", "node", "lab-node", "offline")
+ if f.aptCalls.Load() != 1 || m.nodePendingUpdatesCache["lab-node"] != denied || offline.PendingUpdatesStatus != "not_checked" || offline.PendingUpdatesReason != "node_offline" {
+ t.Error("backoff/offline poll renewed or discarded denial")
+ }
+ for _, target := range [][3]string{{"lab", "other", "lab-other"}, {"peer", "node", "peer-node"}} {
+ n := poll(target[0], target[1], target[2], "online")
+ if n.PendingUpdatesStatus != "checked" || n.PendingUpdates != 0 || n.PendingUpdatesCheckedAt.IsZero() {
+ t.Error("denial leaked to another connection/node")
+ }
+ }
+ denied.deniedAt = time.Now().Add(-pendingUpdatesCacheTTL)
+ m.nodePendingUpdatesCache["lab-node"] = denied
+ recovered := poll("lab", "node", "lab-node", "online")
+ cached := m.nodePendingUpdatesCache["lab-node"]
+ if f.aptCalls.Load() != 4 || recovered.PendingUpdatesStatus != "checked" || recovered.PendingUpdates != 0 || recovered.PendingUpdatesReason != "" || !recovered.PendingUpdatesCheckedAt.After(original) || !cached.deniedAt.IsZero() {
+ t.Errorf("expiry did not recover confirmed zero: %+v / %+v", recovered, cached)
+ }
+ poll("lab", "node", "lab-node", "online")
+ if f.aptCalls.Load() != 4 || m.nodePendingUpdatesCache["lab-node"] != cached {
+ t.Error("successful cache cadence/clock changed")
+ }
+ })
+ }
+}
+
+func TestOptionalAPTRepeatedDenialReprobesWithoutRenewingSuccess(t *testing.T) {
+ f, client := optionalDenialClient(t, "client", http.StatusForbidden)
+ original := time.Now().Add(-2 * pendingUpdatesCacheTTL)
+ m := &Monitor{nodePendingUpdatesCache: map[string]pendingUpdatesCache{"lab-node": {count: 0, checkedAt: original}}}
+ poll := func() models.Node {
+ n := models.Node{}
+ m.applyNodePendingUpdates(context.Background(), "lab", client, proxmox.Node{Node: "node"}, "lab-node", "online", &n)
+ return n
+ }
+ poll()
+ entry := m.nodePendingUpdatesCache["lab-node"]
+ entry.deniedAt = time.Now().Add(-pendingUpdatesCacheTTL)
+ m.nodePendingUpdatesCache["lab-node"] = entry
+ n := poll()
+ renewed := m.nodePendingUpdatesCache["lab-node"]
+ if f.aptCalls.Load() != 2 || !renewed.deniedAt.After(entry.deniedAt) || !renewed.checkedAt.Equal(original) || n.PendingUpdatesStatus != "stale" || !n.PendingUpdatesCheckedAt.Equal(original) {
+ t.Error("repeated denial changed successful origin or failed bounded reprobe")
+ }
+ poll()
+ if f.aptCalls.Load() != 2 {
+ t.Error("reprobe failure was not backed off again")
+ }
+}
+
+func TestOptionalMemoryDenialExpiryRecoveryAndIsolation(t *testing.T) {
+ for _, kind := range []string{"client", "cluster-client"} {
+ t.Run(kind, func(t *testing.T) {
+ f, client := optionalDenialClient(t, kind, http.StatusForbidden)
+ key := guestMemoryCacheKey("lab", "node", 105)
+ original := agentMemCacheEntry{info: memoryLifecycleSample(4096), fetchedAt: time.Now().Add(-2 * vmAgentMemCacheTTL)}
+ m := &Monitor{guestMetadataCache: map[string]guestMetadataCacheEntry{guestMetadataCacheKey("lab", "node", 105): {osName: "Linux"}}, vmAgentMemCache: map[string]agentMemCacheEntry{key: original}}
+ _, err := m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105)
+ denied := m.vmAgentMemCache[key]
+ if status, ok := proxmox.APIErrorStatus(err); !ok || status != http.StatusForbidden || !denied.negative || !denied.permissionDenied || denied.info != (proxmox.LinuxMemoryAvailability{}) {
+ t.Fatalf("denial became success or lost wire status: %+v / %v", denied, err)
+ }
+ f.status.Store(http.StatusOK)
+ for _, target := range []struct {
+ instance, node string
+ vmid int
+ }{{"lab", "node", 106}, {"lab", "other", 105}, {"peer", "node", 105}} {
+ info, err := m.getVMAgentMemoryAvailability(context.Background(), client, target.instance, target.node, target.vmid)
+ if err != nil || info.EffectiveAvailable != 6144*1024 {
+ t.Errorf("denial leaked scope: %+v / %v", info, err)
+ }
+ }
+ // The long retry overrides known-Linux cadence, not the success origin.
+ denied.fetchedAt = time.Now().Add(-vmAgentMemNegativeTTL)
+ m.vmAgentMemCache[key] = denied
+ info, err := m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105)
+ recovered := m.vmAgentMemCache[key]
+ if err != nil || info.EffectiveAvailable != 6144*1024 || recovered.negative || recovered.permissionDenied || !recovered.fetchedAt.After(original.fetchedAt) || f.memoryCalls.Load() != 5 {
+ t.Fatalf("expiry failed to recover: %+v / %v / %+v", info, err, recovered)
+ }
+ _, _ = m.getVMAgentMemoryAvailability(context.Background(), client, "lab", "node", 105)
+ if f.memoryCalls.Load() != 5 || m.vmAgentMemCache[key] != recovered {
+ t.Error("success cache cadence/origin changed")
+ }
+ m.clearGuestMemoryCache("lab", "node", 105)
+ if _, ok := m.vmAgentMemCache[key]; ok {
+ t.Error("lifecycle did not retire memory state")
+ }
+ })
+ }
+}
+
+func TestOptionalMemoryDenialInvalidationKeepsOwnershipFence(t *testing.T) {
+ _, wire := optionalDenialClient(t, "client", http.StatusForbidden)
+ _, denial := wire.(guestAgentMemoryAvailabilityClient).GetVMMemoryAvailabilityFromAgent(context.Background(), "node", 105)
+ for _, disposition := range []string{"guest", "connection", "unavailable", "stopped"} {
+ t.Run(disposition, func(t *testing.T) {
+ m := &Monitor{}
+ _, _, done, release := startBlockedGuestMemory(t, m, "lab", proxmox.LinuxMemoryAvailability{}, denial)
+ defer release()
+ invalidateMemoryLifecycle(m, "lab", disposition)
+ next := &guestMemoryAgentTestClient{stubPVEClient: &stubPVEClient{}}
+ if _, err := m.getVMAgentMemoryAvailability(context.Background(), next, "lab", "node", 105); err != proxmox.ErrGuestAgentDeferred || next.memCalls != 0 {
+ t.Error("retirement released unfinished read")
+ }
+ release()
+ got := readGuestMemoryReply(t, done)
+ if got.err != proxmox.ErrGuestAgentDeferred || len(m.vmAgentMemCache) != 0 || !reflect.DeepEqual(got.info, proxmox.LinuxMemoryAvailability{}) {
+ t.Error("late obsolete denial repopulated replacement cache")
+ }
+ })
+ }
+}