Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep failed Proxmox nodes probes out of recovered health

pulse-triage[bot]4147f48db3bb
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 7ae9c78a0b..6dc321cb3b 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,36 @@
## Purpose
+### Cluster recovery requires success of the operation actually probed
+
+Startup and unhealthy-member recovery read `/nodes`; lazy construction uses the
+same probe. A failed nodes or login request cannot count as successful recovery
+because its response text mentions a guest agent, QMP or `guest-get-*`. The old
+per-guest substring exception is removed from these non-guest operations.
+Failed selection retains its actual sanitised reason and observation time, and
+installs no polling client. Successful probe plus accepted full-timeout session
+is still required. The existing temporary-rate-limit path may admit an ordinary
+request, but does not report the failed probe as passed or clear a known failure.
+
+The single-endpoint last-resort path remains available with its cached client;
+selecting it no longer marks that endpoint healthy before a request succeeds.
+A later accepted ordinary read still clears the failure. Actual per-guest
+errors remain request-level errors, without marking an otherwise working node
+unhealthy. Certificate pins, first-use trust, credential encodings, request and
+recovery cadence, rate-limit backoff and guest/backup fences are unchanged.
+
+`TestClusterNodeProbeRequiresSuccess` covers startup/recovery/selection in token,
+JSON password and form password modes, three guest-like failure bodies at the
+nodes or login stage, no extra login/full client and distinct later recovery.
+`TestClusterSingleEndpointFallbackKeepsFailedHealth` checks cached selection,
+failed ordinary read and later success. `TestClusterNodeProbeRateLimitKeepsOrdinaryRead`
+preserves genuine rate-limit handling. `TestClusterNodeProbeEndpointProjection`
+carries the actual failed and later recovered member through the production
+monitor projection while independent native Online, saved trust and identity
+remain unchanged. These are source/consumer controls, not attribution or
+installed acceptance for a reported PBS/QGA/backup incident.
+
+
### First-use cluster trust includes password construction
Declared, previously untrusted Proxmox members use the existing credential-free
diff --git a/internal/monitoring/monitor_pve_node_probe_test.go b/internal/monitoring/monitor_pve_node_probe_test.go
new file mode 100644
index 0000000000..e6ab0848a4
--- /dev/null
+++ b/internal/monitoring/monitor_pve_node_probe_test.go
@@ -0,0 +1,121 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// A failed member /nodes response cannot establish a healthy polling endpoint
+// merely by quoting a guest error. The primary's accepted inventory still
+// reports both members online. Public reads initiate later ordinary recovery;
+// no guest command, backup, credential change or guest fence is exercised.
+func TestClusterNodeProbeEndpointProjection(t *testing.T) {
+ var acceptMember atomic.Bool
+ var memberLogins, memberNodes atomic.Int32
+ endpoint := func(member bool) *httptest.Server {
+ s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api2/json/access/ticket":
+ if member {
+ memberLogins.Add(1)
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ case "/api2/json/nodes":
+ if r.Header.Get("Cookie") != "PVEAuthCookie=fixture-ticket" {
+ t.Error("ordinary nodes read lost password authentication")
+ }
+ if member {
+ memberNodes.Add(1)
+ if !acceptMember.Load() {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, "QEMU guest agent is not running")
+ return
+ }
+ }
+ fmt.Fprint(w, `{"data":[{"node":"primary","status":"online","cpu":0.125},{"node":"member","status":"online","cpu":0.25}]}`)
+ default:
+ t.Errorf("left ordinary session/nodes reads: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(s.Close)
+ return s
+ }
+ primary, member := endpoint(false), endpoint(true)
+ pin := func(s *httptest.Server) string {
+ v := sha256.Sum256(s.Certificate().Raw)
+ return hex.EncodeToString(v[:])
+ }
+ pins := map[string]string{primary.URL: pin(primary), member.URL: pin(member)}
+ cc := proxmox.NewClusterClient("node-probe-cluster", proxmox.ClientConfig{Host: primary.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: pins[primary.URL], Timeout: 2 * time.Second}, []string{primary.URL, member.URL}, pins)
+ if !cc.GetHealthStatus()[primary.URL] || cc.GetHealthStatus()[member.URL] || memberLogins.Load() != 1 || memberNodes.Load() != 1 {
+ t.Fatalf("failed nodes probe became healthy or advanced to a full session: logins=%d nodes=%d", memberLogins.Load(), memberNodes.Load())
+ }
+ inst := config.PVEInstance{Name: "node-probe-cluster", Host: primary.URL, IsCluster: true, ClusterName: "Fixture cluster", VerifySSL: true, Fingerprint: pins[primary.URL], ClusterEndpoints: []config.ClusterEndpoint{
+ {NodeName: "primary", NodeIdentity: "primary-identity", Host: primary.URL, Fingerprint: pins[primary.URL]},
+ {NodeName: "member", NodeIdentity: "member-identity", Host: member.URL, Fingerprint: pins[member.URL]},
+ }}
+ m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{inst}}}
+ project := func(phase string) {
+ nodes, err := cc.GetNodes(context.Background())
+ if err != nil || len(nodes) != 2 {
+ t.Fatalf("accepted primary read failed: %+v %v", nodes, err)
+ }
+ model := make([]models.Node, len(nodes))
+ for i, n := range nodes {
+ model[i] = models.Node{Name: n.Node, Status: n.Status}
+ }
+ m.updateClusterEndpointStatus(inst.Name, &inst, cc, model)
+ for i, ep := range inst.ClusterEndpoints {
+ if !ep.Online || ep.NodeIdentity != []string{"primary-identity", "member-identity"}[i] || ep.Fingerprint != pins[ep.Host] {
+ t.Fatal("projection changed native online state, identity or saved trust")
+ }
+ }
+ wire, err := json.Marshal(struct {
+ Phase string `json:"phase"`
+ Endpoints []config.ClusterEndpoint `json:"endpoints"`
+ }{phase, inst.ClusterEndpoints})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("NODE_PROBE_ENDPOINT_PROJECTION_JSON=%s", wire)
+ }
+ project("nodes-probe-refused")
+ if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || *ep.PulseReachable || ep.PulseError != "API error 500: QEMU guest agent is not running" {
+ t.Fatalf("consumer lost failed nodes observation: %+v", ep)
+ }
+ acceptMember.Store(true)
+ // Honour the real recovery interval. Do not clear a production admission,
+ // replace clients or change clocks to manufacture the consuming result.
+ time.Sleep(11 * time.Second)
+ if _, err := cc.GetNodes(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for !cc.GetHealthStatus()[member.URL] {
+ if time.Now().After(deadline) {
+ t.Fatal("later ordinary recovery did not accept member")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ if memberLogins.Load() != 3 || memberNodes.Load() != 2 {
+ t.Fatalf("later recovery did not use one distinct probe/full session: logins=%d nodes=%d", memberLogins.Load(), memberNodes.Load())
+ }
+ project("member-recovered")
+ if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || !*ep.PulseReachable || ep.PulseError != "" {
+ t.Fatalf("accepted recovery retained stale failure: %+v", ep)
+ }
+}
diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go
index 441a7c3e01..9dd1e0b39f 100644
--- a/pkg/proxmox/cluster_client.go
+++ b/pkg/proxmox/cluster_client.go
@@ -53,33 +53,6 @@ var transientRateLimitStatusCodes = map[int]struct{}{
504: {},
}
-// isVMSpecificError reports whether an error string is scoped to a single VM/guest agent
-// and should not be treated as a node connectivity failure.
-func isVMSpecificError(errStr string) bool {
- if errStr == "" {
- return false
- }
- lower := strings.ToLower(errStr)
-
- if strings.Contains(lower, "no qemu guest agent") ||
- strings.Contains(lower, "qemu guest agent is not running") ||
- strings.Contains(lower, "guest agent") {
- return true
- }
-
- // QMP guest agent operations can time out or fail per-VM (e.g. guest-get-fsinfo).
- // These aren't node connectivity issues and should not mark endpoints unhealthy.
- if strings.Contains(lower, "qmp command") {
- return true
- }
-
- if strings.Contains(lower, "guest-get-") {
- return true
- }
-
- return false
-}
-
// isEndpointConnectivityError reports whether an error indicates the endpoint
// itself is unreachable (TCP, DNS, TLS failures). Application-level errors
// (HTTP responses, parsing errors) mean the endpoint IS reachable.
@@ -429,10 +402,9 @@ func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {
defer testClient.httpClient.CloseIdleConnections()
}
- // Check if error is VM-specific (shouldn't affect health)
- vmSpecificErr := err != nil && isVMSpecificError(err.Error())
-
- if err == nil || vmSpecificErr {
+ // The probe reads /nodes, not a guest endpoint. Provider text about
+ // QGA cannot turn a failed nodes/login request into usable polling.
+ if err == nil {
// Node is healthy - create a proper client with full timeout for actual use
fullCfg := cc.config
fullCfg.Host = ep
@@ -456,17 +428,10 @@ func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {
cc.lastHealthCheck[ep] = time.Now()
cc.clients[ep] = fullClient // Store the full client, not test client
cc.mu.Unlock()
- if vmSpecificErr {
- log.Debug().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
- Msg("Cluster endpoint healthy despite VM-specific errors")
- } else {
- log.Info().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
- Msg("Cluster endpoint passed initial health check")
- }
+ log.Info().
+ Str("cluster", cc.name).
+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
+ Msg("Cluster endpoint passed initial health check")
}
} else {
// Real connectivity issue
@@ -646,8 +611,8 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(cc.endpoints[0])).
Msg("Single endpoint appears unhealthy but attempting to use it anyway")
healthyEndpoints = cc.endpoints
- // Mark it as healthy optimistically
- cc.nodeHealth[cc.endpoints[0]] = true
+ // Selection is not recovery evidence. Keep the failure until a
+ // successful ordinary request clears it, even with a cached client.
} else {
// Provide detailed error with endpoint status. Surfacing the
// per-endpoint reason (already sanitized) preserves the original
@@ -731,41 +696,29 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
Msg("Ignoring transient rate limit error during connectivity test")
// Continue with client creation since the node is accessible, just rate limited
} else {
- // Check if this is a VM-specific error that shouldn't mark the node unhealthy
- testErrStr := testErr.Error()
- if strings.Contains(testErrStr, "No QEMU guest agent") ||
- strings.Contains(testErrStr, "QEMU guest agent is not running") ||
- strings.Contains(testErrStr, "guest agent") {
- // This is a VM-specific issue, not a connectivity problem
- // The node is actually healthy, so don't mark it unhealthy
- log.Debug().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
- Err(testErr).
- Msg("Ignoring VM-specific error during connectivity test")
- // Continue with client creation since the node is actually accessible
- } else {
- // Mark as unhealthy for real connectivity issues
- cc.nodeHealth[selectedEndpoint] = false
- log.Warn().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
- Err(testErr).
- Msg("Failed to connect to Proxmox endpoint; endpoint removed from rotation until next refresh")
- return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), testErr)
- }
+ cc.nodeHealth[selectedEndpoint] = false
+ cc.lastError[selectedEndpoint] = sanitizeEndpointError(testErr.Error())
+ cc.lastHealthCheck[selectedEndpoint] = time.Now()
+ log.Warn().
+ Str("cluster", cc.name).
+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
+ Err(testErr).
+ Msg("Failed to read Proxmox nodes; endpoint removed from rotation until next refresh")
+ return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), testErr)
}
}
- log.Debug().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
- Func(func(event *zerolog.Event) {
- if testNodes != nil {
- event.Int("nodes", len(testNodes))
- }
- }).
- Msg("Cluster endpoint passed connectivity test")
+ if testErr == nil {
+ log.Debug().
+ Str("cluster", cc.name).
+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).
+ Func(func(event *zerolog.Event) {
+ if testNodes != nil {
+ event.Int("nodes", len(testNodes))
+ }
+ }).
+ Msg("Cluster endpoint passed connectivity test")
+ }
// Use the first-use winner's pin for the full polling session too.
cfg.Fingerprint = cc.getEndpointFingerprintLocked(selectedEndpoint)
@@ -777,12 +730,19 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
}
// This shouldn't happen since we just tested it
cc.nodeHealth[selectedEndpoint] = false
+ cc.lastError[selectedEndpoint] = sanitizeEndpointError(err.Error())
+ cc.lastHealthCheck[selectedEndpoint] = time.Now()
return nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)
}
- // A completed probe cannot clear an error until the polling session is
- // accepted. Ending a caller is not endpoint failure or recovery evidence.
- delete(cc.lastError, selectedEndpoint)
+ // A successful nodes probe and accepted polling session establish
+ // recovery. The existing rate-limit exception only admits an ordinary
+ // request; it cannot clear a recorded failure before that request succeeds.
+ if testErr == nil {
+ cc.nodeHealth[selectedEndpoint] = true
+ cc.lastHealthCheck[selectedEndpoint] = time.Now()
+ delete(cc.lastError, selectedEndpoint)
+ }
cc.clients[selectedEndpoint] = newClient
client = newClient
}
@@ -909,10 +869,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
return
}
- // Check if error is VM-specific (shouldn't prevent recovery)
- vmSpecificErr := err != nil && isVMSpecificError(err.Error())
-
- if err == nil || vmSpecificErr {
+ // Recovery requires the actual nodes probe to succeed; a guest-like
+ // error body is not evidence about this endpoint's nodes inventory.
+ if err == nil {
// Store the client with original timeout
cfg.Timeout = cc.config.Timeout
cfg.Fingerprint = cc.getEndpointFingerprint(ep)
@@ -944,17 +903,10 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
cc.mu.Unlock()
recoveredEndpoints <- ep
- if vmSpecificErr {
- log.Info().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
- Msg("Recovered unhealthy cluster node (ignoring VM-specific errors)")
- } else {
- log.Info().
- Str("cluster", cc.name).
- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
- Msg("Recovered unhealthy cluster node")
- }
+ log.Info().
+ Str("cluster", cc.name).
+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
+ Msg("Recovered unhealthy cluster node")
} else {
recordFailure(ep, err)
log.Debug().
diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.go
index 8034b4275f..b460d77f76 100644
--- a/pkg/proxmox/cluster_client_api_test.go
+++ b/pkg/proxmox/cluster_client_api_test.go
@@ -754,3 +754,160 @@ func TestClusterClientFirstUseCaptureCancellation(t *testing.T) {
func TestClusterFirstUseSessionAdmission(t *testing.T) {
testClusterFirstUseSessionAdmission(t)
}
+
+// A nodes/login failure is not a per-guest failure, even when the provider's
+// body mentions QGA. Exercise the real constructors/probes in all three phases;
+// none of these handlers can enqueue a guest command or perform a backup.
+func TestClusterNodeProbeRequiresSuccess(t *testing.T) {
+ for _, phase := range []string{"startup", "recovery", "selection"} {
+ for _, mode := range []string{"token", "json", "form"} {
+ for _, stage := range []string{"nodes", "login"} {
+ if stage == "login" && mode == "token" {
+ continue
+ }
+ for i, message := range []string{"QEMU guest agent is not running", "VM 105 qmp command 'guest-get-fsinfo' failed", "proxy: guest-get-osinfo unavailable"} {
+ t.Run(fmt.Sprintf("%s/%s/%s/body-%d", phase, mode, stage, i), func(t *testing.T) {
+ f := newNodeProbeFixture(t, mode, stage, message)
+ cfg := f.config()
+ saved := map[string]string{f.server.URL: cfg.Fingerprint}
+ f.reject.Store(true)
+ var cc *ClusterClient
+ if phase == "startup" {
+ sibling := newNodeProbeFixture(t, mode, "nodes", "")
+ saved[sibling.server.URL] = sibling.config().Fingerprint
+ cc = NewClusterClient("node-probe", cfg, []string{f.server.URL, sibling.server.URL}, saved)
+ if !cc.GetHealthStatus()[sibling.server.URL] {
+ t.Fatal("failed peer blocked accepted sibling")
+ }
+ } else {
+ cc = NewClusterClient("node-probe", cfg, []string{f.server.URL}, saved)
+ if phase == "recovery" {
+ cc.nodeHealth[f.server.URL] = false
+ cc.lastError[f.server.URL] = "previous failed observation"
+ cc.recoverUnhealthyNodes(context.Background())
+ } else {
+ c, err := cc.getHealthyClient(context.Background())
+ if c != nil || err == nil {
+ t.Error("failed nodes probe installed a polling client")
+ }
+ }
+ }
+ t.Cleanup(func() {
+ cc.mu.RLock()
+ defer cc.mu.RUnlock()
+ for _, c := range cc.clients {
+ if c != nil {
+ c.httpClient.CloseIdleConnections()
+ }
+ }
+ })
+ health := cc.GetHealthStatusWithErrors()[f.server.URL]
+ if health.Healthy || health.LastError == "" || health.LastCheck.IsZero() {
+ t.Errorf("failed probe published healthy or lost its reason/time: %+v", health)
+ }
+ if cc.clients[f.server.URL] != nil {
+ t.Error("failed probe installed a full polling client")
+ }
+ wantNodes := int32(1)
+ if stage == "login" {
+ wantNodes = 0
+ }
+ if f.logins.Load() != f.loginAttempts() || f.nodes.Load() != wantNodes {
+ t.Errorf("failed probe was retried/advanced: logins=%d want=%d nodes=%d want=%d", f.logins.Load(), f.loginAttempts(), f.nodes.Load(), wantNodes)
+ }
+ if cc.getEndpointFingerprint(f.server.URL) != cfg.Fingerprint || saved[f.server.URL] != cfg.Fingerprint || cc.config.VerifySSL != true {
+ t.Error("failed probe changed saved trust")
+ }
+ // A distinct later recovery uses this fixture's normal accepted reply.
+ // Only endpoint recovery eligibility advances, never a guest fence.
+ f.reject.Store(false)
+ cc.mu.Lock()
+ delete(cc.lastHealthCheck, f.server.URL)
+ cc.mu.Unlock()
+ cc.recoverUnhealthyNodes(context.Background())
+ if c := cc.clients[f.server.URL]; c == nil || c.config.Timeout != cfg.Timeout {
+ t.Fatal("accepted later recovery has no full-timeout polling client")
+ }
+ cc.mu.Lock()
+ for ep := range cc.nodeHealth {
+ if ep != f.server.URL {
+ cc.nodeHealth[ep] = false
+ cc.lastHealthCheck[ep] = time.Now()
+ }
+ }
+ cc.mu.Unlock()
+ nodes, err := cc.GetNodes(context.Background())
+ health = cc.GetHealthStatusWithErrors()[f.server.URL]
+ if err != nil || len(nodes) != 1 || nodes[0].CPU != 0.25 || !health.Healthy || health.LastError != "" || f.nodes.Load() != wantNodes+2 || f.logins.Load() != 3*f.loginAttempts() {
+ t.Errorf("separate ordinary recovery did not restore coherent readings: err=%v health=%+v logins=%d nodes=%d", err, health, f.logins.Load(), f.nodes.Load())
+ }
+ })
+ }
+ }
+ }
+ }
+}
+
+// Keep the existing single-endpoint last-resort read, but do not publish
+// recovery simply for selecting its cached client. A failed read is still a
+// failure; only a later successful ordinary read clears the observed error.
+func TestClusterSingleEndpointFallbackKeepsFailedHealth(t *testing.T) {
+ f := newNodeProbeFixture(t, "token", "nodes", "QEMU guest agent is not running")
+ cc := NewClusterClient("single-probe", f.config(), []string{f.server.URL}, nil)
+ if _, err := cc.GetNodes(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ original := cc.clients[f.server.URL]
+ defer original.httpClient.CloseIdleConnections()
+ f.reject.Store(true)
+ cc.markUnhealthyWithError(f.server.URL, "previous connection failure")
+ c, err := cc.getHealthyClient(context.Background())
+ if err != nil || c != original {
+ t.Fatal("last-resort selection lost the existing client")
+ }
+ if h := cc.GetHealthStatusWithErrors()[f.server.URL]; h.Healthy || h.LastError == "" {
+ t.Errorf("selection invented endpoint recovery: %+v", h)
+ }
+ before := f.nodes.Load()
+ if _, err := cc.GetNodes(context.Background()); err == nil || !strings.Contains(err.Error(), "API error 500") {
+ t.Errorf("failed ordinary read was hidden: %v", err)
+ }
+ if h := cc.GetHealthStatusWithErrors()[f.server.URL]; h.Healthy || h.LastError == "" || f.nodes.Load() != before+1 {
+ t.Errorf("failed fallback read published recovery or replayed: %+v reads=%d", h, f.nodes.Load())
+ }
+ f.reject.Store(false)
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 {
+ t.Fatalf("accepted later read did not recover: %+v %v", nodes, err)
+ }
+ if h := cc.GetHealthStatusWithErrors()[f.server.URL]; !h.Healthy || h.LastError != "" || f.nodes.Load() != before+2 || cc.clients[f.server.URL] != original {
+ t.Errorf("accepted fallback did not restore health with the same client: %+v", h)
+ }
+}
+
+// A genuine temporary nodes rate limit retains the established lazy-selection
+// route: the actual ordinary read, not the failed probe, supplies its readings.
+func TestClusterNodeProbeRateLimitKeepsOrdinaryRead(t *testing.T) {
+ var calls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api2/json/nodes" {
+ t.Errorf("unexpected request: %s", r.URL.Path)
+ http.NotFound(w, r)
+ return
+ }
+ if calls.Add(1) == 1 {
+ w.WriteHeader(http.StatusTooManyRequests)
+ fmt.Fprint(w, "fixture busy")
+ return
+ }
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online","cpu":0.25}]}`)
+ }))
+ defer server.Close()
+ cc := NewClusterClient("probe-rate-limit", ClientConfig{Host: server.URL, TokenName: "fixture@pam!pulse", TokenValue: "fixture-token"}, []string{server.URL}, nil)
+ nodes, err := cc.GetNodes(context.Background())
+ if c := cc.clients[server.URL]; c != nil {
+ defer c.httpClient.CloseIdleConnections()
+ }
+ if err != nil || len(nodes) != 1 || nodes[0].CPU != 0.25 || calls.Load() != 2 || !cc.GetHealthStatus()[server.URL] {
+ t.Fatalf("temporary probe rate limit blocked ordinary read: %+v err=%v calls=%d", nodes, err, calls.Load())
+ }
+}
diff --git a/pkg/proxmox/cluster_client_initial_health_test.go b/pkg/proxmox/cluster_client_initial_health_test.go
index d65f6a708a..24a88bcbe2 100644
--- a/pkg/proxmox/cluster_client_initial_health_test.go
+++ b/pkg/proxmox/cluster_client_initial_health_test.go
@@ -8,7 +8,7 @@ import (
"testing"
)
-func TestClusterClient_InitialHealthCheck_VMErrorTreatedHealthy(t *testing.T) {
+func TestClusterClient_InitialHealthCheck_FailedNodesProbeStaysUnhealthy(t *testing.T) {
server1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api2/json/nodes" {
w.WriteHeader(http.StatusInternalServerError)
@@ -32,11 +32,14 @@ func TestClusterClient_InitialHealthCheck_VMErrorTreatedHealthy(t *testing.T) {
cc := NewClusterClient("test", cfg, []string{server1.URL, server2.URL}, nil)
health := cc.GetHealthStatusWithErrors()
- if !health[server1.URL].Healthy {
- t.Fatalf("expected vm-specific error endpoint to be healthy, got %+v", health[server1.URL])
+ if health[server1.URL].Healthy {
+ t.Fatalf("failed nodes probe must stay unhealthy, got %+v", health[server1.URL])
}
- if health[server1.URL].LastError != "" {
- t.Fatalf("expected no last error, got %q", health[server1.URL].LastError)
+ if health[server1.URL].LastError == "" {
+ t.Fatalf("expected failed nodes reason, got %q", health[server1.URL].LastError)
+ }
+ if cc.clients[server1.URL] != nil {
+ t.Fatal("failed nodes probe installed a full client")
}
if !health[server2.URL].Healthy {
t.Fatalf("expected second endpoint healthy, got %+v", health[server2.URL])
diff --git a/pkg/proxmox/cluster_client_node_probe_test.go b/pkg/proxmox/cluster_client_node_probe_test.go
new file mode 100644
index 0000000000..a5b00719f1
--- /dev/null
+++ b/pkg/proxmox/cluster_client_node_probe_test.go
@@ -0,0 +1,85 @@
+package proxmox
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+type nodeProbeFixture struct {
+ server *httptest.Server
+ reject atomic.Bool
+ logins, nodes atomic.Int32
+ mode, stage, message string
+}
+
+func newNodeProbeFixture(t *testing.T, mode, stage, message string) *nodeProbeFixture {
+ t.Helper()
+ f := &nodeProbeFixture{mode: mode, stage: stage, message: message}
+ f.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api2/json/access/ticket":
+ f.logins.Add(1)
+ if mode == "token" {
+ t.Error("token mode attempted password authentication")
+ }
+ if mode == "form" && r.Header.Get("Content-Type") == "application/json" {
+ w.WriteHeader(http.StatusUnsupportedMediaType)
+ return
+ }
+ if f.reject.Load() && stage == "login" {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, message)
+ return
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ case "/api2/json/nodes":
+ f.nodes.Add(1)
+ if mode == "token" {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pam!pulse=fixture-token" {
+ t.Error("nodes read lost token authentication")
+ }
+ } else if r.Header.Get("Cookie") != "PVEAuthCookie=fixture-ticket" {
+ t.Error("nodes read lost the accepted password session")
+ }
+ if f.reject.Load() && stage == "nodes" {
+ w.WriteHeader(http.StatusInternalServerError)
+ fmt.Fprint(w, message)
+ return
+ }
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online","cpu":0.25}]}`)
+ default:
+ t.Errorf("probe left ordinary session/nodes paths: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(f.server.Close)
+ return f
+}
+
+func (f *nodeProbeFixture) config() ClientConfig {
+ sum := sha256.Sum256(f.server.Certificate().Raw)
+ cfg := ClientConfig{Host: f.server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(sum[:]), Timeout: 2 * time.Second}
+ if f.mode == "token" {
+ cfg.Password = ""
+ cfg.TokenName = "fixture@pam!pulse"
+ cfg.TokenValue = "fixture-token"
+ }
+ return cfg
+}
+
+func (f *nodeProbeFixture) loginAttempts() int32 {
+ if f.mode == "token" {
+ return 0
+ }
+ if f.mode == "form" {
+ return 2
+ }
+ return 1
+}
diff --git a/pkg/proxmox/cluster_client_test.go b/pkg/proxmox/cluster_client_test.go
index 19ebf4f59c..b6a87e08af 100644
--- a/pkg/proxmox/cluster_client_test.go
+++ b/pkg/proxmox/cluster_client_test.go
@@ -226,40 +226,6 @@ func TestIsNotImplementedError(t *testing.T) {
}
}
-func TestIsVMSpecificError(t *testing.T) {
- tests := []struct {
- name string
- errStr string
- expected bool
- }{
- {"empty string", "", false},
- {"generic connection error", "connection refused", false},
- {"no qemu guest agent", "no qemu guest agent", true},
- {"No QEMU guest agent uppercase", "No QEMU guest agent running", true},
- {"qemu guest agent is not running", "qemu guest agent is not running", true},
- {"QEMU guest agent is not running", "QEMU guest agent is not running", true},
- {"guest agent generic", "guest agent error", true},
- {"qmp command timeout", "qmp command 'guest-get-fsinfo' failed - got timeout", true},
- {"QMP command uppercase", "QMP Command failed", true},
- {"guest-get-fsinfo", "guest-get-fsinfo returned empty", true},
- {"guest-get-memory-blocks", "guest-get-memory-blocks failed", true},
- {"guest-get-vcpus", "error in guest-get-vcpus", true},
- {"authentication error not VM specific", "authentication failed", false},
- {"connection timeout not VM specific", "connection timeout", false},
- {"node offline not VM specific", "node offline", false},
- {"vm in error message but not agent", "VM 100 is offline", false},
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- result := isVMSpecificError(tt.errStr)
- if result != tt.expected {
- t.Errorf("isVMSpecificError(%q) = %v, want %v", tt.errStr, result, tt.expected)
- }
- })
- }
-}
-
func TestIsEndpointConnectivityError(t *testing.T) {
tests := []struct {
name string