Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep failed Proxmox nodes probes out of recovered health
pulse-triage[bot]4147f48db3bb
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 7ae9c78a0b..6dc321cb3b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,36 @@## Purpose+### Cluster recovery requires success of the operation actually probed++Startup and unhealthy-member recovery read `/nodes`; lazy construction uses the+same probe. A failed nodes or login request cannot count as successful recovery+because its response text mentions a guest agent, QMP or `guest-get-*`. The old+per-guest substring exception is removed from these non-guest operations.+Failed selection retains its actual sanitised reason and observation time, and+installs no polling client. Successful probe plus accepted full-timeout session+is still required. The existing temporary-rate-limit path may admit an ordinary+request, but does not report the failed probe as passed or clear a known failure.++The single-endpoint last-resort path remains available with its cached client;+selecting it no longer marks that endpoint healthy before a request succeeds.+A later accepted ordinary read still clears the failure. Actual per-guest+errors remain request-level errors, without marking an otherwise working node+unhealthy. Certificate pins, first-use trust, credential encodings, request and+recovery cadence, rate-limit backoff and guest/backup fences are unchanged.++`TestClusterNodeProbeRequiresSuccess` covers startup/recovery/selection in token,+JSON password and form password modes, three guest-like failure bodies at the+nodes or login stage, no extra login/full client and distinct later recovery.+`TestClusterSingleEndpointFallbackKeepsFailedHealth` checks cached selection,+failed ordinary read and later success. `TestClusterNodeProbeRateLimitKeepsOrdinaryRead`+preserves genuine rate-limit handling. `TestClusterNodeProbeEndpointProjection`+carries the actual failed and later recovered member through the production+monitor projection while independent native Online, saved trust and identity+remain unchanged. These are source/consumer controls, not attribution or+installed acceptance for a reported PBS/QGA/backup incident.++### First-use cluster trust includes password constructionDeclared, previously untrusted Proxmox members use the existing credential-freediff --git a/internal/monitoring/monitor_pve_node_probe_test.go b/internal/monitoring/monitor_pve_node_probe_test.gonew file mode 100644index 0000000000..e6ab0848a4--- /dev/null+++ b/internal/monitoring/monitor_pve_node_probe_test.go@@ -0,0 +1,121 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// A failed member /nodes response cannot establish a healthy polling endpoint+// merely by quoting a guest error. The primary's accepted inventory still+// reports both members online. Public reads initiate later ordinary recovery;+// no guest command, backup, credential change or guest fence is exercised.+func TestClusterNodeProbeEndpointProjection(t *testing.T) {+ var acceptMember atomic.Bool+ var memberLogins, memberNodes atomic.Int32+ endpoint := func(member bool) *httptest.Server {+ s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api2/json/access/ticket":+ if member {+ memberLogins.Add(1)+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ case "/api2/json/nodes":+ if r.Header.Get("Cookie") != "PVEAuthCookie=fixture-ticket" {+ t.Error("ordinary nodes read lost password authentication")+ }+ if member {+ memberNodes.Add(1)+ if !acceptMember.Load() {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, "QEMU guest agent is not running")+ return+ }+ }+ fmt.Fprint(w, `{"data":[{"node":"primary","status":"online","cpu":0.125},{"node":"member","status":"online","cpu":0.25}]}`)+ default:+ t.Errorf("left ordinary session/nodes reads: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(s.Close)+ return s+ }+ primary, member := endpoint(false), endpoint(true)+ pin := func(s *httptest.Server) string {+ v := sha256.Sum256(s.Certificate().Raw)+ return hex.EncodeToString(v[:])+ }+ pins := map[string]string{primary.URL: pin(primary), member.URL: pin(member)}+ cc := proxmox.NewClusterClient("node-probe-cluster", proxmox.ClientConfig{Host: primary.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: pins[primary.URL], Timeout: 2 * time.Second}, []string{primary.URL, member.URL}, pins)+ if !cc.GetHealthStatus()[primary.URL] || cc.GetHealthStatus()[member.URL] || memberLogins.Load() != 1 || memberNodes.Load() != 1 {+ t.Fatalf("failed nodes probe became healthy or advanced to a full session: logins=%d nodes=%d", memberLogins.Load(), memberNodes.Load())+ }+ inst := config.PVEInstance{Name: "node-probe-cluster", Host: primary.URL, IsCluster: true, ClusterName: "Fixture cluster", VerifySSL: true, Fingerprint: pins[primary.URL], ClusterEndpoints: []config.ClusterEndpoint{+ {NodeName: "primary", NodeIdentity: "primary-identity", Host: primary.URL, Fingerprint: pins[primary.URL]},+ {NodeName: "member", NodeIdentity: "member-identity", Host: member.URL, Fingerprint: pins[member.URL]},+ }}+ m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{inst}}}+ project := func(phase string) {+ nodes, err := cc.GetNodes(context.Background())+ if err != nil || len(nodes) != 2 {+ t.Fatalf("accepted primary read failed: %+v %v", nodes, err)+ }+ model := make([]models.Node, len(nodes))+ for i, n := range nodes {+ model[i] = models.Node{Name: n.Node, Status: n.Status}+ }+ m.updateClusterEndpointStatus(inst.Name, &inst, cc, model)+ for i, ep := range inst.ClusterEndpoints {+ if !ep.Online || ep.NodeIdentity != []string{"primary-identity", "member-identity"}[i] || ep.Fingerprint != pins[ep.Host] {+ t.Fatal("projection changed native online state, identity or saved trust")+ }+ }+ wire, err := json.Marshal(struct {+ Phase string `json:"phase"`+ Endpoints []config.ClusterEndpoint `json:"endpoints"`+ }{phase, inst.ClusterEndpoints})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("NODE_PROBE_ENDPOINT_PROJECTION_JSON=%s", wire)+ }+ project("nodes-probe-refused")+ if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || *ep.PulseReachable || ep.PulseError != "API error 500: QEMU guest agent is not running" {+ t.Fatalf("consumer lost failed nodes observation: %+v", ep)+ }+ acceptMember.Store(true)+ // Honour the real recovery interval. Do not clear a production admission,+ // replace clients or change clocks to manufacture the consuming result.+ time.Sleep(11 * time.Second)+ if _, err := cc.GetNodes(context.Background()); err != nil {+ t.Fatal(err)+ }+ deadline := time.Now().Add(5 * time.Second)+ for !cc.GetHealthStatus()[member.URL] {+ if time.Now().After(deadline) {+ t.Fatal("later ordinary recovery did not accept member")+ }+ time.Sleep(5 * time.Millisecond)+ }+ if memberLogins.Load() != 3 || memberNodes.Load() != 2 {+ t.Fatalf("later recovery did not use one distinct probe/full session: logins=%d nodes=%d", memberLogins.Load(), memberNodes.Load())+ }+ project("member-recovered")+ if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || !*ep.PulseReachable || ep.PulseError != "" {+ t.Fatalf("accepted recovery retained stale failure: %+v", ep)+ }+}diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.goindex 441a7c3e01..9dd1e0b39f 100644--- a/pkg/proxmox/cluster_client.go+++ b/pkg/proxmox/cluster_client.go@@ -53,33 +53,6 @@ var transientRateLimitStatusCodes = map[int]struct{}{504: {},}-// isVMSpecificError reports whether an error string is scoped to a single VM/guest agent-// and should not be treated as a node connectivity failure.-func isVMSpecificError(errStr string) bool {- if errStr == "" {- return false- }- lower := strings.ToLower(errStr)-- if strings.Contains(lower, "no qemu guest agent") ||- strings.Contains(lower, "qemu guest agent is not running") ||- strings.Contains(lower, "guest agent") {- return true- }-- // QMP guest agent operations can time out or fail per-VM (e.g. guest-get-fsinfo).- // These aren't node connectivity issues and should not mark endpoints unhealthy.- if strings.Contains(lower, "qmp command") {- return true- }-- if strings.Contains(lower, "guest-get-") {- return true- }-- return false-}-// isEndpointConnectivityError reports whether an error indicates the endpoint// itself is unreachable (TCP, DNS, TLS failures). Application-level errors// (HTTP responses, parsing errors) mean the endpoint IS reachable.@@ -429,10 +402,9 @@ func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {defer testClient.httpClient.CloseIdleConnections()}- // Check if error is VM-specific (shouldn't affect health)- vmSpecificErr := err != nil && isVMSpecificError(err.Error())-- if err == nil || vmSpecificErr {+ // The probe reads /nodes, not a guest endpoint. Provider text about+ // QGA cannot turn a failed nodes/login request into usable polling.+ if err == nil {// Node is healthy - create a proper client with full timeout for actual usefullCfg := cc.configfullCfg.Host = ep@@ -456,17 +428,10 @@ func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {cc.lastHealthCheck[ep] = time.Now()cc.clients[ep] = fullClient // Store the full client, not test clientcc.mu.Unlock()- if vmSpecificErr {- log.Debug().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).- Msg("Cluster endpoint healthy despite VM-specific errors")- } else {- log.Info().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).- Msg("Cluster endpoint passed initial health check")- }+ log.Info().+ Str("cluster", cc.name).+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).+ Msg("Cluster endpoint passed initial health check")}} else {// Real connectivity issue@@ -646,8 +611,8 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(cc.endpoints[0])).Msg("Single endpoint appears unhealthy but attempting to use it anyway")healthyEndpoints = cc.endpoints- // Mark it as healthy optimistically- cc.nodeHealth[cc.endpoints[0]] = true+ // Selection is not recovery evidence. Keep the failure until a+ // successful ordinary request clears it, even with a cached client.} else {// Provide detailed error with endpoint status. Surfacing the// per-endpoint reason (already sanitized) preserves the original@@ -731,41 +696,29 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)Msg("Ignoring transient rate limit error during connectivity test")// Continue with client creation since the node is accessible, just rate limited} else {- // Check if this is a VM-specific error that shouldn't mark the node unhealthy- testErrStr := testErr.Error()- if strings.Contains(testErrStr, "No QEMU guest agent") ||- strings.Contains(testErrStr, "QEMU guest agent is not running") ||- strings.Contains(testErrStr, "guest agent") {- // This is a VM-specific issue, not a connectivity problem- // The node is actually healthy, so don't mark it unhealthy- log.Debug().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).- Err(testErr).- Msg("Ignoring VM-specific error during connectivity test")- // Continue with client creation since the node is actually accessible- } else {- // Mark as unhealthy for real connectivity issues- cc.nodeHealth[selectedEndpoint] = false- log.Warn().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).- Err(testErr).- Msg("Failed to connect to Proxmox endpoint; endpoint removed from rotation until next refresh")- return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), testErr)- }+ cc.nodeHealth[selectedEndpoint] = false+ cc.lastError[selectedEndpoint] = sanitizeEndpointError(testErr.Error())+ cc.lastHealthCheck[selectedEndpoint] = time.Now()+ log.Warn().+ Str("cluster", cc.name).+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).+ Err(testErr).+ Msg("Failed to read Proxmox nodes; endpoint removed from rotation until next refresh")+ return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), testErr)}}- log.Debug().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).- Func(func(event *zerolog.Event) {- if testNodes != nil {- event.Int("nodes", len(testNodes))- }- }).- Msg("Cluster endpoint passed connectivity test")+ if testErr == nil {+ log.Debug().+ Str("cluster", cc.name).+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint)).+ Func(func(event *zerolog.Event) {+ if testNodes != nil {+ event.Int("nodes", len(testNodes))+ }+ }).+ Msg("Cluster endpoint passed connectivity test")+ }// Use the first-use winner's pin for the full polling session too.cfg.Fingerprint = cc.getEndpointFingerprintLocked(selectedEndpoint)@@ -777,12 +730,19 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)}// This shouldn't happen since we just tested itcc.nodeHealth[selectedEndpoint] = false+ cc.lastError[selectedEndpoint] = sanitizeEndpointError(err.Error())+ cc.lastHealthCheck[selectedEndpoint] = time.Now()return nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)}- // A completed probe cannot clear an error until the polling session is- // accepted. Ending a caller is not endpoint failure or recovery evidence.- delete(cc.lastError, selectedEndpoint)+ // A successful nodes probe and accepted polling session establish+ // recovery. The existing rate-limit exception only admits an ordinary+ // request; it cannot clear a recorded failure before that request succeeds.+ if testErr == nil {+ cc.nodeHealth[selectedEndpoint] = true+ cc.lastHealthCheck[selectedEndpoint] = time.Now()+ delete(cc.lastError, selectedEndpoint)+ }cc.clients[selectedEndpoint] = newClientclient = newClient}@@ -909,10 +869,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {return}- // Check if error is VM-specific (shouldn't prevent recovery)- vmSpecificErr := err != nil && isVMSpecificError(err.Error())-- if err == nil || vmSpecificErr {+ // Recovery requires the actual nodes probe to succeed; a guest-like+ // error body is not evidence about this endpoint's nodes inventory.+ if err == nil {// Store the client with original timeoutcfg.Timeout = cc.config.Timeoutcfg.Fingerprint = cc.getEndpointFingerprint(ep)@@ -944,17 +903,10 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {cc.mu.Unlock()recoveredEndpoints <- ep- if vmSpecificErr {- log.Info().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).- Msg("Recovered unhealthy cluster node (ignoring VM-specific errors)")- } else {- log.Info().- Str("cluster", cc.name).- Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).- Msg("Recovered unhealthy cluster node")- }+ log.Info().+ Str("cluster", cc.name).+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).+ Msg("Recovered unhealthy cluster node")} else {recordFailure(ep, err)log.Debug().diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.goindex 8034b4275f..b460d77f76 100644--- a/pkg/proxmox/cluster_client_api_test.go+++ b/pkg/proxmox/cluster_client_api_test.go@@ -754,3 +754,160 @@ func TestClusterClientFirstUseCaptureCancellation(t *testing.T) {func TestClusterFirstUseSessionAdmission(t *testing.T) {testClusterFirstUseSessionAdmission(t)}++// A nodes/login failure is not a per-guest failure, even when the provider's+// body mentions QGA. Exercise the real constructors/probes in all three phases;+// none of these handlers can enqueue a guest command or perform a backup.+func TestClusterNodeProbeRequiresSuccess(t *testing.T) {+ for _, phase := range []string{"startup", "recovery", "selection"} {+ for _, mode := range []string{"token", "json", "form"} {+ for _, stage := range []string{"nodes", "login"} {+ if stage == "login" && mode == "token" {+ continue+ }+ for i, message := range []string{"QEMU guest agent is not running", "VM 105 qmp command 'guest-get-fsinfo' failed", "proxy: guest-get-osinfo unavailable"} {+ t.Run(fmt.Sprintf("%s/%s/%s/body-%d", phase, mode, stage, i), func(t *testing.T) {+ f := newNodeProbeFixture(t, mode, stage, message)+ cfg := f.config()+ saved := map[string]string{f.server.URL: cfg.Fingerprint}+ f.reject.Store(true)+ var cc *ClusterClient+ if phase == "startup" {+ sibling := newNodeProbeFixture(t, mode, "nodes", "")+ saved[sibling.server.URL] = sibling.config().Fingerprint+ cc = NewClusterClient("node-probe", cfg, []string{f.server.URL, sibling.server.URL}, saved)+ if !cc.GetHealthStatus()[sibling.server.URL] {+ t.Fatal("failed peer blocked accepted sibling")+ }+ } else {+ cc = NewClusterClient("node-probe", cfg, []string{f.server.URL}, saved)+ if phase == "recovery" {+ cc.nodeHealth[f.server.URL] = false+ cc.lastError[f.server.URL] = "previous failed observation"+ cc.recoverUnhealthyNodes(context.Background())+ } else {+ c, err := cc.getHealthyClient(context.Background())+ if c != nil || err == nil {+ t.Error("failed nodes probe installed a polling client")+ }+ }+ }+ t.Cleanup(func() {+ cc.mu.RLock()+ defer cc.mu.RUnlock()+ for _, c := range cc.clients {+ if c != nil {+ c.httpClient.CloseIdleConnections()+ }+ }+ })+ health := cc.GetHealthStatusWithErrors()[f.server.URL]+ if health.Healthy || health.LastError == "" || health.LastCheck.IsZero() {+ t.Errorf("failed probe published healthy or lost its reason/time: %+v", health)+ }+ if cc.clients[f.server.URL] != nil {+ t.Error("failed probe installed a full polling client")+ }+ wantNodes := int32(1)+ if stage == "login" {+ wantNodes = 0+ }+ if f.logins.Load() != f.loginAttempts() || f.nodes.Load() != wantNodes {+ t.Errorf("failed probe was retried/advanced: logins=%d want=%d nodes=%d want=%d", f.logins.Load(), f.loginAttempts(), f.nodes.Load(), wantNodes)+ }+ if cc.getEndpointFingerprint(f.server.URL) != cfg.Fingerprint || saved[f.server.URL] != cfg.Fingerprint || cc.config.VerifySSL != true {+ t.Error("failed probe changed saved trust")+ }+ // A distinct later recovery uses this fixture's normal accepted reply.+ // Only endpoint recovery eligibility advances, never a guest fence.+ f.reject.Store(false)+ cc.mu.Lock()+ delete(cc.lastHealthCheck, f.server.URL)+ cc.mu.Unlock()+ cc.recoverUnhealthyNodes(context.Background())+ if c := cc.clients[f.server.URL]; c == nil || c.config.Timeout != cfg.Timeout {+ t.Fatal("accepted later recovery has no full-timeout polling client")+ }+ cc.mu.Lock()+ for ep := range cc.nodeHealth {+ if ep != f.server.URL {+ cc.nodeHealth[ep] = false+ cc.lastHealthCheck[ep] = time.Now()+ }+ }+ cc.mu.Unlock()+ nodes, err := cc.GetNodes(context.Background())+ health = cc.GetHealthStatusWithErrors()[f.server.URL]+ if err != nil || len(nodes) != 1 || nodes[0].CPU != 0.25 || !health.Healthy || health.LastError != "" || f.nodes.Load() != wantNodes+2 || f.logins.Load() != 3*f.loginAttempts() {+ t.Errorf("separate ordinary recovery did not restore coherent readings: err=%v health=%+v logins=%d nodes=%d", err, health, f.logins.Load(), f.nodes.Load())+ }+ })+ }+ }+ }+ }+}++// Keep the existing single-endpoint last-resort read, but do not publish+// recovery simply for selecting its cached client. A failed read is still a+// failure; only a later successful ordinary read clears the observed error.+func TestClusterSingleEndpointFallbackKeepsFailedHealth(t *testing.T) {+ f := newNodeProbeFixture(t, "token", "nodes", "QEMU guest agent is not running")+ cc := NewClusterClient("single-probe", f.config(), []string{f.server.URL}, nil)+ if _, err := cc.GetNodes(context.Background()); err != nil {+ t.Fatal(err)+ }+ original := cc.clients[f.server.URL]+ defer original.httpClient.CloseIdleConnections()+ f.reject.Store(true)+ cc.markUnhealthyWithError(f.server.URL, "previous connection failure")+ c, err := cc.getHealthyClient(context.Background())+ if err != nil || c != original {+ t.Fatal("last-resort selection lost the existing client")+ }+ if h := cc.GetHealthStatusWithErrors()[f.server.URL]; h.Healthy || h.LastError == "" {+ t.Errorf("selection invented endpoint recovery: %+v", h)+ }+ before := f.nodes.Load()+ if _, err := cc.GetNodes(context.Background()); err == nil || !strings.Contains(err.Error(), "API error 500") {+ t.Errorf("failed ordinary read was hidden: %v", err)+ }+ if h := cc.GetHealthStatusWithErrors()[f.server.URL]; h.Healthy || h.LastError == "" || f.nodes.Load() != before+1 {+ t.Errorf("failed fallback read published recovery or replayed: %+v reads=%d", h, f.nodes.Load())+ }+ f.reject.Store(false)+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 {+ t.Fatalf("accepted later read did not recover: %+v %v", nodes, err)+ }+ if h := cc.GetHealthStatusWithErrors()[f.server.URL]; !h.Healthy || h.LastError != "" || f.nodes.Load() != before+2 || cc.clients[f.server.URL] != original {+ t.Errorf("accepted fallback did not restore health with the same client: %+v", h)+ }+}++// A genuine temporary nodes rate limit retains the established lazy-selection+// route: the actual ordinary read, not the failed probe, supplies its readings.+func TestClusterNodeProbeRateLimitKeepsOrdinaryRead(t *testing.T) {+ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/api2/json/nodes" {+ t.Errorf("unexpected request: %s", r.URL.Path)+ http.NotFound(w, r)+ return+ }+ if calls.Add(1) == 1 {+ w.WriteHeader(http.StatusTooManyRequests)+ fmt.Fprint(w, "fixture busy")+ return+ }+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online","cpu":0.25}]}`)+ }))+ defer server.Close()+ cc := NewClusterClient("probe-rate-limit", ClientConfig{Host: server.URL, TokenName: "fixture@pam!pulse", TokenValue: "fixture-token"}, []string{server.URL}, nil)+ nodes, err := cc.GetNodes(context.Background())+ if c := cc.clients[server.URL]; c != nil {+ defer c.httpClient.CloseIdleConnections()+ }+ if err != nil || len(nodes) != 1 || nodes[0].CPU != 0.25 || calls.Load() != 2 || !cc.GetHealthStatus()[server.URL] {+ t.Fatalf("temporary probe rate limit blocked ordinary read: %+v err=%v calls=%d", nodes, err, calls.Load())+ }+}diff --git a/pkg/proxmox/cluster_client_initial_health_test.go b/pkg/proxmox/cluster_client_initial_health_test.goindex d65f6a708a..24a88bcbe2 100644--- a/pkg/proxmox/cluster_client_initial_health_test.go+++ b/pkg/proxmox/cluster_client_initial_health_test.go@@ -8,7 +8,7 @@ import ("testing")-func TestClusterClient_InitialHealthCheck_VMErrorTreatedHealthy(t *testing.T) {+func TestClusterClient_InitialHealthCheck_FailedNodesProbeStaysUnhealthy(t *testing.T) {server1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if r.URL.Path == "/api2/json/nodes" {w.WriteHeader(http.StatusInternalServerError)@@ -32,11 +32,14 @@ func TestClusterClient_InitialHealthCheck_VMErrorTreatedHealthy(t *testing.T) {cc := NewClusterClient("test", cfg, []string{server1.URL, server2.URL}, nil)health := cc.GetHealthStatusWithErrors()- if !health[server1.URL].Healthy {- t.Fatalf("expected vm-specific error endpoint to be healthy, got %+v", health[server1.URL])+ if health[server1.URL].Healthy {+ t.Fatalf("failed nodes probe must stay unhealthy, got %+v", health[server1.URL])}- if health[server1.URL].LastError != "" {- t.Fatalf("expected no last error, got %q", health[server1.URL].LastError)+ if health[server1.URL].LastError == "" {+ t.Fatalf("expected failed nodes reason, got %q", health[server1.URL].LastError)+ }+ if cc.clients[server1.URL] != nil {+ t.Fatal("failed nodes probe installed a full client")}if !health[server2.URL].Healthy {t.Fatalf("expected second endpoint healthy, got %+v", health[server2.URL])diff --git a/pkg/proxmox/cluster_client_node_probe_test.go b/pkg/proxmox/cluster_client_node_probe_test.gonew file mode 100644index 0000000000..a5b00719f1--- /dev/null+++ b/pkg/proxmox/cluster_client_node_probe_test.go@@ -0,0 +1,85 @@+package proxmox++import (+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+ "time"+)++type nodeProbeFixture struct {+ server *httptest.Server+ reject atomic.Bool+ logins, nodes atomic.Int32+ mode, stage, message string+}++func newNodeProbeFixture(t *testing.T, mode, stage, message string) *nodeProbeFixture {+ t.Helper()+ f := &nodeProbeFixture{mode: mode, stage: stage, message: message}+ f.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api2/json/access/ticket":+ f.logins.Add(1)+ if mode == "token" {+ t.Error("token mode attempted password authentication")+ }+ if mode == "form" && r.Header.Get("Content-Type") == "application/json" {+ w.WriteHeader(http.StatusUnsupportedMediaType)+ return+ }+ if f.reject.Load() && stage == "login" {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, message)+ return+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ case "/api2/json/nodes":+ f.nodes.Add(1)+ if mode == "token" {+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pam!pulse=fixture-token" {+ t.Error("nodes read lost token authentication")+ }+ } else if r.Header.Get("Cookie") != "PVEAuthCookie=fixture-ticket" {+ t.Error("nodes read lost the accepted password session")+ }+ if f.reject.Load() && stage == "nodes" {+ w.WriteHeader(http.StatusInternalServerError)+ fmt.Fprint(w, message)+ return+ }+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online","cpu":0.25}]}`)+ default:+ t.Errorf("probe left ordinary session/nodes paths: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(f.server.Close)+ return f+}++func (f *nodeProbeFixture) config() ClientConfig {+ sum := sha256.Sum256(f.server.Certificate().Raw)+ cfg := ClientConfig{Host: f.server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(sum[:]), Timeout: 2 * time.Second}+ if f.mode == "token" {+ cfg.Password = ""+ cfg.TokenName = "fixture@pam!pulse"+ cfg.TokenValue = "fixture-token"+ }+ return cfg+}++func (f *nodeProbeFixture) loginAttempts() int32 {+ if f.mode == "token" {+ return 0+ }+ if f.mode == "form" {+ return 2+ }+ return 1+}diff --git a/pkg/proxmox/cluster_client_test.go b/pkg/proxmox/cluster_client_test.goindex 19ebf4f59c..b6a87e08af 100644--- a/pkg/proxmox/cluster_client_test.go+++ b/pkg/proxmox/cluster_client_test.go@@ -226,40 +226,6 @@ func TestIsNotImplementedError(t *testing.T) {}}-func TestIsVMSpecificError(t *testing.T) {- tests := []struct {- name string- errStr string- expected bool- }{- {"empty string", "", false},- {"generic connection error", "connection refused", false},- {"no qemu guest agent", "no qemu guest agent", true},- {"No QEMU guest agent uppercase", "No QEMU guest agent running", true},- {"qemu guest agent is not running", "qemu guest agent is not running", true},- {"QEMU guest agent is not running", "QEMU guest agent is not running", true},- {"guest agent generic", "guest agent error", true},- {"qmp command timeout", "qmp command 'guest-get-fsinfo' failed - got timeout", true},- {"QMP command uppercase", "QMP Command failed", true},- {"guest-get-fsinfo", "guest-get-fsinfo returned empty", true},- {"guest-get-memory-blocks", "guest-get-memory-blocks failed", true},- {"guest-get-vcpus", "error in guest-get-vcpus", true},- {"authentication error not VM specific", "authentication failed", false},- {"connection timeout not VM specific", "connection timeout", false},- {"node offline not VM specific", "node offline", false},- {"vm in error message but not agent", "VM 100 is offline", false},- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- result := isVMSpecificError(tt.errStr)- if result != tt.expected {- t.Errorf("isVMSpecificError(%q) = %v, want %v", tt.errStr, result, tt.expected)- }- })- }-}-func TestIsEndpointConnectivityError(t *testing.T) {tests := []struct {name string