Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require a usable polling session before cluster recovery

pulse-triage[bot]8eb0daddc94c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 1b6cca89a6..e838f423fb 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,27 @@
## Purpose
+### Cluster recovery requires a usable polling session
+
+A short endpoint probe cannot establish successful recovery until the client
+retained for ordinary polling has been constructed successfully. A refused or
+malformed password login for that second client preserves the existing client
+(or its absence), records the failure and does not count or publish recovery.
+An endpoint must never become healthy with a nil polling client. A later
+ordinary eligible recovery can install an accepted full-timeout session and
+clear its own error; usable siblings continue to serve polling throughout.
+
+`TestClusterRecoveryRequiresUsableSession` covers pinned HTTPS 401/403,
+malformed success and JSON-to-form refusal, with and without a retained client,
+then accepted recovery and ordinary reads. `TestClusterRecoverySessionEndpointProjection`
+drives public cluster reads and background recovery through the monitor's
+per-endpoint projection: native Online state, identity and saved pins survive,
+while PulseReachable and PulseError reflect the actual polling session. The
+existing recovery interval, authentication encodings, TLS trust/first-use
+policy, failover order and guest-command safety state are unchanged. These
+synthetic controls establish neither installed trust recovery nor a reported
+incident's cause.
+
### Rejected infrastructure URLs stay out of diagnostics
PVE/PBS constructors validate a host before logging it. URL syntax errors omit
diff --git a/internal/monitoring/monitor_pve_recovery_session_test.go b/internal/monitoring/monitor_pve_recovery_session_test.go
new file mode 100644
index 0000000000..0dc903aa8e
--- /dev/null
+++ b/internal/monitoring/monitor_pve_recovery_session_test.go
@@ -0,0 +1,148 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
+)
+
+// Public cluster reads drive the actual background recovery and the production
+// monitor's per-endpoint projection. The independent sibling stays usable,
+// while Proxmox's online state remains distinct from Pulse's usable session.
+func TestClusterRecoverySessionEndpointProjection(t *testing.T) {
+ var phase atomic.Int32
+ var recoveryLogins atomic.Int32
+ newEndpoint := func(primary bool) *httptest.Server {
+ return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api2/json/access/ticket":
+ if primary && phase.Load() == 1 && recoveryLogins.Add(1) > 1 {
+ w.WriteHeader(http.StatusForbidden)
+ fmt.Fprint(w, `{"message":"fixture polling session refused"}`)
+ return
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-session","CSRFPreventionToken":"fixture-csrf"}}`)
+ case "/api2/json/nodes":
+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-session") {
+ t.Error("ordinary polling lost fixture authentication")
+ }
+ if primary && phase.Load() == 0 {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ fmt.Fprint(w, `{"message":"fixture warming"}`)
+ return
+ }
+ fmt.Fprint(w, `{"data":[{"node":"primary","status":"online"},{"node":"sibling","status":"online"}]}`)
+ default:
+ t.Errorf("recovery dispatched outside ordinary nodes/session reads: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ }
+ primary, sibling := newEndpoint(true), newEndpoint(false)
+ defer primary.Close()
+ defer sibling.Close()
+ pin := func(s *httptest.Server) string {
+ value := sha256.Sum256(s.Certificate().Raw)
+ return hex.EncodeToString(value[:])
+ }
+ pins := map[string]string{primary.URL: pin(primary), sibling.URL: pin(sibling)}
+ cc := proxmox.NewClusterClient("recovery-projection", proxmox.ClientConfig{
+ Host: primary.URL, User: "fixture@pam", Password: "fixture-password",
+ Fingerprint: pins[primary.URL], VerifySSL: true, Timeout: time.Second,
+ }, []string{primary.URL, sibling.URL}, pins)
+ if cc.GetHealthStatus()[primary.URL] || !cc.GetHealthStatus()[sibling.URL] {
+ t.Fatal("fixture did not establish an unreachable primary with a usable sibling")
+ }
+ instance := config.PVEInstance{
+ Name: "recovery-projection", Host: primary.URL, IsCluster: true, VerifySSL: true,
+ Fingerprint: pins[primary.URL], ClusterEndpoints: []config.ClusterEndpoint{
+ {NodeName: "primary", NodeIdentity: "primary-identity", Host: primary.URL, Fingerprint: pins[primary.URL]},
+ {NodeName: "sibling", NodeIdentity: "sibling-identity", Host: sibling.URL, Fingerprint: pins[sibling.URL]},
+ },
+ }
+ monitor := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{instance}}}
+ project := func(stage string) {
+ monitor.updateClusterEndpointStatus(instance.Name, &instance, cc, []models.Node{
+ {Name: "primary", Status: "online"}, {Name: "sibling", Status: "online"},
+ })
+ if !instance.ClusterEndpoints[0].Online || !instance.ClusterEndpoints[1].Online {
+ t.Fatal("Pulse session evidence changed native cluster online state")
+ }
+ for i, endpoint := range instance.ClusterEndpoints {
+ if endpoint.NodeIdentity != []string{"primary-identity", "sibling-identity"}[i] || endpoint.Fingerprint != pins[endpoint.Host] {
+ t.Fatal("projection changed identity or saved TLS trust")
+ }
+ }
+ wire, err := json.Marshal(struct {
+ Phase string `json:"phase"`
+ Endpoints []config.ClusterEndpoint `json:"endpoints"`
+ }{stage, instance.ClusterEndpoints})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("CLUSTER_SESSION_PROJECTION_JSON=%s", wire)
+ }
+ project("initial")
+
+ // Honour the existing ten-second endpoint recovery interval. No process
+ // clock, monitor, credentials or guest-command fence is reset for this test.
+ phase.Store(1)
+ time.Sleep(11 * time.Second)
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 2 {
+ t.Fatalf("healthy sibling was blocked by primary recovery: %+v %v", nodes, err)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ health := cc.GetHealthStatusWithErrors()[primary.URL]
+ if health.Healthy || health.LastError == "Authentication failed - check API token or credentials" {
+ if health.Healthy || recoveryLogins.Load() != 2 {
+ t.Fatalf("failed actual polling session became healthy or replayed: %+v logins=%d", health, recoveryLogins.Load())
+ }
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("background recovery did not publish its actual session failure: %+v", health)
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ project("refused")
+ if endpoint := instance.ClusterEndpoints[0]; endpoint.PulseReachable == nil || *endpoint.PulseReachable || endpoint.PulseError == "" {
+ t.Fatalf("monitor lost refused primary's actual session evidence: %+v", endpoint)
+ }
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 2 || recoveryLogins.Load() != 2 {
+ t.Fatalf("ordinary sibling read replayed refused authentication: %+v %v logins=%d", nodes, err, recoveryLogins.Load())
+ }
+
+ phase.Store(2)
+ time.Sleep(11 * time.Second)
+ if _, err := cc.GetNodes(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ deadline = time.Now().Add(5 * time.Second)
+ for !cc.GetHealthStatus()[primary.URL] {
+ if time.Now().After(deadline) {
+ t.Fatal("ordinary later recovery did not accept the usable polling session")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 2 {
+ t.Fatalf("recovered primary cannot serve ordinary polling: %+v %v", nodes, err)
+ }
+ project("recovered")
+ if endpoint := instance.ClusterEndpoints[0]; endpoint.PulseReachable == nil || !*endpoint.PulseReachable || endpoint.PulseError != "" {
+ t.Fatalf("accepted recovery left stale primary failure: %+v", endpoint)
+ }
+}
diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go
index 937cee4e05..040a530aae 100644
--- a/pkg/proxmox/cluster_client.go
+++ b/pkg/proxmox/cluster_client.go
@@ -862,12 +862,23 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
vmSpecificErr := err != nil && isVMSpecificError(err.Error())
if err == nil || vmSpecificErr {
- recoveredEndpoints <- ep
-
// Store the client with original timeout
cfg.Timeout = cc.config.Timeout
cfg.Fingerprint = cc.getEndpointFingerprint(ep)
- fullClient, _ := NewClient(cfg)
+ fullClient, clientErr := NewClient(cfg)
+ if clientErr != nil {
+ // A short probe is not a usable polling session. Password
+ // authentication may fail when constructing the actual client;
+ // retain the existing client and failure instead of publishing
+ // a healthy endpoint with a nil client.
+ recordFailure(ep, clientErr)
+ log.Debug().
+ Str("cluster", cc.name).
+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).
+ Err(clientErr).
+ Msg("Failed to create full client after successful recovery probe")
+ return
+ }
cc.mu.Lock()
cc.nodeHealth[ep] = true
@@ -875,6 +886,7 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
cc.lastHealthCheck[ep] = time.Now()
cc.clients[ep] = fullClient
cc.mu.Unlock()
+ recoveredEndpoints <- ep
if vmSpecificErr {
log.Info().
diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.go
index e8d9477cac..02719163bb 100644
--- a/pkg/proxmox/cluster_client_api_test.go
+++ b/pkg/proxmox/cluster_client_api_test.go
@@ -12,6 +12,10 @@ import (
"time"
)
+func TestClusterRecoveryRequiresUsableSession(t *testing.T) {
+ testClusterRecoveryRequiresUsableSession(t)
+}
+
func TestClusterClient_GetCephStatus(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
diff --git a/pkg/proxmox/cluster_client_recovery_session_test.go b/pkg/proxmox/cluster_client_recovery_session_test.go
new file mode 100644
index 0000000000..bed66b6f1a
--- /dev/null
+++ b/pkg/proxmox/cluster_client_recovery_session_test.go
@@ -0,0 +1,119 @@
+package proxmox
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// Recovery constructs two password sessions: the short connectivity probe and
+// the client retained for ordinary polling. A successful first login cannot
+// establish that the second was accepted. All credentials and TLS peers here
+// are guest-local fixtures, not an installed authentication/recovery attempt.
+func testClusterRecoveryRequiresUsableSession(t *testing.T) {
+ for _, refusal := range []string{"unauthorised", "forbidden", "malformed", "form-refused"} {
+ for _, retained := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/retained-%t", refusal, retained), func(t *testing.T) {
+ var failing atomic.Bool
+ var logins, nodeReads atomic.Int32
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api2/json/access/ticket":
+ login := logins.Add(1)
+ if failing.Load() && login > 1 {
+ switch refusal {
+ case "unauthorised":
+ w.WriteHeader(http.StatusUnauthorized)
+ fmt.Fprint(w, `{"message":"fixture session refused"}`)
+ case "forbidden":
+ w.WriteHeader(http.StatusForbidden)
+ fmt.Fprint(w, `{"message":"fixture session refused"}`)
+ case "malformed":
+ fmt.Fprint(w, `{"data":`)
+ case "form-refused":
+ if r.Header.Get("Content-Type") == "application/json" {
+ w.WriteHeader(http.StatusUnsupportedMediaType)
+ } else {
+ w.WriteHeader(http.StatusForbidden)
+ }
+ fmt.Fprint(w, `{"message":"fixture session refused"}`)
+ }
+ return
+ }
+ fmt.Fprintf(w, `{"data":{"ticket":"fixture-ticket-%d","CSRFPreventionToken":"fixture-csrf-%d"}}`, login, login)
+ case "/api2/json/nodes":
+ nodeReads.Add(1)
+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket-") {
+ t.Error("ordinary node read lost the accepted fixture session")
+ }
+ fmt.Fprint(w, `{"data":[{"node":"node1","status":"online"}]}`)
+ default:
+ t.Errorf("unexpected request %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: time.Second}
+ cc := NewClusterClient("session-recovery", cfg, []string{server.URL}, nil)
+ var original *Client
+ if retained {
+ var err error
+ original, err = NewClient(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer original.httpClient.CloseIdleConnections()
+ cc.clients[server.URL] = original
+ }
+ logins.Store(0)
+ failing.Store(true)
+ cc.nodeHealth[server.URL] = false
+ cc.lastError[server.URL] = "previous connectivity failure"
+ cc.recoverUnhealthyNodes(context.Background())
+
+ health := cc.GetHealthStatusWithErrors()[server.URL]
+ if health.Healthy || health.LastError != "Authentication failed - check API token or credentials" || health.LastCheck.IsZero() {
+ t.Fatalf("failed polling session published recovery or lost the refusal: %+v", health)
+ }
+ client, exists := cc.clients[server.URL]
+ if exists != retained || client != original {
+ t.Fatalf("failed replacement changed retained client: exists=%t retained=%t nil=%t", exists, retained, client == nil)
+ }
+ wantLogins := int32(2)
+ if refusal == "form-refused" {
+ wantLogins = 3
+ }
+ if logins.Load() != wantLogins || nodeReads.Load() != 1 || cc.getEndpointFingerprint(server.URL) != cfg.Fingerprint {
+ t.Fatalf("refusal replayed, changed trust or lost its probe: logins=%d nodes=%d", logins.Load(), nodeReads.Load())
+ }
+
+ // Advance only this fixture's recovery eligibility. No credential,
+ // trust, persistent state or uncertainty fence is reset.
+ failing.Store(false)
+ delete(cc.lastHealthCheck, server.URL)
+ cc.recoverUnhealthyNodes(context.Background())
+ actual := cc.clients[server.URL]
+ if actual == nil || actual == original || actual.config.Timeout != cfg.Timeout {
+ t.Fatal("accepted later recovery did not publish a usable full-timeout client")
+ }
+ defer actual.httpClient.CloseIdleConnections()
+ if health := cc.GetHealthStatusWithErrors()[server.URL]; !health.Healthy || health.LastError != "" {
+ t.Fatalf("accepted later session did not clear its own failure: %+v", health)
+ }
+ nodes, err := cc.GetNodes(context.Background())
+ if err != nil || len(nodes) != 1 || nodes[0].Node != "node1" || nodeReads.Load() != 3 {
+ t.Fatalf("ordinary polling did not use recovered session: %+v %v reads=%d", nodes, err, nodeReads.Load())
+ }
+ })
+ }
+ }
+}