Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require a usable polling session before cluster recovery
pulse-triage[bot]8eb0daddc94c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 1b6cca89a6..e838f423fb 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,27 @@## Purpose+### Cluster recovery requires a usable polling session++A short endpoint probe cannot establish successful recovery until the client+retained for ordinary polling has been constructed successfully. A refused or+malformed password login for that second client preserves the existing client+(or its absence), records the failure and does not count or publish recovery.+An endpoint must never become healthy with a nil polling client. A later+ordinary eligible recovery can install an accepted full-timeout session and+clear its own error; usable siblings continue to serve polling throughout.++`TestClusterRecoveryRequiresUsableSession` covers pinned HTTPS 401/403,+malformed success and JSON-to-form refusal, with and without a retained client,+then accepted recovery and ordinary reads. `TestClusterRecoverySessionEndpointProjection`+drives public cluster reads and background recovery through the monitor's+per-endpoint projection: native Online state, identity and saved pins survive,+while PulseReachable and PulseError reflect the actual polling session. The+existing recovery interval, authentication encodings, TLS trust/first-use+policy, failover order and guest-command safety state are unchanged. These+synthetic controls establish neither installed trust recovery nor a reported+incident's cause.+### Rejected infrastructure URLs stay out of diagnosticsPVE/PBS constructors validate a host before logging it. URL syntax errors omitdiff --git a/internal/monitoring/monitor_pve_recovery_session_test.go b/internal/monitoring/monitor_pve_recovery_session_test.gonew file mode 100644index 0000000000..0dc903aa8e--- /dev/null+++ b/internal/monitoring/monitor_pve_recovery_session_test.go@@ -0,0 +1,148 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Public cluster reads drive the actual background recovery and the production+// monitor's per-endpoint projection. The independent sibling stays usable,+// while Proxmox's online state remains distinct from Pulse's usable session.+func TestClusterRecoverySessionEndpointProjection(t *testing.T) {+ var phase atomic.Int32+ var recoveryLogins atomic.Int32+ newEndpoint := func(primary bool) *httptest.Server {+ return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api2/json/access/ticket":+ if primary && phase.Load() == 1 && recoveryLogins.Add(1) > 1 {+ w.WriteHeader(http.StatusForbidden)+ fmt.Fprint(w, `{"message":"fixture polling session refused"}`)+ return+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-session","CSRFPreventionToken":"fixture-csrf"}}`)+ case "/api2/json/nodes":+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-session") {+ t.Error("ordinary polling lost fixture authentication")+ }+ if primary && phase.Load() == 0 {+ w.WriteHeader(http.StatusServiceUnavailable)+ fmt.Fprint(w, `{"message":"fixture warming"}`)+ return+ }+ fmt.Fprint(w, `{"data":[{"node":"primary","status":"online"},{"node":"sibling","status":"online"}]}`)+ default:+ t.Errorf("recovery dispatched outside ordinary nodes/session reads: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ }+ primary, sibling := newEndpoint(true), newEndpoint(false)+ defer primary.Close()+ defer sibling.Close()+ pin := func(s *httptest.Server) string {+ value := sha256.Sum256(s.Certificate().Raw)+ return hex.EncodeToString(value[:])+ }+ pins := map[string]string{primary.URL: pin(primary), sibling.URL: pin(sibling)}+ cc := proxmox.NewClusterClient("recovery-projection", proxmox.ClientConfig{+ Host: primary.URL, User: "fixture@pam", Password: "fixture-password",+ Fingerprint: pins[primary.URL], VerifySSL: true, Timeout: time.Second,+ }, []string{primary.URL, sibling.URL}, pins)+ if cc.GetHealthStatus()[primary.URL] || !cc.GetHealthStatus()[sibling.URL] {+ t.Fatal("fixture did not establish an unreachable primary with a usable sibling")+ }+ instance := config.PVEInstance{+ Name: "recovery-projection", Host: primary.URL, IsCluster: true, VerifySSL: true,+ Fingerprint: pins[primary.URL], ClusterEndpoints: []config.ClusterEndpoint{+ {NodeName: "primary", NodeIdentity: "primary-identity", Host: primary.URL, Fingerprint: pins[primary.URL]},+ {NodeName: "sibling", NodeIdentity: "sibling-identity", Host: sibling.URL, Fingerprint: pins[sibling.URL]},+ },+ }+ monitor := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{instance}}}+ project := func(stage string) {+ monitor.updateClusterEndpointStatus(instance.Name, &instance, cc, []models.Node{+ {Name: "primary", Status: "online"}, {Name: "sibling", Status: "online"},+ })+ if !instance.ClusterEndpoints[0].Online || !instance.ClusterEndpoints[1].Online {+ t.Fatal("Pulse session evidence changed native cluster online state")+ }+ for i, endpoint := range instance.ClusterEndpoints {+ if endpoint.NodeIdentity != []string{"primary-identity", "sibling-identity"}[i] || endpoint.Fingerprint != pins[endpoint.Host] {+ t.Fatal("projection changed identity or saved TLS trust")+ }+ }+ wire, err := json.Marshal(struct {+ Phase string `json:"phase"`+ Endpoints []config.ClusterEndpoint `json:"endpoints"`+ }{stage, instance.ClusterEndpoints})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("CLUSTER_SESSION_PROJECTION_JSON=%s", wire)+ }+ project("initial")++ // Honour the existing ten-second endpoint recovery interval. No process+ // clock, monitor, credentials or guest-command fence is reset for this test.+ phase.Store(1)+ time.Sleep(11 * time.Second)+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 2 {+ t.Fatalf("healthy sibling was blocked by primary recovery: %+v %v", nodes, err)+ }+ deadline := time.Now().Add(5 * time.Second)+ for {+ health := cc.GetHealthStatusWithErrors()[primary.URL]+ if health.Healthy || health.LastError == "Authentication failed - check API token or credentials" {+ if health.Healthy || recoveryLogins.Load() != 2 {+ t.Fatalf("failed actual polling session became healthy or replayed: %+v logins=%d", health, recoveryLogins.Load())+ }+ break+ }+ if time.Now().After(deadline) {+ t.Fatalf("background recovery did not publish its actual session failure: %+v", health)+ }+ time.Sleep(5 * time.Millisecond)+ }+ project("refused")+ if endpoint := instance.ClusterEndpoints[0]; endpoint.PulseReachable == nil || *endpoint.PulseReachable || endpoint.PulseError == "" {+ t.Fatalf("monitor lost refused primary's actual session evidence: %+v", endpoint)+ }+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 2 || recoveryLogins.Load() != 2 {+ t.Fatalf("ordinary sibling read replayed refused authentication: %+v %v logins=%d", nodes, err, recoveryLogins.Load())+ }++ phase.Store(2)+ time.Sleep(11 * time.Second)+ if _, err := cc.GetNodes(context.Background()); err != nil {+ t.Fatal(err)+ }+ deadline = time.Now().Add(5 * time.Second)+ for !cc.GetHealthStatus()[primary.URL] {+ if time.Now().After(deadline) {+ t.Fatal("ordinary later recovery did not accept the usable polling session")+ }+ time.Sleep(5 * time.Millisecond)+ }+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 2 {+ t.Fatalf("recovered primary cannot serve ordinary polling: %+v %v", nodes, err)+ }+ project("recovered")+ if endpoint := instance.ClusterEndpoints[0]; endpoint.PulseReachable == nil || !*endpoint.PulseReachable || endpoint.PulseError != "" {+ t.Fatalf("accepted recovery left stale primary failure: %+v", endpoint)+ }+}diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.goindex 937cee4e05..040a530aae 100644--- a/pkg/proxmox/cluster_client.go+++ b/pkg/proxmox/cluster_client.go@@ -862,12 +862,23 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {vmSpecificErr := err != nil && isVMSpecificError(err.Error())if err == nil || vmSpecificErr {- recoveredEndpoints <- ep-// Store the client with original timeoutcfg.Timeout = cc.config.Timeoutcfg.Fingerprint = cc.getEndpointFingerprint(ep)- fullClient, _ := NewClient(cfg)+ fullClient, clientErr := NewClient(cfg)+ if clientErr != nil {+ // A short probe is not a usable polling session. Password+ // authentication may fail when constructing the actual client;+ // retain the existing client and failure instead of publishing+ // a healthy endpoint with a nil client.+ recordFailure(ep, clientErr)+ log.Debug().+ Str("cluster", cc.name).+ Str("endpoint", securityutil.InfrastructureEndpointDiagnostic(ep)).+ Err(clientErr).+ Msg("Failed to create full client after successful recovery probe")+ return+ }cc.mu.Lock()cc.nodeHealth[ep] = true@@ -875,6 +886,7 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {cc.lastHealthCheck[ep] = time.Now()cc.clients[ep] = fullClientcc.mu.Unlock()+ recoveredEndpoints <- epif vmSpecificErr {log.Info().diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.goindex e8d9477cac..02719163bb 100644--- a/pkg/proxmox/cluster_client_api_test.go+++ b/pkg/proxmox/cluster_client_api_test.go@@ -12,6 +12,10 @@ import ("time")+func TestClusterRecoveryRequiresUsableSession(t *testing.T) {+ testClusterRecoveryRequiresUsableSession(t)+}+func TestClusterClient_GetCephStatus(t *testing.T) {server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.Header().Set("Content-Type", "application/json")diff --git a/pkg/proxmox/cluster_client_recovery_session_test.go b/pkg/proxmox/cluster_client_recovery_session_test.gonew file mode 100644index 0000000000..bed66b6f1a--- /dev/null+++ b/pkg/proxmox/cluster_client_recovery_session_test.go@@ -0,0 +1,119 @@+package proxmox++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++// Recovery constructs two password sessions: the short connectivity probe and+// the client retained for ordinary polling. A successful first login cannot+// establish that the second was accepted. All credentials and TLS peers here+// are guest-local fixtures, not an installed authentication/recovery attempt.+func testClusterRecoveryRequiresUsableSession(t *testing.T) {+ for _, refusal := range []string{"unauthorised", "forbidden", "malformed", "form-refused"} {+ for _, retained := range []bool{false, true} {+ t.Run(fmt.Sprintf("%s/retained-%t", refusal, retained), func(t *testing.T) {+ var failing atomic.Bool+ var logins, nodeReads atomic.Int32+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api2/json/access/ticket":+ login := logins.Add(1)+ if failing.Load() && login > 1 {+ switch refusal {+ case "unauthorised":+ w.WriteHeader(http.StatusUnauthorized)+ fmt.Fprint(w, `{"message":"fixture session refused"}`)+ case "forbidden":+ w.WriteHeader(http.StatusForbidden)+ fmt.Fprint(w, `{"message":"fixture session refused"}`)+ case "malformed":+ fmt.Fprint(w, `{"data":`)+ case "form-refused":+ if r.Header.Get("Content-Type") == "application/json" {+ w.WriteHeader(http.StatusUnsupportedMediaType)+ } else {+ w.WriteHeader(http.StatusForbidden)+ }+ fmt.Fprint(w, `{"message":"fixture session refused"}`)+ }+ return+ }+ fmt.Fprintf(w, `{"data":{"ticket":"fixture-ticket-%d","CSRFPreventionToken":"fixture-csrf-%d"}}`, login, login)+ case "/api2/json/nodes":+ nodeReads.Add(1)+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket-") {+ t.Error("ordinary node read lost the accepted fixture session")+ }+ fmt.Fprint(w, `{"data":[{"node":"node1","status":"online"}]}`)+ default:+ t.Errorf("unexpected request %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: time.Second}+ cc := NewClusterClient("session-recovery", cfg, []string{server.URL}, nil)+ var original *Client+ if retained {+ var err error+ original, err = NewClient(cfg)+ if err != nil {+ t.Fatal(err)+ }+ defer original.httpClient.CloseIdleConnections()+ cc.clients[server.URL] = original+ }+ logins.Store(0)+ failing.Store(true)+ cc.nodeHealth[server.URL] = false+ cc.lastError[server.URL] = "previous connectivity failure"+ cc.recoverUnhealthyNodes(context.Background())++ health := cc.GetHealthStatusWithErrors()[server.URL]+ if health.Healthy || health.LastError != "Authentication failed - check API token or credentials" || health.LastCheck.IsZero() {+ t.Fatalf("failed polling session published recovery or lost the refusal: %+v", health)+ }+ client, exists := cc.clients[server.URL]+ if exists != retained || client != original {+ t.Fatalf("failed replacement changed retained client: exists=%t retained=%t nil=%t", exists, retained, client == nil)+ }+ wantLogins := int32(2)+ if refusal == "form-refused" {+ wantLogins = 3+ }+ if logins.Load() != wantLogins || nodeReads.Load() != 1 || cc.getEndpointFingerprint(server.URL) != cfg.Fingerprint {+ t.Fatalf("refusal replayed, changed trust or lost its probe: logins=%d nodes=%d", logins.Load(), nodeReads.Load())+ }++ // Advance only this fixture's recovery eligibility. No credential,+ // trust, persistent state or uncertainty fence is reset.+ failing.Store(false)+ delete(cc.lastHealthCheck, server.URL)+ cc.recoverUnhealthyNodes(context.Background())+ actual := cc.clients[server.URL]+ if actual == nil || actual == original || actual.config.Timeout != cfg.Timeout {+ t.Fatal("accepted later recovery did not publish a usable full-timeout client")+ }+ defer actual.httpClient.CloseIdleConnections()+ if health := cc.GetHealthStatusWithErrors()[server.URL]; !health.Healthy || health.LastError != "" {+ t.Fatalf("accepted later session did not clear its own failure: %+v", health)+ }+ nodes, err := cc.GetNodes(context.Background())+ if err != nil || len(nodes) != 1 || nodes[0].Node != "node1" || nodeReads.Load() != 3 {+ t.Fatalf("ordinary polling did not use recovered session: %+v %v reads=%d", nodes, err, nodeReads.Load())+ }+ })+ }+ }+}