Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep diagnostic TLS recovery within saved trust
pulse-triage[bot]c0e11cf44a5a
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.mdindex f46404af3b..40a872088d 100644--- a/docs/release-control/v6/internal/subsystems/api-contracts.md+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md@@ -8098,6 +8098,21 @@ machine-readable `errorKind` values with operator guidance incertificate mismatch, refused connections, unreachable networks, DNS failures,timeouts, and unknown connection failures must remain distinguishable insupport bundles instead of collapsing into a generic connection failure string.+TLS recovery guidance must preserve certificate verification: check the+configured endpoint's hostname, validity and trusted chain, and independently+verify certificate identity before adding or updating a saved fingerprint.+Neither a trusted network nor a failed probe justifies advice to disable+verification or switch to HTTP. Wrapped cancellation and typed network/deadline+timeouts take precedence over certificate words in endpoint/error text;+serialized scheduler timeout messages retain the same distinction. The existing+`errorKind` vocabulary and trust settings remain unchanged. Production loopback+PVE/PBS probes must reject untrusted chains and changed pins before sending HTTP+credentials, preserve saved trust, and resume only when the fixture operator+supplies the independently known pin. A successful PBS probe still does not+establish scheduler recovery without a completed poll. Verification:+`TestComputeDiagnosticsTLSRefusalAndVerifiedRecovery`,+`TestComputeDiagnosticsTLSHandshakeDeadline`, and the+`TestContract_DiagnosticsTLS*` controls, including JSON/LastError round trips.PBS connection state follows the same scheduler-ledger authority as theSettings connections response and connection-alert snapshots.`InstancePollStatus.LastError` is a current outstanding failure that is cleareddiff --git a/internal/api/contract_test.go b/internal/api/contract_test.goindex 63c6700805..5b91d26210 100644--- a/internal/api/contract_test.go+++ b/internal/api/contract_test.go@@ -10,6 +10,7 @@ import ("errors""fmt""math"+ "net""net/http""net/http/httptest""net/url"@@ -27317,3 +27318,51 @@ func TestContract_DiagnosticsTLSDeadlineIsNotCertificateEvidence(t *testing.T) {})}}++func TestContract_DiagnosticsTLSTypedTimeout(t *testing.T) {+ for _, err := range []error{+ &url.Error{Op: "Get", URL: "https://certificate.example:8006", Err: &net.DNSError{Err: "fixture stalled", Name: "certificate.example", IsTimeout: true}},+ &net.OpError{Op: "dial", Net: "tcp", Err: fmt.Errorf("certificate endpoint: %w", os.ErrDeadlineExceeded)},+ } {+ t.Run(err.Error(), func(t *testing.T) {+ failure := classifyDiagnosticsConnectionFailure(err, "probe failed")+ if failure.kind != "connection_timeout" || strings.Contains(failure.troubleshooting, "fingerprint") {+ t.Fatalf("typed timeout lost to endpoint text: %+v", failure)+ }+ })+ }+}++func TestContract_DiagnosticsTLSGuidanceSurvivesSerialization(t *testing.T) {+ for _, message := range []string{+ "certificate fingerprint mismatch: expected fixture-old, got fixture-new",+ `Get "https://certificate.example:8007/api2/json/version": net/http: TLS handshake timeout`,+ "connection timed out",+ } {+ t.Run(message, func(t *testing.T) {+ failure := classifyDiagnosticsConnectionFailure(errors.New(message), "probe failed")+ diag := EmptyDiagnosticsInfo()+ diag.Nodes = []NodeDiagnostic{{ID: "pve-fixture"}}+ diag.PBS = []PBSDiagnostic{{ID: "pbs-fixture"}}+ applyNodeDiagnosticFailure(&diag.Nodes[0], failure)+ applyPBSDiagnosticFailure(&diag.PBS[0], failure)+ applyPBSConnectionDiagnostic(&diag.PBS[0], Connection{State: ConnectionStateUnreachable,+ StateReason: "poll failed", LastError: &ConnectionError{Message: message}})+ rec := httptest.NewRecorder()+ writeDiagnosticsResponse(rec, diag, time.Time{})+ var payload DiagnosticsInfo+ if err := json.Unmarshal(rec.Body.Bytes(), &payload); err != nil {+ t.Fatal(err)+ }+ if payload.Nodes[0].ErrorKind != failure.kind || payload.PBS[0].ErrorKind != failure.kind || payload.PBS[0].Probe.ErrorKind != failure.kind {+ t.Fatal("serialized PVE, PBS scheduler and PBS probe diagnoses disagree")+ }+ if payload.Nodes[0].Troubleshooting != failure.troubleshooting || payload.PBS[0].Troubleshooting != failure.troubleshooting || payload.PBS[0].Probe.Troubleshooting != failure.troubleshooting {+ t.Fatal("safe guidance was lost between diagnostics and support JSON")+ }+ if payload.Nodes[0].Connected || payload.PBS[0].Connected || payload.PBS[0].Probe.Connected || payload.PBS[0].State != ConnectionStateUnreachable || payload.PBS[0].LastError.Message != message {+ t.Fatal("diagnostic guidance changed the observed connection failure")+ }+ })+ }+}diff --git a/internal/api/diagnostics.go b/internal/api/diagnostics.goindex 276c87a292..3876201258 100644--- a/internal/api/diagnostics.go+++ b/internal/api/diagnostics.go@@ -6,6 +6,7 @@ import ("errors""fmt""math"+ "net""net/http""os""os/user"@@ -929,6 +930,33 @@ func classifyDiagnosticsConnectionFailure(err error, fallbackMessage string) dia}lower := strings.ToLower(raw)+ // A stopped handshake is not evidence that a certificate is wrong. Check+ // typed causes before text, which also contains the requested URL and can+ // mention "certificate" or "TLS" in an otherwise ordinary endpoint name.+ if errors.Is(err, context.Canceled) {+ return diagnosticsConnectionFailure{+ kind: "connection_failed",+ message: "Connection check was cancelled",+ troubleshooting: "The diagnostic request was cancelled; no certificate-trust failure was established. Keep the saved TLS settings.",+ }+ }+ var networkError net.Error+ if errors.Is(err, context.DeadlineExceeded) ||+ (errors.As(err, &networkError) && networkError.Timeout()) ||+ strings.Contains(lower, "context deadline exceeded") ||+ strings.Contains(lower, "client.timeout exceeded") ||+ strings.Contains(lower, "i/o timeout") ||+ strings.Contains(lower, "tls handshake timeout") ||+ strings.Contains(lower, "tls: handshake timeout") ||+ strings.Contains(lower, "timeout awaiting response headers") ||+ strings.Contains(lower, "connection timed out") {+ return diagnosticsConnectionFailure{+ kind: "connection_timeout",+ message: "Connection timed out",+ troubleshooting: "Check network latency, API responsiveness, and slow or unreachable backend storage from the Pulse server.",+ }+ }+if strings.Contains(lower, "invalid") && strings.Contains(lower, "host") {return diagnosticsConnectionFailure{kind: "invalid_host_url",@@ -968,7 +996,7 @@ func classifyDiagnosticsConnectionFailure(err error, fallbackMessage string) diareturn diagnosticsConnectionFailure{kind: "tls_certificate_mismatch",message: "TLS certificate mismatch or untrusted certificate",- troubleshooting: "Update the stored certificate fingerprint, fix the node certificate, or disable verification only on a trusted network.",+ troubleshooting: "Check the configured endpoint's certificate hostname, expiry and trusted chain. If a fingerprint is needed or has changed, independently verify the certificate's identity and add or update the saved fingerprint only after that verification. Do not disable verification or switch to HTTP to diagnose the failure.",}}if strings.Contains(lower, "authentication") ||@@ -985,18 +1013,6 @@ func classifyDiagnosticsConnectionFailure(err error, fallbackMessage string) diatroubleshooting: "Rotate the node credentials in Settings, then verify the user or token has the required Proxmox permissions.",}}- if strings.Contains(lower, "context deadline exceeded") ||- strings.Contains(lower, "client.timeout exceeded") ||- strings.Contains(lower, "i/o timeout") ||- strings.Contains(lower, "tls handshake timeout") ||- strings.Contains(lower, "timeout awaiting response headers") {- return diagnosticsConnectionFailure{- kind: "connection_timeout",- message: "Connection timed out",- troubleshooting: "Check network latency, API responsiveness, and slow or unreachable backend storage from the Pulse server.",- }- }-if fallbackMessage == "" {fallbackMessage = "Connection check failed"}diff --git a/internal/api/diagnostics_additional_test.go b/internal/api/diagnostics_additional_test.goindex ddafea8a7e..3f51c4e079 100644--- a/internal/api/diagnostics_additional_test.go+++ b/internal/api/diagnostics_additional_test.go@@ -3,8 +3,11 @@ package apiimport ("context""crypto/sha256"+ "crypto/tls""encoding/hex""encoding/json"+ "io"+ "log""net/http""net/http/httptest""strings"@@ -314,6 +317,124 @@ func TestComputeDiagnostics_PVE_HonoursStoredFingerprint(t *testing.T) {}}+// Real loopback TLS probes exercise the production PVE/PBS diagnostics, not+// just the guidance helper. No native guest, backup or trust setting is changed.+func TestComputeDiagnosticsTLSRefusalAndVerifiedRecovery(t *testing.T) {+ for _, initialPin := range []string{"", strings.Repeat("0", 64)} {+ name := "untrusted-chain"+ if initialPin != "" {+ name = "changed-pin"+ }+ t.Run(name, func(t *testing.T) {+ var pveReads, pbsReads atomic.Int32+ server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if req.URL.Path == "/api2/json/version" {+ pbsReads.Add(1)+ if req.Header.Get("Authorization") != "PBSAPIToken=fixture@pbs!diagnostics:fixture-token" {+ http.Error(w, "unexpected fixture auth", http.StatusUnauthorized)+ return+ }+ io.WriteString(w, `{"data":{"version":"3.4.2","release":"1"}}`)+ return+ }+ pveReads.Add(1)+ if req.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!diagnostics=fixture-token" {+ http.Error(w, "unexpected fixture auth", http.StatusUnauthorized)+ return+ }+ if strings.Contains(req.URL.Path, "/agent/") {+ t.Error("empty fixture inventory unexpectedly queued a guest command")+ }+ io.WriteString(w, `{"data":[]}`)+ }))+ server.Config.ErrorLog = log.New(io.Discard, "", 0)+ server.StartTLS()+ t.Cleanup(server.Close)+ fingerprint := sha256.Sum256(server.Certificate().Raw)+ verifiedPin := hex.EncodeToString(fingerprint[:])+ cfg := &config.Config{+ DataPath: t.TempDir(),+ PVEInstances: []config.PVEInstance{{Name: "pve-fixture", Host: server.URL,+ TokenName: "fixture@pve!diagnostics", TokenValue: "fixture-token", VerifySSL: true, Fingerprint: initialPin}},+ PBSInstances: []config.PBSInstance{{Name: "pbs-fixture", Host: server.URL,+ TokenName: "fixture@pbs!diagnostics", TokenValue: "fixture-token", VerifySSL: true, Fingerprint: initialPin}},+ }+ router := &Router{config: cfg}+ diag := router.computeDiagnostics(context.Background())+ if len(diag.Nodes) != 1 || len(diag.PBS) != 1 || diag.Nodes[0].Connected || diag.PBS[0].Probe.Connected {+ t.Fatalf("TLS-refused probes reported connectivity: %+v %+v", diag.Nodes, diag.PBS)+ }+ for _, failure := range []struct{ kind, guidance string }{+ {diag.Nodes[0].ErrorKind, diag.Nodes[0].Troubleshooting},+ {diag.PBS[0].Probe.ErrorKind, diag.PBS[0].Probe.Troubleshooting},+ } {+ if failure.kind != "tls_certificate_mismatch" || !strings.Contains(failure.guidance, "independently verify") || !strings.Contains(failure.guidance, "Do not disable verification") {+ t.Errorf("unsafe or lost TLS diagnosis: %+v", failure)+ }+ }+ if pveReads.Load() != 0 || pbsReads.Load() != 0 {+ t.Fatal("HTTP credentials reached an endpoint rejected by TLS")+ }+ if !cfg.PVEInstances[0].VerifySSL || !cfg.PBSInstances[0].VerifySSL || cfg.PVEInstances[0].Fingerprint != initialPin || cfg.PBSInstances[0].Fingerprint != initialPin {+ t.Fatal("failed diagnostics changed saved trust")+ }++ // Only this fixture operator supplies the known peer identity; the+ // diagnostic path must not capture or replace the pin on its own.+ cfg.PVEInstances[0].Fingerprint = verifiedPin+ cfg.PBSInstances[0].Fingerprint = verifiedPin+ recovered := router.computeDiagnostics(context.Background())+ if !recovered.Nodes[0].Connected || !recovered.PBS[0].Probe.Connected || pveReads.Load() == 0 || pbsReads.Load() == 0 {+ t.Fatalf("verified same-endpoint recovery failed: %+v %+v", recovered.Nodes, recovered.PBS)+ }+ if recovered.PBS[0].Connected {+ t.Fatal("a successful support probe established PBS scheduler recovery without a completed poll")+ }+ if !cfg.PVEInstances[0].VerifySSL || !cfg.PBSInstances[0].VerifySSL || cfg.PVEInstances[0].Fingerprint != verifiedPin || cfg.PBSInstances[0].Fingerprint != verifiedPin || cfg.PVEInstances[0].TokenValue != "fixture-token" || cfg.PBSInstances[0].TokenValue != "fixture-token" {+ t.Fatal("successful diagnostics changed trust or credentials")+ }+ })+ }+}++func TestComputeDiagnosticsTLSHandshakeDeadline(t *testing.T) {+ for _, provider := range []string{"pve", "pbs"} {+ t.Run(provider, func(t *testing.T) {+ var handshakes, httpReads atomic.Int32+ release := make(chan struct{})+ server := httptest.NewUnstartedServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { httpReads.Add(1) }))+ server.Config.ErrorLog = log.New(io.Discard, "", 0)+ server.TLS = &tls.Config{GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {+ handshakes.Add(1)+ <-release+ return nil, nil+ }}+ server.StartTLS()+ defer server.Close()+ defer close(release)+ cfg := &config.Config{DataPath: t.TempDir()}+ if provider == "pve" {+ cfg.PVEInstances = []config.PVEInstance{{Name: "pve-fixture", Host: server.URL, TokenName: "fixture@pve!diagnostics", TokenValue: "fixture-token", VerifySSL: true}}+ } else {+ cfg.PBSInstances = []config.PBSInstance{{Name: "pbs-fixture", Host: server.URL, TokenName: "fixture@pbs!diagnostics", TokenValue: "fixture-token", VerifySSL: true}}+ }+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ diag := (&Router{config: cfg}).computeDiagnostics(ctx)+ var kind, guidance string+ if provider == "pve" {+ kind, guidance = diag.Nodes[0].ErrorKind, diag.Nodes[0].Troubleshooting+ } else {+ kind, guidance = diag.PBS[0].Probe.ErrorKind, diag.PBS[0].Probe.Troubleshooting+ }+ if handshakes.Load() == 0 || httpReads.Load() != 0 || kind != "connection_timeout" || strings.Contains(guidance, "fingerprint") {+ t.Fatalf("handshake deadline became a trust fault: handshakes=%d HTTP=%d kind=%q guidance=%q", handshakes.Load(), httpReads.Load(), kind, guidance)+ }+ })+ }+}+func TestDiagnosticsInfo_UsesCanonicalEmptyCollections(t *testing.T) {payload, err := json.Marshal(EmptyDiagnosticsInfo())if err != nil {