Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Require complete Proxmox node inventory before recovery
pulse-triage[bot]ba9bb34d096c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 6dc321cb3b..905b41c152 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -47,6 +47,35 @@ remain unchanged. These are source/consumer controls, not attribution orinstalled acceptance for a reported PBS/QGA/backup incident.+### Node inventory admission consumes the complete response++The PVE `/nodes` reader consumes and closes the entire existing 8 MiB-bounded+body before returning inventory or allowing startup/recovery/lazy probes to+succeed. A valid JSON prefix with a missing transport tail, extra JSON values,+oversized suffix, malformed UTF-8, missing/null data, conflicting errors or+ambiguous/case-overwritten fields is not an authoritative inventory. Each row+must have a canonical nonempty, unique node name; null/unnamed/duplicate rows+and ambiguous supported node fields reject the whole response rather than+publish partial or invented empty inventory. Validation errors contain no body+material. Unique provider metadata and explicit empty arrays remain accepted,+including scoped tokens with no visible nodes. Existing node fields and offline+status are preserved without changing provider permissions.++`node_inventory_response_test.go` exercises actual HTTP reads and all supported+fields, the explicit-empty controls, byte/transport bounds and no replay.+`cluster_node_inventory_test.go` checks pinned HTTPS in token, JSON-password and+form-password modes at startup, recovery and lazy selection: failed success+installs no client and never advances to a full session; distinct complete+recovery remains usable. The cached fallback keeps its earlier failure until a+complete ordinary read succeeds. `TestClusterNodeInventoryEndpointProjection`+drives missing/duplicate/incomplete responses and subsequent ordinary recovery+through the production monitor projection while native Online, identity and+saved pins remain intact. Recovery cadence, auth encodings, TLS admission,+rate-limit policy, other API readers and guest/backup fences are unchanged.+These are source and consuming-surface proofs, not attribution or installed+acceptance for any reported PBS, guest restart or workload-safety incident.++### First-use cluster trust includes password constructionDeclared, previously untrusted Proxmox members use the existing credential-freediff --git a/internal/monitoring/monitor_pve_node_probe_test.go b/internal/monitoring/monitor_pve_node_probe_test.goindex e6ab0848a4..bf457f2774 100644--- a/internal/monitoring/monitor_pve_node_probe_test.go+++ b/internal/monitoring/monitor_pve_node_probe_test.go@@ -22,6 +22,23 @@ import (// reports both members online. Public reads initiate later ordinary recovery;// no guest command, backup, credential change or guest fence is exercised.func TestClusterNodeProbeEndpointProjection(t *testing.T) {+ testClusterNodeProbeEndpointProjection(t, http.StatusInternalServerError, "QEMU guest agent is not running", true)+}++func TestClusterNodeInventoryEndpointProjection(t *testing.T) {+ for _, tc := range []struct {+ name, body string+ complete bool+ }{+ {"missing", `{}`, true},+ {"duplicate", `{"data":{},"data":[]}`, true},+ {"truncated", `{"data":[{"node":"member","status":"online"}]}`, false},+ } {+ t.Run(tc.name, func(t *testing.T) { testClusterNodeProbeEndpointProjection(t, http.StatusOK, tc.body, tc.complete) })+ }+}++func testClusterNodeProbeEndpointProjection(t *testing.T, status int, failureBody string, complete bool) {var acceptMember atomic.Boolvar memberLogins, memberNodes atomic.Int32endpoint := func(member bool) *httptest.Server {@@ -40,8 +57,11 @@ func TestClusterNodeProbeEndpointProjection(t *testing.T) {if member {memberNodes.Add(1)if !acceptMember.Load() {- w.WriteHeader(http.StatusInternalServerError)- fmt.Fprint(w, "QEMU guest agent is not running")+ if !complete {+ w.Header().Set("Content-Length", fmt.Sprint(len(failureBody)+100))+ }+ w.WriteHeader(status)+ fmt.Fprint(w, failureBody)return}}@@ -86,15 +106,23 @@ func TestClusterNodeProbeEndpointProjection(t *testing.T) {}wire, err := json.Marshal(struct {Phase string `json:"phase"`+ Case string `json:"case"`Endpoints []config.ClusterEndpoint `json:"endpoints"`- }{phase, inst.ClusterEndpoints})+ }{phase, t.Name(), inst.ClusterEndpoints})if err != nil {t.Fatal(err)}t.Logf("NODE_PROBE_ENDPOINT_PROJECTION_JSON=%s", wire)}project("nodes-probe-refused")- if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || *ep.PulseReachable || ep.PulseError != "API error 500: QEMU guest agent is not running" {+ expectedError := "API error 500: QEMU guest agent is not running"+ if status == http.StatusOK {+ expectedError = "invalid Proxmox node inventory response"+ if !complete {+ expectedError = "read Proxmox node inventory: unexpected EOF"+ }+ }+ if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || *ep.PulseReachable || ep.PulseError != expectedError {t.Fatalf("consumer lost failed nodes observation: %+v", ep)}acceptMember.Store(true)diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex c2c98abb9e..9138ac63da 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -4,6 +4,7 @@ import ("bytes""context""encoding/json"+ "errors""fmt""io""math"@@ -950,17 +951,14 @@ func (c *Client) GetNodes(ctx context.Context) ([]Node, error) {if err != nil {return nil, err}- defer resp.Body.Close()-- var result struct {- Data []Node `json:"data"`+ // /nodes is also the cluster's startup/recovery probe. A decodable JSON+ // prefix or absent inventory cannot establish usable polling health.+ body, readErr := readResponseBodyLimited(resp.Body)+ closeErr := resp.Body.Close()+ if readErr != nil || closeErr != nil {+ return nil, fmt.Errorf("read Proxmox node inventory: %w", errors.Join(readErr, closeErr))}-- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {- return nil, err- }-- return result.Data, nil+ return decodeNodeInventory(body)}// GetNodeStatus returns detailed status for a specific nodediff --git a/pkg/proxmox/cluster_node_inventory_test.go b/pkg/proxmox/cluster_node_inventory_test.gonew file mode 100644index 0000000000..3329d73f05--- /dev/null+++ b/pkg/proxmox/cluster_node_inventory_test.go@@ -0,0 +1,164 @@+package proxmox++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"+)++func nodeInventoryProbeFixture(t *testing.T, mode, body string, complete bool) *nodeProbeFixture {+ t.Helper()+ f := &nodeProbeFixture{mode: mode}+ f.reject.Store(true)+ f.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api2/json/access/ticket":+ f.logins.Add(1)+ if mode == "token" {+ t.Error("token inventory probe logged in")+ }+ if mode == "form" && r.Header.Get("Content-Type") == "application/json" {+ w.WriteHeader(415)+ return+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ case "/api2/json/nodes":+ f.nodes.Add(1)+ if mode == "token" {+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pam!pulse=fixture-token" {+ t.Error("token authentication changed")+ }+ } else if r.Header.Get("Cookie") != "PVEAuthCookie=fixture-ticket" {+ t.Error("password authentication changed")+ }+ if f.reject.Load() {+ if !complete {+ w.Header().Set("Content-Length", fmt.Sprint(len(body)+100))+ }+ fmt.Fprint(w, body)+ } else {+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online","cpu":0.25}]}`)+ }+ default:+ t.Errorf("inventory admission left ordinary session/nodes paths: %s", r.URL.Path)+ http.NotFound(w, r)+ }+ }))+ t.Cleanup(f.server.Close)+ return f+}++func TestClusterNodeInventoryRequiresCompleteResponse(t *testing.T) {+ cases := []struct {+ name, body string+ complete bool+ }{+ {"missing", `{}`, true},+ {"truncated-prefix", `{"data":[{"node":"member"}]}`, false},+ {"duplicate", `{"data":{},"data":[]}`, true},+ {"error", `{"data":null,"errors":{"message":"fixture-private-detail"}}`, true},+ }+ for _, phase := range []string{"startup", "recovery", "selection"} {+ for _, mode := range []string{"token", "json", "form"} {+ for _, tc := range cases {+ t.Run(phase+"/"+mode+"/"+tc.name, func(t *testing.T) {+ f := nodeInventoryProbeFixture(t, mode, tc.body, tc.complete)+ cfg := f.config()+ saved := map[string]string{f.server.URL: cfg.Fingerprint}+ var cc *ClusterClient+ if phase == "startup" {+ sibling := nodeInventoryProbeFixture(t, mode, ``, true)+ sibling.reject.Store(false)+ saved[sibling.server.URL] = sibling.config().Fingerprint+ cc = NewClusterClient("inventory", cfg, []string{f.server.URL, sibling.server.URL}, saved)+ if !cc.GetHealthStatus()[sibling.server.URL] {+ t.Fatal("invalid member blocked independently accepted sibling")+ }+ } else {+ cc = NewClusterClient("inventory", cfg, []string{f.server.URL}, saved)+ if phase == "recovery" {+ cc.nodeHealth[f.server.URL] = false+ cc.lastError[f.server.URL] = "previous failure"+ cc.recoverUnhealthyNodes(context.Background())+ } else if c, err := cc.getHealthyClient(context.Background()); c != nil || err == nil {+ t.Error("invalid first selection installed polling client")+ }+ }+ t.Cleanup(func() {+ cc.mu.RLock()+ defer cc.mu.RUnlock()+ for _, c := range cc.clients {+ if c != nil {+ c.httpClient.CloseIdleConnections()+ }+ }+ })+ health := cc.GetHealthStatusWithErrors()[f.server.URL]+ if health.Healthy || health.LastError == "" || health.LastCheck.IsZero() || strings.Contains(health.LastError, "fixture-private-detail") || cc.clients[f.server.URL] != nil {+ t.Errorf("invalid probe published usable health/client or leaked body: %+v", health)+ }+ if f.nodes.Load() != 1 || f.logins.Load() != f.loginAttempts() {+ t.Fatalf("invalid probe replayed or advanced: nodes=%d logins=%d", f.nodes.Load(), f.logins.Load())+ }+ if cc.getEndpointFingerprint(f.server.URL) != cfg.Fingerprint || saved[f.server.URL] != cfg.Fingerprint || !cc.config.VerifySSL {+ t.Fatal("invalid inventory changed saved trust")+ }+ // Advance only this synthetic endpoint's eligibility for a distinct accepted+ // response. Connected consuming proof below honours the real interval.+ f.reject.Store(false)+ cc.mu.Lock()+ delete(cc.lastHealthCheck, f.server.URL)+ cc.mu.Unlock()+ cc.recoverUnhealthyNodes(context.Background())+ health = cc.GetHealthStatusWithErrors()[f.server.URL]+ if c := cc.clients[f.server.URL]; c == nil || c.config.Timeout != cfg.Timeout || !health.Healthy || health.LastError != "" {+ t.Fatalf("later complete recovery failed: %+v", health)+ }+ if f.nodes.Load() != 2 || f.logins.Load() != 3*f.loginAttempts() {+ t.Fatalf("later recovery lost probe/full-session boundary: nodes=%d logins=%d", f.nodes.Load(), f.logins.Load())+ }+ })+ }+ }+ }+}++func TestClusterNodeInventoryCachedFallbackCannotInventRecovery(t *testing.T) {+ f := nodeInventoryProbeFixture(t, "json", `{}`, true)+ f.reject.Store(false)+ cc := NewClusterClient("inventory", f.config(), []string{f.server.URL}, nil)+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 {+ t.Fatal(nodes, err)+ }+ original := cc.clients[f.server.URL]+ defer original.httpClient.CloseIdleConnections()+ f.reject.Store(true)+ cc.markUnhealthyWithError(f.server.URL, "previous failure")+ if c, err := cc.getHealthyClient(context.Background()); err != nil || c != original {+ t.Fatal("cached fallback unavailable", err)+ }+ before := f.nodes.Load()+ if nodes, err := cc.GetNodes(context.Background()); err == nil || nodes != nil {+ t.Fatal("invalid cached read invented an authoritative empty inventory", nodes, err)+ }+ h := cc.GetHealthStatusWithErrors()[f.server.URL]+ if h.Healthy || h.LastError != "previous failure" || f.nodes.Load() != before+1 {+ t.Fatal("cached invalid read invented recovery or replayed", h)+ }+ f.reject.Store(false)+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 || nodes[0].CPU != 0.25 {+ t.Fatal("distinct complete read did not recover", nodes, err)+ }+ h = cc.GetHealthStatusWithErrors()[f.server.URL]+ if !h.Healthy || h.LastError != "" || cc.clients[f.server.URL] != original || f.nodes.Load() != before+2 {+ t.Fatal("complete read lost normal same-client recovery", h)+ }+ if cc.clients[f.server.URL].config.Timeout != 2*time.Second {+ t.Fatal("full timeout changed")+ }+}diff --git a/pkg/proxmox/node_inventory_response.go b/pkg/proxmox/node_inventory_response.gonew file mode 100644index 0000000000..a8b05b3b43--- /dev/null+++ b/pkg/proxmox/node_inventory_response.go@@ -0,0 +1,73 @@+package proxmox++import (+ "bytes"+ "encoding/json"+ "errors"+ "strings"+ "unicode/utf8"+)++// Do not copy a malformed successful response into health diagnostics. It may+// contain private proxy details; it is not an authentication refusal either.+var errInvalidNodeInventoryResponse = errors.New("invalid Proxmox node inventory response")++func decodeNodeInventory(body []byte) ([]Node, error) {+ if !utf8.Valid(body) {+ return nil, errInvalidNodeInventoryResponse+ }+ envelope, valid := nodeInventoryObject(body, "data", "errors")+ if !valid {+ return nil, errInvalidNodeInventoryResponse+ }+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {+ return nil, errInvalidNodeInventoryResponse+ }+ data := bytes.TrimSpace(envelope["data"])+ // An explicit empty array remains valid for scoped tokens. Missing/null+ // data is not an empty authoritative inventory and must not erase nodes.+ if len(data) == 0 || data[0] != '[' {+ return nil, errInvalidNodeInventoryResponse+ }+ var rows []json.RawMessage+ if json.Unmarshal(data, &rows) != nil {+ return nil, errInvalidNodeInventoryResponse+ }+ nodes := make([]Node, 0, len(rows))+ seen := make(map[string]bool, len(rows))+ for _, row := range rows {+ fields, valid := nodeInventoryObject(row, "node", "status", "cpu", "maxcpu", "mem", "maxmem", "disk", "maxdisk", "uptime", "level")+ if !valid {+ return nil, errInvalidNodeInventoryResponse+ }+ var name string+ if json.Unmarshal(fields["node"], &name) != nil || strings.TrimSpace(name) == "" || seen[name] {+ return nil, errInvalidNodeInventoryResponse+ }+ var node Node+ if json.Unmarshal(row, &node) != nil {+ return nil, errInvalidNodeInventoryResponse+ }+ seen[name] = true+ nodes = append(nodes, node)+ }+ return nodes, nil+}++// The existing object reader rejects duplicates and extra JSON values. Keep+// unique provider metadata, but never let encoding/json's case-insensitive+// struct matching replace canonical inventory fields with another spelling.+func nodeInventoryObject(body []byte, canonical ...string) (map[string]json.RawMessage, bool) {+ fields, valid := guestAgentResponseFields(body)+ if !valid {+ return nil, false+ }+ for name := range fields {+ for _, expected := range canonical {+ if name != expected && strings.EqualFold(name, expected) {+ return nil, false+ }+ }+ }+ return fields, true+}diff --git a/pkg/proxmox/node_inventory_response_test.go b/pkg/proxmox/node_inventory_response_test.gonew file mode 100644index 0000000000..306661ab26--- /dev/null+++ b/pkg/proxmox/node_inventory_response_test.go@@ -0,0 +1,94 @@+package proxmox++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic"+ "testing"+)++// Exercise the real bounded HTTP reader, not just a detached JSON parser.+func TestNodeInventoryResponseAdmission(t *testing.T) {+ const inventory = `{"data":[{"node":"member","status":"online","cpu":0.25}]}`+ cases := []struct {+ name, body string+ complete bool+ }{+ {"missing-data", `{}`, true},+ {"error-as-success", `{"errors":{"message":"fixture-private-detail"}}`, true},+ {"conflicting-error", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, true},+ {"null-inventory", `{"data":null}`, true},+ {"null-envelope", `null`, true},+ {"case-data", `{"Data":[]}`, true},+ {"duplicate-data", `{"data":{},"data":[]}`, true},+ {"escaped-duplicate-data", `{"data":{},"\u0064ata":[]}`, true},+ {"case-overwrite", `{"data":[],"Data":null}`, true},+ {"trailing-value", inventory + ` {"errors":{"message":"fixture-private-detail"}}`, true},+ {"trailing-garbage", inventory + `not-json`, true},+ {"truncated-after-prefix", inventory, false},+ {"oversized-after-prefix", inventory + strings.Repeat(" ", int(maxResponseBodyBytes)), true},+ {"invalid-utf8", `{"data":[],"metadata":"` + string([]byte{0xff}) + `"}`, true},+ {"null-row", `{"data":[null]}`, true},+ {"unnamed-row", `{"data":[{}]}`, true},+ {"empty-node", `{"data":[{"node":""}]}`, true},+ {"null-node", `{"data":[{"node":null}]}`, true},+ {"duplicate-node-field", `{"data":[{"node":"other","node":"member"}]}`, true},+ {"case-node-field", `{"data":[{"node":"member","Node":"other"}]}`, true},+ {"duplicate-node-row", `{"data":[{"node":"member"},{"node":"member"}]}`, true},+ {"wrong-node-type", `{"data":[{"node":42}]}`, true},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ if r.URL.Path != "/api2/json/nodes" {+ t.Errorf("unexpected path: %s", r.URL.Path)+ }+ if !tc.complete {+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))+ }+ fmt.Fprint(w, tc.body)+ }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ nodes, err := c.GetNodes(context.Background())+ if err == nil || nodes != nil {+ t.Errorf("unverified inventory admitted: nodes=%+v err=%v", nodes, err)+ }+ if err != nil && strings.Contains(err.Error(), "fixture-private-detail") {+ t.Error("response body leaked into admission error")+ }+ if requests.Load() != 1 {+ t.Errorf("invalid success replayed: %d requests", requests.Load())+ }+ })+ }+}++func TestNodeInventoryAcceptedScopedAndExtendedResponses(t *testing.T) {+ for _, body := range []string{+ `{"data":[]}`,+ `{"data":[],"errors":null,"total":0}`,+ `{"data":[{"node":"member","status":"offline","cpu":0.25,"maxcpu":8,"mem":128,"maxmem":512,"disk":256,"maxdisk":1024,"uptime":42,"level":"c","id":"node/member","type":"node"}]}`,+ } {+ t.Run(body, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) }))+ defer server.Close()+ c := backupTestClient(t, server.URL)+ nodes, err := c.GetNodes(context.Background())+ if err != nil {+ t.Fatalf("accepted scoped/extended inventory refused: %v", err)+ }+ if strings.Contains(body, `"node"`) && (len(nodes) != 1 || nodes[0].Node != "member" || nodes[0].Status != "offline" || nodes[0].CPU != 0.25 || nodes[0].MaxCPU != 8 || nodes[0].Mem != 128 || nodes[0].MaxMem != 512 || nodes[0].Disk != 256 || nodes[0].MaxDisk != 1024 || nodes[0].Uptime != 42 || nodes[0].Level != "c") {+ t.Fatalf("accepted node fields changed: %+v", nodes)+ }+ if !strings.Contains(body, `"node"`) && len(nodes) != 0 {+ t.Fatalf("scoped empty inventory not empty: %+v", nodes)+ }+ })+ }+}