Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Require complete Proxmox node inventory before recovery

pulse-triage[bot]ba9bb34d096c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 6dc321cb3b..905b41c152 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -47,6 +47,35 @@ remain unchanged. These are source/consumer controls, not attribution or
installed acceptance for a reported PBS/QGA/backup incident.
+### Node inventory admission consumes the complete response
+
+The PVE `/nodes` reader consumes and closes the entire existing 8 MiB-bounded
+body before returning inventory or allowing startup/recovery/lazy probes to
+succeed. A valid JSON prefix with a missing transport tail, extra JSON values,
+oversized suffix, malformed UTF-8, missing/null data, conflicting errors or
+ambiguous/case-overwritten fields is not an authoritative inventory. Each row
+must have a canonical nonempty, unique node name; null/unnamed/duplicate rows
+and ambiguous supported node fields reject the whole response rather than
+publish partial or invented empty inventory. Validation errors contain no body
+material. Unique provider metadata and explicit empty arrays remain accepted,
+including scoped tokens with no visible nodes. Existing node fields and offline
+status are preserved without changing provider permissions.
+
+`node_inventory_response_test.go` exercises actual HTTP reads and all supported
+fields, the explicit-empty controls, byte/transport bounds and no replay.
+`cluster_node_inventory_test.go` checks pinned HTTPS in token, JSON-password and
+form-password modes at startup, recovery and lazy selection: failed success
+installs no client and never advances to a full session; distinct complete
+recovery remains usable. The cached fallback keeps its earlier failure until a
+complete ordinary read succeeds. `TestClusterNodeInventoryEndpointProjection`
+drives missing/duplicate/incomplete responses and subsequent ordinary recovery
+through the production monitor projection while native Online, identity and
+saved pins remain intact. Recovery cadence, auth encodings, TLS admission,
+rate-limit policy, other API readers and guest/backup fences are unchanged.
+These are source and consuming-surface proofs, not attribution or installed
+acceptance for any reported PBS, guest restart or workload-safety incident.
+
+
### First-use cluster trust includes password construction
Declared, previously untrusted Proxmox members use the existing credential-free
diff --git a/internal/monitoring/monitor_pve_node_probe_test.go b/internal/monitoring/monitor_pve_node_probe_test.go
index e6ab0848a4..bf457f2774 100644
--- a/internal/monitoring/monitor_pve_node_probe_test.go
+++ b/internal/monitoring/monitor_pve_node_probe_test.go
@@ -22,6 +22,23 @@ import (
// reports both members online. Public reads initiate later ordinary recovery;
// no guest command, backup, credential change or guest fence is exercised.
func TestClusterNodeProbeEndpointProjection(t *testing.T) {
+ testClusterNodeProbeEndpointProjection(t, http.StatusInternalServerError, "QEMU guest agent is not running", true)
+}
+
+func TestClusterNodeInventoryEndpointProjection(t *testing.T) {
+ for _, tc := range []struct {
+ name, body string
+ complete bool
+ }{
+ {"missing", `{}`, true},
+ {"duplicate", `{"data":{},"data":[]}`, true},
+ {"truncated", `{"data":[{"node":"member","status":"online"}]}`, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) { testClusterNodeProbeEndpointProjection(t, http.StatusOK, tc.body, tc.complete) })
+ }
+}
+
+func testClusterNodeProbeEndpointProjection(t *testing.T, status int, failureBody string, complete bool) {
var acceptMember atomic.Bool
var memberLogins, memberNodes atomic.Int32
endpoint := func(member bool) *httptest.Server {
@@ -40,8 +57,11 @@ func TestClusterNodeProbeEndpointProjection(t *testing.T) {
if member {
memberNodes.Add(1)
if !acceptMember.Load() {
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprint(w, "QEMU guest agent is not running")
+ if !complete {
+ w.Header().Set("Content-Length", fmt.Sprint(len(failureBody)+100))
+ }
+ w.WriteHeader(status)
+ fmt.Fprint(w, failureBody)
return
}
}
@@ -86,15 +106,23 @@ func TestClusterNodeProbeEndpointProjection(t *testing.T) {
}
wire, err := json.Marshal(struct {
Phase string `json:"phase"`
+ Case string `json:"case"`
Endpoints []config.ClusterEndpoint `json:"endpoints"`
- }{phase, inst.ClusterEndpoints})
+ }{phase, t.Name(), inst.ClusterEndpoints})
if err != nil {
t.Fatal(err)
}
t.Logf("NODE_PROBE_ENDPOINT_PROJECTION_JSON=%s", wire)
}
project("nodes-probe-refused")
- if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || *ep.PulseReachable || ep.PulseError != "API error 500: QEMU guest agent is not running" {
+ expectedError := "API error 500: QEMU guest agent is not running"
+ if status == http.StatusOK {
+ expectedError = "invalid Proxmox node inventory response"
+ if !complete {
+ expectedError = "read Proxmox node inventory: unexpected EOF"
+ }
+ }
+ if ep := inst.ClusterEndpoints[1]; ep.PulseReachable == nil || *ep.PulseReachable || ep.PulseError != expectedError {
t.Fatalf("consumer lost failed nodes observation: %+v", ep)
}
acceptMember.Store(true)
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go
index c2c98abb9e..9138ac63da 100644
--- a/pkg/proxmox/client.go
+++ b/pkg/proxmox/client.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"math"
@@ -950,17 +951,14 @@ func (c *Client) GetNodes(ctx context.Context) ([]Node, error) {
if err != nil {
return nil, err
}
- defer resp.Body.Close()
-
- var result struct {
- Data []Node `json:"data"`
+ // /nodes is also the cluster's startup/recovery probe. A decodable JSON
+ // prefix or absent inventory cannot establish usable polling health.
+ body, readErr := readResponseBodyLimited(resp.Body)
+ closeErr := resp.Body.Close()
+ if readErr != nil || closeErr != nil {
+ return nil, fmt.Errorf("read Proxmox node inventory: %w", errors.Join(readErr, closeErr))
}
-
- if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
- return nil, err
- }
-
- return result.Data, nil
+ return decodeNodeInventory(body)
}
// GetNodeStatus returns detailed status for a specific node
diff --git a/pkg/proxmox/cluster_node_inventory_test.go b/pkg/proxmox/cluster_node_inventory_test.go
new file mode 100644
index 0000000000..3329d73f05
--- /dev/null
+++ b/pkg/proxmox/cluster_node_inventory_test.go
@@ -0,0 +1,164 @@
+package proxmox
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+func nodeInventoryProbeFixture(t *testing.T, mode, body string, complete bool) *nodeProbeFixture {
+ t.Helper()
+ f := &nodeProbeFixture{mode: mode}
+ f.reject.Store(true)
+ f.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api2/json/access/ticket":
+ f.logins.Add(1)
+ if mode == "token" {
+ t.Error("token inventory probe logged in")
+ }
+ if mode == "form" && r.Header.Get("Content-Type") == "application/json" {
+ w.WriteHeader(415)
+ return
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ case "/api2/json/nodes":
+ f.nodes.Add(1)
+ if mode == "token" {
+ if r.Header.Get("Authorization") != "PVEAPIToken=fixture@pam!pulse=fixture-token" {
+ t.Error("token authentication changed")
+ }
+ } else if r.Header.Get("Cookie") != "PVEAuthCookie=fixture-ticket" {
+ t.Error("password authentication changed")
+ }
+ if f.reject.Load() {
+ if !complete {
+ w.Header().Set("Content-Length", fmt.Sprint(len(body)+100))
+ }
+ fmt.Fprint(w, body)
+ } else {
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online","cpu":0.25}]}`)
+ }
+ default:
+ t.Errorf("inventory admission left ordinary session/nodes paths: %s", r.URL.Path)
+ http.NotFound(w, r)
+ }
+ }))
+ t.Cleanup(f.server.Close)
+ return f
+}
+
+func TestClusterNodeInventoryRequiresCompleteResponse(t *testing.T) {
+ cases := []struct {
+ name, body string
+ complete bool
+ }{
+ {"missing", `{}`, true},
+ {"truncated-prefix", `{"data":[{"node":"member"}]}`, false},
+ {"duplicate", `{"data":{},"data":[]}`, true},
+ {"error", `{"data":null,"errors":{"message":"fixture-private-detail"}}`, true},
+ }
+ for _, phase := range []string{"startup", "recovery", "selection"} {
+ for _, mode := range []string{"token", "json", "form"} {
+ for _, tc := range cases {
+ t.Run(phase+"/"+mode+"/"+tc.name, func(t *testing.T) {
+ f := nodeInventoryProbeFixture(t, mode, tc.body, tc.complete)
+ cfg := f.config()
+ saved := map[string]string{f.server.URL: cfg.Fingerprint}
+ var cc *ClusterClient
+ if phase == "startup" {
+ sibling := nodeInventoryProbeFixture(t, mode, ``, true)
+ sibling.reject.Store(false)
+ saved[sibling.server.URL] = sibling.config().Fingerprint
+ cc = NewClusterClient("inventory", cfg, []string{f.server.URL, sibling.server.URL}, saved)
+ if !cc.GetHealthStatus()[sibling.server.URL] {
+ t.Fatal("invalid member blocked independently accepted sibling")
+ }
+ } else {
+ cc = NewClusterClient("inventory", cfg, []string{f.server.URL}, saved)
+ if phase == "recovery" {
+ cc.nodeHealth[f.server.URL] = false
+ cc.lastError[f.server.URL] = "previous failure"
+ cc.recoverUnhealthyNodes(context.Background())
+ } else if c, err := cc.getHealthyClient(context.Background()); c != nil || err == nil {
+ t.Error("invalid first selection installed polling client")
+ }
+ }
+ t.Cleanup(func() {
+ cc.mu.RLock()
+ defer cc.mu.RUnlock()
+ for _, c := range cc.clients {
+ if c != nil {
+ c.httpClient.CloseIdleConnections()
+ }
+ }
+ })
+ health := cc.GetHealthStatusWithErrors()[f.server.URL]
+ if health.Healthy || health.LastError == "" || health.LastCheck.IsZero() || strings.Contains(health.LastError, "fixture-private-detail") || cc.clients[f.server.URL] != nil {
+ t.Errorf("invalid probe published usable health/client or leaked body: %+v", health)
+ }
+ if f.nodes.Load() != 1 || f.logins.Load() != f.loginAttempts() {
+ t.Fatalf("invalid probe replayed or advanced: nodes=%d logins=%d", f.nodes.Load(), f.logins.Load())
+ }
+ if cc.getEndpointFingerprint(f.server.URL) != cfg.Fingerprint || saved[f.server.URL] != cfg.Fingerprint || !cc.config.VerifySSL {
+ t.Fatal("invalid inventory changed saved trust")
+ }
+ // Advance only this synthetic endpoint's eligibility for a distinct accepted
+ // response. Connected consuming proof below honours the real interval.
+ f.reject.Store(false)
+ cc.mu.Lock()
+ delete(cc.lastHealthCheck, f.server.URL)
+ cc.mu.Unlock()
+ cc.recoverUnhealthyNodes(context.Background())
+ health = cc.GetHealthStatusWithErrors()[f.server.URL]
+ if c := cc.clients[f.server.URL]; c == nil || c.config.Timeout != cfg.Timeout || !health.Healthy || health.LastError != "" {
+ t.Fatalf("later complete recovery failed: %+v", health)
+ }
+ if f.nodes.Load() != 2 || f.logins.Load() != 3*f.loginAttempts() {
+ t.Fatalf("later recovery lost probe/full-session boundary: nodes=%d logins=%d", f.nodes.Load(), f.logins.Load())
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestClusterNodeInventoryCachedFallbackCannotInventRecovery(t *testing.T) {
+ f := nodeInventoryProbeFixture(t, "json", `{}`, true)
+ f.reject.Store(false)
+ cc := NewClusterClient("inventory", f.config(), []string{f.server.URL}, nil)
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 {
+ t.Fatal(nodes, err)
+ }
+ original := cc.clients[f.server.URL]
+ defer original.httpClient.CloseIdleConnections()
+ f.reject.Store(true)
+ cc.markUnhealthyWithError(f.server.URL, "previous failure")
+ if c, err := cc.getHealthyClient(context.Background()); err != nil || c != original {
+ t.Fatal("cached fallback unavailable", err)
+ }
+ before := f.nodes.Load()
+ if nodes, err := cc.GetNodes(context.Background()); err == nil || nodes != nil {
+ t.Fatal("invalid cached read invented an authoritative empty inventory", nodes, err)
+ }
+ h := cc.GetHealthStatusWithErrors()[f.server.URL]
+ if h.Healthy || h.LastError != "previous failure" || f.nodes.Load() != before+1 {
+ t.Fatal("cached invalid read invented recovery or replayed", h)
+ }
+ f.reject.Store(false)
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 || nodes[0].CPU != 0.25 {
+ t.Fatal("distinct complete read did not recover", nodes, err)
+ }
+ h = cc.GetHealthStatusWithErrors()[f.server.URL]
+ if !h.Healthy || h.LastError != "" || cc.clients[f.server.URL] != original || f.nodes.Load() != before+2 {
+ t.Fatal("complete read lost normal same-client recovery", h)
+ }
+ if cc.clients[f.server.URL].config.Timeout != 2*time.Second {
+ t.Fatal("full timeout changed")
+ }
+}
diff --git a/pkg/proxmox/node_inventory_response.go b/pkg/proxmox/node_inventory_response.go
new file mode 100644
index 0000000000..a8b05b3b43
--- /dev/null
+++ b/pkg/proxmox/node_inventory_response.go
@@ -0,0 +1,73 @@
+package proxmox
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "strings"
+ "unicode/utf8"
+)
+
+// Do not copy a malformed successful response into health diagnostics. It may
+// contain private proxy details; it is not an authentication refusal either.
+var errInvalidNodeInventoryResponse = errors.New("invalid Proxmox node inventory response")
+
+func decodeNodeInventory(body []byte) ([]Node, error) {
+ if !utf8.Valid(body) {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ envelope, valid := nodeInventoryObject(body, "data", "errors")
+ if !valid {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ if failure, exists := envelope["errors"]; exists && !bytes.Equal(bytes.TrimSpace(failure), []byte("null")) {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ data := bytes.TrimSpace(envelope["data"])
+ // An explicit empty array remains valid for scoped tokens. Missing/null
+ // data is not an empty authoritative inventory and must not erase nodes.
+ if len(data) == 0 || data[0] != '[' {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ var rows []json.RawMessage
+ if json.Unmarshal(data, &rows) != nil {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ nodes := make([]Node, 0, len(rows))
+ seen := make(map[string]bool, len(rows))
+ for _, row := range rows {
+ fields, valid := nodeInventoryObject(row, "node", "status", "cpu", "maxcpu", "mem", "maxmem", "disk", "maxdisk", "uptime", "level")
+ if !valid {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ var name string
+ if json.Unmarshal(fields["node"], &name) != nil || strings.TrimSpace(name) == "" || seen[name] {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ var node Node
+ if json.Unmarshal(row, &node) != nil {
+ return nil, errInvalidNodeInventoryResponse
+ }
+ seen[name] = true
+ nodes = append(nodes, node)
+ }
+ return nodes, nil
+}
+
+// The existing object reader rejects duplicates and extra JSON values. Keep
+// unique provider metadata, but never let encoding/json's case-insensitive
+// struct matching replace canonical inventory fields with another spelling.
+func nodeInventoryObject(body []byte, canonical ...string) (map[string]json.RawMessage, bool) {
+ fields, valid := guestAgentResponseFields(body)
+ if !valid {
+ return nil, false
+ }
+ for name := range fields {
+ for _, expected := range canonical {
+ if name != expected && strings.EqualFold(name, expected) {
+ return nil, false
+ }
+ }
+ }
+ return fields, true
+}
diff --git a/pkg/proxmox/node_inventory_response_test.go b/pkg/proxmox/node_inventory_response_test.go
new file mode 100644
index 0000000000..306661ab26
--- /dev/null
+++ b/pkg/proxmox/node_inventory_response_test.go
@@ -0,0 +1,94 @@
+package proxmox
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
+ "testing"
+)
+
+// Exercise the real bounded HTTP reader, not just a detached JSON parser.
+func TestNodeInventoryResponseAdmission(t *testing.T) {
+ const inventory = `{"data":[{"node":"member","status":"online","cpu":0.25}]}`
+ cases := []struct {
+ name, body string
+ complete bool
+ }{
+ {"missing-data", `{}`, true},
+ {"error-as-success", `{"errors":{"message":"fixture-private-detail"}}`, true},
+ {"conflicting-error", `{"data":[],"errors":{"message":"fixture-private-detail"}}`, true},
+ {"null-inventory", `{"data":null}`, true},
+ {"null-envelope", `null`, true},
+ {"case-data", `{"Data":[]}`, true},
+ {"duplicate-data", `{"data":{},"data":[]}`, true},
+ {"escaped-duplicate-data", `{"data":{},"\u0064ata":[]}`, true},
+ {"case-overwrite", `{"data":[],"Data":null}`, true},
+ {"trailing-value", inventory + ` {"errors":{"message":"fixture-private-detail"}}`, true},
+ {"trailing-garbage", inventory + `not-json`, true},
+ {"truncated-after-prefix", inventory, false},
+ {"oversized-after-prefix", inventory + strings.Repeat(" ", int(maxResponseBodyBytes)), true},
+ {"invalid-utf8", `{"data":[],"metadata":"` + string([]byte{0xff}) + `"}`, true},
+ {"null-row", `{"data":[null]}`, true},
+ {"unnamed-row", `{"data":[{}]}`, true},
+ {"empty-node", `{"data":[{"node":""}]}`, true},
+ {"null-node", `{"data":[{"node":null}]}`, true},
+ {"duplicate-node-field", `{"data":[{"node":"other","node":"member"}]}`, true},
+ {"case-node-field", `{"data":[{"node":"member","Node":"other"}]}`, true},
+ {"duplicate-node-row", `{"data":[{"node":"member"},{"node":"member"}]}`, true},
+ {"wrong-node-type", `{"data":[{"node":42}]}`, true},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ if r.URL.Path != "/api2/json/nodes" {
+ t.Errorf("unexpected path: %s", r.URL.Path)
+ }
+ if !tc.complete {
+ w.Header().Set("Content-Length", fmt.Sprint(len(tc.body)+100))
+ }
+ fmt.Fprint(w, tc.body)
+ }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ nodes, err := c.GetNodes(context.Background())
+ if err == nil || nodes != nil {
+ t.Errorf("unverified inventory admitted: nodes=%+v err=%v", nodes, err)
+ }
+ if err != nil && strings.Contains(err.Error(), "fixture-private-detail") {
+ t.Error("response body leaked into admission error")
+ }
+ if requests.Load() != 1 {
+ t.Errorf("invalid success replayed: %d requests", requests.Load())
+ }
+ })
+ }
+}
+
+func TestNodeInventoryAcceptedScopedAndExtendedResponses(t *testing.T) {
+ for _, body := range []string{
+ `{"data":[]}`,
+ `{"data":[],"errors":null,"total":0}`,
+ `{"data":[{"node":"member","status":"offline","cpu":0.25,"maxcpu":8,"mem":128,"maxmem":512,"disk":256,"maxdisk":1024,"uptime":42,"level":"c","id":"node/member","type":"node"}]}`,
+ } {
+ t.Run(body, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, body) }))
+ defer server.Close()
+ c := backupTestClient(t, server.URL)
+ nodes, err := c.GetNodes(context.Background())
+ if err != nil {
+ t.Fatalf("accepted scoped/extended inventory refused: %v", err)
+ }
+ if strings.Contains(body, `"node"`) && (len(nodes) != 1 || nodes[0].Node != "member" || nodes[0].Status != "offline" || nodes[0].CPU != 0.25 || nodes[0].MaxCPU != 8 || nodes[0].Mem != 128 || nodes[0].MaxMem != 512 || nodes[0].Disk != 256 || nodes[0].MaxDisk != 1024 || nodes[0].Uptime != 42 || nodes[0].Level != "c") {
+ t.Fatalf("accepted node fields changed: %+v", nodes)
+ }
+ if !strings.Contains(body, `"node"`) && len(nodes) != 0 {
+ t.Fatalf("scoped empty inventory not empty: %+v", nodes)
+ }
+ })
+ }
+}