Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Own cluster endpoint membership before routing and trust checks

pulse-triage[bot]496b37ec716c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 0bf1a4ad1e..2e41e917aa 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,27 @@
## Purpose
+### Cluster endpoint membership belongs to its client
+
+`NewClusterClient` snapshots the supplied endpoint slice before registering
+shared guest-command aliases or starting health and trust checks. After the
+constructor returns, reusing caller configuration storage cannot replace live
+failover targets, declare a new first-use TLS member, or route a guest read
+outside its registered serial channel. Routing, health recovery and trust
+admission use that same client-owned snapshot. Order, duplicate entries and
+nil/empty inputs retain their previous semantics; actual configuration changes
+still construct a replacement client rather than editing an existing client
+through a borrowed slice.
+
+`TestClusterClientEndpointOwnershipContract` exercises pinned ordinary reads,
+typed TLS mismatch/first-use refusal, all six guest-read paths while a response
+body is in flight, independent-guest and completed-command resumption, input
+shape and concurrent caller reuse. Its fixtures use only synthetic credentials
+and local sockets. Endpoint URLs, saved pins, CA/opt-out and first-use policy,
+alias equivalence, backup locks, uncertainty fences and non-replay are unchanged.
+This is source-bound admission safety, not native workload thaw, Windows restart
+recovery, the cause of a reported guest failure, or installed/release acceptance.
+
### Repeatable resource wiring publishes a complete binding once
Router and tenant initialization applies saved notification policy before the
diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go
index 63339172f9..baeb0574d1 100644
--- a/pkg/proxmox/cluster_client.go
+++ b/pkg/proxmox/cluster_client.go
@@ -6,6 +6,7 @@ import (
"fmt"
"math/rand"
"regexp"
+ "slices"
"strconv"
"strings"
"sync"
@@ -177,6 +178,10 @@ func sanitizeEndpointError(errMsg string) string {
// endpointFingerprints is an optional map of endpoint URL -> TLS fingerprint for per-node certificate verification.
// This enables TOFU (Trust On First Use) for clusters with unique self-signed certs per node.
func NewClusterClient(name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string) *ClusterClient {
+ // The caller may reuse configuration storage after construction. Keep live
+ // routing and first-use trust on the same membership snapshot registered for
+ // guest-command coordination, including during initial health checks.
+ endpoints = slices.Clone(endpoints)
registerGuestAgentEndpoints(config.Host, endpoints)
// The client owns subsequent first-use captures. Do not mutate the caller's
// saved trust map or let another client overwrite it without our mutex.
diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.go
index 7b6b13a1db..31c6a24a33 100644
--- a/pkg/proxmox/cluster_client_api_test.go
+++ b/pkg/proxmox/cluster_client_api_test.go
@@ -639,6 +639,7 @@ func TestClusterClientEndpointOwnershipContract(t *testing.T) {
t.Run("routing-and-first-use-trust", testClusterClientEndpointOwnershipRoutingAndTrust)
t.Run("shared-guest-admission", testClusterClientEndpointOwnershipGuestAdmission)
t.Run("input-shape", testClusterClientEndpointOwnershipShape)
+ t.Run("concurrent-ordinary-reads", testClusterClientEndpointOwnershipConcurrentReads)
}
// Guest admission equivalence must not broaden the existing certificate trust scope.
diff --git a/pkg/proxmox/cluster_client_endpoint_ownership_test.go b/pkg/proxmox/cluster_client_endpoint_ownership_test.go
index b35925cdee..cb72a96c04 100644
--- a/pkg/proxmox/cluster_client_endpoint_ownership_test.go
+++ b/pkg/proxmox/cluster_client_endpoint_ownership_test.go
@@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
+ "runtime"
"slices"
"strings"
"sync/atomic"
@@ -51,7 +52,7 @@ func testClusterClientEndpointOwnershipRoutingAndTrust(t *testing.T) {
if captured || cc.getEndpointFingerprint(declared.server.URL) != declaredPin {
t.Error("caller-owned endpoint edit changed the client's trust state")
}
- t.Logf("original pinned monitoring preserved; injected endpoint authenticated requests=%d", other.requests.Load())
+ t.Logf("observed injected endpoint authenticated requests=%d; original pin unchanged=%t", other.requests.Load(), cc.getEndpointFingerprint(declared.server.URL) == declaredPin)
}
func testClusterClientEndpointOwnershipGuestAdmission(t *testing.T) {
@@ -150,7 +151,7 @@ func testClusterClientEndpointOwnershipGuestAdmission(t *testing.T) {
if originalCommands.Load() != 3 || otherRequests.Load() != 0 {
t.Errorf("recovery left declared targets: original commands=%d, other requests=%d", originalCommands.Load(), otherRequests.Load())
}
- t.Logf("six busy reads fenced; independent guest and completed guest resume; other requests=%d", otherRequests.Load())
+ t.Logf("observed original commands=%d, other requests=%d; six busy-read results checked", originalCommands.Load(), otherRequests.Load())
}
func testClusterClientEndpointOwnershipShape(t *testing.T) {
@@ -165,3 +166,49 @@ func testClusterClientEndpointOwnershipShape(t *testing.T) {
}
}
}
+
+func testClusterClientEndpointOwnershipConcurrentReads(t *testing.T) {
+ var declaredRequests, otherRequests atomic.Int32
+ newServer := func(name string, requests *atomic.Int32) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ if r.URL.Path != "/api2/json/nodes" || r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {
+ t.Error("ordinary monitoring path or fixture authentication changed")
+ }
+ fmt.Fprintf(w, `{"data":[{"node":%q,"status":"online"}]}`, name)
+ }))
+ }
+ declared := newServer("declared", &declaredRequests)
+ defer declared.Close()
+ other := newServer("other", &otherRequests)
+ defer other.Close()
+ input := []string{declared.URL}
+ cc := NewClusterClient("owned-concurrent", ClientConfig{Host: declared.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}, input, nil)
+ started, stop, joined := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ go func() {
+ defer close(joined)
+ close(started)
+ for {
+ select {
+ case <-stop:
+ return
+ default:
+ input[0] = other.URL
+ runtime.Gosched()
+ input[0] = declared.URL
+ }
+ }
+ }()
+ defer func() { close(stop); <-joined }()
+ <-started
+ for i := 0; i < 32; i++ {
+ nodes, err := cc.GetNodes(context.Background())
+ if err != nil || len(nodes) != 1 || nodes[0].Node != "declared" {
+ t.Fatalf("concurrent configuration reuse changed ordinary monitoring: %v", err)
+ }
+ }
+ if declaredRequests.Load() != 33 || otherRequests.Load() != 0 {
+ t.Error("concurrent configuration reuse changed the authenticated target")
+ }
+ t.Logf("observed declared requests=%d, other requests=%d during 32 concurrent-input reads", declaredRequests.Load(), otherRequests.Load())
+}