Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Own cluster endpoint membership before routing and trust checks
pulse-triage[bot]496b37ec716c
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 0bf1a4ad1e..2e41e917aa 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,27 @@## Purpose+### Cluster endpoint membership belongs to its client++`NewClusterClient` snapshots the supplied endpoint slice before registering+shared guest-command aliases or starting health and trust checks. After the+constructor returns, reusing caller configuration storage cannot replace live+failover targets, declare a new first-use TLS member, or route a guest read+outside its registered serial channel. Routing, health recovery and trust+admission use that same client-owned snapshot. Order, duplicate entries and+nil/empty inputs retain their previous semantics; actual configuration changes+still construct a replacement client rather than editing an existing client+through a borrowed slice.++`TestClusterClientEndpointOwnershipContract` exercises pinned ordinary reads,+typed TLS mismatch/first-use refusal, all six guest-read paths while a response+body is in flight, independent-guest and completed-command resumption, input+shape and concurrent caller reuse. Its fixtures use only synthetic credentials+and local sockets. Endpoint URLs, saved pins, CA/opt-out and first-use policy,+alias equivalence, backup locks, uncertainty fences and non-replay are unchanged.+This is source-bound admission safety, not native workload thaw, Windows restart+recovery, the cause of a reported guest failure, or installed/release acceptance.+### Repeatable resource wiring publishes a complete binding onceRouter and tenant initialization applies saved notification policy before thediff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.goindex 63339172f9..baeb0574d1 100644--- a/pkg/proxmox/cluster_client.go+++ b/pkg/proxmox/cluster_client.go@@ -6,6 +6,7 @@ import ("fmt""math/rand""regexp"+ "slices""strconv""strings""sync"@@ -177,6 +178,10 @@ func sanitizeEndpointError(errMsg string) string {// endpointFingerprints is an optional map of endpoint URL -> TLS fingerprint for per-node certificate verification.// This enables TOFU (Trust On First Use) for clusters with unique self-signed certs per node.func NewClusterClient(name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string) *ClusterClient {+ // The caller may reuse configuration storage after construction. Keep live+ // routing and first-use trust on the same membership snapshot registered for+ // guest-command coordination, including during initial health checks.+ endpoints = slices.Clone(endpoints)registerGuestAgentEndpoints(config.Host, endpoints)// The client owns subsequent first-use captures. Do not mutate the caller's// saved trust map or let another client overwrite it without our mutex.diff --git a/pkg/proxmox/cluster_client_api_test.go b/pkg/proxmox/cluster_client_api_test.goindex 7b6b13a1db..31c6a24a33 100644--- a/pkg/proxmox/cluster_client_api_test.go+++ b/pkg/proxmox/cluster_client_api_test.go@@ -639,6 +639,7 @@ func TestClusterClientEndpointOwnershipContract(t *testing.T) {t.Run("routing-and-first-use-trust", testClusterClientEndpointOwnershipRoutingAndTrust)t.Run("shared-guest-admission", testClusterClientEndpointOwnershipGuestAdmission)t.Run("input-shape", testClusterClientEndpointOwnershipShape)+ t.Run("concurrent-ordinary-reads", testClusterClientEndpointOwnershipConcurrentReads)}// Guest admission equivalence must not broaden the existing certificate trust scope.diff --git a/pkg/proxmox/cluster_client_endpoint_ownership_test.go b/pkg/proxmox/cluster_client_endpoint_ownership_test.goindex b35925cdee..cb72a96c04 100644--- a/pkg/proxmox/cluster_client_endpoint_ownership_test.go+++ b/pkg/proxmox/cluster_client_endpoint_ownership_test.go@@ -6,6 +6,7 @@ import ("fmt""net/http""net/http/httptest"+ "runtime""slices""strings""sync/atomic"@@ -51,7 +52,7 @@ func testClusterClientEndpointOwnershipRoutingAndTrust(t *testing.T) {if captured || cc.getEndpointFingerprint(declared.server.URL) != declaredPin {t.Error("caller-owned endpoint edit changed the client's trust state")}- t.Logf("original pinned monitoring preserved; injected endpoint authenticated requests=%d", other.requests.Load())+ t.Logf("observed injected endpoint authenticated requests=%d; original pin unchanged=%t", other.requests.Load(), cc.getEndpointFingerprint(declared.server.URL) == declaredPin)}func testClusterClientEndpointOwnershipGuestAdmission(t *testing.T) {@@ -150,7 +151,7 @@ func testClusterClientEndpointOwnershipGuestAdmission(t *testing.T) {if originalCommands.Load() != 3 || otherRequests.Load() != 0 {t.Errorf("recovery left declared targets: original commands=%d, other requests=%d", originalCommands.Load(), otherRequests.Load())}- t.Logf("six busy reads fenced; independent guest and completed guest resume; other requests=%d", otherRequests.Load())+ t.Logf("observed original commands=%d, other requests=%d; six busy-read results checked", originalCommands.Load(), otherRequests.Load())}func testClusterClientEndpointOwnershipShape(t *testing.T) {@@ -165,3 +166,49 @@ func testClusterClientEndpointOwnershipShape(t *testing.T) {}}}++func testClusterClientEndpointOwnershipConcurrentReads(t *testing.T) {+ var declaredRequests, otherRequests atomic.Int32+ newServer := func(name string, requests *atomic.Int32) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ if r.URL.Path != "/api2/json/nodes" || r.Header.Get("Authorization") != "PVEAPIToken=fixture@pve!pulse=fixture" {+ t.Error("ordinary monitoring path or fixture authentication changed")+ }+ fmt.Fprintf(w, `{"data":[{"node":%q,"status":"online"}]}`, name)+ }))+ }+ declared := newServer("declared", &declaredRequests)+ defer declared.Close()+ other := newServer("other", &otherRequests)+ defer other.Close()+ input := []string{declared.URL}+ cc := NewClusterClient("owned-concurrent", ClientConfig{Host: declared.URL, TokenName: "fixture@pve!pulse", TokenValue: "fixture", Timeout: time.Second}, input, nil)+ started, stop, joined := make(chan struct{}), make(chan struct{}), make(chan struct{})+ go func() {+ defer close(joined)+ close(started)+ for {+ select {+ case <-stop:+ return+ default:+ input[0] = other.URL+ runtime.Gosched()+ input[0] = declared.URL+ }+ }+ }()+ defer func() { close(stop); <-joined }()+ <-started+ for i := 0; i < 32; i++ {+ nodes, err := cc.GetNodes(context.Background())+ if err != nil || len(nodes) != 1 || nodes[0].Node != "declared" {+ t.Fatalf("concurrent configuration reuse changed ordinary monitoring: %v", err)+ }+ }+ if declaredRequests.Load() != 33 || otherRequests.Load() != 0 {+ t.Error("concurrent configuration reuse changed the authenticated target")+ }+ t.Logf("observed declared requests=%d, other requests=%d during 32 concurrent-input reads", declaredRequests.Load(), otherRequests.Load())+}