Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep Proxmox recovery authentication within its caller budget
pulse-triage[bot]13984070ff03
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex b0dccf390d..8f3d0725a0 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,37 @@## Purpose+### Proxmox recovery sessions honour the owning caller++PVE/PBS context-aware constructors bind initial password login, including the+existing JSON/form fallback, to the initiating context. Cluster selection,+first-use pinned retry, synchronous construction and unhealthy-member recovery+carry that context through both the short probe and the retained polling+session. The six-second background recovery budget also bounds construction;+ending a caller is neither a credential refusal nor successful recovery.+Cancelled selection/recovery preserves prior errors and does not install a+partial or nil client. Startup retry exits without publishing a new health+outcome or advancing another provider after its monitor stops. Discarded probe+transports close their idle connections.++The compatibility constructors retain their background-context behaviour.+Accepted clients retain their configured full timeout, and subsequent ordinary+reads use their own contexts, not the ended construction context. TLS pins,+CA verification, redirects, first-use admission, auth encodings, retry cadence,+provider permissions, guest/backup fences and History are unchanged.++`TestClientConstructionHonoursContext` in both client packages covers ended+password/token construction, in-flight JSON/form cancellation/deadlines,+independent later reads and wrong-pin refusal. Cluster caller-budget controls+cover selection/recovery at first login, probe and full login, then trusted+ordinary recovery. Startup cancellation covers PVE, PBS and multi-endpoint+cluster construction. The connected PBS first-poll control emits the actual+broadcast for desktop/phone consumption: cancellation cannot turn an offline+server healthy, and accepted construction alone still cannot do so. These are+secret-free local fixtures, not installed recovery, native guest safety or an+attribution of any reporter's failure.++### Cluster recovery requires a usable polling sessionA short endpoint probe cannot establish successful recovery until the clientdiff --git a/internal/monitoring/mock_mode_config_test.go b/internal/monitoring/mock_mode_config_test.goindex b54ac015ca..f958aca929 100644--- a/internal/monitoring/mock_mode_config_test.go+++ b/internal/monitoring/mock_mode_config_test.go@@ -1,6 +1,7 @@package monitoringimport (+ "context""testing""github.com/rcourtman/pulse-go-rewrite/internal/config"@@ -50,7 +51,7 @@ func TestNew_MockModeClientInitializationRespectsKeepRealPollingSetting(t *testiorig := newProxmoxClientFunct.Cleanup(func() { newProxmoxClientFunc = orig })called := false- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {called = truereturn &mockPVEClient{}, nil}diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.goindex 625f7cd32a..902e1d4db1 100644--- a/internal/monitoring/monitor.go+++ b/internal/monitoring/monitor.go@@ -78,8 +78,8 @@ func alertManagerRestoreOptions(mockEnabled bool) []alerts.ManagerOption {// newProxmoxClientFunc is a variable that holds the function to create a new Proxmox client.// It is used to allow mocking the client creation in tests.-var newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {- return proxmox.NewClient(cfg)+var newProxmoxClientFunc = func(ctx context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ return proxmox.NewClientWithContext(ctx, cfg)}// PVEClientInterface defines the interface for PVE clients (both regular and cluster)diff --git a/internal/monitoring/monitor_client_cancellation_test.go b/internal/monitoring/monitor_client_cancellation_test.gonew file mode 100644index 0000000000..d61296b50d--- /dev/null+++ b/internal/monitoring/monitor_client_cancellation_test.go@@ -0,0 +1,224 @@+package monitoring++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "reflect"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"+)++func testStartupClientConstructionHonoursCancellation(t *testing.T) {+ for _, provider := range []string{"pve", "pbs", "cluster"} {+ t.Run(provider, func(t *testing.T) {+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once, endedOnce, releaseOnce sync.Once+ var requests atomic.Int32+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ requests.Add(1)+ if !strings.HasSuffix(r.URL.Path, "/api2/json/access/ticket") {+ t.Errorf("ended startup construction advanced to another request: %s", r.URL.Path)+ }+ once.Do(func() { close(entered) })+ select {+ case <-r.Context().Done():+ endedOnce.Do(func() { close(requestEnded) })+ return+ case <-release:+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ }))+ defer server.Close()+ defer releaseOnce.Do(func() { close(release) })+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := &config.Config{ConnectionTimeout: 8 * time.Second}+ if provider == "pbs" {+ cfg.PBSInstances = []config.PBSInstance{{Name: "cancelled-pbs", Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:])}}+ } else {+ instance := config.PVEInstance{Name: "cancelled-pve", Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:])}+ if provider == "cluster" {+ // Two distinct base paths on the same trusted local TLS origin+ // exercise synchronous initial health/session checks too.+ instance.IsCluster = true+ instance.ClusterEndpoints = []config.ClusterEndpoint{{Host: server.URL, Fingerprint: instance.Fingerprint}}+ // Canonical authority deduplication would otherwise skip the check.+ instance.ClusterEndpoints[0].Host += "/member"+ }+ cfg.PVEInstances = []config.PVEInstance{instance}+ }+ monitor := &Monitor{config: cfg, state: models.NewState(), pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client)}+ originalDelays := connRetryDelays+ connRetryDelays = []time.Duration{time.Millisecond}+ defer func() { connRetryDelays = originalDelays }()+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ done := make(chan struct{})+ go func() { monitor.retryFailedConnections(ctx); close(done) }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ releaseOnce.Do(func() { close(release) })+ cancel()+ <-done+ t.Fatal("startup constructor did not reach local password login")+ }+ cancel()+ select {+ case <-done:+ case <-time.After(time.Second):+ t.Error("retry shutdown waited for detached constructor authentication")+ releaseOnce.Do(func() { close(release) })+ <-done+ }+ // Both cluster constructor requests may have started before cancellation.+ if provider != "cluster" {+ select {+ case <-requestEnded:+ case <-time.After(time.Second):+ t.Error("cancelled constructor left the password request active")+ }+ }+ if len(monitor.pveClients) != 0 || len(monitor.pbsClients) != 0 || len(monitor.state.GetSnapshot().ConnectionHealth) != 0 {+ t.Error("cancelled startup construction published a polling client or health outcome")+ }+ if provider != "cluster" && requests.Load() != 1 {+ t.Errorf("shutdown sent more than the admitted fixture login: %d", requests.Load())+ }+ })+ }+}++// Real pinned PBS construction, ordinary retry admission, first poll and the+// production broadcast feed the consuming browser proof. This is not an+// installed/reporter recovery or a reason to restart monitoring on a workload.+func testPBSStartupCancellationKeepsFirstPollBoundary(t *testing.T) {+ var logins, reads atomic.Int32+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var releaseOnce sync.Once+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ w.Header().Set("Content-Type", "application/json")+ if r.URL.Path == "/api2/json/access/ticket" {+ switch logins.Add(1) {+ case 1:+ w.WriteHeader(http.StatusServiceUnavailable)+ fmt.Fprint(w, `{"message":"fixture starting"}`)+ return+ case 2:+ close(entered)+ select {+ case <-r.Context().Done():+ close(requestEnded)+ return+ case <-release:+ }+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ return+ }+ reads.Add(1)+ if r.Method != http.MethodGet || !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {+ t.Errorf("non-ordinary or unauthenticated PBS fixture read: %s %s", r.Method, r.URL.Path)+ }+ switch r.URL.Path {+ case "/api2/json/version":+ fmt.Fprint(w, `{"data":{"version":"3.4.2"}}`)+ case "/api2/json/nodes/localhost/status":+ fmt.Fprint(w, `{"data":{"cpu":0.15,"memory":{"used":512,"total":1024},"uptime":120}}`)+ case "/api2/json/admin/datastore":+ fmt.Fprint(w, `{"data":[{"store":"backups","total":1000,"used":250,"avail":750}]}`)+ case "/api2/json/admin/datastore/backups/status":+ fmt.Fprint(w, `{"data":{"status":"available","total":1000,"used":250,"avail":750}}`)+ default:+ fmt.Fprint(w, `{"data":[]}`)+ }+ }))+ defer server.Close()+ defer releaseOnce.Do(func() { close(release) })+ pin := sha256.Sum256(server.Certificate().Raw)+ instance := config.PBSInstance{Name: "cancellable-pbs", Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), MonitorDatastores: true}+ monitor := newPBSHealthAuthorityMonitor([]config.PBSInstance{instance})+ project := func(phase string) {+ snapshot, scope := monitor.currentStateWithScope()+ frontend := monitor.buildBroadcastFrontendStateFromSnapshot(snapshot, scope)+ wire, err := json.Marshal(struct {+ Phase string `json:"phase"`+ State models.StateFrontend `json:"state"`+ }{phase, frontend})+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("PBS_CANCELLATION_PROJECTION_JSON=%s", wire)+ }+ monitor.initPBSClients(monitor.config)+ assertPBSConnectionProjection(t, monitor, instance.Name, false, "offline")+ project("unavailable")+ originalDelays := connRetryDelays+ connRetryDelays = []time.Duration{time.Millisecond, time.Millisecond}+ defer func() { connRetryDelays = originalDelays }()+ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ done := make(chan struct{})+ go func() { monitor.retryFailedConnections(ctx); close(done) }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ releaseOnce.Do(func() { close(release) })+ cancel()+ <-done+ t.Fatal("fixture did not reach the reconnect password request")+ }+ cancel()+ select {+ case <-done:+ case <-time.After(time.Second):+ t.Error("ended retry caller still owns a detached password request")+ releaseOnce.Do(func() { close(release) })+ <-done+ }+ select {+ case <-requestEnded:+ case <-time.After(time.Second):+ t.Error("ended retry caller did not cancel its HTTP request")+ }+ if logins.Load() != 2 || reads.Load() != 0 || len(monitor.pbsClients) != 0 || len(newPBSPollProvider().ListInstances(monitor)) != 0 {+ t.Fatalf("cancelled construction published a session or advanced polling: logins=%d reads=%d clients=%d", logins.Load(), reads.Load(), len(monitor.pbsClients))+ }+ assertPBSConnectionProjection(t, monitor, instance.Name, false, "offline")+ project("cancelled")++ releaseOnce.Do(func() { close(release) })+ laterCtx, laterCancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer laterCancel()+ monitor.retryFailedConnections(laterCtx)+ if logins.Load() != 3 || len(newPBSPollProvider().ListInstances(monitor)) != 1 {+ t.Fatal("later admitted retry did not retain its accepted session exactly once")+ }+ assertPBSConnectionProjection(t, monitor, instance.Name, false, "offline")+ project("awaiting-poll")+ task, err := newPBSPollProvider().BuildPollTask(monitor, instance.Name)+ if err != nil {+ t.Fatal(err)+ }+ newRealExecutor(monitor).Execute(laterCtx, task)+ assertPBSConnectionProjection(t, monitor, instance.Name, true, "online")+ actual := pbsInstanceByName(t, monitor.state.GetSnapshot(), instance.Name)+ if actual.CPU != 0.15 || actual.Memory != 50 || len(actual.Datastores) != 1 || actual.Datastores[0].Usage != 25 || logins.Load() != 3 || !reflect.DeepEqual(instance, monitor.config.PBSInstances[0]) {+ t.Fatalf("ordinary first poll lost readings, identity or saved trust: %+v logins=%d", actual, logins.Load())+ }+ project("recovered")+}diff --git a/internal/monitoring/monitor_client_init.go b/internal/monitoring/monitor_client_init.goindex 7a0b318dd0..05754727d3 100644--- a/internal/monitoring/monitor_client_init.go+++ b/internal/monitoring/monitor_client_init.go@@ -1,6 +1,7 @@package monitoringimport (+ "context""time""github.com/rcourtman/pulse-go-rewrite/internal/config"@@ -58,7 +59,7 @@ func (m *Monitor) initPVEClients(cfg *config.Config) {// Create regular clientclientConfig := config.CreateProxmoxConfig(&pve)clientConfig.Timeout = cfg.ConnectionTimeout- client, err := newProxmoxClientFunc(clientConfig)+ client, err := newProxmoxClientFunc(context.Background(), clientConfig)if err != nil {monErr := errors.WrapConnectionError("create_pve_client", pve.Name, err)log.Error().diff --git a/internal/monitoring/monitor_client_reconnect.go b/internal/monitoring/monitor_client_reconnect.goindex eb0296137c..7734b21dfb 100644--- a/internal/monitoring/monitor_client_reconnect.go+++ b/internal/monitoring/monitor_client_reconnect.go@@ -103,13 +103,23 @@ func (m *Monitor) retryFailedConnections(ctx context.Context) {// Try to recreate PVE clientsfor _, pve := range missingPVE {+ if ctx.Err() != nil {+ return+ }if pve.IsCluster && len(pve.ClusterEndpoints) > 0 {// Create cluster clientendpoints, endpointFingerprints := m.buildClusterEndpointsForReconnect(pve)clientConfig := config.CreateProxmoxConfig(&pve)clientConfig.Timeout = connectionTimeout- clusterClient := proxmox.NewClusterClient(pve.Name, clientConfig, endpoints, endpointFingerprints)+ clusterClient, err := proxmox.NewClusterClientWithContext(ctx, pve.Name, clientConfig, endpoints, endpointFingerprints)+ if ctx.Err() != nil {+ return+ }+ if err != nil {+ log.Warn().Err(err).Str("instance", pve.Name).Msg("Failed to reconnect cluster client, will retry")+ continue+ }m.replacePVEClient(pve.Name, clusterClient)m.setProviderConnectionHealth(InstanceTypePVE, pve.Name, true)@@ -124,7 +134,10 @@ func (m *Monitor) retryFailedConnections(ctx context.Context) {// Create regular clientclientConfig := config.CreateProxmoxConfig(&pve)clientConfig.Timeout = connectionTimeout- client, err := newProxmoxClientFunc(clientConfig)+ client, err := newProxmoxClientFunc(ctx, clientConfig)+ if ctx.Err() != nil {+ return+ }if err != nil {log.Warn().Err(err).@@ -143,9 +156,15 @@ func (m *Monitor) retryFailedConnections(ctx context.Context) {// Try to recreate PBS clientsfor _, pbsInst := range missingPBS {+ if ctx.Err() != nil {+ return+ }clientConfig := config.CreatePBSConfig(&pbsInst)clientConfig.Timeout = 60 * time.Second- client, err := pbs.NewClient(clientConfig)+ client, err := pbs.NewClientWithContext(ctx, clientConfig)+ if ctx.Err() != nil {+ return+ }if err != nil {monErr := monitorerrors.WrapConnectionError("recreate_pbs_client", pbsInst.Name, err)log.Warn().diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.goindex 1d675570ed..d151305fc4 100644--- a/internal/monitoring/monitor_extra_coverage_test.go+++ b/internal/monitoring/monitor_extra_coverage_test.go@@ -2416,7 +2416,7 @@ func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {cause := fmt.Errorf("dial tcp 127.0.0.1:8006: connect: connection refused")// 1. Success case- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {if strings.Contains(cfg.Host, "8006") {return nil, fmt.Errorf("should not be called with 8006 in fallback")}@@ -2439,7 +2439,7 @@ func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {_ = nodes // ignore// 2. Failure to create client- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {return nil, fmt.Errorf("create failed")}_, _, err = m.retryPVEPortFallback(context.Background(), "pve1", instanceCfg, currentClient, cause)@@ -2448,7 +2448,7 @@ func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {}// 3. Failure to get nodes- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {// Return a client that fails GetNodesreturn &mockPVEClientFailNodes{}, nil}diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.goindex 3cac9a7b7c..5815241c1e 100644--- a/internal/monitoring/monitor_pve.go+++ b/internal/monitoring/monitor_pve.go@@ -412,7 +412,7 @@ func (m *Monitor) retryPVEPortFallback(ctx context.Context, instanceName string,m.mu.RUnlock()}- fallbackClient, err := newProxmoxClientFunc(clientCfg)+ fallbackClient, err := newProxmoxClientFunc(ctx, clientCfg)if err != nil {return nil, currentClient, cause}diff --git a/internal/monitoring/monitor_pve_cluster_refresh_test.go b/internal/monitoring/monitor_pve_cluster_refresh_test.goindex 4305732a3f..b2edb65796 100644--- a/internal/monitoring/monitor_pve_cluster_refresh_test.go+++ b/internal/monitoring/monitor_pve_cluster_refresh_test.go@@ -413,3 +413,12 @@ func TestClusterEndpointIdentityChanged(t *testing.T) {t.Fatal("an added node must count as an identity change")}}++// Retry admission and first-poll health must not outlive the owning monitor.+func TestStartupClientConstructionHonoursCancellation(t *testing.T) {+ testStartupClientConstructionHonoursCancellation(t)+}++func TestPBSStartupCancellationKeepsFirstPollBoundary(t *testing.T) {+ testPBSStartupCancellationKeepsFirstPollBoundary(t)+}diff --git a/internal/monitoring/monitor_reconnect_test.go b/internal/monitoring/monitor_reconnect_test.goindex 4e2c2e9973..0f36ba8b21 100644--- a/internal/monitoring/monitor_reconnect_test.go+++ b/internal/monitoring/monitor_reconnect_test.go@@ -46,7 +46,7 @@ func TestMonitor_RetryFailedConnections_Detailed_Extra(t *testing.T) {// 1. Test Successful Reconnectioncalled := false- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {called = trueif !strings.Contains(cfg.Host, "pve1") {return nil, fmt.Errorf("unexpected host: %s", cfg.Host)@@ -74,7 +74,7 @@ func TestMonitor_RetryFailedConnections_Detailed_Extra(t *testing.T) {{Name: "pve2", Host: "https://pve2:8006"},}- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {return nil, fmt.Errorf("connection failed")}diff --git a/internal/monitoring/monitor_startup_recovery_test.go b/internal/monitoring/monitor_startup_recovery_test.goindex 04e21aa74a..2ee4d5a588 100644--- a/internal/monitoring/monitor_startup_recovery_test.go+++ b/internal/monitoring/monitor_startup_recovery_test.go@@ -32,7 +32,7 @@ func TestConnectionRetrySurvivesLongStartupOutage(t *testing.T) {m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{instance}, ConnectionTimeout: 3 * time.Second}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}started := time.Now()var attempts []time.Duration- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {want := config.CreateProxmoxConfig(&instance)want.Timeout = 3 * time.Secondif !reflect.DeepEqual(cfg, want) {@@ -66,7 +66,7 @@ func TestConnectionRetryLongOutageStopsOnCancellation(t *testing.T) {synctest.Test(t, func(t *testing.T) {m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{{Name: "still-unavailable", Host: "https://pve.local:8006"}}}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}calls := 0- newProxmoxClientFunc = func(proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(context.Context, proxmox.ClientConfig) (PVEClientInterface, error) {calls++return nil, fmt.Errorf("fixture provider still unavailable")}@@ -85,7 +85,7 @@ func TestConnectionRetryDoesNotResumePausedConnections(t *testing.T) {defer func() { newProxmoxClientFunc = original }()synctest.Test(t, func(t *testing.T) {m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{{Name: "paused-pve", Disabled: true}}, PBSInstances: []config.PBSInstance{{Name: "paused-pbs", Disabled: true, Host: "://invalid"}}}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}- newProxmoxClientFunc = func(proxmox.ClientConfig) (PVEClientInterface, error) {+ newProxmoxClientFunc = func(context.Context, proxmox.ClientConfig) (PVEClientInterface, error) {t.Fatal("retry attempted a paused connection")return nil, nil}diff --git a/pkg/pbs/client.go b/pkg/pbs/client.goindex 4473091f8f..a6ab8d2fcc 100644--- a/pkg/pbs/client.go+++ b/pkg/pbs/client.go@@ -117,6 +117,15 @@ type auth struct {// NewClient creates a new PBS API clientfunc NewClient(cfg ClientConfig) (*Client, error) {+ return NewClientWithContext(context.Background(), cfg)+}++// NewClientWithContext binds initial password authentication to its caller.+// Later polling remains independent of this construction context.+func NewClientWithContext(ctx context.Context, cfg ClientConfig) (*Client, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }// Normalize host URL - ensure it has a protocoldefaultedScheme := !strings.HasPrefix(cfg.Host, "http://") && !strings.HasPrefix(cfg.Host, "https://")if defaultedScheme {@@ -205,10 +214,15 @@ func NewClient(cfg ClientConfig) (*Client, error) {// Authenticate if using passwordif cfg.Password != "" && cfg.TokenName == "" {- if err := client.authenticate(context.Background()); err != nil {+ if err := client.authenticate(ctx); err != nil {+ httpClient.CloseIdleConnections()return nil, fmt.Errorf("authentication failed: %w", err)}}+ if err := ctx.Err(); err != nil {+ httpClient.CloseIdleConnections()+ return nil, err+ }return client, nil}diff --git a/pkg/pbs/client_construction_context_test.go b/pkg/pbs/client_construction_context_test.gonew file mode 100644index 0000000000..237083f61b--- /dev/null+++ b/pkg/pbs/client_construction_context_test.go@@ -0,0 +1,156 @@+package pbs++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++func testClientConstructionHonoursContext(t *testing.T) {+ for _, mode := range []string{"password", "token"} {+ t.Run("already-ended/"+mode, func(t *testing.T) {+ var requests atomic.Int32+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ http.Error(w, "unexpected fixture request", http.StatusForbidden)+ }))+ defer server.Close()+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}+ if mode == "token" {+ cfg.Password, cfg.TokenName, cfg.TokenValue = "", "fixture@pbs!pulse", "fixture-token"+ }+ for _, end := range []string{"cancel", "deadline"} {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ if end == "deadline" {+ ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(-time.Second))+ defer cancel()+ }+ client, err := NewClientWithContext(ctx, cfg)+ if client != nil || !errors.Is(err, ctx.Err()) || requests.Load() != 0 {+ t.Errorf("ended %s constructor: client=%t error=%v requests=%d", end, client != nil, err, requests.Load())+ }+ }+ })+ }+ for _, encoding := range []string{"json", "form"} {+ for _, end := range []string{"cancel", "deadline"} {+ t.Run(encoding+"/"+end, func(t *testing.T) {+ var logins, reads atomic.Int32+ var blocking atomic.Bool+ blocking.Store(true)+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var releaseOnce sync.Once+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ w.Header().Set("Content-Type", "application/json")+ if r.URL.Path != "/api2/json/access/ticket" {+ reads.Add(1)+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {+ t.Error("later ordinary read lost its session")+ }+ fmt.Fprint(w, `{"data":{"version":"3.4.2"}}`)+ return+ }+ logins.Add(1)+ if encoding == "form" && r.Header.Get("Content-Type") == "application/json" {+ w.WriteHeader(http.StatusUnsupportedMediaType)+ fmt.Fprint(w, `{"message":"fixture requires form"}`)+ return+ }+ if blocking.Load() {+ close(entered)+ select {+ case <-r.Context().Done():+ close(requestEnded)+ return+ case <-release:+ }+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ }))+ defer server.Close()+ defer releaseOnce.Do(func() { close(release) })+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}+ ctx, cancel := context.WithCancel(context.Background())+ if end == "deadline" {+ cancel()+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)+ }+ defer cancel()+ type result struct {+ client *Client+ err error+ }+ done := make(chan result, 1)+ go func() { c, err := NewClientWithContext(ctx, cfg); done <- result{c, err} }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ releaseOnce.Do(func() { close(release) })+ <-done+ t.Fatal("fixture never reached selected login encoding")+ }+ if end == "cancel" {+ cancel()+ } else {+ <-ctx.Done()+ }+ var got result+ select {+ case got = <-done:+ case <-time.After(time.Second):+ t.Error("constructor exceeded ended caller budget")+ releaseOnce.Do(func() { close(release) })+ got = <-done+ }+ if got.client != nil || !errors.Is(got.err, ctx.Err()) {+ t.Errorf("cancelled construction returned session or lost context: client=%t error=%v", got.client != nil, got.err)+ }+ select {+ case <-requestEnded:+ case <-time.After(time.Second):+ t.Error("cancelled password request remained active")+ }+ want := int32(1)+ if encoding == "form" {+ want = 2+ }+ if logins.Load() != want || reads.Load() != 0 {+ t.Errorf("cancellation replayed or advanced login: logins=%d reads=%d", logins.Load(), reads.Load())+ }+ blocking.Store(false)+ releaseOnce.Do(func() { close(release) })+ laterCtx, laterCancel := context.WithCancel(context.Background())+ client, err := NewClientWithContext(laterCtx, cfg)+ if err != nil {+ laterCancel()+ t.Fatal(err)+ }+ defer client.httpClient.CloseIdleConnections()+ laterCancel() // Cancelling construction later must not poison ordinary reads.+ if _, err := client.GetVersion(context.Background()); err != nil || reads.Load() != 1 || client.config.Timeout != cfg.Timeout {+ t.Errorf("retained client lost full timeout or ordinary independent reads: %v reads=%d", err, reads.Load())+ }+ before := logins.Load()+ bad := cfg+ bad.Fingerprint = strings.Repeat("0", 64)+ if refused, err := NewClientWithContext(context.Background(), bad); refused != nil || err == nil || logins.Load() != before {+ t.Error("context-aware construction weakened the saved TLS pin")+ }+ })+ }+ }+}diff --git a/pkg/pbs/host_diagnostics_regression_test.go b/pkg/pbs/host_diagnostics_regression_test.goindex a57b0a71b7..6c1c0060e4 100644--- a/pkg/pbs/host_diagnostics_regression_test.go+++ b/pkg/pbs/host_diagnostics_regression_test.go@@ -78,3 +78,8 @@ func TestClientAcceptedHostDiagnosticsRemainUseful(t *testing.T) {}}}++// Initial authentication must obey the same caller lifetime as ordinary reads.+func TestClientConstructionHonoursContext(t *testing.T) {+ testClientConstructionHonoursContext(t)+}diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex f28c0ac5a6..3b8597adec 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -317,6 +317,15 @@ type auth struct {// NewClient creates a new Proxmox VE API clientfunc NewClient(cfg ClientConfig) (*Client, error) {+ return NewClientWithContext(context.Background(), cfg)+}++// NewClientWithContext binds initial password authentication to its caller.+// The retained client's later reads keep their own contexts and full timeout.+func NewClientWithContext(ctx context.Context, cfg ClientConfig) (*Client, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }var user, realm string// For token authentication, we don't need user@realm format@@ -410,10 +419,15 @@ func NewClient(cfg ClientConfig) (*Client, error) {// Authenticate if using passwordif cfg.Password != "" && cfg.TokenName == "" {- if err := client.authenticate(context.Background()); err != nil {+ if err := client.authenticate(ctx); err != nil {+ httpClient.CloseIdleConnections()return nil, fmt.Errorf("authentication failed: %w", err)}}+ if err := ctx.Err(); err != nil {+ httpClient.CloseIdleConnections()+ return nil, err+ }return client, nil}diff --git a/pkg/proxmox/client_construction_context_test.go b/pkg/proxmox/client_construction_context_test.gonew file mode 100644index 0000000000..f8e95ececf--- /dev/null+++ b/pkg/proxmox/client_construction_context_test.go@@ -0,0 +1,156 @@+package proxmox++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++func testClientConstructionHonoursContext(t *testing.T) {+ for _, mode := range []string{"password", "token"} {+ t.Run("already-ended/"+mode, func(t *testing.T) {+ var requests atomic.Int32+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ http.Error(w, "unexpected fixture request", http.StatusForbidden)+ }))+ defer server.Close()+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}+ if mode == "token" {+ cfg.Password, cfg.TokenName, cfg.TokenValue = "", "fixture@pam!pulse", "fixture-token"+ }+ for _, end := range []string{"cancel", "deadline"} {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ if end == "deadline" {+ ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(-time.Second))+ defer cancel()+ }+ client, err := NewClientWithContext(ctx, cfg)+ if client != nil || !errors.Is(err, ctx.Err()) || requests.Load() != 0 {+ t.Errorf("ended %s constructor: client=%t error=%v requests=%d", end, client != nil, err, requests.Load())+ }+ }+ })+ }+ for _, encoding := range []string{"json", "form"} {+ for _, end := range []string{"cancel", "deadline"} {+ t.Run(encoding+"/"+end, func(t *testing.T) {+ var logins, reads atomic.Int32+ var blocking atomic.Bool+ blocking.Store(true)+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var releaseOnce sync.Once+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ w.Header().Set("Content-Type", "application/json")+ if r.URL.Path != "/api2/json/access/ticket" {+ reads.Add(1)+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {+ t.Error("later ordinary read lost its session")+ }+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)+ return+ }+ logins.Add(1)+ if encoding == "form" && r.Header.Get("Content-Type") == "application/json" {+ w.WriteHeader(http.StatusUnsupportedMediaType)+ fmt.Fprint(w, `{"message":"fixture requires form"}`)+ return+ }+ if blocking.Load() {+ close(entered)+ select {+ case <-r.Context().Done():+ close(requestEnded)+ return+ case <-release:+ }+ }+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ }))+ defer server.Close()+ defer releaseOnce.Do(func() { close(release) })+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}+ ctx, cancel := context.WithCancel(context.Background())+ if end == "deadline" {+ cancel()+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)+ }+ defer cancel()+ type result struct {+ client *Client+ err error+ }+ done := make(chan result, 1)+ go func() { c, err := NewClientWithContext(ctx, cfg); done <- result{c, err} }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ releaseOnce.Do(func() { close(release) })+ <-done+ t.Fatal("fixture never reached selected login encoding")+ }+ if end == "cancel" {+ cancel()+ } else {+ <-ctx.Done()+ }+ var got result+ select {+ case got = <-done:+ case <-time.After(time.Second):+ t.Error("constructor exceeded ended caller budget")+ releaseOnce.Do(func() { close(release) })+ got = <-done+ }+ if got.client != nil || !errors.Is(got.err, ctx.Err()) {+ t.Errorf("cancelled construction returned session or lost context: client=%t error=%v", got.client != nil, got.err)+ }+ select {+ case <-requestEnded:+ case <-time.After(time.Second):+ t.Error("cancelled password request remained active")+ }+ want := int32(1)+ if encoding == "form" {+ want = 2+ }+ if logins.Load() != want || reads.Load() != 0 {+ t.Errorf("cancellation replayed or advanced login: logins=%d reads=%d", logins.Load(), reads.Load())+ }+ blocking.Store(false)+ releaseOnce.Do(func() { close(release) })+ laterCtx, laterCancel := context.WithCancel(context.Background())+ client, err := NewClientWithContext(laterCtx, cfg)+ if err != nil {+ laterCancel()+ t.Fatal(err)+ }+ defer client.httpClient.CloseIdleConnections()+ laterCancel() // Cancelling construction later must not poison ordinary reads.+ if _, err := client.GetNodes(context.Background()); err != nil || reads.Load() != 1 || client.config.Timeout != cfg.Timeout {+ t.Errorf("retained client lost full timeout or ordinary independent reads: %v reads=%d", err, reads.Load())+ }+ before := logins.Load()+ bad := cfg+ bad.Fingerprint = strings.Repeat("0", 64)+ if refused, err := NewClientWithContext(context.Background(), bad); refused != nil || err == nil || logins.Load() != before {+ t.Error("context-aware construction weakened the saved TLS pin")+ }+ })+ }+ }+}diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.goindex 040a530aae..03a26ae803 100644--- a/pkg/proxmox/cluster_client.go+++ b/pkg/proxmox/cluster_client.go@@ -179,6 +179,17 @@ func sanitizeEndpointError(errMsg string) string {// endpointFingerprints is an optional map of endpoint URL -> TLS fingerprint for per-node certificate verification.// This enables TOFU (Trust On First Use) for clusters with unique self-signed certs per node.func NewClusterClient(name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string) *ClusterClient {+ cc, _ := NewClusterClientWithContext(context.Background(), name, config, endpoints, endpointFingerprints)+ return cc+}++// NewClusterClientWithContext keeps synchronous startup checks and password+// session construction within the caller's lifetime. Endpoint failures still+// produce a recoverable cluster; cancellation does not publish a partial one.+func NewClusterClientWithContext(ctx context.Context, name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string) (*ClusterClient, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }// The caller may reuse configuration storage after construction. Keep live// routing and first-use trust on the same membership snapshot registered for// guest-command coordination, including during initial health checks.@@ -211,9 +222,15 @@ func NewClusterClient(name string, config ClientConfig, endpoints []string, endp// Do a quick parallel health check on initialization (synchronous to avoid race)// This will mark unhealthy nodes but won't prevent trying them later- cc.initialHealthCheck()+ cc.initialHealthCheck(ctx)+ if err := ctx.Err(); err != nil {+ for _, client := range cc.clients {+ client.httpClient.CloseIdleConnections()+ }+ return nil, err+ }- return cc+ return cc, nil}// getEndpointFingerprint returns the TLS fingerprint to use for a specific endpoint.@@ -343,7 +360,7 @@ func (cc *ClusterClient) refreshTOFUFingerprintAndRetry(ctx context.Context, endcfg.Fingerprint = newFingerprintcfg.Timeout = timeout- retryClient, err := NewClient(cfg)+ retryClient, err := NewClientWithContext(ctx, cfg)if err != nil {return nil, err, true}@@ -356,7 +373,7 @@ func (cc *ClusterClient) refreshTOFUFingerprintAndRetry(ctx context.Context, end}// initialHealthCheck performs a quick parallel health check on all endpoints-func (cc *ClusterClient) initialHealthCheck() {+func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {// Skip initial health check if there's only one endpoint// For single-endpoint clusters (using main host for routing), assume healthyif len(cc.endpoints) == 1 {@@ -382,7 +399,7 @@ func (cc *ClusterClient) initialHealthCheck() {cfg.Fingerprint = cc.getEndpointFingerprint(ep)cfg.Timeout = 5 * time.Second- testClient, err := NewClient(cfg)+ testClient, err := NewClientWithContext(ctx, cfg)if err != nil {cc.mu.Lock()cc.nodeHealth[ep] = false@@ -396,13 +413,17 @@ func (cc *ClusterClient) initialHealthCheck() {Msg("Cluster endpoint marked unhealthy on initialization")return}+ defer testClient.httpClient.CloseIdleConnections()// Quick test with slightly longer timeout for initial check- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)- _, err = testClient.GetNodes(ctx)+ testCtx, cancel := context.WithTimeout(ctx, 5*time.Second)+ _, err = testClient.GetNodes(testCtx)cancel()- if _, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(context.Background(), ep, 5*time.Second, err); refreshed {+ if retryClient, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(ctx, ep, 5*time.Second, err); refreshed {+ if retryClient != nil {+ defer retryClient.httpClient.CloseIdleConnections()+ }err = retryErr}@@ -414,7 +435,7 @@ func (cc *ClusterClient) initialHealthCheck() {fullCfg := cc.configfullCfg.Host = epfullCfg.Fingerprint = cc.getEndpointFingerprint(ep)- fullClient, clientErr := NewClient(fullCfg)+ fullClient, clientErr := NewClientWithContext(ctx, fullCfg)cc.mu.Lock()if clientErr != nil {@@ -540,8 +561,14 @@ func (cc *ClusterClient) endpointHealthDiagnosticsLocked() []endpointHealthDiagn// member addresses are failover candidates rather than a random load-balancing// pool.func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }cc.mu.Lock()defer cc.mu.Unlock()+ if err := ctx.Err(); err != nil {+ return nil, err+ }now := time.Now()healthyEndpoints, coolingEndpoints := cc.availableEndpointsLocked(now)@@ -596,6 +623,9 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)defer cc.mu.Lock()cc.recoverUnhealthyNodes(ctx)}()+ if err := ctx.Err(); err != nil {+ return nil, err+ }now = time.Now()healthyEndpoints, coolingEndpoints = cc.availableEndpointsLocked(now)@@ -658,8 +688,11 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)testCfg := cfgtestCfg.Timeout = 3 * time.Second- testClient, err := NewClient(testCfg)+ testClient, err := NewClientWithContext(ctx, testCfg)if err != nil {+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }// Mark as unhealthycc.nodeHealth[selectedEndpoint] = falselog.Debug().@@ -669,11 +702,15 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)Msg("Failed to create client for cluster endpoint")return nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)}+ defer testClient.httpClient.CloseIdleConnections()// Connectivity test - 5 seconds to allow for TLS handshake (~3s typical)- testCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ testCtx, cancel := context.WithTimeout(ctx, 5*time.Second)testNodes, testErr := testClient.GetNodes(testCtx)cancel()+ if err := ctx.Err(); err != nil {+ return nil, err+ }if testErr != nil {// Check if this is a transient rate limit error that shouldn't mark the node unhealthy@@ -717,17 +754,20 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)Int("nodes", len(testNodes)).Msg("Cluster endpoint passed connectivity test")- // Clear any stale error from previous failures now that connectivity succeeded- delete(cc.lastError, selectedEndpoint)-// Create the actual client with full timeout- newClient, err := NewClient(cfg)+ newClient, err := NewClientWithContext(ctx, cfg)if err != nil {+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }// This shouldn't happen since we just tested itcc.nodeHealth[selectedEndpoint] = falsereturn nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)}+ // A completed probe cannot clear an error until the polling session is+ // accepted. Ending a caller is not endpoint failure or recovery evidence.+ delete(cc.lastError, selectedEndpoint)cc.clients[selectedEndpoint] = newClientclient = newClient}@@ -768,6 +808,9 @@ func (cc *ClusterClient) clearEndpointError(endpoint string) {// recoverUnhealthyNodes attempts to recover unhealthy nodesfunc (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {+ if ctx.Err() != nil {+ return+ }cc.mu.RLock()unhealthyEndpoints := make([]string, 0)throttledEndpoints := make([]string, 0)@@ -810,6 +853,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {var failureMu sync.MutexfailureReasons := make(map[string]string, len(unhealthyEndpoints))recordFailure := func(ep string, err error) {+ if ctx.Err() != nil {+ return+ }sanitized := sanitizeEndpointError(err.Error())failureMu.Lock()failureReasons[ep] = sanitized@@ -823,6 +869,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {wg.Add(1)go func(ep string) {defer wg.Done()+ if ctx.Err() != nil {+ return+ }// Update last check timecc.mu.Lock()@@ -837,7 +886,7 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {cfg.Fingerprint = cc.getEndpointFingerprint(ep)cfg.Timeout = 5 * time.Second- testClient, err := NewClient(cfg)+ testClient, err := NewClientWithContext(ctx, cfg)if err != nil {recordFailure(ep, err)log.Debug().@@ -847,16 +896,23 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {Msg("Failed to create client during recovery attempt")return}+ defer testClient.httpClient.CloseIdleConnections()// Try a simple API calltestCtx, cancel := context.WithTimeout(ctx, 5*time.Second)_, err = testClient.GetNodes(testCtx)cancel()- if _, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(ctx, ep, 5*time.Second, err); refreshed {+ if retryClient, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(ctx, ep, 5*time.Second, err); refreshed {+ if retryClient != nil {+ defer retryClient.httpClient.CloseIdleConnections()+ }err = retryErrcfg.Fingerprint = cc.getEndpointFingerprint(ep)}+ if ctx.Err() != nil {+ return+ }// Check if error is VM-specific (shouldn't prevent recovery)vmSpecificErr := err != nil && isVMSpecificError(err.Error())@@ -865,7 +921,7 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {// Store the client with original timeoutcfg.Timeout = cc.config.Timeoutcfg.Fingerprint = cc.getEndpointFingerprint(ep)- fullClient, clientErr := NewClient(cfg)+ fullClient, clientErr := NewClientWithContext(ctx, cfg)if clientErr != nil {// A short probe is not a usable polling session. Password// authentication may fail when constructing the actual client;@@ -881,6 +937,11 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {}cc.mu.Lock()+ if ctx.Err() != nil {+ cc.mu.Unlock()+ fullClient.httpClient.CloseIdleConnections()+ return+ }cc.nodeHealth[ep] = truedelete(cc.lastError, ep)cc.lastHealthCheck[ep] = time.Now()diff --git a/pkg/proxmox/cluster_client_cancellation_test.go b/pkg/proxmox/cluster_client_cancellation_test.gonew file mode 100644index 0000000000..c01b2ca999--- /dev/null+++ b/pkg/proxmox/cluster_client_cancellation_test.go@@ -0,0 +1,172 @@+package proxmox++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++// The public read/recovery paths must not manufacture a password session after+// their caller has stopped. These peers and credentials are local fixtures.+func testClusterCancelledCallerStartsNoSession(t *testing.T) {+ for _, authMode := range []string{"password", "token"} {+ t.Run(authMode, func(t *testing.T) {+ var requests atomic.Int32+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ if r.URL.Path == "/api2/json/access/ticket" {+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)+ } else {+ fmt.Fprint(w, `{"data":[]}`)+ }+ }))+ defer server.Close()+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}+ if authMode == "token" {+ cfg.Password, cfg.TokenName, cfg.TokenValue = "", "fixture@pam!pulse", "fixture-token"+ }+ cc := NewClusterClient("cancelled", cfg, []string{server.URL}, nil)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ if _, err := cc.GetNodes(ctx); !errors.Is(err, context.Canceled) {+ t.Errorf("cancelled ordinary read = %v, want context cancellation", err)+ }+ if requests.Load() != 0 || len(cc.clients) != 0 || !cc.nodeHealth[server.URL] || cc.lastError[server.URL] != "" {+ t.Errorf("cancelled caller changed session/health: requests=%d clients=%d healthy=%t error=%q", requests.Load(), len(cc.clients), cc.nodeHealth[server.URL], cc.lastError[server.URL])+ }+ })+ }+}++func testClusterSessionConstructionHonoursCallerBudget(t *testing.T) {+ for _, path := range []string{"selection", "recovery"} {+ for _, stage := range []string{"first-login", "probe", "full-login"} {+ for _, end := range []string{"cancel", "deadline"} {+ t.Run(path+"/"+stage+"/"+end, func(t *testing.T) {+ var logins, reads atomic.Int32+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var enteredOnce, releaseOnce sync.Once+ var block atomic.Bool+ block.Store(true)+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = io.Copy(io.Discard, r.Body)+ w.Header().Set("Content-Type", "application/json")+ login := r.URL.Path == "/api2/json/access/ticket"+ var ordinal int32+ if login {+ ordinal = logins.Add(1)+ } else {+ ordinal = reads.Add(1)+ if r.URL.Path != "/api2/json/nodes" || !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket-") {+ t.Errorf("non-ordinary or unauthenticated fixture read: %s", r.URL.Path)+ }+ }+ target := (stage == "first-login" && login && ordinal == 1) || (stage == "probe" && !login && ordinal == 1) || (stage == "full-login" && login && ordinal == 2)+ if block.Load() && target {+ enteredOnce.Do(func() { close(entered) })+ select {+ case <-r.Context().Done():+ close(requestEnded)+ return+ case <-release:+ }+ }+ if login {+ fmt.Fprintf(w, `{"data":{"ticket":"fixture-ticket-%d","CSRFPreventionToken":"fixture-csrf-%d"}}`, ordinal, ordinal)+ } else {+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)+ }+ }))+ defer server.Close()+ defer releaseOnce.Do(func() { close(release) })+ pin := sha256.Sum256(server.Certificate().Raw)+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}+ cc := NewClusterClient("budget", cfg, []string{server.URL}, nil)+ cc.nodeHealth[server.URL] = path == "selection"+ cc.lastError[server.URL] = "previous fixture outage"+ ctx, cancel := context.WithCancel(context.Background())+ if end == "deadline" {+ cancel()+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)+ }+ defer cancel()+ done := make(chan error, 1)+ go func() {+ if path == "recovery" {+ cc.recoverUnhealthyNodes(ctx)+ done <- nil+ } else {+ _, err := cc.GetNodes(ctx)+ done <- err+ }+ }()+ select {+ case <-entered:+ case <-time.After(5 * time.Second):+ releaseOnce.Do(func() { close(release) })+ <-done+ t.Fatal("fixture did not reach selected session phase")+ }+ if end == "cancel" {+ cancel()+ } else {+ <-ctx.Done()+ }+ var err error+ select {+ case err = <-done:+ case <-time.After(time.Second):+ t.Error("session construction outlived the ended caller by over one second")+ releaseOnce.Do(func() { close(release) })+ err = <-done // Join the old implementation too; no leaked fixture worker.+ }+ if path == "selection" && !errors.Is(err, ctx.Err()) {+ t.Errorf("selection lost caller context: %v, want %v", err, ctx.Err())+ }+ select {+ case <-requestEnded:+ case <-time.After(time.Second):+ t.Error("ended caller left its fixture request active")+ }+ if len(cc.clients) != 0 || cc.nodeHealth[server.URL] != (path == "selection") || cc.lastError[server.URL] != "previous fixture outage" || cc.getEndpointFingerprint(server.URL) != cfg.Fingerprint {+ t.Errorf("cancelled construction published a session/recovery or changed prior trust/error: clients=%d healthy=%t error=%q", len(cc.clients), cc.nodeHealth[server.URL], cc.lastError[server.URL])+ }+ wantLogins, wantReads := int32(1), int32(0)+ if stage == "probe" || stage == "full-login" {+ wantReads = 1+ if stage == "full-login" {+ wantLogins = 2+ }+ }+ if logins.Load() != wantLogins || reads.Load() != wantReads {+ t.Errorf("ended construction dispatched another phase: logins=%d reads=%d", logins.Load(), reads.Load())+ }+ // A later ordinary caller can recover under the unchanged pin.+ block.Store(false)+ releaseOnce.Do(func() { close(release) })+ cc.nodeHealth[server.URL] = false+ delete(cc.lastHealthCheck, server.URL) // Only advance this fixture's eligibility.+ cc.recoverUnhealthyNodes(context.Background())+ if client := cc.clients[server.URL]; client == nil || client.config.Timeout != cfg.Timeout || !cc.nodeHealth[server.URL] || cc.lastError[server.URL] != "" {+ t.Fatal("ordinary later recovery did not publish its accepted full-timeout session")+ }+ defer cc.clients[server.URL].httpClient.CloseIdleConnections()+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 || nodes[0].Node != "member" {+ t.Fatalf("later recovered session cannot serve ordinary reads: %+v %v", nodes, err)+ }+ })+ }+ }+ }+}diff --git a/pkg/proxmox/host_diagnostics_regression_test.go b/pkg/proxmox/host_diagnostics_regression_test.goindex 947a00e119..2de7bebec9 100644--- a/pkg/proxmox/host_diagnostics_regression_test.go+++ b/pkg/proxmox/host_diagnostics_regression_test.go@@ -157,3 +157,16 @@ func TestClusterDiagnosticHealthSnapshotKeepsEveryEndpoint(t *testing.T) {t.Error("display snapshot lost endpoint status observations")}}++// Initial authentication must obey the same caller lifetime as ordinary reads.+func TestClientConstructionHonoursContext(t *testing.T) {+ testClientConstructionHonoursContext(t)+}++func TestClusterCancelledCallerStartsNoSession(t *testing.T) {+ testClusterCancelledCallerStartsNoSession(t)+}++func TestClusterSessionConstructionHonoursCallerBudget(t *testing.T) {+ testClusterSessionConstructionHonoursCallerBudget(t)+}