Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep Proxmox recovery authentication within its caller budget

pulse-triage[bot]13984070ff03
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index b0dccf390d..8f3d0725a0 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,37 @@
## Purpose
+### Proxmox recovery sessions honour the owning caller
+
+PVE/PBS context-aware constructors bind initial password login, including the
+existing JSON/form fallback, to the initiating context. Cluster selection,
+first-use pinned retry, synchronous construction and unhealthy-member recovery
+carry that context through both the short probe and the retained polling
+session. The six-second background recovery budget also bounds construction;
+ending a caller is neither a credential refusal nor successful recovery.
+Cancelled selection/recovery preserves prior errors and does not install a
+partial or nil client. Startup retry exits without publishing a new health
+outcome or advancing another provider after its monitor stops. Discarded probe
+transports close their idle connections.
+
+The compatibility constructors retain their background-context behaviour.
+Accepted clients retain their configured full timeout, and subsequent ordinary
+reads use their own contexts, not the ended construction context. TLS pins,
+CA verification, redirects, first-use admission, auth encodings, retry cadence,
+provider permissions, guest/backup fences and History are unchanged.
+
+`TestClientConstructionHonoursContext` in both client packages covers ended
+password/token construction, in-flight JSON/form cancellation/deadlines,
+independent later reads and wrong-pin refusal. Cluster caller-budget controls
+cover selection/recovery at first login, probe and full login, then trusted
+ordinary recovery. Startup cancellation covers PVE, PBS and multi-endpoint
+cluster construction. The connected PBS first-poll control emits the actual
+broadcast for desktop/phone consumption: cancellation cannot turn an offline
+server healthy, and accepted construction alone still cannot do so. These are
+secret-free local fixtures, not installed recovery, native guest safety or an
+attribution of any reporter's failure.
+
+
### Cluster recovery requires a usable polling session
A short endpoint probe cannot establish successful recovery until the client
diff --git a/internal/monitoring/mock_mode_config_test.go b/internal/monitoring/mock_mode_config_test.go
index b54ac015ca..f958aca929 100644
--- a/internal/monitoring/mock_mode_config_test.go
+++ b/internal/monitoring/mock_mode_config_test.go
@@ -1,6 +1,7 @@
package monitoring
import (
+ "context"
"testing"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
@@ -50,7 +51,7 @@ func TestNew_MockModeClientInitializationRespectsKeepRealPollingSetting(t *testi
orig := newProxmoxClientFunc
t.Cleanup(func() { newProxmoxClientFunc = orig })
called := false
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
called = true
return &mockPVEClient{}, nil
}
diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go
index 625f7cd32a..902e1d4db1 100644
--- a/internal/monitoring/monitor.go
+++ b/internal/monitoring/monitor.go
@@ -78,8 +78,8 @@ func alertManagerRestoreOptions(mockEnabled bool) []alerts.ManagerOption {
// newProxmoxClientFunc is a variable that holds the function to create a new Proxmox client.
// It is used to allow mocking the client creation in tests.
-var newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
- return proxmox.NewClient(cfg)
+var newProxmoxClientFunc = func(ctx context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ return proxmox.NewClientWithContext(ctx, cfg)
}
// PVEClientInterface defines the interface for PVE clients (both regular and cluster)
diff --git a/internal/monitoring/monitor_client_cancellation_test.go b/internal/monitoring/monitor_client_cancellation_test.go
new file mode 100644
index 0000000000..d61296b50d
--- /dev/null
+++ b/internal/monitoring/monitor_client_cancellation_test.go
@@ -0,0 +1,224 @@
+package monitoring
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/models"
+ "github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
+)
+
+func testStartupClientConstructionHonoursCancellation(t *testing.T) {
+ for _, provider := range []string{"pve", "pbs", "cluster"} {
+ t.Run(provider, func(t *testing.T) {
+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once, endedOnce, releaseOnce sync.Once
+ var requests atomic.Int32
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ requests.Add(1)
+ if !strings.HasSuffix(r.URL.Path, "/api2/json/access/ticket") {
+ t.Errorf("ended startup construction advanced to another request: %s", r.URL.Path)
+ }
+ once.Do(func() { close(entered) })
+ select {
+ case <-r.Context().Done():
+ endedOnce.Do(func() { close(requestEnded) })
+ return
+ case <-release:
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ }))
+ defer server.Close()
+ defer releaseOnce.Do(func() { close(release) })
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := &config.Config{ConnectionTimeout: 8 * time.Second}
+ if provider == "pbs" {
+ cfg.PBSInstances = []config.PBSInstance{{Name: "cancelled-pbs", Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:])}}
+ } else {
+ instance := config.PVEInstance{Name: "cancelled-pve", Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:])}
+ if provider == "cluster" {
+ // Two distinct base paths on the same trusted local TLS origin
+ // exercise synchronous initial health/session checks too.
+ instance.IsCluster = true
+ instance.ClusterEndpoints = []config.ClusterEndpoint{{Host: server.URL, Fingerprint: instance.Fingerprint}}
+ // Canonical authority deduplication would otherwise skip the check.
+ instance.ClusterEndpoints[0].Host += "/member"
+ }
+ cfg.PVEInstances = []config.PVEInstance{instance}
+ }
+ monitor := &Monitor{config: cfg, state: models.NewState(), pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client)}
+ originalDelays := connRetryDelays
+ connRetryDelays = []time.Duration{time.Millisecond}
+ defer func() { connRetryDelays = originalDelays }()
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan struct{})
+ go func() { monitor.retryFailedConnections(ctx); close(done) }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ releaseOnce.Do(func() { close(release) })
+ cancel()
+ <-done
+ t.Fatal("startup constructor did not reach local password login")
+ }
+ cancel()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Error("retry shutdown waited for detached constructor authentication")
+ releaseOnce.Do(func() { close(release) })
+ <-done
+ }
+ // Both cluster constructor requests may have started before cancellation.
+ if provider != "cluster" {
+ select {
+ case <-requestEnded:
+ case <-time.After(time.Second):
+ t.Error("cancelled constructor left the password request active")
+ }
+ }
+ if len(monitor.pveClients) != 0 || len(monitor.pbsClients) != 0 || len(monitor.state.GetSnapshot().ConnectionHealth) != 0 {
+ t.Error("cancelled startup construction published a polling client or health outcome")
+ }
+ if provider != "cluster" && requests.Load() != 1 {
+ t.Errorf("shutdown sent more than the admitted fixture login: %d", requests.Load())
+ }
+ })
+ }
+}
+
+// Real pinned PBS construction, ordinary retry admission, first poll and the
+// production broadcast feed the consuming browser proof. This is not an
+// installed/reporter recovery or a reason to restart monitoring on a workload.
+func testPBSStartupCancellationKeepsFirstPollBoundary(t *testing.T) {
+ var logins, reads atomic.Int32
+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var releaseOnce sync.Once
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path == "/api2/json/access/ticket" {
+ switch logins.Add(1) {
+ case 1:
+ w.WriteHeader(http.StatusServiceUnavailable)
+ fmt.Fprint(w, `{"message":"fixture starting"}`)
+ return
+ case 2:
+ close(entered)
+ select {
+ case <-r.Context().Done():
+ close(requestEnded)
+ return
+ case <-release:
+ }
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ return
+ }
+ reads.Add(1)
+ if r.Method != http.MethodGet || !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {
+ t.Errorf("non-ordinary or unauthenticated PBS fixture read: %s %s", r.Method, r.URL.Path)
+ }
+ switch r.URL.Path {
+ case "/api2/json/version":
+ fmt.Fprint(w, `{"data":{"version":"3.4.2"}}`)
+ case "/api2/json/nodes/localhost/status":
+ fmt.Fprint(w, `{"data":{"cpu":0.15,"memory":{"used":512,"total":1024},"uptime":120}}`)
+ case "/api2/json/admin/datastore":
+ fmt.Fprint(w, `{"data":[{"store":"backups","total":1000,"used":250,"avail":750}]}`)
+ case "/api2/json/admin/datastore/backups/status":
+ fmt.Fprint(w, `{"data":{"status":"available","total":1000,"used":250,"avail":750}}`)
+ default:
+ fmt.Fprint(w, `{"data":[]}`)
+ }
+ }))
+ defer server.Close()
+ defer releaseOnce.Do(func() { close(release) })
+ pin := sha256.Sum256(server.Certificate().Raw)
+ instance := config.PBSInstance{Name: "cancellable-pbs", Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), MonitorDatastores: true}
+ monitor := newPBSHealthAuthorityMonitor([]config.PBSInstance{instance})
+ project := func(phase string) {
+ snapshot, scope := monitor.currentStateWithScope()
+ frontend := monitor.buildBroadcastFrontendStateFromSnapshot(snapshot, scope)
+ wire, err := json.Marshal(struct {
+ Phase string `json:"phase"`
+ State models.StateFrontend `json:"state"`
+ }{phase, frontend})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("PBS_CANCELLATION_PROJECTION_JSON=%s", wire)
+ }
+ monitor.initPBSClients(monitor.config)
+ assertPBSConnectionProjection(t, monitor, instance.Name, false, "offline")
+ project("unavailable")
+ originalDelays := connRetryDelays
+ connRetryDelays = []time.Duration{time.Millisecond, time.Millisecond}
+ defer func() { connRetryDelays = originalDelays }()
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ done := make(chan struct{})
+ go func() { monitor.retryFailedConnections(ctx); close(done) }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ releaseOnce.Do(func() { close(release) })
+ cancel()
+ <-done
+ t.Fatal("fixture did not reach the reconnect password request")
+ }
+ cancel()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Error("ended retry caller still owns a detached password request")
+ releaseOnce.Do(func() { close(release) })
+ <-done
+ }
+ select {
+ case <-requestEnded:
+ case <-time.After(time.Second):
+ t.Error("ended retry caller did not cancel its HTTP request")
+ }
+ if logins.Load() != 2 || reads.Load() != 0 || len(monitor.pbsClients) != 0 || len(newPBSPollProvider().ListInstances(monitor)) != 0 {
+ t.Fatalf("cancelled construction published a session or advanced polling: logins=%d reads=%d clients=%d", logins.Load(), reads.Load(), len(monitor.pbsClients))
+ }
+ assertPBSConnectionProjection(t, monitor, instance.Name, false, "offline")
+ project("cancelled")
+
+ releaseOnce.Do(func() { close(release) })
+ laterCtx, laterCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer laterCancel()
+ monitor.retryFailedConnections(laterCtx)
+ if logins.Load() != 3 || len(newPBSPollProvider().ListInstances(monitor)) != 1 {
+ t.Fatal("later admitted retry did not retain its accepted session exactly once")
+ }
+ assertPBSConnectionProjection(t, monitor, instance.Name, false, "offline")
+ project("awaiting-poll")
+ task, err := newPBSPollProvider().BuildPollTask(monitor, instance.Name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ newRealExecutor(monitor).Execute(laterCtx, task)
+ assertPBSConnectionProjection(t, monitor, instance.Name, true, "online")
+ actual := pbsInstanceByName(t, monitor.state.GetSnapshot(), instance.Name)
+ if actual.CPU != 0.15 || actual.Memory != 50 || len(actual.Datastores) != 1 || actual.Datastores[0].Usage != 25 || logins.Load() != 3 || !reflect.DeepEqual(instance, monitor.config.PBSInstances[0]) {
+ t.Fatalf("ordinary first poll lost readings, identity or saved trust: %+v logins=%d", actual, logins.Load())
+ }
+ project("recovered")
+}
diff --git a/internal/monitoring/monitor_client_init.go b/internal/monitoring/monitor_client_init.go
index 7a0b318dd0..05754727d3 100644
--- a/internal/monitoring/monitor_client_init.go
+++ b/internal/monitoring/monitor_client_init.go
@@ -1,6 +1,7 @@
package monitoring
import (
+ "context"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
@@ -58,7 +59,7 @@ func (m *Monitor) initPVEClients(cfg *config.Config) {
// Create regular client
clientConfig := config.CreateProxmoxConfig(&pve)
clientConfig.Timeout = cfg.ConnectionTimeout
- client, err := newProxmoxClientFunc(clientConfig)
+ client, err := newProxmoxClientFunc(context.Background(), clientConfig)
if err != nil {
monErr := errors.WrapConnectionError("create_pve_client", pve.Name, err)
log.Error().
diff --git a/internal/monitoring/monitor_client_reconnect.go b/internal/monitoring/monitor_client_reconnect.go
index eb0296137c..7734b21dfb 100644
--- a/internal/monitoring/monitor_client_reconnect.go
+++ b/internal/monitoring/monitor_client_reconnect.go
@@ -103,13 +103,23 @@ func (m *Monitor) retryFailedConnections(ctx context.Context) {
// Try to recreate PVE clients
for _, pve := range missingPVE {
+ if ctx.Err() != nil {
+ return
+ }
if pve.IsCluster && len(pve.ClusterEndpoints) > 0 {
// Create cluster client
endpoints, endpointFingerprints := m.buildClusterEndpointsForReconnect(pve)
clientConfig := config.CreateProxmoxConfig(&pve)
clientConfig.Timeout = connectionTimeout
- clusterClient := proxmox.NewClusterClient(pve.Name, clientConfig, endpoints, endpointFingerprints)
+ clusterClient, err := proxmox.NewClusterClientWithContext(ctx, pve.Name, clientConfig, endpoints, endpointFingerprints)
+ if ctx.Err() != nil {
+ return
+ }
+ if err != nil {
+ log.Warn().Err(err).Str("instance", pve.Name).Msg("Failed to reconnect cluster client, will retry")
+ continue
+ }
m.replacePVEClient(pve.Name, clusterClient)
m.setProviderConnectionHealth(InstanceTypePVE, pve.Name, true)
@@ -124,7 +134,10 @@ func (m *Monitor) retryFailedConnections(ctx context.Context) {
// Create regular client
clientConfig := config.CreateProxmoxConfig(&pve)
clientConfig.Timeout = connectionTimeout
- client, err := newProxmoxClientFunc(clientConfig)
+ client, err := newProxmoxClientFunc(ctx, clientConfig)
+ if ctx.Err() != nil {
+ return
+ }
if err != nil {
log.Warn().
Err(err).
@@ -143,9 +156,15 @@ func (m *Monitor) retryFailedConnections(ctx context.Context) {
// Try to recreate PBS clients
for _, pbsInst := range missingPBS {
+ if ctx.Err() != nil {
+ return
+ }
clientConfig := config.CreatePBSConfig(&pbsInst)
clientConfig.Timeout = 60 * time.Second
- client, err := pbs.NewClient(clientConfig)
+ client, err := pbs.NewClientWithContext(ctx, clientConfig)
+ if ctx.Err() != nil {
+ return
+ }
if err != nil {
monErr := monitorerrors.WrapConnectionError("recreate_pbs_client", pbsInst.Name, err)
log.Warn().
diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.go
index 1d675570ed..d151305fc4 100644
--- a/internal/monitoring/monitor_extra_coverage_test.go
+++ b/internal/monitoring/monitor_extra_coverage_test.go
@@ -2416,7 +2416,7 @@ func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {
cause := fmt.Errorf("dial tcp 127.0.0.1:8006: connect: connection refused")
// 1. Success case
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
if strings.Contains(cfg.Host, "8006") {
return nil, fmt.Errorf("should not be called with 8006 in fallback")
}
@@ -2439,7 +2439,7 @@ func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {
_ = nodes // ignore
// 2. Failure to create client
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
return nil, fmt.Errorf("create failed")
}
_, _, err = m.retryPVEPortFallback(context.Background(), "pve1", instanceCfg, currentClient, cause)
@@ -2448,7 +2448,7 @@ func TestMonitor_RetryPVEPortFallback_Detailed_Extra(t *testing.T) {
}
// 3. Failure to get nodes
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
// Return a client that fails GetNodes
return &mockPVEClientFailNodes{}, nil
}
diff --git a/internal/monitoring/monitor_pve.go b/internal/monitoring/monitor_pve.go
index 3cac9a7b7c..5815241c1e 100644
--- a/internal/monitoring/monitor_pve.go
+++ b/internal/monitoring/monitor_pve.go
@@ -412,7 +412,7 @@ func (m *Monitor) retryPVEPortFallback(ctx context.Context, instanceName string,
m.mu.RUnlock()
}
- fallbackClient, err := newProxmoxClientFunc(clientCfg)
+ fallbackClient, err := newProxmoxClientFunc(ctx, clientCfg)
if err != nil {
return nil, currentClient, cause
}
diff --git a/internal/monitoring/monitor_pve_cluster_refresh_test.go b/internal/monitoring/monitor_pve_cluster_refresh_test.go
index 4305732a3f..b2edb65796 100644
--- a/internal/monitoring/monitor_pve_cluster_refresh_test.go
+++ b/internal/monitoring/monitor_pve_cluster_refresh_test.go
@@ -413,3 +413,12 @@ func TestClusterEndpointIdentityChanged(t *testing.T) {
t.Fatal("an added node must count as an identity change")
}
}
+
+// Retry admission and first-poll health must not outlive the owning monitor.
+func TestStartupClientConstructionHonoursCancellation(t *testing.T) {
+ testStartupClientConstructionHonoursCancellation(t)
+}
+
+func TestPBSStartupCancellationKeepsFirstPollBoundary(t *testing.T) {
+ testPBSStartupCancellationKeepsFirstPollBoundary(t)
+}
diff --git a/internal/monitoring/monitor_reconnect_test.go b/internal/monitoring/monitor_reconnect_test.go
index 4e2c2e9973..0f36ba8b21 100644
--- a/internal/monitoring/monitor_reconnect_test.go
+++ b/internal/monitoring/monitor_reconnect_test.go
@@ -46,7 +46,7 @@ func TestMonitor_RetryFailedConnections_Detailed_Extra(t *testing.T) {
// 1. Test Successful Reconnection
called := false
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
called = true
if !strings.Contains(cfg.Host, "pve1") {
return nil, fmt.Errorf("unexpected host: %s", cfg.Host)
@@ -74,7 +74,7 @@ func TestMonitor_RetryFailedConnections_Detailed_Extra(t *testing.T) {
{Name: "pve2", Host: "https://pve2:8006"},
}
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
return nil, fmt.Errorf("connection failed")
}
diff --git a/internal/monitoring/monitor_startup_recovery_test.go b/internal/monitoring/monitor_startup_recovery_test.go
index 04e21aa74a..2ee4d5a588 100644
--- a/internal/monitoring/monitor_startup_recovery_test.go
+++ b/internal/monitoring/monitor_startup_recovery_test.go
@@ -32,7 +32,7 @@ func TestConnectionRetrySurvivesLongStartupOutage(t *testing.T) {
m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{instance}, ConnectionTimeout: 3 * time.Second}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}
started := time.Now()
var attempts []time.Duration
- newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(_ context.Context, cfg proxmox.ClientConfig) (PVEClientInterface, error) {
want := config.CreateProxmoxConfig(&instance)
want.Timeout = 3 * time.Second
if !reflect.DeepEqual(cfg, want) {
@@ -66,7 +66,7 @@ func TestConnectionRetryLongOutageStopsOnCancellation(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{{Name: "still-unavailable", Host: "https://pve.local:8006"}}}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}
calls := 0
- newProxmoxClientFunc = func(proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(context.Context, proxmox.ClientConfig) (PVEClientInterface, error) {
calls++
return nil, fmt.Errorf("fixture provider still unavailable")
}
@@ -85,7 +85,7 @@ func TestConnectionRetryDoesNotResumePausedConnections(t *testing.T) {
defer func() { newProxmoxClientFunc = original }()
synctest.Test(t, func(t *testing.T) {
m := &Monitor{config: &config.Config{PVEInstances: []config.PVEInstance{{Name: "paused-pve", Disabled: true}}, PBSInstances: []config.PBSInstance{{Name: "paused-pbs", Disabled: true, Host: "://invalid"}}}, pveClients: make(map[string]PVEClientInterface), pbsClients: make(map[string]*pbs.Client), state: models.NewState()}
- newProxmoxClientFunc = func(proxmox.ClientConfig) (PVEClientInterface, error) {
+ newProxmoxClientFunc = func(context.Context, proxmox.ClientConfig) (PVEClientInterface, error) {
t.Fatal("retry attempted a paused connection")
return nil, nil
}
diff --git a/pkg/pbs/client.go b/pkg/pbs/client.go
index 4473091f8f..a6ab8d2fcc 100644
--- a/pkg/pbs/client.go
+++ b/pkg/pbs/client.go
@@ -117,6 +117,15 @@ type auth struct {
// NewClient creates a new PBS API client
func NewClient(cfg ClientConfig) (*Client, error) {
+ return NewClientWithContext(context.Background(), cfg)
+}
+
+// NewClientWithContext binds initial password authentication to its caller.
+// Later polling remains independent of this construction context.
+func NewClientWithContext(ctx context.Context, cfg ClientConfig) (*Client, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
// Normalize host URL - ensure it has a protocol
defaultedScheme := !strings.HasPrefix(cfg.Host, "http://") && !strings.HasPrefix(cfg.Host, "https://")
if defaultedScheme {
@@ -205,10 +214,15 @@ func NewClient(cfg ClientConfig) (*Client, error) {
// Authenticate if using password
if cfg.Password != "" && cfg.TokenName == "" {
- if err := client.authenticate(context.Background()); err != nil {
+ if err := client.authenticate(ctx); err != nil {
+ httpClient.CloseIdleConnections()
return nil, fmt.Errorf("authentication failed: %w", err)
}
}
+ if err := ctx.Err(); err != nil {
+ httpClient.CloseIdleConnections()
+ return nil, err
+ }
return client, nil
}
diff --git a/pkg/pbs/client_construction_context_test.go b/pkg/pbs/client_construction_context_test.go
new file mode 100644
index 0000000000..237083f61b
--- /dev/null
+++ b/pkg/pbs/client_construction_context_test.go
@@ -0,0 +1,156 @@
+package pbs
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func testClientConstructionHonoursContext(t *testing.T) {
+ for _, mode := range []string{"password", "token"} {
+ t.Run("already-ended/"+mode, func(t *testing.T) {
+ var requests atomic.Int32
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ http.Error(w, "unexpected fixture request", http.StatusForbidden)
+ }))
+ defer server.Close()
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}
+ if mode == "token" {
+ cfg.Password, cfg.TokenName, cfg.TokenValue = "", "fixture@pbs!pulse", "fixture-token"
+ }
+ for _, end := range []string{"cancel", "deadline"} {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if end == "deadline" {
+ ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
+ defer cancel()
+ }
+ client, err := NewClientWithContext(ctx, cfg)
+ if client != nil || !errors.Is(err, ctx.Err()) || requests.Load() != 0 {
+ t.Errorf("ended %s constructor: client=%t error=%v requests=%d", end, client != nil, err, requests.Load())
+ }
+ }
+ })
+ }
+ for _, encoding := range []string{"json", "form"} {
+ for _, end := range []string{"cancel", "deadline"} {
+ t.Run(encoding+"/"+end, func(t *testing.T) {
+ var logins, reads atomic.Int32
+ var blocking atomic.Bool
+ blocking.Store(true)
+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var releaseOnce sync.Once
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path != "/api2/json/access/ticket" {
+ reads.Add(1)
+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {
+ t.Error("later ordinary read lost its session")
+ }
+ fmt.Fprint(w, `{"data":{"version":"3.4.2"}}`)
+ return
+ }
+ logins.Add(1)
+ if encoding == "form" && r.Header.Get("Content-Type") == "application/json" {
+ w.WriteHeader(http.StatusUnsupportedMediaType)
+ fmt.Fprint(w, `{"message":"fixture requires form"}`)
+ return
+ }
+ if blocking.Load() {
+ close(entered)
+ select {
+ case <-r.Context().Done():
+ close(requestEnded)
+ return
+ case <-release:
+ }
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ }))
+ defer server.Close()
+ defer releaseOnce.Do(func() { close(release) })
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pbs", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}
+ ctx, cancel := context.WithCancel(context.Background())
+ if end == "deadline" {
+ cancel()
+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+ }
+ defer cancel()
+ type result struct {
+ client *Client
+ err error
+ }
+ done := make(chan result, 1)
+ go func() { c, err := NewClientWithContext(ctx, cfg); done <- result{c, err} }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ releaseOnce.Do(func() { close(release) })
+ <-done
+ t.Fatal("fixture never reached selected login encoding")
+ }
+ if end == "cancel" {
+ cancel()
+ } else {
+ <-ctx.Done()
+ }
+ var got result
+ select {
+ case got = <-done:
+ case <-time.After(time.Second):
+ t.Error("constructor exceeded ended caller budget")
+ releaseOnce.Do(func() { close(release) })
+ got = <-done
+ }
+ if got.client != nil || !errors.Is(got.err, ctx.Err()) {
+ t.Errorf("cancelled construction returned session or lost context: client=%t error=%v", got.client != nil, got.err)
+ }
+ select {
+ case <-requestEnded:
+ case <-time.After(time.Second):
+ t.Error("cancelled password request remained active")
+ }
+ want := int32(1)
+ if encoding == "form" {
+ want = 2
+ }
+ if logins.Load() != want || reads.Load() != 0 {
+ t.Errorf("cancellation replayed or advanced login: logins=%d reads=%d", logins.Load(), reads.Load())
+ }
+ blocking.Store(false)
+ releaseOnce.Do(func() { close(release) })
+ laterCtx, laterCancel := context.WithCancel(context.Background())
+ client, err := NewClientWithContext(laterCtx, cfg)
+ if err != nil {
+ laterCancel()
+ t.Fatal(err)
+ }
+ defer client.httpClient.CloseIdleConnections()
+ laterCancel() // Cancelling construction later must not poison ordinary reads.
+ if _, err := client.GetVersion(context.Background()); err != nil || reads.Load() != 1 || client.config.Timeout != cfg.Timeout {
+ t.Errorf("retained client lost full timeout or ordinary independent reads: %v reads=%d", err, reads.Load())
+ }
+ before := logins.Load()
+ bad := cfg
+ bad.Fingerprint = strings.Repeat("0", 64)
+ if refused, err := NewClientWithContext(context.Background(), bad); refused != nil || err == nil || logins.Load() != before {
+ t.Error("context-aware construction weakened the saved TLS pin")
+ }
+ })
+ }
+ }
+}
diff --git a/pkg/pbs/host_diagnostics_regression_test.go b/pkg/pbs/host_diagnostics_regression_test.go
index a57b0a71b7..6c1c0060e4 100644
--- a/pkg/pbs/host_diagnostics_regression_test.go
+++ b/pkg/pbs/host_diagnostics_regression_test.go
@@ -78,3 +78,8 @@ func TestClientAcceptedHostDiagnosticsRemainUseful(t *testing.T) {
}
}
}
+
+// Initial authentication must obey the same caller lifetime as ordinary reads.
+func TestClientConstructionHonoursContext(t *testing.T) {
+ testClientConstructionHonoursContext(t)
+}
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go
index f28c0ac5a6..3b8597adec 100644
--- a/pkg/proxmox/client.go
+++ b/pkg/proxmox/client.go
@@ -317,6 +317,15 @@ type auth struct {
// NewClient creates a new Proxmox VE API client
func NewClient(cfg ClientConfig) (*Client, error) {
+ return NewClientWithContext(context.Background(), cfg)
+}
+
+// NewClientWithContext binds initial password authentication to its caller.
+// The retained client's later reads keep their own contexts and full timeout.
+func NewClientWithContext(ctx context.Context, cfg ClientConfig) (*Client, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
var user, realm string
// For token authentication, we don't need user@realm format
@@ -410,10 +419,15 @@ func NewClient(cfg ClientConfig) (*Client, error) {
// Authenticate if using password
if cfg.Password != "" && cfg.TokenName == "" {
- if err := client.authenticate(context.Background()); err != nil {
+ if err := client.authenticate(ctx); err != nil {
+ httpClient.CloseIdleConnections()
return nil, fmt.Errorf("authentication failed: %w", err)
}
}
+ if err := ctx.Err(); err != nil {
+ httpClient.CloseIdleConnections()
+ return nil, err
+ }
return client, nil
}
diff --git a/pkg/proxmox/client_construction_context_test.go b/pkg/proxmox/client_construction_context_test.go
new file mode 100644
index 0000000000..f8e95ececf
--- /dev/null
+++ b/pkg/proxmox/client_construction_context_test.go
@@ -0,0 +1,156 @@
+package proxmox
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func testClientConstructionHonoursContext(t *testing.T) {
+ for _, mode := range []string{"password", "token"} {
+ t.Run("already-ended/"+mode, func(t *testing.T) {
+ var requests atomic.Int32
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ http.Error(w, "unexpected fixture request", http.StatusForbidden)
+ }))
+ defer server.Close()
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}
+ if mode == "token" {
+ cfg.Password, cfg.TokenName, cfg.TokenValue = "", "fixture@pam!pulse", "fixture-token"
+ }
+ for _, end := range []string{"cancel", "deadline"} {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if end == "deadline" {
+ ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
+ defer cancel()
+ }
+ client, err := NewClientWithContext(ctx, cfg)
+ if client != nil || !errors.Is(err, ctx.Err()) || requests.Load() != 0 {
+ t.Errorf("ended %s constructor: client=%t error=%v requests=%d", end, client != nil, err, requests.Load())
+ }
+ }
+ })
+ }
+ for _, encoding := range []string{"json", "form"} {
+ for _, end := range []string{"cancel", "deadline"} {
+ t.Run(encoding+"/"+end, func(t *testing.T) {
+ var logins, reads atomic.Int32
+ var blocking atomic.Bool
+ blocking.Store(true)
+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var releaseOnce sync.Once
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ w.Header().Set("Content-Type", "application/json")
+ if r.URL.Path != "/api2/json/access/ticket" {
+ reads.Add(1)
+ if !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket") {
+ t.Error("later ordinary read lost its session")
+ }
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)
+ return
+ }
+ logins.Add(1)
+ if encoding == "form" && r.Header.Get("Content-Type") == "application/json" {
+ w.WriteHeader(http.StatusUnsupportedMediaType)
+ fmt.Fprint(w, `{"message":"fixture requires form"}`)
+ return
+ }
+ if blocking.Load() {
+ close(entered)
+ select {
+ case <-r.Context().Done():
+ close(requestEnded)
+ return
+ case <-release:
+ }
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ }))
+ defer server.Close()
+ defer releaseOnce.Do(func() { close(release) })
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}
+ ctx, cancel := context.WithCancel(context.Background())
+ if end == "deadline" {
+ cancel()
+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+ }
+ defer cancel()
+ type result struct {
+ client *Client
+ err error
+ }
+ done := make(chan result, 1)
+ go func() { c, err := NewClientWithContext(ctx, cfg); done <- result{c, err} }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ releaseOnce.Do(func() { close(release) })
+ <-done
+ t.Fatal("fixture never reached selected login encoding")
+ }
+ if end == "cancel" {
+ cancel()
+ } else {
+ <-ctx.Done()
+ }
+ var got result
+ select {
+ case got = <-done:
+ case <-time.After(time.Second):
+ t.Error("constructor exceeded ended caller budget")
+ releaseOnce.Do(func() { close(release) })
+ got = <-done
+ }
+ if got.client != nil || !errors.Is(got.err, ctx.Err()) {
+ t.Errorf("cancelled construction returned session or lost context: client=%t error=%v", got.client != nil, got.err)
+ }
+ select {
+ case <-requestEnded:
+ case <-time.After(time.Second):
+ t.Error("cancelled password request remained active")
+ }
+ want := int32(1)
+ if encoding == "form" {
+ want = 2
+ }
+ if logins.Load() != want || reads.Load() != 0 {
+ t.Errorf("cancellation replayed or advanced login: logins=%d reads=%d", logins.Load(), reads.Load())
+ }
+ blocking.Store(false)
+ releaseOnce.Do(func() { close(release) })
+ laterCtx, laterCancel := context.WithCancel(context.Background())
+ client, err := NewClientWithContext(laterCtx, cfg)
+ if err != nil {
+ laterCancel()
+ t.Fatal(err)
+ }
+ defer client.httpClient.CloseIdleConnections()
+ laterCancel() // Cancelling construction later must not poison ordinary reads.
+ if _, err := client.GetNodes(context.Background()); err != nil || reads.Load() != 1 || client.config.Timeout != cfg.Timeout {
+ t.Errorf("retained client lost full timeout or ordinary independent reads: %v reads=%d", err, reads.Load())
+ }
+ before := logins.Load()
+ bad := cfg
+ bad.Fingerprint = strings.Repeat("0", 64)
+ if refused, err := NewClientWithContext(context.Background(), bad); refused != nil || err == nil || logins.Load() != before {
+ t.Error("context-aware construction weakened the saved TLS pin")
+ }
+ })
+ }
+ }
+}
diff --git a/pkg/proxmox/cluster_client.go b/pkg/proxmox/cluster_client.go
index 040a530aae..03a26ae803 100644
--- a/pkg/proxmox/cluster_client.go
+++ b/pkg/proxmox/cluster_client.go
@@ -179,6 +179,17 @@ func sanitizeEndpointError(errMsg string) string {
// endpointFingerprints is an optional map of endpoint URL -> TLS fingerprint for per-node certificate verification.
// This enables TOFU (Trust On First Use) for clusters with unique self-signed certs per node.
func NewClusterClient(name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string) *ClusterClient {
+ cc, _ := NewClusterClientWithContext(context.Background(), name, config, endpoints, endpointFingerprints)
+ return cc
+}
+
+// NewClusterClientWithContext keeps synchronous startup checks and password
+// session construction within the caller's lifetime. Endpoint failures still
+// produce a recoverable cluster; cancellation does not publish a partial one.
+func NewClusterClientWithContext(ctx context.Context, name string, config ClientConfig, endpoints []string, endpointFingerprints map[string]string) (*ClusterClient, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
// The caller may reuse configuration storage after construction. Keep live
// routing and first-use trust on the same membership snapshot registered for
// guest-command coordination, including during initial health checks.
@@ -211,9 +222,15 @@ func NewClusterClient(name string, config ClientConfig, endpoints []string, endp
// Do a quick parallel health check on initialization (synchronous to avoid race)
// This will mark unhealthy nodes but won't prevent trying them later
- cc.initialHealthCheck()
+ cc.initialHealthCheck(ctx)
+ if err := ctx.Err(); err != nil {
+ for _, client := range cc.clients {
+ client.httpClient.CloseIdleConnections()
+ }
+ return nil, err
+ }
- return cc
+ return cc, nil
}
// getEndpointFingerprint returns the TLS fingerprint to use for a specific endpoint.
@@ -343,7 +360,7 @@ func (cc *ClusterClient) refreshTOFUFingerprintAndRetry(ctx context.Context, end
cfg.Fingerprint = newFingerprint
cfg.Timeout = timeout
- retryClient, err := NewClient(cfg)
+ retryClient, err := NewClientWithContext(ctx, cfg)
if err != nil {
return nil, err, true
}
@@ -356,7 +373,7 @@ func (cc *ClusterClient) refreshTOFUFingerprintAndRetry(ctx context.Context, end
}
// initialHealthCheck performs a quick parallel health check on all endpoints
-func (cc *ClusterClient) initialHealthCheck() {
+func (cc *ClusterClient) initialHealthCheck(ctx context.Context) {
// Skip initial health check if there's only one endpoint
// For single-endpoint clusters (using main host for routing), assume healthy
if len(cc.endpoints) == 1 {
@@ -382,7 +399,7 @@ func (cc *ClusterClient) initialHealthCheck() {
cfg.Fingerprint = cc.getEndpointFingerprint(ep)
cfg.Timeout = 5 * time.Second
- testClient, err := NewClient(cfg)
+ testClient, err := NewClientWithContext(ctx, cfg)
if err != nil {
cc.mu.Lock()
cc.nodeHealth[ep] = false
@@ -396,13 +413,17 @@ func (cc *ClusterClient) initialHealthCheck() {
Msg("Cluster endpoint marked unhealthy on initialization")
return
}
+ defer testClient.httpClient.CloseIdleConnections()
// Quick test with slightly longer timeout for initial check
- ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
- _, err = testClient.GetNodes(ctx)
+ testCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ _, err = testClient.GetNodes(testCtx)
cancel()
- if _, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(context.Background(), ep, 5*time.Second, err); refreshed {
+ if retryClient, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(ctx, ep, 5*time.Second, err); refreshed {
+ if retryClient != nil {
+ defer retryClient.httpClient.CloseIdleConnections()
+ }
err = retryErr
}
@@ -414,7 +435,7 @@ func (cc *ClusterClient) initialHealthCheck() {
fullCfg := cc.config
fullCfg.Host = ep
fullCfg.Fingerprint = cc.getEndpointFingerprint(ep)
- fullClient, clientErr := NewClient(fullCfg)
+ fullClient, clientErr := NewClientWithContext(ctx, fullCfg)
cc.mu.Lock()
if clientErr != nil {
@@ -540,8 +561,14 @@ func (cc *ClusterClient) endpointHealthDiagnosticsLocked() []endpointHealthDiagn
// member addresses are failover candidates rather than a random load-balancing
// pool.
func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
cc.mu.Lock()
defer cc.mu.Unlock()
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
now := time.Now()
healthyEndpoints, coolingEndpoints := cc.availableEndpointsLocked(now)
@@ -596,6 +623,9 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
defer cc.mu.Lock()
cc.recoverUnhealthyNodes(ctx)
}()
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
now = time.Now()
healthyEndpoints, coolingEndpoints = cc.availableEndpointsLocked(now)
@@ -658,8 +688,11 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
testCfg := cfg
testCfg.Timeout = 3 * time.Second
- testClient, err := NewClient(testCfg)
+ testClient, err := NewClientWithContext(ctx, testCfg)
if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
// Mark as unhealthy
cc.nodeHealth[selectedEndpoint] = false
log.Debug().
@@ -669,11 +702,15 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
Msg("Failed to create client for cluster endpoint")
return nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)
}
+ defer testClient.httpClient.CloseIdleConnections()
// Connectivity test - 5 seconds to allow for TLS handshake (~3s typical)
- testCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ testCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
testNodes, testErr := testClient.GetNodes(testCtx)
cancel()
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
if testErr != nil {
// Check if this is a transient rate limit error that shouldn't mark the node unhealthy
@@ -717,17 +754,20 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
Int("nodes", len(testNodes)).
Msg("Cluster endpoint passed connectivity test")
- // Clear any stale error from previous failures now that connectivity succeeded
- delete(cc.lastError, selectedEndpoint)
-
// Create the actual client with full timeout
- newClient, err := NewClient(cfg)
+ newClient, err := NewClientWithContext(ctx, cfg)
if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
// This shouldn't happen since we just tested it
cc.nodeHealth[selectedEndpoint] = false
return nil, fmt.Errorf("failed to create client for %s: %w", securityutil.InfrastructureEndpointDiagnostic(selectedEndpoint), err)
}
+ // A completed probe cannot clear an error until the polling session is
+ // accepted. Ending a caller is not endpoint failure or recovery evidence.
+ delete(cc.lastError, selectedEndpoint)
cc.clients[selectedEndpoint] = newClient
client = newClient
}
@@ -768,6 +808,9 @@ func (cc *ClusterClient) clearEndpointError(endpoint string) {
// recoverUnhealthyNodes attempts to recover unhealthy nodes
func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
+ if ctx.Err() != nil {
+ return
+ }
cc.mu.RLock()
unhealthyEndpoints := make([]string, 0)
throttledEndpoints := make([]string, 0)
@@ -810,6 +853,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
var failureMu sync.Mutex
failureReasons := make(map[string]string, len(unhealthyEndpoints))
recordFailure := func(ep string, err error) {
+ if ctx.Err() != nil {
+ return
+ }
sanitized := sanitizeEndpointError(err.Error())
failureMu.Lock()
failureReasons[ep] = sanitized
@@ -823,6 +869,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
wg.Add(1)
go func(ep string) {
defer wg.Done()
+ if ctx.Err() != nil {
+ return
+ }
// Update last check time
cc.mu.Lock()
@@ -837,7 +886,7 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
cfg.Fingerprint = cc.getEndpointFingerprint(ep)
cfg.Timeout = 5 * time.Second
- testClient, err := NewClient(cfg)
+ testClient, err := NewClientWithContext(ctx, cfg)
if err != nil {
recordFailure(ep, err)
log.Debug().
@@ -847,16 +896,23 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
Msg("Failed to create client during recovery attempt")
return
}
+ defer testClient.httpClient.CloseIdleConnections()
// Try a simple API call
testCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_, err = testClient.GetNodes(testCtx)
cancel()
- if _, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(ctx, ep, 5*time.Second, err); refreshed {
+ if retryClient, retryErr, refreshed := cc.refreshTOFUFingerprintAndRetry(ctx, ep, 5*time.Second, err); refreshed {
+ if retryClient != nil {
+ defer retryClient.httpClient.CloseIdleConnections()
+ }
err = retryErr
cfg.Fingerprint = cc.getEndpointFingerprint(ep)
}
+ if ctx.Err() != nil {
+ return
+ }
// Check if error is VM-specific (shouldn't prevent recovery)
vmSpecificErr := err != nil && isVMSpecificError(err.Error())
@@ -865,7 +921,7 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
// Store the client with original timeout
cfg.Timeout = cc.config.Timeout
cfg.Fingerprint = cc.getEndpointFingerprint(ep)
- fullClient, clientErr := NewClient(cfg)
+ fullClient, clientErr := NewClientWithContext(ctx, cfg)
if clientErr != nil {
// A short probe is not a usable polling session. Password
// authentication may fail when constructing the actual client;
@@ -881,6 +937,11 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
}
cc.mu.Lock()
+ if ctx.Err() != nil {
+ cc.mu.Unlock()
+ fullClient.httpClient.CloseIdleConnections()
+ return
+ }
cc.nodeHealth[ep] = true
delete(cc.lastError, ep)
cc.lastHealthCheck[ep] = time.Now()
diff --git a/pkg/proxmox/cluster_client_cancellation_test.go b/pkg/proxmox/cluster_client_cancellation_test.go
new file mode 100644
index 0000000000..c01b2ca999
--- /dev/null
+++ b/pkg/proxmox/cluster_client_cancellation_test.go
@@ -0,0 +1,172 @@
+package proxmox
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+// The public read/recovery paths must not manufacture a password session after
+// their caller has stopped. These peers and credentials are local fixtures.
+func testClusterCancelledCallerStartsNoSession(t *testing.T) {
+ for _, authMode := range []string{"password", "token"} {
+ t.Run(authMode, func(t *testing.T) {
+ var requests atomic.Int32
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ if r.URL.Path == "/api2/json/access/ticket" {
+ fmt.Fprint(w, `{"data":{"ticket":"fixture-ticket","CSRFPreventionToken":"fixture-csrf"}}`)
+ } else {
+ fmt.Fprint(w, `{"data":[]}`)
+ }
+ }))
+ defer server.Close()
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}
+ if authMode == "token" {
+ cfg.Password, cfg.TokenName, cfg.TokenValue = "", "fixture@pam!pulse", "fixture-token"
+ }
+ cc := NewClusterClient("cancelled", cfg, []string{server.URL}, nil)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if _, err := cc.GetNodes(ctx); !errors.Is(err, context.Canceled) {
+ t.Errorf("cancelled ordinary read = %v, want context cancellation", err)
+ }
+ if requests.Load() != 0 || len(cc.clients) != 0 || !cc.nodeHealth[server.URL] || cc.lastError[server.URL] != "" {
+ t.Errorf("cancelled caller changed session/health: requests=%d clients=%d healthy=%t error=%q", requests.Load(), len(cc.clients), cc.nodeHealth[server.URL], cc.lastError[server.URL])
+ }
+ })
+ }
+}
+
+func testClusterSessionConstructionHonoursCallerBudget(t *testing.T) {
+ for _, path := range []string{"selection", "recovery"} {
+ for _, stage := range []string{"first-login", "probe", "full-login"} {
+ for _, end := range []string{"cancel", "deadline"} {
+ t.Run(path+"/"+stage+"/"+end, func(t *testing.T) {
+ var logins, reads atomic.Int32
+ entered, release, requestEnded := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var enteredOnce, releaseOnce sync.Once
+ var block atomic.Bool
+ block.Store(true)
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = io.Copy(io.Discard, r.Body)
+ w.Header().Set("Content-Type", "application/json")
+ login := r.URL.Path == "/api2/json/access/ticket"
+ var ordinal int32
+ if login {
+ ordinal = logins.Add(1)
+ } else {
+ ordinal = reads.Add(1)
+ if r.URL.Path != "/api2/json/nodes" || !strings.Contains(r.Header.Get("Cookie"), "fixture-ticket-") {
+ t.Errorf("non-ordinary or unauthenticated fixture read: %s", r.URL.Path)
+ }
+ }
+ target := (stage == "first-login" && login && ordinal == 1) || (stage == "probe" && !login && ordinal == 1) || (stage == "full-login" && login && ordinal == 2)
+ if block.Load() && target {
+ enteredOnce.Do(func() { close(entered) })
+ select {
+ case <-r.Context().Done():
+ close(requestEnded)
+ return
+ case <-release:
+ }
+ }
+ if login {
+ fmt.Fprintf(w, `{"data":{"ticket":"fixture-ticket-%d","CSRFPreventionToken":"fixture-csrf-%d"}}`, ordinal, ordinal)
+ } else {
+ fmt.Fprint(w, `{"data":[{"node":"member","status":"online"}]}`)
+ }
+ }))
+ defer server.Close()
+ defer releaseOnce.Do(func() { close(release) })
+ pin := sha256.Sum256(server.Certificate().Raw)
+ cfg := ClientConfig{Host: server.URL, User: "fixture@pam", Password: "fixture-password", VerifySSL: true, Fingerprint: hex.EncodeToString(pin[:]), Timeout: 8 * time.Second}
+ cc := NewClusterClient("budget", cfg, []string{server.URL}, nil)
+ cc.nodeHealth[server.URL] = path == "selection"
+ cc.lastError[server.URL] = "previous fixture outage"
+ ctx, cancel := context.WithCancel(context.Background())
+ if end == "deadline" {
+ cancel()
+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+ }
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ if path == "recovery" {
+ cc.recoverUnhealthyNodes(ctx)
+ done <- nil
+ } else {
+ _, err := cc.GetNodes(ctx)
+ done <- err
+ }
+ }()
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ releaseOnce.Do(func() { close(release) })
+ <-done
+ t.Fatal("fixture did not reach selected session phase")
+ }
+ if end == "cancel" {
+ cancel()
+ } else {
+ <-ctx.Done()
+ }
+ var err error
+ select {
+ case err = <-done:
+ case <-time.After(time.Second):
+ t.Error("session construction outlived the ended caller by over one second")
+ releaseOnce.Do(func() { close(release) })
+ err = <-done // Join the old implementation too; no leaked fixture worker.
+ }
+ if path == "selection" && !errors.Is(err, ctx.Err()) {
+ t.Errorf("selection lost caller context: %v, want %v", err, ctx.Err())
+ }
+ select {
+ case <-requestEnded:
+ case <-time.After(time.Second):
+ t.Error("ended caller left its fixture request active")
+ }
+ if len(cc.clients) != 0 || cc.nodeHealth[server.URL] != (path == "selection") || cc.lastError[server.URL] != "previous fixture outage" || cc.getEndpointFingerprint(server.URL) != cfg.Fingerprint {
+ t.Errorf("cancelled construction published a session/recovery or changed prior trust/error: clients=%d healthy=%t error=%q", len(cc.clients), cc.nodeHealth[server.URL], cc.lastError[server.URL])
+ }
+ wantLogins, wantReads := int32(1), int32(0)
+ if stage == "probe" || stage == "full-login" {
+ wantReads = 1
+ if stage == "full-login" {
+ wantLogins = 2
+ }
+ }
+ if logins.Load() != wantLogins || reads.Load() != wantReads {
+ t.Errorf("ended construction dispatched another phase: logins=%d reads=%d", logins.Load(), reads.Load())
+ }
+ // A later ordinary caller can recover under the unchanged pin.
+ block.Store(false)
+ releaseOnce.Do(func() { close(release) })
+ cc.nodeHealth[server.URL] = false
+ delete(cc.lastHealthCheck, server.URL) // Only advance this fixture's eligibility.
+ cc.recoverUnhealthyNodes(context.Background())
+ if client := cc.clients[server.URL]; client == nil || client.config.Timeout != cfg.Timeout || !cc.nodeHealth[server.URL] || cc.lastError[server.URL] != "" {
+ t.Fatal("ordinary later recovery did not publish its accepted full-timeout session")
+ }
+ defer cc.clients[server.URL].httpClient.CloseIdleConnections()
+ if nodes, err := cc.GetNodes(context.Background()); err != nil || len(nodes) != 1 || nodes[0].Node != "member" {
+ t.Fatalf("later recovered session cannot serve ordinary reads: %+v %v", nodes, err)
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/pkg/proxmox/host_diagnostics_regression_test.go b/pkg/proxmox/host_diagnostics_regression_test.go
index 947a00e119..2de7bebec9 100644
--- a/pkg/proxmox/host_diagnostics_regression_test.go
+++ b/pkg/proxmox/host_diagnostics_regression_test.go
@@ -157,3 +157,16 @@ func TestClusterDiagnosticHealthSnapshotKeepsEveryEndpoint(t *testing.T) {
t.Error("display snapshot lost endpoint status observations")
}
}
+
+// Initial authentication must obey the same caller lifetime as ordinary reads.
+func TestClientConstructionHonoursContext(t *testing.T) {
+ testClientConstructionHonoursContext(t)
+}
+
+func TestClusterCancelledCallerStartsNoSession(t *testing.T) {
+ testClusterCancelledCallerStartsNoSession(t)
+}
+
+func TestClusterSessionConstructionHonoursCallerBudget(t *testing.T) {
+ testClusterSessionConstructionHonoursCallerBudget(t)
+}