Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep shared Proxmox login alive when its starter cancels

rcourtmanaf3b578c0609
diff --git a/pkg/proxmox/auth_session_concurrency_test.go b/pkg/proxmox/auth_session_concurrency_test.go
index cd83e654ec..1e3a6a4784 100644
--- a/pkg/proxmox/auth_session_concurrency_test.go
+++ b/pkg/proxmox/auth_session_concurrency_test.go
@@ -323,3 +323,168 @@ func TestClientSessionTokenAuthDoesNotRenew(t *testing.T) {
t.Errorf("token auth/API calls=%d/%d", authCalls.Load(), apiCalls.Load())
}
}
+
+// The caller that happens to start a shared renewal may be a short-lived read
+// (a guest-agent call, a cancelled metadata fetch, a disconnected API client).
+// Its cancellation ends only its own wait: live waiters keep the login, a
+// genuine login failure still reaches them, and a login whose starter has left
+// still completes and is reused instead of being repeated.
+func TestClientSessionRenewalStarterCancellation(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ waiter bool
+ failure bool
+ }{
+ {name: "live waiter keeps login", waiter: true},
+ {name: "login failure reaches live waiter", waiter: true, failure: true},
+ {name: "abandoned login is reused", waiter: false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ entered, finish := make(chan struct{}), make(chan struct{})
+ var authCalls, apiCalls atomic.Int32
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/access/ticket") {
+ n := authCalls.Add(1)
+ if n == 2 {
+ close(entered)
+ select {
+ case <-finish:
+ case <-ctx.Done():
+ return
+ }
+ if tc.failure {
+ w.WriteHeader(http.StatusUnauthorized)
+ fmt.Fprint(w, "synthetic credentials rejected")
+ return
+ }
+ }
+ fmt.Fprintf(w, `{"data":{"ticket":"session-%d","CSRFPreventionToken":"csrf-%d"}}`, n, n)
+ return
+ }
+ apiCalls.Add(1)
+ fmt.Fprint(w, `{"data":[{"node":"session-safe-node","status":"online"}]}`)
+ }))
+ defer server.Close()
+ c, err := NewClient(ClientConfig{Host: server.URL, User: "synthetic@pam", Password: "synthetic-password", Timeout: 5 * time.Second})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer c.httpClient.CloseIdleConnections()
+ expireSessionFixture(c)
+
+ starterCtx, cancelStarter := context.WithCancel(ctx)
+ defer cancelStarter()
+ starter := make(chan error, 1)
+ go func() { _, err := c.GetNodes(starterCtx); starter <- err }()
+ awaitSessionSignal(t, entered)
+
+ waiter := make(chan error, 1)
+ if tc.waiter {
+ joined := &authJoinContext{Context: ctx, joined: make(chan struct{})}
+ go func() { _, err := c.GetNodes(joined); waiter <- err }()
+ awaitSessionSignal(t, joined.joined)
+ }
+
+ cancelStarter()
+ select {
+ case err := <-starter:
+ if !errors.Is(err, context.Canceled) {
+ t.Errorf("cancelled starter result: %v", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("cancelled starter remained blocked behind its shared login")
+ }
+ if tc.waiter {
+ select {
+ case err := <-waiter:
+ t.Fatalf("starter cancellation ended a live waiter: %v", err)
+ default:
+ }
+ }
+
+ close(finish)
+ if tc.waiter {
+ err := <-waiter
+ switch {
+ case tc.failure && (err == nil || !strings.Contains(err.Error(), "synthetic credentials rejected")):
+ t.Errorf("login failure not shared with live waiter: %v", err)
+ case tc.failure && errors.Is(err, context.Canceled):
+ t.Errorf("waiter received the starter's cancellation: %v", err)
+ case !tc.failure && err != nil:
+ t.Errorf("live waiter failed: %v", err)
+ }
+ }
+ if tc.failure {
+ if apiCalls.Load() != 0 {
+ t.Error("failed login admitted API reads")
+ }
+ return
+ }
+ // A later read joins or reuses the detached login; it never logs in again.
+ if _, err := c.GetNodes(ctx); err != nil {
+ t.Fatal(err)
+ }
+ snapshot, _ := c.authSnapshot()
+ if snapshot.ticket != "session-2" || !snapshot.expiresAt.After(time.Now()) {
+ t.Errorf("detached login not published: ticket=%q", snapshot.ticket)
+ }
+ wantAPI := int32(1)
+ if tc.waiter {
+ wantAPI = 2
+ }
+ if authCalls.Load() != 2 || apiCalls.Load() != wantAPI {
+ t.Errorf("auth/api=%d/%d, want 2/%d", authCalls.Load(), apiCalls.Load(), wantAPI)
+ }
+ })
+ }
+}
+
+// A detached login cannot outlive the client's request timeout: a stalled
+// Proxmox login ends, releases the in-flight slot, and a later read can renew.
+func TestClientSessionRenewalStalledLoginIsBounded(t *testing.T) {
+ var authCalls atomic.Int32
+ release := make(chan struct{})
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/access/ticket") {
+ if authCalls.Add(1) == 2 {
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ }
+ return
+ }
+ fmt.Fprint(w, `{"data":{"ticket":"session","CSRFPreventionToken":"csrf"}}`)
+ return
+ }
+ fmt.Fprint(w, `{"data":[]}`)
+ }))
+ defer server.Close()
+ // The stalled handler never reads its body, so the server cannot see the
+ // client hang up; release it before server.Close waits for it.
+ defer close(release)
+ c, err := NewClient(ClientConfig{Host: server.URL, User: "synthetic@pam", Password: "synthetic-password", Timeout: 200 * time.Millisecond})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer c.httpClient.CloseIdleConnections()
+ expireSessionFixture(c)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if _, err := c.GetNodes(ctx); err == nil {
+ t.Fatal("stalled login reported success")
+ }
+ c.authMu.Lock()
+ inFlight := c.authInFlight
+ c.authMu.Unlock()
+ if inFlight != nil {
+ t.Fatal("stalled login kept the shared renewal slot")
+ }
+ if _, err := c.GetNodes(ctx); err != nil {
+ t.Fatalf("later renewal after stalled login: %v", err)
+ }
+ if authCalls.Load() != 3 {
+ t.Errorf("auth calls=%d, want initial, stalled and recovery", authCalls.Load())
+ }
+}
diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go
index 9613d2cca2..433986306f 100644
--- a/pkg/proxmox/client.go
+++ b/pkg/proxmox/client.go
@@ -270,6 +270,10 @@ type authRefresh struct {
err error
}
+// defaultClientTimeout is the per-request timeout when ClientConfig.Timeout
+// is unset.
+const defaultClientTimeout = 60 * time.Second
+
func (c *Client) authSnapshot() (auth, uint64) {
c.authMu.Lock()
defer c.authMu.Unlock()
@@ -356,7 +360,7 @@ func NewClient(cfg ClientConfig) (*Client, error) {
// Use configured timeout or default to 60 seconds
timeout := cfg.Timeout
if timeout <= 0 {
- timeout = 60 * time.Second
+ timeout = defaultClientTimeout
}
httpClient := tlsutil.CreateHTTPClientWithTimeout(cfg.VerifySSL, cfg.Fingerprint, timeout)
@@ -447,17 +451,41 @@ func (c *Client) renewAuthentication(ctx context.Context, expected *uint64) erro
username := c.auth.user + "@" + c.auth.realm
c.authMu.Unlock()
- password := c.config.Password
- err := c.authenticateJSON(ctx, username, password)
- if shouldFallbackToForm(err) {
- err = c.authenticateForm(ctx, username, password)
+ // The login is shared, so no single caller's cancellation may end it: a
+ // cancelled guest-agent or API read that happened to start the renewal
+ // must not fail every live read waiting on it. The detached attempt keeps
+ // the caller's values, is bounded by the client's request timeout, and
+ // publishes its result for later reads even after its starter has left.
+ loginCtx, cancelLogin := context.WithTimeout(context.WithoutCancel(ctx), c.sessionRenewalTimeout())
+ go func() {
+ defer cancelLogin()
+ password := c.config.Password
+ err := c.authenticateJSON(loginCtx, username, password)
+ if shouldFallbackToForm(err) {
+ err = c.authenticateForm(loginCtx, username, password)
+ }
+ c.authMu.Lock()
+ pending.err = err
+ c.authInFlight = nil
+ close(pending.done)
+ c.authMu.Unlock()
+ }()
+
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-pending.done:
+ return pending.err
}
- c.authMu.Lock()
- pending.err = err
- c.authInFlight = nil
- close(pending.done)
- c.authMu.Unlock()
- return err
+}
+
+// sessionRenewalTimeout bounds one shared login attempt (JSON plus any form
+// fallback) by the same per-request timeout the client applies to API reads.
+func (c *Client) sessionRenewalTimeout() time.Duration {
+ if c.httpClient != nil && c.httpClient.Timeout > 0 {
+ return c.httpClient.Timeout
+ }
+ return defaultClientTimeout
}
func (c *Client) authenticateJSON(ctx context.Context, username, password string) error {