Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep shared Proxmox login alive when its starter cancels
rcourtmanaf3b578c0609
diff --git a/pkg/proxmox/auth_session_concurrency_test.go b/pkg/proxmox/auth_session_concurrency_test.goindex cd83e654ec..1e3a6a4784 100644--- a/pkg/proxmox/auth_session_concurrency_test.go+++ b/pkg/proxmox/auth_session_concurrency_test.go@@ -323,3 +323,168 @@ func TestClientSessionTokenAuthDoesNotRenew(t *testing.T) {t.Errorf("token auth/API calls=%d/%d", authCalls.Load(), apiCalls.Load())}}++// The caller that happens to start a shared renewal may be a short-lived read+// (a guest-agent call, a cancelled metadata fetch, a disconnected API client).+// Its cancellation ends only its own wait: live waiters keep the login, a+// genuine login failure still reaches them, and a login whose starter has left+// still completes and is reused instead of being repeated.+func TestClientSessionRenewalStarterCancellation(t *testing.T) {+ for _, tc := range []struct {+ name string+ waiter bool+ failure bool+ }{+ {name: "live waiter keeps login", waiter: true},+ {name: "login failure reaches live waiter", waiter: true, failure: true},+ {name: "abandoned login is reused", waiter: false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ entered, finish := make(chan struct{}), make(chan struct{})+ var authCalls, apiCalls atomic.Int32+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/access/ticket") {+ n := authCalls.Add(1)+ if n == 2 {+ close(entered)+ select {+ case <-finish:+ case <-ctx.Done():+ return+ }+ if tc.failure {+ w.WriteHeader(http.StatusUnauthorized)+ fmt.Fprint(w, "synthetic credentials rejected")+ return+ }+ }+ fmt.Fprintf(w, `{"data":{"ticket":"session-%d","CSRFPreventionToken":"csrf-%d"}}`, n, n)+ return+ }+ apiCalls.Add(1)+ fmt.Fprint(w, `{"data":[{"node":"session-safe-node","status":"online"}]}`)+ }))+ defer server.Close()+ c, err := NewClient(ClientConfig{Host: server.URL, User: "synthetic@pam", Password: "synthetic-password", Timeout: 5 * time.Second})+ if err != nil {+ t.Fatal(err)+ }+ defer c.httpClient.CloseIdleConnections()+ expireSessionFixture(c)++ starterCtx, cancelStarter := context.WithCancel(ctx)+ defer cancelStarter()+ starter := make(chan error, 1)+ go func() { _, err := c.GetNodes(starterCtx); starter <- err }()+ awaitSessionSignal(t, entered)++ waiter := make(chan error, 1)+ if tc.waiter {+ joined := &authJoinContext{Context: ctx, joined: make(chan struct{})}+ go func() { _, err := c.GetNodes(joined); waiter <- err }()+ awaitSessionSignal(t, joined.joined)+ }++ cancelStarter()+ select {+ case err := <-starter:+ if !errors.Is(err, context.Canceled) {+ t.Errorf("cancelled starter result: %v", err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("cancelled starter remained blocked behind its shared login")+ }+ if tc.waiter {+ select {+ case err := <-waiter:+ t.Fatalf("starter cancellation ended a live waiter: %v", err)+ default:+ }+ }++ close(finish)+ if tc.waiter {+ err := <-waiter+ switch {+ case tc.failure && (err == nil || !strings.Contains(err.Error(), "synthetic credentials rejected")):+ t.Errorf("login failure not shared with live waiter: %v", err)+ case tc.failure && errors.Is(err, context.Canceled):+ t.Errorf("waiter received the starter's cancellation: %v", err)+ case !tc.failure && err != nil:+ t.Errorf("live waiter failed: %v", err)+ }+ }+ if tc.failure {+ if apiCalls.Load() != 0 {+ t.Error("failed login admitted API reads")+ }+ return+ }+ // A later read joins or reuses the detached login; it never logs in again.+ if _, err := c.GetNodes(ctx); err != nil {+ t.Fatal(err)+ }+ snapshot, _ := c.authSnapshot()+ if snapshot.ticket != "session-2" || !snapshot.expiresAt.After(time.Now()) {+ t.Errorf("detached login not published: ticket=%q", snapshot.ticket)+ }+ wantAPI := int32(1)+ if tc.waiter {+ wantAPI = 2+ }+ if authCalls.Load() != 2 || apiCalls.Load() != wantAPI {+ t.Errorf("auth/api=%d/%d, want 2/%d", authCalls.Load(), apiCalls.Load(), wantAPI)+ }+ })+ }+}++// A detached login cannot outlive the client's request timeout: a stalled+// Proxmox login ends, releases the in-flight slot, and a later read can renew.+func TestClientSessionRenewalStalledLoginIsBounded(t *testing.T) {+ var authCalls atomic.Int32+ release := make(chan struct{})+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if strings.HasSuffix(r.URL.Path, "/access/ticket") {+ if authCalls.Add(1) == 2 {+ select {+ case <-release:+ case <-r.Context().Done():+ }+ return+ }+ fmt.Fprint(w, `{"data":{"ticket":"session","CSRFPreventionToken":"csrf"}}`)+ return+ }+ fmt.Fprint(w, `{"data":[]}`)+ }))+ defer server.Close()+ // The stalled handler never reads its body, so the server cannot see the+ // client hang up; release it before server.Close waits for it.+ defer close(release)+ c, err := NewClient(ClientConfig{Host: server.URL, User: "synthetic@pam", Password: "synthetic-password", Timeout: 200 * time.Millisecond})+ if err != nil {+ t.Fatal(err)+ }+ defer c.httpClient.CloseIdleConnections()+ expireSessionFixture(c)+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ if _, err := c.GetNodes(ctx); err == nil {+ t.Fatal("stalled login reported success")+ }+ c.authMu.Lock()+ inFlight := c.authInFlight+ c.authMu.Unlock()+ if inFlight != nil {+ t.Fatal("stalled login kept the shared renewal slot")+ }+ if _, err := c.GetNodes(ctx); err != nil {+ t.Fatalf("later renewal after stalled login: %v", err)+ }+ if authCalls.Load() != 3 {+ t.Errorf("auth calls=%d, want initial, stalled and recovery", authCalls.Load())+ }+}diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.goindex 9613d2cca2..433986306f 100644--- a/pkg/proxmox/client.go+++ b/pkg/proxmox/client.go@@ -270,6 +270,10 @@ type authRefresh struct {err error}+// defaultClientTimeout is the per-request timeout when ClientConfig.Timeout+// is unset.+const defaultClientTimeout = 60 * time.Second+func (c *Client) authSnapshot() (auth, uint64) {c.authMu.Lock()defer c.authMu.Unlock()@@ -356,7 +360,7 @@ func NewClient(cfg ClientConfig) (*Client, error) {// Use configured timeout or default to 60 secondstimeout := cfg.Timeoutif timeout <= 0 {- timeout = 60 * time.Second+ timeout = defaultClientTimeout}httpClient := tlsutil.CreateHTTPClientWithTimeout(cfg.VerifySSL, cfg.Fingerprint, timeout)@@ -447,17 +451,41 @@ func (c *Client) renewAuthentication(ctx context.Context, expected *uint64) errousername := c.auth.user + "@" + c.auth.realmc.authMu.Unlock()- password := c.config.Password- err := c.authenticateJSON(ctx, username, password)- if shouldFallbackToForm(err) {- err = c.authenticateForm(ctx, username, password)+ // The login is shared, so no single caller's cancellation may end it: a+ // cancelled guest-agent or API read that happened to start the renewal+ // must not fail every live read waiting on it. The detached attempt keeps+ // the caller's values, is bounded by the client's request timeout, and+ // publishes its result for later reads even after its starter has left.+ loginCtx, cancelLogin := context.WithTimeout(context.WithoutCancel(ctx), c.sessionRenewalTimeout())+ go func() {+ defer cancelLogin()+ password := c.config.Password+ err := c.authenticateJSON(loginCtx, username, password)+ if shouldFallbackToForm(err) {+ err = c.authenticateForm(loginCtx, username, password)+ }+ c.authMu.Lock()+ pending.err = err+ c.authInFlight = nil+ close(pending.done)+ c.authMu.Unlock()+ }()++ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-pending.done:+ return pending.err}- c.authMu.Lock()- pending.err = err- c.authInFlight = nil- close(pending.done)- c.authMu.Unlock()- return err+}++// sessionRenewalTimeout bounds one shared login attempt (JSON plus any form+// fallback) by the same per-request timeout the client applies to API reads.+func (c *Client) sessionRenewalTimeout() time.Duration {+ if c.httpClient != nil && c.httpClient.Timeout > 0 {+ return c.httpClient.Timeout+ }+ return defaultClientTimeout}func (c *Client) authenticateJSON(ctx context.Context, username, password string) error {