Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: prevent logout and skipped environments during update all (#3000)
Kyle Mendellaacc3d2bb4ec
diff --git a/backend/api/middleware/auth.go b/backend/api/middleware/auth.goindex 4a6d48dba..17aba7ca3 100644--- a/backend/api/middleware/auth.go+++ b/backend/api/middleware/auth.go@@ -321,7 +321,14 @@ func handleBearerAuthInternal(api huma.API, ctx huma.Context, authService *servinewCtx = context.WithValue(newCtx, ContextKeyCurrentSessionID, sessionID)return huma.WithContext(ctx, newCtx), true}- if errors.Is(err, services.ErrTokenVersionMismatch) || common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {+ if errors.Is(err, services.ErrTokenVersionMismatch) {+ // The app version changed (a self-update). The session is still valid — the+ // refresh path tolerates the version change and rotates the token — so do NOT+ // clear the auth cookies. Return a recoverable 401 the frontend refreshes from.+ _ = huma.WriteErr(api, ctx, http.StatusUnauthorized, "Application has been updated. Refreshing session.")+ return nil, true+ }+ if common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {for _, cookieHeader := range cookie.BuildClearTokenCookieStringsFor(cookie.SecureCookieFromContext(ctx.Context())) {ctx.AppendHeader("Set-Cookie", cookieHeader)}diff --git a/backend/api/middleware/auth_test.go b/backend/api/middleware/auth_test.goindex c30e73e4d..b54e67107 100644--- a/backend/api/middleware/auth_test.go+++ b/backend/api/middleware/auth_test.go@@ -285,3 +285,85 @@ func TestNewAuthBridge_OpportunisticAuthOnPublicRoute(t *testing.T) {require.Equal(t, "", sawSessionID)})}++// After a self-update the app version changes and old access tokens fail the version+// check. That must be RECOVERABLE (the refresh path rotates the token), so the+// middleware returns a refreshable 401 and must NOT clear the auth cookies — otherwise+// the user is logged out on every update.+func TestNewAuthBridge_VersionMismatchIsRecoverable(t *testing.T) {+ db := setupAuthMiddlewareTestDBInternal(t)+ userSvc := services.NewUserService(db)+ sessionSvc := services.NewSessionService(db)++ jwtSecret := "test-secret-please-do-not-use-in-prod"+ cfg := &config.Config{JWTRefreshExpiry: 24 * time.Hour}+ authSvc := services.NewAuthService(userSvc, nil, nil, sessionSvc, nil, jwtSecret, cfg)++ _, err := userSvc.CreateUser(context.Background(), &models.User{+ BaseModel: models.BaseModel{ID: "u-ver"},+ Username: "vertest",+ })+ require.NoError(t, err)++ exp := time.Now().Add(5 * time.Minute)+ session, _, err := sessionSvc.CreateSession(context.Background(), "u-ver", exp, auth.SessionMeta{})+ require.NoError(t, err)++ // An empty appVersion omits the claim, which passes the version check (no pin).+ mintToken := func(appVersion string) string {+ claims := jwt.MapClaims{+ "jti": "u-ver",+ "sub": "access",+ "iat": time.Now().Unix(),+ "exp": exp.Unix(),+ "sid": session.ID,+ "user_id": "u-ver",+ "username": "vertest",+ }+ if appVersion != "" {+ claims["app_version"] = appVersion+ }+ token, signErr := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(jwtSecret))+ require.NoError(t, signErr)+ return token+ }++ router := echo.New()+ apiGroup := router.Group("/api")+ humaConfig := huma.DefaultConfig("test", "1.0.0")+ humaConfig.Components.SecuritySchemes = map[string]*huma.SecurityScheme{+ "BearerAuth": {Type: "http", Scheme: "bearer"},+ }+ api := humaecho.NewWithGroup(router, apiGroup, humaConfig)+ api.UseMiddleware(NewAuthBridge(api, authSvc, nil, nil, nil, &config.Config{}))++ huma.Register(api, huma.Operation{+ OperationID: "protected",+ Method: http.MethodGet,+ Path: "/protected",+ Security: []map[string][]string{{"BearerAuth": {}}},+ }, func(_ context.Context, _ *secureInput) (*secureOutput, error) {+ return &secureOutput{}, nil+ })++ t.Run("version mismatch returns a recoverable 401 without clearing cookies", func(t *testing.T) {+ req := httptest.NewRequest(http.MethodGet, "/api/protected", nil)+ req.Header.Set("Authorization", "Bearer "+mintToken("v0.0.0-stale"))+ rec := httptest.NewRecorder()+ router.ServeHTTP(rec, req)++ require.Equal(t, http.StatusUnauthorized, rec.Code)+ require.Contains(t, rec.Body.String(), "Application has been updated")+ // The frontend recovers via refresh; clearing the cookies here would log the+ // user out on every self-update.+ require.Empty(t, rec.Header().Values("Set-Cookie"))+ })++ t.Run("token without a version pin still authenticates", func(t *testing.T) {+ req := httptest.NewRequest(http.MethodGet, "/api/protected", nil)+ req.Header.Set("Authorization", "Bearer "+mintToken(""))+ rec := httptest.NewRecorder()+ router.ServeHTTP(rec, req)+ require.Equal(t, http.StatusOK, rec.Code)+ })+}diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex bcd65df28..1ca1c7cec 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -33,11 +33,11 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul}}- // Resume a fleet-wide update-all job interrupted by the manager's own upgrade- // restart (manager only). Runs off the app context so it can upgrade agents- // without blocking bootstrap.- if !appConfig.AgentMode && appServices.SystemUpgrade != nil && appServices.Environment != nil {- go appServices.SystemUpgrade.ResumeUpdateAllOnStartup(appCtx, appServices.Environment)+ // Finalize a fleet-wide update-all job whose manager self-upgrade restarted the+ // process as its final step (manager only). Runs off the app context so it does+ // not block bootstrap.+ if !appConfig.AgentMode && appServices.SystemUpgrade != nil {+ go appServices.SystemUpgrade.ResumeUpdateAllOnStartup(appCtx)}newScheduler.RegisterJob(jobs.AutoUpdate)diff --git a/backend/internal/middleware/auth_middleware.go b/backend/internal/middleware/auth_middleware.goindex 4c274f6c4..2121c0c27 100644--- a/backend/internal/middleware/auth_middleware.go+++ b/backend/internal/middleware/auth_middleware.go@@ -203,7 +203,17 @@ func (m *AuthMiddleware) managerAuth(ctx context.Context, c echo.Context, next euser, sessionID, err := m.authService.VerifyToken(ctx, token)if err != nil {- if errors.Is(err, services.ErrTokenVersionMismatch) || common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {+ // A version mismatch means the app self-updated; the session is still valid+ // (the refresh path tolerates the version change and rotates the token), so do+ // NOT clear the cookies. Return a recoverable 401 the frontend refreshes from.+ if errors.Is(err, services.ErrTokenVersionMismatch) {+ return c.JSON(http.StatusUnauthorized, models.APIError{+ Code: models.APIErrorCodeUnauthorized,+ Message: "Application has been updated. Refreshing session.",+ })+ }++ if common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {cookie.ClearTokenCookie(c.Response().Writer, req)return c.JSON(http.StatusUnauthorized, models.APIError{Code: models.APIErrorCodeUnauthorized,diff --git a/backend/internal/models/environment_update_job.go b/backend/internal/models/environment_update_job.goindex 17dabb0dd..81f688310 100644--- a/backend/internal/models/environment_update_job.go+++ b/backend/internal/models/environment_update_job.go@@ -12,9 +12,9 @@ import (type EnvironmentUpdateJobStatus stringconst (- // EnvironmentUpdateJobStatusPendingRestart means the manager has triggered its- // own self-upgrade and is waiting to restart on the new version, after which- // the job resumes to upgrade the remote agents.+ // EnvironmentUpdateJobStatusPendingRestart means the remote agents have been+ // updated and the manager has triggered its own self-upgrade as the final step;+ // it is waiting to restart on the new version, after which the job is finalized.EnvironmentUpdateJobStatusPendingRestart EnvironmentUpdateJobStatus = "pending_restart"// EnvironmentUpdateJobStatusRunning means the agents phase is in progress.EnvironmentUpdateJobStatusRunning EnvironmentUpdateJobStatus = "running"@@ -89,8 +89,8 @@ func (r *EnvironmentUpdateResults) Scan(value any) error {}// EnvironmentUpdateJob is a persisted fleet-wide update orchestration record. It-// survives the manager's self-upgrade restart so the agents phase can resume on-// the next boot. See [EnvironmentUpdateJobStatus].+// survives the manager's final self-upgrade restart so the manager result can be+// finalized on the next boot. See [EnvironmentUpdateJobStatus].type EnvironmentUpdateJob struct {BaseModeldiff --git a/backend/internal/services/system_upgrade_service.go b/backend/internal/services/system_upgrade_service.goindex fcab820d2..d530d519f 100644--- a/backend/internal/services/system_upgrade_service.go+++ b/backend/internal/services/system_upgrade_service.go@@ -406,11 +406,13 @@ func (s *SystemUpgradeService) findArcaneContainerInternal(ctx context.Context,// --- Fleet-wide "update all environments" orchestration ---//-// The manager upgrades itself first, which recreates its own container. Because-// the browser loses the backend across that restart, the orchestration is a-// persisted EnvironmentUpdateJob: StartUpdateAll triggers the manager self-upgrade-// and leaves the job in pending_restart; on the next boot ResumeUpdateAllOnStartup-// picks the job back up and upgrades each remote agent sequentially.+// Remote agents upgrade first, while the manager is still up and can orchestrate+// and report live progress. The manager upgrades itself LAST, which recreates its+// own container. Because the browser loses the backend across that final restart,+// the orchestration is a persisted EnvironmentUpdateJob: StartUpdateAll runs the+// agents phase and, when the manager itself has an update, triggers the manager+// self-upgrade as the last step and leaves the job in pending_restart; on the next+// boot ResumeUpdateAllOnStartup finalizes the manager result and closes the job.const (localEnvironmentIDInternal = "0"@@ -423,9 +425,10 @@ const (updateAllErrorMaxLenInternal = 500)-// StartUpdateAll begins a fleet-wide update. If the manager has an update available-// it self-upgrades first (job left pending_restart, resumed at next boot); otherwise-// the agents phase runs immediately in the background.+// StartUpdateAll begins a fleet-wide update. The agents phase runs first in the+// background (while the manager is up); when the manager itself has an update, the+// agents goroutine triggers the manager self-upgrade as its final step (job left+// pending_restart, finalized at next boot).func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.User, env *EnvironmentService) (*models.EnvironmentUpdateJob, error) {// Guard the check-then-create against concurrent callers (e.g. a double-click).// A dedicated flag rather than s.upgrading, because the manager branch below@@ -466,29 +469,19 @@ func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.UManagerTargetVersion: updateAllTargetVersionInternal(info),}- if info.UpdateAvailable {+ managerHasUpdate := info.UpdateAvailable+ if managerHasUpdate {+ // Manager upgrades LAST. Seed its row pending; the agents-phase goroutine+ // flips it to updating right before triggering the self-upgrade.managerResult.Status = models.EnvironmentUpdateResultStatusPendingmanagerResult.ToVersion = job.ManagerTargetVersion- job.Status = models.EnvironmentUpdateJobStatusPendingRestart- job.Results = append(models.EnvironmentUpdateResults{managerResult}, remoteResults...)-- if err := s.db.WithContext(ctx).Create(job).Error; err != nil {- return nil, fmt.Errorf("create update-all job: %w", err)- }-- // Trigger the manager self-upgrade. The manager restarts shortly after and- // the job resumes on the next boot via ResumeUpdateAllOnStartup.- if err := s.TriggerUpgradeViaCLI(ctx, user, updatertypes.SelfUpdateTarget{}); err != nil {- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))- return nil, err- }-- slog.InfoContext(ctx, "Update-all started; manager self-upgrade triggered", "jobId", job.ID, "user", user.Username)- return job, nil+ } else {+ managerResult.Status = models.EnvironmentUpdateResultStatusSkippedUpToDate}- // Manager already up to date — go straight to the agents phase.- managerResult.Status = models.EnvironmentUpdateResultStatusSkippedUpToDate+ // Running from the start in both cases: the agents phase happens first, while the+ // backend is up and can report progress. The manager self-upgrade (if any) fires+ // at the very end of the agents goroutine.job.Status = models.EnvironmentUpdateJobStatusRunningjob.Results = append(models.EnvironmentUpdateResults{managerResult}, remoteResults...)@@ -496,16 +489,17 @@ func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.Ureturn nil, fmt.Errorf("create update-all job: %w", err)}- slog.InfoContext(ctx, "Update-all started; manager up to date, upgrading agents", "jobId", job.ID, "user", user.Username)- go s.runAgentsPhaseInternal(context.WithoutCancel(ctx), job.ID, env)+ slog.InfoContext(ctx, "Update-all started; upgrading agents first", "jobId", job.ID, "managerHasUpdate", managerHasUpdate, "user", user.Username)+ go s.runAgentsPhaseInternal(context.WithoutCancel(ctx), job.ID, env, user, managerHasUpdate)return job, nil}-// ResumeUpdateAllOnStartup is called once at manager startup. It transitions any-// non-terminal update-all job out of its waiting state and, when appropriate, runs-// the agents phase. It is a no-op when there is nothing pending.-func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context, env *EnvironmentService) {+// ResumeUpdateAllOnStartup is called once at manager startup. When the manager+// self-upgraded as the final step of an update-all (job left pending_restart), the+// agents phase already ran before the restart — so this finalizes the manager's own+// result and closes the job. It is a no-op when there is nothing pending.+func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context) {job, err := s.activeUpdateAllJobInternal(ctx)if err != nil {slog.WarnContext(ctx, "Failed to load pending update-all job on startup", "error", err)@@ -515,8 +509,8 @@ func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context, envreturn}- // A job left running means the manager died mid-agents-phase; we can't safely- // resume partial progress, so fail it.+ // A job left running means the manager died mid-agents-phase, before it reached+ // the manager step; we can't safely resume partial progress, so fail it.if job.Status == models.EnvironmentUpdateJobStatusRunning {s.markUpdateAllFailedInternal(ctx, job, "interrupted by manager restart")return@@ -530,15 +524,20 @@ func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context, envreturn}+ // The agents phase already ran before the restart. Finalize the manager's own+ // result and close the job — do not re-run the agents phase.s.recordManagerResultInternal(job, action.managerSucceeded, info.CurrentVersion)- job.Status = models.EnvironmentUpdateJobStatusRunning++ completedAt := time.Now()+ job.Status = models.EnvironmentUpdateJobStatusCompleted+ job.CompletedAt = &completedAtif err := s.persistUpdateAllJobInternal(ctx, job); err != nil {- slog.WarnContext(ctx, "Failed to mark resumed update-all job as running", "jobId", job.ID, "error", err)+ slog.WarnContext(ctx, "Failed to finalize resumed update-all job", "jobId", job.ID, "error", err)return}- slog.InfoContext(ctx, "Resuming update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)- s.runAgentsPhaseInternal(ctx, job.ID, env)+ s.logUpdateAllEventInternal(ctx, job)+ slog.InfoContext(ctx, "Finalized update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)}type resumeActionInternal struct {@@ -563,8 +562,10 @@ func resolveResumeActionInternal(job *models.EnvironmentUpdateJob, currentVersio// runAgentsPhaseInternal upgrades every online remote environment that has an update// available, sequentially, persisting progress after each one so the status endpoint-// can report live progress.-func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID string, env *EnvironmentService) {+// can report live progress. When managerHasUpdate is set, it triggers the manager's+// own self-upgrade as the final step (leaving the job pending_restart for the next+// boot to finalize); otherwise it marks the job completed in-process.+func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID string, env *EnvironmentService, user models.User, managerHasUpdate bool) {job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)if err != nil || job == nil {slog.WarnContext(ctx, "update-all: failed to reload job for agents phase", "jobId", jobID, "error", err)@@ -596,6 +597,33 @@ func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID}}+ // All remote agents processed. Handle the manager LAST.+ if managerHasUpdate {+ // Flip the manager row pending -> updating and move the job to pending_restart,+ // persisting BEFORE the trigger so that if the manager dies the instant the+ // upgrader starts, the next boot sees pending_restart and finalizes it.+ for i := range job.Results {+ if job.Results[i].EnvironmentID == localEnvironmentIDInternal {+ job.Results[i].Status = models.EnvironmentUpdateResultStatusUpdating+ break+ }+ }+ job.Status = models.EnvironmentUpdateJobStatusPendingRestart+ if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {+ slog.WarnContext(ctx, "update-all: failed to persist pending_restart before manager upgrade", "jobId", job.ID, "error", err)+ }++ if err := s.TriggerUpgradeViaCLI(ctx, user, updatertypes.SelfUpdateTarget{}); err != nil {+ // Agents already ran and no restart is coming, so finalize now. This flips+ // the manager's updating row to failed with the reason.+ s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))+ return+ }++ slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered; finalizing on next boot", "jobId", job.ID)+ return+ }+completedAt := time.Now()job.Status = models.EnvironmentUpdateJobStatusCompletedjob.CompletedAt = &completedAtdiff --git a/backend/internal/services/system_upgrade_update_all_test.go b/backend/internal/services/system_upgrade_update_all_test.goindex 7f1a60ec2..1e9b24d6c 100644--- a/backend/internal/services/system_upgrade_update_all_test.go+++ b/backend/internal/services/system_upgrade_update_all_test.go@@ -154,3 +154,53 @@ func TestUpdateAllFailedJobMarksUpdatingResultsFailed(t *testing.T) {require.Equal(t, models.EnvironmentUpdateResultStatusFailed, got.Results[3].Status)require.Equal(t, "already failed", got.Results[3].Error)}++// With the manager-last ordering, a resumed pending_restart job means the agents+// phase already ran before the restart: resume must finalize the manager's own row+// and complete the job, NOT re-run the agents phase.+func TestResumeUpdateAllFinalizesManagerWithoutRerunningAgents(t *testing.T) {+ ctx := context.Background()+ gormDB, err := gorm.Open(glsqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)++ db := &database.DB{DB: gormDB}+ require.NoError(t, db.AutoMigrate(&models.EnvironmentUpdateJob{}, &models.Event{}))++ // disabled=true keeps GetAppVersionInfo offline; nil docker => empty current digest.+ // The reported version differs from ManagerVersionAtStart, so the manager upgrade+ // is judged successful.+ versionSvc := NewVersionService(nil, true, "v9.9.9-new", "", nil, nil, nil)+ svc := NewSystemUpgradeService(db, nil, versionSvc, NewEventService(db, nil, nil), nil)++ job := &models.EnvironmentUpdateJob{+ Status: models.EnvironmentUpdateJobStatusPendingRestart,+ UserID: "user-1",+ Username: "arcane",+ ManagerVersionAtStart: "v1.0.0-old",+ Results: models.EnvironmentUpdateResults{+ {EnvironmentID: "0", EnvironmentName: "Local", Status: models.EnvironmentUpdateResultStatusUpdating},+ {EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: models.EnvironmentUpdateResultStatusUpdated},+ {EnvironmentID: "remote-2", EnvironmentName: "oracle-cloud", Status: models.EnvironmentUpdateResultStatusSkippedUpToDate},+ },+ }+ require.NoError(t, db.WithContext(ctx).Create(job).Error)++ svc.ResumeUpdateAllOnStartup(ctx)++ var got models.EnvironmentUpdateJob+ require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)++ // Job is finalized in-process (no re-run, not left running/pending).+ require.Equal(t, models.EnvironmentUpdateJobStatusCompleted, got.Status)+ require.NotNil(t, got.CompletedAt)+ require.Len(t, got.Results, 3)++ // Manager row transitioned updating -> updated (version changed across the restart).+ require.Equal(t, "0", got.Results[0].EnvironmentID)+ require.Equal(t, models.EnvironmentUpdateResultStatusUpdated, got.Results[0].Status)+ require.NotEmpty(t, got.Results[0].ToVersion)++ // Remote rows are untouched — proving the agents phase was NOT re-run on resume.+ require.Equal(t, models.EnvironmentUpdateResultStatusUpdated, got.Results[1].Status)+ require.Equal(t, models.EnvironmentUpdateResultStatusSkippedUpToDate, got.Results[2].Status)+}diff --git a/frontend/src/lib/components/dialogs/update-center-dialog.svelte b/frontend/src/lib/components/dialogs/update-center-dialog.svelteindex fee3ec3dc..f548e8929 100644--- a/frontend/src/lib/components/dialogs/update-center-dialog.svelte+++ b/frontend/src/lib/components/dialogs/update-center-dialog.svelte@@ -7,6 +7,7 @@import { queryKeys } from '$lib/query/query-keys';import { onDestroy } from 'svelte';import systemUpgradeService from '$lib/services/api/system-upgrade-service';+ import BaseAPIService from '$lib/services/api-service';import { cn } from '$lib/utils';import { ExternalLinkIcon, SuccessIcon } from '$lib/icons';import type { AppVersionInformation } from '$lib/types/settings';@@ -321,6 +322,7 @@} else {upgradeStatus = 'upgrading';upgrading = false;+ BaseAPIService.setUpgradeInProgress(false);}}}@@ -365,6 +367,9 @@async function handleConfirm() {upgrading = true;upgradeStatus = 'upgrading';+ // A local upgrade restarts THIS manager; flag it so the restart's version-+ // mismatch 401s are treated as a recoverable reconnect, not a logout.+ if (!isRemoteEnvironment) BaseAPIService.setUpgradeInProgress(true);log('confirm', {isRemoteEnvironment,environmentId: environmentId ?? '0',@@ -398,6 +403,7 @@if (triggerErrored) {upgrading = false;+ BaseAPIService.setUpgradeInProgress(false);return;}if (!upgrading) {@@ -430,6 +436,7 @@function closeAfterComplete() {upgrading = false;+ BaseAPIService.setUpgradeInProgress(false);open = false;}@@ -477,6 +484,7 @@if (countdownInterval) clearInterval(countdownInterval);if (fallbackTimeout) clearTimeout(fallbackTimeout);if (pollAbort) pollAbort.aborted = true;+ BaseAPIService.setUpgradeInProgress(false);});</script>diff --git a/frontend/src/lib/services/api-service.ts b/frontend/src/lib/services/api-service.tsindex 4452b30b5..bf94eae30 100644--- a/frontend/src/lib/services/api-service.ts+++ b/frontend/src/lib/services/api-service.ts@@ -161,6 +161,10 @@ function getRequestPath(url: string, baseURL: string): string {}let tokenRefreshHandler: (() => Promise<string | null>) | null = null;+// Set true while a manager self-update / fleet "Update All" is running. During that+// window the backend briefly restarts and returns version-mismatch 401s; we must not+// bounce the user to /login — the session is recoverable once the backend is back.+let upgradeInProgressInternal = false;const skipAuthPathsInternal = ['/auth/login','/auth/logout',@@ -193,18 +197,29 @@ export async function handleUnauthorizedResponseInternal(return 'none';}- const isVersionMismatch = serverMsg?.toLowerCase().includes('application has been updated');+ const isVersionMismatch = serverMsg?.toLowerCase().includes('application has been updated') ?? false;const isAuthApi = skipAuthPathsInternal.some((path) => requestPath.startsWith(path));const pathname = window.location.pathname || '/';const isOnAuthPage = isAuthPagePathInternal(pathname);- if (!isAuthApi && !isOnAuthPage && tokenRefreshHandler) {+ if (isAuthApi || isOnAuthPage) {+ return 'none';+ }++ // During a server self-update the backend briefly returns version-mismatch 401s+ // (and is momentarily unreachable mid-restart). The session is recoverable — the+ // refresh path rotates the token across the version change — so never bounce to+ // /login for these: refresh and retry when possible, otherwise let the caller+ // (e.g. the update poller) keep retrying until the backend is back.+ const recoverable = isVersionMismatch || upgradeInProgressInternal;++ if (tokenRefreshHandler) {try {await tokenRefreshHandler();return 'retry';} catch {- if (isVersionMismatch) {- toast.info('Application has been updated. Please log in again.');+ if (recoverable) {+ return 'none';}const redirectTo = encodeURIComponent(pathname);window.location.replace(`/login?redirect=${redirectTo}`);@@ -212,13 +227,6 @@ export async function handleUnauthorizedResponseInternal(}}- if (!isAuthApi && !isOnAuthPage && isVersionMismatch) {- toast.info('Application has been updated. Please log in again.');- const redirectTo = encodeURIComponent(pathname);- window.location.replace(`/login?redirect=${redirectTo}`);- return 'redirect';- }-return 'none';}@@ -392,6 +400,12 @@ abstract class BaseAPIService {tokenRefreshHandler = handler;}+ // Toggled by the update flows (Update All / local update center) so an in-progress+ // self-update restart is treated as a recoverable reconnect, not a logout.+ static setUpgradeInProgress(value: boolean) {+ upgradeInProgressInternal = value;+ }+protected async handleResponse<T>(promise: Promise<APIResponse>): Promise<T> {const response = await promise;const payload = response.data;diff --git a/frontend/src/lib/services/api/system-upgrade-service.ts b/frontend/src/lib/services/api/system-upgrade-service.tsindex 6a94cffe5..dedf13385 100644--- a/frontend/src/lib/services/api/system-upgrade-service.ts+++ b/frontend/src/lib/services/api/system-upgrade-service.ts@@ -71,9 +71,9 @@ async function triggerUpgrade(environmentId: string = '0'): Promise<UpgradeRespo}/**- * Trigger a fleet-wide update, upgrading the manager first and then every online- * remote environment that has an update available. No client timeout is set: the- * manager pulls the upgrader image before responding.+ * Trigger a fleet-wide update, upgrading every online remote environment that has an+ * update available first and then the manager itself (last) when it has an update.+ * No client timeout is set: the manager pulls the upgrader image before responding.*/async function triggerUpdateAll(): Promise<UpdateAllJob> {const res = await apiClient.post<ApiResponse<UpdateAllJob>>('/environments/0/system/upgrade/all');diff --git a/frontend/src/lib/services/auth-service.ts b/frontend/src/lib/services/auth-service.tsindex 18335e5bb..832db6cf3 100644--- a/frontend/src/lib/services/auth-service.ts+++ b/frontend/src/lib/services/auth-service.ts@@ -1,5 +1,5 @@import { goto, invalidateAll } from '$app/navigation';-import BaseAPIService from './api-service';+import BaseAPIService, { APIError } from './api-service';import userStore from '$lib/stores/user-store';import type { User } from '$lib/types/auth';import type { OidcStatusInfo } from '$lib/types/settings';@@ -111,7 +111,12 @@ class AuthService extends BaseAPIService {} catch (error) {const err = error instanceof Error ? error : new Error('Token refresh failed');console.error('Token refresh failed:', err);- this.clearTokenData();+ // Only drop the stored refresh token when the server definitively rejects it+ // (HTTP 401). Transient failures — a network error or the backend briefly+ // down during a self-update — must keep the token so a later retry recovers.+ if (error instanceof APIError && error.status === 401) {+ this.clearTokenData();+ }this.refreshSubscribers.forEach((callback) => callback(null, err));this.refreshSubscribers = [];this.isRefreshing = false;diff --git a/frontend/src/routes/(app)/environments/update-all-dialog.svelte b/frontend/src/routes/(app)/environments/update-all-dialog.svelteindex b6d340f93..88800c59d 100644--- a/frontend/src/routes/(app)/environments/update-all-dialog.svelte+++ b/frontend/src/routes/(app)/environments/update-all-dialog.svelte@@ -12,6 +12,7 @@type UpdateAllEnvironmentStatus} from '$lib/services/api/system-upgrade-service';import { SuccessIcon, CheckIcon, ClockIcon, AlertIcon, AlertTriangleIcon } from '$lib/icons';+ import BaseAPIService from '$lib/services/api-service';// open has no $bindable fallback: upstream binds can start out undefined, and// binding undefined to a $bindable with a fallback throws props_invalid_value.@@ -40,6 +41,7 @@// renders job/reconnecting, so clearing them here is safe.function resetState() {stopPolling();+ BaseAPIService.setUpgradeInProgress(false);phase = 'confirm';job = null;reconnecting = false;@@ -58,6 +60,7 @@job = next;if (next.status === 'completed' || next.status === 'failed') {stopPolling();+ BaseAPIService.setUpgradeInProgress(false);phase = 'finished';return;}@@ -72,6 +75,7 @@async function handleConfirm() {phase = 'running';reconnecting = false;+ BaseAPIService.setUpgradeInProgress(true);try {job = await systemUpgradeService.triggerUpdateAll();} catch {@@ -83,6 +87,7 @@if (phase !== 'running') return;if (job && (job.status === 'completed' || job.status === 'failed')) {+ BaseAPIService.setUpgradeInProgress(false);phase = 'finished';return;}@@ -92,12 +97,24 @@}async function handleClose() {+ // Capture before resetState() clears the job: the manager (env "0") upgrading+ // means our own frontend assets are now stale and need a reload.+ const managerUpdated = job?.results?.some((r) => r.environmentId === '0' && r.status === 'updated') ?? false;resetState();open = false;+ if (managerUpdated) {+ // Reload to pick up the new frontend assets. The session survives — the+ // cookie was re-minted by the recovered refresh during the restart.+ window.location.reload();+ return;+ }await onFinished?.();}- onDestroy(stopPolling);+ onDestroy(() => {+ stopPolling();+ BaseAPIService.setUpgradeInProgress(false);+ });const title = $derived.by(() => {if (phase === 'confirm') return m.environments_update_all_title();