Skip to content

Modern Docker Management, Designed for Everyone

fix: prevent logout and skipped environments during update all (#3000)

Kyle Mendellaacc3d2bb4ec
diff --git a/backend/api/middleware/auth.go b/backend/api/middleware/auth.go
index 4a6d48dba..17aba7ca3 100644
--- a/backend/api/middleware/auth.go
+++ b/backend/api/middleware/auth.go
@@ -321,7 +321,14 @@ func handleBearerAuthInternal(api huma.API, ctx huma.Context, authService *servi
newCtx = context.WithValue(newCtx, ContextKeyCurrentSessionID, sessionID)
return huma.WithContext(ctx, newCtx), true
}
- if errors.Is(err, services.ErrTokenVersionMismatch) || common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {
+ if errors.Is(err, services.ErrTokenVersionMismatch) {
+ // The app version changed (a self-update). The session is still valid — the
+ // refresh path tolerates the version change and rotates the token — so do NOT
+ // clear the auth cookies. Return a recoverable 401 the frontend refreshes from.
+ _ = huma.WriteErr(api, ctx, http.StatusUnauthorized, "Application has been updated. Refreshing session.")
+ return nil, true
+ }
+ if common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {
for _, cookieHeader := range cookie.BuildClearTokenCookieStringsFor(cookie.SecureCookieFromContext(ctx.Context())) {
ctx.AppendHeader("Set-Cookie", cookieHeader)
}
diff --git a/backend/api/middleware/auth_test.go b/backend/api/middleware/auth_test.go
index c30e73e4d..b54e67107 100644
--- a/backend/api/middleware/auth_test.go
+++ b/backend/api/middleware/auth_test.go
@@ -285,3 +285,85 @@ func TestNewAuthBridge_OpportunisticAuthOnPublicRoute(t *testing.T) {
require.Equal(t, "", sawSessionID)
})
}
+
+// After a self-update the app version changes and old access tokens fail the version
+// check. That must be RECOVERABLE (the refresh path rotates the token), so the
+// middleware returns a refreshable 401 and must NOT clear the auth cookies — otherwise
+// the user is logged out on every update.
+func TestNewAuthBridge_VersionMismatchIsRecoverable(t *testing.T) {
+ db := setupAuthMiddlewareTestDBInternal(t)
+ userSvc := services.NewUserService(db)
+ sessionSvc := services.NewSessionService(db)
+
+ jwtSecret := "test-secret-please-do-not-use-in-prod"
+ cfg := &config.Config{JWTRefreshExpiry: 24 * time.Hour}
+ authSvc := services.NewAuthService(userSvc, nil, nil, sessionSvc, nil, jwtSecret, cfg)
+
+ _, err := userSvc.CreateUser(context.Background(), &models.User{
+ BaseModel: models.BaseModel{ID: "u-ver"},
+ Username: "vertest",
+ })
+ require.NoError(t, err)
+
+ exp := time.Now().Add(5 * time.Minute)
+ session, _, err := sessionSvc.CreateSession(context.Background(), "u-ver", exp, auth.SessionMeta{})
+ require.NoError(t, err)
+
+ // An empty appVersion omits the claim, which passes the version check (no pin).
+ mintToken := func(appVersion string) string {
+ claims := jwt.MapClaims{
+ "jti": "u-ver",
+ "sub": "access",
+ "iat": time.Now().Unix(),
+ "exp": exp.Unix(),
+ "sid": session.ID,
+ "user_id": "u-ver",
+ "username": "vertest",
+ }
+ if appVersion != "" {
+ claims["app_version"] = appVersion
+ }
+ token, signErr := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(jwtSecret))
+ require.NoError(t, signErr)
+ return token
+ }
+
+ router := echo.New()
+ apiGroup := router.Group("/api")
+ humaConfig := huma.DefaultConfig("test", "1.0.0")
+ humaConfig.Components.SecuritySchemes = map[string]*huma.SecurityScheme{
+ "BearerAuth": {Type: "http", Scheme: "bearer"},
+ }
+ api := humaecho.NewWithGroup(router, apiGroup, humaConfig)
+ api.UseMiddleware(NewAuthBridge(api, authSvc, nil, nil, nil, &config.Config{}))
+
+ huma.Register(api, huma.Operation{
+ OperationID: "protected",
+ Method: http.MethodGet,
+ Path: "/protected",
+ Security: []map[string][]string{{"BearerAuth": {}}},
+ }, func(_ context.Context, _ *secureInput) (*secureOutput, error) {
+ return &secureOutput{}, nil
+ })
+
+ t.Run("version mismatch returns a recoverable 401 without clearing cookies", func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "/api/protected", nil)
+ req.Header.Set("Authorization", "Bearer "+mintToken("v0.0.0-stale"))
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+
+ require.Equal(t, http.StatusUnauthorized, rec.Code)
+ require.Contains(t, rec.Body.String(), "Application has been updated")
+ // The frontend recovers via refresh; clearing the cookies here would log the
+ // user out on every self-update.
+ require.Empty(t, rec.Header().Values("Set-Cookie"))
+ })
+
+ t.Run("token without a version pin still authenticates", func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "/api/protected", nil)
+ req.Header.Set("Authorization", "Bearer "+mintToken(""))
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+ require.Equal(t, http.StatusOK, rec.Code)
+ })
+}
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index bcd65df28..1ca1c7cec 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -33,11 +33,11 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
}
}
- // Resume a fleet-wide update-all job interrupted by the manager's own upgrade
- // restart (manager only). Runs off the app context so it can upgrade agents
- // without blocking bootstrap.
- if !appConfig.AgentMode && appServices.SystemUpgrade != nil && appServices.Environment != nil {
- go appServices.SystemUpgrade.ResumeUpdateAllOnStartup(appCtx, appServices.Environment)
+ // Finalize a fleet-wide update-all job whose manager self-upgrade restarted the
+ // process as its final step (manager only). Runs off the app context so it does
+ // not block bootstrap.
+ if !appConfig.AgentMode && appServices.SystemUpgrade != nil {
+ go appServices.SystemUpgrade.ResumeUpdateAllOnStartup(appCtx)
}
newScheduler.RegisterJob(jobs.AutoUpdate)
diff --git a/backend/internal/middleware/auth_middleware.go b/backend/internal/middleware/auth_middleware.go
index 4c274f6c4..2121c0c27 100644
--- a/backend/internal/middleware/auth_middleware.go
+++ b/backend/internal/middleware/auth_middleware.go
@@ -203,7 +203,17 @@ func (m *AuthMiddleware) managerAuth(ctx context.Context, c echo.Context, next e
user, sessionID, err := m.authService.VerifyToken(ctx, token)
if err != nil {
- if errors.Is(err, services.ErrTokenVersionMismatch) || common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {
+ // A version mismatch means the app self-updated; the session is still valid
+ // (the refresh path tolerates the version change and rotates the token), so do
+ // NOT clear the cookies. Return a recoverable 401 the frontend refreshes from.
+ if errors.Is(err, services.ErrTokenVersionMismatch) {
+ return c.JSON(http.StatusUnauthorized, models.APIError{
+ Code: models.APIErrorCodeUnauthorized,
+ Message: "Application has been updated. Refreshing session.",
+ })
+ }
+
+ if common.IsSessionRevokedError(err) || common.IsTokenValidationError(err) {
cookie.ClearTokenCookie(c.Response().Writer, req)
return c.JSON(http.StatusUnauthorized, models.APIError{
Code: models.APIErrorCodeUnauthorized,
diff --git a/backend/internal/models/environment_update_job.go b/backend/internal/models/environment_update_job.go
index 17dabb0dd..81f688310 100644
--- a/backend/internal/models/environment_update_job.go
+++ b/backend/internal/models/environment_update_job.go
@@ -12,9 +12,9 @@ import (
type EnvironmentUpdateJobStatus string
const (
- // EnvironmentUpdateJobStatusPendingRestart means the manager has triggered its
- // own self-upgrade and is waiting to restart on the new version, after which
- // the job resumes to upgrade the remote agents.
+ // EnvironmentUpdateJobStatusPendingRestart means the remote agents have been
+ // updated and the manager has triggered its own self-upgrade as the final step;
+ // it is waiting to restart on the new version, after which the job is finalized.
EnvironmentUpdateJobStatusPendingRestart EnvironmentUpdateJobStatus = "pending_restart"
// EnvironmentUpdateJobStatusRunning means the agents phase is in progress.
EnvironmentUpdateJobStatusRunning EnvironmentUpdateJobStatus = "running"
@@ -89,8 +89,8 @@ func (r *EnvironmentUpdateResults) Scan(value any) error {
}
// EnvironmentUpdateJob is a persisted fleet-wide update orchestration record. It
-// survives the manager's self-upgrade restart so the agents phase can resume on
-// the next boot. See [EnvironmentUpdateJobStatus].
+// survives the manager's final self-upgrade restart so the manager result can be
+// finalized on the next boot. See [EnvironmentUpdateJobStatus].
type EnvironmentUpdateJob struct {
BaseModel
diff --git a/backend/internal/services/system_upgrade_service.go b/backend/internal/services/system_upgrade_service.go
index fcab820d2..d530d519f 100644
--- a/backend/internal/services/system_upgrade_service.go
+++ b/backend/internal/services/system_upgrade_service.go
@@ -406,11 +406,13 @@ func (s *SystemUpgradeService) findArcaneContainerInternal(ctx context.Context,
// --- Fleet-wide "update all environments" orchestration ---
//
-// The manager upgrades itself first, which recreates its own container. Because
-// the browser loses the backend across that restart, the orchestration is a
-// persisted EnvironmentUpdateJob: StartUpdateAll triggers the manager self-upgrade
-// and leaves the job in pending_restart; on the next boot ResumeUpdateAllOnStartup
-// picks the job back up and upgrades each remote agent sequentially.
+// Remote agents upgrade first, while the manager is still up and can orchestrate
+// and report live progress. The manager upgrades itself LAST, which recreates its
+// own container. Because the browser loses the backend across that final restart,
+// the orchestration is a persisted EnvironmentUpdateJob: StartUpdateAll runs the
+// agents phase and, when the manager itself has an update, triggers the manager
+// self-upgrade as the last step and leaves the job in pending_restart; on the next
+// boot ResumeUpdateAllOnStartup finalizes the manager result and closes the job.
const (
localEnvironmentIDInternal = "0"
@@ -423,9 +425,10 @@ const (
updateAllErrorMaxLenInternal = 500
)
-// StartUpdateAll begins a fleet-wide update. If the manager has an update available
-// it self-upgrades first (job left pending_restart, resumed at next boot); otherwise
-// the agents phase runs immediately in the background.
+// StartUpdateAll begins a fleet-wide update. The agents phase runs first in the
+// background (while the manager is up); when the manager itself has an update, the
+// agents goroutine triggers the manager self-upgrade as its final step (job left
+// pending_restart, finalized at next boot).
func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.User, env *EnvironmentService) (*models.EnvironmentUpdateJob, error) {
// Guard the check-then-create against concurrent callers (e.g. a double-click).
// A dedicated flag rather than s.upgrading, because the manager branch below
@@ -466,29 +469,19 @@ func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.U
ManagerTargetVersion: updateAllTargetVersionInternal(info),
}
- if info.UpdateAvailable {
+ managerHasUpdate := info.UpdateAvailable
+ if managerHasUpdate {
+ // Manager upgrades LAST. Seed its row pending; the agents-phase goroutine
+ // flips it to updating right before triggering the self-upgrade.
managerResult.Status = models.EnvironmentUpdateResultStatusPending
managerResult.ToVersion = job.ManagerTargetVersion
- job.Status = models.EnvironmentUpdateJobStatusPendingRestart
- job.Results = append(models.EnvironmentUpdateResults{managerResult}, remoteResults...)
-
- if err := s.db.WithContext(ctx).Create(job).Error; err != nil {
- return nil, fmt.Errorf("create update-all job: %w", err)
- }
-
- // Trigger the manager self-upgrade. The manager restarts shortly after and
- // the job resumes on the next boot via ResumeUpdateAllOnStartup.
- if err := s.TriggerUpgradeViaCLI(ctx, user, updatertypes.SelfUpdateTarget{}); err != nil {
- s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))
- return nil, err
- }
-
- slog.InfoContext(ctx, "Update-all started; manager self-upgrade triggered", "jobId", job.ID, "user", user.Username)
- return job, nil
+ } else {
+ managerResult.Status = models.EnvironmentUpdateResultStatusSkippedUpToDate
}
- // Manager already up to date — go straight to the agents phase.
- managerResult.Status = models.EnvironmentUpdateResultStatusSkippedUpToDate
+ // Running from the start in both cases: the agents phase happens first, while the
+ // backend is up and can report progress. The manager self-upgrade (if any) fires
+ // at the very end of the agents goroutine.
job.Status = models.EnvironmentUpdateJobStatusRunning
job.Results = append(models.EnvironmentUpdateResults{managerResult}, remoteResults...)
@@ -496,16 +489,17 @@ func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.U
return nil, fmt.Errorf("create update-all job: %w", err)
}
- slog.InfoContext(ctx, "Update-all started; manager up to date, upgrading agents", "jobId", job.ID, "user", user.Username)
- go s.runAgentsPhaseInternal(context.WithoutCancel(ctx), job.ID, env)
+ slog.InfoContext(ctx, "Update-all started; upgrading agents first", "jobId", job.ID, "managerHasUpdate", managerHasUpdate, "user", user.Username)
+ go s.runAgentsPhaseInternal(context.WithoutCancel(ctx), job.ID, env, user, managerHasUpdate)
return job, nil
}
-// ResumeUpdateAllOnStartup is called once at manager startup. It transitions any
-// non-terminal update-all job out of its waiting state and, when appropriate, runs
-// the agents phase. It is a no-op when there is nothing pending.
-func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context, env *EnvironmentService) {
+// ResumeUpdateAllOnStartup is called once at manager startup. When the manager
+// self-upgraded as the final step of an update-all (job left pending_restart), the
+// agents phase already ran before the restart — so this finalizes the manager's own
+// result and closes the job. It is a no-op when there is nothing pending.
+func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context) {
job, err := s.activeUpdateAllJobInternal(ctx)
if err != nil {
slog.WarnContext(ctx, "Failed to load pending update-all job on startup", "error", err)
@@ -515,8 +509,8 @@ func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context, env
return
}
- // A job left running means the manager died mid-agents-phase; we can't safely
- // resume partial progress, so fail it.
+ // A job left running means the manager died mid-agents-phase, before it reached
+ // the manager step; we can't safely resume partial progress, so fail it.
if job.Status == models.EnvironmentUpdateJobStatusRunning {
s.markUpdateAllFailedInternal(ctx, job, "interrupted by manager restart")
return
@@ -530,15 +524,20 @@ func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context, env
return
}
+ // The agents phase already ran before the restart. Finalize the manager's own
+ // result and close the job — do not re-run the agents phase.
s.recordManagerResultInternal(job, action.managerSucceeded, info.CurrentVersion)
- job.Status = models.EnvironmentUpdateJobStatusRunning
+
+ completedAt := time.Now()
+ job.Status = models.EnvironmentUpdateJobStatusCompleted
+ job.CompletedAt = &completedAt
if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
- slog.WarnContext(ctx, "Failed to mark resumed update-all job as running", "jobId", job.ID, "error", err)
+ slog.WarnContext(ctx, "Failed to finalize resumed update-all job", "jobId", job.ID, "error", err)
return
}
- slog.InfoContext(ctx, "Resuming update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)
- s.runAgentsPhaseInternal(ctx, job.ID, env)
+ s.logUpdateAllEventInternal(ctx, job)
+ slog.InfoContext(ctx, "Finalized update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)
}
type resumeActionInternal struct {
@@ -563,8 +562,10 @@ func resolveResumeActionInternal(job *models.EnvironmentUpdateJob, currentVersio
// runAgentsPhaseInternal upgrades every online remote environment that has an update
// available, sequentially, persisting progress after each one so the status endpoint
-// can report live progress.
-func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID string, env *EnvironmentService) {
+// can report live progress. When managerHasUpdate is set, it triggers the manager's
+// own self-upgrade as the final step (leaving the job pending_restart for the next
+// boot to finalize); otherwise it marks the job completed in-process.
+func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID string, env *EnvironmentService, user models.User, managerHasUpdate bool) {
job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)
if err != nil || job == nil {
slog.WarnContext(ctx, "update-all: failed to reload job for agents phase", "jobId", jobID, "error", err)
@@ -596,6 +597,33 @@ func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID
}
}
+ // All remote agents processed. Handle the manager LAST.
+ if managerHasUpdate {
+ // Flip the manager row pending -> updating and move the job to pending_restart,
+ // persisting BEFORE the trigger so that if the manager dies the instant the
+ // upgrader starts, the next boot sees pending_restart and finalizes it.
+ for i := range job.Results {
+ if job.Results[i].EnvironmentID == localEnvironmentIDInternal {
+ job.Results[i].Status = models.EnvironmentUpdateResultStatusUpdating
+ break
+ }
+ }
+ job.Status = models.EnvironmentUpdateJobStatusPendingRestart
+ if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
+ slog.WarnContext(ctx, "update-all: failed to persist pending_restart before manager upgrade", "jobId", job.ID, "error", err)
+ }
+
+ if err := s.TriggerUpgradeViaCLI(ctx, user, updatertypes.SelfUpdateTarget{}); err != nil {
+ // Agents already ran and no restart is coming, so finalize now. This flips
+ // the manager's updating row to failed with the reason.
+ s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))
+ return
+ }
+
+ slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered; finalizing on next boot", "jobId", job.ID)
+ return
+ }
+
completedAt := time.Now()
job.Status = models.EnvironmentUpdateJobStatusCompleted
job.CompletedAt = &completedAt
diff --git a/backend/internal/services/system_upgrade_update_all_test.go b/backend/internal/services/system_upgrade_update_all_test.go
index 7f1a60ec2..1e9b24d6c 100644
--- a/backend/internal/services/system_upgrade_update_all_test.go
+++ b/backend/internal/services/system_upgrade_update_all_test.go
@@ -154,3 +154,53 @@ func TestUpdateAllFailedJobMarksUpdatingResultsFailed(t *testing.T) {
require.Equal(t, models.EnvironmentUpdateResultStatusFailed, got.Results[3].Status)
require.Equal(t, "already failed", got.Results[3].Error)
}
+
+// With the manager-last ordering, a resumed pending_restart job means the agents
+// phase already ran before the restart: resume must finalize the manager's own row
+// and complete the job, NOT re-run the agents phase.
+func TestResumeUpdateAllFinalizesManagerWithoutRerunningAgents(t *testing.T) {
+ ctx := context.Background()
+ gormDB, err := gorm.Open(glsqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+
+ db := &database.DB{DB: gormDB}
+ require.NoError(t, db.AutoMigrate(&models.EnvironmentUpdateJob{}, &models.Event{}))
+
+ // disabled=true keeps GetAppVersionInfo offline; nil docker => empty current digest.
+ // The reported version differs from ManagerVersionAtStart, so the manager upgrade
+ // is judged successful.
+ versionSvc := NewVersionService(nil, true, "v9.9.9-new", "", nil, nil, nil)
+ svc := NewSystemUpgradeService(db, nil, versionSvc, NewEventService(db, nil, nil), nil)
+
+ job := &models.EnvironmentUpdateJob{
+ Status: models.EnvironmentUpdateJobStatusPendingRestart,
+ UserID: "user-1",
+ Username: "arcane",
+ ManagerVersionAtStart: "v1.0.0-old",
+ Results: models.EnvironmentUpdateResults{
+ {EnvironmentID: "0", EnvironmentName: "Local", Status: models.EnvironmentUpdateResultStatusUpdating},
+ {EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: models.EnvironmentUpdateResultStatusUpdated},
+ {EnvironmentID: "remote-2", EnvironmentName: "oracle-cloud", Status: models.EnvironmentUpdateResultStatusSkippedUpToDate},
+ },
+ }
+ require.NoError(t, db.WithContext(ctx).Create(job).Error)
+
+ svc.ResumeUpdateAllOnStartup(ctx)
+
+ var got models.EnvironmentUpdateJob
+ require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)
+
+ // Job is finalized in-process (no re-run, not left running/pending).
+ require.Equal(t, models.EnvironmentUpdateJobStatusCompleted, got.Status)
+ require.NotNil(t, got.CompletedAt)
+ require.Len(t, got.Results, 3)
+
+ // Manager row transitioned updating -> updated (version changed across the restart).
+ require.Equal(t, "0", got.Results[0].EnvironmentID)
+ require.Equal(t, models.EnvironmentUpdateResultStatusUpdated, got.Results[0].Status)
+ require.NotEmpty(t, got.Results[0].ToVersion)
+
+ // Remote rows are untouched — proving the agents phase was NOT re-run on resume.
+ require.Equal(t, models.EnvironmentUpdateResultStatusUpdated, got.Results[1].Status)
+ require.Equal(t, models.EnvironmentUpdateResultStatusSkippedUpToDate, got.Results[2].Status)
+}
diff --git a/frontend/src/lib/components/dialogs/update-center-dialog.svelte b/frontend/src/lib/components/dialogs/update-center-dialog.svelte
index fee3ec3dc..f548e8929 100644
--- a/frontend/src/lib/components/dialogs/update-center-dialog.svelte
+++ b/frontend/src/lib/components/dialogs/update-center-dialog.svelte
@@ -7,6 +7,7 @@
import { queryKeys } from '$lib/query/query-keys';
import { onDestroy } from 'svelte';
import systemUpgradeService from '$lib/services/api/system-upgrade-service';
+ import BaseAPIService from '$lib/services/api-service';
import { cn } from '$lib/utils';
import { ExternalLinkIcon, SuccessIcon } from '$lib/icons';
import type { AppVersionInformation } from '$lib/types/settings';
@@ -321,6 +322,7 @@
} else {
upgradeStatus = 'upgrading';
upgrading = false;
+ BaseAPIService.setUpgradeInProgress(false);
}
}
}
@@ -365,6 +367,9 @@
async function handleConfirm() {
upgrading = true;
upgradeStatus = 'upgrading';
+ // A local upgrade restarts THIS manager; flag it so the restart's version-
+ // mismatch 401s are treated as a recoverable reconnect, not a logout.
+ if (!isRemoteEnvironment) BaseAPIService.setUpgradeInProgress(true);
log('confirm', {
isRemoteEnvironment,
environmentId: environmentId ?? '0',
@@ -398,6 +403,7 @@
if (triggerErrored) {
upgrading = false;
+ BaseAPIService.setUpgradeInProgress(false);
return;
}
if (!upgrading) {
@@ -430,6 +436,7 @@
function closeAfterComplete() {
upgrading = false;
+ BaseAPIService.setUpgradeInProgress(false);
open = false;
}
@@ -477,6 +484,7 @@
if (countdownInterval) clearInterval(countdownInterval);
if (fallbackTimeout) clearTimeout(fallbackTimeout);
if (pollAbort) pollAbort.aborted = true;
+ BaseAPIService.setUpgradeInProgress(false);
});
</script>
diff --git a/frontend/src/lib/services/api-service.ts b/frontend/src/lib/services/api-service.ts
index 4452b30b5..bf94eae30 100644
--- a/frontend/src/lib/services/api-service.ts
+++ b/frontend/src/lib/services/api-service.ts
@@ -161,6 +161,10 @@ function getRequestPath(url: string, baseURL: string): string {
}
let tokenRefreshHandler: (() => Promise<string | null>) | null = null;
+// Set true while a manager self-update / fleet "Update All" is running. During that
+// window the backend briefly restarts and returns version-mismatch 401s; we must not
+// bounce the user to /login — the session is recoverable once the backend is back.
+let upgradeInProgressInternal = false;
const skipAuthPathsInternal = [
'/auth/login',
'/auth/logout',
@@ -193,18 +197,29 @@ export async function handleUnauthorizedResponseInternal(
return 'none';
}
- const isVersionMismatch = serverMsg?.toLowerCase().includes('application has been updated');
+ const isVersionMismatch = serverMsg?.toLowerCase().includes('application has been updated') ?? false;
const isAuthApi = skipAuthPathsInternal.some((path) => requestPath.startsWith(path));
const pathname = window.location.pathname || '/';
const isOnAuthPage = isAuthPagePathInternal(pathname);
- if (!isAuthApi && !isOnAuthPage && tokenRefreshHandler) {
+ if (isAuthApi || isOnAuthPage) {
+ return 'none';
+ }
+
+ // During a server self-update the backend briefly returns version-mismatch 401s
+ // (and is momentarily unreachable mid-restart). The session is recoverable — the
+ // refresh path rotates the token across the version change — so never bounce to
+ // /login for these: refresh and retry when possible, otherwise let the caller
+ // (e.g. the update poller) keep retrying until the backend is back.
+ const recoverable = isVersionMismatch || upgradeInProgressInternal;
+
+ if (tokenRefreshHandler) {
try {
await tokenRefreshHandler();
return 'retry';
} catch {
- if (isVersionMismatch) {
- toast.info('Application has been updated. Please log in again.');
+ if (recoverable) {
+ return 'none';
}
const redirectTo = encodeURIComponent(pathname);
window.location.replace(`/login?redirect=${redirectTo}`);
@@ -212,13 +227,6 @@ export async function handleUnauthorizedResponseInternal(
}
}
- if (!isAuthApi && !isOnAuthPage && isVersionMismatch) {
- toast.info('Application has been updated. Please log in again.');
- const redirectTo = encodeURIComponent(pathname);
- window.location.replace(`/login?redirect=${redirectTo}`);
- return 'redirect';
- }
-
return 'none';
}
@@ -392,6 +400,12 @@ abstract class BaseAPIService {
tokenRefreshHandler = handler;
}
+ // Toggled by the update flows (Update All / local update center) so an in-progress
+ // self-update restart is treated as a recoverable reconnect, not a logout.
+ static setUpgradeInProgress(value: boolean) {
+ upgradeInProgressInternal = value;
+ }
+
protected async handleResponse<T>(promise: Promise<APIResponse>): Promise<T> {
const response = await promise;
const payload = response.data;
diff --git a/frontend/src/lib/services/api/system-upgrade-service.ts b/frontend/src/lib/services/api/system-upgrade-service.ts
index 6a94cffe5..dedf13385 100644
--- a/frontend/src/lib/services/api/system-upgrade-service.ts
+++ b/frontend/src/lib/services/api/system-upgrade-service.ts
@@ -71,9 +71,9 @@ async function triggerUpgrade(environmentId: string = '0'): Promise<UpgradeRespo
}
/**
- * Trigger a fleet-wide update, upgrading the manager first and then every online
- * remote environment that has an update available. No client timeout is set: the
- * manager pulls the upgrader image before responding.
+ * Trigger a fleet-wide update, upgrading every online remote environment that has an
+ * update available first and then the manager itself (last) when it has an update.
+ * No client timeout is set: the manager pulls the upgrader image before responding.
*/
async function triggerUpdateAll(): Promise<UpdateAllJob> {
const res = await apiClient.post<ApiResponse<UpdateAllJob>>('/environments/0/system/upgrade/all');
diff --git a/frontend/src/lib/services/auth-service.ts b/frontend/src/lib/services/auth-service.ts
index 18335e5bb..832db6cf3 100644
--- a/frontend/src/lib/services/auth-service.ts
+++ b/frontend/src/lib/services/auth-service.ts
@@ -1,5 +1,5 @@
import { goto, invalidateAll } from '$app/navigation';
-import BaseAPIService from './api-service';
+import BaseAPIService, { APIError } from './api-service';
import userStore from '$lib/stores/user-store';
import type { User } from '$lib/types/auth';
import type { OidcStatusInfo } from '$lib/types/settings';
@@ -111,7 +111,12 @@ class AuthService extends BaseAPIService {
} catch (error) {
const err = error instanceof Error ? error : new Error('Token refresh failed');
console.error('Token refresh failed:', err);
- this.clearTokenData();
+ // Only drop the stored refresh token when the server definitively rejects it
+ // (HTTP 401). Transient failures — a network error or the backend briefly
+ // down during a self-update — must keep the token so a later retry recovers.
+ if (error instanceof APIError && error.status === 401) {
+ this.clearTokenData();
+ }
this.refreshSubscribers.forEach((callback) => callback(null, err));
this.refreshSubscribers = [];
this.isRefreshing = false;
diff --git a/frontend/src/routes/(app)/environments/update-all-dialog.svelte b/frontend/src/routes/(app)/environments/update-all-dialog.svelte
index b6d340f93..88800c59d 100644
--- a/frontend/src/routes/(app)/environments/update-all-dialog.svelte
+++ b/frontend/src/routes/(app)/environments/update-all-dialog.svelte
@@ -12,6 +12,7 @@
type UpdateAllEnvironmentStatus
} from '$lib/services/api/system-upgrade-service';
import { SuccessIcon, CheckIcon, ClockIcon, AlertIcon, AlertTriangleIcon } from '$lib/icons';
+ import BaseAPIService from '$lib/services/api-service';
// open has no $bindable fallback: upstream binds can start out undefined, and
// binding undefined to a $bindable with a fallback throws props_invalid_value.
@@ -40,6 +41,7 @@
// renders job/reconnecting, so clearing them here is safe.
function resetState() {
stopPolling();
+ BaseAPIService.setUpgradeInProgress(false);
phase = 'confirm';
job = null;
reconnecting = false;
@@ -58,6 +60,7 @@
job = next;
if (next.status === 'completed' || next.status === 'failed') {
stopPolling();
+ BaseAPIService.setUpgradeInProgress(false);
phase = 'finished';
return;
}
@@ -72,6 +75,7 @@
async function handleConfirm() {
phase = 'running';
reconnecting = false;
+ BaseAPIService.setUpgradeInProgress(true);
try {
job = await systemUpgradeService.triggerUpdateAll();
} catch {
@@ -83,6 +87,7 @@
if (phase !== 'running') return;
if (job && (job.status === 'completed' || job.status === 'failed')) {
+ BaseAPIService.setUpgradeInProgress(false);
phase = 'finished';
return;
}
@@ -92,12 +97,24 @@
}
async function handleClose() {
+ // Capture before resetState() clears the job: the manager (env "0") upgrading
+ // means our own frontend assets are now stale and need a reload.
+ const managerUpdated = job?.results?.some((r) => r.environmentId === '0' && r.status === 'updated') ?? false;
resetState();
open = false;
+ if (managerUpdated) {
+ // Reload to pick up the new frontend assets. The session survives — the
+ // cookie was re-minted by the recovered refresh during the restart.
+ window.location.reload();
+ return;
+ }
await onFinished?.();
}
- onDestroy(stopPolling);
+ onDestroy(() => {
+ stopPolling();
+ BaseAPIService.setUpgradeInProgress(false);
+ });
const title = $derived.by(() => {
if (phase === 'confirm') return m.environments_update_all_title();