Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: show upgrade status for arcanes self-upgrade (#4278)
Kyle Mendelladc37dd5e0ce
diff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.goindex 6eb9bc4a7..2fd83b1f8 100644--- a/backend/cli/upgrade/upgrade.go+++ b/backend/cli/upgrade/upgrade.go@@ -545,7 +545,12 @@ func UpgradeContainer(ctx context.Context, dockerClient *client.Client, oldContafmt.Println("PROGRESS:70:Stopping old container")slog.Info("Stopping old container", "name", oldName)- if _, containerStopErr := dockerClient.ContainerStop(ctx, oldContainer.ID, client.ContainerStopOptions{Timeout: new(10)}); containerStopErr != nil {+ // Allow graceful shutdown and longer finite timeouts, but never wait forever.+ stopTimeout := 40+ if configured := oldContainer.Config.StopTimeout; configured != nil && *configured > stopTimeout {+ stopTimeout = *configured+ }+ if _, containerStopErr := dockerClient.ContainerStop(ctx, oldContainer.ID, client.ContainerStopOptions{Timeout: &stopTimeout}); containerStopErr != nil {_, _ = dockerClient.ContainerRename(ctx, oldContainer.ID, client.ContainerRenameOptions{NewName: originalName})return fmt.Errorf("stop old container: %w", containerStopErr)}diff --git a/backend/internal/system/children/upgrade/model.go b/backend/internal/system/children/upgrade/model.goindex e00cf5d1c..0dfb3f05e 100644--- a/backend/internal/system/children/upgrade/model.go+++ b/backend/internal/system/children/upgrade/model.go@@ -48,6 +48,16 @@ const (// EnvironmentUpdateResultStatus is the per-environment outcome within a job.type EnvironmentUpdateResultStatus string+// EnvironmentUpdateStage is the live step an updating environment is in.+type EnvironmentUpdateStage string++const (+ EnvironmentUpdateStageChecking EnvironmentUpdateStage = "checking"+ EnvironmentUpdateStageStarting EnvironmentUpdateStage = "starting"+ EnvironmentUpdateStageReconnecting EnvironmentUpdateStage = "reconnecting"+ EnvironmentUpdateStageVerifying EnvironmentUpdateStage = "verifying"+)+// EnvironmentUpdateResult is the outcome for a single environment within a job.// The manager appears as the first entry with EnvironmentID "0".type EnvironmentUpdateResult struct {@@ -56,9 +66,17 @@ type EnvironmentUpdateResult struct {Status EnvironmentUpdateResultStatus `json:"status"`FromVersion string `json:"fromVersion,omitempty"`ToVersion string `json:"toVersion,omitempty"`+ Stage EnvironmentUpdateStage `json:"stage,omitempty"`+ StageStartedAt *time.Time `json:"stageStartedAt,omitempty"`Error string `json:"error,omitempty"`}+// clearStageInternal drops live-progress fields once the row leaves updating.+func (r *EnvironmentUpdateResult) clearStageInternal() {+ r.Stage = ""+ r.StageStartedAt = nil+}+// EnvironmentUpdateResults is a JSON-serialized slice of per-environment results,// stored in a single TEXT column.//diff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.goindex 848625e67..87f982e24 100644--- a/backend/internal/system/children/upgrade/service.go+++ b/backend/internal/system/children/upgrade/service.go@@ -764,9 +764,7 @@ func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, envslog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistUpdateAllJobErr)}- result := job.Results[idx]- s.upgradeAgentInternal(ctx, env, remote.ID, &result)- job.Results[idx] = result+ s.upgradeAgentInternal(ctx, env, remote.ID, job, &job.Results[idx])if saveRemoteProgressErr := s.persistUpdateAllJobInternal(ctx, job); saveRemoteProgressErr != nil {slog.WarnContext(ctx, "update-all: failed to persist progress", "jobId", job.ID, "environmentId", remote.ID, "error", saveRemoteProgressErr)@@ -777,11 +775,11 @@ func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, env// updating and move the job to pending_restart, persisting BEFORE the trigger so// that if the manager dies the instant the upgrader starts, the next boot sees// pending_restart and finalizes it.- for i := range job.Results {- if job.Results[i].EnvironmentID == environment.LocalEnvironmentID {- job.Results[i].Status = EnvironmentUpdateResultStatusUpdating- break- }+ manager := managerResultInternal(job)+ if manager != nil {+ manager.Status = EnvironmentUpdateResultStatusUpdating+ manager.Stage = EnvironmentUpdateStageStarting+ manager.StageStartedAt = new(time.Now())}job.Status = EnvironmentUpdateJobStatusPendingRestartif savePendingRestartErr := s.persistUpdateAllJobInternal(ctx, job); savePendingRestartErr != nil {@@ -801,6 +799,10 @@ func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, envreturn}+ if manager != nil {+ s.setUpdateStageInternal(ctx, job, manager, EnvironmentUpdateStageReconnecting)+ }+slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)s.watchManagerUpgraderInternal(ctx, job.ID, upgraderID, wasAlreadyNewest)}@@ -983,10 +985,8 @@ func (s *Service) seedRemoteResultsInternal(ctx context.Context, env *environmen// appending a new pending row when seeding missed it (e.g. the seed list failed, or a// new environment was registered after the job started).func upsertPendingResultInternal(job *EnvironmentUpdateJob, envID, envName string) int {- for i := range job.Results {- if job.Results[i].EnvironmentID == envID {- return i- }+ if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {+ return idx}job.Results = append(job.Results, EnvironmentUpdateResult{EnvironmentID: envID,@@ -996,10 +996,21 @@ func upsertPendingResultInternal(job *EnvironmentUpdateJob, envID, envName strinreturn len(job.Results) - 1}+// managerResultInternal returns the manager's (env "0") row, or nil when the job has none.+func managerResultInternal(job *EnvironmentUpdateJob) *EnvironmentUpdateResult {+ idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })+ if idx < 0 {+ return nil+ }+ return &job.Results[idx]+}+// upgradeAgentInternal triggers and confirms a single remote environment's// self-upgrade, recording the outcome on result. The upgrade always runs — the// agent pulls the latest image even when it reports no update available.-func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.EnvironmentService, envID string, result *EnvironmentUpdateResult) {+func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.EnvironmentService, envID string, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult) {+ defer result.clearStageInternal()+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageChecking)versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)var info versiontypes.Infoerr := env.ProxyJSONRequest(versionCtx, envID, http.MethodGet, "/api/app-version", nil, &info)@@ -1026,6 +1037,7 @@ func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.Envresult.FromVersion = info.CurrentVersionresult.ToVersion = updateAllTargetVersionInternal(&info)+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageStarting)triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)resp, err := env.ExecuteRemoteRequest(triggerCtx, envID, http.MethodPost, "/api/environments/0/system/upgrade", triggerBody)cancel()@@ -1050,7 +1062,8 @@ func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.Envreturn}- if s.confirmAgentUpgradedInternal(ctx, env, envID, info) {+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)+ if s.confirmAgentUpgradedInternal(ctx, env, envID, info, job, result) {result.Status = EnvironmentUpdateResultStatusUpdated} else {// Upgrade fired but the new version was not confirmed within the wait window.@@ -1083,7 +1096,10 @@ func updateAllAgentFailureStatusInternal(err error) EnvironmentUpdateResultStatu// force-update of an already-latest agent — including one whose version check// could not determine the latest release — confirms on a same-image recreation// instead of timing out to triggered.-func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environment.EnvironmentService, envID string, baseline versiontypes.Info) bool {+func (s *Service) confirmAgentUpgradedInternal(+ ctx context.Context, env *environment.EnvironmentService, envID string,+ baseline versiontypes.Info, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult,+) bool {target := updateAllTargetVersionInternal(&baseline)deadline := time.Now().Add(updateAllConfirmTimeoutInternal)ticker := time.NewTicker(updateAllConfirmPollIntervalInternal)@@ -1099,6 +1115,7 @@ func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environerr := env.ProxyJSONRequest(reqCtx, envID, http.MethodGet, "/api/app-version", nil, &info)cancel()if err == nil {+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageVerifying)versionChanged := baseline.CurrentVersion != "" && info.CurrentVersion != baseline.CurrentVersiondigestChanged := baseline.CurrentDigest != "" && info.CurrentDigest != baseline.CurrentDigestonTarget := target != "" &&@@ -1107,6 +1124,8 @@ func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environif versionChanged || digestChanged || onTarget {return true}+ } else {+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)}if time.Now().After(deadline) {return false@@ -1115,6 +1134,19 @@ func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environ}}+// setUpdateStageInternal records which step result is in and persists the job so+// the status endpoint reflects it; re-entering the current stage keeps its start time.+func (s *Service) setUpdateStageInternal(ctx context.Context, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult, stage EnvironmentUpdateStage) {+ if result.Stage == stage {+ return+ }+ result.Stage = stage+ result.StageStartedAt = new(time.Now())+ if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {+ slog.WarnContext(ctx, "update-all: failed to persist stage", "jobId", job.ID, "environmentId", result.EnvironmentID, "stage", stage, "error", err)+ }+}+// GetLatestUpdateAllJob returns the most recently created update-all job, or nil.func (s *Service) GetLatestUpdateAllJob(ctx context.Context) (*EnvironmentUpdateJob, error) {var jobs []EnvironmentUpdateJob@@ -1171,6 +1203,7 @@ func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *Environmjob.Error = &reasonjob.CompletedAt = new(time.Now())for i := range job.Results {+ job.Results[i].clearStageInternal()if job.Results[i].Status == EnvironmentUpdateResultStatusUpdating {job.Results[i].Status = EnvironmentUpdateResultStatusFailedjob.Results[i].Error = reason@@ -1207,25 +1240,23 @@ func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *Environm// updated after a confirmed restart, up_to_date when the pull found nothing to swap// in, or failed otherwise.func (s *Service) recordManagerResultInternal(job *EnvironmentUpdateJob, status EnvironmentUpdateResultStatus, currentVersion string) {- for i := range job.Results {- if job.Results[i].EnvironmentID != environment.LocalEnvironmentID {- continue- }- job.Results[i].Status = status- switch status {- case EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusUpToDate:- job.Results[i].ToVersion = currentVersion- case EnvironmentUpdateResultStatusFailed:- job.Results[i].Error = "manager version did not change after upgrade"- case EnvironmentUpdateResultStatusPending,- EnvironmentUpdateResultStatusUpdating,- EnvironmentUpdateResultStatusTriggered,- EnvironmentUpdateResultStatusSkippedOffline:- // Nothing more to record: the row keeps the target version it was seeded- // with, since none of these outcomes establishes what it ended up running.- }+ manager := managerResultInternal(job)+ if manager == nil {return}+ manager.Status = status+ switch status {+ case EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusUpToDate:+ manager.ToVersion = currentVersion+ case EnvironmentUpdateResultStatusFailed:+ manager.Error = "manager version did not change after upgrade"+ case EnvironmentUpdateResultStatusPending,+ EnvironmentUpdateResultStatusUpdating,+ EnvironmentUpdateResultStatusTriggered,+ EnvironmentUpdateResultStatusSkippedOffline:+ // Nothing more to record: the row keeps the target version it was seeded+ // with, since none of these outcomes establishes what it ended up running.+ }}// finalizeUpdateAllJobInternal closes a job out as completed and records the audit@@ -1233,6 +1264,9 @@ func (s *Service) recordManagerResultInternal(job *EnvironmentUpdateJob, statusfunc (s *Service) finalizeUpdateAllJobInternal(ctx context.Context, job *EnvironmentUpdateJob) {job.Status = EnvironmentUpdateJobStatusCompletedjob.CompletedAt = new(time.Now())+ for i := range job.Results {+ job.Results[i].clearStageInternal()+ }if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)returndiff --git a/backend/internal/system/children/upgrade/service_test.go b/backend/internal/system/children/upgrade/service_test.goindex 2ad9991d4..0294f0dbd 100644--- a/backend/internal/system/children/upgrade/service_test.go+++ b/backend/internal/system/children/upgrade/service_test.go@@ -448,7 +448,10 @@ func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {Username: "arcane",ManagerVersionAtStart: "v1.0.0",Results: EnvironmentUpdateResults{- {EnvironmentID: "0", EnvironmentName: "Local", Status: EnvironmentUpdateResultStatusUpdating, FromVersion: "v1.0.0"},+ {+ EnvironmentID: "0", EnvironmentName: "Local", Status: EnvironmentUpdateResultStatusUpdating, FromVersion: "v1.0.0",+ Stage: EnvironmentUpdateStageReconnecting, StageStartedAt: new(time.Now()),+ },{EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: EnvironmentUpdateResultStatusUpToDate},},}@@ -470,6 +473,43 @@ func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {require.Equal(t, "v1.0.0", got.Results[0].ToVersion)require.Empty(t, got.Results[0].Error)require.Equal(t, EnvironmentUpdateResultStatusUpToDate, got.Results[1].Status)+ require.Empty(t, got.Results[0].Stage)+ require.Nil(t, got.Results[0].StageStartedAt)+}++// Each stage change is persisted immediately so the status endpoint shows live+// progress; re-entering the current stage must not restart its clock.+func TestUpdateAllStageChangesPersist(t *testing.T) {+ ctx := t.Context()+ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)++ db := &database.DB{DB: gormDB}+ require.NoError(t, db.AutoMigrate(&EnvironmentUpdateJob{}))++ svc := NewService(db, nil, nil, nil, nil, nil)+ job := &EnvironmentUpdateJob{+ Status: EnvironmentUpdateJobStatusRunning,+ Results: EnvironmentUpdateResults{+ {EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: EnvironmentUpdateResultStatusUpdating},+ },+ }+ require.NoError(t, db.WithContext(ctx).Create(job).Error)++ svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)+ started := job.Results[0].StageStartedAt+ require.NotNil(t, started)+ require.WithinDuration(t, time.Now(), *started, time.Second)++ svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)+ require.Same(t, started, job.Results[0].StageStartedAt)++ svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageStarting)+ var got EnvironmentUpdateJob+ require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)+ require.Equal(t, EnvironmentUpdateStageStarting, got.Results[0].Stage)+ require.NotNil(t, got.Results[0].StageStartedAt)+ require.NotSame(t, started, job.Results[0].StageStartedAt)}// The up-to-date short-circuit must only fire when the version check is conclusive:diff --git a/backend/pkg/libarcane/startup/runtime_identity_linux.go b/backend/pkg/libarcane/startup/runtime_identity_linux.goindex db1f3e784..87a6161f3 100644--- a/backend/pkg/libarcane/startup/runtime_identity_linux.go+++ b/backend/pkg/libarcane/startup/runtime_identity_linux.go@@ -8,7 +8,6 @@ import ("fmt""os""os/exec"- "os/signal""syscall""github.com/samber/mo"@@ -23,6 +22,8 @@ func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityRgroups := runtimeIdentitySupplementaryGroupsInternal(req.DockerHost, resolveSocketGroupInternal)cmd := exec.CommandContext(ctx, executable, os.Args[1:]...) //nolint:gosec // re-executing our own binary with the same args under a different UID/GID+ // Forward shutdown instead of the default SIGKILL so the child can unregister its actor host.+ cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }cmd.Env = os.Environ()cmd.Stdin = os.Stdincmd.Stdout = os.Stdout@@ -39,39 +40,23 @@ func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityRreturn fmt.Errorf("start runtime identity child: %w", startErr)}- sigCh := make(chan os.Signal, 2)- signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)-- done := make(chan error, 1)- go func() {- done <- cmd.Wait()- }()-- for {- select {- case sig := <-sigCh:- if cmd.Process != nil {- _ = cmd.Process.Signal(sig)- }- case waitErr := <-done:- signal.Stop(sigCh)- if waitErr == nil {- os.Exit(0)- }+ waitErr := cmd.Wait()+ // After a forwarded shutdown Wait reports ctx.Err() even when the child exited cleanly.+ if cmd.ProcessState != nil && cmd.ProcessState.Success() {+ os.Exit(0)+ }- if exitErr, ok := errors.AsType[*exec.ExitError](waitErr); ok {- if status, hasStatus := exitErr.Sys().(syscall.WaitStatus); hasStatus {- if status.Signaled() {- os.Exit(128 + int(status.Signal()))- }- os.Exit(status.ExitStatus())- }- os.Exit(exitErr.ExitCode())+ if exitErr, ok := errors.AsType[*exec.ExitError](waitErr); ok {+ if status, hasStatus := exitErr.Sys().(syscall.WaitStatus); hasStatus {+ if status.Signaled() {+ os.Exit(128 + int(status.Signal()))}-- return fmt.Errorf("wait for runtime identity child: %w", waitErr)+ os.Exit(status.ExitStatus())}+ os.Exit(exitErr.ExitCode())}++ return fmt.Errorf("wait for runtime identity child: %w", waitErr)}func resolveSocketGroupInternal(socketPath string) mo.Option[uint32] {diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 6d1c9f3de..8c3aa79f9 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -1538,13 +1538,24 @@"environments_update_all_title": "Update all environments","environments_update_all_message": "This pulls the latest Arcane image and updates every environment, whether or not an update is reported. Agents update first, then this manager, which restarts only if the pull brought a new image. Offline environments are skipped.","environments_update_all_manager_restarting": "Manager is restarting, reconnecting…",- "environments_update_all_manager_note": "Agents update first, then this manager. Arcane refreshes on its own when the run finishes.",+ "environments_update_all_manager_note": "Agents update first; the manager updates last. You can close this view and reopen it to follow the active update.","environments_update_all_in_progress": "Updating environments…","environments_update_all_progress": "{done} of {total}",+ "environments_update_all_stage_checking": "Checking version",+ "environments_update_all_stage_starting": "Starting update",+ "environments_update_all_stage_reconnecting": "Waiting for reconnection",+ "environments_update_all_stage_verifying": "Verifying version",+ "environments_update_all_stage_manager_restart": "Waiting for manager restart",+ "environments_update_all_elapsed": "Elapsed {duration}",+ "environments_update_all_progress_unavailable": "Progress unavailable",+ "environments_update_all_loading_status": "Checking update status…",+ "environments_update_all_finished_with_issues": "Finished with issues",+ "environments_update_all_reconnection_note": "Temporary disconnection is expected while restarting. Agents have up to five minutes to reconnect and confirm their version. Progress refreshes automatically.",+ "environments_update_all_summary": "{updated} confirmed updates · {current} already current · {skipped} skipped · {failed} failed · {unconfirmed} unconfirmed","environments_update_all_completed": "All environments processed","environments_update_all_failed": "Update all failed","environments_update_all_trigger_failed": "Failed to start update all",- "environments_update_all_status_triggered": "Update triggered",+ "environments_update_all_status_triggered": "Update requested; version not confirmed","environments_update_all_status_skipped_offline": "Offline — skipped","environments_edge_control_plane_label": "Control Plane","environments_edge_polling_active": "Polling Active",diff --git a/frontend/src/lib/components/dialogs/update-all-dialog.svelte b/frontend/src/lib/components/dialogs/update-all-dialog.svelteindex 72fd26c7b..c45ddf939 100644--- a/frontend/src/lib/components/dialogs/update-all-dialog.svelte+++ b/frontend/src/lib/components/dialogs/update-all-dialog.svelte@@ -1,6 +1,7 @@<script lang="ts">import { refreshAll } from '$app/navigation';import { onDestroy } from 'svelte';+ import type { Attachment } from 'svelte/attachments';import ReleaseNotes from '#lib/components/release-notes.svelte';import { Button } from '#lib/components/ui/button/index.js';@@ -8,16 +9,18 @@import Spinner from '#lib/components/ui/spinner/spinner.svelte';import { SuccessIcon, ClockIcon, AlertIcon, AlertTriangleIcon, ExternalLinkIcon } from '#lib/icons/index.js';import { m } from '#lib/paraglide/messages.js';- import BaseAPIService, { APIError } from '#lib/services/api-service.js';- import systemUpgradeService, {- type UpdateAllJob,- type UpdateAllEnvironmentResult,- type UpdateAllEnvironmentStatus- } from '#lib/services/api/system-upgrade-service.js';+ import { APIError } from '#lib/services/api-service.js';+ import systemUpgradeService from '#lib/services/api/system-upgrade-service.js';import type { AppVersionInformation } from '#lib/types/settings.js';+ import type {+ UpdateAllJob,+ UpdateAllEnvironmentResult,+ UpdateAllEnvironmentStatus,+ UpdateAllStage+ } from '#lib/types/system-upgrade.js';import { cn } from '#lib/utils.js';- import { handleApiResultWithCallbacks } from '#lib/utils/api.js';- import { formatRelativeTime, nowInstantString } from '#lib/utils/formatting.js';+ import { extractApiErrorMessage, handleApiResultWithCallbacks } from '#lib/utils/api.js';+ import { formatElapsedTime, formatRelativeTime, nowInstantString } from '#lib/utils/formatting.js';import { tryCatch } from '#lib/utils/try-catch.js';import VersionUpdateSummary from './version-update-summary.svelte';@@ -51,6 +54,9 @@let phase = $state<Phase>('confirm');let job = $state<UpdateAllJob | null>(null);let reconnecting = $state(false);+ let progressError = $state('');+ // Ticks once a second while the dialog is open so elapsed stage times advance.+ let clock = $state<string>();let pollActive = false;let pollTimer: ReturnType<typeof setTimeout> | null = null;// Bumped on every confirm and reset so a startup/status response that lands after@@ -66,16 +72,14 @@}}- // Reset on close (not on open) so a reopened dialog always starts at the confirm- // step, without mutating $state from inside an $effect. The confirm step never- // renders job/reconnecting, so clearing them here is safe.+ // Closing stops observing; the backend job and restart recovery keep running.function resetState() {startAttempt++;stopPolling();- BaseAPIService.setUpgradeInProgress(false);phase = 'confirm';job = null;reconnecting = false;+ progressError = '';}function schedulePoll() {@@ -83,56 +87,98 @@pollTimer = setTimeout(poll, POLL_INTERVAL_MS);}- function finishTerminalJob(terminalJob: UpdateAllJob) {- const managerRestarted =- terminalJob.results?.some((result) => result.environmentId === MANAGER_ENVIRONMENT_ID && result.status === 'updated') ??- false;- // Keep the upgrade flag armed when the manager actually restarted: the refresh in- // handleClose may hit the new backend with a stale token, and api-service only- // recovers that version-mismatch 401 while the flag is set.- if (!managerRestarted) {- BaseAPIService.setUpgradeInProgress(false);+ function isActiveJob(candidate: UpdateAllJob | null | undefined): candidate is UpdateAllJob {+ return candidate?.status === 'running' || candidate?.status === 'pending_restart';+ }++ function followJob(next: UpdateAllJob) {+ job = next;+ reconnecting = false;+ progressError = '';+ if (!isActiveJob(next)) {+ stopPolling();+ phase = 'finished';+ return;}- phase = 'finished';+ phase = 'running';+ if (!debugDemo) systemUpgradeService.monitorUpdateAllRecovery(next);+ pollActive = true;+ schedulePoll();+ }++ // A status read that never reached the backend, or hit a gateway with no backend+ // behind it, is what a manager restart looks like from the browser.+ function isConnectionLost(error: unknown): boolean {+ if (error instanceof TypeError) return true;+ if (!(error instanceof APIError)) return false;+ return !error.status || [502, 503, 504].includes(error.status);+ }++ function handleStatusError(error: unknown) {+ // No job has ever run: nothing to follow.+ if (!job && error instanceof APIError && error.status === 404) {+ stopPolling();+ phase = 'confirm';+ progressError = '';+ return;+ }+ reconnecting = job?.status === 'pending_restart' && isConnectionLost(error);+ progressError = reconnecting ? '' : extractApiErrorMessage(error);+ pollActive = true;+ schedulePoll();}async function poll() {if (!pollActive) return;-- const requestResult1 = await tryCatch(- (async () => {- const next = await systemUpgradeService.getUpdateAllStatus();- reconnecting = false;- job = next;- if (next.status === 'completed' || next.status === 'failed') {- stopPolling();- finishTerminalJob(next);- return true;- }- })()- );- if (requestResult1.error !== null) {- // The manager is likely restarting after its own upgrade — keep retrying- // until the backend answers again.- reconnecting = true;+ const attempt = startAttempt;+ const response = await tryCatch(systemUpgradeService.getUpdateAllStatus());+ if (!pollActive || attempt !== startAttempt) return;+ if (response.error !== null) {+ handleStatusError(response.error);+ return;}- if (requestResult1.data) return;+ followJob(response.data);+ }- schedulePoll();+ // Reopening the dialog picks up an update that is still running on the backend.+ async function restoreActiveJob() {+ const attempt = ++startAttempt;+ phase = 'running';+ const response = await tryCatch(systemUpgradeService.getUpdateAllStatus());+ if (attempt !== startAttempt) return;+ if (response.error !== null) {+ handleStatusError(response.error);+ return;+ }+ if (isActiveJob(response.data)) {+ followJob(response.data);+ } else {+ phase = 'confirm';+ }}+ // Dialog.Content only exists while the dialog is open, so this runs once per+ // opening and its cleanup once per close.+ const observeUpdate: Attachment = () => {+ if (!debugDemo) void restoreActiveJob();+ const ticker = setInterval(() => (clock = nowInstantString()), 1000);+ return () => {+ clearInterval(ticker);+ resetState();+ };+ };+async function handleConfirm() {const attempt = ++startAttempt;phase = 'running';reconnecting = false;+ progressError = '';if (debugDemo) {void runDebugDemo();return;}- BaseAPIService.setUpgradeInProgress(true);-const startResult = await tryCatch(systemUpgradeService.triggerUpdateAll());let next = startResult.data;// The backend refuses a second job while one is active (409). Adopt that job@@ -143,7 +189,7 @@for (let read = 0; read < CONFLICT_STATUS_READS && attempt === startAttempt; read++) {if (read > 0) await new Promise((resolve) => setTimeout(resolve, CONFLICT_STATUS_RETRY_MS));const active = (await tryCatch(systemUpgradeService.getUpdateAllStatus())).data;- if (active?.status === 'running' || active?.status === 'pending_restart') {+ if (isActiveJob(active)) {next = active;break;}@@ -158,17 +204,11 @@return;}- job = next;- if (job.status === 'completed' || job.status === 'failed') {- finishTerminalJob(job);- return;- }-- pollActive = true;- schedulePoll();+ followJob(next);}async function handleClose() {+ const attempt = ++startAttempt;stopPolling();// Refresh through SvelteKit instead of reloading the document: a hard reload// lands while agents are still reconnecting after the fleet restart, which@@ -183,6 +223,7 @@// A failed refresh must not trap the dialog open.}}+ if (attempt !== startAttempt) return;resetState();open = false;await onFinished?.();@@ -191,7 +232,6 @@onDestroy(() => {startAttempt++;stopPolling();- BaseAPIService.setUpgradeInProgress(false);});// Version/release presentation for the confirm step, rendered when the caller@@ -203,21 +243,31 @@return at ? formatRelativeTime(at) : '';});+ // "Done" is every environment that has reached a terminal state — i.e. anything+ // that isn't still queued (pending) or actively being worked on (updating).+ const results = $derived(job?.results ?? []);+ const managerResult = $derived(results.find((r) => r.environmentId === MANAGER_ENVIRONMENT_ID));+ const totalCount = $derived(results.length);+ const doneCount = $derived(results.filter((r) => r.status !== 'pending' && r.status !== 'updating').length);+ const outcomeCounts = $derived({+ updated: results.filter((r) => r.status === 'updated').length,+ current: results.filter((r) => r.status === 'up_to_date').length,+ skipped: results.filter((r) => r.status === 'skipped_offline').length,+ failed: results.filter((r) => r.status === 'failed').length,+ unconfirmed: results.filter((r) => r.status === 'triggered').length+ });+ const hasIssues = $derived(outcomeCounts.failed > 0 || outcomeCounts.unconfirmed > 0);+ const failed = $derived(job?.status === 'failed' || hasIssues);+const title = $derived.by(() => {if (phase === 'confirm') return m.environments_update_all_title();if (phase === 'finished') {- return job?.status === 'failed' ? m.environments_update_all_failed() : m.environments_update_all_completed();+ if (job?.status === 'failed') return m.environments_update_all_failed();+ return hasIssues ? m.environments_update_all_finished_with_issues() : m.environments_update_all_completed();}return m.environments_update_all_in_progress();});- // "Done" is every environment that has reached a terminal state — i.e. anything- // that isn't still queued (pending) or actively being worked on (updating).- const results = $derived(job?.results ?? []);- const totalCount = $derived(results.length);- const doneCount = $derived(results.filter((r) => r.status !== 'pending' && r.status !== 'updating').length);- const failed = $derived(job?.status === 'failed');-const completedColors = {segment: 'bg-success',badge: 'border-success/40 bg-success/10 text-success',@@ -240,7 +290,12 @@},updated: { label: m.common_updated, ...completedColors },up_to_date: { label: m.image_update_up_to_date_title, ...completedColors },- triggered: { label: m.environments_update_all_status_triggered, ...completedColors },+ triggered: {+ label: m.environments_update_all_status_triggered,+ segment: 'bg-warning',+ badge: 'border-warning/40 bg-warning/10 text-warning',+ text: 'text-warning'+ },skipped_offline: {label: m.environments_update_all_status_skipped_offline,segment: 'bg-warning',@@ -260,6 +315,23 @@return environmentStatusDisplay.get(status) ?? pendingDisplay;}+ const stageLabels: Record<UpdateAllStage, () => string> = {+ checking: m.environments_update_all_stage_checking,+ starting: m.environments_update_all_stage_starting,+ reconnecting: m.environments_update_all_stage_reconnecting,+ verifying: m.environments_update_all_stage_verifying+ };++ // While a row is updating its stage replaces the generic status label.+ function rowLabel(result: UpdateAllEnvironmentResult): string {+ const statusLabel = statusPresentation(result.status).label;+ if (result.status !== 'updating' || !result.stage) return statusLabel();+ if (result.stage === 'reconnecting' && result.environmentId === MANAGER_ENVIRONMENT_ID) {+ return m.environments_update_all_stage_manager_restart();+ }+ return (stageLabels[result.stage] ?? statusLabel)();+ }+// An environment that was already current reports the same version twice; show it// once rather than as a "v1.0.0 → v1.0.0" no-op transition.function versionLine(result: UpdateAllEnvironmentResult): string {@@ -273,7 +345,17 @@// designed and reviewed without triggering a real fleet update. Only ever enabled// from a dev-guarded callsite.async function runDebugDemo() {- const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));+ const attempt = startAttempt;+ const live = () => attempt === startAttempt && phase === 'running';+ // Sleeps, then reports whether the demo is still the one being watched.+ const pause = async (ms: number) => {+ await new Promise((resolve) => setTimeout(resolve, ms));+ return live();+ };+ const enterStage = (row: UpdateAllEnvironmentResult, stage: UpdateAllStage) => {+ row.stage = stage;+ row.stageStartedAt = nowInstantString();+ };// Big enough fleet (10) to exercise the scrolling list (#3655).const demo: Array<{ name: string; outcome: UpdateAllEnvironmentStatus; to: string; error?: string }> = [{ name: 'Local Docker', outcome: 'updated', to: 'v0.9.2' },@@ -303,50 +385,49 @@// Remotes first, manager last — the real processing order.const steps = [...demo.entries()].map(([index, entry]) => ({ index, entry }));for (const { index, entry } of [...steps.slice(1), ...steps.slice(0, 1)]) {- const starting = job?.results?.[index];- if (phase !== 'running' || !starting) return;- starting.status = 'updating';- await wait(700);-+ const row = job?.results?.[index];+ if (!live() || !row) return;+ row.status = 'updating';+ enterStage(row, 'checking');+ if (!(await pause(700))) return;+ enterStage(row, 'starting');+ if (!(await pause(700))) return;+ enterStage(row, 'reconnecting');+ // One slow reconnect and one status-read outage, to exercise both displays.+ if (index === 1 && !(await pause(5000))) return;+ if (index === 2) {+ progressError = m.environments_update_all_progress_unavailable();+ if (!(await pause(2000))) return;+ progressError = '';+ }// The manager restart is the one moment the reconnecting banner shows.- if (index === 0) {+ if (index === 0 && job) {+ job.status = 'pending_restart';reconnecting = true;- await wait(1600);+ if (!(await pause(3000))) return;reconnecting = false;}-- const row = job?.results?.[index];- if (phase !== 'running' || !row) return;+ enterStage(row, 'verifying');+ if (!(await pause(700))) return;row.status = entry.outcome;+ row.stage = undefined;+ row.stageStartedAt = undefined;row.toVersion = entry.to;if (entry.error) row.error = entry.error;}- if (phase !== 'running' || !job) return;+ if (!live() || !job) return;job.status = 'completed';job.completedAt = nowInstantString();phase = 'finished';}</script>-<Dialog.Root- {open}- onOpenChange={(next) => {- if (!next) {- resetState();- }- open = next;- }}->+<Dialog.Root {open} onOpenChange={(next) => (open = next)}><Dialog.Content+ {@attach observeUpdate}sectioned- class={cn(- 'flex max-h-(--max-height-dscreen-90) flex-col overflow-hidden sm:max-w-130',- phase === 'running' && '[&>button]:hidden'- )}- onInteractOutside={(e: Event) => {- if (phase === 'running') e.preventDefault();- }}+ class="flex max-h-(--max-height-dscreen-90) flex-col overflow-hidden sm:max-w-130">{#if phase === 'confirm'}<div class="px-6 pt-6 pb-4">@@ -416,12 +497,30 @@</div>{/if}+ {#if phase === 'finished'}+ <p class="mt-3 text-sm text-muted-foreground">{m.environments_update_all_summary(outcomeCounts)}</p>+ {#if job?.error}<p class="mt-2 text-sm text-destructive">{job.error}</p>{/if}+ {/if}+ {#if progressError}+ <div role="status" class="mt-4 rounded-lg border border-warning/20 bg-warning/5 px-3 py-2.5 text-sm">+ <p class="font-medium">{m.environments_update_all_progress_unavailable()}</p>+ <p class="mt-1 break-words text-muted-foreground">{progressError}</p>+ </div>+ {/if}+{#if reconnecting}<divclass="mt-4 flex items-center gap-2 rounded-lg border border-warning/20 bg-warning/5 px-3 py-2.5 text-sm text-warning"><Spinner class="size-4" />- <span>{m.environments_update_all_manager_restarting()}</span>+ <div>+ <span>{m.environments_update_all_manager_restarting()}</span>+ <span class="block text-xs">+ {m.environments_update_all_elapsed({+ duration: formatElapsedTime(managerResult?.stageStartedAt ?? job?.createdAt, { base: clock })+ })}+ </span>+ </div></div>{/if}</div>@@ -441,11 +540,11 @@statusPresentation(result.status).badge)}>- {#if result.status === 'updated' || result.status === 'triggered' || result.status === 'up_to_date'}+ {#if result.status === 'updated' || result.status === 'up_to_date'}<SuccessIcon class="size-3.5" />{:else if result.status === 'updating'}<Spinner class="size-3.5" />- {:else if result.status === 'skipped_offline'}+ {:else if result.status === 'skipped_offline' || result.status === 'triggered'}<AlertIcon class="size-3.5" />{:else if result.status === 'failed'}<AlertTriangleIcon class="size-3.5" />@@ -468,9 +567,16 @@{/if}</div>- <span class={cn('shrink-0 text-xs', statusPresentation(result.status).text)}>- {statusPresentation(result.status).label()}- </span>+ <div class={cn('max-w-48 shrink-0 text-right text-xs', statusPresentation(result.status).text)}>+ <span>{rowLabel(result)}</span>+ {#if result.status === 'updating' && result.stageStartedAt}+ <span class="block text-muted-foreground">+ {m.environments_update_all_elapsed({+ duration: formatElapsedTime(result.stageStartedAt, { base: clock })+ })}+ </span>+ {/if}+ </div></li>{/each}</ul>@@ -479,13 +585,14 @@{:else}<div class="flex items-center gap-2 border-t border-border/60 px-6 py-4 text-sm text-muted-foreground"><Spinner class="size-4" />- <span>{m.environments_update_all_in_progress()}</span>+ <span>{job ? m.environments_update_all_in_progress() : m.environments_update_all_loading_status()}</span></div>{/if}{#if phase === 'running'}<div class="border-t border-border/60 bg-muted/30 px-6 py-3"><p class="text-xs leading-relaxed text-muted-foreground">{m.environments_update_all_manager_note()}</p>+ <p class="mt-1 text-xs leading-relaxed text-muted-foreground">{m.environments_update_all_reconnection_note()}</p></div>{/if}{/if}diff --git a/frontend/src/lib/services/api-service.ts b/frontend/src/lib/services/api-service.tsindex ca7a01ec9..f22f179c9 100644--- a/frontend/src/lib/services/api-service.ts+++ b/frontend/src/lib/services/api-service.ts@@ -219,7 +219,10 @@ let tokenRefreshHandler: (() => Promise<string | null>) | null = null;// window the backend briefly restarts and returns version-mismatch 401s; we must not// bounce the user to /login — the session is recoverable once the backend is back.let upgradeInProgressInternal = false;+// A confirmed manager restart still needs document recovery after polling ends.+let upgradeReloadPendingInternal = false;let upgradeReloadStartedInternal = false;+let unauthorizedRedirectStartedInternal = false;const skipAuthPathsInternal = ['/auth/login','/auth/logout',@@ -257,6 +260,7 @@ export async function handleUnauthorizedResponseInternal(if (isAuthApi || isOnAuthPage) {return 'none';}+ if (unauthorizedRedirectStartedInternal) return 'redirect';// During a server self-update the backend briefly returns version-mismatch 401s.// Refresh first so the new backend rotates both tokens, then replace the stale@@ -269,7 +273,7 @@ export async function handleUnauthorizedResponseInternal(const operationResult = await tryCatch((async () => {await tokenRefreshHandler();- if (!isVersionMismatch || !upgradeInProgressInternal) return 'retry';+ if (!isVersionMismatch || (!upgradeInProgressInternal && !upgradeReloadPendingInternal)) return 'retry';if (!upgradeReloadStartedInternal) {upgradeReloadStartedInternal = true;window.location.reload();@@ -282,7 +286,10 @@ export async function handleUnauthorizedResponseInternal(const isTransientRefreshFailure =error instanceof APIError && (error.name === 'NetworkError' || error.name === 'TimeoutError');if (recoverable && isTransientRefreshFailure) return 'none';- window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);+ if (!unauthorizedRedirectStartedInternal) {+ unauthorizedRedirectStartedInternal = true;+ window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);+ }return 'redirect';} else {return operationResult.data;@@ -494,15 +501,16 @@ abstract class BaseAPIService {// Toggled by the update flows (Update All / local update center) so an in-progress// self-update restart is treated as a recoverable reconnect, not a logout.static setUpgradeInProgress(value: boolean) {- if (value && !upgradeInProgressInternal) {- upgradeReloadStartedInternal = false;- }- if (!value) {+ if (!upgradeReloadPendingInternal && (!value || !upgradeInProgressInternal)) {upgradeReloadStartedInternal = false;}upgradeInProgressInternal = value;}+ static confirmUpgradeRestart() {+ upgradeReloadPendingInternal = true;+ }+protected async handleResponse<T>(promise: Promise<APIResponse>): Promise<T> {const response = await promise;const payload = response.data;diff --git a/frontend/src/lib/services/api/system-upgrade-service.ts b/frontend/src/lib/services/api/system-upgrade-service.tsindex 40a40af01..4e375fb8b 100644--- a/frontend/src/lib/services/api/system-upgrade-service.ts+++ b/frontend/src/lib/services/api/system-upgrade-service.ts@@ -1,7 +1,14 @@import type { AppVersionInformation } from '#lib/types/settings.js';+import type { UpdateAllJob } from '#lib/types/system-upgrade.js';import { tryCatch } from '#lib/utils/try-catch.js';-import { apiClient } from '../api-service';+import BaseAPIService, { APIError, apiClient } from '../api-service';++const UPDATE_ALL_RECOVERY_POLL_MS = 3000;+const UPDATE_ALL_RECOVERY_MAX_UNAVAILABLE_MS = 5 * 60 * 1000;+let updateAllRecovery: Promise<void> | null = null;+let updateAllRecoveryAttempt = 0;+let updateAllRecoveryJobId: string | undefined;export interface UpgradeCheckResponse {canUpgrade: boolean;@@ -24,35 +31,6 @@ export interface HealthCheckResult {healthy: boolean;}-export type UpdateAllEnvironmentStatus =- | 'pending'- | 'updating'- | 'updated'- | 'up_to_date'- | 'triggered'- | 'skipped_offline'- | 'failed';--export interface UpdateAllEnvironmentResult {- environmentId: string;- environmentName: string;- status: UpdateAllEnvironmentStatus;- fromVersion?: string;- toVersion?: string;- error?: string;-}--export type UpdateAllJobStatus = 'pending_restart' | 'running' | 'completed' | 'failed';--export interface UpdateAllJob {- id: string;- status: UpdateAllJobStatus;- results?: UpdateAllEnvironmentResult[];- error?: string;- createdAt: string;- completedAt?: string;-}-type ApiResponse<T> = {success: boolean;data: T;@@ -89,8 +67,24 @@ async function triggerUpgrade(environmentId: string = '0'): Promise<UpgradeRespo* No client timeout is set: the manager pulls the upgrader image before responding.*/async function triggerUpdateAll(): Promise<UpdateAllJob> {- const res = await apiClient.post<ApiResponse<UpdateAllJob>>('/environments/0/system/upgrade/all');- return res.data.data;+ BaseAPIService.setUpgradeInProgress(true);+ const result = await tryCatch(apiClient.post<ApiResponse<UpdateAllJob>>('/environments/0/system/upgrade/all'));+ if (result.error !== null) {+ const error = result.error;+ const uncertain = !(error instanceof APIError) || !error.status || error.status >= 500 || [408, 409].includes(error.status);+ if (!updateAllRecovery) {+ BaseAPIService.setUpgradeInProgress(false);+ const active = uncertain ? (await tryCatch(getUpdateAllStatus())).data : null;+ // A concurrent successful start may have established recovery during this lookup.+ if (!updateAllRecovery && active && (active.status === 'running' || active.status === 'pending_restart')) {+ monitorUpdateAllRecovery(active);+ }+ }+ throw error;+ }+ const job = result.data.data.data;+ monitorUpdateAllRecovery(job);+ return job;}/**@@ -103,6 +97,42 @@ async function getUpdateAllStatus(): Promise<UpdateAllJob> {return res.data.data;}+// Recovery follows the job even after its dialog closes or unmounts.+function monitorUpdateAllRecovery(job: UpdateAllJob): void {+ if (typeof window === 'undefined') return;+ if (updateAllRecovery && job.id === updateAllRecoveryJobId) return;+ const attempt = ++updateAllRecoveryAttempt;+ updateAllRecoveryJobId = job.id;+ BaseAPIService.setUpgradeInProgress(true);+ updateAllRecovery = (async () => {+ let deadline = Date.now() + UPDATE_ALL_RECOVERY_MAX_UNAVAILABLE_MS;+ for (;;) {+ await new Promise((resolve) => setTimeout(resolve, UPDATE_ALL_RECOVERY_POLL_MS));+ if (attempt !== updateAllRecoveryAttempt) return;+ // An authenticated status read also drives refresh-and-reload after restart.+ const result = await tryCatch(getUpdateAllStatus());+ if (attempt !== updateAllRecoveryAttempt) return;+ if (result.error === null) {+ if (result.data.id !== job.id) break;+ if (result.data.status !== 'running' && result.data.status !== 'pending_restart') {+ if (result.data.results?.some((item) => item.environmentId === '0' && item.status === 'updated')) {+ BaseAPIService.confirmUpgradeRestart();+ }+ break;+ }+ deadline = Date.now() + UPDATE_ALL_RECOVERY_MAX_UNAVAILABLE_MS;+ } else if (Date.now() >= deadline) {+ break;+ }+ }+ })().finally(() => {+ if (attempt !== updateAllRecoveryAttempt) return;+ BaseAPIService.setUpgradeInProgress(false);+ updateAllRecovery = null;+ updateAllRecoveryJobId = undefined;+ });+}+/*** Check system health* @param environmentId - Optional environment ID for remote environments (defaults to local system)@@ -146,6 +176,7 @@ export default {triggerUpgrade,triggerUpdateAll,getUpdateAllStatus,+ monitorUpdateAllRecovery,checkHealth,getVersionInfo};diff --git a/frontend/src/lib/types/system-upgrade.ts b/frontend/src/lib/types/system-upgrade.tsnew file mode 100644index 000000000..fccf36806--- /dev/null+++ b/frontend/src/lib/types/system-upgrade.ts@@ -0,0 +1,32 @@+export type UpdateAllEnvironmentStatus =+ | 'pending'+ | 'updating'+ | 'updated'+ | 'up_to_date'+ | 'triggered'+ | 'skipped_offline'+ | 'failed';++export type UpdateAllStage = 'checking' | 'starting' | 'reconnecting' | 'verifying';++export interface UpdateAllEnvironmentResult {+ environmentId: string;+ environmentName: string;+ status: UpdateAllEnvironmentStatus;+ stage?: UpdateAllStage;+ stageStartedAt?: string;+ fromVersion?: string;+ toVersion?: string;+ error?: string;+}++export type UpdateAllJobStatus = 'pending_restart' | 'running' | 'completed' | 'failed';++export interface UpdateAllJob {+ id: string;+ status: UpdateAllJobStatus;+ results?: UpdateAllEnvironmentResult[];+ error?: string;+ createdAt: string;+ completedAt?: string;+}diff --git a/tests/spec/updates.spec.ts b/tests/spec/updates.spec.tsindex c47ec81fa..4e76d11ef 100644--- a/tests/spec/updates.spec.ts+++ b/tests/spec/updates.spec.ts@@ -448,6 +448,15 @@ test.describe('Arcane upgrades', () => {}async function openAndConfirmUpdateAll(page: Page) {+ // Opening checks for an existing run before showing confirmation. These tests+ // start a new run; their status fixtures apply only after this initial lookup.+ await page.route(+ '**/api/environments/0/system/upgrade/all/status',+ async (route) => {+ await fulfillProblem(route, 404, 'no update-all job found');+ },+ { times: 1 }+ );await page.getByRole('button', { name: 'Update All', exact: true }).first().click();const dialog = page.getByRole('dialog');await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();@@ -479,7 +488,7 @@ test.describe('Arcane upgrades', () => {});await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {statusCalls++;- await fulfillJob(route, 'running', 'updating');+ await fulfillJob(route, 'completed', 'updated');});await page.goto('/environments');@@ -491,11 +500,11 @@ test.describe('Arcane upgrades', () => {await expect(toast).toBeVisible({ timeout: 10_000 });await expect(toast).toContainText('pull access denied');- // Back at the confirm step, ready for another attempt — and no polling started.+ // The failed POST checks for an active job once, then returns to confirmation.const dialog = page.getByRole('dialog');await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();await expect(dialog.getByRole('button', { name: 'Update All', exact: true })).toBeVisible();- expect(statusCalls).toBe(0);+ expect(statusCalls).toBe(1);});test('a 409 conflict adopts the active job and follows it to completion', async ({ page }) => {@@ -507,8 +516,8 @@ test.describe('Arcane upgrades', () => {});await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {statusCalls++;- // First read (adoption) reports the job still running; the poll completes it.- if (statusCalls === 1) {+ // Recovery and the dialog each adopt the active job; later polls complete it.+ if (statusCalls <= 2) {await fulfillJob(route, 'running', 'updating');return;}@@ -560,10 +569,10 @@ test.describe('Arcane upgrades', () => {await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();// Wait past one poll interval to prove neither polling nor a retry kicked in.- // The three status reads are the persistence-window grace, not polling.+ // One recovery lookup and three dialog adoption reads precede this interval.await page.clock.runFor(4_000);expect(startCalls).toBe(1);- expect(statusCalls).toBe(3);+ expect(statusCalls).toBe(4);});test('a 409 conflict adopts a job that is persisted shortly after the conflict', async ({@@ -575,15 +584,15 @@ test.describe('Arcane upgrades', () => {});await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {statusCalls++;- // The winning client's job row does not exist yet on the first read.- if (statusCalls === 1) {+ // Recovery and the first dialog read precede persistence; the dialog retries.+ if (statusCalls <= 2) {await fulfillProblem(route, 404, 'no update-all job found');return;}await fulfillJob(route,- statusCalls === 2 ? 'running' : 'completed',- statusCalls === 2 ? 'updating' : 'updated'+ statusCalls === 3 ? 'running' : 'completed',+ statusCalls === 3 ? 'updating' : 'updated');});@@ -599,7 +608,7 @@ test.describe('Arcane upgrades', () => {await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);});- test('closing the dialog during a pending start ignores the late response', async ({+ test('closing during a pending start keeps recovery running without reopening the dialog', async ({page}) => {await page.clock.install();@@ -608,28 +617,39 @@ test.describe('Arcane upgrades', () => {releaseStart = resolve;});let statusCalls = 0;+ let startCalls = 0;await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {+ startCalls++;await startReleased;await fulfillJob(route, 'running', 'updating');});await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {statusCalls++;- await fulfillJob(route, 'running', 'updating');+ await fulfillJob(route, 'completed', 'updated');});await page.goto('/environments');await openAndConfirmUpdateAll(page);+ await expect.poll(() => startCalls).toBe(1);const dialog = page.getByRole('dialog');await expect(dialog.getByRole('heading', { name: 'Updating environments…' })).toBeVisible();await dialog.getByRole('button', { name: 'Close', exact: true }).first().click();await expect(dialog).toBeHidden();- // The start response arrives after closure: it must not reopen or start polling.+ // The late response starts service-owned recovery without reopening the dialog.+ const startResponse = page.waitForResponse(/\/api\/environments\/0\/system\/upgrade\/all$/);releaseStart();+ await (await startResponse).finished();+ await expect+ .poll(async () => {+ await page.clock.runFor(4_000);+ return statusCalls;+ })+ .toBe(1);await page.clock.runFor(4_000);await expect(dialog).toBeHidden();- expect(statusCalls).toBe(0);+ expect(statusCalls).toBe(1);await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);});});@@ -934,11 +954,17 @@ test.describe('Arcane upgrades', () => {}) => {await registerTokenSeeding(page);let refreshCalls = 0;+ let updateStarted = false;- await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {+ await page.route('**/api/environments/0/system/upgrade/all', async (route) => {+ updateStarted = true;await fulfillJob(route, 'running', 'updating');});- await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {+ await page.route('**/api/environments/0/system/upgrade/all/status', async (route) => {+ if (!updateStarted) {+ await fulfillProblem(route, 404, 'no update-all job found');+ return;+ }await route.fulfill({status: 401,contentType: 'application/json',@@ -946,9 +972,11 @@ test.describe('Arcane upgrades', () => {});});- await page.goto('/environments');- await openAndConfirmUpdateAll(page);- await page.route(/\/api\/auth\/refresh$/, async (route) => {+ await page.route('**/api/auth/refresh', async (route) => {+ if (!updateStarted) {+ await route.continue();+ return;+ }refreshCalls++;await route.fulfill({status: 401,@@ -956,7 +984,11 @@ test.describe('Arcane upgrades', () => {body: JSON.stringify({ message: 'Invalid or expired refresh token' })});});- await page.route(/\/api\/auth\/me$/, async (route) => {+ await page.route('**/api/auth/me', async (route) => {+ if (!updateStarted) {+ await route.continue();+ return;+ }await route.fulfill({status: 401,contentType: 'application/json',@@ -964,6 +996,10 @@ test.describe('Arcane upgrades', () => {});});+ await page.goto('/environments');+ await openAndConfirmUpdateAll(page);+ await expect.poll(() => updateStarted).toBe(true);+await page.waitForURL(/\/login(?:\?|$)/, { timeout: 10_000 });expect(refreshCalls).toBe(1);await expect(@@ -975,14 +1011,24 @@ test.describe('Arcane upgrades', () => {page}) => {let refreshCalls = 0;- await page.route(/\/api\/auth\/refresh$/, async (route) => {+ let updateStarted = false;+ await page.route('**/api/auth/refresh', async (route) => {+ if (!updateStarted) {+ await route.continue();+ return;+ }refreshCalls++;await route.continue();});- await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {+ await page.route('**/api/environments/0/system/upgrade/all', async (route) => {+ updateStarted = true;await fulfillJob(route, 'running', 'updating');});- await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {+ await page.route('**/api/environments/0/system/upgrade/all/status', async (route) => {+ if (!updateStarted) {+ await fulfillProblem(route, 404, 'no update-all job found');+ return;+ }await route.fulfill({status: 401,contentType: 'application/json',@@ -990,9 +1036,11 @@ test.describe('Arcane upgrades', () => {});});- await page.goto('/environments');- await openAndConfirmUpdateAll(page);- await page.route(/\/api\/auth\/me$/, async (route) => {+ await page.route('**/api/auth/me', async (route) => {+ if (!updateStarted) {+ await route.continue();+ return;+ }await route.fulfill({status: 401,contentType: 'application/json',@@ -1000,6 +1048,10 @@ test.describe('Arcane upgrades', () => {});});+ await page.goto('/environments');+ await openAndConfirmUpdateAll(page);+ await expect.poll(() => updateStarted).toBe(true);+await page.waitForURL(/\/login(?:\?|$)/, { timeout: 10_000 });expect(refreshCalls).toBe(0);await expect(