Skip to content

Modern Docker Management, Designed for Everyone

fix: show upgrade status for arcanes self-upgrade (#4278)

Kyle Mendelladc37dd5e0ce
diff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.go
index 6eb9bc4a7..2fd83b1f8 100644
--- a/backend/cli/upgrade/upgrade.go
+++ b/backend/cli/upgrade/upgrade.go
@@ -545,7 +545,12 @@ func UpgradeContainer(ctx context.Context, dockerClient *client.Client, oldConta
fmt.Println("PROGRESS:70:Stopping old container")
slog.Info("Stopping old container", "name", oldName)
- if _, containerStopErr := dockerClient.ContainerStop(ctx, oldContainer.ID, client.ContainerStopOptions{Timeout: new(10)}); containerStopErr != nil {
+ // Allow graceful shutdown and longer finite timeouts, but never wait forever.
+ stopTimeout := 40
+ if configured := oldContainer.Config.StopTimeout; configured != nil && *configured > stopTimeout {
+ stopTimeout = *configured
+ }
+ if _, containerStopErr := dockerClient.ContainerStop(ctx, oldContainer.ID, client.ContainerStopOptions{Timeout: &stopTimeout}); containerStopErr != nil {
_, _ = dockerClient.ContainerRename(ctx, oldContainer.ID, client.ContainerRenameOptions{NewName: originalName})
return fmt.Errorf("stop old container: %w", containerStopErr)
}
diff --git a/backend/internal/system/children/upgrade/model.go b/backend/internal/system/children/upgrade/model.go
index e00cf5d1c..0dfb3f05e 100644
--- a/backend/internal/system/children/upgrade/model.go
+++ b/backend/internal/system/children/upgrade/model.go
@@ -48,6 +48,16 @@ const (
// EnvironmentUpdateResultStatus is the per-environment outcome within a job.
type EnvironmentUpdateResultStatus string
+// EnvironmentUpdateStage is the live step an updating environment is in.
+type EnvironmentUpdateStage string
+
+const (
+ EnvironmentUpdateStageChecking EnvironmentUpdateStage = "checking"
+ EnvironmentUpdateStageStarting EnvironmentUpdateStage = "starting"
+ EnvironmentUpdateStageReconnecting EnvironmentUpdateStage = "reconnecting"
+ EnvironmentUpdateStageVerifying EnvironmentUpdateStage = "verifying"
+)
+
// EnvironmentUpdateResult is the outcome for a single environment within a job.
// The manager appears as the first entry with EnvironmentID "0".
type EnvironmentUpdateResult struct {
@@ -56,9 +66,17 @@ type EnvironmentUpdateResult struct {
Status EnvironmentUpdateResultStatus `json:"status"`
FromVersion string `json:"fromVersion,omitempty"`
ToVersion string `json:"toVersion,omitempty"`
+ Stage EnvironmentUpdateStage `json:"stage,omitempty"`
+ StageStartedAt *time.Time `json:"stageStartedAt,omitempty"`
Error string `json:"error,omitempty"`
}
+// clearStageInternal drops live-progress fields once the row leaves updating.
+func (r *EnvironmentUpdateResult) clearStageInternal() {
+ r.Stage = ""
+ r.StageStartedAt = nil
+}
+
// EnvironmentUpdateResults is a JSON-serialized slice of per-environment results,
// stored in a single TEXT column.
//
diff --git a/backend/internal/system/children/upgrade/service.go b/backend/internal/system/children/upgrade/service.go
index 848625e67..87f982e24 100644
--- a/backend/internal/system/children/upgrade/service.go
+++ b/backend/internal/system/children/upgrade/service.go
@@ -764,9 +764,7 @@ func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, env
slog.WarnContext(ctx, "update-all: failed to persist updating status", "jobId", job.ID, "environmentId", remote.ID, "error", persistUpdateAllJobErr)
}
- result := job.Results[idx]
- s.upgradeAgentInternal(ctx, env, remote.ID, &result)
- job.Results[idx] = result
+ s.upgradeAgentInternal(ctx, env, remote.ID, job, &job.Results[idx])
if saveRemoteProgressErr := s.persistUpdateAllJobInternal(ctx, job); saveRemoteProgressErr != nil {
slog.WarnContext(ctx, "update-all: failed to persist progress", "jobId", job.ID, "environmentId", remote.ID, "error", saveRemoteProgressErr)
@@ -777,11 +775,11 @@ func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, env
// updating and move the job to pending_restart, persisting BEFORE the trigger so
// that if the manager dies the instant the upgrader starts, the next boot sees
// pending_restart and finalizes it.
- for i := range job.Results {
- if job.Results[i].EnvironmentID == environment.LocalEnvironmentID {
- job.Results[i].Status = EnvironmentUpdateResultStatusUpdating
- break
- }
+ manager := managerResultInternal(job)
+ if manager != nil {
+ manager.Status = EnvironmentUpdateResultStatusUpdating
+ manager.Stage = EnvironmentUpdateStageStarting
+ manager.StageStartedAt = new(time.Now())
}
job.Status = EnvironmentUpdateJobStatusPendingRestart
if savePendingRestartErr := s.persistUpdateAllJobInternal(ctx, job); savePendingRestartErr != nil {
@@ -801,6 +799,10 @@ func (s *Service) runAgentsPhaseInternal(ctx context.Context, jobID string, env
return
}
+ if manager != nil {
+ s.setUpdateStageInternal(ctx, job, manager, EnvironmentUpdateStageReconnecting)
+ }
+
slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)
s.watchManagerUpgraderInternal(ctx, job.ID, upgraderID, wasAlreadyNewest)
}
@@ -983,10 +985,8 @@ func (s *Service) seedRemoteResultsInternal(ctx context.Context, env *environmen
// appending a new pending row when seeding missed it (e.g. the seed list failed, or a
// new environment was registered after the job started).
func upsertPendingResultInternal(job *EnvironmentUpdateJob, envID, envName string) int {
- for i := range job.Results {
- if job.Results[i].EnvironmentID == envID {
- return i
- }
+ if idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == envID }); idx >= 0 {
+ return idx
}
job.Results = append(job.Results, EnvironmentUpdateResult{
EnvironmentID: envID,
@@ -996,10 +996,21 @@ func upsertPendingResultInternal(job *EnvironmentUpdateJob, envID, envName strin
return len(job.Results) - 1
}
+// managerResultInternal returns the manager's (env "0") row, or nil when the job has none.
+func managerResultInternal(job *EnvironmentUpdateJob) *EnvironmentUpdateResult {
+ idx := slices.IndexFunc(job.Results, func(r EnvironmentUpdateResult) bool { return r.EnvironmentID == environment.LocalEnvironmentID })
+ if idx < 0 {
+ return nil
+ }
+ return &job.Results[idx]
+}
+
// upgradeAgentInternal triggers and confirms a single remote environment's
// self-upgrade, recording the outcome on result. The upgrade always runs — the
// agent pulls the latest image even when it reports no update available.
-func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.EnvironmentService, envID string, result *EnvironmentUpdateResult) {
+func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.EnvironmentService, envID string, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult) {
+ defer result.clearStageInternal()
+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageChecking)
versionCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)
var info versiontypes.Info
err := env.ProxyJSONRequest(versionCtx, envID, http.MethodGet, "/api/app-version", nil, &info)
@@ -1026,6 +1037,7 @@ func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.Env
result.FromVersion = info.CurrentVersion
result.ToVersion = updateAllTargetVersionInternal(&info)
+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageStarting)
triggerCtx, cancel := context.WithTimeout(ctx, updateAllAgentRequestTimeoutInternal)
resp, err := env.ExecuteRemoteRequest(triggerCtx, envID, http.MethodPost, "/api/environments/0/system/upgrade", triggerBody)
cancel()
@@ -1050,7 +1062,8 @@ func (s *Service) upgradeAgentInternal(ctx context.Context, env *environment.Env
return
}
- if s.confirmAgentUpgradedInternal(ctx, env, envID, info) {
+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)
+ if s.confirmAgentUpgradedInternal(ctx, env, envID, info, job, result) {
result.Status = EnvironmentUpdateResultStatusUpdated
} else {
// Upgrade fired but the new version was not confirmed within the wait window.
@@ -1083,7 +1096,10 @@ func updateAllAgentFailureStatusInternal(err error) EnvironmentUpdateResultStatu
// force-update of an already-latest agent — including one whose version check
// could not determine the latest release — confirms on a same-image recreation
// instead of timing out to triggered.
-func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environment.EnvironmentService, envID string, baseline versiontypes.Info) bool {
+func (s *Service) confirmAgentUpgradedInternal(
+ ctx context.Context, env *environment.EnvironmentService, envID string,
+ baseline versiontypes.Info, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult,
+) bool {
target := updateAllTargetVersionInternal(&baseline)
deadline := time.Now().Add(updateAllConfirmTimeoutInternal)
ticker := time.NewTicker(updateAllConfirmPollIntervalInternal)
@@ -1099,6 +1115,7 @@ func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environ
err := env.ProxyJSONRequest(reqCtx, envID, http.MethodGet, "/api/app-version", nil, &info)
cancel()
if err == nil {
+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageVerifying)
versionChanged := baseline.CurrentVersion != "" && info.CurrentVersion != baseline.CurrentVersion
digestChanged := baseline.CurrentDigest != "" && info.CurrentDigest != baseline.CurrentDigest
onTarget := target != "" &&
@@ -1107,6 +1124,8 @@ func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environ
if versionChanged || digestChanged || onTarget {
return true
}
+ } else {
+ s.setUpdateStageInternal(ctx, job, result, EnvironmentUpdateStageReconnecting)
}
if time.Now().After(deadline) {
return false
@@ -1115,6 +1134,19 @@ func (s *Service) confirmAgentUpgradedInternal(ctx context.Context, env *environ
}
}
+// setUpdateStageInternal records which step result is in and persists the job so
+// the status endpoint reflects it; re-entering the current stage keeps its start time.
+func (s *Service) setUpdateStageInternal(ctx context.Context, job *EnvironmentUpdateJob, result *EnvironmentUpdateResult, stage EnvironmentUpdateStage) {
+ if result.Stage == stage {
+ return
+ }
+ result.Stage = stage
+ result.StageStartedAt = new(time.Now())
+ if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
+ slog.WarnContext(ctx, "update-all: failed to persist stage", "jobId", job.ID, "environmentId", result.EnvironmentID, "stage", stage, "error", err)
+ }
+}
+
// GetLatestUpdateAllJob returns the most recently created update-all job, or nil.
func (s *Service) GetLatestUpdateAllJob(ctx context.Context) (*EnvironmentUpdateJob, error) {
var jobs []EnvironmentUpdateJob
@@ -1171,6 +1203,7 @@ func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *Environm
job.Error = &reason
job.CompletedAt = new(time.Now())
for i := range job.Results {
+ job.Results[i].clearStageInternal()
if job.Results[i].Status == EnvironmentUpdateResultStatusUpdating {
job.Results[i].Status = EnvironmentUpdateResultStatusFailed
job.Results[i].Error = reason
@@ -1207,25 +1240,23 @@ func (s *Service) markUpdateAllFailedInternal(ctx context.Context, job *Environm
// updated after a confirmed restart, up_to_date when the pull found nothing to swap
// in, or failed otherwise.
func (s *Service) recordManagerResultInternal(job *EnvironmentUpdateJob, status EnvironmentUpdateResultStatus, currentVersion string) {
- for i := range job.Results {
- if job.Results[i].EnvironmentID != environment.LocalEnvironmentID {
- continue
- }
- job.Results[i].Status = status
- switch status {
- case EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusUpToDate:
- job.Results[i].ToVersion = currentVersion
- case EnvironmentUpdateResultStatusFailed:
- job.Results[i].Error = "manager version did not change after upgrade"
- case EnvironmentUpdateResultStatusPending,
- EnvironmentUpdateResultStatusUpdating,
- EnvironmentUpdateResultStatusTriggered,
- EnvironmentUpdateResultStatusSkippedOffline:
- // Nothing more to record: the row keeps the target version it was seeded
- // with, since none of these outcomes establishes what it ended up running.
- }
+ manager := managerResultInternal(job)
+ if manager == nil {
return
}
+ manager.Status = status
+ switch status {
+ case EnvironmentUpdateResultStatusUpdated, EnvironmentUpdateResultStatusUpToDate:
+ manager.ToVersion = currentVersion
+ case EnvironmentUpdateResultStatusFailed:
+ manager.Error = "manager version did not change after upgrade"
+ case EnvironmentUpdateResultStatusPending,
+ EnvironmentUpdateResultStatusUpdating,
+ EnvironmentUpdateResultStatusTriggered,
+ EnvironmentUpdateResultStatusSkippedOffline:
+ // Nothing more to record: the row keeps the target version it was seeded
+ // with, since none of these outcomes establishes what it ended up running.
+ }
}
// finalizeUpdateAllJobInternal closes a job out as completed and records the audit
@@ -1233,6 +1264,9 @@ func (s *Service) recordManagerResultInternal(job *EnvironmentUpdateJob, status
func (s *Service) finalizeUpdateAllJobInternal(ctx context.Context, job *EnvironmentUpdateJob) {
job.Status = EnvironmentUpdateJobStatusCompleted
job.CompletedAt = new(time.Now())
+ for i := range job.Results {
+ job.Results[i].clearStageInternal()
+ }
if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {
slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)
return
diff --git a/backend/internal/system/children/upgrade/service_test.go b/backend/internal/system/children/upgrade/service_test.go
index 2ad9991d4..0294f0dbd 100644
--- a/backend/internal/system/children/upgrade/service_test.go
+++ b/backend/internal/system/children/upgrade/service_test.go
@@ -448,7 +448,10 @@ func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {
Username: "arcane",
ManagerVersionAtStart: "v1.0.0",
Results: EnvironmentUpdateResults{
- {EnvironmentID: "0", EnvironmentName: "Local", Status: EnvironmentUpdateResultStatusUpdating, FromVersion: "v1.0.0"},
+ {
+ EnvironmentID: "0", EnvironmentName: "Local", Status: EnvironmentUpdateResultStatusUpdating, FromVersion: "v1.0.0",
+ Stage: EnvironmentUpdateStageReconnecting, StageStartedAt: new(time.Now()),
+ },
{EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: EnvironmentUpdateResultStatusUpToDate},
},
}
@@ -470,6 +473,43 @@ func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {
require.Equal(t, "v1.0.0", got.Results[0].ToVersion)
require.Empty(t, got.Results[0].Error)
require.Equal(t, EnvironmentUpdateResultStatusUpToDate, got.Results[1].Status)
+ require.Empty(t, got.Results[0].Stage)
+ require.Nil(t, got.Results[0].StageStartedAt)
+}
+
+// Each stage change is persisted immediately so the status endpoint shows live
+// progress; re-entering the current stage must not restart its clock.
+func TestUpdateAllStageChangesPersist(t *testing.T) {
+ ctx := t.Context()
+ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ require.NoError(t, err)
+
+ db := &database.DB{DB: gormDB}
+ require.NoError(t, db.AutoMigrate(&EnvironmentUpdateJob{}))
+
+ svc := NewService(db, nil, nil, nil, nil, nil)
+ job := &EnvironmentUpdateJob{
+ Status: EnvironmentUpdateJobStatusRunning,
+ Results: EnvironmentUpdateResults{
+ {EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: EnvironmentUpdateResultStatusUpdating},
+ },
+ }
+ require.NoError(t, db.WithContext(ctx).Create(job).Error)
+
+ svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)
+ started := job.Results[0].StageStartedAt
+ require.NotNil(t, started)
+ require.WithinDuration(t, time.Now(), *started, time.Second)
+
+ svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageChecking)
+ require.Same(t, started, job.Results[0].StageStartedAt)
+
+ svc.setUpdateStageInternal(ctx, job, &job.Results[0], EnvironmentUpdateStageStarting)
+ var got EnvironmentUpdateJob
+ require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)
+ require.Equal(t, EnvironmentUpdateStageStarting, got.Results[0].Stage)
+ require.NotNil(t, got.Results[0].StageStartedAt)
+ require.NotSame(t, started, job.Results[0].StageStartedAt)
}
// The up-to-date short-circuit must only fire when the version check is conclusive:
diff --git a/backend/pkg/libarcane/startup/runtime_identity_linux.go b/backend/pkg/libarcane/startup/runtime_identity_linux.go
index db1f3e784..87a6161f3 100644
--- a/backend/pkg/libarcane/startup/runtime_identity_linux.go
+++ b/backend/pkg/libarcane/startup/runtime_identity_linux.go
@@ -8,7 +8,6 @@ import (
"fmt"
"os"
"os/exec"
- "os/signal"
"syscall"
"github.com/samber/mo"
@@ -23,6 +22,8 @@ func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityR
groups := runtimeIdentitySupplementaryGroupsInternal(req.DockerHost, resolveSocketGroupInternal)
cmd := exec.CommandContext(ctx, executable, os.Args[1:]...) //nolint:gosec // re-executing our own binary with the same args under a different UID/GID
+ // Forward shutdown instead of the default SIGKILL so the child can unregister its actor host.
+ cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
cmd.Env = os.Environ()
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
@@ -39,39 +40,23 @@ func reexecWithRuntimeIdentityInternal(ctx context.Context, req runtimeIdentityR
return fmt.Errorf("start runtime identity child: %w", startErr)
}
- sigCh := make(chan os.Signal, 2)
- signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
-
- done := make(chan error, 1)
- go func() {
- done <- cmd.Wait()
- }()
-
- for {
- select {
- case sig := <-sigCh:
- if cmd.Process != nil {
- _ = cmd.Process.Signal(sig)
- }
- case waitErr := <-done:
- signal.Stop(sigCh)
- if waitErr == nil {
- os.Exit(0)
- }
+ waitErr := cmd.Wait()
+ // After a forwarded shutdown Wait reports ctx.Err() even when the child exited cleanly.
+ if cmd.ProcessState != nil && cmd.ProcessState.Success() {
+ os.Exit(0)
+ }
- if exitErr, ok := errors.AsType[*exec.ExitError](waitErr); ok {
- if status, hasStatus := exitErr.Sys().(syscall.WaitStatus); hasStatus {
- if status.Signaled() {
- os.Exit(128 + int(status.Signal()))
- }
- os.Exit(status.ExitStatus())
- }
- os.Exit(exitErr.ExitCode())
+ if exitErr, ok := errors.AsType[*exec.ExitError](waitErr); ok {
+ if status, hasStatus := exitErr.Sys().(syscall.WaitStatus); hasStatus {
+ if status.Signaled() {
+ os.Exit(128 + int(status.Signal()))
}
-
- return fmt.Errorf("wait for runtime identity child: %w", waitErr)
+ os.Exit(status.ExitStatus())
}
+ os.Exit(exitErr.ExitCode())
}
+
+ return fmt.Errorf("wait for runtime identity child: %w", waitErr)
}
func resolveSocketGroupInternal(socketPath string) mo.Option[uint32] {
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 6d1c9f3de..8c3aa79f9 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -1538,13 +1538,24 @@
"environments_update_all_title": "Update all environments",
"environments_update_all_message": "This pulls the latest Arcane image and updates every environment, whether or not an update is reported. Agents update first, then this manager, which restarts only if the pull brought a new image. Offline environments are skipped.",
"environments_update_all_manager_restarting": "Manager is restarting, reconnecting…",
- "environments_update_all_manager_note": "Agents update first, then this manager. Arcane refreshes on its own when the run finishes.",
+ "environments_update_all_manager_note": "Agents update first; the manager updates last. You can close this view and reopen it to follow the active update.",
"environments_update_all_in_progress": "Updating environments…",
"environments_update_all_progress": "{done} of {total}",
+ "environments_update_all_stage_checking": "Checking version",
+ "environments_update_all_stage_starting": "Starting update",
+ "environments_update_all_stage_reconnecting": "Waiting for reconnection",
+ "environments_update_all_stage_verifying": "Verifying version",
+ "environments_update_all_stage_manager_restart": "Waiting for manager restart",
+ "environments_update_all_elapsed": "Elapsed {duration}",
+ "environments_update_all_progress_unavailable": "Progress unavailable",
+ "environments_update_all_loading_status": "Checking update status…",
+ "environments_update_all_finished_with_issues": "Finished with issues",
+ "environments_update_all_reconnection_note": "Temporary disconnection is expected while restarting. Agents have up to five minutes to reconnect and confirm their version. Progress refreshes automatically.",
+ "environments_update_all_summary": "{updated} confirmed updates · {current} already current · {skipped} skipped · {failed} failed · {unconfirmed} unconfirmed",
"environments_update_all_completed": "All environments processed",
"environments_update_all_failed": "Update all failed",
"environments_update_all_trigger_failed": "Failed to start update all",
- "environments_update_all_status_triggered": "Update triggered",
+ "environments_update_all_status_triggered": "Update requested; version not confirmed",
"environments_update_all_status_skipped_offline": "Offline — skipped",
"environments_edge_control_plane_label": "Control Plane",
"environments_edge_polling_active": "Polling Active",
diff --git a/frontend/src/lib/components/dialogs/update-all-dialog.svelte b/frontend/src/lib/components/dialogs/update-all-dialog.svelte
index 72fd26c7b..c45ddf939 100644
--- a/frontend/src/lib/components/dialogs/update-all-dialog.svelte
+++ b/frontend/src/lib/components/dialogs/update-all-dialog.svelte
@@ -1,6 +1,7 @@
<script lang="ts">
import { refreshAll } from '$app/navigation';
import { onDestroy } from 'svelte';
+ import type { Attachment } from 'svelte/attachments';
import ReleaseNotes from '#lib/components/release-notes.svelte';
import { Button } from '#lib/components/ui/button/index.js';
@@ -8,16 +9,18 @@
import Spinner from '#lib/components/ui/spinner/spinner.svelte';
import { SuccessIcon, ClockIcon, AlertIcon, AlertTriangleIcon, ExternalLinkIcon } from '#lib/icons/index.js';
import { m } from '#lib/paraglide/messages.js';
- import BaseAPIService, { APIError } from '#lib/services/api-service.js';
- import systemUpgradeService, {
- type UpdateAllJob,
- type UpdateAllEnvironmentResult,
- type UpdateAllEnvironmentStatus
- } from '#lib/services/api/system-upgrade-service.js';
+ import { APIError } from '#lib/services/api-service.js';
+ import systemUpgradeService from '#lib/services/api/system-upgrade-service.js';
import type { AppVersionInformation } from '#lib/types/settings.js';
+ import type {
+ UpdateAllJob,
+ UpdateAllEnvironmentResult,
+ UpdateAllEnvironmentStatus,
+ UpdateAllStage
+ } from '#lib/types/system-upgrade.js';
import { cn } from '#lib/utils.js';
- import { handleApiResultWithCallbacks } from '#lib/utils/api.js';
- import { formatRelativeTime, nowInstantString } from '#lib/utils/formatting.js';
+ import { extractApiErrorMessage, handleApiResultWithCallbacks } from '#lib/utils/api.js';
+ import { formatElapsedTime, formatRelativeTime, nowInstantString } from '#lib/utils/formatting.js';
import { tryCatch } from '#lib/utils/try-catch.js';
import VersionUpdateSummary from './version-update-summary.svelte';
@@ -51,6 +54,9 @@
let phase = $state<Phase>('confirm');
let job = $state<UpdateAllJob | null>(null);
let reconnecting = $state(false);
+ let progressError = $state('');
+ // Ticks once a second while the dialog is open so elapsed stage times advance.
+ let clock = $state<string>();
let pollActive = false;
let pollTimer: ReturnType<typeof setTimeout> | null = null;
// Bumped on every confirm and reset so a startup/status response that lands after
@@ -66,16 +72,14 @@
}
}
- // Reset on close (not on open) so a reopened dialog always starts at the confirm
- // step, without mutating $state from inside an $effect. The confirm step never
- // renders job/reconnecting, so clearing them here is safe.
+ // Closing stops observing; the backend job and restart recovery keep running.
function resetState() {
startAttempt++;
stopPolling();
- BaseAPIService.setUpgradeInProgress(false);
phase = 'confirm';
job = null;
reconnecting = false;
+ progressError = '';
}
function schedulePoll() {
@@ -83,56 +87,98 @@
pollTimer = setTimeout(poll, POLL_INTERVAL_MS);
}
- function finishTerminalJob(terminalJob: UpdateAllJob) {
- const managerRestarted =
- terminalJob.results?.some((result) => result.environmentId === MANAGER_ENVIRONMENT_ID && result.status === 'updated') ??
- false;
- // Keep the upgrade flag armed when the manager actually restarted: the refresh in
- // handleClose may hit the new backend with a stale token, and api-service only
- // recovers that version-mismatch 401 while the flag is set.
- if (!managerRestarted) {
- BaseAPIService.setUpgradeInProgress(false);
+ function isActiveJob(candidate: UpdateAllJob | null | undefined): candidate is UpdateAllJob {
+ return candidate?.status === 'running' || candidate?.status === 'pending_restart';
+ }
+
+ function followJob(next: UpdateAllJob) {
+ job = next;
+ reconnecting = false;
+ progressError = '';
+ if (!isActiveJob(next)) {
+ stopPolling();
+ phase = 'finished';
+ return;
}
- phase = 'finished';
+ phase = 'running';
+ if (!debugDemo) systemUpgradeService.monitorUpdateAllRecovery(next);
+ pollActive = true;
+ schedulePoll();
+ }
+
+ // A status read that never reached the backend, or hit a gateway with no backend
+ // behind it, is what a manager restart looks like from the browser.
+ function isConnectionLost(error: unknown): boolean {
+ if (error instanceof TypeError) return true;
+ if (!(error instanceof APIError)) return false;
+ return !error.status || [502, 503, 504].includes(error.status);
+ }
+
+ function handleStatusError(error: unknown) {
+ // No job has ever run: nothing to follow.
+ if (!job && error instanceof APIError && error.status === 404) {
+ stopPolling();
+ phase = 'confirm';
+ progressError = '';
+ return;
+ }
+ reconnecting = job?.status === 'pending_restart' && isConnectionLost(error);
+ progressError = reconnecting ? '' : extractApiErrorMessage(error);
+ pollActive = true;
+ schedulePoll();
}
async function poll() {
if (!pollActive) return;
-
- const requestResult1 = await tryCatch(
- (async () => {
- const next = await systemUpgradeService.getUpdateAllStatus();
- reconnecting = false;
- job = next;
- if (next.status === 'completed' || next.status === 'failed') {
- stopPolling();
- finishTerminalJob(next);
- return true;
- }
- })()
- );
- if (requestResult1.error !== null) {
- // The manager is likely restarting after its own upgrade — keep retrying
- // until the backend answers again.
- reconnecting = true;
+ const attempt = startAttempt;
+ const response = await tryCatch(systemUpgradeService.getUpdateAllStatus());
+ if (!pollActive || attempt !== startAttempt) return;
+ if (response.error !== null) {
+ handleStatusError(response.error);
+ return;
}
- if (requestResult1.data) return;
+ followJob(response.data);
+ }
- schedulePoll();
+ // Reopening the dialog picks up an update that is still running on the backend.
+ async function restoreActiveJob() {
+ const attempt = ++startAttempt;
+ phase = 'running';
+ const response = await tryCatch(systemUpgradeService.getUpdateAllStatus());
+ if (attempt !== startAttempt) return;
+ if (response.error !== null) {
+ handleStatusError(response.error);
+ return;
+ }
+ if (isActiveJob(response.data)) {
+ followJob(response.data);
+ } else {
+ phase = 'confirm';
+ }
}
+ // Dialog.Content only exists while the dialog is open, so this runs once per
+ // opening and its cleanup once per close.
+ const observeUpdate: Attachment = () => {
+ if (!debugDemo) void restoreActiveJob();
+ const ticker = setInterval(() => (clock = nowInstantString()), 1000);
+ return () => {
+ clearInterval(ticker);
+ resetState();
+ };
+ };
+
async function handleConfirm() {
const attempt = ++startAttempt;
phase = 'running';
reconnecting = false;
+ progressError = '';
if (debugDemo) {
void runDebugDemo();
return;
}
- BaseAPIService.setUpgradeInProgress(true);
-
const startResult = await tryCatch(systemUpgradeService.triggerUpdateAll());
let next = startResult.data;
// The backend refuses a second job while one is active (409). Adopt that job
@@ -143,7 +189,7 @@
for (let read = 0; read < CONFLICT_STATUS_READS && attempt === startAttempt; read++) {
if (read > 0) await new Promise((resolve) => setTimeout(resolve, CONFLICT_STATUS_RETRY_MS));
const active = (await tryCatch(systemUpgradeService.getUpdateAllStatus())).data;
- if (active?.status === 'running' || active?.status === 'pending_restart') {
+ if (isActiveJob(active)) {
next = active;
break;
}
@@ -158,17 +204,11 @@
return;
}
- job = next;
- if (job.status === 'completed' || job.status === 'failed') {
- finishTerminalJob(job);
- return;
- }
-
- pollActive = true;
- schedulePoll();
+ followJob(next);
}
async function handleClose() {
+ const attempt = ++startAttempt;
stopPolling();
// Refresh through SvelteKit instead of reloading the document: a hard reload
// lands while agents are still reconnecting after the fleet restart, which
@@ -183,6 +223,7 @@
// A failed refresh must not trap the dialog open.
}
}
+ if (attempt !== startAttempt) return;
resetState();
open = false;
await onFinished?.();
@@ -191,7 +232,6 @@
onDestroy(() => {
startAttempt++;
stopPolling();
- BaseAPIService.setUpgradeInProgress(false);
});
// Version/release presentation for the confirm step, rendered when the caller
@@ -203,21 +243,31 @@
return at ? formatRelativeTime(at) : '';
});
+ // "Done" is every environment that has reached a terminal state — i.e. anything
+ // that isn't still queued (pending) or actively being worked on (updating).
+ const results = $derived(job?.results ?? []);
+ const managerResult = $derived(results.find((r) => r.environmentId === MANAGER_ENVIRONMENT_ID));
+ const totalCount = $derived(results.length);
+ const doneCount = $derived(results.filter((r) => r.status !== 'pending' && r.status !== 'updating').length);
+ const outcomeCounts = $derived({
+ updated: results.filter((r) => r.status === 'updated').length,
+ current: results.filter((r) => r.status === 'up_to_date').length,
+ skipped: results.filter((r) => r.status === 'skipped_offline').length,
+ failed: results.filter((r) => r.status === 'failed').length,
+ unconfirmed: results.filter((r) => r.status === 'triggered').length
+ });
+ const hasIssues = $derived(outcomeCounts.failed > 0 || outcomeCounts.unconfirmed > 0);
+ const failed = $derived(job?.status === 'failed' || hasIssues);
+
const title = $derived.by(() => {
if (phase === 'confirm') return m.environments_update_all_title();
if (phase === 'finished') {
- return job?.status === 'failed' ? m.environments_update_all_failed() : m.environments_update_all_completed();
+ if (job?.status === 'failed') return m.environments_update_all_failed();
+ return hasIssues ? m.environments_update_all_finished_with_issues() : m.environments_update_all_completed();
}
return m.environments_update_all_in_progress();
});
- // "Done" is every environment that has reached a terminal state — i.e. anything
- // that isn't still queued (pending) or actively being worked on (updating).
- const results = $derived(job?.results ?? []);
- const totalCount = $derived(results.length);
- const doneCount = $derived(results.filter((r) => r.status !== 'pending' && r.status !== 'updating').length);
- const failed = $derived(job?.status === 'failed');
-
const completedColors = {
segment: 'bg-success',
badge: 'border-success/40 bg-success/10 text-success',
@@ -240,7 +290,12 @@
},
updated: { label: m.common_updated, ...completedColors },
up_to_date: { label: m.image_update_up_to_date_title, ...completedColors },
- triggered: { label: m.environments_update_all_status_triggered, ...completedColors },
+ triggered: {
+ label: m.environments_update_all_status_triggered,
+ segment: 'bg-warning',
+ badge: 'border-warning/40 bg-warning/10 text-warning',
+ text: 'text-warning'
+ },
skipped_offline: {
label: m.environments_update_all_status_skipped_offline,
segment: 'bg-warning',
@@ -260,6 +315,23 @@
return environmentStatusDisplay.get(status) ?? pendingDisplay;
}
+ const stageLabels: Record<UpdateAllStage, () => string> = {
+ checking: m.environments_update_all_stage_checking,
+ starting: m.environments_update_all_stage_starting,
+ reconnecting: m.environments_update_all_stage_reconnecting,
+ verifying: m.environments_update_all_stage_verifying
+ };
+
+ // While a row is updating its stage replaces the generic status label.
+ function rowLabel(result: UpdateAllEnvironmentResult): string {
+ const statusLabel = statusPresentation(result.status).label;
+ if (result.status !== 'updating' || !result.stage) return statusLabel();
+ if (result.stage === 'reconnecting' && result.environmentId === MANAGER_ENVIRONMENT_ID) {
+ return m.environments_update_all_stage_manager_restart();
+ }
+ return (stageLabels[result.stage] ?? statusLabel)();
+ }
+
// An environment that was already current reports the same version twice; show it
// once rather than as a "v1.0.0 → v1.0.0" no-op transition.
function versionLine(result: UpdateAllEnvironmentResult): string {
@@ -273,7 +345,17 @@
// designed and reviewed without triggering a real fleet update. Only ever enabled
// from a dev-guarded callsite.
async function runDebugDemo() {
- const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+ const attempt = startAttempt;
+ const live = () => attempt === startAttempt && phase === 'running';
+ // Sleeps, then reports whether the demo is still the one being watched.
+ const pause = async (ms: number) => {
+ await new Promise((resolve) => setTimeout(resolve, ms));
+ return live();
+ };
+ const enterStage = (row: UpdateAllEnvironmentResult, stage: UpdateAllStage) => {
+ row.stage = stage;
+ row.stageStartedAt = nowInstantString();
+ };
// Big enough fleet (10) to exercise the scrolling list (#3655).
const demo: Array<{ name: string; outcome: UpdateAllEnvironmentStatus; to: string; error?: string }> = [
{ name: 'Local Docker', outcome: 'updated', to: 'v0.9.2' },
@@ -303,50 +385,49 @@
// Remotes first, manager last — the real processing order.
const steps = [...demo.entries()].map(([index, entry]) => ({ index, entry }));
for (const { index, entry } of [...steps.slice(1), ...steps.slice(0, 1)]) {
- const starting = job?.results?.[index];
- if (phase !== 'running' || !starting) return;
- starting.status = 'updating';
- await wait(700);
-
+ const row = job?.results?.[index];
+ if (!live() || !row) return;
+ row.status = 'updating';
+ enterStage(row, 'checking');
+ if (!(await pause(700))) return;
+ enterStage(row, 'starting');
+ if (!(await pause(700))) return;
+ enterStage(row, 'reconnecting');
+ // One slow reconnect and one status-read outage, to exercise both displays.
+ if (index === 1 && !(await pause(5000))) return;
+ if (index === 2) {
+ progressError = m.environments_update_all_progress_unavailable();
+ if (!(await pause(2000))) return;
+ progressError = '';
+ }
// The manager restart is the one moment the reconnecting banner shows.
- if (index === 0) {
+ if (index === 0 && job) {
+ job.status = 'pending_restart';
reconnecting = true;
- await wait(1600);
+ if (!(await pause(3000))) return;
reconnecting = false;
}
-
- const row = job?.results?.[index];
- if (phase !== 'running' || !row) return;
+ enterStage(row, 'verifying');
+ if (!(await pause(700))) return;
row.status = entry.outcome;
+ row.stage = undefined;
+ row.stageStartedAt = undefined;
row.toVersion = entry.to;
if (entry.error) row.error = entry.error;
}
- if (phase !== 'running' || !job) return;
+ if (!live() || !job) return;
job.status = 'completed';
job.completedAt = nowInstantString();
phase = 'finished';
}
</script>
-<Dialog.Root
- {open}
- onOpenChange={(next) => {
- if (!next) {
- resetState();
- }
- open = next;
- }}
->
+<Dialog.Root {open} onOpenChange={(next) => (open = next)}>
<Dialog.Content
+ {@attach observeUpdate}
sectioned
- class={cn(
- 'flex max-h-(--max-height-dscreen-90) flex-col overflow-hidden sm:max-w-130',
- phase === 'running' && '[&>button]:hidden'
- )}
- onInteractOutside={(e: Event) => {
- if (phase === 'running') e.preventDefault();
- }}
+ class="flex max-h-(--max-height-dscreen-90) flex-col overflow-hidden sm:max-w-130"
>
{#if phase === 'confirm'}
<div class="px-6 pt-6 pb-4">
@@ -416,12 +497,30 @@
</div>
{/if}
+ {#if phase === 'finished'}
+ <p class="mt-3 text-sm text-muted-foreground">{m.environments_update_all_summary(outcomeCounts)}</p>
+ {#if job?.error}<p class="mt-2 text-sm text-destructive">{job.error}</p>{/if}
+ {/if}
+ {#if progressError}
+ <div role="status" class="mt-4 rounded-lg border border-warning/20 bg-warning/5 px-3 py-2.5 text-sm">
+ <p class="font-medium">{m.environments_update_all_progress_unavailable()}</p>
+ <p class="mt-1 break-words text-muted-foreground">{progressError}</p>
+ </div>
+ {/if}
+
{#if reconnecting}
<div
class="mt-4 flex items-center gap-2 rounded-lg border border-warning/20 bg-warning/5 px-3 py-2.5 text-sm text-warning"
>
<Spinner class="size-4" />
- <span>{m.environments_update_all_manager_restarting()}</span>
+ <div>
+ <span>{m.environments_update_all_manager_restarting()}</span>
+ <span class="block text-xs">
+ {m.environments_update_all_elapsed({
+ duration: formatElapsedTime(managerResult?.stageStartedAt ?? job?.createdAt, { base: clock })
+ })}
+ </span>
+ </div>
</div>
{/if}
</div>
@@ -441,11 +540,11 @@
statusPresentation(result.status).badge
)}
>
- {#if result.status === 'updated' || result.status === 'triggered' || result.status === 'up_to_date'}
+ {#if result.status === 'updated' || result.status === 'up_to_date'}
<SuccessIcon class="size-3.5" />
{:else if result.status === 'updating'}
<Spinner class="size-3.5" />
- {:else if result.status === 'skipped_offline'}
+ {:else if result.status === 'skipped_offline' || result.status === 'triggered'}
<AlertIcon class="size-3.5" />
{:else if result.status === 'failed'}
<AlertTriangleIcon class="size-3.5" />
@@ -468,9 +567,16 @@
{/if}
</div>
- <span class={cn('shrink-0 text-xs', statusPresentation(result.status).text)}>
- {statusPresentation(result.status).label()}
- </span>
+ <div class={cn('max-w-48 shrink-0 text-right text-xs', statusPresentation(result.status).text)}>
+ <span>{rowLabel(result)}</span>
+ {#if result.status === 'updating' && result.stageStartedAt}
+ <span class="block text-muted-foreground">
+ {m.environments_update_all_elapsed({
+ duration: formatElapsedTime(result.stageStartedAt, { base: clock })
+ })}
+ </span>
+ {/if}
+ </div>
</li>
{/each}
</ul>
@@ -479,13 +585,14 @@
{:else}
<div class="flex items-center gap-2 border-t border-border/60 px-6 py-4 text-sm text-muted-foreground">
<Spinner class="size-4" />
- <span>{m.environments_update_all_in_progress()}</span>
+ <span>{job ? m.environments_update_all_in_progress() : m.environments_update_all_loading_status()}</span>
</div>
{/if}
{#if phase === 'running'}
<div class="border-t border-border/60 bg-muted/30 px-6 py-3">
<p class="text-xs leading-relaxed text-muted-foreground">{m.environments_update_all_manager_note()}</p>
+ <p class="mt-1 text-xs leading-relaxed text-muted-foreground">{m.environments_update_all_reconnection_note()}</p>
</div>
{/if}
{/if}
diff --git a/frontend/src/lib/services/api-service.ts b/frontend/src/lib/services/api-service.ts
index ca7a01ec9..f22f179c9 100644
--- a/frontend/src/lib/services/api-service.ts
+++ b/frontend/src/lib/services/api-service.ts
@@ -219,7 +219,10 @@ let tokenRefreshHandler: (() => Promise<string | null>) | null = null;
// window the backend briefly restarts and returns version-mismatch 401s; we must not
// bounce the user to /login — the session is recoverable once the backend is back.
let upgradeInProgressInternal = false;
+// A confirmed manager restart still needs document recovery after polling ends.
+let upgradeReloadPendingInternal = false;
let upgradeReloadStartedInternal = false;
+let unauthorizedRedirectStartedInternal = false;
const skipAuthPathsInternal = [
'/auth/login',
'/auth/logout',
@@ -257,6 +260,7 @@ export async function handleUnauthorizedResponseInternal(
if (isAuthApi || isOnAuthPage) {
return 'none';
}
+ if (unauthorizedRedirectStartedInternal) return 'redirect';
// During a server self-update the backend briefly returns version-mismatch 401s.
// Refresh first so the new backend rotates both tokens, then replace the stale
@@ -269,7 +273,7 @@ export async function handleUnauthorizedResponseInternal(
const operationResult = await tryCatch(
(async () => {
await tokenRefreshHandler();
- if (!isVersionMismatch || !upgradeInProgressInternal) return 'retry';
+ if (!isVersionMismatch || (!upgradeInProgressInternal && !upgradeReloadPendingInternal)) return 'retry';
if (!upgradeReloadStartedInternal) {
upgradeReloadStartedInternal = true;
window.location.reload();
@@ -282,7 +286,10 @@ export async function handleUnauthorizedResponseInternal(
const isTransientRefreshFailure =
error instanceof APIError && (error.name === 'NetworkError' || error.name === 'TimeoutError');
if (recoverable && isTransientRefreshFailure) return 'none';
- window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);
+ if (!unauthorizedRedirectStartedInternal) {
+ unauthorizedRedirectStartedInternal = true;
+ window.location.replace(`/login?redirect=${encodeURIComponent(pathname)}`);
+ }
return 'redirect';
} else {
return operationResult.data;
@@ -494,15 +501,16 @@ abstract class BaseAPIService {
// Toggled by the update flows (Update All / local update center) so an in-progress
// self-update restart is treated as a recoverable reconnect, not a logout.
static setUpgradeInProgress(value: boolean) {
- if (value && !upgradeInProgressInternal) {
- upgradeReloadStartedInternal = false;
- }
- if (!value) {
+ if (!upgradeReloadPendingInternal && (!value || !upgradeInProgressInternal)) {
upgradeReloadStartedInternal = false;
}
upgradeInProgressInternal = value;
}
+ static confirmUpgradeRestart() {
+ upgradeReloadPendingInternal = true;
+ }
+
protected async handleResponse<T>(promise: Promise<APIResponse>): Promise<T> {
const response = await promise;
const payload = response.data;
diff --git a/frontend/src/lib/services/api/system-upgrade-service.ts b/frontend/src/lib/services/api/system-upgrade-service.ts
index 40a40af01..4e375fb8b 100644
--- a/frontend/src/lib/services/api/system-upgrade-service.ts
+++ b/frontend/src/lib/services/api/system-upgrade-service.ts
@@ -1,7 +1,14 @@
import type { AppVersionInformation } from '#lib/types/settings.js';
+import type { UpdateAllJob } from '#lib/types/system-upgrade.js';
import { tryCatch } from '#lib/utils/try-catch.js';
-import { apiClient } from '../api-service';
+import BaseAPIService, { APIError, apiClient } from '../api-service';
+
+const UPDATE_ALL_RECOVERY_POLL_MS = 3000;
+const UPDATE_ALL_RECOVERY_MAX_UNAVAILABLE_MS = 5 * 60 * 1000;
+let updateAllRecovery: Promise<void> | null = null;
+let updateAllRecoveryAttempt = 0;
+let updateAllRecoveryJobId: string | undefined;
export interface UpgradeCheckResponse {
canUpgrade: boolean;
@@ -24,35 +31,6 @@ export interface HealthCheckResult {
healthy: boolean;
}
-export type UpdateAllEnvironmentStatus =
- | 'pending'
- | 'updating'
- | 'updated'
- | 'up_to_date'
- | 'triggered'
- | 'skipped_offline'
- | 'failed';
-
-export interface UpdateAllEnvironmentResult {
- environmentId: string;
- environmentName: string;
- status: UpdateAllEnvironmentStatus;
- fromVersion?: string;
- toVersion?: string;
- error?: string;
-}
-
-export type UpdateAllJobStatus = 'pending_restart' | 'running' | 'completed' | 'failed';
-
-export interface UpdateAllJob {
- id: string;
- status: UpdateAllJobStatus;
- results?: UpdateAllEnvironmentResult[];
- error?: string;
- createdAt: string;
- completedAt?: string;
-}
-
type ApiResponse<T> = {
success: boolean;
data: T;
@@ -89,8 +67,24 @@ async function triggerUpgrade(environmentId: string = '0'): Promise<UpgradeRespo
* No client timeout is set: the manager pulls the upgrader image before responding.
*/
async function triggerUpdateAll(): Promise<UpdateAllJob> {
- const res = await apiClient.post<ApiResponse<UpdateAllJob>>('/environments/0/system/upgrade/all');
- return res.data.data;
+ BaseAPIService.setUpgradeInProgress(true);
+ const result = await tryCatch(apiClient.post<ApiResponse<UpdateAllJob>>('/environments/0/system/upgrade/all'));
+ if (result.error !== null) {
+ const error = result.error;
+ const uncertain = !(error instanceof APIError) || !error.status || error.status >= 500 || [408, 409].includes(error.status);
+ if (!updateAllRecovery) {
+ BaseAPIService.setUpgradeInProgress(false);
+ const active = uncertain ? (await tryCatch(getUpdateAllStatus())).data : null;
+ // A concurrent successful start may have established recovery during this lookup.
+ if (!updateAllRecovery && active && (active.status === 'running' || active.status === 'pending_restart')) {
+ monitorUpdateAllRecovery(active);
+ }
+ }
+ throw error;
+ }
+ const job = result.data.data.data;
+ monitorUpdateAllRecovery(job);
+ return job;
}
/**
@@ -103,6 +97,42 @@ async function getUpdateAllStatus(): Promise<UpdateAllJob> {
return res.data.data;
}
+// Recovery follows the job even after its dialog closes or unmounts.
+function monitorUpdateAllRecovery(job: UpdateAllJob): void {
+ if (typeof window === 'undefined') return;
+ if (updateAllRecovery && job.id === updateAllRecoveryJobId) return;
+ const attempt = ++updateAllRecoveryAttempt;
+ updateAllRecoveryJobId = job.id;
+ BaseAPIService.setUpgradeInProgress(true);
+ updateAllRecovery = (async () => {
+ let deadline = Date.now() + UPDATE_ALL_RECOVERY_MAX_UNAVAILABLE_MS;
+ for (;;) {
+ await new Promise((resolve) => setTimeout(resolve, UPDATE_ALL_RECOVERY_POLL_MS));
+ if (attempt !== updateAllRecoveryAttempt) return;
+ // An authenticated status read also drives refresh-and-reload after restart.
+ const result = await tryCatch(getUpdateAllStatus());
+ if (attempt !== updateAllRecoveryAttempt) return;
+ if (result.error === null) {
+ if (result.data.id !== job.id) break;
+ if (result.data.status !== 'running' && result.data.status !== 'pending_restart') {
+ if (result.data.results?.some((item) => item.environmentId === '0' && item.status === 'updated')) {
+ BaseAPIService.confirmUpgradeRestart();
+ }
+ break;
+ }
+ deadline = Date.now() + UPDATE_ALL_RECOVERY_MAX_UNAVAILABLE_MS;
+ } else if (Date.now() >= deadline) {
+ break;
+ }
+ }
+ })().finally(() => {
+ if (attempt !== updateAllRecoveryAttempt) return;
+ BaseAPIService.setUpgradeInProgress(false);
+ updateAllRecovery = null;
+ updateAllRecoveryJobId = undefined;
+ });
+}
+
/**
* Check system health
* @param environmentId - Optional environment ID for remote environments (defaults to local system)
@@ -146,6 +176,7 @@ export default {
triggerUpgrade,
triggerUpdateAll,
getUpdateAllStatus,
+ monitorUpdateAllRecovery,
checkHealth,
getVersionInfo
};
diff --git a/frontend/src/lib/types/system-upgrade.ts b/frontend/src/lib/types/system-upgrade.ts
new file mode 100644
index 000000000..fccf36806
--- /dev/null
+++ b/frontend/src/lib/types/system-upgrade.ts
@@ -0,0 +1,32 @@
+export type UpdateAllEnvironmentStatus =
+ | 'pending'
+ | 'updating'
+ | 'updated'
+ | 'up_to_date'
+ | 'triggered'
+ | 'skipped_offline'
+ | 'failed';
+
+export type UpdateAllStage = 'checking' | 'starting' | 'reconnecting' | 'verifying';
+
+export interface UpdateAllEnvironmentResult {
+ environmentId: string;
+ environmentName: string;
+ status: UpdateAllEnvironmentStatus;
+ stage?: UpdateAllStage;
+ stageStartedAt?: string;
+ fromVersion?: string;
+ toVersion?: string;
+ error?: string;
+}
+
+export type UpdateAllJobStatus = 'pending_restart' | 'running' | 'completed' | 'failed';
+
+export interface UpdateAllJob {
+ id: string;
+ status: UpdateAllJobStatus;
+ results?: UpdateAllEnvironmentResult[];
+ error?: string;
+ createdAt: string;
+ completedAt?: string;
+}
diff --git a/tests/spec/updates.spec.ts b/tests/spec/updates.spec.ts
index c47ec81fa..4e76d11ef 100644
--- a/tests/spec/updates.spec.ts
+++ b/tests/spec/updates.spec.ts
@@ -448,6 +448,15 @@ test.describe('Arcane upgrades', () => {
}
async function openAndConfirmUpdateAll(page: Page) {
+ // Opening checks for an existing run before showing confirmation. These tests
+ // start a new run; their status fixtures apply only after this initial lookup.
+ await page.route(
+ '**/api/environments/0/system/upgrade/all/status',
+ async (route) => {
+ await fulfillProblem(route, 404, 'no update-all job found');
+ },
+ { times: 1 }
+ );
await page.getByRole('button', { name: 'Update All', exact: true }).first().click();
const dialog = page.getByRole('dialog');
await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();
@@ -479,7 +488,7 @@ test.describe('Arcane upgrades', () => {
});
await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
statusCalls++;
- await fulfillJob(route, 'running', 'updating');
+ await fulfillJob(route, 'completed', 'updated');
});
await page.goto('/environments');
@@ -491,11 +500,11 @@ test.describe('Arcane upgrades', () => {
await expect(toast).toBeVisible({ timeout: 10_000 });
await expect(toast).toContainText('pull access denied');
- // Back at the confirm step, ready for another attempt — and no polling started.
+ // The failed POST checks for an active job once, then returns to confirmation.
const dialog = page.getByRole('dialog');
await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();
await expect(dialog.getByRole('button', { name: 'Update All', exact: true })).toBeVisible();
- expect(statusCalls).toBe(0);
+ expect(statusCalls).toBe(1);
});
test('a 409 conflict adopts the active job and follows it to completion', async ({ page }) => {
@@ -507,8 +516,8 @@ test.describe('Arcane upgrades', () => {
});
await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
statusCalls++;
- // First read (adoption) reports the job still running; the poll completes it.
- if (statusCalls === 1) {
+ // Recovery and the dialog each adopt the active job; later polls complete it.
+ if (statusCalls <= 2) {
await fulfillJob(route, 'running', 'updating');
return;
}
@@ -560,10 +569,10 @@ test.describe('Arcane upgrades', () => {
await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();
// Wait past one poll interval to prove neither polling nor a retry kicked in.
- // The three status reads are the persistence-window grace, not polling.
+ // One recovery lookup and three dialog adoption reads precede this interval.
await page.clock.runFor(4_000);
expect(startCalls).toBe(1);
- expect(statusCalls).toBe(3);
+ expect(statusCalls).toBe(4);
});
test('a 409 conflict adopts a job that is persisted shortly after the conflict', async ({
@@ -575,15 +584,15 @@ test.describe('Arcane upgrades', () => {
});
await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
statusCalls++;
- // The winning client's job row does not exist yet on the first read.
- if (statusCalls === 1) {
+ // Recovery and the first dialog read precede persistence; the dialog retries.
+ if (statusCalls <= 2) {
await fulfillProblem(route, 404, 'no update-all job found');
return;
}
await fulfillJob(
route,
- statusCalls === 2 ? 'running' : 'completed',
- statusCalls === 2 ? 'updating' : 'updated'
+ statusCalls === 3 ? 'running' : 'completed',
+ statusCalls === 3 ? 'updating' : 'updated'
);
});
@@ -599,7 +608,7 @@ test.describe('Arcane upgrades', () => {
await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);
});
- test('closing the dialog during a pending start ignores the late response', async ({
+ test('closing during a pending start keeps recovery running without reopening the dialog', async ({
page
}) => {
await page.clock.install();
@@ -608,28 +617,39 @@ test.describe('Arcane upgrades', () => {
releaseStart = resolve;
});
let statusCalls = 0;
+ let startCalls = 0;
await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ startCalls++;
await startReleased;
await fulfillJob(route, 'running', 'updating');
});
await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
statusCalls++;
- await fulfillJob(route, 'running', 'updating');
+ await fulfillJob(route, 'completed', 'updated');
});
await page.goto('/environments');
await openAndConfirmUpdateAll(page);
+ await expect.poll(() => startCalls).toBe(1);
const dialog = page.getByRole('dialog');
await expect(dialog.getByRole('heading', { name: 'Updating environments…' })).toBeVisible();
await dialog.getByRole('button', { name: 'Close', exact: true }).first().click();
await expect(dialog).toBeHidden();
- // The start response arrives after closure: it must not reopen or start polling.
+ // The late response starts service-owned recovery without reopening the dialog.
+ const startResponse = page.waitForResponse(/\/api\/environments\/0\/system\/upgrade\/all$/);
releaseStart();
+ await (await startResponse).finished();
+ await expect
+ .poll(async () => {
+ await page.clock.runFor(4_000);
+ return statusCalls;
+ })
+ .toBe(1);
await page.clock.runFor(4_000);
await expect(dialog).toBeHidden();
- expect(statusCalls).toBe(0);
+ expect(statusCalls).toBe(1);
await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);
});
});
@@ -934,11 +954,17 @@ test.describe('Arcane upgrades', () => {
}) => {
await registerTokenSeeding(page);
let refreshCalls = 0;
+ let updateStarted = false;
- await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ await page.route('**/api/environments/0/system/upgrade/all', async (route) => {
+ updateStarted = true;
await fulfillJob(route, 'running', 'updating');
});
- await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ await page.route('**/api/environments/0/system/upgrade/all/status', async (route) => {
+ if (!updateStarted) {
+ await fulfillProblem(route, 404, 'no update-all job found');
+ return;
+ }
await route.fulfill({
status: 401,
contentType: 'application/json',
@@ -946,9 +972,11 @@ test.describe('Arcane upgrades', () => {
});
});
- await page.goto('/environments');
- await openAndConfirmUpdateAll(page);
- await page.route(/\/api\/auth\/refresh$/, async (route) => {
+ await page.route('**/api/auth/refresh', async (route) => {
+ if (!updateStarted) {
+ await route.continue();
+ return;
+ }
refreshCalls++;
await route.fulfill({
status: 401,
@@ -956,7 +984,11 @@ test.describe('Arcane upgrades', () => {
body: JSON.stringify({ message: 'Invalid or expired refresh token' })
});
});
- await page.route(/\/api\/auth\/me$/, async (route) => {
+ await page.route('**/api/auth/me', async (route) => {
+ if (!updateStarted) {
+ await route.continue();
+ return;
+ }
await route.fulfill({
status: 401,
contentType: 'application/json',
@@ -964,6 +996,10 @@ test.describe('Arcane upgrades', () => {
});
});
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+ await expect.poll(() => updateStarted).toBe(true);
+
await page.waitForURL(/\/login(?:\?|$)/, { timeout: 10_000 });
expect(refreshCalls).toBe(1);
await expect(
@@ -975,14 +1011,24 @@ test.describe('Arcane upgrades', () => {
page
}) => {
let refreshCalls = 0;
- await page.route(/\/api\/auth\/refresh$/, async (route) => {
+ let updateStarted = false;
+ await page.route('**/api/auth/refresh', async (route) => {
+ if (!updateStarted) {
+ await route.continue();
+ return;
+ }
refreshCalls++;
await route.continue();
});
- await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ await page.route('**/api/environments/0/system/upgrade/all', async (route) => {
+ updateStarted = true;
await fulfillJob(route, 'running', 'updating');
});
- await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ await page.route('**/api/environments/0/system/upgrade/all/status', async (route) => {
+ if (!updateStarted) {
+ await fulfillProblem(route, 404, 'no update-all job found');
+ return;
+ }
await route.fulfill({
status: 401,
contentType: 'application/json',
@@ -990,9 +1036,11 @@ test.describe('Arcane upgrades', () => {
});
});
- await page.goto('/environments');
- await openAndConfirmUpdateAll(page);
- await page.route(/\/api\/auth\/me$/, async (route) => {
+ await page.route('**/api/auth/me', async (route) => {
+ if (!updateStarted) {
+ await route.continue();
+ return;
+ }
await route.fulfill({
status: 401,
contentType: 'application/json',
@@ -1000,6 +1048,10 @@ test.describe('Arcane upgrades', () => {
});
});
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+ await expect.poll(() => updateStarted).toBe(true);
+
await page.waitForURL(/\/login(?:\?|$)/, { timeout: 10_000 });
expect(refreshCalls).toBe(0);
await expect(