Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
refactor: redesign update view and show environment version in the tables (#3389)
Kyle Mendell6118af4f0724
diff --git a/backend/api/handlers/activities.go b/backend/api/handlers/activities.goindex bf6be679d..6624048cd 100644--- a/backend/api/handlers/activities.go+++ b/backend/api/handlers/activities.go@@ -581,32 +581,19 @@ func applyActivitySourceLabelsForEnvironmentInternal(environment models.Environmfunc activitySourceFromEnvironmentInternal(environment models.Environment) (string, string) {environmentID := environment.IDif environmentID == "" {- environmentID = "0"- }- environmentName := environment.Name- if environmentName == "" {- if environmentID == "0" {- environmentName = "Local"- } else {- environmentName = environmentID- }+ environmentID = services.LocalEnvironmentID}- return environmentID, environmentName+ return environmentID, services.EnvironmentDisplayName(environmentID, environment.Name)}func (h *ActivityHandler) resolveActivitySourceInternal(ctx context.Context, environmentID string) (string, string) {if environmentID == "" {- environmentID = "0"+ environmentID = services.LocalEnvironmentID}if h.environmentService != nil {- if env, err := h.environmentService.GetEnvironmentByID(ctx, environmentID); err == nil && env != nil {- return env.ID, env.Name- }- }- if environmentID == "0" {- return "0", "Local"+ return environmentID, h.environmentService.ResolveEnvironmentName(ctx, environmentID)}- return environmentID, environmentID+ return environmentID, services.EnvironmentDisplayName(environmentID, "")}func applyActivitySourceInternal(item *activity.Activity, sourceID, sourceName string) {diff --git a/backend/api/handlers/system.go b/backend/api/handlers/system.goindex b42064f1a..644f79e45 100644--- a/backend/api/handlers/system.go+++ b/backend/api/handlers/system.go@@ -119,8 +119,16 @@ type TriggerUpgradeInput struct {EnvironmentID string `path:"id" doc:"Environment ID"`}+// TriggerUpgradeData reports the upgrade was accepted. UpToDate lets a client skip+// waiting for a restart: the upgrader still pulls, but when the environment already+// runs the newest image it finds nothing to swap in and no restart follows.+type TriggerUpgradeData struct {+ Message string `json:"message" doc:"Response message"`+ UpToDate bool `json:"upToDate" doc:"Environment already runs the newest image, so no restart is expected"`+}+type TriggerUpgradeOutput struct {- Body base.ApiResponse[base.MessageResponse]+ Body base.ApiResponse[TriggerUpgradeData]}type TriggerUpdateAllInput struct {@@ -504,7 +512,11 @@ func (h *SystemHandler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradslog.Info("System upgrade triggered", "user", user.Username, "userId", user.ID)- err = h.upgradeService.TriggerUpgradeViaCLI(ctx, *user, updater.SelfUpdateTarget{})+ // Resolved before triggering, while the version check still describes the running+ // container: the upgrade itself may replace it.+ upToDate := h.upgradeService.AlreadyOnNewestImage(ctx)++ _, err = h.upgradeService.TriggerUpgradeViaCLI(ctx, *user, updater.SelfUpdateTarget{})if err != nil {slog.Error("System upgrade failed", "error", err, "user", user.Username)@@ -515,11 +527,17 @@ func (h *SystemHandler) TriggerUpgrade(ctx context.Context, input *TriggerUpgradreturn nil, huma.Error500InternalServerError(errors.WithMessage(err, "Failed to initiate upgrade").Error())}+ message := "Upgrade initiated successfully. A new container is being created and will replace this one shortly."+ if upToDate {+ message = "Already running the newest image. The upgrade pulled it again and left the container in place."+ }+return &TriggerUpgradeOutput{- Body: base.ApiResponse[base.MessageResponse]{+ Body: base.ApiResponse[TriggerUpgradeData]{Success: true,- Data: base.MessageResponse{- Message: "Upgrade initiated successfully. A new container is being created and will replace this one shortly.",+ Data: TriggerUpgradeData{+ Message: message,+ UpToDate: upToDate,},},}, nildiff --git a/backend/cli/upgrade/upgrade.go b/backend/cli/upgrade/upgrade.goindex 641a0f180..e6e9038e4 100644--- a/backend/cli/upgrade/upgrade.go+++ b/backend/cli/upgrade/upgrade.go@@ -105,6 +105,15 @@ func runUpgrade(cmd *cobra.Command, args []string) error {return errors.WrapIf(err, "failed to pull image")}+ // The pull always runs so a mutable tag gets re-resolved, but when it lands on+ // the image the container already runs there is nothing to swap in. Skipping the+ // recreate avoids needless downtime and, for agents, a dropped connection.+ if same, pulledID := pulledImageAlreadyRunning(ctx, dockerClient, imageToPull, targetContainer.Image); same {+ slog.Info("Image unchanged after pull; already up to date, skipping recreate",+ "container", containerName, "image", imageToPull, "imageId", pulledID)+ return nil+ }+// Perform the upgradeslog.Info("Starting container upgrade", "container", containerName)if err := upgradeContainer(ctx, dockerClient, targetContainer, imageToPull); err != nil {@@ -338,6 +347,22 @@ func ensureDefaultTag(imageName string) string {return imageName + ":latest"}+// pulledImageAlreadyRunning reports whether the freshly pulled reference resolves to+// the image the target container is already running, along with the resolved image ID.+// An inspect failure reports false: recreating unnecessarily is safer than skipping a+// real update.+func pulledImageAlreadyRunning(ctx context.Context, dockerClient *client.Client, imageRef, runningImageID string) (bool, string) {+ if runningImageID == "" {+ return false, ""+ }+ inspect, err := dockerClient.ImageInspect(ctx, imageRef)+ if err != nil {+ slog.Warn("Could not inspect pulled image; continuing with recreate", "image", imageRef, "error", err)+ return false, ""+ }+ return inspect.ID == runningImageID, inspect.ID+}+func pullImage(ctx context.Context, dockerClient *client.Client, imageName string) error {reader, err := dockerClient.ImagePull(ctx, imageName, client.ImagePullOptions{})if err != nil {diff --git a/backend/internal/models/environment_update_job.go b/backend/internal/models/environment_update_job.goindex dc28d9634..385de5762 100644--- a/backend/internal/models/environment_update_job.go+++ b/backend/internal/models/environment_update_job.go@@ -42,6 +42,9 @@ const (// EnvironmentUpdateResultStatusTriggered means the upgrade was triggered but not// confirmed within the wait window (still likely succeeding in the background).EnvironmentUpdateResultStatusTriggered EnvironmentUpdateResultStatus = "triggered"+ // EnvironmentUpdateResultStatusUpToDate means the environment already ran the+ // target image, so the pull found nothing new and no container was recreated.+ EnvironmentUpdateResultStatusUpToDate EnvironmentUpdateResultStatus = "up_to_date"// EnvironmentUpdateResultStatusSkippedOffline means the environment was unreachable.EnvironmentUpdateResultStatusSkippedOffline EnvironmentUpdateResultStatus = "skipped_offline"// EnvironmentUpdateResultStatusFailed means the upgrade trigger failed.diff --git a/backend/internal/services/environment_service.go b/backend/internal/services/environment_service.goindex 94e1dda16..a2bd85421 100644--- a/backend/internal/services/environment_service.go+++ b/backend/internal/services/environment_service.go@@ -485,11 +485,47 @@ func (s *EnvironmentService) updateRemoteEnvironmentSnapshotInternal(environment}}-func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrl string) error {- const localEnvID = "0"+// LocalEnvironmentID is the reserved ID of the environment Arcane manages directly —+// the manager itself, as opposed to a remote agent.+const LocalEnvironmentID = "0"++// localEnvironmentFallbackNameInternal labels the local environment when its stored+// name is unreadable or blank. Its real name is user-editable, so this is only ever a+// fallback — never assume it, resolve the name instead.+const localEnvironmentFallbackNameInternal = "Local"++// EnvironmentDisplayName returns the label to show for an environment: its stored name+// when set, otherwise a readable fallback — the local environment gets a friendly+// label, any other environment falls back to its ID.+func EnvironmentDisplayName(environmentID, storedName string) string {+ if name := strings.TrimSpace(storedName); name != "" {+ return name+ }+ id := strings.TrimSpace(environmentID)+ if id == "" || id == LocalEnvironmentID {+ return localEnvironmentFallbackNameInternal+ }+ return id+}+// ResolveEnvironmentName looks up an environment and returns the label to show for it.+// Use this instead of hardcoding a name for a known ID: names are user-editable, so+// even the local environment's is not fixed.+func (s *EnvironmentService) ResolveEnvironmentName(ctx context.Context, environmentID string) string {+ if strings.TrimSpace(environmentID) == "" {+ environmentID = LocalEnvironmentID+ }+ env, err := s.GetEnvironmentByID(ctx, environmentID)+ if err != nil || env == nil {+ slog.WarnContext(ctx, "failed to resolve environment name", "environmentID", environmentID, "error", err)+ return EnvironmentDisplayName(environmentID, "")+ }+ return EnvironmentDisplayName(env.ID, env.Name)+}++func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrl string) error {var existingEnv models.Environment- err := s.db.WithContext(ctx).Where("id = ?", localEnvID).First(&existingEnv).Error+ err := s.db.WithContext(ctx).Where("id = ?", LocalEnvironmentID).First(&existingEnv).Errorif err == nil {// Local environment already exists, ensure ApiUrl matches current appUrl@@ -497,7 +533,7 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrlif err := s.db.WithContext(ctx).Model(&existingEnv).Update("api_url", appUrl).Error; err != nil {return errors.WrapIf(err, "failed to update local environment api url")}- slog.InfoContext(ctx, "updated local environment api url", "id", localEnvID, "url", appUrl)+ slog.InfoContext(ctx, "updated local environment api url", "id", LocalEnvironmentID, "url", appUrl)}return nil}@@ -510,7 +546,7 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrlnow := time.Now()localEnv := &models.Environment{BaseModel: models.BaseModel{- ID: localEnvID,+ ID: LocalEnvironmentID,CreatedAt: now,UpdatedAt: new(now),},@@ -524,7 +560,7 @@ func (s *EnvironmentService) EnsureLocalEnvironment(ctx context.Context, appUrlreturn errors.WrapIf(err, "failed to create local environment")}- slog.InfoContext(ctx, "created local environment record", "id", localEnvID)+ slog.InfoContext(ctx, "created local environment record", "id", LocalEnvironmentID)return nil}diff --git a/backend/internal/services/swarm_service.go b/backend/internal/services/swarm_service.goindex a455d26c5..2825010d5 100644--- a/backend/internal/services/swarm_service.go+++ b/backend/internal/services/swarm_service.go@@ -94,6 +94,9 @@ type swarmNodeAgentCoverage struct {boundEnvsByNodeID map[string][]models.EnvironmentcandidatesByNodeID map[string][]models.EnvironmentlocalIdentity *SwarmNodeIdentity+ // Resolved once per request rather than per node: the name is user-editable, and+ // applyNodeAgentCoverageInternal runs in a loop with no context to look it up.+ localEnvironmentName string}func (s *SwarmService) IsEnabled(ctx context.Context) (bool, error) {@@ -736,11 +739,12 @@ func (s *SwarmService) resolveNodeAgentCoverageInternal(ctx context.Context, envboundEnvsByNodeID: boundEnvsByNodeID,candidatesByNodeID: candidatesByNodeID,}- if environmentID == "0" {+ if environmentID == LocalEnvironmentID {identity, identityErr := s.GetLocalNodeIdentity(ctx)if identityErr == nil {coverage.localIdentity = identity}+ coverage.localEnvironmentName = s.environmentService.ResolveEnvironmentName(ctx, LocalEnvironmentID)}return coverage, nil}@@ -750,10 +754,11 @@ func (s *SwarmService) applyNodeAgentCoverageInternal(environmentID string, itemreturn}nodeID := strings.TrimSpace(item.ID)- if environmentID == "0" && coverage.localIdentity != nil && coverage.localIdentity.SwarmActive && strings.TrimSpace(coverage.localIdentity.SwarmNodeID) == nodeID {+ if environmentID == LocalEnvironmentID && coverage.localIdentity != nil && coverage.localIdentity.SwarmActive && strings.TrimSpace(coverage.localIdentity.SwarmNodeID) == nodeID {connected := truebindingKind := swarmtypes.NodeAgentBindingKindLocal- localID, localName, localType := "0", "Local", "local"+ localID, localType := LocalEnvironmentID, "local"+ localName := EnvironmentDisplayName(LocalEnvironmentID, coverage.localEnvironmentName)item.Agent = swarmtypes.NodeAgentStatus{State: swarmtypes.NodeAgentStateConnected,BindingKind: &bindingKind,diff --git a/backend/internal/services/system_upgrade_service.go b/backend/internal/services/system_upgrade_service.goindex b2def8e06..e492de59c 100644--- a/backend/internal/services/system_upgrade_service.go+++ b/backend/internal/services/system_upgrade_service.go@@ -15,6 +15,7 @@ import ("emperror.dev/errors"+ cerrdefs "github.com/containerd/errdefs""github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/models"@@ -90,13 +91,26 @@ func (s *SystemUpgradeService) CanUpgrade(ctx context.Context) (bool, error) {return true, nil}-// TriggerUpgradeViaCLI spawns the upgrade CLI command in a separate container-// This avoids self-termination issues by running the upgrade from outside.-// A zero-value target upgrades the current container to its own image tag;-// the updater engine passes an explicit target with the resolved new image.-func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user models.User, target updater.SelfUpdateTarget) error {+// AlreadyOnNewestImage reports whether this environment's version check is confident+// it already runs the newest image. A triggered upgrade still pulls, but will then+// find nothing to swap in and skip the recreate — so callers can use this to stop+// waiting for a restart that is not coming.+func (s *SystemUpgradeService) AlreadyOnNewestImage(ctx context.Context) bool {+ if s.versionService == nil {+ return false+ }+ return agentAlreadyOnTargetInternal(s.versionService.GetAppVersionInfo(ctx))+}++// TriggerUpgradeViaCLI spawns the upgrade CLI command in a separate container and+// returns that upgrader container's ID. This avoids self-termination issues by running+// the upgrade from outside. A zero-value target upgrades the current container to its+// own image tag; the updater engine passes an explicit target with the resolved new+// image. Update-all uses the returned ID to tell an upgrade that recreated this+// container from one that found nothing to do — see watchManagerUpgraderInternal.+func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user models.User, target updater.SelfUpdateTarget) (string, error) {if !s.upgrading.CompareAndSwap(false, true) {- return common.Classify(common.ErrUpgradeInProgress, errors.New("an upgrade is already in progress"))+ return "", common.Classify(common.ErrUpgradeInProgress, errors.New("an upgrade is already in progress"))}defer s.upgrading.Store(false)@@ -106,13 +120,13 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user movar err errorcontainerId, err = s.getCurrentContainerIDInternal()if err != nil {- return errors.WrapIf(err, "get current container")+ return "", errors.WrapIf(err, "get current container")}}currentContainer, err := s.findArcaneContainerInternal(ctx, containerId)if err != nil {- return errors.WrapIf(err, "inspect container")+ return "", errors.WrapIf(err, "inspect container")}containerName := strings.TrimPrefix(currentContainer.Name, "/")@@ -157,7 +171,7 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user mo// This will run independently of the current containerdockerClient, err := s.dockerService.GetClient(ctx)if err != nil {- return errors.WrapIf(err, "failed to connect to Docker")+ return "", errors.WrapIf(err, "failed to connect to Docker")}// Pull the upgrader image first to ensure it exists@@ -170,14 +184,14 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user mopullReader, err := dockerClient.ImagePull(pullCtx, upgraderImage, client.ImagePullOptions{})if err != nil {if errors.Is(pullCtx.Err(), context.DeadlineExceeded) {- return errors.Errorf("upgrader image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", upgraderImage)+ return "", errors.Errorf("upgrader image pull timed out for %s (increase DOCKER_IMAGE_PULL_TIMEOUT or setting)", upgraderImage)}- return errors.WrapIf(err, "pull upgrader image")+ return "", errors.WrapIf(err, "pull upgrader image")}// Drain and validate the JSON stream to complete the pull.if err := dockerutils.RenderJSONMessageStream(pullReader, io.Discard); err != nil {_ = pullReader.Close()- return errors.WrapIf(err, "failed to complete upgrader image pull")+ return "", errors.WrapIf(err, "failed to complete upgrader image pull")}if closeErr := pullReader.Close(); closeErr != nil {slog.Warn("Failed to close upgrader image pull reader", "error", closeErr)@@ -206,7 +220,7 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user mo},)if err != nil {- return errors.WrapIf(err, "resolve upgrader docker runtime")+ return "", errors.WrapIf(err, "resolve upgrader docker runtime")}upgradeCmd := []string{binaryPath, "upgrade", "--container", containerName}@@ -265,18 +279,18 @@ func (s *SystemUpgradeService) TriggerUpgradeViaCLI(ctx context.Context, user moName: containerName,})if err != nil {- return errors.WrapIf(err, "create upgrader container")+ return "", errors.WrapIf(err, "create upgrader container")}// Start the upgrader container - it will run the upgrade and auto-removeif _, err := dockerClient.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}); err != nil {_, _ = dockerClient.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{Force: true})- return errors.WrapIf(err, "start upgrader container")+ return "", errors.WrapIf(err, "start upgrader container")}slog.Info("Upgrade container started", "upgraderId", resp.ID[:12], "upgraderName", containerName)- return nil+ return resp.ID, nil}// hasSELinuxLabelOptInternal reports whether the security options already set@@ -423,14 +437,18 @@ func (s *SystemUpgradeService) findArcaneContainerInternal(ctx context.Context,}const (- localEnvironmentIDInternal = "0"- managerEnvironmentNameInternal = "Local"-updateAllStaleThresholdInternal = time.HourupdateAllAgentRequestTimeoutInternal = 15 * time.SecondupdateAllConfirmPollIntervalInternal = 10 * time.SecondupdateAllConfirmTimeoutInternal = 2 * time.MinuteupdateAllErrorMaxLenInternal = 500++ // How long to watch the manager's upgrader container for an exit. Generous: it+ // pulls the target image before recreating anything.+ updateAllManagerWatchTimeoutInternal = 15 * time.Minute+ // How long to let a recreate that is already underway stop this container before+ // concluding the manager was up to date and no restart is coming.+ updateAllManagerNoRestartGraceInternal = 15 * time.Second)// StartUpdateAll begins a fleet-wide update. The agents phase runs first in the@@ -460,8 +478,8 @@ func (s *SystemUpgradeService) StartUpdateAll(ctx context.Context, user models.Uinfo := s.versionService.GetAppVersionInfo(ctx)managerResult := models.EnvironmentUpdateResult{- EnvironmentID: localEnvironmentIDInternal,- EnvironmentName: managerEnvironmentNameInternal,+ EnvironmentID: LocalEnvironmentID,+ EnvironmentName: env.ResolveEnvironmentName(ctx, LocalEnvironmentID),FromVersion: info.CurrentVersion,}@@ -530,16 +548,13 @@ func (s *SystemUpgradeService) ResumeUpdateAllOnStartup(ctx context.Context) {// The agents phase already ran before the restart. Finalize the manager's own// result and close the job — do not re-run the agents phase.- s.recordManagerResultInternal(job, action.managerSucceeded, info.CurrentVersion)-- job.Status = models.EnvironmentUpdateJobStatusCompleted- job.CompletedAt = new(time.Now())- if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {- slog.WarnContext(ctx, "Failed to finalize resumed update-all job", "jobId", job.ID, "error", err)- return+ managerStatus := models.EnvironmentUpdateResultStatusFailed+ if action.managerSucceeded {+ managerStatus = models.EnvironmentUpdateResultStatusUpdated}+ s.recordManagerResultInternal(job, managerStatus, info.CurrentVersion)+ s.finalizeUpdateAllJobInternal(ctx, job)- s.logUpdateAllEventInternal(ctx, job)slog.InfoContext(ctx, "Finalized update-all job after manager restart", "jobId", job.ID, "managerUpgraded", action.managerSucceeded)}@@ -611,7 +626,7 @@ func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobID// that if the manager dies the instant the upgrader starts, the next boot sees// pending_restart and finalizes it.for i := range job.Results {- if job.Results[i].EnvironmentID == localEnvironmentIDInternal {+ if job.Results[i].EnvironmentID == LocalEnvironmentID {job.Results[i].Status = models.EnvironmentUpdateResultStatusUpdatingbreak}@@ -621,14 +636,187 @@ func (s *SystemUpgradeService) runAgentsPhaseInternal(ctx context.Context, jobIDslog.WarnContext(ctx, "update-all: failed to persist pending_restart before manager upgrade", "jobId", job.ID, "error", err)}- if err := s.TriggerUpgradeViaCLI(ctx, user, updater.SelfUpdateTarget{}); err != nil {+ // Snapshot whether the manager was already on the newest image BEFORE triggering:+ // only that makes a no-op upgrader run an expected outcome. Checked after the fact+ // the same answer proves nothing, since it was already true going in.+ wasAlreadyNewest := s.AlreadyOnNewestImage(ctx)++ upgraderID, err := s.TriggerUpgradeViaCLI(ctx, user, updater.SelfUpdateTarget{})+ if err != nil {// Agents already ran and no restart is coming, so finalize now. This flips// the manager's updating row to failed with the reason.s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade trigger failed: %v", err))return}- slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered; finalizing on next boot", "jobId", job.ID)+ slog.InfoContext(ctx, "Update-all: agents done, manager self-upgrade triggered", "jobId", job.ID, "upgraderId", upgraderID)+ s.watchManagerUpgraderInternal(ctx, job.ID, upgraderID, wasAlreadyNewest)+}++// watchManagerUpgraderInternal closes out a job whose manager never restarted. The+// upgrader skips the container recreate when the pull lands on the image already+// running, so this process survives it — and the job, persisted pending_restart for+// the next boot to finalize, would otherwise wait for a restart that never comes.+//+// Runs at the tail of the agents-phase goroutine. When the manager IS recreated this+// process is stopped instead and ResumeUpdateAllOnStartup finalizes on the next boot.+// wasAlreadyNewest is the pre-trigger version check, the only evidence that a run+// which changed nothing was supposed to change nothing.+func (s *SystemUpgradeService) watchManagerUpgraderInternal(ctx context.Context, jobID, upgraderID string, wasAlreadyNewest bool) {+ exit, exitCode := s.waitForUpgraderExitInternal(ctx, upgraderID)+ if exit == upgraderExitUnobservedInternal {+ // Never saw it finish, so a recreate may still be coming — but it may equally+ // never come, and pending_restart is only ever resolved at boot. Wait the run+ // out rather than returning: an unobserved exit must not strand the job.+ s.closeOutUnobservedUpgradeInternal(ctx, jobID)+ return+ }++ // A recreate stops this container, but not instantly — the stop carries a grace+ // period during which this goroutine still runs. Wait it out so a restart that is+ // already underway wins: if the manager is being replaced, the process dies here.+ time.Sleep(updateAllManagerNoRestartGraceInternal)++ job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)+ if err != nil || job == nil {+ slog.WarnContext(ctx, "update-all: failed to reload job after upgrader exit", "jobId", jobID, "error", err)+ return+ }+ if job.Status != models.EnvironmentUpdateJobStatusPendingRestart {+ return+ }++ if exit == upgraderExitFailedInternal {+ s.markUpdateAllFailedInternal(ctx, job, fmt.Sprintf("manager upgrade failed: upgrader exited with code %d", exitCode))+ return+ }++ info := s.versionService.GetAppVersionInfo(ctx)++ // Still running here means the container was never recreated, so the upgrader either+ // found the image already current or died before it got that far. With no exit code+ // to tell those apart, a no-op is only credible when the pre-trigger check already+ // said there was nothing to swap in and the post-run check still agrees: an after-+ // the-fact "already newest" on its own restates what was true before the run, so it+ // cannot vouch for it — and on a manager that was due an update it would launder a+ // dead upgrader (stale newest-image state, a check that regressed to the running+ // image) into a green row and zero logged failures, leaving a broken upgrade with no+ // retry path. Anything less is a failure; the job closes either way, since no restart+ // is coming.+ if exit == upgraderExitCodeLostInternal && (!wasAlreadyNewest || !agentAlreadyOnTargetInternal(info)) {+ s.markUpdateAllFailedInternal(ctx, job, "manager upgrade could not be confirmed: the upgrader exited without a readable status and this manager was not a confirmed no-op upgrade")+ return+ }++ s.recordManagerResultInternal(job, models.EnvironmentUpdateResultStatusUpToDate, info.CurrentVersion)+ slog.InfoContext(ctx, "Update-all: manager was already up to date; no restart needed", "jobId", job.ID, "version", info.CurrentVersion)+ s.finalizeUpdateAllJobInternal(ctx, job)+}++// closeOutUnobservedUpgradeInternal fails a job whose upgrader outcome was never+// observed and whose manager then never restarted. Waiting is what makes the answer+// safe: a recreate kills this process, so still being here once the job would be+// considered stale proves no restart is coming. Nothing else would close it — only a+// boot resolves pending_restart — and until it closes, every later update-all is+// refused as already in progress.+func (s *SystemUpgradeService) closeOutUnobservedUpgradeInternal(ctx context.Context, jobID string) {+ job, err := s.getUpdateAllJobByIDInternal(ctx, jobID)+ if err != nil || job == nil {+ slog.WarnContext(ctx, "update-all: failed to reload job after unobserved upgrader exit", "jobId", jobID, "error", err)+ return+ }+ if job.Status != models.EnvironmentUpdateJobStatusPendingRestart {+ return+ }++ // Hold until the job hits the same staleness bound a resumed job is judged by, so a+ // slow upgrader still gets its full window to recreate this container.+ if remaining := time.Until(job.CreatedAt.Add(updateAllStaleThresholdInternal)); remaining > 0 {+ select {+ case <-ctx.Done():+ return+ case <-time.After(remaining):+ }+ }++ job, err = s.getUpdateAllJobByIDInternal(ctx, jobID)+ if err != nil || job == nil {+ slog.WarnContext(ctx, "update-all: failed to reload stale job", "jobId", jobID, "error", err)+ return+ }+ if job.Status != models.EnvironmentUpdateJobStatusPendingRestart {+ return+ }++ slog.WarnContext(ctx, "update-all: manager upgrade was never observed and no restart followed", "jobId", job.ID)+ s.markUpdateAllFailedInternal(ctx, job, "manager upgrade could not be observed: the upgrader run was never seen to finish and the manager did not restart")+}++// upgraderExitInternal is how much the watcher managed to learn about an upgrader run.+type upgraderExitInternal int++const (+ // The run never finished observably: nothing may be concluded from it.+ upgraderExitUnobservedInternal upgraderExitInternal = iota+ // The upgrader exited 0.+ upgraderExitSucceededInternal+ // The upgrader exited non-zero.+ upgraderExitFailedInternal+ // The upgrader is gone, but its exit code was lost with it — success and failure+ // are indistinguishable.+ upgraderExitCodeLostInternal+)++// waitForUpgraderExitInternal blocks until the upgrader container stops, reporting what+// could be learned about how it ended along with its exit code when one was observed.+func (s *SystemUpgradeService) waitForUpgraderExitInternal(ctx context.Context, upgraderID string) (upgraderExitInternal, int64) {+ dockerClient, err := s.dockerService.GetClient(ctx)+ if err != nil {+ slog.WarnContext(ctx, "update-all: cannot watch upgrader container", "upgraderId", upgraderID, "error", err)+ return upgraderExitUnobservedInternal, 0+ }++ waitCtx, cancel := context.WithTimeout(ctx, updateAllManagerWatchTimeoutInternal)+ defer cancel()++ wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{+ Condition: container.WaitConditionNotRunning,+ })+ select {+ case result, ok := <-wait.Result:+ // A closed channel hands back a zero-value response, which is indistinguishable+ // from a clean exit 0 — nothing was delivered, so nothing may be concluded.+ if !ok {+ slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)+ return upgraderExitUnobservedInternal, 0+ }+ if result.StatusCode != 0 {+ return upgraderExitFailedInternal, result.StatusCode+ }+ // The daemon fills in status code 0 alongside a wait error (the container was+ // removed mid-wait, the wait itself broke): the exit is real but its code is not.+ if result.Error != nil {+ slog.WarnContext(ctx, "update-all: upgrader wait reported an error", "upgraderId", upgraderID, "error", result.Error.Message)+ return upgraderExitCodeLostInternal, 0+ }+ return upgraderExitSucceededInternal, 0+ case err, ok := <-wait.Error:+ // The upgrader runs with AutoRemove, so a fast run can be gone before the wait+ // is acknowledged. It exited, but the code went with it — a failed upgrade must+ // not be mistaken for a clean no-op, so report the ambiguity rather than a code.+ if cerrdefs.IsNotFound(err) {+ return upgraderExitCodeLostInternal, 0+ }+ if !ok || err == nil {+ slog.WarnContext(ctx, "update-all: upgrader wait ended without a status", "upgraderId", upgraderID)+ } else {+ slog.WarnContext(ctx, "update-all: failed waiting for upgrader container", "upgraderId", upgraderID, "error", err)+ }+ return upgraderExitUnobservedInternal, 0+ case <-waitCtx.Done():+ slog.WarnContext(ctx, "update-all: timed out waiting for upgrader container", "upgraderId", upgraderID)+ return upgraderExitUnobservedInternal, 0+ }}// seedRemoteResultsInternal builds a pending result row for every remote environment@@ -698,6 +886,16 @@ func (s *SystemUpgradeService) upgradeAgentInternal(ctx context.Context, env *Enreturn}+ // An agent that reports no update available and already runs the target has+ // nothing to swap in: its upgrader pulls, finds the same image and skips the+ // recreate. Confirming that would only burn the poll window waiting for a version+ // change that cannot come, so record it directly.+ if agentAlreadyOnTargetInternal(&info) {+ result.Status = models.EnvironmentUpdateResultStatusUpToDate+ result.ToVersion = info.CurrentVersion+ return+ }+if s.confirmAgentUpgradedInternal(ctx, env, envID, info) {result.Status = models.EnvironmentUpdateResultStatusUpdated} else {@@ -706,6 +904,34 @@ func (s *SystemUpgradeService) upgradeAgentInternal(ctx context.Context, env *En}}+// agentAlreadyOnTargetInternal reports whether a version check is confident that the+// environment already runs the newest image. Only a positive answer is actionable:+// anything unresolved or contradictory reports false, which keeps the full+// trigger-and-confirm flow rather than claiming there was nothing to do.+func agentAlreadyOnTargetInternal(info *version.Info) bool {+ if info == nil || info.UpdateAvailable {+ return false+ }++ currentDigest := strings.TrimSpace(info.CurrentDigest)+ newestDigest := strings.TrimSpace(info.NewestDigest)++ // Digests are the only thing that catches a mutable tag rebuilt at the same version,+ // since the semver track's UpdateAvailable ignores them entirely. Once either side+ // resolves, both must resolve and agree: with only one in hand there is no way to+ // rule out that the pull is about to replace the image, so a matching version tag+ // must not be trusted on its own.+ if currentDigest != "" || newestDigest != "" {+ return currentDigest == newestDigest+ }++ // No digest information at all — the resolved version tag is all there is to go on.+ if info.NewestVersion != "" {+ return strings.TrimPrefix(info.NewestVersion, "v") == strings.TrimPrefix(info.CurrentVersion, "v")+ }+ return false+}+// updateAllAgentFailureStatusInternal classifies a failed agent pre-check. An// environment we actually reached but whose request failed or timed out is a real// failure, not an offline skip: poll-mode agents connect on demand, so a slow@@ -851,23 +1077,43 @@ func (s *SystemUpgradeService) markUpdateAllFailedInternal(ctx context.Context,slog.WarnContext(ctx, "Update-all job failed", "jobId", job.ID, "reason", reason)}-// recordManagerResultInternal updates the manager (env "0") entry after the restart.-func (s *SystemUpgradeService) recordManagerResultInternal(job *models.EnvironmentUpdateJob, succeeded bool, currentVersion string) {+// recordManagerResultInternal sets the manager (env "0") entry to its final status:+// updated after a confirmed restart, up_to_date when the pull found nothing to swap+// in, or failed otherwise.+func (s *SystemUpgradeService) recordManagerResultInternal(job *models.EnvironmentUpdateJob, status models.EnvironmentUpdateResultStatus, currentVersion string) {for i := range job.Results {- if job.Results[i].EnvironmentID != localEnvironmentIDInternal {+ if job.Results[i].EnvironmentID != LocalEnvironmentID {continue}- if succeeded {- job.Results[i].Status = models.EnvironmentUpdateResultStatusUpdated+ job.Results[i].Status = status+ switch status {+ case models.EnvironmentUpdateResultStatusUpdated, models.EnvironmentUpdateResultStatusUpToDate:job.Results[i].ToVersion = currentVersion- } else {- job.Results[i].Status = models.EnvironmentUpdateResultStatusFailed+ case models.EnvironmentUpdateResultStatusFailed:job.Results[i].Error = "manager version did not change after upgrade"+ case models.EnvironmentUpdateResultStatusPending,+ models.EnvironmentUpdateResultStatusUpdating,+ models.EnvironmentUpdateResultStatusTriggered,+ models.EnvironmentUpdateResultStatusSkippedOffline:+ // Nothing more to record: the row keeps the target version it was seeded+ // with, since none of these outcomes establishes what it ended up running.}return}}+// finalizeUpdateAllJobInternal closes a job out as completed and records the audit+// event. The per-environment rows must already carry their final statuses.+func (s *SystemUpgradeService) finalizeUpdateAllJobInternal(ctx context.Context, job *models.EnvironmentUpdateJob) {+ job.Status = models.EnvironmentUpdateJobStatusCompleted+ job.CompletedAt = new(time.Now())+ if err := s.persistUpdateAllJobInternal(ctx, job); err != nil {+ slog.WarnContext(ctx, "update-all: failed to finalize job", "jobId", job.ID, "error", err)+ return+ }+ s.logUpdateAllEventInternal(ctx, job)+}+func (s *SystemUpgradeService) logUpdateAllEventInternal(ctx context.Context, job *models.EnvironmentUpdateJob) {failed := 0for _, r := range job.Results {diff --git a/backend/internal/services/system_upgrade_update_all_test.go b/backend/internal/services/system_upgrade_update_all_test.goindex 675f5ebab..6df2f2516 100644--- a/backend/internal/services/system_upgrade_update_all_test.go+++ b/backend/internal/services/system_upgrade_update_all_test.go@@ -176,6 +176,110 @@ func TestUpdateAllFailedJobMarksUpdatingResultsFailed(t *testing.T) {require.Equal(t, "already failed", got.Results[3].Error)}+// An up-to-date manager never restarts: its upgrader pulls, finds the image already+// running and skips the recreate. The pending_restart job must then be finalized in+// place, because no next boot is coming to do it.+func TestUpdateAllFinalizesUpToDateManagerWithoutRestart(t *testing.T) {+ ctx := context.Background()+ gormDB, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ require.NoError(t, err)++ db := &database.DB{DB: gormDB}+ require.NoError(t, db.AutoMigrate(&models.EnvironmentUpdateJob{}, &models.Event{}))++ svc := NewSystemUpgradeService(db, nil, nil, NewEventService(db, nil, nil), nil)+ job := &models.EnvironmentUpdateJob{+ Status: models.EnvironmentUpdateJobStatusPendingRestart,+ UserID: "user-1",+ Username: "arcane",+ ManagerVersionAtStart: "v1.0.0",+ Results: models.EnvironmentUpdateResults{+ {EnvironmentID: "0", EnvironmentName: "Local", Status: models.EnvironmentUpdateResultStatusUpdating, FromVersion: "v1.0.0"},+ {EnvironmentID: "remote-1", EnvironmentName: "palladium", Status: models.EnvironmentUpdateResultStatusUpToDate},+ },+ }+ require.NoError(t, db.WithContext(ctx).Create(job).Error)++ svc.recordManagerResultInternal(job, models.EnvironmentUpdateResultStatusUpToDate, "v1.0.0")+ svc.finalizeUpdateAllJobInternal(ctx, job)++ var got models.EnvironmentUpdateJob+ require.NoError(t, db.WithContext(ctx).First(&got, "id = ?", job.ID).Error)++ // Nothing is left waiting on a restart that will never happen.+ require.Equal(t, models.EnvironmentUpdateJobStatusCompleted, got.Status)+ require.NotNil(t, got.CompletedAt)+ require.Nil(t, got.Error)++ require.Equal(t, "0", got.Results[0].EnvironmentID)+ require.Equal(t, models.EnvironmentUpdateResultStatusUpToDate, got.Results[0].Status)+ require.Equal(t, "v1.0.0", got.Results[0].ToVersion)+ require.Empty(t, got.Results[0].Error)+ require.Equal(t, models.EnvironmentUpdateResultStatusUpToDate, got.Results[1].Status)+}++// The up-to-date short-circuit must only fire when the version check is conclusive:+// an agent that could not resolve a newest version or digest keeps the full+// trigger-and-confirm flow rather than being reported as already current.+func TestAgentAlreadyOnTarget(t *testing.T) {+ tests := []struct {+ name string+ info version.Info+ want bool+ }{+ {+ name: "on newest version",+ info: version.Info{CurrentVersion: "1.2.3", NewestVersion: "v1.2.3"},+ want: true,+ },+ {+ name: "update available",+ info: version.Info{CurrentVersion: "1.2.3", NewestVersion: "v1.3.0", UpdateAvailable: true},+ want: false,+ },+ {+ name: "on newest digest",+ info: version.Info{CurrentDigest: "sha256:abc", NewestDigest: "sha256:abc"},+ want: true,+ },+ {+ // A mutable tag rebuilt at the same version: the semver track reports no+ // update, but the differing digest means the pull really will replace the+ // image, so this must not be treated as already current.+ name: "same version, rebuilt digest",+ info: version.Info{+ CurrentVersion: "1.2.3",+ NewestVersion: "v1.2.3",+ CurrentDigest: "sha256:old",+ NewestDigest: "sha256:new",+ },+ want: false,+ },+ {+ // Only one digest resolved, so a rebuild cannot be ruled out: the matching+ // version tag must not be enough on its own.+ name: "same version, remote digest unresolved",+ info: version.Info{+ CurrentVersion: "1.2.3",+ NewestVersion: "v1.2.3",+ CurrentDigest: "sha256:running",+ },+ want: false,+ },+ {+ name: "inconclusive check resolves nothing",+ info: version.Info{CurrentVersion: "1.2.3"},+ want: false,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ require.Equal(t, tt.want, agentAlreadyOnTargetInternal(&tt.info))+ })+ }+}+// With the manager-last ordering, a resumed pending_restart job means the agents// phase already ran before the restart: resume must finalize the manager's own row// and complete the job, NOT re-run the agents phase.diff --git a/backend/internal/services/updater_service.go b/backend/internal/services/updater_service.goindex 61ae2347b..dd9bbfe08 100644--- a/backend/internal/services/updater_service.go+++ b/backend/internal/services/updater_service.go@@ -56,7 +56,9 @@ type updaterDependenciesInternal struct {}type selfUpgradeServiceInternal interface {- TriggerUpgradeViaCLI(ctx context.Context, user models.User, target updater.SelfUpdateTarget) error+ // TriggerUpgradeViaCLI returns the spawned upgrader container's ID, which this+ // service does not need — only update-all's manager step uses it.+ TriggerUpgradeViaCLI(ctx context.Context, user models.User, target updater.SelfUpdateTarget) (string, error)}// NewUpdaterService constructs the Arcane updater facade.@@ -676,7 +678,7 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.Ss.markSelfUpdateTriggeredInternal(ctx, target)}- if err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, target); err != nil {+ if _, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, target); err != nil {return errors.WrapIf(err, "CLI upgrade failed")}return nildiff --git a/backend/internal/services/updater_service_test.go b/backend/internal/services/updater_service_test.goindex 2e8575df1..42dbd6fc1 100644--- a/backend/internal/services/updater_service_test.go+++ b/backend/internal/services/updater_service_test.go@@ -181,11 +181,11 @@ type mockSystemUpgradeServiceInternal struct {capturedTarget *updater.SelfUpdateTarget}-func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Context, user models.User, target updater.SelfUpdateTarget) error {+func (m *mockSystemUpgradeServiceInternal) TriggerUpgradeViaCLI(_ context.Context, user models.User, target updater.SelfUpdateTarget) (string, error) {m.triggerCalled = truem.capturedUser = &userm.capturedTarget = &target- return m.triggerError+ return "upgrader-container-id", m.triggerError}func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 7891bacb1..9e8e64ff7 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -1218,8 +1218,9 @@"environments_download_edge_ca": "Download Edge CA","update_all": "Update All","environments_update_all_title": "Update all environments",- "environments_update_all_message": "This pulls the latest Arcane image and updates every environment, whether or not an update is reported. Agents update first, then this manager, which will restart. Offline environments are skipped.",+ "environments_update_all_message": "This pulls the latest Arcane image and updates every environment, whether or not an update is reported. Agents update first, then this manager, which restarts only if the pull brought a new image. Offline environments are skipped.","environments_update_all_manager_restarting": "Manager is restarting, reconnecting…",+ "environments_update_all_manager_note": "Agents update first, then this manager. Arcane refreshes on its own when the run finishes.","environments_update_all_in_progress": "Updating environments…","environments_update_all_progress": "{done} of {total}","environments_update_all_completed": "All environments processed",diff --git a/frontend/src/lib/components/dialogs/update-all-dialog.svelte b/frontend/src/lib/components/dialogs/update-all-dialog.svelteindex 0256d4dd1..3bc29029c 100644--- a/frontend/src/lib/components/dialogs/update-all-dialog.svelte+++ b/frontend/src/lib/components/dialogs/update-all-dialog.svelte@@ -7,8 +7,10 @@import { m } from '#lib/paraglide/messages';import { toast } from 'svelte-sonner';import { onDestroy } from 'svelte';+ import { refreshAll } from '$app/navigation';import systemUpgradeService, {type UpdateAllJob,+ type UpdateAllEnvironmentResult,type UpdateAllEnvironmentStatus} from '#lib/services/api/system-upgrade-service';import { SuccessIcon, ClockIcon, AlertIcon, AlertTriangleIcon, ExternalLinkIcon } from '#lib/icons';@@ -24,17 +26,21 @@open = $bindable(undefined),versionInformation,canConfirm = true,+ debugDemo = false,onFinished}: {open?: boolean;versionInformation?: AppVersionInformation;canConfirm?: boolean;+ /** Dev-only: run a scripted fake fleet instead of calling the API. */+ debugDemo?: boolean;onFinished?: () => void | Promise<void>;} = $props();type Phase = 'confirm' | 'running' | 'finished';const POLL_INTERVAL_MS = 3000;+ const MANAGER_ENVIRONMENT_ID = '0';let phase = $state<Phase>('confirm');let job = $state<UpdateAllJob | null>(null);@@ -67,14 +73,15 @@}function finishTerminalJob(terminalJob: UpdateAllJob) {- const managerUpdated =- terminalJob.status === 'completed' &&- (terminalJob.results?.some((result) => result.environmentId === '0' && result.status === 'updated') ?? false);- if (managerUpdated) {- window.location.reload();- return;+ const managerRestarted =+ terminalJob.results?.some((result) => result.environmentId === MANAGER_ENVIRONMENT_ID && result.status === 'updated') ??+ false;+ // Keep the upgrade flag armed when the manager actually restarted: the refresh in+ // handleClose may hit the new backend with a stale token, and api-service only+ // recovers that version-mismatch 401 while the flag is set.+ if (!managerRestarted) {+ BaseAPIService.setUpgradeInProgress(false);}- BaseAPIService.setUpgradeInProgress(false);phase = 'finished';}@@ -100,6 +107,12 @@async function handleConfirm() {phase = 'running';reconnecting = false;++ if (debugDemo) {+ void runDebugDemo();+ return;+ }+BaseAPIService.setUpgradeInProgress(true);try {job = await systemUpgradeService.triggerUpdateAll();@@ -121,6 +134,17 @@}async function handleClose() {+ stopPolling();+ // Refresh through SvelteKit instead of reloading the document: a hard reload+ // lands while agents are still reconnecting after the fleet restart, which+ // renders every environment as offline for a moment.+ if (job && !debugDemo) {+ try {+ await refreshAll();+ } catch {+ // A failed refresh must not trap the dialog open.+ }+ }resetState();open = false;await onFinished?.();@@ -153,9 +177,10 @@// "Done" is every environment that has reached a terminal state — i.e. anything// that isn't still queued (pending) or actively being worked on (updating).- const totalCount = $derived(job?.results?.length ?? 0);- const doneCount = $derived(job?.results?.filter((r) => r.status !== 'pending' && r.status !== 'updating').length ?? 0);- const progressPct = $derived(totalCount > 0 ? Math.round((doneCount / totalCount) * 100) : 0);+ const results = $derived(job?.results ?? []);+ const totalCount = $derived(results.length);+ const doneCount = $derived(results.filter((r) => r.status !== 'pending' && r.status !== 'updating').length);+ const failed = $derived(job?.status === 'failed');function statusLabel(status: UpdateAllEnvironmentStatus): string {switch (status) {@@ -163,6 +188,8 @@return m.common_action_updating();case 'updated':return m.common_updated();+ case 'up_to_date':+ return m.image_update_up_to_date_title();case 'triggered':return m.environments_update_all_status_triggered();case 'skipped_offline':@@ -173,6 +200,118 @@return m.common_pending();}}++ // One segment per environment, colored by outcome: the whole fleet's state reads+ // at a glance, and an in-flight environment stays indeterminate because the+ // backend reports status transitions, never a percentage.+ function segmentClass(status: UpdateAllEnvironmentStatus): string {+ switch (status) {+ case 'updating':+ return 'bg-primary animate-pulse';+ case 'updated':+ case 'triggered':+ case 'up_to_date':+ return 'bg-green-500';+ case 'skipped_offline':+ return 'bg-amber-500';+ case 'failed':+ return 'bg-destructive';+ default:+ return 'bg-muted';+ }+ }++ function badgeClass(status: UpdateAllEnvironmentStatus): string {+ switch (status) {+ case 'updated':+ case 'triggered':+ case 'up_to_date':+ return 'border-green-500/40 bg-green-500/10 text-green-600 dark:text-green-400';+ case 'updating':+ return 'border-primary/40 bg-primary/10 text-primary';+ case 'skipped_offline':+ return 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400';+ case 'failed':+ return 'border-destructive/40 bg-destructive/10 text-destructive';+ default:+ return 'border-border bg-muted/40 text-muted-foreground/60';+ }+ }++ function labelClass(status: UpdateAllEnvironmentStatus): string {+ switch (status) {+ case 'updating':+ return 'text-primary';+ case 'skipped_offline':+ return 'text-amber-600 dark:text-amber-400';+ case 'failed':+ return 'text-destructive';+ default:+ return 'text-muted-foreground';+ }+ }++ // An environment that was already current reports the same version twice; show it+ // once rather than as a "v1.0.0 → v1.0.0" no-op transition.+ function versionLine(result: UpdateAllEnvironmentResult): string {+ const from = result.fromVersion?.trim() ?? '';+ const to = result.toVersion?.trim() ?? '';+ if (!from || !to || from === to) return to || from;+ return `${from} → ${to}`;+ }++ // Dev-only preview: step a fake fleet through every row state so this dialog can be+ // designed and reviewed without triggering a real fleet update. Only ever enabled+ // from a dev-guarded callsite.+ async function runDebugDemo() {+ const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));+ const demo: Array<{ name: string; outcome: UpdateAllEnvironmentStatus; to: string; error?: string }> = [+ { name: 'Local Docker', outcome: 'updated', to: 'v0.9.2' },+ { name: 'palladium', outcome: 'updated', to: 'v0.9.2' },+ { name: 'naswidc1.ofkm.us', outcome: 'up_to_date', to: 'v0.9.1' },+ { name: 'ofkm-cloud', outcome: 'skipped_offline', to: '' },+ { name: 'parquetide', outcome: 'failed', to: '', error: 'dial tcp 10.0.0.9:3552: connect: connection refused' }+ ];++ job = {+ id: 'demo',+ status: 'running',+ createdAt: new Date().toISOString(),+ results: demo.map((entry, index) => ({+ environmentId: index === 0 ? MANAGER_ENVIRONMENT_ID : `demo-${index}`,+ environmentName: entry.name,+ status: 'pending',+ fromVersion: 'v0.9.1'+ }))+ };++ // Remotes first, manager last — the real processing order.+ const steps = [...demo.entries()].map(([index, entry]) => ({ index, entry }));+ for (const { index, entry } of [...steps.slice(1), ...steps.slice(0, 1)]) {+ const starting = job?.results?.[index];+ if (phase !== 'running' || !starting) return;+ starting.status = 'updating';+ await wait(1400);++ // The manager restart is the one moment the reconnecting banner shows.+ if (index === 0) {+ reconnecting = true;+ await wait(1600);+ reconnecting = false;+ }++ const row = job?.results?.[index];+ if (phase !== 'running' || !row) return;+ row.status = entry.outcome;+ row.toVersion = entry.to;+ if (entry.error) row.error = entry.error;+ }++ if (phase !== 'running' || !job) return;+ job.status = 'completed';+ job.completedAt = new Date().toISOString();+ phase = 'finished';+ }</script><Dialog.Root@@ -185,24 +324,25 @@}}><Dialog.Content- class="sm:max-w-[520px]"+ class={cn('gap-0 overflow-hidden p-0 sm:max-w-[520px]', phase === 'running' && '[&>button]:hidden')}onInteractOutside={(e: Event) => {if (phase === 'running') e.preventDefault();}}>- <Dialog.Header>- <Dialog.Title>{title}</Dialog.Title>- {#if phase === 'confirm'}- <Dialog.Description>{m.environments_update_all_message()}</Dialog.Description>- {/if}- </Dialog.Header>-- {#if phase === 'confirm' && versionInformation}- <div class="space-y-4">- <VersionUpdateSummary {versionInformation} {releasedAgo} />+ {#if phase === 'confirm'}+ <div class="px-6 pt-6 pb-4">+ <Dialog.Header class="space-y-3">+ <Dialog.Title class="text-lg">{title}</Dialog.Title>+ <Dialog.Description>{m.environments_update_all_message()}</Dialog.Description>+ {#if versionInformation}+ <VersionUpdateSummary {versionInformation} {releasedAgo} />+ {/if}+ </Dialog.Header>+ </div>- <div class="border-t border-border/60 pt-3">- <div class="flex items-center justify-between pb-2">+ {#if versionInformation}+ <div class="border-t border-border/60">+ <div class="flex items-center justify-between px-6 pt-4 pb-2"><h3 class="text-sm font-semibold text-foreground">{m.update_center_whats_new()}</h3>{#if releaseUrl}<a@@ -216,7 +356,7 @@</a>{/if}</div>- <ScrollArea.Root class="h-[220px]">+ <ScrollArea.Root class="h-[220px] px-6 pb-4">{#if releaseNotes}<ReleaseNotes markdown={releaseNotes} />{:else}@@ -224,47 +364,58 @@{/if}</ScrollArea.Root></div>- </div>- {/if}+ {/if}+ {:else}+ <div class="px-6 pt-6 pb-4">+ <Dialog.Header>+ <div class="flex items-center justify-between gap-3">+ <div class="flex min-w-0 items-center gap-2">+ {#if phase === 'finished'}+ {#if failed}+ <AlertTriangleIcon class="size-4 shrink-0 text-destructive" />+ {:else}+ <SuccessIcon class="size-4 shrink-0 text-green-600 dark:text-green-400" />+ {/if}+ {/if}+ <Dialog.Title class="truncate text-lg">{title}</Dialog.Title>+ </div>+ {#if totalCount > 0}+ <span class="shrink-0 text-xs text-muted-foreground tabular-nums">+ {m.environments_update_all_progress({ done: doneCount, total: totalCount })}+ </span>+ {/if}+ </div>+ </Dialog.Header>++ {#if totalCount > 0}+ <div class="mt-3 flex gap-1">+ {#each results as result (result.environmentId)}+ <div class={cn('h-1.5 flex-1 rounded-full transition-colors', segmentClass(result.status))}></div>+ {/each}+ </div>+ {/if}- {#if phase !== 'confirm'}- <div class="space-y-3">{#if reconnecting}- <div class="flex items-center gap-2 text-sm text-muted-foreground">+ <div+ class="mt-4 flex items-center gap-2 rounded-lg border border-amber-500/20 bg-amber-500/5 px-3 py-2.5 text-sm text-amber-700 dark:text-amber-400"+ ><Spinner class="size-4" /><span>{m.environments_update_all_manager_restarting()}</span></div>{/if}+ </div>- {#if job?.results?.length}- {#if phase === 'running'}- <div class="space-y-1.5">- <div class="flex items-center justify-end text-xs text-muted-foreground">- <span>{m.environments_update_all_progress({ done: doneCount, total: totalCount })}</span>- </div>- <div class="h-1.5 overflow-hidden rounded-full bg-muted">- <div class="h-full rounded-full bg-primary transition-all duration-500" style="width: {progressPct}%"></div>- </div>- </div>- {/if}-+ {#if totalCount > 0}+ <div class="border-t border-border/60"><ScrollArea.Root class="max-h-72">- <ul class="divide-y divide-border">- {#each job.results as result (result.environmentId)}- <li class="flex items-center gap-3 py-2.5 text-sm">+ <ul class="divide-y divide-border/50">+ {#each results as result (result.environmentId)}+ {@const versions = versionLine(result)}+ <li class="flex items-center gap-3 px-6 py-2.5 text-sm"><span- class={cn(- 'flex size-7 shrink-0 items-center justify-center rounded-full border',- (result.status === 'updated' || result.status === 'triggered') &&- 'border-green-500/40 bg-green-500/10 text-green-600 dark:text-green-400',- result.status === 'updating' && 'border-primary/40 bg-primary/10 text-primary',- result.status === 'skipped_offline' &&- 'border-amber-500/40 bg-amber-500/10 text-amber-600 dark:text-amber-400',- result.status === 'failed' && 'border-destructive/40 bg-destructive/10 text-destructive',- result.status === 'pending' && 'border-border bg-muted/40 text-muted-foreground/60'- )}+ class={cn('flex size-7 shrink-0 items-center justify-center rounded-full border', badgeClass(result.status))}>- {#if result.status === 'updated' || result.status === 'triggered'}+ {#if result.status === 'updated' || result.status === 'triggered' || result.status === 'up_to_date'}<SuccessIcon class="size-3.5" />{:else if result.status === 'updating'}<Spinner class="size-3.5" />@@ -278,40 +429,42 @@</span><div class="min-w-0 flex-1">- <span class="block truncate font-medium">{result.environmentName}</span>- {#if result.status === 'updating'}- <div class="mt-1.5 h-1 overflow-hidden rounded-full bg-muted">- <div class="update-all-capbar h-full rounded-full bg-primary"></div>- </div>- {:else if result.error}+ <div class="flex items-center gap-1.5">+ <span class="truncate font-medium">{result.environmentName}</span>+ {#if result.environmentId === MANAGER_ENVIRONMENT_ID}+ <span class="shrink-0 rounded border px-1 text-[10px] text-muted-foreground">{m.manager()}</span>+ {/if}+ </div>+ {#if result.status === 'failed' && result.error}<span class="block truncate text-xs text-muted-foreground" title={result.error}>{result.error}</span>+ {:else if versions}+ <span class="block truncate text-xs text-muted-foreground tabular-nums">{versions}</span>{/if}</div>- <span- class={cn(- 'shrink-0 text-xs',- result.status === 'updating' ? 'text-primary' : 'text-muted-foreground',- result.status === 'skipped_offline' && 'text-amber-600 dark:text-amber-400',- result.status === 'failed' && 'text-destructive'- )}- >+ <span class={cn('shrink-0 text-xs', labelClass(result.status))}>{statusLabel(result.status)}</span></li>{/each}</ul></ScrollArea.Root>- {:else}- <div class="flex items-center gap-2 py-2 text-sm text-muted-foreground">- <Spinner class="size-4" />- <span>{m.environments_update_all_in_progress()}</span>- </div>- {/if}- </div>+ </div>+ {:else}+ <div class="flex items-center gap-2 border-t border-border/60 px-6 py-4 text-sm text-muted-foreground">+ <Spinner class="size-4" />+ <span>{m.environments_update_all_in_progress()}</span>+ </div>+ {/if}++ {#if phase === 'running'}+ <div class="border-t border-border/60 bg-muted/30 px-6 py-3">+ <p class="text-xs leading-relaxed text-muted-foreground">{m.environments_update_all_manager_note()}</p>+ </div>+ {/if}{/if}- <Dialog.Footer>+ <Dialog.Footer class="border-t border-border/60 px-6 py-4">{#if phase === 'confirm'}<Button variant="outline" onclick={() => (open = false)}>{m.common_cancel()}</Button>{#if canConfirm}@@ -323,21 +476,3 @@</Dialog.Footer></Dialog.Content></Dialog.Root>--<style>- /* Indeterminate, capped progress: a short segment slides across the track so a- busy environment never reads as "100% complete" while the upgrade is still- running (we have no real per-step percentage from the backend). */- @keyframes update-all-indeterminate {- 0% {- transform: translateX(-100%);- }- 100% {- transform: translateX(250%);- }- }- .update-all-capbar {- width: 40%;- animation: update-all-indeterminate 1.4s ease-in-out infinite;- }-</style>diff --git a/frontend/src/lib/components/dialogs/update-center-dialog.svelte b/frontend/src/lib/components/dialogs/update-center-dialog.svelteindex 32a2ed91b..c9b60e024 100644--- a/frontend/src/lib/components/dialogs/update-center-dialog.svelte+++ b/frontend/src/lib/components/dialogs/update-center-dialog.svelte@@ -31,7 +31,11 @@}: {open?: boolean;versionInformation?: AppVersionInformation;- onConfirm: () => void | Promise<void>;+ /**+ * Triggers the upgrade. Resolving with `upToDate` lets this dialog settle+ * immediately instead of waiting out the no-op detection below.+ */+ onConfirm: () => void | Promise<void | { upToDate?: boolean }>;environmentName?: string;environmentId?: string;canInstall?: boolean;@@ -115,6 +119,10 @@// to decide whether enough time has passed since the agent was told to// upgrade for "no version change" to be conclusive.let triggerResolvedAt = $state<number | null>(null);+ // The backend told us this environment already ran the newest image, so the pull+ // found nothing to swap in and no restart is coming. Lets the no-op detection below+ // conclude at once instead of waiting out its grace period.+ let reportedUpToDate = $state(false);function short(v?: string | null, n = 12): string {if (!v) return '';@@ -270,9 +278,10 @@!!baselineVersionInfo &&!changed &&sinceTriggerMs !== null &&- sinceTriggerMs >= NO_OP_GRACE_MS+ (reportedUpToDate || sinceTriggerMs >= NO_OP_GRACE_MS)) {log('no-op-detected', {+ reportedUpToDate,sinceTriggerMs,currentDigest: short(info.currentDigest),baselineDigest: short(baselineVersionInfo?.currentDigest)@@ -358,12 +367,14 @@// trigger eventually returns successfully we record the timestamp so the// poll loop can run its no-op detection against it.triggerResolvedAt = null;+ reportedUpToDate = false;let triggerErrored = false;const triggerPromise = Promise.resolve().then(() => onConfirm())- .then(() => {+ .then((result) => {triggerResolvedAt = Date.now();- log('trigger-resolved');+ reportedUpToDate = !!result && result.upToDate === true;+ log('trigger-resolved', { upToDate: reportedUpToDate });}).catch((err) => {log('trigger-error', err);diff --git a/frontend/src/lib/services/api/system-upgrade-service.ts b/frontend/src/lib/services/api/system-upgrade-service.tsindex a1af00070..b1af717b1 100644--- a/frontend/src/lib/services/api/system-upgrade-service.ts+++ b/frontend/src/lib/services/api/system-upgrade-service.ts@@ -11,13 +11,25 @@ export interface UpgradeResponse {message: string;success: boolean;error?: string;+ /**+ * The environment already runs the newest image, so the upgrade pulled it again and+ * left the container in place. Callers can stop waiting for a restart.+ */+ upToDate: boolean;}export interface HealthCheckResult {healthy: boolean;}-export type UpdateAllEnvironmentStatus = 'pending' | 'updating' | 'updated' | 'triggered' | 'skipped_offline' | 'failed';+export type UpdateAllEnvironmentStatus =+ | 'pending'+ | 'updating'+ | 'updated'+ | 'up_to_date'+ | 'triggered'+ | 'skipped_offline'+ | 'failed';export interface UpdateAllEnvironmentResult {environmentId: string;@@ -59,8 +71,14 @@ async function checkUpgradeAvailable(environmentId: string = '0'): Promise<Upgra* @param environmentId - Environment ID (defaults to the local manager, '0')*/async function triggerUpgrade(environmentId: string = '0'): Promise<UpgradeResponse> {- const res = await apiClient.post<UpgradeResponse>(`/environments/${environmentId}/system/upgrade`);- return res.data;+ const res = await apiClient.post<ApiResponse<{ message?: string; upToDate?: boolean }>>(+ `/environments/${environmentId}/system/upgrade`+ );+ return {+ success: res.data.success,+ message: res.data.data?.message ?? res.data.message ?? '',+ upToDate: res.data.data?.upToDate ?? false+ };}/**diff --git a/frontend/src/routes/(app)/dashboard/dashboard-environment-upgrade-action.svelte b/frontend/src/routes/(app)/dashboard/dashboard-environment-upgrade-action.svelteindex 0891de205..a655d9d0f 100644--- a/frontend/src/routes/(app)/dashboard/dashboard-environment-upgrade-action.svelte+++ b/frontend/src/routes/(app)/dashboard/dashboard-environment-upgrade-action.svelte@@ -102,6 +102,7 @@}toast.success(m.upgrade_success());await onRefreshRequested?.();+ return { upToDate: result.upToDate };} catch (error) {toastUpgradeError(error, m.upgrade_failed);throw error;diff --git a/frontend/src/routes/(app)/environments/+page.svelte b/frontend/src/routes/(app)/environments/+page.svelteindex c8d254cfb..b90af43a7 100644--- a/frontend/src/routes/(app)/environments/+page.svelte+++ b/frontend/src/routes/(app)/environments/+page.svelte@@ -1,5 +1,7 @@<script lang="ts">import { toast } from 'svelte-sonner';+ import { dev } from '$app/env';+ import { page } from '$app/state';import NewEnvironmentSheet from '#lib/components/sheets/new-environment-sheet.svelte';import EnvironmentTable from './environment-table.svelte';import { tryCatch } from '#lib/utils/api';@@ -79,6 +81,14 @@});}+ // Dev-only: `?updateAllDemo=1` opens the update-all dialog against a scripted fake+ // fleet, so its progress and result states can be reviewed under `just dev` without+ // updating anything. `dev` is compiled out of production builds.+ const updateAllDemo = $derived(dev && page.url.searchParams.get('updateAllDemo') === '1');+ $effect(() => {+ if (updateAllDemo) showUpdateAllDialog = true;+ });+async function onEnvironmentCreated() {showEnvironmentSheet = false;environments = await environmentManagementService.getEnvironments(requestOptions);@@ -150,6 +160,6 @@{#snippet additionalContent()}<NewEnvironmentSheet bind:open={showEnvironmentSheet} {onEnvironmentCreated} />- <UpdateAllDialog bind:open={showUpdateAllDialog} onFinished={refresh} />+ <UpdateAllDialog bind:open={showUpdateAllDialog} debugDemo={updateAllDemo} onFinished={refresh} />{/snippet}</ResourcePageLayout>diff --git a/frontend/src/routes/(app)/environments/environment-table.svelte b/frontend/src/routes/(app)/environments/environment-table.svelteindex e87025c92..e1e028bc9 100644--- a/frontend/src/routes/(app)/environments/environment-table.svelte+++ b/frontend/src/routes/(app)/environments/environment-table.svelte@@ -19,6 +19,7 @@import systemUpgradeService from '#lib/services/api/system-upgrade-service';import UpdateCenterDialog from '#lib/components/dialogs/update-center-dialog.svelte';import EnvironmentUpgradeMenuItem from './environment-upgrade-menu-item.svelte';+ import EnvironmentVersionCell from './environment-version-cell.svelte';import type { AppVersionInformation } from '#lib/types/settings';import { hasPermission } from '#lib/utils/auth';import { environmentStore } from '#lib/stores/environment.store.svelte';@@ -197,6 +198,7 @@throw new Error(result.error || m.upgrade_failed({ error: 'Unknown error' }));}toast.success(m.upgrade_success());+ return { upToDate: result.upToDate };} catch (error) {toastUpgradeError(error, m.upgrade_failed);throw error;@@ -264,9 +266,10 @@cell: EnabledCell},{- accessorKey: 'apiUrl',- title: m.environments_api_url(),- cell: ApiCell+ id: 'version',+ title: m.common_version(),+ accessorFn: (row) => row,+ cell: VersionCell}] satisfies ColumnSpec<Environment>[];@@ -341,8 +344,8 @@<Badge variant={typeVariant} minWidth="20">{typeLabel}</Badge>{/snippet}-{#snippet ApiCell({ value }: { value: unknown })}- <span class="font-mono text-sm text-muted-foreground">{String(value)}</span>+{#snippet VersionCell({ item }: { item: Environment })}+ <EnvironmentVersionCell environment={item} isOnline={isEnvironmentOnline(item)} />{/snippet}{#snippet EnabledCell({ value }: { value: unknown })}diff --git a/frontend/src/routes/(app)/environments/environment-version-cell.svelte b/frontend/src/routes/(app)/environments/environment-version-cell.sveltenew file mode 100644index 000000000..f55acf06f--- /dev/null+++ b/frontend/src/routes/(app)/environments/environment-version-cell.svelte@@ -0,0 +1,41 @@+<script lang="ts">+ import { m } from '#lib/paraglide/messages';+ import { queryKeys } from '#lib/query/query-keys';+ import systemUpgradeService from '#lib/services/api/system-upgrade-service';+ import type { Environment } from '#lib/types/environment';+ import { UpdateIcon } from '#lib/icons';+ import { createQuery } from '@tanstack/svelte-query';++ let { environment, isOnline }: { environment: Environment; isOnline: boolean } = $props();++ // Shares its key with the upgrade menu item, so opening a row's actions reuses this+ // result instead of asking the agent again. Offline environments are never queried:+ // remote version lookups are proxied through the agent and would just time out.+ const versionQuery = createQuery(() => ({+ queryKey: queryKeys.system.versionInfo(environment.id),+ queryFn: () => systemUpgradeService.getVersionInfo(environment.id),+ enabled: isOnline,+ retry: false+ }));++ const info = $derived(versionQuery.data ?? null);+ const label = $derived(info?.displayVersion || info?.currentVersion || info?.currentTag || '');+</script>++{#if label}+ <div class="flex items-center gap-1.5">+ <span class="font-mono text-sm">{label}</span>+ {#if info?.updateAvailable}+ <span+ class="inline-flex items-center text-amber-600 dark:text-amber-400"+ title={info.newestVersion+ ? m.sidebar_update_available_tooltip({ version: info.newestVersion })+ : m.sidebar_update_available()}+ >+ <UpdateIcon class="size-3.5" />+ </span>+ {/if}+ </div>+{:else}+ <span class="text-sm text-muted-foreground">—</span>+{/if}diff --git a/tests/spec/image-updates.spec.ts b/tests/spec/image-updates.spec.tsindex e8169f360..2e56b42b4 100644--- a/tests/spec/image-updates.spec.ts+++ b/tests/spec/image-updates.spec.ts@@ -114,6 +114,13 @@ test.describe('Image Update UI - Check All Updates Button', () => {});test('should trigger bulk update check when clicking Check Updates button', async ({ page }) => {+ await page.route(`**${ROUTES.apiImageUpdatesCheckAll}`, async (route) => {+ await route.fulfill({+ status: 200,+ contentType: 'application/json',+ body: JSON.stringify({ success: true, data: {} })+ });+ });await navigateToImages(page);const checkUpdatesButton = await getCheckUpdatesAction(page);diff --git a/tests/spec/swarm-ui.spec.ts b/tests/spec/swarm-ui.spec.tsindex 28e10b04d..6875d31db 100644--- a/tests/spec/swarm-ui.spec.ts+++ b/tests/spec/swarm-ui.spec.ts@@ -30,15 +30,17 @@ test.describe('Swarm UI', () => {test('cluster page renders correct lifecycle controls for current swarm state', async ({page}) => {+ const swarmInfoResponsePromise = page.waitForResponse('**/api/environments/*/swarm/info');await page.goto('/swarm/cluster');await page.waitForLoadState('load');+ const swarmInfoResponse = await swarmInfoResponsePromise;await expect(page.getByRole('heading', { name: 'Cluster', level: 1 })).toBeVisible();const initializeCard = page.locator('[data-slot="card-title"]').filter({ hasText: 'Initialize Cluster' });- if ((await initializeCard.count()) === 0) {+ if (swarmInfoResponse.ok()) {await expect(page.getByRole('button', { name: 'Actions' })).toBeVisible();await expect(page.getByText('Initialize Cluster')).toHaveCount(0);await expect(page.getByText('Join Existing Cluster')).toHaveCount(0);diff --git a/tests/spec/system-upgrade.spec.ts b/tests/spec/system-upgrade.spec.tsindex ba645186c..d10632e3e 100644--- a/tests/spec/system-upgrade.spec.ts+++ b/tests/spec/system-upgrade.spec.ts@@ -100,7 +100,9 @@ async function currentReloadCount(page: Page) {}test.describe('Manager self-update recovery', () => {- test('Update All reloads automatically when the manager is updated', async ({ page }) => {+ test('Update All keeps the manager result visible and refreshes without a document reload', async ({+ page+ }) => {await registerReloadCounter(page);await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {await fulfillJob(route, 'running', 'updating');@@ -112,7 +114,19 @@ test.describe('Manager self-update recovery', () => {await page.goto('/environments');await openAndConfirmUpdateAll(page);- await expect.poll(() => currentReloadCount(page), { timeout: 10_000 }).toBe(2);+ // The finished result stays on screen instead of the page being yanked away.+ const dialog = page.getByRole('dialog');+ await expect(dialog.getByRole('heading', { name: 'All environments processed' })).toBeVisible({+ timeout: 10_000+ });+ // A hard reload here would re-run every load function while the agents are still+ // reconnecting, briefly rendering the whole fleet as offline.+ await expect.poll(() => currentReloadCount(page)).toBe(1);++ // Closing refreshes through SvelteKit, which reloads data but not the document.+ await dialog.getByRole('button', { name: 'Close', exact: true }).first().click();+ await expect(dialog).toBeHidden();+ await expect.poll(() => currentReloadCount(page)).toBe(1);await expect(page).toHaveURL('/environments');});