Skip to content

Modern Docker Management, Designed for Everyone

fix: surface Update All start errors and adopt an already-running job (#4104)

Kyle Mendelldc89ea0b1920
diff --git a/frontend/src/lib/components/dialogs/update-all-dialog.svelte b/frontend/src/lib/components/dialogs/update-all-dialog.svelte
index f9fc41d0d..5d3b993ef 100644
--- a/frontend/src/lib/components/dialogs/update-all-dialog.svelte
+++ b/frontend/src/lib/components/dialogs/update-all-dialog.svelte
@@ -6,7 +6,6 @@
import Spinner from '#lib/components/ui/spinner/spinner.svelte';
import { cn } from '#lib/utils.js';
import { m } from '#lib/paraglide/messages.js';
- import { toast } from 'svelte-sonner';
import { onDestroy } from 'svelte';
import { refreshAll } from '$app/navigation';
import systemUpgradeService, {
@@ -15,7 +14,8 @@
type UpdateAllEnvironmentStatus
} from '#lib/services/api/system-upgrade-service.js';
import { SuccessIcon, ClockIcon, AlertIcon, AlertTriangleIcon, ExternalLinkIcon } from '#lib/icons/index.js';
- import BaseAPIService from '#lib/services/api-service.js';
+ import BaseAPIService, { APIError } from '#lib/services/api-service.js';
+ import { handleApiResultWithCallbacks } from '#lib/utils/api.js';
import ReleaseNotes from '#lib/components/release-notes.svelte';
import type { AppVersionInformation } from '#lib/types/settings.js';
import { formatRelativeTime, nowInstantString } from '#lib/utils/formatting.js';
@@ -41,6 +41,10 @@
type Phase = 'confirm' | 'running' | 'finished';
const POLL_INTERVAL_MS = 3000;
+ // After a 409 the winning job may not be persisted yet; re-read status this many
+ // times, this far apart, before concluding there is nothing to adopt.
+ const CONFLICT_STATUS_READS = 3;
+ const CONFLICT_STATUS_RETRY_MS = 1000;
const MANAGER_ENVIRONMENT_ID = '0';
let phase = $state<Phase>('confirm');
@@ -48,6 +52,10 @@
let reconnecting = $state(false);
let pollActive = false;
let pollTimer: ReturnType<typeof setTimeout> | null = null;
+ // Bumped on every confirm and reset so a startup/status response that lands after
+ // the dialog closed (or a newer attempt began) is ignored instead of restarting
+ // polling for a dialog nobody is looking at.
+ let startAttempt = 0;
function stopPolling() {
pollActive = false;
@@ -61,6 +69,7 @@
// step, without mutating $state from inside an $effect. The confirm step never
// renders job/reconnecting, so clearing them here is safe.
function resetState() {
+ startAttempt++;
stopPolling();
BaseAPIService.setUpgradeInProgress(false);
phase = 'confirm';
@@ -112,6 +121,7 @@
}
async function handleConfirm() {
+ const attempt = ++startAttempt;
phase = 'running';
reconnecting = false;
@@ -122,20 +132,33 @@
BaseAPIService.setUpgradeInProgress(true);
- const operationResult1 = await tryCatch(
- (async () => {
- job = await systemUpgradeService.triggerUpdateAll();
- })()
- );
- if (operationResult1.error !== null) {
- toast.error(m.environments_update_all_trigger_failed());
+ const startResult = await tryCatch(systemUpgradeService.triggerUpdateAll());
+ let next = startResult.data;
+ // The backend refuses a second job while one is active (409). Adopt that job
+ // and follow it instead of failing; never retry the POST.
+ if (startResult.error instanceof APIError && startResult.error.status === 409) {
+ // A concurrent start answers 409 before its job row is persisted, so the first
+ // read can still show no job (404) or the previous terminal one.
+ for (let read = 0; read < CONFLICT_STATUS_READS && attempt === startAttempt; read++) {
+ if (read > 0) await new Promise((resolve) => setTimeout(resolve, CONFLICT_STATUS_RETRY_MS));
+ const active = (await tryCatch(systemUpgradeService.getUpdateAllStatus())).data;
+ if (active?.status === 'running' || active?.status === 'pending_restart') {
+ next = active;
+ break;
+ }
+ }
+ }
+ // The dialog closed (or was re-confirmed) while the request was in flight.
+ if (attempt !== startAttempt) return;
+
+ if (!next) {
+ await handleApiResultWithCallbacks({ result: startResult, message: m.environments_update_all_trigger_failed() });
resetState();
return;
}
- if (phase !== 'running') return;
-
- if (job && (job.status === 'completed' || job.status === 'failed')) {
+ job = next;
+ if (job.status === 'completed' || job.status === 'failed') {
finishTerminalJob(job);
return;
}
@@ -165,6 +188,7 @@
}
onDestroy(() => {
+ startAttempt++;
stopPolling();
BaseAPIService.setUpgradeInProgress(false);
});
diff --git a/tests/spec/system-upgrade.spec.ts b/tests/spec/system-upgrade.spec.ts
index bfc4d91b7..59a9b40bc 100644
--- a/tests/spec/system-upgrade.spec.ts
+++ b/tests/spec/system-upgrade.spec.ts
@@ -99,6 +99,174 @@ async function currentReloadCount(page: Page) {
);
}
+// Huma renders errors as RFC 7807 problem documents; the dialog surfaces `detail`.
+async function fulfillProblem(route: Route, status: number, detail: string) {
+ await route.fulfill({
+ status,
+ contentType: 'application/problem+json',
+ body: JSON.stringify({ title: 'Error', status, detail })
+ });
+}
+
+test.describe('Update All startup', () => {
+ const CONFLICT_DETAIL = 'an update-all job is already in progress';
+
+ test('a failed start surfaces the API error and returns to the confirm step', async ({
+ page
+ }) => {
+ let statusCalls = 0;
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ await fulfillProblem(route, 500, 'Failed to initiate upgrade: pull access denied');
+ });
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ statusCalls++;
+ await fulfillJob(route, 'running', 'updating');
+ });
+
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+
+ const toast = page
+ .locator('li[data-sonner-toast]')
+ .filter({ hasText: 'Failed to start update all' });
+ await expect(toast).toBeVisible({ timeout: 10_000 });
+ await expect(toast).toContainText('pull access denied');
+
+ // Back at the confirm step, ready for another attempt — and no polling started.
+ const dialog = page.getByRole('dialog');
+ await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();
+ await expect(dialog.getByRole('button', { name: 'Update All', exact: true })).toBeVisible();
+ expect(statusCalls).toBe(0);
+ });
+
+ test('a 409 conflict adopts the active job and follows it to completion', async ({ page }) => {
+ let startCalls = 0;
+ let statusCalls = 0;
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ startCalls++;
+ await fulfillProblem(route, 409, CONFLICT_DETAIL);
+ });
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ statusCalls++;
+ // First read (adoption) reports the job still running; the poll completes it.
+ if (statusCalls === 1) {
+ await fulfillJob(route, 'running', 'updating');
+ return;
+ }
+ await fulfillJob(route, 'completed', 'updated');
+ });
+
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+
+ const dialog = page.getByRole('dialog');
+ await expect(dialog.getByRole('heading', { name: 'All environments processed' })).toBeVisible({
+ timeout: 10_000
+ });
+ await expect(dialog.getByText('1 of 1', { exact: true })).toBeVisible();
+ await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);
+ // The conflict is never retried; the existing job is tracked instead.
+ expect(startCalls).toBe(1);
+ expect(statusCalls).toBeGreaterThanOrEqual(2);
+ });
+
+ test('a 409 conflict without an active job reports the conflict and does not retry', async ({
+ page
+ }) => {
+ let startCalls = 0;
+ let statusCalls = 0;
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ startCalls++;
+ await fulfillProblem(route, 409, CONFLICT_DETAIL);
+ });
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ statusCalls++;
+ // The guard tripped but the latest job is already terminal: nothing to adopt.
+ await fulfillJob(route, 'completed', 'updated');
+ });
+
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+
+ const toast = page
+ .locator('li[data-sonner-toast]')
+ .filter({ hasText: 'Failed to start update all' });
+ await expect(toast).toBeVisible({ timeout: 10_000 });
+ await expect(toast).toContainText(CONFLICT_DETAIL);
+
+ const dialog = page.getByRole('dialog');
+ await expect(dialog.getByRole('heading', { name: 'Update all environments' })).toBeVisible();
+
+ // Wait past one poll interval to prove neither polling nor a retry kicked in.
+ // The three status reads are the persistence-window grace, not polling.
+ await page.waitForTimeout(4_000);
+ expect(startCalls).toBe(1);
+ expect(statusCalls).toBe(3);
+ });
+
+ test('a 409 conflict adopts a job that is persisted shortly after the conflict', async ({
+ page
+ }) => {
+ let statusCalls = 0;
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ await fulfillProblem(route, 409, CONFLICT_DETAIL);
+ });
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ statusCalls++;
+ // The winning client's job row does not exist yet on the first read.
+ if (statusCalls === 1) {
+ await fulfillProblem(route, 404, 'no update-all job found');
+ return;
+ }
+ await fulfillJob(
+ route,
+ statusCalls === 2 ? 'running' : 'completed',
+ statusCalls === 2 ? 'updating' : 'updated'
+ );
+ });
+
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+
+ const dialog = page.getByRole('dialog');
+ await expect(dialog.getByRole('heading', { name: 'All environments processed' })).toBeVisible({
+ timeout: 10_000
+ });
+ await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);
+ });
+
+ test('closing the dialog during a pending start ignores the late response', async ({ page }) => {
+ let releaseStart!: () => void;
+ const startReleased = new Promise<void>((resolve) => {
+ releaseStart = resolve;
+ });
+ let statusCalls = 0;
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all$/, async (route) => {
+ await startReleased;
+ await fulfillJob(route, 'running', 'updating');
+ });
+ await page.route(/\/api\/environments\/0\/system\/upgrade\/all\/status$/, async (route) => {
+ statusCalls++;
+ await fulfillJob(route, 'running', 'updating');
+ });
+
+ await page.goto('/environments');
+ await openAndConfirmUpdateAll(page);
+
+ const dialog = page.getByRole('dialog');
+ await expect(dialog.getByRole('heading', { name: 'Updating environments…' })).toBeVisible();
+ await dialog.getByRole('button', { name: 'Close', exact: true }).first().click();
+ await expect(dialog).toBeHidden();
+
+ // The start response arrives after closure: it must not reopen or start polling.
+ releaseStart();
+ await page.waitForTimeout(4_000);
+ await expect(dialog).toBeHidden();
+ expect(statusCalls).toBe(0);
+ await expect(page.locator('li[data-sonner-toast]')).toHaveCount(0);
+ });
+});
+
test.describe('Manager self-update recovery', () => {
test('Update All keeps the manager result visible and refreshes without a document reload', async ({
page