Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(dashboard): retry a hung readiness check and bound the restart call

Yudhi Armyndharis2819b9fa1933
diff --git a/dashboard/src/hooks/useRestartFlow.test.ts b/dashboard/src/hooks/useRestartFlow.test.ts
index 1a11e7304..bfe07e8da 100644
--- a/dashboard/src/hooks/useRestartFlow.test.ts
+++ b/dashboard/src/hooks/useRestartFlow.test.ts
@@ -306,3 +306,72 @@ test('a successful readiness response retires its deadline', async () => {
assert.equal(result.current.restartStatus, 'success');
assert.equal(readySignals.length, 1);
});
+
+test('a readiness attempt that hangs is given up on and the next one is made before the deadline', async () => {
+ mock.timers.enable({ apis: ['setTimeout', 'setInterval', 'Date'], now: 1800000000000 });
+ // A warning keeps the success view open instead of scheduling a page reload.
+ restartAnswer = () =>
+ Promise.resolve(
+ jsonResponse({
+ message: 'Restarting',
+ restarting: true,
+ estimatedTime: 5,
+ orchestration: { errors: ['A service needs attention'] },
+ }),
+ );
+ const hung = deferred();
+ let attempt = 0;
+ readyAnswer = signal =>
+ ++attempt === 1 ? abortReady(hung.promise, signal) : Promise.resolve(jsonResponse({ status: 'ok', details: {} }));
+ const { result } = rtl.renderHook(() => useRestartFlow());
+ await rtl.act(async () => {
+ void result.current.start();
+ await settleRestartReply();
+ });
+ await rtl.act(async () => {
+ mock.timers.tick(3000);
+ await settleRestartReply();
+ });
+ assert.equal(readySignals.length, 1);
+ assert.equal(result.current.restartStatus, 'waiting');
+ await rtl.act(async () => {
+ mock.timers.tick(5000);
+ await settleRestartReply();
+ });
+ assert.equal(readySignals[0]?.aborted, true);
+ assert.equal(result.current.restartStatus, 'waiting');
+ await rtl.act(async () => {
+ mock.timers.tick(1000);
+ await settleRestartReply();
+ });
+ assert.equal(readySignals.length, 2);
+ assert.equal(result.current.restartStatus, 'success');
+});
+
+test('a restart request that never answers ends in the error view at its bound, with no readiness poll', async () => {
+ mock.timers.enable({ apis: ['setTimeout', 'setInterval', 'Date'], now: 1800000000000 });
+ restartAnswer = signal => abortReady(deferred().promise, signal);
+ const { result } = rtl.renderHook(() => useRestartFlow());
+ await rtl.act(async () => {
+ void result.current.start();
+ await settleRestartReply();
+ });
+ assert.equal(result.current.restartStatus, 'restarting');
+ await rtl.act(async () => {
+ mock.timers.tick(4 * 60 * 1000);
+ await settleRestartReply();
+ });
+ assert.equal(result.current.restartStatus, 'restarting');
+ await rtl.act(async () => {
+ mock.timers.tick(61 * 1000);
+ await settleRestartReply();
+ });
+ assert.equal(result.current.restartStatus, 'error');
+ assert.equal(result.current.restartError, null);
+ assert.equal(result.current.restartCountdown, 0);
+ await rtl.act(async () => {
+ mock.timers.tick(120000);
+ await settleRestartReply();
+ });
+ assert.equal(readySignals.length, 0);
+});
diff --git a/dashboard/src/hooks/useRestartFlow.ts b/dashboard/src/hooks/useRestartFlow.ts
index 84a4b8f7c..68a0bf133 100644
--- a/dashboard/src/hooks/useRestartFlow.ts
+++ b/dashboard/src/hooks/useRestartFlow.ts
@@ -4,6 +4,13 @@ import { restartPollAttempts } from '../utils/restartPoll';
// 'unknown': a proxy answered in the gateway's place (a 504, a Cloudflare 52x, or a 502 with no gateway
// code), so whether the restart went through cannot be told.
+// One readiness request is given this long before it counts as a failed attempt and the next one is made; the
+// elapsed deadline of the whole wait is still what ends it.
+const READINESS_ATTEMPT_TIMEOUT_MS = 5000;
+// The restart request can take a while (a first-time enable pulls an image before it answers), but not forever:
+// the restarting view has no close button.
+const RESTART_REQUEST_TIMEOUT_MS = 5 * 60 * 1000;
+
export type RestartStatus = 'idle' | 'restarting' | 'waiting' | 'success' | 'error' | 'unknown';
export interface RestartOpenRequest {
@@ -145,8 +152,14 @@ export function useRestartFlow(): RestartFlow {
const check = async () => {
if (!isCurrent()) return;
+ // Each attempt has its own signal, so one that hangs is given up on and counted, and the wait goes on.
+ const attempt = new AbortController();
+ const abortAttempt = () => attempt.abort();
+ controller.signal.addEventListener('abort', abortAttempt, { once: true });
+ const timer = setTimeout(abortAttempt, READINESS_ATTEMPT_TIMEOUT_MS);
+ pollTimeoutsRef.current.add(timer);
try {
- await infraApi.healthCheck(controller.signal);
+ await infraApi.healthCheck(attempt.signal);
if (!settle('success')) return;
// With warnings on screen the operator reloads by hand, after reading them.
if (!hasWarnings) {
@@ -159,6 +172,10 @@ export function useRestartFlow(): RestartFlow {
attempts++;
if (attempts < maxAttempts) schedulePollTimeout(check, 1000);
else settle('error');
+ } finally {
+ clearTimeout(timer);
+ pollTimeoutsRef.current.delete(timer);
+ controller.signal.removeEventListener('abort', abortAttempt);
}
};
@@ -184,8 +201,13 @@ export function useRestartFlow(): RestartFlow {
// fail sometimes (the server may go down before it answers).
let estimatedTime: number | undefined;
let warnings: string[] = [];
+ const request = new AbortController();
+ const abortRequest = () => request.abort();
+ controller.signal.addEventListener('abort', abortRequest, { once: true });
+ const requestTimer = setTimeout(abortRequest, RESTART_REQUEST_TIMEOUT_MS);
+ pollTimeoutsRef.current.add(requestTimer);
try {
- const response = await infraApi.restart(profiles.pending, profilesToRemove);
+ const response = await infraApi.restart(profiles.pending, profilesToRemove, request.signal);
if (!isCurrent()) return;
estimatedTime = response.estimatedTime;
if (response.estimatedTime) {
@@ -196,6 +218,13 @@ export function useRestartFlow(): RestartFlow {
setRestartWarnings(warnings);
} catch (err) {
if (!isCurrent()) return;
+ // Given up on after its bound: the server never answered, which the poll below could only misread.
+ if (request.signal.aborted) {
+ stopCountdown();
+ setRestartCountdown(0);
+ setRestartStatus('error');
+ return;
+ }
// An HTTP status (a proxy 503 included) means no shutdown was confirmed and the old
// process may still be serving, so a readiness poll would report a restart that never happened.
// Only a status-less network failure is the expected sign of the server going down mid-answer.
@@ -217,6 +246,10 @@ export function useRestartFlow(): RestartFlow {
setRestartStatus('error');
return;
}
+ } finally {
+ clearTimeout(requestTimer);
+ pollTimeoutsRef.current.delete(requestTimer);
+ controller.signal.removeEventListener('abort', abortRequest);
}
if (!isCurrent()) return;
diff --git a/dashboard/src/services/api.ts b/dashboard/src/services/api.ts
index 343940cbb..22ea17a33 100644
--- a/dashboard/src/services/api.ts
+++ b/dashboard/src/services/api.ts
@@ -1388,7 +1388,7 @@ export const infraApi = {
method: 'PUT',
body: JSON.stringify(config),
}),
- restart: (profiles?: string[], profilesToRemove?: string[]) =>
+ restart: (profiles?: string[], profilesToRemove?: string[], signal?: AbortSignal) =>
request<{
message: string;
restarting: boolean;
@@ -1401,6 +1401,7 @@ export const infraApi = {
}>('/infra/restart', {
method: 'POST',
body: JSON.stringify({ profiles: profiles || [], profilesToRemove: profilesToRemove || [] }),
+ signal,
}),
// Readiness, not the plain /infra/health ping. The restart poll must not be able to latch onto the
// process it just asked to shut down: /infra/health answers 200 for the whole drain,