Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(backends): quick-retry a transient backend health probe (#1582)

Vasco Schiavo4088a23bcba3
diff --git a/__tests__/hooks/query/use-backends-health.test.tsx b/__tests__/hooks/query/use-backends-health.test.tsx
index 7bcc29491d..bc8b11445e 100644
--- a/__tests__/hooks/query/use-backends-health.test.tsx
+++ b/__tests__/hooks/query/use-backends-health.test.tsx
@@ -103,12 +103,15 @@ describe("useBackendsHealth", () => {
wrapper,
});
- await waitFor(() =>
- expect(result.current[localBackend.id]).toMatchObject({
- isConnected: false,
- lastError:
- "Agent Canvas requires agent-server 1.28.0 or newer; this backend is running 1.27.1. Please upgrade the agent-server backend.",
- }),
+ await waitFor(
+ () =>
+ expect(result.current[localBackend.id]).toMatchObject({
+ isConnected: false,
+ lastError:
+ "Agent Canvas requires agent-server 1.28.0 or newer; this backend is running 1.27.1. Please upgrade the agent-server backend.",
+ }),
+ // Failing probes now retry a couple of times before settling.
+ { timeout: 3000 },
);
});
@@ -119,11 +122,33 @@ describe("useBackendsHealth", () => {
wrapper,
});
- await waitFor(() =>
- expect(result.current[localBackend.id].isConnected).toBe(false),
+ await waitFor(
+ () => expect(result.current[localBackend.id].isConnected).toBe(false),
+ { timeout: 3000 },
);
});
+ it("recovers when a transient first probe fails, then succeeds on retry", async () => {
+ // The first probe attempt rejects (agent-server still warming up right
+ // after navigation); the quick-retry inside the query function re-probes
+ // and succeeds, so the backend reports connected without waiting for the
+ // 10s poll β€” and because the probe ultimately succeeded, zero failures are
+ // recorded toward the disabled cap.
+ getSettingsMock.mockRejectedValueOnce(new Error("ECONNREFUSED"));
+ getSettingsMock.mockResolvedValue({});
+
+ const { result } = renderHook(() => useBackendsHealth([localBackend]), {
+ wrapper,
+ });
+
+ await waitFor(
+ () => expect(result.current[localBackend.id].isConnected).toBe(true),
+ { timeout: 3000 },
+ );
+ expect(result.current[localBackend.id].consecutiveFailures).toBe(0);
+ expect(getSettingsMock).toHaveBeenCalledTimes(2);
+ });
+
it("reports invalid API key when the authenticated local probe returns 401", async () => {
getSettingsMock.mockRejectedValue(
Object.assign(new Error("Unauthorized"), {
@@ -143,6 +168,8 @@ describe("useBackendsHealth", () => {
}),
);
expect(getServerInfoMock).not.toHaveBeenCalled();
+ // A definitive auth rejection is not retried β€” it won't self-heal.
+ expect(getSettingsMock).toHaveBeenCalledTimes(1);
});
it("probes cloud backends via getCurrentCloudApiKey", async () => {
@@ -169,8 +196,9 @@ describe("useBackendsHealth", () => {
wrapper,
});
- await waitFor(() =>
- expect(result.current[cloudBackend.id].isConnected).toBe(false),
+ await waitFor(
+ () => expect(result.current[cloudBackend.id].isConnected).toBe(false),
+ { timeout: 3000 },
);
});
@@ -227,13 +255,16 @@ describe("useBackendsHealth", () => {
// Assert β€” one failed probe surfaces the new metadata fields on
// the hook's return value and persists them to localStorage; the
// disabled flag stays false because we're below the cap.
- await waitFor(() =>
- expect(result.current[localBackend.id]).toMatchObject({
- isConnected: false,
- consecutiveFailures: 1,
- lastError: "ECONNREFUSED",
- disabled: false,
- }),
+ await waitFor(
+ () =>
+ expect(result.current[localBackend.id]).toMatchObject({
+ isConnected: false,
+ consecutiveFailures: 1,
+ lastError: "ECONNREFUSED",
+ disabled: false,
+ }),
+ // Retries settle before a single failure is recorded.
+ { timeout: 3000 },
);
const persisted = JSON.parse(
window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY) ?? "{}",
diff --git a/src/hooks/query/use-backends-health.ts b/src/hooks/query/use-backends-health.ts
index 6919bf031a..4467721a58 100644
--- a/src/hooks/query/use-backends-health.ts
+++ b/src/hooks/query/use-backends-health.ts
@@ -115,6 +115,58 @@ async function probeBackend(backend: Backend): Promise<true> {
return true;
}
+/**
+ * How many extra times to re-run a failed probe before giving up, and how
+ * long to wait between attempts.
+ */
+const PROBE_RETRY_ATTEMPTS = 2;
+const PROBE_RETRY_DELAY_MS = 300;
+
+/**
+ * Probe a backend, retrying a couple of times on failure before giving up.
+ *
+ * The connectivity indicator (and the onboarding "backend connected" banner)
+ * only flips green once a probe succeeds. With `retry: false` at the query
+ * level and a REFRESH_INTERVAL_MS (10s) poll, a single transient first-probe
+ * miss β€” the agent-server still warming up right after navigation, a momentary
+ * proxy 5xx, a dropped connection β€” would otherwise leave the banner stuck for
+ * a full 10s until the next scheduled refetch. That is the root cause of the
+ * flaky onboarding e2e "backend health probe should report connected" timeout.
+ *
+ * Retrying here, inside the query function rather than via React Query's
+ * `retry`, is deliberate: the success/failure recording below runs once per
+ * settled query, so a single logical probe still records exactly one outcome.
+ * Retrying at the query level instead would call `recordBackendFailure` on
+ * every internal attempt and reach the disabled cap several times too fast.
+ *
+ * Definitive auth failures (logged out, invalid/missing key) are NOT retried:
+ * they are a decided server response, not a transient miss, so retrying only
+ * delays surfacing the correct "disconnected" state (and the recovery UI).
+ */
+function isRetryableProbeError(error: unknown): boolean {
+ if (!(error instanceof Error)) return true;
+ return (
+ error.message !== INVALID_BACKEND_API_KEY_ERROR &&
+ error.message !== MISSING_BACKEND_API_KEY_ERROR &&
+ error.message !== CLOUD_BACKEND_LOGGED_OUT_ERROR
+ );
+}
+
+async function probeBackendWithQuickRetry(backend: Backend): Promise<true> {
+ for (let attempt = 0; ; attempt += 1) {
+ try {
+ return await probeBackend(backend);
+ } catch (error) {
+ if (attempt >= PROBE_RETRY_ATTEMPTS || !isRetryableProbeError(error)) {
+ throw error;
+ }
+ await new Promise((resolve) => {
+ setTimeout(resolve, PROBE_RETRY_DELAY_MS);
+ });
+ }
+ }
+}
+
export interface BackendHealth {
/** `null` while the first probe is in flight; then `true` / `false`. */
isConnected: boolean | null;
@@ -190,7 +242,7 @@ export function useBackendsHealth(
] as const,
queryFn: async () => {
try {
- const result = await probeBackend(b);
+ const result = await probeBackendWithQuickRetry(b);
recordBackendSuccess(b.id);
return result;
} catch (err) {