π OpenHands: AI-Driven Development
fix(backends): quick-retry a transient backend health probe (#1582)
Vasco Schiavo4088a23bcba3
diff --git a/__tests__/hooks/query/use-backends-health.test.tsx b/__tests__/hooks/query/use-backends-health.test.tsxindex 7bcc29491d..bc8b11445e 100644--- a/__tests__/hooks/query/use-backends-health.test.tsx+++ b/__tests__/hooks/query/use-backends-health.test.tsx@@ -103,12 +103,15 @@ describe("useBackendsHealth", () => {wrapper,});- await waitFor(() =>- expect(result.current[localBackend.id]).toMatchObject({- isConnected: false,- lastError:- "Agent Canvas requires agent-server 1.28.0 or newer; this backend is running 1.27.1. Please upgrade the agent-server backend.",- }),+ await waitFor(+ () =>+ expect(result.current[localBackend.id]).toMatchObject({+ isConnected: false,+ lastError:+ "Agent Canvas requires agent-server 1.28.0 or newer; this backend is running 1.27.1. Please upgrade the agent-server backend.",+ }),+ // Failing probes now retry a couple of times before settling.+ { timeout: 3000 },);});@@ -119,11 +122,33 @@ describe("useBackendsHealth", () => {wrapper,});- await waitFor(() =>- expect(result.current[localBackend.id].isConnected).toBe(false),+ await waitFor(+ () => expect(result.current[localBackend.id].isConnected).toBe(false),+ { timeout: 3000 },);});+ it("recovers when a transient first probe fails, then succeeds on retry", async () => {+ // The first probe attempt rejects (agent-server still warming up right+ // after navigation); the quick-retry inside the query function re-probes+ // and succeeds, so the backend reports connected without waiting for the+ // 10s poll β and because the probe ultimately succeeded, zero failures are+ // recorded toward the disabled cap.+ getSettingsMock.mockRejectedValueOnce(new Error("ECONNREFUSED"));+ getSettingsMock.mockResolvedValue({});++ const { result } = renderHook(() => useBackendsHealth([localBackend]), {+ wrapper,+ });++ await waitFor(+ () => expect(result.current[localBackend.id].isConnected).toBe(true),+ { timeout: 3000 },+ );+ expect(result.current[localBackend.id].consecutiveFailures).toBe(0);+ expect(getSettingsMock).toHaveBeenCalledTimes(2);+ });+it("reports invalid API key when the authenticated local probe returns 401", async () => {getSettingsMock.mockRejectedValue(Object.assign(new Error("Unauthorized"), {@@ -143,6 +168,8 @@ describe("useBackendsHealth", () => {}),);expect(getServerInfoMock).not.toHaveBeenCalled();+ // A definitive auth rejection is not retried β it won't self-heal.+ expect(getSettingsMock).toHaveBeenCalledTimes(1);});it("probes cloud backends via getCurrentCloudApiKey", async () => {@@ -169,8 +196,9 @@ describe("useBackendsHealth", () => {wrapper,});- await waitFor(() =>- expect(result.current[cloudBackend.id].isConnected).toBe(false),+ await waitFor(+ () => expect(result.current[cloudBackend.id].isConnected).toBe(false),+ { timeout: 3000 },);});@@ -227,13 +255,16 @@ describe("useBackendsHealth", () => {// Assert β one failed probe surfaces the new metadata fields on// the hook's return value and persists them to localStorage; the// disabled flag stays false because we're below the cap.- await waitFor(() =>- expect(result.current[localBackend.id]).toMatchObject({- isConnected: false,- consecutiveFailures: 1,- lastError: "ECONNREFUSED",- disabled: false,- }),+ await waitFor(+ () =>+ expect(result.current[localBackend.id]).toMatchObject({+ isConnected: false,+ consecutiveFailures: 1,+ lastError: "ECONNREFUSED",+ disabled: false,+ }),+ // Retries settle before a single failure is recorded.+ { timeout: 3000 },);const persisted = JSON.parse(window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY) ?? "{}",diff --git a/src/hooks/query/use-backends-health.ts b/src/hooks/query/use-backends-health.tsindex 6919bf031a..4467721a58 100644--- a/src/hooks/query/use-backends-health.ts+++ b/src/hooks/query/use-backends-health.ts@@ -115,6 +115,58 @@ async function probeBackend(backend: Backend): Promise<true> {return true;}+/**+ * How many extra times to re-run a failed probe before giving up, and how+ * long to wait between attempts.+ */+const PROBE_RETRY_ATTEMPTS = 2;+const PROBE_RETRY_DELAY_MS = 300;++/**+ * Probe a backend, retrying a couple of times on failure before giving up.+ *+ * The connectivity indicator (and the onboarding "backend connected" banner)+ * only flips green once a probe succeeds. With `retry: false` at the query+ * level and a REFRESH_INTERVAL_MS (10s) poll, a single transient first-probe+ * miss β the agent-server still warming up right after navigation, a momentary+ * proxy 5xx, a dropped connection β would otherwise leave the banner stuck for+ * a full 10s until the next scheduled refetch. That is the root cause of the+ * flaky onboarding e2e "backend health probe should report connected" timeout.+ *+ * Retrying here, inside the query function rather than via React Query's+ * `retry`, is deliberate: the success/failure recording below runs once per+ * settled query, so a single logical probe still records exactly one outcome.+ * Retrying at the query level instead would call `recordBackendFailure` on+ * every internal attempt and reach the disabled cap several times too fast.+ *+ * Definitive auth failures (logged out, invalid/missing key) are NOT retried:+ * they are a decided server response, not a transient miss, so retrying only+ * delays surfacing the correct "disconnected" state (and the recovery UI).+ */+function isRetryableProbeError(error: unknown): boolean {+ if (!(error instanceof Error)) return true;+ return (+ error.message !== INVALID_BACKEND_API_KEY_ERROR &&+ error.message !== MISSING_BACKEND_API_KEY_ERROR &&+ error.message !== CLOUD_BACKEND_LOGGED_OUT_ERROR+ );+}++async function probeBackendWithQuickRetry(backend: Backend): Promise<true> {+ for (let attempt = 0; ; attempt += 1) {+ try {+ return await probeBackend(backend);+ } catch (error) {+ if (attempt >= PROBE_RETRY_ATTEMPTS || !isRetryableProbeError(error)) {+ throw error;+ }+ await new Promise((resolve) => {+ setTimeout(resolve, PROBE_RETRY_DELAY_MS);+ });+ }+ }+}+export interface BackendHealth {/** `null` while the first probe is in flight; then `true` / `false`. */isConnected: boolean | null;@@ -190,7 +242,7 @@ export function useBackendsHealth(] as const,queryFn: async () => {try {- const result = await probeBackend(b);+ const result = await probeBackendWithQuickRetry(b);recordBackendSuccess(b.id);return result;} catch (err) {