Skip to content

🙌 OpenHands: AI-Driven Development

fix(frontend): cap backend-health retries and persist failure state (#411)

Hiep Le05d226a4700a
diff --git a/__tests__/api/backend-registry/health-store.test.ts b/__tests__/api/backend-registry/health-store.test.ts
new file mode 100644
index 0000000000..ac1c92ebe8
--- /dev/null
+++ b/__tests__/api/backend-registry/health-store.test.ts
@@ -0,0 +1,89 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import {
+ BACKEND_HEALTH_STORAGE_KEY,
+ MAX_CONSECUTIVE_FAILURES,
+} from "#/api/backend-registry/health-storage";
+import {
+ __resetHealthStoreForTests,
+ getBackendHealthEntry,
+ recordBackendFailure,
+ recordBackendSuccess,
+ resetBackendHealth,
+ subscribeBackendHealth,
+} from "#/api/backend-registry/health-store";
+
+const BACKEND_ID = "backend-under-test";
+
+beforeEach(() => {
+ window.localStorage.clear();
+ __resetHealthStoreForTests();
+});
+
+afterEach(() => {
+ window.localStorage.clear();
+ __resetHealthStoreForTests();
+});
+
+describe("backend health store", () => {
+ it("increments the failure count and persists to localStorage without disabling polling below the cap", () => {
+ // Arrange / Act
+ recordBackendFailure(BACKEND_ID, new Error("ECONNREFUSED"));
+ recordBackendFailure(BACKEND_ID, new Error("ECONNREFUSED"));
+
+ // Assert
+ const entry = getBackendHealthEntry(BACKEND_ID);
+ expect(entry).toMatchObject({
+ consecutiveFailures: 2,
+ disabled: false,
+ lastError: "ECONNREFUSED",
+ });
+
+ const persisted = JSON.parse(
+ window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY) ?? "{}",
+ );
+ expect(persisted[BACKEND_ID]).toMatchObject({
+ consecutiveFailures: 2,
+ disabled: false,
+ });
+ });
+
+ it("flips disabled=true once consecutive failures hit the cap and persists that flag", () => {
+ // Arrange / Act
+ for (let i = 0; i < MAX_CONSECUTIVE_FAILURES; i += 1) {
+ recordBackendFailure(BACKEND_ID, new Error("timeout"));
+ }
+
+ // Assert — the cap is reached; polling-consumers will see disabled.
+ const entry = getBackendHealthEntry(BACKEND_ID);
+ expect(entry?.consecutiveFailures).toBe(MAX_CONSECUTIVE_FAILURES);
+ expect(entry?.disabled).toBe(true);
+
+ const persisted = JSON.parse(
+ window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY) ?? "{}",
+ );
+ expect(persisted[BACKEND_ID].disabled).toBe(true);
+ });
+
+ it("clears the entry (and storage) and notifies subscribers when the backend recovers or the user edits its config", () => {
+ // Arrange — record one failure so there is something to clear, and
+ // subscribe so we can confirm listeners get notified.
+ recordBackendFailure(BACKEND_ID, new Error("boom"));
+ let notifications = 0;
+ const unsubscribe = subscribeBackendHealth(() => {
+ notifications += 1;
+ });
+
+ // Act — `recordBackendSuccess` (probe recovers) and
+ // `resetBackendHealth` (user edits host/apiKey) share the same
+ // clear-entry semantics, so we cover both in one go.
+ recordBackendSuccess(BACKEND_ID);
+ recordBackendFailure(BACKEND_ID, new Error("again"));
+ resetBackendHealth(BACKEND_ID);
+ unsubscribe();
+
+ // Assert
+ expect(getBackendHealthEntry(BACKEND_ID)).toBeNull();
+ expect(window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY)).toBeNull();
+ expect(notifications).toBeGreaterThanOrEqual(3);
+ });
+});
diff --git a/__tests__/contexts/active-backend-context.test.tsx b/__tests__/contexts/active-backend-context.test.tsx
index 3e67a88fdd..4f3fc11c3b 100644
--- a/__tests__/contexts/active-backend-context.test.tsx
+++ b/__tests__/contexts/active-backend-context.test.tsx
@@ -4,6 +4,12 @@ import { act, render, renderHook } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { __resetActiveStoreForTests } from "#/api/backend-registry/active-store";
import { DEFAULT_LOCAL_BACKEND_ID } from "#/api/backend-registry/default-backend";
+import { MAX_CONSECUTIVE_FAILURES } from "#/api/backend-registry/health-storage";
+import {
+ __resetHealthStoreForTests,
+ getBackendHealthEntry,
+ recordBackendFailure,
+} from "#/api/backend-registry/health-store";
import {
ActiveBackendProvider,
useActiveBackendContext,
@@ -23,11 +29,13 @@ function makeWrapper(queryClient = new QueryClient()) {
beforeEach(() => {
window.localStorage.clear();
__resetActiveStoreForTests();
+ __resetHealthStoreForTests();
});
afterEach(() => {
window.localStorage.clear();
__resetActiveStoreForTests();
+ __resetHealthStoreForTests();
});
describe("ActiveBackendProvider", () => {
@@ -155,4 +163,67 @@ describe("ActiveBackendProvider", () => {
expect(() => render(<HookConsumer />)).toThrow(/ActiveBackendProvider/);
errorSpy.mockRestore();
});
+
+ it("updateBackend re-arms health polling when host or apiKey changes but leaves cosmetic edits alone", () => {
+ // Arrange — register a backend that has hit the failure cap.
+ const { result } = renderHook(() => useActiveBackendContext(), {
+ wrapper: makeWrapper(),
+ });
+
+ let id = "";
+ act(() => {
+ id = result.current.addBackend({
+ name: "Stale",
+ host: "http://localhost:9000",
+ apiKey: "old-key",
+ kind: "local",
+ }).id;
+ });
+ for (let i = 0; i < MAX_CONSECUTIVE_FAILURES; i += 1) {
+ recordBackendFailure(id, new Error("timeout"));
+ }
+ expect(getBackendHealthEntry(id)?.disabled).toBe(true);
+
+ // Act — renaming is cosmetic; it must NOT silently re-enable
+ // polling against an unreachable backend.
+ act(() => {
+ result.current.updateBackend(id, { name: "Renamed" });
+ });
+ expect(getBackendHealthEntry(id)?.disabled).toBe(true);
+
+ // Act — changing host is the explicit "fix the config" signal and
+ // must clear the entry so polling resumes.
+ act(() => {
+ result.current.updateBackend(id, { host: "http://localhost:9001" });
+ });
+
+ // Assert
+ expect(getBackendHealthEntry(id)).toBeNull();
+ });
+
+ it("removeBackend drops the backend's persisted health entry", () => {
+ // Arrange
+ const { result } = renderHook(() => useActiveBackendContext(), {
+ wrapper: makeWrapper(),
+ });
+ let id = "";
+ act(() => {
+ id = result.current.addBackend({
+ name: "Doomed",
+ host: "http://localhost:9000",
+ apiKey: "k",
+ kind: "local",
+ }).id;
+ });
+ recordBackendFailure(id, new Error("boom"));
+ expect(getBackendHealthEntry(id)).not.toBeNull();
+
+ // Act
+ act(() => {
+ result.current.removeBackend(id);
+ });
+
+ // Assert
+ expect(getBackendHealthEntry(id)).toBeNull();
+ });
});
diff --git a/__tests__/hooks/query/use-backends-health.test.tsx b/__tests__/hooks/query/use-backends-health.test.tsx
index 4b0922287d..c2f96a3619 100644
--- a/__tests__/hooks/query/use-backends-health.test.tsx
+++ b/__tests__/hooks/query/use-backends-health.test.tsx
@@ -1,9 +1,17 @@
import { ServerClient } from "@openhands/typescript-client/clients";
import React from "react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
-import { renderHook, waitFor } from "@testing-library/react";
+import { act, renderHook, waitFor } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_LOCAL_BACKEND_ID } from "#/api/backend-registry/default-backend";
+import {
+ BACKEND_HEALTH_STORAGE_KEY,
+ MAX_CONSECUTIVE_FAILURES,
+} from "#/api/backend-registry/health-storage";
+import {
+ __resetHealthStoreForTests,
+ resetBackendHealth,
+} from "#/api/backend-registry/health-store";
import type { Backend } from "#/api/backend-registry/types";
import { useBackendsHealth } from "#/hooks/query/use-backends-health";
@@ -48,10 +56,14 @@ beforeEach(() => {
getServerInfoMock.mockReset();
getCurrentCloudApiKeyMock.mockReset();
vi.mocked(ServerClient).mockClear();
+ window.localStorage.clear();
+ __resetHealthStoreForTests();
});
afterEach(() => {
vi.useRealTimers();
+ window.localStorage.clear();
+ __resetHealthStoreForTests();
});
describe("useBackendsHealth", () => {
@@ -130,4 +142,102 @@ describe("useBackendsHealth", () => {
expect(result.current[localBackend.id].isConnected).toBe(true),
);
});
+
+ it("records the failure count and last error to the health store after a failed probe", async () => {
+ // Arrange
+ getServerInfoMock.mockRejectedValue(new Error("ECONNREFUSED"));
+
+ // Act
+ const { result } = renderHook(() => useBackendsHealth([localBackend]), {
+ wrapper,
+ });
+
+ // Assert — one failed probe surfaces the new metadata fields on
+ // the hook's return value and persists them to localStorage; the
+ // disabled flag stays false because we're below the cap.
+ await waitFor(() =>
+ expect(result.current[localBackend.id]).toMatchObject({
+ isConnected: false,
+ consecutiveFailures: 1,
+ lastError: "ECONNREFUSED",
+ disabled: false,
+ }),
+ );
+ const persisted = JSON.parse(
+ window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY) ?? "{}",
+ );
+ expect(persisted[localBackend.id]).toMatchObject({
+ consecutiveFailures: 1,
+ disabled: false,
+ });
+ });
+
+ it("does not probe a backend whose disabled state was persisted before the GUI mounted (refresh case)", async () => {
+ // Arrange — simulate a prior session that already exhausted retries
+ // by seeding localStorage before the hook subscribes.
+ window.localStorage.setItem(
+ BACKEND_HEALTH_STORAGE_KEY,
+ JSON.stringify({
+ [localBackend.id]: {
+ consecutiveFailures: MAX_CONSECUTIVE_FAILURES,
+ lastError: "ECONNREFUSED",
+ lastFailureAt: Date.now(),
+ disabled: true,
+ },
+ }),
+ );
+ __resetHealthStoreForTests();
+ getServerInfoMock.mockResolvedValue({ version: "1.18.0" });
+
+ // Act
+ const { result } = renderHook(() => useBackendsHealth([localBackend]), {
+ wrapper,
+ });
+ // Let any microtasks drain so a stray probe would have fired by now.
+ await act(async () => {
+ await Promise.resolve();
+ });
+
+ // Assert — polling is gated off; no probe goes out.
+ expect(getServerInfoMock).not.toHaveBeenCalled();
+ expect(result.current[localBackend.id]).toMatchObject({
+ isConnected: false,
+ disabled: true,
+ });
+ });
+
+ it("re-arms polling after the user edits the backend (resetBackendHealth clears the disabled flag)", async () => {
+ // Arrange — start from the persisted-disabled state.
+ window.localStorage.setItem(
+ BACKEND_HEALTH_STORAGE_KEY,
+ JSON.stringify({
+ [localBackend.id]: {
+ consecutiveFailures: MAX_CONSECUTIVE_FAILURES,
+ lastError: "ECONNREFUSED",
+ lastFailureAt: Date.now(),
+ disabled: true,
+ },
+ }),
+ );
+ __resetHealthStoreForTests();
+ getServerInfoMock.mockResolvedValue({ version: "1.18.0" });
+
+ const { result } = renderHook(() => useBackendsHealth([localBackend]), {
+ wrapper,
+ });
+ expect(getServerInfoMock).not.toHaveBeenCalled();
+
+ // Act — the active-backend-context calls resetBackendHealth when
+ // host or apiKey changes; do that directly so we don't have to
+ // spin up the whole context.
+ act(() => {
+ resetBackendHealth(localBackend.id);
+ });
+
+ // Assert — a fresh probe fires and the hook reports connected.
+ await waitFor(() =>
+ expect(result.current[localBackend.id].isConnected).toBe(true),
+ );
+ expect(getServerInfoMock).toHaveBeenCalled();
+ });
});
diff --git a/src/api/backend-registry/health-storage.ts b/src/api/backend-registry/health-storage.ts
new file mode 100644
index 0000000000..85ca85dc6b
--- /dev/null
+++ b/src/api/backend-registry/health-storage.ts
@@ -0,0 +1,87 @@
+export const BACKEND_HEALTH_STORAGE_KEY = "openhands-backend-health";
+
+/**
+ * Once a backend has failed this many probes in a row, polling stops
+ * until the user edits the backend's host or apiKey (which resets the
+ * counter via the active-backend store).
+ */
+export const MAX_CONSECUTIVE_FAILURES = 5;
+
+/** Max characters of `error.message` we persist — keeps localStorage tidy. */
+const MAX_ERROR_MESSAGE_LENGTH = 500;
+
+export interface BackendHealthEntry {
+ consecutiveFailures: number;
+ lastError: string | null;
+ lastFailureAt: number | null;
+ disabled: boolean;
+}
+
+export type BackendHealthMap = Record<string, BackendHealthEntry>;
+
+function isValidEntry(value: unknown): value is BackendHealthEntry {
+ if (typeof value !== "object" || value === null) return false;
+ const v = value as Partial<BackendHealthEntry>;
+ // localStorage is user-writable. Reject anything outside the range
+ // our own writer produces — a tampered `-1` would never reach the
+ // cap and would defeat the whole disable mechanism, and a giant
+ // value would clutter the UI for no reason.
+ return (
+ Number.isInteger(v.consecutiveFailures) &&
+ (v.consecutiveFailures as number) >= 0 &&
+ (v.consecutiveFailures as number) <= MAX_CONSECUTIVE_FAILURES &&
+ (v.lastError === null || typeof v.lastError === "string") &&
+ (v.lastFailureAt === null || typeof v.lastFailureAt === "number") &&
+ typeof v.disabled === "boolean"
+ );
+}
+
+export function readStoredHealth(): BackendHealthMap {
+ if (typeof window === "undefined") return {};
+ try {
+ const raw = window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY);
+ if (!raw) return {};
+ const parsed = JSON.parse(raw);
+ if (typeof parsed !== "object" || parsed === null) return {};
+
+ const out: BackendHealthMap = {};
+ for (const [id, entry] of Object.entries(parsed)) {
+ if (typeof id === "string" && id.length > 0 && isValidEntry(entry)) {
+ out[id] = entry;
+ }
+ }
+ return out;
+ } catch {
+ return {};
+ }
+}
+
+export function writeStoredHealth(map: BackendHealthMap): void {
+ if (typeof window === "undefined") return;
+ try {
+ if (Object.keys(map).length === 0) {
+ window.localStorage.removeItem(BACKEND_HEALTH_STORAGE_KEY);
+ return;
+ }
+ window.localStorage.setItem(
+ BACKEND_HEALTH_STORAGE_KEY,
+ JSON.stringify(map),
+ );
+ } catch {
+ /* ignore quota / serialization errors */
+ }
+}
+
+export function truncateErrorMessage(error: unknown): string {
+ let message: string;
+ if (error instanceof Error) {
+ message = error.message || error.name || "Unknown error";
+ } else if (typeof error === "string") {
+ message = error;
+ } else {
+ message = "Unknown error";
+ }
+ return message.length > MAX_ERROR_MESSAGE_LENGTH
+ ? `${message.slice(0, MAX_ERROR_MESSAGE_LENGTH)}…`
+ : message;
+}
diff --git a/src/api/backend-registry/health-store.ts b/src/api/backend-registry/health-store.ts
new file mode 100644
index 0000000000..c488ed8e88
--- /dev/null
+++ b/src/api/backend-registry/health-store.ts
@@ -0,0 +1,96 @@
+import {
+ MAX_CONSECUTIVE_FAILURES,
+ readStoredHealth,
+ truncateErrorMessage,
+ writeStoredHealth,
+ type BackendHealthEntry,
+ type BackendHealthMap,
+} from "./health-storage";
+
+type Listener = () => void;
+
+let healthMap: BackendHealthMap = readStoredHealth();
+const listeners = new Set<Listener>();
+
+function notify(): void {
+ listeners.forEach((listener) => listener());
+}
+
+function commit(next: BackendHealthMap): void {
+ healthMap = next;
+ writeStoredHealth(next);
+ notify();
+}
+
+export function getHealthSnapshot(): BackendHealthMap {
+ return healthMap;
+}
+
+export function getBackendHealthEntry(id: string): BackendHealthEntry | null {
+ return healthMap[id] ?? null;
+}
+
+export function subscribeBackendHealth(listener: Listener): () => void {
+ listeners.add(listener);
+ return () => {
+ listeners.delete(listener);
+ };
+}
+
+/**
+ * Increment the per-backend failure counter. Once the count reaches
+ * `MAX_CONSECUTIVE_FAILURES`, the entry is marked `disabled: true`
+ * which the polling hook reads to stop firing probes — including on a
+ * fresh page load, since the state lives in localStorage.
+ */
+export function recordBackendFailure(id: string, error: unknown): void {
+ const prev = healthMap[id];
+ const consecutiveFailures = (prev?.consecutiveFailures ?? 0) + 1;
+ const disabled =
+ prev?.disabled === true || consecutiveFailures >= MAX_CONSECUTIVE_FAILURES;
+
+ const nextEntry: BackendHealthEntry = {
+ consecutiveFailures,
+ lastError: truncateErrorMessage(error),
+ lastFailureAt: Date.now(),
+ disabled,
+ };
+
+ commit({ ...healthMap, [id]: nextEntry });
+}
+
+/**
+ * Clear the entry so the next 10s tick can mark it healthy again.
+ * Called on a successful probe — also covers the case where a backend
+ * had a few failures but recovered before hitting the cap.
+ */
+export function recordBackendSuccess(id: string): void {
+ if (!(id in healthMap)) return;
+ const { [id]: _removed, ...rest } = healthMap;
+ commit(rest);
+}
+
+/**
+ * Re-arm polling for a backend after the user updates its config.
+ * Identical to `recordBackendSuccess` today, but kept distinct so the
+ * call sites read clearly.
+ */
+export function resetBackendHealth(id: string): void {
+ if (!(id in healthMap)) return;
+ const { [id]: _removed, ...rest } = healthMap;
+ commit(rest);
+}
+
+/** Drop the entry entirely — used when the backend is deleted. */
+export function dropBackendHealth(id: string): void {
+ if (!(id in healthMap)) return;
+ const { [id]: _removed, ...rest } = healthMap;
+ commit(rest);
+}
+
+/** Test-only: re-read storage and clear listeners. */
+
+export function __resetHealthStoreForTests(): void {
+ healthMap = readStoredHealth();
+ listeners.clear();
+}
diff --git a/src/components/features/backends/backend-form-modal.tsx b/src/components/features/backends/backend-form-modal.tsx
index 1bc1aa5644..eb0f55fd51 100644
--- a/src/components/features/backends/backend-form-modal.tsx
+++ b/src/components/features/backends/backend-form-modal.tsx
@@ -52,7 +52,11 @@ function BackendStatusBadge({
}) {
const { t } = useTranslation("openhands");
const healthByBackendId = useBackendsHealth([backend]);
- const isConnected = healthByBackendId[backend.id]?.isConnected ?? null;
+ const health = healthByBackendId[backend.id];
+ const isConnected = health?.isConnected ?? null;
+ const disabled = health?.disabled === true;
+ const consecutiveFailures = health?.consecutiveFailures ?? 0;
+ const lastError = health?.lastError ?? null;
const { data: version } = useQuery({
queryKey: ["backend-version", backend.host, backend.apiKey],
@@ -68,7 +72,7 @@ function BackendStatusBadge({
},
retry: false,
staleTime: 60_000,
- enabled: backend.kind === "local",
+ enabled: backend.kind === "local" && !disabled,
});
let statusLabel: string;
@@ -86,23 +90,49 @@ function BackendStatusBadge({
: t(I18nKey.BACKEND$KIND_LOCAL);
return (
- <div
- data-testid={`${testIdRoot}-status`}
- className="flex items-center gap-3 text-sm"
- >
- <BackendStatusDot isConnected={isConnected} />
- <span className="text-white" data-testid={`${testIdRoot}-status-label`}>
- {statusLabel}
- </span>
- <span className="text-tertiary-alt">·</span>
- <span className="text-gray-300">{kindLabel}</span>
- {version ? (
- <span
- className="text-xs text-gray-400"
- data-testid={`${testIdRoot}-version`}
- >
- {t(I18nKey.BACKEND$VERSION_LABEL, { version })}
+ <div className="flex flex-col gap-2">
+ <div
+ data-testid={`${testIdRoot}-status`}
+ className="flex items-center gap-3 text-sm"
+ >
+ <BackendStatusDot isConnected={isConnected} />
+ <span className="text-white" data-testid={`${testIdRoot}-status-label`}>
+ {statusLabel}
</span>
+ <span className="text-tertiary-alt">·</span>
+ <span className="text-gray-300">{kindLabel}</span>
+ {version ? (
+ <span
+ className="text-xs text-gray-400"
+ data-testid={`${testIdRoot}-version`}
+ >
+ {t(I18nKey.BACKEND$VERSION_LABEL, { version })}
+ </span>
+ ) : null}
+ </div>
+
+ {disabled ? (
+ <div
+ data-testid={`${testIdRoot}-status-error`}
+ className="flex flex-col gap-1 rounded-md border border-red-500/40 bg-red-500/10 p-3 text-sm"
+ >
+ <span className="font-semibold text-red-300">
+ {t(I18nKey.BACKEND$HEALTH_FAILED_TITLE)}
+ </span>
+ <span className="text-xs text-gray-300">
+ {t(I18nKey.BACKEND$HEALTH_FAILED_DETAIL, {
+ count: consecutiveFailures,
+ })}
+ </span>
+ {lastError ? (
+ <span
+ data-testid={`${testIdRoot}-status-error-message`}
+ className="text-xs text-red-300 whitespace-pre-wrap break-words"
+ >
+ {lastError}
+ </span>
+ ) : null}
+ </div>
) : null}
</div>
);
diff --git a/src/contexts/active-backend-context.tsx b/src/contexts/active-backend-context.tsx
index 2b4798b3f4..26077df427 100644
--- a/src/contexts/active-backend-context.tsx
+++ b/src/contexts/active-backend-context.tsx
@@ -8,6 +8,10 @@ import {
subscribeActiveBackend,
} from "#/api/backend-registry/active-store";
import { makeDefaultLocalBackend } from "#/api/backend-registry/default-backend";
+import {
+ dropBackendHealth,
+ resetBackendHealth,
+} from "#/api/backend-registry/health-store";
import {
type Backend,
type BackendSelection,
@@ -81,10 +85,26 @@ export function ActiveBackendProvider({
const updateBackend = React.useCallback(
(id: string, patch: Partial<Omit<Backend, "id">>) => {
+ const prev = getRegisteredBackends().find((b) => b.id === id);
const list = getRegisteredBackends().map((b) =>
b.id === id ? { ...b, ...patch } : b,
);
setRegisteredBackends(list);
+
+ // Re-arm health polling when the user edits the fields that
+ // actually drive the probe. Cosmetic edits (name) shouldn't
+ // re-enable a backend that was disabled for being unreachable.
+ const hostChanged =
+ patch.host !== undefined &&
+ prev !== undefined &&
+ patch.host !== prev.host;
+ const apiKeyChanged =
+ patch.apiKey !== undefined &&
+ prev !== undefined &&
+ patch.apiKey !== prev.apiKey;
+ if (hostChanged || apiKeyChanged) {
+ resetBackendHealth(id);
+ }
},
[],
);
@@ -92,6 +112,7 @@ export function ActiveBackendProvider({
const removeBackend = React.useCallback((id: string) => {
const list = getRegisteredBackends().filter((b) => b.id !== id);
setRegisteredBackends(list);
+ dropBackendHealth(id);
// If the active selection pointed at this backend, the active
// store falls back to the first remaining local backend (or the
// env-derived default if no locals exist); consumer hooks re-key
diff --git a/src/hooks/query/use-backends-health.ts b/src/hooks/query/use-backends-health.ts
index e9b6c9dc30..d7e102c008 100644
--- a/src/hooks/query/use-backends-health.ts
+++ b/src/hooks/query/use-backends-health.ts
@@ -1,8 +1,16 @@
+import React from "react";
import { useQueries } from "@tanstack/react-query";
import { ServerClient } from "@openhands/typescript-client/clients";
import { getCurrentCloudApiKey } from "#/api/cloud/organization-service.api";
import type { Backend } from "#/api/backend-registry/types";
import { getAgentServerClientOptions } from "#/api/agent-server-client-options";
+import {
+ getHealthSnapshot,
+ recordBackendFailure,
+ recordBackendSuccess,
+ subscribeBackendHealth,
+} from "#/api/backend-registry/health-store";
+import { MAX_CONSECUTIVE_FAILURES } from "#/api/backend-registry/health-storage";
const REFRESH_INTERVAL_MS = 10000;
const PROBE_TIMEOUT_MS = 4000;
@@ -43,6 +51,17 @@ async function probeBackend(backend: Backend): Promise<true> {
export interface BackendHealth {
/** `null` while the first probe is in flight; then `true` / `false`. */
isConnected: boolean | null;
+ /** Number of consecutive failed probes since the last success. */
+ consecutiveFailures: number;
+ /** Last error message captured from a failed probe, if any. */
+ lastError: string | null;
+ /**
+ * `true` once `consecutiveFailures` reaches the cap. While disabled,
+ * the hook stops polling and survives a page refresh in that state.
+ * Polling resumes only when the user edits the backend's host or
+ * api key (see `active-backend-context.tsx`).
+ */
+ disabled: boolean;
}
/**
@@ -52,38 +71,76 @@ export interface BackendHealth {
* The query key includes `host` and `apiKey` so editing a backend's
* connection details re-keys the query and triggers an immediate
* refetch instead of waiting for the next tick.
+ *
+ * After `MAX_CONSECUTIVE_FAILURES` failures in a row, polling stops
+ * for that backend until the user updates its host / apiKey. The
+ * failure count and last error live in localStorage so a page refresh
+ * does not silently re-arm polling against a backend that's known to
+ * be unreachable.
*/
export function useBackendsHealth(
backends: Backend[],
): Record<string, BackendHealth> {
+ const healthMap = React.useSyncExternalStore(
+ subscribeBackendHealth,
+ getHealthSnapshot,
+ getHealthSnapshot,
+ );
+
const results = useQueries({
- queries: backends.map((b) => ({
- queryKey: [
- "backend-health",
- b.id,
- b.kind,
- b.host,
- b.apiKey ?? "",
- ] as const,
- queryFn: () => probeBackend(b),
- refetchInterval: REFRESH_INTERVAL_MS,
- refetchIntervalInBackground: false,
- retry: false,
- // Keep the previous verdict visible while the next probe is in
- // flight so the indicator doesn't flicker every 2s.
- staleTime: REFRESH_INTERVAL_MS,
- meta: { disableToast: true },
- })),
+ queries: backends.map((b) => {
+ const isDisabled = healthMap[b.id]?.disabled === true;
+ return {
+ queryKey: [
+ "backend-health",
+ b.id,
+ b.kind,
+ b.host,
+ b.apiKey ?? "",
+ ] as const,
+ queryFn: async () => {
+ try {
+ const result = await probeBackend(b);
+ recordBackendSuccess(b.id);
+ return result;
+ } catch (err) {
+ recordBackendFailure(b.id, err);
+ throw err;
+ }
+ },
+ enabled: !isDisabled,
+ refetchInterval: isDisabled ? (false as const) : REFRESH_INTERVAL_MS,
+ refetchIntervalInBackground: false,
+ retry: false,
+ // Keep the previous verdict visible while the next probe is in
+ // flight so the indicator doesn't flicker every 2s.
+ staleTime: REFRESH_INTERVAL_MS,
+ meta: { disableToast: true },
+ };
+ }),
});
const out: Record<string, BackendHealth> = {};
backends.forEach((b, i) => {
const r = results[i];
+ const entry = healthMap[b.id];
+ const disabled = entry?.disabled === true;
+ const consecutiveFailures = entry?.consecutiveFailures ?? 0;
+ const lastError = entry?.lastError ?? null;
+
let isConnected: boolean | null;
- if (r.isSuccess) isConnected = true;
+ if (disabled) {
+ // Polling stopped after hitting the cap — treat as disconnected
+ // so existing consumers (dot, badge) render red without needing
+ // to know about the new fields.
+ isConnected = false;
+ } else if (r.isSuccess) isConnected = true;
else if (r.isError) isConnected = false;
else isConnected = null;
- out[b.id] = { isConnected };
+
+ out[b.id] = { isConnected, consecutiveFailures, lastError, disabled };
});
return out;
}
+
+export { MAX_CONSECUTIVE_FAILURES };
diff --git a/src/i18n/translation.json b/src/i18n/translation.json
index 9a2cdc3ac1..e8a6730181 100644
--- a/src/i18n/translation.json
+++ b/src/i18n/translation.json
@@ -23765,6 +23765,40 @@
"uk": "v{{version}}",
"ca": "v{{version}}"
},
+ "BACKEND$HEALTH_FAILED_TITLE": {
+ "en": "Connection failed",
+ "ja": "接続に失敗しました",
+ "zh-CN": "连接失败",
+ "zh-TW": "連線失敗",
+ "ko-KR": "연결 실패",
+ "no": "Tilkobling mislyktes",
+ "ar": "فشل الاتصال",
+ "de": "Verbindung fehlgeschlagen",
+ "fr": "Échec de la connexion",
+ "it": "Connessione non riuscita",
+ "pt": "Falha na conexão",
+ "es": "Conexión fallida",
+ "ca": "Ha fallat la connexió",
+ "tr": "Bağlantı başarısız",
+ "uk": "Не вдалося з'єднатися"
+ },
+ "BACKEND$HEALTH_FAILED_DETAIL": {
+ "en": "Stopped retrying after {{count}} attempts. Update the host or API key to retry.",
+ "ja": "{{count}} 回試行後に再試行を停止しました。再試行するにはホストまたは API キーを更新してください。",
+ "zh-CN": "在 {{count}} 次尝试后已停止重试。请更新主机或 API 密钥后重试。",
+ "zh-TW": "在 {{count}} 次嘗試後已停止重試。請更新主機或 API 金鑰後重試。",
+ "ko-KR": "{{count}}회 시도 후 재시도를 중단했습니다. 호스트 또는 API 키를 업데이트하여 다시 시도하세요.",
+ "no": "Sluttet å prøve på nytt etter {{count}} forsøk. Oppdater verten eller API-nøkkelen for å prøve igjen.",
+ "ar": "تم إيقاف المحاولات بعد {{count}} محاولات. حدّث المضيف أو مفتاح API لإعادة المحاولة.",
+ "de": "Wiederholungsversuche nach {{count}} Versuchen gestoppt. Aktualisieren Sie Host oder API-Schlüssel, um es erneut zu versuchen.",
+ "fr": "Tentatives arrêtées après {{count}} essais. Mettez à jour l'hôte ou la clé API pour réessayer.",
+ "it": "Tentativi interrotti dopo {{count}} tentativi. Aggiorna l'host o la chiave API per riprovare.",
+ "pt": "As tentativas pararam após {{count}} tentativas. Atualize o host ou a chave de API para tentar novamente.",
+ "es": "Se detuvieron los intentos después de {{count}} intentos. Actualiza el host o la clave de API para reintentar.",
+ "ca": "S'han aturat els intents després de {{count}} intents. Actualitzeu l'amfitrió o la clau d'API per tornar-ho a provar.",
+ "tr": "{{count}} denemeden sonra yeniden deneme durduruldu. Yeniden denemek için sunucuyu veya API anahtarını güncelleyin.",
+ "uk": "Зупинено повторні спроби після {{count}} спроб. Оновіть хост або API-ключ, щоб повторити."
+ },
"SETTINGS$BACKEND_SYNCED_BADGE": {
"en": "These settings are synced from {{name}} backend ({{host}})",
"ja": "これらの設定は {{name}} バックエンド ({{host}}) と同期されています",