π OpenHands: AI-Driven Development
Fix stale backend health errors after recovery (#424)
Devina1befdd89db3
diff --git a/AGENTS.md b/AGENTS.mdindex 85e6d8c721..371633c11d 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -235,7 +235,7 @@- The terminal tab (`components/features/terminal/terminal.tsx`) is `React.lazy`'d in `conversation-tab-content.tsx` alongside the other tabs, so xterm + addon-fit + xterm.css don't enter the conversation route's eager graph (they ship as a separate `terminal-*.js` chunk now).- Avoid importing app code through `#/components/conversation-events/chat` or its `event-message-components/index.ts` barrel β they exist for `lib/index.ts` (npm subpath) consumers only. Internal callers use deep paths (`./messages`, `./event-message-components/<name>`, `./event-content-helpers/should-render-event`) so Vite dev doesn't fan out the barrel.-- Backend dropdown connectivity indicator: `useBackendsHealth` (`src/hooks/query/use-backends-health.ts`) polls each registered backend every 10s. Local agent-server backends are probed via `ServerClient.getServerInfo()` (`/server_info`); cloud SaaS backends are probed via `getCurrentCloudApiKey()` (`/api/keys/current` through the bundled `/api/cloud-proxy`). Verdicts are surfaced as a colored dot rendered through `DropdownOption.prefix` (added to `src/ui/dropdown/types.ts`); the trigger reads its prefix from the live `options` array (not downshift's frozen `selectedItem`) so the indicator updates without remounting. The same dot is also rendered in each row of `ManageBackendsModal`. Tests live in `__tests__/hooks/query/use-backends-health.test.tsx`, the `connection indicator` block of `__tests__/components/backends/backend-selector.test.tsx`, and `__tests__/components/backends/manage-backends-modal.test.tsx`.+- Backend dropdown connectivity indicator: `useBackendsHealth` (`src/hooks/query/use-backends-health.ts`) polls each registered backend every 10s. Local agent-server backends are probed via `ServerClient.getServerInfo()` (`/server_info`); cloud SaaS backends are probed via `getCurrentCloudApiKey()` (`/api/keys/current` through the bundled `/api/cloud-proxy`). Verdicts are surfaced as a colored dot rendered through `DropdownOption.prefix` (added to `src/ui/dropdown/types.ts`); the trigger reads its prefix from the live `options` array (not downshift's frozen `selectedItem`) so the indicator updates without remounting. The same dot is also rendered in each row of `ManageBackendsModal`, which now opts into a one-shot re-probe for previously disabled backends so opening the modal can clear stale persisted error state when a server has recovered. Tests live in `__tests__/hooks/query/use-backends-health.test.tsx`, the `connection indicator` block of `__tests__/components/backends/backend-selector.test.tsx`, and `__tests__/components/backends/manage-backends-modal.test.tsx`.- Manage Backends modal: `src/components/features/backends/manage-backends-modal.tsx` lets users edit (host/name/api-key/kind) and remove existing backends, plus add new ones inline via a "+ Add Backend" footer button that opens a `BackendFormModal`. Both the dropdown footer's "Add backend" and the manage modal's "+ Add Backend" reuse `BackendFormModal` (see `backend-form-modal.tsx`), with `mode="add"` or `mode="edit"`; `AddBackendModal` is now a thin compatibility wrapper for `BackendFormModal mode="add"`. The modal is also auto-rendered (with a no-op `onClose`) by `src/root.tsx` when the active backend is unreachable, replacing the old full-screen `MissingAgentServerNotice` onboarding screen.@@ -254,4 +254,3 @@- Collapsible thinking: `ThinkAction` events and LLM extended reasoning (`reasoning_content` / `thinking_blocks` on `ActionEvent`) are rendered as collapsible sections via `CollapsibleThinking` (`src/components/conversation-events/chat/event-message-components/collapsible-thinking.tsx`). Collapsed by default to keep the chat compact β the thinking is often in English regardless of the user's conversation language. The `getReasoningContent()` helper in `event-thought-helpers.ts` extracts the content, preferring `reasoning_content` (plain string) and falling back to Anthropic `thinking_blocks`. i18n keys: `THINKING$TITLE`, `THINKING$EXPAND`, `THINKING$COLLAPSE`. Tests: `__tests__/components/conversation-events/chat/event-message-think-action.test.tsx`.- ESLint config (flat, ESLint 9): the project uses `eslint.config.js` (not `.eslintrc`) and runs on `[email protected]`, not 10. The constraint pinning us below 10 is `[email protected]`, which still calls `context.getFilename()` at rule-load time β that API was removed in ESLint 10 and `@eslint/compat`'s `fixupPluginRules` does NOT shim it. Don't try to bump eslint past 9 until eslint-plugin-react ships a v10-compatible release. Import rules come from `eslint-plugin-import-x` (the maintained fork of `eslint-plugin-import`) but are registered under both `import-x/` and `import/` prefixes via `plugins: { import: importXPlugin, ... }` so existing `// eslint-disable-next-line import/...` directives keep working. `linterOptions.reportUnusedDisableDirectives` is set to `"warn"` (not "off") so stale airbnb-era disable comments still surface in lint output without failing CI. The TS-overrides block has an `ignores: ["src/hooks/query/query-keys.ts"]` so the `no-restricted-syntax` rule banning raw `["settings", ...]` query keys doesn't fire on the file that defines the helpers themselves. No `.npmrc` / `legacy-peer-deps` flag is needed β all our plugins declare ESLint 9 peer compatibility.-diff --git a/__tests__/api/backend-registry/health-store.test.ts b/__tests__/api/backend-registry/health-store.test.tsindex ac1c92ebe8..d6788332c8 100644--- a/__tests__/api/backend-registry/health-store.test.ts+++ b/__tests__/api/backend-registry/health-store.test.ts@@ -64,6 +64,28 @@ describe("backend health store", () => {expect(persisted[BACKEND_ID].disabled).toBe(true);});+ it("caps the failure count at the max when a disabled backend fails another recheck", () => {+ for (let i = 0; i < MAX_CONSECUTIVE_FAILURES; i += 1) {+ recordBackendFailure(BACKEND_ID, new Error("timeout"));+ }++ recordBackendFailure(BACKEND_ID, new Error("still down"));++ const entry = getBackendHealthEntry(BACKEND_ID);+ expect(entry).toMatchObject({+ consecutiveFailures: MAX_CONSECUTIVE_FAILURES,+ disabled: true,+ lastError: "still down",+ });++ const persisted = JSON.parse(+ window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY) ?? "{}",+ );+ expect(persisted[BACKEND_ID].consecutiveFailures).toBe(+ MAX_CONSECUTIVE_FAILURES,+ );+ });+it("clears the entry (and storage) and notifies subscribers when the backend recovers or the user edits its config", () => {// Arrange β record one failure so there is something to clear, and// subscribe so we can confirm listeners get notified.diff --git a/__tests__/components/backends/manage-backends-modal.test.tsx b/__tests__/components/backends/manage-backends-modal.test.tsxindex 70f6d062f4..de80d31776 100644--- a/__tests__/components/backends/manage-backends-modal.test.tsx+++ b/__tests__/components/backends/manage-backends-modal.test.tsx@@ -4,17 +4,25 @@ import { render, screen, waitFor } from "@testing-library/react";import userEvent from "@testing-library/user-event";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import { DEFAULT_LOCAL_BACKEND_ID } from "#/api/backend-registry/default-backend";+import {+ BACKEND_HEALTH_STORAGE_KEY,+ MAX_CONSECUTIVE_FAILURES,+} from "#/api/backend-registry/health-storage";import { __resetActiveStoreForTests } from "#/api/backend-registry/active-store";+import { __resetHealthStoreForTests } from "#/api/backend-registry/health-store";import {ActiveBackendProvider,useActiveBackendContext,} from "#/contexts/active-backend-context";import { ManageBackendsModal } from "#/components/features/backends/manage-backends-modal";+const getServerInfoMock = vi.fn().mockResolvedValue({ version: "1.18.0" });+vi.mock("@openhands/typescript-client/clients", () => ({- ServerClient: vi.fn(() => ({- getServerInfo: vi.fn().mockResolvedValue({ version: "1.18.0" }),- })),+ ServerClient: vi.fn(function ServerClientMock() {+ return { getServerInfo: getServerInfoMock };+ }),}));vi.mock("#/api/cloud/organization-service.api", () => ({@@ -53,11 +61,13 @@ function TestSeed({beforeEach(() => {window.localStorage.clear();__resetActiveStoreForTests();+ __resetHealthStoreForTests();});afterEach(() => {window.localStorage.clear();__resetActiveStoreForTests();+ __resetHealthStoreForTests();});describe("ManageBackendsModal", () => {@@ -83,6 +93,29 @@ describe("ManageBackendsModal", () => {expect(await screen.findByTestId("add-backend-modal")).toBeInTheDocument();});+ it("re-checks a disabled backend on open and clears stale persisted health when it recovers", async () => {+ window.localStorage.setItem(+ BACKEND_HEALTH_STORAGE_KEY,+ JSON.stringify({+ [DEFAULT_LOCAL_BACKEND_ID]: {+ consecutiveFailures: MAX_CONSECUTIVE_FAILURES,+ lastError: "Network Error",+ lastFailureAt: Date.now(),+ disabled: true,+ },+ }),+ );+ __resetHealthStoreForTests();++ renderWithProviders(<ManageBackendsModal onClose={vi.fn()} />);++ await waitFor(() => {+ expect(+ window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY),+ ).toBeNull();+ });+ });+it("opens an edit form pre-filled with the row's backend, and persists changes via updateBackend", async () => {const user = userEvent.setup();diff --git a/__tests__/hooks/query/use-backends-health.test.tsx b/__tests__/hooks/query/use-backends-health.test.tsxindex c2f96a3619..6edec7f12e 100644--- a/__tests__/hooks/query/use-backends-health.test.tsx+++ b/__tests__/hooks/query/use-backends-health.test.tsx@@ -240,4 +240,36 @@ describe("useBackendsHealth", () => {);expect(getServerInfoMock).toHaveBeenCalled();});++ it("re-probes a persisted-disabled backend when explicitly asked and clears the stale health entry on success", async () => {+ window.localStorage.setItem(+ BACKEND_HEALTH_STORAGE_KEY,+ JSON.stringify({+ [localBackend.id]: {+ consecutiveFailures: MAX_CONSECUTIVE_FAILURES,+ lastError: "Network Error",+ lastFailureAt: Date.now(),+ disabled: true,+ },+ }),+ );+ __resetHealthStoreForTests();+ getServerInfoMock.mockResolvedValue({ version: "1.18.0" });++ const { result } = renderHook(+ () => useBackendsHealth([localBackend], { probeDisabledOnce: true }),+ {+ wrapper,+ },+ );++ await waitFor(() =>+ expect(result.current[localBackend.id]).toMatchObject({+ isConnected: true,+ disabled: false,+ }),+ );+ expect(getServerInfoMock).toHaveBeenCalled();+ expect(window.localStorage.getItem(BACKEND_HEALTH_STORAGE_KEY)).toBeNull();+ });});diff --git a/src/api/backend-registry/health-store.ts b/src/api/backend-registry/health-store.tsindex c488ed8e88..5b4a470b03 100644--- a/src/api/backend-registry/health-store.ts+++ b/src/api/backend-registry/health-store.ts@@ -45,7 +45,10 @@ export function subscribeBackendHealth(listener: Listener): () => void {*/export function recordBackendFailure(id: string, error: unknown): void {const prev = healthMap[id];- const consecutiveFailures = (prev?.consecutiveFailures ?? 0) + 1;+ const consecutiveFailures = Math.min(+ (prev?.consecutiveFailures ?? 0) + 1,+ MAX_CONSECUTIVE_FAILURES,+ );const disabled =prev?.disabled === true || consecutiveFailures >= MAX_CONSECUTIVE_FAILURES;diff --git a/src/components/features/backends/manage-backends-modal.tsx b/src/components/features/backends/manage-backends-modal.tsxindex 50ea099066..400fbab5c4 100644--- a/src/components/features/backends/manage-backends-modal.tsx+++ b/src/components/features/backends/manage-backends-modal.tsx@@ -112,7 +112,9 @@ function BackendRow({ backend, health, onEdit, onRemove }: BackendRowProps) {export function ManageBackendsModal({ onClose }: ManageBackendsModalProps) {const { t } = useTranslation("openhands");const { backends, removeBackend } = useActiveBackendContext();- const healthByBackendId = useBackendsHealth(backends);+ const healthByBackendId = useBackendsHealth(backends, {+ probeDisabledOnce: true,+ });const [pendingRemoval, setPendingRemoval] =React.useState<PendingRemoval | null>(null);const [editingBackend, setEditingBackend] = React.useState<Backend | null>(diff --git a/src/hooks/query/use-backends-health.ts b/src/hooks/query/use-backends-health.tsindex d7e102c008..e33082d343 100644--- a/src/hooks/query/use-backends-health.ts+++ b/src/hooks/query/use-backends-health.ts@@ -57,13 +57,21 @@ export interface BackendHealth {lastError: string | null;/*** `true` once `consecutiveFailures` reaches the cap. While disabled,- * the hook stops polling and survives a page refresh in that state.- * Polling resumes only when the user edits the backend's host or- * api key (see `active-backend-context.tsx`).+ * ordinary background polling stops and survives a page refresh in+ * that state.*/disabled: boolean;}+export interface UseBackendsHealthOptions {+ /**+ * Re-probe disabled backends once when the hook mounts. Used by the+ * Manage Backends modal so a recovered backend can clear its stale+ * persisted error state without forcing the user to edit the config.+ */+ probeDisabledOnce?: boolean;+}+/*** Poll every backend in `backends` once every 10s and report a simple* connected / disconnected verdict per backend id.@@ -72,15 +80,19 @@ export interface BackendHealth {* connection details re-keys the query and triggers an immediate* refetch instead of waiting for the next tick.*- * After `MAX_CONSECUTIVE_FAILURES` failures in a row, polling stops- * for that backend until the user updates its host / apiKey. The+ * After `MAX_CONSECUTIVE_FAILURES` failures in a row, ordinary polling+ * stops for that backend until the user updates its host / apiKey.+ * Callers can still opt into a one-shot recheck for disabled backends+ * (for example when the user explicitly opens Manage Backends). The* failure count and last error live in localStorage so a page refresh* does not silently re-arm polling against a backend that's known to* be unreachable.*/export function useBackendsHealth(backends: Backend[],+ options: UseBackendsHealthOptions = {},): Record<string, BackendHealth> {+ const { probeDisabledOnce = false } = options;const healthMap = React.useSyncExternalStore(subscribeBackendHealth,getHealthSnapshot,@@ -90,6 +102,7 @@ export function useBackendsHealth(const results = useQueries({queries: backends.map((b) => {const isDisabled = healthMap[b.id]?.disabled === true;+ const shouldProbe = !isDisabled || probeDisabledOnce;return {queryKey: ["backend-health",@@ -108,13 +121,16 @@ export function useBackendsHealth(throw err;}},- enabled: !isDisabled,+ enabled: shouldProbe,refetchInterval: isDisabled ? (false as const) : REFRESH_INTERVAL_MS,refetchIntervalInBackground: false,+ refetchOnMount: isDisabled && probeDisabledOnce ? "always" : true,+ refetchOnReconnect: !isDisabled,+ refetchOnWindowFocus: !isDisabled,retry: false,// Keep the previous verdict visible while the next probe is in- // flight so the indicator doesn't flicker every 2s.- staleTime: REFRESH_INTERVAL_MS,+ // flight so the indicator doesn't flicker on routine polling.+ staleTime: isDisabled ? 0 : REFRESH_INTERVAL_MS,meta: { disableToast: true },};}),