Skip to content

πŸ™Œ OpenHands: AI-Driven Development

11.5 KB
Raw Blame
import React from "react";
import axios from "axios";
import { useQueries } from "@tanstack/react-query";
import { HttpError } from "@openhands/typescript-client";
import {
  getCloudOrganizations,
  getCurrentCloudApiKey,
} from "#/api/cloud/organization-service.api";
import {
  isDefinitiveLocalBackendError,
  validateLocalBackend,
  INVALID_BACKEND_API_KEY_ERROR,
} from "#/api/agent-server-compatibility";
import type { Backend } from "#/api/backend-registry/types";
import { useActiveBackend } from "#/contexts/active-backend-context";
import {
  isCorsOrNetworkError,
  isCorsOrNetworkErrorMessage,
} from "#/utils/user-facing-error";
import {
  getHealthSnapshot,
  recordBackendFailure,
  recordBackendSuccess,
  subscribeBackendHealth,
} from "#/api/backend-registry/health-store";
import { MAX_CONSECUTIVE_FAILURES } from "#/api/backend-registry/health-storage";

// 30s: each local tick costs two sequential requests per backend (settings +
// server_info), which on slow links measurably competes with conversation
// traffic for the browser's per-origin HTTP/1.1 connection pool.
const REFRESH_INTERVAL_MS = 30000;
// Cloud health probes hit app-server auth/org endpoints that are shared with the
// main UI. They do not need local-runtime-level freshness, and a slower cadence
// avoids competing with conversation, sandbox, and repository requests.
const CLOUD_REFRESH_INTERVAL_MS = 5 * 60 * 1000;
const PROBE_TIMEOUT_MS = 4000;
export { INVALID_BACKEND_API_KEY_ERROR } from "#/api/agent-server-compatibility";
export const MISSING_BACKEND_API_KEY_ERROR = "API key required";
export const CLOUD_BACKEND_API_KEY_OR_NETWORK_ERROR =
  "Cloud API key or network issue";
export const CLOUD_BACKEND_LOGGED_OUT_ERROR = "Logged out";

export function isInvalidBackendApiKeyHealthError(
  error: string | null | undefined,
): boolean {
  return error === INVALID_BACKEND_API_KEY_ERROR;
}

export function isMissingBackendApiKeyHealthError(
  error: string | null | undefined,
): boolean {
  return error === MISSING_BACKEND_API_KEY_ERROR;
}

export function isCloudBackendApiKeyOrNetworkHealthError(
  error: string | null | undefined,
): boolean {
  return error === CLOUD_BACKEND_API_KEY_OR_NETWORK_ERROR;
}

function hasMissingBackendApiKey(backend: Backend): boolean {
  return (
    backend.kind === "cloud" &&
    backend.authMode !== "cookie" &&
    !backend.apiKey.trim()
  );
}

export function isCloudBackendLoggedOutHealthError(
  error: string | null | undefined,
): boolean {
  return error === CLOUD_BACKEND_LOGGED_OUT_ERROR;
}

/**
 * Probe a single backend for connectivity. The probe path differs by
 * backend kind:
 *
 *  - Local agent-server: GET `/api/settings`, then `/server_info` via the
 *    typescript-client. The settings call validates the configured session
 *    API key; the server info call validates the version compatibility floor.
 *  - Cloud: GET `/api/keys/current` directly against the cloud host. That
 *    endpoint is lightweight, requires auth, and `getCurrentCloudApiKey`
 *    already absorbs the legacy-key 400 fallback so we treat that as
 *    "connected" too. Missing / rejected keys are reported explicitly instead
 *    of falling through to the browser's opaque CORS/network error shape.
 *
 * Throws on failure so React Query marks the query as errored β€” the
 * dropdown reads `isSuccess` to flip the indicator green.
 */
async function probeBackend(backend: Backend): Promise<true> {
  if (backend.kind === "cloud") {
    if (backend.authMode !== "cookie" && !backend.apiKey?.trim()) {
      throw new Error(MISSING_BACKEND_API_KEY_ERROR);
    }

    try {
      if (backend.authMode === "cookie") {
        await getCloudOrganizations(backend);
      } else {
        await getCurrentCloudApiKey(backend);
      }
    } catch (error) {
      if (
        (axios.isAxiosError(error) && error.response?.status === 401) ||
        (error instanceof HttpError && error.status === 401)
      ) {
        throw new Error(CLOUD_BACKEND_LOGGED_OUT_ERROR);
      }
      if (isCorsOrNetworkError(error)) {
        throw new Error(CLOUD_BACKEND_API_KEY_OR_NETWORK_ERROR);
      }
      throw error;
    }
    return true;
  }

  await validateLocalBackend(backend, PROBE_TIMEOUT_MS);
  return true;
}

/**
 * How many extra times to re-run a failed probe before giving up, and how
 * long to wait between attempts.
 */
const PROBE_RETRY_ATTEMPTS = 2;
const PROBE_RETRY_DELAY_MS = 300;

/**
 * Probe a backend, retrying a couple of times on failure before giving up.
 *
 * The connectivity indicator (and the onboarding "backend connected" banner)
 * only flips green once a probe succeeds. With `retry: false` at the query
 * level and a REFRESH_INTERVAL_MS (30s) poll, a single transient first-probe
 * miss β€” the agent-server still warming up right after navigation, a momentary
 * proxy 5xx, a dropped connection β€” would otherwise leave the banner stuck
 * until the next scheduled refetch. That is the root cause of the
 * flaky onboarding e2e "backend health probe should report connected" timeout.
 *
 * Retrying here, inside the query function rather than via React Query's
 * `retry`, is deliberate: the success/failure recording below runs once per
 * settled query, so a single logical probe still records exactly one outcome.
 * Retrying at the query level instead would call `recordBackendFailure` on
 * every internal attempt and reach the disabled cap several times too fast.
 *
 * Definitive failures (logged out, invalid/missing key, unsupported version)
 * are NOT retried: they are a decided server response, not a transient miss,
 * so retrying only delays surfacing the correct "disconnected" state (and the
 * recovery UI).
 */
function isRetryableProbeError(error: unknown): boolean {
  if (!(error instanceof Error)) return true;
  return (
    !isDefinitiveLocalBackendError(error) &&
    error.message !== MISSING_BACKEND_API_KEY_ERROR &&
    error.message !== CLOUD_BACKEND_LOGGED_OUT_ERROR
  );
}

async function probeBackendWithQuickRetry(backend: Backend): Promise<true> {
  for (let attempt = 0; ; attempt += 1) {
    try {
      return await probeBackend(backend);
    } catch (error) {
      if (attempt >= PROBE_RETRY_ATTEMPTS || !isRetryableProbeError(error)) {
        throw error;
      }
      await new Promise((resolve) => {
        setTimeout(resolve, PROBE_RETRY_DELAY_MS);
      });
    }
  }
}

export interface BackendHealth {
  /** `null` while the first probe is in flight; then `true` / `false`. */
  isConnected: boolean | null;
  /** Number of consecutive failed probes since the last success. */
  consecutiveFailures: number;
  /** Last error message captured from a failed probe, if any. */
  lastError: string | null;
  /**
   * `true` once `consecutiveFailures` reaches the cap. While disabled,
   * ordinary background polling stops and survives a page refresh in
   * that state.
   */
  disabled: boolean;
}

export interface UseBackendsHealthOptions {
  /**
   * Re-probe disabled backends once when the hook mounts. Used by the
   * Manage Backends modal so a recovered backend can clear its stale
   * persisted error state without forcing the user to edit the config.
   */
  probeDisabledOnce?: boolean;
}

/**
 * Poll every backend in `backends` once every 10s and report a simple
 * connected / disconnected verdict per backend id.
 *
 * The query key includes `host` and `apiKey` so editing a backend's
 * connection details re-keys the query and triggers an immediate
 * refetch instead of waiting for the next tick.
 *
 * After `MAX_CONSECUTIVE_FAILURES` failures in a row, ordinary polling
 * stops for that backend until the user updates its host / apiKey.
 * Callers can still opt into a one-shot recheck for disabled backends
 * (for example when the user explicitly opens Manage Backends). The
 * failure count and last error live in localStorage so a page refresh
 * does not silently re-arm polling against a backend that's known to
 * be unreachable.
 */
export function useBackendsHealth(
  backends: Backend[],
  options: UseBackendsHealthOptions = {},
): Record<string, BackendHealth> {
  const { probeDisabledOnce = false } = options;
  const healthMap = React.useSyncExternalStore(
    subscribeBackendHealth,
    getHealthSnapshot,
    getHealthSnapshot,
  );

  const activeBackendId = useActiveBackend().backend.id;

  const results = useQueries({
    queries: backends.map((b) => {
      const entry = healthMap[b.id];
      const hasMissingCloudApiKey = hasMissingBackendApiKey(b);
      // Every probe of a MARS session travels its tunnel into the sandbox;
      // polling sessions the user is not on would keep them from ever
      // idle-pausing (and billing), so those are probed once on mount and
      // only the active one is watched.
      const isIdleMarsSession =
        Boolean(b.marsSessionId) && b.id !== activeBackendId;
      const isDisabled = entry?.disabled === true;
      const refreshInterval =
        b.kind === "cloud" ? CLOUD_REFRESH_INTERVAL_MS : REFRESH_INTERVAL_MS;
      const shouldReprobeStaleCloudNetworkError =
        isDisabled &&
        b.kind === "cloud" &&
        isCorsOrNetworkErrorMessage(entry?.lastError);
      const shouldProbe =
        !hasMissingCloudApiKey &&
        (!isDisabled ||
          probeDisabledOnce ||
          shouldReprobeStaleCloudNetworkError);
      return {
        queryKey: [
          "backend-health",
          b.id,
          b.kind,
          b.host,
          b.apiKey ?? "",
        ] as const,
        queryFn: async () => {
          try {
            const result = await probeBackendWithQuickRetry(b);
            recordBackendSuccess(b.id);
            return result;
          } catch (err) {
            recordBackendFailure(b.id, err);
            throw err;
          }
        },
        enabled: shouldProbe,
        refetchInterval:
          isDisabled || hasMissingCloudApiKey || isIdleMarsSession
            ? (false as const)
            : refreshInterval,
        refetchIntervalInBackground: false,
        refetchOnMount: isDisabled && probeDisabledOnce ? "always" : true,
        refetchOnReconnect:
          b.kind !== "cloud" &&
          !isDisabled &&
          !hasMissingCloudApiKey &&
          !isIdleMarsSession,
        refetchOnWindowFocus:
          b.kind !== "cloud" &&
          !isDisabled &&
          !hasMissingCloudApiKey &&
          !isIdleMarsSession,
        retry: false,
        // Keep the previous verdict visible while the next probe is in
        // flight so the indicator doesn't flicker on routine polling.
        staleTime: isDisabled ? 0 : refreshInterval,
        meta: { disableToast: true },
      };
    }),
  });

  const out: Record<string, BackendHealth> = {};
  backends.forEach((b, i) => {
    const r = results[i];
    const entry = healthMap[b.id];
    const hasMissingCloudApiKey = hasMissingBackendApiKey(b);
    const disabled = hasMissingCloudApiKey ? false : entry?.disabled === true;
    const consecutiveFailures = hasMissingCloudApiKey
      ? 0
      : (entry?.consecutiveFailures ?? 0);
    const lastError = hasMissingCloudApiKey
      ? MISSING_BACKEND_API_KEY_ERROR
      : (entry?.lastError ?? null);

    let isConnected: boolean | null;
    if (hasMissingCloudApiKey) {
      isConnected = false;
    } else if (disabled) {
      // Polling stopped after hitting the cap β€” treat as disconnected
      // so existing consumers (dot, badge) render red without needing
      // to know about the new fields.
      isConnected = false;
    } else if (r.isSuccess) isConnected = true;
    else if (r.isError) isConnected = false;
    else isConnected = null;

    out[b.id] = { isConnected, consecutiveFailures, lastError, disabled };
  });
  return out;
}

export { MAX_CONSECUTIVE_FAILURES };