π OpenHands: AI-Driven Development
fix: cloud sandbox resume retry behavior (#17665)
Rohit Malhotra53c8b4d9ca38
diff --git a/__tests__/hooks/mutation/pause-conversation-mutation-behavior.test.tsx b/__tests__/hooks/mutation/pause-conversation-mutation-behavior.test.tsxindex 8a2efa1c25..10d12c3ba1 100644--- a/__tests__/hooks/mutation/pause-conversation-mutation-behavior.test.tsx+++ b/__tests__/hooks/mutation/pause-conversation-mutation-behavior.test.tsx@@ -4,23 +4,50 @@ import { AxiosError } from "axios";import type { ReactNode } from "react";import { describe, expect, it, vi } from "vitest";import { usePauseConversation } from "#/hooks/mutation/use-pause-conversation";--const { pauseConversationMock } = vi.hoisted(() => ({+import { ExecutionStatus } from "#/types/agent-server/core";++const {+ activeBackendState,+ patchConversationInCacheMock,+ pauseConversationMock,+ suppressNextCloudAutoResumeMock,+} = vi.hoisted(() => ({+ activeBackendState: { kind: "local" as "local" | "cloud" },+ patchConversationInCacheMock: vi.fn(),pauseConversationMock: vi.fn(),+ suppressNextCloudAutoResumeMock: vi.fn(),+}));++vi.mock("#/contexts/active-backend-context", () => ({+ useActiveBackend: () => ({+ backend: { id: "backend-1", kind: activeBackendState.kind },+ }),+}));++vi.mock("#/api/cloud/cloud-sandbox-resume-suppression", () => ({+ suppressNextCloudAutoResume: (...args: unknown[]) =>+ suppressNextCloudAutoResumeMock(...args),}));vi.mock("#/hooks/mutation/conversation-mutation-utils", () => ({+ patchConversationInCache: (...args: unknown[]) =>+ patchConversationInCacheMock(...args),pauseConversation: (...args: unknown[]) => pauseConversationMock(...args),}));const CONVERSATIONS_QUERY_KEY = ["user", "conversations"] as const;interface SetupOptions {+ backendKind?: "local" | "cloud";previousConversations?: unknown;}-const setup = ({ previousConversations }: SetupOptions = {}) => {+const setup = ({+ backendKind = "local",+ previousConversations,+}: SetupOptions = {}) => {vi.clearAllMocks();+ activeBackendState.kind = backendKind;pauseConversationMock.mockResolvedValue({ success: true });const queryClient = new QueryClient({@@ -118,6 +145,28 @@ describe("pause conversation mutation behavior", () => {);});+ it("suppresses cloud auto-resume and clears the runtime URL after pausing", async () => {+ const { queryClient, result } = setup({ backendKind: "cloud" });++ await act(async () => {+ await result.current.mutateAsync({+ conversationId: "conversation-cloud",+ });+ });++ expect(suppressNextCloudAutoResumeMock).toHaveBeenCalledWith(+ "conversation-cloud",+ );+ expect(patchConversationInCacheMock).toHaveBeenCalledWith(+ queryClient,+ "conversation-cloud",+ {+ execution_status: ExecutionStatus.PAUSED,+ conversation_url: null,+ },+ );+ });+it("restores the exact conversation list snapshot when pausing fails", async () => {const previousConversations = {pages: [{ items: [{ id: "conversation-1", title: "Original" }] }],diff --git a/__tests__/hooks/mutation/use-unified-stop-conversation.test.tsx b/__tests__/hooks/mutation/use-unified-stop-conversation.test.tsxindex 5721357cf8..6a6874155c 100644--- a/__tests__/hooks/mutation/use-unified-stop-conversation.test.tsx+++ b/__tests__/hooks/mutation/use-unified-stop-conversation.test.tsx@@ -100,7 +100,7 @@ beforeEach(() => {});describe("useUnifiedPauseConversation", () => {- it("pauses the active conversation, patches both paused statuses, and navigates home", async () => {+ it("pauses the active conversation, clears its runtime URL, and navigates home", async () => {const previousConversations = {pages: [{ items: [{ id: "conversation-1" }] }],};@@ -137,7 +137,7 @@ describe("useUnifiedPauseConversation", () => {"conversation-1",{execution_status: ExecutionStatus.PAUSED,- sandbox_status: "PAUSED",+ conversation_url: null,},);expect(navigateMock).toHaveBeenCalledWith("/conversations");@@ -160,7 +160,7 @@ describe("useUnifiedPauseConversation", () => {"conversation-1",{execution_status: ExecutionStatus.PAUSED,- sandbox_status: "PAUSED",+ conversation_url: null,},);expect(navigateMock).not.toHaveBeenCalled();diff --git a/__tests__/routes/conversation-backend-switch.test.tsx b/__tests__/routes/conversation-backend-switch.test.tsxindex 8e2ac4fbd3..33866c1ca7 100644--- a/__tests__/routes/conversation-backend-switch.test.tsx+++ b/__tests__/routes/conversation-backend-switch.test.tsx@@ -1,6 +1,6 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import React from "react";-import { render, screen, waitFor } from "@testing-library/react";+import { act, render, screen, waitFor } from "@testing-library/react";import { MemoryRouter, Route, Routes } from "react-router";import type { SharedConversation } from "@openhands/typescript-client";import { QueryClient, QueryClientProvider } from "@tanstack/react-query";@@ -14,6 +14,11 @@ import {import { ActiveBackendProvider } from "#/contexts/active-backend-context";import { NavigationProvider } from "#/context/navigation-context";import ConversationView from "#/routes/conversation";+import { resumeCloudSandbox } from "#/api/cloud/conversation-service.api";+import {+ __clearCloudAutoResumeSuppressionsForTests,+ suppressNextCloudAutoResume,+} from "#/api/cloud/cloud-sandbox-resume-suppression";import type { Backend } from "#/api/backend-registry/types";import type { AppConversation } from "#/api/conversation-service/agent-server-conversation-service.types";import { getCloudSharedConversation } from "#/api/cloud/shared-conversation-service.api";@@ -92,7 +97,10 @@ const cloudBackend: Backend = {const CLOUD_CONVERSATION_ID = "conv-cloud";-function makeConversation(id: string): AppConversation {+function makeConversation(+ id: string,+ overrides: Partial<AppConversation> = {},+): AppConversation {return {id,created_by_user_id: null,@@ -112,6 +120,7 @@ function makeConversation(id: string): AppConversation {session_api_key: null,sandbox_id: null,sub_conversation_ids: [],+ ...overrides,};}@@ -204,6 +213,8 @@ beforeEach(() => {vi.mocked(AgentServerConversationService.batchGetAppConversations,).mockReset();+ vi.mocked(resumeCloudSandbox).mockReset();+ vi.mocked(resumeCloudSandbox).mockResolvedValue(undefined);vi.mocked(AgentServerConversationService.batchGetAppConversations,).mockResolvedValue([makeConversation(CLOUD_CONVERSATION_ID)]);@@ -211,6 +222,8 @@ beforeEach(() => {});afterEach(() => {+ vi.useRealTimers();+ __clearCloudAutoResumeSuppressionsForTests();window.localStorage.clear();__resetActiveStoreForTests();});@@ -234,6 +247,145 @@ describe("conversation route β backend switch", () => {});});+describe("conversation route β cloud sandbox resume", () => {+ it("resumes an already-paused cloud sandbox when the conversation is opened", async () => {+ setActiveSelection({ backendId: cloudBackend.id, orgId: "org-a" });+ vi.mocked(+ AgentServerConversationService.batchGetAppConversations,+ ).mockResolvedValue([+ makeConversation(CLOUD_CONVERSATION_ID, {+ sandbox_status: "PAUSED",+ sandbox_id: "sandbox-paused-1",+ }),+ ]);++ renderConversation();++ await waitFor(() => {+ expect(resumeCloudSandbox).toHaveBeenCalledWith("sandbox-paused-1");+ });+ expect(resumeCloudSandbox).toHaveBeenCalledTimes(1);+ });++ it("does not auto-resume a cloud sandbox intentionally paused in this tab", async () => {+ setActiveSelection({ backendId: cloudBackend.id, orgId: "org-a" });+ suppressNextCloudAutoResume(CLOUD_CONVERSATION_ID);+ vi.mocked(+ AgentServerConversationService.batchGetAppConversations,+ ).mockResolvedValue([+ makeConversation(CLOUD_CONVERSATION_ID, {+ sandbox_status: "PAUSED",+ sandbox_id: "sandbox-paused-suppressed",+ }),+ ]);++ renderConversation();++ expect(await screen.findByTestId("conversation-main")).toBeInTheDocument();+ expect(resumeCloudSandbox).not.toHaveBeenCalled();+ });++ it("clears stale suppression when the route unmounts before PAUSED is observed", async () => {+ setActiveSelection({ backendId: cloudBackend.id, orgId: "org-a" });+ suppressNextCloudAutoResume(CLOUD_CONVERSATION_ID);+ vi.mocked(+ AgentServerConversationService.batchGetAppConversations,+ ).mockResolvedValue([+ makeConversation(CLOUD_CONVERSATION_ID, {+ sandbox_status: "RUNNING",+ sandbox_id: "sandbox-running-before-unmount",+ }),+ ]);++ const { unmount } = renderConversation();++ expect(await screen.findByTestId("conversation-main")).toBeInTheDocument();+ expect(resumeCloudSandbox).not.toHaveBeenCalled();++ unmount();+ vi.mocked(resumeCloudSandbox).mockClear();+ vi.mocked(+ AgentServerConversationService.batchGetAppConversations,+ ).mockResolvedValue([+ makeConversation(CLOUD_CONVERSATION_ID, {+ sandbox_status: "PAUSED",+ sandbox_id: "sandbox-paused-after-reopen",+ }),+ ]);++ renderConversation();++ await waitFor(() => {+ expect(resumeCloudSandbox).toHaveBeenCalledWith(+ "sandbox-paused-after-reopen",+ );+ });+ });++ it("retries a failed cloud sandbox resume while the conversation remains paused", async () => {+ setActiveSelection({ backendId: cloudBackend.id, orgId: "org-a" });+ vi.mocked(+ AgentServerConversationService.batchGetAppConversations,+ ).mockResolvedValue([+ makeConversation(CLOUD_CONVERSATION_ID, {+ sandbox_status: "PAUSED",+ sandbox_id: "sandbox-paused-2",+ }),+ ]);+ vi.mocked(resumeCloudSandbox)+ .mockRejectedValueOnce(new Error("resume failed"))+ .mockResolvedValueOnce(undefined);++ renderConversation();++ await waitFor(() => {+ expect(resumeCloudSandbox).toHaveBeenCalled();+ });++ await waitFor(() => {+ expect(resumeCloudSandbox).toHaveBeenCalledTimes(2);+ });+ expect(resumeCloudSandbox).toHaveBeenNthCalledWith(2, "sandbox-paused-2");+ });++ it("does not retry or toast when a pending cloud resume fails after unmount", async () => {+ setActiveSelection({ backendId: cloudBackend.id, orgId: "org-a" });+ vi.mocked(+ AgentServerConversationService.batchGetAppConversations,+ ).mockResolvedValue([+ makeConversation(CLOUD_CONVERSATION_ID, {+ sandbox_status: "PAUSED",+ sandbox_id: "sandbox-paused-unmounted",+ }),+ ]);+ let rejectResume!: (reason?: unknown) => void;+ vi.mocked(resumeCloudSandbox).mockReturnValueOnce(+ new Promise<void>((_, reject) => {+ rejectResume = reject;+ }),+ );++ const { unmount } = renderConversation();++ await waitFor(() => {+ expect(resumeCloudSandbox).toHaveBeenCalledTimes(1);+ });++ unmount();+ await act(async () => {+ rejectResume(new Error("resume failed after unmount"));+ await Promise.resolve();+ });++ expect(displayErrorToast).not.toHaveBeenCalled();++ await new Promise((resolve) => {+ window.setTimeout(resolve, 20);+ });+ expect(resumeCloudSandbox).toHaveBeenCalledTimes(1);+ });+});+describe("conversation route β shared read-only fallback", () => {it("sends an org member to the read-only shared view when only the shared lookup can see the conversation", async () => {// Arrange β the owner lookup misses on cloud, but the conversation isdiff --git a/src/api/cloud/cloud-sandbox-resume-suppression.ts b/src/api/cloud/cloud-sandbox-resume-suppression.tsnew file mode 100644index 0000000000..407e3b957e--- /dev/null+++ b/src/api/cloud/cloud-sandbox-resume-suppression.ts@@ -0,0 +1,19 @@+const suppressedConversationIds = new Set<string>();++export function suppressNextCloudAutoResume(conversationId: string): void {+ suppressedConversationIds.add(conversationId);+}++export function consumeCloudAutoResumeSuppression(+ conversationId: string,+): boolean {+ return suppressedConversationIds.delete(conversationId);+}++export function clearCloudAutoResumeSuppression(conversationId: string): void {+ suppressedConversationIds.delete(conversationId);+}++export function __clearCloudAutoResumeSuppressionsForTests(): void {+ suppressedConversationIds.clear();+}diff --git a/src/hooks/mutation/use-cloud-sandbox-auto-resume.ts b/src/hooks/mutation/use-cloud-sandbox-auto-resume.tsnew file mode 100644index 0000000000..2e1bab0fee--- /dev/null+++ b/src/hooks/mutation/use-cloud-sandbox-auto-resume.ts@@ -0,0 +1,134 @@+import { useEffect, useRef } from "react";+import { useMutation } from "@tanstack/react-query";+import { useTranslation } from "react-i18next";+import { resumeCloudSandbox } from "#/api/cloud/conversation-service.api";+import {+ clearCloudAutoResumeSuppression,+ consumeCloudAutoResumeSuppression,+} from "#/api/cloud/cloud-sandbox-resume-suppression";+import type { AppConversation } from "#/api/conversation-service/agent-server-conversation-service.types";+import type { BackendKind } from "#/api/backend-registry/types";+import { I18nKey } from "#/i18n/declaration";+import { displayErrorToast } from "#/utils/custom-toast-handlers";++const CLOUD_RESUME_RETRY_DELAY_MS =+ import.meta.env.MODE === "test" ? 10 : 10_000;++type ResumeVariables = {+ resumeKey: string;+ sandboxId: string;+};++type ResumeAttemptState = "pending" | "succeeded";++type UseCloudSandboxAutoResumeOptions = {+ backendChanged: boolean;+ backendKind: BackendKind;+ conversation: AppConversation | null | undefined;+ conversationId: string;+ isFetched: boolean;+};++export function useCloudSandboxAutoResume({+ backendChanged,+ backendKind,+ conversation,+ conversationId,+ isFetched,+}: UseCloudSandboxAutoResumeOptions): void {+ const { t } = useTranslation("openhands");+ const activeResumeKeyRef = useRef<string | null>(null);+ const resumeAttemptRef = useRef<{+ key: string;+ state: ResumeAttemptState;+ } | null>(null);++ // TanStack can schedule a retry after the route has moved on; keep the+ // mutation function as the final guard so stale retries become no-ops.+ const { mutate: resumeSandbox } = useMutation({+ mutationFn: async ({ resumeKey, sandboxId }: ResumeVariables) => {+ if (activeResumeKeyRef.current !== resumeKey) return;++ try {+ await resumeCloudSandbox(sandboxId);+ } catch (error) {+ if (activeResumeKeyRef.current === resumeKey) {+ displayErrorToast(t(I18nKey.CONVERSATION$FAILED_TO_START_FROM_TASK));+ }+ throw error;+ }+ },+ retry: () => activeResumeKeyRef.current !== null,+ retryDelay: () => CLOUD_RESUME_RETRY_DELAY_MS,+ onSuccess: (_, variables) => {+ if (activeResumeKeyRef.current === variables.resumeKey) {+ resumeAttemptRef.current = {+ key: variables.resumeKey,+ state: "succeeded",+ };+ }+ },+ meta: { disableToast: true },+ });++ useEffect(+ () => () => {+ clearCloudAutoResumeSuppression(conversationId);+ },+ [conversationId],+ );++ useEffect(() => {+ if (+ backendChanged ||+ !isFetched ||+ !conversation ||+ backendKind !== "cloud" ||+ conversation.sandbox_status !== "PAUSED" ||+ !conversation.sandbox_id+ ) {+ activeResumeKeyRef.current = null;+ return;+ }++ if (consumeCloudAutoResumeSuppression(conversation.id)) {+ activeResumeKeyRef.current = null;+ return;+ }++ const resumeKey = `${conversation.id}:${conversation.sandbox_id}`;+ activeResumeKeyRef.current = resumeKey;++ const currentAttempt = resumeAttemptRef.current;+ if (+ currentAttempt?.key === resumeKey &&+ (currentAttempt.state === "pending" ||+ currentAttempt.state === "succeeded")+ ) {+ return;+ }++ resumeAttemptRef.current = { key: resumeKey, state: "pending" };+ resumeSandbox({ resumeKey, sandboxId: conversation.sandbox_id });++ return () => {+ if (activeResumeKeyRef.current === resumeKey) {+ activeResumeKeyRef.current = null;+ }+ if (+ resumeAttemptRef.current?.key === resumeKey &&+ resumeAttemptRef.current.state !== "succeeded"+ ) {+ resumeAttemptRef.current = null;+ }+ };+ }, [+ backendChanged,+ backendKind,+ conversation?.id,+ conversation?.sandbox_status,+ conversation?.sandbox_id,+ isFetched,+ resumeSandbox,+ ]);+}diff --git a/src/hooks/mutation/use-pause-conversation.ts b/src/hooks/mutation/use-pause-conversation.tsindex bdc4725afe..9db79dd494 100644--- a/src/hooks/mutation/use-pause-conversation.ts+++ b/src/hooks/mutation/use-pause-conversation.ts@@ -1,8 +1,15 @@import { useMutation, useQueryClient } from "@tanstack/react-query";-import { pauseConversation } from "./conversation-mutation-utils";+import { ExecutionStatus } from "#/types/agent-server/core";+import { useActiveBackend } from "#/contexts/active-backend-context";+import { suppressNextCloudAutoResume } from "#/api/cloud/cloud-sandbox-resume-suppression";+import {+ pauseConversation,+ patchConversationInCache,+} from "./conversation-mutation-utils";export const usePauseConversation = () => {const queryClient = useQueryClient();+ const { backend } = useActiveBackend();return useMutation({mutationFn: (variables: { conversationId: string }) =>@@ -24,6 +31,15 @@ export const usePauseConversation = () => {);}},+ onSuccess: (_, variables) => {+ if (backend.kind !== "cloud") return;++ suppressNextCloudAutoResume(variables.conversationId);+ patchConversationInCache(queryClient, variables.conversationId, {+ execution_status: ExecutionStatus.PAUSED,+ conversation_url: null,+ });+ },onSettled: (_, __, variables) => {// Invalidate the specific conversation query to trigger automatic refetchqueryClient.invalidateQueries({diff --git a/src/hooks/mutation/use-unified-stop-conversation.ts b/src/hooks/mutation/use-unified-stop-conversation.tsindex 61b304f45a..574a977225 100644--- a/src/hooks/mutation/use-unified-stop-conversation.ts+++ b/src/hooks/mutation/use-unified-stop-conversation.ts@@ -55,14 +55,13 @@ export const useUnifiedPauseConversation = () => {}toast.success(t(I18nKey.TOAST$CONVERSATION_STOPPED), TOAST_OPTIONS);- // Update both execution_status and sandbox_status together so that- // WebSocketProviderWrapper's sandbox_status === "PAUSED" gate fires- // immediately when the user reopens this conversation β preventing a- // WebSocket connection attempt against the now-paused sandbox host- // before the next useActiveConversation poll returns.+ // Drop the runtime URL immediately so this mounted route cannot reconnect+ // to the sandbox being paused. Leave sandbox_status server-authored: the+ // conversation route uses PAUSED as the signal to auto-resume when a user+ // later reopens an already-paused cloud conversation.patchConversationInCache(queryClient, variables.conversationId, {execution_status: ExecutionStatus.PAUSED,- sandbox_status: "PAUSED",+ conversation_url: null,});if (currentConversationId === variables.conversationId) {diff --git a/src/routes/conversation.tsx b/src/routes/conversation.tsxindex 9dfcc65cf8..7dcf247e74 100644--- a/src/routes/conversation.tsx+++ b/src/routes/conversation.tsx@@ -30,7 +30,7 @@ import { ConversationOverviewDrawerProvider } from "#/components/features/converimport { WebSocketProviderWrapper } from "#/contexts/websocket-provider-wrapper";import { useErrorMessageStore } from "#/stores/error-message-store";import { I18nKey } from "#/i18n/declaration";-import { resumeCloudSandbox } from "#/api/cloud/conversation-service.api";+import { useCloudSandboxAutoResume } from "#/hooks/mutation/use-cloud-sandbox-auto-resume";function AppContent() {const { t } = useTranslation("openhands");@@ -162,42 +162,13 @@ function AppContent() {setLastConversationId(active.backend.id, active.orgId, conversationId);}, [conversationId, backendChanged, active.backend.id, active.orgId]);- // Cloud conversation resume: mirrors OpenHands' useSandboxRecovery.- //- // When the cloud API reports sandbox_status === "PAUSED" the sandbox is- // sleeping. The correct wake-up call is POST /api/v1/sandboxes/{id}/resume- // (a lightweight unpause). The previous approach β creating a new start task- // via POST /api/v1/app-conversations β was wrong: it tries to provision a- // fresh conversation in the sandbox and is subject to a 120-second cold-start- // timeout that can fail. The resume endpoint simply unpauses the existing one.- //- // After calling resume we stay on the current URL. The 3-second refetch- // interval in useActiveConversation (active while conversation_url is null)- // polls until conversation_url populates, then the WebSocket connects.- //- // A ref guards against duplicate triggers per unique conversation.id within- // the same route-mount lifetime.- const resumeTriggeredForRef = React.useRef<string | null>(null);- React.useEffect(() => {- if (!isFetched || !conversation) return;- if (active.backend.kind !== "cloud") return;- if (conversation.sandbox_status !== "PAUSED") return; // only resume PAUSED sandboxes- if (!conversation.sandbox_id) return; // no sandbox to resume- if (resumeTriggeredForRef.current === conversation.id) return; // already sent-- resumeTriggeredForRef.current = conversation.id;-- resumeCloudSandbox(conversation.sandbox_id).catch(() => {- displayErrorToast(t(I18nKey.CONVERSATION$FAILED_TO_START_FROM_TASK));- });- }, [+ useCloudSandboxAutoResume({+ backendChanged,+ backendKind: active.backend.kind,+ conversation,+ conversationId,isFetched,- conversation?.id,- conversation?.sandbox_status,- conversation?.sandbox_id,- active.backend.kind,- t,- ]);+ });// A backend switch is in flight (BackendSelector flips the active backend// and redirects to /conversations on the next tick). The conversationId in