π OpenHands: AI-Driven Development
fix: PLTF-2956 batch_get_sandboxes to gracefully handle runtime API failures (#14853)
aivong-openhands8757aa2187fe
diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.pyindex 9ff8cc10b8..dc44c53a73 100644--- a/openhands/app_server/sandbox/remote_sandbox_service.py+++ b/openhands/app_server/sandbox/remote_sandbox_service.py@@ -669,7 +669,11 @@ class RemoteSandboxService(SandboxService):async def batch_get_sandboxes(self, sandbox_ids: list[str]) -> list[SandboxInfo | None]:- """Get a batch of sandboxes, returning None for any which were not found."""+ """Get a batch of sandboxes, returning None for any which were not found.++ Falls back to returning sandboxes with missing/unknown runtime status if the+ runtime API is unavailable, rather than failing the entire batch request.+ """if not sandbox_ids:return []query = await self._secure_select()@@ -679,9 +683,20 @@ class RemoteSandboxService(SandboxService):stored_remote_sandbox[0].id: stored_remote_sandbox[0]for stored_remote_sandbox in stored_remote_sandboxes}- runtimes_by_id = await self._get_runtimes_batch(- list(stored_remote_sandboxes_by_id)- )++ # Gracefully handle runtime API failures by falling back to empty runtimes.+ # This mirrors the behavior of get_sandbox which falls back to runtime=None.+ try:+ runtimes_by_id = await self._get_runtimes_batch(+ list(stored_remote_sandboxes_by_id)+ )+ except Exception:+ _logger.exception(+ 'Error getting runtimes batch, falling back to empty runtimes',+ stack_info=True,+ )+ runtimes_by_id = {}+results = []for sandbox_id in sandbox_ids:stored_remote_sandbox = stored_remote_sandboxes_by_id.get(sandbox_id)diff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.pyindex 2d470d6e3c..93f7e6f1a0 100644--- a/tests/unit/app_server/test_remote_sandbox_service.py+++ b/tests/unit/app_server/test_remote_sandbox_service.py@@ -1763,3 +1763,159 @@ class TestPollAgentServersSessionScoping:'expected refresh_conversation to open a write session')assert tracker.open == 0+++class TestBatchGetSandboxes:+ """Test cases for batch_get_sandboxes method."""++ @pytest.mark.asyncio+ async def test_batch_get_sandboxes_success(self, remote_sandbox_service):+ """Test successful batch retrieval of sandboxes."""+ # Setup+ sandbox_ids = ['sandbox-1', 'sandbox-2']+ stored_sandbox_1 = create_stored_sandbox(sandbox_id='sandbox-1')+ stored_sandbox_2 = create_stored_sandbox(sandbox_id='sandbox-2')+ runtime_1 = create_runtime_data(session_id='sandbox-1', status='running')++ # Mock DB query result+ mock_result = MagicMock()+ mock_result.__iter__ = MagicMock(+ return_value=iter([(stored_sandbox_1,), (stored_sandbox_2,)])+ )+ remote_sandbox_service.db_session.execute = AsyncMock(return_value=mock_result)++ # Mock successful runtime batch response+ remote_sandbox_service._get_runtimes_batch = AsyncMock(+ return_value={'sandbox-1': runtime_1}+ )++ # Execute+ results = await remote_sandbox_service.batch_get_sandboxes(sandbox_ids)++ # Verify+ assert len(results) == 2+ assert results[0] is not None+ assert results[0].id == 'sandbox-1'+ assert results[0].status == SandboxStatus.RUNNING+ assert results[1] is not None+ assert results[1].id == 'sandbox-2'+ # sandbox-2 has no runtime, so it's marked as MISSING+ assert results[1].status == SandboxStatus.MISSING++ @pytest.mark.asyncio+ async def test_batch_get_sandboxes_empty_ids(self, remote_sandbox_service):+ """Test batch retrieval with empty sandbox IDs list."""+ # Execute+ results = await remote_sandbox_service.batch_get_sandboxes([])++ # Verify+ assert results == []++ @pytest.mark.asyncio+ async def test_batch_get_sandboxes_graceful_fallback_on_timeout(+ self, remote_sandbox_service+ ):+ """Test that batch_get_sandboxes gracefully handles runtime API timeout.++ This is the key regression test: when the runtime API times out,+ batch_get_sandboxes should not raise but should return sandboxes+ with MISSING status (matching the behavior of get_sandbox).+ """+ # Setup+ sandbox_ids = ['sandbox-1', 'sandbox-2']+ stored_sandbox_1 = create_stored_sandbox(sandbox_id='sandbox-1')+ stored_sandbox_2 = create_stored_sandbox(sandbox_id='sandbox-2')++ # Mock DB query result+ mock_result = MagicMock()+ mock_result.__iter__ = MagicMock(+ return_value=iter([(stored_sandbox_1,), (stored_sandbox_2,)])+ )+ remote_sandbox_service.db_session.execute = AsyncMock(return_value=mock_result)++ # Mock runtime API timeout+ remote_sandbox_service._get_runtimes_batch = AsyncMock(+ side_effect=httpx.TimeoutException('Request timeout')+ )++ # Execute - should NOT raise, should gracefully fall back+ results = await remote_sandbox_service.batch_get_sandboxes(sandbox_ids)++ # Verify - all sandboxes should be returned with MISSING status+ assert len(results) == 2+ assert results[0] is not None+ assert results[0].id == 'sandbox-1'+ assert results[0].status == SandboxStatus.MISSING+ assert results[1] is not None+ assert results[1].id == 'sandbox-2'+ assert results[1].status == SandboxStatus.MISSING++ @pytest.mark.asyncio+ async def test_batch_get_sandboxes_graceful_fallback_on_http_error(+ self, remote_sandbox_service+ ):+ """Test that batch_get_sandboxes gracefully handles runtime API HTTP error.++ When the runtime API returns an error (e.g., 500 Internal Server Error),+ batch_get_sandboxes should not raise but should return sandboxes+ with MISSING status.+ """+ # Setup+ sandbox_ids = ['sandbox-1']+ stored_sandbox_1 = create_stored_sandbox(sandbox_id='sandbox-1')++ # Mock DB query result+ mock_result = MagicMock()+ mock_result.__iter__ = MagicMock(return_value=iter([(stored_sandbox_1,)]))+ remote_sandbox_service.db_session.execute = AsyncMock(return_value=mock_result)++ # Mock runtime API HTTP error+ remote_sandbox_service._get_runtimes_batch = AsyncMock(+ side_effect=httpx.HTTPError('Internal server error')+ )++ # Execute - should NOT raise, should gracefully fall back+ results = await remote_sandbox_service.batch_get_sandboxes(sandbox_ids)++ # Verify - sandbox should be returned with MISSING status+ assert len(results) == 1+ assert results[0] is not None+ assert results[0].id == 'sandbox-1'+ assert results[0].status == SandboxStatus.MISSING++ @pytest.mark.asyncio+ async def test_batch_get_sandboxes_graceful_fallback_on_raise_for_status(+ self, remote_sandbox_service+ ):+ """Test graceful fallback when raise_for_status() fails in _get_runtimes_batch.++ The _get_runtimes_batch method calls response.raise_for_status().+ When this raises an httpx.HTTPStatusError, batch_get_sandboxes should+ catch it and fall back gracefully.+ """+ # Setup+ sandbox_ids = ['sandbox-1']+ stored_sandbox_1 = create_stored_sandbox(sandbox_id='sandbox-1')++ # Mock DB query result+ mock_result = MagicMock()+ mock_result.__iter__ = MagicMock(return_value=iter([(stored_sandbox_1,)]))+ remote_sandbox_service.db_session.execute = AsyncMock(return_value=mock_result)++ # Mock HTTP status error from raise_for_status()+ remote_sandbox_service._get_runtimes_batch = AsyncMock(+ side_effect=httpx.HTTPStatusError(+ message='500 Internal Server Error',+ request=MagicMock(),+ response=MagicMock(status_code=500),+ )+ )++ # Execute - should NOT raise, should gracefully fall back+ results = await remote_sandbox_service.batch_get_sandboxes(sandbox_ids)++ # Verify - sandbox should be returned with MISSING status+ assert len(results) == 1+ assert results[0] is not None+ assert results[0].id == 'sandbox-1'+ assert results[0].status == SandboxStatus.MISSING