π OpenHands: AI-Driven Development
fix: retry idempotent runtime-api reads once on timeout (#15266)
Alona9c5d4dce546e
diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.pyindex 19d34e60a8..27b10461a6 100644--- a/openhands/app_server/sandbox/remote_sandbox_service.py+++ b/openhands/app_server/sandbox/remote_sandbox_service.py@@ -127,17 +127,28 @@ class RemoteSandboxService(SandboxService):self, method: str, path: str, **kwargs: Any) -> httpx.Response:"""Send a request to the remote runtime API."""- try:- url = self.api_url + path- return await self.httpx_client.request(- method, url, headers={'X-API-Key': self.api_key}, **kwargs- )- except httpx.TimeoutException:- _logger.error(f'No response received within timeout for URL: {url}')- raise- except httpx.HTTPError as e:- _logger.error(f'HTTP error for URL {url}: {e}')- raise+ url = self.api_url + path+ # A stalled runtime-api request (e.g. dead pooled DB connection on its+ # side) times out once, while a retry on a fresh connection succeeds β+ # so retry idempotent reads once instead of failing conversation start.+ attempts = 2 if method in ('GET', 'HEAD') else 1+ last_exc: httpx.TimeoutException | None = None+ for attempt in range(attempts):+ try:+ return await self.httpx_client.request(+ method, url, headers={'X-API-Key': self.api_key}, **kwargs+ )+ except httpx.TimeoutException as e:+ last_exc = e+ if attempt + 1 < attempts:+ _logger.warning(f'Timeout for URL {url}; retrying')+ continue+ _logger.error(f'No response received within timeout for URL: {url}')+ raise+ except httpx.HTTPError as e:+ _logger.error(f'HTTP error for URL {url}: {e}')+ raise+ raise last_exc # type: ignore[misc] # unreachable; keeps mypy happydef _to_sandbox_info(self, stored: StoredRemoteSandbox, runtime: dict[str, Any] | None = Nonediff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.pyindex ebfbddfbd9..8c8cb6fcff 100644--- a/tests/unit/app_server/test_remote_sandbox_service.py+++ b/tests/unit/app_server/test_remote_sandbox_service.py@@ -211,6 +211,42 @@ class TestRemoteSandboxService:with pytest.raises(httpx.TimeoutException):await remote_sandbox_service._send_runtime_api_request('GET', '/test')+ @pytest.mark.asyncio+ async def test_send_runtime_api_request_get_retries_once_on_timeout(+ self, remote_sandbox_service+ ):+ """A timed-out idempotent read is retried once and can succeed."""+ # Setup+ mock_response = MagicMock()+ remote_sandbox_service.httpx_client.request.side_effect = [+ httpx.TimeoutException('Request timeout'),+ mock_response,+ ]++ # Execute+ response = await remote_sandbox_service._send_runtime_api_request(+ 'GET', '/test'+ )++ # Verify+ assert response == mock_response+ assert remote_sandbox_service.httpx_client.request.call_count == 2++ @pytest.mark.asyncio+ async def test_send_runtime_api_request_post_does_not_retry(+ self, remote_sandbox_service+ ):+ """Non-idempotent requests are not retried on timeout."""+ # Setup+ remote_sandbox_service.httpx_client.request.side_effect = (+ httpx.TimeoutException('Request timeout')+ )++ # Execute & Verify+ with pytest.raises(httpx.TimeoutException):+ await remote_sandbox_service._send_runtime_api_request('POST', '/start')+ assert remote_sandbox_service.httpx_client.request.call_count == 1+@pytest.mark.asyncioasync def test_send_runtime_api_request_http_error(self, remote_sandbox_service):"""Test API request HTTP error handling."""