Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: retry idempotent runtime-api reads once on timeout (#15266)

Alona9c5d4dce546e
diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.py
index 19d34e60a8..27b10461a6 100644
--- a/openhands/app_server/sandbox/remote_sandbox_service.py
+++ b/openhands/app_server/sandbox/remote_sandbox_service.py
@@ -127,17 +127,28 @@ class RemoteSandboxService(SandboxService):
self, method: str, path: str, **kwargs: Any
) -> httpx.Response:
"""Send a request to the remote runtime API."""
- try:
- url = self.api_url + path
- return await self.httpx_client.request(
- method, url, headers={'X-API-Key': self.api_key}, **kwargs
- )
- except httpx.TimeoutException:
- _logger.error(f'No response received within timeout for URL: {url}')
- raise
- except httpx.HTTPError as e:
- _logger.error(f'HTTP error for URL {url}: {e}')
- raise
+ url = self.api_url + path
+ # A stalled runtime-api request (e.g. dead pooled DB connection on its
+ # side) times out once, while a retry on a fresh connection succeeds β€”
+ # so retry idempotent reads once instead of failing conversation start.
+ attempts = 2 if method in ('GET', 'HEAD') else 1
+ last_exc: httpx.TimeoutException | None = None
+ for attempt in range(attempts):
+ try:
+ return await self.httpx_client.request(
+ method, url, headers={'X-API-Key': self.api_key}, **kwargs
+ )
+ except httpx.TimeoutException as e:
+ last_exc = e
+ if attempt + 1 < attempts:
+ _logger.warning(f'Timeout for URL {url}; retrying')
+ continue
+ _logger.error(f'No response received within timeout for URL: {url}')
+ raise
+ except httpx.HTTPError as e:
+ _logger.error(f'HTTP error for URL {url}: {e}')
+ raise
+ raise last_exc # type: ignore[misc] # unreachable; keeps mypy happy
def _to_sandbox_info(
self, stored: StoredRemoteSandbox, runtime: dict[str, Any] | None = None
diff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.py
index ebfbddfbd9..8c8cb6fcff 100644
--- a/tests/unit/app_server/test_remote_sandbox_service.py
+++ b/tests/unit/app_server/test_remote_sandbox_service.py
@@ -211,6 +211,42 @@ class TestRemoteSandboxService:
with pytest.raises(httpx.TimeoutException):
await remote_sandbox_service._send_runtime_api_request('GET', '/test')
+ @pytest.mark.asyncio
+ async def test_send_runtime_api_request_get_retries_once_on_timeout(
+ self, remote_sandbox_service
+ ):
+ """A timed-out idempotent read is retried once and can succeed."""
+ # Setup
+ mock_response = MagicMock()
+ remote_sandbox_service.httpx_client.request.side_effect = [
+ httpx.TimeoutException('Request timeout'),
+ mock_response,
+ ]
+
+ # Execute
+ response = await remote_sandbox_service._send_runtime_api_request(
+ 'GET', '/test'
+ )
+
+ # Verify
+ assert response == mock_response
+ assert remote_sandbox_service.httpx_client.request.call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_send_runtime_api_request_post_does_not_retry(
+ self, remote_sandbox_service
+ ):
+ """Non-idempotent requests are not retried on timeout."""
+ # Setup
+ remote_sandbox_service.httpx_client.request.side_effect = (
+ httpx.TimeoutException('Request timeout')
+ )
+
+ # Execute & Verify
+ with pytest.raises(httpx.TimeoutException):
+ await remote_sandbox_service._send_runtime_api_request('POST', '/start')
+ assert remote_sandbox_service.httpx_client.request.call_count == 1
+
@pytest.mark.asyncio
async def test_send_runtime_api_request_http_error(self, remote_sandbox_service):
"""Test API request HTTP error handling."""