π OpenHands: AI-Driven Development
feat: surface sandbox start-failure reason in conversation start errors (#14885)
Alona234b819be8e4
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.pyindex e174b6e847..acbc9b0a22 100644--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py@@ -209,6 +209,12 @@ The selected repository was cloned as a shallow clone. Git history may be incomp</GIT_WORKSPACE_CONTEXT>"""+def _exception_detail(exc: Exception) -> str:+ """HTTPException.__str__ prepends '<status>: '; prefer its clean .detail."""+ detail = getattr(exc, 'detail', None)+ return detail if isinstance(detail, str) else str(exc)++def append_system_context(existing: str | None, block: str) -> str:if not existing:return block@@ -629,7 +635,9 @@ class LiveStatusAppConversationService(AppConversationServiceBase):except Exception as exc:_logger.exception('Error starting conversation', stack_info=True)task.status = AppConversationStartTaskStatus.ERROR- task.detail = redact_text_secrets(redact_api_key_literals(str(exc)))+ task.detail = redact_text_secrets(+ redact_api_key_literals(_exception_detail(exc))+ )yield taskasync def _build_app_conversations(diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.pyindex a1f5b3dbd9..0481bcd2be 100644--- a/openhands/app_server/sandbox/remote_sandbox_service.py+++ b/openhands/app_server/sandbox/remote_sandbox_service.py@@ -205,6 +205,7 @@ class RemoteSandboxService(SandboxService):session_api_key=session_api_key,exposed_urls=exposed_urls,created_at=stored.created_at,+ status_detail=runtime.get('status_detail') if runtime else None,)def _get_sandbox_status_from_runtime(diff --git a/openhands/app_server/sandbox/sandbox_models.py b/openhands/app_server/sandbox/sandbox_models.pyindex e399d3ec78..f008868e11 100644--- a/openhands/app_server/sandbox/sandbox_models.py+++ b/openhands/app_server/sandbox/sandbox_models.py@@ -69,6 +69,13 @@ class SandboxInfo(BaseModel):),)created_at: datetime = Field(default_factory=utc_now)+ status_detail: str | None = Field(+ default=None,+ description=(+ 'Last pod/scheduling reason from the runtime (e.g. insufficient kvm, '+ 'ImagePullBackOff), surfaced when a sandbox is stuck or errored.'+ ),+ )class SandboxPage(BaseModel):diff --git a/openhands/app_server/sandbox/sandbox_service.py b/openhands/app_server/sandbox/sandbox_service.pyindex 3fc0655865..f99dc3d69e 100644--- a/openhands/app_server/sandbox/sandbox_service.py+++ b/openhands/app_server/sandbox/sandbox_service.py@@ -26,6 +26,88 @@ SESSION_API_KEY_VARIABLE = 'OH_SESSION_API_KEYS_0'WEBHOOK_CALLBACK_VARIABLE = 'OH_WEBHOOKS_0_BASE_URL'ALLOW_CORS_ORIGINS_VARIABLE = 'OH_ALLOW_CORS_ORIGINS_0'+# Known start-failure classes we translate into short, user-safe messages. Raw+# runtime status_detail (k8s pod/scheduling text) can leak internal registry+# hosts, secret/configmap names, node taints/labels, cluster size, etc., so it+# is logged for debugging but never surfaced to end users. Only the standard+# resource names below are echoed back; extended/device resources are not (the+# name can reveal the device plugin in use).+_CAPACITY_MARKERS = (+ 'Insufficient cpu',+ 'Insufficient memory',+ 'Insufficient ephemeral-storage',+ 'Insufficient pods',+)+_IMAGE_PULL_MARKERS = ('ImagePullBackOff', 'ErrImagePull')+# Deterministic placement failures -- retrying won't help. Substrings match the+# scheduler's predicate messages without echoing specific taint/label values.+_SCHEDULING_MARKERS = (+ 'untolerated taint',+ 'node affinity',+ 'node selector',+ 'volume node affinity conflict',+ 'topology spread',+ 'unbound', # e.g. "unbound immediate PersistentVolumeClaims"+)++_GENERIC_START_FAILURE = (+ 'The sandbox failed to start for an unexpected reason. Please try again.'+)+++def _classify_start_failure(detail: str | None) -> str | None:+ """Translate a raw runtime status_detail into a user-safe message.++ Describes the failure class and likely cause so an owner knows where to+ look, but never echoes the raw detail (it can leak registry hosts, secret+ names, taints, cluster size). Returns None when nothing safe can be said,+ so the caller falls back to a generic message; the raw detail is logged.+ """+ if not detail:+ return None+ if any(m in detail for m in _IMAGE_PULL_MARKERS):+ return (+ 'The sandbox image could not be pulled. The image tag may be missing, '+ 'the registry credentials may be invalid, or the registry may be '+ 'unreachable.'+ )+ if 'CreateContainerConfigError' in detail:+ return (+ 'The sandbox failed to start due to a configuration issue (e.g. a '+ 'missing secret or config value).'+ )+ if 'CreateContainerError' in detail:+ return (+ 'The sandbox container could not be created, possibly due to an '+ 'invalid mount, device, or startup command.'+ )+ resource = next((m for m in _CAPACITY_MARKERS if m in detail), None)+ if resource:+ return (+ f'The system is at capacity right now ({resource.lower()}). '+ 'Please try again in a few minutes.'+ )+ if any(m in detail for m in _SCHEDULING_MARKERS):+ return (+ 'The sandbox could not be scheduled onto any available node. No node '+ 'met its placement requirements (node affinity/selector, taints, or '+ 'volume constraints).'+ )+ if 'Insufficient ' in detail:+ return (+ 'The sandbox could not be scheduled because a required device or '+ 'resource is currently unavailable.'+ )+ return None+++def _start_failure_error(sandbox_id: str, detail: str | None) -> SandboxError:+ """Build the user-facing start-failure error: a safe, descriptive message+ tagged with the sandbox id so an owner can find the raw reason in the logs.+ """+ message = _classify_start_failure(detail) or _GENERIC_START_FAILURE+ return SandboxError(f'{message} (reference: {sandbox_id})')+class SandboxService(ABC):"""Service for accessing sandboxes in which conversations may be run."""@@ -117,13 +199,19 @@ class SandboxService(ABC):SandboxError: If sandbox not found, enters ERROR state, or times out"""start = time.time()+ sandbox: SandboxInfo | None = Nonewhile time.time() - start <= timeout:sandbox = await self.get_sandbox(sandbox_id)if sandbox is None:raise SandboxError(f'Sandbox not found: {sandbox_id}')if sandbox.status == SandboxStatus.ERROR:- raise SandboxError(f'Sandbox entered error state: {sandbox_id}')+ _logger.warning(+ 'Sandbox %s entered error state; status_detail=%r',+ sandbox_id,+ sandbox.status_detail,+ )+ raise _start_failure_error(sandbox_id, sandbox.status_detail)if sandbox.status == SandboxStatus.RUNNING:# Optionally verify agent server is alive to avoid race conditions@@ -137,7 +225,14 @@ class SandboxService(ABC):await asyncio.sleep(poll_interval)- raise SandboxError(f'Sandbox failed to start within {timeout}s: {sandbox_id}')+ status_detail = sandbox.status_detail if sandbox is not None else None+ _logger.warning(+ 'Sandbox %s did not start within %ss; status_detail=%r',+ sandbox_id,+ timeout,+ status_detail,+ )+ raise _start_failure_error(sandbox_id, status_detail)async def _check_agent_server_alive(self, sandbox: SandboxInfo, httpx_client: httpx.AsyncClientdiff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.pyindex de8b0b330e..3f8edfe521 100644--- a/tests/unit/app_server/test_live_status_app_conversation_service.py+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py@@ -32,8 +32,10 @@ from openhands.app_server.app_conversation.app_conversation_service import ()from openhands.app_server.app_conversation.live_status_app_conversation_service import (LiveStatusAppConversationService,+ _exception_detail,effective_disabled_skills,)+from openhands.app_server.errors import SandboxErrorfrom openhands.app_server.integrations.provider import ProviderToken, ProviderTypefrom openhands.app_server.integrations.service_types import SuggestedTask, TaskTypefrom openhands.app_server.sandbox.sandbox_models import (@@ -4501,3 +4503,12 @@ class TestBuildAcpStartConversationRequestSecrets:'conversation_id': str(request.conversation_id),'agent_kind': 'acp',}+++def test_exception_detail_strips_http_status_prefix():+ """SandboxError.__str__ carries a '500: ' prefix; task.detail should not."""+ assert (+ _exception_detail(SandboxError('The system is at capacity right now.'))+ == 'The system is at capacity right now.'+ )+ assert _exception_detail(ValueError('boom')) == 'boom'diff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.pyindex 8c8cb6fcff..1e8c1ac330 100644--- a/tests/unit/app_server/test_remote_sandbox_service.py+++ b/tests/unit/app_server/test_remote_sandbox_service.py@@ -444,6 +444,34 @@ class TestSandboxInfoConversion:assert sandbox_info.session_api_key == 'test-session-key'assert sandbox_info.exposed_urls is None+ @pytest.mark.asyncio+ async def test_to_sandbox_info_carries_status_detail(self, remote_sandbox_service):+ """status_detail from the runtime payload flows onto SandboxInfo."""+ runtime_data = create_runtime_data(status='starting')+ runtime_data['status_detail'] = (+ '0/1 nodes are available: 1 Insufficient smarter-devices/kvm.'+ )+ sandbox_info = remote_sandbox_service._to_sandbox_info(+ create_stored_sandbox(), runtime_data+ )+ assert sandbox_info.status_detail == (+ '0/1 nodes are available: 1 Insufficient smarter-devices/kvm.'+ )++ @pytest.mark.asyncio+ async def test_to_sandbox_info_status_detail_defaults_none(+ self, remote_sandbox_service+ ):+ """Absent status_detail (and no runtime) -> None, no crash."""+ stored = create_stored_sandbox()+ assert (+ remote_sandbox_service._to_sandbox_info(stored, None).status_detail is None+ )+ running = remote_sandbox_service._to_sandbox_info(+ stored, create_runtime_data(status='running')+ )+ assert running.status_detail is None+@pytest.mark.asyncioasync def test_to_sandbox_info_loads_runtime_when_none_provided(self, remote_sandbox_servicediff --git a/tests/unit/app_server/test_sandbox_service.py b/tests/unit/app_server/test_sandbox_service.pyindex f5464196a3..afa15f9e7f 100644--- a/tests/unit/app_server/test_sandbox_service.py+++ b/tests/unit/app_server/test_sandbox_service.py@@ -13,13 +13,18 @@ from unittest.mock import AsyncMockimport pytest+from openhands.app_server.errors import SandboxErrorfrom openhands.app_server.sandbox.sandbox_models import (SandboxInfo,SandboxPage,SandboxRecord,SandboxStatus,)-from openhands.app_server.sandbox.sandbox_service import SandboxService+from openhands.app_server.sandbox.sandbox_service import (+ SandboxService,+ _classify_start_failure,+ _start_failure_error,+)class MockSandboxService(SandboxService):@@ -92,6 +97,201 @@ def mock_sandbox_service():return MockSandboxService()+class TestWaitForSandboxRunning:+ """wait_for_sandbox_running maps runtime status_detail to user-safe errors.++ Raw status_detail is logged, not surfaced: only classified/generic text+ reaches the user, so internal infra details (registry hosts, secret names,+ node taints/labels, cluster size, ...) can't leak.+ """++ def _sandbox(self, status, detail):+ return SandboxInfo(+ id='sb1',+ created_by_user_id=None,+ sandbox_spec_id='test-spec',+ status=status,+ session_api_key=None,+ created_at=datetime.now(timezone.utc),+ status_detail=detail,+ )++ def _patch_timeout(self, monkeypatch):+ calls = {'n': 0}++ def fake_time():+ calls['n'] += 1+ return 1000.0 if calls['n'] <= 3 else 2000.0++ monkeypatch.setattr(+ 'openhands.app_server.sandbox.sandbox_service.time.time', fake_time+ )+ monkeypatch.setattr(+ 'openhands.app_server.sandbox.sandbox_service.asyncio.sleep', AsyncMock()+ )++ @pytest.mark.asyncio+ async def test_capacity_error_state_is_friendly_and_trimmed(+ self, mock_sandbox_service+ ):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR,+ '0/1 nodes are available: 1 Insufficient ephemeral-storage. '+ 'preemption: 0/1 nodes are available.',+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'at capacity' in msg+ assert 'insufficient ephemeral-storage' in msg+ # reference id lets an owner find the raw reason in the logs+ assert 'reference: sb1' in msg+ # node counts / preemption noise are not leaked+ assert '0/1 nodes' not in msg+ assert 'preemption' not in msg+ assert 'entered error state' not in msg++ @pytest.mark.asyncio+ async def test_capacity_at_timeout_is_retryable(+ self, mock_sandbox_service, monkeypatch+ ):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.STARTING, '0/1 nodes are available: 1 Insufficient cpu.'+ )+ self._patch_timeout(monkeypatch)+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=1)+ msg = str(ei.value)+ assert 'at capacity' in msg and 'try again' in msg+ assert 'insufficient cpu' in msg+ assert 'failed to start within' not in msg+ assert mock_sandbox_service.get_sandbox_mock.call_count > 1++ @pytest.mark.asyncio+ async def test_image_pull_detail_is_redacted(self, mock_sandbox_service):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR,+ 'ImagePullBackOff: Back-off pulling image '+ '"registry.internal.corp/team/secret-project:tag"',+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'could not be pulled' in msg+ assert 'registry.internal.corp' not in msg+ assert 'secret-project' not in msg++ @pytest.mark.asyncio+ async def test_config_error_detail_is_redacted(self, mock_sandbox_service):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR,+ "CreateContainerConfigError: couldn't find key db-password "+ 'in secret "prod-db-creds"',+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'configuration issue' in msg+ assert 'prod-db-creds' not in msg+ assert 'db-password' not in msg++ @pytest.mark.asyncio+ async def test_container_create_error_is_distinct_and_redacted(+ self, mock_sandbox_service+ ):+ # CreateContainerError is broader than a missing secret: don't mislabel+ # it as a config issue, and don't leak the path.+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR,+ 'CreateContainerError: failed to mount /var/secrets/prod-token',+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'could not be created' in msg+ assert 'configuration issue' not in msg+ assert 'prod-token' not in msg++ @pytest.mark.asyncio+ async def test_scheduling_constraint_is_redacted(self, mock_sandbox_service):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR,+ 'node(s) had untolerated taint {dedicated: internal-gpu-pool}',+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'could not be scheduled' in msg+ assert 'internal-gpu-pool' not in msg+ assert 'dedicated' not in msg+ # deterministic placement failure -> no misleading 'try again'+ assert 'try again' not in msg++ @pytest.mark.asyncio+ async def test_device_resource_shortage_is_redacted(self, mock_sandbox_service):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR,+ '0/3 nodes are available: 3 Insufficient smarter-devices/kvm.',+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'device or resource' in msg+ # the device-plugin name must not leak+ assert 'smarter-devices' not in msg+ assert 'kvm' not in msg++ @pytest.mark.asyncio+ async def test_unrecognized_detail_is_generic(self, mock_sandbox_service):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR, 'the sandbox imploded for reasons unknown'+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ msg = str(ei.value)+ assert 'unexpected reason' in msg+ assert 'imploded' not in msg++ @pytest.mark.asyncio+ async def test_no_detail_is_generic(self, mock_sandbox_service):+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(+ SandboxStatus.ERROR, None+ )+ with pytest.raises(SandboxError) as ei:+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)+ assert 'failed to start' in str(ei.value)+++def test_classify_start_failure_maps_each_class():+ assert 'at capacity' in _classify_start_failure('x Insufficient memory y')+ assert 'insufficient memory' in _classify_start_failure('x Insufficient memory y')+ assert 'could not be pulled' in _classify_start_failure('ImagePullBackOff: x')+ assert 'configuration issue' in _classify_start_failure(+ 'CreateContainerConfigError: x'+ )+ assert 'could not be created' in _classify_start_failure('CreateContainerError: x')+ assert 'could not be scheduled' in _classify_start_failure(+ "0/2 nodes: 2 node(s) didn't match Pod's node affinity/selector."+ )+ assert 'device or resource' in _classify_start_failure(+ '0/1 nodes: 1 Insufficient nvidia.com/gpu.'+ )+ # unrecognized / empty -> None so the caller uses a generic message+ assert _classify_start_failure('some novel reason') is None+ assert _classify_start_failure(None) is None+ assert _classify_start_failure('') is None+++def test_start_failure_error_appends_reference():+ err = _start_failure_error('sb-xyz', 'ImagePullBackOff: secret/path')+ assert 'could not be pulled' in str(err)+ assert '(reference: sb-xyz)' in str(err)+ # unknown detail still gets the generic message + the reference id+ generic = _start_failure_error('sb-xyz', 'mystery')+ assert 'unexpected reason' in str(generic)+ assert '(reference: sb-xyz)' in str(generic)++class TestCleanupOldSandboxes:"""Test cases for the pause_old_sandboxes method."""