Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat: surface sandbox start-failure reason in conversation start errors (#14885)

Alona234b819be8e4
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
index e174b6e847..acbc9b0a22 100644
--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py
+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
@@ -209,6 +209,12 @@ The selected repository was cloned as a shallow clone. Git history may be incomp
</GIT_WORKSPACE_CONTEXT>"""
+def _exception_detail(exc: Exception) -> str:
+ """HTTPException.__str__ prepends '<status>: '; prefer its clean .detail."""
+ detail = getattr(exc, 'detail', None)
+ return detail if isinstance(detail, str) else str(exc)
+
+
def append_system_context(existing: str | None, block: str) -> str:
if not existing:
return block
@@ -629,7 +635,9 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
except Exception as exc:
_logger.exception('Error starting conversation', stack_info=True)
task.status = AppConversationStartTaskStatus.ERROR
- task.detail = redact_text_secrets(redact_api_key_literals(str(exc)))
+ task.detail = redact_text_secrets(
+ redact_api_key_literals(_exception_detail(exc))
+ )
yield task
async def _build_app_conversations(
diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.py
index a1f5b3dbd9..0481bcd2be 100644
--- a/openhands/app_server/sandbox/remote_sandbox_service.py
+++ b/openhands/app_server/sandbox/remote_sandbox_service.py
@@ -205,6 +205,7 @@ class RemoteSandboxService(SandboxService):
session_api_key=session_api_key,
exposed_urls=exposed_urls,
created_at=stored.created_at,
+ status_detail=runtime.get('status_detail') if runtime else None,
)
def _get_sandbox_status_from_runtime(
diff --git a/openhands/app_server/sandbox/sandbox_models.py b/openhands/app_server/sandbox/sandbox_models.py
index e399d3ec78..f008868e11 100644
--- a/openhands/app_server/sandbox/sandbox_models.py
+++ b/openhands/app_server/sandbox/sandbox_models.py
@@ -69,6 +69,13 @@ class SandboxInfo(BaseModel):
),
)
created_at: datetime = Field(default_factory=utc_now)
+ status_detail: str | None = Field(
+ default=None,
+ description=(
+ 'Last pod/scheduling reason from the runtime (e.g. insufficient kvm, '
+ 'ImagePullBackOff), surfaced when a sandbox is stuck or errored.'
+ ),
+ )
class SandboxPage(BaseModel):
diff --git a/openhands/app_server/sandbox/sandbox_service.py b/openhands/app_server/sandbox/sandbox_service.py
index 3fc0655865..f99dc3d69e 100644
--- a/openhands/app_server/sandbox/sandbox_service.py
+++ b/openhands/app_server/sandbox/sandbox_service.py
@@ -26,6 +26,88 @@ SESSION_API_KEY_VARIABLE = 'OH_SESSION_API_KEYS_0'
WEBHOOK_CALLBACK_VARIABLE = 'OH_WEBHOOKS_0_BASE_URL'
ALLOW_CORS_ORIGINS_VARIABLE = 'OH_ALLOW_CORS_ORIGINS_0'
+# Known start-failure classes we translate into short, user-safe messages. Raw
+# runtime status_detail (k8s pod/scheduling text) can leak internal registry
+# hosts, secret/configmap names, node taints/labels, cluster size, etc., so it
+# is logged for debugging but never surfaced to end users. Only the standard
+# resource names below are echoed back; extended/device resources are not (the
+# name can reveal the device plugin in use).
+_CAPACITY_MARKERS = (
+ 'Insufficient cpu',
+ 'Insufficient memory',
+ 'Insufficient ephemeral-storage',
+ 'Insufficient pods',
+)
+_IMAGE_PULL_MARKERS = ('ImagePullBackOff', 'ErrImagePull')
+# Deterministic placement failures -- retrying won't help. Substrings match the
+# scheduler's predicate messages without echoing specific taint/label values.
+_SCHEDULING_MARKERS = (
+ 'untolerated taint',
+ 'node affinity',
+ 'node selector',
+ 'volume node affinity conflict',
+ 'topology spread',
+ 'unbound', # e.g. "unbound immediate PersistentVolumeClaims"
+)
+
+_GENERIC_START_FAILURE = (
+ 'The sandbox failed to start for an unexpected reason. Please try again.'
+)
+
+
+def _classify_start_failure(detail: str | None) -> str | None:
+ """Translate a raw runtime status_detail into a user-safe message.
+
+ Describes the failure class and likely cause so an owner knows where to
+ look, but never echoes the raw detail (it can leak registry hosts, secret
+ names, taints, cluster size). Returns None when nothing safe can be said,
+ so the caller falls back to a generic message; the raw detail is logged.
+ """
+ if not detail:
+ return None
+ if any(m in detail for m in _IMAGE_PULL_MARKERS):
+ return (
+ 'The sandbox image could not be pulled. The image tag may be missing, '
+ 'the registry credentials may be invalid, or the registry may be '
+ 'unreachable.'
+ )
+ if 'CreateContainerConfigError' in detail:
+ return (
+ 'The sandbox failed to start due to a configuration issue (e.g. a '
+ 'missing secret or config value).'
+ )
+ if 'CreateContainerError' in detail:
+ return (
+ 'The sandbox container could not be created, possibly due to an '
+ 'invalid mount, device, or startup command.'
+ )
+ resource = next((m for m in _CAPACITY_MARKERS if m in detail), None)
+ if resource:
+ return (
+ f'The system is at capacity right now ({resource.lower()}). '
+ 'Please try again in a few minutes.'
+ )
+ if any(m in detail for m in _SCHEDULING_MARKERS):
+ return (
+ 'The sandbox could not be scheduled onto any available node. No node '
+ 'met its placement requirements (node affinity/selector, taints, or '
+ 'volume constraints).'
+ )
+ if 'Insufficient ' in detail:
+ return (
+ 'The sandbox could not be scheduled because a required device or '
+ 'resource is currently unavailable.'
+ )
+ return None
+
+
+def _start_failure_error(sandbox_id: str, detail: str | None) -> SandboxError:
+ """Build the user-facing start-failure error: a safe, descriptive message
+ tagged with the sandbox id so an owner can find the raw reason in the logs.
+ """
+ message = _classify_start_failure(detail) or _GENERIC_START_FAILURE
+ return SandboxError(f'{message} (reference: {sandbox_id})')
+
class SandboxService(ABC):
"""Service for accessing sandboxes in which conversations may be run."""
@@ -117,13 +199,19 @@ class SandboxService(ABC):
SandboxError: If sandbox not found, enters ERROR state, or times out
"""
start = time.time()
+ sandbox: SandboxInfo | None = None
while time.time() - start <= timeout:
sandbox = await self.get_sandbox(sandbox_id)
if sandbox is None:
raise SandboxError(f'Sandbox not found: {sandbox_id}')
if sandbox.status == SandboxStatus.ERROR:
- raise SandboxError(f'Sandbox entered error state: {sandbox_id}')
+ _logger.warning(
+ 'Sandbox %s entered error state; status_detail=%r',
+ sandbox_id,
+ sandbox.status_detail,
+ )
+ raise _start_failure_error(sandbox_id, sandbox.status_detail)
if sandbox.status == SandboxStatus.RUNNING:
# Optionally verify agent server is alive to avoid race conditions
@@ -137,7 +225,14 @@ class SandboxService(ABC):
await asyncio.sleep(poll_interval)
- raise SandboxError(f'Sandbox failed to start within {timeout}s: {sandbox_id}')
+ status_detail = sandbox.status_detail if sandbox is not None else None
+ _logger.warning(
+ 'Sandbox %s did not start within %ss; status_detail=%r',
+ sandbox_id,
+ timeout,
+ status_detail,
+ )
+ raise _start_failure_error(sandbox_id, status_detail)
async def _check_agent_server_alive(
self, sandbox: SandboxInfo, httpx_client: httpx.AsyncClient
diff --git a/tests/unit/app_server/test_live_status_app_conversation_service.py b/tests/unit/app_server/test_live_status_app_conversation_service.py
index de8b0b330e..3f8edfe521 100644
--- a/tests/unit/app_server/test_live_status_app_conversation_service.py
+++ b/tests/unit/app_server/test_live_status_app_conversation_service.py
@@ -32,8 +32,10 @@ from openhands.app_server.app_conversation.app_conversation_service import (
)
from openhands.app_server.app_conversation.live_status_app_conversation_service import (
LiveStatusAppConversationService,
+ _exception_detail,
effective_disabled_skills,
)
+from openhands.app_server.errors import SandboxError
from openhands.app_server.integrations.provider import ProviderToken, ProviderType
from openhands.app_server.integrations.service_types import SuggestedTask, TaskType
from openhands.app_server.sandbox.sandbox_models import (
@@ -4501,3 +4503,12 @@ class TestBuildAcpStartConversationRequestSecrets:
'conversation_id': str(request.conversation_id),
'agent_kind': 'acp',
}
+
+
+def test_exception_detail_strips_http_status_prefix():
+ """SandboxError.__str__ carries a '500: ' prefix; task.detail should not."""
+ assert (
+ _exception_detail(SandboxError('The system is at capacity right now.'))
+ == 'The system is at capacity right now.'
+ )
+ assert _exception_detail(ValueError('boom')) == 'boom'
diff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.py
index 8c8cb6fcff..1e8c1ac330 100644
--- a/tests/unit/app_server/test_remote_sandbox_service.py
+++ b/tests/unit/app_server/test_remote_sandbox_service.py
@@ -444,6 +444,34 @@ class TestSandboxInfoConversion:
assert sandbox_info.session_api_key == 'test-session-key'
assert sandbox_info.exposed_urls is None
+ @pytest.mark.asyncio
+ async def test_to_sandbox_info_carries_status_detail(self, remote_sandbox_service):
+ """status_detail from the runtime payload flows onto SandboxInfo."""
+ runtime_data = create_runtime_data(status='starting')
+ runtime_data['status_detail'] = (
+ '0/1 nodes are available: 1 Insufficient smarter-devices/kvm.'
+ )
+ sandbox_info = remote_sandbox_service._to_sandbox_info(
+ create_stored_sandbox(), runtime_data
+ )
+ assert sandbox_info.status_detail == (
+ '0/1 nodes are available: 1 Insufficient smarter-devices/kvm.'
+ )
+
+ @pytest.mark.asyncio
+ async def test_to_sandbox_info_status_detail_defaults_none(
+ self, remote_sandbox_service
+ ):
+ """Absent status_detail (and no runtime) -> None, no crash."""
+ stored = create_stored_sandbox()
+ assert (
+ remote_sandbox_service._to_sandbox_info(stored, None).status_detail is None
+ )
+ running = remote_sandbox_service._to_sandbox_info(
+ stored, create_runtime_data(status='running')
+ )
+ assert running.status_detail is None
+
@pytest.mark.asyncio
async def test_to_sandbox_info_loads_runtime_when_none_provided(
self, remote_sandbox_service
diff --git a/tests/unit/app_server/test_sandbox_service.py b/tests/unit/app_server/test_sandbox_service.py
index f5464196a3..afa15f9e7f 100644
--- a/tests/unit/app_server/test_sandbox_service.py
+++ b/tests/unit/app_server/test_sandbox_service.py
@@ -13,13 +13,18 @@ from unittest.mock import AsyncMock
import pytest
+from openhands.app_server.errors import SandboxError
from openhands.app_server.sandbox.sandbox_models import (
SandboxInfo,
SandboxPage,
SandboxRecord,
SandboxStatus,
)
-from openhands.app_server.sandbox.sandbox_service import SandboxService
+from openhands.app_server.sandbox.sandbox_service import (
+ SandboxService,
+ _classify_start_failure,
+ _start_failure_error,
+)
class MockSandboxService(SandboxService):
@@ -92,6 +97,201 @@ def mock_sandbox_service():
return MockSandboxService()
+class TestWaitForSandboxRunning:
+ """wait_for_sandbox_running maps runtime status_detail to user-safe errors.
+
+ Raw status_detail is logged, not surfaced: only classified/generic text
+ reaches the user, so internal infra details (registry hosts, secret names,
+ node taints/labels, cluster size, ...) can't leak.
+ """
+
+ def _sandbox(self, status, detail):
+ return SandboxInfo(
+ id='sb1',
+ created_by_user_id=None,
+ sandbox_spec_id='test-spec',
+ status=status,
+ session_api_key=None,
+ created_at=datetime.now(timezone.utc),
+ status_detail=detail,
+ )
+
+ def _patch_timeout(self, monkeypatch):
+ calls = {'n': 0}
+
+ def fake_time():
+ calls['n'] += 1
+ return 1000.0 if calls['n'] <= 3 else 2000.0
+
+ monkeypatch.setattr(
+ 'openhands.app_server.sandbox.sandbox_service.time.time', fake_time
+ )
+ monkeypatch.setattr(
+ 'openhands.app_server.sandbox.sandbox_service.asyncio.sleep', AsyncMock()
+ )
+
+ @pytest.mark.asyncio
+ async def test_capacity_error_state_is_friendly_and_trimmed(
+ self, mock_sandbox_service
+ ):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR,
+ '0/1 nodes are available: 1 Insufficient ephemeral-storage. '
+ 'preemption: 0/1 nodes are available.',
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'at capacity' in msg
+ assert 'insufficient ephemeral-storage' in msg
+ # reference id lets an owner find the raw reason in the logs
+ assert 'reference: sb1' in msg
+ # node counts / preemption noise are not leaked
+ assert '0/1 nodes' not in msg
+ assert 'preemption' not in msg
+ assert 'entered error state' not in msg
+
+ @pytest.mark.asyncio
+ async def test_capacity_at_timeout_is_retryable(
+ self, mock_sandbox_service, monkeypatch
+ ):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.STARTING, '0/1 nodes are available: 1 Insufficient cpu.'
+ )
+ self._patch_timeout(monkeypatch)
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=1)
+ msg = str(ei.value)
+ assert 'at capacity' in msg and 'try again' in msg
+ assert 'insufficient cpu' in msg
+ assert 'failed to start within' not in msg
+ assert mock_sandbox_service.get_sandbox_mock.call_count > 1
+
+ @pytest.mark.asyncio
+ async def test_image_pull_detail_is_redacted(self, mock_sandbox_service):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR,
+ 'ImagePullBackOff: Back-off pulling image '
+ '"registry.internal.corp/team/secret-project:tag"',
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'could not be pulled' in msg
+ assert 'registry.internal.corp' not in msg
+ assert 'secret-project' not in msg
+
+ @pytest.mark.asyncio
+ async def test_config_error_detail_is_redacted(self, mock_sandbox_service):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR,
+ "CreateContainerConfigError: couldn't find key db-password "
+ 'in secret "prod-db-creds"',
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'configuration issue' in msg
+ assert 'prod-db-creds' not in msg
+ assert 'db-password' not in msg
+
+ @pytest.mark.asyncio
+ async def test_container_create_error_is_distinct_and_redacted(
+ self, mock_sandbox_service
+ ):
+ # CreateContainerError is broader than a missing secret: don't mislabel
+ # it as a config issue, and don't leak the path.
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR,
+ 'CreateContainerError: failed to mount /var/secrets/prod-token',
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'could not be created' in msg
+ assert 'configuration issue' not in msg
+ assert 'prod-token' not in msg
+
+ @pytest.mark.asyncio
+ async def test_scheduling_constraint_is_redacted(self, mock_sandbox_service):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR,
+ 'node(s) had untolerated taint {dedicated: internal-gpu-pool}',
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'could not be scheduled' in msg
+ assert 'internal-gpu-pool' not in msg
+ assert 'dedicated' not in msg
+ # deterministic placement failure -> no misleading 'try again'
+ assert 'try again' not in msg
+
+ @pytest.mark.asyncio
+ async def test_device_resource_shortage_is_redacted(self, mock_sandbox_service):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR,
+ '0/3 nodes are available: 3 Insufficient smarter-devices/kvm.',
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'device or resource' in msg
+ # the device-plugin name must not leak
+ assert 'smarter-devices' not in msg
+ assert 'kvm' not in msg
+
+ @pytest.mark.asyncio
+ async def test_unrecognized_detail_is_generic(self, mock_sandbox_service):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR, 'the sandbox imploded for reasons unknown'
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ msg = str(ei.value)
+ assert 'unexpected reason' in msg
+ assert 'imploded' not in msg
+
+ @pytest.mark.asyncio
+ async def test_no_detail_is_generic(self, mock_sandbox_service):
+ mock_sandbox_service.get_sandbox_mock.return_value = self._sandbox(
+ SandboxStatus.ERROR, None
+ )
+ with pytest.raises(SandboxError) as ei:
+ await mock_sandbox_service.wait_for_sandbox_running('sb1', timeout=5)
+ assert 'failed to start' in str(ei.value)
+
+
+def test_classify_start_failure_maps_each_class():
+ assert 'at capacity' in _classify_start_failure('x Insufficient memory y')
+ assert 'insufficient memory' in _classify_start_failure('x Insufficient memory y')
+ assert 'could not be pulled' in _classify_start_failure('ImagePullBackOff: x')
+ assert 'configuration issue' in _classify_start_failure(
+ 'CreateContainerConfigError: x'
+ )
+ assert 'could not be created' in _classify_start_failure('CreateContainerError: x')
+ assert 'could not be scheduled' in _classify_start_failure(
+ "0/2 nodes: 2 node(s) didn't match Pod's node affinity/selector."
+ )
+ assert 'device or resource' in _classify_start_failure(
+ '0/1 nodes: 1 Insufficient nvidia.com/gpu.'
+ )
+ # unrecognized / empty -> None so the caller uses a generic message
+ assert _classify_start_failure('some novel reason') is None
+ assert _classify_start_failure(None) is None
+ assert _classify_start_failure('') is None
+
+
+def test_start_failure_error_appends_reference():
+ err = _start_failure_error('sb-xyz', 'ImagePullBackOff: secret/path')
+ assert 'could not be pulled' in str(err)
+ assert '(reference: sb-xyz)' in str(err)
+ # unknown detail still gets the generic message + the reference id
+ generic = _start_failure_error('sb-xyz', 'mystery')
+ assert 'unexpected reason' in str(generic)
+ assert '(reference: sb-xyz)' in str(generic)
+
+
class TestCleanupOldSandboxes:
"""Test cases for the pause_old_sandboxes method."""