π OpenHands: AI-Driven Development
fix(app_server): derive agent server image from package version (#15168)
Tim O'Farrell20a721e23b05
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.pyindex 77d09c7e37..cf56283496 100644--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py@@ -3,7 +3,6 @@ import importlib.metadataimport ioimport jsonimport logging-import osimport zipfilefrom collections import defaultdictfrom collections.abc import Mapping@@ -14,7 +13,6 @@ from uuid import UUID, uuid4import httpxfrom fastapi import Request-from packaging.version import InvalidVersion, Versionfrom pydantic import Field, SecretStr, TypeAdapterfrom openhands.agent_server.models import (@@ -92,8 +90,7 @@ from openhands.app_server.sandbox.sandbox_models import (from openhands.app_server.sandbox.sandbox_service import SandboxServicefrom openhands.app_server.sandbox.sandbox_spec_service import (SandboxSpecService,- get_agent_server_image,- is_custom_agent_server_image,+ is_custom_sandbox_spec,)from openhands.app_server.services.injector import InjectorStatefrom openhands.app_server.services.jwt_service import JwtService@@ -383,12 +380,6 @@ class LiveStatusAppConversationService(AppConversationServiceBase):assert sandbox is not Noneagent_server_url = self._get_agent_server_url(sandbox)- # Custom sandbox images can ship an incompatible openhands-sdk; fail- # fast with a clear error instead of an opaque 500 on create.- await self._verify_agent_server_version(- agent_server_url, sandbox.session_api_key- )-# Mirror the user's LLM profiles into the sandbox so the agent's# built-in switch_llm tool can resolve them (in SaaS profiles live# on the app-server, not the sandbox filesystem). Before conversation@@ -482,12 +473,12 @@ class LiveStatusAppConversationService(AppConversationServiceBase):except httpx.HTTPStatusError as exc:# A custom image that 500s on create is usually an openhands-sdk# mismatch /server_info couldn't reveal; add an actionable hint.- if is_custom_agent_server_image():+ if is_custom_sandbox_spec(sandbox.sandbox_spec_id):expected = _expected_sdk_version()raise SandboxError(f'Conversation create failed (HTTP 'f'{exc.response.status_code}) on custom sandbox image '- f'{get_agent_server_image()}. Verify its openhands-sdk '+ f'{sandbox.sandbox_spec_id}. Verify its openhands-sdk 'f'matches this release'+ (f' ({expected})' if expected else '')+ '; rebuild/re-pin the image if not.'@@ -912,57 +903,6 @@ class LiveStatusAppConversationService(AppConversationServiceBase):httpx_client=self.httpx_client,)- async def _verify_agent_server_version(- self, agent_server_url: str, session_api_key: str | None- ) -> None:- """Fail fast with a clear error when an admin-pinned custom sandbox image- runs a different openhands-sdk minor than this app, instead of the opaque- 500 the agent-server returns on create. Best-effort: only custom images are- checked, and we fail open on anything we can't read."""- if os.getenv('OH_SKIP_AGENT_SERVER_VERSION_CHECK', '').strip().lower() in (- '1',- 'true',- 'yes',- ):- return- # Proxy-default images move with the release; only custom-pinned can drift.- if not is_custom_agent_server_image():- return- expected = _expected_sdk_version()- if not expected:- return- try:- headers = {'X-Session-API-Key': session_api_key} if session_api_key else {}- resp = await self.httpx_client.get(- f'{agent_server_url.rstrip("/")}/server_info',- headers=headers,- timeout=30.0,- )- resp.raise_for_status()- reported = str(resp.json().get('sdk_version', '')).strip()- except Exception:- # 404 (image predates /server_info) or transient errors: can't verify,- # so don't block β the create POST still surfaces a custom-image hint.- _logger.warning(- 'Could not read /server_info to verify agent-server SDK version',- exc_info=True,- )- return- # Endpoint present but metadata missing -> nothing to compare against.- if reported in ('', 'unknown'):- return- try:- if Version(reported).release[:2] == Version(expected).release[:2]:- return- except InvalidVersion:- return- raise SandboxError(- f'Sandbox image {get_agent_server_image()} runs openhands-sdk '- f'{reported}, but this release requires {expected}. Rebuild/re-pin the '- 'custom sandbox image to a matching openhands-sdk, or set '- 'OH_SKIP_AGENT_SERVER_VERSION_CHECK=1 to bypass.'- )-async def _seed_sandbox_profiles(self, agent_server_url: str, session_api_key: str | None) -> None:diff --git a/openhands/app_server/sandbox/sandbox_spec_service.py b/openhands/app_server/sandbox/sandbox_spec_service.pyindex 79cb0c9ff0..c9b5dd326e 100644--- a/openhands/app_server/sandbox/sandbox_spec_service.py+++ b/openhands/app_server/sandbox/sandbox_spec_service.py@@ -1,7 +1,9 @@import asyncio+import importlibimport loggingimport osfrom abc import ABC, abstractmethod+from functools import cachefrom openhands.agent_server import env_parserfrom openhands.app_server.errors import SandboxError@@ -12,9 +14,25 @@ from openhands.app_server.sandbox.sandbox_spec_models import (from openhands.app_server.services.injector import Injectorfrom openhands.sdk.utils.models import DiscriminatedUnionMixin-# The version of the agent server to use for deployments.-# Typically this will be the same as the values from the pyproject.toml-AGENT_SERVER_IMAGE = 'ghcr.io/openhands/agent-server:1.32.0-python'+_DEFAULT_REPOSITORY = 'ghcr.io/openhands/agent-server'+++@cache+def _bundled_agent_server_version() -> str:+ """Version of the installed openhands-agent-server package.++ openhands-agent-server is a hard runtime dependency of the V1 app server;+ if it's not installed the V1 app server has no business starting. Let+ PackageNotFoundError surface at first call rather than degrading silently+ to a wrong default. Memoised via ``functools.cache`` so the lookup runs+ once per process and remains mockable in tests.+ """+ return importlib.metadata.version('openhands-agent-server')+++def _bundled_default_image() -> str:+ """Image URL of the bundled agent-server: ``<repo>:<version>-python``."""+ return f'{_DEFAULT_REPOSITORY}:{_bundled_agent_server_version()}-python'class SandboxSpecService(ABC):@@ -98,19 +116,34 @@ async def resolve_sandbox_spec(raise ValueError(f'Sandbox Spec {effective_id!r} not found')+@cache # memoize so the auto-correct warning fires once per processdef get_agent_server_image() -> str:- agent_server_image_repository = os.getenv('AGENT_SERVER_IMAGE_REPOSITORY')- agent_server_image_tag = os.getenv('AGENT_SERVER_IMAGE_TAG')- if agent_server_image_repository and agent_server_image_tag:- return f'{agent_server_image_repository}:{agent_server_image_tag}'- return AGENT_SERVER_IMAGE---def is_custom_agent_server_image() -> bool:- """True only when an admin pinned a custom sandbox image (tag differs from the- release-default tag). Default/upgrade installs keep the release tag, never gated."""- tag = os.getenv('AGENT_SERVER_IMAGE_TAG')- return bool(tag) and tag != AGENT_SERVER_IMAGE.rsplit(':', 1)[-1]+ repository = os.getenv('AGENT_SERVER_IMAGE_REPOSITORY') or _DEFAULT_REPOSITORY+ bundled_version = _bundled_agent_server_version()+ tag = os.getenv('AGENT_SERVER_IMAGE_TAG') or f'{bundled_version}-python'++ if repository == _DEFAULT_REPOSITORY:+ parts = tag.split('-', 1)+ if len(parts) == 2 and parts[0] != bundled_version:+ parts[0] = bundled_version+ updated_tag = '-'.join(parts)+ logging.getLogger(__name__).warning(+ f'AGENT_SERVER_IMAGE_TAG={tag} does not match the installed '+ f'openhands-sdk (using {updated_tag} instead). Pin to a custom '+ f'image repository via AGENT_SERVER_IMAGE_REPOSITORY if you '+ f'need to use a different build.'+ )+ tag = updated_tag++ return f'{repository}:{tag}'+++def is_custom_sandbox_spec(sandbox_spec_id: str) -> bool:+ """True when the sandbox spec differs from the bundled release default (e.g. a+ custom image registered via runtime-api). The bundled spec always matches the+ SDK the app server ships with, so non-default specs are the only ones worth+ flagging on a sandbox-create failure."""+ return sandbox_spec_id != _bundled_default_image()# Prefixes for environment variables that should be auto-forwarded to agent-serverdiff --git a/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py b/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.pyindex ece6a93500..0448c6cd85 100644--- a/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py+++ b/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py@@ -193,6 +193,37 @@ class TestFetchSpecs:# is the intended signal for get_default_sandbox_spec to fall back too.assert service._name_to_spec == {}+ async def test_empty_configs_fallback_uses_bundled_default_not_env_override(+ self,+ ):+ """Regression test for the staging 500.++ When runtime-api returns empty configs, the fallback spec's image id+ must equal the bundled default (whatever ``get_agent_server_image()``+ currently returns), NOT whatever the legacy AGENT_SERVER_IMAGE_TAG env+ var happens to say. Otherwise an admin pinning a stale tag would+ cause a sandbox-create 500.+ """+ # Pin the bundled default to a specific image, regardless of what's+ # installed in the test env, so we can assert against it directly.+ # Patch via the module attribute (not patch.object on the function+ # itself) because the @cache decorator wraps __call__ and intercepts+ # the standard mock hook.+ expected_image = 'ghcr.io/openhands/agent-server:9.9.9-python'++ ctx, _ = _make_async_client_mock(_make_http_response({'configs': []}))+ service = _make_service()++ with patch(+ 'openhands.app_server.sandbox.remote_sandbox_spec_service.get_agent_server_image',+ return_value=expected_image,+ ):+ with patch('httpx.AsyncClient', return_value=ctx):+ specs = await service._fetch_specs()++ assert len(specs) == 1+ assert specs[0].id == expected_image+async def test_fallback_specs_are_cached(self):"""Default fallback specs must be cached like normal results."""from openhands.app_server.sandbox.remote_sandbox_spec_service import (diff --git a/tests/unit/app_server/test_sandbox_spec_service.py b/tests/unit/app_server/test_sandbox_spec_service.pynew file mode 100644index 0000000000..764cc02762--- /dev/null+++ b/tests/unit/app_server/test_sandbox_spec_service.py@@ -0,0 +1,181 @@+"""Tests for sandbox_spec_service helpers.++Covers ``get_agent_server_image`` (derived from the installed+``openhands-agent-server`` package, with auto-correction of stale canonical+tags for self-hosted installs) and ``is_custom_sandbox_spec`` (which compares+a sandbox spec id against the bundled default).+"""++import importlib.metadata+from unittest.mock import patch++import pytest++from openhands.app_server.sandbox.sandbox_spec_service import (+ _bundled_agent_server_version,+ get_agent_server_image,+ is_custom_sandbox_spec,+)++[email protected](autouse=True)+def _clear_caches():+ """All the @cache'd helpers in this module need their caches cleared around+ every test so we don't leak env-var or mocked-version state between cases.+ (_bundled_default_image isn't @cache'd; it just calls _bundled_agent_server_version.)"""+ get_agent_server_image.cache_clear()+ _bundled_agent_server_version.cache_clear()+ yield+ get_agent_server_image.cache_clear()+ _bundled_agent_server_version.cache_clear()+++def test_get_agent_server_image_derived_from_package_version():+ """The URL must be built from the installed openhands-agent-server version,+ not a hand-maintained constant β that's the whole point of removing the+ drift-prone AGENT_SERVER_IMAGE string."""+ fake_version = '9.9.9'+ with patch.object(+ importlib.metadata,+ 'version',+ return_value=fake_version,+ ):+ assert get_agent_server_image() == (+ f'ghcr.io/openhands/agent-server:{fake_version}-python'+ )+++def test_get_agent_server_image_returns_consistent_value_within_process():+ """``@cache`` should make repeat calls return the same object without+ re-reading importlib.metadata or re-checking env vars."""+ with patch.object(+ importlib.metadata,+ 'version',+ return_value='1.0.0',+ ) as version_mock:+ first = get_agent_server_image()+ # Call many times β version_mock should only be hit once thanks to @cache.+ for _ in range(5):+ assert get_agent_server_image() is first+ assert version_mock.call_count == 1+++def test_get_agent_server_image_honours_custom_repository():+ """Self-hosted customers can pin to their own image repository via+ AGENT_SERVER_IMAGE_REPOSITORY; the result must be honoured verbatim,+ including a non-default tag."""+ fake_version = '1.0.0'+ with patch.object(importlib.metadata, 'version', return_value=fake_version):+ with patch.dict(+ 'os.environ',+ {+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'example.com/agent-server',+ 'AGENT_SERVER_IMAGE_TAG': '9.9.9-python',+ },+ clear=False,+ ):+ assert get_agent_server_image() == 'example.com/agent-server:9.9.9-python'+++def test_get_agent_server_image_auto_corrects_stale_canonical_tag():+ """Regression test for the staging 500.++ A canonical-repo + stale ``-python`` tag (the exact staging failure mode)+ must be silently re-pinned to the SDK-matching tag, so self-hosted+ installs keep working when they ship a stale AGENT_SERVER_IMAGE_TAG.+ """+ fake_version = '9.9.9'+ with patch.object(importlib.metadata, 'version', return_value=fake_version):+ with patch.dict(+ 'os.environ',+ {+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'ghcr.io/openhands/agent-server',+ 'AGENT_SERVER_IMAGE_TAG': '1.31.1-python',+ },+ clear=False,+ ):+ assert get_agent_server_image() == (+ f'ghcr.io/openhands/agent-server:{fake_version}-python'+ )+++def test_get_agent_server_image_passes_through_canonical_tag_without_suffix():+ """Tags without a ``-`` separator (e.g. ``1.32.0``, ``nightly``) must not+ be silently rewritten by the auto-correct heuristic β that would lose+ legitimate canonical tags. The post-hoc 500-hint will still flag any+ actual SDK mismatch downstream."""+ fake_version = '9.9.9'+ with patch.object(importlib.metadata, 'version', return_value=fake_version):+ with patch.dict(+ 'os.environ',+ {+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'ghcr.io/openhands/agent-server',+ 'AGENT_SERVER_IMAGE_TAG': 'nightly',+ },+ clear=False,+ ):+ assert get_agent_server_image() == (+ 'ghcr.io/openhands/agent-server:nightly'+ )+++def test_get_agent_server_image_passes_through_canonical_repo_with_matching_tag():+ """Canonical repo + already-matching tag must be a no-op (no warning, no+ rewrite) β the most common case for fresh installs."""+ fake_version = '1.2.3'+ with patch.object(importlib.metadata, 'version', return_value=fake_version):+ with patch.dict(+ 'os.environ',+ {+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'ghcr.io/openhands/agent-server',+ 'AGENT_SERVER_IMAGE_TAG': f'{fake_version}-python',+ },+ clear=False,+ ):+ assert get_agent_server_image() == (+ f'ghcr.io/openhands/agent-server:{fake_version}-python'+ )+++def test_is_custom_sandbox_spec_false_for_bundled_default():+ """A spec id equal to the bundled default is, by definition, not custom."""+ with patch.object(importlib.metadata, 'version', return_value='1.0.0'):+ bundled = get_agent_server_image()+ assert is_custom_sandbox_spec(bundled) is False+++def test_is_custom_sandbox_spec_true_for_runtime_api_image():+ """A spec id from runtime-api (any non-default image) must be flagged custom."""+ with patch.object(importlib.metadata, 'version', return_value='1.0.0'):+ assert is_custom_sandbox_spec('ghcr.io/some/custom:0.0.1') is True+++def test_is_custom_sandbox_spec_true_for_self_hosted_custom_repo():+ """A self-hosted customer who set AGENT_SERVER_IMAGE_REPOSITORY to a+ custom repo still has a non-default image; the post-hoc 500-hint must+ fire for them when their image's SDK doesn't match."""+ fake_version = '1.0.0'+ with patch.object(importlib.metadata, 'version', return_value=fake_version):+ with patch.dict(+ 'os.environ',+ {+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'example.com/agent-server',+ 'AGENT_SERVER_IMAGE_TAG': '9.9.9-python',+ },+ clear=False,+ ):+ assert (+ is_custom_sandbox_spec('example.com/agent-server:9.9.9-python') is True+ )+++def test_get_agent_server_image_propagates_package_not_found():+ """openhands-agent-server is a hard runtime dependency; a missing install+ must surface as PackageNotFoundError at first call, not silently degrade."""+ with patch.object(+ importlib.metadata,+ 'version',+ side_effect=importlib.metadata.PackageNotFoundError('openhands-agent-server'),+ ):+ with pytest.raises(importlib.metadata.PackageNotFoundError):+ get_agent_server_image()diff --git a/tests/unit/app_server/test_verify_agent_server_version.py b/tests/unit/app_server/test_verify_agent_server_version.pydeleted file mode 100644index 30ceaf1219..0000000000--- a/tests/unit/app_server/test_verify_agent_server_version.py+++ /dev/null@@ -1,136 +0,0 @@-"""Tests for the agent-server SDK version check on conversation start.--Covers _verify_agent_server_version (only enforced for custom sandbox images,-fail-open on anything it can't read) and the is_custom_agent_server_image gate.-"""--import os-from unittest.mock import AsyncMock, Mock, patch--import httpx-import pytest--from openhands.app_server.app_conversation.live_status_app_conversation_service import (- LiveStatusAppConversationService,- _expected_sdk_version,-)-from openhands.app_server.errors import SandboxError-from openhands.app_server.sandbox.sandbox_spec_service import (- AGENT_SERVER_IMAGE,- is_custom_agent_server_image,-)--MODULE = 'openhands.app_server.app_conversation.live_status_app_conversation_service'-PINNED_TAG = AGENT_SERVER_IMAGE.rsplit(':', 1)[-1]-# A custom pin = a tag different from the release-default one.-CUSTOM_ENV = {- 'AGENT_SERVER_IMAGE_REPOSITORY': 'harbor.example/agent-server',- 'AGENT_SERVER_IMAGE_TAG': 'custom-9',-}---def _service(client):- # Only httpx_client is touched by the method under test.- svc = LiveStatusAppConversationService.__new__(LiveStatusAppConversationService)- svc.httpx_client = client- return svc---def _resp(sdk_version):- return Mock(- raise_for_status=Mock(),- json=Mock(return_value={'sdk_version': sdk_version}),- )---async def _verify(client, env):- with patch.dict(os.environ, env, clear=False):- await _service(client)._verify_agent_server_version('http://agent.test/', 'k')---async def test_default_image_skips_check():- client = Mock(get=AsyncMock())- await _verify(client, {'AGENT_SERVER_IMAGE_TAG': PINNED_TAG})- client.get.assert_not_awaited()--[email protected]('val', ['1', 'true', 'TRUE', 'yes'])-async def test_opt_out_skips_check(val):- client = Mock(get=AsyncMock())- await _verify(client, {**CUSTOM_ENV, 'OH_SKIP_AGENT_SERVER_VERSION_CHECK': val})- client.get.assert_not_awaited()---async def test_matching_version_passes():- expected = _expected_sdk_version()- assert expected- client = Mock(get=AsyncMock(return_value=_resp(expected)))- await _verify(client, CUSTOM_ENV)- client.get.assert_awaited_once()--[email protected]('suffix', ['.99', '.0+c24'])-async def test_patch_or_local_retag_tolerated(suffix):- major, minor = _expected_sdk_version().split('.')[:2]- client = Mock(get=AsyncMock(return_value=_resp(f'{major}.{minor}{suffix}')))- await _verify(client, CUSTOM_ENV) # major.minor equal -> no raise---async def test_minor_mismatch_raises():- expected = _expected_sdk_version()- reported = f'{int(expected.split(".")[0]) + 1}.0.0'- client = Mock(get=AsyncMock(return_value=_resp(reported)))- with pytest.raises(SandboxError) as ei:- await _verify(client, CUSTOM_ENV)- # str() of a SandboxError (HTTPException) is '500: <msg>'; assert substrings.- assert reported in str(ei.value)- assert 'OH_SKIP_AGENT_SERVER_VERSION_CHECK' in str(ei.value)--[email protected]('reported', ['unknown', '', 'not-a-version'])-async def test_unverifiable_versions_pass(reported):- client = Mock(get=AsyncMock(return_value=_resp(reported)))- await _verify(client, CUSTOM_ENV)---async def test_non_200_server_info_fails_open():- resp = Mock(- raise_for_status=Mock(- side_effect=httpx.HTTPStatusError(- 'x', request=Mock(), response=Mock(status_code=404)- )- )- )- client = Mock(get=AsyncMock(return_value=resp))- await _verify(client, CUSTOM_ENV)- client.get.assert_awaited_once()---async def test_get_transport_error_fails_open():- client = Mock(get=AsyncMock(side_effect=httpx.ConnectError('down')))- await _verify(client, CUSTOM_ENV)---async def test_no_expected_version_skips_request():- client = Mock(get=AsyncMock())- with patch(f'{MODULE}._expected_sdk_version', return_value=None):- await _verify(client, CUSTOM_ENV)- client.get.assert_not_awaited()---def test_is_custom_agent_server_image():- with patch.dict(os.environ, {}, clear=True):- assert is_custom_agent_server_image() is False- with patch.dict(os.environ, {'AGENT_SERVER_IMAGE_TAG': PINNED_TAG}, clear=False):- assert is_custom_agent_server_image() is False- with patch.dict(os.environ, {'AGENT_SERVER_IMAGE_TAG': 'custom-9'}, clear=False):- assert is_custom_agent_server_image() is True