Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(app_server): derive agent server image from package version (#15168)

Tim O'Farrell20a721e23b05
diff --git a/openhands/app_server/app_conversation/live_status_app_conversation_service.py b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
index 77d09c7e37..cf56283496 100644
--- a/openhands/app_server/app_conversation/live_status_app_conversation_service.py
+++ b/openhands/app_server/app_conversation/live_status_app_conversation_service.py
@@ -3,7 +3,6 @@ import importlib.metadata
import io
import json
import logging
-import os
import zipfile
from collections import defaultdict
from collections.abc import Mapping
@@ -14,7 +13,6 @@ from uuid import UUID, uuid4
import httpx
from fastapi import Request
-from packaging.version import InvalidVersion, Version
from pydantic import Field, SecretStr, TypeAdapter
from openhands.agent_server.models import (
@@ -92,8 +90,7 @@ from openhands.app_server.sandbox.sandbox_models import (
from openhands.app_server.sandbox.sandbox_service import SandboxService
from openhands.app_server.sandbox.sandbox_spec_service import (
SandboxSpecService,
- get_agent_server_image,
- is_custom_agent_server_image,
+ is_custom_sandbox_spec,
)
from openhands.app_server.services.injector import InjectorState
from openhands.app_server.services.jwt_service import JwtService
@@ -383,12 +380,6 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
assert sandbox is not None
agent_server_url = self._get_agent_server_url(sandbox)
- # Custom sandbox images can ship an incompatible openhands-sdk; fail
- # fast with a clear error instead of an opaque 500 on create.
- await self._verify_agent_server_version(
- agent_server_url, sandbox.session_api_key
- )
-
# Mirror the user's LLM profiles into the sandbox so the agent's
# built-in switch_llm tool can resolve them (in SaaS profiles live
# on the app-server, not the sandbox filesystem). Before conversation
@@ -482,12 +473,12 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
except httpx.HTTPStatusError as exc:
# A custom image that 500s on create is usually an openhands-sdk
# mismatch /server_info couldn't reveal; add an actionable hint.
- if is_custom_agent_server_image():
+ if is_custom_sandbox_spec(sandbox.sandbox_spec_id):
expected = _expected_sdk_version()
raise SandboxError(
f'Conversation create failed (HTTP '
f'{exc.response.status_code}) on custom sandbox image '
- f'{get_agent_server_image()}. Verify its openhands-sdk '
+ f'{sandbox.sandbox_spec_id}. Verify its openhands-sdk '
f'matches this release'
+ (f' ({expected})' if expected else '')
+ '; rebuild/re-pin the image if not.'
@@ -912,57 +903,6 @@ class LiveStatusAppConversationService(AppConversationServiceBase):
httpx_client=self.httpx_client,
)
- async def _verify_agent_server_version(
- self, agent_server_url: str, session_api_key: str | None
- ) -> None:
- """Fail fast with a clear error when an admin-pinned custom sandbox image
- runs a different openhands-sdk minor than this app, instead of the opaque
- 500 the agent-server returns on create. Best-effort: only custom images are
- checked, and we fail open on anything we can't read."""
- if os.getenv('OH_SKIP_AGENT_SERVER_VERSION_CHECK', '').strip().lower() in (
- '1',
- 'true',
- 'yes',
- ):
- return
- # Proxy-default images move with the release; only custom-pinned can drift.
- if not is_custom_agent_server_image():
- return
- expected = _expected_sdk_version()
- if not expected:
- return
- try:
- headers = {'X-Session-API-Key': session_api_key} if session_api_key else {}
- resp = await self.httpx_client.get(
- f'{agent_server_url.rstrip("/")}/server_info',
- headers=headers,
- timeout=30.0,
- )
- resp.raise_for_status()
- reported = str(resp.json().get('sdk_version', '')).strip()
- except Exception:
- # 404 (image predates /server_info) or transient errors: can't verify,
- # so don't block β€” the create POST still surfaces a custom-image hint.
- _logger.warning(
- 'Could not read /server_info to verify agent-server SDK version',
- exc_info=True,
- )
- return
- # Endpoint present but metadata missing -> nothing to compare against.
- if reported in ('', 'unknown'):
- return
- try:
- if Version(reported).release[:2] == Version(expected).release[:2]:
- return
- except InvalidVersion:
- return
- raise SandboxError(
- f'Sandbox image {get_agent_server_image()} runs openhands-sdk '
- f'{reported}, but this release requires {expected}. Rebuild/re-pin the '
- 'custom sandbox image to a matching openhands-sdk, or set '
- 'OH_SKIP_AGENT_SERVER_VERSION_CHECK=1 to bypass.'
- )
-
async def _seed_sandbox_profiles(
self, agent_server_url: str, session_api_key: str | None
) -> None:
diff --git a/openhands/app_server/sandbox/sandbox_spec_service.py b/openhands/app_server/sandbox/sandbox_spec_service.py
index 79cb0c9ff0..c9b5dd326e 100644
--- a/openhands/app_server/sandbox/sandbox_spec_service.py
+++ b/openhands/app_server/sandbox/sandbox_spec_service.py
@@ -1,7 +1,9 @@
import asyncio
+import importlib
import logging
import os
from abc import ABC, abstractmethod
+from functools import cache
from openhands.agent_server import env_parser
from openhands.app_server.errors import SandboxError
@@ -12,9 +14,25 @@ from openhands.app_server.sandbox.sandbox_spec_models import (
from openhands.app_server.services.injector import Injector
from openhands.sdk.utils.models import DiscriminatedUnionMixin
-# The version of the agent server to use for deployments.
-# Typically this will be the same as the values from the pyproject.toml
-AGENT_SERVER_IMAGE = 'ghcr.io/openhands/agent-server:1.32.0-python'
+_DEFAULT_REPOSITORY = 'ghcr.io/openhands/agent-server'
+
+
+@cache
+def _bundled_agent_server_version() -> str:
+ """Version of the installed openhands-agent-server package.
+
+ openhands-agent-server is a hard runtime dependency of the V1 app server;
+ if it's not installed the V1 app server has no business starting. Let
+ PackageNotFoundError surface at first call rather than degrading silently
+ to a wrong default. Memoised via ``functools.cache`` so the lookup runs
+ once per process and remains mockable in tests.
+ """
+ return importlib.metadata.version('openhands-agent-server')
+
+
+def _bundled_default_image() -> str:
+ """Image URL of the bundled agent-server: ``<repo>:<version>-python``."""
+ return f'{_DEFAULT_REPOSITORY}:{_bundled_agent_server_version()}-python'
class SandboxSpecService(ABC):
@@ -98,19 +116,34 @@ async def resolve_sandbox_spec(
raise ValueError(f'Sandbox Spec {effective_id!r} not found')
+@cache # memoize so the auto-correct warning fires once per process
def get_agent_server_image() -> str:
- agent_server_image_repository = os.getenv('AGENT_SERVER_IMAGE_REPOSITORY')
- agent_server_image_tag = os.getenv('AGENT_SERVER_IMAGE_TAG')
- if agent_server_image_repository and agent_server_image_tag:
- return f'{agent_server_image_repository}:{agent_server_image_tag}'
- return AGENT_SERVER_IMAGE
-
-
-def is_custom_agent_server_image() -> bool:
- """True only when an admin pinned a custom sandbox image (tag differs from the
- release-default tag). Default/upgrade installs keep the release tag, never gated."""
- tag = os.getenv('AGENT_SERVER_IMAGE_TAG')
- return bool(tag) and tag != AGENT_SERVER_IMAGE.rsplit(':', 1)[-1]
+ repository = os.getenv('AGENT_SERVER_IMAGE_REPOSITORY') or _DEFAULT_REPOSITORY
+ bundled_version = _bundled_agent_server_version()
+ tag = os.getenv('AGENT_SERVER_IMAGE_TAG') or f'{bundled_version}-python'
+
+ if repository == _DEFAULT_REPOSITORY:
+ parts = tag.split('-', 1)
+ if len(parts) == 2 and parts[0] != bundled_version:
+ parts[0] = bundled_version
+ updated_tag = '-'.join(parts)
+ logging.getLogger(__name__).warning(
+ f'AGENT_SERVER_IMAGE_TAG={tag} does not match the installed '
+ f'openhands-sdk (using {updated_tag} instead). Pin to a custom '
+ f'image repository via AGENT_SERVER_IMAGE_REPOSITORY if you '
+ f'need to use a different build.'
+ )
+ tag = updated_tag
+
+ return f'{repository}:{tag}'
+
+
+def is_custom_sandbox_spec(sandbox_spec_id: str) -> bool:
+ """True when the sandbox spec differs from the bundled release default (e.g. a
+ custom image registered via runtime-api). The bundled spec always matches the
+ SDK the app server ships with, so non-default specs are the only ones worth
+ flagging on a sandbox-create failure."""
+ return sandbox_spec_id != _bundled_default_image()
# Prefixes for environment variables that should be auto-forwarded to agent-server
diff --git a/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py b/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py
index ece6a93500..0448c6cd85 100644
--- a/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py
+++ b/tests/unit/app_server/test_dynamic_remote_sandbox_spec_service.py
@@ -193,6 +193,37 @@ class TestFetchSpecs:
# is the intended signal for get_default_sandbox_spec to fall back too.
assert service._name_to_spec == {}
+ async def test_empty_configs_fallback_uses_bundled_default_not_env_override(
+ self,
+ ):
+ """Regression test for the staging 500.
+
+ When runtime-api returns empty configs, the fallback spec's image id
+ must equal the bundled default (whatever ``get_agent_server_image()``
+ currently returns), NOT whatever the legacy AGENT_SERVER_IMAGE_TAG env
+ var happens to say. Otherwise an admin pinning a stale tag would
+ cause a sandbox-create 500.
+ """
+ # Pin the bundled default to a specific image, regardless of what's
+ # installed in the test env, so we can assert against it directly.
+ # Patch via the module attribute (not patch.object on the function
+ # itself) because the @cache decorator wraps __call__ and intercepts
+ # the standard mock hook.
+ expected_image = 'ghcr.io/openhands/agent-server:9.9.9-python'
+
+ ctx, _ = _make_async_client_mock(_make_http_response({'configs': []}))
+ service = _make_service()
+
+ with patch(
+ 'openhands.app_server.sandbox.remote_sandbox_spec_service.get_agent_server_image',
+ return_value=expected_image,
+ ):
+ with patch('httpx.AsyncClient', return_value=ctx):
+ specs = await service._fetch_specs()
+
+ assert len(specs) == 1
+ assert specs[0].id == expected_image
+
async def test_fallback_specs_are_cached(self):
"""Default fallback specs must be cached like normal results."""
from openhands.app_server.sandbox.remote_sandbox_spec_service import (
diff --git a/tests/unit/app_server/test_sandbox_spec_service.py b/tests/unit/app_server/test_sandbox_spec_service.py
new file mode 100644
index 0000000000..764cc02762
--- /dev/null
+++ b/tests/unit/app_server/test_sandbox_spec_service.py
@@ -0,0 +1,181 @@
+"""Tests for sandbox_spec_service helpers.
+
+Covers ``get_agent_server_image`` (derived from the installed
+``openhands-agent-server`` package, with auto-correction of stale canonical
+tags for self-hosted installs) and ``is_custom_sandbox_spec`` (which compares
+a sandbox spec id against the bundled default).
+"""
+
+import importlib.metadata
+from unittest.mock import patch
+
+import pytest
+
+from openhands.app_server.sandbox.sandbox_spec_service import (
+ _bundled_agent_server_version,
+ get_agent_server_image,
+ is_custom_sandbox_spec,
+)
+
+
[email protected](autouse=True)
+def _clear_caches():
+ """All the @cache'd helpers in this module need their caches cleared around
+ every test so we don't leak env-var or mocked-version state between cases.
+ (_bundled_default_image isn't @cache'd; it just calls _bundled_agent_server_version.)"""
+ get_agent_server_image.cache_clear()
+ _bundled_agent_server_version.cache_clear()
+ yield
+ get_agent_server_image.cache_clear()
+ _bundled_agent_server_version.cache_clear()
+
+
+def test_get_agent_server_image_derived_from_package_version():
+ """The URL must be built from the installed openhands-agent-server version,
+ not a hand-maintained constant β€” that's the whole point of removing the
+ drift-prone AGENT_SERVER_IMAGE string."""
+ fake_version = '9.9.9'
+ with patch.object(
+ importlib.metadata,
+ 'version',
+ return_value=fake_version,
+ ):
+ assert get_agent_server_image() == (
+ f'ghcr.io/openhands/agent-server:{fake_version}-python'
+ )
+
+
+def test_get_agent_server_image_returns_consistent_value_within_process():
+ """``@cache`` should make repeat calls return the same object without
+ re-reading importlib.metadata or re-checking env vars."""
+ with patch.object(
+ importlib.metadata,
+ 'version',
+ return_value='1.0.0',
+ ) as version_mock:
+ first = get_agent_server_image()
+ # Call many times β€” version_mock should only be hit once thanks to @cache.
+ for _ in range(5):
+ assert get_agent_server_image() is first
+ assert version_mock.call_count == 1
+
+
+def test_get_agent_server_image_honours_custom_repository():
+ """Self-hosted customers can pin to their own image repository via
+ AGENT_SERVER_IMAGE_REPOSITORY; the result must be honoured verbatim,
+ including a non-default tag."""
+ fake_version = '1.0.0'
+ with patch.object(importlib.metadata, 'version', return_value=fake_version):
+ with patch.dict(
+ 'os.environ',
+ {
+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'example.com/agent-server',
+ 'AGENT_SERVER_IMAGE_TAG': '9.9.9-python',
+ },
+ clear=False,
+ ):
+ assert get_agent_server_image() == 'example.com/agent-server:9.9.9-python'
+
+
+def test_get_agent_server_image_auto_corrects_stale_canonical_tag():
+ """Regression test for the staging 500.
+
+ A canonical-repo + stale ``-python`` tag (the exact staging failure mode)
+ must be silently re-pinned to the SDK-matching tag, so self-hosted
+ installs keep working when they ship a stale AGENT_SERVER_IMAGE_TAG.
+ """
+ fake_version = '9.9.9'
+ with patch.object(importlib.metadata, 'version', return_value=fake_version):
+ with patch.dict(
+ 'os.environ',
+ {
+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'ghcr.io/openhands/agent-server',
+ 'AGENT_SERVER_IMAGE_TAG': '1.31.1-python',
+ },
+ clear=False,
+ ):
+ assert get_agent_server_image() == (
+ f'ghcr.io/openhands/agent-server:{fake_version}-python'
+ )
+
+
+def test_get_agent_server_image_passes_through_canonical_tag_without_suffix():
+ """Tags without a ``-`` separator (e.g. ``1.32.0``, ``nightly``) must not
+ be silently rewritten by the auto-correct heuristic β€” that would lose
+ legitimate canonical tags. The post-hoc 500-hint will still flag any
+ actual SDK mismatch downstream."""
+ fake_version = '9.9.9'
+ with patch.object(importlib.metadata, 'version', return_value=fake_version):
+ with patch.dict(
+ 'os.environ',
+ {
+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'ghcr.io/openhands/agent-server',
+ 'AGENT_SERVER_IMAGE_TAG': 'nightly',
+ },
+ clear=False,
+ ):
+ assert get_agent_server_image() == (
+ 'ghcr.io/openhands/agent-server:nightly'
+ )
+
+
+def test_get_agent_server_image_passes_through_canonical_repo_with_matching_tag():
+ """Canonical repo + already-matching tag must be a no-op (no warning, no
+ rewrite) β€” the most common case for fresh installs."""
+ fake_version = '1.2.3'
+ with patch.object(importlib.metadata, 'version', return_value=fake_version):
+ with patch.dict(
+ 'os.environ',
+ {
+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'ghcr.io/openhands/agent-server',
+ 'AGENT_SERVER_IMAGE_TAG': f'{fake_version}-python',
+ },
+ clear=False,
+ ):
+ assert get_agent_server_image() == (
+ f'ghcr.io/openhands/agent-server:{fake_version}-python'
+ )
+
+
+def test_is_custom_sandbox_spec_false_for_bundled_default():
+ """A spec id equal to the bundled default is, by definition, not custom."""
+ with patch.object(importlib.metadata, 'version', return_value='1.0.0'):
+ bundled = get_agent_server_image()
+ assert is_custom_sandbox_spec(bundled) is False
+
+
+def test_is_custom_sandbox_spec_true_for_runtime_api_image():
+ """A spec id from runtime-api (any non-default image) must be flagged custom."""
+ with patch.object(importlib.metadata, 'version', return_value='1.0.0'):
+ assert is_custom_sandbox_spec('ghcr.io/some/custom:0.0.1') is True
+
+
+def test_is_custom_sandbox_spec_true_for_self_hosted_custom_repo():
+ """A self-hosted customer who set AGENT_SERVER_IMAGE_REPOSITORY to a
+ custom repo still has a non-default image; the post-hoc 500-hint must
+ fire for them when their image's SDK doesn't match."""
+ fake_version = '1.0.0'
+ with patch.object(importlib.metadata, 'version', return_value=fake_version):
+ with patch.dict(
+ 'os.environ',
+ {
+ 'AGENT_SERVER_IMAGE_REPOSITORY': 'example.com/agent-server',
+ 'AGENT_SERVER_IMAGE_TAG': '9.9.9-python',
+ },
+ clear=False,
+ ):
+ assert (
+ is_custom_sandbox_spec('example.com/agent-server:9.9.9-python') is True
+ )
+
+
+def test_get_agent_server_image_propagates_package_not_found():
+ """openhands-agent-server is a hard runtime dependency; a missing install
+ must surface as PackageNotFoundError at first call, not silently degrade."""
+ with patch.object(
+ importlib.metadata,
+ 'version',
+ side_effect=importlib.metadata.PackageNotFoundError('openhands-agent-server'),
+ ):
+ with pytest.raises(importlib.metadata.PackageNotFoundError):
+ get_agent_server_image()
diff --git a/tests/unit/app_server/test_verify_agent_server_version.py b/tests/unit/app_server/test_verify_agent_server_version.py
deleted file mode 100644
index 30ceaf1219..0000000000
--- a/tests/unit/app_server/test_verify_agent_server_version.py
+++ /dev/null
@@ -1,136 +0,0 @@
-"""Tests for the agent-server SDK version check on conversation start.
-
-Covers _verify_agent_server_version (only enforced for custom sandbox images,
-fail-open on anything it can't read) and the is_custom_agent_server_image gate.
-"""
-
-import os
-from unittest.mock import AsyncMock, Mock, patch
-
-import httpx
-import pytest
-
-from openhands.app_server.app_conversation.live_status_app_conversation_service import (
- LiveStatusAppConversationService,
- _expected_sdk_version,
-)
-from openhands.app_server.errors import SandboxError
-from openhands.app_server.sandbox.sandbox_spec_service import (
- AGENT_SERVER_IMAGE,
- is_custom_agent_server_image,
-)
-
-MODULE = 'openhands.app_server.app_conversation.live_status_app_conversation_service'
-PINNED_TAG = AGENT_SERVER_IMAGE.rsplit(':', 1)[-1]
-# A custom pin = a tag different from the release-default one.
-CUSTOM_ENV = {
- 'AGENT_SERVER_IMAGE_REPOSITORY': 'harbor.example/agent-server',
- 'AGENT_SERVER_IMAGE_TAG': 'custom-9',
-}
-
-
-def _service(client):
- # Only httpx_client is touched by the method under test.
- svc = LiveStatusAppConversationService.__new__(LiveStatusAppConversationService)
- svc.httpx_client = client
- return svc
-
-
-def _resp(sdk_version):
- return Mock(
- raise_for_status=Mock(),
- json=Mock(return_value={'sdk_version': sdk_version}),
- )
-
-
-async def _verify(client, env):
- with patch.dict(os.environ, env, clear=False):
- await _service(client)._verify_agent_server_version('http://agent.test/', 'k')
-
-
-async def test_default_image_skips_check():
- client = Mock(get=AsyncMock())
- await _verify(client, {'AGENT_SERVER_IMAGE_TAG': PINNED_TAG})
- client.get.assert_not_awaited()
-
-
[email protected]('val', ['1', 'true', 'TRUE', 'yes'])
-async def test_opt_out_skips_check(val):
- client = Mock(get=AsyncMock())
- await _verify(client, {**CUSTOM_ENV, 'OH_SKIP_AGENT_SERVER_VERSION_CHECK': val})
- client.get.assert_not_awaited()
-
-
-async def test_matching_version_passes():
- expected = _expected_sdk_version()
- assert expected
- client = Mock(get=AsyncMock(return_value=_resp(expected)))
- await _verify(client, CUSTOM_ENV)
- client.get.assert_awaited_once()
-
-
[email protected]('suffix', ['.99', '.0+c24'])
-async def test_patch_or_local_retag_tolerated(suffix):
- major, minor = _expected_sdk_version().split('.')[:2]
- client = Mock(get=AsyncMock(return_value=_resp(f'{major}.{minor}{suffix}')))
- await _verify(client, CUSTOM_ENV) # major.minor equal -> no raise
-
-
-async def test_minor_mismatch_raises():
- expected = _expected_sdk_version()
- reported = f'{int(expected.split(".")[0]) + 1}.0.0'
- client = Mock(get=AsyncMock(return_value=_resp(reported)))
- with pytest.raises(SandboxError) as ei:
- await _verify(client, CUSTOM_ENV)
- # str() of a SandboxError (HTTPException) is '500: <msg>'; assert substrings.
- assert reported in str(ei.value)
- assert 'OH_SKIP_AGENT_SERVER_VERSION_CHECK' in str(ei.value)
-
-
[email protected]('reported', ['unknown', '', 'not-a-version'])
-async def test_unverifiable_versions_pass(reported):
- client = Mock(get=AsyncMock(return_value=_resp(reported)))
- await _verify(client, CUSTOM_ENV)
-
-
-async def test_non_200_server_info_fails_open():
- resp = Mock(
- raise_for_status=Mock(
- side_effect=httpx.HTTPStatusError(
- 'x', request=Mock(), response=Mock(status_code=404)
- )
- )
- )
- client = Mock(get=AsyncMock(return_value=resp))
- await _verify(client, CUSTOM_ENV)
- client.get.assert_awaited_once()
-
-
-async def test_get_transport_error_fails_open():
- client = Mock(get=AsyncMock(side_effect=httpx.ConnectError('down')))
- await _verify(client, CUSTOM_ENV)
-
-
-async def test_no_expected_version_skips_request():
- client = Mock(get=AsyncMock())
- with patch(f'{MODULE}._expected_sdk_version', return_value=None):
- await _verify(client, CUSTOM_ENV)
- client.get.assert_not_awaited()
-
-
-def test_is_custom_agent_server_image():
- with patch.dict(os.environ, {}, clear=True):
- assert is_custom_agent_server_image() is False
- with patch.dict(os.environ, {'AGENT_SERVER_IMAGE_TAG': PINNED_TAG}, clear=False):
- assert is_custom_agent_server_image() is False
- with patch.dict(os.environ, {'AGENT_SERVER_IMAGE_TAG': 'custom-9'}, clear=False):
- assert is_custom_agent_server_image() is True