Skip to content

πŸ™Œ OpenHands: AI-Driven Development

PLTF-2895: scope DB sessions tightly in poll_agent_servers to prevent idle-in-transaction (#14637)

aivong-openhands63746da2dcff
diff --git a/openhands/app_server/sandbox/remote_sandbox_service.py b/openhands/app_server/sandbox/remote_sandbox_service.py
index 33248372d6..4532efcd75 100644
--- a/openhands/app_server/sandbox/remote_sandbox_service.py
+++ b/openhands/app_server/sandbox/remote_sandbox_service.py
@@ -21,17 +21,10 @@ from openhands.agent_server.models import (
EventPage,
)
from openhands.agent_server.utils import utc_now
-from openhands.app_server.app_conversation.app_conversation_info_service import (
- AppConversationInfoService,
-)
from openhands.app_server.app_conversation.app_conversation_models import (
AppConversationInfo,
)
from openhands.app_server.errors import SandboxError
-from openhands.app_server.event.event_service import EventService
-from openhands.app_server.event_callback.event_callback_service import (
- EventCallbackService,
-)
from openhands.app_server.sandbox.sandbox_models import (
AGENT_SERVER,
VSCODE,
@@ -677,18 +670,24 @@ async def poll_agent_servers(api_url: str, api_key: str, sleep_interval: int):
"""When the app server does not have a public facing url, we poll the agent
servers for the most recent data.
- This is because webhook callbacks cannot be invoked."""
+ This is because webhook callbacks cannot be invoked.
+
+ IMPORTANT: DB sessions are scoped tightly to avoid holding connections across
+ network I/O. Services are imported locally inside the function bodies to
+ ensure they are resolved in the correct context. We use a
+ "fetch -> release -> network -> re-acquire -> write" pattern.
+ """
from openhands.app_server.config import (
get_app_conversation_info_service,
- get_event_callback_service,
- get_event_service,
+ get_db_session,
get_httpx_client,
)
while True:
try:
- # Refresh the conversations associated with those sandboxes.
state = InjectorState()
+ # We allow access to all items here
+ setattr(state, USER_CONTEXT_ATTR, ADMIN)
try:
# Get the list of running sandboxes using the runtime api /list endpoint.
@@ -707,29 +706,35 @@ async def poll_agent_servers(api_url: str, api_key: str, sleep_interval: int):
if runtime['status'] == 'running'
}
- # We allow access to all items here
- setattr(state, USER_CONTEXT_ATTR, ADMIN)
+ # Phase 1: Read - fetch all conversations into a list with a short DB session
+ # This releases the DB session before any network I/O
+ conversations_to_refresh: list[AppConversationInfo] = []
async with (
get_app_conversation_info_service(
state
) as app_conversation_info_service,
- get_event_service(state) as event_service,
- get_event_callback_service(state) as event_callback_service,
- get_httpx_client(state) as httpx_client,
+ get_db_session(state) as _db_session,
):
- matches = 0
async for app_conversation_info in page_iterator(
app_conversation_info_service.search_app_conversation_info
):
+ conversations_to_refresh.append(app_conversation_info)
+
+ _logger.debug(
+ f'Found {len(conversations_to_refresh)} conversations to check'
+ )
+
+ # Phase 2: Network I/O - fetch httpx client and do all network operations
+ # WITHOUT any DB session held
+ async with get_httpx_client(state) as httpx_client:
+ matches = 0
+ for app_conversation_info in conversations_to_refresh:
runtime = runtimes_by_sandbox_id.get(
app_conversation_info.sandbox_id
)
if runtime:
matches += 1
await refresh_conversation(
- app_conversation_info_service=app_conversation_info_service,
- event_service=event_service,
- event_callback_service=event_callback_service,
app_conversation_info=app_conversation_info,
runtime=runtime,
httpx_client=httpx_client,
@@ -751,9 +756,6 @@ async def poll_agent_servers(api_url: str, api_key: str, sleep_interval: int):
async def refresh_conversation(
- app_conversation_info_service: AppConversationInfoService,
- event_service: EventService,
- event_callback_service: EventCallbackService,
app_conversation_info: AppConversationInfo,
runtime: dict[str, Any],
httpx_client: httpx.AsyncClient,
@@ -761,14 +763,29 @@ async def refresh_conversation(
"""Refresh a conversation.
Grab ConversationInfo and all events from the agent server and make sure they
- exist in the app server."""
+ exist in the app server.
+
+ IMPORTANT: This function acquires its own short-lived DB sessions for writes,
+ never holding a session across network I/O. Uses a "fetch -> release -> write"
+ pattern per conversation.
+ """
+ from openhands.app_server.config import (
+ get_app_conversation_info_service,
+ get_db_session,
+ get_event_callback_service,
+ get_event_service,
+ )
+
+ state = InjectorState()
+ setattr(state, USER_CONTEXT_ATTR, ADMIN)
+
_logger.debug(f'Started Refreshing Conversation {app_conversation_info.id}')
try:
url = runtime['url']
# TODO: Maybe we can use RemoteConversation here?
- # First get conversation...
+ # Phase 1: Network I/O - First get conversation...
conversation_url = f'{url}/api/conversations/{app_conversation_info.id.hex}'
response = await httpx_client.get(
conversation_url, headers={'X-Session-API-Key': runtime['session_api_key']}
@@ -789,12 +806,17 @@ async def refresh_conversation(
except Exception:
_logger.exception('error_updating_conversation_metrics', stack_info=True)
- # TODO: Update other appropriate attributes...
-
- await app_conversation_info_service.save_app_conversation_info(
- app_conversation_info
- )
+ # Phase 2: Write - acquire DB session and save conversation info
+ # (short-lived session, no network I/O held)
+ async with (
+ get_db_session(state) as _db_session,
+ get_app_conversation_info_service(state) as app_conversation_info_service,
+ ):
+ await app_conversation_info_service.save_app_conversation_info(
+ app_conversation_info
+ )
+ # Phase 3: Network I/O - fetch events (no DB session held)
# TODO: It would be nice to have an updated_at__gte filter parameter in the
# agent server so that we don't pull the full event list each time
event_url = (
@@ -815,14 +837,21 @@ async def refresh_conversation(
return EventPage.model_validate(response.json())
async for event in page_iterator(fetch_events_page):
- existing = await event_service.get_event(
- app_conversation_info.id, UUID(event.id)
- )
- if existing is None:
- await event_service.save_event(app_conversation_info.id, event)
- await event_callback_service.execute_callbacks(
- app_conversation_info.id, event
+ # Phase 4: Write - acquire DB session for each event save
+ # (short-lived session per event, no network I/O held)
+ async with (
+ get_db_session(state) as _db_session,
+ get_event_service(state) as event_service,
+ get_event_callback_service(state) as event_callback_service,
+ ):
+ existing = await event_service.get_event(
+ app_conversation_info.id, UUID(event.id)
)
+ if existing is None:
+ await event_service.save_event(app_conversation_info.id, event)
+ await event_callback_service.execute_callbacks(
+ app_conversation_info.id, event
+ )
_logger.debug(f'Finished Refreshing Conversation {app_conversation_info.id}')
diff --git a/tests/unit/app_server/test_remote_sandbox_service.py b/tests/unit/app_server/test_remote_sandbox_service.py
index f316559e19..7ee7ec9f8b 100644
--- a/tests/unit/app_server/test_remote_sandbox_service.py
+++ b/tests/unit/app_server/test_remote_sandbox_service.py
@@ -11,9 +11,12 @@ This module tests the RemoteSandboxService implementation, focusing on:
- Error handling for HTTP failures and edge cases
"""
+import asyncio
+from contextlib import ExitStack, asynccontextmanager
from datetime import datetime, timezone
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
+from uuid import uuid4
import httpx
import pytest
@@ -1152,3 +1155,377 @@ class TestConstants:
"""Test that environment variable constants are defined."""
assert WEBHOOK_CALLBACK_VARIABLE == 'OH_WEBHOOKS_0_BASE_URL'
assert ALLOW_CORS_ORIGINS_VARIABLE == 'OH_ALLOW_CORS_ORIGINS_0'
+
+
+def _async_cm_factory(value):
+ """Return a callable that yields ``value`` as an async context manager.
+
+ Mirrors ``get_*_service(state)``, which is used as
+ ``async with get_x(state) as y``.
+ """
+
+ @asynccontextmanager
+ async def _cm(*args, **kwargs):
+ yield value
+
+ return _cm
+
+
+def _make_page(items, next_page_id=None):
+ """Build a minimal object satisfying the page_iterator protocol."""
+ page = MagicMock()
+ page.items = items
+ page.next_page_id = next_page_id
+ return page
+
+
+class _SessionTracker:
+ """Counts how many mocked DB sessions are open at any instant.
+
+ Used as the ``side_effect`` for a patched ``get_db_session``: every
+ ``async with get_db_session(state)`` increments ``open`` on enter and
+ decrements it on exit. Network-call probes record ``open`` at the moment
+ they fire, so the regression guard ``open == 0 during network I/O`` can be
+ asserted directly.
+ """
+
+ def __init__(self):
+ self.open = 0
+ self.enter_count = 0
+ self.max_open = 0
+
+ def __call__(self, *args, **kwargs):
+ tracker = self
+
+ class _Session:
+ async def __aenter__(self):
+ tracker.open += 1
+ tracker.enter_count += 1
+ tracker.max_open = max(tracker.max_open, tracker.open)
+ return MagicMock()
+
+ async def __aexit__(self, *exc):
+ tracker.open -= 1
+ return False
+
+ return _Session()
+
+
+class TestPollAgentServersSessionScoping:
+ """Test cases for DB session scoping in poll_agent_servers and refresh_conversation.
+
+ These tests verify that DB sessions are released before network I/O to
+ prevent 'idle in transaction' issues. The key invariant:
+
+ - No DB session/transaction is held open across an await'ed agent-server
+ network call.
+
+ A shared :class:`_SessionTracker` counts how many DB sessions are open at
+ any instant, and every mocked agent-server network call records the count
+ it observes. The regression guard is that this count is always ``0`` during
+ network I/O. The mocks deliberately drive the *real* poll/refresh code
+ paths (including the DB-write phases) so the assertions are not vacuous.
+ """
+
+ def _patches(
+ self,
+ tracker,
+ httpx_client,
+ conv_service,
+ event_service,
+ callback_service,
+ validated_conv,
+ event_pages,
+ ):
+ """Common patches shared by the tests in this class.
+
+ Returns a list of ``patch`` context managers. ``ConversationInfo`` and
+ ``EventPage`` validation is stubbed so the refresh code reaches its
+ DB-write phases without needing fully-formed agent-server payloads.
+ """
+ mock_conv_info = MagicMock()
+ mock_conv_info.model_validate.return_value = validated_conv
+ mock_event_page = MagicMock()
+ mock_event_page.model_validate.side_effect = list(event_pages)
+ return [
+ patch('openhands.app_server.config.get_db_session', side_effect=tracker),
+ patch(
+ 'openhands.app_server.config.get_app_conversation_info_service',
+ side_effect=_async_cm_factory(conv_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_event_service',
+ side_effect=_async_cm_factory(event_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_event_callback_service',
+ side_effect=_async_cm_factory(callback_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_httpx_client',
+ side_effect=_async_cm_factory(httpx_client),
+ ),
+ patch(
+ 'openhands.app_server.sandbox.remote_sandbox_service.ConversationInfo',
+ mock_conv_info,
+ ),
+ patch(
+ 'openhands.app_server.sandbox.remote_sandbox_service.EventPage',
+ mock_event_page,
+ ),
+ patch('openhands.app_server.sandbox.remote_sandbox_service.InjectorState'),
+ patch('openhands.app_server.sandbox.remote_sandbox_service.ADMIN'),
+ patch(
+ 'openhands.app_server.sandbox.remote_sandbox_service.USER_CONTEXT_ATTR',
+ 'user_context',
+ ),
+ ]
+
+ @staticmethod
+ def _validated_conv():
+ validated_conv = MagicMock()
+ validated_conv.updated_at = datetime(2024, 1, 1, tzinfo=timezone.utc)
+ validated_conv.stats.get_combined_metrics.return_value = MagicMock()
+ return validated_conv
+
+ @staticmethod
+ def _app_conv(sandbox_id='sandbox-1'):
+ app_conv = MagicMock()
+ app_conv.id = MagicMock(hex='c0ffee')
+ app_conv.sandbox_id = sandbox_id
+ app_conv.metrics = None
+ return app_conv
+
+ @pytest.mark.asyncio
+ async def test_poll_agent_servers_releases_db_before_network_io(self):
+ """poll_agent_servers must release the read session before network I/O."""
+ from openhands.app_server.sandbox.remote_sandbox_service import (
+ poll_agent_servers,
+ )
+
+ tracker = _SessionTracker()
+ network_open_counts: list[int] = []
+
+ app_conv = self._app_conv()
+ list_payload = {
+ 'runtimes': [
+ {
+ 'session_id': 'sandbox-1',
+ 'status': 'running',
+ 'url': 'https://sandbox.example.com',
+ 'session_api_key': 'key1',
+ }
+ ]
+ }
+
+ async def probe_get(url, *args, **kwargs):
+ # Record open DB sessions at the moment of every agent-server call.
+ network_open_counts.append(tracker.open)
+ resp = MagicMock()
+ resp.raise_for_status = MagicMock()
+ if url.endswith('/list'):
+ resp.json.return_value = list_payload
+ else:
+ resp.json.return_value = {}
+ return resp
+
+ httpx_client = AsyncMock()
+ httpx_client.get.side_effect = probe_get
+
+ conv_service = AsyncMock()
+ conv_service.search_app_conversation_info = AsyncMock(
+ return_value=_make_page([app_conv])
+ )
+ conv_service.save_app_conversation_info = AsyncMock()
+ event_service = AsyncMock()
+ event_service.get_event = AsyncMock(return_value=None)
+ callback_service = AsyncMock()
+
+ patches = self._patches(
+ tracker,
+ httpx_client,
+ conv_service,
+ event_service,
+ callback_service,
+ self._validated_conv(),
+ event_pages=[_make_page([], None)],
+ )
+ with ExitStack() as stack:
+ for patch_cm in patches:
+ stack.enter_context(patch_cm)
+ task = asyncio.create_task(
+ poll_agent_servers(
+ api_url='https://api.example.com',
+ api_key='test-key',
+ sleep_interval=3600, # long, so cancellation ends the loop
+ )
+ )
+ await asyncio.sleep(0.1)
+ task.cancel()
+ await task # poll swallows CancelledError and returns
+
+ # The runtime-list call and the per-conversation refresh both ran.
+ assert network_open_counts, 'expected agent-server network calls to fire'
+ assert len(network_open_counts) >= 2, (
+ 'expected at least the /list call plus a conversation refresh, '
+ f'got {len(network_open_counts)} calls'
+ )
+ # Core regression guard: no DB session may be open during network I/O.
+ assert all(count == 0 for count in network_open_counts), (
+ 'DB session held during network I/O (idle-in-transaction risk); '
+ f'observed open-session counts at network calls: {network_open_counts}'
+ )
+ # Non-vacuous: the read session (Phase 1) and a write session were used.
+ assert tracker.enter_count >= 2, (
+ 'expected the read session plus at least one write session to open'
+ )
+ assert tracker.max_open >= 1, 'expected at least one DB session to open'
+ assert tracker.open == 0, 'all DB sessions must be released after polling'
+ conv_service.save_app_conversation_info.assert_awaited()
+
+ @pytest.mark.asyncio
+ async def test_refresh_conversation_acquires_own_db_session(self):
+ """refresh_conversation must open its own short-lived write sessions."""
+ from openhands.app_server.sandbox.remote_sandbox_service import (
+ refresh_conversation,
+ )
+
+ tracker = _SessionTracker()
+ network_open_counts: list[int] = []
+
+ async def probe_get(url, *args, **kwargs):
+ network_open_counts.append(tracker.open)
+ resp = MagicMock()
+ resp.raise_for_status = MagicMock()
+ resp.json.return_value = {}
+ return resp
+
+ httpx_client = AsyncMock()
+ httpx_client.get.side_effect = probe_get
+
+ conv_service = AsyncMock()
+ conv_service.save_app_conversation_info = AsyncMock()
+ event_service = AsyncMock()
+ event_service.get_event = AsyncMock(return_value=None)
+ event_service.save_event = AsyncMock()
+ callback_service = AsyncMock()
+ callback_service.execute_callbacks = AsyncMock()
+
+ # One event on the first page, then an empty page to end pagination.
+ event = MagicMock()
+ event.id = str(uuid4())
+ event_pages = [_make_page([event], 'page-2'), _make_page([], None)]
+
+ patches = self._patches(
+ tracker,
+ httpx_client,
+ conv_service,
+ event_service,
+ callback_service,
+ self._validated_conv(),
+ event_pages=event_pages,
+ )
+ runtime = {
+ 'url': 'https://sandbox.example.com',
+ 'session_api_key': 'test-key',
+ }
+ with ExitStack() as stack:
+ for patch_cm in patches:
+ stack.enter_context(patch_cm)
+ await refresh_conversation(
+ app_conversation_info=self._app_conv(),
+ runtime=runtime,
+ httpx_client=httpx_client,
+ )
+
+ # The write paths actually ran (otherwise the assertions are vacuous).
+ conv_service.save_app_conversation_info.assert_awaited_once()
+ event_service.save_event.assert_awaited_once()
+ callback_service.execute_callbacks.assert_awaited_once()
+ # refresh_conversation opened its own sessions: one for the conversation
+ # save, one for the single new event.
+ assert tracker.enter_count >= 2, (
+ 'refresh_conversation should acquire its own DB sessions for writes'
+ )
+ # Those sessions were short-lived and never overlapped network I/O.
+ assert all(count == 0 for count in network_open_counts), (
+ 'DB session held during network I/O; observed open counts: '
+ f'{network_open_counts}'
+ )
+ assert tracker.open == 0, 'all DB sessions must be released afterwards'
+
+ @pytest.mark.asyncio
+ async def test_db_session_not_held_across_network_call(self):
+ """The key regression test: no DB session is open during a network call.
+
+ Uses an artificial network delay so a held session would visibly span
+ the await; the open-session count is sampled inside that window.
+ """
+ from openhands.app_server.sandbox.remote_sandbox_service import (
+ refresh_conversation,
+ )
+
+ tracker = _SessionTracker()
+ open_counts_during_network: list[int] = []
+
+ async def slow_get(url, *args, **kwargs):
+ # Sample the open-session count while the "network call" is mid-flight.
+ await asyncio.sleep(0.01)
+ open_counts_during_network.append(tracker.open)
+ await asyncio.sleep(0.01)
+ resp = MagicMock()
+ resp.raise_for_status = MagicMock()
+ resp.json.return_value = {}
+ return resp
+
+ httpx_client = AsyncMock()
+ httpx_client.get.side_effect = slow_get
+
+ conv_service = AsyncMock()
+ conv_service.save_app_conversation_info = AsyncMock()
+ event_service = AsyncMock()
+ event_service.get_event = AsyncMock(return_value=None)
+ event_service.save_event = AsyncMock()
+ callback_service = AsyncMock()
+ callback_service.execute_callbacks = AsyncMock()
+
+ event = MagicMock()
+ event.id = str(uuid4())
+ event_pages = [_make_page([event], 'page-2'), _make_page([], None)]
+
+ patches = self._patches(
+ tracker,
+ httpx_client,
+ conv_service,
+ event_service,
+ callback_service,
+ self._validated_conv(),
+ event_pages=event_pages,
+ )
+ runtime = {
+ 'url': 'https://sandbox.example.com',
+ 'session_api_key': 'test-key',
+ }
+ with ExitStack() as stack:
+ for patch_cm in patches:
+ stack.enter_context(patch_cm)
+ await refresh_conversation(
+ app_conversation_info=self._app_conv(),
+ runtime=runtime,
+ httpx_client=httpx_client,
+ )
+
+ # The conversation fetch and at least one events fetch happened...
+ assert len(open_counts_during_network) >= 2, (
+ 'expected the conversation fetch and an events fetch'
+ )
+ # ...and no DB session was open during any of them.
+ assert all(count == 0 for count in open_counts_during_network), (
+ 'DB session must NOT be active during network I/O to prevent '
+ f"'idle in transaction' issues; observed: {open_counts_during_network}"
+ )
+ # Non-vacuous: a write session really did open at some point.
+ assert tracker.max_open >= 1, (
+ 'expected refresh_conversation to open a write session'
+ )
+ assert tracker.open == 0