π OpenHands: AI-Driven Development
fix(app-server): prevent webhook callbacks from starving the database pool (#15379)
Alona652503005093
diff --git a/openhands/app_server/config.py b/openhands/app_server/config.pyindex b899313dd9..0b674aba23 100644--- a/openhands/app_server/config.py+++ b/openhands/app_server/config.py@@ -2,7 +2,7 @@import osfrom pathlib import Path-from typing import AsyncContextManager+from typing import AsyncContextManager, Literalimport httpxfrom fastapi import Depends, Request@@ -524,10 +524,10 @@ def get_app_lifespan_service() -> AppLifespanService | None:return config.lifespan-def depends_event_service():+def depends_event_service(*, scope: Literal['function', 'request'] | None = None):injector = get_global_config().eventassert injector is not None- return Depends(injector.depends)+ return Depends(injector.depends, scope=scope)def depends_event_callback_service():@@ -548,10 +548,12 @@ def depends_sandbox_spec_service():return Depends(injector.depends)-def depends_app_conversation_info_service():+def depends_app_conversation_info_service(+ *, scope: Literal['function', 'request'] | None = None+):injector = get_global_config().app_conversation_infoassert injector is not None- return Depends(injector.depends)+ return Depends(injector.depends, scope=scope)def depends_app_conversation_start_task_service():diff --git a/openhands/app_server/event_callback/set_title_callback_processor.py b/openhands/app_server/event_callback/set_title_callback_processor.pyindex 975f9bb98e..8b7e78ca43 100644--- a/openhands/app_server/event_callback/set_title_callback_processor.py+++ b/openhands/app_server/event_callback/set_title_callback_processor.py@@ -5,9 +5,6 @@ from uuid import UUIDimport httpx-from openhands.app_server.app_conversation.app_conversation_models import (- AppConversationInfo,-)from openhands.app_server.event_callback.event_callback_models import (EventCallback,EventCallbackProcessor,@@ -32,6 +29,10 @@ _logger = logging.getLogger(__name__)_POLL_DELAY_S = 3# Number of attempts to poll title_NUM_POLL_ATTEMPTS = 4+# Avoid starting one slow title poll per webhook event. This set is scoped to a+# worker process; with multiple workers, at most one poll per worker and+# conversation can run at a time.+_CONVERSATIONS_BEING_POLLED: set[UUID] = set()async def _poll_for_title(@@ -92,6 +93,30 @@ class SetTitleCallbackProcessor(EventCallbackProcessor):) -> EventCallbackResult | None:if not isinstance(event, MessageEvent):return None++ if conversation_id in _CONVERSATIONS_BEING_POLLED:+ _logger.debug(+ 'Skipping duplicate title poll for conversation %s',+ conversation_id,+ )+ return None++ _CONVERSATIONS_BEING_POLLED.add(conversation_id)+ try:+ return await self._process_callback(+ conversation_id,+ callback,+ event,+ )+ finally:+ _CONVERSATIONS_BEING_POLLED.discard(conversation_id)++ async def _process_callback(+ self,+ conversation_id: UUID,+ callback: EventCallback,+ event: MessageEvent,+ ) -> EventCallbackResult | None:from openhands.app_server.config import (get_app_conversation_info_service,get_app_conversation_service,@@ -105,14 +130,9 @@ class SetTitleCallbackProcessor(EventCallbackProcessor):redact_text_secrets(str(event)),)- state = InjectorState()- setattr(state, USER_CONTEXT_ATTR, ADMIN)- async with (- get_event_callback_service(state) as event_callback_service,- get_app_conversation_service(state) as app_conversation_service,- get_app_conversation_info_service(state) as app_conversation_info_service,- get_httpx_client(state) as httpx_client,- ):+ read_state = InjectorState()+ setattr(read_state, USER_CONTEXT_ATTR, ADMIN)+ async with get_app_conversation_service(read_state) as app_conversation_service:app_conversation = await app_conversation_service.get_app_conversation(conversation_id)@@ -122,32 +142,40 @@ class SetTitleCallbackProcessor(EventCallbackProcessor):app_conversation_url = replace_localhost_hostname_for_docker(app_conversation_url)+ session_api_key = app_conversation.session_api_key+ http_state = InjectorState()+ setattr(http_state, USER_CONTEXT_ATTR, ADMIN)+ async with get_httpx_client(http_state) as httpx_client:title = await _poll_for_title(httpx_client,app_conversation_url,- app_conversation.session_api_key,+ session_api_key,)- if not title:- # Keep the callback active so later message events can retry.- _logger.info(- f'Conversation {conversation_id} title not available yet; '- 'will retry on a future message event.'- )- return None-- # Save the conversation info- info = AppConversationInfo(- **{- name: getattr(app_conversation, name)- for name in AppConversationInfo.model_fields- }+ if not title:+ # Keep the callback active so later message events can retry.+ _logger.info(+ f'Conversation {conversation_id} title not available yet; '+ 'will retry on a future message event.'+ )+ return None++ info_state = InjectorState()+ setattr(info_state, USER_CONTEXT_ATTR, ADMIN)+ async with get_app_conversation_info_service(+ info_state+ ) as app_conversation_info_service:+ info = await app_conversation_info_service.get_app_conversation_info(+ conversation_id)+ assert info is not Noneinfo.title = titleawait app_conversation_info_service.save_app_conversation_info(info)- # Disable callback - we have already set the status+ callback_state = InjectorState()+ setattr(callback_state, USER_CONTEXT_ATTR, ADMIN)+ async with get_event_callback_service(callback_state) as event_callback_service:callback.status = EventCallbackStatus.DISABLEDawait event_callback_service.save_event_callback(callback)diff --git a/openhands/app_server/event_callback/webhook_router.py b/openhands/app_server/event_callback/webhook_router.pyindex 78cc9317d0..c7d9fe7103 100644--- a/openhands/app_server/event_callback/webhook_router.py+++ b/openhands/app_server/event_callback/webhook_router.py@@ -82,8 +82,10 @@ from openhands.sdk.settings.acp_providers import detect_acp_provider_by_commandfrom openhands.sdk.tool.builtins import SwitchLLMObservationrouter = APIRouter(prefix='/webhooks', tags=['Webhooks'])-event_service_dependency = depends_event_service()-app_conversation_info_service_dependency = depends_app_conversation_info_service()+event_service_dependency = depends_event_service(scope='function')+app_conversation_info_service_dependency = depends_app_conversation_info_service(+ scope='function'+)jwt_dependency = depends_jwt_service()app_mode = get_global_config().app_mode_logger = logging.getLogger(__name__)diff --git a/tests/unit/app_server/test_set_title_callback_processor.py b/tests/unit/app_server/test_set_title_callback_processor.pyindex ef0257be82..5cb82f8e52 100644--- a/tests/unit/app_server/test_set_title_callback_processor.py+++ b/tests/unit/app_server/test_set_title_callback_processor.py@@ -1,5 +1,6 @@from __future__ import annotations+import asynciofrom contextlib import asynccontextmanagerfrom unittest.mock import AsyncMock, patchfrom uuid import uuid4@@ -134,6 +135,148 @@ async def test_set_title_callback_processor_fetches_title_from_conversation():event_callback_service.save_event_callback.assert_called_once()+async def test_set_title_callback_processor_releases_db_context_before_polling():+ conversation_id = uuid4()+ conversation_url = f'http://localhost:8000/api/conversations/{conversation_id.hex}'+ app_conversation = AppConversation(+ id=conversation_id,+ created_by_user_id='user',+ sandbox_id='sandbox',+ title=f'Conversation {conversation_id.hex[:5]}',+ conversation_url=conversation_url,+ session_api_key='test-session-key',+ )+ app_conversation_service = AsyncMock()+ app_conversation_service.get_app_conversation.return_value = app_conversation+ app_conversation_info_service = AsyncMock()+ event_callback_service = AsyncMock()+ httpx_client = _FakeHttpxClient(titles=[None])+ open_db_contexts = 0++ @asynccontextmanager+ async def tracked_db_ctx(obj):+ nonlocal open_db_contexts+ open_db_contexts += 1+ try:+ yield obj+ finally:+ open_db_contexts -= 1++ async def poll_for_title(*_args):+ assert open_db_contexts == 0+ return None++ callback = EventCallback(+ conversation_id=conversation_id, processor=SetTitleCallbackProcessor()+ )+ event = MessageEvent(+ source='user',+ llm_message=Message(role='user', content=[TextContent(text='hi')]),+ )++ with (+ patch(+ 'openhands.app_server.config.get_app_conversation_service',+ side_effect=lambda _state: tracked_db_ctx(app_conversation_service),+ ),+ patch(+ 'openhands.app_server.config.get_app_conversation_info_service',+ side_effect=lambda _state: tracked_db_ctx(app_conversation_info_service),+ ),+ patch(+ 'openhands.app_server.config.get_event_callback_service',+ side_effect=lambda _state: tracked_db_ctx(event_callback_service),+ ),+ patch(+ 'openhands.app_server.config.get_httpx_client',+ side_effect=lambda _state: _ctx(httpx_client),+ ),+ patch(+ 'openhands.app_server.event_callback.'+ 'set_title_callback_processor._poll_for_title',+ side_effect=poll_for_title,+ ),+ ):+ result = await SetTitleCallbackProcessor()(conversation_id, callback, event)++ assert result is None+ assert open_db_contexts == 0+++async def test_set_title_callback_processor_deduplicates_concurrent_polls():+ conversation_id = uuid4()+ conversation_url = f'http://localhost:8000/api/conversations/{conversation_id.hex}'+ app_conversation = AppConversation(+ id=conversation_id,+ created_by_user_id='user',+ sandbox_id='sandbox',+ title=f'Conversation {conversation_id.hex[:5]}',+ conversation_url=conversation_url,+ session_api_key='test-session-key',+ )+ app_conversation_service = AsyncMock()+ app_conversation_service.get_app_conversation.return_value = app_conversation+ app_conversation_info_service = AsyncMock()+ event_callback_service = AsyncMock()+ httpx_client = _FakeHttpxClient(titles=[None])+ poll_started = asyncio.Event()+ finish_poll = asyncio.Event()++ async def poll_for_title(*_args):+ poll_started.set()+ await finish_poll.wait()+ return None++ def make_callback():+ return EventCallback(+ conversation_id=conversation_id, processor=SetTitleCallbackProcessor()+ )++ event = MessageEvent(+ source='user',+ llm_message=Message(role='user', content=[TextContent(text='hi')]),+ )+ processor = SetTitleCallbackProcessor()++ with (+ patch(+ 'openhands.app_server.config.get_app_conversation_service',+ side_effect=lambda _state: _ctx(app_conversation_service),+ ),+ patch(+ 'openhands.app_server.config.get_app_conversation_info_service',+ side_effect=lambda _state: _ctx(app_conversation_info_service),+ ),+ patch(+ 'openhands.app_server.config.get_event_callback_service',+ side_effect=lambda _state: _ctx(event_callback_service),+ ),+ patch(+ 'openhands.app_server.config.get_httpx_client',+ side_effect=lambda _state: _ctx(httpx_client),+ ),+ patch(+ 'openhands.app_server.event_callback.'+ 'set_title_callback_processor._poll_for_title',+ side_effect=poll_for_title,+ ) as poll_mock,+ ):+ first = asyncio.create_task(processor(conversation_id, make_callback(), event))+ await poll_started.wait()+ second_result = await asyncio.wait_for(+ processor(conversation_id, make_callback(), event),+ timeout=0.1,+ )+ finish_poll.set()+ first_result = await first++ assert first_result is None+ assert second_result is None+ poll_mock.assert_awaited_once()++@pytest.mark.asyncioasync def test_set_title_callback_processor_no_title_yet_returns_none():conversation_id = uuid4()diff --git a/tests/unit/app_server/test_webhook_router_auth.py b/tests/unit/app_server/test_webhook_router_auth.pyindex 706b34e4d8..c09ed70f75 100644--- a/tests/unit/app_server/test_webhook_router_auth.py+++ b/tests/unit/app_server/test_webhook_router_auth.py@@ -13,12 +13,14 @@ from fastapi.testclient import TestClientfrom openhands.app_server.config_api.config_models import AppModefrom openhands.app_server.event_callback.webhook_router import (- router as webhook_router,-)-from openhands.app_server.event_callback.webhook_router import (+ app_conversation_info_service_dependency,+ event_service_dependency,valid_conversation,valid_sandbox,)+from openhands.app_server.event_callback.webhook_router import (+ router as webhook_router,+)from openhands.app_server.sandbox.sandbox_models import SandboxRecordfrom openhands.app_server.user.specifiy_user_context import (USER_CONTEXT_ATTR,@@ -54,6 +56,11 @@ def create_mock_request():return request+def test_webhook_db_dependencies_close_before_background_tasks():+ assert app_conversation_info_service_dependency.scope == 'function'+ assert event_service_dependency.scope == 'function'++def create_sandbox_service_context_manager(sandbox_service):"""Create an async context manager that yields the given sandbox service."""