Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix(app-server): prevent webhook callbacks from starving the database pool (#15379)

Alona652503005093
diff --git a/openhands/app_server/config.py b/openhands/app_server/config.py
index b899313dd9..0b674aba23 100644
--- a/openhands/app_server/config.py
+++ b/openhands/app_server/config.py
@@ -2,7 +2,7 @@
import os
from pathlib import Path
-from typing import AsyncContextManager
+from typing import AsyncContextManager, Literal
import httpx
from fastapi import Depends, Request
@@ -524,10 +524,10 @@ def get_app_lifespan_service() -> AppLifespanService | None:
return config.lifespan
-def depends_event_service():
+def depends_event_service(*, scope: Literal['function', 'request'] | None = None):
injector = get_global_config().event
assert injector is not None
- return Depends(injector.depends)
+ return Depends(injector.depends, scope=scope)
def depends_event_callback_service():
@@ -548,10 +548,12 @@ def depends_sandbox_spec_service():
return Depends(injector.depends)
-def depends_app_conversation_info_service():
+def depends_app_conversation_info_service(
+ *, scope: Literal['function', 'request'] | None = None
+):
injector = get_global_config().app_conversation_info
assert injector is not None
- return Depends(injector.depends)
+ return Depends(injector.depends, scope=scope)
def depends_app_conversation_start_task_service():
diff --git a/openhands/app_server/event_callback/set_title_callback_processor.py b/openhands/app_server/event_callback/set_title_callback_processor.py
index 975f9bb98e..8b7e78ca43 100644
--- a/openhands/app_server/event_callback/set_title_callback_processor.py
+++ b/openhands/app_server/event_callback/set_title_callback_processor.py
@@ -5,9 +5,6 @@ from uuid import UUID
import httpx
-from openhands.app_server.app_conversation.app_conversation_models import (
- AppConversationInfo,
-)
from openhands.app_server.event_callback.event_callback_models import (
EventCallback,
EventCallbackProcessor,
@@ -32,6 +29,10 @@ _logger = logging.getLogger(__name__)
_POLL_DELAY_S = 3
# Number of attempts to poll title
_NUM_POLL_ATTEMPTS = 4
+# Avoid starting one slow title poll per webhook event. This set is scoped to a
+# worker process; with multiple workers, at most one poll per worker and
+# conversation can run at a time.
+_CONVERSATIONS_BEING_POLLED: set[UUID] = set()
async def _poll_for_title(
@@ -92,6 +93,30 @@ class SetTitleCallbackProcessor(EventCallbackProcessor):
) -> EventCallbackResult | None:
if not isinstance(event, MessageEvent):
return None
+
+ if conversation_id in _CONVERSATIONS_BEING_POLLED:
+ _logger.debug(
+ 'Skipping duplicate title poll for conversation %s',
+ conversation_id,
+ )
+ return None
+
+ _CONVERSATIONS_BEING_POLLED.add(conversation_id)
+ try:
+ return await self._process_callback(
+ conversation_id,
+ callback,
+ event,
+ )
+ finally:
+ _CONVERSATIONS_BEING_POLLED.discard(conversation_id)
+
+ async def _process_callback(
+ self,
+ conversation_id: UUID,
+ callback: EventCallback,
+ event: MessageEvent,
+ ) -> EventCallbackResult | None:
from openhands.app_server.config import (
get_app_conversation_info_service,
get_app_conversation_service,
@@ -105,14 +130,9 @@ class SetTitleCallbackProcessor(EventCallbackProcessor):
redact_text_secrets(str(event)),
)
- state = InjectorState()
- setattr(state, USER_CONTEXT_ATTR, ADMIN)
- async with (
- get_event_callback_service(state) as event_callback_service,
- get_app_conversation_service(state) as app_conversation_service,
- get_app_conversation_info_service(state) as app_conversation_info_service,
- get_httpx_client(state) as httpx_client,
- ):
+ read_state = InjectorState()
+ setattr(read_state, USER_CONTEXT_ATTR, ADMIN)
+ async with get_app_conversation_service(read_state) as app_conversation_service:
app_conversation = await app_conversation_service.get_app_conversation(
conversation_id
)
@@ -122,32 +142,40 @@ class SetTitleCallbackProcessor(EventCallbackProcessor):
app_conversation_url = replace_localhost_hostname_for_docker(
app_conversation_url
)
+ session_api_key = app_conversation.session_api_key
+ http_state = InjectorState()
+ setattr(http_state, USER_CONTEXT_ATTR, ADMIN)
+ async with get_httpx_client(http_state) as httpx_client:
title = await _poll_for_title(
httpx_client,
app_conversation_url,
- app_conversation.session_api_key,
+ session_api_key,
)
- if not title:
- # Keep the callback active so later message events can retry.
- _logger.info(
- f'Conversation {conversation_id} title not available yet; '
- 'will retry on a future message event.'
- )
- return None
-
- # Save the conversation info
- info = AppConversationInfo(
- **{
- name: getattr(app_conversation, name)
- for name in AppConversationInfo.model_fields
- }
+ if not title:
+ # Keep the callback active so later message events can retry.
+ _logger.info(
+ f'Conversation {conversation_id} title not available yet; '
+ 'will retry on a future message event.'
+ )
+ return None
+
+ info_state = InjectorState()
+ setattr(info_state, USER_CONTEXT_ATTR, ADMIN)
+ async with get_app_conversation_info_service(
+ info_state
+ ) as app_conversation_info_service:
+ info = await app_conversation_info_service.get_app_conversation_info(
+ conversation_id
)
+ assert info is not None
info.title = title
await app_conversation_info_service.save_app_conversation_info(info)
- # Disable callback - we have already set the status
+ callback_state = InjectorState()
+ setattr(callback_state, USER_CONTEXT_ATTR, ADMIN)
+ async with get_event_callback_service(callback_state) as event_callback_service:
callback.status = EventCallbackStatus.DISABLED
await event_callback_service.save_event_callback(callback)
diff --git a/openhands/app_server/event_callback/webhook_router.py b/openhands/app_server/event_callback/webhook_router.py
index 78cc9317d0..c7d9fe7103 100644
--- a/openhands/app_server/event_callback/webhook_router.py
+++ b/openhands/app_server/event_callback/webhook_router.py
@@ -82,8 +82,10 @@ from openhands.sdk.settings.acp_providers import detect_acp_provider_by_command
from openhands.sdk.tool.builtins import SwitchLLMObservation
router = APIRouter(prefix='/webhooks', tags=['Webhooks'])
-event_service_dependency = depends_event_service()
-app_conversation_info_service_dependency = depends_app_conversation_info_service()
+event_service_dependency = depends_event_service(scope='function')
+app_conversation_info_service_dependency = depends_app_conversation_info_service(
+ scope='function'
+)
jwt_dependency = depends_jwt_service()
app_mode = get_global_config().app_mode
_logger = logging.getLogger(__name__)
diff --git a/tests/unit/app_server/test_set_title_callback_processor.py b/tests/unit/app_server/test_set_title_callback_processor.py
index ef0257be82..5cb82f8e52 100644
--- a/tests/unit/app_server/test_set_title_callback_processor.py
+++ b/tests/unit/app_server/test_set_title_callback_processor.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from contextlib import asynccontextmanager
from unittest.mock import AsyncMock, patch
from uuid import uuid4
@@ -134,6 +135,148 @@ async def test_set_title_callback_processor_fetches_title_from_conversation():
event_callback_service.save_event_callback.assert_called_once()
+async def test_set_title_callback_processor_releases_db_context_before_polling():
+ conversation_id = uuid4()
+ conversation_url = f'http://localhost:8000/api/conversations/{conversation_id.hex}'
+ app_conversation = AppConversation(
+ id=conversation_id,
+ created_by_user_id='user',
+ sandbox_id='sandbox',
+ title=f'Conversation {conversation_id.hex[:5]}',
+ conversation_url=conversation_url,
+ session_api_key='test-session-key',
+ )
+ app_conversation_service = AsyncMock()
+ app_conversation_service.get_app_conversation.return_value = app_conversation
+ app_conversation_info_service = AsyncMock()
+ event_callback_service = AsyncMock()
+ httpx_client = _FakeHttpxClient(titles=[None])
+ open_db_contexts = 0
+
+ @asynccontextmanager
+ async def tracked_db_ctx(obj):
+ nonlocal open_db_contexts
+ open_db_contexts += 1
+ try:
+ yield obj
+ finally:
+ open_db_contexts -= 1
+
+ async def poll_for_title(*_args):
+ assert open_db_contexts == 0
+ return None
+
+ callback = EventCallback(
+ conversation_id=conversation_id, processor=SetTitleCallbackProcessor()
+ )
+ event = MessageEvent(
+ source='user',
+ llm_message=Message(role='user', content=[TextContent(text='hi')]),
+ )
+
+ with (
+ patch(
+ 'openhands.app_server.config.get_app_conversation_service',
+ side_effect=lambda _state: tracked_db_ctx(app_conversation_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_app_conversation_info_service',
+ side_effect=lambda _state: tracked_db_ctx(app_conversation_info_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_event_callback_service',
+ side_effect=lambda _state: tracked_db_ctx(event_callback_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_httpx_client',
+ side_effect=lambda _state: _ctx(httpx_client),
+ ),
+ patch(
+ 'openhands.app_server.event_callback.'
+ 'set_title_callback_processor._poll_for_title',
+ side_effect=poll_for_title,
+ ),
+ ):
+ result = await SetTitleCallbackProcessor()(conversation_id, callback, event)
+
+ assert result is None
+ assert open_db_contexts == 0
+
+
+async def test_set_title_callback_processor_deduplicates_concurrent_polls():
+ conversation_id = uuid4()
+ conversation_url = f'http://localhost:8000/api/conversations/{conversation_id.hex}'
+ app_conversation = AppConversation(
+ id=conversation_id,
+ created_by_user_id='user',
+ sandbox_id='sandbox',
+ title=f'Conversation {conversation_id.hex[:5]}',
+ conversation_url=conversation_url,
+ session_api_key='test-session-key',
+ )
+ app_conversation_service = AsyncMock()
+ app_conversation_service.get_app_conversation.return_value = app_conversation
+ app_conversation_info_service = AsyncMock()
+ event_callback_service = AsyncMock()
+ httpx_client = _FakeHttpxClient(titles=[None])
+ poll_started = asyncio.Event()
+ finish_poll = asyncio.Event()
+
+ async def poll_for_title(*_args):
+ poll_started.set()
+ await finish_poll.wait()
+ return None
+
+ def make_callback():
+ return EventCallback(
+ conversation_id=conversation_id, processor=SetTitleCallbackProcessor()
+ )
+
+ event = MessageEvent(
+ source='user',
+ llm_message=Message(role='user', content=[TextContent(text='hi')]),
+ )
+ processor = SetTitleCallbackProcessor()
+
+ with (
+ patch(
+ 'openhands.app_server.config.get_app_conversation_service',
+ side_effect=lambda _state: _ctx(app_conversation_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_app_conversation_info_service',
+ side_effect=lambda _state: _ctx(app_conversation_info_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_event_callback_service',
+ side_effect=lambda _state: _ctx(event_callback_service),
+ ),
+ patch(
+ 'openhands.app_server.config.get_httpx_client',
+ side_effect=lambda _state: _ctx(httpx_client),
+ ),
+ patch(
+ 'openhands.app_server.event_callback.'
+ 'set_title_callback_processor._poll_for_title',
+ side_effect=poll_for_title,
+ ) as poll_mock,
+ ):
+ first = asyncio.create_task(processor(conversation_id, make_callback(), event))
+ await poll_started.wait()
+ second_result = await asyncio.wait_for(
+ processor(conversation_id, make_callback(), event),
+ timeout=0.1,
+ )
+ finish_poll.set()
+ first_result = await first
+
+ assert first_result is None
+ assert second_result is None
+ poll_mock.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_set_title_callback_processor_no_title_yet_returns_none():
conversation_id = uuid4()
diff --git a/tests/unit/app_server/test_webhook_router_auth.py b/tests/unit/app_server/test_webhook_router_auth.py
index 706b34e4d8..c09ed70f75 100644
--- a/tests/unit/app_server/test_webhook_router_auth.py
+++ b/tests/unit/app_server/test_webhook_router_auth.py
@@ -13,12 +13,14 @@ from fastapi.testclient import TestClient
from openhands.app_server.config_api.config_models import AppMode
from openhands.app_server.event_callback.webhook_router import (
- router as webhook_router,
-)
-from openhands.app_server.event_callback.webhook_router import (
+ app_conversation_info_service_dependency,
+ event_service_dependency,
valid_conversation,
valid_sandbox,
)
+from openhands.app_server.event_callback.webhook_router import (
+ router as webhook_router,
+)
from openhands.app_server.sandbox.sandbox_models import SandboxRecord
from openhands.app_server.user.specifiy_user_context import (
USER_CONTEXT_ATTR,
@@ -54,6 +56,11 @@ def create_mock_request():
return request
+def test_webhook_db_dependencies_close_before_background_tasks():
+ assert app_conversation_info_service_dependency.scope == 'function'
+ assert event_service_dependency.scope == 'function'
+
+
def create_sandbox_service_context_manager(sandbox_service):
"""Create an async context manager that yields the given sandbox service."""