Skip to content

πŸ™Œ OpenHands: AI-Driven Development

fix: prevent webhook-driven DB connection leaks (#15212)

Tim O'Farrellecb9b4ad369d
diff --git a/enterprise/server/app_lifespan/saas_app_lifespan_service.py b/enterprise/server/app_lifespan/saas_app_lifespan_service.py
index 85b6ed42c8..12d55132d9 100644
--- a/enterprise/server/app_lifespan/saas_app_lifespan_service.py
+++ b/enterprise/server/app_lifespan/saas_app_lifespan_service.py
@@ -42,3 +42,14 @@ class SaasAppLifespanService(AppLifespanService):
svc.shutdown()
except Exception:
logger.exception('Error shutting down analytics service')
+
+ # Release long-lived database resources: the GCP Cloud SQL connector
+ # (background cert-refresh tasks + aiohttp ClientSession) and the
+ # SQLAlchemy async engine's pool. Without this, every worker respawn
+ # leaves the connector's tasks running on the previous event loop.
+ try:
+ from openhands.app_server.config import get_global_config
+
+ await get_global_config().db_session.close()
+ except Exception:
+ logger.exception('Error closing DB session injector')
diff --git a/openhands/app_server/event_callback/event_callback_service.py b/openhands/app_server/event_callback/event_callback_service.py
index 2d27884bc9..3e8b567ee2 100644
--- a/openhands/app_server/event_callback/event_callback_service.py
+++ b/openhands/app_server/event_callback/event_callback_service.py
@@ -61,6 +61,28 @@ class EventCallbackService(ABC):
async def execute_callbacks(self, conversation_id: UUID, event: Event) -> None:
"""Execute any applicable callbacks for the event and store the results."""
+ @abstractmethod
+ async def get_active_callbacks(
+ self, conversation_id: UUID, event: Event
+ ) -> list[EventCallback]:
+ """Return active callbacks registered for the conversation+event kind.
+
+ Implementations should keep the DB session short-lived so the caller
+ can run (potentially slow) callback processors without holding a pool
+ connection. Pair with :meth:`persist_callback_results` afterwards."""
+
+ @abstractmethod
+ async def persist_callback_results(
+ self,
+ callbacks: list[EventCallback],
+ results: list,
+ ) -> None:
+ """Persist callback status changes and their results.
+
+ Pairs up ``callbacks`` and ``results`` index-wise. ``results[i]`` may be
+ ``None`` if the callback returned ``None``. Implementations should keep
+ the DB session short-lived."""
+
class EventCallbackServiceInjector(
DiscriminatedUnionMixin, Injector[EventCallbackService], ABC
diff --git a/openhands/app_server/event_callback/sql_event_callback_service.py b/openhands/app_server/event_callback/sql_event_callback_service.py
index 0b36fb53ca..4e963f8f21 100644
--- a/openhands/app_server/event_callback/sql_event_callback_service.py
+++ b/openhands/app_server/event_callback/sql_event_callback_service.py
@@ -12,7 +12,7 @@ from uuid import UUID, uuid4
from fastapi import Request
from sqlalchemy import Enum, Index, String, and_, func, select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlalchemy.orm import Mapped, mapped_column
from openhands.agent_server.utils import utc_now
@@ -91,49 +91,61 @@ class StoredEventCallbackResult(Base):
@dataclass
class SQLEventCallbackService(EventCallbackService):
- """SQL implementation of EventCallbackService."""
+ """SQL implementation of EventCallbackService.
- db_session: AsyncSession
+ The service does **not** hold a long-lived ``AsyncSession``. Each public
+ method opens a short-lived session via ``self.async_session_maker`` for the
+ duration of its own DB work. This means:
+
+ * The pool connection is only checked out while the method runs an
+ actual SQL statement β€” it is released as soon as the method returns,
+ so callers never accidentally pin a connection across slow logic
+ (e.g. webhook deliveries, ``asyncio.gather`` of callback processors).
+ * Methods are independent: a failure in one (e.g. a flush error) cannot
+ leave another with a session in an unknown transactional state.
+ * The service itself is no longer an async-context-manager. There is
+ no per-service ``__aenter__``/``__aexit__`` lifecycle to get wrong.
+ """
+
+ async_session_maker: async_sessionmaker
async def create_event_callback(
self, request: CreateEventCallbackRequest
) -> EventCallback:
"""Create a new event callback."""
- # Create EventCallback from request
event_callback = EventCallback(
conversation_id=request.conversation_id,
processor=request.processor,
event_kind=request.event_kind,
)
-
- # Create stored version and add to db_session
- stored_callback = StoredEventCallback(**event_callback.model_dump())
- self.db_session.add(stored_callback)
- await self.db_session.commit()
- await self.db_session.refresh(stored_callback)
- return EventCallback.model_validate(row2dict(stored_callback))
+ async with self.async_session_maker() as db_session:
+ stored_callback = StoredEventCallback(**event_callback.model_dump())
+ db_session.add(stored_callback)
+ await db_session.commit()
+ await db_session.refresh(stored_callback)
+ return EventCallback.model_validate(row2dict(stored_callback))
async def get_event_callback(self, id: UUID) -> EventCallback | None:
"""Get a single event callback, returning None if not found."""
- stmt = select(StoredEventCallback).where(StoredEventCallback.id == id)
- result = await self.db_session.execute(stmt)
- stored_callback = result.scalar_one_or_none()
- if stored_callback:
- return EventCallback.model_validate(row2dict(stored_callback))
- return None
+ async with self.async_session_maker() as db_session:
+ stmt = select(StoredEventCallback).where(StoredEventCallback.id == id)
+ result = await db_session.execute(stmt)
+ stored_callback = result.scalar_one_or_none()
+ if stored_callback:
+ return EventCallback.model_validate(row2dict(stored_callback))
+ return None
async def delete_event_callback(self, id: UUID) -> bool:
"""Delete an event callback, returning True if deleted, False if not found."""
- stmt = select(StoredEventCallback).where(StoredEventCallback.id == id)
- result = await self.db_session.execute(stmt)
- stored_callback = result.scalar_one_or_none()
-
- if stored_callback is None:
- return False
-
- await self.db_session.delete(stored_callback)
- await self.db_session.commit()
- return True
+ async with self.async_session_maker() as db_session:
+ stmt = select(StoredEventCallback).where(StoredEventCallback.id == id)
+ result = await db_session.execute(stmt)
+ stored_callback = result.scalar_one_or_none()
+ if stored_callback is None:
+ return False
+ await db_session.delete(stored_callback)
+ await db_session.commit()
+ return True
async def search_event_callbacks(
self,
@@ -144,56 +156,38 @@ class SQLEventCallbackService(EventCallbackService):
limit: int = 100,
) -> EventCallbackPage:
"""Search for event callbacks, optionally filtered by parameters."""
- # Build the query with filters
conditions = []
-
if conversation_id__eq is not None:
conditions.append(
StoredEventCallback.conversation_id == conversation_id__eq
)
-
if event_kind__eq is not None:
conditions.append(StoredEventCallback.event_kind == event_kind__eq)
+ # Note: event_id__eq is not stored in the event_callbacks table; kept
+ # in the signature for ABI compatibility with the abstract service.
- # Note: event_id__eq is not stored in the event_callbacks table
- # This parameter might be used for filtering results after retrieval
- # or might be intended for a different use case
-
- # Build the base query
stmt = select(StoredEventCallback)
-
if conditions:
stmt = stmt.where(and_(*conditions))
- # Handle pagination
if page_id is not None:
- # Parse page_id to get offset or cursor
try:
offset = int(page_id)
stmt = stmt.offset(offset)
except ValueError:
- # If page_id is not a valid integer, start from beginning
offset = 0
else:
offset = 0
-
- # Apply limit and get one extra to check if there are more results
stmt = stmt.limit(limit + 1).order_by(StoredEventCallback.created_at.desc())
- result = await self.db_session.execute(stmt)
- stored_callbacks = result.scalars().all()
+ async with self.async_session_maker() as db_session:
+ result = await db_session.execute(stmt)
+ stored_callbacks = result.scalars().all()
- # Check if there are more results
has_more = len(stored_callbacks) > limit
if has_more:
stored_callbacks = stored_callbacks[:limit]
-
- # Calculate next page ID
- next_page_id = None
- if has_more:
- next_page_id = str(offset + limit)
-
- # Convert stored callbacks to domain models
+ next_page_id = str(offset + limit) if has_more else None
callbacks = [
EventCallback.model_validate(row2dict(cb)) for cb in stored_callbacks
]
@@ -201,64 +195,129 @@ class SQLEventCallbackService(EventCallbackService):
async def save_event_callback(self, event_callback: EventCallback) -> EventCallback:
event_callback.updated_at = utc_now()
- stored_callback = StoredEventCallback(**event_callback.model_dump())
- await self.db_session.merge(stored_callback)
+ async with self.async_session_maker() as db_session:
+ stored_callback = StoredEventCallback(**event_callback.model_dump())
+ await db_session.merge(stored_callback)
+ await db_session.commit()
return event_callback
- async def execute_callbacks(self, conversation_id: UUID, event: Event) -> None:
+ async def get_active_callbacks(
+ self, conversation_id: UUID, event: Event
+ ) -> list[EventCallback]:
+ """Return the active callbacks registered for this conversation+event kind.
+
+ Each returned ``EventCallback`` is detached from the session, so the
+ caller can run the (potentially slow) callback processors without
+ holding a pool connection. Use :meth:`persist_callback_results`
+ afterwards to save any status changes the callbacks made to themselves
+ plus the ``EventCallbackResult`` rows they produced.
+ """
query = (
select(StoredEventCallback)
.where(StoredEventCallback.status == EventCallbackStatus.ACTIVE)
.where(StoredEventCallback.event_kind == event.kind)
.where(StoredEventCallback.conversation_id == conversation_id)
)
- result = await self.db_session.execute(query)
- stored_callbacks = result.scalars().all()
- if stored_callbacks:
- callbacks = [
- EventCallback.model_validate(row2dict(cb)) for cb in stored_callbacks
- ]
- await asyncio.gather(
- *[
- self.execute_callback(conversation_id, callback, event)
- for callback in callbacks
- ]
- )
+ async with self.async_session_maker() as db_session:
+ result = await db_session.execute(query)
+ stored_callbacks = result.scalars().all()
+ return [EventCallback.model_validate(row2dict(cb)) for cb in stored_callbacks]
- # Persist any new changes callbacks may have made to itself
- for callback in callbacks:
- await self.save_event_callback(callback)
- await self.db_session.commit()
-
- async def execute_callback(
- self, conversation_id: UUID, callback: EventCallback, event: Event
- ):
- try:
- result = await callback.processor(conversation_id, callback, event)
- if result is None:
- return
- stored_result = StoredEventCallbackResult(**result.model_dump())
- except Exception as exc:
- _logger.exception(f'Exception in callback {callback.id}', stack_info=True)
- stored_result = StoredEventCallbackResult(
- status=EventCallbackResultStatus.ERROR,
- event_callback_id=callback.id,
- event_id=event.id,
- conversation_id=conversation_id,
- detail=str(exc),
- )
- self.db_session.add(stored_result)
+ async def persist_callback_results(
+ self,
+ callbacks: list[EventCallback],
+ results: list[StoredEventCallbackResult | None],
+ ) -> None:
+ """Persist callback status changes and their results.
+
+ Pairs up ``callbacks`` and ``results`` index-wise. ``results[i]`` may be
+ ``None`` if the callback returned ``None``. Opens its own short-lived
+ session so the pool connection is only held for the duration of the
+ COMMIT, not the entire callback execution.
+ """
+ async with self.async_session_maker() as db_session:
+ for callback, stored_result in zip(callbacks, results, strict=False):
+ callback.updated_at = utc_now()
+ stored_callback = StoredEventCallback(**callback.model_dump())
+ await db_session.merge(stored_callback)
+ if stored_result is not None:
+ db_session.add(stored_result)
+ if any(r is not None for r in results):
+ await db_session.commit()
- async def __aexit__(self, exc_type, exc_value, traceback):
- """Stop using this event callback service."""
- pass
+ async def execute_callbacks(self, conversation_id: UUID, event: Event) -> None:
+ """Run all active callbacks for the event and persist their results.
+
+ Each step (``get_active_callbacks``, ``invoke_callback``,
+ ``persist_callback_results``) opens and closes its own session, so the
+ pool connection is never pinned while the (potentially slow) callback
+ processors run. Failed callbacks are turned into ERROR result rows so a
+ single failure doesn't abort the rest.
+ """
+ callbacks = await self.get_active_callbacks(conversation_id, event)
+ if not callbacks:
+ return
+ outcomes = await asyncio.gather(
+ *[invoke_callback(cb, conversation_id, event) for cb in callbacks],
+ return_exceptions=True,
+ )
+ normalised: list[StoredEventCallbackResult | None] = []
+ for callback, outcome in zip(callbacks, outcomes, strict=False):
+ if isinstance(outcome, BaseException):
+ _logger.exception(
+ f'Exception in callback {callback.id}', stack_info=True
+ )
+ normalised.append(
+ StoredEventCallbackResult(
+ status=EventCallbackResultStatus.ERROR,
+ event_callback_id=callback.id,
+ event_id=event.id,
+ conversation_id=conversation_id,
+ detail=str(outcome),
+ )
+ )
+ else:
+ normalised.append(outcome)
+ await self.persist_callback_results(callbacks, normalised)
+
+
+async def invoke_callback(
+ callback: EventCallback, conversation_id: UUID, event: Event
+) -> StoredEventCallbackResult | None:
+ """Run a single callback processor and convert its outcome into a stored
+ result row. No DB session is required to call the processor β€” the row is
+ only added to a session later by ``persist_callback_results``. This lets
+ the caller release the pool connection while the (potentially slow)
+ callbacks run, avoiding pool exhaustion when several webhooks fire in
+ burst."""
+ try:
+ result = await callback.processor(conversation_id, callback, event)
+ except Exception as exc:
+ _logger.exception(f'Exception in callback {callback.id}', stack_info=True)
+ return StoredEventCallbackResult(
+ status=EventCallbackResultStatus.ERROR,
+ event_callback_id=callback.id,
+ event_id=event.id,
+ conversation_id=conversation_id,
+ detail=str(exc),
+ )
+ if result is None:
+ return None
+ return StoredEventCallbackResult(**result.model_dump())
class SQLEventCallbackServiceInjector(EventCallbackServiceInjector):
async def inject(
self, state: InjectorState, request: Request | None = None
) -> AsyncGenerator[EventCallbackService, None]:
- from openhands.app_server.config import get_db_session
-
- async with get_db_session(state) as db_session:
- yield SQLEventCallbackService(db_session=db_session)
+ # No async-context-manager needed here: the service does not hold a
+ # session, so there is nothing to release on __aexit__. We just hand
+ # it the cached ``async_sessionmaker`` (which does not open a
+ # connection on its own) and let the service open short-lived
+ # sessions per method call.
+ from openhands.app_server.config import get_global_config
+
+ async_session_maker = (
+ await get_global_config().db_session.get_async_session_maker()
+ )
+ yield SQLEventCallbackService(async_session_maker=async_session_maker)
diff --git a/openhands/app_server/event_callback/webhook_router.py b/openhands/app_server/event_callback/webhook_router.py
index 15b6d94c5e..bd58b60317 100644
--- a/openhands/app_server/event_callback/webhook_router.py
+++ b/openhands/app_server/event_callback/webhook_router.py
@@ -7,7 +7,15 @@ import pkgutil
from typing import Any
from uuid import UUID
-from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
+from fastapi import (
+ APIRouter,
+ BackgroundTasks,
+ Depends,
+ HTTPException,
+ Request,
+ Response,
+ status,
+)
from fastapi.security import APIKeyHeader
from jwt import InvalidTokenError
from pydantic import SecretStr
@@ -36,9 +44,16 @@ from openhands.app_server.config_api.config_models import AppMode
from openhands.app_server.errors import AuthError
from openhands.app_server.event.event_service import EventService
from openhands.app_server.event_callback.event_callback_models import EventCallback
+from openhands.app_server.event_callback.event_callback_result_models import (
+ EventCallbackResultStatus,
+)
from openhands.app_server.event_callback.set_title_callback_processor import (
SetTitleCallbackProcessor,
)
+from openhands.app_server.event_callback.sql_event_callback_service import (
+ StoredEventCallbackResult,
+ invoke_callback,
+)
from openhands.app_server.integrations.provider import ProviderType
from openhands.app_server.sandbox.sandbox_models import SandboxRecord
from openhands.app_server.services.injector import InjectorState
@@ -452,6 +467,7 @@ async def on_conversation_update(
@router.post('/events/{conversation_id}')
async def on_event(
+ background_tasks: BackgroundTasks,
events: list[Event],
conversation_id: UUID,
app_conversation_info: AppConversationInfo = Depends(valid_conversation),
@@ -515,10 +531,11 @@ async def on_event(
except Exception:
_logger.exception('analytics:conversation_terminal:failed')
- asyncio.create_task(
- _run_callbacks_in_bg_and_close(
- conversation_id, app_conversation_info.created_by_user_id, events
- )
+ background_tasks.add_task(
+ _run_callbacks_in_bg_and_close,
+ conversation_id,
+ app_conversation_info.created_by_user_id,
+ events,
)
except Exception:
@@ -568,14 +585,46 @@ async def _run_callbacks_in_bg_and_close(
user_id: str | None,
events: list[Event],
):
- """Run all callbacks and close the session"""
+ """Run all active callbacks for the given events.
+
+ The ``SQLEventCallbackService`` opens its own short-lived ``AsyncSession``
+ per method call, so the SQLAlchemy pool connection is only checked out for
+ the SELECT (in :meth:`get_active_callbacks`) and the COMMIT (in
+ :meth:`persist_callback_results`). The slow callback processors run with
+ **no** connection held β€” which is what stops webhook bursts from
+ exhausting the 25+10 pool and starving subsequent ``asyncpg.connect``
+ attempts.
+ """
state = InjectorState()
setattr(state, USER_CONTEXT_ATTR, SpecifyUserContext(user_id=user_id))
- async with get_event_callback_service(state) as event_callback_service:
- # We don't use asynio.gather here because callbacks must be run in sequence.
+ async with get_event_callback_service(state) as service:
for event in events:
- await event_callback_service.execute_callbacks(conversation_id, event)
+ callbacks = await service.get_active_callbacks(conversation_id, event)
+ if not callbacks:
+ continue
+ outcomes = await asyncio.gather(
+ *[invoke_callback(cb, conversation_id, event) for cb in callbacks],
+ return_exceptions=True,
+ )
+ normalised: list[StoredEventCallbackResult | None] = []
+ for callback, outcome in zip(callbacks, outcomes, strict=False):
+ if isinstance(outcome, BaseException):
+ _logger.exception(
+ f'Exception in callback {callback.id}', stack_info=True
+ )
+ normalised.append(
+ StoredEventCallbackResult(
+ status=EventCallbackResultStatus.ERROR,
+ event_callback_id=callback.id,
+ event_id=event.id,
+ conversation_id=conversation_id,
+ detail=str(outcome),
+ )
+ )
+ else:
+ normalised.append(outcome)
+ await service.persist_callback_results(callbacks, normalised)
def _import_all_tools():
diff --git a/openhands/app_server/services/db_session_injector.py b/openhands/app_server/services/db_session_injector.py
index a0b85a0779..5351200d8f 100644
--- a/openhands/app_server/services/db_session_injector.py
+++ b/openhands/app_server/services/db_session_injector.py
@@ -1,6 +1,7 @@
"""Database configuration and session management for OpenHands App Server."""
import asyncio
+import contextlib
import logging
import os
from pathlib import Path
@@ -320,7 +321,46 @@ class DbSessionInjector(BaseModel, Injector[AsyncSession]):
# Clean up the session from request state
if hasattr(state, DB_SESSION_ATTR):
delattr(state, DB_SESSION_ATTR)
- await db_session.close()
+ # ``close`` is async, so it can still fail (e.g. the pool
+ # is in an inconsistent state because we were cancelled
+ # mid-query). Suppress secondary failures so the original
+ # exception keeps propagating.
+ with contextlib.suppress(Exception):
+ await db_session.close()
+
+ async def close(self) -> None:
+ """Release long-lived resources owned by this injector.
+
+ Currently this closes the GCP Cloud SQL connector (which in turn stops
+ its background cert-refresh tasks and aiohttp ClientSession) and
+ disposes the async SQLAlchemy engine. ``close`` is idempotent and
+ safe to call multiple times.
+
+ Call this from the application lifespan ``__aexit__`` so that workers
+ exiting gracefully don't leak the connector's background resources
+ across respawns.
+ """
+ connector = self._gcp_connector
+ if connector is not None:
+ # ``close_async`` may not exist on older connector versions; fall
+ # back to ``close`` if so.
+ close = getattr(connector, 'close_async', None)
+ if close is None:
+ close = connector.close
+ try:
+ result = close()
+ if asyncio.iscoroutine(result):
+ await result
+ except Exception:
+ _logger.exception('Error closing GCP Cloud SQL connector')
+ self._gcp_connector = None
+
+ engine = self._async_engine
+ if engine is not None:
+ try:
+ await engine.dispose()
+ except Exception:
+ _logger.exception('Error disposing async DB engine')
def set_db_session_keep_open(state: InjectorState, keep_open: bool):
diff --git a/tests/unit/app_server/test_db_session_injector.py b/tests/unit/app_server/test_db_session_injector.py
index 81e28fed3c..bddd88df4d 100644
--- a/tests/unit/app_server/test_db_session_injector.py
+++ b/tests/unit/app_server/test_db_session_injector.py
@@ -602,3 +602,108 @@ class TestDbSessionInjectorEdgeCases:
await session_gen2.__anext__()
except StopAsyncIteration:
pass
+
+ @pytest.mark.asyncio
+ async def test_inject_cleanup_on_cancelled_session(self, basic_db_session_injector):
+ """Cancellation mid-yield should still close the session.
+
+ The injector must call ``db_session.close()`` even when the outer
+ coroutine is being torn down by ``asyncio.CancelledError`` β€” otherwise
+ the SQLAlchemy pool will hold a checked-out connection.
+ """
+
+ closed = []
+ rollback_called = []
+
+ class FakeSession:
+ async def close(self):
+ closed.append(True)
+
+ async def rollback(self):
+ rollback_called.append(True)
+
+ async def commit(self):
+ pass
+
+ fake_session = FakeSession()
+
+ class _FakeSessionMaker:
+ """Stand-in for an ``async_sessionmaker`` callable."""
+
+ def __init__(self, session):
+ self._session = session
+
+ def __call__(self):
+ return self._session
+
+ async def fake_session_maker():
+ class Maker:
+ def __call__(self):
+ return fake_session
+
+ return Maker()
+
+ # Patch ``get_async_session_maker`` on the class so pydantic's frozen
+ # model doesn't refuse the assignment. ``patch.object`` with a
+ # non-callable new value bypasses the auto-wrap-into-MagicMock path.
+ maker_factory = AsyncMock(return_value=_FakeSessionMaker(fake_session))
+ with patch.object(DbSessionInjector, 'get_async_session_maker', maker_factory):
+ from starlette.datastructures import State
+
+ state = State()
+ gen = basic_db_session_injector.inject(state)
+ await gen.__anext__() # enter the try-block
+
+ # Simulate cancellation: send GeneratorExit into the yield.
+ await gen.aclose()
+
+ assert closed, 'close() must run during generator cleanup'
+ assert len(closed) == 1
+
+ @pytest.mark.asyncio
+ async def test_close_releases_gcp_connector_and_engine(self, temp_persistence_dir):
+ """DbSessionInjector.close() must call close_async on the connector
+ and dispose() on the async engine, so that worker respawns don't leak
+ the connector's background tasks."""
+ from openhands.app_server.services.db_session_injector import (
+ DbSessionInjector,
+ )
+
+ connector = MagicMock()
+ connector.close_async = AsyncMock()
+ engine = MagicMock()
+ engine.dispose = AsyncMock()
+
+ injector = DbSessionInjector(persistence_dir=temp_persistence_dir)
+ injector._gcp_connector = connector # type: ignore[attr-defined]
+ injector._async_engine = engine # type: ignore[attr-defined]
+
+ await injector.close()
+
+ connector.close_async.assert_awaited_once()
+ engine.dispose.assert_awaited_once()
+ assert injector._gcp_connector is None
+
+ @pytest.mark.asyncio
+ async def test_close_tolerates_missing_close_async(self, temp_persistence_dir):
+ """Older connector versions only expose ``close`` (sync). Fall back to
+ that and don't crash."""
+
+ class SyncOnlyConnector:
+ closed = 0
+
+ def close(self):
+ type(self).closed += 1
+
+ from openhands.app_server.services.db_session_injector import (
+ DbSessionInjector,
+ )
+
+ connector = SyncOnlyConnector()
+ injector = DbSessionInjector(persistence_dir=temp_persistence_dir)
+ injector._gcp_connector = connector # type: ignore[attr-defined]
+
+ await injector.close()
+
+ assert SyncOnlyConnector.closed == 1
+ assert injector._gcp_connector is None
diff --git a/tests/unit/app_server/test_sql_event_callback_service.py b/tests/unit/app_server/test_sql_event_callback_service.py
index fe3a91100c..9ff2c1a127 100644
--- a/tests/unit/app_server/test_sql_event_callback_service.py
+++ b/tests/unit/app_server/test_sql_event_callback_service.py
@@ -6,7 +6,6 @@ using SQLite as a mock database.
"""
from datetime import datetime, timezone
-from typing import AsyncGenerator
from uuid import uuid4
import pytest
@@ -51,19 +50,15 @@ async def async_engine():
@pytest.fixture
-async def async_db_session(async_engine) -> AsyncGenerator[AsyncSession, None]:
- """Create an async db_session for testing."""
- async_db_session_maker = async_sessionmaker(
- async_engine, class_=AsyncSession, expire_on_commit=False
- )
- async with async_db_session_maker() as db_session:
- yield db_session
+async def async_session_maker(async_engine):
+ """Create an async_sessionmaker that points at the test SQLite engine."""
+ return async_sessionmaker(async_engine, class_=AsyncSession, expire_on_commit=False)
@pytest.fixture
-def service(async_db_session: AsyncSession) -> SQLEventCallbackService:
+def service(async_session_maker) -> SQLEventCallbackService:
"""Create a SQLEventCallbackService instance for testing."""
- return SQLEventCallbackService(db_session=async_db_session)
+ return SQLEventCallbackService(async_session_maker=async_session_maker)
@pytest.fixture
@@ -389,6 +384,7 @@ class TestSQLEventCallbackService:
async def test_execute_callbacks_runs_conversation_specific_callback(
self,
service: SQLEventCallbackService,
+ async_session_maker,
sample_processor: EventCallbackProcessor,
):
"""Test executing callbacks for the matching conversation."""
@@ -407,8 +403,9 @@ class TestSQLEventCallbackService:
await service.execute_callbacks(conversation_id, event)
- result = await service.db_session.execute(select(StoredEventCallbackResult))
- callback_results = result.scalars().all()
+ async with async_session_maker() as db_session:
+ result = await db_session.execute(select(StoredEventCallbackResult))
+ callback_results = result.scalars().all()
assert len(callback_results) == 1
assert callback_results[0].event_callback_id == callback.id
assert callback_results[0].conversation_id == conversation_id
@@ -416,20 +413,23 @@ class TestSQLEventCallbackService:
async def test_execute_callbacks_ignores_legacy_null_conversation_callback(
self,
service: SQLEventCallbackService,
+ async_session_maker,
sample_processor: EventCallbackProcessor,
):
"""Legacy callbacks without a conversation_id should not execute globally."""
conversation_id = uuid4()
- service.db_session.add(
- StoredEventCallback(
- id=uuid4(),
- conversation_id=None,
- status=EventCallbackStatus.ACTIVE,
- processor=sample_processor,
- event_kind='MessageEvent',
+ async with async_session_maker() as db_session:
+ db_session.add(
+ StoredEventCallback(
+ id=uuid4(),
+ conversation_id=None,
+ status=EventCallbackStatus.ACTIVE,
+ processor=sample_processor,
+ event_kind='MessageEvent',
+ )
)
- )
- await service.db_session.commit()
+ await db_session.commit()
+
event = MessageEvent(
source='user',
llm_message=Message(role='user', content=[TextContent(text='hi')]),
@@ -437,27 +437,31 @@ class TestSQLEventCallbackService:
await service.execute_callbacks(conversation_id, event)
- result = await service.db_session.execute(select(StoredEventCallbackResult))
- callback_results = result.scalars().all()
+ async with async_session_maker() as db_session:
+ result = await db_session.execute(select(StoredEventCallbackResult))
+ callback_results = result.scalars().all()
assert callback_results == []
async def test_execute_callbacks_ignores_legacy_null_event_kind_callback(
self,
service: SQLEventCallbackService,
+ async_session_maker,
sample_processor: EventCallbackProcessor,
):
"""Legacy callbacks without an event_kind should not execute as event wildcards."""
conversation_id = uuid4()
- service.db_session.add(
- StoredEventCallback(
- id=uuid4(),
- conversation_id=conversation_id,
- status=EventCallbackStatus.ACTIVE,
- processor=sample_processor,
- event_kind=None,
+ async with async_session_maker() as db_session:
+ db_session.add(
+ StoredEventCallback(
+ id=uuid4(),
+ conversation_id=conversation_id,
+ status=EventCallbackStatus.ACTIVE,
+ processor=sample_processor,
+ event_kind=None,
+ )
)
- )
- await service.db_session.commit()
+ await db_session.commit()
+
event = MessageEvent(
source='user',
llm_message=Message(role='user', content=[TextContent(text='hi')]),
@@ -465,8 +469,9 @@ class TestSQLEventCallbackService:
await service.execute_callbacks(conversation_id, event)
- result = await service.db_session.execute(select(StoredEventCallbackResult))
- callback_results = result.scalars().all()
+ async with async_session_maker() as db_session:
+ result = await db_session.execute(select(StoredEventCallbackResult))
+ callback_results = result.scalars().all()
assert callback_results == []
async def test_save_event_callback_new(
@@ -500,9 +505,6 @@ class TestSQLEventCallbackService:
)
assert saved_utc >= original_utc
- # Commit the transaction to persist changes
- await service.db_session.commit()
-
# Verify the callback can be retrieved
retrieved_callback = await service.get_event_callback(sample_callback.id)
assert retrieved_callback is not None
@@ -549,9 +551,6 @@ class TestSQLEventCallbackService:
)
assert saved_utc >= original_utc
- # Commit the transaction to persist changes
- await service.db_session.commit()
-
# Verify the changes were persisted
retrieved_callback = await service.get_event_callback(created_callback.id)
assert retrieved_callback is not None
@@ -678,8 +677,6 @@ class TestSQLEventCallbackService:
# Verify the status was preserved
assert saved_callback.status == status
- # Commit and verify persistence
- await service.db_session.commit()
retrieved_callback = await service.get_event_callback(callback.id)
assert retrieved_callback is not None
assert retrieved_callback.status == status
@@ -734,8 +731,6 @@ class TestSQLEventCallbackService:
assert first_save.id == second_save.id
assert first_save is second_save # Same object instance
- # Commit and verify only one record exists
- await service.db_session.commit()
retrieved_callback = await service.get_event_callback(sample_callback.id)
assert retrieved_callback is not None
assert retrieved_callback.id == sample_callback.id
diff --git a/tests/unit/app_server/test_webhook_router_stats.py b/tests/unit/app_server/test_webhook_router_stats.py
index 76a5d17332..56d314ce6f 100644
--- a/tests/unit/app_server/test_webhook_router_stats.py
+++ b/tests/unit/app_server/test_webhook_router_stats.py
@@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
from uuid import uuid4
import pytest
+from fastapi import BackgroundTasks
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.pool import StaticPool
@@ -543,8 +544,12 @@ class TestOnEventStatsProcessing:
with patch(
'openhands.app_server.event_callback.webhook_router._run_callbacks_in_bg_and_close'
) as mock_callbacks:
+ # on_event now takes a BackgroundTasks dependency. We pass a real
+ # instance and verify it was scheduled rather than mocking it out.
+ background_tasks = BackgroundTasks()
# Call on_event directly with dependencies
await on_event(
+ background_tasks=background_tasks,
events=events,
conversation_id=conversation_id,
app_conversation_info=mock_app_conversation_info,
@@ -558,8 +563,9 @@ class TestOnEventStatsProcessing:
# Verify stats event was processed
mock_app_conversation_info_service.update_conversation_statistics.assert_called_once()
- # Verify callbacks were scheduled
- mock_callbacks.assert_called_once()
+ # Verify callbacks were scheduled via BackgroundTasks.add_task
+ assert len(background_tasks.tasks) == 1
+ assert background_tasks.tasks[0].func is mock_callbacks
@pytest.mark.asyncio
async def test_on_event_skips_non_stats_events(self):
@@ -593,6 +599,7 @@ class TestOnEventStatsProcessing:
):
# Call on_event directly with dependencies
await on_event(
+ background_tasks=BackgroundTasks(),
events=events,
conversation_id=conversation_id,
app_conversation_info=mock_app_conversation_info,