Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(activity): one process runs the session-activity sweeps

raullopez-sandboxd9600a0b9be2
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index b1c0795f3..bd946b43b 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -203,7 +203,7 @@ from switch_core.room_changes import RoomChanges
from switch_core.room_service import RoomService
from switch_core.session_activity.listener import AgentSessionActivityListener
from switch_core.session_activity.maintenance import (
- maintenance_loop as session_activity_maintenance_loop,
+ own_maintenance as own_session_activity_maintenance,
)
from switch_core.session_activity.outcomes import ApprovalOutcomes
from switch_core.session_activity.service import AgentSessionActivityService
@@ -700,6 +700,7 @@ async def run(config: SwitchConfig, process_id: str) -> None:
store=OwnershipLeaseStore(),
processes=SwitchCoreProcessStore(),
)
+ own_session_activity_maintenance(ownership, session_factory)
# Every owned consumer's progress, written in batches
# (`transport/cursors.py`).
cursor_writer = CursorWriter(
@@ -1385,9 +1386,6 @@ async def run(config: SwitchConfig, process_id: str) -> None:
attachment_group_task = asyncio.create_task(
_attachment_group_sweep_loop(session_factory, router)
)
- session_activity_task = asyncio.create_task(
- session_activity_maintenance_loop(session_factory)
- )
connection_sweep_task = asyncio.create_task(
_connection_sweep_loop(protocol)
)
@@ -1436,7 +1434,6 @@ async def run(config: SwitchConfig, process_id: str) -> None:
retention_task.cancel()
attachment_group_task.cancel()
router.close()
- session_activity_task.cancel()
connection_sweep_task.cancel()
service_token_task.cancel()
if snapshot_task is not None:
diff --git a/core/switch_core/session_activity/maintenance.py b/core/switch_core/session_activity/maintenance.py
index 50068aa40..e7c25fd44 100644
--- a/core/switch_core/session_activity/maintenance.py
+++ b/core/switch_core/session_activity/maintenance.py
@@ -12,12 +12,14 @@ like every other background task that spans them.
from __future__ import annotations
import asyncio
+import contextlib
import logging
from datetime import timedelta
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.tenant_lookup import all_tenant_ids
+from switch_core.ownership import Fence, OwnedKind, Ownership
from switch_core.session_activity.service import AgentSessionActivityService
from switch_core.tenant_context import no_tenant, tenant_scope
@@ -95,3 +97,35 @@ async def maintenance_loop(session_factory: async_sessionmaker[AsyncSession]) ->
else:
if prune:
since_prune = 0.0
+
+
+# The sweeps are owned work (`ownership.py`): one process runs them for every
+# tenant, and the next takes over when it stops or dies, rather than each
+# process sweeping every tenant on its own clock.
+SWEEP_KIND = "session_activity_sweep"
+SWEEP_KEY = "all"
+
+
+def own_maintenance(
+ ownership: Ownership, session_factory: async_sessionmaker[AsyncSession]
+) -> OwnedKind:
+ """Register the session-activity sweeps as owned work, wanted by this
+ process; they run here only while it holds them."""
+ running: dict[str, asyncio.Task[None]] = {}
+
+ async def start(key: str, fence: Fence) -> None:
+ running[key] = asyncio.create_task(
+ maintenance_loop(session_factory), name="session-activity-sweeps"
+ )
+
+ async def stop(key: str) -> None:
+ task = running.pop(key, None)
+ if task is None:
+ return
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+
+ kind = ownership.kind(SWEEP_KIND, start=start, stop=stop)
+ kind.want(SWEEP_KEY)
+ return kind
diff --git a/core/tests/switch_core/session_activity/test_sweep_ownership.py b/core/tests/switch_core/session_activity/test_sweep_ownership.py
new file mode 100644
index 000000000..5cb91c907
--- /dev/null
+++ b/core/tests/switch_core/session_activity/test_sweep_ownership.py
@@ -0,0 +1,80 @@
+"""The session-activity sweeps run in one process at a time, and move to
+another when that one stops or its lease lapses."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import timedelta
+from typing import Any
+
+import pytest
+from sqlalchemy import func, update
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.db.models import SwitchCoreProcess
+from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStore
+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
+from switch_core.ownership import Ownership
+from switch_core.process_lease import LEASE_TTL
+from switch_core.session_activity import maintenance
+from switch_core.signals import SignalBus
+from tests.ownership import process_lease_on
+
+
+def _never_listening() -> Any:
+ raise AssertionError("these processes' signal buses are never started")
+
+
+async def test_one_process_sweeps_and_another_takes_over(
+ session_factory: async_sessionmaker[AsyncSession],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ sweeping: list[str] = []
+
+ async def loop(_factory: Any) -> None:
+ process = asyncio.current_task().get_name() # type: ignore[union-attr]
+ sweeping.append(process)
+ try:
+ await asyncio.Event().wait()
+ finally:
+ sweeping.remove(process)
+
+ monkeypatch.setattr(maintenance, "maintenance_loop", loop)
+
+ processes = []
+ for _ in range(2):
+ lease = process_lease_on(session_factory)
+ await lease.renew()
+ ownership = Ownership(
+ lease=lease,
+ session_factory=session_factory,
+ bus=SignalBus(
+ session_factory, _never_listening, process_id=lease.process_id
+ ),
+ store=OwnershipLeaseStore(),
+ processes=SwitchCoreProcessStore(),
+ )
+ kind = maintenance.own_maintenance(ownership, session_factory)
+ processes.append((lease, ownership, kind))
+
+ for _, ownership, _ in processes:
+ await ownership.balance()
+ await asyncio.sleep(0)
+ holders = [kind for _, _, kind in processes if kind.held()]
+ assert len(holders) == 1 and len(sweeping) == 1
+
+ holder = next(p for p in processes if p[2].held())
+ other = next(p for p in processes if not p[2].held())
+ # The holder dies: its lease lapses and the other process claims the sweeps.
+ async with session_factory() as session:
+ await session.execute(
+ update(SwitchCoreProcess)
+ .where(SwitchCoreProcess.id == holder[0].process_id)
+ .values(beat_at=func.now() - LEASE_TTL - timedelta(seconds=1))
+ )
+ await session.commit()
+ await other[1].balance()
+ await asyncio.sleep(0)
+ assert other[2].held() == {maintenance.SWEEP_KEY}
+ for _, ownership, _ in processes:
+ await ownership.stop()