Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(core): keep the session-activity sweeps running through an error

raullopez-sandboxd33ad5f6fd31
diff --git a/core/switch_core/session_activity/maintenance.py b/core/switch_core/session_activity/maintenance.py
index e7c25fd44..1a62bbaf3 100644
--- a/core/switch_core/session_activity/maintenance.py
+++ b/core/switch_core/session_activity/maintenance.py
@@ -85,12 +85,12 @@ async def maintenance_loop(session_factory: async_sessionmaker[AsyncSession]) ->
while True:
await asyncio.sleep(EXPIRY_INTERVAL_SECONDS)
since_orphans += EXPIRY_INTERVAL_SECONDS
- if since_orphans >= ORPHANS_EVERY_SECONDS:
- since_orphans = 0.0
- await end_orphaned_turns_once(session_factory)
since_prune += EXPIRY_INTERVAL_SECONDS
prune = since_prune >= PRUNE_EVERY_SECONDS
try:
+ if since_orphans >= ORPHANS_EVERY_SECONDS:
+ since_orphans = 0.0
+ await end_orphaned_turns_once(session_factory)
await maintain_once(session_factory, prune=prune)
except Exception:
logger.exception("Session-activity upkeep failed")
@@ -113,10 +113,24 @@ def own_maintenance(
process; they run here only while it holds them."""
running: dict[str, asyncio.Task[None]] = {}
- async def start(key: str, fence: Fence) -> None:
- running[key] = asyncio.create_task(
+ def launch(key: str) -> None:
+ task = asyncio.create_task(
maintenance_loop(session_factory), name="session-activity-sweeps"
)
+ running[key] = task
+ task.add_done_callback(lambda done: ended(key, done))
+
+ def ended(key: str, task: asyncio.Task[None]) -> None:
+ if task.cancelled() or running.get(key) is not task:
+ return
+ logger.error(
+ "Session-activity sweeps stopped unexpectedly; restarting",
+ exc_info=task.exception(),
+ )
+ launch(key)
+
+ async def start(key: str, fence: Fence) -> None:
+ launch(key)
async def stop(key: str) -> None:
task = running.pop(key, None)
diff --git a/core/tests/switch_core/session_activity/test_sweep_ownership.py b/core/tests/switch_core/session_activity/test_sweep_ownership.py
index 5cb91c907..1e9fe4362 100644
--- a/core/tests/switch_core/session_activity/test_sweep_ownership.py
+++ b/core/tests/switch_core/session_activity/test_sweep_ownership.py
@@ -78,3 +78,37 @@ async def test_one_process_sweeps_and_another_takes_over(
assert other[2].held() == {maintenance.SWEEP_KEY}
for _, ownership, _ in processes:
await ownership.stop()
+
+
+async def test_sweeps_that_end_with_an_error_are_restarted(
+ session_factory: async_sessionmaker[AsyncSession],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ runs: list[int] = []
+ second_run = asyncio.Event()
+
+ async def loop(_factory: Any) -> None:
+ runs.append(len(runs))
+ if len(runs) == 1:
+ raise RuntimeError("sweep broke")
+ second_run.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(maintenance, "maintenance_loop", loop)
+
+ starts: dict[str, Any] = {}
+
+ class _Kind:
+ def want(self, key: str) -> None:
+ pass
+
+ class _Ownership:
+ def kind(self, name: str, *, start: Any, stop: Any) -> _Kind:
+ starts["start"], starts["stop"] = start, stop
+ return _Kind()
+
+ maintenance.own_maintenance(_Ownership(), session_factory) # type: ignore[arg-type]
+ await starts["start"](maintenance.SWEEP_KEY, None)
+ await asyncio.wait_for(second_run.wait(), timeout=1)
+ assert runs == [0, 1]
+ await starts["stop"](maintenance.SWEEP_KEY)