Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Wait for the retention and erasure loops to stop before closing their engine

Petr Bauch846825f60978
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 0a2c1326b..9c6c74309 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -567,6 +567,23 @@ async def _close_by(
)
+async def _stop_tasks(*tasks: asyncio.Task[None]) -> None:
+ """Cancel `tasks` and wait until each has stopped, so nothing they were in
+ the middle of still runs against what is closed next. A task that had
+ already failed is logged, not raised: shutdown goes on."""
+ for task in tasks:
+ task.cancel()
+ for task, outcome in zip(
+ tasks, await asyncio.gather(*tasks, return_exceptions=True), strict=True
+ ):
+ if isinstance(outcome, Exception):
+ logger.error(
+ "Background task %s had failed before shutdown",
+ task.get_name(),
+ exc_info=outcome,
+ )
+
+
async def _snapshot_loop(reporter: SnapshotReporter) -> None:
# `no_tenant` for the reason every other long-lived task does it: the
# snapshot binds each tenant in turn as it counts, and must not inherit
@@ -1580,10 +1597,9 @@ async def run(config: SwitchConfig, process_id: str) -> None:
router.close()
connection_sweep_task.cancel()
service_token_task.cancel()
- data_retention_task.cancel()
+ await _stop_tasks(data_retention_task, erasure_task)
if history_engine is not None:
await history_engine.dispose()
- erasure_task.cancel()
if snapshot_task is not None:
snapshot_task.cancel()
shared_connections_task.cancel()
diff --git a/core/tests/switch_core/test_main_stop_tasks.py b/core/tests/switch_core/test_main_stop_tasks.py
new file mode 100644
index 000000000..c6bc76e28
--- /dev/null
+++ b/core/tests/switch_core/test_main_stop_tasks.py
@@ -0,0 +1,51 @@
+"""Stopping background work at shutdown: each task has finished before the
+next step closes what it uses."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+import pytest
+
+from switch_core import main
+
+
+async def test_every_task_has_stopped_when_it_returns() -> None:
+ cleaned_up: list[str] = []
+
+ async def loop(name: str) -> None:
+ try:
+ await asyncio.sleep(3600)
+ finally:
+ await asyncio.sleep(0.01)
+ cleaned_up.append(name)
+
+ first = asyncio.create_task(loop("first"))
+ second = asyncio.create_task(loop("second"))
+ await asyncio.sleep(0)
+
+ await main._stop_tasks(first, second)
+
+ assert sorted(cleaned_up) == ["first", "second"]
+ assert first.done() and second.done()
+
+
+async def test_a_task_that_had_failed_is_logged_and_the_rest_still_stop(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ async def failed() -> None:
+ raise RuntimeError("broken")
+
+ async def waits() -> None:
+ await asyncio.sleep(3600)
+
+ broken = asyncio.create_task(failed(), name="broken")
+ waiting = asyncio.create_task(waits())
+ await asyncio.sleep(0)
+
+ with caplog.at_level(logging.ERROR, logger=main.logger.name):
+ await main._stop_tasks(broken, waiting)
+
+ assert waiting.cancelled()
+ assert "broken had failed" in caplog.text