Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
feat(backfill): carry a bounded amount of each room's history (CHOO-1436) (#379)
Louis Amaudruz3fe3a678d696
diff --git a/core/switch_core/cli/backfill.py b/core/switch_core/cli/backfill.pyindex b7fc13d31..dbb2c55e3 100644--- a/core/switch_core/cli/backfill.py+++ b/core/switch_core/cli/backfill.py@@ -39,6 +39,7 @@ from switch_core.db.models import Roomfrom switch_core.db.stores.client_store import ClientStorefrom switch_core.db.stores.message_store import MessageStorefrom switch_core.messages import backfill_room+from switch_core.messages.backfill import MESSAGES_PER_ROOMfrom switch_core.transport.matrix import MatrixTransport# The admin client is a member of every room by design, which is what makes it@@ -70,6 +71,18 @@ def _parse_args() -> argparse.Namespace:"history in pages to learn there is nothing to write."),)+ parser.add_argument(+ "--messages-per-room",+ type=int,+ default=MESSAGES_PER_ROOM,+ metavar="N",+ help=(+ "How many of a room's messages to carry across, newest first "+ f"(default {MESSAGES_PER_ROOM}). 0 walks every room to its start. "+ "Older messages are left on the homeserver, which is going away, "+ "so this is what is kept rather than what is done first."+ ),+ )parser.add_argument("--allow-empty",action="store_true",@@ -153,7 +166,13 @@ async def _run(args: argparse.Namespace) -> int:# reporting it as done would be the one answer that misleads.unreadable.append(f"{room.name} ({room.matrix_room_id})")continue- report = await backfill_room(transport, session_factory, room, store=store)+ report = await backfill_room(+ transport,+ session_factory,+ room,+ store=store,+ messages_per_room=args.messages_per_room,+ )print(report.summary())written += report.writtenincomplete += 1 if report.incomplete else 0diff --git a/core/switch_core/messages/backfill.py b/core/switch_core/messages/backfill.pyindex d401756c0..099d0cc2c 100644--- a/core/switch_core/messages/backfill.py+++ b/core/switch_core/messages/backfill.py@@ -66,6 +66,17 @@ PAGE_SIZE = 100# room that ends returns no next token and stops on its own.MAX_PAGES = 10_000+# How much of a room's past is worth carrying across. The walk runs newest+# first, so this keeps the most recent messages and abandons what is older —+# and abandons it for good, because the homeserver goes away after this.+#+# It is a product decision rather than a technical limit: recent context is+# what an agent reads, a migration that finishes in minutes is one an operator+# will actually run, and an unbounded walk over a room with years in it is the+# reason someone reaches for `--force` and then waits. Raise it with+# `--messages-per-room`, or pass 0 to walk a room to its start.+MESSAGES_PER_ROOM = 50+@dataclassclass BackfillReport:@@ -81,6 +92,11 @@ class BackfillReport:# because the room ended. The room is then partially reconstructed, and# saying so matters more than the count does.incomplete: bool = False+ # True when the walk stopped at `messages_per_room`. Deliberately not+ # `incomplete`: that one reports a walk that failed to finish and makes the+ # command exit non-zero, and Console fails an upgrade on that. Stopping+ # where we were told to stop is success, and the room is marked done.+ capped: bool = Falsedef summary(self) -> str:line = (@@ -94,6 +110,8 @@ class BackfillReport:for name, count in sorted(self.skipped_by_type.items()))line += f"; not part of the log: {counts}"+ if self.capped:+ line += "; stopped at the per-room limit, older history not carried"if self.incomplete:line += "; INCOMPLETE — hit the page limit before the room ended"return line@@ -119,17 +137,33 @@ async def _mark_backfilled(await session.commit()+def _in_log(report: BackfillReport) -> int:+ """How many of this room's messages the log now holds from this walk.++ Written plus already recorded, because the cap has to mean the same thing+ on a second run as on the first. Counting only what this run wrote would+ make every re-run carry another `messages_per_room` of older history.+ """+ return report.written + report.already_present++async def backfill_room(transport: MessageTransport,session_factory: async_sessionmaker[AsyncSession],room: Room,*,store: MessageStore,+ messages_per_room: int = MESSAGES_PER_ROOM,) -> BackfillReport:- """Walk one room to its start, writing every message with no row.+ """Walk a room backwards, writing the messages that have no row.The transport must already be connected and a member of the room.+ Stops at `messages_per_room` loggable messages, or at the room's start if+ that comes first; 0 means walk to the start. The count is of messages the+ log would hold, found rather than written, so a second run over a room that+ is already done stops in the same place instead of carrying fifty more.+Each page is its own transaction. A walk that fails partway thereforeleaves the pages it finished written rather than rolling back an hour ofwork, and re-running picks up where it stopped — which is the behaviour@@ -153,6 +187,12 @@ async def backfill_room(# room's start leaves the oldest message with the lowest number. Any# other order would number the room backwards.for raw in page.events:+ if messages_per_room and _in_log(report) >= messages_per_room:+ # Told where to stop, so stopping is a finished room rather+ # than an abandoned one: mark it, or every start walks it again.+ report.capped = True+ await _mark_backfilled(session_factory, room, store)+ return reportawait _consider(raw, session_factory, room, store, report)end = page.next_tokendiff --git a/core/tests/switch_core/messages/test_backfill.py b/core/tests/switch_core/messages/test_backfill.pyindex 9da30a4d8..c74e087ad 100644--- a/core/tests/switch_core/messages/test_backfill.py+++ b/core/tests/switch_core/messages/test_backfill.py@@ -453,3 +453,136 @@ class TestStopping:assert transport.reads == 2assert report.written == 1+++class TestPerRoomLimit:+ """A room's history is carried across newest-first, up to a limit.++ The limit is the whole point of the release it shipped in: an unbounded+ walk over a room with years in it is the difference between a migration an+ operator runs and one they abandon. What it leaves behind is left behind+ for good, because the homeserver goes away afterwards.+ """++ async def test_it_keeps_the_newest_and_leaves_the_rest(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _make_room(session)++ transport = PagingTransport(+ [+ HistoryPage(+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(5)],+ next_token=None,+ )+ ]+ )+ async with session_factory() as session:+ room = await session.get(Room, room.id) # type: ignore[assignment]+ report = await backfill_room(+ transport,+ session_factory,+ room,+ store=MessageStore(),+ messages_per_room=2,+ )++ assert report.written == 2+ assert report.capped is True+ rows = await _rows(session_factory, room.id)+ assert [r.body for r in rows] == ["m1", "m0"]++ async def test_a_capped_room_is_finished_not_abandoned(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """Marked, or every start walks it again — and `incomplete` stays+ false, because the command exits non-zero on that and Console fails an+ upgrade on a non-zero exit."""+ async with session_factory() as session:+ room = await _make_room(session)++ transport = PagingTransport(+ [+ HistoryPage(+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(4)],+ next_token=None,+ )+ ]+ )+ async with session_factory() as session:+ room = await session.get(Room, room.id) # type: ignore[assignment]+ report = await backfill_room(+ transport,+ session_factory,+ room,+ store=MessageStore(),+ messages_per_room=1,+ )++ assert report.incomplete is False+ async with session_factory() as session:+ marked = await session.get(Room, room.id)+ assert marked is not None+ assert marked.history_backfilled_at is not None++ async def test_a_second_run_does_not_carry_another_batch(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """The cap counts what the log holds, not what this run wrote. Counting+ only writes would make each re-run reach further back."""+ async with session_factory() as session:+ room = await _make_room(session)++ pages = [+ HistoryPage(+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(5)],+ next_token=None,+ )+ ]+ async with session_factory() as session:+ room = await session.get(Room, room.id) # type: ignore[assignment]+ await backfill_room(+ PagingTransport(list(pages)),+ session_factory,+ room,+ store=MessageStore(),+ messages_per_room=2,+ )+ second = await backfill_room(+ PagingTransport(list(pages)),+ session_factory,+ room,+ store=MessageStore(),+ messages_per_room=2,+ )++ assert second.written == 0+ assert len(await _rows(session_factory, room.id)) == 2++ async def test_zero_walks_the_room_to_its_start(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _make_room(session)++ transport = PagingTransport(+ [+ HistoryPage(+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(5)],+ next_token=None,+ )+ ]+ )+ async with session_factory() as session:+ room = await session.get(Room, room.id) # type: ignore[assignment]+ report = await backfill_room(+ transport,+ session_factory,+ room,+ store=MessageStore(),+ messages_per_room=0,+ )++ assert report.written == 5+ assert report.capped is False