Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(backfill): carry a bounded amount of each room's history (CHOO-1436) (#379)

Louis Amaudruz3fe3a678d696
diff --git a/core/switch_core/cli/backfill.py b/core/switch_core/cli/backfill.py
index b7fc13d31..dbb2c55e3 100644
--- a/core/switch_core/cli/backfill.py
+++ b/core/switch_core/cli/backfill.py
@@ -39,6 +39,7 @@ from switch_core.db.models import Room
from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.message_store import MessageStore
from switch_core.messages import backfill_room
+from switch_core.messages.backfill import MESSAGES_PER_ROOM
from switch_core.transport.matrix import MatrixTransport
# The admin client is a member of every room by design, which is what makes it
@@ -70,6 +71,18 @@ def _parse_args() -> argparse.Namespace:
"history in pages to learn there is nothing to write."
),
)
+ parser.add_argument(
+ "--messages-per-room",
+ type=int,
+ default=MESSAGES_PER_ROOM,
+ metavar="N",
+ help=(
+ "How many of a room's messages to carry across, newest first "
+ f"(default {MESSAGES_PER_ROOM}). 0 walks every room to its start. "
+ "Older messages are left on the homeserver, which is going away, "
+ "so this is what is kept rather than what is done first."
+ ),
+ )
parser.add_argument(
"--allow-empty",
action="store_true",
@@ -153,7 +166,13 @@ async def _run(args: argparse.Namespace) -> int:
# reporting it as done would be the one answer that misleads.
unreadable.append(f"{room.name} ({room.matrix_room_id})")
continue
- report = await backfill_room(transport, session_factory, room, store=store)
+ report = await backfill_room(
+ transport,
+ session_factory,
+ room,
+ store=store,
+ messages_per_room=args.messages_per_room,
+ )
print(report.summary())
written += report.written
incomplete += 1 if report.incomplete else 0
diff --git a/core/switch_core/messages/backfill.py b/core/switch_core/messages/backfill.py
index d401756c0..099d0cc2c 100644
--- a/core/switch_core/messages/backfill.py
+++ b/core/switch_core/messages/backfill.py
@@ -66,6 +66,17 @@ PAGE_SIZE = 100
# room that ends returns no next token and stops on its own.
MAX_PAGES = 10_000
+# How much of a room's past is worth carrying across. The walk runs newest
+# first, so this keeps the most recent messages and abandons what is older —
+# and abandons it for good, because the homeserver goes away after this.
+#
+# It is a product decision rather than a technical limit: recent context is
+# what an agent reads, a migration that finishes in minutes is one an operator
+# will actually run, and an unbounded walk over a room with years in it is the
+# reason someone reaches for `--force` and then waits. Raise it with
+# `--messages-per-room`, or pass 0 to walk a room to its start.
+MESSAGES_PER_ROOM = 50
+
@dataclass
class BackfillReport:
@@ -81,6 +92,11 @@ class BackfillReport:
# because the room ended. The room is then partially reconstructed, and
# saying so matters more than the count does.
incomplete: bool = False
+ # True when the walk stopped at `messages_per_room`. Deliberately not
+ # `incomplete`: that one reports a walk that failed to finish and makes the
+ # command exit non-zero, and Console fails an upgrade on that. Stopping
+ # where we were told to stop is success, and the room is marked done.
+ capped: bool = False
def summary(self) -> str:
line = (
@@ -94,6 +110,8 @@ class BackfillReport:
for name, count in sorted(self.skipped_by_type.items())
)
line += f"; not part of the log: {counts}"
+ if self.capped:
+ line += "; stopped at the per-room limit, older history not carried"
if self.incomplete:
line += "; INCOMPLETE — hit the page limit before the room ended"
return line
@@ -119,17 +137,33 @@ async def _mark_backfilled(
await session.commit()
+def _in_log(report: BackfillReport) -> int:
+ """How many of this room's messages the log now holds from this walk.
+
+ Written plus already recorded, because the cap has to mean the same thing
+ on a second run as on the first. Counting only what this run wrote would
+ make every re-run carry another `messages_per_room` of older history.
+ """
+ return report.written + report.already_present
+
+
async def backfill_room(
transport: MessageTransport,
session_factory: async_sessionmaker[AsyncSession],
room: Room,
*,
store: MessageStore,
+ messages_per_room: int = MESSAGES_PER_ROOM,
) -> BackfillReport:
- """Walk one room to its start, writing every message with no row.
+ """Walk a room backwards, writing the messages that have no row.
The transport must already be connected and a member of the room.
+ Stops at `messages_per_room` loggable messages, or at the room's start if
+ that comes first; 0 means walk to the start. The count is of messages the
+ log would hold, found rather than written, so a second run over a room that
+ is already done stops in the same place instead of carrying fifty more.
+
Each page is its own transaction. A walk that fails partway therefore
leaves the pages it finished written rather than rolling back an hour of
work, and re-running picks up where it stopped — which is the behaviour
@@ -153,6 +187,12 @@ async def backfill_room(
# room's start leaves the oldest message with the lowest number. Any
# other order would number the room backwards.
for raw in page.events:
+ if messages_per_room and _in_log(report) >= messages_per_room:
+ # Told where to stop, so stopping is a finished room rather
+ # than an abandoned one: mark it, or every start walks it again.
+ report.capped = True
+ await _mark_backfilled(session_factory, room, store)
+ return report
await _consider(raw, session_factory, room, store, report)
end = page.next_token
diff --git a/core/tests/switch_core/messages/test_backfill.py b/core/tests/switch_core/messages/test_backfill.py
index 9da30a4d8..c74e087ad 100644
--- a/core/tests/switch_core/messages/test_backfill.py
+++ b/core/tests/switch_core/messages/test_backfill.py
@@ -453,3 +453,136 @@ class TestStopping:
assert transport.reads == 2
assert report.written == 1
+
+
+class TestPerRoomLimit:
+ """A room's history is carried across newest-first, up to a limit.
+
+ The limit is the whole point of the release it shipped in: an unbounded
+ walk over a room with years in it is the difference between a migration an
+ operator runs and one they abandon. What it leaves behind is left behind
+ for good, because the homeserver goes away afterwards.
+ """
+
+ async def test_it_keeps_the_newest_and_leaves_the_rest(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _make_room(session)
+
+ transport = PagingTransport(
+ [
+ HistoryPage(
+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(5)],
+ next_token=None,
+ )
+ ]
+ )
+ async with session_factory() as session:
+ room = await session.get(Room, room.id) # type: ignore[assignment]
+ report = await backfill_room(
+ transport,
+ session_factory,
+ room,
+ store=MessageStore(),
+ messages_per_room=2,
+ )
+
+ assert report.written == 2
+ assert report.capped is True
+ rows = await _rows(session_factory, room.id)
+ assert [r.body for r in rows] == ["m1", "m0"]
+
+ async def test_a_capped_room_is_finished_not_abandoned(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Marked, or every start walks it again — and `incomplete` stays
+ false, because the command exits non-zero on that and Console fails an
+ upgrade on a non-zero exit."""
+ async with session_factory() as session:
+ room = await _make_room(session)
+
+ transport = PagingTransport(
+ [
+ HistoryPage(
+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(4)],
+ next_token=None,
+ )
+ ]
+ )
+ async with session_factory() as session:
+ room = await session.get(Room, room.id) # type: ignore[assignment]
+ report = await backfill_room(
+ transport,
+ session_factory,
+ room,
+ store=MessageStore(),
+ messages_per_room=1,
+ )
+
+ assert report.incomplete is False
+ async with session_factory() as session:
+ marked = await session.get(Room, room.id)
+ assert marked is not None
+ assert marked.history_backfilled_at is not None
+
+ async def test_a_second_run_does_not_carry_another_batch(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The cap counts what the log holds, not what this run wrote. Counting
+ only writes would make each re-run reach further back."""
+ async with session_factory() as session:
+ room = await _make_room(session)
+
+ pages = [
+ HistoryPage(
+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(5)],
+ next_token=None,
+ )
+ ]
+ async with session_factory() as session:
+ room = await session.get(Room, room.id) # type: ignore[assignment]
+ await backfill_room(
+ PagingTransport(list(pages)),
+ session_factory,
+ room,
+ store=MessageStore(),
+ messages_per_room=2,
+ )
+ second = await backfill_room(
+ PagingTransport(list(pages)),
+ session_factory,
+ room,
+ store=MessageStore(),
+ messages_per_room=2,
+ )
+
+ assert second.written == 0
+ assert len(await _rows(session_factory, room.id)) == 2
+
+ async def test_zero_walks_the_room_to_its_start(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _make_room(session)
+
+ transport = PagingTransport(
+ [
+ HistoryPage(
+ events=[_event(f"$e{i}", 100 - i, f"m{i}") for i in range(5)],
+ next_token=None,
+ )
+ ]
+ )
+ async with session_factory() as session:
+ room = await session.get(Room, room.id) # type: ignore[assignment]
+ report = await backfill_room(
+ transport,
+ session_factory,
+ room,
+ store=MessageStore(),
+ messages_per_room=0,
+ )
+
+ assert report.written == 5
+ assert report.capped is False