Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Ask which rooms moved with two arrays, not a parameter per room

Abel Dantasd6598e1e3a11
diff --git a/core/switch_core/db/stores/message_store.py b/core/switch_core/db/stores/message_store.py
index 2312f4de4..d0cd58f55 100644
--- a/core/switch_core/db/stores/message_store.py
+++ b/core/switch_core/db/stores/message_store.py
@@ -2,16 +2,11 @@ from collections.abc import Collection
from datetime import datetime
from sqlalchemy import (
- BigInteger,
- Text,
- and_,
bindparam,
- column,
func,
or_,
select,
text,
- values,
)
from sqlalchemy.ext.asyncio import AsyncSession
@@ -240,22 +235,25 @@ class MessageStore:
) -> set[str]:
"""Of the rooms in `cursors`, the ones with a row after their cursor,
in one query whose result is only those rooms. With `types`, a row
- of one of those types."""
+ of one of those types.
+
+ The rooms and cursors go as two arrays, so the statement is the same
+ however many rooms there are: a reader in every room of a workspace
+ asks about tens of thousands, past the 32,767 parameters a statement
+ may have, and building a statement that size would hold the event
+ loop for a good part of a second."""
if not cursors:
return set()
- position = values(
- column("room_id", Text), column("seq", BigInteger), name="position"
- ).data(list(cursors.items()))
- query = select(Message.room_id).join(
- position,
- and_(
- Message.room_id == position.c.room_id,
- Message.seq > position.c.seq,
- ),
- )
- if types is not None:
- query = query.where(Message.event_type.in_(list(types)))
- result = await session.execute(query.distinct())
+ params: dict[str, object] = {
+ "room_ids": list(cursors),
+ "seqs": list(cursors.values()),
+ }
+ if types is None:
+ statement = _ROOMS_PAST
+ else:
+ statement = _ROOMS_PAST_OF_TYPES
+ params["types"] = list(types)
+ result = await session.execute(statement, params)
return set(result.scalars().all())
async def get_by_transport_event_id(
@@ -414,6 +412,19 @@ class MessageStore:
# The two statements a message's number costs, run by asyncpg directly
# (`db/raw.py`): every message an agent or a person sends takes the room's
# lock and reads its head.
+_ROOMS_PAST_SQL = (
+ "SELECT position.room_id "
+ "FROM unnest(CAST(:room_ids AS text[]), CAST(:seqs AS bigint[])) "
+ "AS position(room_id, seq) "
+ "WHERE EXISTS (SELECT 1 FROM messages "
+ "WHERE messages.room_id = position.room_id AND messages.seq > position.seq{types})"
+)
+_ROOMS_PAST = text(_ROOMS_PAST_SQL.format(types=""))
+_ROOMS_PAST_OF_TYPES = text(
+ _ROOMS_PAST_SQL.format(
+ types=" AND messages.event_type = ANY(CAST(:types AS text[]))"
+ )
+)
_LOCK_ROOM = RawStatement(text("SELECT pg_advisory_xact_lock(hashtext(:room_id))"))
_LOCK_ROOM_ANNOUNCED_BY_WRITER = RawStatement(
text(f"SELECT pg_advisory_xact_lock(hashtext(:room_id)), {SET_WRITER_ANNOUNCES}")
diff --git a/core/tests/switch_core/db/stores/test_message_store.py b/core/tests/switch_core/db/stores/test_message_store.py
index 400a9d47e..a7baeaaf6 100644
--- a/core/tests/switch_core/db/stores/test_message_store.py
+++ b/core/tests/switch_core/db/stores/test_message_store.py
@@ -390,6 +390,62 @@ class TestListForRoom:
assert [m.transport_event_id for m in following] == ["$a1"]
+class TestRoomsPast:
+ async def test_names_only_the_rooms_with_a_row_after_their_cursor(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ store = MessageStore()
+ async with session_factory() as session:
+ moved = await _make_room(session, "moved")
+ still = await _make_room(session, "still")
+ for room in (moved, still):
+ await store.create(session, _message(room.id, f"$a-{room.id}"), [])
+ await store.create(session, _message(moved.id, "$b"), [])
+ await session.commit()
+
+ past = await store.rooms_past(session, {moved.id: 1, still.id: 1})
+
+ assert past == {moved.id}
+
+ async def test_with_types_only_a_row_of_those_types_counts(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ store = MessageStore()
+ command = "com.switch.command"
+ async with session_factory() as session:
+ room = await _make_room(session, "chatty")
+ await store.create(session, _message(room.id, "$chat"), [])
+ await session.commit()
+
+ chat_only = await store.rooms_past(session, {room.id: 0}, types=[command])
+ await store.create(
+ session, _message(room.id, "$cmd", event_type=command), []
+ )
+ await session.commit()
+ with_command = await store.rooms_past(
+ session, {room.id: 0}, types=[command]
+ )
+
+ assert chat_only == set()
+ assert with_command == {room.id}
+
+ async def test_asks_about_more_rooms_than_a_statement_has_parameters(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """A reader in every room of a workspace can watch tens of thousands."""
+ store = MessageStore()
+ async with session_factory() as session:
+ room = await _make_room(session, "one-of-many")
+ await store.create(session, _message(room.id, "$x"), [])
+ await session.commit()
+ cursors = {str(uuid.uuid4()): 0 for _ in range(40_000)}
+ cursors[room.id] = 0
+
+ past = await store.rooms_past(session, cursors)
+
+ assert past == {room.id}
+
+
class TestConversationHistory:
"""The two reads following the conversation decides from."""