Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Read a workspace's telemetry facts at once, not one room at a time (CHOO-3667)

Abel Dantas97b1a7fc63a3
diff --git a/CHANGELOG.md b/CHANGELOG.md
index de9220ae0..c47dcb533 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -232,6 +232,10 @@ version of their own to them without also giving them a release of their own.
group whose subgroup would move up next to a namesake is refused too, and
the dashboard now shows why. Names can no longer contain `/` or be blank.
Existing duplicates are left as they are and can be reached by id.
+- **Message telemetry reads a workspace's rooms, clients and agents at once.**
+ The first lookup that misses reads all of them, where each room, sender and
+ agent was read on its own, so in a large workspace most messages no longer
+ cost a lookup.
#### Fixed
- **On Slack, the 👀 and the activity card no longer arrive after the agent's
diff --git a/core/switch_core/telemetry/messages.py b/core/switch_core/telemetry/messages.py
index 31ee6e176..fe54a6d63 100644
--- a/core/switch_core/telemetry/messages.py
+++ b/core/switch_core/telemetry/messages.py
@@ -188,6 +188,9 @@ class MessageTelemetry:
)
self._failures_in_a_row = 0
self._all_lookups_resume_at: float | None = None
+ # When each tenant may next be read whole (`_read_tenant`), on the
+ # monotonic clock.
+ self._tenants_read: dict[str, float] = {}
# Taken off the queue and not yet reported. Not `qsize`, which leaves
# out the one the worker is cancelled in the middle of.
self._in_hand = 0
@@ -453,6 +456,98 @@ class MessageTelemetry:
misses,
)
+ async def _read_tenant(self, tenant_id: str) -> None:
+ """Read the facts of every open room, the kind of every client and the
+ runtime of every agent of the tenant at once, the first time a lookup
+ of it misses, and again once `_ROOM_TTL_SECONDS` have passed.
+
+ In a workspace of thousands of agents and rooms, a process reports a
+ message about most of them before it reports a second one, so with one
+ lookup each a cache that kept every answer still missed for most
+ messages. A read that fails raises, and the lookup that asked counts
+ it as its own failure; it is tried again once a failed lookup would be.
+ """
+ next_read = self._tenants_read.get(tenant_id)
+ if next_read is not None and time.monotonic() < next_read:
+ return
+ # Not again for as long as a failed lookup waits; a read that succeeds
+ # puts it off for as long as a room is kept, below.
+ self._tenants_read[tenant_id] = time.monotonic() + _LOOKUP_BACKOFF_SECONDS
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ open_room = and_(Room.tenant_id == tenant_id, Room.archived_at.is_(None))
+ rooms = (
+ await session.execute(
+ select(Room.id, Room.channel_type, CollaborationBridge.type)
+ .select_from(Room)
+ .outerjoin(
+ CollaborationBridge,
+ and_(
+ CollaborationBridge.id == Room.bridge_id,
+ CollaborationBridge.tenant_id == tenant_id,
+ ),
+ )
+ .where(open_room)
+ )
+ ).all()
+ members = (
+ await session.execute(
+ select(ClientRoom.room_id, Client.type, func.count())
+ .select_from(ClientRoom)
+ .join(Client, Client.id == ClientRoom.client_id)
+ .join(Room, Room.id == ClientRoom.room_id)
+ .where(
+ ClientRoom.tenant_id == tenant_id,
+ Client.tenant_id == tenant_id,
+ open_room,
+ )
+ .group_by(ClientRoom.room_id, Client.type)
+ )
+ ).all()
+ clients = (
+ await session.execute(
+ select(Client.transport_user_id, Client.type).where(
+ Client.tenant_id == tenant_id
+ )
+ )
+ ).all()
+ agents = (
+ await session.execute(
+ select(
+ Agent.client_id,
+ Agent.metadata_["known_agent_type"].astext,
+ ).where(Agent.tenant_id == tenant_id)
+ )
+ ).all()
+ self._tenants_read[tenant_id] = time.monotonic() + _ROOM_TTL_SECONDS
+ by_room: dict[str, dict[str, int]] = {}
+ for room_id, client_type, count in members:
+ by_room.setdefault(room_id, {})[client_type] = int(count)
+ for room_id, channel_type, platform in rooms:
+ by_type = by_room.get(room_id, {})
+ self._rooms.put(
+ (tenant_id, room_id),
+ _RoomFacts(
+ bridge_platform=normalise_platform(platform),
+ channel_type=normalise_channel_type(channel_type),
+ user_count=by_type.get(HUMAN_CLIENT_TYPE, 0),
+ agent_count=by_type.get(AGENT_CLIENT_TYPE, 0),
+ ),
+ )
+ for transport_user_id, client_type in clients:
+ self._sender_kinds.put(
+ (tenant_id, transport_user_id),
+ _CLIENT_TYPE_KIND.get(client_type, "unknown"),
+ )
+ for client_id, known_agent_type in agents:
+ self._agent_runtimes.put(
+ (tenant_id, client_id),
+ normalise_known_agent_type(
+ None
+ if known_agent_type is None
+ else {"known_agent_type": known_agent_type}
+ ),
+ )
+
async def _room_facts(self, tenant_id: str, room_id: str) -> _RoomFacts:
key = (tenant_id, room_id)
cached = self._rooms.get(key)
@@ -461,6 +556,14 @@ class MessageTelemetry:
failure_key = ("room", tenant_id, room_id)
if not self._may_look_up(failure_key):
return _UNKNOWN_ROOM
+ try:
+ await self._read_tenant(tenant_id)
+ except Exception:
+ self._lookup_failed(failure_key, f"room {room_id}")
+ return _UNKNOWN_ROOM
+ cached = self._rooms.get(key)
+ if cached is not None:
+ return cached
try:
facts = await self._load_room_facts(tenant_id, room_id)
except Exception:
@@ -520,6 +623,14 @@ class MessageTelemetry:
failure_key = ("sender", tenant_id, transport_user_id)
if not self._may_look_up(failure_key):
return "unknown"
+ try:
+ await self._read_tenant(tenant_id)
+ except Exception:
+ self._lookup_failed(failure_key, "a message sender")
+ return "unknown"
+ cached = self._sender_kinds.get(key)
+ if cached is not None:
+ return cached
try:
async with tenant_session(self._session_factory, tenant_id) as session:
client_type = (
@@ -550,6 +661,14 @@ class MessageTelemetry:
failure_key = ("agent", tenant_id, client_id)
if not self._may_look_up(failure_key):
return _UNKNOWN_RUNTIME
+ try:
+ await self._read_tenant(tenant_id)
+ except Exception:
+ self._lookup_failed(failure_key, "an agent's runtime")
+ return _UNKNOWN_RUNTIME
+ cached = self._agent_runtimes.get(key)
+ if cached is not None:
+ return cached
try:
async with tenant_session(self._session_factory, tenant_id) as session:
result = await session.execute(
diff --git a/core/tests/switch_core/telemetry/test_message_events.py b/core/tests/switch_core/telemetry/test_message_events.py
index 23fee7851..05cb84edb 100644
--- a/core/tests/switch_core/telemetry/test_message_events.py
+++ b/core/tests/switch_core/telemetry/test_message_events.py
@@ -31,6 +31,7 @@ from switch_core.telemetry.messages import MessageTelemetry, _RoomFacts, _TtlCac
from switch_core.telemetry.service import TelemetryService
from switch_core.telemetry.sink import TelemetryRecord
from switch_core.transport.observer import ParticipantMessage
+from tests.session_counting import CountingSessionFactory
TENANT_ZERO = "00000000-0000-0000-0000-000000000000"
@@ -200,6 +201,17 @@ class _Unreachable:
raise ConnectionError("database unreachable")
+def _one_at_a_time(messages: MessageTelemetry) -> None:
+ """Leave out the read of the whole workspace (`MessageTelemetry._read_tenant`),
+ so each room, sender and agent is looked up on its own, as these tests
+ count."""
+
+ async def _not_read(tenant_id: str) -> None:
+ return None
+
+ messages._read_tenant = _not_read # type: ignore[method-assign]
+
+
async def _reported(
messages: MessageTelemetry, service: TelemetryService, sink: _RecordingSink
) -> list[tuple[str, dict[str, object]]]:
@@ -429,6 +441,8 @@ class TestFailingLookups:
messages._load_room_facts = _broken # type: ignore[method-assign]
+ _one_at_a_time(messages)
+
with caplog.at_level("WARNING"):
for _ in range(3):
messages.observe(_said(world, world.humans[0].id, "human"))
@@ -459,6 +473,8 @@ class TestFailingLookups:
return await _a_slack_room(tenant_id, room_id)
messages._load_room_facts = _bad_room_fails # type: ignore[method-assign]
+
+ _one_at_a_time(messages)
other_tenant = str(uuid.uuid4())
messages.observe(_said_in("bad"))
@@ -507,6 +523,8 @@ class TestFailingLookups:
messages._load_room_facts = _broken # type: ignore[method-assign]
+ _one_at_a_time(messages)
+
with caplog.at_level("WARNING"):
for room in ("a", "b", "c", "d", "e"):
messages.observe(_said_in(room))
@@ -533,6 +551,8 @@ class TestFailingLookups:
messages._load_room_facts = _only_good_answers # type: ignore[method-assign]
+ _one_at_a_time(messages)
+
for room in ("a", "b", "good", "c", "d"):
messages.observe(_said_in(room))
await _reported(messages, service, sink)
@@ -614,6 +634,8 @@ class TestCaching:
messages._load_room_facts = _gone # type: ignore[method-assign]
+ _one_at_a_time(messages)
+
for _ in range(3):
messages.observe(_said_in("deleted-room"))
reported = await _reported(messages, service, sink)
@@ -671,6 +693,99 @@ class TestCaching:
] == ["unknown", "unknown"]
+class TestTheWholeWorkspace:
+ """The first lookup that misses reads every open room, client and agent
+ of the workspace at once (`MessageTelemetry._read_tenant`): in a workspace
+ of thousands, a process reports on most of them before it reports a second
+ message about any, so one lookup each missed for most messages."""
+
+ async def test_other_rooms_senders_and_agents_cost_no_read(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ async with session_factory() as session:
+ other = await _room(session, bridge_platform=None, channel_type="direct")
+ await _join(session, world.humans[1].id, other)
+ await session.commit()
+ counting = CountingSessionFactory(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=counting) # type: ignore[arg-type]
+
+ messages.observe(_said(world, world.agent.client_id, "agent"))
+ await messages._queue.join()
+ assert counting.opened == 1
+ messages.observe(_said_in(other.id, sender_client_id=world.humans[1].id))
+ messages.agent_addressed(
+ tenant_id=TENANT_ZERO,
+ room_id=other.id,
+ sender_transport_user_id=world.humans[1].transport_user_id,
+ from_platform=False,
+ agent_metadata=None,
+ agent_live=True,
+ has_attachment=False,
+ )
+ reported = await _reported(messages, service, sink)
+
+ assert counting.opened == 1
+ facts = {
+ (name, p.get("bridge_platform"), p.get("room_user_count"))
+ for name, p in reported
+ }
+ assert ("switch_core.room_message_sent", "slack", 2) in facts
+ assert ("switch_core.room_message_sent", "none", 1) in facts
+ assert [
+ p["known_agent_type"]
+ for name, p in reported
+ if name == "switch_core.agent_message_sent"
+ ] == ["codex"]
+ assert [
+ p["sender_kind"]
+ for name, p in reported
+ if name == "switch_core.agent_message_received"
+ ] == ["user"]
+
+ async def test_a_room_made_after_the_read_is_looked_up_on_its_own(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ world = await _world(session_factory)
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)
+ messages.observe(_said(world, world.humans[0].id, "human"))
+ await messages._queue.join()
+ async with session_factory() as session:
+ later = await _room(session, bridge_platform=None, channel_type="direct")
+ await _join(session, world.humans[0].id, later)
+ await session.commit()
+
+ messages.observe(_said_in(later.id, sender_client_id=world.humans[0].id))
+ reported = await _reported(messages, service, sink)
+
+ assert [p["room_user_count"] for _, p in reported] == [2, 1]
+
+ async def test_a_read_that_fails_is_the_lookups_failure(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Counted once, not followed by the lookup it was for, and not tried
+ again while a failed lookup would not be."""
+ database = _Unreachable()
+ sink = _RecordingSink()
+ service = _service(sink)
+ messages = MessageTelemetry(
+ telemetry=service,
+ session_factory=database, # type: ignore[arg-type]
+ )
+
+ for _ in range(2):
+ messages.observe(_said_in("room"))
+ with caplog.at_level("WARNING"):
+ reported = await _reported(messages, service, sink)
+
+ assert [p["bridge_platform"] for _, p in reported] == ["unknown", "unknown"]
+ assert database.attempts == 1
+
+
class TestHowLongLookupsAreKept:
def test_a_room_for_a_quarter_hour_and_who_someone_is_for_an_hour(
self, monkeypatch: pytest.MonkeyPatch
@@ -740,6 +855,7 @@ class TestBackpressure:
session_factory=None, # type: ignore[arg-type]
)
messages._load_room_facts = _a_slack_room # type: ignore[method-assign]
+ _one_at_a_time(messages)
with caplog.at_level("WARNING"):
for _ in range(4):
@@ -769,6 +885,7 @@ class TestFailingSenderAndRuntimeLookups:
session_factory=database, # type: ignore[arg-type]
)
messages._load_room_facts = _a_slack_room # type: ignore[method-assign]
+ _one_at_a_time(messages)
for sender in ("@alice:test", "@alice:test"):
messages.agent_addressed(
@@ -798,6 +915,7 @@ class TestFailingSenderAndRuntimeLookups:
session_factory=database, # type: ignore[arg-type]
)
messages._load_room_facts = _a_slack_room # type: ignore[method-assign]
+ _one_at_a_time(messages)
for _ in range(2):
messages.observe(
@@ -833,6 +951,8 @@ class TestFailingSenderAndRuntimeLookups:
messages._load_room_facts = _broken # type: ignore[method-assign]
+ _one_at_a_time(messages)
+
with caplog.at_level("WARNING"):
for _ in range(3):
messages.observe(_said_in("room"))
@@ -859,6 +979,8 @@ class TestFailingSenderAndRuntimeLookups:
messages._load_room_facts = _gone # type: ignore[method-assign]
+ _one_at_a_time(messages)
+
with caplog.at_level("WARNING"):
messages.observe(_said_in("room-a"))
messages.observe(_said_in("room-b"))
@@ -980,6 +1102,7 @@ class TestShutdown:
session_factory=None, # type: ignore[arg-type]
)
messages._load_room_facts = _slow_room # type: ignore[method-assign]
+ _one_at_a_time(messages)
messages.observe(_said_in("room"))
with caplog.at_level("WARNING"):
closing = asyncio.create_task(messages.aclose())
@@ -1020,6 +1143,8 @@ class TestShutdown:
raise
messages._load_room_facts = _slow_to_cancel # type: ignore[method-assign]
+
+ _one_at_a_time(messages)
messages.observe(
ParticipantMessage(
tenant_id=TENANT_ZERO,