Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Read a workspace's telemetry facts at once, not one room at a time (CHOO-3667)
Abel Dantas97b1a7fc63a3
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex de9220ae0..c47dcb533 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -232,6 +232,10 @@ version of their own to them without also giving them a release of their own.group whose subgroup would move up next to a namesake is refused too, andthe dashboard now shows why. Names can no longer contain `/` or be blank.Existing duplicates are left as they are and can be reached by id.+- **Message telemetry reads a workspace's rooms, clients and agents at once.**+ The first lookup that misses reads all of them, where each room, sender and+ agent was read on its own, so in a large workspace most messages no longer+ cost a lookup.#### Fixed- **On Slack, the 👀 and the activity card no longer arrive after the agent'sdiff --git a/core/switch_core/telemetry/messages.py b/core/switch_core/telemetry/messages.pyindex 31ee6e176..fe54a6d63 100644--- a/core/switch_core/telemetry/messages.py+++ b/core/switch_core/telemetry/messages.py@@ -188,6 +188,9 @@ class MessageTelemetry:)self._failures_in_a_row = 0self._all_lookups_resume_at: float | None = None+ # When each tenant may next be read whole (`_read_tenant`), on the+ # monotonic clock.+ self._tenants_read: dict[str, float] = {}# Taken off the queue and not yet reported. Not `qsize`, which leaves# out the one the worker is cancelled in the middle of.self._in_hand = 0@@ -453,6 +456,98 @@ class MessageTelemetry:misses,)+ async def _read_tenant(self, tenant_id: str) -> None:+ """Read the facts of every open room, the kind of every client and the+ runtime of every agent of the tenant at once, the first time a lookup+ of it misses, and again once `_ROOM_TTL_SECONDS` have passed.++ In a workspace of thousands of agents and rooms, a process reports a+ message about most of them before it reports a second one, so with one+ lookup each a cache that kept every answer still missed for most+ messages. A read that fails raises, and the lookup that asked counts+ it as its own failure; it is tried again once a failed lookup would be.+ """+ next_read = self._tenants_read.get(tenant_id)+ if next_read is not None and time.monotonic() < next_read:+ return+ # Not again for as long as a failed lookup waits; a read that succeeds+ # puts it off for as long as a room is kept, below.+ self._tenants_read[tenant_id] = time.monotonic() + _LOOKUP_BACKOFF_SECONDS+ async with tenant_session(self._session_factory, tenant_id) as session:+ open_room = and_(Room.tenant_id == tenant_id, Room.archived_at.is_(None))+ rooms = (+ await session.execute(+ select(Room.id, Room.channel_type, CollaborationBridge.type)+ .select_from(Room)+ .outerjoin(+ CollaborationBridge,+ and_(+ CollaborationBridge.id == Room.bridge_id,+ CollaborationBridge.tenant_id == tenant_id,+ ),+ )+ .where(open_room)+ )+ ).all()+ members = (+ await session.execute(+ select(ClientRoom.room_id, Client.type, func.count())+ .select_from(ClientRoom)+ .join(Client, Client.id == ClientRoom.client_id)+ .join(Room, Room.id == ClientRoom.room_id)+ .where(+ ClientRoom.tenant_id == tenant_id,+ Client.tenant_id == tenant_id,+ open_room,+ )+ .group_by(ClientRoom.room_id, Client.type)+ )+ ).all()+ clients = (+ await session.execute(+ select(Client.transport_user_id, Client.type).where(+ Client.tenant_id == tenant_id+ )+ )+ ).all()+ agents = (+ await session.execute(+ select(+ Agent.client_id,+ Agent.metadata_["known_agent_type"].astext,+ ).where(Agent.tenant_id == tenant_id)+ )+ ).all()+ self._tenants_read[tenant_id] = time.monotonic() + _ROOM_TTL_SECONDS+ by_room: dict[str, dict[str, int]] = {}+ for room_id, client_type, count in members:+ by_room.setdefault(room_id, {})[client_type] = int(count)+ for room_id, channel_type, platform in rooms:+ by_type = by_room.get(room_id, {})+ self._rooms.put(+ (tenant_id, room_id),+ _RoomFacts(+ bridge_platform=normalise_platform(platform),+ channel_type=normalise_channel_type(channel_type),+ user_count=by_type.get(HUMAN_CLIENT_TYPE, 0),+ agent_count=by_type.get(AGENT_CLIENT_TYPE, 0),+ ),+ )+ for transport_user_id, client_type in clients:+ self._sender_kinds.put(+ (tenant_id, transport_user_id),+ _CLIENT_TYPE_KIND.get(client_type, "unknown"),+ )+ for client_id, known_agent_type in agents:+ self._agent_runtimes.put(+ (tenant_id, client_id),+ normalise_known_agent_type(+ None+ if known_agent_type is None+ else {"known_agent_type": known_agent_type}+ ),+ )+async def _room_facts(self, tenant_id: str, room_id: str) -> _RoomFacts:key = (tenant_id, room_id)cached = self._rooms.get(key)@@ -461,6 +556,14 @@ class MessageTelemetry:failure_key = ("room", tenant_id, room_id)if not self._may_look_up(failure_key):return _UNKNOWN_ROOM+ try:+ await self._read_tenant(tenant_id)+ except Exception:+ self._lookup_failed(failure_key, f"room {room_id}")+ return _UNKNOWN_ROOM+ cached = self._rooms.get(key)+ if cached is not None:+ return cachedtry:facts = await self._load_room_facts(tenant_id, room_id)except Exception:@@ -520,6 +623,14 @@ class MessageTelemetry:failure_key = ("sender", tenant_id, transport_user_id)if not self._may_look_up(failure_key):return "unknown"+ try:+ await self._read_tenant(tenant_id)+ except Exception:+ self._lookup_failed(failure_key, "a message sender")+ return "unknown"+ cached = self._sender_kinds.get(key)+ if cached is not None:+ return cachedtry:async with tenant_session(self._session_factory, tenant_id) as session:client_type = (@@ -550,6 +661,14 @@ class MessageTelemetry:failure_key = ("agent", tenant_id, client_id)if not self._may_look_up(failure_key):return _UNKNOWN_RUNTIME+ try:+ await self._read_tenant(tenant_id)+ except Exception:+ self._lookup_failed(failure_key, "an agent's runtime")+ return _UNKNOWN_RUNTIME+ cached = self._agent_runtimes.get(key)+ if cached is not None:+ return cachedtry:async with tenant_session(self._session_factory, tenant_id) as session:result = await session.execute(diff --git a/core/tests/switch_core/telemetry/test_message_events.py b/core/tests/switch_core/telemetry/test_message_events.pyindex 23fee7851..05cb84edb 100644--- a/core/tests/switch_core/telemetry/test_message_events.py+++ b/core/tests/switch_core/telemetry/test_message_events.py@@ -31,6 +31,7 @@ from switch_core.telemetry.messages import MessageTelemetry, _RoomFacts, _TtlCacfrom switch_core.telemetry.service import TelemetryServicefrom switch_core.telemetry.sink import TelemetryRecordfrom switch_core.transport.observer import ParticipantMessage+from tests.session_counting import CountingSessionFactoryTENANT_ZERO = "00000000-0000-0000-0000-000000000000"@@ -200,6 +201,17 @@ class _Unreachable:raise ConnectionError("database unreachable")+def _one_at_a_time(messages: MessageTelemetry) -> None:+ """Leave out the read of the whole workspace (`MessageTelemetry._read_tenant`),+ so each room, sender and agent is looked up on its own, as these tests+ count."""++ async def _not_read(tenant_id: str) -> None:+ return None++ messages._read_tenant = _not_read # type: ignore[method-assign]++async def _reported(messages: MessageTelemetry, service: TelemetryService, sink: _RecordingSink) -> list[tuple[str, dict[str, object]]]:@@ -429,6 +441,8 @@ class TestFailingLookups:messages._load_room_facts = _broken # type: ignore[method-assign]+ _one_at_a_time(messages)+with caplog.at_level("WARNING"):for _ in range(3):messages.observe(_said(world, world.humans[0].id, "human"))@@ -459,6 +473,8 @@ class TestFailingLookups:return await _a_slack_room(tenant_id, room_id)messages._load_room_facts = _bad_room_fails # type: ignore[method-assign]++ _one_at_a_time(messages)other_tenant = str(uuid.uuid4())messages.observe(_said_in("bad"))@@ -507,6 +523,8 @@ class TestFailingLookups:messages._load_room_facts = _broken # type: ignore[method-assign]+ _one_at_a_time(messages)+with caplog.at_level("WARNING"):for room in ("a", "b", "c", "d", "e"):messages.observe(_said_in(room))@@ -533,6 +551,8 @@ class TestFailingLookups:messages._load_room_facts = _only_good_answers # type: ignore[method-assign]+ _one_at_a_time(messages)+for room in ("a", "b", "good", "c", "d"):messages.observe(_said_in(room))await _reported(messages, service, sink)@@ -614,6 +634,8 @@ class TestCaching:messages._load_room_facts = _gone # type: ignore[method-assign]+ _one_at_a_time(messages)+for _ in range(3):messages.observe(_said_in("deleted-room"))reported = await _reported(messages, service, sink)@@ -671,6 +693,99 @@ class TestCaching:] == ["unknown", "unknown"]+class TestTheWholeWorkspace:+ """The first lookup that misses reads every open room, client and agent+ of the workspace at once (`MessageTelemetry._read_tenant`): in a workspace+ of thousands, a process reports on most of them before it reports a second+ message about any, so one lookup each missed for most messages."""++ async def test_other_rooms_senders_and_agents_cost_no_read(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ async with session_factory() as session:+ other = await _room(session, bridge_platform=None, channel_type="direct")+ await _join(session, world.humans[1].id, other)+ await session.commit()+ counting = CountingSessionFactory(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=counting) # type: ignore[arg-type]++ messages.observe(_said(world, world.agent.client_id, "agent"))+ await messages._queue.join()+ assert counting.opened == 1+ messages.observe(_said_in(other.id, sender_client_id=world.humans[1].id))+ messages.agent_addressed(+ tenant_id=TENANT_ZERO,+ room_id=other.id,+ sender_transport_user_id=world.humans[1].transport_user_id,+ from_platform=False,+ agent_metadata=None,+ agent_live=True,+ has_attachment=False,+ )+ reported = await _reported(messages, service, sink)++ assert counting.opened == 1+ facts = {+ (name, p.get("bridge_platform"), p.get("room_user_count"))+ for name, p in reported+ }+ assert ("switch_core.room_message_sent", "slack", 2) in facts+ assert ("switch_core.room_message_sent", "none", 1) in facts+ assert [+ p["known_agent_type"]+ for name, p in reported+ if name == "switch_core.agent_message_sent"+ ] == ["codex"]+ assert [+ p["sender_kind"]+ for name, p in reported+ if name == "switch_core.agent_message_received"+ ] == ["user"]++ async def test_a_room_made_after_the_read_is_looked_up_on_its_own(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ world = await _world(session_factory)+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(telemetry=service, session_factory=session_factory)+ messages.observe(_said(world, world.humans[0].id, "human"))+ await messages._queue.join()+ async with session_factory() as session:+ later = await _room(session, bridge_platform=None, channel_type="direct")+ await _join(session, world.humans[0].id, later)+ await session.commit()++ messages.observe(_said_in(later.id, sender_client_id=world.humans[0].id))+ reported = await _reported(messages, service, sink)++ assert [p["room_user_count"] for _, p in reported] == [2, 1]++ async def test_a_read_that_fails_is_the_lookups_failure(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ """Counted once, not followed by the lookup it was for, and not tried+ again while a failed lookup would not be."""+ database = _Unreachable()+ sink = _RecordingSink()+ service = _service(sink)+ messages = MessageTelemetry(+ telemetry=service,+ session_factory=database, # type: ignore[arg-type]+ )++ for _ in range(2):+ messages.observe(_said_in("room"))+ with caplog.at_level("WARNING"):+ reported = await _reported(messages, service, sink)++ assert [p["bridge_platform"] for _, p in reported] == ["unknown", "unknown"]+ assert database.attempts == 1++class TestHowLongLookupsAreKept:def test_a_room_for_a_quarter_hour_and_who_someone_is_for_an_hour(self, monkeypatch: pytest.MonkeyPatch@@ -740,6 +855,7 @@ class TestBackpressure:session_factory=None, # type: ignore[arg-type])messages._load_room_facts = _a_slack_room # type: ignore[method-assign]+ _one_at_a_time(messages)with caplog.at_level("WARNING"):for _ in range(4):@@ -769,6 +885,7 @@ class TestFailingSenderAndRuntimeLookups:session_factory=database, # type: ignore[arg-type])messages._load_room_facts = _a_slack_room # type: ignore[method-assign]+ _one_at_a_time(messages)for sender in ("@alice:test", "@alice:test"):messages.agent_addressed(@@ -798,6 +915,7 @@ class TestFailingSenderAndRuntimeLookups:session_factory=database, # type: ignore[arg-type])messages._load_room_facts = _a_slack_room # type: ignore[method-assign]+ _one_at_a_time(messages)for _ in range(2):messages.observe(@@ -833,6 +951,8 @@ class TestFailingSenderAndRuntimeLookups:messages._load_room_facts = _broken # type: ignore[method-assign]+ _one_at_a_time(messages)+with caplog.at_level("WARNING"):for _ in range(3):messages.observe(_said_in("room"))@@ -859,6 +979,8 @@ class TestFailingSenderAndRuntimeLookups:messages._load_room_facts = _gone # type: ignore[method-assign]+ _one_at_a_time(messages)+with caplog.at_level("WARNING"):messages.observe(_said_in("room-a"))messages.observe(_said_in("room-b"))@@ -980,6 +1102,7 @@ class TestShutdown:session_factory=None, # type: ignore[arg-type])messages._load_room_facts = _slow_room # type: ignore[method-assign]+ _one_at_a_time(messages)messages.observe(_said_in("room"))with caplog.at_level("WARNING"):closing = asyncio.create_task(messages.aclose())@@ -1020,6 +1143,8 @@ class TestShutdown:raisemessages._load_room_facts = _slow_to_cancel # type: ignore[method-assign]++ _one_at_a_time(messages)messages.observe(ParticipantMessage(tenant_id=TENANT_ZERO,