Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Route file posts through follow-the-conversation

Simon Flack3d2959c5eaf7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b264908fb..321e6d9f3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -453,6 +453,13 @@ version of their own to them without also giving them a release of their own.
token again; revocation is still checked on every request.
#### Fixed
+- **A file that names nobody now follows the conversation like text does.**
+ A post with one or more files and no mention never reached the room's
+ assigned agent, the agent whose thread it was posted in, or the agent the
+ sender was just talking to, even with follow-ups on; the same words as a
+ text message did. A post is now decided by the same rules, read from its
+ first file's text and thread, and the agent gets the post with every file
+ on it.
- **A client in more than 32,767 rooms starts reading them.** The reads a
client makes as it starts matched its rooms with one parameter each, and
Postgres takes at most 32,767 per statement. Switch's own client is in
diff --git a/core/switch_core/delivery/conversation.py b/core/switch_core/delivery/conversation.py
index d582789d9..bb26b53e3 100644
--- a/core/switch_core/delivery/conversation.py
+++ b/core/switch_core/delivery/conversation.py
@@ -20,10 +20,11 @@ A room can turn the two follow-up rules off with `!follow-ups off`; the main
agent rule is separate and unaffected. Both settings live in the room's
`metadata` (`db/stores/room_store.py`).
-The rules only ever consider a person's text message that has no `@` in it
-at all: a mention always wins, and a message from an agent, an app or Switch
+The rules only ever consider a person's message that has no `@` in it at
+all: a mention always wins, and a message from an agent, an app or Switch
itself never reaches anyone by these rules, so agents cannot set each other
-off. Permissions are not decided here; the caller applies the agent's policy
+off. A post of files counts as one message, read from its first file, which
+carries the post's text and thread. Permissions are not decided here; the caller applies the agent's policy
afterwards, as it does for a mention.
The router (`delivery/router.py`) asks these rules about a message that no
diff --git a/core/switch_core/delivery/recipient.py b/core/switch_core/delivery/recipient.py
index e05c9c000..ef2034be6 100644
--- a/core/switch_core/delivery/recipient.py
+++ b/core/switch_core/delivery/recipient.py
@@ -672,6 +672,33 @@ class AgentRecipient:
can be followed in the server's log.
"""
meta = self._meta
+ if not await self._follows(session, event, decision):
+ return NOTHING
+
+ async def after(row: InboundEvent) -> None:
+ assert isinstance(row, InboundMessage)
+ await self._after_follow_up(row, has_attachment=False)
+
+ sender_role = await self._sender_role(session, event.sender, event.content)
+ notice = await self._switch_notice(session)
+ return Delivery(
+ own_event=_message_own_event(
+ meta, event, event.thread_root_id, self.agent.id, sender_role, notice
+ ),
+ after=after,
+ )
+
+ async def _follows(
+ self,
+ session: AsyncSession,
+ event: InboundMessage,
+ decision: ConversationDecision,
+ ) -> bool:
+ """Whether a message (or a post's first file) that names nobody is
+ delivered to the agent by following the conversation: a rule chose
+ the agent, and its addressing policy and budget let the sender
+ through. Every outcome is logged."""
+ meta = self._meta
outcome = "nobody"
delivered = False
if decision.route is not None:
@@ -695,23 +722,11 @@ class AgentRecipient:
if outcome != "dropped_not_permitted"
else "the sender may not instruct this agent, or it is over budget",
)
- if not delivered:
- return NOTHING
-
- async def after(row: InboundEvent) -> None:
- assert isinstance(row, InboundMessage)
- await self._after_follow_up(row)
-
- sender_role = await self._sender_role(session, event.sender, event.content)
- notice = await self._switch_notice(session)
- return Delivery(
- own_event=_message_own_event(
- meta, event, event.thread_root_id, self.agent.id, sender_role, notice
- ),
- after=after,
- )
+ return delivered
- async def _after_follow_up(self, event: InboundMessage) -> None:
+ async def _after_follow_up(
+ self, event: InboundMessage, *, has_attachment: bool
+ ) -> None:
"""What a follow-up leaves to do once it has committed: wake the
agent's cloud machine if it is asleep, so a session can start, and
report that the agent was asked something. Nothing is posted."""
@@ -721,7 +736,9 @@ class AgentRecipient:
agent_live = await self._is_available(session, agent, meta.room_id)
if not agent_live:
await self._wake_for_follow_up()
- self._report_addressed(meta, event, agent_live=agent_live, has_attachment=False)
+ self._report_addressed(
+ meta, event, agent_live=agent_live, has_attachment=has_attachment
+ )
async def _wake_for_follow_up(self) -> None:
"""Start the cloud machine of an agent run by a controller that is
@@ -829,10 +846,16 @@ class AgentRecipient:
lead: InboundMedia,
complete: bool,
message_key: str,
+ follow: ConversationDecision | None = None,
) -> Delivery:
"""Whether a post of one or more files is one of the agent's own
events, delivered as one message carrying every file.
+ `follow` is what following the conversation decided for the agent,
+ given only when no file of the post named any agent in the room: the
+ post is then delivered as a follow-up would be (`decide_follow_up`),
+ quietly, with no refusal for a sender the policy turns away.
+
`files` are the post's rows, in the order they were written; a file
still being written is given as it will be stored, and replaced by
the stored row when the event is built. `named` is whether any of them
@@ -849,11 +872,15 @@ class AgentRecipient:
body how many files arrived.
"""
meta = self._meta
- anchor = _first_file(files) or lead
+ anchor = first_file(files) or lead
gate = _GateOutcome(addressed=False, refusal=None)
+ followed = False
if named:
agent = await self._fresh_agent(session)
gate = await self._gate_addressed(session, agent, anchor, meta)
+ elif follow is not None:
+ followed = await self._follows(session, anchor, follow)
+ delivered = gate.addressed or followed
sender_name = lead.sender_name
if not sender_name:
@@ -878,14 +905,14 @@ class AgentRecipient:
opening = opening_message_for(meta, self.agent.id)
sender_role = (
await self._sender_role(session, anchor.sender, anchor.content)
- if gate.addressed
+ if delivered
else None
)
- switch_notice = await self._switch_notice(session) if gate.addressed else None
+ switch_notice = await self._switch_notice(session) if delivered else None
def own_event(row: InboundEvent) -> OwnEvent:
parts = written(row)
- first = _first_file(parts)
+ first = first_file(parts)
anchored = first or next(p for p in parts if p.event_id == lead.event_id)
body = (first.body if first is not None else "") or anchored.body
if not complete:
@@ -922,14 +949,17 @@ class AgentRecipient:
async def after(row: InboundEvent) -> None:
parts = written(row)
- anchored = _first_file(parts) or next(
+ anchored = first_file(parts) or next(
p for p in parts if p.event_id == lead.event_id
)
- await self._after_post(anchored, gate=gate, thread_id=thread_id)
+ if followed:
+ await self._after_follow_up(anchored, has_attachment=True)
+ else:
+ await self._after_post(anchored, gate=gate, thread_id=thread_id)
return Delivery(
- own_event=own_event if gate.addressed else None,
- after=after if named else None,
+ own_event=own_event if delivered else None,
+ after=after if named or followed else None,
named=named,
)
@@ -1901,7 +1931,7 @@ def file_index(part: InboundMedia) -> int:
return group[1] if group is not None else 0
-def _first_file(parts: list[InboundMedia]) -> InboundMedia | None:
+def first_file(parts: list[InboundMedia]) -> InboundMedia | None:
"""The file a post is anchored on, index 0, when it is among `parts`."""
return next((part for part in parts if file_index(part) == 0), None)
diff --git a/core/switch_core/delivery/router.py b/core/switch_core/delivery/router.py
index c6d397e39..5c0c6f252 100644
--- a/core/switch_core/delivery/router.py
+++ b/core/switch_core/delivery/router.py
@@ -103,6 +103,7 @@ from switch_core.delivery.addressing import AddressingResolver, SenderPrincipal
from switch_core.delivery.conversation import (
ConversationMessage,
ConversationRules,
+ RoomConversation,
may_follow,
)
from switch_core.delivery.recipient import (
@@ -113,6 +114,7 @@ from switch_core.delivery.recipient import (
RoomMember,
command_event,
file_index,
+ first_file,
)
from switch_core.logging_context import log_context
from switch_core.observability.catalogue import ROUTING_FAILURES
@@ -1167,6 +1169,36 @@ class MessageRouter:
in `delivery/conversation.py`. Most chatter is settled by `may_follow`
without a read; past it, what the rules read is read once for the
room and shared by every agent's decision."""
+ # Not stamped until written (`_Unwritten`); the database stamps it
+ # with its own clock a moment from now.
+ conversation = self._conversation_of(
+ reads, recipients, event, sent_at=datetime.now(UTC)
+ )
+ if conversation is None:
+ return _Plan()
+ deliveries: list[tuple[AgentRecipient, Delivery]] = []
+ for recipient in recipients:
+ decision = await conversation.decide(
+ session,
+ agent=recipient.agent,
+ agent_user_id=recipient.transport_user_id,
+ )
+ delivery = await recipient.decide_follow_up(session, event, decision)
+ if delivery.own_event is not None:
+ deliveries.append((recipient, delivery))
+ return _Plan(deliveries=deliveries)
+
+ def _conversation_of(
+ self,
+ reads: RoomReads,
+ recipients: list[AgentRecipient],
+ event: InboundMessage,
+ *,
+ sent_at: datetime,
+ ) -> RoomConversation | None:
+ """`event`'s room as the follow-the-conversation rules read it, for a
+ message (or a post's first file) that names nobody; None when
+ `may_follow` settles that no rule can apply, without a read."""
rules = self._conversation
meta = reads.meta
message = ConversationMessage(
@@ -1175,15 +1207,13 @@ class MessageRouter:
body=event.body or "",
content=event.content,
thread_root_id=event.thread_root_id,
- # Not stamped until written (`_Unwritten`); the database stamps it
- # with its own clock a moment from now.
- sent_at=datetime.now(UTC),
+ sent_at=sent_at,
)
if not recipients or not may_follow(
rules.settings, channel_type=meta.channel_type, message=message
):
- return _Plan()
- conversation = rules.for_message(
+ return None
+ return rules.for_message(
addressing=reads.addressing,
room_id=reads.room_id,
channel_type=meta.channel_type,
@@ -1191,17 +1221,6 @@ class MessageRouter:
main_agent_id=meta.main_agent_id,
message=message,
)
- deliveries: list[tuple[AgentRecipient, Delivery]] = []
- for recipient in recipients:
- decision = await conversation.decide(
- session,
- agent=recipient.agent,
- agent_user_id=recipient.transport_user_id,
- )
- delivery = await recipient.decide_follow_up(session, event, decision)
- if delivery.own_event is not None:
- deliveries.append((recipient, delivery))
- return _Plan(deliveries=deliveries)
async def _decide_file(
self,
@@ -1241,37 +1260,85 @@ class MessageRouter:
key = group_id
return _Plan(
deliveries=await self._decide_post(
- session, recipients, files, lead=event, complete=True, key=key
+ session,
+ reads,
+ recipients,
+ files,
+ lead=event,
+ complete=True,
+ key=key,
+ # Decided as it is written, as a text message is.
+ sent_at=datetime.now(UTC),
),
group=step,
)
- @staticmethod
async def _decide_post(
+ self,
session: AsyncSession,
+ reads: RoomReads,
recipients: list[AgentRecipient],
files: list[InboundMedia],
*,
lead: InboundMedia,
complete: bool,
key: str,
+ sent_at: datetime,
) -> list[tuple[AgentRecipient, Delivery]]:
- deliveries = []
+ """Each agent's delivery of a post of one or more files.
+
+ When no file names any agent in the room, the post is the room's
+ conversation, and following the conversation may still hand it to
+ one, as it does a text message (`_follow_the_conversation`): the rules
+ read the first file, whose text and thread are the post's, and the
+ agent is given the post with all its files (CHOO-4067).
+ """
+
+ # Whether any file names the agent: the text and its mentions ride on
+ # the first file alone, so every file is asked, not only the lead.
+ # An agent this cannot be worked out for misses the post.
+ named: dict[str, bool] = {}
for recipient in recipients:
- async def decide(recipient: AgentRecipient = recipient) -> Delivery:
- named = False
+ async def name(recipient: AgentRecipient = recipient) -> Delivery:
for part in files:
if await recipient.addressed_by(session, part):
- named = True
- break
+ return Delivery(own_event=None, after=None, named=True)
+ return _NAMES_NOBODY
+
+ named_here = await _decided(recipient, lead, name)
+ if named_here is not NOTHING:
+ named[recipient.agent.id] = named_here.named
+ recipients = [r for r in recipients if r.agent.id in named]
+
+ conversation = (
+ None
+ if any(named.values())
+ else self._conversation_of(
+ reads, recipients, first_file(files) or lead, sent_at=sent_at
+ )
+ )
+ deliveries = []
+ for recipient in recipients:
+
+ async def decide(recipient: AgentRecipient = recipient) -> Delivery:
+ follow = (
+ None
+ if conversation is None
+ else await conversation.decide(
+ session,
+ agent=recipient.agent,
+ agent_user_id=recipient.transport_user_id,
+ )
+ )
return await recipient.decide_post(
session,
files,
- named=named,
+ named=named[recipient.agent.id],
lead=lead,
complete=complete,
message_key=key,
+ follow=follow,
)
deliveries.append((recipient, await _decided(recipient, lead, decide)))
@@ -1711,14 +1778,20 @@ class MessageRouter:
if member.client_id != sender
and int(member.joined_at.timestamp() * 1000) <= began
]
+ first = first_file(files) or lead
plan = _Plan(
deliveries=await self._decide_post(
session,
+ reads,
recipients,
files,
lead=lead,
complete=complete,
key=group_id,
+ # When its first file was sent: the post is decided only now,
+ # its wait over, and how long ago that was is not a gap in
+ # the conversation.
+ sent_at=datetime.fromtimestamp(first.timestamp / 1000, UTC),
)
)
stored = await self._store_own_events(session, reads, plan.deliveries, lead)
@@ -1846,6 +1919,11 @@ def _unwritten(message: Message) -> _Unwritten:
)
+# What naming a post's files found for an agent no file named; distinct from
+# `NOTHING`, which `_decided` gives an agent it could not decide for.
+_NAMES_NOBODY = Delivery(own_event=None, after=None)
+
+
# The time on a row not written yet, which nothing deciding it reads.
_NOT_STAMPED = datetime.fromtimestamp(0, UTC)
diff --git a/core/tests/routing.py b/core/tests/routing.py
index 169b52076..23b211d15 100644
--- a/core/tests/routing.py
+++ b/core/tests/routing.py
@@ -153,6 +153,7 @@ class Process:
sender: Client,
caption: str | None = None,
group: dict[str, object] | None = None,
+ thread_root_id: str | None = None,
) -> str:
"""Write one file as `sender`, a part of a multi-file post when
`group` says so, and return its event id."""
@@ -166,6 +167,7 @@ class Process:
metered=False,
msgtype="m.image",
caption=caption,
+ thread_root_id=thread_root_id,
group=group,
)
return result.event_id
diff --git a/core/tests/switch_core/delivery/test_conversation_journeys.py b/core/tests/switch_core/delivery/test_conversation_journeys.py
index 4e94d84fe..2f0ba905f 100644
--- a/core/tests/switch_core/delivery/test_conversation_journeys.py
+++ b/core/tests/switch_core/delivery/test_conversation_journeys.py
@@ -17,7 +17,7 @@ from types import SimpleNamespace
from typing import Any
import pytest
-from sqlalchemy import ColumnElement, func, select, update
+from sqlalchemy import ColumnElement, delete, func, select, update
from switch_core.addressing import owner_only_policy
from switch_core.bridges.agent.commands import (
@@ -25,14 +25,25 @@ from switch_core.bridges.agent.commands import (
_cmd_follow_ups,
_cmd_unassign_agent,
)
+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
from switch_core.clients.admin_messages import ADMIN_MARKER, AUTO_REPLY_FLAG
from switch_core.clients.room_meta import RoomMeta
-from switch_core.db.models import AgentEventRow, Client, ClientRoom, Message, Room
+from switch_core.db.models import (
+ TENANT_ZERO_ID,
+ AgentEventRow,
+ Client,
+ ClientRoom,
+ Message,
+ PendingAttachmentGroup,
+ Room,
+)
+from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.external_user_store import ExternalUserStore
from switch_core.db.stores.room_role_store import RoomRoleStore
from switch_core.db.stores.room_store import RoomStore
+from switch_core.delivery import router as router_module
from switch_core.delivery.conversation import ConversationSettings
from switch_core.transport import RoomRef
from tests.routing import Process, process_on
@@ -108,6 +119,63 @@ class Journey:
post = say
+ async def send_file(
+ self,
+ who: str,
+ filename: str,
+ *,
+ at: float,
+ thread: str | None = None,
+ caption: str | None = None,
+ group: dict[str, object] | None = None,
+ ) -> str:
+ """Write one file as `who`, `at` seconds in, and return its event id."""
+ await self._set_the_clock(at)
+ event_id = await self.process.send_file(
+ self.room,
+ filename,
+ sender=self.clients[who],
+ caption=caption,
+ group=group,
+ thread_root_id=thread,
+ )
+ self.at[event_id] = at
+ await self.process.idle()
+ return event_id
+
+ async def payloads(self, message_key: str) -> dict[str, MessagePayload]:
+ """Each agent's own event for `message_key`, by agent name."""
+ ids = {agent.id: name for name, agent in self.world.agents.items()}
+ async with self.world.session_factory() as session:
+ rows = (
+ await session.scalars(
+ select(AgentEventRow).where(
+ AgentEventRow.message_key == message_key
+ )
+ )
+ ).all()
+ payloads = {}
+ for row in rows:
+ payload = AgentEvent.model_validate(row.payload).payload
+ assert isinstance(payload, MessagePayload)
+ payloads[ids[row.agent_id]] = payload
+ return payloads
+
+ async def after_the_wait(self) -> None:
+ """Every multi-file post still waiting for files is given up on and
+ decided with the files that arrived."""
+ async with tenant_session(
+ self.world.session_factory, TENANT_ZERO_ID
+ ) as session:
+ await session.execute(
+ update(PendingAttachmentGroup).values(
+ deadline=PendingAttachmentGroup.deadline - timedelta(minutes=10)
+ )
+ )
+ await session.commit()
+ await self.process.router.flush_due_groups(TENANT_ZERO_ID)
+ await self.process.idle()
+
async def reached(self, event_id: str) -> set[str]:
"""The agents given `event_id` as one of their own events."""
ids = {agent.id: name for name, agent in self.world.agents.items()}
@@ -347,6 +415,154 @@ class TestJourneys:
assert await room.reached(after) == set()
+class TestAttachments:
+ """A post with files follows the conversation as a text message does
+ (CHOO-4067): the rules read the first file's text and thread, and the
+ agent is given the post with every file on it."""
+
+ async def test_the_assigned_agent_gets_a_file(self, room: Journey) -> None:
+ await room.command("alice", "assign-agent", "@helper", at=0)
+
+ told = await room.send_file("bob", "deploy.log", at=10)
+
+ assert await room.reached(told) == {"helper"}
+
+ async def test_a_file_follows_the_agents_answer(self, room: Journey) -> None:
+ await room.say("alice", "@helper can you read a screenshot?", at=0)
+ await room.post("helper", "Yes, send it", at=10)
+
+ told = await room.send_file("alice", "screen.png", at=20, caption="here")
+
+ assert await room.reached(told) == {"helper"}
+
+ async def test_a_file_in_the_agents_thread_reaches_it(self, room: Journey) -> None:
+ answer = await room.post("helper", "Which log?", at=0)
+
+ told = await room.send_file("alice", "app.log", at=10, thread=answer)
+
+ assert await room.reached(told) == {"helper"}
+ payload = (await room.payloads(told))["helper"]
+ assert payload.thread_id == answer
+
+ async def test_a_file_that_mentions_someone_else_follows_nothing(
+ self, room: Journey
+ ) -> None:
+ answer = await room.post("helper", "Which log?", at=0)
+
+ told = await room.send_file(
+ "alice", "app.log", at=10, thread=answer, caption="@bob have a look"
+ )
+
+ assert await room.reached(told) == set()
+
+ async def test_a_mention_on_a_file_still_wins(self, room: Journey) -> None:
+ await room.command("alice", "assign-agent", "@helper", at=0)
+
+ told = await room.send_file(
+ "alice", "pr.diff", at=10, caption="@reviewer can you check this?"
+ )
+
+ assert await room.reached(told) == {"reviewer"}
+
+ async def test_a_file_from_someone_not_allowed_is_dropped_quietly(
+ self, world: World, room: Journey
+ ) -> None:
+ await _owner_only(world, "helper")
+ answer = await room.post("helper", "Which log?", at=0)
+
+ told = await room.send_file("bob", "app.log", at=10, thread=answer)
+
+ assert await room.reached(told) == set()
+ refusals = await room.auto_replies("helper")
+ assert not any(
+ "not permitted" in body or "only take" in body for body in refusals
+ )
+
+ async def test_a_file_to_the_room_reaches_nobody(self, room: Journey) -> None:
+ told = await room.send_file("bob", "lunch-menu.pdf", at=0)
+
+ assert await room.reached(told) == set()
+
+ async def test_follow_ups_off_stops_a_file(self, room: Journey) -> None:
+ answer = await room.post("helper", "Which log?", at=0)
+ await room.command("alice", "follow-ups", "off", at=5)
+
+ told = await room.send_file("alice", "app.log", at=10, thread=answer)
+
+ assert await room.reached(told) == set()
+
+ async def test_a_multi_file_post_is_delivered_once_with_every_file(
+ self, room: Journey
+ ) -> None:
+ answer = await room.post("helper", "Send me the screenshots", at=0)
+ for index, name in enumerate(("one.png", "two.png", "three.png")):
+ await room.send_file(
+ "alice",
+ name,
+ at=10,
+ thread=answer,
+ caption="the three screens" if index == 0 else None,
+ group={"id": "grp-1", "index": index, "total": 3},
+ )
+
+ assert await room.reached("grp-1") == {"helper"}
+ payload = (await room.payloads("grp-1"))["helper"]
+ assert [a.filename for a in payload.attachments] == [
+ "one.png",
+ "two.png",
+ "three.png",
+ ]
+ assert payload.thread_id == answer
+
+ async def test_a_multi_file_post_given_up_on_still_follows(
+ self, room: Journey, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(router_module, "ATTACHMENT_GROUP_TIMEOUT_SECONDS", 60.0)
+ await room.command("alice", "assign-agent", "@helper", at=0)
+ await room.send_file(
+ "bob", "one.png", at=10, group={"id": "grp-2", "index": 0, "total": 2}
+ )
+
+ assert await room.reached("grp-2") == set()
+ await room.after_the_wait()
+
+ assert await room.reached("grp-2") == {"helper"}
+ payload = (await room.payloads("grp-2"))["helper"]
+ assert [a.filename for a in payload.attachments] == ["one.png"]
+
+ async def test_a_file_in_a_direct_room_reaches_its_agent_once(
+ self, world: World, room: Journey
+ ) -> None:
+ async with world.session_factory() as session:
+ await session.execute(
+ update(Room)
+ .where(Room.id == world.room_id)
+ .values(channel_type="direct")
+ )
+ # A direct room is one agent's: helper is left alone in it.
+ await RoomStore().remove_agents(
+ session, world.room_id, [world.agents["reviewer"].id]
+ )
+ await session.execute(
+ delete(ClientRoom).where(
+ ClientRoom.client_id == room.clients["reviewer"].id,
+ ClientRoom.room_id == world.room_id,
+ )
+ )
+ await session.commit()
+
+ told = await room.send_file("alice", "notes.md", at=0)
+
+ assert await room.reached(told) == {"helper"}
+ async with world.session_factory() as session:
+ count = await session.scalar(
+ select(func.count())
+ .select_from(AgentEventRow)
+ .where(AgentEventRow.message_key == told)
+ )
+ assert count == 1
+
+
class TestWithTheFlagOff:
async def test_nothing_changes(self, world: World) -> None:
room = await Journey(