Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Cross-replica idempotency ids + minor fixes (#858)

Abel Dantas525bc17b9d16
diff --git a/core/switch_core/bridges/agent/api/handlers.py b/core/switch_core/bridges/agent/api/handlers.py
index c720ffd6a..594c0c5f7 100644
--- a/core/switch_core/bridges/agent/api/handlers.py
+++ b/core/switch_core/bridges/agent/api/handlers.py
@@ -1007,11 +1007,13 @@ async def _open_connection(
# stream 409s and retries forever.
async with protocol.connections.slots(agent.id):
protocol.connections.claim_room(conn, room_id, takeover=True)
- # The room's unread count follows the slot: whoever is told how far
- # behind the room is has to be the one whose reading clears it.
- await protocol.event_buffer.take_counting(
- agent.tenant_id, agent.id, conn.id, room_id
- )
+ # The room's unread count follows the slot: whoever is told how
+ # far behind the room is has to be the one whose reading clears
+ # it. Recorded with the slots still held, so a later claim of
+ # the room cannot have its count overwritten by this one.
+ await protocol.event_buffer.take_counting(
+ agent.tenant_id, agent.id, conn.id, room_id
+ )
except (ValueError, PermissionError) as exc:
protocol.connections.close(
conn.id,
@@ -1418,12 +1420,14 @@ async def connection_subscribe(
for departed in departing:
protocol.connections.release_room(conn, departed)
- # A room slot changes hands here as much as it does on the stream URL or in
- # connect_to_room, and the room's unread count follows it: the holder being
- # told how far behind the room is has to be the one whose reading clears it.
- await protocol.event_buffer.take_counting(
- agent.tenant_id, agent.id, conn.id, req.room_id
- )
+ # A room slot changes hands here as much as it does on the stream URL
+ # or in connect_to_room, and the room's unread count follows it: the
+ # holder being told how far behind the room is has to be the one whose
+ # reading clears it. Recorded with the slots still held, so a later
+ # claim of the room cannot have its count overwritten by this one.
+ await protocol.event_buffer.take_counting(
+ agent.tenant_id, agent.id, conn.id, req.room_id
+ )
if evicted is not None:
logger.warning(
@@ -1501,14 +1505,17 @@ async def connection_placements(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
- for session_id, room_id in req.placements.items():
- if before.get(session_id) != room_id:
- await protocol.event_buffer.hand_counting_to(
- agent.tenant_id,
- agent.id,
- Reader(id=session_id, is_session=True),
- room_id,
- )
+ # Each moved session takes its room's unread count, with the slots
+ # still held, so a later claim of the room cannot have its count
+ # overwritten by this one.
+ for session_id, room_id in req.placements.items():
+ if before.get(session_id) != room_id:
+ await protocol.event_buffer.hand_counting_to(
+ agent.tenant_id,
+ agent.id,
+ Reader(id=session_id, is_session=True),
+ room_id,
+ )
for lost in released:
logger.warning(
"[CONN] agent=%s connection=%s took room %s from connection %s "
diff --git a/core/switch_core/bridges/agent/protocol/controller_stream.py b/core/switch_core/bridges/agent/protocol/controller_stream.py
index 11736a9cc..1b37a7c33 100644
--- a/core/switch_core/bridges/agent/protocol/controller_stream.py
+++ b/core/switch_core/bridges/agent/protocol/controller_stream.py
@@ -437,14 +437,20 @@ class _ControllerStream:
attached.cursor = max(attached.cursor, bound)
self._conn.sent_cursors[agent_id] = attached.cursor
return []
+ # The unread counts sent with the events below, counted for the whole
+ # batch at once, so that catching up on a few hundred events costs a
+ # few statements per room rather than several per event.
+ missed = await self._buffer.unread_batch(
+ self._tenant,
+ agent_id,
+ [(item.room_id, item.room_seq) for item in pending if item.notifiable],
+ )
frames: list[ControllerFrame] = []
for item in pending:
data = item.event.model_dump(mode="json")
data["sequence"] = item.seq
if item.notifiable:
- unread = await self._buffer.unread(
- self._tenant, agent_id, item.room_id, item.room_seq
- )
+ unread = missed[(item.room_id, item.room_seq)]
data["missed"] = {"count": unread.count, "reason": unread.reason}
attached.cursor = item.seq
self._conn.sent_cursors[agent_id] = item.seq
diff --git a/core/switch_core/bridges/agent/protocol/event_buffer.py b/core/switch_core/bridges/agent/protocol/event_buffer.py
index 7a8867b42..55c8c70c9 100644
--- a/core/switch_core/bridges/agent/protocol/event_buffer.py
+++ b/core/switch_core/bridges/agent/protocol/event_buffer.py
@@ -47,12 +47,13 @@ from __future__ import annotations
import asyncio
import logging
import time
+from collections.abc import Iterable
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING, Any, cast
-from sqlalchemy import BigInteger, delete, func, literal, select, text
-from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import BigInteger, bindparam, delete, func, literal, select, text
+from sqlalchemy.dialects.postgresql import ARRAY, insert
from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
@@ -137,16 +138,10 @@ CONVERSATION_GROUP_GRACE_SECONDS = 5.0
_READ = {"operation": "read"}
_UNREAD = {"operation": "unread"}
-# The stored poll position moved forward by the new one, source by source,
-# never back: `positions.merged`, written in SQL so that it happens inside the
-# upsert and two overlapping saves cannot undo each other.
-_MERGED_POLL_POSITION = text(
- "(SELECT jsonb_object_agg(source, GREATEST("
- "COALESCE((agent_poll_positions.position ->> source)::bigint, -1), "
- "COALESCE((excluded.position ->> source)::bigint, -1))) "
- "FROM (SELECT jsonb_object_keys(agent_poll_positions.position) AS source "
- "UNION SELECT jsonb_object_keys(excluded.position)) AS sources)"
-)
+# How many times a long poll reads again after another poll of the same reader
+# saved a position first. Each lost save means another poll moved the reader,
+# so the next read starts from where that one stopped.
+_POLL_SAVE_ATTEMPTS = 5
# How long `enqueue` waits before each retry of a write that failed because the
# database connection or the connection pool did. After the last, the error is
@@ -249,10 +244,14 @@ class Conversation:
that are not conversation (the agent's own messages, other kinds of event,
rows that are already own events) are skipped, but they still count as read, so
the reader's position moves past them and they are not examined again.
+ `cut` maps each room whose read stopped at the row limit, with rows still
+ to read after it, to when the last row read was sent (seconds since the
+ epoch).
"""
events: list[BufferedEvent] = field(default_factory=list)
through: dict[str, int] = field(default_factory=dict)
+ cut: dict[str, float] = field(default_factory=dict)
@dataclass(frozen=True)
@@ -306,26 +305,44 @@ _MESSAGE_KEY_SQL = (
"COALESCE(m.content->'com.switch.attachment_group'->>'id', m.transport_event_id)"
)
+# The unread counts of one room at several positions (`:throughs`, each a
+# `messages.seq` in the room), in one statement. A message counts at a
+# position when it is a message, not sent by the agent, not one of its own
+# events, and inside (`:baseline`, position]. Several rows of one message share
+# a key and count once, so each key is reduced to its first qualifying row, and
+# the count at a position is the number of keys whose first row is at or
+# before it. That is the same number as counting the distinct keys of the
+# qualifying rows up to that position, but the room's rows are read once for
+# all the positions instead of once per position. `:through` is the highest
+# of `:throughs`. `firsts` is MATERIALIZED because Postgres otherwise copies it
+# into the per-position count, and under a cached generic plan that reruns the
+# room scan once per position (224 ms instead of 5 ms for 200 positions).
_UNREAD_SQL = text(
f"""
- SELECT count(DISTINCT {_MESSAGE_KEY_SQL})
- FROM messages m
- WHERE m.room_id = :room_id
- AND m.seq > :baseline
- AND m.seq <= :through
- AND m.seq > 0
- AND m.event_type = 'm.room.message'
- AND m.sender_id <> :me
- AND NOT EXISTS (
- SELECT 1 FROM agent_events e
- WHERE e.agent_id = :agent_id
- AND e.room_id = :room_id
- AND e.type = 'message'
- AND e.message_key = {_MESSAGE_KEY_SQL}
- AND e.room_seq >= m.seq
- )
+ WITH firsts AS MATERIALIZED (
+ SELECT min(m.seq) AS first_seq
+ FROM messages m
+ WHERE m.room_id = :room_id
+ AND m.seq > :baseline
+ AND m.seq <= :through
+ AND m.seq > 0
+ AND m.event_type = 'm.room.message'
+ AND m.sender_id <> :me
+ AND NOT EXISTS (
+ SELECT 1 FROM agent_events e
+ WHERE e.agent_id = :agent_id
+ AND e.room_id = :room_id
+ AND e.type = 'message'
+ AND e.message_key = {_MESSAGE_KEY_SQL}
+ AND e.room_seq >= m.seq
+ )
+ GROUP BY {_MESSAGE_KEY_SQL}
+ )
+ SELECT t.through,
+ (SELECT count(*) FROM firsts f WHERE f.first_seq <= t.through)
+ FROM unnest(:throughs) AS t(through)
"""
-)
+).bindparams(bindparam("throughs", type_=ARRAY(BigInteger)))
class EventBuffer:
@@ -424,8 +441,35 @@ class EventBuffer:
room_seq: int,
message_key: str | None,
) -> int:
- """Number one own event and insert it, in one transaction."""
+ """Number one own event and insert it, in one transaction, unless an
+ event from the same source is already stored, and return its number.
+
+ The source is `(message_key, room_id, room_seq, type)`
+ (`uq_agent_events_source`):
+ the consumer of another switch-core process makes the same decision
+ for the same room row, and a write retried after a commit whose
+ acknowledgement was lost repeats one that was made. Either would
+ otherwise give the agent the same event twice under two numbers. When
+ the other write lands between the lookup and the insert, the insert
+ does nothing and the transaction is rolled back, which also gives back
+ the number it took.
+ """
+ source = (
+ select(AgentEventRow.seq).where(
+ AgentEventRow.agent_id == agent_id,
+ AgentEventRow.message_key == message_key,
+ AgentEventRow.room_id == room_id,
+ AgentEventRow.room_seq == room_seq,
+ AgentEventRow.type == event.type,
+ )
+ if message_key is not None
+ else None
+ )
async with tenant_session(self._session_factory, tenant_id) as session:
+ if source is not None:
+ stored = await session.scalar(source)
+ if stored is not None:
+ return int(stored)
claim = (
insert(AgentEventHead)
.values(
@@ -441,8 +485,9 @@ class EventBuffer:
.returning(AgentEventHead.last_seq)
)
seq = (await session.execute(claim)).scalar_one()
- session.add(
- AgentEventRow(
+ inserted = await session.scalar(
+ insert(AgentEventRow)
+ .values(
tenant_id=tenant_id,
agent_id=agent_id,
seq=seq,
@@ -453,7 +498,31 @@ class EventBuffer:
payload=event.model_dump(mode="json"),
notifiable=is_notifiable(event),
)
+ .on_conflict_do_nothing(
+ index_elements=[
+ AgentEventRow.agent_id,
+ AgentEventRow.message_key,
+ AgentEventRow.room_id,
+ AgentEventRow.room_seq,
+ AgentEventRow.type,
+ ],
+ index_where=AgentEventRow.message_key.is_not(None),
+ )
+ .returning(AgentEventRow.seq)
)
+ if inserted is None:
+ await session.rollback()
+ assert source is not None
+ stored = await session.scalar(source)
+ assert stored is not None
+ logger.info(
+ "[EVENT-BUF] own event for agent %s from %s was already stored "
+ "as %s",
+ agent_id,
+ message_key,
+ stored,
+ )
+ return int(stored)
await session.commit()
return seq
@@ -777,6 +846,7 @@ class EventBuffer:
started = time.perf_counter()
events: list[BufferedEvent] = []
through: dict[str, int] = {}
+ cut: dict[str, float] = {}
async with tenant_session(self._session_factory, tenant_id) as session:
me = await session.execute(
select(Client.transport_user_id, Client.id)
@@ -838,8 +908,7 @@ class EventBuffer:
room = rooms.get(room_id)
if room is None or room_id not in joined:
continue
- rows: list[Message] = []
- for row in (
+ fetched = (
await session.scalars(
select(Message)
.where(
@@ -847,9 +916,11 @@ class EventBuffer:
Message.seq > max(after_seq, 0),
)
.order_by(Message.seq)
- .limit(limit)
+ .limit(limit + 1)
)
- ).all():
+ ).all()
+ rows: list[Message] = []
+ for row in fetched[:limit]:
if (
row.seq > decided_through.get(room_id, 0)
and cast(datetime, row.sent_at) >= settled_before
@@ -858,6 +929,8 @@ class EventBuffer:
rows.append(row)
if not rows:
continue
+ if len(fetched) > limit and len(rows) == limit:
+ cut[room_id] = _epoch(cast(datetime, rows[-1].sent_at))
room_events, room_through = await self._conversation_in(
session,
room,
@@ -871,7 +944,7 @@ class EventBuffer:
if room_through > after_seq:
through[room_id] = room_through
self._record_scan(_READ, len(events), started)
- return Conversation(events=events, through=through)
+ return Conversation(events=events, through=through, cut=cut)
async def _conversation_in(
self,
@@ -942,6 +1015,21 @@ class EventBuffer:
if not complete and cast(datetime, lead.sent_at) >= group_deadline:
first = min(row.seq for _, _, row in parts)
held_from = first if held_from is None else min(held_from, first)
+ # A group goes out at its last file, so one with a file before the
+ # hold and its last file at or after it would not go out in this read,
+ # while the read's place moved past its earlier files. The next read
+ # would then find only the later files and send the group without
+ # them. Such a group is held from its first file too, which can reach
+ # back to another group that starts earlier.
+ spans = [
+ (min(row.seq for _, _, row in parts), max(row.seq for _, _, row in parts))
+ for parts in groups.values()
+ ]
+ while held_from is not None:
+ spanning = [first for first, last in spans if first < held_from <= last]
+ if not spanning:
+ break
+ held_from = min(spanning)
member_names = await self._member_names(
session,
@@ -1149,7 +1237,29 @@ class EventBuffer:
)
for room_id, seq in conversation.through.items():
looked[room_source(room_id)] = seq
- return _interleaved([own_events, *_by_source(conversation.events)]), looked
+ events = _interleaved([own_events, *_by_source(conversation.events)])
+ # A source cut short at `limit` has more to read after the last row it
+ # returned. An event from another source that happened later than that
+ # row waits for the next read, or it would overtake the cut source's
+ # later events. The events go out as a prefix of the merged order,
+ # with at least one event, so every read moves forward even when
+ # clocks disagree. A source with an event held back keeps its position
+ # at the last event sent, so the next read finds the event again.
+ cut = {room_source(room_id): at for room_id, at in conversation.cut.items()}
+ if len(own_events) >= limit:
+ cut[OWN_EVENTS] = own_events[-1].at
+ if cut:
+ kept = len(events)
+ for index, item in enumerate(events[1:], start=1):
+ if any(
+ item.at > at for source, at in cut.items() if source != item.source
+ ):
+ kept = index
+ break
+ for source in {item.source for item in events[kept:]}:
+ looked.pop(source, None)
+ events = events[:kept]
+ return events, looked
# ------------------------------------------------------------------
# Tickets, for clients that hold one number
@@ -1342,38 +1452,92 @@ class EventBuffer:
can no longer be told apart. A count cut at the window says so in its
`reason`.
"""
+ counts = await self.unread_batch(tenant_id, agent_id, [(room_id, through_seq)])
+ return counts[(room_id, through_seq)]
+
+ async def unread_batch(
+ self,
+ tenant_id: str,
+ agent_id: str,
+ positions: Iterable[tuple[str, int]],
+ ) -> dict[tuple[str, int], Unread]:
+ """`unread` for every (room, `messages.seq`) pair in `positions`, keyed
+ by the pair.
+
+ A stream catching an agent up reports a count alongside every event it
+ is woken for, up to a few hundred in one batch. Counted one at a time,
+ each count read the room's baseline, its retention window and the
+ agent's own user id again, then rescanned the same stretch of the
+ room's log. Here the agent's baselines and user id are read once, and
+ each room in the batch costs two statements however many positions are
+ asked of it: its window start, and one scan that counts at all of its
+ positions together (`_UNREAD_SQL`). The counts are the ones `unread`
+ gives for each pair on its own.
+ """
+ wanted: dict[str, set[int]] = {}
+ for room_id, through_seq in positions:
+ wanted.setdefault(room_id, set()).add(through_seq)
+ if not wanted:
+ return {}
+ counts: dict[tuple[str, int], Unread] = {}
async with tenant_session(self._session_factory, tenant_id) as session:
- baseline = await session.scalar(
- select(AgentRoomRead.caught_up_seq).where(
- AgentRoomRead.agent_id == agent_id,
- AgentRoomRead.room_id == room_id,
- )
- )
- if baseline is None:
- return Unread(count=None, reason=NO_BASELINE)
- cutoff = datetime.now(UTC) - timedelta(seconds=self._retention_seconds)
- window_start = await session.scalar(_window_start(room_id, cutoff)) or 0
- reason = None
- if baseline < window_start:
- baseline, reason = window_start, COUNTED_FROM_WINDOW
+ baselines: dict[str, int] = {
+ room_id: seq
+ for room_id, seq in (
+ await session.execute(
+ select(
+ AgentRoomRead.room_id, AgentRoomRead.caught_up_seq
+ ).where(
+ AgentRoomRead.agent_id == agent_id,
+ AgentRoomRead.room_id.in_(wanted),
+ )
+ )
+ ).all()
+ }
+ for room_id in wanted.keys() - baselines.keys():
+ for through_seq in wanted[room_id]:
+ counts[(room_id, through_seq)] = Unread(
+ count=None, reason=NO_BASELINE
+ )
+ if not baselines:
+ return counts
me = await session.scalar(
select(Client.transport_user_id)
.join(Agent, Agent.client_id == Client.id)
.where(Agent.id == agent_id)
)
- started = time.perf_counter()
- count = await session.scalar(
- _UNREAD_SQL,
- {
- "room_id": room_id,
- "baseline": baseline,
- "through": through_seq,
- "me": me or "",
- "agent_id": agent_id,
- },
- )
- self._record_scan(_UNREAD, count or 0, started)
- return Unread(count=count or 0, reason=reason)
+ cutoff = datetime.now(UTC) - timedelta(seconds=self._retention_seconds)
+ for room_id, baseline in baselines.items():
+ window_start = await session.scalar(_window_start(room_id, cutoff)) or 0
+ reason = None
+ if baseline < window_start:
+ baseline, reason = window_start, COUNTED_FROM_WINDOW
+ throughs = sorted(wanted[room_id])
+ started = time.perf_counter()
+ at = {
+ through: count
+ for through, count in (
+ await session.execute(
+ _UNREAD_SQL,
+ {
+ "room_id": room_id,
+ "baseline": baseline,
+ "through": throughs[-1],
+ "throughs": throughs,
+ "me": me or "",
+ "agent_id": agent_id,
+ },
+ )
+ ).all()
+ }
+ # One scan of the room's log, recorded with what it counted at
+ # the furthest position asked, which every other count is part of.
+ self._record_scan(_UNREAD, at[throughs[-1]] or 0, started)
+ for through_seq in throughs:
+ counts[(room_id, through_seq)] = Unread(
+ count=at[through_seq] or 0, reason=reason
+ )
+ return counts
# ------------------------------------------------------------------
# Leaving
@@ -1589,6 +1753,53 @@ class EventBuffer:
"""Read once for a long-polling reader, save its new position, and
return the events.
+ Two polls of the same reader can overlap. Each saves its new position
+ only if the stored one is still the one it read from, so of two polls
+ that read the same events, one returns them and the other reads again
+ from where the first stopped. Returning nothing after the last attempt
+ loses nothing, because the position did not move.
+ """
+ for _ in range(_POLL_SAVE_ATTEMPTS):
+ stored, position = await self._poll_position(
+ tenant_id, agent_id, reader_id, rooms if conversation else set()
+ )
+ events, moved = await self._legacy_read(
+ tenant_id,
+ agent_id,
+ reader_id,
+ position,
+ rooms,
+ conversation,
+ notifiable_only,
+ rooms_are_membership,
+ )
+ if await self._save_poll_position(
+ tenant_id, agent_id, reader_id, moved, expected=stored
+ ):
+ return events
+ logger.warning(
+ "[EVENT-BUF] legacy poller %s for agent %s lost the save of its "
+ "position %s times in a row to overlapping polls; returning nothing",
+ reader_id,
+ agent_id,
+ _POLL_SAVE_ATTEMPTS,
+ )
+ return []
+
+ async def _legacy_read(
+ self,
+ tenant_id: str,
+ agent_id: str,
+ reader_id: str,
+ position: Position,
+ rooms: set[str],
+ conversation: bool,
+ notifiable_only: bool,
+ rooms_are_membership: bool,
+ ) -> tuple[list[AgentEvent], Position]:
+ """Read for a long-polling reader from `position`, and return the
+ events with the position to save once they are returned.
+
The bound on own event numbers is read before the own events and the read stops
at it, so the saved position covers exactly the own events returned. When
`rooms` is the agent's membership, the bound is the newest own event in
@@ -1598,9 +1809,6 @@ class EventBuffer:
`poll_room` reads a fixed room chosen by the reader, so it passes
`rooms_are_membership=False` and moves to the head.
"""
- position = await self._poll_position(
- tenant_id, agent_id, reader_id, rooms if conversation else set()
- )
after = position[OWN_EVENTS]
if rooms_are_membership:
through = await self._last_seq_in_rooms(tenant_id, agent_id, after, rooms)
@@ -1650,14 +1858,14 @@ class EventBuffer:
moved,
{room_source(room_id): seq for room_id, seq in said.through.items()},
)
- await self._save_poll_position(tenant_id, agent_id, reader_id, moved)
- return [item.event for item in _interleaved(found)]
+ return [item.event for item in _interleaved(found)], moved
async def _poll_position(
self, tenant_id: str, agent_id: str, reader_id: str, rooms: set[str]
- ) -> Position:
- """A long-polling reader's position from `agent_poll_positions`, with a
- number for each room in `rooms`.
+ ) -> tuple[Position | None, Position]:
+ """A long-polling reader's stored position from `agent_poll_positions`,
+ or None when it has none, and the position to read from, with a number
+ for each room in `rooms`.
A reader seen for the first time starts just before the agent's oldest
own event still kept, so own events written before the client first polled are
@@ -1666,13 +1874,14 @@ class EventBuffer:
window, for the same reason.
"""
async with tenant_session(self._session_factory, tenant_id) as session:
- stored = await session.scalar(
+ row = await session.scalar(
select(AgentPollPosition.position).where(
AgentPollPosition.agent_id == agent_id,
AgentPollPosition.reader_id == reader_id,
)
)
- position: Position = dict(stored) if stored is not None else {}
+ stored: Position | None = dict(row) if row is not None else None
+ position: Position = dict(stored or {})
if OWN_EVENTS not in position:
position[OWN_EVENTS] = await self.before_retained(tenant_id, agent_id)
missing = {room_id for room_id in rooms if room_source(room_id) not in position}
@@ -1680,20 +1889,32 @@ class EventBuffer:
starts = await self.room_window_starts(tenant_id, missing)
for room_id, start in starts.items():
position[room_source(room_id)] = start
- return position
+ return stored, position
async def _save_poll_position(
- self, tenant_id: str, agent_id: str, reader_id: str, position: Position
- ) -> None:
- # The row is inserted from a SELECT on `agents`, so if the agent was
- # deleted while the poll was in flight, nothing is saved and the insert
- # does not fail on the foreign key. An existing row is merged with the
- # new position, source by source, rather than replaced: two polls of
- # the same reader can overlap, and the one that finishes last may have
- # read less, so replacing would move the reader back and deliver the
- # same events again.
+ self,
+ tenant_id: str,
+ agent_id: str,
+ reader_id: str,
+ position: Position,
+ *,
+ expected: Position | None,
+ ) -> bool:
+ """Store a long-polling reader's new position if its stored position is
+ still `expected` (None for no row), and return whether it was stored.
+
+ The row is inserted from a SELECT on `agents`, so if the agent was
+ deleted while the poll was in flight, nothing is saved and the insert
+ does not fail on the foreign key.
+ """
+ unchanged = (
+ AgentPollPosition.position
+ == literal(expected, AgentPollPosition.position.type)
+ if expected is not None
+ else literal(False)
+ )
async with tenant_session(self._session_factory, tenant_id) as session:
- await session.execute(
+ saved = await session.scalar(
insert(AgentPollPosition)
.from_select(
[
@@ -1714,10 +1935,15 @@ class EventBuffer:
AgentPollPosition.agent_id,
AgentPollPosition.reader_id,
],
- set_={"position": _MERGED_POLL_POSITION},
+ set_={
+ "position": literal(position, AgentPollPosition.position.type)
+ },
+ where=unchanged,
)
+ .returning(AgentPollPosition.reader_id)
)
await session.commit()
+ return saved is not None
async def _last_seq_in_rooms(
self, tenant_id: str, agent_id: str, after_seq: int, rooms: set[str]
diff --git a/core/switch_core/bridges/agent/protocol/stream.py b/core/switch_core/bridges/agent/protocol/stream.py
index ef057952d..03462caf3 100644
--- a/core/switch_core/bridges/agent/protocol/stream.py
+++ b/core/switch_core/bridges/agent/protocol/stream.py
@@ -376,26 +376,46 @@ async def _event_stream(
# Room coverage is evaluated per event rather than up front: a
# sibling claiming a room must take effect immediately, without
- # tearing the stream down. One of the agent's own events for a room
- # this connection does not cover is not sent, but the position
- # still moves past it, in order: only once every event before it
- # has gone out, so the position never says the client was sent
- # an event that is still to come. Room messages need no check
- # here, because `_conversation_rooms` only returned rooms this
- # connection covers.
+ # tearing the stream down. An event for a room this connection does
+ # not cover is not sent, but the position still moves past it, in
+ # order: only once every event before it has gone out, so the
+ # position never says the client was sent an event that is still
+ # to come. `covered` is the plan when the batch is read, which the
+ # tickets are minted for: room messages are in it because
+ # `_conversation_rooms` only returned rooms this connection covers.
+ # Each event is checked again just before it is sent, because the
+ # loop suspends at every yield and a sibling can claim the room
+ # while it does. The check reads the registry in memory, so it
+ # costs no query.
covered = [
item.source != OWN_EVENTS or registry.covers(conn, item.room_id)
for item in pending
]
tickets = await _tickets_for(conn, buffer, tenant, pending, covered)
+ # The unread counts sent with the events below, counted for the
+ # whole batch at once, so that catching up on a few hundred events
+ # costs a few statements per room rather than several per event.
+ missed = await buffer.unread_batch(
+ tenant,
+ agent_id,
+ [
+ (item.room_id, item.room_seq)
+ for item, send in zip(pending, covered, strict=True)
+ if send and item.notifiable
+ ],
+ )
delivered = False
index = -1
for item, send in zip(pending, covered, strict=True):
- if not send:
- if item.seq > conn.position.get(OWN_EVENTS, -1):
- conn.position[OWN_EVENTS] = item.seq
+ if send:
+ # Counted before the re-check below, so a ticket stays
+ # with the event it was minted for even when an event
+ # before it in the batch is no longer sent.
+ index += 1
+ if not send or not registry.covers(conn, item.room_id):
+ if item.seq > conn.position.get(item.source, -1):
+ conn.position[item.source] = item.seq
continue
- index += 1
payload = item.event.model_dump(mode="json")
frame_seq = _number_for(conn, item, tickets, index)
if frame_seq is not None:
@@ -407,9 +427,7 @@ async def _event_stream(
# woken for anyway. A count of its own would be a wake
# spent on "you may have missed something you may not care
# about".
- unread = await buffer.unread(
- tenant, agent_id, item.room_id, item.room_seq
- )
+ unread = missed[(item.room_id, item.room_seq)]
payload["missed"] = {
"count": unread.count,
"reason": unread.reason,
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index 72a49db7c..024724e97 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -2400,6 +2400,14 @@ class AgentEventRow(TenantScoped, Base):
the room's conversation uses the pair to leave out the rows of `messages`
it has already received from here.
+ `message_key`, `room_id`, `room_seq` and `type` together name the source of
+ an event,
+ and an agent has at most one event per source (`uq_agent_events_source`).
+ `EventBuffer.enqueue` hands back the number of the event already stored
+ instead of storing a second copy. A file of a multi-file message that
+ arrives after the message was delivered has a later `room_seq`, so it is
+ stored as an event of its own.
+
The retention loop in `main.py` deletes rows older than
`agent_event_retention_seconds`. It never deletes from `messages`.
"""
@@ -2415,7 +2423,19 @@ class AgentEventRow(TenantScoped, Base):
"room_id",
"room_seq",
),
- Index("ix_agent_events_agent_message_key", "agent_id", "message_key"),
+ # One decision of the consumer is stored once. The same room row, read
+ # by the consumer of a second process or written again after a commit
+ # whose acknowledgement was lost, finds this row already there.
+ Index(
+ "uq_agent_events_source",
+ "agent_id",
+ "message_key",
+ "room_id",
+ "room_seq",
+ "type",
+ unique=True,
+ postgresql_where=text("message_key IS NOT NULL"),
+ ),
ForeignKeyConstraint(
["tenant_id", "agent_id"],
["agents.tenant_id", "agents.id"],
diff --git a/core/switch_core/migrations/versions/c4d8e2f6a913_agent_events_source_key.py b/core/switch_core/migrations/versions/c4d8e2f6a913_agent_events_source_key.py
new file mode 100644
index 000000000..0bdf7a4c9
--- /dev/null
+++ b/core/switch_core/migrations/versions/c4d8e2f6a913_agent_events_source_key.py
@@ -0,0 +1,68 @@
+"""agent_events: at most one event per agent and source
+
+An agent's consumer decides whether a room row is for the agent and stores the
+event in `agent_events`. With more than one switch-core process, the consumer
+of each process makes the same decision for the same row, and a write retried
+after a commit whose acknowledgement was lost stores the same event again.
+Either way the agent would receive the event twice, with two numbers, so
+nothing on the reading side could tell the copies apart.
+
+A unique index on the event's source, `(agent_id, message_key, room_id,
+room_seq, type)`, lets the second write find the first. Events with no
+`message_key` are left out. The index starts with `agent_id, message_key`, so
+it also serves the lookups `ix_agent_events_agent_message_key` served, and
+replaces it.
+
+Copies stored before this migration are removed first, keeping the lowest
+number of each, so the index can always be built. The table is locked against
+writes first, so a copy an older release stores while the migration runs
+waits for it and then finds the index, rather than landing after the delete
+and failing the index build.
+
+Revision ID: c4d8e2f6a913
+Revises: b3c7e1d9f2a5
+Create Date: 2026-10-13 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "c4d8e2f6a913"
+down_revision: str | None = "b3c7e1d9f2a5"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+INDEX = "uq_agent_events_source"
+REPLACED = "ix_agent_events_agent_message_key"
+
+
+def upgrade() -> None:
+ op.execute("LOCK TABLE agent_events IN SHARE ROW EXCLUSIVE MODE")
+ op.execute(
+ """
+ DELETE FROM agent_events AS later
+ USING agent_events AS earlier
+ WHERE later.agent_id = earlier.agent_id
+ AND later.message_key = earlier.message_key
+ AND later.room_id = earlier.room_id
+ AND later.room_seq = earlier.room_seq
+ AND later.type = earlier.type
+ AND later.seq > earlier.seq
+ """
+ )
+ op.create_index(
+ INDEX,
+ "agent_events",
+ ["agent_id", "message_key", "room_id", "room_seq", "type"],
+ unique=True,
+ postgresql_where=sa.text("message_key IS NOT NULL"),
+ )
+ op.drop_index(REPLACED, table_name="agent_events")
+
+
+def downgrade() -> None:
+ op.create_index(REPLACED, "agent_events", ["agent_id", "message_key"])
+ op.drop_index(INDEX, table_name="agent_events")
diff --git a/core/tests/switch_core/bridges/agent/api/test_connection_subscribe_rooms.py b/core/tests/switch_core/bridges/agent/api/test_connection_subscribe_rooms.py
index 495d6cd74..66e00de6a 100644
--- a/core/tests/switch_core/bridges/agent/api/test_connection_subscribe_rooms.py
+++ b/core/tests/switch_core/bridges/agent/api/test_connection_subscribe_rooms.py
@@ -256,3 +256,60 @@ async def test_a_subscribe_waits_while_the_agents_room_slots_are_held(
await repoint
assert protocol.connections.claimant_of(AGENT_ID, ROOM_A) is conn
+
+
+async def test_a_slow_subscribe_cannot_take_the_count_back_from_a_later_one(
+ db_buffer: EventBuffer,
+ session_factory: async_sessionmaker[AsyncSession],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Two subscribes to one room, the first slow to record its unread count.
+
+ The second takes the room after the first, so the count must end up with
+ the second. Were the first recorded after the second finished, the room's
+ holder could never clear a room it has read to the end of.
+ """
+ protocol = _Protocol(db_buffer)
+ buffer = protocol.event_buffer
+ first = _open(protocol, "single", "conn-a")
+ second = _open(protocol, "single", "conn-b")
+
+ reached = asyncio.Event()
+ proceed = asyncio.Event()
+ take_counting = buffer.take_counting
+
+ async def slow_for_the_first(
+ tenant_id: str, agent_id: str, reader_id: str, room_id: str
+ ) -> None:
+ if reader_id == "conn-a":
+ reached.set()
+ await proceed.wait()
+ await take_counting(tenant_id, agent_id, reader_id, room_id)
+
+ monkeypatch.setattr(buffer, "take_counting", slow_for_the_first)
+
+ slow = asyncio.create_task(
+ _subscribe(protocol, ROOM_A, first.stream_generation, "conn-a", False)
+ )
+ await reached.wait()
+ later = asyncio.create_task(
+ _subscribe(protocol, ROOM_A, second.stream_generation, "conn-b", True)
+ )
+ await asyncio.sleep(0.05)
+ proceed.set()
+ await slow
+ await later
+ head = await _chatter(session_factory, ROOM_A, 3)
+
+ assert protocol.connections.claimant_of(AGENT_ID, ROOM_A) is second
+ assert (await buffer.unread(TENANT_ZERO_ID, AGENT_ID, ROOM_A, head)).count == 3
+ await buffer.caught_up(
+ TENANT_ZERO_ID,
+ AGENT_ID,
+ Reader(id="conn-b", is_session=False),
+ ROOM_A,
+ head,
+ "conn-b",
+ )
+ assert (await buffer.unread(TENANT_ZERO_ID, AGENT_ID, ROOM_A, head)).count == 0
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_conversation_reading.py b/core/tests/switch_core/bridges/agent/protocol/test_conversation_reading.py
index fc2466644..5f201fe58 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_conversation_reading.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_conversation_reading.py
@@ -18,7 +18,7 @@ from __future__ import annotations
from datetime import UTC, datetime, timedelta
import pytest
-from sqlalchemy import delete
+from sqlalchemy import delete, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.attachments import ATTACHMENT_GROUP_KEY
@@ -73,6 +73,46 @@ async def _keep(buffer: EventBuffer, room_seq: int, message_key: str) -> None:
)
+async def _file(
+ session_factory: SessionFactory,
+ group: str,
+ index: int,
+ name: str,
+ sent_at: datetime,
+ *,
+ caption: str | None = None,
+) -> Message:
+ """One file of a two-file message, as the bridge stores it: a row of its
+ own, numbered within its attachment group."""
+ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:
+ row = await MessageStore().create(
+ session,
+ Message(
+ room_id=ROOM,
+ transport_event_id=f"${group}-{index}",
+ sender_id="@alice:test",
+ sender_name="Alice",
+ event_type="m.room.message",
+ msgtype="m.file",
+ body=caption or name,
+ formatted_body=None,
+ content={
+ "body": caption or name,
+ ATTACHMENT_GROUP_KEY: {"id": group, "index": index, "total": 2},
+ },
+ sent_at=sent_at,
+ ),
+ [MessageAttachment(uri=f"mxc://s/{name}", filename=name)],
+ )
+ await session.commit()
+ return row
+
+
+def _files(item: BufferedEvent) -> set[str]:
+ attachments = getattr(item.event.payload, "attachments", None) or []
+ return {ref.filename for ref in attachments}
+
+
def _bodies(items: list[BufferedEvent]) -> list[str | None]:
return [getattr(item.event.payload, "body", None) for item in items]
@@ -115,6 +155,37 @@ async def test_a_row_not_yet_decided_holds_back_the_rows_after_it(
assert _bodies(read.events) == ["settled"]
+async def test_a_backlog_longer_than_one_read_keeps_its_order(
+ db_buffer: EventBuffer, session_factory: SessionFactory
+) -> None:
+ """A read takes at most `limit` rows of a room. The agent's own event for a
+ message posted after the rows left over waits for them, rather than going
+ out ahead of conversation that came before it."""
+ buffer = db_buffer
+ long_ago = datetime.now(UTC) - timedelta(minutes=1)
+ for index in range(3):
+ await say(session_factory, ROOM, f"chatter {index}", sent_at=long_ago)
+ asked = await say(session_factory, ROOM, "@agent over to you")
+ await _keep(buffer, asked.seq, asked.transport_event_id)
+
+ position: dict[str, int] = {room_source(ROOM): 0}
+ sent: list[list[str | None]] = []
+ for _ in range(2):
+ events, looked = await buffer.read(
+ TENANT_ZERO_ID, AGENT, position, conversation_rooms={ROOM}, limit=2
+ )
+ sent.append(_bodies(events))
+ for item in events:
+ position[item.source] = max(position.get(item.source, -1), item.seq)
+ for source, seq in looked.items():
+ position[source] = max(position.get(source, -1), seq)
+
+ assert sent == [
+ ["chatter 0", "chatter 1"],
+ ["chatter 2", "@agent over to you"],
+ ]
+
+
async def test_a_file_that_arrives_after_its_group_was_delivered_is_still_read(
db_buffer: EventBuffer, session_factory: SessionFactory
) -> None:
@@ -150,33 +221,9 @@ async def test_a_message_posted_between_the_files_of_a_group_goes_out_first(
buffer = db_buffer
long_ago = datetime.now(UTC) - timedelta(minutes=1)
- async def file(index: int, name: str) -> Message:
- async with tenant_session(session_factory, TENANT_ZERO_ID) as session:
- row = await MessageStore().create(
- session,
- Message(
- room_id=ROOM,
- transport_event_id=f"$part{index}",
- sender_id="@alice:test",
- sender_name="Alice",
- event_type="m.room.message",
- msgtype="m.file",
- body=name,
- formatted_body=None,
- content={
- "body": name,
- ATTACHMENT_GROUP_KEY: {"id": "g1", "index": index, "total": 2},
- },
- sent_at=long_ago,
- ),
- [MessageAttachment(uri=f"mxc://s/{name}", filename=name)],
- )
- await session.commit()
- return row
-
- await file(0, "one.pdf")
+ await _file(session_factory, "g1", 0, "one.pdf", long_ago)
between = await say(session_factory, ROOM, "in between", sent_at=long_ago)
- last = await file(1, "two.pdf")
+ last = await _file(session_factory, "g1", 1, "two.pdf", long_ago)
read = await buffer.read_conversation(TENANT_ZERO_ID, AGENT, {ROOM: 0}, limit=100)
@@ -184,6 +231,52 @@ async def test_a_message_posted_between_the_files_of_a_group_goes_out_first(
assert _bodies(read.events)[0] == "in between"
+async def test_a_group_held_back_by_another_keeps_its_first_file(
+ db_buffer: EventBuffer, session_factory: SessionFactory
+) -> None:
+ """Two people post two files each and the files interleave: A's first, B's
+ first, A's second. B is still waiting for its second file, so it holds the
+ read at its first file. A goes out at its last file, which is past the
+ hold, so the read must not move past A's first file either, or the next
+ read would send A with only its second file."""
+ buffer = db_buffer
+ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:
+ now = await session.scalar(select(func.now()))
+ assert now is not None
+ # Past the settle time (0.2 s for `db_buffer`) and within the group grace
+ # time (5 s), so B, missing its second file, is held.
+ recent = now - timedelta(seconds=3)
+ a_first = await _file(
+ session_factory, "A", 0, "a-one.pdf", recent, caption="see these"
+ )
+ await _file(session_factory, "B", 0, "b-one.pdf", recent)
+ await _file(session_factory, "A", 1, "a-two.pdf", recent)
+
+ first = await buffer.read_conversation(TENANT_ZERO_ID, AGENT, {ROOM: 0}, limit=100)
+
+ assert first.events == []
+ place = first.through.get(ROOM, 0)
+ assert place < a_first.seq
+
+ # B's grace time runs out without its second file.
+ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:
+ await session.execute(
+ update(Message)
+ .where(Message.room_id == ROOM)
+ .values(sent_at=Message.sent_at - timedelta(minutes=1))
+ )
+ await session.commit()
+
+ second = await buffer.read_conversation(
+ TENANT_ZERO_ID, AGENT, {ROOM: place}, limit=100
+ )
+
+ assert [_files(item) for item in second.events] == [
+ {"b-one.pdf"},
+ {"a-one.pdf", "a-two.pdf"},
+ ]
+
+
async def test_a_place_past_the_end_of_a_room_is_brought_back_to_it(
db_buffer: EventBuffer, session_factory: SessionFactory
) -> None:
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py b/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py
index 5a6771f31..63c147d73 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py
@@ -12,8 +12,10 @@ the room's log, and the per-room and all-rooms polls read it from there.
from __future__ import annotations
+import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
+from typing import Any
import pytest
from sqlalchemy import delete, select
@@ -430,22 +432,61 @@ async def test_overlapping_polls_never_move_a_reader_back(
db_buffer: EventBuffer, session_factory: async_sessionmaker[AsyncSession]
) -> None:
"""Two polls of the same reader can overlap, and the one that finishes
- last may have read less. Its save is merged with the stored position,
- source by source, so the reader is never moved back to events it was
- already sent."""
+ last may have read less. Its save is refused, because the stored position
+ is no longer the one it read from, so the reader is never moved back to
+ events it was already sent."""
room = f"r:{ROOM}"
- await db_buffer._save_poll_position(
- TENANT_ZERO_ID, AGENT, "legacy:all", {OWN_EVENTS: 5, room: 7}
+ first = {OWN_EVENTS: 5, room: 7}
+ assert await db_buffer._save_poll_position(
+ TENANT_ZERO_ID, AGENT, "legacy:all", first, expected=None
)
- await db_buffer._save_poll_position(
- TENANT_ZERO_ID, AGENT, "legacy:all", {OWN_EVENTS: 3, room: 9, "r:other": 1}
+ assert not await db_buffer._save_poll_position(
+ TENANT_ZERO_ID, AGENT, "legacy:all", {OWN_EVENTS: 3, room: 9}, expected=None
+ )
+ assert not await db_buffer._save_poll_position(
+ TENANT_ZERO_ID,
+ AGENT,
+ "legacy:all",
+ {OWN_EVENTS: 3, room: 9},
+ expected={OWN_EVENTS: 2, room: 7},
+ )
+ assert await _poll_position(session_factory, "legacy:all") == first
+
+ after = {OWN_EVENTS: 6, room: 8}
+ assert await db_buffer._save_poll_position(
+ TENANT_ZERO_ID, AGENT, "legacy:all", after, expected=first
+ )
+ assert await _poll_position(session_factory, "legacy:all") == after
+
+
+async def test_overlapping_polls_return_each_event_once(
+ db_buffer: EventBuffer,
+ session_factory: async_sessionmaker[AsyncSession],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Two polls of the same reader that both read before either saves would
+ both return what they read. The second to save reads again from where the
+ first stopped instead."""
+ await _asked(db_buffer, session_factory)
+ both_read = asyncio.Barrier(2)
+ read_from = db_buffer.read_from
+ calls = [0]
+
+ async def read_together(*args: Any, **kwargs: Any) -> Any:
+ calls[0] += 1
+ found = await read_from(*args, **kwargs)
+ if calls[0] <= 2:
+ await both_read.wait()
+ return found
+
+ monkeypatch.setattr(db_buffer, "read_from", read_together)
+
+ polled = await asyncio.gather(
+ db_buffer.poll_notifications(TENANT_ZERO_ID, AGENT, timeout=0, rooms={ROOM}),
+ db_buffer.poll_notifications(TENANT_ZERO_ID, AGENT, timeout=0, rooms={ROOM}),
)
- assert await _poll_position(session_factory, "legacy:all") == {
- OWN_EVENTS: 5,
- room: 9,
- "r:other": 1,
- }
+ assert sorted(len(events) for events in polled) == [0, 1]
async def test_a_write_that_fails_on_the_connection_is_retried(
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_own_event_sources.py b/core/tests/switch_core/bridges/agent/protocol/test_own_event_sources.py
new file mode 100644
index 000000000..d0c6d3cec
--- /dev/null
+++ b/core/tests/switch_core/bridges/agent/protocol/test_own_event_sources.py
@@ -0,0 +1,127 @@
+"""An agent has at most one own event per source.
+
+The consumer of every switch-core process decides, for each room row, whether
+it is for the agent, and stores the event in `agent_events`. Two processes make
+that decision for the same row, and a write retried after a commit whose
+acknowledgement was lost repeats one that was made. The agent must still
+receive the event once, under one number.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
+from switch_core.db.models import TENANT_ZERO_ID
+from tests.db_buffer import db_buffer_on, seed_agent
+
+AGENT = "agent-1"
+ROOM = "room-1"
+
+SessionFactory = async_sessionmaker[AsyncSession]
+
+
[email protected](autouse=True)
+async def _agent_row(session_factory: SessionFactory) -> None:
+ await seed_agent(session_factory, AGENT)
+
+
+def _message(body: str = "for you") -> AgentEvent:
+ return AgentEvent(
+ type="message",
+ room_id=ROOM,
+ payload=MessagePayload(
+ addressed=True,
+ sender="@alice:test",
+ sender_name="Alice",
+ message_id="$m",
+ body=body,
+ timestamp=0,
+ ),
+ )
+
+
+async def _keep(
+ buffer: EventBuffer, *, room_seq: int = 7, message_key: str | None = "$m"
+) -> int:
+ return await buffer.enqueue(
+ TENANT_ZERO_ID,
+ AGENT,
+ ROOM,
+ _message(),
+ room_seq=room_seq,
+ message_key=message_key,
+ )
+
+
+async def _numbers(buffer: EventBuffer) -> list[int]:
+ return [item.seq for item in await buffer.read_from(TENANT_ZERO_ID, AGENT, 0)]
+
+
+async def test_two_processes_storing_the_same_source_store_one_event(
+ session_factory: SessionFactory,
+) -> None:
+ """Each process's consumer makes the same decision for the same row; the
+ second write is handed the first one's number."""
+ first = db_buffer_on(session_factory)
+ second = db_buffer_on(session_factory)
+
+ stored = await _keep(first)
+ again = await _keep(second)
+
+ assert again == stored
+ assert await _numbers(first) == [stored]
+
+
+async def test_a_write_repeated_after_its_commit_stores_nothing_more(
+ db_buffer: EventBuffer,
+) -> None:
+ """A write retried because its commit's acknowledgement was lost, though
+ the commit was made, finds the event already there."""
+ stored = await _keep(db_buffer)
+
+ assert await _keep(db_buffer) == stored
+ assert await _numbers(db_buffer) == [stored]
+
+
+async def test_writes_racing_for_one_source_store_one_event_and_waste_no_number(
+ session_factory: SessionFactory,
+) -> None:
+ """Two processes write the same source at once. One stores it; the other
+ finds it at insert time, gives its number back, and is handed the stored
+ one, so the next event is numbered right after it."""
+ first = db_buffer_on(session_factory)
+ second = db_buffer_on(session_factory)
+
+ results = await asyncio.gather(_keep(first), _keep(second))
+ following = await _keep(first, room_seq=8, message_key="$next")
+
+ assert results[0] == results[1]
+ assert await _numbers(first) == [results[0], following]
+ assert following == results[0] + 1
+
+
+async def test_a_file_arriving_after_its_message_was_delivered_is_its_own_event(
+ db_buffer: EventBuffer,
+) -> None:
+ """Several files posted as one message share a key. When the message was
+ delivered before all of them arrived, a later file comes from a later room
+ row and is stored as an event of its own."""
+ delivered = await _keep(db_buffer, room_seq=7, message_key="group-1")
+ late = await _keep(db_buffer, room_seq=9, message_key="group-1")
+
+ assert late != delivered
+ assert await _numbers(db_buffer) == [delivered, late]
+
+
+async def test_events_without_a_source_key_are_never_merged(
+ db_buffer: EventBuffer,
+) -> None:
+ first = await _keep(db_buffer, message_key=None)
+ second = await _keep(db_buffer, message_key=None)
+
+ assert await _numbers(db_buffer) == [first, second]
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_stream.py b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
index 8da24cbba..224719aaf 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_stream.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
@@ -592,6 +592,39 @@ async def test_events_for_rooms_the_connection_does_not_cover_are_skipped(
assert frames[1][1]["payload"]["body"] == "mine"
+async def test_a_room_claimed_while_a_batch_goes_out_is_not_sent_by_its_old_holder(
+ db_buffer: EventBuffer, session_factory: async_sessionmaker[AsyncSession]
+) -> None:
+ """The stream suspends after each event it sends. A session of the same
+ agent that claims the room in that moment takes the rest of the batch: the
+ connection that covered the room before must not send it too."""
+ await seed_member(session_factory, AGENT, ROOM_A)
+ registry = AgentConnectionRegistry()
+ watcher = _open(
+ registry, connection_id="watcher", scope="all", delivery_filter="addressed"
+ )
+ await _keep(db_buffer, _message("first"))
+ await _keep(db_buffer, _message("second"))
+
+ stream = event_frames(
+ conn=watcher, registry=registry, buffer=db_buffer, approvals=None
+ )
+ assert (await anext(stream)).event == "connection_state"
+ assert (await anext(stream)).data["payload"]["body"] == "first"
+
+ session = _open(
+ registry, connection_id="session", scope="single", delivery_filter="addressed"
+ )
+ registry.claim_room(session, ROOM_A)
+
+ try:
+ after: Frame | None = await asyncio.wait_for(anext(stream), timeout=1.0)
+ except TimeoutError:
+ after = None
+ assert after is None or after is KEEPALIVE
+ assert watcher.position[OWN_EVENTS] == await db_buffer.head(TENANT_ZERO_ID, AGENT)
+
+
async def test_addressed_filter_drops_ambient_chatter(
db_buffer: EventBuffer, session_factory: async_sessionmaker[AsyncSession]
) -> None:
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_unread_counts.py b/core/tests/switch_core/bridges/agent/protocol/test_unread_counts.py
index 8c2a942d3..fe138c276 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_unread_counts.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_unread_counts.py
@@ -37,11 +37,13 @@ from tests.db_buffer import (
seed_room,
sender_of,
)
+from tests.switch_core.statement_counts import StatementCounts
AGENT = "agent-1"
READER = "connection-1"
ROOM_A = "room-a"
ROOM_B = "room-b"
+ROOM_C = "room-c"
SessionFactory = async_sessionmaker[AsyncSession]
@@ -569,3 +571,124 @@ async def test_a_room_not_caught_up_on_for_longer_than_the_window_counts_over_it
await _chatter(session_factory, ROOM_A, times=2)
assert await _unread(buffer, ROOM_A) == Unread(count=2, reason=COUNTED_FROM_WINDOW)
+
+
+async def _history_for_a_catch_up(
+ buffer: EventBuffer, session_factory: SessionFactory
+) -> dict[str, list[int]]:
+ """Rooms with every kind of row a count has to tell apart, as a stream
+ catching up meets them. The positions in each room a count is asked at:
+ every row in it, so that counts are asked in the middle of groups of files
+ and right after the agent's own messages."""
+ agent_sender = await sender_of(session_factory, AGENT)
+ # Room A: counted from the start, with chatter, messages addressed to the
+ # agent, what the agent said itself, files posted as one message, and
+ # files addressed to the agent as one message.
+ await buffer.ensure_counting(TENANT_ZERO_ID, AGENT, READER, ROOM_A)
+ await _chatter(session_factory, ROOM_A, times=2)
+ await _addressed(buffer, session_factory, ROOM_A)
+ await say(session_factory, ROOM_A, "on it", sender=agent_sender)
+ for index in range(3):
+ await say(session_factory, ROOM_A, "chart", content=_part("posted", index, 3))
+ await _chatter(session_factory, ROOM_A)
+ for index in range(2):
+ last = await say(
+ session_factory, ROOM_A, "brief", content=_part("asked", index, 2)
+ )
+ await buffer.enqueue(
+ TENANT_ZERO_ID,
+ AGENT,
+ ROOM_A,
+ _asked(ROOM_A, "asked"),
+ room_seq=last.seq,
+ message_key="asked",
+ )
+ await _chatter(session_factory, ROOM_A, times=2)
+ await _addressed(buffer, session_factory, ROOM_A)
+ await _chatter(session_factory, ROOM_A)
+
+ # Room B: last caught up before the retention window, so its counts start
+ # at the window and say so.
+ await buffer.ensure_counting(TENANT_ZERO_ID, AGENT, READER, ROOM_B)
+ long_ago = datetime.now(UTC) - timedelta(hours=2)
+ await say(session_factory, ROOM_B, "@agent over to you", sent_at=long_ago)
+ await say(session_factory, ROOM_B, "old chatter", sent_at=long_ago)
+ await _chatter(session_factory, ROOM_B, times=2)
+ await _addressed(buffer, session_factory, ROOM_B)
+ await say(session_factory, ROOM_B, "noted", sender=agent_sender)
+ await _chatter(session_factory, ROOM_B)
+
+ # Room C: nothing recorded a baseline, so it has no count at all.
+ await seed_room(session_factory, ROOM_C)
+ await _chatter(session_factory, ROOM_C, times=2)
+
+ return {
+ room: list(range(1, await _head(buffer, room) + 1))
+ for room in (ROOM_A, ROOM_B, ROOM_C)
+ }
+
+
+async def test_a_batch_counts_what_counting_each_event_alone_counts(
+ db_buffer: EventBuffer, session_factory: SessionFactory
+) -> None:
+ """A catch-up counts every event it is woken for in one batch, and must
+ tell each event exactly what counting it on its own would have.
+
+ The batch is counted by a second buffer on the same database, as another
+ process would after the agent's stream moved there."""
+ buffer = db_buffer
+ rooms = await _history_for_a_catch_up(buffer, session_factory)
+ # Part of room A caught up on, so some positions asked fall below the
+ # baseline.
+ await buffer.caught_up(TENANT_ZERO_ID, AGENT, _connection(READER), ROOM_A, 3, None)
+ # Asked out of order and with repeats, as events from several rooms
+ # interleave in a stream.
+ positions = [
+ (room, seq) for seq in range(30, 0, -1) for room in rooms if seq in rooms[room]
+ ]
+ positions += positions[:5]
+
+ elsewhere = db_buffer_on(session_factory)
+ batch = await elsewhere.unread_batch(TENANT_ZERO_ID, AGENT, positions)
+
+ one_by_one = {
+ (room, seq): await buffer.unread(TENANT_ZERO_ID, AGENT, room, seq)
+ for room, seq in positions
+ }
+ assert batch == one_by_one
+ # The mix covered every kind of answer, so the comparison means something.
+ assert {unread.reason for unread in batch.values()} == {
+ None,
+ COUNTED_FROM_WINDOW,
+ NO_BASELINE,
+ }
+ assert batch[(ROOM_A, rooms[ROOM_A][-1])] == Unread(count=7, reason=None)
+ assert batch[(ROOM_B, rooms[ROOM_B][-1])] == Unread(
+ count=3, reason=COUNTED_FROM_WINDOW
+ )
+
+
+async def test_a_batch_costs_the_same_statements_however_many_events_it_counts(
+ db_buffer: EventBuffer,
+ session_factory: SessionFactory,
+ statement_counts: StatementCounts,
+) -> None:
+ """Counting each event on its own cost four statements an event, so a
+ catch-up of two hundred events ran about eight hundred. A batch costs a
+ fixed number per room, whatever the number of events in it."""
+ buffer = db_buffer
+ rooms = await _history_for_a_catch_up(buffer, session_factory)
+
+ with statement_counts.scope("one event a room") as few:
+ await buffer.unread_batch(
+ TENANT_ZERO_ID, AGENT, [(room, seqs[-1]) for room, seqs in rooms.items()]
+ )
+ with statement_counts.scope("every event") as many:
+ await buffer.unread_batch(
+ TENANT_ZERO_ID,
+ AGENT,
+ [(room, seq) for room, seqs in rooms.items() for seq in seqs],
+ )
+
+ assert sum(len(seqs) for seqs in rooms.values()) > 3 * len(rooms)
+ assert many.count == few.count
diff --git a/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py b/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py
index d902b58af..681b90527 100644
--- a/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py
+++ b/core/tests/switch_core/clients/test_agent_consumer_attachment_groups.py
@@ -591,6 +591,8 @@ async def test_a_post_split_by_the_timeout_is_reported_once(
body=f"f{index}.png",
event_id=f"$part-{index}",
group={"id": "grp-late", "index": index, "total": 3},
+ # Each part is a row of its own in the room's log.
+ seq=index + 1,
),
)
await asyncio.sleep(0.15)
diff --git a/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py b/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py
index 67e9153fc..1de2c63b0 100644
--- a/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py
+++ b/core/tests/switch_core/clients/test_agent_consumer_reports_addressed.py
@@ -119,10 +119,10 @@ async def _ignore_post(*_args: object, **_kwargs: object) -> None:
return None
-def _message(**content: object) -> InboundMessage:
+def _message(event_id: str = "$evt", **content: object) -> InboundMessage:
return InboundMessage(
room_id="!room:s",
- event_id="$evt",
+ event_id=event_id,
sender="@alice:s",
timestamp=0,
content={"sender_name": "alice", **content},
@@ -218,8 +218,8 @@ async def test_a_telemetry_failure_does_not_cost_the_agent_the_message(
try:
with caplog.at_level("ERROR"):
- await _deliver(consumer, _message())
- await _deliver(consumer, _message())
+ await _deliver(consumer, _message(event_id="$first"))
+ await _deliver(consumer, _message(event_id="$second"))
failures = next(
p
for p in registry.collect()