Skip to content

The agent that grows with you

fix(gateway): send a resumed viewer what its session published after its replay

Tekniumf57628bcb20c
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index 862d345b77c..fdea81c9fc4 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -351,10 +351,11 @@ class SessionAuthority:
raise RuntimeStoreError('not_found') from None
raise
- async def attach(self, actor, ref, *, history=True):
+ async def attach(self, actor, ref, *, history=True, replay_from=None):
"""Join *ref*'s event stream and snapshot it. ``history=False``: the snapshot carries no transcript
(``history == ()``), only the physical ``transcript_target``, so the display-row lineage read is
- skipped; for a viewer that pages the transcript itself (Desktop over REST)."""
+ skipped; for a viewer that pages the transcript itself (Desktop over REST).
+ *replay_from*: ``(epoch, sequence)`` this connection's reconnect replay answered (SessionEvents.catch_up)."""
self.authorize(actor, ref, 'session:read')
from gateway.session_local_recovery import local_history, prepare_history, transcript_target
# The lineage read stays off the loop and the lock; without history the target is resolved under it.
@@ -379,6 +380,8 @@ class SessionAuthority:
allow_for_session((self.profile_id, ref.session_id)) # a closed session's work may start again
transport = self.events.get(actor.transport_id)
if transport is not None:
+ if replay_from is not None and not live.event_stream.fanout.contains(transport):
+ live.event_stream.catch_up(transport, *replay_from)
live.event_stream.fanout.attach(transport)
live.event_stream.on_overflow = partial(self._retire_overflowed, ref.session_id)
active_generation = handle.execution_generation if handle.execution_state == "running" else None
diff --git a/gateway/session_contract.py b/gateway/session_contract.py
index 71107a5f82a..3ccab60afb1 100644
--- a/gateway/session_contract.py
+++ b/gateway/session_contract.py
@@ -87,7 +87,8 @@ class SubscriptionSnapshot:
class SessionAuthority(Protocol):
async def create(self, actor: Principal, request: SessionCreate) -> SessionHandle: ...
async def resolve(self, actor: Principal, ref: SessionRef) -> SessionHandle: ...
- async def attach(self, actor: Principal, ref: SessionRef, *, history: bool = True) -> SubscriptionSnapshot: ...
+ async def attach(self, actor: Principal, ref: SessionRef, *, history: bool = True,
+ replay_from: tuple[str, int] | None = None) -> SubscriptionSnapshot: ...
async def detach(self, actor: Principal, subscription_id: str, *, end_reason: str | None = None) -> None: ...
async def submit(self, actor: Principal, request: Submission) -> AdmissionReceipt: ...
async def receipt(self, actor: Principal, ref: SessionRef, admission_id: str) -> AdmissionReceipt: ...
diff --git a/gateway/session_controls.py b/gateway/session_controls.py
index 733216380ef..fdea539c2fe 100644
--- a/gateway/session_controls.py
+++ b/gateway/session_controls.py
@@ -158,6 +158,8 @@ class AuthorityConnection:
bind_native_transport(authority, self.actor, identity)
self.native_owner = identity.get('native_bootstrap') is True
self.subscriptions = {}
+ # session_id -> (epoch, sequence) the last reconnect replay answered; spent by the next resume.
+ self.replay_cursors = {}
self.handed_back = set() # sessions this live client detached without closing (the TUI's switcher)
self._identity, self._operator, self._siblings = identity, operator, {}
authority.events[self.actor.transport_id] = transport
@@ -605,6 +607,8 @@ class AuthorityConnection:
async def resume(self, ref, params):
from gateway.session_local_inventory import snapshot_info
+ replay_from = self.replay_cursors.pop(ref.session_id, None)
+ requested = ref.session_id
if 'latest' in params:
# Bare `-c` / `--resume latest`: the most recent session of that surface family the
# actor may read (workspace first), resolved by the owner like a title.
@@ -663,7 +667,8 @@ class AuthorityConnection:
# ``omit_messages``: the client pages the transcript itself (Desktop over REST), so the attach reads no
# compression lineage (a whole-lineage read here made every open of a long chat cost seconds).
omit = params.get('omit_messages') is True
- snapshot = await self.authority.attach(self.actor, ref, history=not omit)
+ snapshot = await self.authority.attach(
+ self.actor, ref, history=not omit, replay_from=replay_from if ref.session_id == requested else None)
self.subscriptions[ref.session_id] = snapshot.subscription_id
self.handed_back.discard(ref.session_id)
self._spawn_notice(ref)
@@ -737,7 +742,11 @@ class AuthorityConnection:
raise RuntimeStoreError('invalid_params')
from gateway.session_server_requests import open_requests
stream = self.authority.sessions[ref.session_id].event_stream
- return {**stream.since(epoch, sequence), 'open_requests': open_requests(stream, ref.session_id)}
+ replay = stream.since(epoch, sequence)
+ if not replay['snapshot_required']:
+ self.replay_cursors[ref.session_id] = (
+ replay['replay_epoch'], max((frame['seq'] for frame in replay['events']), default=sequence))
+ return {**replay, 'open_requests': open_requests(stream, ref.session_id)}
async def submit(self, ref, params):
if self._voice_stop_replay(ref, params) or await self._typed_voice_stop(ref, params):
diff --git a/gateway/session_events.py b/gateway/session_events.py
index f669acfb44e..f887260018e 100644
--- a/gateway/session_events.py
+++ b/gateway/session_events.py
@@ -74,6 +74,18 @@ class SessionEvents:
# delivery (a detached finite viewer reads nothing until it reattaches).
return delivered
+ def catch_up(self, transport, epoch, sequence):
+ """Caller holds ``lock``, before *transport* joins ``fanout``: write it the frames after the
+ cursor its connection's replay answered. A reconnect asks ``session.events.since`` and then
+ ``session.resume``; whatever the session published in between went to no live peer of this
+ socket, and the replay was already over, so a reply's tail and ``message.complete`` were lost
+ and the view stayed on its first chunk. Under the lock nothing can publish ahead of them."""
+ replay = self.since(epoch, sequence)
+ if replay['snapshot_required']:
+ return
+ for frame in replay['events']:
+ transport.write({'jsonrpc': '2.0', 'method': 'event', 'params': frame})
+
def since(self, epoch, sequence):
with self.lock:
current_epoch, latest = self.watermark()
diff --git a/tests/gateway/test_authority_reconnect_gap.py b/tests/gateway/test_authority_reconnect_gap.py
new file mode 100644
index 00000000000..5ce58e13a06
--- /dev/null
+++ b/tests/gateway/test_authority_reconnect_gap.py
@@ -0,0 +1,70 @@
+"""A reconnecting viewer receives what its session published between the replay answer and its resume."""
+import asyncio
+import queue
+from types import SimpleNamespace
+
+import pytest
+
+
+def _drain(peer):
+ frames = []
+ while not peer.frames.empty():
+ frames.append(peer.frames.get_nowait()['params'])
+ return frames
+
+
+async def test_resume_after_replay_delivers_the_frames_published_in_between(tmp_path, monkeypatch):
+ from gateway.config import GatewayConfig, Platform
+ from gateway.session import SessionSource, SessionStore
+ from gateway.session_authority import LiveSession, initialize_session_authority
+ from gateway.session_contract import SessionRef, Submission
+ from gateway.session_controls import AuthorityConnection
+
+ monkeypatch.setenv('HERMES_HOME', str(tmp_path))
+ store = SessionStore(tmp_path / 'sessions', GatewayConfig())
+ db = store._db
+
+ async def answer(event):
+ return f'reply to {event.text}'
+ runner = SimpleNamespace(_session_db=db, session_store=store, _draining=False,
+ _handle_message=answer, _adapter_for_source=lambda source: None)
+ authority = await initialize_session_authority(runner, profile_id='gap', instance_id='owner')
+ ref = SessionRef('gap', 'chat')
+ db.create_session('chat', source='telegram')
+ authority.sessions['chat'] = LiveSession(SessionSource(platform=Platform.TELEGRAM, chat_id='gap'), 'chat')
+
+ class Peer:
+ def __init__(self):
+ self.frames = queue.Queue()
+
+ def write(self, frame):
+ self.frames.put(frame)
+ return True
+
+ dropped, reconnected, fresh = Peer(), Peer(), Peer()
+ old = AuthorityConnection(authority, dropped, {'user_id': 'viewer'})
+ new = AuthorityConnection(authority, reconnected, {'user_id': 'viewer'})
+
+ async def turn(text):
+ await authority.submit(old.actor, Submission(text, ref, {'text': text}, 'queue'))
+ await authority.sessions['chat'].task
+
+ attached = await old.resume(ref, {})
+ await turn('one')
+ seen = _drain(dropped)[-1]['seq']
+ # The reconnect: the replay is answered on the new socket before its resume rebinds it.
+ replay = (await new.dispatch({'id': 1, 'method': 'session.events.since', 'params': {
+ 'session_id': 'chat', 'replay_epoch': attached['replay_epoch'], 'last_sequence': seen}}))['result']
+ assert replay['events'] == []
+ await turn('two') # commits in the gap: only the dead socket is subscribed
+ await new.resume(ref, {})
+ caught_up = _drain(reconnected)
+ assert caught_up and caught_up[0]['seq'] == seen + 1
+ assert [f['seq'] for f in caught_up] == list(range(seen + 1, seen + 1 + len(caught_up)))
+ complete = [f for f in caught_up if f['type'] == 'message.complete']
+ assert [f['payload']['text'] for f in complete] == ['reply to two']
+
+ # A viewer that never replayed opens from the snapshot alone: no backlog is pushed to it.
+ await AuthorityConnection(authority, fresh, {'user_id': 'viewer'}).resume(ref, {})
+ assert _drain(fresh) == []