Skip to content

The agent that grows with you

fix(gateway): resume's prepare window answers a delete and counts new rows

Tekniumc72966db57c0
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index fa15ef4a0f9..7216f21acd9 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -262,6 +262,8 @@ class SessionAuthority:
self.authorize(actor, ref, 'session:read')
from gateway.session_local_recovery import local_history, prepare_history
prepared = await prepare_history(self, ref) # the lineage read stays off the loop and the lock
+ # A delete or eviction can land during that await: re-check, so it answers not_found, never a KeyError.
+ self.authorize(actor, ref, 'session:read')
live = self.sessions[ref.session_id]
from gateway.session_queued_controls import pending_rows
with live.event_stream.lock:
diff --git a/gateway/session_hosted_rpc.py b/gateway/session_hosted_rpc.py
index 87e7ec61081..b0942487ff4 100644
--- a/gateway/session_hosted_rpc.py
+++ b/gateway/session_hosted_rpc.py
@@ -198,6 +198,9 @@ class HostedRoomAuthorityRPC:
async def _history(self, params):
from gateway.session_local_recovery import local_history, prepare_history
+ # Authorize (restoring a cold room session) before the read names its live entry, and again after
+ # the await (``_rows``): a delete in between answers not_found, never a KeyError.
+ self.authority.authorize(self.principal, self.ref, 'session:read')
prepared = await prepare_history(self.authority, self.ref)
rows = self._rows()
history = list(local_history(self.authority, self.ref, prepared))
diff --git a/gateway/session_local_recovery.py b/gateway/session_local_recovery.py
index 8b5cec6ba6a..0661918b215 100644
--- a/gateway/session_local_recovery.py
+++ b/gateway/session_local_recovery.py
@@ -227,16 +227,24 @@ def display_rows(db, target):
db.assert_resume_safe(target)
except SessionResumeTooLargeError:
return live, [], False
+ kept, truncated = _fit_archive(live_rows, db.get_messages_as_conversation(
+ target, include_ancestors=True, include_compacted=True, include_row_ids=True))
+ return live, kept, truncated
+
+
+def _fit_archive(live_rows, display):
+ """``(display rows kept, truncated)``: every live row of *display*, and its newest archived rows that fit
+ what *live_rows* leave of ``RESUME_HISTORY_BYTES``."""
+ live = {row['_row_id'] for row in live_rows}
kept, used, truncated = [], sum(len(json.dumps(row, default=str)) for row in live_rows), False
- for row in reversed(db.get_messages_as_conversation(target, include_ancestors=True, include_compacted=True,
- include_row_ids=True)):
+ for row in reversed(display):
if row['_row_id'] not in live:
used += len(json.dumps(row, default=str))
truncated = truncated or used > RESUME_HISTORY_BYTES
if truncated:
continue
kept.append(row)
- return live, kept[::-1], truncated
+ return kept[::-1], truncated
async def prepare_history(authority, ref):
@@ -292,9 +300,14 @@ def local_history(authority, ref, prepared=None):
read_live, display, truncated = (prepared[1] if prepared is not None and prepared[0] == target
else display_rows(db, target))
live = db.get_messages_as_conversation(target, include_row_ids=True)
- if not read_live <= {row['_row_id'] for row in live}:
+ live_ids = {row['_row_id'] for row in live}
+ if not read_live <= live_ids:
read_live, display, truncated = display_rows(db, target)
live = db.get_messages_as_conversation(target, include_row_ids=True)
+ elif live_ids - read_live:
+ # Rows appended since the prepare count against the archive budget too (no lineage re-read).
+ display, cut = _fit_archive(live, display)
+ truncated = truncated or cut
rows = [{**{k: v for k, v in row.items() if k != '_row_id'}, 'row_id': row['_row_id']}
for row in _merge_live_in_stored_order(display, live)]
if truncated and rows:
diff --git a/tests/gateway/test_resume_display_history.py b/tests/gateway/test_resume_display_history.py
index 9f022472a29..a813cf1d59c 100644
--- a/tests/gateway/test_resume_display_history.py
+++ b/tests/gateway/test_resume_display_history.py
@@ -168,3 +168,60 @@ async def test_a_long_archived_history_keeps_the_resume_reply_bounded(tmp_path,
assert [r['row_id'] for r in rows if not r.get('archived')] == [m['id'] for m in db.get_messages(tip)]
archived = [r['content'][:4] for r in rows if r.get('archived') and r['role'] == 'user']
assert archived and archived[-1] == 'Q39 ' and 'Q0 x' not in archived # the newest are kept
+
+
+async def test_a_row_appended_during_prepare_counts_against_the_archive_budget(tmp_path, monkeypatch):
+ """A live row landing between the off-loop prepare and the locked read shrinks what the archive may
+ use, as a fresh read would: the raced reply is no larger than the fresh one (review of #140971)."""
+ import json
+ from gateway import session_local_recovery as recovery
+ authority, actor, ref, physical = await _local_session(tmp_path, monkeypatch)
+ db = authority.db
+ for n in range(28): # ~2.8 MiB of history, then summarized away in place (all archived)
+ db.append_message(physical, 'user' if n % 2 == 0 else 'assistant', f'{n}:' + 'h' * 100 * 1024)
+ db.archive_and_compact(physical, [{'role': 'user', 'content': SUMMARY}])
+ real, raced = recovery.prepare_history, []
+
+ async def prepare_then_append(authority_, ref_):
+ prepared = await real(authority_, ref_)
+ if not raced:
+ raced.append(db.append_message(physical, 'assistant', 'n' * 2 * 1024 * 1024))
+ return prepared
+ monkeypatch.setattr(recovery, 'prepare_history', prepare_then_append)
+ reply = await _resume(authority, actor, ref, display_projection=True)
+ fresh = await _resume(authority, actor, ref, display_projection=True)
+ assert raced and reply['messages'][-1]['content'].startswith('nnn')
+ assert reply['messages'][0].get('archived_truncated')
+ assert len(json.dumps(reply['messages'])) <= len(json.dumps(fresh['messages']))
+ assert len(json.dumps(reply)) < 8 * 1024 * 1024
+
+
+async def test_a_delete_during_the_resume_prepare_answers_not_found(tmp_path, monkeypatch):
+ """``attach`` awaits the lineage read between its authorize and the live-entry lookup: a delete committed
+ by another connection in that window answers ``not_found``, never an internal error (review of #140971)."""
+ from gateway import session_local_recovery as recovery
+ from gateway.session_controls import AuthorityConnection
+ authority, _actor, ref, physical = await _local_session(tmp_path, monkeypatch)
+ authority.runner.session_authority = authority # one process owns both writes of a delete
+ authority.db.append_message(physical, 'user', 'hello')
+
+ def connection():
+ return AuthorityConnection(authority, object(), {
+ 'provider': 'local', 'user_id': 'owner', 'profile_id': 'fixture', 'instance_id': authority.instance_id,
+ 'capabilities': ['session:create', 'session:read', 'session:control']})
+ real, deleted = recovery.prepare_history, []
+
+ async def prepare_then_delete(authority_, ref_):
+ prepared = await real(authority_, ref_)
+ if not deleted:
+ row = authority.db.get_session(ref.session_id)
+ deleted.append(await connection().dispatch({'id': 9, 'method': 'session.mutate', 'params': {
+ 'session_id': ref.session_id, 'request_id': 'delete-1', 'expected_revision': row['runtime_revision'],
+ 'expected_generation': row['runtime_generation'], 'operation': 'delete', 'payload': {}}}))
+ return prepared
+ monkeypatch.setattr(recovery, 'prepare_history', prepare_then_delete)
+ reply = await connection().dispatch({'id': 1, 'method': 'session.resume', 'params': {'session_id': ref.session_id}})
+ assert 'result' in deleted[0], deleted # the delete really committed inside the window
+ assert reply.get('error', {}).get('message') == 'not_found', reply