Skip to content

The agent that grows with you

fix(gateway): settle a refused head after a storage retry; sync the /new race test

unsupportedpastels04fae30eebb3
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index 256582304c1..6edca211676 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -1115,17 +1115,25 @@ class SessionAuthority:
"""Settle the queued head a preclaim check refused as cancelled: its waiters get *reason* (the
ingress notice, the API's refusal) and its successors revalidate on their own merits."""
from gateway.session_runtime_workers import tracked_write
+ before = get_session_admission(self.db, admission_id=admission_id)
+ if before is not None and before['status'] == 'queued':
+ # Owed from before the write, as in _commit_cancel: storage failing after the commit leaves
+ # it for the drain's retry of this exact refusal (or an explicit cancel) to pay.
+ self.cancel_obligations.add(admission_id)
try:
row = await tracked_write(self, partial(cancel_session_input, self.db, epoch=self.epoch,
admission_id=admission_id))
except RuntimeStoreError:
+ self.cancel_obligations.discard(admission_id)
return False
if (row['status'], row['outcome']) != ('terminal', 'cancelled'):
+ self.cancel_obligations.discard(admission_id)
return False
- waiter = self.waiters.pop(admission_id, None)
- if waiter is not None and not waiter.done():
- waiter.set_exception(RuntimeStoreError(reason))
- self._settle_cancelled(ref, admission_id)
+ if admission_id in self.cancel_obligations:
+ waiter = self.waiters.pop(admission_id, None)
+ if waiter is not None and not waiter.done():
+ waiter.set_exception(RuntimeStoreError(reason))
+ self._settle_cancelled(ref, admission_id)
from gateway.session_ingress_media import release_admission_media
await asyncio.to_thread(release_admission_media, self.db, admission_id)
return True
@@ -1133,11 +1141,14 @@ class SessionAuthority:
async def _claim_with_retry(self, ref, live):
"""The drain's next ``(row, first)``, waiting out transient storage and a classic turn on the
route; None when the drain must exit (paused, or stopping with committed rows left queued)."""
- backoff, conflicted = 0.0, None
+ backoff, conflicted, refusing = 0.0, None, None
while True:
- # The refusal handling's own reads and writes share the claim's storage retry: the retry's
- # claim conflicts on the same head again (``conflicted`` kept) and refuses that exact row.
+ # The refusal handling's own reads and writes share the claim's storage retry, which finishes
+ # that exact refusal first: once its cancellation commits, the claim no longer sees the row.
try:
+ if refusing is not None:
+ await self._refuse_head(ref, *refusing)
+ refusing = None
try:
row, first = await self._claim_next(ref, live)
except RuntimeStoreError as exc:
@@ -1156,8 +1167,12 @@ class SessionAuthority:
if head is not None and head != conflicted:
conflicted = head
continue
- if head is not None and await self._refuse_head(ref, head, exc.reason):
- continue
+ if head is not None:
+ refusing = head, exc.reason
+ refused = await self._refuse_head(ref, *refusing)
+ refusing = None
+ if refused:
+ continue
import logging
logging.getLogger(__name__).warning('Session %s paused: %s', ref.session_id, exc.reason)
self._pause(ref, exc.reason)
diff --git a/tests/gateway/test_new_moves_automation_wake.py b/tests/gateway/test_new_moves_automation_wake.py
index cbf5bf8f927..6cd87839367 100644
--- a/tests/gateway/test_new_moves_automation_wake.py
+++ b/tests/gateway/test_new_moves_automation_wake.py
@@ -52,21 +52,37 @@ async def test_new_moves_a_queued_wake_to_the_fresh_session(monkeypatch, settles
captured['result'] = {'final_response': 'reply', 'messages': []}
return 'reply'
monkeypatch.setattr(runner, '_handle_message_with_agent', turn)
+ await adapter.handle_message(MessageEvent(text='A', source=source, message_id='A'))
+ await asyncio.wait_for(running.wait(), 10)
+ old = runner.session_store.peek_session_id(key)
+ assert await admit_route_wake(runner, adapter, source, 'WAKE', 'kanban-wake')
if settles_mid_reset:
# The old turn settles after the rotation, before the wake moves: its drain wakes up there.
+ live = authority.sessions[authority.logical_owner(old)]
+ blocked = []
+
+ class ClaimLock(asyncio.Lock):
+ async def acquire(self):
+ if finish.is_set() and asyncio.current_task() is live.task:
+ blocked.append(self.locked()) # the old drain's next claim, after the old turn settled
+ return await super().acquire()
+ live.mutation_lock = ClaimLock()
reset = runner.async_session_store.reset_session
+ def old_statuses(wake):
+ return [r['status'] for r in list_session_admissions(authority.db, session_id=str(old), pending_only=False)
+ if r['principal_id'].startswith('automation:') == wake]
+
async def reset_then_settle(session_key):
entry = await reset(session_key)
finish.set()
- await asyncio.sleep(0.5) # health: allow HX013 -- the window the old drain must NOT claim in
+ # The window the old drain must NOT claim in: it waits behind /new, or (unlocked) has
+ # already taken the wake off the old FIFO.
+ await async_wait_until(lambda: any(blocked) or old_statuses(True) != ['queued'],
+ timeout=10, what='the old drain to try its next claim')
+ assert old_statuses(False) == ['terminal']
return entry
monkeypatch.setattr(runner.async_session_store, 'reset_session', reset_then_settle)
-
- await adapter.handle_message(MessageEvent(text='A', source=source, message_id='A'))
- await asyncio.wait_for(running.wait(), 10)
- old = runner.session_store.peek_session_id(key)
- assert await admit_route_wake(runner, adapter, source, 'WAKE', 'kanban-wake')
await adapter.handle_message(MessageEvent(text='/new', source=source, message_id='new'))
new = runner.session_store.peek_session_id(key)
assert new != old
diff --git a/tests/gateway/test_preclaim_conflict_settles_head.py b/tests/gateway/test_preclaim_conflict_settles_head.py
index 574c6aac876..3960e37da77 100644
--- a/tests/gateway/test_preclaim_conflict_settles_head.py
+++ b/tests/gateway/test_preclaim_conflict_settles_head.py
@@ -86,3 +86,57 @@ async def test_refusal_write_storage_error_retries_and_still_settles_that_head(t
for r in list_session_admissions(db, session_id='s', pending_only=False)}
assert statuses == {'stale': ('terminal', 'cancelled'), 'follower': ('terminal', 'completed')}
assert waiter.exception().reason == 'admission_conflict'
+
+
+async def test_storage_error_after_the_refusal_commits_still_publishes_its_completion_once(tmp_path, monkeypatch):
+ import sqlite3
+
+ from gateway import session_api_turn, session_authority, session_finite, session_queued_controls
+ from gateway.config import Platform
+
+ db, authority = _authority(tmp_path, monkeypatch, platform=Platform.API_SERVER)
+ monkeypatch.setattr(session_authority, '_CLAIM_RETRY_MIN_S', 0.0)
+ executed, frames, committed = [], [], []
+
+ def check_api_turn(authority, ref, payload):
+ if payload['text'] == 'stale':
+ raise RuntimeStoreError('admission_conflict')
+ monkeypatch.setattr(session_api_turn, 'check_api_turn', check_api_turn)
+
+ async def execute(authority, ref, row):
+ executed.append(row['payload']['text'])
+ return 'done'
+ monkeypatch.setattr(session_finite, 'execute_finite_admission', execute)
+
+ cancel, rows = session_authority.cancel_session_input, session_queued_controls.pending_rows
+
+ def cancel_and_note(db, **kwargs):
+ row = cancel(db, **kwargs)
+ committed.append(row['status'])
+ return row
+
+ def locked_once_after_the_commit(*args, **kwargs):
+ if committed == ['terminal']:
+ committed.append('locked')
+ raise sqlite3.OperationalError('database is locked') # inside the settlement's pending read
+ return rows(*args, **kwargs)
+ monkeypatch.setattr(session_authority, 'cancel_session_input', cancel_and_note)
+ monkeypatch.setattr(session_queued_controls, 'pending_rows', locked_once_after_the_commit)
+
+ with db:
+ authority._schedule = lambda ref: None
+ head = await _submit(authority, 'stale')
+ await _submit(authority, 'follower')
+ authority.sessions['s'].event_stream.observers.add(frames.append)
+ del authority._schedule
+ authority._schedule(REF)
+ await asyncio.wait_for(authority.sessions['s'].task, 5)
+
+ assert 'locked' in committed
+ assert executed == ['follower']
+ completions = [f['params'] for f in frames if f['params']['type'] == 'message.complete'
+ and f['params']['admission_id'] == head.admission_id]
+ assert len(completions) == 1, frames
+ assert completions[0]['payload']['outcome'] == 'cancelled'
+ assert head.admission_id not in authority.cancel_obligations