The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): settle a refused head after a storage retry; sync the /new race test
unsupportedpastels04fae30eebb3
diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex 256582304c1..6edca211676 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -1115,17 +1115,25 @@ class SessionAuthority:"""Settle the queued head a preclaim check refused as cancelled: its waiters get *reason* (theingress notice, the API's refusal) and its successors revalidate on their own merits."""from gateway.session_runtime_workers import tracked_write+ before = get_session_admission(self.db, admission_id=admission_id)+ if before is not None and before['status'] == 'queued':+ # Owed from before the write, as in _commit_cancel: storage failing after the commit leaves+ # it for the drain's retry of this exact refusal (or an explicit cancel) to pay.+ self.cancel_obligations.add(admission_id)try:row = await tracked_write(self, partial(cancel_session_input, self.db, epoch=self.epoch,admission_id=admission_id))except RuntimeStoreError:+ self.cancel_obligations.discard(admission_id)return Falseif (row['status'], row['outcome']) != ('terminal', 'cancelled'):+ self.cancel_obligations.discard(admission_id)return False- waiter = self.waiters.pop(admission_id, None)- if waiter is not None and not waiter.done():- waiter.set_exception(RuntimeStoreError(reason))- self._settle_cancelled(ref, admission_id)+ if admission_id in self.cancel_obligations:+ waiter = self.waiters.pop(admission_id, None)+ if waiter is not None and not waiter.done():+ waiter.set_exception(RuntimeStoreError(reason))+ self._settle_cancelled(ref, admission_id)from gateway.session_ingress_media import release_admission_mediaawait asyncio.to_thread(release_admission_media, self.db, admission_id)return True@@ -1133,11 +1141,14 @@ class SessionAuthority:async def _claim_with_retry(self, ref, live):"""The drain's next ``(row, first)``, waiting out transient storage and a classic turn on theroute; None when the drain must exit (paused, or stopping with committed rows left queued)."""- backoff, conflicted = 0.0, None+ backoff, conflicted, refusing = 0.0, None, Nonewhile True:- # The refusal handling's own reads and writes share the claim's storage retry: the retry's- # claim conflicts on the same head again (``conflicted`` kept) and refuses that exact row.+ # The refusal handling's own reads and writes share the claim's storage retry, which finishes+ # that exact refusal first: once its cancellation commits, the claim no longer sees the row.try:+ if refusing is not None:+ await self._refuse_head(ref, *refusing)+ refusing = Nonetry:row, first = await self._claim_next(ref, live)except RuntimeStoreError as exc:@@ -1156,8 +1167,12 @@ class SessionAuthority:if head is not None and head != conflicted:conflicted = headcontinue- if head is not None and await self._refuse_head(ref, head, exc.reason):- continue+ if head is not None:+ refusing = head, exc.reason+ refused = await self._refuse_head(ref, *refusing)+ refusing = None+ if refused:+ continueimport logginglogging.getLogger(__name__).warning('Session %s paused: %s', ref.session_id, exc.reason)self._pause(ref, exc.reason)diff --git a/tests/gateway/test_new_moves_automation_wake.py b/tests/gateway/test_new_moves_automation_wake.pyindex cbf5bf8f927..6cd87839367 100644--- a/tests/gateway/test_new_moves_automation_wake.py+++ b/tests/gateway/test_new_moves_automation_wake.py@@ -52,21 +52,37 @@ async def test_new_moves_a_queued_wake_to_the_fresh_session(monkeypatch, settlescaptured['result'] = {'final_response': 'reply', 'messages': []}return 'reply'monkeypatch.setattr(runner, '_handle_message_with_agent', turn)+ await adapter.handle_message(MessageEvent(text='A', source=source, message_id='A'))+ await asyncio.wait_for(running.wait(), 10)+ old = runner.session_store.peek_session_id(key)+ assert await admit_route_wake(runner, adapter, source, 'WAKE', 'kanban-wake')if settles_mid_reset:# The old turn settles after the rotation, before the wake moves: its drain wakes up there.+ live = authority.sessions[authority.logical_owner(old)]+ blocked = []++ class ClaimLock(asyncio.Lock):+ async def acquire(self):+ if finish.is_set() and asyncio.current_task() is live.task:+ blocked.append(self.locked()) # the old drain's next claim, after the old turn settled+ return await super().acquire()+ live.mutation_lock = ClaimLock()reset = runner.async_session_store.reset_session+ def old_statuses(wake):+ return [r['status'] for r in list_session_admissions(authority.db, session_id=str(old), pending_only=False)+ if r['principal_id'].startswith('automation:') == wake]+async def reset_then_settle(session_key):entry = await reset(session_key)finish.set()- await asyncio.sleep(0.5) # health: allow HX013 -- the window the old drain must NOT claim in+ # The window the old drain must NOT claim in: it waits behind /new, or (unlocked) has+ # already taken the wake off the old FIFO.+ await async_wait_until(lambda: any(blocked) or old_statuses(True) != ['queued'],+ timeout=10, what='the old drain to try its next claim')+ assert old_statuses(False) == ['terminal']return entrymonkeypatch.setattr(runner.async_session_store, 'reset_session', reset_then_settle)-- await adapter.handle_message(MessageEvent(text='A', source=source, message_id='A'))- await asyncio.wait_for(running.wait(), 10)- old = runner.session_store.peek_session_id(key)- assert await admit_route_wake(runner, adapter, source, 'WAKE', 'kanban-wake')await adapter.handle_message(MessageEvent(text='/new', source=source, message_id='new'))new = runner.session_store.peek_session_id(key)assert new != olddiff --git a/tests/gateway/test_preclaim_conflict_settles_head.py b/tests/gateway/test_preclaim_conflict_settles_head.pyindex 574c6aac876..3960e37da77 100644--- a/tests/gateway/test_preclaim_conflict_settles_head.py+++ b/tests/gateway/test_preclaim_conflict_settles_head.py@@ -86,3 +86,57 @@ async def test_refusal_write_storage_error_retries_and_still_settles_that_head(tfor r in list_session_admissions(db, session_id='s', pending_only=False)}assert statuses == {'stale': ('terminal', 'cancelled'), 'follower': ('terminal', 'completed')}assert waiter.exception().reason == 'admission_conflict'+++async def test_storage_error_after_the_refusal_commits_still_publishes_its_completion_once(tmp_path, monkeypatch):+ import sqlite3++ from gateway import session_api_turn, session_authority, session_finite, session_queued_controls+ from gateway.config import Platform++ db, authority = _authority(tmp_path, monkeypatch, platform=Platform.API_SERVER)+ monkeypatch.setattr(session_authority, '_CLAIM_RETRY_MIN_S', 0.0)+ executed, frames, committed = [], [], []++ def check_api_turn(authority, ref, payload):+ if payload['text'] == 'stale':+ raise RuntimeStoreError('admission_conflict')+ monkeypatch.setattr(session_api_turn, 'check_api_turn', check_api_turn)++ async def execute(authority, ref, row):+ executed.append(row['payload']['text'])+ return 'done'+ monkeypatch.setattr(session_finite, 'execute_finite_admission', execute)++ cancel, rows = session_authority.cancel_session_input, session_queued_controls.pending_rows++ def cancel_and_note(db, **kwargs):+ row = cancel(db, **kwargs)+ committed.append(row['status'])+ return row++ def locked_once_after_the_commit(*args, **kwargs):+ if committed == ['terminal']:+ committed.append('locked')+ raise sqlite3.OperationalError('database is locked') # inside the settlement's pending read+ return rows(*args, **kwargs)+ monkeypatch.setattr(session_authority, 'cancel_session_input', cancel_and_note)+ monkeypatch.setattr(session_queued_controls, 'pending_rows', locked_once_after_the_commit)++ with db:+ authority._schedule = lambda ref: None+ head = await _submit(authority, 'stale')+ await _submit(authority, 'follower')+ authority.sessions['s'].event_stream.observers.add(frames.append)+ del authority._schedule+ authority._schedule(REF)+ await asyncio.wait_for(authority.sessions['s'].task, 5)++ assert 'locked' in committed+ assert executed == ['follower']+ completions = [f['params'] for f in frames if f['params']['type'] == 'message.complete'+ and f['params']['admission_id'] == head.admission_id]+ assert len(completions) == 1, frames+ assert completions[0]['payload']['outcome'] == 'cancelled'+ assert head.admission_id not in authority.cancel_obligations