Skip to content

The agent that grows with you

fix(gateway): settle a pending route row on every exit of the transition that published it

Tekniumfe0e58b67f2c
diff --git a/gateway/session.py b/gateway/session.py
index e43c5c9990e..7626b1880dd 100644
--- a/gateway/session.py
+++ b/gateway/session.py
@@ -971,17 +971,21 @@ class SessionStore(
create_kwargs = self._route_create(
decision, session_key, source, now, force_new, observed
)
- if decision.needs_save:
- if decision.metadata_only_save:
- self._save_entry(session_key)
- else:
- self._save_entries()
-
- self._finish_route_transition(
- session_key, end_session_id=decision.prev_session_id,
- end_reason=decision.reset_reason or "session_reset", create_kwargs=create_kwargs,
- origin=source, display_name=decision.entry.display_name,
- )
+ try:
+ if decision.needs_save:
+ if decision.metadata_only_save:
+ self._save_entry(session_key)
+ else:
+ self._save_entries()
+
+ self._finish_route_transition(
+ session_key, end_session_id=decision.prev_session_id,
+ end_reason=decision.reset_reason or "session_reset", create_kwargs=create_kwargs,
+ origin=source, display_name=decision.entry.display_name,
+ )
+ finally:
+ if create_kwargs:
+ self._row_settled(create_kwargs["session_id"])
return decision.entry
def _apply_route_checks(
@@ -1057,20 +1061,19 @@ class SessionStore(
auto_reset_reason=decision.reset_reason, reset_had_activity=decision.reset_had_activity,
prev_session_id=decision.prev_session_id, transport_profile=transport_profile_of(source),
)
+ create_kwargs = self._session_create_kwargs(
+ session_id=session_id, session_key=session_key, origin=source,
+ source_value=source.platform.value, display_name=source.chat_name,
+ parent_session_id=decision.prev_session_id,
+ )
with self._lock:
current = self._entries.get(session_key)
if current is None or (force_new and current is observed):
self._entries[session_key] = current = candidate
- self._rows_pending.add(session_id)
+ self._rows_pending.add(session_id) # the caller settles it once create_kwargs land
decision.entry = current
decision.needs_save = True
- if current is not candidate:
- return None
- return self._session_create_kwargs(
- session_id=session_id, session_key=session_key, origin=source,
- source_value=source.platform.value, display_name=source.chat_name,
- parent_session_id=decision.prev_session_id,
- )
+ return create_kwargs if current is candidate else None
def update_session(
self, session_key: str, last_prompt_tokens: int | None = None, touch_activity: bool = True,
@@ -1152,21 +1155,24 @@ class SessionStore(
if old_entry.origin and old_entry.origin.platform == Platform.LOCAL:
from gateway.session_local_recovery import reset_local_session
return reset_local_session(self, old_entry, session_id, now, display_name)
+ db_create_kwargs = self._session_create_kwargs(
+ session_id=session_id, session_key=session_key, origin=old_entry.origin,
+ source_value=old_entry.platform.value if old_entry.platform else "unknown",
+ display_name=old_entry.display_name, parent_session_id=old_entry.session_id,
+ )
new_entry = self._replace_route_locked(
session_key, old_entry, session_id, now,
display_name=display_name if display_name is not None else old_entry.display_name,
is_fresh_reset=True,
)
- db_create_kwargs = self._session_create_kwargs(
- session_id=session_id, session_key=session_key, origin=old_entry.origin,
- source_value=old_entry.platform.value if old_entry.platform else "unknown",
- display_name=old_entry.display_name, parent_session_id=old_entry.session_id,
+ try:
+ self._finish_route_transition(
+ session_key, end_session_id=old_entry.session_id, end_reason="session_reset",
+ create_kwargs=db_create_kwargs, origin=old_entry.origin,
+ display_name=new_entry.display_name, during=" during reset",
)
- self._finish_route_transition(
- session_key, end_session_id=old_entry.session_id, end_reason="session_reset",
- create_kwargs=db_create_kwargs, origin=old_entry.origin,
- display_name=new_entry.display_name, during=" during reset",
- )
+ finally:
+ self._row_settled(session_id)
return new_entry
def _replace_route_locked(self, session_key, old_entry, session_id, now, **fields) -> SessionEntry:
@@ -1177,8 +1183,8 @@ class SessionStore(
transport_profile=old_entry.transport_profile, **fields,
)
self._entries[session_key] = new_entry
- self._rows_pending.add(session_id)
self._save()
+ self._rows_pending.add(session_id) # after the save: a raising save leaves nothing pending
return new_entry
def rekey_profile_routing(self, old_name: str, new_name: str) -> int:
diff --git a/gateway/session_recovery.py b/gateway/session_recovery.py
index 1d6cf0d15a0..187827d20ae 100644
--- a/gateway/session_recovery.py
+++ b/gateway/session_recovery.py
@@ -389,34 +389,30 @@ class SessionRecoveryMixin:
explicit reset boundary (with the specific reason so state.db is auditable, e.g.
``suspended`` vs plain ``session_reset``), then INSERT the new row + routing
peer. Both best-effort: failures are warned and self-healed by the next peer refresh."""
- try:
- if end_session_id:
- from hermes_cli.observability.relay_shared_metrics import close_session_run
-
- close_session_run(end_session_id)
- if self._db_for_key(session_key) and end_session_id:
- self._promote_session_reset(
- session_key, end_session_id, end_reason,
- log=lambda e: logger.warning(
- "Failed to end predecessor session row %s for %s%s: %s — the old row remains "
- "open and may win restart recovery until the next successful peer refresh",
- end_session_id, session_key, during, e),
- )
- if self._db_for_key(session_key) and create_kwargs:
- self._create_session_row(
- session_key, create_kwargs, origin, display_name,
- log=lambda e: logger.warning(
- "Failed to create session row %s for %s%s: %s — deferring to the "
- "self-healing peer refresh on the next turn",
- create_kwargs.get("session_id"), session_key, during, e),
- )
- finally:
- if create_kwargs:
- self._row_settled(create_kwargs["session_id"])
+ if end_session_id:
+ from hermes_cli.observability.relay_shared_metrics import close_session_run
+
+ close_session_run(end_session_id)
+ if self._db_for_key(session_key) and end_session_id:
+ self._promote_session_reset(
+ session_key, end_session_id, end_reason,
+ log=lambda e: logger.warning(
+ "Failed to end predecessor session row %s for %s%s: %s — the old row remains "
+ "open and may win restart recovery until the next successful peer refresh",
+ end_session_id, session_key, during, e),
+ )
+ if self._db_for_key(session_key) and create_kwargs:
+ self._create_session_row(
+ session_key, create_kwargs, origin, display_name,
+ log=lambda e: logger.warning(
+ "Failed to create session row %s for %s%s: %s — deferring to the "
+ "self-healing peer refresh on the next turn",
+ create_kwargs.get("session_id"), session_key, during, e),
+ )
def _row_settled(self, session_id: str) -> None:
- """The route transition that published *session_id* finished its state.db half: from here
- a missing or ended row is real again (see ``_is_session_ended_in_db``)."""
+ """The route transition that published *session_id* returned (its state.db half done or
+ failed): from here a missing or ended row is real again (see ``_is_session_ended_in_db``)."""
with self._lock:
self._rows_pending.discard(session_id)
diff --git a/tests/gateway/test_route_transition_racing_turn.py b/tests/gateway/test_route_transition_racing_turn.py
index 9b8fed5b282..538f7518229 100644
--- a/tests/gateway/test_route_transition_racing_turn.py
+++ b/tests/gateway/test_route_transition_racing_turn.py
@@ -12,7 +12,6 @@ handlers. The window is held open at the real SQL statement that ends the predec
"""
import asyncio
import threading
-import time
import weakref
from pathlib import Path
@@ -25,6 +24,7 @@ from gateway.run_runtime import initialize_gateway_runtime
from gateway.runtime_ownership import process_ownership
from gateway.session import SessionSource
from gateway.session_ingress_context import register_transport_home
+from tests.waiting import async_wait_until
class _Adapter:
@@ -93,13 +93,6 @@ class _HoldPredecessorEnd:
self.conn.set_trace_callback(None)
-async def _until(predicate, what):
- deadline = time.monotonic() + 10
- while not predicate():
- assert time.monotonic() < deadline, what
- await asyncio.sleep(0.01)
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize('command', ['/new', '/resume First chat'])
async def test_a_turn_finishing_mid_transition_leaves_the_chat_on_the_conversation_it_named(served, command):
@@ -115,7 +108,7 @@ async def test_a_turn_finishing_mid_transition_leaves_the_chat_on_the_conversati
with _HoldPredecessorEnd(authority.db) as hold:
transition = asyncio.create_task(_command(runner, chat, command))
- await _until(hold.reached.is_set, 'the transition reaches state.db')
+ await async_wait_until(hold.reached.is_set, what='the transition reaches state.db')
named = store.lookup_by_session_key(key)
assert named.session_id not in (None, before)
stamp = named.updated_at
@@ -124,8 +117,9 @@ async def test_a_turn_finishing_mid_transition_leaves_the_chat_on_the_conversati
finishing = asyncio.create_task(runner._run_post_turn_hooks(
agent_result={'final_response': 'the answer on the old conversation'}, source=chat,
is_internal=False))
- await _until(lambda: store.peek_session_id(key) != named.session_id or named.updated_at != stamp,
- 'the finishing turn resolved the route')
+ await async_wait_until(
+ lambda: store.peek_session_id(key) != named.session_id or named.updated_at != stamp,
+ what='the finishing turn resolved the route')
hold.release.set()
await transition
await finishing