Skip to content

The agent that grows with you

fix(gateway): retired-session polls answer not_found; one scope build path

kshitijk4poor86cc60932fb2
diff --git a/gateway/session_ancillary.py b/gateway/session_ancillary.py
index 3683400a267..4a88f90654c 100644
--- a/gateway/session_ancillary.py
+++ b/gateway/session_ancillary.py
@@ -243,18 +243,30 @@ async def read(connection, ref, params, *, kind):
def _read_projection(authority, live, ref, params, kind):
with live.event_stream.lock:
- if kind == 'control':
- # Persisted per session id, never the route's live objects: readable after a messaging
- # route moved on to another conversation.
- return {'control': control_snapshot(authority, ref)}
- _require_current_route(authority, ref)
- agent, records = _owner_objects(authority, ref, include_fallback_evicted=kind == 'subagents')
- if kind == 'subagents':
- return {'subagents': [{key: record.get(key) for key in _SUBAGENT_FIELDS}
- for record in records], 'delegations': []}
- if kind == 'tail':
- return subagent_tail(records, params['subagent_id'])
- return {'processes': process_snapshot(authority, ref, agent, records)}
+ try:
+ return _project(authority, ref, params, kind)
+ except KeyError:
+ # Deleted/pruned while this read waited for a worker or the lock (retire_live_sessions pops
+ # on the loop): the projection re-indexes authority.sessions, and a gone session is
+ # not_found, as the loop-side check answers.
+ if authority.sessions.get(ref.session_id) is not live:
+ raise RuntimeStoreError('not_found') from None
+ raise
+
+
+def _project(authority, ref, params, kind):
+ if kind == 'control':
+ # Persisted per session id, never the route's live objects: readable after a messaging
+ # route moved on to another conversation.
+ return {'control': control_snapshot(authority, ref)}
+ _require_current_route(authority, ref)
+ agent, records = _owner_objects(authority, ref, include_fallback_evicted=kind == 'subagents')
+ if kind == 'subagents':
+ return {'subagents': [{key: record.get(key) for key in _SUBAGENT_FIELDS}
+ for record in records], 'delegations': []}
+ if kind == 'tail':
+ return subagent_tail(records, params['subagent_id'])
+ return {'processes': process_snapshot(authority, ref, agent, records)}
def _owned_targets(connection, ref, params, *, select):
diff --git a/gateway/session_authorities.py b/gateway/session_authorities.py
index 3cf0e143156..0ff207262b4 100644
--- a/gateway/session_authorities.py
+++ b/gateway/session_authorities.py
@@ -284,25 +284,16 @@ def holds_admission(authority, session_id) -> bool:
def _prepare_owner_scope(home: Path):
- """``(secrets, terminal scope)`` of the profile at *home*, built the way ``_profile_runtime_scope``
- builds them on entry (under the home override, the terminal policy under the secret scope);
- ``None`` when *home* is not a directory (``owner_scope`` binds nothing then)."""
+ """Enter ``owner_scope``'s profile scope in this (worker) thread and return what it bound:
+ ``(secrets, terminal scope)``, or ``None`` when *home* is not a directory (``owner_scope`` binds
+ nothing then). Entering the real scope keeps one definition of how it is built."""
if not home.is_dir():
return None
- from agent.secret_scope import reset_secret_scope, set_secret_scope
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- from tools.terminal_scope import prepare_profile_terminal_scope
- from tui_gateway.launch_profile_policy import owner_secret_scope
- home_token = set_hermes_home_override(str(home))
- try:
- secrets = owner_secret_scope(home)
- secret_token = set_secret_scope(secrets, profile_home=str(home))
- try:
- return secrets, prepare_profile_terminal_scope(home)
- finally:
- reset_secret_scope(secret_token)
- finally:
- reset_hermes_home_override(home_token)
+ from agent.secret_scope import current_secret_scope
+ from gateway.run import _profile_runtime_scope
+ from tools.terminal_scope import get_terminal_scope
+ with _profile_runtime_scope(home, hydrate_secrets=False):
+ return current_secret_scope(), get_terminal_scope()
@asynccontextmanager
diff --git a/tests/gateway/test_rpc_scope_off_loop.py b/tests/gateway/test_rpc_scope_off_loop.py
index f56a759abdb..58db2512dc3 100644
--- a/tests/gateway/test_rpc_scope_off_loop.py
+++ b/tests/gateway/test_rpc_scope_off_loop.py
@@ -24,8 +24,11 @@ def _bound():
from tools.terminal_scope import get_terminal_scope
secrets = current_secret_scope()
terminal = get_terminal_scope()
- return (get_hermes_home_override(), dict(secrets) if secrets is not None else None,
- current_secret_scope_home(), dict(terminal) if terminal is not None else None, type(terminal).__name__)
+ return {"home": get_hermes_home_override(),
+ "secrets": dict(secrets) if secrets is not None else None,
+ "secret_home": current_secret_scope_home(),
+ "terminal": dict(terminal) if terminal is not None else None,
+ "terminal_type": type(terminal).__name__}
@pytest.fixture
@@ -45,8 +48,8 @@ async def test_async_owner_scope_binds_what_owner_scope_binds_with_the_reads_off
authority = _authority(profile)
with session_authorities.owner_scope(authority):
expected = _bound()
- assert expected[0] == str(profile) and expected[1]["LOOPSTALL_PROBE_KEY"] == "from-profile"
- assert expected[3]["TERMINAL_TIMEOUT"] == "77"
+ assert expected["home"] == str(profile) and expected["secrets"]["LOOPSTALL_PROBE_KEY"] == "from-profile"
+ assert expected["terminal"]["TERMINAL_TIMEOUT"] == "77"
loop_thread = threading.get_ident()
build_threads = []
@@ -81,23 +84,67 @@ async def test_async_owner_scope_keeps_owner_scope_for_unrouted_owners(tmp_path)
assert _bound() == expected
-async def test_ancillary_polls_project_off_the_loop(monkeypatch):
- """``subagent.list`` / ``process.list`` / ``session.control.read`` build their projection in a worker:
- it takes the session's event lock (held by every streamed token) and reads state.db."""
- from gateway import session_ancillary
+class _RecordingLock:
+ """``event_stream.lock`` stand-in: records the thread that takes it, optionally runs a hook first."""
- loop_thread = threading.get_ident()
- seen = []
+ def __init__(self, on_enter=None):
+ self.threads = []
+ self._on_enter = on_enter
+
+ def __enter__(self):
+ self.threads.append(threading.get_ident())
+ if self._on_enter:
+ self._on_enter()
+ return self
+
+ def __exit__(self, *exc):
+ return False
- def projection(authority, live, ref, params, kind):
- seen.append((threading.get_ident(), kind))
- return {'subagents': [], 'delegations': []}
- monkeypatch.setattr(session_ancillary, "_read_projection", projection)
+def _ancillary_fixture(monkeypatch, *, on_enter=None, snapshot=None):
+ from gateway import session_ancillary
+ lock = _RecordingLock(on_enter)
+ live = SimpleNamespace(event_stream=SimpleNamespace(lock=lock))
+ authority = SimpleNamespace(sessions={"s1": live})
monkeypatch.setattr(session_ancillary, "authorize", lambda *a, **k: None)
+ monkeypatch.setattr(session_ancillary, "control_snapshot", snapshot or (lambda authority, ref: {"revision": "r1"}))
ref = SimpleNamespace(session_id="s1", profile_id="fixture")
- connection = SimpleNamespace(authority=SimpleNamespace(sessions={"s1": object()}))
- result = await session_ancillary.read(connection, ref, {"session_id": "s1"}, kind="subagents")
- assert result == {'subagents': [], 'delegations': []}
- assert seen and seen[0][0] != loop_thread, "the projection ran on the event loop"
+ return session_ancillary, SimpleNamespace(authority=authority), ref, lock
+
+
+async def test_ancillary_polls_take_the_event_lock_off_the_loop(monkeypatch):
+ """The projection takes the session's event lock (held by every streamed token) and reads state.db:
+ it must do both in a worker, not on the event loop."""
+ session_ancillary, connection, ref, lock = _ancillary_fixture(monkeypatch)
+ result = await session_ancillary.read(connection, ref, {"session_id": "s1"}, kind="control")
+ assert result == {"control": {"revision": "r1"}}
+ assert lock.threads and threading.get_ident() not in lock.threads, "the event lock was taken on the loop"
+
+
+async def test_ancillary_poll_of_a_session_retired_mid_read_is_not_found(monkeypatch):
+ """A chat deleted or pruned while its poll waits in the worker answers not_found (what the loop-side
+ check answers), not an internal error from the projection re-indexing ``authority.sessions``."""
+ from hermes_state_runtime import RuntimeStoreError
+
+ def retire_then_index(authority, ref):
+ authority.sessions.pop("s1")
+ return authority.sessions["s1"] # physical_target / _require_current_route re-index like this
+
+ session_ancillary, connection, ref, _lock = _ancillary_fixture(monkeypatch, snapshot=retire_then_index)
+ with pytest.raises(RuntimeStoreError) as exc:
+ await session_ancillary.read(connection, ref, {"session_id": "s1"}, kind="control")
+ assert str(exc.value) == "not_found"
+
+
+async def test_ancillary_projection_keyerror_of_a_live_session_still_raises(monkeypatch):
+ """Only a retired session maps to not_found: a KeyError from a bug in a live session's projection
+ still surfaces."""
+ def broken(authority, ref):
+ return {}["missing"]
+
+ session_ancillary, connection, ref, _lock = _ancillary_fixture(monkeypatch, snapshot=broken)
+ with pytest.raises(KeyError):
+ await session_ancillary.read(connection, ref, {"session_id": "s1"}, kind="control")
diff --git a/tools/terminal_scope.py b/tools/terminal_scope.py
index c9a631dec22..549326285ac 100644
--- a/tools/terminal_scope.py
+++ b/tools/terminal_scope.py
@@ -219,8 +219,9 @@ def _resolve_scope_cwd_placeholder(scope: dict[str, str]) -> None:
def prepare_profile_terminal_scope(
hermes_home: Any, *, env_overlay: Optional[dict[str, str]] = None) -> dict[str, str]:
"""The scope ``install_profile_terminal_scope`` installs: the profile's policy, or the refusal scope
- when it cannot be resolved. Never raises. File reads only, so it can run in a worker thread and be
- installed on the loop (``install_and_reset_profile_terminal_scope(prepared=...)``)."""
+ when it cannot be resolved. Never raises. Reads the profile files under the current context's home
+ override and secret scope; the result can be built in a worker and installed on the loop with
+ ``install_and_reset_profile_terminal_scope(prepared=...)``."""
try:
return build_profile_terminal_scope(hermes_home, env_overlay=env_overlay)
except TerminalPolicyUnavailable as exc: