Skip to content

The agent that grows with you

fix(dashboard): let the offline reservation decide when discovery has no endpoint

unsupportedpastels79213a33192c
diff --git a/hermes_cli/web_server_sessions.py b/hermes_cli/web_server_sessions.py
index 59a3c39e11a..0ea01c36057 100644
--- a/hermes_cli/web_server_sessions.py
+++ b/hermes_cli/web_server_sessions.py
@@ -192,11 +192,12 @@ async def _mutate_via_gateway_owner(profile, actor, params):
dashboard`` is not the store owner, so the offline path's
exclusive-maintenance reservation can never win and every write action
dies on a bare 409 (#138915). The owner exposes the same revision-fenced
- verb over the control socket. Only a home discovery reads as owner-less
- (``absent``) returns None, so the caller falls back to the offline store. An owner that
- exists but could not be identified (a control timeout, a profile mismatch, a refused peer,
- one starting or draining) answers 503 with discovery's reason: the offline reservation would
- lose to it and the edit would vanish behind the bare 409 (#144141).
+ verb over the control socket. With no endpoint the offline store's reservation
+ decides: no endpoint does not prove a live owner (a stale bad-mode socket, a parked
+ secondary read ``inaccessible`` with the lock free). When the reservation loses and
+ discovery read anything but ``absent`` (a control timeout, a profile mismatch, one
+ starting or draining), the edit answers 503 with discovery's reason rather than the
+ bare 409 (#144141).
A dial, mint or RPC that dies on an endpoint discovery just identified is
not owner-less (a slow first dial, or an owner restart between discovery
@@ -211,6 +212,7 @@ async def _mutate_via_gateway_owner(profile, actor, params):
the mutation when the forward acts as that account.
"""
from fastapi import HTTPException
+ from gateway.runtime_ownership import OwnershipConflict
from hermes_cli.gateway_client import GatewayClientError, GatewayRPCError, connect_gateway
from hermes_cli.gateway_runtime import discover_gateway_endpoint
from hermes_cli.gateway_runtime_discovery import DiscoveryError
@@ -222,12 +224,15 @@ async def _mutate_via_gateway_owner(profile, actor, params):
discovery = await asyncio.to_thread(discover_gateway_endpoint, home, timeout=1.5)
endpoint = discovery.endpoint
if endpoint is None:
- if discovery.state == 'absent':
- return None
- reason = discovery.reason_code or discovery.state
- _log.warning('session write for %s: gateway owner not reachable (%s, %s): %s',
- home, discovery.state, reason, discovery.detail)
- raise HTTPException(status_code=503, detail=f'owner_unavailable: {reason}')
+ try:
+ return await asyncio.to_thread(_mutate_offline_store, profile, actor, params)
+ except HTTPException as exc:
+ if discovery.state == 'absent' or not isinstance(exc.__cause__, OwnershipConflict):
+ raise
+ known = ''.join(f', {value}' for value in (discovery.reason_code, discovery.detail) if value)
+ _log.warning('session write for %s: gateway owner not reachable (%s%s)', home, discovery.state, known)
+ reason = discovery.reason_code or discovery.state
+ raise HTTPException(status_code=503, detail=f'owner_unavailable: {reason}: {exc.detail}') from exc
if not getattr(endpoint, 'control_home', True):
# The selected home answered on its own socket (a multiplexer-served profile already
# names the multiplexer root). Pin it, so the ticket is minted on the socket that just
@@ -287,11 +292,9 @@ async def _mutate_session_request(request, profile, session_id, *, request_id,
params['expected_generation'] = expected_generation
try:
if authority is None:
- # A live gateway owner answers first; only an owner-less home takes
- # the offline store (whose reservation loses to that same owner).
+ # A live gateway owner answers first; a home without an endpoint takes
+ # the offline store (whose reservation loses to any owner holding the lock).
result = await _mutate_via_gateway_owner(profile, actor, params)
- if result is None:
- result = await asyncio.to_thread(_mutate_offline_store, profile, actor, params)
else:
with _native_history_transport(request, authority, actor):
result = await mutate_session(authority, actor, SessionRef(authority.profile_id, session_id), params)
diff --git a/tests/hermes_cli/test_session_mutation_standalone.py b/tests/hermes_cli/test_session_mutation_standalone.py
index c23f437edd2..61e0607259c 100644
--- a/tests/hermes_cli/test_session_mutation_standalone.py
+++ b/tests/hermes_cli/test_session_mutation_standalone.py
@@ -74,8 +74,9 @@ async def test_standalone_edit_refuses_while_a_gateway_owns_the_store(tmp_path,
# The held lock without a control socket reads as a starting owner: retryable, never offline.
starting = await client.patch('/api/sessions/owned',
json={'request_id': 'r1', 'expected_revision': 0, 'title': 'X'})
- assert (starting.status_code, starting.json()['detail']) == (503, 'owner_unavailable: starting')
- assert 'gateway owner not reachable' in caplog.text
+ assert starting.status_code == 503, starting.text
+ assert starting.json()['detail'].startswith('owner_unavailable: starting: Exclusive maintenance refused')
+ assert 'gateway owner not reachable (starting)' in caplog.text
# An owner that reserves after discovery read absent still wins the offline reservation.
monkeypatch.setattr('hermes_cli.gateway_runtime.discover_gateway_endpoint',
lambda home, timeout=2.0: GatewayDiscovery('absent'))
@@ -93,3 +94,24 @@ async def test_standalone_edit_refuses_while_a_gateway_owns_the_store(tmp_path,
owner.close()
assert db.get_session('owned')['title'] is None
assert db.get_session('owned')['runtime_revision'] == 0
+
+
+async def test_a_stale_socket_without_a_gateway_still_edits_offline(tmp_path, monkeypatch):
+ """A crashed gateway can leave a ``gateway.sock`` discovery refuses on its mode before it reads
+ ``gateway.lock``. No process owns the store, so the offline reservation wins and the edit lands."""
+ import socket
+ app = _standalone_app(tmp_path, monkeypatch)
+ monkeypatch.chdir(tmp_path) # a relative bind keeps the socket path under AF_UNIX's length limit
+ stale = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ stale.bind('gateway.sock')
+ stale.close()
+ (tmp_path / 'gateway.sock').chmod(0o755)
+ with SessionDB(db_path=tmp_path / 'state.db') as db:
+ db.create_session('stale', source='cli')
+ async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app),
+ base_url='http://localhost', headers=AUTH) as client:
+ pinned = await client.patch('/api/sessions/stale',
+ json={'request_id': 'r1', 'expected_revision': 0, 'pinned': True})
+ assert pinned.status_code == 200, pinned.text
+ assert db.get_session('stale')['pinned'] == 1
diff --git a/tests/hermes_cli/test_web_server_session_owner_forward.py b/tests/hermes_cli/test_web_server_session_owner_forward.py
index 316954a1ca8..ffe2bc4b239 100644
--- a/tests/hermes_cli/test_web_server_session_owner_forward.py
+++ b/tests/hermes_cli/test_web_server_session_owner_forward.py
@@ -107,24 +107,49 @@ async def test_standalone_mutation_falls_back_to_offline_without_an_owner(tmp_pa
('inaccessible', 'profile_mismatch'), ('starting', None)])
async def test_an_owner_discovery_cannot_identify_is_not_owner_less(tmp_path, monkeypatch, state, reason_code):
"""#144141: over SSH ``serve --isolated`` discovery of a running gateway came back without an
- endpoint, and every such verdict took the offline store, whose reservation the live owner
- always wins (409): pins and renames looked applied, then vanished. Only an ``absent`` owner is
- owner-less; anything else answers a retryable 503 that names discovery's reason."""
+ endpoint, and the offline reservation that owner wins answered a bare 409: pins and renames
+ looked applied, then vanished. A lost reservation behind any verdict but ``absent`` answers a
+ retryable 503 that names discovery's reason and keeps the reservation's remedy."""
+ from contextlib import contextmanager
+ from gateway.runtime_ownership import OwnershipConflict
from hermes_cli import web_server_sessions
from hermes_cli.gateway_runtime import GatewayDiscovery
request = _standalone_request(monkeypatch, tmp_path)
monkeypatch.setattr('hermes_cli.gateway_runtime.discover_gateway_endpoint',
lambda home, timeout=2.0: GatewayDiscovery(state, reason_code=reason_code))
- offline = []
- monkeypatch.setattr(web_server_sessions, '_mutate_offline_store',
- lambda profile, actor, params: offline.append(params))
+
+ @contextmanager
+ def owner_holds_the_lock(homes):
+ raise OwnershipConflict('Exclusive maintenance refused: held. Drain and stop the gateway, then retry.')
+ yield
+
+ monkeypatch.setattr('gateway.runtime_ownership.exclusive_maintenance', owner_holds_the_lock)
with pytest.raises(HTTPException) as exc:
await web_server_sessions._mutate_session_request(
request, None, 'session-7', request_id='req-7', expected_revision=0,
- operation='sidebar', payload={})
- assert (exc.value.status_code, exc.value.detail) == (503, f'owner_unavailable: {reason_code or state}')
- assert offline == []
+ operation='sidebar', payload={'pinned': True})
+ assert exc.value.status_code == 503
+ assert exc.value.detail.startswith(f'owner_unavailable: {reason_code or state}: ')
+ assert 'Drain and stop the gateway' in exc.value.detail
+
+
+async def test_a_verdict_without_a_live_owner_still_edits_offline(tmp_path, monkeypatch):
+ """Path checks read ``inaccessible`` before ``gateway.lock`` (a stale bad-mode socket, a parked
+ secondary), so no endpoint does not prove an owner: the offline reservation decides, as on main."""
+ from hermes_cli import web_server_sessions
+ from hermes_cli.gateway_runtime import GatewayDiscovery
+
+ request = _standalone_request(monkeypatch, tmp_path)
+ monkeypatch.setattr('hermes_cli.gateway_runtime.discover_gateway_endpoint', lambda home, timeout=2.0:
+ GatewayDiscovery('inaccessible', reason_code='unsafe_control_permissions'))
+ monkeypatch.setattr(web_server_sessions, '_mutate_offline_store', lambda profile, actor, params:
+ {'session_id': params['session_id'], 'operation': 'sidebar', 'revision': 1})
+ result = await web_server_sessions._mutate_session_request(
+ request, None, 'session-8', request_id='req-8', expected_revision=0,
+ operation='sidebar', payload={'pinned': True})
+ assert result == {'ok': True, 'session_id': 'session-8', 'operation': 'sidebar', 'revision': 1}
@pytest.mark.asyncio