Skip to content

The agent that grows with you

fix(kanban): the owner refuses a worker that dials into a restart wait opened after discovery

Tekniumdbde5a991ae0
diff --git a/gateway/control_socket.py b/gateway/control_socket.py
index d9e3d74788b..ddcb81aa7ec 100644
--- a/gateway/control_socket.py
+++ b/gateway/control_socket.py
@@ -373,7 +373,8 @@ class GatewayControlServer:
raise PermissionError("invalid bootstrap envelope")
params = request.get("params")
if (not isinstance(params, dict) or not {"profile_id", "instance_id", "purpose"} <= set(params)
- or set(params) - {"profile_id", "instance_id", "purpose", "scope", "subject"}):
+ or set(params) - {"profile_id", "instance_id", "purpose", "scope", "subject", "restart_wait"}
+ or params.get("restart_wait", "refuse") != "refuse"):
raise PermissionError("invalid bootstrap parameters")
if params["instance_id"] != self.ticket_store.instance_id:
raise PermissionError("stale runtime instance")
@@ -389,7 +390,8 @@ class GatewayControlServer:
if subject is not peer_subject and not _structured_login_subject(subject):
raise PermissionError("invalid bootstrap subject")
ticket = self.ticket_store.mint(profile_id=params["profile_id"], subject=subject,
- purpose=params["purpose"], scope=params.get("scope", "profile"))
+ purpose=params["purpose"], scope=params.get("scope", "profile"),
+ refuse_restart_wait="restart_wait" in params)
return {"ticket": ticket, "expires_in_seconds": 30,
"instance_id": self.ticket_store.instance_id,
"profile_id": params["profile_id"], "runtime_protocol": 1}
diff --git a/gateway/run_api.py b/gateway/run_api.py
index f7a09da45a1..2051c93313c 100644
--- a/gateway/run_api.py
+++ b/gateway/run_api.py
@@ -459,7 +459,10 @@ class GatewayRuntimeAPI:
return
return await self.app(scope, receive, send)
allow_draining = not operator
- admitted_in_wait = restart_wait_open(self.runner)
+ # Decided here, at admission, not at the mint: a restart wait that opened after the client's
+ # discovery still refuses the machine poller that asked to stay out of it (a Kanban worker),
+ # which would otherwise hold the turn the wait is waiting on.
+ admitted_in_wait = restart_wait_open(self.runner) and not grant.get('refuse_restart_wait')
# A restart's after-turn wait still owns its turns: an interactive client attaches to answer
# a parked approval/clarify (#139041); stop() closes this window.
if ((descriptor['state'] != 'ready' or self.runner._draining) and not allow_draining
diff --git a/gateway/runtime_bootstrap.py b/gateway/runtime_bootstrap.py
index 564757e3b86..97e860e480f 100644
--- a/gateway/runtime_bootstrap.py
+++ b/gateway/runtime_bootstrap.py
@@ -34,9 +34,11 @@ class TicketStore:
self._entries: dict[str, tuple[float, dict]] = {}
self._lock = threading.Lock()
- def mint(self, *, profile_id, subject, purpose, scope='profile') -> str:
+ def mint(self, *, profile_id, subject, purpose, scope='profile', refuse_restart_wait=False) -> str:
"""``scope='host'`` lets an interactive connection select every profile this process
- serves (the Desktop's one shared socket); the default binds exactly ``profile_id``."""
+ serves (the Desktop's one shared socket); the default binds exactly ``profile_id``.
+ ``refuse_restart_wait``: the socket redeeming it is refused while a restart's after-turn
+ wait is open, however long after the mint that wait began (a machine poller, never a human)."""
if (profile_id not in self.profile_ids or purpose not in _PURPOSE_CAPABILITIES or not subject
or scope not in _SCOPES or (scope == 'host' and purpose != 'interactive')):
raise PermissionError('bootstrap binding rejected')
@@ -49,7 +51,8 @@ class TicketStore:
self._entries[hashlib.sha256(ticket.encode()).hexdigest()] = (
now + self.TTL_SECONDS,
{'instance_id': self.instance_id, 'profile_id': profile_id, 'subject': subject,
- 'purpose': purpose, 'capabilities': _PURPOSE_CAPABILITIES[purpose], 'scope': scope})
+ 'purpose': purpose, 'capabilities': _PURPOSE_CAPABILITIES[purpose], 'scope': scope,
+ 'refuse_restart_wait': bool(refuse_restart_wait)})
return ticket
def redeem(self, ticket, *, profile_id, purpose) -> dict:
diff --git a/hermes_cli/gateway_client.py b/hermes_cli/gateway_client.py
index fa7cc3f7e27..f7897479870 100644
--- a/hermes_cli/gateway_client.py
+++ b/hermes_cli/gateway_client.py
@@ -248,18 +248,32 @@ def private_control_query(home: Path, request: bytes, timeout: float = 5) -> byt
return bytes(data)
-def _session_ticket(home: Path, endpoint, *, purpose="interactive", scope=None, subject=None) -> str:
+# An owner that predates ``restart_wait`` refuses the unknown parameter with exactly this answer.
+_UNKNOWN_TICKET_PARAMS = "PermissionError: invalid bootstrap parameters"
+
+
+def _session_ticket(home: Path, endpoint, *, purpose="interactive", scope=None, subject=None,
+ refuse_restart_wait=False) -> str:
from hermes_cli.gateway_runtime import control_home_for
from hermes_cli.gateway_runtime_discovery import _identify_response
# A served secondary's ticket is minted by the multiplexer's socket, bound to the secondary.
home = control_home_for(home, endpoint)
# ``subject`` names the principal a same-user caller acts for (the standalone dashboard's
# authenticated user) instead of the socket UID; the owner's peer gate stays the boundary.
- request = json.dumps({"protocol": 1, "id": 1, "verb": "session-ticket", "params": {
- "profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "purpose": purpose,
- **({"scope": scope} if scope else {}), **({"subject": subject} if subject else {}),
- }}).encode() + b"\n"
- grant = _identify_response(private_control_query(home, request))
+ # ``restart_wait``: the owner refuses the socket at admission if a restart wait has opened by then.
+ params = {"profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "purpose": purpose,
+ **({"scope": scope} if scope else {}), **({"subject": subject} if subject else {}),
+ **({"restart_wait": "refuse"} if refuse_restart_wait else {})}
+ request = json.dumps({"protocol": 1, "id": 1, "verb": "session-ticket", "params": params}).encode() + b"\n"
+ answer = private_control_query(home, request)
+ refused = json.loads(answer.split(b"\n", 1)[0]) if refuse_restart_wait else None
+ if isinstance(refused, dict) and refused.get("error") == _UNKNOWN_TICKET_PARAMS:
+ # An owner older than this client (the code on disk was updated under it) cannot hold the
+ # policy; refusing it outright would strand the caller until it restarts. The pre-dial check stands.
+ del params["restart_wait"]
+ answer = private_control_query(home, json.dumps(
+ {"protocol": 1, "id": 1, "verb": "session-ticket", "params": params}).encode() + b"\n")
+ grant = _identify_response(answer)
if any(grant.get(k) != v for k, v in {
"profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "runtime_protocol": 1,
}.items()) or not isinstance(grant.get("ticket"), str) or not grant["ticket"]:
@@ -291,7 +305,8 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10
rather than the socket identity; only a peer the owner already trusts may name one.
``max_size``: the largest frame accepted; a ``session.resume`` answer carries the whole history.
Inside ``outside_restart_wait()`` an owner in a restart's after-turn wait is refused as
- ``draining`` (``restart_wait``) instead of attached."""
+ ``draining`` (``restart_wait``) instead of attached, and the ticket carries that policy so the
+ owner also refuses the socket when the wait opens between discovery and the dial."""
from websockets.asyncio.client import connect
from hermes_constants import get_hermes_home
from hermes_cli.gateway_runtime import attachable, ensure_gateway_runtime, gateway_profile_home
@@ -325,7 +340,8 @@ async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 10
result.state, result.reason_code)
endpoint = result.endpoint
try:
- ticket = await asyncio.to_thread(_session_ticket, home, endpoint)
+ policy: dict = {"refuse_restart_wait": True} if _REFUSE_RESTART_WAIT.get() else {}
+ ticket = await asyncio.to_thread(_session_ticket, home, endpoint, **policy)
except DiscoveryError as exc:
# The control socket refused the ticket: the verdict discovery gives the same refusal.
detail = exc.bounded_detail(500)
diff --git a/tests/hermes_cli/test_kanban_worker_restart_wait.py b/tests/hermes_cli/test_kanban_worker_restart_wait.py
index a38116b42f4..58c1a0cb5e8 100644
--- a/tests/hermes_cli/test_kanban_worker_restart_wait.py
+++ b/tests/hermes_cli/test_kanban_worker_restart_wait.py
@@ -107,3 +107,97 @@ async def test_a_worker_cut_by_a_restart_wait_waits_outside_it(monkeypatch, tmp_
await kanban_worker_client.run({"task_id": "t1"}, str(tmp_path / "kanban.db"))
assert owner.receipts_in_wait == 0
assert owner.dials > 2
+
+
+class _Attached(Exception):
+ """The owner admitted the socket into its handler."""
+
+
+def _owner_rig(tmp_path, monkeypatch, *, old_owner=False):
+ """A ready owner whose restart wait opens while the client mints its ticket: after discovery
+ said ``ready`` and before the WebSocket dials. Tickets come from the real control-socket verb
+ and the dial is admitted (or refused) by the real ``GatewayRuntimeAPI``."""
+ import json
+
+ from gateway.control_socket import GatewayControlServer
+ from gateway.run_api import GatewayRuntimeAPI
+ from gateway.runtime_bootstrap import TicketStore
+ from hermes_cli import web_server as web
+ from hermes_cli import web_server_chat
+
+ profile = str(tmp_path)
+ tickets = TicketStore("owner", [profile])
+ control = GatewayControlServer(tmp_path)
+ control.ticket_store = tickets
+ runner = SimpleNamespace(_draining=False, _running=True, session_runtime_descriptor={"state": "ready"},
+ session_authority=SimpleNamespace(db=SimpleNamespace(db_path=tmp_path / "state.db"),
+ profile_id=profile, runner=None),
+ session_ticket_store=tickets)
+ endpoint = gateway_runtime.GatewayEndpoint(profile, "owner", 1, 1, "http://127.0.0.1:1", "none", frozenset())
+ monkeypatch.setattr(gateway_runtime, "ensure_gateway_runtime",
+ lambda *a, **k: gateway_runtime.GatewayDiscovery("ready", endpoint))
+
+ def control_query(home, request, timeout=5):
+ params = json.loads(request)["params"]
+ if old_owner and "restart_wait" in params: # an owner from before the policy existed
+ return json.dumps({"ok": False, "error": gateway_client._UNKNOWN_TICKET_PARAMS, "protocol": 1,
+ "id": 1}).encode() + b"\n"
+ answer = control.handle_request_line(request, "uid:test")
+ runner._draining = True # the restart begins: its after-turn wait is open by the dial
+ runner.session_runtime_descriptor.update(state="draining", drain_reason=RESTART_WAIT_REASON)
+ return answer
+ monkeypatch.setattr(gateway_client, "private_control_query", control_query)
+
+ async def handle(ws, **kwargs):
+ raise _Attached
+
+ async def fallback(*args):
+ pytest.fail("a ticketed native socket must not enter the legacy fallback")
+ monkeypatch.setattr("tui_gateway.ws.handle_ws", handle)
+ monkeypatch.setattr(web, "_DASHBOARD_EMBEDDED_CHAT_ENABLED", True)
+ monkeypatch.setattr(web_server_chat, "_ws_request_is_allowed", lambda ws: True)
+ api = GatewayRuntimeAPI(fallback, runner, web.app)
+
+ @asynccontextmanager
+ async def dial(url, *, subprotocols, **_):
+ closed = []
+
+ async def receive():
+ return {"type": "websocket.connect"}
+
+ async def send(message):
+ closed.append(message)
+ scope = {"type": "websocket", "path": "/api/ws", "scheme": "ws", "query_string": b"",
+ "client": ("127.0.0.1", 1234), "server": ("127.0.0.1", 1),
+ "headers": [(b"sec-websocket-protocol", ", ".join(subprotocols).encode())],
+ "subprotocols": subprotocols}
+ await api(scope, receive, send)
+ assert [(m["type"], m.get("code")) for m in closed] == [("websocket.close", 1013)], closed
+ raise OSError("owner refused the upgrade (1013)")
+ yield
+ monkeypatch.setattr("websockets.asyncio.client.connect", dial)
+
+
+async def test_a_restart_that_begins_after_discovery_still_refuses_the_worker_at_admission(tmp_path, monkeypatch):
+ _owner_rig(tmp_path, monkeypatch)
+ with pytest.raises(gateway_client.GatewayClientError, match="connection failed"), \
+ gateway_client.outside_restart_wait():
+ async with gateway_client.connect_gateway():
+ pytest.fail("attached inside the restart wait")
+
+ # An interactive client dialing the same race still attaches to answer a parked prompt (#139041).
+ _owner_rig(tmp_path, monkeypatch)
+ with pytest.raises(_Attached):
+ async with gateway_client.connect_gateway():
+ pass
+
+
+async def test_an_owner_older_than_the_policy_still_mints_the_worker_a_ticket(tmp_path, monkeypatch):
+ # The code on disk was updated under a running owner: the worker still attaches to it (the
+ # pre-dial check is all that owner can honour) instead of being stranded until it restarts.
+ _owner_rig(tmp_path, monkeypatch, old_owner=True)
+ with pytest.raises(_Attached), gateway_client.outside_restart_wait():
+ async with gateway_client.connect_gateway():
+ pass