The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a cancelled hot-serve reserve can no longer take the profile after cleanup
Teknium839415bd112b
diff --git a/gateway/run_profile_reconcile.py b/gateway/run_profile_reconcile.pyindex 59753a5c212..68434a1549a 100644--- a/gateway/run_profile_reconcile.py+++ b/gateway/run_profile_reconcile.py@@ -404,10 +404,10 @@ class GatewayProfileReconcileMixin:steps). Returns the park reason — reservation released — when another gateway owns the home(a stray per-profile daemon) or its store cannot be opened; None when served. Anadapters-only runner (no authority registry) grows the reservation alone."""- from gateway.run_runtime import release_profile_home, reserve_profile_home, serve_profile_runtime+ from gateway.run_runtime import release_profile_home, reserve_profile_home_in_worker, serve_profile_runtimefrom gateway.runtime_ownership import OwnershipConflicttry:- await asyncio.to_thread(reserve_profile_home, self, name, home)+ await reserve_profile_home_in_worker(self, name, home)except (OwnershipConflict, OSError) as exc:logger.error("[MULTIPLEX] Profile '%s' not served: %s", name, exc)return f"home owned by another gateway: {exc}"diff --git a/gateway/run_runtime.py b/gateway/run_runtime.pyindex 527c8867980..9157d68a43a 100644--- a/gateway/run_runtime.py+++ b/gateway/run_runtime.py@@ -12,8 +12,10 @@ is parked (logged, left out of ``served_profiles``) instead of aborting every otfrom __future__ import annotationsimport asyncio+from contextlib import suppressimport loggingfrom pathlib import Path+import threadingimport uuidlogger = logging.getLogger(__name__)@@ -232,21 +234,45 @@ def _publish_served_set(runner):runner.session_ticket_store.profile_ids = registry.profile_ids()-def reserve_profile_home(runner, name, home):+def reserve_profile_home(runner, name, home, cancel=None):"""Grow the process reservation by one profile (hot-serve): take its ``gateway.lock`` and add itto the frozen boot set, so every reader of the reservation — and the next restart'sall-or-nothing reserve — sees it. Waits out a transient holder within boot's window (blocking:- callers on the event loop run it in a worker thread). ``OwnershipConflict`` when another- gateway owns the home."""+ the event loop calls :func:`reserve_profile_home_in_worker`). ``OwnershipConflict`` when another+ gateway owns the home; *cancel* set ends the wait with nothing reserved."""from gateway.run_bootstrap import _browse_reconcile_reserve_wait_sfrom gateway.runtime_ownership import process_ownership, reserve_waiting_out_transient_holdershome = Path(home).resolve()- reserve_waiting_out_transient_holders(process_ownership, [home], wait_s=_browse_reconcile_reserve_wait_s())+ reserve_waiting_out_transient_holders(process_ownership, [home], wait_s=_browse_reconcile_reserve_wait_s(),+ cancel=cancel)reserved = getattr(runner.config, '_runtime_profile_homes', None)if reserved is not None and all(Path(h).resolve() != home for _n, h in reserved):runner.config._runtime_profile_homes = (*reserved, (name, home))+async def reserve_profile_home_in_worker(runner, name, home):+ """:func:`reserve_profile_home` off the event loop, settled on cancellation. A cancelled bare+ ``to_thread`` await leaves its worker retrying, free to take the flock after the canceller's+ ownership cleanup with no coroutine left to serve or release it. So a cancel stops the worker,+ waits for it to finish (each further cancel is absorbed until then), undoes a reservation it+ completed first, and only then propagates."""+ from gateway.runtime_ownership import process_ownership+ home = await asyncio.to_thread(Path(home).resolve)+ owned_before = process_ownership.owns(home)+ cancel = threading.Event()+ worker = asyncio.ensure_future(asyncio.to_thread(reserve_profile_home, runner, name, home, cancel))+ try:+ await asyncio.shield(worker)+ except asyncio.CancelledError:+ cancel.set()+ while not worker.done():+ with suppress(asyncio.CancelledError):+ await asyncio.wait({worker})+ if not worker.cancelled() and worker.exception() is None and not owned_before:+ release_profile_home(runner, home)+ raise++def release_profile_home(runner, home):"""Shrink the reservation by one profile (deleted, or parked because it cannot be served)."""from gateway.runtime_ownership import process_ownershipdiff --git a/gateway/runtime_ownership.py b/gateway/runtime_ownership.pyindex 00ae835a31a..41a80f60213 100644--- a/gateway/runtime_ownership.py+++ b/gateway/runtime_ownership.py@@ -24,6 +24,10 @@ class OwnershipConflict(RuntimeError):pass+class ReservationCancelled(OwnershipConflict):+ """The waiting reserve's caller gave up: nothing reserved, nothing left held."""++def canonical_home(home: Path) -> Path:return Path(os.path.normcase(str(Path(home).expanduser().resolve())))@@ -267,18 +271,25 @@ def _conflicting_homes(homes):return conflicting-def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float) -> None:+def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float,+ cancel: threading.Event | None = None) -> None:"""``ownership.reserve(homes)``, retried until *wait_s* passes while every contender istransient: a client's lock probe (gone by the re-check), a browse reconcile, or a holder that hasnot published its record yet (flock taken, record unwritten). A positively identified live owneron any home refuses at once. Every grant is a real ``reserve``: the flock is never bypassed.- Shared by the gateway boot and the running multiplexer's hot-serve of a new profile."""+ Shared by the gateway boot and the running multiplexer's hot-serve of a new profile.++ *cancel* set (a worker thread's caller gave up) ends the wait at once and releases a grant+ that landed meanwhile, raising :class:`ReservationCancelled`: the caller may already have run+ its ownership cleanup, so nobody would ever release a home taken after it."""from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE+ cancel = threading.Event() if cancel is None else canceldeadline = time.monotonic() + wait_slogged = False- while True:+ while not cancel.is_set():+ before = set(ownership.homes)try:- return ownership.reserve(homes)+ ownership.reserve(homes)except OwnershipConflict:# Only the homes that actually conflict decide: a real owner on any of them refuses at# once, even while another home is mid-browse (multiplexer boot, all-or-nothing).@@ -288,13 +299,19 @@ def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float) ->if len(holders) < len(conflicting) or time.monotonic() >= deadline:raiseif not conflicting:- time.sleep(PROBE_RETRY_BACKOFF_S) # the holder already let go: a lock probe, never an owner+ cancel.wait(PROBE_RETRY_BACKOFF_S) # the holder already let go: a lock probe, never an ownercontinueif not logged:logger.info("Profile %s is held by a short session-store schema catch-up; waiting up to %.0fs",holders[0], wait_s)logged = True- time.sleep(TRANSIENT_HOLDER_POLL_S)+ cancel.wait(TRANSIENT_HOLDER_POLL_S)+ continue+ if not cancel.is_set():+ return+ for home in {canonical_home(p) for p in homes} - before:+ ownership.release(home)+ raise ReservationCancelled(f'Reservation of {", ".join(map(str, homes))} cancelled')def live_runtime_owner(home: Path) -> int | None:diff --git a/tests/gateway/test_boot_reserve_lock_probe.py b/tests/gateway/test_boot_reserve_lock_probe.pyindex 99aca55df3d..b23054804a2 100644--- a/tests/gateway/test_boot_reserve_lock_probe.py+++ b/tests/gateway/test_boot_reserve_lock_probe.py@@ -12,6 +12,7 @@ import fcntlfrom pathlib import Pathimport subprocessimport sys+import threadingimport timefrom types import SimpleNamespace@@ -219,3 +220,46 @@ def test_hot_serve_still_parks_a_profile_another_gateway_owns(tmp_path):assert time.monotonic() - started < _browse_reconcile_reserve_wait_s() / 2assert not process_ownership.owns(fresh)assert runner.config._runtime_profile_homes == (("default", root),)+++def test_cancelled_hot_serve_never_takes_the_profile_after_cleanup(tmp_path, monkeypatch):+ """The gateway stop cancels the reconcile mid-wait, then releases ownership. The reservation+ worker must not outlive that cancel: a worker still retrying took the flock once the holder let+ go, after cleanup, with no coroutine left to serve or release it."""+ from gateway import runtime_ownership++ root, fresh = _homes(tmp_path)+ retrying, worker_done = threading.Event(), threading.Event()+ real_conflicts = runtime_ownership._conflicting_homes+ real_reserve = runtime_ownership.reserve_waiting_out_transient_holders++ def conflicts(homes):+ found = real_conflicts(homes)+ retrying.set() # the worker refused once and is waiting the holder out+ return found++ def reserve(*args, **kwargs):+ try:+ return real_reserve(*args, **kwargs)+ finally:+ worker_done.set()++ monkeypatch.setattr(runtime_ownership, "_conflicting_homes", conflicts)+ monkeypatch.setattr(runtime_ownership, "reserve_waiting_out_transient_holders", reserve)++ async def scenario(probe):+ task = asyncio.ensure_future(_HotServeRunner(root)._serve_profile_runtime("boot", fresh))+ assert await asyncio.to_thread(retrying.wait, 30)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ process_ownership.release(fresh) # the canceller's ownership cleanup+ _release(probe) # the holder lets go after cleanup+ assert await asyncio.to_thread(worker_done.wait, 30)++ try:+ with _holder("probe", fresh) as probe:+ asyncio.run(scenario(probe))+ assert not process_ownership.owns(fresh)+ finally:+ process_ownership.release(fresh)