Skip to content

The agent that grows with you

fix(gateway): a cancelled hot-serve reserve can no longer take the profile after cleanup

Teknium839415bd112b
diff --git a/gateway/run_profile_reconcile.py b/gateway/run_profile_reconcile.py
index 59753a5c212..68434a1549a 100644
--- a/gateway/run_profile_reconcile.py
+++ b/gateway/run_profile_reconcile.py
@@ -404,10 +404,10 @@ class GatewayProfileReconcileMixin:
steps). Returns the park reason — reservation released — when another gateway owns the home
(a stray per-profile daemon) or its store cannot be opened; None when served. An
adapters-only runner (no authority registry) grows the reservation alone."""
- from gateway.run_runtime import release_profile_home, reserve_profile_home, serve_profile_runtime
+ from gateway.run_runtime import release_profile_home, reserve_profile_home_in_worker, serve_profile_runtime
from gateway.runtime_ownership import OwnershipConflict
try:
- await asyncio.to_thread(reserve_profile_home, self, name, home)
+ await reserve_profile_home_in_worker(self, name, home)
except (OwnershipConflict, OSError) as exc:
logger.error("[MULTIPLEX] Profile '%s' not served: %s", name, exc)
return f"home owned by another gateway: {exc}"
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index 527c8867980..9157d68a43a 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -12,8 +12,10 @@ is parked (logged, left out of ``served_profiles``) instead of aborting every ot
from __future__ import annotations
import asyncio
+from contextlib import suppress
import logging
from pathlib import Path
+import threading
import uuid
logger = logging.getLogger(__name__)
@@ -232,21 +234,45 @@ def _publish_served_set(runner):
runner.session_ticket_store.profile_ids = registry.profile_ids()
-def reserve_profile_home(runner, name, home):
+def reserve_profile_home(runner, name, home, cancel=None):
"""Grow the process reservation by one profile (hot-serve): take its ``gateway.lock`` and add it
to the frozen boot set, so every reader of the reservation — and the next restart's
all-or-nothing reserve — sees it. Waits out a transient holder within boot's window (blocking:
- callers on the event loop run it in a worker thread). ``OwnershipConflict`` when another
- gateway owns the home."""
+ the event loop calls :func:`reserve_profile_home_in_worker`). ``OwnershipConflict`` when another
+ gateway owns the home; *cancel* set ends the wait with nothing reserved."""
from gateway.run_bootstrap import _browse_reconcile_reserve_wait_s
from gateway.runtime_ownership import process_ownership, reserve_waiting_out_transient_holders
home = Path(home).resolve()
- reserve_waiting_out_transient_holders(process_ownership, [home], wait_s=_browse_reconcile_reserve_wait_s())
+ reserve_waiting_out_transient_holders(process_ownership, [home], wait_s=_browse_reconcile_reserve_wait_s(),
+ cancel=cancel)
reserved = getattr(runner.config, '_runtime_profile_homes', None)
if reserved is not None and all(Path(h).resolve() != home for _n, h in reserved):
runner.config._runtime_profile_homes = (*reserved, (name, home))
+async def reserve_profile_home_in_worker(runner, name, home):
+ """:func:`reserve_profile_home` off the event loop, settled on cancellation. A cancelled bare
+ ``to_thread`` await leaves its worker retrying, free to take the flock after the canceller's
+ ownership cleanup with no coroutine left to serve or release it. So a cancel stops the worker,
+ waits for it to finish (each further cancel is absorbed until then), undoes a reservation it
+ completed first, and only then propagates."""
+ from gateway.runtime_ownership import process_ownership
+ home = await asyncio.to_thread(Path(home).resolve)
+ owned_before = process_ownership.owns(home)
+ cancel = threading.Event()
+ worker = asyncio.ensure_future(asyncio.to_thread(reserve_profile_home, runner, name, home, cancel))
+ try:
+ await asyncio.shield(worker)
+ except asyncio.CancelledError:
+ cancel.set()
+ while not worker.done():
+ with suppress(asyncio.CancelledError):
+ await asyncio.wait({worker})
+ if not worker.cancelled() and worker.exception() is None and not owned_before:
+ release_profile_home(runner, home)
+ raise
+
+
def release_profile_home(runner, home):
"""Shrink the reservation by one profile (deleted, or parked because it cannot be served)."""
from gateway.runtime_ownership import process_ownership
diff --git a/gateway/runtime_ownership.py b/gateway/runtime_ownership.py
index 00ae835a31a..41a80f60213 100644
--- a/gateway/runtime_ownership.py
+++ b/gateway/runtime_ownership.py
@@ -24,6 +24,10 @@ class OwnershipConflict(RuntimeError):
pass
+class ReservationCancelled(OwnershipConflict):
+ """The waiting reserve's caller gave up: nothing reserved, nothing left held."""
+
+
def canonical_home(home: Path) -> Path:
return Path(os.path.normcase(str(Path(home).expanduser().resolve())))
@@ -267,18 +271,25 @@ def _conflicting_homes(homes):
return conflicting
-def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float) -> None:
+def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float,
+ cancel: threading.Event | None = None) -> None:
"""``ownership.reserve(homes)``, retried until *wait_s* passes while every contender is
transient: a client's lock probe (gone by the re-check), a browse reconcile, or a holder that has
not published its record yet (flock taken, record unwritten). A positively identified live owner
on any home refuses at once. Every grant is a real ``reserve``: the flock is never bypassed.
- Shared by the gateway boot and the running multiplexer's hot-serve of a new profile."""
+ Shared by the gateway boot and the running multiplexer's hot-serve of a new profile.
+
+ *cancel* set (a worker thread's caller gave up) ends the wait at once and releases a grant
+ that landed meanwhile, raising :class:`ReservationCancelled`: the caller may already have run
+ its ownership cleanup, so nobody would ever release a home taken after it."""
from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
+ cancel = threading.Event() if cancel is None else cancel
deadline = time.monotonic() + wait_s
logged = False
- while True:
+ while not cancel.is_set():
+ before = set(ownership.homes)
try:
- return ownership.reserve(homes)
+ ownership.reserve(homes)
except OwnershipConflict:
# Only the homes that actually conflict decide: a real owner on any of them refuses at
# once, even while another home is mid-browse (multiplexer boot, all-or-nothing).
@@ -288,13 +299,19 @@ def reserve_waiting_out_transient_holders(ownership, homes, *, wait_s: float) ->
if len(holders) < len(conflicting) or time.monotonic() >= deadline:
raise
if not conflicting:
- time.sleep(PROBE_RETRY_BACKOFF_S) # the holder already let go: a lock probe, never an owner
+ cancel.wait(PROBE_RETRY_BACKOFF_S) # the holder already let go: a lock probe, never an owner
continue
if not logged:
logger.info("Profile %s is held by a short session-store schema catch-up; waiting up to %.0fs",
holders[0], wait_s)
logged = True
- time.sleep(TRANSIENT_HOLDER_POLL_S)
+ cancel.wait(TRANSIENT_HOLDER_POLL_S)
+ continue
+ if not cancel.is_set():
+ return
+ for home in {canonical_home(p) for p in homes} - before:
+ ownership.release(home)
+ raise ReservationCancelled(f'Reservation of {", ".join(map(str, homes))} cancelled')
def live_runtime_owner(home: Path) -> int | None:
diff --git a/tests/gateway/test_boot_reserve_lock_probe.py b/tests/gateway/test_boot_reserve_lock_probe.py
index 99aca55df3d..b23054804a2 100644
--- a/tests/gateway/test_boot_reserve_lock_probe.py
+++ b/tests/gateway/test_boot_reserve_lock_probe.py
@@ -12,6 +12,7 @@ import fcntl
from pathlib import Path
import subprocess
import sys
+import threading
import time
from types import SimpleNamespace
@@ -219,3 +220,46 @@ def test_hot_serve_still_parks_a_profile_another_gateway_owns(tmp_path):
assert time.monotonic() - started < _browse_reconcile_reserve_wait_s() / 2
assert not process_ownership.owns(fresh)
assert runner.config._runtime_profile_homes == (("default", root),)
+
+
+def test_cancelled_hot_serve_never_takes_the_profile_after_cleanup(tmp_path, monkeypatch):
+ """The gateway stop cancels the reconcile mid-wait, then releases ownership. The reservation
+ worker must not outlive that cancel: a worker still retrying took the flock once the holder let
+ go, after cleanup, with no coroutine left to serve or release it."""
+ from gateway import runtime_ownership
+
+ root, fresh = _homes(tmp_path)
+ retrying, worker_done = threading.Event(), threading.Event()
+ real_conflicts = runtime_ownership._conflicting_homes
+ real_reserve = runtime_ownership.reserve_waiting_out_transient_holders
+
+ def conflicts(homes):
+ found = real_conflicts(homes)
+ retrying.set() # the worker refused once and is waiting the holder out
+ return found
+
+ def reserve(*args, **kwargs):
+ try:
+ return real_reserve(*args, **kwargs)
+ finally:
+ worker_done.set()
+
+ monkeypatch.setattr(runtime_ownership, "_conflicting_homes", conflicts)
+ monkeypatch.setattr(runtime_ownership, "reserve_waiting_out_transient_holders", reserve)
+
+ async def scenario(probe):
+ task = asyncio.ensure_future(_HotServeRunner(root)._serve_profile_runtime("boot", fresh))
+ assert await asyncio.to_thread(retrying.wait, 30)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ process_ownership.release(fresh) # the canceller's ownership cleanup
+ _release(probe) # the holder lets go after cleanup
+ assert await asyncio.to_thread(worker_done.wait, 30)
+
+ try:
+ with _holder("probe", fresh) as probe:
+ asyncio.run(scenario(probe))
+ assert not process_ownership.owns(fresh)
+ finally:
+ process_ownership.release(fresh)