Skip to content

The agent that grows with you

fix(gateway): boot waits out a browse schema catch-up instead of exiting

Teknium1e23addffcac
diff --git a/gateway/run_bootstrap.py b/gateway/run_bootstrap.py
index b32d9c862f6..25aa6997c88 100644
--- a/gateway/run_bootstrap.py
+++ b/gateway/run_bootstrap.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import logging
import threading
+import time
from pathlib import Path
from typing import Any, Optional
@@ -544,12 +545,38 @@ def _start_gateway_resolve_terminal_cwd() -> None:
os.environ["TERMINAL_CWD"] = resolved_cwd
+# A browse that catches a lagging state.db up holds the profile's gateway.lock for at most its
+# sqlite busy budget (hermes_state_reconcile._BROWSE_RECONCILE_BUSY_S, 2 s). A gateway booting in
+# that window waits it out instead of exiting as if another gateway owned the profile.
+_BROWSE_RECONCILE_RESERVE_WAIT_S = 5.0
+
+
+def _reserve_waiting_out_browse_reconcile(ownership, homes, *, wait_s: float = _BROWSE_RECONCILE_RESERVE_WAIT_S):
+ from gateway.runtime_ownership import OwnershipConflict, maintenance_holder
+ from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
+ deadline = time.monotonic() + wait_s
+ logged = False
+ while True:
+ try:
+ return ownership.reserve(homes)
+ except OwnershipConflict:
+ holders = [h for h in homes if maintenance_holder(h, BROWSE_RECONCILE_PURPOSE)]
+ if not holders or time.monotonic() >= deadline:
+ raise
+ if not logged:
+ logger.info("Profile %s is held by a short session-store schema catch-up; waiting up to %.0fs",
+ holders[0], wait_s)
+ logged = True
+ time.sleep(0.1)
+
+
def _start_gateway_reserve_profiles(profile_homes) -> bool:
"""Reserve the launch home plus every served profile home; False when startup must abort."""
from gateway.run import get_hermes_home
from gateway.runtime_ownership import process_ownership, OwnershipConflict
+ homes = [get_hermes_home(), *(home for _, home in profile_homes)]
try:
- process_ownership.reserve([get_hermes_home(), *(home for _, home in profile_homes)])
+ _reserve_waiting_out_browse_reconcile(process_ownership, homes)
except OwnershipConflict as exc:
logger.error("Cannot reserve gateway profiles: %s", exc)
# A named profile that the LIVE default multiplexer already serves is a configuration
diff --git a/gateway/runtime_ownership.py b/gateway/runtime_ownership.py
index 33b678d4cd1..f4701dcaff4 100644
--- a/gateway/runtime_ownership.py
+++ b/gateway/runtime_ownership.py
@@ -216,6 +216,17 @@ process_ownership = ProfileOwnership()
_maintenance = threading.local()
+def maintenance_holder(home: Path, purpose: str) -> dict | None:
+ """The live holder record of *home*'s ``gateway.lock`` when it is a ``purpose`` maintenance
+ reservation (see :meth:`ProfileOwnership.reserve`), else None. Advisory only: it names the
+ holder for a refusal or a short wait, it never grants the lock."""
+ from gateway.status import _live_pid_from_record, _read_gateway_lock_record
+ record = _read_gateway_lock_record(canonical_home(home) / 'gateway.lock')
+ if not record or record.get('purpose') != purpose or _live_pid_from_record(record) is None:
+ return None
+ return record
+
+
def live_runtime_owner(home: Path) -> int | None:
"""PID of a live runtime owner of *home* (a gateway, or a reservation that is not marked as
maintenance), else None. A purpose-marked maintenance holder is never an owner."""
diff --git a/tests/hermes_cli/test_browse_schema_drift.py b/tests/hermes_cli/test_browse_schema_drift.py
index f47690f0e05..950fbfba59f 100644
--- a/tests/hermes_cli/test_browse_schema_drift.py
+++ b/tests/hermes_cli/test_browse_schema_drift.py
@@ -410,3 +410,39 @@ def test_stuck_maintenance_holder_gets_busy_retry_not_a_gateway_remedy(tmp_path)
assert "Retry" in detail
assert "gateway" not in detail.lower()
assert _file_state(path) == before
+
+
[email protected]("linux", "macos")
+def test_gateway_boot_reserve_waits_out_a_browse_reconcile(tmp_path):
+ """P3-1: a gateway booting while a browse reconcile holds gateway.lock waits for it (bounded)
+ instead of exiting as though another gateway owned the profile. A real owner still refuses."""
+ import time
+
+ from gateway.run_bootstrap import _reserve_waiting_out_browse_reconcile
+ from gateway.runtime_ownership import OwnershipConflict, ProfileOwnership
+ from hermes_state_reconcile import BROWSE_RECONCILE_PURPOSE
+
+ home = _profile_home(tmp_path)
+ holder = _hold(home, 1.0, BROWSE_RECONCILE_PURPOSE)
+ booting = ProfileOwnership()
+ try:
+ started = time.monotonic()
+ _reserve_waiting_out_browse_reconcile(booting, [home], wait_s=10)
+ assert booting.owns(home)
+ assert time.monotonic() - started < 9
+ finally:
+ booting.close()
+ holder.kill()
+ holder.wait()
+
+ gateway = _hold(home, 30) # an unmarked holder is a runtime owner: refuse at once
+ contender = ProfileOwnership()
+ try:
+ started = time.monotonic()
+ with pytest.raises(OwnershipConflict):
+ _reserve_waiting_out_browse_reconcile(contender, [home], wait_s=10)
+ assert time.monotonic() - started < 2
+ finally:
+ contender.close()
+ gateway.kill()
+ gateway.wait()