Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): start the engine ready only once its OAuth grants list their models (#2823)
Call Whitea0f9a20c3ef3
diff --git a/docs/plans/model-hub.md b/docs/plans/model-hub.mdindex 293196a1b..a063a0b6a 100644--- a/docs/plans/model-hub.md+++ b/docs/plans/model-hub.md@@ -1996,6 +1996,16 @@ both health surfaces in 7.4 seconds. It adds bounded cold-start margin withoutclaiming an unobserved internal engine phase or introducing a retry loop. Ifreadiness is not established, Avibe terminates the child and preserves theexisting runtime failure contract.+The engine answers before its file watcher has registered the auth directory's+OAuth grants, and until then lists each grant with no model and routes none of+them (#2756). So after the health check a start also waits, for at most the+full 5-second config reload bound, until every grant the engine loaded (except a+disabled one) lists a model. The wait has two outcomes. A listing the engine+refuses as a credential or route it does not hold (401/403/404), or an engine+that exited, fails the start. Anything else is ready once every grant lists a+model or the bound expires; grants still unlisted and failed or malformed+listings are logged as a degraded listing. API-key Sources are registered before+the engine answers.CLIProxyAPI stdout and stderr are directed to the operating system's null sinkwhen the process is created. Child bytes therefore never enter Avibe's Pythonmemory or service logs and cannot create pipe backpressure. Startup diagnosticsdiff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 1feaf95f5..27c491769 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -1155,11 +1155,13 @@ scenarios:missing_layer: scenariotest: tests/test_model_hub_runtime.py::test_mh_runtime_001_service_restart_reports_installed_engine_as_not_started- id: MH-RUNTIME-005- name: First CLIProxyAPI cold start remains pending until both health surfaces are ready+ name: First CLIProxyAPI cold start remains pending until both health surfaces are ready and, within a bound, each OAuth grant it loaded lists its modelsstatus: coveredkind: recoverylayer: unittest: tests/test_model_hub_runtime.py::test_mh_runtime_005_first_cold_start_waits_for_readiness_within_bounded_budget+ related_tests:+ - tests/test_model_hub_runtime.py::test_a_started_engine_is_ready_once_it_lists_the_models_of_each_oauth_grant_it_loaded- id: MH-RUNTIME-006name: CLIProxyAPI readiness timeout stops the child, discards its output, and leaves bounded structured diagnosticsstatus: covereddiff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.pyindex 7b16c849e..ee8623a4a 100644--- a/tests/test_model_hub_runtime.py+++ b/tests/test_model_hub_runtime.py@@ -3055,6 +3055,11 @@ def _write_mock_engine(reload_delay: float = 0.0,drop_reload_ack: bool = False,lose_config_on_reload: str | None = None,+ grants_load_after: float = 0.0,+ grant_listing_delay: float = 0.0,+ auth_files_status: int = 200,+ auth_files_malformed: bool = False,+ grant_models_malformed: bool = False,) -> None:script = f"""#!{sys.executable}import json@@ -3089,6 +3094,7 @@ exit_before_ready = {exit_before_ready!r}if exit_before_ready is not None:raise SystemExit(exit_before_ready)time.sleep({startup_delay!r})+answering_since = time.monotonic()health_surfaces = set()defaulted = False@@ -3133,8 +3139,17 @@ class Handler(BaseHTTPRequestHandler):mark_health_surface('management')self._json(200, {{'host': config['host']}})return+ if self.path == '/v0/management/auth-files' and self.headers.get('X-Management-Key') == management:+ grants = sorted(os.listdir(config['auth-dir'])) if os.path.isdir(config['auth-dir']) else []+ files = [{{'name': name}} for name in grants if name.endswith('.json')]+ self._json({auth_files_status!r}, {{'files': {{}} if {auth_files_malformed!r} else files}})+ returnif self.path.startswith('/v0/management/auth-files/models'):- self._json(200, {{'models': [{{'id': 'model-a'}}]}})+ time.sleep({grant_listing_delay!r})+ # CPA answers before its watcher loads the auth directory, listing each grant with no model until then.+ loaded = time.monotonic() - answering_since >= {grants_load_after!r}+ models = ([{{}}] if {grant_models_malformed!r} else [{{'id': 'model-a'}}]) if loaded else []+ self._json(200, {{'models': models}})returnself._json(401, {{'error': {{'type': 'unauthorized'}}}})@@ -3312,6 +3327,11 @@ def _fixture_supervisor(reload_delay: float = 0.0,drop_reload_ack: bool = False,lose_config_on_reload: str | None = None,+ grants_load_after: float = 0.0,+ grant_listing_delay: float = 0.0,+ auth_files_status: int = 200,+ auth_files_malformed: bool = False,+ grant_models_malformed: bool = False,) -> tuple[EngineSupervisor, EngineStateStore]:tmp_path.mkdir(parents=True, exist_ok=True)binary = tmp_path / "mock-engine"@@ -3325,6 +3345,11 @@ def _fixture_supervisor(reload_delay=reload_delay,drop_reload_ack=drop_reload_ack,lose_config_on_reload=lose_config_on_reload,+ grants_load_after=grants_load_after,+ grant_listing_delay=grant_listing_delay,+ auth_files_status=auth_files_status,+ auth_files_malformed=auth_files_malformed,+ grant_models_malformed=grant_models_malformed,)installer = _FixtureInstaller(binary, tmp_path / "versions" / "install-1")store = EngineStateStore(tmp_path / "state")@@ -4665,6 +4690,91 @@ def test_supervisor_process_exit_discards_all_child_output(assert not (instance_dir / "health-surfaces-complete").exists()+def test_a_started_engine_is_ready_once_it_lists_the_models_of_each_oauth_grant_it_loaded(tmp_path: Path) -> None:+ """MH-RUNTIME-005 (#2756): the engine answers before it loads its auth directory, and lists each grant with no+ model until then; a refresh or turn right after the start must not find a grant that offers nothing."""++ supervisor, store = _fixture_supervisor(tmp_path, grants_load_after=0.5)+ auth_name = "antigravity-account.json"+ store.write_oauth_auth_file(auth_name, {"type": "antigravity", "access_token": "oauth-fixture"})+ try:+ connection = supervisor.ensure_running()+ listing = EngineClient(connection).management_request("GET", "/auth-files/models", query={"name": auth_name})+ assert listing["models"] == [{"id": "model-a"}]+ finally:+ supervisor.stop()+++def test_the_wait_for_oauth_grants_is_bounded_as_a_whole_however_many_grants_answer_slowly(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch,+) -> None:+ """MH-RUNTIME-005 (#2756): one deadline covers every grant request, so slow grants cannot stretch the start."""++ monkeypatch.setattr(supervisor_module, "MODEL_HUB_CONFIG_RELOAD_TIMEOUT_SECONDS", 1.0)+ supervisor, store = _fixture_supervisor(tmp_path, grants_load_after=60.0, grant_listing_delay=0.4)+ for number in range(12):+ store.write_oauth_auth_file(f"account-{number}.json", {"type": "antigravity", "access_token": "oauth-fixture"})+ try:+ started = time.monotonic()+ supervisor.ensure_running()+ # One pass over twelve grants at 0.4 s each would take 4.8 s; the start's own spawn and health check get the rest.+ assert time.monotonic() - started < 3.5+ finally:+ supervisor.stop()+++ ("listing", "degraded"),+ [+ ({"auth_files_status": 404}, None),+ ({"auth_files_status": 401}, None),+ ({"auth_files_status": 503}, "failure=503"),+ ({"auth_files_malformed": True}, "failure=malformed_inventory"),+ ({"grant_models_malformed": True}, "failure=malformed_listing"),+ ],+ ids=["disowned-404", "disowned-401", "degraded-503", "degraded-malformed-inventory", "degraded-malformed-models"],+)+def test_a_failed_grant_listing_fails_the_start_only_when_the_engine_disowns_its_config(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture,+ listing: dict, degraded: str | None,+) -> None:+ """MH-RUNTIME-005 (#2756): an engine that answers its health check but refuses the grant listing as a credential+ or route it does not hold never starts as ready, and is stopped. Any other failure, a malformed inventory or+ model listing included, is a degraded listing, not a failed engine: the start polls it through the bound, says+ why, and is ready."""++ monkeypatch.setattr(supervisor_module, "MODEL_HUB_CONFIG_RELOAD_TIMEOUT_SECONDS", 1.5)+ caplog.set_level(logging.WARNING, logger="vibe.model_hub_runtime.supervisor")+ supervisor, store = _fixture_supervisor(tmp_path, startup_timeout=2.0, **listing)+ store.write_oauth_auth_file("antigravity-account.json", {"type": "antigravity", "access_token": "oauth-fixture"})+ try:+ if degraded is not None:+ supervisor.ensure_running()+ assert supervisor.status()["status"]["health"] == "ok"+ assert any(degraded in record.getMessage() for record in caplog.records), caplog.text+ else:+ with pytest.raises(EngineUnavailableError):+ supervisor.ensure_running()+ assert supervisor.status()["status"]["health"] != "ok"+ finally:+ supervisor.stop()+++def test_a_start_ready_late_in_its_budget_still_waits_the_full_bound_for_its_grants(tmp_path: Path) -> None:+ """MH-RUNTIME-005 (#2756): the grant wait always gets its full bound after the health check, however little of+ the startup budget is left, so a start that is healthy late still waits for its grants."""++ supervisor, store = _fixture_supervisor(tmp_path, startup_timeout=3.0, startup_delay=2.0, grants_load_after=1.5)+ auth_name = "antigravity-account.json"+ store.write_oauth_auth_file(auth_name, {"type": "antigravity", "access_token": "oauth-fixture"})+ try:+ connection = supervisor.ensure_running()+ listing = EngineClient(connection).management_request("GET", "/auth-files/models", query={"name": auth_name})+ assert listing["models"] == [{"id": "model-a"}]+ finally:+ supervisor.stop()++def test_mh_runtime_006_timeout_stops_engine_with_bounded_structured_diagnostics(tmp_path: Path,caplog: pytest.LogCaptureFixture,diff --git a/vibe/model_hub_runtime/supervisor.py b/vibe/model_hub_runtime/supervisor.pyindex d4c62d741..570161edc 100644--- a/vibe/model_hub_runtime/supervisor.py+++ b/vibe/model_hub_runtime/supervisor.py@@ -103,6 +103,17 @@ class EngineConfigLostError(EngineUnavailableError):super().__init__("models.engine.health_failed", reason="engine_config_lost")+def _lists_a_model(listing: dict[str, Any]) -> bool:+ """Whether a grant's listing holds a model the discovery reader (``adapter._discovered_models``) reads as one."""++ models = listing.get("models")+ return isinstance(models, list) and any(+ isinstance(value := (item.get("id") or item.get("alias") or item.get("name")) if isinstance(item, dict) else item, str)+ and bool(value)+ for item in models+ )++class EngineSupervisor:"""Start-on-demand supervisor for one loopback-only Model Hub engine."""@@ -587,6 +598,9 @@ class EngineSupervisor:except Exception as exc:self._stop_locked()raise EngineUnavailableError("models.engine.unsafe_permissions") from exc+ if not self._oauth_grants_listed_locked(connection) or process.poll() is not None:+ time.sleep(min(_STARTUP_POLL_INTERVAL_SECONDS, max(0.0, deadline - time.monotonic())))+ continueself._last_check = _utc_now()self._verified_at = time.monotonic()self._engine_answered = True@@ -612,6 +626,83 @@ class EngineSupervisor:)raise EngineUnavailableError("models.engine.health_failed", os_errno=os_errno)+ def _oauth_grants_listed_locked(self, connection: EngineConnection) -> bool:+ """Wait, bounded, until a started engine lists a model for each OAuth grant it loaded.++ CPA answers before its watcher has registered the auth directory's+ grants, and until then lists each with no model and routes none of its+ models: a refresh or turn in that window finds a grant that offers+ nothing (#2756). API-key Sources are registered before it answers.++ The wait has exactly two outcomes:++ * Not ready (False): the engine refused a listing as a credential or+ route it does not hold (``_disowns_config_locked``). The caller also+ treats an engine that exited as not ready. Either way the start fails+ like any start that never becomes ready.+ * Ready (True): every loaded grant lists a model the discovery reader+ can read, or the reload bound expired, after one log line naming why.+ Grants left unlisted, and listings that failed any other way or+ answered malformed, are a degraded listing, not a failed engine.++ The bound is that watcher's reload bound. It always runs in full after+ the health check, and covers every request in the wait.+ """++ client = EngineClient(connection)+ deadline = time.monotonic() + MODEL_HUB_CONFIG_RELOAD_TIMEOUT_SECONDS+ failure: str | None = None++ def listing(path: str, query: dict[str, str] | None = None) -> dict[str, Any] | None:+ nonlocal failure+ try:+ return client.management_request(+ "GET", path, query=query, timeout=min(1.0, max(0.01, deadline - time.monotonic())),+ )+ except EngineClientError as exc:+ if self._disowns_config_locked(exc.status_code, local_os_errno(exc)):+ raise+ # The request the deadline cut short times out whatever the engine does: an answer says more.+ if failure is None or exc.error_type != "TimeoutError":+ failure = str(exc.status_code or exc.error_type)+ return None++ pending: list[str] | None = None+ try:+ while deadline - time.monotonic() > 0:+ if pending is None and (inventory := listing("/auth-files")) is not None:+ files = inventory.get("files")+ if isinstance(files, list):+ pending = sorted(+ item["name"] for item in files+ if isinstance(item, dict) and isinstance(item.get("name"), str) and item["name"]+ and item.get("disabled") is not True+ )+ else:+ failure = "malformed_inventory"+ listed = set()+ for name in pending or ():+ if deadline - time.monotonic() <= 0:+ break+ models = listing("/auth-files/models", {"name": name})+ if models is not None and _lists_a_model(models):+ listed.add(name)+ elif models is not None and models.get("models"):+ failure = "malformed_listing"+ if pending is not None:+ pending = [name for name in pending if name not in listed]+ if not pending:+ return True+ time.sleep(min(_STARTUP_POLL_INTERVAL_SECONDS, max(0.0, deadline - time.monotonic())))+ except EngineClientError:+ return False+ logger.warning(+ "Model Hub engine startup: OAuth grant listing degraded after %.1fs: "+ "inventory_read=%s grants_without_models=%d failure=%s",+ MODEL_HUB_CONFIG_RELOAD_TIMEOUT_SECONDS, pending is not None, len(pending or ()), failure,+ )+ return True+def _healthy_locked(self) -> bool:if not self._is_running_locked() or self._connection is None:return False