Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): restore an engine whose own reload tore its config, as the save that wrote it (#2774)
Call White5e52250613cc
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 70ffacbac..348f23785 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -1186,7 +1186,8 @@ scenarios:test: tests/test_model_hub_retry_lifecycle.py::test_a_silent_engine_stream_is_cancelled_and_retried_at_its_idle_bound- id: MH-RUNTIME-014name: >-- An engine that refuses the credentials Avibe launched it with is reported once as having lost its configuration,+ An engine that no longer runs the config Avibe launched it with (it refuses Avibe's credentials, serves no+ management route, or refuses connections on its port, as after its own torn reload) is reported once as having lost its configuration,admits no new work, and restarts from the rendered config only after its open streams end; the waiting save orturn then runs on the restarted enginestatus: covered@@ -1195,6 +1196,7 @@ scenarios:test:- tests/test_model_hub_transport_leases.py::test_an_engine_that_lost_its_configuration_restarts_only_once_its_streams_end- tests/test_model_hub_runtime.py::test_an_engine_that_lost_its_configuration_is_reported_once_and_restarted_by_its_drainer+ - tests/test_model_hub_runtime.py::test_a_source_save_restores_an_engine_whose_reload_tore_its_config- id: MH-RUNTIME-015name: >-A pre-output 401 is the engine's own only when its engine-local model listing refuses the same gateway key;diff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.pyindex ee128de17..ab5a8d00a 100644--- a/tests/test_model_hub_runtime.py+++ b/tests/test_model_hub_runtime.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import errnoimport hashlibimport ioimport json@@ -56,7 +57,13 @@ from vibe.model_hub_runtime import installer as runtime_installer_modulefrom vibe.model_hub_runtime import supervisor as supervisor_modulefrom vibe.model_hub_runtime.adapter import CLIProxyEngineAdapterfrom vibe.model_hub_runtime.api_key_vendors import api_key_vendor_catalog, official_models_dev_provider-from vibe.model_hub_runtime.client import EngineClient, EngineClientError, EngineConnection, completed_handle+from vibe.model_hub_runtime.client import (+ EngineClient,+ EngineClientError,+ EngineConnection,+ completed_handle,+ engine_disowned_call,+)from vibe.model_hub_runtime.config import expected_model_names, write_engine_configfrom vibe.model_hub_runtime.environment import engine_subprocess_environmentfrom vibe.model_hub_runtime.installer import (@@ -2999,7 +3006,7 @@ def _write_mock_engine(exit_before_ready: int | None = None,reload_delay: float = 0.0,drop_reload_ack: bool = False,- lose_config_on_reload: bool = False,+ lose_config_on_reload: str | None = None,) -> None:script = f"""#!{sys.executable}import json@@ -3035,6 +3042,7 @@ if exit_before_ready is not None:raise SystemExit(exit_before_ready)time.sleep({startup_delay!r})health_surfaces = set()+defaulted = Falsedef configured_models():for section in ('claude-api-key', 'codex-api-key', 'openai-compatibility'):@@ -3061,6 +3069,14 @@ class Handler(BaseHTTPRequestHandler):self.wfile.write(body)def do_GET(self):+ if defaulted:+ # Its defaults hold no api-keys, so it lists to anyone, and no+ # management secret, so it serves no management routes.+ if self.path == '/v1/models':+ self._json(200, {{'object': 'list', 'data': []}})+ else:+ self._json(404, {{'error': 'not found'}})+ returnif self.path == '/v1/models' and self.headers.get('Authorization') == f'Bearer {{gateway}}':mark_health_surface('gateway')self._json(200, {{'object': 'list', 'data': [{{'id': model, 'name': name}} for model, name in configured_models()]}})@@ -3075,22 +3091,31 @@ class Handler(BaseHTTPRequestHandler):self._json(401, {{'error': {{'type': 'unauthorized'}}}})def do_PUT(self):- global config+ global config, defaultedlength = int(self.headers.get('Content-Length', '0'))body = self.rfile.read(length)+ if defaulted:+ self._json(404, {{'error': 'not found'}})+ returnif self.path != '/v0/management/config.yaml' or self.headers.get('X-Management-Key') != management:self._json(401, {{'error': {{'type': 'unauthorized'}}}})returnwith open(config_path, 'wb') as handle:handle.write(body)+ loss = {lose_config_on_reload!r}+ torn = loss is not None and not os.path.exists('config-lost')+ if torn:+ open('config-lost', 'w').close()+ if torn and loss == 'defaults':+ # The 3.2.2rc2 race (#2686): the watcher read the file this PUT was+ # still writing and applied it as defaults before the PUT answered.+ defaulted = Truedef apply():global config, gateway, managementconfig = yaml.safe_load(body)- if {lose_config_on_reload!r} and not os.path.exists('config-lost'):- # The 3.2.1rc8 race (#2686): the watcher applied a half-written- # file, so the engine runs its defaults, not the config written.- open('config-lost', 'w').close()+ if torn and loss == 'refused_keys':+ # The 3.2.1rc8 race: a later reload kept the port but not the keys.gateway = management = 'engine-default'with open('config-reloads', 'a', encoding='utf-8') as handle:handle.write('reload\\n')@@ -3101,6 +3126,10 @@ class Handler(BaseHTTPRequestHandler):self.close_connection = Truereturnself._json(200, {{'ok': True, 'changed': ['config']}})+ if torn and loss == 'listener':+ # Its defaults name another port: the process lives on, but nothing+ # answers where Avibe connects.+ threading.Thread(target=server.shutdown, daemon=True).start()def do_POST(self):length = int(self.headers.get('Content-Length', '0'))@@ -3175,7 +3204,11 @@ class Handler(BaseHTTPRequestHandler):returnself._json(200, {{'id': 'response-1', 'model': payload['model']}})-HTTPServer((config['host'], config['port']), Handler).serve_forever()+server = HTTPServer((config['host'], config['port']), Handler)+server.serve_forever()+server.server_close()+while True:+ time.sleep(1)"""path.write_text(script, encoding="utf-8")path.chmod(0o755)@@ -3230,7 +3263,7 @@ def _fixture_supervisor(exit_before_ready: int | None = None,reload_delay: float = 0.0,drop_reload_ack: bool = False,- lose_config_on_reload: bool = False,+ lose_config_on_reload: str | None = None,) -> tuple[EngineSupervisor, EngineStateStore]:tmp_path.mkdir(parents=True, exist_ok=True)binary = tmp_path / "mock-engine"@@ -3291,7 +3324,7 @@ def test_an_engine_that_lost_its_configuration_is_reported_once_and_restarted_byno caller restarts it under the streams it still serves, and the restartits drainer asks for renders the config again."""- supervisor, store = _fixture_supervisor(tmp_path, lose_config_on_reload=True)+ supervisor, store = _fixture_supervisor(tmp_path, lose_config_on_reload="refused_keys")credential_ref = store.store_api_key("upstream-secret", base_url="https://api.example.test/v1")store.sync_sources([_binding(credential_ref)])connection = supervisor.ensure_running()@@ -3322,6 +3355,99 @@ def test_an_engine_that_lost_its_configuration_is_reported_once_and_restarted_bysupervisor.stop()[email protected](("loss", "reported"), [+ # The repro on the pinned engine: its defaults keep the port, list models+ # to anyone, and serve no management routes.+ ("defaults", "endpoint=/v0/management/config http_status=404"),+ # The 3.2.2rc2 occurrence: the defaults name another port, so nothing answers.+ ("listener", f"http_status=None os_errno={errno.ECONNREFUSED}"),+])+def test_a_source_save_restores_an_engine_whose_reload_tore_its_config(+ tmp_path: Path, caplog: pytest.LogCaptureFixture, loss: str, reported: str,+) -> None:+ """Scenario: MH-RUNTIME-014 -- the engine's own reload race, injected after a PUT, is caught by the save.++ The torn reload proves itself only by the engine answering as no config of+ Avibe's would: a 404 for its management route, or no listener on its port.+ The save that wrote the config finds it, the engine restarts from the+ rendered config, and the save applies on the restarted engine (#2686, #2753).+ """+ supervisor, store = _fixture_supervisor(tmp_path, lose_config_on_reload=loss)+ credential_ref = store.store_api_key("upstream-secret", base_url="https://api.example.test/v1")+ store.sync_sources([_binding(credential_ref)])+ supervisor.ensure_running()+ torn = supervisor._process+ adapter = CLIProxyEngineAdapter(supervisor=supervisor, state_store=store)+ try:+ with caplog.at_level(logging.WARNING, logger=supervisor_module.__name__):+ asyncio.run(adapter.sync_sources([_binding(credential_ref, route_model_ids=("model-b",))]))+ assert supervisor._process is not torn and torn.poll() is not None+ prefix = store.get_source("src_fixture123").prefix+ assert EngineClient(supervisor.ensure_running()).list_model_ids() == {f"{prefix}/model-a", f"{prefix}/model-b"}+ [report] = [record.getMessage() for record in caplog.records if "lost its configuration" in record.getMessage()]+ assert reported in report+ finally:+ supervisor.stop()++[email protected](("kind", "fields", "disowned"), [+ # The engine refused Avibe's key, witnessed on its own listing (#2755).+ (RawOutcomeKind.NETWORK_ERROR, {"http_status": 401, "error_code": "engine_down"}, True),+ # The engine did not know the routed model Avibe registered: an engine on its defaults.+ (RawOutcomeKind.NETWORK_ERROR, {"http_status": 400, "error_code": "engine_down"}, True),+ # Nothing answers on the engine's port.+ (RawOutcomeKind.NETWORK_ERROR, {"http_status": None, "os_errno": errno.ECONNREFUSED}, True),+ # The engine could not reach the upstream: the engine is fine.+ (RawOutcomeKind.NETWORK_ERROR, {"http_status": 502}, False),+ # An upstream's own 401, 400 or model-not-found 404.+ (RawOutcomeKind.HTTP_ERROR, {"http_status": 401, "error_code": "invalid_api_key"}, False),+ (RawOutcomeKind.HTTP_ERROR, {"http_status": 400, "error_code": "invalid_request_error"}, False),+ (RawOutcomeKind.HTTP_ERROR, {"http_status": 404, "error_code": "model_not_found"}, False),+])+def test_only_a_call_the_engine_itself_disowned_rechecks_the_engine(kind, fields, disowned) -> None:+ """Scenario: MH-RUNTIME-014 -- a turn that met an engine not running Avibe's config has it checked again."""+ outcome = RawCallOutcome(+ kind=kind, error_code=fields.get("error_code"), redacted_message=None, stream_started=False,+ model_id="model-a", source_id="src_fixture123", http_status=fields.get("http_status"),+ os_errno=fields.get("os_errno"),+ )+ assert engine_disowned_call(outcome) is disowned+++def test_a_recheck_alone_never_restarts_an_engine_that_answers(tmp_path: Path) -> None:+ """Scenario: MH-RUNTIME-014 -- a call the engine disowned asks for a check, and only the check decides."""+ supervisor, _store = _fixture_supervisor(tmp_path)+ supervisor.ensure_running()+ running = supervisor._process+ try:+ supervisor.recheck_engine()+ supervisor.ensure_running()+ assert supervisor._process is running and running.poll() is None+ finally:+ supervisor.stop()+++def test_a_refused_port_is_no_loss_before_the_engine_first_answers(tmp_path: Path) -> None:+ """Scenario: MH-RUNTIME-014 -- a cold start still binding its port is starting, not lost; once it answered, it is lost."""+ supervisor, _store = _fixture_supervisor(tmp_path, startup_delay=0.6)+ started = threading.Thread(target=supervisor.ensure_running)+ started.start()+ try:+ deadline = time.monotonic() + 5+ while supervisor._process is None and time.monotonic() < deadline:+ time.sleep(0.01)+ starting = supervisor._process+ assert starting is not None and starting.poll() is None+ assert supervisor._disowns_config_locked(None, errno.ECONNREFUSED) is False+ started.join(timeout=10)+ # One start: the window did not restart it.+ assert supervisor._process is starting+ assert supervisor._disowns_config_locked(None, errno.ECONNREFUSED) is True+ finally:+ started.join(timeout=10)+ supervisor.stop()++def test_source_sync_waits_for_an_id_preserving_reload_to_apply(tmp_path: Path) -> None:supervisor, store = _fixture_supervisor(tmp_path, reload_delay=0.3)credential_ref = store.store_api_key("upstream-secret", base_url="https://api.example.test/v1")diff --git a/vibe/model_hub_runtime/adapter.py b/vibe/model_hub_runtime/adapter.pyindex 933f694d7..90656736c 100644--- a/vibe/model_hub_runtime/adapter.py+++ b/vibe/model_hub_runtime/adapter.py@@ -55,7 +55,7 @@ from vibe.model_hub_runtime.client import (EngineClientError,EngineInvokeHandle,completed_handle,- engine_refused_gateway_key,+ engine_disowned_call,probe_models,upstream_api_url,)@@ -2037,7 +2037,12 @@ class CLIProxyEngineAdapter:}.get(normalized_vendor)if expected_provider is None:raise EngineStateError("OAuth credential validation is unsupported")- client = await run_owned_in_thread(self.supervisor.client)+ try:+ client = await run_owned_in_thread(self.supervisor.client)+ except EngineConfigLostError:+ # Already drained: the routing lock is held and transports are idle.+ await self._restart_lost_engine()+ client = await run_owned_in_thread(self.supervisor.client)try:inventory = await run_owned_in_thread(_auth_inventory, client)except EngineClientError as exc:@@ -2903,7 +2908,7 @@ class CLIProxyEngineAdapter:on_transport_done=release,on_request_sent=request_sent,)- if handle.outcome_available and engine_refused_gateway_key(await handle.outcome()):+ if handle.outcome_available and engine_disowned_call(await handle.outcome()):# The engine's last health check vouches for it no longer: the# next admission checks it, finds the lost configuration, and# restores it once the open streams end (#2753, #2755).diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.pyindex 6d404bb25..8864e6cf9 100644--- a/vibe/model_hub_runtime/client.py+++ b/vibe/model_hub_runtime/client.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import errnoimport ioimport jsonimport logging@@ -1395,13 +1396,19 @@ def _project_model_inventory_before_deadline()-def engine_refused_gateway_key(outcome: RawCallOutcome) -> bool:- """Whether a call ended on the engine refusing Avibe's own gateway key, which the client witnessed."""+def engine_disowned_call(outcome: RawCallOutcome) -> bool:+ """Whether a call ended on the engine itself showing it may not run Avibe's config.- return (- outcome.kind is RawOutcomeKind.NETWORK_ERROR- and outcome.error_code == "engine_down"- and outcome.http_status == 401+ It refused Avibe's gateway key (a 401 the client witnessed), did not know+ the routed model Avibe registered (the local registration failure), or+ refused the connection on the port Avibe gave it. The owner checks the+ engine before the next call instead of trusting its last health check.+ """++ if outcome.kind is not RawOutcomeKind.NETWORK_ERROR:+ return False+ return (outcome.error_code == "engine_down" and outcome.http_status in (400, 401)) or (+ outcome.os_errno == errno.ECONNREFUSED)diff --git a/vibe/model_hub_runtime/supervisor.py b/vibe/model_hub_runtime/supervisor.pyindex 00056ac15..d4c62d741 100644--- a/vibe/model_hub_runtime/supervisor.py+++ b/vibe/model_hub_runtime/supervisor.py@@ -1,6 +1,7 @@from __future__ import annotationsimport atexit+import errnoimport jsonimport loggingimport signal@@ -56,8 +57,10 @@ MODEL_HUB_CONFIG_RELOAD_TIMEOUT_SECONDS = 5.0# remembered: every call checks again, and a request restarts the engine, unless# the engine refused Avibe's credentials (``EngineConfigLostError``).MODEL_HUB_HEALTH_TTL_SECONDS = 30.0-# What an engine-local endpoint answers a credential it does not hold.-_CREDENTIALS_REFUSED_STATUSES = frozenset({401, 403})+# What an engine-local endpoint answers when its config is not Avibe's: a+# credential it does not hold (401, 403), or management routes it no longer+# serves because its config has no management secret at all (404).+_CONFIG_DISOWNED_STATUSES = frozenset({401, 403, 404})_STARTUP_POLL_INTERVAL_SECONDS = 0.05# Durable identity of the engine this state root last spawned. A service that# dies without running ``atexit`` (SIGKILL, crash, forced restart) leaves its@@ -86,13 +89,14 @@ class EngineUnavailableError(RuntimeError):class EngineConfigLostError(EngineUnavailableError):- """The running engine refuses the credentials Avibe launched it with.-- Its gateway key and management secret come only from the config Avibe- rendered, so an engine-local endpoint refusing them proves it no longer runs- that config. That is a broken invariant, not a transient failure: the owner- of admissions restores it by admitting no new work, letting the streams the- engine still serves end, and then calling ``restart_lost_engine``.+ """The running engine no longer runs the config Avibe launched it with.++ Its port, gateway key and management secret come only from the config Avibe+ rendered, so one of its own endpoints refusing them, or the still-running+ process refusing connections on that port, proves it lost that config. That+ is a broken invariant, not a transient failure: the owner of admissions+ restores it by admitting no new work, letting the streams the engine still+ serves end, and then calling ``restart_lost_engine``."""def __init__(self) -> None:@@ -128,6 +132,9 @@ class EngineSupervisor:self._record_os_errno: int | None = Noneself._config_generation = 0self._config_loss_reported = False+ # Whether this engine process has answered a health check: before its+ # first answer a refused connection is a start still binding its port.+ self._engine_answered = Falsedef ensure_running(self) -> EngineConnection:with self._lock:@@ -135,10 +142,10 @@ class EngineSupervisor:if self._healthy_locked():assert self._connection is not Nonereturn self._connection- failure = self._health_failure_signature- if failure is not None and failure[1] == "http_error" and failure[2] in _CREDENTIALS_REFUSED_STATUSES:+ lost = self._health_shows_config_lost_locked()+ if lost is not None:# Stopping it here would cut the streams it still serves.- raise self._config_lost_locked(failure[0], failure[2])+ raise lostself._stop_locked()return self._start_locked()@@ -165,15 +172,51 @@ class EngineSupervisor:self._stop_locked()self._start_locked()- def _config_lost_locked(self, endpoint: str, http_status: int) -> EngineConfigLostError:+ def _disowns_config_locked(self, http_status: int | None, os_errno: int | None) -> bool:+ """Whether one answer of the running engine proves it no longer runs Avibe's config.++ It answers Avibe's own credentials or management route as one it does not+ hold, or the process is alive yet refuses connections on the port Avibe+ gave it, after it once answered there: before that it is still starting.+ A timeout or a server error proves neither.+ """++ if http_status in _CONFIG_DISOWNED_STATUSES:+ return True+ return (+ http_status is None and os_errno == errno.ECONNREFUSED+ and self._engine_answered and self._is_running_locked()+ )++ def _health_shows_config_lost_locked(self) -> EngineConfigLostError | None:+ """The loss the last failed health check proved, if it proved one."""++ failure = self._health_failure_signature+ if failure is None or not self._disowns_config_locked(failure[2], self._health_os_errno):+ return None+ return self._config_lost_locked(failure[0], failure[2], self._health_os_errno)++ def _require_launch_config_locked(self) -> None:+ """Check the engine and raise if it no longer runs Avibe's config; a check that proves nothing passes."""++ assert self._connection is not None+ if self._check_health_locked(EngineClient(self._connection, timeout=1.0)):+ return+ lost = self._health_shows_config_lost_locked()+ if lost is not None:+ raise lost++ def _config_lost_locked(+ self, endpoint: str, http_status: int | None, os_errno: int | None = None,+ ) -> EngineConfigLostError:# Nothing it answered vouches for it any more, so the next call checks it.self._verified_at = Noneif not self._config_loss_reported:self._config_loss_reported = Trueassert self._connection is not Nonelogger.warning(- "Model Hub engine lost its configuration endpoint=%s http_status=%s port=%s",- endpoint, http_status, self._connection.base_url.rsplit(":", 1)[1],+ "Model Hub engine lost its configuration endpoint=%s http_status=%s os_errno=%s port=%s",+ endpoint, http_status, os_errno, self._connection.base_url.rsplit(":", 1)[1],)return EngineConfigLostError()@@ -247,8 +290,10 @@ class EngineSupervisor:try:client.put_config_yaml(text)except EngineClientError as exc:- if exc.status_code in _CREDENTIALS_REFUSED_STATUSES:- raise self._config_lost_locked("/v0/management/config.yaml", exc.status_code) from exc+ if self._disowns_config_locked(exc.status_code, local_os_errno(exc)):+ raise self._config_lost_locked(+ "/v0/management/config.yaml", exc.status_code, local_os_errno(exc),+ ) from excif exc.status_code is not None:raise EngineUnavailableError("models.engine.health_failed") from excif not self._is_running_locked():@@ -260,6 +305,11 @@ class EngineSupervisor:# A lost acknowledgment may still have applied, and a stalled# management path does not prove live streams dead. The# generation check decides; the process is never killed here.+ else:+ # The engine's watcher reads the file it just wrote, and has applied+ # a torn read of it as its defaults while answering this very PUT+ # (#2686). Checking now finds that instead of waiting the reload out.+ self._require_launch_config_locked()expected = expected_model_names(sources, self.state_store, generation)stale = self._previous_model_names(previous_sources or (), generation) - set(expected)self._await_models_locked(client, expected, stale)@@ -313,9 +363,9 @@ class EngineSupervisor:try:listed = client.list_model_names(timeout=1.0)except EngineClientError as exc:- if exc.status_code in _CREDENTIALS_REFUSED_STATUSES:+ if self._disowns_config_locked(exc.status_code, local_os_errno(exc)):# The applied file was not the one written: waiting cannot help.- raise self._config_lost_locked("/v1/models", exc.status_code) from exc+ raise self._config_lost_locked("/v1/models", exc.status_code, local_os_errno(exc)) from exclisted = Noneos_errno = local_os_errno(exc)if (@@ -325,6 +375,9 @@ class EngineSupervisor:):returnif time.monotonic() >= deadline:+ # An engine on its defaults may still list models to anyone; its+ # management route shows what its listing cannot.+ self._require_launch_config_locked()raise EngineUnavailableError("models.engine.health_failed", os_errno=os_errno)time.sleep(_STARTUP_POLL_INTERVAL_SECONDS)@@ -536,6 +589,7 @@ class EngineSupervisor:raise EngineUnavailableError("models.engine.unsafe_permissions") from excself._last_check = _utc_now()self._verified_at = time.monotonic()+ self._engine_answered = Truelogger.info("Model Hub engine startup outcome=ready managed_version=%s ""elapsed_seconds=%.3f child_output_retained=false",@@ -608,6 +662,7 @@ class EngineSupervisor:self._health_failure_signature = Noneself._health_os_errno = Noneself._config_loss_reported = False+ self._engine_answered = Falseif process is not None and process.poll() is None:signal_process_tree(process, signal.SIGTERM, logger, "Model Hub engine")try: