Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): list a Source again when its listed executor model is refused (#2765)
Call White43827d473d8d
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.pyindex 65635c477..83194a692 100644--- a/core/handlers/model_hub/service.py+++ b/core/handlers/model_hub/service.py@@ -383,6 +383,8 @@ _BACKGROUND_DISCOVERY_RETRY_BASE = timedelta(minutes=15)_BACKGROUND_DISCOVERY_RETRY_CAP = timedelta(hours=6)# A viewer of Source inventory asks for listings at most this old._VIEWED_DISCOVERY_AGE = timedelta(minutes=30)+# A Source marked stale is listed again once its listing is this old.+_STALE_LISTING_AGE = timedelta(minutes=15)# A credential waiting on the user cannot list models until the user acts._UNDISCOVERABLE_SOURCE_DETAILS = frozenset({"models.source.needs_action.oauth_expired",@@ -1018,6 +1020,14 @@ class _DiscoveryRetry:delay: timedelta+@dataclass(frozen=True)+class _StaleListing:+ """The discovered rows a Source refused as models it does not serve, under the identity that refused them."""++ identity: tuple+ model_ids: frozenset[str]++def _background_discovery_period(source_id: str) -> timedelta:"""A stable per-Source period within the jitter window around the interval.@@ -1374,6 +1384,23 @@ def runtime_dependency_payload(return payload++def _drop_discovered_models(source: ModelHubSourceConfig, model_ids: Iterable[str]) -> bool:+ """Remove these listed discovered rows; whether one was there.++ A manual row is the user's, and a retired row is the tombstone that keeps+ a model the user removed from being listed again, so both stay.+ """++ dropping = set(model_ids)+ kept = [+ model for model in source.models+ if not (model.id in dropping and model.provenance == "discovered" and not model.retired)+ ]+ dropped = len(kept) != len(source.models)+ source.models = kept+ return dropped+class ModelHubService:def __init__(self,@@ -1503,6 +1530,7 @@ class ModelHubService:self._background_discovery_caught_up = Falseself._background_discovery_inflight: set[str] = set()self._viewed_discovery_task: asyncio.Task[None] | None = None+ self._stale_listings: dict[str, _StaleListing] = {}self._builtin_snapshot_generations: dict[BackendName, str] = {}self._builtin_snapshot_cache: dict[BackendName, list[dict[str, Any]]] = {}self._pending_builtin_catalog_refresh: set[BackendName] = set()@@ -4850,6 +4878,26 @@ class ModelHubService:shutdown and never starts work for a stopped runtime."""+ self._request_discovery_pass(max_age=_VIEWED_DISCOVERY_AGE)++ def mark_stale_listing(self, source: ModelHubSourceConfig, model_id: str) -> None:+ """A Source refused a model it lists: have the schedule list it again soon (#2682).++ Only a mark: the schedule's own due rule lists the Source once its+ listing is older than ``_STALE_LISTING_AGE``, under its eligibility,+ backoff, in-flight exclusion and shutdown, and that listing drops each+ marked row if the identity that refused it still holds and the listing+ no longer names it either.+ """++ identity = source_identity(source)+ marked = self._stale_listings.get(source.id)+ # Turns frozen on different executor models of one Source may each be refused.+ earlier = marked.model_ids if marked is not None and marked.identity == identity else frozenset()+ self._stale_listings[source.id] = _StaleListing(identity, earlier | {model_id})+ self._request_discovery_pass(max_age=None)++ def _request_discovery_pass(self, *, max_age: timedelta | None) -> None:schedule = self._background_discovery_taskstopping = self._background_discovery_stopif schedule is None or schedule.done() or stopping is None or stopping.is_set():@@ -4858,7 +4906,7 @@ class ModelHubService:if viewed is not None and not viewed.done():returnself._viewed_discovery_task = asyncio.get_running_loop().create_task(- self._refresh_due_sources_logged(max_age=_VIEWED_DISCOVERY_AGE),+ self._refresh_due_sources_logged(max_age=max_age),name="model-hub-viewed-discovery",)@@ -4878,7 +4926,7 @@ class ModelHubService:if self._background_discovery_eligible(source)]live_ids = {source.id for source in sources}- for table in (self._background_discovery_checked, self._background_discovery_retries):+ for table in (self._background_discovery_checked, self._background_discovery_retries, self._stale_listings):for source_id in [item for item in table if item not in live_ids]:del table[source_id]now = self.now()@@ -4944,6 +4992,9 @@ class ModelHubService:period = min(period, _BACKGROUND_DISCOVERY_INTERVAL)if max_age is not None:period = min(period, max_age)+ mark = self._stale_listings.get(source.id)+ if mark is not None and mark.identity == source_identity(source):+ period = min(period, _STALE_LISTING_AGE)return listed_at + periodasync def _refresh_source_in_background(self, source_id: str) -> None:@@ -4951,6 +5002,9 @@ class ModelHubService:A listing is not inference evidence, and removals move supply, so Sourcestate, removals, and their guards stay with a manual `refresh_source`.+ A stale mark's models are the exception: each leaves if this listing no+ longer names it either, under the identity that refused it, and only+ when its removal moves no route."""if source_id in self._background_discovery_inflight:@@ -4974,6 +5028,7 @@ class ModelHubService:if observed is None or not self._background_discovery_eligible(observed):returnbasis = (source_identity(observed), observed.last_discovered_at)+ mark: _StaleListing | None = Nonetry:discovered = await asyncio.wait_for(self._discover(observed, start_engine=False),@@ -4991,25 +5046,50 @@ class ModelHubService:# The listing describes a Source another owner has changed# since; the next tick lists the current one.return+ # Read now, not at the start: a refusal that arrived while this+ # listing ran is answered by it.+ mark = self._stale_listings.get(source_id)+ if mark is not None and mark.identity != basis[0]:+ # The identity that refused the model is gone, and its refusal with it.+ del self._stale_listings[source_id]+ mark = Noneupdated = self._clone_config(previous)source = self._source(updated, source_id)overrides = self._merge_discovered_models(source, discovered)+ disowned = (mark.model_ids if mark is not None else frozenset()) - {+ canonical_model_id(model.id) for model in discovered+ }+ if disowned:+ _drop_discovered_models(source, disowned)if self._inventory_fingerprint(source) != self._inventory_fingerprint(current):- if self._inventory_removed_hops(previous, updated) or self._introduced_interruptions(- previous, updated,- ):+ if not self._moves_supply(previous, updated):+ await self._commit_synced(previous, updated)+ self._record_reasoning_tier_overrides(source, overrides)+ else:+ dropped = self._clone_config(previous)+ if disowned and _drop_discovered_models(self._source(dropped, source_id), disowned) and (+ not self._moves_supply(previous, dropped)+ ):+ # The disowned row may leave on its own when the+ # rest of this listing has to wait.+ await self._commit_synced(previous, dropped)logger.info("Model Hub background discovery left a supply change ""for Source %s to a manual refresh",source_id,)- else:- await self._commit_synced(previous, updated)- self._record_reasoning_tier_overrides(source, overrides)except Exception as exc:self._record_background_discovery_failure(observed, exc)returnself._background_discovery_checked[source_id] = self.now()+ if mark is not None and self._stale_listings.get(source_id) is mark:+ # This listing answered the mark; a later refusal marks again.+ del self._stale_listings[source_id]++ def _moves_supply(self, previous: ModelHubConfig, updated: ModelHubConfig) -> bool:+ """Whether an inventory change removes a route hop or interrupts supply, which only a manual refresh may."""++ return bool(self._inventory_removed_hops(previous, updated) or self._introduced_interruptions(previous, updated))def _record_background_discovery_failure(self,@@ -6951,6 +7031,11 @@ class ModelHubService:if learned is not None and learned.feature.kind == "hosted_tool":return Trueif not outcome.stream_started and names_model_refusal(outcome):+ if any(model.id == model_id and model.provenance == "discovered" for model in source.models):+ # An executor runs only on a model its Source lists, so the+ # listing may be stale: mark it for the schedule (#2682). A+ # manual row is the user's, and only they remove it.+ self.mark_stale_listing(source, model_id)return Truedecision = classify_outcome(replace(outcome, stream_started=False))return decision.action in {"fallback", "refresh"}diff --git a/docs/plans/vibey-hosted-tools.md b/docs/plans/vibey-hosted-tools.mdindex b7a7d7f1e..abc2fba37 100644--- a/docs/plans/vibey-hosted-tools.md+++ b/docs/plans/vibey-hosted-tools.md@@ -503,7 +503,11 @@ These rules are C-1 rows in `cross-provider.md` and per-protocol rows in `converrouting's classifier reads it; L1 hands no prose to it) skips that Source for this call without settling itshealth, as `_InvocationUnsupported` does. It is not a refusal of the tool and writes no evidence: only thenarrow rule, an error naming the feature, does. With the executor model taken from the Source's listing, it- means a stale listing, which the Source's next inventory refresh corrects. A failure mid-stream moves on too: nothing+ means a stale listing, so a discovered row marks that Source for the background schedule (#2682). The+ schedule's own due rule lists a marked Source once its listing is 15 minutes old, under its eligibility,+ backoff and shutdown. The row leaves only if that listing, under the identity that refused it, no longer+ names it either and its removal moves no route; the rest of the listing merges as the schedule's always+ does. A manual row is the user's, and stays. A failure mid-stream moves on too: nothingreaches Vibey until the gateway returns the record, so "before output" here means before the result isreturned, and the partial call is still metered on its Source. A completed response that never ran the tool(the model chose not to; Anthropic is asked in the message alone) ends the chain with a `failed` record: it isdiff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 348f23785..886afaf58 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -2570,6 +2570,15 @@ scenarios:test: tests/test_model_hub_hosted_evidence.py::test_a_served_response_proves_only_its_own_completed_search_itemrelated_tests:- tests/test_model_hub_hosted_evidence.py::test_a_native_success_is_recorded_at_settlement_for_a_tool_the_request_declared+ - id: MH-HOSTED-GATEWAY-009+ name: >-+ A model-not-found for an executor model its Source lists marks that Source, and the background schedule's own+ due rule lists it once its listing is 15 minutes old; the row leaves only when that listing, under the identity+ that refused it, no longer names it either and its removal moves no route, so the next chain skips it+ status: covered+ kind: recovery+ layer: contract+ test: tests/test_model_hub_hosted_executor.py::test_a_refused_listed_executor_model_is_listed_again_and_leaves_once_unlisted- id: MH-HOSTED-GATEWAY-007name: image_generate runs directly on the images endpoint of a model its turn's frozen enum names, admitted on the final snapshot with no fallback; an argument the model does not take or a model no longer admitted fails before anything runs; only usable base64 images complete; only an upstream refusal of the model or its endpoint is learned; each offered path is verified through the pinned enginestatus: covereddiff --git a/tests/test_model_hub_hosted_executor.py b/tests/test_model_hub_hosted_executor.pyindex a9170e95e..c80460e4e 100644--- a/tests/test_model_hub_hosted_executor.py+++ b/tests/test_model_hub_hosted_executor.py@@ -22,7 +22,7 @@ import pytestfrom config.v2_config import ModelHubModelConfig, ModelHubHostedToolConfigfrom core.agent_core.messages import HostedCost, message_from_dictfrom core.handlers.model_hub import service as service_module-from core.handlers.model_hub.adapter import InvokeCancelledError, RawOutcomeKind, UpstreamRejection+from core.handlers.model_hub.adapter import DiscoveredModel, InvokeCancelledError, RawOutcomeKind, UpstreamRejectionfrom core.handlers.model_hub.capability_evidence import EvidenceFeaturefrom core.handlers.model_hub.hosted_executor import ResponsesReadout, anthropic_result, excerpt, responses_resultfrom core.handlers.model_hub import hosted_tools as hosted_tools_module@@ -504,6 +504,195 @@ async def test_a_model_the_source_does_not_serve_moves_on_and_refuses_no_tool(tmassert [slot.source_id for slot in chain] == [source_id, XAI]+def _stale_listing_service(tmp_path, handles=()):+ """Vibey with OpenAI's executor row on no route: its turns route to gpt-6-chat."""++ sources = _sources()+ for source in sources:+ # Listed recently, so only a stale mark makes a Source due.+ source.last_discovered_at = "2099-01-01T00:00:00+00:00"+ next(item for item in sources if item.id == OPENAI).models.insert(+ 0, ModelHubModelConfig(id="gpt-6-chat", provenance="discovered"),+ )+ service = _vibey_service(tmp_path, sources, handles=handles)+ _chain(service, "web_search", (OPENAI, "gpt-6-astra"), (XAI, "grok-4.6"))+ service.adapter.stop = lambda: asyncio.sleep(0)+ return service+++def _openai(service):+ return next(item for item in service.store.config.sources if item.id == OPENAI)+++async def _settled(service):+ async with asyncio.timeout(2):+ await asyncio.sleep(0)+ while service._viewed_discovery_task is not None and not service._viewed_discovery_task.done():+ await asyncio.sleep(0.01)++[email protected]("listed_again", [False, True], ids=["unlisted", "still_listed"])+async def test_a_refused_listed_executor_model_is_listed_again_and_leaves_once_unlisted(tmp_path, listed_again):+ """MH-HOSTED-GATEWAY-009: a model-not-found for an executor model its Source lists marks that Source (#2682).++ The schedule's own due rule lists a marked Source once its listing is older+ than the debounce. Only a model both the call and that listing disown+ leaves, and the rest of the listing merges as the schedule's always does.+ """++ source_id, model_id, protocol, body = MODEL_NOT_SERVED["openai"]+ not_served = replace(await _engine_outcome(protocol, 404, body), source_id=source_id)+ served = LiveInvokeHandle(_outcome(RawOutcomeKind.SUCCESS, source_id=XAI), _responses(WEB_CALL, WEB_ANSWER))+ service = _stale_listing_service(tmp_path, handles=[InvokeHandle(not_served), served])+ _openai(service).last_discovered_at = None+ listed = [DiscoveredModel(id="gpt-6-chat"), *([DiscoveredModel(id=model_id)] if listed_again else [])]+ listings = []++ async def discover_models(vendor, *_args, **_kwargs):+ listings.append(vendor)+ return listed++ service.adapter.discover_models = discover_models+ service.start_background_discovery()+ try:+ status, answer = await _call(service, "web_search", {"arguments": {"query": "q"}})+ assert (status, answer["execution"]["executor_source_id"]) == (200, XAI)+ await _settled(service)+ assert listings == ["openai"]+ # The Source was just listed, so the same refusal again lists nothing.+ service.mark_stale_listing(_openai(service), model_id)+ await _settled(service)+ assert listings == ["openai"]+ finally:+ await service.stop()+ assert [model.id for model in _openai(service).models] == (["gpt-6-chat", model_id] if listed_again else ["gpt-6-chat"])+ chain = service.freeze_turn_hosted_tools(service.store.config).executors["web_search"]+ assert [slot.source_id for slot in chain] == ([source_id, XAI] if listed_again else [XAI])+++async def test_a_stale_mark_is_answered_by_a_running_listing_and_keeps_the_schedules_backoff(tmp_path):+ """MH-HOSTED-GATEWAY-009: a listing already running answers the mark, and a failing one keeps its backoff."""++ service = _stale_listing_service(tmp_path)+ _openai(service).last_discovered_at = None+ base = service.now()+ clock = [base]+ service.now = lambda: clock[0]+ release = asyncio.Event()+ fail = False+ listings = []++ async def discover_models(vendor, *_args, **_kwargs):+ listings.append(vendor)+ await release.wait()+ if fail:+ raise RuntimeError("listing unavailable")+ return [DiscoveredModel(id="gpt-6-chat")]++ service.adapter.discover_models = discover_models+ service.start_background_discovery()+ try:+ # A viewer's pass is listing the Source when the refusal arrives: that listing answers it.+ service.request_inventory_refresh()+ async with asyncio.timeout(2):+ while not listings:+ await asyncio.sleep(0.01)+ service.mark_stale_listing(_openai(service), "gpt-6-astra")+ release.set()+ await _settled(service)+ assert listings == ["openai"]+ assert [model.id for model in _openai(service).models] == ["gpt-6-chat"]+ # A failing listing backs off past the debounce, whoever marks it.+ fail = True+ for minutes in (16, 32):+ clock[0] = base + timedelta(minutes=minutes)+ service.mark_stale_listing(_openai(service), "gpt-6-chat")+ await _settled(service)+ clock[0] = base + timedelta(minutes=48)+ service.mark_stale_listing(_openai(service), "gpt-6-chat")+ await _settled(service)+ assert listings == ["openai"] * 3+ finally:+ await service.stop()+++async def test_a_stale_mark_drops_nothing_once_its_source_identity_changed(tmp_path):+ """MH-HOSTED-GATEWAY-009: a refusal belongs to the identity that gave it; a replaced key or URL voids it."""++ service = _stale_listing_service(tmp_path)+ _openai(service).last_discovered_at = None++ async def discover_models(*_args, **_kwargs):+ return [DiscoveredModel(id="gpt-6-chat")]++ service.adapter.discover_models = discover_models+ service.start_background_discovery()+ try:+ service.mark_stale_listing(_openai(service), "gpt-6-astra")+ _openai(service).base_url = "https://replaced.example/v1"+ await _settled(service)+ finally:+ await service.stop()+ assert [model.id for model in _openai(service).models] == ["gpt-6-chat", "gpt-6-astra"]+ assert service._stale_listings == {}+++async def test_every_refused_row_leaves_but_a_retired_tombstone_stays(tmp_path):+ """MH-HOSTED-GATEWAY-009: refusals of two executor models both count, and a row the user retired is kept."""++ service = _stale_listing_service(tmp_path)+ openai = _openai(service)+ openai.last_discovered_at = None+ openai.models.append(ModelHubModelConfig(id="gpt-6-luna", provenance="discovered"))+ openai.models.append(ModelHubModelConfig(id="gpt-6-sol", provenance="discovered"))++ async def discover_models(*_args, **_kwargs):+ return [DiscoveredModel(id="gpt-6-chat")]++ service.adapter.discover_models = discover_models+ service.start_background_discovery()+ try:+ # Two turns frozen on different executor models are each refused.+ service.mark_stale_listing(_openai(service), "gpt-6-astra")+ service.mark_stale_listing(_openai(service), "gpt-6-luna")+ service.mark_stale_listing(_openai(service), "gpt-6-sol")+ # The user retires gpt-6-sol before the listing returns.+ next(model for model in _openai(service).models if model.id == "gpt-6-sol").retired = True+ await _settled(service)+ finally:+ await service.stop()+ models = {model.id: bool(model.retired) for model in _openai(service).models}+ assert models == {"gpt-6-chat": False, "gpt-6-sol": True}+++async def test_a_disowned_row_leaves_even_when_the_rest_of_the_listing_must_wait(tmp_path, monkeypatch):+ """MH-HOSTED-GATEWAY-009: a supply change elsewhere in the listing does not hold back the disowned row."""++ service = _stale_listing_service(tmp_path)+ _openai(service).last_discovered_at = None++ async def discover_models(*_args, **_kwargs):+ return [DiscoveredModel(id="gpt-6-chat"), DiscoveredModel(id="gpt-6-new")]++ service.adapter.discover_models = discover_models+ # Admitting gpt-6-new is the change a manual refresh must confirm.+ monkeypatch.setattr(service, "_moves_supply", lambda _previous, updated: any(+ model.id == "gpt-6-new" for source in updated.sources for model in source.models+ ))+ service.start_background_discovery()+ try:+ service.mark_stale_listing(_openai(service), "gpt-6-astra")+ await _settled(service)+ finally:+ await service.stop()+ assert [model.id for model in _openai(service).models] == ["gpt-6-chat"]++@pytest.mark.asyncio@pytest.mark.parametrize(("case", "handles", "invoked", "status", "served_by", "failed_attempts"), [# A failure normal routing fails over from moves on; a refusal before output that reported no tokens is unbilled.