Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): list a Source again when its listed executor model is refused (#2765)

Call White43827d473d8d
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index 65635c477..83194a692 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -383,6 +383,8 @@ _BACKGROUND_DISCOVERY_RETRY_BASE = timedelta(minutes=15)
_BACKGROUND_DISCOVERY_RETRY_CAP = timedelta(hours=6)
# A viewer of Source inventory asks for listings at most this old.
_VIEWED_DISCOVERY_AGE = timedelta(minutes=30)
+# A Source marked stale is listed again once its listing is this old.
+_STALE_LISTING_AGE = timedelta(minutes=15)
# A credential waiting on the user cannot list models until the user acts.
_UNDISCOVERABLE_SOURCE_DETAILS = frozenset({
"models.source.needs_action.oauth_expired",
@@ -1018,6 +1020,14 @@ class _DiscoveryRetry:
delay: timedelta
+@dataclass(frozen=True)
+class _StaleListing:
+ """The discovered rows a Source refused as models it does not serve, under the identity that refused them."""
+
+ identity: tuple
+ model_ids: frozenset[str]
+
+
def _background_discovery_period(source_id: str) -> timedelta:
"""A stable per-Source period within the jitter window around the interval.
@@ -1374,6 +1384,23 @@ def runtime_dependency_payload(
return payload
+
+def _drop_discovered_models(source: ModelHubSourceConfig, model_ids: Iterable[str]) -> bool:
+ """Remove these listed discovered rows; whether one was there.
+
+ A manual row is the user's, and a retired row is the tombstone that keeps
+ a model the user removed from being listed again, so both stay.
+ """
+
+ dropping = set(model_ids)
+ kept = [
+ model for model in source.models
+ if not (model.id in dropping and model.provenance == "discovered" and not model.retired)
+ ]
+ dropped = len(kept) != len(source.models)
+ source.models = kept
+ return dropped
+
class ModelHubService:
def __init__(
self,
@@ -1503,6 +1530,7 @@ class ModelHubService:
self._background_discovery_caught_up = False
self._background_discovery_inflight: set[str] = set()
self._viewed_discovery_task: asyncio.Task[None] | None = None
+ self._stale_listings: dict[str, _StaleListing] = {}
self._builtin_snapshot_generations: dict[BackendName, str] = {}
self._builtin_snapshot_cache: dict[BackendName, list[dict[str, Any]]] = {}
self._pending_builtin_catalog_refresh: set[BackendName] = set()
@@ -4850,6 +4878,26 @@ class ModelHubService:
shutdown and never starts work for a stopped runtime.
"""
+ self._request_discovery_pass(max_age=_VIEWED_DISCOVERY_AGE)
+
+ def mark_stale_listing(self, source: ModelHubSourceConfig, model_id: str) -> None:
+ """A Source refused a model it lists: have the schedule list it again soon (#2682).
+
+ Only a mark: the schedule's own due rule lists the Source once its
+ listing is older than ``_STALE_LISTING_AGE``, under its eligibility,
+ backoff, in-flight exclusion and shutdown, and that listing drops each
+ marked row if the identity that refused it still holds and the listing
+ no longer names it either.
+ """
+
+ identity = source_identity(source)
+ marked = self._stale_listings.get(source.id)
+ # Turns frozen on different executor models of one Source may each be refused.
+ earlier = marked.model_ids if marked is not None and marked.identity == identity else frozenset()
+ self._stale_listings[source.id] = _StaleListing(identity, earlier | {model_id})
+ self._request_discovery_pass(max_age=None)
+
+ def _request_discovery_pass(self, *, max_age: timedelta | None) -> None:
schedule = self._background_discovery_task
stopping = self._background_discovery_stop
if schedule is None or schedule.done() or stopping is None or stopping.is_set():
@@ -4858,7 +4906,7 @@ class ModelHubService:
if viewed is not None and not viewed.done():
return
self._viewed_discovery_task = asyncio.get_running_loop().create_task(
- self._refresh_due_sources_logged(max_age=_VIEWED_DISCOVERY_AGE),
+ self._refresh_due_sources_logged(max_age=max_age),
name="model-hub-viewed-discovery",
)
@@ -4878,7 +4926,7 @@ class ModelHubService:
if self._background_discovery_eligible(source)
]
live_ids = {source.id for source in sources}
- for table in (self._background_discovery_checked, self._background_discovery_retries):
+ for table in (self._background_discovery_checked, self._background_discovery_retries, self._stale_listings):
for source_id in [item for item in table if item not in live_ids]:
del table[source_id]
now = self.now()
@@ -4944,6 +4992,9 @@ class ModelHubService:
period = min(period, _BACKGROUND_DISCOVERY_INTERVAL)
if max_age is not None:
period = min(period, max_age)
+ mark = self._stale_listings.get(source.id)
+ if mark is not None and mark.identity == source_identity(source):
+ period = min(period, _STALE_LISTING_AGE)
return listed_at + period
async def _refresh_source_in_background(self, source_id: str) -> None:
@@ -4951,6 +5002,9 @@ class ModelHubService:
A listing is not inference evidence, and removals move supply, so Source
state, removals, and their guards stay with a manual `refresh_source`.
+ A stale mark's models are the exception: each leaves if this listing no
+ longer names it either, under the identity that refused it, and only
+ when its removal moves no route.
"""
if source_id in self._background_discovery_inflight:
@@ -4974,6 +5028,7 @@ class ModelHubService:
if observed is None or not self._background_discovery_eligible(observed):
return
basis = (source_identity(observed), observed.last_discovered_at)
+ mark: _StaleListing | None = None
try:
discovered = await asyncio.wait_for(
self._discover(observed, start_engine=False),
@@ -4991,25 +5046,50 @@ class ModelHubService:
# The listing describes a Source another owner has changed
# since; the next tick lists the current one.
return
+ # Read now, not at the start: a refusal that arrived while this
+ # listing ran is answered by it.
+ mark = self._stale_listings.get(source_id)
+ if mark is not None and mark.identity != basis[0]:
+ # The identity that refused the model is gone, and its refusal with it.
+ del self._stale_listings[source_id]
+ mark = None
updated = self._clone_config(previous)
source = self._source(updated, source_id)
overrides = self._merge_discovered_models(source, discovered)
+ disowned = (mark.model_ids if mark is not None else frozenset()) - {
+ canonical_model_id(model.id) for model in discovered
+ }
+ if disowned:
+ _drop_discovered_models(source, disowned)
if self._inventory_fingerprint(source) != self._inventory_fingerprint(current):
- if self._inventory_removed_hops(previous, updated) or self._introduced_interruptions(
- previous, updated,
- ):
+ if not self._moves_supply(previous, updated):
+ await self._commit_synced(previous, updated)
+ self._record_reasoning_tier_overrides(source, overrides)
+ else:
+ dropped = self._clone_config(previous)
+ if disowned and _drop_discovered_models(self._source(dropped, source_id), disowned) and (
+ not self._moves_supply(previous, dropped)
+ ):
+ # The disowned row may leave on its own when the
+ # rest of this listing has to wait.
+ await self._commit_synced(previous, dropped)
logger.info(
"Model Hub background discovery left a supply change "
"for Source %s to a manual refresh",
source_id,
)
- else:
- await self._commit_synced(previous, updated)
- self._record_reasoning_tier_overrides(source, overrides)
except Exception as exc:
self._record_background_discovery_failure(observed, exc)
return
self._background_discovery_checked[source_id] = self.now()
+ if mark is not None and self._stale_listings.get(source_id) is mark:
+ # This listing answered the mark; a later refusal marks again.
+ del self._stale_listings[source_id]
+
+ def _moves_supply(self, previous: ModelHubConfig, updated: ModelHubConfig) -> bool:
+ """Whether an inventory change removes a route hop or interrupts supply, which only a manual refresh may."""
+
+ return bool(self._inventory_removed_hops(previous, updated) or self._introduced_interruptions(previous, updated))
def _record_background_discovery_failure(
self,
@@ -6951,6 +7031,11 @@ class ModelHubService:
if learned is not None and learned.feature.kind == "hosted_tool":
return True
if not outcome.stream_started and names_model_refusal(outcome):
+ if any(model.id == model_id and model.provenance == "discovered" for model in source.models):
+ # An executor runs only on a model its Source lists, so the
+ # listing may be stale: mark it for the schedule (#2682). A
+ # manual row is the user's, and only they remove it.
+ self.mark_stale_listing(source, model_id)
return True
decision = classify_outcome(replace(outcome, stream_started=False))
return decision.action in {"fallback", "refresh"}
diff --git a/docs/plans/vibey-hosted-tools.md b/docs/plans/vibey-hosted-tools.md
index b7a7d7f1e..abc2fba37 100644
--- a/docs/plans/vibey-hosted-tools.md
+++ b/docs/plans/vibey-hosted-tools.md
@@ -503,7 +503,11 @@ These rules are C-1 rows in `cross-provider.md` and per-protocol rows in `conver
routing's classifier reads it; L1 hands no prose to it) skips that Source for this call without settling its
health, as `_InvocationUnsupported` does. It is not a refusal of the tool and writes no evidence: only the
narrow rule, an error naming the feature, does. With the executor model taken from the Source's listing, it
- means a stale listing, which the Source's next inventory refresh corrects. A failure mid-stream moves on too: nothing
+ means a stale listing, so a discovered row marks that Source for the background schedule (#2682). The
+ schedule's own due rule lists a marked Source once its listing is 15 minutes old, under its eligibility,
+ backoff and shutdown. The row leaves only if that listing, under the identity that refused it, no longer
+ names it either and its removal moves no route; the rest of the listing merges as the schedule's always
+ does. A manual row is the user's, and stays. A failure mid-stream moves on too: nothing
reaches Vibey until the gateway returns the record, so "before output" here means before the result is
returned, and the partial call is still metered on its Source. A completed response that never ran the tool
(the model chose not to; Anthropic is asked in the message alone) ends the chain with a `failed` record: it is
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 348f23785..886afaf58 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -2570,6 +2570,15 @@ scenarios:
test: tests/test_model_hub_hosted_evidence.py::test_a_served_response_proves_only_its_own_completed_search_item
related_tests:
- tests/test_model_hub_hosted_evidence.py::test_a_native_success_is_recorded_at_settlement_for_a_tool_the_request_declared
+ - id: MH-HOSTED-GATEWAY-009
+ name: >-
+ A model-not-found for an executor model its Source lists marks that Source, and the background schedule's own
+ due rule lists it once its listing is 15 minutes old; the row leaves only when that listing, under the identity
+ that refused it, no longer names it either and its removal moves no route, so the next chain skips it
+ status: covered
+ kind: recovery
+ layer: contract
+ test: tests/test_model_hub_hosted_executor.py::test_a_refused_listed_executor_model_is_listed_again_and_leaves_once_unlisted
- id: MH-HOSTED-GATEWAY-007
name: image_generate runs directly on the images endpoint of a model its turn's frozen enum names, admitted on the final snapshot with no fallback; an argument the model does not take or a model no longer admitted fails before anything runs; only usable base64 images complete; only an upstream refusal of the model or its endpoint is learned; each offered path is verified through the pinned engine
status: covered
diff --git a/tests/test_model_hub_hosted_executor.py b/tests/test_model_hub_hosted_executor.py
index a9170e95e..c80460e4e 100644
--- a/tests/test_model_hub_hosted_executor.py
+++ b/tests/test_model_hub_hosted_executor.py
@@ -22,7 +22,7 @@ import pytest
from config.v2_config import ModelHubModelConfig, ModelHubHostedToolConfig
from core.agent_core.messages import HostedCost, message_from_dict
from core.handlers.model_hub import service as service_module
-from core.handlers.model_hub.adapter import InvokeCancelledError, RawOutcomeKind, UpstreamRejection
+from core.handlers.model_hub.adapter import DiscoveredModel, InvokeCancelledError, RawOutcomeKind, UpstreamRejection
from core.handlers.model_hub.capability_evidence import EvidenceFeature
from core.handlers.model_hub.hosted_executor import ResponsesReadout, anthropic_result, excerpt, responses_result
from core.handlers.model_hub import hosted_tools as hosted_tools_module
@@ -504,6 +504,195 @@ async def test_a_model_the_source_does_not_serve_moves_on_and_refuses_no_tool(tm
assert [slot.source_id for slot in chain] == [source_id, XAI]
+def _stale_listing_service(tmp_path, handles=()):
+ """Vibey with OpenAI's executor row on no route: its turns route to gpt-6-chat."""
+
+ sources = _sources()
+ for source in sources:
+ # Listed recently, so only a stale mark makes a Source due.
+ source.last_discovered_at = "2099-01-01T00:00:00+00:00"
+ next(item for item in sources if item.id == OPENAI).models.insert(
+ 0, ModelHubModelConfig(id="gpt-6-chat", provenance="discovered"),
+ )
+ service = _vibey_service(tmp_path, sources, handles=handles)
+ _chain(service, "web_search", (OPENAI, "gpt-6-astra"), (XAI, "grok-4.6"))
+ service.adapter.stop = lambda: asyncio.sleep(0)
+ return service
+
+
+def _openai(service):
+ return next(item for item in service.store.config.sources if item.id == OPENAI)
+
+
+async def _settled(service):
+ async with asyncio.timeout(2):
+ await asyncio.sleep(0)
+ while service._viewed_discovery_task is not None and not service._viewed_discovery_task.done():
+ await asyncio.sleep(0.01)
+
+
[email protected]("listed_again", [False, True], ids=["unlisted", "still_listed"])
+async def test_a_refused_listed_executor_model_is_listed_again_and_leaves_once_unlisted(tmp_path, listed_again):
+ """MH-HOSTED-GATEWAY-009: a model-not-found for an executor model its Source lists marks that Source (#2682).
+
+ The schedule's own due rule lists a marked Source once its listing is older
+ than the debounce. Only a model both the call and that listing disown
+ leaves, and the rest of the listing merges as the schedule's always does.
+ """
+
+ source_id, model_id, protocol, body = MODEL_NOT_SERVED["openai"]
+ not_served = replace(await _engine_outcome(protocol, 404, body), source_id=source_id)
+ served = LiveInvokeHandle(_outcome(RawOutcomeKind.SUCCESS, source_id=XAI), _responses(WEB_CALL, WEB_ANSWER))
+ service = _stale_listing_service(tmp_path, handles=[InvokeHandle(not_served), served])
+ _openai(service).last_discovered_at = None
+ listed = [DiscoveredModel(id="gpt-6-chat"), *([DiscoveredModel(id=model_id)] if listed_again else [])]
+ listings = []
+
+ async def discover_models(vendor, *_args, **_kwargs):
+ listings.append(vendor)
+ return listed
+
+ service.adapter.discover_models = discover_models
+ service.start_background_discovery()
+ try:
+ status, answer = await _call(service, "web_search", {"arguments": {"query": "q"}})
+ assert (status, answer["execution"]["executor_source_id"]) == (200, XAI)
+ await _settled(service)
+ assert listings == ["openai"]
+ # The Source was just listed, so the same refusal again lists nothing.
+ service.mark_stale_listing(_openai(service), model_id)
+ await _settled(service)
+ assert listings == ["openai"]
+ finally:
+ await service.stop()
+ assert [model.id for model in _openai(service).models] == (["gpt-6-chat", model_id] if listed_again else ["gpt-6-chat"])
+ chain = service.freeze_turn_hosted_tools(service.store.config).executors["web_search"]
+ assert [slot.source_id for slot in chain] == ([source_id, XAI] if listed_again else [XAI])
+
+
+async def test_a_stale_mark_is_answered_by_a_running_listing_and_keeps_the_schedules_backoff(tmp_path):
+ """MH-HOSTED-GATEWAY-009: a listing already running answers the mark, and a failing one keeps its backoff."""
+
+ service = _stale_listing_service(tmp_path)
+ _openai(service).last_discovered_at = None
+ base = service.now()
+ clock = [base]
+ service.now = lambda: clock[0]
+ release = asyncio.Event()
+ fail = False
+ listings = []
+
+ async def discover_models(vendor, *_args, **_kwargs):
+ listings.append(vendor)
+ await release.wait()
+ if fail:
+ raise RuntimeError("listing unavailable")
+ return [DiscoveredModel(id="gpt-6-chat")]
+
+ service.adapter.discover_models = discover_models
+ service.start_background_discovery()
+ try:
+ # A viewer's pass is listing the Source when the refusal arrives: that listing answers it.
+ service.request_inventory_refresh()
+ async with asyncio.timeout(2):
+ while not listings:
+ await asyncio.sleep(0.01)
+ service.mark_stale_listing(_openai(service), "gpt-6-astra")
+ release.set()
+ await _settled(service)
+ assert listings == ["openai"]
+ assert [model.id for model in _openai(service).models] == ["gpt-6-chat"]
+ # A failing listing backs off past the debounce, whoever marks it.
+ fail = True
+ for minutes in (16, 32):
+ clock[0] = base + timedelta(minutes=minutes)
+ service.mark_stale_listing(_openai(service), "gpt-6-chat")
+ await _settled(service)
+ clock[0] = base + timedelta(minutes=48)
+ service.mark_stale_listing(_openai(service), "gpt-6-chat")
+ await _settled(service)
+ assert listings == ["openai"] * 3
+ finally:
+ await service.stop()
+
+
+async def test_a_stale_mark_drops_nothing_once_its_source_identity_changed(tmp_path):
+ """MH-HOSTED-GATEWAY-009: a refusal belongs to the identity that gave it; a replaced key or URL voids it."""
+
+ service = _stale_listing_service(tmp_path)
+ _openai(service).last_discovered_at = None
+
+ async def discover_models(*_args, **_kwargs):
+ return [DiscoveredModel(id="gpt-6-chat")]
+
+ service.adapter.discover_models = discover_models
+ service.start_background_discovery()
+ try:
+ service.mark_stale_listing(_openai(service), "gpt-6-astra")
+ _openai(service).base_url = "https://replaced.example/v1"
+ await _settled(service)
+ finally:
+ await service.stop()
+ assert [model.id for model in _openai(service).models] == ["gpt-6-chat", "gpt-6-astra"]
+ assert service._stale_listings == {}
+
+
+async def test_every_refused_row_leaves_but_a_retired_tombstone_stays(tmp_path):
+ """MH-HOSTED-GATEWAY-009: refusals of two executor models both count, and a row the user retired is kept."""
+
+ service = _stale_listing_service(tmp_path)
+ openai = _openai(service)
+ openai.last_discovered_at = None
+ openai.models.append(ModelHubModelConfig(id="gpt-6-luna", provenance="discovered"))
+ openai.models.append(ModelHubModelConfig(id="gpt-6-sol", provenance="discovered"))
+
+ async def discover_models(*_args, **_kwargs):
+ return [DiscoveredModel(id="gpt-6-chat")]
+
+ service.adapter.discover_models = discover_models
+ service.start_background_discovery()
+ try:
+ # Two turns frozen on different executor models are each refused.
+ service.mark_stale_listing(_openai(service), "gpt-6-astra")
+ service.mark_stale_listing(_openai(service), "gpt-6-luna")
+ service.mark_stale_listing(_openai(service), "gpt-6-sol")
+ # The user retires gpt-6-sol before the listing returns.
+ next(model for model in _openai(service).models if model.id == "gpt-6-sol").retired = True
+ await _settled(service)
+ finally:
+ await service.stop()
+ models = {model.id: bool(model.retired) for model in _openai(service).models}
+ assert models == {"gpt-6-chat": False, "gpt-6-sol": True}
+
+
+async def test_a_disowned_row_leaves_even_when_the_rest_of_the_listing_must_wait(tmp_path, monkeypatch):
+ """MH-HOSTED-GATEWAY-009: a supply change elsewhere in the listing does not hold back the disowned row."""
+
+ service = _stale_listing_service(tmp_path)
+ _openai(service).last_discovered_at = None
+
+ async def discover_models(*_args, **_kwargs):
+ return [DiscoveredModel(id="gpt-6-chat"), DiscoveredModel(id="gpt-6-new")]
+
+ service.adapter.discover_models = discover_models
+ # Admitting gpt-6-new is the change a manual refresh must confirm.
+ monkeypatch.setattr(service, "_moves_supply", lambda _previous, updated: any(
+ model.id == "gpt-6-new" for source in updated.sources for model in source.models
+ ))
+ service.start_background_discovery()
+ try:
+ service.mark_stale_listing(_openai(service), "gpt-6-astra")
+ await _settled(service)
+ finally:
+ await service.stop()
+ assert [model.id for model in _openai(service).models] == ["gpt-6-chat"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(("case", "handles", "invoked", "status", "served_by", "failed_attempts"), [
# A failure normal routing fails over from moves on; a refusal before output that reported no tokens is unbilled.