Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(model-hub): state the effort menu's refused tiers in menu values on the picker read (#2590)

Call White50b4754ff2a1
diff --git a/config/v2_config.py b/config/v2_config.py
index f5b6dbc6c..1161f4d21 100644
--- a/config/v2_config.py
+++ b/config/v2_config.py
@@ -2636,6 +2636,7 @@ class ModelHubBackendModelConfig:
# server-owned policy into persisted state.
"locked",
"routeable",
+ "effort_menu",
}
if set(payload) - allowed:
raise ValueError("Config 'model_hub.agents.models' contains unknown fields")
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index 11946d5b0..a0a97efe7 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -43,6 +43,7 @@ from config.v2_config import (
validate_model_hub_source_client_nonce,
)
from core.agent_auth_service import BackendLoginInProgressError
+from core.agent_core.ai.registry import sent_effort
from core.agent_core.messages import HostedCost
from core.os_errors import format_os_errno, local_error_detail
from core.services.settings import default_config
@@ -752,6 +753,19 @@ def _executor_failure(outcome: RawCallOutcome | None, say: Callable[..., str], *
return say("modelHub.hostedTools.providerRefused", detail=detail or outcome.kind.value)
+def _sent_effort(backend: str, source: ModelHubSourceConfig, model_id: str, effort: str) -> Optional[str]:
+ """The effort token ``backend`` sends a hop for the menu value ``effort``: the key its evidence is held under.
+
+ Vibey's adapter for the hop's protocol says it (``registry.sent_effort``),
+ so the menu and the wire cannot drift; the native CLIs send the value as
+ the menu names it.
+ """
+
+ if backend == "vibey":
+ return sent_effort(source.protocol, model_id, effort)
+ return effort
+
+
def _google_admission_skip_reason(
frontend_protocol: str | None,
source_protocol: str,
@@ -5123,7 +5137,14 @@ class ModelHubService:
unavailable_source_ids=unavailable_source_ids,
live_recovery=live_recovery,
)
- menu_model_ids = [model.id for model in agent.models if model.id not in hidden]
+ menu_models = [model for model in agent.models if model.id not in hidden]
+ model_resolutions = {
+ model.id: resolve_model_hub_turn(
+ config, backend, model.id, now=now, unavailable_source_ids=unavailable_source_ids,
+ live_recovery=live_recovery,
+ )
+ for model in menu_models
+ }
model_supply = [
{
"model_id": model_id,
@@ -5131,12 +5152,17 @@ class ModelHubService:
"route_origin": model_resolution.route_origin,
"has_runnable_hop": bool(model_resolution.candidate_hops),
}
- for model_id in menu_model_ids
- for model_resolution in (resolve_model_hub_turn(
- config, backend, model_id, now=now, unavailable_source_ids=unavailable_source_ids,
- live_recovery=live_recovery,
- ),)
+ for model_id, model_resolution in model_resolutions.items()
]
+ # Only Hub turns reach the evidence; a Direct Agent's menu is the backend's own.
+ effort_menus = {
+ model.id: (
+ self._effort_menu(backend, model, model_resolutions[model.id], now)
+ if agent.mode == "hub"
+ else {"omitted": [], "not_on": {}}
+ )
+ for model in menu_models
+ }
selected_model_id = (
(resolution.requested_model or None)
if agent.mode == "hub"
@@ -5228,7 +5254,9 @@ class ModelHubService:
),
"model_supply": model_supply if agent.mode == "hub" else None,
"named_agents": named_agents,
- "catalog_models": self._catalog_models_payload(agent),
+ "catalog_models": [
+ {**model, "effort_menu": effort_menus[model["id"]]} for model in self._catalog_models_payload(agent)
+ ],
"builtin_models": builtin_models,
}
@@ -5893,14 +5921,58 @@ class ModelHubService:
read its file at startup.
"""
- feature = EvidenceFeature.effort_or_none(wire_effort(request, getattr(request, "protocol", "")))
+ return self._refuses_effort(
+ source, model_id, wire_effort(request, getattr(request, "protocol", "")), self.now(),
+ )
+
+ def _refuses_effort(
+ self, source: ModelHubSourceConfig, model_id: str, token: Optional[str], now: datetime
+ ) -> bool:
+ """Whether this hop has negative evidence for the effort token a request sends; None sends none."""
+
+ feature = EvidenceFeature.effort_or_none(token)
if feature is None:
return False
view = self.capability_evidence.view(
- source, model_id, feature, engine=self.engine_stamp, shape=HOSTED_PROBE_SHAPE, now=self.now(),
+ source, model_id, feature, engine=self.engine_stamp, shape=HOSTED_PROBE_SHAPE, now=now,
)
return view.status == "unsupported"
+ def _effort_menu(
+ self, backend: BackendName, model: ModelHubBackendModelConfig, resolution: ModelHubTurnResolution, now: datetime
+ ) -> dict:
+ """Which of this menu model's efforts routing refuses now, in menu vocabulary (capability model §3.4).
+
+ A tier is ``omitted`` when a request at it would end in
+ ``effort_unsupported``: every hop the route would try or wait for refuses
+ the token the backend sends for it, as ``_effort_refused`` decides for a
+ request. A tier refused on some of those hops only is ``not_on`` their
+ Sources.
+ """
+
+ hops = list(
+ {
+ (hop.source.id, hop.model_id): hop
+ for hop in (*resolution.candidate_hops, *resolution.self_healing_hops(None))
+ if hop.source is not None and hop.model_id is not None
+ }.values()
+ )
+ omitted: list[str] = []
+ not_on: dict[str, list[str]] = {}
+ if not hops:
+ return {"omitted": omitted, "not_on": not_on}
+ # One request carries one token to every hop it may try: the one the backend writes for the hop it builds
+ # the request on, the first. ``_effort_refused`` reads that same token from the request.
+ primary = hops[0]
+ for effort in model.reasoning_efforts:
+ token = _sent_effort(backend, primary.source, primary.model_id, effort)
+ refusing = [hop.source.id for hop in hops if self._refuses_effort(hop.source, hop.model_id, token, now)]
+ if refusing and len(refusing) == len(hops):
+ omitted.append(effort)
+ elif refusing:
+ not_on[effort] = list(dict.fromkeys(refusing))
+ return {"omitted": omitted, "not_on": not_on}
+
def _effort_capable_hop_may_recover(
self, backend: str, model_id: str, supply_channel: Any, request: Any
) -> bool:
diff --git a/docs/plans/model-hub-capability-model.md b/docs/plans/model-hub-capability-model.md
index 9d4d54823..88ff89148 100644
--- a/docs/plans/model-hub-capability-model.md
+++ b/docs/plans/model-hub-capability-model.md
@@ -132,7 +132,7 @@ and is pruned at the next write.
| Consumer | Rule |
| --- | --- |
-| Effort menu for a model on an Agent | each hop's declared tiers, then the union across hops. A hop's declared tiers follow `model-hub-reasoning-tiers.md`'s ladder: an upstream reasoning signal (the protocol's ladder, bounded by the model's catalog row when it has one), else the bundled backend row's efforts minus that row's `client_presets` (provenance `catalog`; it outranks a user list, with a visible override event), else the user's list, else models.dev `reasoning_options` when the catalog row is built. A persisted `catalog` list is also read without `client_presets`, since released builds wrote them, and Vibey's own model rows always drop them at config load (3.2.0 saved rows with Codex's `ultra`; Vibey is never the client a preset belongs to, and no provider documents `ultra` as a wire effort). A user-authored Source list is never filtered. The overlay's `wire_efforts` slots in before the backend row once a vendor needs it; no shipped entry sets it yet. A tier is omitted only if **every** hop that can serve the model has negative evidence for it. A tier with negative evidence on some hops is shown with "not on: <Source>". Model Hub projects this per hop as the chain API's read-only `refused_efforts` (`agent-chain.schema.json`); the UI renders the omission and the hint from it. |
+| Effort menu for a model on an Agent | each hop's declared tiers, then the union across hops. A hop's declared tiers follow `model-hub-reasoning-tiers.md`'s ladder: an upstream reasoning signal (the protocol's ladder, bounded by the model's catalog row when it has one), else the bundled backend row's efforts minus that row's `client_presets` (provenance `catalog`; it outranks a user list, with a visible override event), else the user's list, else models.dev `reasoning_options` when the catalog row is built. A persisted `catalog` list is also read without `client_presets`, since released builds wrote them, and Vibey's own model rows always drop them at config load (3.2.0 saved rows with Codex's `ultra`; Vibey is never the client a preset belongs to, and no provider documents `ultra` as a wire effort). A user-authored Source list is never filtered. The overlay's `wire_efforts` slots in before the backend row once a vendor needs it; no shipped entry sets it yet. A tier is omitted only if **every** hop that can serve the model has negative evidence for it: every hop the route would try or wait for, exactly the hops whose refusal ends a request in `effort_unsupported`. A tier with negative evidence on some hops is shown with "not on: <Source>". Evidence is keyed by the token a client puts on the wire, which can differ from the menu value: Vibey sends menu `minimal` to Anthropic as `low`. So Model Hub decides both in menu values, where routing decides, and the picker's own read (`GET /api/models/agents/<backend>/models`, an Editor surface) carries the result per model as the response-only `effort_menu`. Vibey's adapters map each menu value through the one function their request builders use (`registry.sent_effort`); a native CLI sends the value as the menu names it. The chain API keeps each hop's read-only `refused_efforts` (`agent-chain.schema.json`) for the evidence detail. |
| Effort at request time | Forwarded unchanged (reasoning-intent). A hop with **negative** evidence for that effort is skipped for this request, as if it were not a candidate. If no candidate remains, the request fails before invocation with `effort_unsupported` naming the effort and the Sources, instead of a provider 400. Never clamped silently. |
| Hosted tool mode on a hop | `native` only with positive `call` evidence taken on the current engine and probe shape, and the user's toggle on; otherwise the gateway proxy (`vibey-hosted-tools.md`). A wrong native declaration is a terminal 400 or a silent drop; the proxy is lossless. So this feature needs positive evidence, unlike efforts. |
| Native payload replay | only to the same Source, and only while its `replay` evidence is positive and taken on the current engine and probe shape. The decision is made at every request from `HopResolution`; the transcript never stores it. |
diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.md
index bf7a9791f..cc44f55e7 100644
--- a/docs/plans/model-hub-contracts/api.md
+++ b/docs/plans/model-hub-contracts/api.md
@@ -56,7 +56,7 @@ runtime open. Once models are configured it participates in the runtime Stop gua
| PUT `/api/models/agents/<backend>/sources` | `{order: string[], force?: boolean, would_remove_hops?: RouteHopRef[], would_interrupt?: SupplyGap[]}` → guarded `409` or `{agent: AgentSupply}` | Atomically replaces default membership/order, preserves manual overrides, and guards effective hop removal or protected-supply loss. Pure reordering needs no guard. |
| POST `/api/models/agents/<backend>/chains/reorder` | `{order?: string[], force?: boolean, would_remove_hops?: RouteHopRef[], would_interrupt?: SupplyGap[]}` → guarded `409` or `{agent: AgentSupply}` | Compatibility default-order entry point with the same guards as Sources PUT. Never reorders manual arrays. Without order, returns the current projection. New UI uses Sources PUT. |
| PATCH `/api/models/agents/<backend>/mode` | `{mode}` → `{agent: AgentSupply}` | Explicit `hub` / `direct` switch. Direct-to-Hub serializes with takeover under the shared migration guard, interrupts running work on that backend and retires the managed runtime, and returns `mode_switch_blocked` only for a native config the CLI cannot parse or a guard refusal. Native auth, including an importable login, stays native beside the Hub and remains a migration candidate; CLIs running outside Avibe do not block the switch because it changes no native credential. It does not auto-create or adopt a native Source; native custody takeover is the optional grouped migration transaction. Hub-to-Direct changes only the mode, but under the same guard: it interrupts running work on that backend and retires the managed runtime, so no turn keeps calling the Hub after the backend has left it, and a guard refusal returns `mode_switch_blocked`. |
-| GET `/api/models/agents/<backend>/models` | → `{agent: {backend, mode, catalog_models}}` | Picker-safe catalog read. It exposes no Source order, Route, or credential-bearing supplier data. Every OpenCode row carries required `native_protocol`; Claude and Codex rows omit it. |
+| GET `/api/models/agents/<backend>/models` | → `{agent: {backend, mode, catalog_models}}` | Picker-safe catalog read. It exposes no Source order, Route, or credential-bearing supplier data. Every OpenCode row carries required `native_protocol`; Claude and Codex rows omit it. Every row carries the response-only `effort_menu` (`backend-model.schema.json`): which of its `reasoning_efforts` routing refuses now, in the values the menu offers, so the picker omits or annotates them without reading Sources or Routes. |
| GET `/api/models/agents/<backend>/models/candidates` | → `{candidates: {builtin: Candidate[], providers: Candidate[], in_list: Candidate[]}}` | Server-owned picker projection. It returns addable built-ins, deduplicated ordered-provider inventory, and every current menu row with the same exact supplier projection; it is independent of backend mode and contains no credentials. Every OpenCode Candidate carries server-derived `native_protocol`; other backends omit it. Only `in_list` candidates may carry optional `group_if_removed: "builtin" | "providers" | null`, naming the group where that id would be offered after removal. A `providers` candidate whose id names a cached models.dev model exactly also carries that match's `models_dev_id`, `context_window`, `max_output_tokens`, `input_modalities`, `output_modalities`, `supports_tools`, and `supports_reasoning` together; those seven are absent otherwise. |
| PUT `/api/models/agents/<backend>/models` | `{baseline: BackendModel[], models: BackendModel[], expected_suppliers?: {<id>: [{source_id, model_id}]}, force?: boolean, would_remove_hops?: RouteHopRef[], would_interrupt?: SupplyGap[]}` → guarded `409`, stale-candidate `409`, `{agent: AgentSupply}`, or `{agent: AgentSupply, removed_hops: RouteHopRef[], interrupted: SupplyGap[]}` | Applies one full-list backend catalog edit with optimistic merge. OpenCode rows require a literal `native_protocol: openai_responses | anthropic`; rows for other backends forbid it. Each caller addition still absent from the latest list starts automatic without a route key. A listed supplier-projection mismatch refuses atomically with the separate exact shape `{ok: false, contract_version, error: "candidate_suppliers_changed", detail, changed}`; a concurrently added row keeps its existing Route. A removal whose row has a Route, or whose row an enabled Agent selects, uses the exact echoed-plan guard and then atomically removes that Route (see **Model-list save** under Supply guard). Its success carries `removed_hops` and `interrupted` whenever either is nonempty; any other removal is ordinary `{agent}` success. Supplier inventory remains unchanged. |
| GET `/api/models/catalog/models-dev?query=<text>` | → `{matches: ModelsDevMatch[]}` | Read-only metadata lookup through the server-owned conditional cache. Results keep the shipped provider fields, deduplicate aggregator copies by model id, rank first-party matches first, add `first_party`, derive `native_protocol` from the model id's last path segment through the vendor map, cap results at 8, and never persist automatically. |
diff --git a/docs/plans/model-hub-contracts/backend-model.schema.json b/docs/plans/model-hub-contracts/backend-model.schema.json
index ee1b550db..e6d0a3d5a 100644
--- a/docs/plans/model-hub-contracts/backend-model.schema.json
+++ b/docs/plans/model-hub-contracts/backend-model.schema.json
@@ -90,6 +90,24 @@
"routeable": {
"type": "boolean",
"description": "Server-derived. False only for a backend-owned selector such as Claude Code Default that never names a Route key."
+ },
+ "effort_menu": {
+ "type": "object",
+ "description": "Server-derived on Agent reads, accepted and dropped on write like locked and routeable. Which of reasoning_efforts routing refuses now, in the values the menu offers (capability model §3.4): omitted when every hop the route would try or wait for has negative evidence for the token the backend sends for it, so a request there would end in effort_unsupported; not_on names the Sources of the hops that refuse a tier the others accept. Empty for an Agent in Direct mode.",
+ "required": ["omitted", "not_on"],
+ "additionalProperties": false,
+ "properties": {
+ "omitted": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
+ "not_on": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "array",
+ "items": { "type": "string", "pattern": "^src_[a-z0-9]{8,}$" },
+ "minItems": 1,
+ "uniqueItems": true
+ }
+ }
+ }
}
}
}
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 1e4c273f3..83a4dedd7 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -277,6 +277,7 @@ scenarios:
- tests/test_model_hub_evidence_rules.py::test_a_refused_effort_still_waits_for_a_capable_hop_that_is_cooling_down
- tests/test_model_hub_reasoning_tiers.py::test_the_hop_declares_exactly_its_models_levels
- tests/test_model_hub_reasoning_tiers.py::test_rows_saved_by_3_2_0_are_derived_again_once_at_startup
+ - tests/test_model_hub_evidence_rules.py::test_the_picker_menu_states_which_efforts_routing_refuses_in_menu_values
- id: MH-HOSTED-VIBEY-002
name: Every native hosted item becomes one hosted call with a provider-neutral action and its record; citations stay on the text that cites them; the response's counters and reported cost go to its summary
status: covered
diff --git a/tests/test_model_hub_api.py b/tests/test_model_hub_api.py
index ba022c6a7..531d7cb4d 100644
--- a/tests/test_model_hub_api.py
+++ b/tests/test_model_hub_api.py
@@ -2409,6 +2409,8 @@ def test_backend_catalog_add_edit_and_runtime_refresh(
"reasoning_efforts": ["low", "high"],
"locked": False,
"routeable": True,
+ # Server-derived like ``locked``: a client may send back the row it read, and the read states it again.
+ "effort_menu": {"omitted": [], "not_on": {}},
}
if backend == "opencode":
added["native_protocol"] = "openai_responses"
@@ -2914,6 +2916,7 @@ def test_backend_catalog_add_validates_supplier_echo_and_stores_draft_literally(
"reasoning_efforts": [],
"locked": False,
"routeable": True,
+ "effort_menu": {"omitted": [], "not_on": {}},
}
before = store.config.to_payload()
diff --git a/tests/test_model_hub_evidence_rules.py b/tests/test_model_hub_evidence_rules.py
index 570b4ff6c..6db7b0f19 100644
--- a/tests/test_model_hub_evidence_rules.py
+++ b/tests/test_model_hub_evidence_rules.py
@@ -381,3 +381,83 @@ async def test_a_refusing_runnable_hop_never_makes_recovery_spin(tmp_path):
assert served.source_id == cooling.id
assert clock.delays and all(delay > 0 for delay in clock.delays)
assert [invocation[0] for invocation in service.adapter.invocations] == [cooling.id]
+
+
+MENU_CASES = [
+ # (case, hops as (Source id, protocol, upstream model), refused (Source id, wire token) pairs,
+ # the menu row's efforts, the expected effort_menu)
+ (
+ "every-hop-or-some",
+ [("src_menuhop01", "openai_responses", "gpt-6-sol"), ("src_menuhop02", "openai_responses", "gpt-6-sol")],
+ [("src_menuhop01", "xhigh"), ("src_menuhop02", "xhigh"), ("src_menuhop01", "high")],
+ ["low", "high", "xhigh"],
+ {"omitted": ["xhigh"], "not_on": {"high": ["src_menuhop01"]}},
+ ),
+ # Vibey sends menu `minimal` to Anthropic as `low`, so a refused `low` refuses both menu values.
+ (
+ "menu-value-differs-from-wire",
+ [("src_menuhop03", "anthropic", "claude-opus-4-7")],
+ [("src_menuhop03", "low")],
+ ["minimal", "low", "high"],
+ {"omitted": ["minimal", "low"], "not_on": {}},
+ ),
+ # One request carries one token to every hop: the one written for the hop it is built on. Behind an OpenAI
+ # primary, menu `minimal` reaches an Anthropic fallback as `minimal` too, so that is the evidence it is read under.
+ (
+ "the-primary-hop-decides-the-token",
+ [("src_menuhop05", "openai_responses", "gpt-6-sol"), ("src_menuhop06", "anthropic", "claude-opus-4-7")],
+ [("src_menuhop05", "minimal"), ("src_menuhop06", "minimal")],
+ ["minimal", "low"],
+ {"omitted": ["minimal"], "not_on": {}},
+ ),
+ (
+ "nothing-refused",
+ [("src_menuhop04", "openai_responses", "gpt-6-sol")],
+ [],
+ ["low", "high"],
+ {"omitted": [], "not_on": {}},
+ ),
+]
+
+
[email protected]("hops,refused,efforts,expected", [case[1:] for case in MENU_CASES], ids=[c[0] for c in MENU_CASES])
+def test_the_picker_menu_states_which_efforts_routing_refuses_in_menu_values(tmp_path, hops, refused, efforts, expected):
+ """MH-HOSTED-006: the Editor picker's read omits a tier every hop refuses and names the Sources of a tier only
+ some refuse, in the values the menu offers, mapped to each hop's wire token by Vibey's own adapters."""
+ from config.v2_config import ModelHubBackendModelConfig
+ from core.handlers.model_hub.capability_evidence import EvidenceRecord, identity_digest
+ from core.handlers.model_hub.hosted_tools import HOSTED_PROBE_SHAPE
+ from tests.test_model_hub_vibey_consumer import _vibey_service
+
+ sources = [_source(source_id, source_id, vendor=protocol.split("_")[0], protocol=protocol, model_id=model)
+ for source_id, protocol, model in hops]
+ service = _vibey_service(tmp_path, sources)
+ service.store.config.agents["vibey"].models = [ModelHubBackendModelConfig(id="menu-alias", reasoning_efforts=efforts)]
+ live = service.store.load().sources
+ by_id = {source.id: source for source in live}
+ for source_id, token in refused:
+ source = by_id[source_id]
+ service.capability_evidence.write(
+ EvidenceRecord(
+ source_id=source_id,
+ model_id=source.models[0].id,
+ feature=EvidenceFeature.effort(token),
+ status="unsupported",
+ basis="observed",
+ observed_at=service.now(),
+ engine=service.engine_stamp,
+ shape=HOSTED_PROBE_SHAPE,
+ source_identity=identity_digest(source),
+ detail="400 invalid_request_error",
+ ),
+ live=live,
+ )
+
+ agent = service.get_agent_sources("vibey")
+
+ [row] = agent["catalog_models"]
+ assert row["effort_menu"] == expected
+ # The menu itself is unchanged: the editor saves it, and the read it submits back is accepted.
+ assert row["reasoning_efforts"] == efforts
+ assert list(ModelHubBackendModelConfig.from_payload(row).reasoning_efforts) == efforts
+ _assert_valid("backend-model.schema.json", row)