Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(model-hub): learn capability evidence from upstream errors and stop offering client presets as wire efforts (#2551)

Call White0455b7f459a9
diff --git a/config/v2_config.py b/config/v2_config.py
index 9270e8ce4..f5b6dbc6c 100644
--- a/config/v2_config.py
+++ b/config/v2_config.py
@@ -9,7 +9,7 @@ import shutil
import tempfile
import threading
from contextlib import contextmanager
-from dataclasses import dataclass, field, fields
+from dataclasses import dataclass, field, fields, replace
from datetime import datetime
from pathlib import Path
from typing import (
@@ -3016,6 +3016,26 @@ class ModelHubAgentSupplyConfig:
MODEL_HUB_HOSTED_TOOLS: tuple[str, ...] = HOSTED_TOOLS
+def _drop_vibey_client_presets(agent: "ModelHubAgentSupplyConfig") -> None:
+ """Keep client presets (Codex ``ultra``) out of Vibey's model rows.
+
+ A preset is a name another client maps before sending, never a wire tier,
+ and Vibey is not that client. 3.2.0 saved Vibey rows carrying one (#2539),
+ so every read drops them here, the single owner of those rows: the catalog
+ payload, the picker, the launch and the API then agree, and the next save
+ writes the clean list. No I/O: the preset index is read once per process.
+ """
+
+ from vibe.backend_model_catalog import without_client_presets
+
+ agent.models = [
+ replace(model, reasoning_efforts=without_client_presets((model.id,), model.reasoning_efforts))
+ if model.reasoning_efforts
+ else model
+ for model in agent.models
+ ]
+
+
@dataclass(frozen=True)
class ModelHubHostedToolConfig:
"""One hosted tool's user toggle (``vibey-hosted-tools.md`` §3.3).
@@ -3217,6 +3237,7 @@ class ModelHubConfig:
runtime_default_applied=runtime_default_applied,
hosted_tools=_hosted_tools_from_payload(payload.get("hosted_tools"), sources),
)
+ _drop_vibey_client_presets(config.agents["vibey"])
agents = config.agents
for backend in MODEL_HUB_BACKENDS:
configured_sources = agents[backend].sources
diff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.py
index 7550be7e6..a2d8d90df 100644
--- a/core/handlers/model_hub/adapter.py
+++ b/core/handlers/model_hub/adapter.py
@@ -138,6 +138,25 @@ class RawOutcomeKind(str, Enum):
PROTOCOL_ERROR = "protocol_error" # malformed/unparseable upstream response
+@dataclass(frozen=True)
+class UpstreamRejection:
+ """Closed facts of an HTTP 400 the upstream returned before any output.
+
+ The only input of the capability-evidence rules (``model-hub-capability-model.md``
+ §3.2). L1 fills it from the error envelope: raw machine tokens, and the one
+ token each recognized message names. It never carries prose, and the
+ failover decision never reads it.
+ """
+
+ status: int
+ code: str | None = None # raw machine code, e.g. "invalid_value"
+ error_type: str | None = None # raw error type, e.g. "invalid_request_error"
+ param: str | None = None # raw parameter path, e.g. "reasoning.effort"
+ invalid_value: str | None = None # "Invalid value: '<value>'"
+ unsupported_tool_type: str | None = None # "Unsupported tool type: <type>"
+ invalid_encrypted_content_block: str | None = None # "Invalid encrypted_content in <kind> block"
+
+
@dataclass(frozen=True)
class RawCallOutcome:
"""Uninterpreted per-call result. Classification is L2's job, not L1's."""
@@ -175,6 +194,8 @@ class RawCallOutcome:
# Recognized local OS failure only, never an upstream code or error prose.
# Display-only; does not participate in classification or Source health.
os_errno: int | None = None
+ # A 400 before output, as closed tokens for the capability-evidence rules.
+ rejection: UpstreamRejection | None = None
class ObservationOutcome(str, Enum):
diff --git a/core/handlers/model_hub/capability_evidence.py b/core/handlers/model_hub/capability_evidence.py
index 284234527..3ec280554 100644
--- a/core/handlers/model_hub/capability_evidence.py
+++ b/core/handlers/model_hub/capability_evidence.py
@@ -37,6 +37,9 @@ MAX_RECORDS = 4000
OBSERVED_TTL_SECONDS = 7 * 24 * 3600
PROBE_REVERIFY_AFTER_SECONDS = 30 * 24 * 3600
DETAIL_MAX_CHARS = 64
+# The engine stamp when the engine manifest cannot be read. It names no release,
+# so probe evidence taken under it, or checked against it, never vouches for a wire.
+UNKNOWN_ENGINE_STAMP = "unknown-engine"
FeatureKind = Literal["effort", "hosted_tool"]
EvidenceStatus = Literal["verified", "unsupported"]
@@ -73,6 +76,12 @@ class EvidenceFeature:
def effort(cls, name: str) -> "EvidenceFeature":
return cls("effort", name)
+ @classmethod
+ def effort_or_none(cls, name: object) -> Optional["EvidenceFeature"]:
+ """The effort feature, or None for a name evidence cannot hold (free-form efforts are allowed)."""
+
+ return cls("effort", name) if isinstance(name, str) and _NAME_RE.fullmatch(name) else None
+
@classmethod
def hosted(cls, tool: str, aspect: Literal["call", "replay"]) -> "EvidenceFeature":
return cls("hosted_tool", tool, aspect)
@@ -204,7 +213,11 @@ def resolve(
if record.basis == "observed":
# Relays change; an expired negative record falls back to the priors.
return UNKNOWN if age > OBSERVED_TTL_SECONDS else EvidenceView("unsupported", None, record)
- wire_stale = record.engine != engine or record.shape != shape
+ wire_stale = (
+ record.engine != engine
+ or record.shape != shape
+ or UNKNOWN_ENGINE_STAMP in (record.engine, engine)
+ )
stale: Optional[Staleness] = "wire" if wire_stale else "age" if age > PROBE_REVERIFY_AFTER_SECONDS else None
if record.feature.kind == "hosted_tool" and wire_stale:
# Evidence taken on another wire cannot vouch for this one, and a silent
@@ -218,36 +231,46 @@ class CapabilityEvidenceStore:
Small: bounded by Sources x models x features. The controller is the only
writer (probes and the classifier both run there; other processes reach it
- over RPC), so the file is read once and memory is authoritative afterwards.
- Lookups on the request path therefore never touch the disk.
+ over RPC), so the file is read once, when the store is built at controller
+ startup, and memory is authoritative afterwards. No lookup ever touches the
+ disk, so none can block the loop.
"""
def __init__(self, path: Path, *, max_records: int = MAX_RECORDS):
self.path = path
self.max_records = max_records
self._lock = threading.RLock()
- self._records: dict[tuple, EvidenceRecord] | None = None
-
- def _loaded(self) -> dict[tuple, EvidenceRecord]:
- if self._records is None:
+ self._records: dict[tuple, EvidenceRecord] = {}
+ self._by_hop: dict[tuple[str, str], list[EvidenceRecord]] = {}
+ self._remember(self._read_file())
+
+ def _read_file(self) -> dict[tuple, EvidenceRecord]:
+ try:
+ payload = json.loads(self.path.read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ payload = []
+ records: dict[tuple, EvidenceRecord] = {}
+ for item in payload if isinstance(payload, list) else []:
try:
- payload = json.loads(self.path.read_text(encoding="utf-8"))
- except (OSError, ValueError):
- payload = []
- records: dict[tuple, EvidenceRecord] = {}
- for item in payload if isinstance(payload, list) else []:
- try:
- record = EvidenceRecord.from_payload(item)
- except (TypeError, ValueError):
- # An unreadable record is unknown, never negative.
- continue
- records[record.key] = record
- self._records = records
- return self._records
+ record = EvidenceRecord.from_payload(item)
+ except (TypeError, ValueError):
+ # An unreadable record is unknown, never negative.
+ continue
+ records[record.key] = record
+ return records
+
+ def _remember(self, records: dict[tuple, EvidenceRecord]) -> None:
+ by_hop: dict[tuple[str, str], list[EvidenceRecord]] = {}
+ for record in records.values():
+ by_hop.setdefault((record.source_id, record.model_id), []).append(record)
+ # The index first: a reader that sees the new records also sees their index.
+ self._by_hop = by_hop
+ self._records = records
def get(self, source_id: str, model_id: str, feature: EvidenceFeature) -> Optional[EvidenceRecord]:
- with self._lock:
- return self._loaded().get((source_id, model_id, feature.kind, feature.name, feature.aspect or ""))
+ # Published snapshots are never mutated, so a reader needs no lock: a
+ # write builds the next snapshot and swaps the references.
+ return self._records.get((source_id, model_id, feature.kind, feature.name, feature.aspect or ""))
def view(
self,
@@ -262,8 +285,12 @@ class CapabilityEvidenceStore:
return resolve(self.get(source.id, model_id, feature), source=source, engine=engine, shape=shape, now=now)
def records(self) -> list[EvidenceRecord]:
- with self._lock:
- return list(self._loaded().values())
+ return list(self._records.values())
+
+ def for_hop(self, source_id: str, model_id: str) -> list[EvidenceRecord]:
+ """Every record of one (Source, model), without scanning the others."""
+
+ return list(self._by_hop.get((source_id, model_id), ()))
def write(self, record: EvidenceRecord, *, live: Iterable[ModelHubSourceConfig]) -> None:
"""Replace the record for its key; prune records of removed or re-identified Sources.
@@ -277,10 +304,10 @@ class CapabilityEvidenceStore:
with self._lock:
records = {
key: existing
- for key, existing in self._loaded().items()
+ for key, existing in self._records.items()
if identities.get(existing.source_id) == existing.source_identity
}
records[record.key] = record
kept = sorted(records.values(), key=lambda item: item.observed_at)[-self.max_records :]
write_state_document(self.path, [item.payload() for item in kept])
- self._records = {item.key: item for item in kept}
+ self._remember({item.key: item for item in kept})
diff --git a/core/handlers/model_hub/evidence_rules.py b/core/handlers/model_hub/evidence_rules.py
new file mode 100644
index 000000000..54b7d244e
--- /dev/null
+++ b/core/handlers/model_hub/evidence_rules.py
@@ -0,0 +1,92 @@
+"""Narrow rules that learn negative capability evidence from real upstream errors.
+
+``model-hub-capability-model.md`` §3.2: a rule maps an upstream error to exactly
+one (Source, model, feature), and only when the error names that feature.
+Generic errors (5xx, overload, rate limits, auth, any 400 that names nothing
+the request sent) never create evidence. The rules read only the closed tokens
+L1 put in ``UpstreamRejection`` and the request the gateway sent; they never
+change how the request itself is classified or failed over.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, Mapping, Optional
+
+from config.v2_config import ModelHubSourceConfig
+
+from .adapter import UpstreamRejection
+from .capability_evidence import EvidenceFeature
+from .request import declared_tool_types, replays_history_items
+
+# Where each request protocol carries its reasoning effort on the wire.
+_EFFORT_PATHS: Mapping[str, tuple[str, ...]] = {
+ "openai_responses": ("reasoning", "effort"),
+ "openai_chat": ("reasoning_effort",),
+ "anthropic": ("output_config", "effort"),
+}
+_EFFORT_PARAMS = frozenset({"reasoning.effort", "reasoning_effort", "output_config.effort"})
+
+# The hosted result an "Invalid encrypted_content in <kind> block" message names.
+_ENCRYPTED_BLOCK_TOOLS: Mapping[str, str] = {"search_result": "web_search"}
+# Request items that replay a hosted tool's native result, per tool.
+_REPLAYED_RESULT_TYPES: Mapping[str, frozenset[str]] = {
+ "web_search": frozenset({"web_search_tool_result", "web_search_call"}),
+}
+
+SpecLookup = Callable[[str, str, str, str], Any]
+
+
+@dataclass(frozen=True)
+class LearnedEvidence:
+ feature: EvidenceFeature
+ detail: str
+
+
+def wire_effort(request: Mapping[str, Any], protocol: str) -> Optional[str]:
+ """The reasoning effort this request sends, as the wire spells it."""
+
+ value: Any = request
+ for key in _EFFORT_PATHS.get(protocol, ()):
+ value = value.get(key) if isinstance(value, Mapping) else None
+ return value if isinstance(value, str) and value else None
+
+
+def _detail(rejection: UpstreamRejection) -> str:
+ return f"{rejection.status} {rejection.code or rejection.error_type or 'error'}"[:64]
+
+
+def learn(
+ request: Mapping[str, Any],
+ rejection: UpstreamRejection,
+ *,
+ request_protocol: str,
+ source: ModelHubSourceConfig,
+ model_id: str,
+ spec_for: SpecLookup,
+) -> Optional[LearnedEvidence]:
+ """The one feature this rejection names for this request, if any."""
+
+ effort = EvidenceFeature.effort_or_none(wire_effort(request, request_protocol))
+ names_effort_field = rejection.param is None or rejection.param in _EFFORT_PARAMS
+ if effort is not None and names_effort_field and (
+ rejection.invalid_value == effort.name
+ or (
+ rejection.invalid_value is None
+ and rejection.param in _EFFORT_PARAMS
+ and request_protocol == source.protocol
+ )
+ ):
+ return LearnedEvidence(effort, _detail(rejection))
+
+ rejected_type = rejection.unsupported_tool_type
+ if rejected_type is not None and rejected_type in declared_tool_types(request):
+ for tool in ("web_search", "x_search", "image_generation"):
+ spec = spec_for(source.vendor, source.protocol, model_id, tool)
+ if spec is not None and spec.declaration_type == rejected_type:
+ return LearnedEvidence(EvidenceFeature.hosted(tool, "call"), _detail(rejection))
+
+ tool = _ENCRYPTED_BLOCK_TOOLS.get(rejection.invalid_encrypted_content_block or "")
+ if tool is not None and replays_history_items(request, _REPLAYED_RESULT_TYPES[tool]):
+ return LearnedEvidence(EvidenceFeature.hosted(tool, "replay"), _detail(rejection))
+ return None
diff --git a/core/handlers/model_hub/hosted_tools.py b/core/handlers/model_hub/hosted_tools.py
index 9346b1a8c..75db08670 100644
--- a/core/handlers/model_hub/hosted_tools.py
+++ b/core/handlers/model_hub/hosted_tools.py
@@ -26,7 +26,7 @@ from typing import Any, Callable, Mapping, Optional
from config.v2_config import MODEL_HUB_HOSTED_TOOLS, ModelHubConfig, ModelHubHostedToolConfig, ModelHubSourceConfig
from vibe.backend_model_catalog import HostedToolSpec
-from .capability_evidence import HOSTED_TOOL_ASPECTS, CapabilityEvidenceStore, EvidenceFeature
+from .capability_evidence import HOSTED_TOOL_ASPECTS, UNKNOWN_ENGINE_STAMP, CapabilityEvidenceStore, EvidenceFeature
# The request shape a hosted-tool probe sends. Changing the probe or the native
# request it mirrors bumps this, which makes earlier probe evidence wire-stale.
@@ -70,10 +70,15 @@ def engine_stamp() -> str:
Engine upgrades ship as manifest changes, and a patched rebuild of the same
upstream version gets a new asset release, so this is what a wire change
- looks like from here.
+ looks like from here. Read once, when Model Hub's service is built; a
+ manifest that cannot be read is the fixed unknown engine, whose evidence
+ never matches a real release and never raises on a request.
"""
- return _manifest_engine_stamp(os.environ.get("VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH") or None)
+ try:
+ return _manifest_engine_stamp(os.environ.get("VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH") or None)
+ except (OSError, ValueError, TypeError, AttributeError):
+ return UNKNOWN_ENGINE_STAMP
@dataclass(frozen=True)
diff --git a/core/handlers/model_hub/migration.py b/core/handlers/model_hub/migration.py
index 5c942fd0d..9c01c19f4 100644
--- a/core/handlers/model_hub/migration.py
+++ b/core/handlers/model_hub/migration.py
@@ -51,7 +51,7 @@ from core.handlers.model_hub.migration_journal import (
from core.handlers.model_hub.migration_persisted import PersistedInventory
from vibe.backend_model_catalog import (
backend_model_entries,
- bundled_catalog_reasoning_efforts_by_model,
+ bundled_catalog_wire_efforts_by_model,
load_bundled_catalog,
)
from vibe.codex_config import (
@@ -1742,7 +1742,7 @@ async def _prepare_takeover(
provisioned: list[dict[str, str]] = []
source_ids: list[str] = list(retained_source_ids or ())
item_sources: dict[str, dict[str, str]] = {}
- catalog = bundled_catalog_reasoning_efforts_by_model()
+ catalog = bundled_catalog_wire_efforts_by_model()
handoff_attempted = False
try:
# A completed receipt can authorize cleanup of the exact old material
diff --git a/core/handlers/model_hub/reasoning_tiers.py b/core/handlers/model_hub/reasoning_tiers.py
index a98b7d164..d4e572cff 100644
--- a/core/handlers/model_hub/reasoning_tiers.py
+++ b/core/handlers/model_hub/reasoning_tiers.py
@@ -7,7 +7,8 @@ from typing import Literal, Mapping, Sequence
from vibe.backend_model_catalog import (
PROTOCOL_REASONING_EFFORT_DEFAULTS,
- bundled_catalog_reasoning_efforts_for_model,
+ bundled_catalog_wire_efforts_by_model,
+ without_client_presets,
)
ReasoningEffortsSource = Literal["upstream", "catalog", "user"] | None
@@ -30,7 +31,13 @@ def resolve_reasoning_tiers(
existing_source: ReasoningEffortsSource = None,
catalog_efforts_by_model: Mapping[str, tuple[str, ...]] | None = None,
) -> ReasoningTierResolution:
- """Apply upstream, bundled-catalog, then user provenance in that order."""
+ """Apply upstream, bundled-catalog, then user provenance in that order.
+
+ A catalog row is a client's menu: its ``client_presets`` (Codex ``ultra``)
+ are names the client maps before sending, so a Source's wire tiers never
+ include them (#2539). ``catalog_efforts_by_model`` is therefore the wire
+ index, ``bundled_catalog_wire_efforts_by_model()``.
+ """
if supported_parameters is not None and any(
parameter.strip().lower() in _UPSTREAM_REASONING_PARAMETERS
@@ -41,13 +48,21 @@ def resolve_reasoning_tiers(
return ReasoningTierResolution(defaults, "upstream")
catalog_efforts = (
- bundled_catalog_reasoning_efforts_for_model(model_id)
- if catalog_efforts_by_model is None
- else catalog_efforts_by_model.get(model_id)
- )
+ bundled_catalog_wire_efforts_by_model() if catalog_efforts_by_model is None else catalog_efforts_by_model
+ ).get(model_id)
if catalog_efforts is not None:
return ReasoningTierResolution(catalog_efforts, "catalog")
if existing_source == "user" and existing_efforts:
return ReasoningTierResolution(tuple(existing_efforts), "user")
return ReasoningTierResolution((), None)
+
+
+def catalog_wire_efforts(model_id: str, efforts: Sequence[str]) -> tuple[str, ...]:
+ """A catalog-sourced tier list without the client presets of that model's row.
+
+ Also applied when reading a persisted ``catalog`` list, which a released
+ build may have written before presets were known.
+ """
+
+ return tuple(without_client_presets((model_id,), efforts))
diff --git a/core/handlers/model_hub/request.py b/core/handlers/model_hub/request.py
index 085baf681..bd43c94a3 100644
--- a/core/handlers/model_hub/request.py
+++ b/core/handlers/model_hub/request.py
@@ -107,3 +107,29 @@ def without_opaque_history(request: Mapping[str, Any]) -> Mapping[str, Any]:
primary_origin=request.primary_origin,
)
return payload
+
+
+def declared_tool_types(request: Mapping[str, Any]) -> frozenset[str]:
+ """The ``type`` of every tool the request declares."""
+
+ tools = request.get("tools")
+ if not isinstance(tools, list):
+ return frozenset()
+ return frozenset(
+ kind for tool in tools if isinstance(tool, Mapping) and isinstance(kind := tool.get("type"), str)
+ )
+
+
+def replays_history_items(request: Mapping[str, Any], kinds: frozenset[str]) -> bool:
+ """Whether the request's history carries an item of one of these ``type`` values, at any depth."""
+
+ pending: list[Any] = [request.get("messages", request.get("input"))]
+ while pending:
+ value = pending.pop()
+ if isinstance(value, Mapping):
+ if value.get("type") in kinds:
+ return True
+ pending.extend(value.values())
+ elif isinstance(value, list):
+ pending.extend(value)
+ return False
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index 309dd2330..b18bc9a34 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -50,7 +50,9 @@ from storage.models import MESSAGE_TURN_ID_SQL, agent_sessions, messages
from vibe.backend_model_catalog import (
PROTOCOL_REASONING_EFFORT_DEFAULTS,
builtin_display_name,
- bundled_catalog_reasoning_efforts_by_model,
+ bundled_catalog_wire_efforts_by_model,
+ bundled_client_presets_once,
+ bundled_provider_overlay,
hosted_tool_spec,
)
from vibe.model_hub_runtime.api_key_vendors import (
@@ -104,9 +106,12 @@ from .events import (
build_resolution_event,
contains_credential_material,
)
-from .capability_evidence import CapabilityEvidenceStore
+from .capability_evidence import CapabilityEvidenceStore, EvidenceFeature, EvidenceRecord, identity_digest
from .errors import ModelDiscoveryError
+from .evidence_rules import learn as learn_from_rejection
+from .evidence_rules import wire_effort
from .hosted_tools import (
+ HOSTED_PROBE_SHAPE,
HostedToolSettingRefused,
engine_stamp,
hosted_tools_status,
@@ -155,7 +160,7 @@ from .provenance import (
produce_turn_outcome,
)
from .request import ModelHubRequest, without_opaque_history
-from .reasoning_tiers import resolve_reasoning_tiers
+from .reasoning_tiers import catalog_wire_efforts, resolve_reasoning_tiers
from .stream_wire import ProtocolSSEState
from .resolver import (
BackendName,
@@ -1177,6 +1182,11 @@ class ModelHubService:
self.capability_evidence = capability_evidence or CapabilityEvidenceStore(
events.path.parent / "model_hub_capability_evidence.json"
)
+ # Bundled state that never changes while this process runs, resolved once
+ # here at startup so that no request path reads a file on the loop.
+ self.engine_stamp = engine_stamp()
+ bundled_provider_overlay()
+ bundled_client_presets_once()
self.migration_blocked_backends: set[str] = set()
self.migration_guard = migration_guard or self._unavailable_migration_guard
self.migration_reconcile_auth = migration_reconcile_auth
@@ -2477,7 +2487,7 @@ class ModelHubService:
discovered_by_id = {model.id: model for model in canonical_models}
overrides: list[tuple[str, Literal["upstream", "catalog"]]] = []
if catalog_efforts_by_model is None:
- catalog_efforts_by_model = bundled_catalog_reasoning_efforts_by_model()
+ catalog_efforts_by_model = bundled_catalog_wire_efforts_by_model()
def apply_tiers(
model: ModelHubModelConfig,
@@ -2953,7 +2963,7 @@ class ModelHubService:
)
else:
catalog_efforts_by_model = (
- bundled_catalog_reasoning_efforts_by_model()
+ bundled_catalog_wire_efforts_by_model()
)
for model in source.models:
resolution = resolve_reasoning_tiers(
@@ -3740,7 +3750,7 @@ class ModelHubService:
allow_empty=True,
)
else:
- catalog_efforts_by_model = bundled_catalog_reasoning_efforts_by_model()
+ catalog_efforts_by_model = bundled_catalog_wire_efforts_by_model()
for model in source.models:
resolution = resolve_reasoning_tiers(
protocol=source.protocol,
@@ -5179,7 +5189,11 @@ class ModelHubService:
)
proposed_name, proposed_efforts = _storable_backend_model_metadata(
model.display_name,
- model.reasoning_efforts,
+ (
+ catalog_wire_efforts(model.id, model.reasoning_efforts)
+ if model.reasoning_efforts_source == "catalog"
+ else model.reasoning_efforts
+ ),
)
if display_name is None and proposed_name is not None:
display_name = proposed_name
@@ -5775,7 +5789,7 @@ class ModelHubService:
self.store.load(),
evidence=self.capability_evidence,
spec_for=hosted_tool_spec,
- engine=engine_stamp(),
+ engine=self.engine_stamp,
now=self.now(),
)
}
@@ -5795,6 +5809,77 @@ class ModelHubService:
await self._commit_synced(previous, config)
return await asyncio.to_thread(self.hosted_tools_status)
+ def _effort_refused(self, source: ModelHubSourceConfig, model_id: str, request: Any) -> bool:
+ """Whether this hop has negative evidence for the effort the request sends.
+
+ Efforts are forwarded unchanged; a hop known to refuse one is not a
+ candidate for this request (capability model §3.4). Every place that
+ decides on candidates asks this one predicate, so the resolve loop and
+ the recovery loop never disagree. Memory-only and lock-free: the store
+ read its file at startup.
+ """
+
+ feature = EvidenceFeature.effort_or_none(wire_effort(request, getattr(request, "protocol", "")))
+ if feature is None:
+ return False
+ view = self.capability_evidence.view(
+ source, model_id, feature, engine=self.engine_stamp, shape=HOSTED_PROBE_SHAPE, now=self.now(),
+ )
+ return view.status == "unsupported"
+
+ def _effort_capable_hop_may_recover(
+ self, backend: str, model_id: str, supply_channel: Any, request: Any
+ ) -> bool:
+ """Whether a temporarily blocked hop of this route does not refuse the request's effort."""
+
+ resolution = self._invocation_resolution(
+ self.store.load(), cast(BackendName, backend), model_id, supply_channel,
+ )
+ return any(
+ hop.source is not None
+ and hop.model_id is not None
+ and not self._effort_refused(hop.source, hop.model_id, request)
+ for hop in resolution.self_healing_hops(supply_channel)
+ )
+
+ async def _learn_capability_evidence(
+ self, source: ModelHubSourceConfig, model_id: str, request: Any, outcome: RawCallOutcome
+ ) -> None:
+ """Record the one feature a 400 before output names, through the narrow rules."""
+
+ if outcome.rejection is None:
+ return
+ learned = learn_from_rejection(
+ request,
+ outcome.rejection,
+ request_protocol=getattr(request, "protocol", ""),
+ source=source,
+ model_id=model_id,
+ spec_for=hosted_tool_spec,
+ )
+ if learned is None:
+ return
+ try:
+ await asyncio.to_thread(
+ self.capability_evidence.write,
+ EvidenceRecord(
+ source_id=source.id,
+ model_id=model_id,
+ feature=learned.feature,
+ status="unsupported",
+ basis="observed",
+ observed_at=self.now(),
+ engine=self.engine_stamp,
+ shape=HOSTED_PROBE_SHAPE,
+ source_identity=identity_digest(source),
+ detail=learned.detail,
+ ),
+ live=self.store.load().sources,
+ )
+ except (OSError, ValueError):
+ # Evidence is advisory state: losing one record leaves the priors.
+ logger.warning("Model Hub could not record capability evidence", exc_info=True)
+
def hosted_tools_for_hop(self, config: ModelHubConfig, source: ModelHubSourceConfig, model_id: str) -> dict:
"""``HopResolution.hosted_tools`` for a Vibey launch served by this Source and model."""
@@ -5804,7 +5889,7 @@ class ModelHubService:
model_id=model_id,
evidence=self.capability_evidence,
spec_for=hosted_tool_spec,
- engine=engine_stamp(),
+ engine=self.engine_stamp,
now=self.now(),
)
@@ -6809,6 +6894,7 @@ class ModelHubService:
if status in {"active", "standby", "cooldown"}
else status
)
+ refused = self._refused_efforts(source, inspection.model_id, observed_at)
chain.append(
{
"source_id": source.id,
@@ -6819,6 +6905,7 @@ class ModelHubService:
"reason": inspection.reason,
"retry_at": inspection.retry_at,
**({"recovery": inspection.recovery} if inspection.recovery is not None else {}),
+ **({"refused_efforts": refused} if refused else {}),
}
)
current = next(
@@ -6844,6 +6931,28 @@ class ModelHubService:
),
}
+ def _refused_efforts(self, source: ModelHubSourceConfig, model_id: str, now: datetime) -> list[dict]:
+ """The efforts this hop currently refuses, as routing reads the evidence."""
+
+ refused = []
+ for record in self.capability_evidence.for_hop(source.id, model_id):
+ if record.feature.kind != "effort":
+ continue
+ view = self.capability_evidence.view(
+ source, model_id, record.feature, engine=self.engine_stamp, shape=HOSTED_PROBE_SHAPE, now=now,
+ )
+ if view.status == "unsupported":
+ refused.append(record)
+ return [
+ {
+ "effort": record.feature.name,
+ "basis": record.basis,
+ "observed_at": record.payload()["observed_at"],
+ "detail": record.detail,
+ }
+ for record in sorted(refused, key=lambda item: (item.observed_at, item.feature.name))
+ ]
+
def agent_chain(self, backend: str, model_id: object) -> dict:
if backend not in MODEL_HUB_BACKENDS or not isinstance(model_id, str) or not model_id:
raise ModelHubError("mapping_target_unavailable", status=409)
@@ -9022,6 +9131,7 @@ class ModelHubService:
globally_blocked_source_ids: set[str] = set()
unsupported_hops: set[tuple[str, str]] = set()
unsupported_reason: str | None = None
+ effort_rejected_sources: list[str] = []
while True:
async with self._mutation_lock:
if recovery_request is not None and recovery_request.expired:
@@ -9095,6 +9205,9 @@ class ModelHubService:
)
if reason is not None:
raise _InvocationUnsupported(reason)
+ if self._effort_refused(source, target_model, request):
+ effort_rejected_sources.append(source.id)
+ raise _InvocationUnsupported("effort_unsupported")
await self._prepare_engine_for_demand(already_synced=engine_prepared)
engine_prepared = True
handle, outcome, cancelled = await self._invoke(
@@ -9148,6 +9261,7 @@ class ModelHubService:
origin=origin,
)
decision = await self._classify_source_outcome(source, outcome)
+ await self._learn_capability_evidence(source, target_model, request, outcome)
if cancelled is not None:
await self._settle_cancelled_attempt(
source=source,
@@ -9211,6 +9325,7 @@ class ModelHubService:
origin=origin,
)
decision = classify_outcome(outcome, refresh_attempted=True)
+ await self._learn_capability_evidence(source, target_model, request, outcome)
if cancelled is not None:
await self._settle_cancelled_attempt(
source=source,
@@ -9330,11 +9445,29 @@ class ModelHubService:
turn_outcome=ENGINE_DOWN_TURN_OUTCOME,
local_error_detail=format_os_errno(outcome.os_errno),
)
+ if (
+ unsupported_reason == "effort_unsupported"
+ and failed_source is None
+ and self._effort_capable_hop_may_recover(backend, model_id, supply_channel, request)
+ ):
+ # A hop that does not refuse this effort is only blocked for now:
+ # the request waits for it like any other, rather than ending here.
+ unsupported_reason = None
if unsupported_reason is not None and failed_source is None:
# No upstream attempt exists. Exhaustion would invent a failed
# attempt (and a retryable supply state) for a local limitation.
raise ModelHubError(
- unsupported_reason, status=422, turn_outcome=REQUEST_NONFALLBACK_TURN_OUTCOME,
+ unsupported_reason,
+ status=422,
+ turn_outcome=REQUEST_NONFALLBACK_TURN_OUTCOME,
+ data=(
+ {
+ "effort": wire_effort(request, getattr(request, "protocol", "")),
+ "source_ids": list(dict.fromkeys(effort_rejected_sources)),
+ }
+ if unsupported_reason == "effort_unsupported"
+ else None
+ ),
)
final_config, final_resolution = self._inspect_terminal_chain(
backend=cast(BackendName, backend),
@@ -9410,7 +9543,15 @@ class ModelHubService:
resolution = self._invocation_resolution(
config, kwargs["backend"], kwargs["model_id"], supply_channel,
)
- if resolution.candidate_hops:
+ request = kwargs.get("request")
+ if any(
+ hop.source is not None
+ and hop.model_id is not None
+ and not self._effort_refused(hop.source, hop.model_id, request)
+ for hop in resolution.candidate_hops
+ ):
+ # Retry now only for a hop this request may use: a hop
+ # that refuses its effort would spin the loop.
continue
# An action already owed elsewhere in the chain keeps it
# `interrupted` for display, but must not end this request
diff --git a/docs/plans/model-hub-capability-model.md b/docs/plans/model-hub-capability-model.md
index 48d2f43ad..e8a122fa2 100644
--- a/docs/plans/model-hub-capability-model.md
+++ b/docs/plans/model-hub-capability-model.md
@@ -82,12 +82,18 @@ and is pruned at the next write.
### 3.2 Learning from real errors (negative evidence)
-- **Narrow rules, one per feature, in one place:** the gateway's classifier. It sees every upstream error, and
- nothing else writes evidence. A rule maps an upstream error to exactly one (Source, model, feature) only when the
- error names that feature:
- - `invalid_request_error` / `invalid_value` whose message names the request's `reasoning.effort` value → effort;
- - `Unsupported tool type: <type>` for a type the request declared → hosted tool `call`;
- - `Invalid encrypted_content` inside a hosted result block the request replayed → hosted tool `replay`.
+- **Narrow rules, one per feature, in one place:** `core/handlers/model_hub/evidence_rules.py`, run by the
+ gateway's resolution loop on every upstream outcome. Nothing else writes evidence from real traffic. The engine
+ client (L1) projects a 400 before output into an `UpstreamRejection`: the machine code and type from the one
+ error-field extractor, the envelope's `param`, and the single token each recognized message names. The prose
+ never leaves L1, and the failover decision never reads the rejection. A rule maps it to exactly one (Source,
+ model, feature) only when it names that feature:
+ - `Invalid value: '<v>'` where `<v>` is the effort the request sends on the wire, or `param` naming the effort
+ field on an untranslated path → effort `<v>`;
+ - `Unsupported tool type: <type>` for a type the request declared, matched to a catalog tool through the
+ overlay's declaration for that Source and model → hosted tool `call`;
+ - `Invalid encrypted_content in search_result block` while the request replays a web-search result → hosted
+ tool `web_search` `replay`.
- **Generic errors never create evidence.** That covers 5xx, overload, rate limit, auth, and any 400 that does not
name the feature.
- **What happens to the request.**
@@ -126,15 +132,15 @@ and is pruned at the next write.
| Consumer | Rule |
| --- | --- |
-| Effort menu for a model on an Agent | each hop's declared tiers, then the union across hops. A hop's declared tiers are its Source model's persisted `reasoning_efforts` when their provenance is `user` or `upstream`; otherwise the overlay's `providers` `wire_efforts`; otherwise the backend row's efforts minus its `client_presets` (provenance `catalog`); otherwise models.dev `reasoning_options`. User-authored lists, including free-form names, are never replaced or filtered by a prior; only a catalog-sourced list drops `client_presets`. A tier is omitted only if **every** hop that can serve the model has negative evidence for it. A tier with negative evidence on some hops is shown with "not on: <Source>". |
+| Effort menu for a model on an Agent | each hop's declared tiers, then the union across hops. A hop's declared tiers follow `model-hub-reasoning-tiers.md`'s ladder: an upstream reasoning signal (the protocol's ladder), else the bundled backend row's efforts minus that row's `client_presets` (provenance `catalog`; it outranks a user list, with a visible override event), else the user's list, else models.dev `reasoning_options` when the catalog row is built. A persisted `catalog` list is also read without `client_presets`, since released builds wrote them, and Vibey's own model rows always drop them at config load (3.2.0 saved rows with Codex's `ultra`; Vibey is never the client a preset belongs to, and no provider documents `ultra` as a wire effort). A user-authored Source list is never filtered. The overlay's `wire_efforts` slots in before the backend row once a vendor needs it; no shipped entry sets it yet. A tier is omitted only if **every** hop that can serve the model has negative evidence for it. A tier with negative evidence on some hops is shown with "not on: <Source>". Model Hub projects this per hop as the chain API's read-only `refused_efforts` (`agent-chain.schema.json`); the UI renders the omission and the hint from it. |
| Effort at request time | Forwarded unchanged (reasoning-intent). A hop with **negative** evidence for that effort is skipped for this request, as if it were not a candidate. If no candidate remains, the request fails before invocation with `effort_unsupported` naming the effort and the Sources, instead of a provider 400. Never clamped silently. |
| Hosted tool mode on a hop | `native` only with positive `call` evidence taken on the current engine and probe shape, and the user's toggle on; otherwise the gateway proxy (`vibey-hosted-tools.md`). A wrong native declaration is a terminal 400 or a silent drop; the proxy is lossless. So this feature needs positive evidence, unlike efforts. |
| Native payload replay | only to the same Source, and only while its `replay` evidence is positive and taken on the current engine and probe shape. The decision is made at every request from `HopResolution`; the transcript never stores it. |
| Display | §5 |
-This fixes #2539 in three places:
+This fixes #2539 in three places (Phase 1 PR 1b):
- the overlay lists `ultra` in Codex's `client_presets`, so it is not a wire tier for Vibey;
-- models.dev's wire list (`low … max`) is the base;
+- a catalog-sourced tier list is the backend row minus those presets (`low … max`, which matches models.dev's wire list), also when a released build persisted it with `ultra`;
- the relay's real `400 invalid_value` records `ultra: unsupported` for that Source.
Even without the overlay fix, the next request would skip that hop or fail clearly.
@@ -161,7 +167,7 @@ validates it):
"anthropic": {"models": [
{"match": {"prefix": "claude-"},
"hosted_tools": {"anthropic": {
- "web_search": {"declaration": {"type": "web_search_20250305", "name": "web_search"},
+ "web_search": {"declaration": {"type": "web_search_20250305", "name": "web_search", "max_uses": 5},
"price": {"unit": "search", "usd_per_1k": 10, "as_of": "2026-10-13", "url": "https://platform.claude.com/docs/en/about-claude/pricing"}}}}}
]}
}
diff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.py
index 7550be7e6..a2d8d90df 100644
--- a/docs/plans/model-hub-contracts/adapter-interface.py
+++ b/docs/plans/model-hub-contracts/adapter-interface.py
@@ -138,6 +138,25 @@ class RawOutcomeKind(str, Enum):
PROTOCOL_ERROR = "protocol_error" # malformed/unparseable upstream response
+@dataclass(frozen=True)
+class UpstreamRejection:
+ """Closed facts of an HTTP 400 the upstream returned before any output.
+
+ The only input of the capability-evidence rules (``model-hub-capability-model.md``
+ §3.2). L1 fills it from the error envelope: raw machine tokens, and the one
+ token each recognized message names. It never carries prose, and the
+ failover decision never reads it.
+ """
+
+ status: int
+ code: str | None = None # raw machine code, e.g. "invalid_value"
+ error_type: str | None = None # raw error type, e.g. "invalid_request_error"
+ param: str | None = None # raw parameter path, e.g. "reasoning.effort"
+ invalid_value: str | None = None # "Invalid value: '<value>'"
+ unsupported_tool_type: str | None = None # "Unsupported tool type: <type>"
+ invalid_encrypted_content_block: str | None = None # "Invalid encrypted_content in <kind> block"
+
+
@dataclass(frozen=True)
class RawCallOutcome:
"""Uninterpreted per-call result. Classification is L2's job, not L1's."""
@@ -175,6 +194,8 @@ class RawCallOutcome:
# Recognized local OS failure only, never an upstream code or error prose.
# Display-only; does not participate in classification or Source health.
os_errno: int | None = None
+ # A 400 before output, as closed tokens for the capability-evidence rules.
+ rejection: UpstreamRejection | None = None
class ObservationOutcome(str, Enum):
diff --git a/docs/plans/model-hub-contracts/agent-chain.schema.json b/docs/plans/model-hub-contracts/agent-chain.schema.json
index 9aa131a61..86bbcd5f3 100644
--- a/docs/plans/model-hub-contracts/agent-chain.schema.json
+++ b/docs/plans/model-hub-contracts/agent-chain.schema.json
@@ -97,6 +97,22 @@
"type": ["string", "null"],
"format": "date-time",
"description": "When health is backoff, this live deadline is strictly later than the read assembler's captured assembly time. The API normalizes an expired overlay to its underlying non-backoff health before validation and serialization; JSON Schema enforces only the date-time shape, not the relation to a moving clock."
+ },
+ "refused_efforts": {
+ "type": "array",
+ "minItems": 1,
+ "description": "Reasoning efforts this exact hop is currently known to refuse (capability evidence, model-hub-capability-model.md §3.4), oldest evidence first; absent when none. A request at one of them skips this hop. The effort menu omits a tier only when every hop that can serve the model lists it, and otherwise marks it \"not on: <Source>\". Read-only: routing reads the evidence store itself.",
+ "items": {
+ "type": "object",
+ "required": ["effort", "basis", "observed_at", "detail"],
+ "additionalProperties": false,
+ "properties": {
+ "effort": { "type": "string", "minLength": 1 },
+ "basis": { "enum": ["probe", "observed"] },
+ "observed_at": { "type": "string", "format": "date-time" },
+ "detail": { "type": ["string", "null"], "maxLength": 64 }
+ }
+ }
}
},
"allOf": [
diff --git a/tests/fixtures/model_hub/released_3_2_0_vibey_client_preset.json b/tests/fixtures/model_hub/released_3_2_0_vibey_client_preset.json
new file mode 100644
index 000000000..218b9fadf
--- /dev/null
+++ b/tests/fixtures/model_hub/released_3_2_0_vibey_client_preset.json
@@ -0,0 +1,761 @@
+{
+ "enabled": true,
+ "runtime_default_applied": true,
+ "sources": [
+ {
+ "id": "src_relay3200",
+ "created_at": "2026-10-01T00:00:00Z",
+ "last_discovered_at": null,
+ "kind": "api_key",
+ "vendor": "openai",
+ "display_name": "OpenAI 中转",
+ "protocol": "openai_responses",
+ "base_url": null,
+ "supply_channel": "hub",
+ "billing": "metered",
+ "state": {
+ "status": "active",
+ "retry_at": null,
+ "detail_key": null
+ },
+ "models": [
+ {
+ "id": "gpt-6-astra",
+ "display_name": null,
+ "origin": "discovered",
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultra"
+ ],
+ "reasoning_efforts_source": "catalog",
+ "discovered_at": null
+ }
+ ],
+ "credential_ref": "cred_src_relay3200",
+ "account_label": null,
+ "masked_credential": null
+ }
+ ],
+ "agents": {
+ "vibey": {
+ "backend": "vibey",
+ "mode": "hub",
+ "menu_kind": "fixed",
+ "sources": {
+ "order": [
+ "src_relay3200"
+ ]
+ },
+ "routes": {
+ "gpt-6-astra": {
+ "hops": [
+ {
+ "source_id": "src_relay3200",
+ "model_id": "gpt-6-astra"
+ }
+ ]
+ }
+ },
+ "menu": null,
+ "models": [
+ {
+ "id": "gpt-6-astra",
+ "display_name": "GPT-6 Astra",
+ "origin": "provider",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": true,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultra"
+ ]
+ }
+ ],
+ "removed_model_ids": []
+ },
+ "opencode": {
+ "backend": "opencode",
+ "mode": "hub",
+ "menu_kind": "open",
+ "sources": {
+ "order": []
+ },
+ "routes": {},
+ "menu": {
+ "view": "featured",
+ "checked": []
+ },
+ "models": [],
+ "removed_model_ids": []
+ },
+ "claude": {
+ "backend": "claude",
+ "mode": "hub",
+ "menu_kind": "fixed",
+ "sources": {
+ "order": []
+ },
+ "routes": {},
+ "menu": null,
+ "models": [
+ {
+ "id": "claude-fable-5-1",
+ "display_name": "claude-fable-5-1 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-fable-5",
+ "display_name": "claude-fable-5 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-opus-5-5",
+ "display_name": "claude-opus-5-5 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-opus-5",
+ "display_name": "claude-opus-5 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-opus-4-8",
+ "display_name": "claude-opus-4-8 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-opus-4-7",
+ "display_name": "claude-opus-4-7 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-opus-4-6",
+ "display_name": "claude-opus-4-6 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-sonnet-5-5",
+ "display_name": "claude-sonnet-5-5 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-sonnet-5",
+ "display_name": "claude-sonnet-5 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-sonnet-4-6",
+ "display_name": "claude-sonnet-4-6 [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-haiku-4-5",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high"
+ ]
+ },
+ {
+ "id": "claude-opus-4-5",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high"
+ ]
+ },
+ {
+ "id": "claude-sonnet-4-5",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high"
+ ]
+ },
+ {
+ "id": "opus",
+ "display_name": "opus [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "sonnet",
+ "display_name": "sonnet [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "haiku",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high"
+ ]
+ },
+ {
+ "id": "opus[1m]",
+ "display_name": "opus[1m] [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "sonnet[1m]",
+ "display_name": "sonnet[1m] [1M]",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "claude-opus-4",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ },
+ {
+ "id": "claude-sonnet-4",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ },
+ {
+ "id": "claude-haiku-4",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ },
+ {
+ "id": "claude-sonnet-4-0",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ },
+ {
+ "id": "claude-sonnet-4-20250514",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ },
+ {
+ "id": "claude-sonnet-3-7",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ },
+ {
+ "id": "claude-haiku-3-5",
+ "display_name": null,
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": []
+ }
+ ],
+ "removed_model_ids": []
+ },
+ "codex": {
+ "backend": "codex",
+ "mode": "hub",
+ "menu_kind": "fixed",
+ "sources": {
+ "order": []
+ },
+ "routes": {},
+ "menu": null,
+ "models": [
+ {
+ "id": "gpt-6-astra",
+ "display_name": "GPT-6-Astra",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultra"
+ ]
+ },
+ {
+ "id": "gpt-6.1-sol",
+ "display_name": "GPT-6.1-Sol",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "gpt-6-sol",
+ "display_name": "GPT-6-Sol",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "gpt-6-luna",
+ "display_name": "GPT-6-Luna",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "gpt-5.6-sol",
+ "display_name": "GPT-5.6-Sol",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultra"
+ ]
+ },
+ {
+ "id": "gpt-5.6-terra",
+ "display_name": "GPT-5.6-Terra",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultra"
+ ]
+ },
+ {
+ "id": "gpt-5.6-luna",
+ "display_name": "GPT-5.6-Luna",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max"
+ ]
+ },
+ {
+ "id": "gpt-5.5",
+ "display_name": "GPT-5.5",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh"
+ ]
+ },
+ {
+ "id": "gpt-5.4",
+ "display_name": "GPT-5.4",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh"
+ ]
+ },
+ {
+ "id": "gpt-5.4-mini",
+ "display_name": "GPT-5.4-Mini",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh"
+ ]
+ },
+ {
+ "id": "gpt-5.2",
+ "display_name": "GPT-5.2",
+ "origin": "builtin",
+ "models_dev_id": null,
+ "context_window": null,
+ "max_output_tokens": null,
+ "input_modalities": [],
+ "output_modalities": [],
+ "supports_tools": null,
+ "supports_reasoning": null,
+ "reasoning_efforts": [
+ "low",
+ "medium",
+ "high",
+ "xhigh"
+ ]
+ }
+ ],
+ "removed_model_ids": []
+ }
+ }
+}
diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yaml
index 4855db6fc..0ecbd7294 100644
--- a/tests/scenarios/INDEX.yaml
+++ b/tests/scenarios/INDEX.yaml
@@ -55,6 +55,7 @@ capabilities:
- tests/test_provider_overlay.py
- tests/agent_core/ai/test_hosted_requests.py
- tests/test_vibey_agent.py
+ - tests/test_model_hub_evidence_rules.py
- tests/test_model_hub_admission.py
- tests/test_model_hub_vibey_consumer.py
- tests/test_model_hub_config.py
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index bb2452f81..b0dc24b76 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -11,6 +11,7 @@ capability:
- tests/test_provider_overlay.py
- tests/agent_core/ai/test_hosted_requests.py
- tests/test_vibey_agent.py
+ - tests/test_model_hub_evidence_rules.py
- tests/test_model_hub_config.py
- tests/test_model_hub_legacy_opencode_migration.py
- tests/test_member_management_parity.py
@@ -203,6 +204,18 @@ scenarios:
- tests/agent_core/ai/test_hosted_requests.py::test_a_hosted_spec_on_a_native_route_is_sent_as_the_declaration_in_its_place_and_pinned
- tests/agent_core/ai/test_hosted_requests.py::test_the_probe_entry_builds_exactly_the_request_the_adapter_sends
- tests/agent_core/ai/test_hosted_requests.py::test_tool_choice_forces_the_named_tool_in_each_protocols_form
+ - id: MH-HOSTED-006
+ name: A real upstream 400 teaches only the feature it names, and a refused effort then skips its hop instead of another provider 400
+ status: covered
+ kind: recovery
+ layer: contract
+ test: tests/test_model_hub_evidence_rules.py::test_an_upstream_400_teaches_only_the_feature_it_names
+ related_tests:
+ - tests/test_model_hub_evidence_rules.py::test_a_refused_effort_skips_its_hop_instead_of_another_provider_400
+ - tests/test_model_hub_reasoning_tiers.py::test_catalog_row_outranks_user_and_drops_its_client_presets
+ - tests/test_model_hub_reasoning_tiers.py::test_a_persisted_catalog_list_offers_no_client_preset_to_vibey
+ - tests/test_model_hub_vibey_consumer.py::test_a_released_vibey_row_loads_without_its_client_preset
+ - tests/test_model_hub_evidence_rules.py::test_a_refused_effort_still_waits_for_a_capable_hop_that_is_cooling_down
- id: MH-VIBEY-007
name: Vibey starts once with the user's Sources and built-in Agent models, seeded when its first Source exists
status: covered
diff --git a/tests/test_model_hub_api.py b/tests/test_model_hub_api.py
index cb33f765a..11bbd347b 100644
--- a/tests/test_model_hub_api.py
+++ b/tests/test_model_hub_api.py
@@ -6131,7 +6131,7 @@ def test_create_source_rejects_omitted_protocol_for_an_uncataloged_vendor(tmp_pa
(
"openai_responses",
DiscoveredModel(id="gpt-5.6-sol"),
- ["low", "medium", "high", "xhigh", "max", "ultra"],
+ ["low", "medium", "high", "xhigh", "max"],
"catalog",
),
),
@@ -6237,7 +6237,7 @@ def test_source_create_requires_explicit_consent_for_proven_inventory_failure(
return {"catalog-manual": ("low", "ultra")}
monkeypatch.setattr(
- "core.handlers.model_hub.service.bundled_catalog_reasoning_efforts_by_model",
+ "core.handlers.model_hub.service.bundled_catalog_wire_efforts_by_model",
load_catalog_index,
)
store = MemoryStore()
@@ -7036,14 +7036,7 @@ def test_add_model_upsert_cannot_bypass_catalog_tier_lock(tmp_path):
)
)["models"][0]
- assert created["reasoning_efforts"] == [
- "low",
- "medium",
- "high",
- "xhigh",
- "max",
- "ultra",
- ]
+ assert created["reasoning_efforts"] == ["low", "medium", "high", "xhigh", "max"]
assert created["reasoning_efforts_source"] == "catalog"
with pytest.raises(ModelHubError) as error:
asyncio.run(
@@ -7064,7 +7057,7 @@ def test_add_model_upsert_cannot_bypass_catalog_tier_lock(tmp_path):
(
(
DiscoveredModel(id="gpt-5.6-sol"),
- ["low", "medium", "high", "xhigh", "max", "ultra"],
+ ["low", "medium", "high", "xhigh", "max"],
"catalog",
"catalog_tiers",
),
@@ -9003,14 +8996,7 @@ def test_completed_hub_oauth_persists_only_a_response_proven_protocol(tmp_path):
assert result["source"]["protocol"] == "openai_chat"
assert store.config.sources[0].protocol == "openai_chat"
assert store.config.sources[0].credential_ref == "cred_oauth_proven"
- assert store.config.sources[0].models[0].reasoning_efforts == [
- "low",
- "medium",
- "high",
- "xhigh",
- "max",
- "ultra",
- ]
+ assert store.config.sources[0].models[0].reasoning_efforts == ["low", "medium", "high", "xhigh", "max"]
assert (
store.config.sources[0].models[0].reasoning_efforts_source
== "catalog"
diff --git a/tests/test_model_hub_capability_evidence.py b/tests/test_model_hub_capability_evidence.py
index 98d462dd8..f0c32c905 100644
--- a/tests/test_model_hub_capability_evidence.py
+++ b/tests/test_model_hub_capability_evidence.py
@@ -76,6 +76,7 @@ RESOLUTION = [
("engine-upgrade-effort-negative-still-applies", ULTRA, {"status": "unsupported"}, "model-hub-engine-v9", 1, 0 * DAYS, "unsupported", "wire"),
("observed-negative-within-ttl", ULTRA, {"status": "unsupported", "basis": "observed", "detail": "400 invalid_value"}, "model-hub-engine-v9", 1, 6 * DAYS, "unsupported", None),
("observed-negative-expires", CALL, {"status": "unsupported", "basis": "observed", "detail": "400 invalid_request_error"}, ENGINE, 1, 8 * DAYS, "unknown", None),
+ ("unknown-engine-never-vouches", CALL, {"engine": "unknown-engine"}, "unknown-engine", 1, 0 * DAYS, "unknown", "wire"),
]
@@ -156,8 +157,8 @@ def test_image_generation_has_no_replay_evidence():
EvidenceFeature.hosted("image_generation", "replay")
-def test_lookups_after_the_first_read_never_touch_the_disk(tmp_path, monkeypatch):
- """MH-HOSTED-003: the request path reads evidence from memory, never from the file."""
+def test_lookups_after_construction_never_touch_the_disk(tmp_path, monkeypatch):
+ """MH-HOSTED-003: the store reads its file when built at startup; no lookup or write reads it again."""
path = tmp_path / "model_hub_capability_evidence.json"
source = _source()
CapabilityEvidenceStore(path).write(_record(source, CALL), live=[source])
@@ -184,3 +185,41 @@ def test_lookups_after_the_first_read_never_touch_the_disk(tmp_path, monkeypatch
store.write(_record(source, REPLAY), live=[source])
assert store.view(source, "grok-4.6", REPLAY, engine=ENGINE, shape=1, now=NOW).status == "verified"
assert touched == []
+
+
+def test_a_read_never_waits_for_a_write_on_a_slow_disk(tmp_path, monkeypatch):
+ """MH-HOSTED-003: the request path reads a published snapshot, so a write stuck on the disk cannot stall it."""
+ import threading
+
+ from core.handlers.model_hub import capability_evidence
+
+ path = tmp_path / "model_hub_capability_evidence.json"
+ source = _source()
+ store = CapabilityEvidenceStore(path)
+ store.write(_record(source, CALL), live=[source])
+
+ on_disk = threading.Event()
+ release = threading.Event()
+ persist = capability_evidence.write_state_document
+
+ def slow_disk(target, payload):
+ on_disk.set()
+ assert release.wait(5)
+ persist(target, payload)
+
+ monkeypatch.setattr(capability_evidence, "write_state_document", slow_disk)
+ writer = threading.Thread(target=store.write, args=(_record(source, REPLAY),), kwargs={"live": [source]})
+ writer.start()
+ try:
+ assert on_disk.wait(5)
+ read = []
+ reader = threading.Thread(
+ target=lambda: read.append(store.view(source, "grok-4.6", CALL, engine=ENGINE, shape=1, now=NOW).status)
+ )
+ reader.start()
+ reader.join(1)
+ assert read == ["verified"]
+ finally:
+ release.set()
+ writer.join(5)
+ assert store.view(source, "grok-4.6", REPLAY, engine=ENGINE, shape=1, now=NOW).status == "verified"
diff --git a/tests/test_model_hub_evidence_rules.py b/tests/test_model_hub_evidence_rules.py
new file mode 100644
index 000000000..647776a9e
--- /dev/null
+++ b/tests/test_model_hub_evidence_rules.py
@@ -0,0 +1,369 @@
+"""Negative capability evidence learned from real upstream errors (capability model §3.2, §3.4).
+
+The real engine client parses each upstream 400 through a loopback engine, so the
+whole path is exercised: L1's closed rejection tokens, the narrow rule that names
+exactly one (Source, model, feature), and the hop skip that the next request at a
+refused effort gets instead of another provider 400.
+"""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+from aiohttp import web
+
+from core.handlers.model_hub import service as service_module
+from core.handlers.model_hub.capability_evidence import EvidenceFeature
+from core.handlers.model_hub.request import ModelHubRequest
+from core.handlers.model_hub.service import ModelHubError
+from tests.test_model_hub_l3 import _assert_valid, _source
+from tests.test_model_hub_retry_lifecycle import configured_service, loopback_engine
+from vibe.backend_model_catalog import HostedToolSpec
+
+ULTRA_REFUSED = {
+ "error": {
+ "code": "invalid_value",
+ "type": "invalid_request_error",
+ "message": "Invalid value: 'ultra'. Supported values are: 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', and 'max'.",
+ }
+}
+TOOL_REFUSED = {"error": {"message": "Unsupported tool type: web_search", "type": "invalid_request_error"}}
+REPLAY_REFUSED = {
+ "type": "error",
+ "error": {"type": "invalid_request_error", "message": "Invalid encrypted_content in search_result block"},
+}
+WEB_SEARCH_HISTORY = [
+ {"role": "user", "content": "搜索 Avibe"},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "Avibe"}},
+ {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": [{"encrypted_content": "Eo8B"}]},
+ ],
+ },
+ {"role": "user", "content": "第二条结果是什么?"},
+]
+
+CASES = [
+ # (case, backend, vendor, protocol, request body, upstream status, upstream body, learned feature, detail)
+ (
+ "effort-named-by-the-relay", "codex", "openai", "openai_responses",
+ {"input": "hi", "reasoning": {"effort": "ultra"}}, 400, ULTRA_REFUSED,
+ EvidenceFeature.effort("ultra"), "400 invalid_request_error",
+ ),
+ (
+ "declared-tool-type", "codex", "openai", "openai_responses",
+ {"input": "hi", "tools": [{"type": "web_search"}]}, 400, TOOL_REFUSED,
+ EvidenceFeature.hosted("web_search", "call"), "400 invalid_request_error",
+ ),
+ (
+ "replayed-search-result", "claude", "anthropic", "anthropic",
+ {"max_tokens": 64, "messages": WEB_SEARCH_HISTORY}, 400, REPLAY_REFUSED,
+ EvidenceFeature.hosted("web_search", "replay"), "400 invalid_request_error",
+ ),
+ ("another-effort-named", "codex", "openai", "openai_responses",
+ {"input": "hi", "reasoning": {"effort": "high"}}, 400, ULTRA_REFUSED, None, None),
+ ("same-value-in-another-field", "codex", "openai", "openai_responses",
+ {"input": "hi", "reasoning": {"effort": "high"}}, 400,
+ {"error": {"code": "invalid_value", "param": "service_tier", "message": "Invalid value: 'high'."}}, None, None),
+ ("tool-type-not-declared", "codex", "openai", "openai_responses",
+ {"input": "hi"}, 400, TOOL_REFUSED, None, None),
+ ("no-hosted-result-replayed", "claude", "anthropic", "anthropic",
+ {"max_tokens": 64, "messages": [{"role": "user", "content": "hi"}]}, 400, REPLAY_REFUSED, None, None),
+ ("server-error-names-the-effort", "codex", "openai", "openai_responses",
+ {"input": "hi", "reasoning": {"effort": "ultra"}}, 500, ULTRA_REFUSED, None, None),
+]
+
+
+def _web_search_only(vendor, protocol, model_id, tool):
+ if tool != "web_search":
+ return None
+ declaration = {"anthropic": {"type": "web_search_20250305", "name": "web_search"}}.get(
+ protocol, {"type": "web_search"}
+ )
+ return HostedToolSpec(declaration_json=json.dumps(declaration), price=None, include=())
+
+
+ "backend,vendor,protocol,body,status,upstream,feature,detail",
+ [case[1:] for case in CASES],
+ ids=[case[0] for case in CASES],
+)
+async def test_an_upstream_400_teaches_only_the_feature_it_names(
+ tmp_path, monkeypatch, backend, vendor, protocol, body, status, upstream, feature, detail
+):
+ """MH-HOSTED-006: a narrow rule learns exactly the named feature; anything else learns nothing."""
+ monkeypatch.setattr(service_module, "hosted_tool_spec", _web_search_only)
+ source = _source("src_evidence1", "Relay 中转", vendor=vendor, protocol=protocol)
+ service, _clock, models = configured_service(tmp_path, sources=[source])
+
+ async def respond(_request):
+ return web.json_response(upstream, status=status)
+
+ async with loopback_engine(tmp_path, service, respond):
+ with pytest.raises(ModelHubError):
+ await service.resolve_with_recovery(
+ backend=backend, model_id=models[backend],
+ request=ModelHubRequest(body, protocol=protocol), stream=False,
+ )
+
+ records = service.capability_evidence.records()
+ if feature is None:
+ assert records == []
+ return
+ [record] = records
+ assert (record.source_id, record.model_id, record.feature) == ("src_evidence1", "shared-model", feature)
+ assert (record.status, record.basis, record.detail) == ("unsupported", "observed", detail)
+
+
+async def test_a_refused_effort_skips_its_hop_instead_of_another_provider_400(tmp_path):
+ """MH-HOSTED-006: the next request at a refused effort fails locally, and the chain read shows it."""
+ source = _source("src_evidence1", "Relay", vendor="openai", protocol="openai_responses")
+ service, _clock, models = configured_service(tmp_path, sources=[source])
+ calls = []
+
+ async def respond(request):
+ calls.append(await request.json())
+ if calls[-1].get("reasoning", {}).get("effort") == "ultra":
+ return web.json_response(ULTRA_REFUSED, status=400)
+ return web.json_response(
+ {"object": "response", "status": "completed", "output": [{"content": [{"type": "output_text", "text": "ok"}]}]}
+ )
+
+ def request(effort):
+ return ModelHubRequest({"input": "hi", "reasoning": {"effort": effort}}, protocol="openai_responses")
+
+ async with loopback_engine(tmp_path, service, respond):
+ with pytest.raises(ModelHubError) as first:
+ await service.resolve_with_recovery(
+ backend="codex", model_id=models["codex"], request=request("ultra"), stream=False
+ )
+ assert first.value.code != "effort_unsupported"
+ assert len(calls) == 1
+
+ with pytest.raises(ModelHubError) as second:
+ await service.resolve_with_recovery(
+ backend="codex", model_id=models["codex"], request=request("ultra"), stream=False
+ )
+ assert (second.value.code, second.value.status) == ("effort_unsupported", 422)
+ assert second.value.data == {"effort": "ultra", "source_ids": ["src_evidence1"]}
+ assert len(calls) == 1
+
+ # The chain read projects what routing just used, for the effort menu.
+ chain = service.agent_chain("codex", models["codex"])
+ _assert_valid("agent-chain.schema.json", chain)
+ [hop] = [hop for hop in chain["chain"] if hop["source_id"] == "src_evidence1"]
+ assert hop["refused_efforts"] == [
+ {
+ "effort": "ultra",
+ "basis": "observed",
+ "observed_at": service.capability_evidence.records()[0].payload()["observed_at"],
+ "detail": "400 invalid_request_error",
+ }
+ ]
+
+ served = await service.resolve_with_recovery(
+ backend="codex", model_id=models["codex"], request=request("high"), stream=False
+ )
+ assert served.source_id == "src_evidence1"
+ assert len(calls) == 2
+
+
+async def test_a_refreshed_attempt_teaches_what_it_names(tmp_path):
+ """MH-HOSTED-006: after a credential refresh, the exact retry's 400 is learned too."""
+ from dataclasses import replace as replaced
+
+ from core.handlers.model_hub.adapter import RawOutcomeKind, UpstreamRejection
+ from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _outcome, _service
+
+ source = _source("src_refreshed1", "Refreshed")
+ refused = replaced(
+ _outcome(RawOutcomeKind.HTTP_ERROR, status=400, code="invalid_request_error", source_id=source.id),
+ rejection=UpstreamRejection(status=400, error_type="invalid_request_error", invalid_value="ultra"),
+ )
+ service = _service(
+ tmp_path,
+ sources=[source],
+ outcomes=[_outcome(RawOutcomeKind.HTTP_ERROR, status=401, code="unauthorized", source_id=source.id), refused],
+ )
+ service.adapter.refreshable_credential_refs.add(source.credential_ref)
+ model = _canonicalize_fixed_test_routes(service)["codex"]
+
+ with pytest.raises(ModelHubError):
+ await service.resolve_with_recovery(
+ backend="codex",
+ model_id=model,
+ request=ModelHubRequest({"input": "hi", "reasoning": {"effort": "ultra"}}, protocol="openai_responses"),
+ stream=False,
+ )
+
+ [record] = service.capability_evidence.records()
+ assert (record.source_id, record.feature) == ("src_refreshed1", EvidenceFeature.effort("ultra"))
+
+
+async def test_a_free_form_effort_is_forwarded_and_never_breaks_the_gateway(tmp_path):
+ """MH-HOSTED-006: evidence cannot name a free-form effort, so it learns nothing and the 400 surfaces as is."""
+ source = _source("src_evidence1", "Relay", vendor="openai", protocol="openai_responses")
+ service, _clock, models = configured_service(tmp_path, sources=[source])
+ refused = {"error": {"code": "invalid_value", "type": "invalid_request_error", "param": "reasoning.effort"}}
+
+ async def respond(_request):
+ return web.json_response(refused, status=400)
+
+ async with loopback_engine(tmp_path, service, respond):
+ for _ in range(2):
+ with pytest.raises(ModelHubError) as error:
+ await service.resolve_with_recovery(
+ backend="codex",
+ model_id=models["codex"],
+ request=ModelHubRequest(
+ {"input": "hi", "reasoning": {"effort": "future effort"}}, protocol="openai_responses"
+ ),
+ stream=False,
+ )
+ assert error.value.code != "effort_unsupported"
+ assert service.capability_evidence.records() == []
+
+
+async def test_a_refused_effort_still_waits_for_a_capable_hop_that_is_cooling_down(tmp_path):
+ """MH-HOSTED-006: effort_unsupported only when no hop that could recover accepts the effort."""
+ from datetime import timedelta
+
+ from core.handlers.model_hub.capability_evidence import EvidenceRecord, identity_digest
+ from core.handlers.model_hub.hosted_tools import HOSTED_PROBE_SHAPE, engine_stamp
+ from tests.test_model_hub_l3 import NOW, _canonicalize_fixed_test_routes, _service
+
+ refusing = _source("src_refuses01", "Refuses ultra")
+ cooling = _source(
+ "src_cooling01", "Cooling", status="cooldown", retry_at=(NOW + timedelta(minutes=5)).isoformat()
+ )
+ service = _service(tmp_path, sources=[refusing, cooling])
+ model = _canonicalize_fixed_test_routes(service)["codex"]
+ service.capability_evidence.write(
+ EvidenceRecord(
+ source_id=refusing.id,
+ model_id="shared-model",
+ feature=EvidenceFeature.effort("ultra"),
+ status="unsupported",
+ basis="observed",
+ observed_at=service.now(),
+ engine=engine_stamp(),
+ shape=HOSTED_PROBE_SHAPE,
+ source_identity=identity_digest(refusing),
+ detail="400 invalid_request_error",
+ ),
+ live=[refusing, cooling],
+ )
+ request = ModelHubRequest({"input": "hi", "reasoning": {"effort": "ultra"}}, protocol="openai_responses")
+
+ with pytest.raises(ModelHubError) as waiting:
+ await service.resolve(backend="codex", model_id=model, request=request)
+ assert waiting.value.code != "effort_unsupported"
+ assert waiting.value.supply_state == "waiting"
+ assert service.adapter.invocations == []
+
+
+async def test_no_request_path_reads_a_bundled_file_after_startup(tmp_path, monkeypatch):
+ """MH-HOSTED-006: every immutable bundled input is resolved when the service is built, never on a request."""
+ from dataclasses import replace as replaced
+ from pathlib import Path
+
+ from core.handlers.model_hub.adapter import RawOutcomeKind, UpstreamRejection
+ from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _outcome, _service
+
+ source = _source("src_startup01", "Startup", vendor="openai", protocol="openai_responses")
+ service = _service(tmp_path, sources=[source])
+ model = _canonicalize_fixed_test_routes(service)["codex"]
+ watched = {"backend_models.json", "cliproxyapi_manifest.json"}
+ reads = []
+ read_text, path_open = Path.read_text, Path.open
+
+ def counted_read(self, *args, **kwargs):
+ if self.name in watched:
+ reads.append(self.name)
+ return read_text(self, *args, **kwargs)
+
+ def counted_open(self, *args, **kwargs):
+ if self.name in watched:
+ reads.append(self.name)
+ return path_open(self, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "read_text", counted_read)
+ monkeypatch.setattr(Path, "open", counted_open)
+
+ request = ModelHubRequest(
+ {"input": "hi", "reasoning": {"effort": "ultra"}, "tools": [{"type": "web_search"}]}, protocol="openai_responses"
+ )
+ rejected = replaced(
+ _outcome(RawOutcomeKind.HTTP_ERROR, status=400, source_id=source.id),
+ rejection=UpstreamRejection(status=400, unsupported_tool_type="web_search"),
+ )
+ config = service.store.load()
+ assert service._effort_refused(source, "shared-model", request) is False
+ await service._learn_capability_evidence(source, "shared-model", request, rejected)
+ service.hosted_tools_for_hop(config, source, "shared-model")
+ service.agent_chain("codex", model)
+ service.list_sources()
+ assert reads == []
+
+
+async def test_a_refusing_runnable_hop_never_makes_recovery_spin(tmp_path):
+ """MH-HOSTED-006: recovery waits for the capable cooling hop instead of re-resolving around the refusing one."""
+ from datetime import timedelta as delta
+
+ from core.handlers.model_hub.adapter import RawOutcomeKind
+ from core.handlers.model_hub.capability_evidence import EvidenceRecord, identity_digest
+ from core.handlers.model_hub.hosted_tools import HOSTED_PROBE_SHAPE
+ from tests.test_model_hub_l3 import NOW, _canonicalize_fixed_test_routes, _outcome
+ from tests.test_model_hub_retry_policy import clock_service
+
+ refusing = _source("src_refuses02", "Refuses ultra")
+ cooling = _source("src_cooling02", "Cooling", status="cooldown", retry_at=(NOW + delta(seconds=30)).isoformat())
+ service, clock = clock_service(
+ tmp_path,
+ sources=[refusing, cooling],
+ outcomes=[_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=cooling.id)],
+ )
+ model = _canonicalize_fixed_test_routes(service)["codex"]
+ service.capability_evidence.write(
+ EvidenceRecord(
+ source_id=refusing.id,
+ model_id="shared-model",
+ feature=EvidenceFeature.effort("ultra"),
+ status="unsupported",
+ basis="observed",
+ observed_at=service.now(),
+ engine=service.engine_stamp,
+ shape=HOSTED_PROBE_SHAPE,
+ source_identity=identity_digest(refusing),
+ detail="400 invalid_request_error",
+ ),
+ live=[refusing, cooling],
+ )
+ passes = 0
+ resolve = service.resolve
+
+ async def counted(**kwargs):
+ nonlocal passes
+ passes += 1
+ assert passes <= 5, "recovery re-resolved without waiting"
+ return await resolve(**kwargs)
+
+ service.resolve = counted
+ served = await service.resolve_with_recovery(
+ backend="codex",
+ model_id=model,
+ request=ModelHubRequest({"input": "hi", "reasoning": {"effort": "ultra"}}, protocol="openai_responses"),
+ )
+
+ assert served.source_id == cooling.id
+ assert clock.delays and all(delay > 0 for delay in clock.delays)
+ assert [invocation[0] for invocation in service.adapter.invocations] == [cooling.id]
diff --git a/tests/test_model_hub_reasoning_tiers.py b/tests/test_model_hub_reasoning_tiers.py
index fd5f55ef1..7d28bb05b 100644
--- a/tests/test_model_hub_reasoning_tiers.py
+++ b/tests/test_model_hub_reasoning_tiers.py
@@ -28,7 +28,8 @@ def test_upstream_reasoning_signal_uses_only_the_protocol_default(
assert resolution.source == "upstream"
-def test_catalog_row_outranks_user_and_is_returned_verbatim() -> None:
+def test_catalog_row_outranks_user_and_drops_its_client_presets() -> None:
+ """MH-HOSTED-006 (#2539): Codex's ``ultra`` is a client preset, never a Source's wire tier."""
resolution = resolve_reasoning_tiers(
protocol="openai_responses",
model_id="gpt-5.6-sol",
@@ -36,14 +37,7 @@ def test_catalog_row_outranks_user_and_is_returned_verbatim() -> None:
existing_source="user",
)
- assert resolution.efforts == (
- "low",
- "medium",
- "high",
- "xhigh",
- "max",
- "ultra",
- )
+ assert resolution.efforts == ("low", "medium", "high", "xhigh", "max")
assert resolution.source == "catalog"
@@ -76,8 +70,8 @@ def test_catalog_row_is_not_filtered_through_the_vocabulary(
) -> None:
monkeypatch.setattr(
reasoning_tiers,
- "bundled_catalog_reasoning_efforts_for_model",
- lambda _model_id: pytest.fail("injected catalog index was ignored"),
+ "bundled_catalog_wire_efforts_by_model",
+ lambda: pytest.fail("injected catalog index was ignored"),
)
resolution = resolve_reasoning_tiers(
@@ -121,3 +115,45 @@ def test_upstream_reasoning_parameter_matching_is_case_and_space_tolerant(
)
assert resolution.source == "upstream"
+
+
+ ("provenance", "expected"),
+ (
+ ("catalog", ["low", "medium", "high", "xhigh", "max"]),
+ ("user", ["low", "medium", "high", "xhigh", "max", "ultra"]),
+ ),
+)
+def test_a_persisted_catalog_list_offers_no_client_preset_to_vibey(tmp_path, provenance, expected) -> None:
+ """MH-HOSTED-006 (#2539): a released build persisted Codex's ``ultra`` in catalog lists; only user lists keep it."""
+ from config.v2_config import ModelHubModelConfig, ModelHubSourceConfig, ModelHubSourceStateConfig
+ from tests.test_model_hub_api import _service
+
+ service, store, _adapter = _service(tmp_path)
+ store.config.sources = [
+ ModelHubSourceConfig(
+ id="src_relay0001",
+ kind="api_key",
+ vendor="openai",
+ display_name="OpenAI relay",
+ protocol="openai_responses",
+ supply_channel="hub",
+ billing="metered",
+ state=ModelHubSourceStateConfig(status="standby"),
+ models=[
+ ModelHubModelConfig(
+ id="gpt-5.6-sol",
+ provenance="discovered",
+ reasoning_efforts=["low", "medium", "high", "xhigh", "max", "ultra"],
+ reasoning_efforts_source=provenance,
+ )
+ ],
+ credential_ref="cred_relay",
+ )
+ ]
+ store.config.agents["vibey"].sources.order = ["src_relay0001"]
+
+ providers = service.agent_model_candidates("vibey")["providers"]
+
+ [row] = [row for row in providers if row["id"] == "gpt-5.6-sol"]
+ assert row["reasoning_efforts"] == expected
diff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.py
index 1fb4fb75c..bfec5bca5 100644
--- a/tests/test_model_hub_structure_guards.py
+++ b/tests/test_model_hub_structure_guards.py
@@ -1234,11 +1234,16 @@ def test_machine_error_field_access_has_one_extractor() -> None:
if path.name == "migration.py":
continue # its ``type`` field is an auth-record kind, not an error code
tree = _tree(path)
- # Request history discriminates content blocks, not error envelopes.
- # Exempt only that function's type reads; code reads and the rest of
- # request.py remain subject to the extraction-owner guard.
- history = _functions(tree).get("_plain_history") if path.name == "request.py" else None
- history_nodes = set(ast.walk(history)) if history is not None else set()
+ # Request history and declared tools discriminate request items, not
+ # error envelopes. Exempt only those functions' type reads; code reads
+ # and the rest of request.py remain subject to the extraction-owner guard.
+ functions = _functions(tree) if path.name == "request.py" else {}
+ history_nodes = {
+ node
+ for name in ("_plain_history", "declared_tool_types", "replays_history_items")
+ if (history := functions.get(name)) is not None
+ for node in ast.walk(history)
+ }
for node in ast.walk(tree):
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
continue
diff --git a/tests/test_model_hub_vibey_consumer.py b/tests/test_model_hub_vibey_consumer.py
index b3af2200e..15a778246 100644
--- a/tests/test_model_hub_vibey_consumer.py
+++ b/tests/test_model_hub_vibey_consumer.py
@@ -1505,3 +1505,58 @@ async def test_concurrent_responses_keep_their_own_origin(tmp_path):
pending.cancel()
await asyncio.gather(pending, return_exceptions=True)
await gateway.close()
+
+
+
+def test_a_released_vibey_row_loads_without_its_client_preset(tmp_path, monkeypatch):
+ """MH-HOSTED-006 (#2539): 3.2.0 saved Vibey rows with Codex's ``ultra``; every read drops it, once, at load."""
+ from config.v2_config import V2Config
+ from core.services.settings import default_config
+ from vibe import api
+
+ released = json.loads(
+ (Path(__file__).parent / "fixtures/model_hub/released_3_2_0_vibey_client_preset.json").read_text(encoding="utf-8")
+ )
+ clean = ["low", "medium", "high", "xhigh", "max"]
+
+ config = ModelHubConfig.from_payload(copy.deepcopy(released), repairing=True)
+ assert config.agents["vibey"].models[0].reasoning_efforts == clean
+ written = config.to_payload()
+ assert written["agents"]["vibey"]["models"][0]["reasoning_efforts"] == clean
+ # Only the preset changes: Sources, other Agents and the rest of the row are as 3.2.0 wrote them.
+ expected = copy.deepcopy(released)
+ expected["agents"]["vibey"]["models"][0]["reasoning_efforts"] = clean
+ assert written == expected
+
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ payload = api.config_to_payload(default_config(), include_secrets=True, include_internal=True)
+ payload["model_hub"] = copy.deepcopy(released)
+ config_path = tmp_path / "config.json"
+ config_path.write_text(json.dumps(payload), encoding="utf-8")
+ loaded = V2Config.load(config_path=config_path)
+ assert not loaded.load_warnings
+ assert loaded.model_hub.agents["vibey"].models[0].reasoning_efforts == clean
+
+
+def test_loading_and_cloning_model_hub_config_reads_no_catalog_file(monkeypatch):
+ """MH-HOSTED-006: config load runs on the controller loop (service clones), so the preset index costs no I/O."""
+ from vibe.backend_model_catalog import bundled_client_presets_once, get_bundled_catalog_path
+
+ released = json.loads(
+ (Path(__file__).parent / "fixtures/model_hub/released_3_2_0_vibey_client_preset.json").read_text(encoding="utf-8")
+ )
+ bundled_client_presets_once()
+ bundled = get_bundled_catalog_path()
+ reads = []
+ original = Path.read_text
+
+ def counted(self, *args, **kwargs):
+ if self == bundled:
+ reads.append(self)
+ return original(self, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "read_text", counted)
+ for _ in range(5):
+ config = ModelHubConfig.from_payload(copy.deepcopy(released), repairing=True)
+ ModelHubConfig.from_payload(config.to_payload())
+ assert reads == []
diff --git a/tests/test_provider_overlay.py b/tests/test_provider_overlay.py
index 11d075320..a5f7d8158 100644
--- a/tests/test_provider_overlay.py
+++ b/tests/test_provider_overlay.py
@@ -106,7 +106,7 @@ def test_bundled_providers_section_satisfies_its_schema():
"claude-opus-5-5",
"web_search",
(
- {"type": "web_search_20250305", "name": "web_search"},
+ {"type": "web_search_20250305", "name": "web_search", "max_uses": 5},
(),
HostedToolPrice("search", 10.0, AS_OF, "https://platform.claude.com/docs/en/about-claude/pricing"),
),
diff --git a/vibe/backend_model_catalog.py b/vibe/backend_model_catalog.py
index de50e2781..313f619ff 100644
--- a/vibe/backend_model_catalog.py
+++ b/vibe/backend_model_catalog.py
@@ -562,9 +562,11 @@ def bundled_catalog_client_presets_by_model() -> Mapping[str, tuple[str, ...]]:
return _bundled_catalog_lists_by_model("client_presets")
-def _bundled_catalog_lists_by_model(key: str) -> Mapping[str, tuple[str, ...]]:
+def _bundled_catalog_lists_by_model(
+ key: str, catalog: Mapping[str, Any] | None = None
+) -> Mapping[str, tuple[str, ...]]:
values_by_model: dict[str, tuple[str, ...]] = {}
- backends = load_bundled_catalog().get("backends")
+ backends = (load_bundled_catalog() if catalog is None else catalog).get("backends")
if not isinstance(backends, dict):
return MappingProxyType(values_by_model)
for backend in backends.values():
@@ -593,6 +595,54 @@ def bundled_catalog_reasoning_efforts_for_model(
return bundled_catalog_reasoning_efforts_by_model().get(model_id)
[email protected]_cache(maxsize=4)
+def _bundled_client_presets(path: str) -> Mapping[str, tuple[str, ...]]:
+ catalog = _read_catalog(Path(path))
+ # An unreadable bundle has no presets, like a model without a row.
+ return _bundled_catalog_lists_by_model("client_presets", catalog or {})
+
+
+def bundled_client_presets_once() -> Mapping[str, tuple[str, ...]]:
+ """The bundled client-preset index, read once per process.
+
+ The bundled file never changes while a process runs, and config loading and
+ cloning (on the controller loop) consult this index, so it must cost no I/O
+ after the first read. Keyed by path, so a different bundle is its own entry.
+ """
+
+ return _bundled_client_presets(str(get_bundled_catalog_path()))
+
+
+def without_client_presets(model_ids: Sequence[str], efforts: Sequence[str]) -> list[str]:
+ """Efforts as a wire client may send them: without any listed model's client presets.
+
+ Vibey is not the client a preset belongs to, so a preset is never one of its
+ tiers (#2539).
+ """
+
+ presets = bundled_client_presets_once()
+ dropped = {preset for model_id in model_ids for preset in presets.get(model_id, ())}
+ return [effort for effort in efforts if effort not in dropped]
+
+
+def bundled_catalog_wire_efforts_by_model() -> Mapping[str, tuple[str, ...]]:
+ """Index bundled rows' efforts as a Source's wire tiers: each row's list minus its ``client_presets``.
+
+ One catalog read builds both, so a discovery batch still loads the bundled
+ file once (#2539).
+ """
+
+ catalog = load_bundled_catalog()
+ efforts = _bundled_catalog_lists_by_model("reasoning_efforts", catalog)
+ presets = _bundled_catalog_lists_by_model("client_presets", catalog)
+ return MappingProxyType(
+ {
+ model_id: tuple(effort for effort in values if effort not in presets.get(model_id, ()))
+ for model_id, values in efforts.items()
+ }
+ )
+
+
# The ``providers`` section (model-hub-capability-model.md §4,
# docs/plans/model-hub-contracts/provider-overlay.schema.json): what a provider
# offers per model family and protocol. Priors only; whether a Source carries a
@@ -639,6 +689,12 @@ class HostedToolSpec:
return json.loads(self.declaration_json)
+ @property
+ def declaration_type(self) -> str:
+ """The declaration's tool ``type``, as the provider names it on the wire."""
+
+ return self.declaration["type"]
+
@dataclass(frozen=True)
class ProviderModelOverlay:
diff --git a/vibe/data/backend_models.json b/vibe/data/backend_models.json
index 98145ba9c..522227fa4 100644
--- a/vibe/data/backend_models.json
+++ b/vibe/data/backend_models.json
@@ -55,7 +55,7 @@
"hosted_tools": {
"anthropic": {
"web_search": {
- "declaration": {"type": "web_search_20250305", "name": "web_search"},
+ "declaration": {"type": "web_search_20250305", "name": "web_search", "max_uses": 5},
"price": {"unit": "search", "usd_per_1k": 10, "as_of": "2026-10-13", "url": "https://platform.claude.com/docs/en/about-claude/pricing"}
}
}
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index f50b6e3da..78052acb1 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -28,6 +28,7 @@ from core.handlers.model_hub.adapter import (
InvokeCancelledError,
RawCallOutcome,
RawOutcomeKind,
+ UpstreamRejection,
)
from core.handlers.model_hub.async_owner import run_owned_in_thread
from core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES
@@ -1778,6 +1779,15 @@ def _reduce_protocol_observation(
usage=observation.usage,
recovery_verified=observation.recovery_verified,
upstream_detail=upstream_detail,
+ rejection=_upstream_rejection(
+ http_status,
+ stream_started,
+ error_code,
+ error_type,
+ observation.error_payload or b"",
+ observation.error_envelope_paths or (("error",),),
+ observation.error_message,
+ ),
)
return _outcome(
kind=RawOutcomeKind.PROTOCOL_ERROR,
@@ -1880,6 +1890,7 @@ def _outcome(
recovery_verified: bool = False,
upstream_detail: str | None = None,
os_errno: int | None = None,
+ rejection: UpstreamRejection | None = None,
) -> RawCallOutcome:
return RawCallOutcome(
kind=kind,
@@ -1895,6 +1906,7 @@ def _outcome(
recovery_verified=recovery_verified,
upstream_detail=upstream_detail,
os_errno=os_errno,
+ rejection=rejection,
)
@@ -1944,6 +1956,68 @@ def _upstream_error_detail(
return None
+_REJECTION_TOKEN = re.compile(r"[A-Za-z0-9_.\[\]-]{1,64}")
+_REJECTION_MESSAGE_SCAN_CHARS = 2000
+_INVALID_VALUE_MESSAGE = re.compile(r"Invalid value: '([A-Za-z0-9_.-]{1,64})'")
+_UNSUPPORTED_TOOL_TYPE_MESSAGE = re.compile(r"Unsupported tool type: ([A-Za-z0-9_.-]{1,64})")
+_INVALID_ENCRYPTED_CONTENT_MESSAGE = re.compile(r"Invalid encrypted_content in ([a-z_]{1,64}) block")
+
+
+def _upstream_rejection(
+ http_status: int | None,
+ stream_started: bool,
+ error_code: str | None,
+ error_type: str | None,
+ payload: bytes,
+ envelope_paths: tuple[ErrorEnvelopePath, ...],
+ message: str | None,
+) -> UpstreamRejection | None:
+ """Project a 400 before output into the closed tokens the evidence rules read.
+
+ The machine code and type come from the one extractor (``_raw_error_fields``
+ and its projection); this adds only the first envelope's ``param`` and the
+ single token each recognized message names. The prose itself never leaves.
+ """
+
+ if http_status != 400 or stream_started:
+ return None
+ names = ("param", "message")
+ paths = tuple((*path, name) for path in envelope_paths for name in names)
+ values: dict[JSONPath, str] = {}
+
+ def visit(path: JSONPath, event: JSONEvent, value: object | None, _scope: JSONScope) -> None:
+ if event == "replace":
+ values.pop(path, None)
+ elif event == "scalar" and isinstance(value, str):
+ values[path] = value
+
+ if payload:
+ project_json_reader(io.BytesIO(payload), paths, visit)
+ envelope = next(
+ (path for path in envelope_paths if any((*path, name) in values for name in names)),
+ None,
+ )
+ fields = {name: values.get((*envelope, name)) for name in names} if envelope is not None else {}
+ text = (fields.get("message") or message or "")[:_REJECTION_MESSAGE_SCAN_CHARS]
+
+ def token(value: object) -> str | None:
+ return value if isinstance(value, str) and _REJECTION_TOKEN.fullmatch(value) else None
+
+ def named(pattern: re.Pattern[str]) -> str | None:
+ found = pattern.search(text)
+ return found.group(1) if found else None
+
+ return UpstreamRejection(
+ status=400,
+ code=token(error_code),
+ error_type=token(error_type),
+ param=token(fields.get("param")),
+ invalid_value=named(_INVALID_VALUE_MESSAGE),
+ unsupported_tool_type=named(_UNSUPPORTED_TOOL_TYPE_MESSAGE),
+ invalid_encrypted_content_block=named(_INVALID_ENCRYPTED_CONTENT_MESSAGE),
+ )
+
+
def _upstream_reset_advice(
payload: bytes,
envelope_paths: tuple[ErrorEnvelopePath, ...],