Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(model-hub): learn capability evidence from upstream errors and stop offering client presets as wire efforts (#2551)
Call White0455b7f459a9
diff --git a/config/v2_config.py b/config/v2_config.pyindex 9270e8ce4..f5b6dbc6c 100644--- a/config/v2_config.py+++ b/config/v2_config.py@@ -9,7 +9,7 @@ import shutilimport tempfileimport threadingfrom contextlib import contextmanager-from dataclasses import dataclass, field, fields+from dataclasses import dataclass, field, fields, replacefrom datetime import datetimefrom pathlib import Pathfrom typing import (@@ -3016,6 +3016,26 @@ class ModelHubAgentSupplyConfig:MODEL_HUB_HOSTED_TOOLS: tuple[str, ...] = HOSTED_TOOLS+def _drop_vibey_client_presets(agent: "ModelHubAgentSupplyConfig") -> None:+ """Keep client presets (Codex ``ultra``) out of Vibey's model rows.++ A preset is a name another client maps before sending, never a wire tier,+ and Vibey is not that client. 3.2.0 saved Vibey rows carrying one (#2539),+ so every read drops them here, the single owner of those rows: the catalog+ payload, the picker, the launch and the API then agree, and the next save+ writes the clean list. No I/O: the preset index is read once per process.+ """++ from vibe.backend_model_catalog import without_client_presets++ agent.models = [+ replace(model, reasoning_efforts=without_client_presets((model.id,), model.reasoning_efforts))+ if model.reasoning_efforts+ else model+ for model in agent.models+ ]++@dataclass(frozen=True)class ModelHubHostedToolConfig:"""One hosted tool's user toggle (``vibey-hosted-tools.md`` §3.3).@@ -3217,6 +3237,7 @@ class ModelHubConfig:runtime_default_applied=runtime_default_applied,hosted_tools=_hosted_tools_from_payload(payload.get("hosted_tools"), sources),)+ _drop_vibey_client_presets(config.agents["vibey"])agents = config.agentsfor backend in MODEL_HUB_BACKENDS:configured_sources = agents[backend].sourcesdiff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.pyindex 7550be7e6..a2d8d90df 100644--- a/core/handlers/model_hub/adapter.py+++ b/core/handlers/model_hub/adapter.py@@ -138,6 +138,25 @@ class RawOutcomeKind(str, Enum):PROTOCOL_ERROR = "protocol_error" # malformed/unparseable upstream response+@dataclass(frozen=True)+class UpstreamRejection:+ """Closed facts of an HTTP 400 the upstream returned before any output.++ The only input of the capability-evidence rules (``model-hub-capability-model.md``+ §3.2). L1 fills it from the error envelope: raw machine tokens, and the one+ token each recognized message names. It never carries prose, and the+ failover decision never reads it.+ """++ status: int+ code: str | None = None # raw machine code, e.g. "invalid_value"+ error_type: str | None = None # raw error type, e.g. "invalid_request_error"+ param: str | None = None # raw parameter path, e.g. "reasoning.effort"+ invalid_value: str | None = None # "Invalid value: '<value>'"+ unsupported_tool_type: str | None = None # "Unsupported tool type: <type>"+ invalid_encrypted_content_block: str | None = None # "Invalid encrypted_content in <kind> block"++@dataclass(frozen=True)class RawCallOutcome:"""Uninterpreted per-call result. Classification is L2's job, not L1's."""@@ -175,6 +194,8 @@ class RawCallOutcome:# Recognized local OS failure only, never an upstream code or error prose.# Display-only; does not participate in classification or Source health.os_errno: int | None = None+ # A 400 before output, as closed tokens for the capability-evidence rules.+ rejection: UpstreamRejection | None = Noneclass ObservationOutcome(str, Enum):diff --git a/core/handlers/model_hub/capability_evidence.py b/core/handlers/model_hub/capability_evidence.pyindex 284234527..3ec280554 100644--- a/core/handlers/model_hub/capability_evidence.py+++ b/core/handlers/model_hub/capability_evidence.py@@ -37,6 +37,9 @@ MAX_RECORDS = 4000OBSERVED_TTL_SECONDS = 7 * 24 * 3600PROBE_REVERIFY_AFTER_SECONDS = 30 * 24 * 3600DETAIL_MAX_CHARS = 64+# The engine stamp when the engine manifest cannot be read. It names no release,+# so probe evidence taken under it, or checked against it, never vouches for a wire.+UNKNOWN_ENGINE_STAMP = "unknown-engine"FeatureKind = Literal["effort", "hosted_tool"]EvidenceStatus = Literal["verified", "unsupported"]@@ -73,6 +76,12 @@ class EvidenceFeature:def effort(cls, name: str) -> "EvidenceFeature":return cls("effort", name)+ @classmethod+ def effort_or_none(cls, name: object) -> Optional["EvidenceFeature"]:+ """The effort feature, or None for a name evidence cannot hold (free-form efforts are allowed)."""++ return cls("effort", name) if isinstance(name, str) and _NAME_RE.fullmatch(name) else None+@classmethoddef hosted(cls, tool: str, aspect: Literal["call", "replay"]) -> "EvidenceFeature":return cls("hosted_tool", tool, aspect)@@ -204,7 +213,11 @@ def resolve(if record.basis == "observed":# Relays change; an expired negative record falls back to the priors.return UNKNOWN if age > OBSERVED_TTL_SECONDS else EvidenceView("unsupported", None, record)- wire_stale = record.engine != engine or record.shape != shape+ wire_stale = (+ record.engine != engine+ or record.shape != shape+ or UNKNOWN_ENGINE_STAMP in (record.engine, engine)+ )stale: Optional[Staleness] = "wire" if wire_stale else "age" if age > PROBE_REVERIFY_AFTER_SECONDS else Noneif record.feature.kind == "hosted_tool" and wire_stale:# Evidence taken on another wire cannot vouch for this one, and a silent@@ -218,36 +231,46 @@ class CapabilityEvidenceStore:Small: bounded by Sources x models x features. The controller is the onlywriter (probes and the classifier both run there; other processes reach it- over RPC), so the file is read once and memory is authoritative afterwards.- Lookups on the request path therefore never touch the disk.+ over RPC), so the file is read once, when the store is built at controller+ startup, and memory is authoritative afterwards. No lookup ever touches the+ disk, so none can block the loop."""def __init__(self, path: Path, *, max_records: int = MAX_RECORDS):self.path = pathself.max_records = max_recordsself._lock = threading.RLock()- self._records: dict[tuple, EvidenceRecord] | None = None-- def _loaded(self) -> dict[tuple, EvidenceRecord]:- if self._records is None:+ self._records: dict[tuple, EvidenceRecord] = {}+ self._by_hop: dict[tuple[str, str], list[EvidenceRecord]] = {}+ self._remember(self._read_file())++ def _read_file(self) -> dict[tuple, EvidenceRecord]:+ try:+ payload = json.loads(self.path.read_text(encoding="utf-8"))+ except (OSError, ValueError):+ payload = []+ records: dict[tuple, EvidenceRecord] = {}+ for item in payload if isinstance(payload, list) else []:try:- payload = json.loads(self.path.read_text(encoding="utf-8"))- except (OSError, ValueError):- payload = []- records: dict[tuple, EvidenceRecord] = {}- for item in payload if isinstance(payload, list) else []:- try:- record = EvidenceRecord.from_payload(item)- except (TypeError, ValueError):- # An unreadable record is unknown, never negative.- continue- records[record.key] = record- self._records = records- return self._records+ record = EvidenceRecord.from_payload(item)+ except (TypeError, ValueError):+ # An unreadable record is unknown, never negative.+ continue+ records[record.key] = record+ return records++ def _remember(self, records: dict[tuple, EvidenceRecord]) -> None:+ by_hop: dict[tuple[str, str], list[EvidenceRecord]] = {}+ for record in records.values():+ by_hop.setdefault((record.source_id, record.model_id), []).append(record)+ # The index first: a reader that sees the new records also sees their index.+ self._by_hop = by_hop+ self._records = recordsdef get(self, source_id: str, model_id: str, feature: EvidenceFeature) -> Optional[EvidenceRecord]:- with self._lock:- return self._loaded().get((source_id, model_id, feature.kind, feature.name, feature.aspect or ""))+ # Published snapshots are never mutated, so a reader needs no lock: a+ # write builds the next snapshot and swaps the references.+ return self._records.get((source_id, model_id, feature.kind, feature.name, feature.aspect or ""))def view(self,@@ -262,8 +285,12 @@ class CapabilityEvidenceStore:return resolve(self.get(source.id, model_id, feature), source=source, engine=engine, shape=shape, now=now)def records(self) -> list[EvidenceRecord]:- with self._lock:- return list(self._loaded().values())+ return list(self._records.values())++ def for_hop(self, source_id: str, model_id: str) -> list[EvidenceRecord]:+ """Every record of one (Source, model), without scanning the others."""++ return list(self._by_hop.get((source_id, model_id), ()))def write(self, record: EvidenceRecord, *, live: Iterable[ModelHubSourceConfig]) -> None:"""Replace the record for its key; prune records of removed or re-identified Sources.@@ -277,10 +304,10 @@ class CapabilityEvidenceStore:with self._lock:records = {key: existing- for key, existing in self._loaded().items()+ for key, existing in self._records.items()if identities.get(existing.source_id) == existing.source_identity}records[record.key] = recordkept = sorted(records.values(), key=lambda item: item.observed_at)[-self.max_records :]write_state_document(self.path, [item.payload() for item in kept])- self._records = {item.key: item for item in kept}+ self._remember({item.key: item for item in kept})diff --git a/core/handlers/model_hub/evidence_rules.py b/core/handlers/model_hub/evidence_rules.pynew file mode 100644index 000000000..54b7d244e--- /dev/null+++ b/core/handlers/model_hub/evidence_rules.py@@ -0,0 +1,92 @@+"""Narrow rules that learn negative capability evidence from real upstream errors.++``model-hub-capability-model.md`` §3.2: a rule maps an upstream error to exactly+one (Source, model, feature), and only when the error names that feature.+Generic errors (5xx, overload, rate limits, auth, any 400 that names nothing+the request sent) never create evidence. The rules read only the closed tokens+L1 put in ``UpstreamRejection`` and the request the gateway sent; they never+change how the request itself is classified or failed over.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any, Callable, Mapping, Optional++from config.v2_config import ModelHubSourceConfig++from .adapter import UpstreamRejection+from .capability_evidence import EvidenceFeature+from .request import declared_tool_types, replays_history_items++# Where each request protocol carries its reasoning effort on the wire.+_EFFORT_PATHS: Mapping[str, tuple[str, ...]] = {+ "openai_responses": ("reasoning", "effort"),+ "openai_chat": ("reasoning_effort",),+ "anthropic": ("output_config", "effort"),+}+_EFFORT_PARAMS = frozenset({"reasoning.effort", "reasoning_effort", "output_config.effort"})++# The hosted result an "Invalid encrypted_content in <kind> block" message names.+_ENCRYPTED_BLOCK_TOOLS: Mapping[str, str] = {"search_result": "web_search"}+# Request items that replay a hosted tool's native result, per tool.+_REPLAYED_RESULT_TYPES: Mapping[str, frozenset[str]] = {+ "web_search": frozenset({"web_search_tool_result", "web_search_call"}),+}++SpecLookup = Callable[[str, str, str, str], Any]+++@dataclass(frozen=True)+class LearnedEvidence:+ feature: EvidenceFeature+ detail: str+++def wire_effort(request: Mapping[str, Any], protocol: str) -> Optional[str]:+ """The reasoning effort this request sends, as the wire spells it."""++ value: Any = request+ for key in _EFFORT_PATHS.get(protocol, ()):+ value = value.get(key) if isinstance(value, Mapping) else None+ return value if isinstance(value, str) and value else None+++def _detail(rejection: UpstreamRejection) -> str:+ return f"{rejection.status} {rejection.code or rejection.error_type or 'error'}"[:64]+++def learn(+ request: Mapping[str, Any],+ rejection: UpstreamRejection,+ *,+ request_protocol: str,+ source: ModelHubSourceConfig,+ model_id: str,+ spec_for: SpecLookup,+) -> Optional[LearnedEvidence]:+ """The one feature this rejection names for this request, if any."""++ effort = EvidenceFeature.effort_or_none(wire_effort(request, request_protocol))+ names_effort_field = rejection.param is None or rejection.param in _EFFORT_PARAMS+ if effort is not None and names_effort_field and (+ rejection.invalid_value == effort.name+ or (+ rejection.invalid_value is None+ and rejection.param in _EFFORT_PARAMS+ and request_protocol == source.protocol+ )+ ):+ return LearnedEvidence(effort, _detail(rejection))++ rejected_type = rejection.unsupported_tool_type+ if rejected_type is not None and rejected_type in declared_tool_types(request):+ for tool in ("web_search", "x_search", "image_generation"):+ spec = spec_for(source.vendor, source.protocol, model_id, tool)+ if spec is not None and spec.declaration_type == rejected_type:+ return LearnedEvidence(EvidenceFeature.hosted(tool, "call"), _detail(rejection))++ tool = _ENCRYPTED_BLOCK_TOOLS.get(rejection.invalid_encrypted_content_block or "")+ if tool is not None and replays_history_items(request, _REPLAYED_RESULT_TYPES[tool]):+ return LearnedEvidence(EvidenceFeature.hosted(tool, "replay"), _detail(rejection))+ return Nonediff --git a/core/handlers/model_hub/hosted_tools.py b/core/handlers/model_hub/hosted_tools.pyindex 9346b1a8c..75db08670 100644--- a/core/handlers/model_hub/hosted_tools.py+++ b/core/handlers/model_hub/hosted_tools.py@@ -26,7 +26,7 @@ from typing import Any, Callable, Mapping, Optionalfrom config.v2_config import MODEL_HUB_HOSTED_TOOLS, ModelHubConfig, ModelHubHostedToolConfig, ModelHubSourceConfigfrom vibe.backend_model_catalog import HostedToolSpec-from .capability_evidence import HOSTED_TOOL_ASPECTS, CapabilityEvidenceStore, EvidenceFeature+from .capability_evidence import HOSTED_TOOL_ASPECTS, UNKNOWN_ENGINE_STAMP, CapabilityEvidenceStore, EvidenceFeature# The request shape a hosted-tool probe sends. Changing the probe or the native# request it mirrors bumps this, which makes earlier probe evidence wire-stale.@@ -70,10 +70,15 @@ def engine_stamp() -> str:Engine upgrades ship as manifest changes, and a patched rebuild of the sameupstream version gets a new asset release, so this is what a wire change- looks like from here.+ looks like from here. Read once, when Model Hub's service is built; a+ manifest that cannot be read is the fixed unknown engine, whose evidence+ never matches a real release and never raises on a request."""- return _manifest_engine_stamp(os.environ.get("VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH") or None)+ try:+ return _manifest_engine_stamp(os.environ.get("VIBE_MODEL_HUB_ENGINE_MANIFEST_PATH") or None)+ except (OSError, ValueError, TypeError, AttributeError):+ return UNKNOWN_ENGINE_STAMP@dataclass(frozen=True)diff --git a/core/handlers/model_hub/migration.py b/core/handlers/model_hub/migration.pyindex 5c942fd0d..9c01c19f4 100644--- a/core/handlers/model_hub/migration.py+++ b/core/handlers/model_hub/migration.py@@ -51,7 +51,7 @@ from core.handlers.model_hub.migration_journal import (from core.handlers.model_hub.migration_persisted import PersistedInventoryfrom vibe.backend_model_catalog import (backend_model_entries,- bundled_catalog_reasoning_efforts_by_model,+ bundled_catalog_wire_efforts_by_model,load_bundled_catalog,)from vibe.codex_config import (@@ -1742,7 +1742,7 @@ async def _prepare_takeover(provisioned: list[dict[str, str]] = []source_ids: list[str] = list(retained_source_ids or ())item_sources: dict[str, dict[str, str]] = {}- catalog = bundled_catalog_reasoning_efforts_by_model()+ catalog = bundled_catalog_wire_efforts_by_model()handoff_attempted = Falsetry:# A completed receipt can authorize cleanup of the exact old materialdiff --git a/core/handlers/model_hub/reasoning_tiers.py b/core/handlers/model_hub/reasoning_tiers.pyindex a98b7d164..d4e572cff 100644--- a/core/handlers/model_hub/reasoning_tiers.py+++ b/core/handlers/model_hub/reasoning_tiers.py@@ -7,7 +7,8 @@ from typing import Literal, Mapping, Sequencefrom vibe.backend_model_catalog import (PROTOCOL_REASONING_EFFORT_DEFAULTS,- bundled_catalog_reasoning_efforts_for_model,+ bundled_catalog_wire_efforts_by_model,+ without_client_presets,)ReasoningEffortsSource = Literal["upstream", "catalog", "user"] | None@@ -30,7 +31,13 @@ def resolve_reasoning_tiers(existing_source: ReasoningEffortsSource = None,catalog_efforts_by_model: Mapping[str, tuple[str, ...]] | None = None,) -> ReasoningTierResolution:- """Apply upstream, bundled-catalog, then user provenance in that order."""+ """Apply upstream, bundled-catalog, then user provenance in that order.++ A catalog row is a client's menu: its ``client_presets`` (Codex ``ultra``)+ are names the client maps before sending, so a Source's wire tiers never+ include them (#2539). ``catalog_efforts_by_model`` is therefore the wire+ index, ``bundled_catalog_wire_efforts_by_model()``.+ """if supported_parameters is not None and any(parameter.strip().lower() in _UPSTREAM_REASONING_PARAMETERS@@ -41,13 +48,21 @@ def resolve_reasoning_tiers(return ReasoningTierResolution(defaults, "upstream")catalog_efforts = (- bundled_catalog_reasoning_efforts_for_model(model_id)- if catalog_efforts_by_model is None- else catalog_efforts_by_model.get(model_id)- )+ bundled_catalog_wire_efforts_by_model() if catalog_efforts_by_model is None else catalog_efforts_by_model+ ).get(model_id)if catalog_efforts is not None:return ReasoningTierResolution(catalog_efforts, "catalog")if existing_source == "user" and existing_efforts:return ReasoningTierResolution(tuple(existing_efforts), "user")return ReasoningTierResolution((), None)+++def catalog_wire_efforts(model_id: str, efforts: Sequence[str]) -> tuple[str, ...]:+ """A catalog-sourced tier list without the client presets of that model's row.++ Also applied when reading a persisted ``catalog`` list, which a released+ build may have written before presets were known.+ """++ return tuple(without_client_presets((model_id,), efforts))diff --git a/core/handlers/model_hub/request.py b/core/handlers/model_hub/request.pyindex 085baf681..bd43c94a3 100644--- a/core/handlers/model_hub/request.py+++ b/core/handlers/model_hub/request.py@@ -107,3 +107,29 @@ def without_opaque_history(request: Mapping[str, Any]) -> Mapping[str, Any]:primary_origin=request.primary_origin,)return payload+++def declared_tool_types(request: Mapping[str, Any]) -> frozenset[str]:+ """The ``type`` of every tool the request declares."""++ tools = request.get("tools")+ if not isinstance(tools, list):+ return frozenset()+ return frozenset(+ kind for tool in tools if isinstance(tool, Mapping) and isinstance(kind := tool.get("type"), str)+ )+++def replays_history_items(request: Mapping[str, Any], kinds: frozenset[str]) -> bool:+ """Whether the request's history carries an item of one of these ``type`` values, at any depth."""++ pending: list[Any] = [request.get("messages", request.get("input"))]+ while pending:+ value = pending.pop()+ if isinstance(value, Mapping):+ if value.get("type") in kinds:+ return True+ pending.extend(value.values())+ elif isinstance(value, list):+ pending.extend(value)+ return Falsediff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.pyindex 309dd2330..b18bc9a34 100644--- a/core/handlers/model_hub/service.py+++ b/core/handlers/model_hub/service.py@@ -50,7 +50,9 @@ from storage.models import MESSAGE_TURN_ID_SQL, agent_sessions, messagesfrom vibe.backend_model_catalog import (PROTOCOL_REASONING_EFFORT_DEFAULTS,builtin_display_name,- bundled_catalog_reasoning_efforts_by_model,+ bundled_catalog_wire_efforts_by_model,+ bundled_client_presets_once,+ bundled_provider_overlay,hosted_tool_spec,)from vibe.model_hub_runtime.api_key_vendors import (@@ -104,9 +106,12 @@ from .events import (build_resolution_event,contains_credential_material,)-from .capability_evidence import CapabilityEvidenceStore+from .capability_evidence import CapabilityEvidenceStore, EvidenceFeature, EvidenceRecord, identity_digestfrom .errors import ModelDiscoveryError+from .evidence_rules import learn as learn_from_rejection+from .evidence_rules import wire_effortfrom .hosted_tools import (+ HOSTED_PROBE_SHAPE,HostedToolSettingRefused,engine_stamp,hosted_tools_status,@@ -155,7 +160,7 @@ from .provenance import (produce_turn_outcome,)from .request import ModelHubRequest, without_opaque_history-from .reasoning_tiers import resolve_reasoning_tiers+from .reasoning_tiers import catalog_wire_efforts, resolve_reasoning_tiersfrom .stream_wire import ProtocolSSEStatefrom .resolver import (BackendName,@@ -1177,6 +1182,11 @@ class ModelHubService:self.capability_evidence = capability_evidence or CapabilityEvidenceStore(events.path.parent / "model_hub_capability_evidence.json")+ # Bundled state that never changes while this process runs, resolved once+ # here at startup so that no request path reads a file on the loop.+ self.engine_stamp = engine_stamp()+ bundled_provider_overlay()+ bundled_client_presets_once()self.migration_blocked_backends: set[str] = set()self.migration_guard = migration_guard or self._unavailable_migration_guardself.migration_reconcile_auth = migration_reconcile_auth@@ -2477,7 +2487,7 @@ class ModelHubService:discovered_by_id = {model.id: model for model in canonical_models}overrides: list[tuple[str, Literal["upstream", "catalog"]]] = []if catalog_efforts_by_model is None:- catalog_efforts_by_model = bundled_catalog_reasoning_efforts_by_model()+ catalog_efforts_by_model = bundled_catalog_wire_efforts_by_model()def apply_tiers(model: ModelHubModelConfig,@@ -2953,7 +2963,7 @@ class ModelHubService:)else:catalog_efforts_by_model = (- bundled_catalog_reasoning_efforts_by_model()+ bundled_catalog_wire_efforts_by_model())for model in source.models:resolution = resolve_reasoning_tiers(@@ -3740,7 +3750,7 @@ class ModelHubService:allow_empty=True,)else:- catalog_efforts_by_model = bundled_catalog_reasoning_efforts_by_model()+ catalog_efforts_by_model = bundled_catalog_wire_efforts_by_model()for model in source.models:resolution = resolve_reasoning_tiers(protocol=source.protocol,@@ -5179,7 +5189,11 @@ class ModelHubService:)proposed_name, proposed_efforts = _storable_backend_model_metadata(model.display_name,- model.reasoning_efforts,+ (+ catalog_wire_efforts(model.id, model.reasoning_efforts)+ if model.reasoning_efforts_source == "catalog"+ else model.reasoning_efforts+ ),)if display_name is None and proposed_name is not None:display_name = proposed_name@@ -5775,7 +5789,7 @@ class ModelHubService:self.store.load(),evidence=self.capability_evidence,spec_for=hosted_tool_spec,- engine=engine_stamp(),+ engine=self.engine_stamp,now=self.now(),)}@@ -5795,6 +5809,77 @@ class ModelHubService:await self._commit_synced(previous, config)return await asyncio.to_thread(self.hosted_tools_status)+ def _effort_refused(self, source: ModelHubSourceConfig, model_id: str, request: Any) -> bool:+ """Whether this hop has negative evidence for the effort the request sends.++ Efforts are forwarded unchanged; a hop known to refuse one is not a+ candidate for this request (capability model §3.4). Every place that+ decides on candidates asks this one predicate, so the resolve loop and+ the recovery loop never disagree. Memory-only and lock-free: the store+ read its file at startup.+ """++ feature = EvidenceFeature.effort_or_none(wire_effort(request, getattr(request, "protocol", "")))+ if feature is None:+ return False+ view = self.capability_evidence.view(+ source, model_id, feature, engine=self.engine_stamp, shape=HOSTED_PROBE_SHAPE, now=self.now(),+ )+ return view.status == "unsupported"++ def _effort_capable_hop_may_recover(+ self, backend: str, model_id: str, supply_channel: Any, request: Any+ ) -> bool:+ """Whether a temporarily blocked hop of this route does not refuse the request's effort."""++ resolution = self._invocation_resolution(+ self.store.load(), cast(BackendName, backend), model_id, supply_channel,+ )+ return any(+ hop.source is not None+ and hop.model_id is not None+ and not self._effort_refused(hop.source, hop.model_id, request)+ for hop in resolution.self_healing_hops(supply_channel)+ )++ async def _learn_capability_evidence(+ self, source: ModelHubSourceConfig, model_id: str, request: Any, outcome: RawCallOutcome+ ) -> None:+ """Record the one feature a 400 before output names, through the narrow rules."""++ if outcome.rejection is None:+ return+ learned = learn_from_rejection(+ request,+ outcome.rejection,+ request_protocol=getattr(request, "protocol", ""),+ source=source,+ model_id=model_id,+ spec_for=hosted_tool_spec,+ )+ if learned is None:+ return+ try:+ await asyncio.to_thread(+ self.capability_evidence.write,+ EvidenceRecord(+ source_id=source.id,+ model_id=model_id,+ feature=learned.feature,+ status="unsupported",+ basis="observed",+ observed_at=self.now(),+ engine=self.engine_stamp,+ shape=HOSTED_PROBE_SHAPE,+ source_identity=identity_digest(source),+ detail=learned.detail,+ ),+ live=self.store.load().sources,+ )+ except (OSError, ValueError):+ # Evidence is advisory state: losing one record leaves the priors.+ logger.warning("Model Hub could not record capability evidence", exc_info=True)+def hosted_tools_for_hop(self, config: ModelHubConfig, source: ModelHubSourceConfig, model_id: str) -> dict:"""``HopResolution.hosted_tools`` for a Vibey launch served by this Source and model."""@@ -5804,7 +5889,7 @@ class ModelHubService:model_id=model_id,evidence=self.capability_evidence,spec_for=hosted_tool_spec,- engine=engine_stamp(),+ engine=self.engine_stamp,now=self.now(),)@@ -6809,6 +6894,7 @@ class ModelHubService:if status in {"active", "standby", "cooldown"}else status)+ refused = self._refused_efforts(source, inspection.model_id, observed_at)chain.append({"source_id": source.id,@@ -6819,6 +6905,7 @@ class ModelHubService:"reason": inspection.reason,"retry_at": inspection.retry_at,**({"recovery": inspection.recovery} if inspection.recovery is not None else {}),+ **({"refused_efforts": refused} if refused else {}),})current = next(@@ -6844,6 +6931,28 @@ class ModelHubService:),}+ def _refused_efforts(self, source: ModelHubSourceConfig, model_id: str, now: datetime) -> list[dict]:+ """The efforts this hop currently refuses, as routing reads the evidence."""++ refused = []+ for record in self.capability_evidence.for_hop(source.id, model_id):+ if record.feature.kind != "effort":+ continue+ view = self.capability_evidence.view(+ source, model_id, record.feature, engine=self.engine_stamp, shape=HOSTED_PROBE_SHAPE, now=now,+ )+ if view.status == "unsupported":+ refused.append(record)+ return [+ {+ "effort": record.feature.name,+ "basis": record.basis,+ "observed_at": record.payload()["observed_at"],+ "detail": record.detail,+ }+ for record in sorted(refused, key=lambda item: (item.observed_at, item.feature.name))+ ]+def agent_chain(self, backend: str, model_id: object) -> dict:if backend not in MODEL_HUB_BACKENDS or not isinstance(model_id, str) or not model_id:raise ModelHubError("mapping_target_unavailable", status=409)@@ -9022,6 +9131,7 @@ class ModelHubService:globally_blocked_source_ids: set[str] = set()unsupported_hops: set[tuple[str, str]] = set()unsupported_reason: str | None = None+ effort_rejected_sources: list[str] = []while True:async with self._mutation_lock:if recovery_request is not None and recovery_request.expired:@@ -9095,6 +9205,9 @@ class ModelHubService:)if reason is not None:raise _InvocationUnsupported(reason)+ if self._effort_refused(source, target_model, request):+ effort_rejected_sources.append(source.id)+ raise _InvocationUnsupported("effort_unsupported")await self._prepare_engine_for_demand(already_synced=engine_prepared)engine_prepared = Truehandle, outcome, cancelled = await self._invoke(@@ -9148,6 +9261,7 @@ class ModelHubService:origin=origin,)decision = await self._classify_source_outcome(source, outcome)+ await self._learn_capability_evidence(source, target_model, request, outcome)if cancelled is not None:await self._settle_cancelled_attempt(source=source,@@ -9211,6 +9325,7 @@ class ModelHubService:origin=origin,)decision = classify_outcome(outcome, refresh_attempted=True)+ await self._learn_capability_evidence(source, target_model, request, outcome)if cancelled is not None:await self._settle_cancelled_attempt(source=source,@@ -9330,11 +9445,29 @@ class ModelHubService:turn_outcome=ENGINE_DOWN_TURN_OUTCOME,local_error_detail=format_os_errno(outcome.os_errno),)+ if (+ unsupported_reason == "effort_unsupported"+ and failed_source is None+ and self._effort_capable_hop_may_recover(backend, model_id, supply_channel, request)+ ):+ # A hop that does not refuse this effort is only blocked for now:+ # the request waits for it like any other, rather than ending here.+ unsupported_reason = Noneif unsupported_reason is not None and failed_source is None:# No upstream attempt exists. Exhaustion would invent a failed# attempt (and a retryable supply state) for a local limitation.raise ModelHubError(- unsupported_reason, status=422, turn_outcome=REQUEST_NONFALLBACK_TURN_OUTCOME,+ unsupported_reason,+ status=422,+ turn_outcome=REQUEST_NONFALLBACK_TURN_OUTCOME,+ data=(+ {+ "effort": wire_effort(request, getattr(request, "protocol", "")),+ "source_ids": list(dict.fromkeys(effort_rejected_sources)),+ }+ if unsupported_reason == "effort_unsupported"+ else None+ ),)final_config, final_resolution = self._inspect_terminal_chain(backend=cast(BackendName, backend),@@ -9410,7 +9543,15 @@ class ModelHubService:resolution = self._invocation_resolution(config, kwargs["backend"], kwargs["model_id"], supply_channel,)- if resolution.candidate_hops:+ request = kwargs.get("request")+ if any(+ hop.source is not None+ and hop.model_id is not None+ and not self._effort_refused(hop.source, hop.model_id, request)+ for hop in resolution.candidate_hops+ ):+ # Retry now only for a hop this request may use: a hop+ # that refuses its effort would spin the loop.continue# An action already owed elsewhere in the chain keeps it# `interrupted` for display, but must not end this requestdiff --git a/docs/plans/model-hub-capability-model.md b/docs/plans/model-hub-capability-model.mdindex 48d2f43ad..e8a122fa2 100644--- a/docs/plans/model-hub-capability-model.md+++ b/docs/plans/model-hub-capability-model.md@@ -82,12 +82,18 @@ and is pruned at the next write.### 3.2 Learning from real errors (negative evidence)-- **Narrow rules, one per feature, in one place:** the gateway's classifier. It sees every upstream error, and- nothing else writes evidence. A rule maps an upstream error to exactly one (Source, model, feature) only when the- error names that feature:- - `invalid_request_error` / `invalid_value` whose message names the request's `reasoning.effort` value → effort;- - `Unsupported tool type: <type>` for a type the request declared → hosted tool `call`;- - `Invalid encrypted_content` inside a hosted result block the request replayed → hosted tool `replay`.+- **Narrow rules, one per feature, in one place:** `core/handlers/model_hub/evidence_rules.py`, run by the+ gateway's resolution loop on every upstream outcome. Nothing else writes evidence from real traffic. The engine+ client (L1) projects a 400 before output into an `UpstreamRejection`: the machine code and type from the one+ error-field extractor, the envelope's `param`, and the single token each recognized message names. The prose+ never leaves L1, and the failover decision never reads the rejection. A rule maps it to exactly one (Source,+ model, feature) only when it names that feature:+ - `Invalid value: '<v>'` where `<v>` is the effort the request sends on the wire, or `param` naming the effort+ field on an untranslated path → effort `<v>`;+ - `Unsupported tool type: <type>` for a type the request declared, matched to a catalog tool through the+ overlay's declaration for that Source and model → hosted tool `call`;+ - `Invalid encrypted_content in search_result block` while the request replays a web-search result → hosted+ tool `web_search` `replay`.- **Generic errors never create evidence.** That covers 5xx, overload, rate limit, auth, and any 400 that does notname the feature.- **What happens to the request.**@@ -126,15 +132,15 @@ and is pruned at the next write.| Consumer | Rule || --- | --- |-| Effort menu for a model on an Agent | each hop's declared tiers, then the union across hops. A hop's declared tiers are its Source model's persisted `reasoning_efforts` when their provenance is `user` or `upstream`; otherwise the overlay's `providers` `wire_efforts`; otherwise the backend row's efforts minus its `client_presets` (provenance `catalog`); otherwise models.dev `reasoning_options`. User-authored lists, including free-form names, are never replaced or filtered by a prior; only a catalog-sourced list drops `client_presets`. A tier is omitted only if **every** hop that can serve the model has negative evidence for it. A tier with negative evidence on some hops is shown with "not on: <Source>". |+| Effort menu for a model on an Agent | each hop's declared tiers, then the union across hops. A hop's declared tiers follow `model-hub-reasoning-tiers.md`'s ladder: an upstream reasoning signal (the protocol's ladder), else the bundled backend row's efforts minus that row's `client_presets` (provenance `catalog`; it outranks a user list, with a visible override event), else the user's list, else models.dev `reasoning_options` when the catalog row is built. A persisted `catalog` list is also read without `client_presets`, since released builds wrote them, and Vibey's own model rows always drop them at config load (3.2.0 saved rows with Codex's `ultra`; Vibey is never the client a preset belongs to, and no provider documents `ultra` as a wire effort). A user-authored Source list is never filtered. The overlay's `wire_efforts` slots in before the backend row once a vendor needs it; no shipped entry sets it yet. A tier is omitted only if **every** hop that can serve the model has negative evidence for it. A tier with negative evidence on some hops is shown with "not on: <Source>". Model Hub projects this per hop as the chain API's read-only `refused_efforts` (`agent-chain.schema.json`); the UI renders the omission and the hint from it. || Effort at request time | Forwarded unchanged (reasoning-intent). A hop with **negative** evidence for that effort is skipped for this request, as if it were not a candidate. If no candidate remains, the request fails before invocation with `effort_unsupported` naming the effort and the Sources, instead of a provider 400. Never clamped silently. || Hosted tool mode on a hop | `native` only with positive `call` evidence taken on the current engine and probe shape, and the user's toggle on; otherwise the gateway proxy (`vibey-hosted-tools.md`). A wrong native declaration is a terminal 400 or a silent drop; the proxy is lossless. So this feature needs positive evidence, unlike efforts. || Native payload replay | only to the same Source, and only while its `replay` evidence is positive and taken on the current engine and probe shape. The decision is made at every request from `HopResolution`; the transcript never stores it. || Display | §5 |-This fixes #2539 in three places:+This fixes #2539 in three places (Phase 1 PR 1b):- the overlay lists `ultra` in Codex's `client_presets`, so it is not a wire tier for Vibey;-- models.dev's wire list (`low … max`) is the base;+- a catalog-sourced tier list is the backend row minus those presets (`low … max`, which matches models.dev's wire list), also when a released build persisted it with `ultra`;- the relay's real `400 invalid_value` records `ultra: unsupported` for that Source.Even without the overlay fix, the next request would skip that hop or fail clearly.@@ -161,7 +167,7 @@ validates it):"anthropic": {"models": [{"match": {"prefix": "claude-"},"hosted_tools": {"anthropic": {- "web_search": {"declaration": {"type": "web_search_20250305", "name": "web_search"},+ "web_search": {"declaration": {"type": "web_search_20250305", "name": "web_search", "max_uses": 5},"price": {"unit": "search", "usd_per_1k": 10, "as_of": "2026-10-13", "url": "https://platform.claude.com/docs/en/about-claude/pricing"}}}}}]}}diff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.pyindex 7550be7e6..a2d8d90df 100644--- a/docs/plans/model-hub-contracts/adapter-interface.py+++ b/docs/plans/model-hub-contracts/adapter-interface.py@@ -138,6 +138,25 @@ class RawOutcomeKind(str, Enum):PROTOCOL_ERROR = "protocol_error" # malformed/unparseable upstream response+@dataclass(frozen=True)+class UpstreamRejection:+ """Closed facts of an HTTP 400 the upstream returned before any output.++ The only input of the capability-evidence rules (``model-hub-capability-model.md``+ §3.2). L1 fills it from the error envelope: raw machine tokens, and the one+ token each recognized message names. It never carries prose, and the+ failover decision never reads it.+ """++ status: int+ code: str | None = None # raw machine code, e.g. "invalid_value"+ error_type: str | None = None # raw error type, e.g. "invalid_request_error"+ param: str | None = None # raw parameter path, e.g. "reasoning.effort"+ invalid_value: str | None = None # "Invalid value: '<value>'"+ unsupported_tool_type: str | None = None # "Unsupported tool type: <type>"+ invalid_encrypted_content_block: str | None = None # "Invalid encrypted_content in <kind> block"++@dataclass(frozen=True)class RawCallOutcome:"""Uninterpreted per-call result. Classification is L2's job, not L1's."""@@ -175,6 +194,8 @@ class RawCallOutcome:# Recognized local OS failure only, never an upstream code or error prose.# Display-only; does not participate in classification or Source health.os_errno: int | None = None+ # A 400 before output, as closed tokens for the capability-evidence rules.+ rejection: UpstreamRejection | None = Noneclass ObservationOutcome(str, Enum):diff --git a/docs/plans/model-hub-contracts/agent-chain.schema.json b/docs/plans/model-hub-contracts/agent-chain.schema.jsonindex 9aa131a61..86bbcd5f3 100644--- a/docs/plans/model-hub-contracts/agent-chain.schema.json+++ b/docs/plans/model-hub-contracts/agent-chain.schema.json@@ -97,6 +97,22 @@"type": ["string", "null"],"format": "date-time","description": "When health is backoff, this live deadline is strictly later than the read assembler's captured assembly time. The API normalizes an expired overlay to its underlying non-backoff health before validation and serialization; JSON Schema enforces only the date-time shape, not the relation to a moving clock."+ },+ "refused_efforts": {+ "type": "array",+ "minItems": 1,+ "description": "Reasoning efforts this exact hop is currently known to refuse (capability evidence, model-hub-capability-model.md §3.4), oldest evidence first; absent when none. A request at one of them skips this hop. The effort menu omits a tier only when every hop that can serve the model lists it, and otherwise marks it \"not on: <Source>\". Read-only: routing reads the evidence store itself.",+ "items": {+ "type": "object",+ "required": ["effort", "basis", "observed_at", "detail"],+ "additionalProperties": false,+ "properties": {+ "effort": { "type": "string", "minLength": 1 },+ "basis": { "enum": ["probe", "observed"] },+ "observed_at": { "type": "string", "format": "date-time" },+ "detail": { "type": ["string", "null"], "maxLength": 64 }+ }+ }}},"allOf": [diff --git a/tests/fixtures/model_hub/released_3_2_0_vibey_client_preset.json b/tests/fixtures/model_hub/released_3_2_0_vibey_client_preset.jsonnew file mode 100644index 000000000..218b9fadf--- /dev/null+++ b/tests/fixtures/model_hub/released_3_2_0_vibey_client_preset.json@@ -0,0 +1,761 @@+{+ "enabled": true,+ "runtime_default_applied": true,+ "sources": [+ {+ "id": "src_relay3200",+ "created_at": "2026-10-01T00:00:00Z",+ "last_discovered_at": null,+ "kind": "api_key",+ "vendor": "openai",+ "display_name": "OpenAI 中转",+ "protocol": "openai_responses",+ "base_url": null,+ "supply_channel": "hub",+ "billing": "metered",+ "state": {+ "status": "active",+ "retry_at": null,+ "detail_key": null+ },+ "models": [+ {+ "id": "gpt-6-astra",+ "display_name": null,+ "origin": "discovered",+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max",+ "ultra"+ ],+ "reasoning_efforts_source": "catalog",+ "discovered_at": null+ }+ ],+ "credential_ref": "cred_src_relay3200",+ "account_label": null,+ "masked_credential": null+ }+ ],+ "agents": {+ "vibey": {+ "backend": "vibey",+ "mode": "hub",+ "menu_kind": "fixed",+ "sources": {+ "order": [+ "src_relay3200"+ ]+ },+ "routes": {+ "gpt-6-astra": {+ "hops": [+ {+ "source_id": "src_relay3200",+ "model_id": "gpt-6-astra"+ }+ ]+ }+ },+ "menu": null,+ "models": [+ {+ "id": "gpt-6-astra",+ "display_name": "GPT-6 Astra",+ "origin": "provider",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": true,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max",+ "ultra"+ ]+ }+ ],+ "removed_model_ids": []+ },+ "opencode": {+ "backend": "opencode",+ "mode": "hub",+ "menu_kind": "open",+ "sources": {+ "order": []+ },+ "routes": {},+ "menu": {+ "view": "featured",+ "checked": []+ },+ "models": [],+ "removed_model_ids": []+ },+ "claude": {+ "backend": "claude",+ "mode": "hub",+ "menu_kind": "fixed",+ "sources": {+ "order": []+ },+ "routes": {},+ "menu": null,+ "models": [+ {+ "id": "claude-fable-5-1",+ "display_name": "claude-fable-5-1 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-fable-5",+ "display_name": "claude-fable-5 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-opus-5-5",+ "display_name": "claude-opus-5-5 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-opus-5",+ "display_name": "claude-opus-5 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-opus-4-8",+ "display_name": "claude-opus-4-8 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-opus-4-7",+ "display_name": "claude-opus-4-7 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-opus-4-6",+ "display_name": "claude-opus-4-6 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "max"+ ]+ },+ {+ "id": "claude-sonnet-5-5",+ "display_name": "claude-sonnet-5-5 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-sonnet-5",+ "display_name": "claude-sonnet-5 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-sonnet-4-6",+ "display_name": "claude-sonnet-4-6 [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "max"+ ]+ },+ {+ "id": "claude-haiku-4-5",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high"+ ]+ },+ {+ "id": "claude-opus-4-5",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high"+ ]+ },+ {+ "id": "claude-sonnet-4-5",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high"+ ]+ },+ {+ "id": "opus",+ "display_name": "opus [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "sonnet",+ "display_name": "sonnet [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "haiku",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high"+ ]+ },+ {+ "id": "opus[1m]",+ "display_name": "opus[1m] [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "sonnet[1m]",+ "display_name": "sonnet[1m] [1M]",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "claude-opus-4",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ },+ {+ "id": "claude-sonnet-4",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ },+ {+ "id": "claude-haiku-4",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ },+ {+ "id": "claude-sonnet-4-0",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ },+ {+ "id": "claude-sonnet-4-20250514",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ },+ {+ "id": "claude-sonnet-3-7",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ },+ {+ "id": "claude-haiku-3-5",+ "display_name": null,+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": []+ }+ ],+ "removed_model_ids": []+ },+ "codex": {+ "backend": "codex",+ "mode": "hub",+ "menu_kind": "fixed",+ "sources": {+ "order": []+ },+ "routes": {},+ "menu": null,+ "models": [+ {+ "id": "gpt-6-astra",+ "display_name": "GPT-6-Astra",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max",+ "ultra"+ ]+ },+ {+ "id": "gpt-6.1-sol",+ "display_name": "GPT-6.1-Sol",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "gpt-6-sol",+ "display_name": "GPT-6-Sol",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "gpt-6-luna",+ "display_name": "GPT-6-Luna",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "gpt-5.6-sol",+ "display_name": "GPT-5.6-Sol",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max",+ "ultra"+ ]+ },+ {+ "id": "gpt-5.6-terra",+ "display_name": "GPT-5.6-Terra",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max",+ "ultra"+ ]+ },+ {+ "id": "gpt-5.6-luna",+ "display_name": "GPT-5.6-Luna",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh",+ "max"+ ]+ },+ {+ "id": "gpt-5.5",+ "display_name": "GPT-5.5",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh"+ ]+ },+ {+ "id": "gpt-5.4",+ "display_name": "GPT-5.4",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh"+ ]+ },+ {+ "id": "gpt-5.4-mini",+ "display_name": "GPT-5.4-Mini",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh"+ ]+ },+ {+ "id": "gpt-5.2",+ "display_name": "GPT-5.2",+ "origin": "builtin",+ "models_dev_id": null,+ "context_window": null,+ "max_output_tokens": null,+ "input_modalities": [],+ "output_modalities": [],+ "supports_tools": null,+ "supports_reasoning": null,+ "reasoning_efforts": [+ "low",+ "medium",+ "high",+ "xhigh"+ ]+ }+ ],+ "removed_model_ids": []+ }+ }+}diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yamlindex 4855db6fc..0ecbd7294 100644--- a/tests/scenarios/INDEX.yaml+++ b/tests/scenarios/INDEX.yaml@@ -55,6 +55,7 @@ capabilities:- tests/test_provider_overlay.py- tests/agent_core/ai/test_hosted_requests.py- tests/test_vibey_agent.py+ - tests/test_model_hub_evidence_rules.py- tests/test_model_hub_admission.py- tests/test_model_hub_vibey_consumer.py- tests/test_model_hub_config.pydiff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex bb2452f81..b0dc24b76 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -11,6 +11,7 @@ capability:- tests/test_provider_overlay.py- tests/agent_core/ai/test_hosted_requests.py- tests/test_vibey_agent.py+ - tests/test_model_hub_evidence_rules.py- tests/test_model_hub_config.py- tests/test_model_hub_legacy_opencode_migration.py- tests/test_member_management_parity.py@@ -203,6 +204,18 @@ scenarios:- tests/agent_core/ai/test_hosted_requests.py::test_a_hosted_spec_on_a_native_route_is_sent_as_the_declaration_in_its_place_and_pinned- tests/agent_core/ai/test_hosted_requests.py::test_the_probe_entry_builds_exactly_the_request_the_adapter_sends- tests/agent_core/ai/test_hosted_requests.py::test_tool_choice_forces_the_named_tool_in_each_protocols_form+ - id: MH-HOSTED-006+ name: A real upstream 400 teaches only the feature it names, and a refused effort then skips its hop instead of another provider 400+ status: covered+ kind: recovery+ layer: contract+ test: tests/test_model_hub_evidence_rules.py::test_an_upstream_400_teaches_only_the_feature_it_names+ related_tests:+ - tests/test_model_hub_evidence_rules.py::test_a_refused_effort_skips_its_hop_instead_of_another_provider_400+ - tests/test_model_hub_reasoning_tiers.py::test_catalog_row_outranks_user_and_drops_its_client_presets+ - tests/test_model_hub_reasoning_tiers.py::test_a_persisted_catalog_list_offers_no_client_preset_to_vibey+ - tests/test_model_hub_vibey_consumer.py::test_a_released_vibey_row_loads_without_its_client_preset+ - tests/test_model_hub_evidence_rules.py::test_a_refused_effort_still_waits_for_a_capable_hop_that_is_cooling_down- id: MH-VIBEY-007name: Vibey starts once with the user's Sources and built-in Agent models, seeded when its first Source existsstatus: covereddiff --git a/tests/test_model_hub_api.py b/tests/test_model_hub_api.pyindex cb33f765a..11bbd347b 100644--- a/tests/test_model_hub_api.py+++ b/tests/test_model_hub_api.py@@ -6131,7 +6131,7 @@ def test_create_source_rejects_omitted_protocol_for_an_uncataloged_vendor(tmp_pa("openai_responses",DiscoveredModel(id="gpt-5.6-sol"),- ["low", "medium", "high", "xhigh", "max", "ultra"],+ ["low", "medium", "high", "xhigh", "max"],"catalog",),),@@ -6237,7 +6237,7 @@ def test_source_create_requires_explicit_consent_for_proven_inventory_failure(return {"catalog-manual": ("low", "ultra")}monkeypatch.setattr(- "core.handlers.model_hub.service.bundled_catalog_reasoning_efforts_by_model",+ "core.handlers.model_hub.service.bundled_catalog_wire_efforts_by_model",load_catalog_index,)store = MemoryStore()@@ -7036,14 +7036,7 @@ def test_add_model_upsert_cannot_bypass_catalog_tier_lock(tmp_path):))["models"][0]- assert created["reasoning_efforts"] == [- "low",- "medium",- "high",- "xhigh",- "max",- "ultra",- ]+ assert created["reasoning_efforts"] == ["low", "medium", "high", "xhigh", "max"]assert created["reasoning_efforts_source"] == "catalog"with pytest.raises(ModelHubError) as error:asyncio.run(@@ -7064,7 +7057,7 @@ def test_add_model_upsert_cannot_bypass_catalog_tier_lock(tmp_path):((DiscoveredModel(id="gpt-5.6-sol"),- ["low", "medium", "high", "xhigh", "max", "ultra"],+ ["low", "medium", "high", "xhigh", "max"],"catalog","catalog_tiers",),@@ -9003,14 +8996,7 @@ def test_completed_hub_oauth_persists_only_a_response_proven_protocol(tmp_path):assert result["source"]["protocol"] == "openai_chat"assert store.config.sources[0].protocol == "openai_chat"assert store.config.sources[0].credential_ref == "cred_oauth_proven"- assert store.config.sources[0].models[0].reasoning_efforts == [- "low",- "medium",- "high",- "xhigh",- "max",- "ultra",- ]+ assert store.config.sources[0].models[0].reasoning_efforts == ["low", "medium", "high", "xhigh", "max"]assert (store.config.sources[0].models[0].reasoning_efforts_source== "catalog"diff --git a/tests/test_model_hub_capability_evidence.py b/tests/test_model_hub_capability_evidence.pyindex 98d462dd8..f0c32c905 100644--- a/tests/test_model_hub_capability_evidence.py+++ b/tests/test_model_hub_capability_evidence.py@@ -76,6 +76,7 @@ RESOLUTION = [("engine-upgrade-effort-negative-still-applies", ULTRA, {"status": "unsupported"}, "model-hub-engine-v9", 1, 0 * DAYS, "unsupported", "wire"),("observed-negative-within-ttl", ULTRA, {"status": "unsupported", "basis": "observed", "detail": "400 invalid_value"}, "model-hub-engine-v9", 1, 6 * DAYS, "unsupported", None),("observed-negative-expires", CALL, {"status": "unsupported", "basis": "observed", "detail": "400 invalid_request_error"}, ENGINE, 1, 8 * DAYS, "unknown", None),+ ("unknown-engine-never-vouches", CALL, {"engine": "unknown-engine"}, "unknown-engine", 1, 0 * DAYS, "unknown", "wire"),]@@ -156,8 +157,8 @@ def test_image_generation_has_no_replay_evidence():EvidenceFeature.hosted("image_generation", "replay")-def test_lookups_after_the_first_read_never_touch_the_disk(tmp_path, monkeypatch):- """MH-HOSTED-003: the request path reads evidence from memory, never from the file."""+def test_lookups_after_construction_never_touch_the_disk(tmp_path, monkeypatch):+ """MH-HOSTED-003: the store reads its file when built at startup; no lookup or write reads it again."""path = tmp_path / "model_hub_capability_evidence.json"source = _source()CapabilityEvidenceStore(path).write(_record(source, CALL), live=[source])@@ -184,3 +185,41 @@ def test_lookups_after_the_first_read_never_touch_the_disk(tmp_path, monkeypatchstore.write(_record(source, REPLAY), live=[source])assert store.view(source, "grok-4.6", REPLAY, engine=ENGINE, shape=1, now=NOW).status == "verified"assert touched == []+++def test_a_read_never_waits_for_a_write_on_a_slow_disk(tmp_path, monkeypatch):+ """MH-HOSTED-003: the request path reads a published snapshot, so a write stuck on the disk cannot stall it."""+ import threading++ from core.handlers.model_hub import capability_evidence++ path = tmp_path / "model_hub_capability_evidence.json"+ source = _source()+ store = CapabilityEvidenceStore(path)+ store.write(_record(source, CALL), live=[source])++ on_disk = threading.Event()+ release = threading.Event()+ persist = capability_evidence.write_state_document++ def slow_disk(target, payload):+ on_disk.set()+ assert release.wait(5)+ persist(target, payload)++ monkeypatch.setattr(capability_evidence, "write_state_document", slow_disk)+ writer = threading.Thread(target=store.write, args=(_record(source, REPLAY),), kwargs={"live": [source]})+ writer.start()+ try:+ assert on_disk.wait(5)+ read = []+ reader = threading.Thread(+ target=lambda: read.append(store.view(source, "grok-4.6", CALL, engine=ENGINE, shape=1, now=NOW).status)+ )+ reader.start()+ reader.join(1)+ assert read == ["verified"]+ finally:+ release.set()+ writer.join(5)+ assert store.view(source, "grok-4.6", REPLAY, engine=ENGINE, shape=1, now=NOW).status == "verified"diff --git a/tests/test_model_hub_evidence_rules.py b/tests/test_model_hub_evidence_rules.pynew file mode 100644index 000000000..647776a9e--- /dev/null+++ b/tests/test_model_hub_evidence_rules.py@@ -0,0 +1,369 @@+"""Negative capability evidence learned from real upstream errors (capability model §3.2, §3.4).++The real engine client parses each upstream 400 through a loopback engine, so the+whole path is exercised: L1's closed rejection tokens, the narrow rule that names+exactly one (Source, model, feature), and the hop skip that the next request at a+refused effort gets instead of another provider 400.+"""++from __future__ import annotations++import json++import pytest+from aiohttp import web++from core.handlers.model_hub import service as service_module+from core.handlers.model_hub.capability_evidence import EvidenceFeature+from core.handlers.model_hub.request import ModelHubRequest+from core.handlers.model_hub.service import ModelHubError+from tests.test_model_hub_l3 import _assert_valid, _source+from tests.test_model_hub_retry_lifecycle import configured_service, loopback_engine+from vibe.backend_model_catalog import HostedToolSpec++ULTRA_REFUSED = {+ "error": {+ "code": "invalid_value",+ "type": "invalid_request_error",+ "message": "Invalid value: 'ultra'. Supported values are: 'none', 'minimal', 'low', 'medium', 'high', 'xhigh', and 'max'.",+ }+}+TOOL_REFUSED = {"error": {"message": "Unsupported tool type: web_search", "type": "invalid_request_error"}}+REPLAY_REFUSED = {+ "type": "error",+ "error": {"type": "invalid_request_error", "message": "Invalid encrypted_content in search_result block"},+}+WEB_SEARCH_HISTORY = [+ {"role": "user", "content": "搜索 Avibe"},+ {+ "role": "assistant",+ "content": [+ {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "Avibe"}},+ {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": [{"encrypted_content": "Eo8B"}]},+ ],+ },+ {"role": "user", "content": "第二条结果是什么?"},+]++CASES = [+ # (case, backend, vendor, protocol, request body, upstream status, upstream body, learned feature, detail)+ (+ "effort-named-by-the-relay", "codex", "openai", "openai_responses",+ {"input": "hi", "reasoning": {"effort": "ultra"}}, 400, ULTRA_REFUSED,+ EvidenceFeature.effort("ultra"), "400 invalid_request_error",+ ),+ (+ "declared-tool-type", "codex", "openai", "openai_responses",+ {"input": "hi", "tools": [{"type": "web_search"}]}, 400, TOOL_REFUSED,+ EvidenceFeature.hosted("web_search", "call"), "400 invalid_request_error",+ ),+ (+ "replayed-search-result", "claude", "anthropic", "anthropic",+ {"max_tokens": 64, "messages": WEB_SEARCH_HISTORY}, 400, REPLAY_REFUSED,+ EvidenceFeature.hosted("web_search", "replay"), "400 invalid_request_error",+ ),+ ("another-effort-named", "codex", "openai", "openai_responses",+ {"input": "hi", "reasoning": {"effort": "high"}}, 400, ULTRA_REFUSED, None, None),+ ("same-value-in-another-field", "codex", "openai", "openai_responses",+ {"input": "hi", "reasoning": {"effort": "high"}}, 400,+ {"error": {"code": "invalid_value", "param": "service_tier", "message": "Invalid value: 'high'."}}, None, None),+ ("tool-type-not-declared", "codex", "openai", "openai_responses",+ {"input": "hi"}, 400, TOOL_REFUSED, None, None),+ ("no-hosted-result-replayed", "claude", "anthropic", "anthropic",+ {"max_tokens": 64, "messages": [{"role": "user", "content": "hi"}]}, 400, REPLAY_REFUSED, None, None),+ ("server-error-names-the-effort", "codex", "openai", "openai_responses",+ {"input": "hi", "reasoning": {"effort": "ultra"}}, 500, ULTRA_REFUSED, None, None),+]+++def _web_search_only(vendor, protocol, model_id, tool):+ if tool != "web_search":+ return None+ declaration = {"anthropic": {"type": "web_search_20250305", "name": "web_search"}}.get(+ protocol, {"type": "web_search"}+ )+ return HostedToolSpec(declaration_json=json.dumps(declaration), price=None, include=())+++ "backend,vendor,protocol,body,status,upstream,feature,detail",+ [case[1:] for case in CASES],+ ids=[case[0] for case in CASES],+)+async def test_an_upstream_400_teaches_only_the_feature_it_names(+ tmp_path, monkeypatch, backend, vendor, protocol, body, status, upstream, feature, detail+):+ """MH-HOSTED-006: a narrow rule learns exactly the named feature; anything else learns nothing."""+ monkeypatch.setattr(service_module, "hosted_tool_spec", _web_search_only)+ source = _source("src_evidence1", "Relay 中转", vendor=vendor, protocol=protocol)+ service, _clock, models = configured_service(tmp_path, sources=[source])++ async def respond(_request):+ return web.json_response(upstream, status=status)++ async with loopback_engine(tmp_path, service, respond):+ with pytest.raises(ModelHubError):+ await service.resolve_with_recovery(+ backend=backend, model_id=models[backend],+ request=ModelHubRequest(body, protocol=protocol), stream=False,+ )++ records = service.capability_evidence.records()+ if feature is None:+ assert records == []+ return+ [record] = records+ assert (record.source_id, record.model_id, record.feature) == ("src_evidence1", "shared-model", feature)+ assert (record.status, record.basis, record.detail) == ("unsupported", "observed", detail)+++async def test_a_refused_effort_skips_its_hop_instead_of_another_provider_400(tmp_path):+ """MH-HOSTED-006: the next request at a refused effort fails locally, and the chain read shows it."""+ source = _source("src_evidence1", "Relay", vendor="openai", protocol="openai_responses")+ service, _clock, models = configured_service(tmp_path, sources=[source])+ calls = []++ async def respond(request):+ calls.append(await request.json())+ if calls[-1].get("reasoning", {}).get("effort") == "ultra":+ return web.json_response(ULTRA_REFUSED, status=400)+ return web.json_response(+ {"object": "response", "status": "completed", "output": [{"content": [{"type": "output_text", "text": "ok"}]}]}+ )++ def request(effort):+ return ModelHubRequest({"input": "hi", "reasoning": {"effort": effort}}, protocol="openai_responses")++ async with loopback_engine(tmp_path, service, respond):+ with pytest.raises(ModelHubError) as first:+ await service.resolve_with_recovery(+ backend="codex", model_id=models["codex"], request=request("ultra"), stream=False+ )+ assert first.value.code != "effort_unsupported"+ assert len(calls) == 1++ with pytest.raises(ModelHubError) as second:+ await service.resolve_with_recovery(+ backend="codex", model_id=models["codex"], request=request("ultra"), stream=False+ )+ assert (second.value.code, second.value.status) == ("effort_unsupported", 422)+ assert second.value.data == {"effort": "ultra", "source_ids": ["src_evidence1"]}+ assert len(calls) == 1++ # The chain read projects what routing just used, for the effort menu.+ chain = service.agent_chain("codex", models["codex"])+ _assert_valid("agent-chain.schema.json", chain)+ [hop] = [hop for hop in chain["chain"] if hop["source_id"] == "src_evidence1"]+ assert hop["refused_efforts"] == [+ {+ "effort": "ultra",+ "basis": "observed",+ "observed_at": service.capability_evidence.records()[0].payload()["observed_at"],+ "detail": "400 invalid_request_error",+ }+ ]++ served = await service.resolve_with_recovery(+ backend="codex", model_id=models["codex"], request=request("high"), stream=False+ )+ assert served.source_id == "src_evidence1"+ assert len(calls) == 2+++async def test_a_refreshed_attempt_teaches_what_it_names(tmp_path):+ """MH-HOSTED-006: after a credential refresh, the exact retry's 400 is learned too."""+ from dataclasses import replace as replaced++ from core.handlers.model_hub.adapter import RawOutcomeKind, UpstreamRejection+ from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _outcome, _service++ source = _source("src_refreshed1", "Refreshed")+ refused = replaced(+ _outcome(RawOutcomeKind.HTTP_ERROR, status=400, code="invalid_request_error", source_id=source.id),+ rejection=UpstreamRejection(status=400, error_type="invalid_request_error", invalid_value="ultra"),+ )+ service = _service(+ tmp_path,+ sources=[source],+ outcomes=[_outcome(RawOutcomeKind.HTTP_ERROR, status=401, code="unauthorized", source_id=source.id), refused],+ )+ service.adapter.refreshable_credential_refs.add(source.credential_ref)+ model = _canonicalize_fixed_test_routes(service)["codex"]++ with pytest.raises(ModelHubError):+ await service.resolve_with_recovery(+ backend="codex",+ model_id=model,+ request=ModelHubRequest({"input": "hi", "reasoning": {"effort": "ultra"}}, protocol="openai_responses"),+ stream=False,+ )++ [record] = service.capability_evidence.records()+ assert (record.source_id, record.feature) == ("src_refreshed1", EvidenceFeature.effort("ultra"))+++async def test_a_free_form_effort_is_forwarded_and_never_breaks_the_gateway(tmp_path):+ """MH-HOSTED-006: evidence cannot name a free-form effort, so it learns nothing and the 400 surfaces as is."""+ source = _source("src_evidence1", "Relay", vendor="openai", protocol="openai_responses")+ service, _clock, models = configured_service(tmp_path, sources=[source])+ refused = {"error": {"code": "invalid_value", "type": "invalid_request_error", "param": "reasoning.effort"}}++ async def respond(_request):+ return web.json_response(refused, status=400)++ async with loopback_engine(tmp_path, service, respond):+ for _ in range(2):+ with pytest.raises(ModelHubError) as error:+ await service.resolve_with_recovery(+ backend="codex",+ model_id=models["codex"],+ request=ModelHubRequest(+ {"input": "hi", "reasoning": {"effort": "future effort"}}, protocol="openai_responses"+ ),+ stream=False,+ )+ assert error.value.code != "effort_unsupported"+ assert service.capability_evidence.records() == []+++async def test_a_refused_effort_still_waits_for_a_capable_hop_that_is_cooling_down(tmp_path):+ """MH-HOSTED-006: effort_unsupported only when no hop that could recover accepts the effort."""+ from datetime import timedelta++ from core.handlers.model_hub.capability_evidence import EvidenceRecord, identity_digest+ from core.handlers.model_hub.hosted_tools import HOSTED_PROBE_SHAPE, engine_stamp+ from tests.test_model_hub_l3 import NOW, _canonicalize_fixed_test_routes, _service++ refusing = _source("src_refuses01", "Refuses ultra")+ cooling = _source(+ "src_cooling01", "Cooling", status="cooldown", retry_at=(NOW + timedelta(minutes=5)).isoformat()+ )+ service = _service(tmp_path, sources=[refusing, cooling])+ model = _canonicalize_fixed_test_routes(service)["codex"]+ service.capability_evidence.write(+ EvidenceRecord(+ source_id=refusing.id,+ model_id="shared-model",+ feature=EvidenceFeature.effort("ultra"),+ status="unsupported",+ basis="observed",+ observed_at=service.now(),+ engine=engine_stamp(),+ shape=HOSTED_PROBE_SHAPE,+ source_identity=identity_digest(refusing),+ detail="400 invalid_request_error",+ ),+ live=[refusing, cooling],+ )+ request = ModelHubRequest({"input": "hi", "reasoning": {"effort": "ultra"}}, protocol="openai_responses")++ with pytest.raises(ModelHubError) as waiting:+ await service.resolve(backend="codex", model_id=model, request=request)+ assert waiting.value.code != "effort_unsupported"+ assert waiting.value.supply_state == "waiting"+ assert service.adapter.invocations == []+++async def test_no_request_path_reads_a_bundled_file_after_startup(tmp_path, monkeypatch):+ """MH-HOSTED-006: every immutable bundled input is resolved when the service is built, never on a request."""+ from dataclasses import replace as replaced+ from pathlib import Path++ from core.handlers.model_hub.adapter import RawOutcomeKind, UpstreamRejection+ from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _outcome, _service++ source = _source("src_startup01", "Startup", vendor="openai", protocol="openai_responses")+ service = _service(tmp_path, sources=[source])+ model = _canonicalize_fixed_test_routes(service)["codex"]+ watched = {"backend_models.json", "cliproxyapi_manifest.json"}+ reads = []+ read_text, path_open = Path.read_text, Path.open++ def counted_read(self, *args, **kwargs):+ if self.name in watched:+ reads.append(self.name)+ return read_text(self, *args, **kwargs)++ def counted_open(self, *args, **kwargs):+ if self.name in watched:+ reads.append(self.name)+ return path_open(self, *args, **kwargs)++ monkeypatch.setattr(Path, "read_text", counted_read)+ monkeypatch.setattr(Path, "open", counted_open)++ request = ModelHubRequest(+ {"input": "hi", "reasoning": {"effort": "ultra"}, "tools": [{"type": "web_search"}]}, protocol="openai_responses"+ )+ rejected = replaced(+ _outcome(RawOutcomeKind.HTTP_ERROR, status=400, source_id=source.id),+ rejection=UpstreamRejection(status=400, unsupported_tool_type="web_search"),+ )+ config = service.store.load()+ assert service._effort_refused(source, "shared-model", request) is False+ await service._learn_capability_evidence(source, "shared-model", request, rejected)+ service.hosted_tools_for_hop(config, source, "shared-model")+ service.agent_chain("codex", model)+ service.list_sources()+ assert reads == []+++async def test_a_refusing_runnable_hop_never_makes_recovery_spin(tmp_path):+ """MH-HOSTED-006: recovery waits for the capable cooling hop instead of re-resolving around the refusing one."""+ from datetime import timedelta as delta++ from core.handlers.model_hub.adapter import RawOutcomeKind+ from core.handlers.model_hub.capability_evidence import EvidenceRecord, identity_digest+ from core.handlers.model_hub.hosted_tools import HOSTED_PROBE_SHAPE+ from tests.test_model_hub_l3 import NOW, _canonicalize_fixed_test_routes, _outcome+ from tests.test_model_hub_retry_policy import clock_service++ refusing = _source("src_refuses02", "Refuses ultra")+ cooling = _source("src_cooling02", "Cooling", status="cooldown", retry_at=(NOW + delta(seconds=30)).isoformat())+ service, clock = clock_service(+ tmp_path,+ sources=[refusing, cooling],+ outcomes=[_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=cooling.id)],+ )+ model = _canonicalize_fixed_test_routes(service)["codex"]+ service.capability_evidence.write(+ EvidenceRecord(+ source_id=refusing.id,+ model_id="shared-model",+ feature=EvidenceFeature.effort("ultra"),+ status="unsupported",+ basis="observed",+ observed_at=service.now(),+ engine=service.engine_stamp,+ shape=HOSTED_PROBE_SHAPE,+ source_identity=identity_digest(refusing),+ detail="400 invalid_request_error",+ ),+ live=[refusing, cooling],+ )+ passes = 0+ resolve = service.resolve++ async def counted(**kwargs):+ nonlocal passes+ passes += 1+ assert passes <= 5, "recovery re-resolved without waiting"+ return await resolve(**kwargs)++ service.resolve = counted+ served = await service.resolve_with_recovery(+ backend="codex",+ model_id=model,+ request=ModelHubRequest({"input": "hi", "reasoning": {"effort": "ultra"}}, protocol="openai_responses"),+ )++ assert served.source_id == cooling.id+ assert clock.delays and all(delay > 0 for delay in clock.delays)+ assert [invocation[0] for invocation in service.adapter.invocations] == [cooling.id]diff --git a/tests/test_model_hub_reasoning_tiers.py b/tests/test_model_hub_reasoning_tiers.pyindex fd5f55ef1..7d28bb05b 100644--- a/tests/test_model_hub_reasoning_tiers.py+++ b/tests/test_model_hub_reasoning_tiers.py@@ -28,7 +28,8 @@ def test_upstream_reasoning_signal_uses_only_the_protocol_default(assert resolution.source == "upstream"-def test_catalog_row_outranks_user_and_is_returned_verbatim() -> None:+def test_catalog_row_outranks_user_and_drops_its_client_presets() -> None:+ """MH-HOSTED-006 (#2539): Codex's ``ultra`` is a client preset, never a Source's wire tier."""resolution = resolve_reasoning_tiers(protocol="openai_responses",model_id="gpt-5.6-sol",@@ -36,14 +37,7 @@ def test_catalog_row_outranks_user_and_is_returned_verbatim() -> None:existing_source="user",)- assert resolution.efforts == (- "low",- "medium",- "high",- "xhigh",- "max",- "ultra",- )+ assert resolution.efforts == ("low", "medium", "high", "xhigh", "max")assert resolution.source == "catalog"@@ -76,8 +70,8 @@ def test_catalog_row_is_not_filtered_through_the_vocabulary() -> None:monkeypatch.setattr(reasoning_tiers,- "bundled_catalog_reasoning_efforts_for_model",- lambda _model_id: pytest.fail("injected catalog index was ignored"),+ "bundled_catalog_wire_efforts_by_model",+ lambda: pytest.fail("injected catalog index was ignored"),)resolution = resolve_reasoning_tiers(@@ -121,3 +115,45 @@ def test_upstream_reasoning_parameter_matching_is_case_and_space_tolerant()assert resolution.source == "upstream"+++ ("provenance", "expected"),+ (+ ("catalog", ["low", "medium", "high", "xhigh", "max"]),+ ("user", ["low", "medium", "high", "xhigh", "max", "ultra"]),+ ),+)+def test_a_persisted_catalog_list_offers_no_client_preset_to_vibey(tmp_path, provenance, expected) -> None:+ """MH-HOSTED-006 (#2539): a released build persisted Codex's ``ultra`` in catalog lists; only user lists keep it."""+ from config.v2_config import ModelHubModelConfig, ModelHubSourceConfig, ModelHubSourceStateConfig+ from tests.test_model_hub_api import _service++ service, store, _adapter = _service(tmp_path)+ store.config.sources = [+ ModelHubSourceConfig(+ id="src_relay0001",+ kind="api_key",+ vendor="openai",+ display_name="OpenAI relay",+ protocol="openai_responses",+ supply_channel="hub",+ billing="metered",+ state=ModelHubSourceStateConfig(status="standby"),+ models=[+ ModelHubModelConfig(+ id="gpt-5.6-sol",+ provenance="discovered",+ reasoning_efforts=["low", "medium", "high", "xhigh", "max", "ultra"],+ reasoning_efforts_source=provenance,+ )+ ],+ credential_ref="cred_relay",+ )+ ]+ store.config.agents["vibey"].sources.order = ["src_relay0001"]++ providers = service.agent_model_candidates("vibey")["providers"]++ [row] = [row for row in providers if row["id"] == "gpt-5.6-sol"]+ assert row["reasoning_efforts"] == expecteddiff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.pyindex 1fb4fb75c..bfec5bca5 100644--- a/tests/test_model_hub_structure_guards.py+++ b/tests/test_model_hub_structure_guards.py@@ -1234,11 +1234,16 @@ def test_machine_error_field_access_has_one_extractor() -> None:if path.name == "migration.py":continue # its ``type`` field is an auth-record kind, not an error codetree = _tree(path)- # Request history discriminates content blocks, not error envelopes.- # Exempt only that function's type reads; code reads and the rest of- # request.py remain subject to the extraction-owner guard.- history = _functions(tree).get("_plain_history") if path.name == "request.py" else None- history_nodes = set(ast.walk(history)) if history is not None else set()+ # Request history and declared tools discriminate request items, not+ # error envelopes. Exempt only those functions' type reads; code reads+ # and the rest of request.py remain subject to the extraction-owner guard.+ functions = _functions(tree) if path.name == "request.py" else {}+ history_nodes = {+ node+ for name in ("_plain_history", "declared_tool_types", "replays_history_items")+ if (history := functions.get(name)) is not None+ for node in ast.walk(history)+ }for node in ast.walk(tree):if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):continuediff --git a/tests/test_model_hub_vibey_consumer.py b/tests/test_model_hub_vibey_consumer.pyindex b3af2200e..15a778246 100644--- a/tests/test_model_hub_vibey_consumer.py+++ b/tests/test_model_hub_vibey_consumer.py@@ -1505,3 +1505,58 @@ async def test_concurrent_responses_keep_their_own_origin(tmp_path):pending.cancel()await asyncio.gather(pending, return_exceptions=True)await gateway.close()++++def test_a_released_vibey_row_loads_without_its_client_preset(tmp_path, monkeypatch):+ """MH-HOSTED-006 (#2539): 3.2.0 saved Vibey rows with Codex's ``ultra``; every read drops it, once, at load."""+ from config.v2_config import V2Config+ from core.services.settings import default_config+ from vibe import api++ released = json.loads(+ (Path(__file__).parent / "fixtures/model_hub/released_3_2_0_vibey_client_preset.json").read_text(encoding="utf-8")+ )+ clean = ["low", "medium", "high", "xhigh", "max"]++ config = ModelHubConfig.from_payload(copy.deepcopy(released), repairing=True)+ assert config.agents["vibey"].models[0].reasoning_efforts == clean+ written = config.to_payload()+ assert written["agents"]["vibey"]["models"][0]["reasoning_efforts"] == clean+ # Only the preset changes: Sources, other Agents and the rest of the row are as 3.2.0 wrote them.+ expected = copy.deepcopy(released)+ expected["agents"]["vibey"]["models"][0]["reasoning_efforts"] = clean+ assert written == expected++ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))+ payload = api.config_to_payload(default_config(), include_secrets=True, include_internal=True)+ payload["model_hub"] = copy.deepcopy(released)+ config_path = tmp_path / "config.json"+ config_path.write_text(json.dumps(payload), encoding="utf-8")+ loaded = V2Config.load(config_path=config_path)+ assert not loaded.load_warnings+ assert loaded.model_hub.agents["vibey"].models[0].reasoning_efforts == clean+++def test_loading_and_cloning_model_hub_config_reads_no_catalog_file(monkeypatch):+ """MH-HOSTED-006: config load runs on the controller loop (service clones), so the preset index costs no I/O."""+ from vibe.backend_model_catalog import bundled_client_presets_once, get_bundled_catalog_path++ released = json.loads(+ (Path(__file__).parent / "fixtures/model_hub/released_3_2_0_vibey_client_preset.json").read_text(encoding="utf-8")+ )+ bundled_client_presets_once()+ bundled = get_bundled_catalog_path()+ reads = []+ original = Path.read_text++ def counted(self, *args, **kwargs):+ if self == bundled:+ reads.append(self)+ return original(self, *args, **kwargs)++ monkeypatch.setattr(Path, "read_text", counted)+ for _ in range(5):+ config = ModelHubConfig.from_payload(copy.deepcopy(released), repairing=True)+ ModelHubConfig.from_payload(config.to_payload())+ assert reads == []diff --git a/tests/test_provider_overlay.py b/tests/test_provider_overlay.pyindex 11d075320..a5f7d8158 100644--- a/tests/test_provider_overlay.py+++ b/tests/test_provider_overlay.py@@ -106,7 +106,7 @@ def test_bundled_providers_section_satisfies_its_schema():"claude-opus-5-5","web_search",(- {"type": "web_search_20250305", "name": "web_search"},+ {"type": "web_search_20250305", "name": "web_search", "max_uses": 5},(),HostedToolPrice("search", 10.0, AS_OF, "https://platform.claude.com/docs/en/about-claude/pricing"),),diff --git a/vibe/backend_model_catalog.py b/vibe/backend_model_catalog.pyindex de50e2781..313f619ff 100644--- a/vibe/backend_model_catalog.py+++ b/vibe/backend_model_catalog.py@@ -562,9 +562,11 @@ def bundled_catalog_client_presets_by_model() -> Mapping[str, tuple[str, ...]]:return _bundled_catalog_lists_by_model("client_presets")-def _bundled_catalog_lists_by_model(key: str) -> Mapping[str, tuple[str, ...]]:+def _bundled_catalog_lists_by_model(+ key: str, catalog: Mapping[str, Any] | None = None+) -> Mapping[str, tuple[str, ...]]:values_by_model: dict[str, tuple[str, ...]] = {}- backends = load_bundled_catalog().get("backends")+ backends = (load_bundled_catalog() if catalog is None else catalog).get("backends")if not isinstance(backends, dict):return MappingProxyType(values_by_model)for backend in backends.values():@@ -593,6 +595,54 @@ def bundled_catalog_reasoning_efforts_for_model(return bundled_catalog_reasoning_efforts_by_model().get(model_id)[email protected]_cache(maxsize=4)+def _bundled_client_presets(path: str) -> Mapping[str, tuple[str, ...]]:+ catalog = _read_catalog(Path(path))+ # An unreadable bundle has no presets, like a model without a row.+ return _bundled_catalog_lists_by_model("client_presets", catalog or {})+++def bundled_client_presets_once() -> Mapping[str, tuple[str, ...]]:+ """The bundled client-preset index, read once per process.++ The bundled file never changes while a process runs, and config loading and+ cloning (on the controller loop) consult this index, so it must cost no I/O+ after the first read. Keyed by path, so a different bundle is its own entry.+ """++ return _bundled_client_presets(str(get_bundled_catalog_path()))+++def without_client_presets(model_ids: Sequence[str], efforts: Sequence[str]) -> list[str]:+ """Efforts as a wire client may send them: without any listed model's client presets.++ Vibey is not the client a preset belongs to, so a preset is never one of its+ tiers (#2539).+ """++ presets = bundled_client_presets_once()+ dropped = {preset for model_id in model_ids for preset in presets.get(model_id, ())}+ return [effort for effort in efforts if effort not in dropped]+++def bundled_catalog_wire_efforts_by_model() -> Mapping[str, tuple[str, ...]]:+ """Index bundled rows' efforts as a Source's wire tiers: each row's list minus its ``client_presets``.++ One catalog read builds both, so a discovery batch still loads the bundled+ file once (#2539).+ """++ catalog = load_bundled_catalog()+ efforts = _bundled_catalog_lists_by_model("reasoning_efforts", catalog)+ presets = _bundled_catalog_lists_by_model("client_presets", catalog)+ return MappingProxyType(+ {+ model_id: tuple(effort for effort in values if effort not in presets.get(model_id, ()))+ for model_id, values in efforts.items()+ }+ )++# The ``providers`` section (model-hub-capability-model.md §4,# docs/plans/model-hub-contracts/provider-overlay.schema.json): what a provider# offers per model family and protocol. Priors only; whether a Source carries a@@ -639,6 +689,12 @@ class HostedToolSpec:return json.loads(self.declaration_json)+ @property+ def declaration_type(self) -> str:+ """The declaration's tool ``type``, as the provider names it on the wire."""++ return self.declaration["type"]+@dataclass(frozen=True)class ProviderModelOverlay:diff --git a/vibe/data/backend_models.json b/vibe/data/backend_models.jsonindex 98145ba9c..522227fa4 100644--- a/vibe/data/backend_models.json+++ b/vibe/data/backend_models.json@@ -55,7 +55,7 @@"hosted_tools": {"anthropic": {"web_search": {- "declaration": {"type": "web_search_20250305", "name": "web_search"},+ "declaration": {"type": "web_search_20250305", "name": "web_search", "max_uses": 5},"price": {"unit": "search", "usd_per_1k": 10, "as_of": "2026-10-13", "url": "https://platform.claude.com/docs/en/about-claude/pricing"}}}diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.pyindex f50b6e3da..78052acb1 100644--- a/vibe/model_hub_runtime/client.py+++ b/vibe/model_hub_runtime/client.py@@ -28,6 +28,7 @@ from core.handlers.model_hub.adapter import (InvokeCancelledError,RawCallOutcome,RawOutcomeKind,+ UpstreamRejection,)from core.handlers.model_hub.async_owner import run_owned_in_threadfrom core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES@@ -1778,6 +1779,15 @@ def _reduce_protocol_observation(usage=observation.usage,recovery_verified=observation.recovery_verified,upstream_detail=upstream_detail,+ rejection=_upstream_rejection(+ http_status,+ stream_started,+ error_code,+ error_type,+ observation.error_payload or b"",+ observation.error_envelope_paths or (("error",),),+ observation.error_message,+ ),)return _outcome(kind=RawOutcomeKind.PROTOCOL_ERROR,@@ -1880,6 +1890,7 @@ def _outcome(recovery_verified: bool = False,upstream_detail: str | None = None,os_errno: int | None = None,+ rejection: UpstreamRejection | None = None,) -> RawCallOutcome:return RawCallOutcome(kind=kind,@@ -1895,6 +1906,7 @@ def _outcome(recovery_verified=recovery_verified,upstream_detail=upstream_detail,os_errno=os_errno,+ rejection=rejection,)@@ -1944,6 +1956,68 @@ def _upstream_error_detail(return None+_REJECTION_TOKEN = re.compile(r"[A-Za-z0-9_.\[\]-]{1,64}")+_REJECTION_MESSAGE_SCAN_CHARS = 2000+_INVALID_VALUE_MESSAGE = re.compile(r"Invalid value: '([A-Za-z0-9_.-]{1,64})'")+_UNSUPPORTED_TOOL_TYPE_MESSAGE = re.compile(r"Unsupported tool type: ([A-Za-z0-9_.-]{1,64})")+_INVALID_ENCRYPTED_CONTENT_MESSAGE = re.compile(r"Invalid encrypted_content in ([a-z_]{1,64}) block")+++def _upstream_rejection(+ http_status: int | None,+ stream_started: bool,+ error_code: str | None,+ error_type: str | None,+ payload: bytes,+ envelope_paths: tuple[ErrorEnvelopePath, ...],+ message: str | None,+) -> UpstreamRejection | None:+ """Project a 400 before output into the closed tokens the evidence rules read.++ The machine code and type come from the one extractor (``_raw_error_fields``+ and its projection); this adds only the first envelope's ``param`` and the+ single token each recognized message names. The prose itself never leaves.+ """++ if http_status != 400 or stream_started:+ return None+ names = ("param", "message")+ paths = tuple((*path, name) for path in envelope_paths for name in names)+ values: dict[JSONPath, str] = {}++ def visit(path: JSONPath, event: JSONEvent, value: object | None, _scope: JSONScope) -> None:+ if event == "replace":+ values.pop(path, None)+ elif event == "scalar" and isinstance(value, str):+ values[path] = value++ if payload:+ project_json_reader(io.BytesIO(payload), paths, visit)+ envelope = next(+ (path for path in envelope_paths if any((*path, name) in values for name in names)),+ None,+ )+ fields = {name: values.get((*envelope, name)) for name in names} if envelope is not None else {}+ text = (fields.get("message") or message or "")[:_REJECTION_MESSAGE_SCAN_CHARS]++ def token(value: object) -> str | None:+ return value if isinstance(value, str) and _REJECTION_TOKEN.fullmatch(value) else None++ def named(pattern: re.Pattern[str]) -> str | None:+ found = pattern.search(text)+ return found.group(1) if found else None++ return UpstreamRejection(+ status=400,+ code=token(error_code),+ error_type=token(error_type),+ param=token(fields.get("param")),+ invalid_value=named(_INVALID_VALUE_MESSAGE),+ unsupported_tool_type=named(_UNSUPPORTED_TOOL_TYPE_MESSAGE),+ invalid_encrypted_content_block=named(_INVALID_ENCRYPTED_CONTENT_MESSAGE),+ )++def _upstream_reset_advice(payload: bytes,envelope_paths: tuple[ErrorEnvelopePath, ...],