Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(model-hub): ask relays for the fast tier too and confirm it from each response (#3091)

Call White2d970ed627c8
diff --git a/config/v2_config.py b/config/v2_config.py
index d7a18a9a0..a089a0033 100644
--- a/config/v2_config.py
+++ b/config/v2_config.py
@@ -2733,6 +2733,7 @@ class ModelHubBackendModelConfig:
"routeable",
"effort_menu",
"fast_mode",
+ "fast_mode_confirmed",
}
if set(payload) - allowed:
raise ValueError("Config 'model_hub.agents.models' contains unknown fields")
diff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.py
index 9f2ec184a..3818292da 100644
--- a/core/handlers/model_hub/adapter.py
+++ b/core/handlers/model_hub/adapter.py
@@ -20,7 +20,7 @@ from enum import Enum
import re
from typing import Any, AsyncIterator, Awaitable, Callable, Final, Literal, Mapping, Protocol, Sequence
-from .stream_wire import ProtocolSSEState, ProtocolUsageReport
+from .stream_wire import ProtocolSSEState, ProtocolUsageReport, ServedTier
ENGINE_TRANSPORT_TIMEOUT_SECONDS: Final = 60.0
# Model invocation/observation is a narrower domain than stored Sources.
@@ -303,6 +303,9 @@ class RawCallOutcome:
# The search tools whose own completed item a served response carried, for
# the positive evidence written at settlement (``vibey-hosted-tools.md`` §3.2).
hosted_tools: frozenset[str] = frozenset()
+ # The processing tier a served response reports (``docs/plans/fast-mode.md``),
+ # for confirming that a fast request ran fast; None when it names none.
+ served_tier: ServedTier | None = None
# The response header name is retained in lowercase so relay ``x-request-id``
# and provider ``request-id`` values cannot be conflated. The client and
# provenance writer re-validate this tuple before persistence.
diff --git a/core/handlers/model_hub/fast_mode.py b/core/handlers/model_hub/fast_mode.py
index 11a9fdb78..81778fb75 100644
--- a/core/handlers/model_hub/fast_mode.py
+++ b/core/handlers/model_hub/fast_mode.py
@@ -12,16 +12,22 @@ covers new models without a release:
- Anthropic (Claude Opus): ``speed: "fast"`` on Messages requests. The engine
adds the beta header a model needs.
-Only that vendor's own upstream honours the field: an API key on the vendor's
-official endpoint, or the vendor's subscription held by the Model Hub. A request
-is never translated across protocols to reach a fast tier. Reading the catalog
-never fetches: an absent or unreadable copy offers fast mode nowhere.
+The field is sent on every Hub Source that speaks the vendor's own protocol: an
+API key, on the vendor's endpoint or a relay, or a subscription the Model Hub
+holds. Whether it took effect is the response's to say, not ours to assume: the
+vendor reports the tier it served (``usage.speed``, ``service_tier``), and a
+relay may drop the field or serve a standard tier. ``FastTierObservations``
+keeps what the last fast request on each Source and model reported, so the
+picker can say where fast mode was not confirmed. A request is never translated
+across protocols to reach a fast tier. Reading the catalog never fetches: an
+absent or unreadable copy offers fast mode nowhere.
"""
from __future__ import annotations
import logging
import threading
+from collections import OrderedDict
from dataclasses import dataclass, field
from typing import Any, Final, Mapping, Optional
@@ -85,31 +91,28 @@ class FastModeCatalog:
return max(matches, key=lambda item: len(item[0]))[1] if matches else None
def body_for(self, source: Any, model_id: str, request_protocol: Optional[str]) -> Optional[Mapping[str, str]]:
- """The body fields that put this exact attempt on its vendor's fast tier, or None where none applies."""
+ """The body fields that ask for this exact attempt's fast tier, or None where none applies.
+
+ The model's first-party vendor decides the field, and the Source must speak
+ that vendor's protocol, unchanged from the request's. Where the Source
+ points (official endpoint or relay) does not matter here: the response
+ says whether the tier was served.
+ """
if not model_id or getattr(source, "supply_channel", None) != "hub":
return None
- vendor = getattr(source, "vendor", None)
- protocol = getattr(source, "protocol", None)
- if (
- vendor not in _VENDOR_PROTOCOLS
- or protocol not in _VENDOR_PROTOCOLS[vendor]
- or request_protocol != protocol
- ):
+ if getattr(source, "kind", None) not in ("api_key", "subscription"):
return None
- if getattr(source, "kind", None) == "api_key":
- base_url = getattr(source, "base_url", None)
- if base_url is not None:
- from vibe.model_hub_runtime.api_key_vendors import is_official_api_key_endpoint
-
- if not is_official_api_key_endpoint(vendor, base_url):
- return None
- elif getattr(source, "kind", None) != "subscription":
+ protocol = getattr(source, "protocol", None)
+ if request_protocol != protocol:
return None
- models = self.bodies.get(vendor) or {}
for candidate in dict.fromkeys((model_id, catalog_model_id(model_id))):
- if self.first_party(candidate) == vendor and candidate in models:
- return models[candidate]
+ vendor = self.first_party(candidate)
+ if vendor is None or protocol not in _VENDOR_PROTOCOLS[vendor]:
+ continue
+ body = (self.bodies.get(vendor) or {}).get(candidate)
+ if body is not None:
+ return body
return None
@@ -125,6 +128,35 @@ def _fast_body(entry: Any, field_name: str) -> Optional[Mapping[str, str]]:
return {field_name: value} if isinstance(value, str) and value.strip() else None
+class FastTierObservations:
+ """Whether the last fast request on each Source and model was served fast, as the response reported.
+
+ True when the response named the vendor's fast tier; False when it named
+ another tier or none, which is how a relay that drops the field answers.
+ Absent until a fast request on that hop is served. Only the latest answer
+ counts, so a Source that starts honouring the field is confirmed by its next
+ fast request. Process memory, bounded; a restart forgets it.
+ """
+
+ _LIMIT: Final = 512
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._confirmed: OrderedDict[tuple[str, str], bool] = OrderedDict()
+
+ def observe(self, source_id: str, model_id: str, confirmed: bool) -> None:
+ key = (source_id, model_id)
+ with self._lock:
+ self._confirmed.pop(key, None)
+ self._confirmed[key] = confirmed
+ while len(self._confirmed) > self._LIMIT:
+ self._confirmed.popitem(last=False)
+
+ def confirmed(self, source_id: str, model_id: str) -> Optional[bool]:
+ with self._lock:
+ return self._confirmed.get((source_id, model_id))
+
+
_LOCK = threading.Lock()
_CACHED: tuple[Optional[tuple[str, int, int]], FastModeCatalog] = (None, FastModeCatalog())
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index 8285f6e91..5d6490f7b 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -235,7 +235,7 @@ from .provenance import (
produce_turn_outcome,
)
from .opaque_provenance import OpaqueProvenanceRegistry, filter_opaque_history
-from .fast_mode import FAST_SPEED, FastModeCatalog, load_fast_mode_catalog
+from .fast_mode import FAST_SPEED, FastModeCatalog, FastTierObservations, load_fast_mode_catalog
from .request import ModelHubRequest, without_opaque_history
from .reasoning_tiers import catalog_wire_efforts, rederive_persisted_tiers, resolve_reasoning_tiers
from .stream_wire import ProtocolSSEState, ProtocolUsageReport
@@ -842,7 +842,16 @@ class _InvocationUnsupported(_InvocationPlanChanged):
self.reason = reason
-_FAST_REFUSAL_STATUSES = frozenset({400, 403, 429})
+# 422: a relay that validates the body refuses a field it does not know.
+_FAST_REFUSAL_STATUSES = frozenset({400, 403, 422, 429})
+
+
+def _asks_fast_tier(
+ catalog: FastModeCatalog | None, source: ModelHubSourceConfig, model_id: str, request: Any,
+) -> bool:
+ """Whether an attempt of this request on this hop carries the vendor's fast-tier field."""
+
+ return catalog is not None and catalog.body_for(source, model_id, getattr(request, "protocol", None)) is not None
def _fast_tier_refused(outcome: Optional[RawCallOutcome]) -> bool:
@@ -1056,6 +1065,9 @@ class ResolvedInvocation:
# Vibey: the catalog tools the request declared natively, which a served
# response may prove ran (``vibey-hosted-tools.md`` §3.2).
hosted_calls: frozenset[str] = frozenset()
+ # This hop's request asked for the vendor's fast tier, so its served response
+ # confirms or not that it ran fast (``docs/plans/fast-mode.md``).
+ fast_requested: bool = False
@dataclass(frozen=True)
@@ -1636,6 +1648,9 @@ class ModelHubService:
# Which (Source, model) attempts offer a vendor's fast tier; read off the
# loop when a fast request arrives, rebuilt only when models.dev changes.
self._fast_mode_catalog = FastModeCatalog()
+ # Whether each Source and model served the last fast request fast, as its
+ # response reported; the gateway records it when a fast call settles served.
+ self.fast_tier_observations = FastTierObservations()
self._oauth_start_tasks: dict[
tuple[str, str, OAuthChannel], asyncio.Task[dict]
] = {}
@@ -6244,9 +6259,11 @@ class ModelHubService:
fast_catalog = None
elif fast_catalog is None:
fast_catalog = self._fast_mode_catalog
- fast_models = {
- model.id: fast_catalog is not None and self._offers_fast_mode(backend, model_resolutions[model.id], fast_catalog)
+ # Each menu model's fast-tier hop, and what that hop's last fast response said.
+ fast_hops = {
+ model.id: self._fast_mode_hop(backend, model_resolutions[model.id], fast_catalog)
for model in menu_models
+ if fast_catalog is not None
}
# Only Hub turns reach the evidence; a Direct Agent's menu is the backend's own.
effort_menus = {
@@ -6349,7 +6366,16 @@ class ModelHubService:
"model_supply": model_supply if agent.mode == "hub" else None,
"named_agents": named_agents,
"catalog_models": [
- {**model, "effort_menu": effort_menus[model["id"]], "fast_mode": fast_models.get(model["id"], False)}
+ {
+ **model,
+ "effort_menu": effort_menus[model["id"]],
+ "fast_mode": fast_hops.get(model["id"]) is not None,
+ "fast_mode_confirmed": (
+ self.fast_tier_observations.confirmed(*hop)
+ if (hop := fast_hops.get(model["id"])) is not None
+ else None
+ ),
+ }
for model in self._catalog_models_payload(agent)
],
"builtin_models": builtin_models,
@@ -7197,11 +7223,13 @@ class ModelHubService:
return self.hosted_evidence().unsupported(source, model_id, feature)
@staticmethod
- def _offers_fast_mode(backend: BackendName, resolution: ModelHubTurnResolution, catalog: FastModeCatalog) -> bool:
- """Whether a turn on this menu model would reach its vendor's fast tier on the hop that normally serves it.
+ def _fast_mode_hop(
+ backend: BackendName, resolution: ModelHubTurnResolution, catalog: FastModeCatalog,
+ ) -> tuple[str, str] | None:
+ """The Source and model a turn on this menu model would ask for its vendor's fast tier on, or None.
The first candidate hop is the one a turn starts on; a fallback to a hop
- without the tier runs at the standard tier. OpenCode's shared server
+ without the field runs at the standard tier. OpenCode's shared server
names no turn on its requests, so its turns carry no Session's choice.
"""
@@ -7210,9 +7238,11 @@ class ModelHubService:
None,
)
if hop is None:
- return False
+ return None
protocol = hop.source.protocol if backend == "vibey" else _FIXED_BACKEND_PROTOCOLS.get(backend)
- return catalog.body_for(hop.source, hop.model_id, protocol) is not None
+ if catalog.body_for(hop.source, hop.model_id, protocol) is None:
+ return None
+ return hop.source.id, hop.model_id
def _effort_menu(
self, backend: BackendName, model: ModelHubBackendModelConfig, resolution: ModelHubTurnResolution, now: datetime
@@ -11020,8 +11050,8 @@ class ModelHubService:
registry=self.opaque_provenance,
)
if fast_catalog is not None:
- # Read on this final snapshot: a hop gains the fast tier only
- # where its exact Source and model offer one.
+ # Read on this final snapshot: a hop asks for the fast tier only
+ # where its exact Source and model can carry the field.
fast_body = fast_catalog.body_for(source, model_id, getattr(request, "protocol", None))
if fast_body is not None:
attempt_request = _with_body_fields(attempt_request, fast_body)
@@ -11068,10 +11098,11 @@ class ModelHubService:
hosted: HostedContent = HostedContent(),
fast_catalog: FastModeCatalog | None = None,
) -> tuple[InvokeHandle, Optional[RawCallOutcome], asyncio.CancelledError | None]:
- """``fast_catalog`` puts each attempt that offers it on its vendor's fast tier
- (docs/plans/fast-mode.md); a fast attempt refused before any output is
+ """``fast_catalog`` asks each attempt that can carry it for its vendor's fast
+ tier (docs/plans/fast-mode.md); a fast attempt refused before any output is
retried once on the same hop at the standard tier, and only that retry's
- outcome reaches classification."""
+ outcome reaches classification. Whether the tier ran is the response's to
+ say, recorded by the gateway when the call settles."""
acquired_handle: InvokeHandle | None = None
generation: int | None = None
@@ -11165,10 +11196,9 @@ class ModelHubService:
raise cancelled
if (
- fast_catalog is not None
- and cancelled is None
+ cancelled is None
and _fast_tier_refused(outcome)
- and fast_catalog.body_for(source, model_id, getattr(request, "protocol", None)) is not None
+ and _asks_fast_tier(fast_catalog, source, model_id, request)
):
# The fast tier is an upgrade, not a requirement: an account without
# it (no usage credits, an organization that disabled it, its own
@@ -11594,6 +11624,7 @@ class ModelHubService:
origin=origin,
source_identity=identity,
hosted_calls=frozenset(tool for tool, _type in hosted.calls),
+ fast_requested=_asks_fast_tier(fast_catalog, source, target_model, request),
)
decision = await self._classify_source_outcome(source, outcome)
learned = await self._learn_capability_evidence(source, target_model, request, outcome)
@@ -11662,6 +11693,7 @@ class ModelHubService:
origin=origin,
source_identity=identity,
hosted_calls=frozenset(tool for tool, _type in hosted.calls),
+ fast_requested=_asks_fast_tier(fast_catalog, source, target_model, request),
)
decision = classify_outcome(outcome, refresh_attempted=True)
learned = await self._learn_capability_evidence(source, target_model, request, outcome)
@@ -11718,6 +11750,7 @@ class ModelHubService:
verification_pending=verification_pending,
origin=origin,
source_identity=identity,
+ fast_requested=_asks_fast_tier(fast_catalog, source, target_model, request),
)
if (
pinned_hop is not None
diff --git a/core/handlers/model_hub/stream_wire.py b/core/handlers/model_hub/stream_wire.py
index db29091b4..4dbf7368c 100644
--- a/core/handlers/model_hub/stream_wire.py
+++ b/core/handlers/model_hub/stream_wire.py
@@ -380,6 +380,9 @@ def _google_terminal_event(key: str, message: str, _next_sequence_number: int) -
StreamTerminalOutcome = Literal["served", "failed_terminal"]
+# The processing tier a response reports having served (docs/plans/fast-mode.md):
+# the vendor's fast tier, or any other tier it names.
+ServedTier = Literal["fast", "standard"]
ProtocolObservationOutcome = Literal["served", "failed_terminal", "protocol_error"]
ErrorEnvelopePath = tuple[str, ...]
@@ -528,6 +531,25 @@ class ProtocolObservation:
error_message: str | None = field(default=None, repr=False)
# Search tools whose own completed hosted item this response carried (§3.2).
hosted_tools: frozenset[str] = frozenset()
+ # The tier this frame or buffered response reports having served, if it names one.
+ served_tier: ServedTier | None = None
+
+
+@dataclass(frozen=True)
+class ServedTierPaths:
+ """Where a protocol reports the processing tier it served, and the value that names the fast tier.
+
+ Anthropic reports ``usage.speed`` (``fast`` or ``standard``); OpenAI reports
+ ``service_tier`` (``priority`` for its fast tier, else ``default``, ``flex``,
+ ``scale``). Any other value names a tier that is not the fast one.
+ """
+
+ paths: tuple[JSONPath, ...]
+ fast_value: str
+ # Only the stream's terminal frame states the tier that ran: an earlier one
+ # (Responses ``response.created``) may echo the request, which a relay that
+ # dropped the field can still do. A terminal that names no tier confirms none.
+ settled_by_terminal: bool = False
@dataclass(frozen=True)
@@ -607,6 +629,7 @@ class ProtocolStreamTaxonomy:
buffered_output_array_path: JSONPath | None = None
buffered_terminal_string_paths: tuple[JSONPath, ...] = ()
hosted_items: HostedItemPaths | None = None
+ served_tier: ServedTierPaths | None = None
PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
@@ -707,6 +730,8 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
cache_write_paths=(("cache_creation_input_tokens",),),
cache_write_1h_paths=(("cache_creation", "ephemeral_1h_input_tokens"),),
),
+ # https://platform.claude.com/docs/en/build-with-claude/fast-mode#checking-which-speed-was-used
+ served_tier=ServedTierPaths(paths=(("message", "usage", "speed"), ("usage", "speed")), fast_value="fast"),
),
"openai_responses": ProtocolStreamTaxonomy(
terminal_envelopes=(
@@ -826,6 +851,11 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
cache_write_30m_paths=(("input_tokens_details", "cache_write_tokens_details", "ttl_30m_tokens"),),
cache_write_12h_paths=(("input_tokens_details", "cache_write_tokens_details", "ttl_12h_tokens"),),
),
+ # https://platform.openai.com/docs/api-reference/responses/object#responses/object-service_tier
+ # The terminal event nests the completed response; a buffered response is it.
+ served_tier=ServedTierPaths(
+ paths=(("response", "service_tier"), ("service_tier",)), fast_value="priority", settled_by_terminal=True,
+ ),
),
"openai_chat": ProtocolStreamTaxonomy(
terminal_envelopes=(
@@ -894,6 +924,9 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
cached_input_paths=(("prompt_tokens_details", "cached_tokens"),),
output_paths=(("completion_tokens",),),
),
+ # https://platform.openai.com/docs/api-reference/chat/object#chat/object-service_tier
+ # Every streamed chunk repeats it.
+ served_tier=ServedTierPaths(paths=(("service_tier",),), fast_value="priority"),
),
# The engine's images endpoints (vibey-hosted-tools.md §7.4): a buffered JSON
# answer only; ``image_generate`` never streams, so nothing here is streamed.
@@ -1152,6 +1185,8 @@ def _protocol_projection_paths(protocol: str) -> frozenset[JSONPath]:
items = taxonomy.hosted_items
paths.update((*items.event_item, *leaf) for leaf in items.leaves)
paths.update((*items.buffered_items, "*", *leaf) for leaf in items.leaves)
+ if taxonomy.served_tier is not None:
+ paths.update(taxonomy.served_tier.paths)
for container_path in taxonomy.usage.container_paths:
for leaf_paths in (
taxonomy.usage.input_paths,
@@ -1266,6 +1301,7 @@ class ProtocolFactProjector:
return ProtocolObservation(outcome="served" if not streamed else None)
usage = _usage_from_scalar_paths(self.taxonomy.usage, self._scalars)
+ served_tier = self._served_tier()
hosted_tools = self._hosted_tools(streamed=streamed)
model_output_started = any(
envelope.event_name == event_name and self._selector_matches(envelope)
@@ -1297,6 +1333,7 @@ class ProtocolFactProjector:
not matched_paths
and _buffered_recovery_verified(self.taxonomy, self._scalars, self._arrays, self._nonempty)
),
+ served_tier=served_tier,
)
for envelope in self.taxonomy.terminal_envelopes:
@@ -1346,6 +1383,7 @@ class ProtocolFactProjector:
),
recovery_verified=model_output_started or envelope.terminal_outcome == "served",
hosted_tools=hosted_tools,
+ served_tier=served_tier,
)
return ProtocolObservation(
model_output_started=model_output_started,
@@ -1353,8 +1391,21 @@ class ProtocolFactProjector:
usage=usage,
recovery_verified=model_output_started,
hosted_tools=hosted_tools,
+ served_tier=served_tier,
)
+ def _served_tier(self) -> ServedTier | None:
+ """The tier this frame or buffered response names, by the protocol's own value for its fast tier."""
+
+ declared = self.taxonomy.served_tier
+ if declared is None:
+ return None
+ for path in declared.paths:
+ value = self._scalars.get(path)
+ if isinstance(value, str) and value.strip():
+ return "fast" if value.strip() == declared.fast_value else "standard"
+ return None
+
def _hosted_tools(self, *, streamed: bool) -> frozenset[str]:
"""The search tools whose own completed item this event or buffered response carries."""
@@ -1644,6 +1695,9 @@ class ProtocolSSEState:
error_code_candidates: tuple[str, ...] = ()
error_message: str | None = field(default=None, repr=False)
hosted_tools: frozenset[str] = frozenset()
+ # The tier the stream named: OpenAI repeats it on every Chat chunk and states
+ # it on the completed Responses event alone; Anthropic reports it in usage.
+ served_tier: ServedTier | None = None
def __post_init__(self) -> None:
self.tokenizer = SSEObservationTokenizer(self.protocol)
@@ -1700,6 +1754,12 @@ class ProtocolSSEState:
self.model_output_started = True
if observation.hosted_tools:
self.hosted_tools |= observation.hosted_tools
+ tier_paths = PROTOCOL_STREAM_TAXONOMY[self.protocol].served_tier
+ if tier_paths is not None and tier_paths.settled_by_terminal:
+ if observation.outcome is not None:
+ self.served_tier = observation.served_tier
+ elif observation.served_tier is not None:
+ self.served_tier = observation.served_tier
if observation.usage is not None:
self.usage = (
observation.usage if self.usage is None else self.usage.merge(observation.usage)
@@ -1744,6 +1804,7 @@ class ProtocolSSEState:
recovery_verified=self.reached_model,
error_message=self.error_message,
hosted_tools=self.hosted_tools if self.terminal_outcome == "served" else frozenset(),
+ served_tier=self.served_tier,
)
return None
diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.py
index f85ffb5a7..0ab45e99e 100644
--- a/core/handlers/model_hub/turn_gateway.py
+++ b/core/handlers/model_hub/turn_gateway.py
@@ -55,6 +55,7 @@ from .resolver import parse_model_hub_timestamp
from .stream_wire import (
ProtocolSSEState,
ProtocolUsageReport,
+ ServedTier,
StreamTerminalOutcome,
render_protocol_terminal_event,
render_protocol_terminal_frame,
@@ -356,6 +357,22 @@ class _TurnExecution:
)
return self.handle is not None
+ @property
+ def served_tier(self) -> tuple[bool, ServedTier | None]:
+ """Whether upstream served this call in full, and the processing tier its response named.
+
+ Asked in the same order as the usage report: the engine's tracker, then
+ this gateway's, then a buffered outcome. Only a served response says which
+ tier ran; a call that failed or was cut says nothing about it.
+ """
+
+ for state in (self.upstream_observation, self.wire_state):
+ if state is not None and state.terminal_outcome == "served":
+ return True, state.served_tier
+ if self.handle_outcome is not None and self.handle_outcome.kind is RawOutcomeKind.SUCCESS:
+ return True, self.handle_outcome.served_tier
+ return False, None
+
@property
def owes_metering(self) -> bool:
"""Whether a row for this turn is still owed to the ledger.
@@ -1714,6 +1731,12 @@ class ModelHubTurnGateway:
return
resolved = execution.resolved
assert resolved is not None
+ if resolved.fast_requested:
+ # The response, not the request, says whether the fast tier ran: a
+ # relay may drop the field or serve a standard tier (fast_mode.py).
+ served, tier = execution.served_tier
+ if served:
+ self.service.fast_tier_observations.observe(resolved.source_id, resolved.model_id, tier == "fast")
usage = execution.reported_usage
execution.usage_write = self._usage_writer.record(
source_id=resolved.source_id,
diff --git a/docs/plans/fast-mode.md b/docs/plans/fast-mode.md
index 0c03de9d3..49e993ca6 100644
--- a/docs/plans/fast-mode.md
+++ b/docs/plans/fast-mode.md
@@ -2,7 +2,9 @@
Status: owner-approved design, 2026-10-30. The menu uses option B: a Fast
toggle beside the effort heading, with a hover explanation. The trigger uses
-mark ②: a slim light-amber bolt beside the chevron.
+mark ②: a slim light-amber bolt beside the chevron. Revised 2026-10-31, by
+owner decision: relays are asked for the fast tier too, and each response
+confirms whether it ran.
## Goal
@@ -40,17 +42,19 @@ starts with the choice the user made last.
first value it is given, like its hosted tools, so a change applies from the
next turn.
-### Where the tier applies
+### Where the tier is asked for
The gateway reads the turn's tier for each request it can attribute to a turn
and carries it on `ModelHubRequest.speed`. Each attempt adds the vendor's field
-only where its exact `(Source, model)` offers the tier
+where its exact `(Source, model)` can carry it
(`core/handlers/model_hub/fast_mode.py`):
- the model's first-party vendor, by `vibe/data/model_vendors.json` family
- prefix, is OpenAI or Anthropic, and the Source is that vendor's own upstream:
- an API key on its official endpoint, or its subscription held by the Model Hub
- (not a `native_cli` Source, whose requests never reach the gateway);
+ prefix, is OpenAI or Anthropic, and the Source is a Hub API key or a
+ subscription held by the Model Hub that speaks that vendor's protocol. Where
+ an API key points does not matter: the official endpoint and a relay are both
+ asked, and the response says whether the tier ran. A `native_cli` Source never
+ reaches the gateway;
- the request speaks the Source's protocol, so the tier is never reached through
a translation;
- the vendor's models.dev entry for that model names the field under
@@ -69,40 +73,72 @@ picker reports fast mode as not offered.
A vendor refuses the fast tier without a code of its own. For example,
Anthropic answers an account without usage credits with 429
-`rate_limit_error`. So a fast attempt that fails with 400, 403 or 429 before any
-output is retried once on the same hop at the standard tier. Only that retry's
-outcome is classified, so a refused tier never cools down a healthy Source. A
-real rate limit costs one extra immediate request before it is classified as
-usual.
+`rate_limit_error`, and a relay that validates the body may answer 422. So a
+fast attempt that fails with 400, 403, 422 or 429 before any output is retried
+once on the same hop at the standard tier. Only that retry's outcome is
+classified, so a refused tier never cools down a healthy Source. A real rate
+limit costs one extra immediate request before it is classified as usual.
+
+### Confirmation
+
+Asking is not getting: a relay may drop the field, or pass it to an account
+that serves the standard tier. The vendors report the tier they served, and
+the Hub believes that report rather than the request:
+
+- Anthropic reports `usage.speed` (`fast` or `standard`). OpenAI reports
+ `service_tier` on the completed response or on every Chat chunk (`priority`
+ for its fast tier). In a Responses stream only the completed event counts:
+ `response.created` may echo the request even where the field was dropped. The stream tracker and the buffered projection read it
+ (`stream_wire.ServedTierPaths`) into `ProtocolSSEState.served_tier` and
+ `RawCallOutcome.served_tier`.
+- The resolution marks a hop whose request asked for the tier
+ (`ResolvedInvocation.fast_requested`, also after a refused attempt's standard
+ retry). When the gateway settles that call as served, it records whether
+ the response named the fast tier in `ModelHubService.fast_tier_observations`,
+ keyed by Source and model. A response that names another tier or none counts
+ as not confirmed.
+- Only the latest answer counts, and the field keeps being sent for a Session
+ that asks for it, so a relay that starts honouring the field is confirmed by
+ its next fast response. The record is process memory, bounded; after a
+ restart, a hop is unknown until its next fast response.
### Picker
-- The Hub Agent read states, on each catalog model, whether a turn on it would
- reach the fast tier on the hop that normally serves it (`fast_mode`). The model
- menu remembers these per backend, beside the model labels
- (`avibe.fast-models.v1`).
+- The Hub Agent read states, on each catalog model, whether a fast turn on it
+ asks for the tier on the hop that normally serves it (`fast_mode`), and what
+ that hop's last fast response said (`fast_mode_confirmed`: true, false, or
+ null when none was served yet). The model menu remembers both per backend,
+ beside the model labels (`avibe.fast-models.v2`), and the chat page reads them
+ again when a fast turn ends.
- The compact picker's Fast toggle sits beside the effort heading. Its hover and
- focus tooltip says what it does. While fast mode is on, a touch screen shows
- the same line under the efforts. On a model without the tier, the toggle keeps
- the Session's choice but is disabled, and the trigger shows no mark.
+ keyboard-focus tooltip says what it does. While fast mode is on, a touch screen
+ shows the same line under the efforts. On a model without the tier, or one
+ whose last fast response did not confirm it ran, the toggle keeps the
+ Session's choice but is disabled, the trigger shows no mark, and the tooltip
+ (or, on a touch screen, the line under the efforts) says which.
- The trigger's mark, a slim bolt in light amber, sits beside the chevron,
outside the parts a narrow row drops. It shows only while the Session asks for
- the fast tier and its model would get it.
+ the fast tier, its model's hop is asked for it, and that hop's last fast
+ response did not say otherwise.
## Not in this change
- Per-turn cost estimates. The tooltip says the price is higher; the
multiplier differs by vendor and plan.
- Fast mode for Direct-mode Agents, OpenCode, and the IM settings surfaces.
-- Recording the served tier in usage records. The response's `service_tier` or
- `usage.speed` says which tier ran.
+- Recording the served tier in usage records, and pricing fast requests there.
+ The tier is read on every call; the usage ledger does not keep it yet.
## Validation
- Scenario tests: `MH-FAST-001` (a fast Session's turn reaches the hop that
offers the tier, and only that hop), `MH-FAST-002` (a refused tier is served
- at the standard tier, and its Source stays routable), and `MH-FAST-003` (the
- menu marks the models that would run fast).
+ at the standard tier, and its Source stays routable), `MH-FAST-003` (the
+ menu marks the models that would run fast), and `MH-FAST-004` (a relay is
+ asked too, and its buffered or streamed responses decide whether the menu
+ says the tier ran).
+- Units: the served tier each protocol reports, streamed and buffered
+ (`tests/test_model_hub_served_tier.py`), and the observation record.
- Units: the hop rule (`tests/test_model_hub_fast_mode.py`), the Session field
and its dispatch read, the preference, the picker, and the new-session flow.
- Browser: the mark stays visible on a crowded 320px phone row.
diff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.py
index 9f2ec184a..3818292da 100644
--- a/docs/plans/model-hub-contracts/adapter-interface.py
+++ b/docs/plans/model-hub-contracts/adapter-interface.py
@@ -20,7 +20,7 @@ from enum import Enum
import re
from typing import Any, AsyncIterator, Awaitable, Callable, Final, Literal, Mapping, Protocol, Sequence
-from .stream_wire import ProtocolSSEState, ProtocolUsageReport
+from .stream_wire import ProtocolSSEState, ProtocolUsageReport, ServedTier
ENGINE_TRANSPORT_TIMEOUT_SECONDS: Final = 60.0
# Model invocation/observation is a narrower domain than stored Sources.
@@ -303,6 +303,9 @@ class RawCallOutcome:
# The search tools whose own completed item a served response carried, for
# the positive evidence written at settlement (``vibey-hosted-tools.md`` §3.2).
hosted_tools: frozenset[str] = frozenset()
+ # The processing tier a served response reports (``docs/plans/fast-mode.md``),
+ # for confirming that a fast request ran fast; None when it names none.
+ served_tier: ServedTier | None = None
# The response header name is retained in lowercase so relay ``x-request-id``
# and provider ``request-id`` values cannot be conflated. The client and
# provenance writer re-validate this tuple before persistence.
diff --git a/docs/plans/model-hub-contracts/backend-model.schema.json b/docs/plans/model-hub-contracts/backend-model.schema.json
index 4c592af22..1cd7d3a49 100644
--- a/docs/plans/model-hub-contracts/backend-model.schema.json
+++ b/docs/plans/model-hub-contracts/backend-model.schema.json
@@ -111,7 +111,11 @@
},
"fast_mode": {
"type": "boolean",
- "description": "Server-derived on Agent reads, accepted and dropped on write like effort_menu. Whether a turn on this model reaches its vendor's fast tier on the hop that normally serves it (docs/plans/fast-mode.md): that hop's Source is the model's first-party vendor's own upstream and the vendor's models.dev entry names the tier's request field. False for an Agent in Direct mode and for OpenCode, whose requests name no turn."
+ "description": "Server-derived on Agent reads, accepted and dropped on write like effort_menu. Whether a fast turn on this model asks for its vendor's fast tier on the hop that normally serves it (docs/plans/fast-mode.md): that hop's Source is a Hub API key or subscription speaking the model's first-party vendor's protocol, and the vendor's models.dev entry names the tier's request field. False for an Agent in Direct mode and for OpenCode, whose requests name no turn."
+ },
+ "fast_mode_confirmed": {
+ "type": ["boolean", "null"],
+ "description": "Server-derived on Agent reads, accepted and dropped on write like effort_menu. What the last fast response on that hop reported since the service started: true when it named the vendor's fast tier, false when it named another tier or none (as a relay that drops the field answers), null when no fast request has been served there, or when fast_mode is false."
}
}
}
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 92e1cf957..cfd78c5f4 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -2266,6 +2266,12 @@ scenarios:
kind: projection
layer: scenario
test: tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py::test_mh_fast_003_model_menu_marks_the_models_a_turn_would_run_fast_on
+ - id: MH-FAST-004
+ name: A relay is asked for the fast tier too, and its responses decide whether the menu says it ran
+ status: covered
+ kind: projection
+ layer: scenario
+ test: tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py::test_mh_fast_004_the_response_confirms_whether_the_fast_tier_ran
- id: MH-TURN-RESULT-001
name: All terminal outcomes are closed schema-valid provenance products
status: covered
diff --git a/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py b/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py
index ab8d278a6..d75ff875e 100644
--- a/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py
+++ b/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py
@@ -4,7 +4,7 @@ import asyncio
import json
from contextlib import asynccontextmanager
from collections import deque
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
@@ -92,6 +92,8 @@ class AdapterResult:
body: bytes | None = None
stream_started: bool = False
recovery_verified: bool = False
+ # The processing tier a served response names (stream_wire.ServedTierPaths).
+ served_tier: str | None = None
class ScenarioClock:
@@ -193,6 +195,7 @@ class AdapterBoundaryFake:
model_id=model_id,
source_id=source_id,
recovery_verified=result.recovery_verified,
+ served_tier=result.served_tier,
)
return InvokeHandle(outcome, result.body)
@@ -1660,6 +1663,89 @@ def test_mh_fast_002_refused_fast_tier_is_served_at_the_standard_tier(
asyncio.run(exercise())
+def _responses_sse(service_tier: str) -> bytes:
+ """A served Responses stream whose completed response names the tier it ran at."""
+
+ completed = {"type": "response.completed", "response": {"status": "completed", "service_tier": service_tier}}
+ return (
+ b'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"OK"}\n\n'
+ + b"event: response.completed\ndata: " + json.dumps(completed).encode() + b"\n\n"
+ )
+
+
[email protected]("stream", [False, True], ids=["buffered", "streamed"])
+def test_mh_fast_004_the_response_confirms_whether_the_fast_tier_ran(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ stream: bool,
+) -> None:
+ """MH-FAST-004: a relay is asked for the fast tier too, and its responses decide whether the menu says it ran."""
+
+ from vibe import models_dev_catalog
+
+ monkeypatch.setattr(models_dev_catalog, "_refresh_in_background", lambda: False)
+ model_id = _requested_model("codex")
+ _cache_fast_modes({model_id: {"service_tier": "priority"}})
+
+ def served(tier: str) -> AdapterResult:
+ # A buffered body was classified by the engine, which read its tier; a
+ # stream's tier is read by the gateway from the frames it forwards.
+ if stream:
+ return AdapterResult(RawOutcomeKind.SUCCESS, status=200, body=_responses_sse(tier), stream_started=True)
+ return AdapterResult(
+ RawOutcomeKind.SUCCESS, status=200, body=b'{"ok":true}',
+ served_tier="fast" if tier == "priority" else "standard",
+ )
+
+ async def exercise() -> None:
+ adapter = AdapterBoundaryFake([
+ # The relay drops the field and serves the standard tier.
+ served("default"),
+ # Later it passes the field through.
+ served("priority"),
+ # A standard turn there says nothing about the fast tier.
+ served("default"),
+ ])
+ relay = replace(
+ _source("src_relay1"), vendor="openai", protocol="openai_responses", base_url="https://relay.example/v1",
+ )
+ store = MemoryStore(_config(relay))
+ service = _service(
+ tmp_path, store, adapter, now=lambda: datetime(2026, 10, 30, tzinfo=timezone.utc),
+ )
+ gateway = ModelHubTurnGateway(service)
+ router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
+
+ def menu() -> tuple[bool, bool | None]:
+ row = next(row for row in service.get_agent_sources("codex")["catalog_models"] if row["id"] == model_id)
+ return row["fast_mode"], row["fast_mode_confirmed"]
+
+ async def turn(turn_id: str, speed: str | None) -> None:
+ launch = await router.resolve(
+ "codex", model_id, process_scope="scenario-fast", turn_id=turn_id, speed=speed,
+ )
+ status, _body = await _post_turn(
+ launch,
+ endpoint="responses",
+ payload={"model": launch.runtime_model, "input": [], "stream": stream},
+ )
+ assert status == 200
+
+ try:
+ assert menu() == (True, None)
+ await turn("turn_fast_1", "fast")
+ assert menu() == (True, False)
+ await turn("turn_fast_2", "fast")
+ assert menu() == (True, True)
+ await turn("turn_standard", None)
+ assert menu() == (True, True)
+ assert [request.get("service_tier") for request in adapter.requests] == ["priority", "priority", None]
+ finally:
+ await gateway.close()
+
+ asyncio.run(exercise())
+
+
def test_mh_fast_003_model_menu_marks_the_models_a_turn_would_run_fast_on(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/test_model_hub_api.py b/tests/test_model_hub_api.py
index 214434a55..d57ddba8a 100644
--- a/tests/test_model_hub_api.py
+++ b/tests/test_model_hub_api.py
@@ -2514,6 +2514,7 @@ def test_backend_catalog_add_edit_and_runtime_refresh(
# Server-derived like ``locked``: a client may send back the row it read, and the read states it again.
"effort_menu": {"omitted": [], "not_on": {}},
"fast_mode": False,
+ "fast_mode_confirmed": None,
}
if backend == "opencode":
added["native_protocol"] = "openai_responses"
@@ -3021,6 +3022,7 @@ def test_backend_catalog_add_validates_supplier_echo_and_stores_draft_literally(
"routeable": True,
"effort_menu": {"omitted": [], "not_on": {}},
"fast_mode": False,
+ "fast_mode_confirmed": None,
}
before = store.config.to_payload()
diff --git a/tests/test_model_hub_fast_mode.py b/tests/test_model_hub_fast_mode.py
index 8e89deaa3..6de501034 100644
--- a/tests/test_model_hub_fast_mode.py
+++ b/tests/test_model_hub_fast_mode.py
@@ -1,4 +1,4 @@
-"""Which Model Hub attempts may ask for a vendor's fast tier (core/handlers/model_hub/fast_mode.py)."""
+"""Which Model Hub attempts ask for a vendor's fast tier, and what their responses confirmed (fast_mode.py)."""
from __future__ import annotations
@@ -6,7 +6,7 @@ from types import SimpleNamespace
import pytest
-from core.handlers.model_hub.fast_mode import FastModeCatalog
+from core.handlers.model_hub.fast_mode import FastModeCatalog, FastTierObservations
_VENDOR_MAP = {
"families": [
@@ -45,8 +45,13 @@ def _source(vendor="openai", protocol="openai_responses", kind="api_key", base_u
(_source("anthropic", "anthropic", "subscription"), "claude-opus-5-5", "anthropic", {"speed": "fast"}),
# A dated or context-tagged spelling of a listed model is the same model.
(_source("anthropic", "anthropic"), "claude-opus-5-5[1m]", "anthropic", {"speed": "fast"}),
- # A relay or custom endpoint may front any deployment.
- (_source(base_url="https://relay.example/v1"), "gpt-5.6-sol", "openai_responses", None),
+ # A relay is asked too, whatever vendor its Source names: its response
+ # says whether the tier ran (FastTierObservations).
+ (_source(base_url="https://relay.example/v1"), "gpt-5.6-sol", "openai_responses", {"service_tier": "priority"}),
+ (_source("anthropic", "anthropic", base_url="https://relay.example"), "claude-opus-5-5", "anthropic",
+ {"speed": "fast"}),
+ (_source("custom-relay", "openai_chat", base_url="https://relay.example/v1"), "gpt-5.6-sol", "openai_chat",
+ {"service_tier": "priority"}),
# The native CLI's own login never reaches the gateway.
(_source(kind="subscription", supply_channel="native_cli"), "gpt-5.6-sol", "openai_responses", None),
# Never translated across protocols to reach a fast tier.
@@ -58,7 +63,9 @@ def _source(vendor="openai", protocol="openai_responses", kind="api_key", base_u
(_source("xai", "openai_responses"), "grok-5", "openai_responses", None),
],
)
-def test_fast_tier_is_offered_only_by_the_models_own_vendor_upstream(source, model, request_protocol, expected):
+def test_fast_tier_is_asked_for_on_hub_sources_speaking_the_models_vendor_protocol(
+ source, model, request_protocol, expected,
+):
assert _CATALOG.body_for(source, model, request_protocol) == expected
@@ -75,3 +82,28 @@ def test_catalog_entries_asking_for_other_fields_are_not_applied():
_VENDOR_MAP,
)
assert catalog.bodies == {}
+
+
+def test_the_last_fast_response_on_a_hop_is_what_counts():
+ observations = FastTierObservations()
+ assert observations.confirmed("src_relay", "gpt-5.6-sol") is None
+ observations.observe("src_relay", "gpt-5.6-sol", False)
+ assert observations.confirmed("src_relay", "gpt-5.6-sol") is False
+ # The relay started honouring the field: its next fast response says so.
+ observations.observe("src_relay", "gpt-5.6-sol", True)
+ assert observations.confirmed("src_relay", "gpt-5.6-sol") is True
+ # Per Source and model.
+ assert observations.confirmed("src_relay", "gpt-6-astra") is None
+ assert observations.confirmed("src_other", "gpt-5.6-sol") is None
+
+
+def test_observations_stay_bounded_by_dropping_the_longest_unused():
+ observations = FastTierObservations()
+ limit = FastTierObservations._LIMIT
+ for index in range(limit):
+ observations.observe("src", f"model-{index}", True)
+ observations.observe("src", "model-0", False) # refreshed: now the newest
+ observations.observe("src", "one-more", True)
+ assert observations.confirmed("src", "model-1") is None
+ assert observations.confirmed("src", "model-0") is False
+ assert observations.confirmed("src", "one-more") is True
diff --git a/tests/test_model_hub_served_tier.py b/tests/test_model_hub_served_tier.py
new file mode 100644
index 000000000..e6158ffab
--- /dev/null
+++ b/tests/test_model_hub_served_tier.py
@@ -0,0 +1,111 @@
+"""The processing tier a response reports having served (stream_wire.ServedTierPaths, docs/plans/fast-mode.md).
+
+A fast request is only confirmed by its response: Anthropic reports ``usage.speed``
+and OpenAI ``service_tier``. A relay that drops the request field answers with a
+standard tier or with none, and both must read as "not fast".
+"""
+
+from __future__ import annotations
+
+import io
+import json
+
+import pytest
+
+from core.handlers.model_hub.stream_wire import ProtocolSSEState, observe_buffered_protocol_response
+
+
+def _frame(document: dict) -> bytes:
+ named = f"event: {document['type']}\n".encode() if "type" in document else b""
+ return named + b"data: " + json.dumps(document).encode() + b"\n\n"
+
+
+def _streamed(protocol: str, *documents: dict | bytes) -> ProtocolSSEState:
+ state = ProtocolSSEState(protocol)
+ for document in documents:
+ state.observe(document if isinstance(document, bytes) else _frame(document))
+ return state
+
+
+def _anthropic_stream(speed: str | None) -> tuple[dict, ...]:
+ usage = {"output_tokens": 5, **({"speed": speed} if speed is not None else {})}
+ return (
+ {"type": "message_start", "message": {"usage": {"input_tokens": 9}}},
+ {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
+ {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "OK"}},
+ {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": usage},
+ {"type": "message_stop"},
+ )
+
+
+def _responses_stream(tier: str | None, echoed: str = "auto") -> tuple[dict, ...]:
+ completed = {"status": "completed", "usage": {"input_tokens": 9, "output_tokens": 5}}
+ if tier is not None:
+ completed["service_tier"] = tier
+ return (
+ # The created event echoes what was asked; only the completed one says what ran.
+ {"type": "response.created", "response": {"status": "in_progress", "service_tier": echoed}},
+ {"type": "response.output_text.delta", "delta": "OK"},
+ {"type": "response.completed", "response": completed},
+ )
+
+
+ ("speed", "expected"),
+ [("fast", "fast"), ("standard", "standard"), (None, None)],
+)
+def test_anthropic_stream_reports_the_speed_it_served(speed, expected):
+ state = _streamed("anthropic", *_anthropic_stream(speed))
+ assert state.terminal_outcome == "served"
+ assert state.served_tier == expected
+ observation = state.terminal_observation()
+ assert observation is not None and observation.served_tier == expected
+
+
+ ("tier", "echoed", "expected"),
+ [
+ ("priority", "priority", "fast"),
+ ("default", "priority", "standard"),
+ ("flex", "auto", "standard"),
+ # Only the completed response says what ran: a relay may echo the request
+ # on the created event and then name no tier, which confirms none.
+ (None, "priority", None),
+ ],
+)
+def test_responses_stream_reports_the_tier_it_served(tier, echoed, expected):
+ state = _streamed("openai_responses", *_responses_stream(tier, echoed))
+ assert state.terminal_outcome == "served"
+ assert state.served_tier == expected
+
+
[email protected](("tier", "expected"), [("priority", "fast"), ("default", "standard")])
+def test_chat_stream_names_its_tier_on_every_chunk(tier, expected):
+ state = _streamed(
+ "openai_chat",
+ {"choices": [{"delta": {"content": "OK"}}], "service_tier": tier},
+ {"choices": [{"delta": {}, "finish_reason": "stop"}], "service_tier": tier},
+ b"data: [DONE]\n\n",
+ )
+ assert state.terminal_outcome == "served"
+ assert state.served_tier == expected
+
+
+ ("protocol", "body", "expected"),
+ [
+ ("anthropic", {"type": "message", "content": [], "usage": {"output_tokens": 1, "speed": "fast"}}, "fast"),
+ ("anthropic", {"type": "message", "content": [], "usage": {"output_tokens": 1}}, None),
+ ("openai_responses", {"object": "response", "status": "completed", "output": [], "service_tier": "priority"},
+ "fast"),
+ ("openai_responses", {"object": "response", "status": "completed", "output": [], "service_tier": "default"},
+ "standard"),
+ ("openai_chat", {"object": "chat.completion", "choices": [], "service_tier": "priority"}, "fast"),
+ # A relay that answers with another vendor's shape names nothing this reads.
+ ("google", {"candidates": [{"finishReason": "STOP"}]}, None),
+ ],
+)
+def test_buffered_response_reports_the_tier_it_served(protocol, body, expected):
+ observation = observe_buffered_protocol_response(protocol, io.BytesIO(json.dumps(body).encode()))
+ assert observation.outcome == "served"
+ assert observation.served_tier == expected
diff --git a/ui/src/components/settings/models/types.ts b/ui/src/components/settings/models/types.ts
index 6383b22c5..feb78a9e1 100644
--- a/ui/src/components/settings/models/types.ts
+++ b/ui/src/components/settings/models/types.ts
@@ -247,8 +247,11 @@ export type EffortMenu = { omitted: string[]; not_on: Record<string, string[]> }
/** A catalog row as the model picker reads it. */
export type PickerCatalogModel = BackendModel & {
effort_menu?: EffortMenu;
- /** Whether a turn on this model reaches its vendor's fast tier on the hop that normally serves it. */
+ /** Whether a fast turn on this model asks for its vendor's fast tier on the hop that normally serves it. */
fast_mode?: boolean;
+ /** What that hop's last fast response reported: true ran fast, false did not
+ * confirm it (a relay may drop the field), null not seen yet. */
+ fast_mode_confirmed?: boolean | null;
};
/** The model picker's read of one backend's catalog (`GET /api/models/agents/<backend>/models`). */
diff --git a/ui/src/components/workbench/AgentRoutePicker.test.tsx b/ui/src/components/workbench/AgentRoutePicker.test.tsx
index d860520ce..f25a9f22e 100644
--- a/ui/src/components/workbench/AgentRoutePicker.test.tsx
+++ b/ui/src/components/workbench/AgentRoutePicker.test.tsx
@@ -45,8 +45,10 @@ vi.mock('../../context/ApiContext', async (importOriginal) => ({
let catalogReasoning: Record<string, { value: string; label: string }[]> = {};
let catalogMenus: Record<string, { omitted: string[]; not_on: Record<string, string[]> }> = {};
let catalogLabels: Record<string, string> = {};
-// The models a Hub catalog says run at their vendor's fast tier.
+// The models a Hub catalog asks for their vendor's fast tier on, and those whose
+// last fast response did not confirm it ran.
let catalogFast: string[] = [];
+let catalogFastUnconfirmed: string[] = [];
// How many catalog reads were made, and whether the next ones wait to be answered.
let catalogReads = 0;
let deferReads = false;
@@ -65,6 +67,7 @@ vi.mock('../../lib/backendModels', async (importOriginal) => ({
reasoningOptions: Record<string, { value: string; label: string }[]>;
effortMenus?: Record<string, { omitted: string[]; not_on: Record<string, string[]> }>;
fastModels?: string[];
+ fastUnconfirmed?: string[];
hubUnread?: true;
catalogRefreshPending: boolean;
}) => void,
@@ -77,7 +80,9 @@ vi.mock('../../lib/backendModels', async (importOriginal) => ({
// As the real loader answers: only a Hub catalog carries effort menus.
...(hubUnread
? { hubUnread: true as const }
- : hubMode === 'hub' ? { effortMenus: catalogMenus, fastModels: catalogFast } : {}),
+ : hubMode === 'hub'
+ ? { effortMenus: catalogMenus, fastModels: catalogFast, fastUnconfirmed: catalogFastUnconfirmed }
+ : {}),
catalogRefreshPending: false,
});
if (deferReads) pendingReads.push(answer);
@@ -604,6 +609,7 @@ describe('AgentRoutePicker compact presentation', () => {
catalogReasoning = {};
catalogMenus = {};
catalogFast = [];
+ catalogFastUnconfirmed = [];
window.localStorage.clear();
sourceNames = null;
hubMode = 'direct';
@@ -634,6 +640,23 @@ describe('AgentRoutePicker compact presentation', () => {
);
};
+ it('says so where the last fast response did not confirm the fast tier ran, and keeps the choice', async () => {
+ // Opus's hop is asked for the fast tier, but a relay answered at the standard one.
+ hubMode = 'hub';
+ catalogReasoning = { sonnet: [{ value: 'low', label: 'low' }], opus: [{ value: 'low', label: 'low' }] };
+ catalogFastUnconfirmed = ['opus'];
+ const user = userEvent.setup();
+ render(<FastOwner initial={{ ...CLAUDE_ROUTE, model: 'opus' }} initialSpeed="fast" onSpeed={vi.fn()} />);
+ await openMenu(user);
+ const toggle = screen.getByRole('button', { name: 'chat.picker.fast' }) as HTMLButtonElement;
+ expect(toggle.disabled).toBe(true);
+ expect(toggle.getAttribute('aria-pressed')).toBe('true');
+ expect(screen.getByRole('tooltip').textContent).toContain('chat.picker.fastUnconfirmed');
+ // No mark: nothing says the route runs fast. A touch screen reads why.
+ expect(screen.queryByLabelText('chat.picker.fastOn')).toBeNull();
+ expect(screen.getByText('chat.picker.fastUnconfirmedCaption')).not.toBeNull();
+ });
+
it("judges Fast on the Agent's model for a route that names none", async () => {
// A Session that materialized its Agent at the first turn keeps no model of its own.
hubMode = 'hub';
diff --git a/ui/src/components/workbench/AgentRoutePicker.tsx b/ui/src/components/workbench/AgentRoutePicker.tsx
index 60f090d49..83e5b409b 100644
--- a/ui/src/components/workbench/AgentRoutePicker.tsx
+++ b/ui/src/components/workbench/AgentRoutePicker.tsx
@@ -193,14 +193,16 @@ export const AgentRoutePicker: React.FC<AgentRoutePickerProps> = ({
// across pickers and loads; the raw id stays one hover away.
const currentModelLabel = useRememberedModelLabel(backend, currentModel) ?? currentModel;
// Fast mode is the route's to ask for and the model's hop's to grant: the mark
- // and the toggle follow whether this model would actually run fast, so asking
- // on a model without it keeps the choice for the next model that has it.
+ // and the toggle follow whether this model's hop is asked for the fast tier
+ // and its last fast response did not say otherwise, so asking on a model
+ // without it keeps the choice for the next model that has it.
const fastControls = compact && Boolean(onSpeedChange);
const fastOn = speed === 'fast';
// The model a turn runs on: the route's own, else its Agent's default, which a
// Session that materialized its Agent at the first turn keeps without copying.
const effectiveModel = currentModel || displayedAgent?.model || null;
- const fastOffered = useRememberedFastModel(backend, effectiveModel, fastControls && (fastOn || open));
+ const fastState = useRememberedFastModel(backend, effectiveModel, fastControls && (fastOn || open));
+ const fastOffered = fastState === 'offered';
const fastActive = fastControls && fastOn && fastOffered;
const currentAgentLabel = displayedAgent?.display_name || currentAgent;
const triggerLabel = (routeIsDefault && defaultLabel) || currentAgentLabel || defaultLabel || t('chat.pickAgent');
@@ -296,7 +298,9 @@ export const AgentRoutePicker: React.FC<AgentRoutePickerProps> = ({
// Closed pickers anywhere name their model from this answer.
rememberModelLabels(backend, modelLabels ?? {});
// An unread Hub says nothing about which models run fast.
- if (!result.hubUnread) rememberFastModels(backend, result.fastModels ?? []);
+ if (!result.hubUnread) {
+ rememberFastModels(backend, { offered: result.fastModels ?? [], unconfirmed: result.fastUnconfirmed ?? [] });
+ }
},
() => {
loadedBackends.delete(backend);
@@ -652,6 +656,7 @@ export const AgentRoutePicker: React.FC<AgentRoutePickerProps> = ({
<FastToggle
on={fastOn}
offered={fastOffered}
+ unconfirmed={fastState === 'unconfirmed'}
active={fastActive}
disabled={loadingModels}
onToggle={() => onSpeedChange?.(fastOn ? null : 'fast')}
@@ -696,10 +701,10 @@ export const AgentRoutePicker: React.FC<AgentRoutePickerProps> = ({
</>
)}
{/* A touch screen has no hover for the toggle's explanation: it is
- written out while the route runs fast. */}
- {fastActive && (
+ written out while the route asks for the fast tier. */}
+ {fastControls && fastOn && (fastOffered || fastState === 'unconfirmed') && (
<p className="pt-1.5 text-[10px] leading-snug text-muted [@media(hover:hover)]:hidden">
- {t('chat.picker.fastCaption')}
+ {fastOffered ? t('chat.picker.fastCaption') : t('chat.picker.fastUnconfirmedCaption')}
</p>
)}
</div>
@@ -936,14 +941,16 @@ const FastMark: React.FC<React.SVGProps<SVGSVGElement>> = ({ className, ...props
// The Fast toggle beside the effort heading. Hover or keyboard focus explains
// it; a click's focus does not, so the tip never lingers over the model list. A
-// model whose hop offers no fast tier keeps the choice but cannot change it here.
+// model whose hop offers no fast tier, or whose last fast response did not
+// confirm it ran, keeps the choice but cannot change it here.
const FastToggle: React.FC<{
on: boolean;
offered: boolean;
+ unconfirmed: boolean;
active: boolean;
disabled?: boolean;
onToggle: () => void;
-}> = ({ on, offered, active, disabled, onToggle }) => {
+}> = ({ on, offered, unconfirmed, active, disabled, onToggle }) => {
const { t } = useTranslation();
const tipId = useId();
return (
@@ -971,7 +978,11 @@ const FastToggle: React.FC<{
<span className="block font-semibold">{t('chat.picker.fastTitle')}</span>
<span className="block">{t('chat.picker.fastDescription')}</span>
<span className="block text-background/65">
- {offered ? t('chat.picker.fastScope') : t('chat.picker.fastUnavailable')}
+ {offered
+ ? t('chat.picker.fastScope')
+ : unconfirmed
+ ? t('chat.picker.fastUnconfirmed')
+ : t('chat.picker.fastUnavailable')}
</span>
</span>
</span>
diff --git a/ui/src/components/workbench/ChatPage.tsx b/ui/src/components/workbench/ChatPage.tsx
index 6f4fe7fed..b105277cc 100644
--- a/ui/src/components/workbench/ChatPage.tsx
+++ b/ui/src/components/workbench/ChatPage.tsx
@@ -20,6 +20,7 @@ import { apiFetch } from '../../lib/apiFetch';
import { readChatViewMode, writeChatViewMode } from '../../lib/chatViewMemory';
import { normalizeChatMessageFontSize } from '../../lib/chatDisplay';
import { setConfigField } from '../../lib/configMutations';
+import { refreshFastModels } from '../../lib/modelLabelMemory';
import { annotationStandIn, annotationTitleKey, readAnnotationView } from '../../lib/annotationView';
import { isAgentActivityBoundaryMessage, isRetriedFailureNotice, isRetryableFailureNotice, isTerminalAgentMessage, offersContinueOnVibey, isTranscriptMessage, shouldRefreshAgentActivityForMessage } from '../../lib/chatMessageTypes';
import { chatRowKind, drawsEmptyBodyPlaceholder, isAgentAuthored } from '../../lib/chatRowKind';
@@ -3678,6 +3679,14 @@ const ChatRoutePicker: React.FC<ChatRoutePickerProps> = ({ session, agents, defa
// settles. Idle blank-backend rows with a native (legacy, pre-backfill) stay
// enabled: the server allows their one-time "initial pin".
const concreteBackend = session.agent_backend?.trim() || null;
+ // A fast turn's response confirms or contradicts the fast tier on its hop, so
+ // the picker reads that answer again when one ends: the mark follows what ran.
+ const fastBackend = session.speed === 'fast' ? (concreteBackend ?? defaultAgent?.backend ?? null) : null;
+ const wasWorkingRef = useRef(working);
+ useEffect(() => {
+ if (wasWorkingRef.current && !working && fastBackend) refreshFastModels(api, fastBackend);
+ wasWorkingRef.current = working;
+ }, [api, fastBackend, working]);
const nativeBound = Boolean(session.native_session_id);
const pendingForkBackend =
!session.native_session_id &&
diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json
index 0365fbab7..9ca486c93 100644
--- a/ui/src/i18n/en.json
+++ b/ui/src/i18n/en.json
@@ -5187,7 +5187,9 @@
"fastDescription": "Same model, faster output, higher price.",
"fastScope": "Offered on some models and providers only",
"fastUnavailable": "Not offered for this model or provider",
+ "fastUnconfirmed": "This provider didn't confirm the last request ran fast",
"fastCaption": "Fast: faster output, higher price",
+ "fastUnconfirmedCaption": "Fast: this provider didn't confirm it ran fast",
"newAgent": "New agent",
"addModel": "Add model",
"noModels": "No models found",
diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.json
index d19a0a230..be4cc9797 100644
--- a/ui/src/i18n/zh.json
+++ b/ui/src/i18n/zh.json
@@ -5187,7 +5187,9 @@
"fastDescription": "同一个模型,输出更快,费用更高。",
"fastScope": "仅部分模型和服务商支持",
"fastUnavailable": "当前模型或服务商不支持",
+ "fastUnconfirmed": "这个服务商上次没有确认以 Fast 运行",
"fastCaption": "Fast:输出更快,费用更高",
+ "fastUnconfirmedCaption": "Fast:这个服务商没有确认以 Fast 运行",
"newAgent": "新建 Agent",
"addModel": "添加模型",
"noModels": "未找到模型",
diff --git a/ui/src/lib/backendModels.test.ts b/ui/src/lib/backendModels.test.ts
index 33b1f3ae3..13b701e70 100644
--- a/ui/src/lib/backendModels.test.ts
+++ b/ui/src/lib/backendModels.test.ts
@@ -148,6 +148,7 @@ describe('fetchBackendModels in gateway mode', () => {
expect(await fetchBackendModels(api, 'vibey')).toEqual({
models: ['local-model'], modelLabels: { 'local-model': 'Local model' }, reasoningOptions: { 'local-model': [] }, effortMenus: {},
fastModels: [],
+ fastUnconfirmed: [],
});
expect(readModelHubAgentCatalogForModelPicker).toHaveBeenCalledWith('vibey');
readModelHubAgentCatalogForModelPicker.mockResolvedValue(null);
@@ -176,8 +177,10 @@ describe('fetchBackendModels in gateway mode', () => {
supports_reasoning: true,
reasoning_efforts: ['low', 'high'],
}),
- // A turn on beta would reach its vendor's fast tier.
- { ...model('beta'), fast_mode: true },
+ // A fast turn on beta asks for its vendor's fast tier, and one on
+ // gamma did too, but gamma's last fast response did not confirm it ran.
+ { ...model('beta'), fast_mode: true, fast_mode_confirmed: true },
+ { ...model('gamma'), fast_mode: true, fast_mode_confirmed: false },
],
}),
),
@@ -188,17 +191,19 @@ describe('fetchBackendModels in gateway mode', () => {
expect(claudeModels).not.toHaveBeenCalled();
expect(result).toEqual({
- models: ['alpha', 'beta'],
+ models: ['alpha', 'beta', 'gamma'],
modelLabels: { alpha: 'Alpha' },
reasoningOptions: {
alpha: [{ value: 'low', label: 'low' }, { value: 'high', label: 'high' }],
// Present and empty: `beta` states it does not reason, and only a key
// the catalog never wrote may fall back to the generic ladder.
beta: [],
+ gamma: [],
},
// No row states an effort menu, so nothing is left out or marked.
effortMenus: {},
fastModels: ['beta'],
+ fastUnconfirmed: ['gamma'],
});
});
@@ -366,6 +371,7 @@ describe('fetchBackendModels in gateway mode', () => {
reasoningOptions: {},
effortMenus: {},
fastModels: [],
+ fastUnconfirmed: [],
});
expect(codexModels).not.toHaveBeenCalled();
});
diff --git a/ui/src/lib/backendModels.ts b/ui/src/lib/backendModels.ts
index 00595b3e7..05a5174d7 100644
--- a/ui/src/lib/backendModels.ts
+++ b/ui/src/lib/backendModels.ts
@@ -22,9 +22,13 @@ export interface BackendModels {
* `effort_menu`): the efforts left out above, and by Source id the hops that
* refuse one the others still serve. Hub catalogs only. */
effortMenus?: Record<string, EffortMenu>;
- /** The models a turn would run on at its vendor's fast tier (Model Hub's
- * `fast_mode`). Hub catalogs only: a Direct backend offers none. */
+ /** The models a fast turn asks for its vendor's fast tier on (Model Hub's
+ * `fast_mode`), minus those whose last fast response did not confirm it ran.
+ * Hub catalogs only: a Direct backend offers none. */
fastModels?: string[];
+ /** The models whose hop was asked for the fast tier and whose last fast
+ * response did not confirm it ran (`fast_mode_confirmed: false`). */
+ fastUnconfirmed?: string[];
/** True while a background remote-catalog refresh may produce a newer snapshot. */
catalogRefreshPending?: boolean;
/** Model Hub gave no answer (the read failed, or the server predates the read),
@@ -59,6 +63,7 @@ const hubCatalogModels = (agent: PickerAgentCatalog | null, backend: string): Ba
const reasoningOptions = Object.create(null) as Record<string, { value: string; label: string }[]>;
const effortMenus = Object.create(null) as Record<string, EffortMenu>;
const fastModels: string[] = [];
+ const fastUnconfirmed: string[] = [];
for (const id of models) {
const row = rows.get(id);
if (!row) continue;
@@ -73,9 +78,9 @@ const hubCatalogModels = (agent: PickerAgentCatalog | null, backend: string): Ba
? []
: row.reasoning_efforts.filter((effort) => !omitted.has(effort)).map((effort) => ({ value: effort, label: effort }));
if (row.effort_menu) effortMenus[id] = row.effort_menu;
- if (row.fast_mode === true) fastModels.push(id);
+ if (row.fast_mode === true) (row.fast_mode_confirmed === false ? fastUnconfirmed : fastModels).push(id);
}
- return { models, modelLabels, reasoningOptions, effortMenus, fastModels };
+ return { models, modelLabels, reasoningOptions, effortMenus, fastModels, fastUnconfirmed };
};
/**
@@ -148,10 +153,15 @@ export async function fetchBackendModels(
* from Model Hub's persisted catalog only. A backend Hub does not serve, or
* whose catalog it cannot answer, offers none.
*/
-export async function fetchBackendFastModels(api: ApiContextType, backend: string): Promise<string[]> {
- return hubCatalogModels(await api.readModelHubAgentCatalogForModelPicker(backend), backend)?.fastModels ?? [];
+export async function fetchBackendFastModels(api: ApiContextType, backend: string): Promise<FastModels> {
+ const hub = hubCatalogModels(await api.readModelHubAgentCatalogForModelPicker(backend), backend);
+ return { offered: hub?.fastModels ?? [], unconfirmed: hub?.fastUnconfirmed ?? [] };
}
+/** Where a backend's models stand on the fast tier: asked for and not
+ * contradicted, or asked for and not confirmed by the last fast response. */
+export type FastModels = { offered: string[]; unconfirmed: string[] };
+
export async function fetchBackendModelLabels(
api: ApiContextType,
backend: string,
diff --git a/ui/src/lib/modelLabelMemory.test.ts b/ui/src/lib/modelLabelMemory.test.ts
index f04c3b2e5..70a7ac4f9 100644
--- a/ui/src/lib/modelLabelMemory.test.ts
+++ b/ui/src/lib/modelLabelMemory.test.ts
@@ -91,31 +91,54 @@ describe('remembered model labels', () => {
});
describe('remembered fast-tier models', () => {
- const hubCatalog = (fast: string[]) => ({
+ const hubCatalog = (fast: string[], unconfirmed: string[] = []) => ({
backend: 'claude',
mode: 'hub',
catalog_models: ['claude-opus-5-5', 'claude-sonnet-5-5'].map((id) => ({
id, origin: 'builtin', display_name: null, models_dev_id: null, context_window: null, max_output_tokens: null,
input_modalities: [], output_modalities: [], supports_tools: null, supports_reasoning: null, reasoning_efforts: [],
- locked: false, routeable: true, fast_mode: fast.includes(id),
+ locked: false, routeable: true,
+ fast_mode: fast.includes(id) || unconfirmed.includes(id),
+ fast_mode_confirmed: unconfirmed.includes(id) ? false : null,
})),
});
+ const STORED = 'avibe.fast-models.v2';
it('paints from a stored answer, then reads it again once per load since routing may have moved', async () => {
- // Stored while the route went through a relay: no fast tier then.
- window.localStorage.setItem('avibe.fast-models.v1', JSON.stringify({ claude: [] }));
+ // Stored while the route went through a Source without the tier.
+ window.localStorage.setItem(STORED, JSON.stringify({ claude: { offered: [], unconfirmed: [] } }));
api.readModelHubAgentCatalogForModelPicker.mockResolvedValue(hubCatalog(['claude-opus-5-5']));
const { useRememberedFastModel } = await freshMemory();
const { result, rerender } = renderHook(({ wanted }) => useRememberedFastModel('claude', 'claude-opus-5-5', wanted), {
initialProps: { wanted: false },
});
- expect(result.current).toBe(false);
+ expect(result.current).toBeNull();
expect(api.readModelHubAgentCatalogForModelPicker).not.toHaveBeenCalled();
rerender({ wanted: true });
- await waitFor(() => expect(result.current).toBe(true));
+ await waitFor(() => expect(result.current).toBe('offered'));
rerender({ wanted: true });
expect(api.readModelHubAgentCatalogForModelPicker).toHaveBeenCalledTimes(1);
});
+
+ it('follows what the last fast response said, read again when a fast turn ends', async () => {
+ api.readModelHubAgentCatalogForModelPicker.mockResolvedValue(hubCatalog(['claude-opus-5-5']));
+ const { useRememberedFastModel, refreshFastModels } = await freshMemory();
+ const { result } = renderHook(() => useRememberedFastModel('claude', 'claude-opus-5-5', true));
+ await waitFor(() => expect(result.current).toBe('offered'));
+
+ // The turn ran through a relay that answered at the standard tier.
+ api.readModelHubAgentCatalogForModelPicker.mockResolvedValue(hubCatalog([], ['claude-opus-5-5']));
+ refreshFastModels(api as never, 'claude');
+ await waitFor(() => expect(result.current).toBe('unconfirmed'));
+ expect(JSON.parse(window.localStorage.getItem(STORED) ?? '{}')).toEqual({
+ claude: { offered: [], unconfirmed: ['claude-opus-5-5'] },
+ });
+
+ // A later fast turn there ran fast.
+ api.readModelHubAgentCatalogForModelPicker.mockResolvedValue(hubCatalog(['claude-opus-5-5']));
+ refreshFastModels(api as never, 'claude');
+ await waitFor(() => expect(result.current).toBe('offered'));
+ });
});
diff --git a/ui/src/lib/modelLabelMemory.ts b/ui/src/lib/modelLabelMemory.ts
index b613f33f4..6bc862601 100644
--- a/ui/src/lib/modelLabelMemory.ts
+++ b/ui/src/lib/modelLabelMemory.ts
@@ -1,7 +1,7 @@
import { useEffect, useSyncExternalStore } from 'react';
-import { useApi } from '../context/ApiContext';
-import { fetchBackendFastModels, fetchBackendModelLabels } from './backendModels';
+import { useApi, type ApiContextType } from '../context/ApiContext';
+import { fetchBackendFastModels, fetchBackendModelLabels, type FastModels } from './backendModels';
// Friendly model names ("Opus 5.5") for closed pickers, which otherwise only
// know the raw id the route stores. Every catalog answer a picker already reads
@@ -83,16 +83,26 @@ export function useRememberedModelLabel(backend: string, model: string | null |
return label;
}
-// Which models run at their vendor's fast tier, for a closed picker's fast-mode
-// mark: it shows only while the Session asks for the fast tier AND its model
-// would get it. Remembered from every catalog answer a picker reads, so the mark
-// paints from the first frame. Unlike a label, the answer follows routing, which
-// Settings or another client may have changed since it was stored: a picker that
-// needs it reads Model Hub's persisted catalog once per backend per page load.
+// Where each model stands on its vendor's fast tier, for a closed picker's
+// fast-mode mark: it shows only while the Session asks for the fast tier AND its
+// model's hop is asked for it without its last fast response saying otherwise.
+// Remembered from every catalog answer a picker reads, so the mark paints from
+// the first frame. Unlike a label, the answer follows routing and what the last
+// fast response reported, which Settings, another client or the last turn may
+// have changed since it was stored: a picker that needs it reads Model Hub's
+// persisted catalog once per backend per page load, and again when a fast turn
+// ends (refreshFastModels).
-const FAST_STORAGE_KEY = 'avibe.fast-models.v1';
+const FAST_STORAGE_KEY = 'avibe.fast-models.v2';
-type FastByBackend = Record<string, string[]>;
+type FastByBackend = Record<string, FastModels>;
+
+/** Where a model stands: offered, asked for but not confirmed by its last fast
+ * response, or neither (no fast tier on its hop, or not known yet). */
+export type FastModelState = 'offered' | 'unconfirmed' | null;
+
+const isModelList = (value: unknown): value is string[] =>
+ Array.isArray(value) && value.every((model) => typeof model === 'string');
function readStoredFast(): FastByBackend {
try {
@@ -100,8 +110,9 @@ function readStoredFast(): FastByBackend {
const parsed = raw ? (JSON.parse(raw) as unknown) : null;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {};
const out = Object.create(null) as FastByBackend;
- for (const [backend, models] of Object.entries(parsed as Record<string, unknown>)) {
- if (Array.isArray(models) && models.every((model) => typeof model === 'string')) out[backend] = models;
+ for (const [backend, entry] of Object.entries(parsed as Record<string, unknown>)) {
+ const { offered, unconfirmed } = (entry ?? {}) as Record<string, unknown>;
+ if (isModelList(offered) && isModelList(unconfirmed)) out[backend] = { offered, unconfirmed };
}
return out;
} catch {
@@ -115,12 +126,12 @@ const fastReadThisLoad = new Set<string>();
const currentFast = (): FastByBackend => (fastByBackend ??= readStoredFast());
-/** Record a backend's complete answer of fast-tier models, replacing what was known for it. */
-export function rememberFastModels(backend: string, models: readonly string[]): void {
+/** Record a backend's complete fast-tier answer, replacing what was known for it. */
+export function rememberFastModels(backend: string, models: FastModels): void {
if (!backend) return;
fastReadThisLoad.add(backend);
const next = Object.assign(Object.create(null), currentFast()) as FastByBackend;
- next[backend] = [...models];
+ next[backend] = { offered: [...models.offered], unconfirmed: [...models.unconfirmed] };
fastByBackend = next;
try {
window.localStorage.setItem(FAST_STORAGE_KEY, JSON.stringify(next));
@@ -130,23 +141,36 @@ export function rememberFastModels(backend: string, models: readonly string[]):
for (const listener of fastListeners) listener();
}
+/** Read a backend's fast-tier answer again now, as after a fast turn whose
+ * response may have confirmed or contradicted it. */
+export function refreshFastModels(api: ApiContextType, backend: string): void {
+ if (!backend) return;
+ fastReadThisLoad.add(backend);
+ fetchBackendFastModels(api, backend).then(
+ (models) => rememberFastModels(backend, models),
+ () => undefined,
+ );
+}
+
const subscribeFast = (listener: () => void) => {
fastListeners.add(listener);
return () => fastListeners.delete(listener);
};
-/** Whether `model` runs at its vendor's fast tier as far as this browser knows;
+/** Where `model` stands on its vendor's fast tier as far as this browser knows;
* when `wanted`, reads the backend's answer again once per page load. */
-export function useRememberedFastModel(backend: string, model: string | null | undefined, wanted: boolean): boolean {
+export function useRememberedFastModel(
+ backend: string,
+ model: string | null | undefined,
+ wanted: boolean,
+): FastModelState {
const api = useApi();
const known = useSyncExternalStore(subscribeFast, () => (backend ? currentFast()[backend] ?? null : null));
useEffect(() => {
if (!wanted || !backend || fastReadThisLoad.has(backend)) return;
- fastReadThisLoad.add(backend);
- fetchBackendFastModels(api, backend).then(
- (models) => rememberFastModels(backend, models),
- () => undefined,
- );
+ refreshFastModels(api, backend);
}, [api, backend, wanted]);
- return Boolean(model && known?.includes(model));
+ if (!model || !known) return null;
+ if (known.offered.includes(model)) return 'offered';
+ return known.unconfirmed.includes(model) ? 'unconfirmed' : null;
}
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index a3d1ab193..df82bd7f0 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -53,6 +53,7 @@ from core.handlers.model_hub.stream_wire import (
ProtocolObservation,
ProtocolSSEState,
ProtocolUsageReport,
+ ServedTier,
observe_buffered_protocol_response,
)
from vibe.model_hub_runtime.api_key_vendors import (
@@ -2032,6 +2033,7 @@ def _reduce_protocol_observation(
usage=observation.usage,
recovery_verified=observation.recovery_verified,
hosted_tools=observation.hosted_tools,
+ served_tier=observation.served_tier,
)
if observation.outcome == "failed_terminal":
projected_types = tuple(
@@ -2209,6 +2211,7 @@ def _outcome(
os_errno: int | None = None,
rejection: UpstreamRejection | None = None,
hosted_tools: frozenset[str] = frozenset(),
+ served_tier: ServedTier | None = None,
) -> RawCallOutcome:
return RawCallOutcome(
kind=kind,
@@ -2227,6 +2230,7 @@ def _outcome(
os_errno=os_errno,
rejection=rejection,
hosted_tools=hosted_tools,
+ served_tier=served_tier,
)