Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): preserve backend failure kinds without inventing protocol facts (#2544)
Call White9a89da41b0f1
diff --git a/core/handlers/model_hub/events.py b/core/handlers/model_hub/events.pyindex 8904155e7..25a59c96b 100644--- a/core/handlers/model_hub/events.py+++ b/core/handlers/model_hub/events.py@@ -11,6 +11,7 @@ from datetime import datetime, timezonefrom pathlib import Pathfrom typing import Literal, Optional, TypeAlias, get_args+from core.message_output import plain_untrusted_textfrom vibe.i18n import t as i18n_tfrom .state_file import write_state_document@@ -140,6 +141,28 @@ def redact_untrusted_text(value: str) -> str:return redacted[: match.end()] + "[redacted]"+# Upper bound on a failure diagnostic Avibe keeps and shows: an upstream's error+# message, or a backend's own report of a failed turn.+_DIAGNOSTIC_DETAIL_CHARS = 400+# C0/C1 controls and every Unicode Bidi_Control character: either can act on+# whatever renders the text (a terminal, a log viewer, a browser).+_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f-\x9f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]")+++def bounded_diagnostic_detail(message: str) -> str | None:+ """The display-safe projection of untrusted failure text, or None when it holds nothing."""++ # Control characters (terminal escapes, BEL) would act on whatever renders+ # the text, a log viewer included; they carry no message content.+ text = " ".join(_CONTROL_CHARACTERS.sub(" ", message).split())+ if not text:+ return None+ text = plain_untrusted_text(redact_untrusted_text(text))+ if len(text) > _DIAGNOSTIC_DETAIL_CHARS:+ text = text[: _DIAGNOSTIC_DETAIL_CHARS - 1].rstrip() + "…"+ return text++def contains_credential_material(value: object) -> bool:rendered = json.dumps(value, ensure_ascii=False, sort_keys=True)return any(pattern.search(rendered) for pattern in _CREDENTIAL_PATTERNS)diff --git a/core/handlers/model_hub/provenance.py b/core/handlers/model_hub/provenance.pyindex b5b05d34c..95021a117 100644--- a/core/handlers/model_hub/provenance.py+++ b/core/handlers/model_hub/provenance.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport asyncioimport jsonimport logging+import reimport secretsimport threadingfrom dataclasses import dataclass, field@@ -12,6 +13,7 @@ from datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any, Callable, Iterable, Literal, Mapping, Optional+from core.agent_core.ai.errors import redact_provider_textfrom config.v2_config import MODEL_HUB_BACKENDS, ModelHubConfigfrom modules.agents.catalog import display_name_for_backendfrom core.os_errors import format_os_errno@@ -36,6 +38,7 @@ from .events import (EVENT_REASON_AUTHORITY,RETIRED_PERSISTED_REASON_DEGRADATIONS,SOURCE_DETAIL_EVENT_REASONS,+ bounded_diagnostic_detail,event_reason_label,)from .resolver import BackendName, ModelHubTurnResolution, parse_model_hub_timestamp, source_eligible_for_backend@@ -43,6 +46,9 @@ from .state_file import write_state_documentSupplyChannel = Literal["native_cli", "hub"]+# A backend's typed failure kind is a member of that backend's closed vocabulary+# (Vibey: ``NAMED_ERROR_KINDS``), never free text: the record keeps only that shape.+_BACKEND_ERROR_KIND = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,63}")SupplyState = Literal["waiting", "interrupted"]ScopeKey = tuple[BackendName, str]logger = logging.getLogger(__name__)@@ -702,7 +708,9 @@ class TurnTrace:ambiguous: bool = Falseterminal_outcome: TurnOutcomeProjectionInput | None = Noneadmission_closed: bool = False- outcome_frozen: bool = False+ # Both causes veto mutation. Backend acknowledgments keep terminal copy+ # readable; explicit user cancellation suppresses live copy as before.+ outcome_frozen: Literal["backend_failure", "user_cancellation"] | None = Nonerecovery_requests: dict[str, dict] = field(default_factory=dict)@property@@ -1075,13 +1083,15 @@ class TurnCorrelationRegistry:self.on_recovery_changed = on_recovery_changeddef _readable_trace(- self, turn_id: str, backend: str | None = None, *, allow_closed: bool = False,+ self, turn_id: str, backend: str | None = None, *,+ allow_closed: bool = False, allow_acknowledged: bool = False,) -> TurnTrace | None:"""Called under the registry lock; attribution is never inferred."""trace = self._traces.get(turn_id)if (- trace is None or trace.ambiguous or trace.outcome_frozen+ trace is None or trace.ambiguous+ or (trace.outcome_frozen and not (allow_acknowledged and trace.outcome_frozen == "backend_failure"))or (trace.admission_closed and not allow_closed)or (backend is not None and trace.agent != backend)):@@ -1098,9 +1108,9 @@ class TurnCorrelationRegistry:def terminal_projection(self, turn_id: str, *, backend: str) -> TurnOutcomeProjectionInput | None:with self._lock:- trace = self._readable_trace(turn_id, backend, allow_closed=True)+ trace = self._readable_trace(turn_id, backend, allow_closed=True, allow_acknowledged=True)if (- trace is None or trace.pending_attempts or trace.terminal_outcome is None+ trace is None or (trace.pending_attempts and not trace.outcome_frozen) or trace.terminal_outcome is Noneor trace.terminal_outcome.outcome in {"served", "canceled"}):return None@@ -2214,11 +2224,13 @@ class TurnCorrelationRegistry:returnif trace.agent == "vibey" and channel != "hub":raise ValueError("vibey attempts require the hub channel")- # Admission supersedes earlier supply failures, not their attempt history.+ # A newly admitted attempt supersedes earlier request/supply+ # terminal facts, not completed fallback history. Otherwise an+ # interrupted successor would be blamed on the preceding request.trace.model_supply_state = Nonetrace.blockers = []- if trace.terminal_outcome is not None and trace.terminal_outcome.outcome in {"no_candidate", "exhausted"}:- trace.terminal_outcome = None+ trace.terminal_error = None+ trace.terminal_outcome = Nonetrace.pending_attempts[request_id] = AttemptIdentity(source_id=source_id,resolved_model_id=resolved_model_id,@@ -2267,12 +2279,27 @@ class TurnCorrelationRegistry:return any(key != request_id for key in trace.pending_attempts)- def fail_hub_attempt(self, turn_id: Optional[str]) -> None:- """Replace a gateway success rejected by the backend terminal result."""+ def fail_hub_attempt(+ self,+ turn_id: Optional[str],+ *,+ error_kind: str | None = None,+ diagnostic: str | None = None,+ ) -> None:+ """Record the backend's terminal failure of a turn the gateway did not end.++ The record holds only what the backend reported: its typed kind and its+ sanitized diagnostic. The backend cannot see the failing request's+ phase, so ``stream_started`` stays unknown.+ """normalized = str(turn_id or "").strip()if not normalized:return+ # Native backends may include credentialed URLs or Basic auth in their+ # transport diagnostic. Redact those before the shared display bound;+ # redacting only upstream-style labeled keys is insufficient here.+ detail = bounded_diagnostic_detail(redact_provider_text(diagnostic or ""))with self._lock:trace = self._traces.get(normalized)if trace is None or trace.outcome_frozen:@@ -2280,8 +2307,15 @@ class TurnCorrelationRegistry:if self.terminal_projection(normalized, backend=trace.agent) is not None:# The native failure is reporting the gateway's completed# failure, not rejecting content that the gateway served.+ # Protect that acknowledged terminal while retaining its copy+ # for callers that read it after reporting backend failure.+ trace.outcome_frozen = "backend_failure"+ return+ if trace.terminal_error is not None and not trace.pending_attempts:+ # The same for a completed failure that has no copy. A request+ # still open after it would supersede it, but none is.+ trace.outcome_frozen = "backend_failure"return- trace.outcome_frozen = Trueidentity = trace.pending_attemptpayload = (identity.payload()@@ -2291,12 +2325,21 @@ class TurnCorrelationRegistry:if payload is None or payload.get("channel") != "hub":return# The backend rejected the turn's result, not one request's.+ # A callback without an attributed payload must not freeze out+ # gateway evidence that has yet to arrive.+ trace.outcome_frozen = "backend_failure"trace.pending_attempts.clear()trace.served = Nonetrace.terminal_error = {**payload,- "reason": "protocol_error",- "stream_started": True,+ "reason": "backend_error",+ "stream_started": None,+ **(+ {"error_kind": error_kind}+ if isinstance(error_kind, str) and _BACKEND_ERROR_KIND.fullmatch(error_kind)+ else {}+ ),+ **({"backend_error_detail": detail} if detail else {}),}def finish_attempt(@@ -2381,7 +2424,7 @@ class TurnCorrelationRegistry:if settled_by in _USER_CANCELLATIONS:# Preserve the exact facts present at Stop. Teardown may expose# a later producer success, but cannot change who ended the turn.- trace.outcome_frozen = True+ trace.outcome_frozen = "user_cancellation"if had_recovery:self._notify_recovery_changed(turn_id)diff --git a/docs/plans/model-hub-backend-failure-facts.md b/docs/plans/model-hub-backend-failure-facts.mdnew file mode 100644index 000000000..f28934825--- /dev/null+++ b/docs/plans/model-hub-backend-failure-facts.md@@ -0,0 +1,144 @@+# Backend-reported Model Hub terminal failures++## Change contract (2026-10-13)++A source-origin Vibey `stalled` failure reached the Hub failure bridge as text+only. `fail_hub_attempt` wrote `protocol_error` and `stream_started: true`, even+when neither fact was established. This change corrects recorded diagnostics;+it does not change retries, timeouts, route selection, Source health, or Stop.++- The existing BaseAgent -> ModelHubRuntimeRouter -> correlation failure bridge+ accepts an optional structured `error_kind`. Vibey supplies its existing+ `RunEnded.cause.kind` only for source-origin failures. Do not classify by+ diagnostic substrings, infer phase from retryability, or use a prior response.+- Backend-only fallback records use `terminal_error.reason = backend_error`,+ optional `error_kind`, and optional `backend_error_detail`. A `stalled` kind+ remains `stalled`, including in copied JSON and localized detail presentation.+ Other backends without typed facts use the neutral backend-error reason.+- `backend_error_detail` is a credential-redacted, control/bidi-stripped,+ whitespace-collapsed, at-most-400-character diagnostic, not an upstream API+ answer. Reuse the existing Model Hub diagnostic sanitizer by moving it to+ an appropriate shared owner rather than maintaining a weaker second policy.+- `stream_started` may be null on a backend-error record: the callback has no+ exact failing-request phase evidence. Unknown is neither false nor true.+ All gateway-owned terminal reasons keep their existing boolean phase contract.+ Propagating additional provider phase telemetry is not required for this fix.+- An already authoritative gateway failure (terminal projection OR concrete+ terminal_error, even when that reason has no copy projection) must survive a+ backend notification. Keep Stop/frozen-outcome and late-success protections.+- Existing released provenance records remain readable without the new fields.+ Do not rewrite old history by guessing what really happened. Update the owning+ JSON schema, TypeScript mirror, API contract, and English/Chinese UI copy.+- Failure details label the backend diagnostic distinctly, never claim it is+ upstream response text, and show the typed kind (a localized stall label for+ stalled; other kinds may be displayed as diagnostic machine identifiers).++## Validation obligations++The primary regression must exercise the real Vibey adapter and shared failure+bridge into persisted provenance: existing coverage stubs out that bridge and+only checks notice copy. A scripted stalled provider must not become a protocol+error, and the old implementation must fail the new assertion.++Extend the existing correlation tests for authoritative gateway errors without+copy projections, generic native-backend notifications, preservation of late+success/Stop rules, and historical reads. Exercise non-ASCII diagnostics and+redaction. Add UI consumption coverage distinguishing backend diagnostics from+upstream API facts; run focused tests, changed-file Ruff, UI build, exact-head+Codex review and full CI. No production-state writes, deployment, release,+service restart, or merge is authorized.++## Implementation and owner verification++- The main regression and the updated late-success/correlation expectations+ fail on the base implementation for the intended reason; the main regression+ uses the real shared bridge, not a mock of its classification.+- The implementation preserves an existing concrete gateway terminal only when+ no pending request supersedes it. It freezes that terminal before the late+ success can replace it. The regression covers both branches and mutation of+ the freeze.+- Owner review found the old upstream-text redactor insufficient for newly+ retained native diagnostics: credentialed URLs, Basic authentication, and+ JWT-shaped tokens survived. Reuse `redact_provider_text` on backend diagnostics+ before the shared display sanitizer. Three consuming cases fail without this+ step and all pass with it. The existing upstream message path is unchanged.+- The stall label describes the observation (a model request timed out waiting+ for data), not an unproved claim about which remote component stopped.+- Focused owner verification includes the Vibey-to-provenance regression,+ gateway terminal preservation and native diagnostics, released-record reads,+ all of `test_model_hub_provenance.py`, late-success/Stop tests, 32 UI tests,+ Model Hub authority closure, changed-file Ruff, and `npm run build`.+- Scenario: `MH-TURN-RESULT-001`. Repository-wide CI and exact-head Codex review+ remain the asynchronous delivery gates. No live provider/Incus regression was+ run; this change introduces no model request or retry behavior.++## Independent audit follow-up++Read-only audit of implementation `53e7da45a` found three boundary defects;+owner reproduced them on the pushed `22b9342c4` before editing. Four new checks+failed for their intended reasons (one case has two projection variants):++1. A newly admitted request retained the preceding request's terminal error+ and copy projection, so an FSM drop could report the wrong Source. New+ actual attempt admission now atomically retires both terminal artifacts,+ while retaining fallback history and the existing frozen/Stop veto.+2. An unattributed backend callback froze the trace before checking for a+ pending/served payload, hiding subsequent gateway evidence. Such a callback+ is now a no-op; only a concrete terminal or a recordable backend error freezes.+3. The schema allowed boolean phase on backend_error despite writing null.+ Its positive branch now requires null; both boolean counterexamples fail.++The owner chose these local corrections, not a lifecycle or retry redesign.+The three changed correlation suites pass **889 tests**, and the two Vibey+adapter/context suites pass **158 tests** after the changes. Ruff and authority+closure pass. GitHub Codex passed the previous head `22b9342c4`; a new exact-head+review is still required after this correction. No GitHub findings-bearing head+has recurred; the independent audit findings were reconciled together once.++## Circuit-breaker decision: terminal evidence ownership++GitHub review `5501580777` on `08f641f42` raised two findings (threads+`PRRT_kwDOPbFPYs6ry_u_` and `PRRT_kwDOPbFPYs6ry_vK`): backend-error schema rows+still admitted upstream-only fields, and acknowledging a gateway projection did+not protect it from a request arriving before Turn settlement. The latter is+the same ownership class as the independent review of `53e7da45a`; the schema+finding also revisits the closed-shape class. Owner paused editing, fetched the+complete review/CI inventory, and re-examined every mutation/read of the trace.++Root cause: `outcome_frozen` conflates a mutation veto with suppression of live+copy reads. Freezing an acknowledged projection as an undifferentiated boolean+would hide the exact error copy callers still need. Leaving it unfrozen permits+later admission to erase it. The existing freeze therefore needs to identify+its cause, not an additional terminal ledger or a retry-policy change.++Owner decision: retain the existing freeze gate, distinguish backend-failure+acknowledgment from explicit user cancellation, and allow only the former's+frozen terminal projection to remain readable. Recovery reads and all writes+remain blocked. Once a backend acknowledges a terminal, later requests cannot+replace it; before acknowledgment, already pending/new actual attempts retain+the supersession rule. Empty callbacks still freeze nothing. Exercise both+projection and concrete terminal paths, late admission/success/interruption,+wrong backend, ordinary closure, and Stop. Backend-error schema shapes exclude+upstream-only fields, and the TypeScript/display consumer follows that same+partition. This is the smallest local correction; no lifecycle, Source-health,+or retry architecture is changed. New regressions must fail before these edits.++Validation of that decision: the four projection/closure combinations, the+backend-only shape check, and the malformed UI record each fail on `08f641f42`.+After correction, 957 correlation/retention/failure-presentation tests, 154 retry+lifecycle tests, 158 Vibey adapter/context tests, and 33 UI tests pass; build,+Ruff and authority closure pass. The schema negative constraints are object-only+so the existing null terminal on served/exhausted/canceled/no-candidate outcomes+continues to validate. Full CI on `08f641f42` was green; the corrected head still+requires its own Codex verdict and CI.++## Known-by-design++- Records remain generation 11 as with earlier additive diagnostic additions;+ old records are not rewritten. Downgrading to an older UI may show an unknown+ label for a new backend-error record, without making stored data unreadable.+- Only Vibey currently supplies a typed kind, from `NAMED_ERROR_KINDS`; generic+ native callbacks retain no guessed kind. The shared identifier-shape check is+ not a classifier or authority for future backend kind vocabularies.+- The 600-second silence bound, gateway idle recovery, Source health and routing,+ and the separate raw warning-log format are outside this change.diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.mdindex 6e26e33e3..d4f409f8a 100644--- a/docs/plans/model-hub-contracts/api.md+++ b/docs/plans/model-hub-contracts/api.md@@ -1357,6 +1357,34 @@ of control and bidi characters, whitespace-collapsed, and bounded to 400 charactThey are omitted when no message was observed and in older records, and are neverread by classification.+`terminal_error.reason: backend_error` is an agent backend's own report of a failed+turn that the gateway did not end itself+(`docs/plans/model-hub-backend-failure-facts.md`). The shared failure bridge records+it on the request still open, else on the served attempt. A gateway terminal already+recorded for the turn, a copy projection or a concrete `terminal_error` with or+without one, is kept instead, unless a later request was still open when the backend+failed. Admission of a new actual attempt atomically retires the preceding+request's terminal error and copy projection, keeping its fallback history.+An unattributed callback with neither a payload nor a concrete gateway terminal+is a no-op; it must not freeze out later gateway evidence. The supersession rule+applies before backend acknowledgment. Once the backend acknowledges an existing+gateway terminal, that terminal is frozen against later requests while its exact+copy remains readable; explicit user cancellation continues to suppress live copy.+Backend-error records prohibit `http_status`, `upstream_error_code`, and+`upstream_detail` (including null-valued members), so no upstream verdict can be+presented as a backend-only fact. A null `terminal_error` remains valid for all+other turn outcomes.+The record carries only the backend's facts: optional `error_kind`, the+backend's typed failure kind from its closed vocabulary (Vibey supplies its named+`AgentError` kind, such as `stalled`, for a failure the served Source produced; other+backends supply none), and optional `backend_error_detail`, the backend's diagnostic+under the same redaction and bound as `upstream_detail`, which is never an upstream+answer. Its `stream_started` is null because the reporter cannot see the failing+request's phase; every other reason keeps its boolean. It changes no classification,+retry, Source health or Stop rule. Records written before this reason existed,+including backend rejections recorded as `protocol_error` with `stream_started: true`,+are read as written.+The dialog independently reads this projection on demand and labels it "Latest recordedturn" / "最近已记录回合". Its error panel and details action use the same structured record,recorded timestamp and exact historical Source/model identifiers. Deleted Sources keepdiff --git a/docs/plans/model-hub-contracts/turn-provenance.schema.json b/docs/plans/model-hub-contracts/turn-provenance.schema.jsonindex 43522fd7c..4730301fb 100644--- a/docs/plans/model-hub-contracts/turn-provenance.schema.json+++ b/docs/plans/model-hub-contracts/turn-provenance.schema.json@@ -172,10 +172,26 @@"protocol_error","tool_incompatible","stream_interrupted",- "engine_down"- ]+ "engine_down",+ "backend_error"+ ],+ "description": "Who ended the turn and how. backend_error is the backend's own report of a failed turn the gateway did not end itself; it carries only what the backend observed, never an upstream verdict or a guessed gateway reason."+ },+ "stream_started": {+ "type": ["boolean", "null"],+ "description": "Whether user-visible model output began on the failing request. Null only on backend_error, whose reporter cannot see that request's phase; unknown is neither false nor true."+ },+ "error_kind": {+ "type": "string",+ "pattern": "^[A-Za-z][A-Za-z0-9_]{0,63}$",+ "description": "Optional backend_error fact: the reporting backend's typed failure kind from its closed vocabulary (Vibey: its named AgentError kinds, for example stalled). Never free text, an unexpected exception class, or an upstream code. Absent when the backend has no typed kind and in historical records."+ },+ "backend_error_detail": {+ "type": "string",+ "minLength": 1,+ "maxLength": 400,+ "description": "Optional backend_error fact: the backend's own diagnostic for the failed turn, under the same redaction and bound as upstream_detail. It is not an upstream API answer. Display only; absent when the backend gave none and in historical records."},- "stream_started": { "type": "boolean" },"local_error_detail": {"type": "string","description": "Optional local OS errno and system message. Produced from numeric errno only; never contains exception paths, upstream text, credentials, or request bodies. Historical records may omit it."@@ -231,6 +247,33 @@"declared_reasoning_efforts": ["stripped_reasoning_efforts"]},"allOf": [+ {+ "if": {+ "properties": { "reason": { "const": "backend_error" } },+ "required": ["reason"]+ },+ "then": {+ "properties": { "stream_started": { "const": null } },+ "not": {+ "type": "object",+ "anyOf": [+ { "required": ["http_status"] },+ { "required": ["upstream_error_code"] },+ { "required": ["upstream_detail"] }+ ]+ }+ },+ "else": {+ "properties": { "stream_started": { "type": "boolean" } },+ "not": {+ "type": "object",+ "anyOf": [+ { "required": ["error_kind"] },+ { "required": ["backend_error_detail"] }+ ]+ }+ }+ },{"if": {"properties": { "reason": { "const": "stream_interrupted" } },diff --git a/modules/agents/base.py b/modules/agents/base.pyindex 9105e131a..5ab746b69 100644--- a/modules/agents/base.py+++ b/modules/agents/base.py@@ -262,15 +262,20 @@ class BaseAgent(ABC):return None- async def record_model_hub_native_failure(self, context: Any, diagnostic: str) -> bool:- """Persist source cooldown state when a terminal turn proves source loss."""+ async def record_model_hub_native_failure(+ self, context: Any, diagnostic: str, *, error_kind: str | None = None+ ) -> bool:+ """Persist source cooldown state when a terminal turn proves source loss.++ ``error_kind`` is the backend's typed failure kind, for a backend that has one.+ """router = getattr(self.controller, "model_hub_runtime", None)recorder = getattr(router, "record_native_failure", None)if not callable(recorder):return Falsetry:- return bool(await recorder(context, diagnostic))+ return bool(await recorder(context, diagnostic, error_kind=error_kind))except Exception:logger.warning("Failed to record Model Hub native cooldown", exc_info=True)return Falsediff --git a/modules/agents/model_hub.py b/modules/agents/model_hub.pyindex 7090b1a0c..f111502ee 100644--- a/modules/agents/model_hub.py+++ b/modules/agents/model_hub.py@@ -1424,11 +1424,15 @@ class ModelHubRuntimeRouter:self._emit_transition(launch, config)return launch- async def record_native_failure(self, context: Any, diagnostic: str) -> bool:+ async def record_native_failure(+ self, context: Any, diagnostic: str, *, error_kind: str | None = None+ ) -> bool:"""Record a terminal source failure for the next per-turn resolution.The method name is retained for existing backend call sites; Hub launchesuse the same cooldown state so the next turn can select a backup source.+ A Hub turn's record keeps the backend's typed ``error_kind``, when it+ has one, beside its diagnostic."""launch = launch_for_context(context)@@ -1443,7 +1447,9 @@ class ModelHubRuntimeRouter:)or "").strip()- self.turn_gateway.correlation.fail_hub_attempt(turn_id)+ self.turn_gateway.correlation.fail_hub_attempt(+ turn_id, error_kind=error_kind, diagnostic=diagnostic+ )setattr(context, _CONTEXT_FAILURE_RECORDED_ATTR, True)return Falsedecision: ResolutionDecision | Nonediff --git a/modules/agents/vibey/agent.py b/modules/agents/vibey/agent.pyindex 69ccdfb99..e89116f19 100644--- a/modules/agents/vibey/agent.py+++ b/modules/agents/vibey/agent.py@@ -40,6 +40,7 @@ from sqlalchemy.engine import Enginefrom config import pathsfrom core.agent_core.agent.events import (+ NAMED_ERROR_KINDS,AgentError,AgentEvent,CompactionFinished,@@ -153,6 +154,11 @@ def _loggable(text: str, limit: int = _LOGGED_DIAGNOSTIC_CHARS) -> str:return line if len(line) <= limit else line[: limit - 1] + "…"+def _hub_error_kind(kind: Optional[str]) -> Optional[str]:+ """The failure's kind as Model Hub records it: a kind the engine names, never an unexpected exception's class."""+ return kind if kind in NAMED_ERROR_KINDS else None++@dataclassclass _Run:"""One Turn's state, published before its first await (``handle_message``)."""@@ -655,7 +661,9 @@ class VibeyAgent(BaseAgent):# of its own (``_display_source``), its row carries the explanation.body = error_text(kind or "empty_response", self._language(), reason=reason)if failed and source:- await self.record_model_hub_native_failure(context, diagnostic or (kind or "failed final"))+ await self.record_model_hub_native_failure(+ context, diagnostic or (kind or "failed final"), error_kind=_hub_error_kind(kind)+ )if body.strip():# The same result path as the other backends; the row is already the message.await self.emit_result_message(@@ -1217,7 +1225,9 @@ class VibeyAgent(BaseAgent):other backends; any other (a context that cannot fit, a Stop, a local error) leaves Model Hub's record as it is."""if source:- await self.record_model_hub_native_failure(request.context, diagnostic or (kind or "error"))+ await self.record_model_hub_native_failure(+ request.context, diagnostic or (kind or "error"), error_kind=_hub_error_kind(kind)+ )output = terminal_output_for(request)if kind:# The kind, never the diagnostic, so a failed Turn can be diagnosed from its notice row.diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 794b18193..9a79fcfcb 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -1594,6 +1594,10 @@ scenarios:kind: turn_lifecyclelayer: scenariotest: tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py::test_mh_turn_outcomes_and_cancel_are_closed_provenance_products+ related_tests:+ - tests/test_vibey_agent.py::test_model_hub_records_a_source_stall_as_the_backend_reported_it+ - tests/test_model_hub_l3.py::test_backend_failure_keeps_the_gateway_terminal_error_no_later_request_superseded+ - tests/test_model_hub_l3.py::test_backend_terminal_failure_overrides_gateway_success- id: MH-TURN-COPY-001name: Every turn decision resolves to its authoritative backend copy key or silencestatus: covereddiff --git a/tests/test_claude_cli_path.py b/tests/test_claude_cli_path.pyindex bef6e2470..d0c73f48a 100644--- a/tests/test_claude_cli_path.py+++ b/tests/test_claude_cli_path.py@@ -2519,7 +2519,7 @@ def test_claude_startup_failure_is_recorded_before_scope_retirement(source_id="src_native01",)- async def record_native_failure(self, context, diagnostic):+ async def record_native_failure(self, context, diagnostic, *, error_kind=None):events.append(("record", diagnostic))return Falsediff --git a/tests/test_model_hub_l3.py b/tests/test_model_hub_l3.pyindex 55451073c..de3cdc7eb 100644--- a/tests/test_model_hub_l3.py+++ b/tests/test_model_hub_l3.py@@ -5457,8 +5457,24 @@ def test_fsm_drop_overrides_a_completed_gateway_attempt(tmp_path: Path) -> None:_assert_valid("turn-provenance.schema.json", record)+ "error_kind,recorded,diagnostic,expected_detail",+ [+ # A backend with no typed kind (Claude, Codex, OpenCode) reports its failure neutrally.+ (None, {}, "API Error: 上游连接已关闭\x07 api_key=sk-proj-detail0secret0value",+ "API Error: 上游连接已关闭 api_key=[redacted]"),+ ("stalled", {"error_kind": "stalled"}, "provider sent no data for 600s", "provider sent no data for 600s"),+ # Only a kind's shape is ever kept: text in the kind field is dropped, never persisted.+ ("Authorization: Bearer sk-proj-kind0secret0value", {}, "", None),+ # Newly retained native diagnostics must not disclose transport credentials.+ (None, {}, "connection failed to https://alice:[email protected]/v1",+ "connection failed to https://example.invalid/v1"),+ (None, {}, "Authorization: Basic ZHVtbXk6ZHVtbXk=", "Authorization: Basic [redacted]"),+ (None, {}, "request failed: eyJkdW1teSI6MX0.eyJkdW1teSI6MX0.ZHVtbXk", "request failed: [redacted]"),+ ],+)def test_backend_terminal_failure_overrides_gateway_success(- tmp_path: Path,+ tmp_path: Path, error_kind: str | None, recorded: dict, diagnostic: str, expected_detail: str | None) -> None:store = BoundedProvenanceStore(tmp_path / "backend-terminal.json")registry = TurnCorrelationRegistry(store)@@ -5473,7 +5489,11 @@ def test_backend_terminal_failure_overrides_gateway_success(decision=classify_outcome(success),)- registry.fail_hub_attempt("turn_backend_terminal")+ registry.fail_hub_attempt(+ "turn_backend_terminal",+ error_kind=error_kind,+ diagnostic=diagnostic,+ )registry.settle("turn_backend_terminal",settled_by=SETTLED_BY_TERMINAL_RESULT,@@ -5484,10 +5504,107 @@ def test_backend_terminal_failure_overrides_gateway_success(assert record is not Noneassert record["outcome"] == "failed_terminal"assert record["served"] is None- assert record["terminal_error"]["reason"] == "protocol_error"+ # The backend saw its turn fail, not the request's phase or an upstream verdict.+ assert record["terminal_error"] == {+ "source_id": "src_primary01",+ "configured_model_id": "shared-model",+ "channel": "hub",+ "reason": "backend_error",+ "stream_started": None,+ **recorded,+ **({"backend_error_detail": expected_detail} if expected_detail else {}),+ }+ _assert_valid("turn-provenance.schema.json", record)+ if error_kind == "stalled":+ # Backend-only callbacks have no phase evidence, so neither boolean is+ # valid even when the rest of a record is otherwise unchanged.+ contradictory = (+ {"stream_started": False}, {"stream_started": True},+ {"http_status": 400}, {"http_status": None},+ {"upstream_error_code": "invalid_request_error"}, {"upstream_error_code": None},+ {"upstream_detail": "not an upstream response"},+ )+ for fields in contradictory:+ invalid = {**record, "terminal_error": {**record["terminal_error"], **fields}}+ with pytest.raises(AssertionError):+ _assert_valid("turn-provenance.schema.json", invalid)++[email protected]("later_request", [False, True])+def test_backend_failure_keeps_the_gateway_terminal_error_no_later_request_superseded(+ tmp_path: Path, later_request: bool+) -> None:+ """A protocol failure has no copy projection; the gateway's exact record of it still outlives the backend's report."""+ store = BoundedProvenanceStore(tmp_path / "gateway-terminal.json")+ registry = TurnCorrelationRegistry(store)+ exact_turn = _begin_hub_attempt(registry, turn_id="turn_gateway_terminal")+ success = _outcome(RawOutcomeKind.SUCCESS)+ registry.finish_attempt(exact_turn, outcome=success, decision=classify_outcome(success))+ registry.begin_attempt(+ exact_turn, source_id="src_backup01", resolved_model_id="shared-model", channel="hub", via_mapping=False,+ )+ failure = _outcome(RawOutcomeKind.PROTOCOL_ERROR, status=200, source_id="src_backup01")+ registry.finish_attempt(exact_turn, outcome=failure, decision=classify_outcome(failure))+ gateway_terminal = dict(registry._traces[exact_turn].terminal_error)+ assert registry.terminal_projection(exact_turn, backend="codex") is None+ if later_request:+ registry.begin_attempt(+ exact_turn, source_id="src_later001", resolved_model_id="shared-model", channel="hub", via_mapping=False,+ )++ registry.fail_hub_attempt(exact_turn, diagnostic="Codex exited")+ # Nothing after the backend's report replaces the turn's record, not even a later success.+ late = _outcome(RawOutcomeKind.SUCCESS, source_id="src_late0001")+ registry.begin_attempt(+ exact_turn, source_id="src_late0001", resolved_model_id="shared-model", channel="hub", via_mapping=False,+ request_id="late",+ )+ registry.finish_attempt(exact_turn, outcome=late, decision=classify_outcome(late), request_id="late")+ registry.settle(exact_turn, settled_by=SETTLED_BY_TERMINAL_RESULT, ts=NOW.isoformat())++ record = store.get(exact_turn)+ assert record is not None and record["outcome"] == "failed_terminal"+ if later_request:+ # The request still open when the backend failed is the one it reports on.+ assert record["terminal_error"] == {+ "source_id": "src_later001", "configured_model_id": "shared-model", "channel": "hub",+ "reason": "backend_error", "stream_started": None, "backend_error_detail": "Codex exited",+ }+ else:+ assert gateway_terminal["reason"] == "protocol_error" and gateway_terminal["stream_started"] is False+ assert record["terminal_error"] == gateway_terminal_assert_valid("turn-provenance.schema.json", record)+def test_released_backend_rejection_records_read_unchanged(tmp_path: Path) -> None:+ """Records the earlier bridge wrote stay as written: no reason or phase is re-derived on read."""+ released = {+ "contract_version": 11,+ "turn_id": "turn_released_rejection",+ "ts": NOW.isoformat(),+ "agent": "vibey",+ "requested_model_id": "shared-model",+ "outcome": "failed_terminal",+ "failed_attempts": [],+ "served": None,+ "terminal_error": {+ "source_id": "src_primary01",+ "configured_model_id": "shared-model",+ "channel": "hub",+ "reason": "protocol_error",+ "stream_started": True,+ },+ "canceled_attempt": None,+ "model_supply_state": None,+ "blockers": [],+ }+ path = tmp_path / "provenance.json"+ path.write_text(json.dumps([released]), encoding="utf-8")++ assert BoundedProvenanceStore(path).get("turn_released_rejection") == released+ _assert_valid("turn-provenance.schema.json", released)++def test_lost_terminal_retains_completed_fallback_attempts(tmp_path: Path,) -> None:diff --git a/tests/test_model_hub_provenance.py b/tests/test_model_hub_provenance.pyindex 481ff5540..037ce0f97 100644--- a/tests/test_model_hub_provenance.py+++ b/tests/test_model_hub_provenance.py@@ -16,7 +16,7 @@ from core.handlers.model_hub.provenance import ()from core.handlers.model_hub.rpc import dispatch_model_hub_rpcfrom core.handlers.model_hub.service import ModelHubError-from core.run_settlement import SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED, SETTLED_BY_NO_TERMINAL_RESULTfrom modules.agents.model_hub import ModelHubLaunch, ModelHubRuntimeRouter, bind_launchfrom tests.test_model_hub_resolution import _config, _service, _sourcefrom tests.test_model_hub_routing_modes import MODEL, _loaded_catalog_config, _sparse_config@@ -97,16 +97,77 @@ def test_recovery_callbacks_are_material_post_lock_and_snapshots_are_copies(tmp_assert not registry.store.path.exists()-def test_native_hub_failure_keeps_the_gateway_terminal_projection_readable(tmp_path):+def test_backend_callback_without_an_attempt_does_not_freeze_gateway_evidence(tmp_path):registry = _live_registry(tmp_path)+ registry.fail_hub_attempt("turn-live", diagnostic="backend exited before attribution")projection = produce_turn_outcome("turn.engine_down")registry.record_turn_outcome("turn-live", projection)- registry.fail_hub_attempt("turn-live")assert registry.terminal_projection("turn-live", backend="codex") == projectionassert not registry._traces["turn-live"].outcome_frozen- registry.close_turn_admission("turn-live", settled_by=SETTLED_BY_STOPPED)++[email protected]("with_projection", [False, True])+def test_new_attempt_retires_previous_request_terminal_before_fsm_drop(tmp_path, with_projection):+ registry = _live_registry(tmp_path)+ registry.begin_attempt(+ "turn-live", source_id="src_previous1", resolved_model_id=MODEL, channel="hub", via_mapping=False,+ )+ outcome = RawCallOutcome(+ kind=RawOutcomeKind.HTTP_ERROR if with_projection else RawOutcomeKind.PROTOCOL_ERROR,+ http_status=400 if with_projection else 200,+ error_code="invalid_request_error" if with_projection else None,+ redacted_message=None, stream_started=False, model_id=MODEL, source_id="src_previous1",+ )+ registry.finish_attempt("turn-live", outcome=outcome, decision=classify_outcome(outcome))+ if with_projection:+ registry.record_turn_outcome("turn-live", produce_turn_outcome("turn.request_nonfallback"))+ registry.begin_attempt(+ "turn-live", source_id="src_successor1", resolved_model_id=MODEL, channel="hub", via_mapping=False,+ )+ assert registry._traces["turn-live"].terminal_outcome is None+ registry.settle("turn-live", settled_by=SETTLED_BY_NO_TERMINAL_RESULT)+ terminal = registry.store.get("turn-live")["terminal_error"]+ assert terminal["source_id"] == "src_successor1"+ assert terminal["reason"] == "stream_interrupted"++[email protected]("late_outcome", [RawOutcomeKind.SUCCESS, RawOutcomeKind.NETWORK_ERROR])[email protected]("closure", [SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED])+def test_native_hub_failure_keeps_the_gateway_terminal_projection_readable(tmp_path, late_outcome, closure):+ registry = _live_registry(tmp_path)+ registry.begin_attempt(+ "turn-live", source_id="src_primary01", resolved_model_id=MODEL, channel="hub", via_mapping=False,+ )+ failure = RawCallOutcome(+ kind=RawOutcomeKind.HTTP_ERROR, http_status=400, error_code="invalid_request_error",+ redacted_message=None, stream_started=False, model_id=MODEL, source_id="src_primary01",+ )+ registry.finish_attempt("turn-live", outcome=failure, decision=classify_outcome(failure))+ projection = produce_turn_outcome("turn.request_nonfallback")+ registry.record_turn_outcome("turn-live", projection)+ terminal = dict(registry._traces["turn-live"].terminal_error)registry.fail_hub_attempt("turn-live")- assert registry.terminal_projection("turn-live", backend="codex") is None+ assert registry.terminal_projection("turn-live", backend="codex") == projection++ # After the backend has acknowledged the terminal, late admission cannot+ # clear it, and late success/interruption cannot contradict the notice.+ registry.begin_attempt(+ "turn-live", source_id="src_late0001", resolved_model_id=MODEL, channel="hub", via_mapping=False,+ )+ late = replace(failure, kind=late_outcome, error_code=None, stream_started=True, source_id="src_late0001")+ registry.finish_attempt("turn-live", outcome=late, decision=classify_outcome(late))+ registry.record_turn_outcome("turn-live", produce_turn_outcome("turn.served"))+ assert registry.terminal_projection("turn-live", backend="codex") == projection+ assert registry.terminal_projection("turn-live", backend="claude") is None+ registry.close_turn_admission("turn-live", settled_by=closure)+ assert registry.terminal_projection("turn-live", backend="codex") == (+ None if closure == SETTLED_BY_STOPPED else projection+ )+ registry.fail_hub_attempt("turn-live")+ registry.settle("turn-live", settled_by=closure)+ record = registry.store.get("turn-live")+ assert record["outcome"] == "failed_terminal"+ assert record["terminal_error"] == terminal@pytest.mark.parametrize("terminal", ["exhausted", "no_candidate"])diff --git a/tests/test_model_hub_retention.py b/tests/test_model_hub_retention.pyindex 21f67d75a..979077be9 100644--- a/tests/test_model_hub_retention.py+++ b/tests/test_model_hub_retention.py@@ -381,7 +381,7 @@ def test_finalization_preserves_stop_and_backend_failure_before_late_success(tmpassert record["outcome"] == expectedassert record["served"] is Noneif backend_rejection:- assert record["terminal_error"]["reason"] == "protocol_error"+ assert record["terminal_error"]["reason"] == "backend_error"elif reason != SETTLED_BY_STOPPED:assert record["terminal_error"]["reason"] == "stream_interrupted"else:diff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.pyindex 2283da229..cb33fbe66 100644--- a/tests/test_model_hub_runtime.py+++ b/tests/test_model_hub_runtime.py@@ -42,7 +42,7 @@ from core.handlers.model_hub.classification import (terminal_outcome_category,)from core.handlers.model_hub.errors import ModelDiscoveryError-from core.handlers.model_hub.events import redact_untrusted_text+from core.handlers.model_hub.events import bounded_diagnostic_detail, redact_untrusted_textfrom core.handlers.model_hub.request import ModelHubRequestfrom core.handlers.model_hub.stream_wire import (ProtocolObservation,@@ -7902,7 +7902,8 @@ def test_engine_upstream_detail_is_bounded_and_keeps_non_ascii_text() -> None:detail = client_module._upstream_error_detail(payload, (("error",),))assert detail is not None- assert len(detail) == client_module._UPSTREAM_DETAIL_CHARS+ # The persisted contract's bound (turn-provenance.schema.json maxLength).+ assert len(detail) == 400assert detail.startswith("模型不可用") and detail.endswith("…")@@ -7922,7 +7923,7 @@ def test_engine_upstream_detail_is_bounded_and_keeps_non_ascii_text() -> None:],)def test_engine_upstream_detail_redacts_labeled_opaque_secrets(message: str, expected: str) -> None:- assert client_module._bounded_upstream_detail(message) == expected+ assert bounded_diagnostic_detail(message) == expecteddef test_engine_upstream_detail_redaction_stays_linear_on_hostile_labels() -> None:@@ -7933,7 +7934,7 @@ def test_engine_upstream_detail_redaction_stays_linear_on_hostile_labels() -> Nodef test_engine_upstream_detail_cannot_ping_the_channel_it_is_rendered_into() -> None:- detail = client_module._bounded_upstream_detail(+ detail = bounded_diagnostic_detail("ask @everyone or @ops_lead, <@123> <@&456> <!here> <#C1>")@@ -7953,7 +7954,7 @@ def test_engine_upstream_detail_cannot_ping_the_channel_it_is_rendered_into() ->],)def test_engine_upstream_detail_cannot_hide_a_link_behind_trusted_copy(message: str) -> None:- detail = client_module._bounded_upstream_detail(message)+ detail = bounded_diagnostic_detail(message)assert detail is not Noneassert "](" not in detail.replace(" ", "")@@ -7963,7 +7964,7 @@ def test_engine_upstream_detail_cannot_hide_a_link_behind_trusted_copy(message:def test_engine_upstream_detail_keeps_email_addresses_usable() -> None:- assert client_module._bounded_upstream_detail("Contact [email protected], cc @ops or x@everyone") == (+ assert bounded_diagnostic_detail("Contact [email protected], cc @ops or x@everyone") == ("Contact [email protected], cc @\u200bops or x@\u200beveryone")diff --git a/tests/test_vibey_agent.py b/tests/test_vibey_agent.pyindex 2d64b7127..49d618492 100644--- a/tests/test_vibey_agent.py+++ b/tests/test_vibey_agent.py@@ -540,7 +540,7 @@ async def test_a_setup_failure_after_the_route_resolved_fails_the_turn_and_not_treported: list[str] = []- async def record_native_failure(context, diagnostic) -> bool:+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:reported.append(diagnostic)return False@@ -1166,7 +1166,7 @@ async def test_a_failed_run_shows_localized_copy_and_a_refusal_shows_its_explana)reported: list[str] = []- async def record_native_failure(context, diagnostic) -> bool:+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:reported.append(diagnostic)return False@@ -1240,7 +1240,7 @@ async def test_a_run_failure_is_reported_to_model_hub_like_the_other_backends(enreported: list[str] = []- async def record_native_failure(context, diagnostic) -> bool:+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:reported.append(diagnostic)return False@@ -1255,6 +1255,49 @@ async def test_a_run_failure_is_reported_to_model_hub_like_the_other_backends(enassert len(reported) == 1 and harness.controller.terminals[-1]["is_error"] is True+async def test_model_hub_records_a_source_stall_as_the_backend_reported_it(engine, session, tmp_path, published) -> None:+ """MH-TURN-RESULT-001: the Turn's Model Hub record holds what Vibey observed through the shared failure bridge:+ the stall's kind and sanitized diagnostic, never a guessed protocol error or failing-request output phase."""+ from core.agent_core.ai.provider import ProviderError+ from core.handlers.model_hub.provenance import BoundedProvenanceStore, TurnCorrelationRegistry+ from core.run_settlement import SETTLED_BY_TERMINAL_RESULT+ from modules.agents.model_hub import ModelHubRuntimeRouter+ from tests.test_model_hub_l3 import _assert_valid++ stall = ProviderError("stalled", "provider sent no data for 600s\n\u202e上游 key=sk-proj-stall0secret0value", False)+ harness = _Harness(engine, tmp_path, "telegram", [[stall]])+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "provenance.json"))+ router = ModelHubRuntimeRouter(+ service=SimpleNamespace(), turn_gateway=SimpleNamespace(correlation=registry), overlay_path=tmp_path / "overlay.json"+ )+ # Only the route is the harness's; the failure goes through the shared router into the real registry.+ router.resolve = harness.controller._resolve+ harness.controller.model_hub_runtime = router+ request = harness.request("write the page")+ turn_id = _turn(request.context)+ token = registry.credentials("vibey", f"vibey:{harness.session_id}", turn_id)+ registry.begin_gateway_request(backend="vibey", token=token, requested_model_id="test-model")+ registry.begin_attempt(+ turn_id, source_id="src_stalled01", resolved_model_id="test-model", channel="hub", via_mapping=False+ )++ await harness.agent.handle_message(request)+ registry.settle(turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT)++ record = registry.store.get(turn_id)+ assert record["outcome"] == "failed_terminal" and record["served"] is None+ assert record["terminal_error"] == {+ "source_id": "src_stalled01",+ "configured_model_id": "test-model",+ "channel": "hub",+ "reason": "backend_error",+ "stream_started": None,+ "error_kind": "stalled",+ "backend_error_detail": "provider sent no data for 600s 上游 key=[redacted]",+ }+ _assert_valid("turn-provenance.schema.json", record)++async def test_an_unreadable_image_is_listed_by_path_and_the_turn_runs(engine, session, tmp_path, published) -> None:from modules.im.base import FileAttachmentdiff --git a/tests/test_vibey_agent_context.py b/tests/test_vibey_agent_context.pyindex 46cfbf894..eab7e6dd9 100644--- a/tests/test_vibey_agent_context.py+++ b/tests/test_vibey_agent_context.py@@ -216,7 +216,7 @@ async def test_a_skill_load_through_bash_is_marked_in_its_context_row(engine, seasync def test_a_context_that_cannot_fit_locally_is_never_reported_as_a_source_failure(engine, session, tmp_path):reported: list[str] = []- async def record_native_failure(context, diagnostic) -> bool:+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:reported.append(diagnostic)return False@@ -283,7 +283,7 @@ _CUT_CALL = [Done(assistant(calls=[ToolCallBlock("cut", "bash", {"command": "ls"async def test_model_hub_hears_only_of_failures_the_served_source_produced(engine, session, tmp_path, case, recorded):reported: list[str] = []- async def record_native_failure(context, diagnostic) -> bool:+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:reported.append(diagnostic)return Falsediff --git a/ui/src/components/settings/models/RouteRecordedTurn.test.tsx b/ui/src/components/settings/models/RouteRecordedTurn.test.tsxindex 4dee8c726..d1102661f 100644--- a/ui/src/components/settings/models/RouteRecordedTurn.test.tsx+++ b/ui/src/components/settings/models/RouteRecordedTurn.test.tsx@@ -75,6 +75,20 @@ describe('recorded turn error', () => {await user.click(screen.getByRole('button', { name: 'View error details' }));expect(screen.getByRole('dialog').querySelector('pre')?.textContent).toBe(JSON.stringify(historical, null, 2));});+ it('names a backend-reported failure and its kind instead of an upstream or protocol fault', async () => {+ const user = userEvent.setup();+ const reported: TurnProvenance = { ...record, terminal_error: {+ source_id: 'src_historical', configured_model_id: 'historical-model', channel: 'hub', reason: 'backend_error',+ stream_started: null, error_kind: 'stalled', backend_error_detail: 'provider sent no data for 600s',+ } };+ vi.spyOn(modelsApi, 'getAgentProvenance').mockResolvedValue(reported);+ render(view());+ expect(await screen.findByText('Latest recorded turn · Agent backend reported a failure')).toBeTruthy();+ expect(screen.getByText('The model request timed out waiting for data')).toBeTruthy();+ expect(screen.queryByText(/Protocol error|model not found/)).toBeNull();+ await user.click(screen.getByRole('button', { name: 'View error details' }));+ expect(screen.getByRole('dialog').querySelector('pre')?.textContent).toBe(JSON.stringify(reported, null, 2));+ });it.each(['served', 'canceled', null] as const)('does not show an old error when latest outcome is %s', async (outcome) => {const read = vi.spyOn(modelsApi, 'getAgentProvenance').mockResolvedValue(outcome ? { ...record, outcome, terminal_error: null } : null);render(view());diff --git a/ui/src/components/settings/models/RouteRecordedTurn.tsx b/ui/src/components/settings/models/RouteRecordedTurn.tsxindex b6af62036..ff31781dd 100644--- a/ui/src/components/settings/models/RouteRecordedTurn.tsx+++ b/ui/src/components/settings/models/RouteRecordedTurn.tsx@@ -7,18 +7,19 @@ import type { RecordedError, RecordedTurn } from './recordedTurn';function useReason(record: RecordedError) {const { t } = useTranslation();const error = record.terminal_error;- return error.upstream_error_code === 'model_not_found'+ return error.reason !== 'backend_error' && error.upstream_error_code === 'model_not_found'? t('settings.models.routing.modelNotFound'): t(`settings.models.routing.errorReason.${error.reason}`);}function RecordedSummary({ record }: { record: RecordedError }) {- const { i18n } = useTranslation();+ const { t, i18n } = useTranslation();const error = record.terminal_error;return <><time dateTime={record.ts}>{new Date(record.ts).toLocaleString(i18n.language)}</time><span className="font-mono">{error.source_id ?? '—'} · {error.configured_model_id ?? '—'}</span>- {error.upstream_error_code && <span className="font-mono">{error.upstream_error_code}</span>}+ {error.reason !== 'backend_error' && error.upstream_error_code && <span className="font-mono">{error.upstream_error_code}</span>}+ {error.error_kind && <span>{t(`settings.models.routing.errorKind.${error.error_kind}`, { defaultValue: error.error_kind })}</span>}</>;}@@ -29,7 +30,8 @@ export function RecordedTurnBadge({ record, onOpen }: { record: RecordedError; oconst label = `${t('settings.models.routing.latestRecorded')} · ${reason}`;return <button type="button" className="model-hub-recorded-turn-badge flex shrink-0 items-center font-mono" aria-label={label} title={label} onClick={onOpen}><AlertTriangle aria-hidden="true" />- {record.terminal_error.http_status ?? record.terminal_error.upstream_error_code ?? t('settings.models.routing.errorShort')}+ {(record.terminal_error.reason !== 'backend_error'+ ? record.terminal_error.http_status ?? record.terminal_error.upstream_error_code : null) ?? t('settings.models.routing.errorShort')}</button>;}diff --git a/ui/src/components/settings/models/types.ts b/ui/src/components/settings/models/types.tsindex a269e02a3..f4ec5696b 100644--- a/ui/src/components/settings/models/types.ts+++ b/ui/src/components/settings/models/types.ts@@ -626,16 +626,30 @@ export type TurnProvenance = {source_id: string | null;configured_model_id: string | null;channel: SupplyChannel | null;+ local_error_detail?: string;+ origin?: { provider: string; api: SourceProtocol; model: string };+ stripped_reasoning_efforts?: string[];+ declared_reasoning_efforts?: string[];+ } & ({reason: 'invalid_parameter' | 'protocol_error' | 'tool_incompatible' | 'stream_interrupted' | 'engine_down';stream_started: boolean;http_status?: number | null;upstream_error_code?: string | null;upstream_detail?: string;- local_error_detail?: string;- origin?: { provider: string; api: SourceProtocol; model: string };- stripped_reasoning_efforts?: string[];- declared_reasoning_efforts?: string[];- } | null;+ error_kind?: never;+ backend_error_detail?: never;+ } | {+ reason: 'backend_error';+ /** The reporter has no failing-request phase evidence or upstream API verdict. */+ stream_started: null;+ http_status?: never;+ upstream_error_code?: never;+ upstream_detail?: never;+ /** The backend's typed failure kind, a machine identifier such as `stalled`. */+ error_kind?: string;+ /** The backend's own redacted diagnostic, never an upstream answer. */+ backend_error_detail?: string;+ }) | null;model_supply_state: 'waiting' | 'interrupted' | null;blockers: Array<RouteHop & { reason: ChainUnavailableReason; retry_at?: string | null }>;};diff --git a/ui/src/components/workbench/FailureDetails.test.tsx b/ui/src/components/workbench/FailureDetails.test.tsxindex 868abbb62..f6b7c7631 100644--- a/ui/src/components/workbench/FailureDetails.test.tsx+++ b/ui/src/components/workbench/FailureDetails.test.tsx@@ -213,6 +213,40 @@ describe('failed-turn upstream details', () => {expect(screen.queryByText(/由上游供应商的 API 返回/)).toBeNull();});+ it.each([+ ['stalled', '模型请求等待数据超时', {}],+ ['ProviderProtocolViolation', 'ProviderProtocolViolation', {}],+ ['stalled', '模型请求等待数据超时', {+ http_status: 418, upstream_error_code: 'invalid_request_error', upstream_detail: 'untrusted mixed record',+ }],+ ])('shows a backend-reported %s failure as the backend\'s diagnostic, never an upstream answer', async (kind, shown, upstreamFields) => {+ vi.spyOn(modelsApi, 'getTurnProvenance').mockResolvedValue({+ ...record,+ agent: 'vibey',+ failed_attempts: [],+ terminal_error: {+ source_id: 'src_b', configured_model_id: 'grok-4.6-beta', channel: 'hub', reason: 'backend_error',+ stream_started: null, error_kind: kind, backend_error_detail: 'provider sent no data for 600s',+ ...upstreamFields,+ },+ blockers: [],+ } as unknown as TurnProvenance);+ mount();+ fireEvent.click(await screen.findByRole('button', { name: '查看详情' }));+ expect(screen.getByText('grok-4.6 的请求失败了,原因见下方。')).toBeTruthy();+ expect(screen.getByText('Agent 后端报告了失败')).toBeTruthy();+ expect(screen.getByText(/失败类型/)).toBeTruthy();+ expect(screen.getByText(shown)).toBeTruthy();+ expect(screen.getByText(/Agent 后端报告:/)).toBeTruthy();+ expect(screen.getByText('provider sent no data for 600s')).toBeTruthy();+ expect(screen.queryByText(/上游返回/)).toBeNull();+ expect(screen.queryByText(/上游 API 拒绝/)).toBeNull();+ expect(screen.queryByText(/由上游供应商的 API 返回/)).toBeNull();+ expect(screen.queryByText('418')).toBeNull();+ expect(screen.queryByText('invalid_request_error')).toBeNull();+ expect(screen.queryByText('untrusted mixed record')).toBeNull();+ });+it('names Avibe-side terminal failures and localizes every blocker reason', async () => {vi.spyOn(modelsApi, 'getTurnProvenance').mockResolvedValue({...record,diff --git a/ui/src/components/workbench/FailureDetails.tsx b/ui/src/components/workbench/FailureDetails.tsxindex 0e13c5477..abacb756c 100644--- a/ui/src/components/workbench/FailureDetails.tsx+++ b/ui/src/components/workbench/FailureDetails.tsx@@ -7,8 +7,10 @@// user see the refusal came from the upstream API and not from the gateway.//// The upstream message is kept only as the engine client's credential-redacted,-// 400-character projection. Local OS reasons are also carried by the notice, so-// details survive an unavailable provenance read.+// 400-character projection. A failure the agent backend reported on its own is+// labeled as the backend's diagnostic, never as an upstream answer. Local OS+// reasons are also carried by the notice, so details survive an unavailable+// provenance read.import * as React from 'react';import { ChevronDown, ChevronRight } from 'lucide-react';import { useTranslation } from 'react-i18next';@@ -27,12 +29,21 @@ type Row = {model: string | null;status: number | null;code: string | null;+ // The agent backend's typed failure kind, for a failure it reported itself.+ kind: string | null;reason: string;detail: string | null;- // A network attempt's text is the connection failure, not an upstream answer.- connection: boolean;+ // Who said `detail`: a network attempt's text is the connection failure and+ // a backend-reported failure's is the backend's, neither an upstream answer.+ said: 'upstream' | 'connection' | 'backend';};+const DETAIL_LABEL = {+ upstream: 'chat.failureDetails.upstreamDetail',+ connection: 'chat.failureDetails.connectionDetail',+ backend: 'chat.failureDetails.backendDetail',+} as const satisfies Record<Row['said'], string>;+const RECORD_RETRIES = 3;const RECORD_RETRY_MS = 1_000;@@ -100,21 +111,24 @@ export function FailureDetails({ message }: { message: WorkbenchMessage }) {model: entry.configured_model_id,status: entry.http_status ?? null,code: null,+ kind: null,reason: t(`chat.failureDetails.reason.${entry.reason}`, { defaultValue: entry.reason }),detail: entry.upstream_detail ?? null,- connection: entry.reason === 'network',+ said: entry.reason === 'network' ? 'connection' : 'upstream',}));const terminal = record.terminal_error;if (terminal) {+ const backend = terminal.reason === 'backend_error';attempts.push({key: 'terminal',source: name(terminal.source_id),model: terminal.configured_model_id,- status: terminal.http_status ?? null,- code: terminal.upstream_error_code ?? null,+ status: backend ? null : terminal.http_status ?? null,+ code: backend ? null : terminal.upstream_error_code ?? null,+ kind: terminal.error_kind ?? null,reason: t(`settings.models.routing.errorReason.${terminal.reason}`),- detail: terminal.upstream_detail ?? null,- connection: false,+ detail: (backend ? terminal.backend_error_detail : terminal.upstream_detail) ?? null,+ said: backend ? 'backend' : 'upstream',});}return attempts;@@ -130,7 +144,7 @@ export function FailureDetails({ message }: { message: WorkbenchMessage }) {? 'failed_local': record.outcome;const upstreamFacts = attempts.some(- (row) => row.status !== null || row.code !== null || (row.detail !== null && !row.connection),+ (row) => row.status !== null || row.code !== null || (row.detail !== null && row.said === 'upstream'),);return (<div className="flex flex-col gap-2">@@ -145,11 +159,12 @@ export function FailureDetails({ message }: { message: WorkbenchMessage }) {<span className="flex flex-wrap gap-x-3 gap-y-0.5 text-gold-ink/70"><span>{t('chat.failureDetails.httpStatus')}: <span className="font-mono text-gold-ink">{row.status ?? '—'}</span></span>{row.code && <span>{t('chat.failureDetails.errorCode')}: <span className="font-mono text-gold-ink">{row.code}</span></span>}+ {row.kind && <span>{t('chat.failureDetails.errorKind')}: <span className="text-gold-ink">{t(`settings.models.routing.errorKind.${row.kind}`, { defaultValue: row.kind })}</span></span>}<span>{t('chat.failureDetails.reasonLabel')}: <span className="text-gold-ink">{row.reason}</span></span></span>{row.detail && (<span className="text-gold-ink/70">- {t(row.connection ? 'chat.failureDetails.connectionDetail' : 'chat.failureDetails.upstreamDetail')}:{' '}+ {t(DETAIL_LABEL[row.said])}:{' '}<span className="whitespace-pre-wrap break-words font-mono text-gold-ink">{row.detail}</span></span>)}diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.jsonindex e2e643cbc..168f88e2f 100644--- a/ui/src/i18n/en.json+++ b/ui/src/i18n/en.json@@ -1324,7 +1324,11 @@"protocol_error": "Protocol error","tool_incompatible": "Tool incompatible","stream_interrupted": "Stream interrupted",- "engine_down": "Gateway unavailable"+ "engine_down": "Gateway unavailable",+ "backend_error": "Agent backend reported a failure"+ },+ "errorKind": {+ "stalled": "The model request timed out waiting for data"},"origin": {"automatic": "Automatic",@@ -4831,10 +4835,12 @@"httpStatus": "HTTP status","errorCode": "Error code","reasonLabel": "Reason",+ "errorKind": "Failure kind","blockers": "Sources that were unavailable","upstreamNote": "These come from the upstream provider's API. Avibe keeps the status, error code, and the provider's error message with credentials removed, up to 400 characters.","upstreamDetail": "Upstream said","connectionDetail": "Connection error",+ "backendDetail": "Agent backend reported","copyRecord": "Copy record","reason": {"quota_exhausted": "Quota exhausted",diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.jsonindex 3034433bb..d58afb584 100644--- a/ui/src/i18n/zh.json+++ b/ui/src/i18n/zh.json@@ -1324,7 +1324,11 @@"protocol_error": "协议错误","tool_incompatible": "工具不兼容","stream_interrupted": "响应中断",- "engine_down": "模型网关不可用"+ "engine_down": "模型网关不可用",+ "backend_error": "Agent 后端报告了失败"+ },+ "errorKind": {+ "stalled": "模型请求等待数据超时"},"origin": {"automatic": "自动",@@ -4831,10 +4835,12 @@"httpStatus": "HTTP 状态码","errorCode": "错误码","reasonLabel": "原因",+ "errorKind": "失败类型","blockers": "当时不可用的供应商","upstreamNote": "以上内容均由上游供应商的 API 返回。Avibe 会保留状态码、错误码,以及去除凭据后的上游报错信息(最多 400 字)。","upstreamDetail": "上游返回","connectionDetail": "连接错误",+ "backendDetail": "Agent 后端报告","copyRecord": "复制记录","reason": {"quota_exhausted": "额度用尽",diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.pyindex 3b896e553..f50b6e3da 100644--- a/vibe/model_hub_runtime/client.py+++ b/vibe/model_hub_runtime/client.py@@ -31,7 +31,7 @@ from core.handlers.model_hub.adapter import ()from core.handlers.model_hub.async_owner import run_owned_in_threadfrom core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES-from core.handlers.model_hub.events import redact_untrusted_text+from core.handlers.model_hub.events import bounded_diagnostic_detailfrom core.handlers.model_hub.json_wire import (JSONEvent,JSONPath,@@ -39,7 +39,6 @@ from core.handlers.model_hub.json_wire import (project_json_reader,)from core.handlers.model_hub.request import FORWARDED_CALLER_HEADERS-from core.message_output import plain_untrusted_textfrom core.os_errors import local_os_errnofrom core.handlers.model_hub.stream_wire import (ErrorEnvelopePath,@@ -57,8 +56,6 @@ from vibe.model_hub_runtime.state import SourceRecord_STREAM_CHUNK_BYTES = 64 * 1024-# Upper bound on upstream error text shown to the user in a terminal message.-_UPSTREAM_DETAIL_CHARS = 400# The engine answers a failed upstream connection itself, before any upstream# response, with exactly its own generic envelope and Go's transport error text.# That is a network fact, not an upstream server verdict.@@ -69,9 +66,6 @@ _ENGINE_TRANSPORT_FAILURE = re.compile(r"http2: (?:client connection lost|server sent GOAWAY)|server closed idle connection|"r"no such host|network is unreachable|proxyconnect")-# C0/C1 controls and every Unicode Bidi_Control character: either can act on-# whatever renders the text (a terminal, a log viewer, a browser).-_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f-\x9f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]")# This threshold only selects memory or a temporary file; it never rejects or# truncates upstream response bytes._PRELUDE_MEMORY_BYTES = 256 * 1024@@ -1746,7 +1740,7 @@ def _reduce_protocol_observation(observation.error_envelope_paths,)upstream_detail = (- _bounded_upstream_detail(observation.error_message)+ bounded_diagnostic_detail(observation.error_message)if observation.error_message is not Noneelse _upstream_error_detail(observation.error_payload or b"",@@ -1944,7 +1938,7 @@ def _upstream_error_detail(if not payload or not project_json_reader(io.BytesIO(payload), message_paths, visit):return Nonefor path in message_paths:- text = _bounded_upstream_detail(values.get(path, ""))+ text = bounded_diagnostic_detail(values.get(path, ""))if text:return textreturn None@@ -1996,18 +1990,6 @@ def _upstream_reset_advice(return None-def _bounded_upstream_detail(message: str) -> str | None:- # Control characters (terminal escapes, BEL) would act on whatever renders- # the text, a log viewer included; they carry no message content.- text = " ".join(_CONTROL_CHARACTERS.sub(" ", message).split())- if not text:- return None- text = plain_untrusted_text(redact_untrusted_text(text))- if len(text) > _UPSTREAM_DETAIL_CHARS:- text = text[: _UPSTREAM_DETAIL_CHARS - 1].rstrip() + "…"- return text--def _safe_error_code(value: object) -> str | None:if not isinstance(value, str) or value not in UPSTREAM_MACHINE_ERROR_CODES:return None