Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): preserve backend failure kinds without inventing protocol facts (#2544)

Call White9a89da41b0f1
diff --git a/core/handlers/model_hub/events.py b/core/handlers/model_hub/events.py
index 8904155e7..25a59c96b 100644
--- a/core/handlers/model_hub/events.py
+++ b/core/handlers/model_hub/events.py
@@ -11,6 +11,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Literal, Optional, TypeAlias, get_args
+from core.message_output import plain_untrusted_text
from vibe.i18n import t as i18n_t
from .state_file import write_state_document
@@ -140,6 +141,28 @@ def redact_untrusted_text(value: str) -> str:
return redacted[: match.end()] + "[redacted]"
+# Upper bound on a failure diagnostic Avibe keeps and shows: an upstream's error
+# message, or a backend's own report of a failed turn.
+_DIAGNOSTIC_DETAIL_CHARS = 400
+# C0/C1 controls and every Unicode Bidi_Control character: either can act on
+# whatever renders the text (a terminal, a log viewer, a browser).
+_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f-\x9f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]")
+
+
+def bounded_diagnostic_detail(message: str) -> str | None:
+ """The display-safe projection of untrusted failure text, or None when it holds nothing."""
+
+ # Control characters (terminal escapes, BEL) would act on whatever renders
+ # the text, a log viewer included; they carry no message content.
+ text = " ".join(_CONTROL_CHARACTERS.sub(" ", message).split())
+ if not text:
+ return None
+ text = plain_untrusted_text(redact_untrusted_text(text))
+ if len(text) > _DIAGNOSTIC_DETAIL_CHARS:
+ text = text[: _DIAGNOSTIC_DETAIL_CHARS - 1].rstrip() + "…"
+ return text
+
+
def contains_credential_material(value: object) -> bool:
rendered = json.dumps(value, ensure_ascii=False, sort_keys=True)
return any(pattern.search(rendered) for pattern in _CREDENTIAL_PATTERNS)
diff --git a/core/handlers/model_hub/provenance.py b/core/handlers/model_hub/provenance.py
index b5b05d34c..95021a117 100644
--- a/core/handlers/model_hub/provenance.py
+++ b/core/handlers/model_hub/provenance.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import json
import logging
+import re
import secrets
import threading
from dataclasses import dataclass, field
@@ -12,6 +13,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Iterable, Literal, Mapping, Optional
+from core.agent_core.ai.errors import redact_provider_text
from config.v2_config import MODEL_HUB_BACKENDS, ModelHubConfig
from modules.agents.catalog import display_name_for_backend
from core.os_errors import format_os_errno
@@ -36,6 +38,7 @@ from .events import (
EVENT_REASON_AUTHORITY,
RETIRED_PERSISTED_REASON_DEGRADATIONS,
SOURCE_DETAIL_EVENT_REASONS,
+ bounded_diagnostic_detail,
event_reason_label,
)
from .resolver import BackendName, ModelHubTurnResolution, parse_model_hub_timestamp, source_eligible_for_backend
@@ -43,6 +46,9 @@ from .state_file import write_state_document
SupplyChannel = Literal["native_cli", "hub"]
+# A backend's typed failure kind is a member of that backend's closed vocabulary
+# (Vibey: ``NAMED_ERROR_KINDS``), never free text: the record keeps only that shape.
+_BACKEND_ERROR_KIND = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,63}")
SupplyState = Literal["waiting", "interrupted"]
ScopeKey = tuple[BackendName, str]
logger = logging.getLogger(__name__)
@@ -702,7 +708,9 @@ class TurnTrace:
ambiguous: bool = False
terminal_outcome: TurnOutcomeProjectionInput | None = None
admission_closed: bool = False
- outcome_frozen: bool = False
+ # Both causes veto mutation. Backend acknowledgments keep terminal copy
+ # readable; explicit user cancellation suppresses live copy as before.
+ outcome_frozen: Literal["backend_failure", "user_cancellation"] | None = None
recovery_requests: dict[str, dict] = field(default_factory=dict)
@property
@@ -1075,13 +1083,15 @@ class TurnCorrelationRegistry:
self.on_recovery_changed = on_recovery_changed
def _readable_trace(
- self, turn_id: str, backend: str | None = None, *, allow_closed: bool = False,
+ self, turn_id: str, backend: str | None = None, *,
+ allow_closed: bool = False, allow_acknowledged: bool = False,
) -> TurnTrace | None:
"""Called under the registry lock; attribution is never inferred."""
trace = self._traces.get(turn_id)
if (
- trace is None or trace.ambiguous or trace.outcome_frozen
+ trace is None or trace.ambiguous
+ or (trace.outcome_frozen and not (allow_acknowledged and trace.outcome_frozen == "backend_failure"))
or (trace.admission_closed and not allow_closed)
or (backend is not None and trace.agent != backend)
):
@@ -1098,9 +1108,9 @@ class TurnCorrelationRegistry:
def terminal_projection(self, turn_id: str, *, backend: str) -> TurnOutcomeProjectionInput | None:
with self._lock:
- trace = self._readable_trace(turn_id, backend, allow_closed=True)
+ trace = self._readable_trace(turn_id, backend, allow_closed=True, allow_acknowledged=True)
if (
- trace is None or trace.pending_attempts or trace.terminal_outcome is None
+ trace is None or (trace.pending_attempts and not trace.outcome_frozen) or trace.terminal_outcome is None
or trace.terminal_outcome.outcome in {"served", "canceled"}
):
return None
@@ -2214,11 +2224,13 @@ class TurnCorrelationRegistry:
return
if trace.agent == "vibey" and channel != "hub":
raise ValueError("vibey attempts require the hub channel")
- # Admission supersedes earlier supply failures, not their attempt history.
+ # A newly admitted attempt supersedes earlier request/supply
+ # terminal facts, not completed fallback history. Otherwise an
+ # interrupted successor would be blamed on the preceding request.
trace.model_supply_state = None
trace.blockers = []
- if trace.terminal_outcome is not None and trace.terminal_outcome.outcome in {"no_candidate", "exhausted"}:
- trace.terminal_outcome = None
+ trace.terminal_error = None
+ trace.terminal_outcome = None
trace.pending_attempts[request_id] = AttemptIdentity(
source_id=source_id,
resolved_model_id=resolved_model_id,
@@ -2267,12 +2279,27 @@ class TurnCorrelationRegistry:
return any(key != request_id for key in trace.pending_attempts)
- def fail_hub_attempt(self, turn_id: Optional[str]) -> None:
- """Replace a gateway success rejected by the backend terminal result."""
+ def fail_hub_attempt(
+ self,
+ turn_id: Optional[str],
+ *,
+ error_kind: str | None = None,
+ diagnostic: str | None = None,
+ ) -> None:
+ """Record the backend's terminal failure of a turn the gateway did not end.
+
+ The record holds only what the backend reported: its typed kind and its
+ sanitized diagnostic. The backend cannot see the failing request's
+ phase, so ``stream_started`` stays unknown.
+ """
normalized = str(turn_id or "").strip()
if not normalized:
return
+ # Native backends may include credentialed URLs or Basic auth in their
+ # transport diagnostic. Redact those before the shared display bound;
+ # redacting only upstream-style labeled keys is insufficient here.
+ detail = bounded_diagnostic_detail(redact_provider_text(diagnostic or ""))
with self._lock:
trace = self._traces.get(normalized)
if trace is None or trace.outcome_frozen:
@@ -2280,8 +2307,15 @@ class TurnCorrelationRegistry:
if self.terminal_projection(normalized, backend=trace.agent) is not None:
# The native failure is reporting the gateway's completed
# failure, not rejecting content that the gateway served.
+ # Protect that acknowledged terminal while retaining its copy
+ # for callers that read it after reporting backend failure.
+ trace.outcome_frozen = "backend_failure"
+ return
+ if trace.terminal_error is not None and not trace.pending_attempts:
+ # The same for a completed failure that has no copy. A request
+ # still open after it would supersede it, but none is.
+ trace.outcome_frozen = "backend_failure"
return
- trace.outcome_frozen = True
identity = trace.pending_attempt
payload = (
identity.payload()
@@ -2291,12 +2325,21 @@ class TurnCorrelationRegistry:
if payload is None or payload.get("channel") != "hub":
return
# The backend rejected the turn's result, not one request's.
+ # A callback without an attributed payload must not freeze out
+ # gateway evidence that has yet to arrive.
+ trace.outcome_frozen = "backend_failure"
trace.pending_attempts.clear()
trace.served = None
trace.terminal_error = {
**payload,
- "reason": "protocol_error",
- "stream_started": True,
+ "reason": "backend_error",
+ "stream_started": None,
+ **(
+ {"error_kind": error_kind}
+ if isinstance(error_kind, str) and _BACKEND_ERROR_KIND.fullmatch(error_kind)
+ else {}
+ ),
+ **({"backend_error_detail": detail} if detail else {}),
}
def finish_attempt(
@@ -2381,7 +2424,7 @@ class TurnCorrelationRegistry:
if settled_by in _USER_CANCELLATIONS:
# Preserve the exact facts present at Stop. Teardown may expose
# a later producer success, but cannot change who ended the turn.
- trace.outcome_frozen = True
+ trace.outcome_frozen = "user_cancellation"
if had_recovery:
self._notify_recovery_changed(turn_id)
diff --git a/docs/plans/model-hub-backend-failure-facts.md b/docs/plans/model-hub-backend-failure-facts.md
new file mode 100644
index 000000000..f28934825
--- /dev/null
+++ b/docs/plans/model-hub-backend-failure-facts.md
@@ -0,0 +1,144 @@
+# Backend-reported Model Hub terminal failures
+
+## Change contract (2026-10-13)
+
+A source-origin Vibey `stalled` failure reached the Hub failure bridge as text
+only. `fail_hub_attempt` wrote `protocol_error` and `stream_started: true`, even
+when neither fact was established. This change corrects recorded diagnostics;
+it does not change retries, timeouts, route selection, Source health, or Stop.
+
+- The existing BaseAgent -> ModelHubRuntimeRouter -> correlation failure bridge
+ accepts an optional structured `error_kind`. Vibey supplies its existing
+ `RunEnded.cause.kind` only for source-origin failures. Do not classify by
+ diagnostic substrings, infer phase from retryability, or use a prior response.
+- Backend-only fallback records use `terminal_error.reason = backend_error`,
+ optional `error_kind`, and optional `backend_error_detail`. A `stalled` kind
+ remains `stalled`, including in copied JSON and localized detail presentation.
+ Other backends without typed facts use the neutral backend-error reason.
+- `backend_error_detail` is a credential-redacted, control/bidi-stripped,
+ whitespace-collapsed, at-most-400-character diagnostic, not an upstream API
+ answer. Reuse the existing Model Hub diagnostic sanitizer by moving it to
+ an appropriate shared owner rather than maintaining a weaker second policy.
+- `stream_started` may be null on a backend-error record: the callback has no
+ exact failing-request phase evidence. Unknown is neither false nor true.
+ All gateway-owned terminal reasons keep their existing boolean phase contract.
+ Propagating additional provider phase telemetry is not required for this fix.
+- An already authoritative gateway failure (terminal projection OR concrete
+ terminal_error, even when that reason has no copy projection) must survive a
+ backend notification. Keep Stop/frozen-outcome and late-success protections.
+- Existing released provenance records remain readable without the new fields.
+ Do not rewrite old history by guessing what really happened. Update the owning
+ JSON schema, TypeScript mirror, API contract, and English/Chinese UI copy.
+- Failure details label the backend diagnostic distinctly, never claim it is
+ upstream response text, and show the typed kind (a localized stall label for
+ stalled; other kinds may be displayed as diagnostic machine identifiers).
+
+## Validation obligations
+
+The primary regression must exercise the real Vibey adapter and shared failure
+bridge into persisted provenance: existing coverage stubs out that bridge and
+only checks notice copy. A scripted stalled provider must not become a protocol
+error, and the old implementation must fail the new assertion.
+
+Extend the existing correlation tests for authoritative gateway errors without
+copy projections, generic native-backend notifications, preservation of late
+success/Stop rules, and historical reads. Exercise non-ASCII diagnostics and
+redaction. Add UI consumption coverage distinguishing backend diagnostics from
+upstream API facts; run focused tests, changed-file Ruff, UI build, exact-head
+Codex review and full CI. No production-state writes, deployment, release,
+service restart, or merge is authorized.
+
+## Implementation and owner verification
+
+- The main regression and the updated late-success/correlation expectations
+ fail on the base implementation for the intended reason; the main regression
+ uses the real shared bridge, not a mock of its classification.
+- The implementation preserves an existing concrete gateway terminal only when
+ no pending request supersedes it. It freezes that terminal before the late
+ success can replace it. The regression covers both branches and mutation of
+ the freeze.
+- Owner review found the old upstream-text redactor insufficient for newly
+ retained native diagnostics: credentialed URLs, Basic authentication, and
+ JWT-shaped tokens survived. Reuse `redact_provider_text` on backend diagnostics
+ before the shared display sanitizer. Three consuming cases fail without this
+ step and all pass with it. The existing upstream message path is unchanged.
+- The stall label describes the observation (a model request timed out waiting
+ for data), not an unproved claim about which remote component stopped.
+- Focused owner verification includes the Vibey-to-provenance regression,
+ gateway terminal preservation and native diagnostics, released-record reads,
+ all of `test_model_hub_provenance.py`, late-success/Stop tests, 32 UI tests,
+ Model Hub authority closure, changed-file Ruff, and `npm run build`.
+- Scenario: `MH-TURN-RESULT-001`. Repository-wide CI and exact-head Codex review
+ remain the asynchronous delivery gates. No live provider/Incus regression was
+ run; this change introduces no model request or retry behavior.
+
+## Independent audit follow-up
+
+Read-only audit of implementation `53e7da45a` found three boundary defects;
+owner reproduced them on the pushed `22b9342c4` before editing. Four new checks
+failed for their intended reasons (one case has two projection variants):
+
+1. A newly admitted request retained the preceding request's terminal error
+ and copy projection, so an FSM drop could report the wrong Source. New
+ actual attempt admission now atomically retires both terminal artifacts,
+ while retaining fallback history and the existing frozen/Stop veto.
+2. An unattributed backend callback froze the trace before checking for a
+ pending/served payload, hiding subsequent gateway evidence. Such a callback
+ is now a no-op; only a concrete terminal or a recordable backend error freezes.
+3. The schema allowed boolean phase on backend_error despite writing null.
+ Its positive branch now requires null; both boolean counterexamples fail.
+
+The owner chose these local corrections, not a lifecycle or retry redesign.
+The three changed correlation suites pass **889 tests**, and the two Vibey
+adapter/context suites pass **158 tests** after the changes. Ruff and authority
+closure pass. GitHub Codex passed the previous head `22b9342c4`; a new exact-head
+review is still required after this correction. No GitHub findings-bearing head
+has recurred; the independent audit findings were reconciled together once.
+
+## Circuit-breaker decision: terminal evidence ownership
+
+GitHub review `5501580777` on `08f641f42` raised two findings (threads
+`PRRT_kwDOPbFPYs6ry_u_` and `PRRT_kwDOPbFPYs6ry_vK`): backend-error schema rows
+still admitted upstream-only fields, and acknowledging a gateway projection did
+not protect it from a request arriving before Turn settlement. The latter is
+the same ownership class as the independent review of `53e7da45a`; the schema
+finding also revisits the closed-shape class. Owner paused editing, fetched the
+complete review/CI inventory, and re-examined every mutation/read of the trace.
+
+Root cause: `outcome_frozen` conflates a mutation veto with suppression of live
+copy reads. Freezing an acknowledged projection as an undifferentiated boolean
+would hide the exact error copy callers still need. Leaving it unfrozen permits
+later admission to erase it. The existing freeze therefore needs to identify
+its cause, not an additional terminal ledger or a retry-policy change.
+
+Owner decision: retain the existing freeze gate, distinguish backend-failure
+acknowledgment from explicit user cancellation, and allow only the former's
+frozen terminal projection to remain readable. Recovery reads and all writes
+remain blocked. Once a backend acknowledges a terminal, later requests cannot
+replace it; before acknowledgment, already pending/new actual attempts retain
+the supersession rule. Empty callbacks still freeze nothing. Exercise both
+projection and concrete terminal paths, late admission/success/interruption,
+wrong backend, ordinary closure, and Stop. Backend-error schema shapes exclude
+upstream-only fields, and the TypeScript/display consumer follows that same
+partition. This is the smallest local correction; no lifecycle, Source-health,
+or retry architecture is changed. New regressions must fail before these edits.
+
+Validation of that decision: the four projection/closure combinations, the
+backend-only shape check, and the malformed UI record each fail on `08f641f42`.
+After correction, 957 correlation/retention/failure-presentation tests, 154 retry
+lifecycle tests, 158 Vibey adapter/context tests, and 33 UI tests pass; build,
+Ruff and authority closure pass. The schema negative constraints are object-only
+so the existing null terminal on served/exhausted/canceled/no-candidate outcomes
+continues to validate. Full CI on `08f641f42` was green; the corrected head still
+requires its own Codex verdict and CI.
+
+## Known-by-design
+
+- Records remain generation 11 as with earlier additive diagnostic additions;
+ old records are not rewritten. Downgrading to an older UI may show an unknown
+ label for a new backend-error record, without making stored data unreadable.
+- Only Vibey currently supplies a typed kind, from `NAMED_ERROR_KINDS`; generic
+ native callbacks retain no guessed kind. The shared identifier-shape check is
+ not a classifier or authority for future backend kind vocabularies.
+- The 600-second silence bound, gateway idle recovery, Source health and routing,
+ and the separate raw warning-log format are outside this change.
diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.md
index 6e26e33e3..d4f409f8a 100644
--- a/docs/plans/model-hub-contracts/api.md
+++ b/docs/plans/model-hub-contracts/api.md
@@ -1357,6 +1357,34 @@ of control and bidi characters, whitespace-collapsed, and bounded to 400 charact
They are omitted when no message was observed and in older records, and are never
read by classification.
+`terminal_error.reason: backend_error` is an agent backend's own report of a failed
+turn that the gateway did not end itself
+(`docs/plans/model-hub-backend-failure-facts.md`). The shared failure bridge records
+it on the request still open, else on the served attempt. A gateway terminal already
+recorded for the turn, a copy projection or a concrete `terminal_error` with or
+without one, is kept instead, unless a later request was still open when the backend
+failed. Admission of a new actual attempt atomically retires the preceding
+request's terminal error and copy projection, keeping its fallback history.
+An unattributed callback with neither a payload nor a concrete gateway terminal
+is a no-op; it must not freeze out later gateway evidence. The supersession rule
+applies before backend acknowledgment. Once the backend acknowledges an existing
+gateway terminal, that terminal is frozen against later requests while its exact
+copy remains readable; explicit user cancellation continues to suppress live copy.
+Backend-error records prohibit `http_status`, `upstream_error_code`, and
+`upstream_detail` (including null-valued members), so no upstream verdict can be
+presented as a backend-only fact. A null `terminal_error` remains valid for all
+other turn outcomes.
+The record carries only the backend's facts: optional `error_kind`, the
+backend's typed failure kind from its closed vocabulary (Vibey supplies its named
+`AgentError` kind, such as `stalled`, for a failure the served Source produced; other
+backends supply none), and optional `backend_error_detail`, the backend's diagnostic
+under the same redaction and bound as `upstream_detail`, which is never an upstream
+answer. Its `stream_started` is null because the reporter cannot see the failing
+request's phase; every other reason keeps its boolean. It changes no classification,
+retry, Source health or Stop rule. Records written before this reason existed,
+including backend rejections recorded as `protocol_error` with `stream_started: true`,
+are read as written.
+
The dialog independently reads this projection on demand and labels it "Latest recorded
turn" / "最近已记录回合". Its error panel and details action use the same structured record,
recorded timestamp and exact historical Source/model identifiers. Deleted Sources keep
diff --git a/docs/plans/model-hub-contracts/turn-provenance.schema.json b/docs/plans/model-hub-contracts/turn-provenance.schema.json
index 43522fd7c..4730301fb 100644
--- a/docs/plans/model-hub-contracts/turn-provenance.schema.json
+++ b/docs/plans/model-hub-contracts/turn-provenance.schema.json
@@ -172,10 +172,26 @@
"protocol_error",
"tool_incompatible",
"stream_interrupted",
- "engine_down"
- ]
+ "engine_down",
+ "backend_error"
+ ],
+ "description": "Who ended the turn and how. backend_error is the backend's own report of a failed turn the gateway did not end itself; it carries only what the backend observed, never an upstream verdict or a guessed gateway reason."
+ },
+ "stream_started": {
+ "type": ["boolean", "null"],
+ "description": "Whether user-visible model output began on the failing request. Null only on backend_error, whose reporter cannot see that request's phase; unknown is neither false nor true."
+ },
+ "error_kind": {
+ "type": "string",
+ "pattern": "^[A-Za-z][A-Za-z0-9_]{0,63}$",
+ "description": "Optional backend_error fact: the reporting backend's typed failure kind from its closed vocabulary (Vibey: its named AgentError kinds, for example stalled). Never free text, an unexpected exception class, or an upstream code. Absent when the backend has no typed kind and in historical records."
+ },
+ "backend_error_detail": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 400,
+ "description": "Optional backend_error fact: the backend's own diagnostic for the failed turn, under the same redaction and bound as upstream_detail. It is not an upstream API answer. Display only; absent when the backend gave none and in historical records."
},
- "stream_started": { "type": "boolean" },
"local_error_detail": {
"type": "string",
"description": "Optional local OS errno and system message. Produced from numeric errno only; never contains exception paths, upstream text, credentials, or request bodies. Historical records may omit it."
@@ -231,6 +247,33 @@
"declared_reasoning_efforts": ["stripped_reasoning_efforts"]
},
"allOf": [
+ {
+ "if": {
+ "properties": { "reason": { "const": "backend_error" } },
+ "required": ["reason"]
+ },
+ "then": {
+ "properties": { "stream_started": { "const": null } },
+ "not": {
+ "type": "object",
+ "anyOf": [
+ { "required": ["http_status"] },
+ { "required": ["upstream_error_code"] },
+ { "required": ["upstream_detail"] }
+ ]
+ }
+ },
+ "else": {
+ "properties": { "stream_started": { "type": "boolean" } },
+ "not": {
+ "type": "object",
+ "anyOf": [
+ { "required": ["error_kind"] },
+ { "required": ["backend_error_detail"] }
+ ]
+ }
+ }
+ },
{
"if": {
"properties": { "reason": { "const": "stream_interrupted" } },
diff --git a/modules/agents/base.py b/modules/agents/base.py
index 9105e131a..5ab746b69 100644
--- a/modules/agents/base.py
+++ b/modules/agents/base.py
@@ -262,15 +262,20 @@ class BaseAgent(ABC):
return None
- async def record_model_hub_native_failure(self, context: Any, diagnostic: str) -> bool:
- """Persist source cooldown state when a terminal turn proves source loss."""
+ async def record_model_hub_native_failure(
+ self, context: Any, diagnostic: str, *, error_kind: str | None = None
+ ) -> bool:
+ """Persist source cooldown state when a terminal turn proves source loss.
+
+ ``error_kind`` is the backend's typed failure kind, for a backend that has one.
+ """
router = getattr(self.controller, "model_hub_runtime", None)
recorder = getattr(router, "record_native_failure", None)
if not callable(recorder):
return False
try:
- return bool(await recorder(context, diagnostic))
+ return bool(await recorder(context, diagnostic, error_kind=error_kind))
except Exception:
logger.warning("Failed to record Model Hub native cooldown", exc_info=True)
return False
diff --git a/modules/agents/model_hub.py b/modules/agents/model_hub.py
index 7090b1a0c..f111502ee 100644
--- a/modules/agents/model_hub.py
+++ b/modules/agents/model_hub.py
@@ -1424,11 +1424,15 @@ class ModelHubRuntimeRouter:
self._emit_transition(launch, config)
return launch
- async def record_native_failure(self, context: Any, diagnostic: str) -> bool:
+ async def record_native_failure(
+ self, context: Any, diagnostic: str, *, error_kind: str | None = None
+ ) -> bool:
"""Record a terminal source failure for the next per-turn resolution.
The method name is retained for existing backend call sites; Hub launches
use the same cooldown state so the next turn can select a backup source.
+ A Hub turn's record keeps the backend's typed ``error_kind``, when it
+ has one, beside its diagnostic.
"""
launch = launch_for_context(context)
@@ -1443,7 +1447,9 @@ class ModelHubRuntimeRouter:
)
or ""
).strip()
- self.turn_gateway.correlation.fail_hub_attempt(turn_id)
+ self.turn_gateway.correlation.fail_hub_attempt(
+ turn_id, error_kind=error_kind, diagnostic=diagnostic
+ )
setattr(context, _CONTEXT_FAILURE_RECORDED_ATTR, True)
return False
decision: ResolutionDecision | None
diff --git a/modules/agents/vibey/agent.py b/modules/agents/vibey/agent.py
index 69ccdfb99..e89116f19 100644
--- a/modules/agents/vibey/agent.py
+++ b/modules/agents/vibey/agent.py
@@ -40,6 +40,7 @@ from sqlalchemy.engine import Engine
from config import paths
from core.agent_core.agent.events import (
+ NAMED_ERROR_KINDS,
AgentError,
AgentEvent,
CompactionFinished,
@@ -153,6 +154,11 @@ def _loggable(text: str, limit: int = _LOGGED_DIAGNOSTIC_CHARS) -> str:
return line if len(line) <= limit else line[: limit - 1] + "…"
+def _hub_error_kind(kind: Optional[str]) -> Optional[str]:
+ """The failure's kind as Model Hub records it: a kind the engine names, never an unexpected exception's class."""
+ return kind if kind in NAMED_ERROR_KINDS else None
+
+
@dataclass
class _Run:
"""One Turn's state, published before its first await (``handle_message``)."""
@@ -655,7 +661,9 @@ class VibeyAgent(BaseAgent):
# of its own (``_display_source``), its row carries the explanation.
body = error_text(kind or "empty_response", self._language(), reason=reason)
if failed and source:
- await self.record_model_hub_native_failure(context, diagnostic or (kind or "failed final"))
+ await self.record_model_hub_native_failure(
+ context, diagnostic or (kind or "failed final"), error_kind=_hub_error_kind(kind)
+ )
if body.strip():
# The same result path as the other backends; the row is already the message.
await self.emit_result_message(
@@ -1217,7 +1225,9 @@ class VibeyAgent(BaseAgent):
other backends; any other (a context that cannot fit, a Stop, a local error) leaves Model Hub's record as it is.
"""
if source:
- await self.record_model_hub_native_failure(request.context, diagnostic or (kind or "error"))
+ await self.record_model_hub_native_failure(
+ request.context, diagnostic or (kind or "error"), error_kind=_hub_error_kind(kind)
+ )
output = terminal_output_for(request)
if kind:
# The kind, never the diagnostic, so a failed Turn can be diagnosed from its notice row.
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 794b18193..9a79fcfcb 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -1594,6 +1594,10 @@ scenarios:
kind: turn_lifecycle
layer: scenario
test: tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py::test_mh_turn_outcomes_and_cancel_are_closed_provenance_products
+ related_tests:
+ - tests/test_vibey_agent.py::test_model_hub_records_a_source_stall_as_the_backend_reported_it
+ - tests/test_model_hub_l3.py::test_backend_failure_keeps_the_gateway_terminal_error_no_later_request_superseded
+ - tests/test_model_hub_l3.py::test_backend_terminal_failure_overrides_gateway_success
- id: MH-TURN-COPY-001
name: Every turn decision resolves to its authoritative backend copy key or silence
status: covered
diff --git a/tests/test_claude_cli_path.py b/tests/test_claude_cli_path.py
index bef6e2470..d0c73f48a 100644
--- a/tests/test_claude_cli_path.py
+++ b/tests/test_claude_cli_path.py
@@ -2519,7 +2519,7 @@ def test_claude_startup_failure_is_recorded_before_scope_retirement(
source_id="src_native01",
)
- async def record_native_failure(self, context, diagnostic):
+ async def record_native_failure(self, context, diagnostic, *, error_kind=None):
events.append(("record", diagnostic))
return False
diff --git a/tests/test_model_hub_l3.py b/tests/test_model_hub_l3.py
index 55451073c..de3cdc7eb 100644
--- a/tests/test_model_hub_l3.py
+++ b/tests/test_model_hub_l3.py
@@ -5457,8 +5457,24 @@ def test_fsm_drop_overrides_a_completed_gateway_attempt(tmp_path: Path) -> None:
_assert_valid("turn-provenance.schema.json", record)
+ "error_kind,recorded,diagnostic,expected_detail",
+ [
+ # A backend with no typed kind (Claude, Codex, OpenCode) reports its failure neutrally.
+ (None, {}, "API Error: 上游连接已关闭\x07 api_key=sk-proj-detail0secret0value",
+ "API Error: 上游连接已关闭 api_key=[redacted]"),
+ ("stalled", {"error_kind": "stalled"}, "provider sent no data for 600s", "provider sent no data for 600s"),
+ # Only a kind's shape is ever kept: text in the kind field is dropped, never persisted.
+ ("Authorization: Bearer sk-proj-kind0secret0value", {}, "", None),
+ # Newly retained native diagnostics must not disclose transport credentials.
+ (None, {}, "connection failed to https://alice:[email protected]/v1",
+ "connection failed to https://example.invalid/v1"),
+ (None, {}, "Authorization: Basic ZHVtbXk6ZHVtbXk=", "Authorization: Basic [redacted]"),
+ (None, {}, "request failed: eyJkdW1teSI6MX0.eyJkdW1teSI6MX0.ZHVtbXk", "request failed: [redacted]"),
+ ],
+)
def test_backend_terminal_failure_overrides_gateway_success(
- tmp_path: Path,
+ tmp_path: Path, error_kind: str | None, recorded: dict, diagnostic: str, expected_detail: str | None
) -> None:
store = BoundedProvenanceStore(tmp_path / "backend-terminal.json")
registry = TurnCorrelationRegistry(store)
@@ -5473,7 +5489,11 @@ def test_backend_terminal_failure_overrides_gateway_success(
decision=classify_outcome(success),
)
- registry.fail_hub_attempt("turn_backend_terminal")
+ registry.fail_hub_attempt(
+ "turn_backend_terminal",
+ error_kind=error_kind,
+ diagnostic=diagnostic,
+ )
registry.settle(
"turn_backend_terminal",
settled_by=SETTLED_BY_TERMINAL_RESULT,
@@ -5484,10 +5504,107 @@ def test_backend_terminal_failure_overrides_gateway_success(
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["served"] is None
- assert record["terminal_error"]["reason"] == "protocol_error"
+ # The backend saw its turn fail, not the request's phase or an upstream verdict.
+ assert record["terminal_error"] == {
+ "source_id": "src_primary01",
+ "configured_model_id": "shared-model",
+ "channel": "hub",
+ "reason": "backend_error",
+ "stream_started": None,
+ **recorded,
+ **({"backend_error_detail": expected_detail} if expected_detail else {}),
+ }
+ _assert_valid("turn-provenance.schema.json", record)
+ if error_kind == "stalled":
+ # Backend-only callbacks have no phase evidence, so neither boolean is
+ # valid even when the rest of a record is otherwise unchanged.
+ contradictory = (
+ {"stream_started": False}, {"stream_started": True},
+ {"http_status": 400}, {"http_status": None},
+ {"upstream_error_code": "invalid_request_error"}, {"upstream_error_code": None},
+ {"upstream_detail": "not an upstream response"},
+ )
+ for fields in contradictory:
+ invalid = {**record, "terminal_error": {**record["terminal_error"], **fields}}
+ with pytest.raises(AssertionError):
+ _assert_valid("turn-provenance.schema.json", invalid)
+
+
[email protected]("later_request", [False, True])
+def test_backend_failure_keeps_the_gateway_terminal_error_no_later_request_superseded(
+ tmp_path: Path, later_request: bool
+) -> None:
+ """A protocol failure has no copy projection; the gateway's exact record of it still outlives the backend's report."""
+ store = BoundedProvenanceStore(tmp_path / "gateway-terminal.json")
+ registry = TurnCorrelationRegistry(store)
+ exact_turn = _begin_hub_attempt(registry, turn_id="turn_gateway_terminal")
+ success = _outcome(RawOutcomeKind.SUCCESS)
+ registry.finish_attempt(exact_turn, outcome=success, decision=classify_outcome(success))
+ registry.begin_attempt(
+ exact_turn, source_id="src_backup01", resolved_model_id="shared-model", channel="hub", via_mapping=False,
+ )
+ failure = _outcome(RawOutcomeKind.PROTOCOL_ERROR, status=200, source_id="src_backup01")
+ registry.finish_attempt(exact_turn, outcome=failure, decision=classify_outcome(failure))
+ gateway_terminal = dict(registry._traces[exact_turn].terminal_error)
+ assert registry.terminal_projection(exact_turn, backend="codex") is None
+ if later_request:
+ registry.begin_attempt(
+ exact_turn, source_id="src_later001", resolved_model_id="shared-model", channel="hub", via_mapping=False,
+ )
+
+ registry.fail_hub_attempt(exact_turn, diagnostic="Codex exited")
+ # Nothing after the backend's report replaces the turn's record, not even a later success.
+ late = _outcome(RawOutcomeKind.SUCCESS, source_id="src_late0001")
+ registry.begin_attempt(
+ exact_turn, source_id="src_late0001", resolved_model_id="shared-model", channel="hub", via_mapping=False,
+ request_id="late",
+ )
+ registry.finish_attempt(exact_turn, outcome=late, decision=classify_outcome(late), request_id="late")
+ registry.settle(exact_turn, settled_by=SETTLED_BY_TERMINAL_RESULT, ts=NOW.isoformat())
+
+ record = store.get(exact_turn)
+ assert record is not None and record["outcome"] == "failed_terminal"
+ if later_request:
+ # The request still open when the backend failed is the one it reports on.
+ assert record["terminal_error"] == {
+ "source_id": "src_later001", "configured_model_id": "shared-model", "channel": "hub",
+ "reason": "backend_error", "stream_started": None, "backend_error_detail": "Codex exited",
+ }
+ else:
+ assert gateway_terminal["reason"] == "protocol_error" and gateway_terminal["stream_started"] is False
+ assert record["terminal_error"] == gateway_terminal
_assert_valid("turn-provenance.schema.json", record)
+def test_released_backend_rejection_records_read_unchanged(tmp_path: Path) -> None:
+ """Records the earlier bridge wrote stay as written: no reason or phase is re-derived on read."""
+ released = {
+ "contract_version": 11,
+ "turn_id": "turn_released_rejection",
+ "ts": NOW.isoformat(),
+ "agent": "vibey",
+ "requested_model_id": "shared-model",
+ "outcome": "failed_terminal",
+ "failed_attempts": [],
+ "served": None,
+ "terminal_error": {
+ "source_id": "src_primary01",
+ "configured_model_id": "shared-model",
+ "channel": "hub",
+ "reason": "protocol_error",
+ "stream_started": True,
+ },
+ "canceled_attempt": None,
+ "model_supply_state": None,
+ "blockers": [],
+ }
+ path = tmp_path / "provenance.json"
+ path.write_text(json.dumps([released]), encoding="utf-8")
+
+ assert BoundedProvenanceStore(path).get("turn_released_rejection") == released
+ _assert_valid("turn-provenance.schema.json", released)
+
+
def test_lost_terminal_retains_completed_fallback_attempts(
tmp_path: Path,
) -> None:
diff --git a/tests/test_model_hub_provenance.py b/tests/test_model_hub_provenance.py
index 481ff5540..037ce0f97 100644
--- a/tests/test_model_hub_provenance.py
+++ b/tests/test_model_hub_provenance.py
@@ -16,7 +16,7 @@ from core.handlers.model_hub.provenance import (
)
from core.handlers.model_hub.rpc import dispatch_model_hub_rpc
from core.handlers.model_hub.service import ModelHubError
-from core.run_settlement import SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED
+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED, SETTLED_BY_NO_TERMINAL_RESULT
from modules.agents.model_hub import ModelHubLaunch, ModelHubRuntimeRouter, bind_launch
from tests.test_model_hub_resolution import _config, _service, _source
from tests.test_model_hub_routing_modes import MODEL, _loaded_catalog_config, _sparse_config
@@ -97,16 +97,77 @@ def test_recovery_callbacks_are_material_post_lock_and_snapshots_are_copies(tmp_
assert not registry.store.path.exists()
-def test_native_hub_failure_keeps_the_gateway_terminal_projection_readable(tmp_path):
+def test_backend_callback_without_an_attempt_does_not_freeze_gateway_evidence(tmp_path):
registry = _live_registry(tmp_path)
+ registry.fail_hub_attempt("turn-live", diagnostic="backend exited before attribution")
projection = produce_turn_outcome("turn.engine_down")
registry.record_turn_outcome("turn-live", projection)
- registry.fail_hub_attempt("turn-live")
assert registry.terminal_projection("turn-live", backend="codex") == projection
assert not registry._traces["turn-live"].outcome_frozen
- registry.close_turn_admission("turn-live", settled_by=SETTLED_BY_STOPPED)
+
+
[email protected]("with_projection", [False, True])
+def test_new_attempt_retires_previous_request_terminal_before_fsm_drop(tmp_path, with_projection):
+ registry = _live_registry(tmp_path)
+ registry.begin_attempt(
+ "turn-live", source_id="src_previous1", resolved_model_id=MODEL, channel="hub", via_mapping=False,
+ )
+ outcome = RawCallOutcome(
+ kind=RawOutcomeKind.HTTP_ERROR if with_projection else RawOutcomeKind.PROTOCOL_ERROR,
+ http_status=400 if with_projection else 200,
+ error_code="invalid_request_error" if with_projection else None,
+ redacted_message=None, stream_started=False, model_id=MODEL, source_id="src_previous1",
+ )
+ registry.finish_attempt("turn-live", outcome=outcome, decision=classify_outcome(outcome))
+ if with_projection:
+ registry.record_turn_outcome("turn-live", produce_turn_outcome("turn.request_nonfallback"))
+ registry.begin_attempt(
+ "turn-live", source_id="src_successor1", resolved_model_id=MODEL, channel="hub", via_mapping=False,
+ )
+ assert registry._traces["turn-live"].terminal_outcome is None
+ registry.settle("turn-live", settled_by=SETTLED_BY_NO_TERMINAL_RESULT)
+ terminal = registry.store.get("turn-live")["terminal_error"]
+ assert terminal["source_id"] == "src_successor1"
+ assert terminal["reason"] == "stream_interrupted"
+
+
[email protected]("late_outcome", [RawOutcomeKind.SUCCESS, RawOutcomeKind.NETWORK_ERROR])
[email protected]("closure", [SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED])
+def test_native_hub_failure_keeps_the_gateway_terminal_projection_readable(tmp_path, late_outcome, closure):
+ registry = _live_registry(tmp_path)
+ registry.begin_attempt(
+ "turn-live", source_id="src_primary01", resolved_model_id=MODEL, channel="hub", via_mapping=False,
+ )
+ failure = RawCallOutcome(
+ kind=RawOutcomeKind.HTTP_ERROR, http_status=400, error_code="invalid_request_error",
+ redacted_message=None, stream_started=False, model_id=MODEL, source_id="src_primary01",
+ )
+ registry.finish_attempt("turn-live", outcome=failure, decision=classify_outcome(failure))
+ projection = produce_turn_outcome("turn.request_nonfallback")
+ registry.record_turn_outcome("turn-live", projection)
+ terminal = dict(registry._traces["turn-live"].terminal_error)
registry.fail_hub_attempt("turn-live")
- assert registry.terminal_projection("turn-live", backend="codex") is None
+ assert registry.terminal_projection("turn-live", backend="codex") == projection
+
+ # After the backend has acknowledged the terminal, late admission cannot
+ # clear it, and late success/interruption cannot contradict the notice.
+ registry.begin_attempt(
+ "turn-live", source_id="src_late0001", resolved_model_id=MODEL, channel="hub", via_mapping=False,
+ )
+ late = replace(failure, kind=late_outcome, error_code=None, stream_started=True, source_id="src_late0001")
+ registry.finish_attempt("turn-live", outcome=late, decision=classify_outcome(late))
+ registry.record_turn_outcome("turn-live", produce_turn_outcome("turn.served"))
+ assert registry.terminal_projection("turn-live", backend="codex") == projection
+ assert registry.terminal_projection("turn-live", backend="claude") is None
+ registry.close_turn_admission("turn-live", settled_by=closure)
+ assert registry.terminal_projection("turn-live", backend="codex") == (
+ None if closure == SETTLED_BY_STOPPED else projection
+ )
+ registry.fail_hub_attempt("turn-live")
+ registry.settle("turn-live", settled_by=closure)
+ record = registry.store.get("turn-live")
+ assert record["outcome"] == "failed_terminal"
+ assert record["terminal_error"] == terminal
@pytest.mark.parametrize("terminal", ["exhausted", "no_candidate"])
diff --git a/tests/test_model_hub_retention.py b/tests/test_model_hub_retention.py
index 21f67d75a..979077be9 100644
--- a/tests/test_model_hub_retention.py
+++ b/tests/test_model_hub_retention.py
@@ -381,7 +381,7 @@ def test_finalization_preserves_stop_and_backend_failure_before_late_success(tmp
assert record["outcome"] == expected
assert record["served"] is None
if backend_rejection:
- assert record["terminal_error"]["reason"] == "protocol_error"
+ assert record["terminal_error"]["reason"] == "backend_error"
elif reason != SETTLED_BY_STOPPED:
assert record["terminal_error"]["reason"] == "stream_interrupted"
else:
diff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.py
index 2283da229..cb33fbe66 100644
--- a/tests/test_model_hub_runtime.py
+++ b/tests/test_model_hub_runtime.py
@@ -42,7 +42,7 @@ from core.handlers.model_hub.classification import (
terminal_outcome_category,
)
from core.handlers.model_hub.errors import ModelDiscoveryError
-from core.handlers.model_hub.events import redact_untrusted_text
+from core.handlers.model_hub.events import bounded_diagnostic_detail, redact_untrusted_text
from core.handlers.model_hub.request import ModelHubRequest
from core.handlers.model_hub.stream_wire import (
ProtocolObservation,
@@ -7902,7 +7902,8 @@ def test_engine_upstream_detail_is_bounded_and_keeps_non_ascii_text() -> None:
detail = client_module._upstream_error_detail(payload, (("error",),))
assert detail is not None
- assert len(detail) == client_module._UPSTREAM_DETAIL_CHARS
+ # The persisted contract's bound (turn-provenance.schema.json maxLength).
+ assert len(detail) == 400
assert detail.startswith("模型不可用") and detail.endswith("…")
@@ -7922,7 +7923,7 @@ def test_engine_upstream_detail_is_bounded_and_keeps_non_ascii_text() -> None:
],
)
def test_engine_upstream_detail_redacts_labeled_opaque_secrets(message: str, expected: str) -> None:
- assert client_module._bounded_upstream_detail(message) == expected
+ assert bounded_diagnostic_detail(message) == expected
def test_engine_upstream_detail_redaction_stays_linear_on_hostile_labels() -> None:
@@ -7933,7 +7934,7 @@ def test_engine_upstream_detail_redaction_stays_linear_on_hostile_labels() -> No
def test_engine_upstream_detail_cannot_ping_the_channel_it_is_rendered_into() -> None:
- detail = client_module._bounded_upstream_detail(
+ detail = bounded_diagnostic_detail(
"ask @everyone or @ops_lead, <@123> <@&456> <!here> <#C1>"
)
@@ -7953,7 +7954,7 @@ def test_engine_upstream_detail_cannot_ping_the_channel_it_is_rendered_into() ->
],
)
def test_engine_upstream_detail_cannot_hide_a_link_behind_trusted_copy(message: str) -> None:
- detail = client_module._bounded_upstream_detail(message)
+ detail = bounded_diagnostic_detail(message)
assert detail is not None
assert "](" not in detail.replace(" ", "")
@@ -7963,7 +7964,7 @@ def test_engine_upstream_detail_cannot_hide_a_link_behind_trusted_copy(message:
def test_engine_upstream_detail_keeps_email_addresses_usable() -> None:
- assert client_module._bounded_upstream_detail("Contact [email protected], cc @ops or x@everyone") == (
+ assert bounded_diagnostic_detail("Contact [email protected], cc @ops or x@everyone") == (
"Contact [email protected], cc @\u200bops or x@\u200beveryone"
)
diff --git a/tests/test_vibey_agent.py b/tests/test_vibey_agent.py
index 2d64b7127..49d618492 100644
--- a/tests/test_vibey_agent.py
+++ b/tests/test_vibey_agent.py
@@ -540,7 +540,7 @@ async def test_a_setup_failure_after_the_route_resolved_fails_the_turn_and_not_t
reported: list[str] = []
- async def record_native_failure(context, diagnostic) -> bool:
+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:
reported.append(diagnostic)
return False
@@ -1166,7 +1166,7 @@ async def test_a_failed_run_shows_localized_copy_and_a_refusal_shows_its_explana
)
reported: list[str] = []
- async def record_native_failure(context, diagnostic) -> bool:
+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:
reported.append(diagnostic)
return False
@@ -1240,7 +1240,7 @@ async def test_a_run_failure_is_reported_to_model_hub_like_the_other_backends(en
reported: list[str] = []
- async def record_native_failure(context, diagnostic) -> bool:
+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:
reported.append(diagnostic)
return False
@@ -1255,6 +1255,49 @@ async def test_a_run_failure_is_reported_to_model_hub_like_the_other_backends(en
assert len(reported) == 1 and harness.controller.terminals[-1]["is_error"] is True
+async def test_model_hub_records_a_source_stall_as_the_backend_reported_it(engine, session, tmp_path, published) -> None:
+ """MH-TURN-RESULT-001: the Turn's Model Hub record holds what Vibey observed through the shared failure bridge:
+ the stall's kind and sanitized diagnostic, never a guessed protocol error or failing-request output phase."""
+ from core.agent_core.ai.provider import ProviderError
+ from core.handlers.model_hub.provenance import BoundedProvenanceStore, TurnCorrelationRegistry
+ from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
+ from modules.agents.model_hub import ModelHubRuntimeRouter
+ from tests.test_model_hub_l3 import _assert_valid
+
+ stall = ProviderError("stalled", "provider sent no data for 600s\n\u202e上游 key=sk-proj-stall0secret0value", False)
+ harness = _Harness(engine, tmp_path, "telegram", [[stall]])
+ registry = TurnCorrelationRegistry(BoundedProvenanceStore(tmp_path / "provenance.json"))
+ router = ModelHubRuntimeRouter(
+ service=SimpleNamespace(), turn_gateway=SimpleNamespace(correlation=registry), overlay_path=tmp_path / "overlay.json"
+ )
+ # Only the route is the harness's; the failure goes through the shared router into the real registry.
+ router.resolve = harness.controller._resolve
+ harness.controller.model_hub_runtime = router
+ request = harness.request("write the page")
+ turn_id = _turn(request.context)
+ token = registry.credentials("vibey", f"vibey:{harness.session_id}", turn_id)
+ registry.begin_gateway_request(backend="vibey", token=token, requested_model_id="test-model")
+ registry.begin_attempt(
+ turn_id, source_id="src_stalled01", resolved_model_id="test-model", channel="hub", via_mapping=False
+ )
+
+ await harness.agent.handle_message(request)
+ registry.settle(turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT)
+
+ record = registry.store.get(turn_id)
+ assert record["outcome"] == "failed_terminal" and record["served"] is None
+ assert record["terminal_error"] == {
+ "source_id": "src_stalled01",
+ "configured_model_id": "test-model",
+ "channel": "hub",
+ "reason": "backend_error",
+ "stream_started": None,
+ "error_kind": "stalled",
+ "backend_error_detail": "provider sent no data for 600s 上游 key=[redacted]",
+ }
+ _assert_valid("turn-provenance.schema.json", record)
+
+
async def test_an_unreadable_image_is_listed_by_path_and_the_turn_runs(engine, session, tmp_path, published) -> None:
from modules.im.base import FileAttachment
diff --git a/tests/test_vibey_agent_context.py b/tests/test_vibey_agent_context.py
index 46cfbf894..eab7e6dd9 100644
--- a/tests/test_vibey_agent_context.py
+++ b/tests/test_vibey_agent_context.py
@@ -216,7 +216,7 @@ async def test_a_skill_load_through_bash_is_marked_in_its_context_row(engine, se
async def test_a_context_that_cannot_fit_locally_is_never_reported_as_a_source_failure(engine, session, tmp_path):
reported: list[str] = []
- async def record_native_failure(context, diagnostic) -> bool:
+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:
reported.append(diagnostic)
return False
@@ -283,7 +283,7 @@ _CUT_CALL = [Done(assistant(calls=[ToolCallBlock("cut", "bash", {"command": "ls"
async def test_model_hub_hears_only_of_failures_the_served_source_produced(engine, session, tmp_path, case, recorded):
reported: list[str] = []
- async def record_native_failure(context, diagnostic) -> bool:
+ async def record_native_failure(context, diagnostic, *, error_kind=None) -> bool:
reported.append(diagnostic)
return False
diff --git a/ui/src/components/settings/models/RouteRecordedTurn.test.tsx b/ui/src/components/settings/models/RouteRecordedTurn.test.tsx
index 4dee8c726..d1102661f 100644
--- a/ui/src/components/settings/models/RouteRecordedTurn.test.tsx
+++ b/ui/src/components/settings/models/RouteRecordedTurn.test.tsx
@@ -75,6 +75,20 @@ describe('recorded turn error', () => {
await user.click(screen.getByRole('button', { name: 'View error details' }));
expect(screen.getByRole('dialog').querySelector('pre')?.textContent).toBe(JSON.stringify(historical, null, 2));
});
+ it('names a backend-reported failure and its kind instead of an upstream or protocol fault', async () => {
+ const user = userEvent.setup();
+ const reported: TurnProvenance = { ...record, terminal_error: {
+ source_id: 'src_historical', configured_model_id: 'historical-model', channel: 'hub', reason: 'backend_error',
+ stream_started: null, error_kind: 'stalled', backend_error_detail: 'provider sent no data for 600s',
+ } };
+ vi.spyOn(modelsApi, 'getAgentProvenance').mockResolvedValue(reported);
+ render(view());
+ expect(await screen.findByText('Latest recorded turn · Agent backend reported a failure')).toBeTruthy();
+ expect(screen.getByText('The model request timed out waiting for data')).toBeTruthy();
+ expect(screen.queryByText(/Protocol error|model not found/)).toBeNull();
+ await user.click(screen.getByRole('button', { name: 'View error details' }));
+ expect(screen.getByRole('dialog').querySelector('pre')?.textContent).toBe(JSON.stringify(reported, null, 2));
+ });
it.each(['served', 'canceled', null] as const)('does not show an old error when latest outcome is %s', async (outcome) => {
const read = vi.spyOn(modelsApi, 'getAgentProvenance').mockResolvedValue(outcome ? { ...record, outcome, terminal_error: null } : null);
render(view());
diff --git a/ui/src/components/settings/models/RouteRecordedTurn.tsx b/ui/src/components/settings/models/RouteRecordedTurn.tsx
index b6af62036..ff31781dd 100644
--- a/ui/src/components/settings/models/RouteRecordedTurn.tsx
+++ b/ui/src/components/settings/models/RouteRecordedTurn.tsx
@@ -7,18 +7,19 @@ import type { RecordedError, RecordedTurn } from './recordedTurn';
function useReason(record: RecordedError) {
const { t } = useTranslation();
const error = record.terminal_error;
- return error.upstream_error_code === 'model_not_found'
+ return error.reason !== 'backend_error' && error.upstream_error_code === 'model_not_found'
? t('settings.models.routing.modelNotFound')
: t(`settings.models.routing.errorReason.${error.reason}`);
}
function RecordedSummary({ record }: { record: RecordedError }) {
- const { i18n } = useTranslation();
+ const { t, i18n } = useTranslation();
const error = record.terminal_error;
return <>
<time dateTime={record.ts}>{new Date(record.ts).toLocaleString(i18n.language)}</time>
<span className="font-mono">{error.source_id ?? '—'} · {error.configured_model_id ?? '—'}</span>
- {error.upstream_error_code && <span className="font-mono">{error.upstream_error_code}</span>}
+ {error.reason !== 'backend_error' && error.upstream_error_code && <span className="font-mono">{error.upstream_error_code}</span>}
+ {error.error_kind && <span>{t(`settings.models.routing.errorKind.${error.error_kind}`, { defaultValue: error.error_kind })}</span>}
</>;
}
@@ -29,7 +30,8 @@ export function RecordedTurnBadge({ record, onOpen }: { record: RecordedError; o
const label = `${t('settings.models.routing.latestRecorded')} · ${reason}`;
return <button type="button" className="model-hub-recorded-turn-badge flex shrink-0 items-center font-mono" aria-label={label} title={label} onClick={onOpen}>
<AlertTriangle aria-hidden="true" />
- {record.terminal_error.http_status ?? record.terminal_error.upstream_error_code ?? t('settings.models.routing.errorShort')}
+ {(record.terminal_error.reason !== 'backend_error'
+ ? record.terminal_error.http_status ?? record.terminal_error.upstream_error_code : null) ?? t('settings.models.routing.errorShort')}
</button>;
}
diff --git a/ui/src/components/settings/models/types.ts b/ui/src/components/settings/models/types.ts
index a269e02a3..f4ec5696b 100644
--- a/ui/src/components/settings/models/types.ts
+++ b/ui/src/components/settings/models/types.ts
@@ -626,16 +626,30 @@ export type TurnProvenance = {
source_id: string | null;
configured_model_id: string | null;
channel: SupplyChannel | null;
+ local_error_detail?: string;
+ origin?: { provider: string; api: SourceProtocol; model: string };
+ stripped_reasoning_efforts?: string[];
+ declared_reasoning_efforts?: string[];
+ } & ({
reason: 'invalid_parameter' | 'protocol_error' | 'tool_incompatible' | 'stream_interrupted' | 'engine_down';
stream_started: boolean;
http_status?: number | null;
upstream_error_code?: string | null;
upstream_detail?: string;
- local_error_detail?: string;
- origin?: { provider: string; api: SourceProtocol; model: string };
- stripped_reasoning_efforts?: string[];
- declared_reasoning_efforts?: string[];
- } | null;
+ error_kind?: never;
+ backend_error_detail?: never;
+ } | {
+ reason: 'backend_error';
+ /** The reporter has no failing-request phase evidence or upstream API verdict. */
+ stream_started: null;
+ http_status?: never;
+ upstream_error_code?: never;
+ upstream_detail?: never;
+ /** The backend's typed failure kind, a machine identifier such as `stalled`. */
+ error_kind?: string;
+ /** The backend's own redacted diagnostic, never an upstream answer. */
+ backend_error_detail?: string;
+ }) | null;
model_supply_state: 'waiting' | 'interrupted' | null;
blockers: Array<RouteHop & { reason: ChainUnavailableReason; retry_at?: string | null }>;
};
diff --git a/ui/src/components/workbench/FailureDetails.test.tsx b/ui/src/components/workbench/FailureDetails.test.tsx
index 868abbb62..f6b7c7631 100644
--- a/ui/src/components/workbench/FailureDetails.test.tsx
+++ b/ui/src/components/workbench/FailureDetails.test.tsx
@@ -213,6 +213,40 @@ describe('failed-turn upstream details', () => {
expect(screen.queryByText(/由上游供应商的 API 返回/)).toBeNull();
});
+ it.each([
+ ['stalled', '模型请求等待数据超时', {}],
+ ['ProviderProtocolViolation', 'ProviderProtocolViolation', {}],
+ ['stalled', '模型请求等待数据超时', {
+ http_status: 418, upstream_error_code: 'invalid_request_error', upstream_detail: 'untrusted mixed record',
+ }],
+ ])('shows a backend-reported %s failure as the backend\'s diagnostic, never an upstream answer', async (kind, shown, upstreamFields) => {
+ vi.spyOn(modelsApi, 'getTurnProvenance').mockResolvedValue({
+ ...record,
+ agent: 'vibey',
+ failed_attempts: [],
+ terminal_error: {
+ source_id: 'src_b', configured_model_id: 'grok-4.6-beta', channel: 'hub', reason: 'backend_error',
+ stream_started: null, error_kind: kind, backend_error_detail: 'provider sent no data for 600s',
+ ...upstreamFields,
+ },
+ blockers: [],
+ } as unknown as TurnProvenance);
+ mount();
+ fireEvent.click(await screen.findByRole('button', { name: '查看详情' }));
+ expect(screen.getByText('grok-4.6 的请求失败了,原因见下方。')).toBeTruthy();
+ expect(screen.getByText('Agent 后端报告了失败')).toBeTruthy();
+ expect(screen.getByText(/失败类型/)).toBeTruthy();
+ expect(screen.getByText(shown)).toBeTruthy();
+ expect(screen.getByText(/Agent 后端报告:/)).toBeTruthy();
+ expect(screen.getByText('provider sent no data for 600s')).toBeTruthy();
+ expect(screen.queryByText(/上游返回/)).toBeNull();
+ expect(screen.queryByText(/上游 API 拒绝/)).toBeNull();
+ expect(screen.queryByText(/由上游供应商的 API 返回/)).toBeNull();
+ expect(screen.queryByText('418')).toBeNull();
+ expect(screen.queryByText('invalid_request_error')).toBeNull();
+ expect(screen.queryByText('untrusted mixed record')).toBeNull();
+ });
+
it('names Avibe-side terminal failures and localizes every blocker reason', async () => {
vi.spyOn(modelsApi, 'getTurnProvenance').mockResolvedValue({
...record,
diff --git a/ui/src/components/workbench/FailureDetails.tsx b/ui/src/components/workbench/FailureDetails.tsx
index 0e13c5477..abacb756c 100644
--- a/ui/src/components/workbench/FailureDetails.tsx
+++ b/ui/src/components/workbench/FailureDetails.tsx
@@ -7,8 +7,10 @@
// user see the refusal came from the upstream API and not from the gateway.
//
// The upstream message is kept only as the engine client's credential-redacted,
-// 400-character projection. Local OS reasons are also carried by the notice, so
-// details survive an unavailable provenance read.
+// 400-character projection. A failure the agent backend reported on its own is
+// labeled as the backend's diagnostic, never as an upstream answer. Local OS
+// reasons are also carried by the notice, so details survive an unavailable
+// provenance read.
import * as React from 'react';
import { ChevronDown, ChevronRight } from 'lucide-react';
import { useTranslation } from 'react-i18next';
@@ -27,12 +29,21 @@ type Row = {
model: string | null;
status: number | null;
code: string | null;
+ // The agent backend's typed failure kind, for a failure it reported itself.
+ kind: string | null;
reason: string;
detail: string | null;
- // A network attempt's text is the connection failure, not an upstream answer.
- connection: boolean;
+ // Who said `detail`: a network attempt's text is the connection failure and
+ // a backend-reported failure's is the backend's, neither an upstream answer.
+ said: 'upstream' | 'connection' | 'backend';
};
+const DETAIL_LABEL = {
+ upstream: 'chat.failureDetails.upstreamDetail',
+ connection: 'chat.failureDetails.connectionDetail',
+ backend: 'chat.failureDetails.backendDetail',
+} as const satisfies Record<Row['said'], string>;
+
const RECORD_RETRIES = 3;
const RECORD_RETRY_MS = 1_000;
@@ -100,21 +111,24 @@ export function FailureDetails({ message }: { message: WorkbenchMessage }) {
model: entry.configured_model_id,
status: entry.http_status ?? null,
code: null,
+ kind: null,
reason: t(`chat.failureDetails.reason.${entry.reason}`, { defaultValue: entry.reason }),
detail: entry.upstream_detail ?? null,
- connection: entry.reason === 'network',
+ said: entry.reason === 'network' ? 'connection' : 'upstream',
}));
const terminal = record.terminal_error;
if (terminal) {
+ const backend = terminal.reason === 'backend_error';
attempts.push({
key: 'terminal',
source: name(terminal.source_id),
model: terminal.configured_model_id,
- status: terminal.http_status ?? null,
- code: terminal.upstream_error_code ?? null,
+ status: backend ? null : terminal.http_status ?? null,
+ code: backend ? null : terminal.upstream_error_code ?? null,
+ kind: terminal.error_kind ?? null,
reason: t(`settings.models.routing.errorReason.${terminal.reason}`),
- detail: terminal.upstream_detail ?? null,
- connection: false,
+ detail: (backend ? terminal.backend_error_detail : terminal.upstream_detail) ?? null,
+ said: backend ? 'backend' : 'upstream',
});
}
return attempts;
@@ -130,7 +144,7 @@ export function FailureDetails({ message }: { message: WorkbenchMessage }) {
? 'failed_local'
: record.outcome;
const upstreamFacts = attempts.some(
- (row) => row.status !== null || row.code !== null || (row.detail !== null && !row.connection),
+ (row) => row.status !== null || row.code !== null || (row.detail !== null && row.said === 'upstream'),
);
return (
<div className="flex flex-col gap-2">
@@ -145,11 +159,12 @@ export function FailureDetails({ message }: { message: WorkbenchMessage }) {
<span className="flex flex-wrap gap-x-3 gap-y-0.5 text-gold-ink/70">
<span>{t('chat.failureDetails.httpStatus')}: <span className="font-mono text-gold-ink">{row.status ?? '—'}</span></span>
{row.code && <span>{t('chat.failureDetails.errorCode')}: <span className="font-mono text-gold-ink">{row.code}</span></span>}
+ {row.kind && <span>{t('chat.failureDetails.errorKind')}: <span className="text-gold-ink">{t(`settings.models.routing.errorKind.${row.kind}`, { defaultValue: row.kind })}</span></span>}
<span>{t('chat.failureDetails.reasonLabel')}: <span className="text-gold-ink">{row.reason}</span></span>
</span>
{row.detail && (
<span className="text-gold-ink/70">
- {t(row.connection ? 'chat.failureDetails.connectionDetail' : 'chat.failureDetails.upstreamDetail')}:{' '}
+ {t(DETAIL_LABEL[row.said])}:{' '}
<span className="whitespace-pre-wrap break-words font-mono text-gold-ink">{row.detail}</span>
</span>
)}
diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json
index e2e643cbc..168f88e2f 100644
--- a/ui/src/i18n/en.json
+++ b/ui/src/i18n/en.json
@@ -1324,7 +1324,11 @@
"protocol_error": "Protocol error",
"tool_incompatible": "Tool incompatible",
"stream_interrupted": "Stream interrupted",
- "engine_down": "Gateway unavailable"
+ "engine_down": "Gateway unavailable",
+ "backend_error": "Agent backend reported a failure"
+ },
+ "errorKind": {
+ "stalled": "The model request timed out waiting for data"
},
"origin": {
"automatic": "Automatic",
@@ -4831,10 +4835,12 @@
"httpStatus": "HTTP status",
"errorCode": "Error code",
"reasonLabel": "Reason",
+ "errorKind": "Failure kind",
"blockers": "Sources that were unavailable",
"upstreamNote": "These come from the upstream provider's API. Avibe keeps the status, error code, and the provider's error message with credentials removed, up to 400 characters.",
"upstreamDetail": "Upstream said",
"connectionDetail": "Connection error",
+ "backendDetail": "Agent backend reported",
"copyRecord": "Copy record",
"reason": {
"quota_exhausted": "Quota exhausted",
diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.json
index 3034433bb..d58afb584 100644
--- a/ui/src/i18n/zh.json
+++ b/ui/src/i18n/zh.json
@@ -1324,7 +1324,11 @@
"protocol_error": "协议错误",
"tool_incompatible": "工具不兼容",
"stream_interrupted": "响应中断",
- "engine_down": "模型网关不可用"
+ "engine_down": "模型网关不可用",
+ "backend_error": "Agent 后端报告了失败"
+ },
+ "errorKind": {
+ "stalled": "模型请求等待数据超时"
},
"origin": {
"automatic": "自动",
@@ -4831,10 +4835,12 @@
"httpStatus": "HTTP 状态码",
"errorCode": "错误码",
"reasonLabel": "原因",
+ "errorKind": "失败类型",
"blockers": "当时不可用的供应商",
"upstreamNote": "以上内容均由上游供应商的 API 返回。Avibe 会保留状态码、错误码,以及去除凭据后的上游报错信息(最多 400 字)。",
"upstreamDetail": "上游返回",
"connectionDetail": "连接错误",
+ "backendDetail": "Agent 后端报告",
"copyRecord": "复制记录",
"reason": {
"quota_exhausted": "额度用尽",
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index 3b896e553..f50b6e3da 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -31,7 +31,7 @@ from core.handlers.model_hub.adapter import (
)
from core.handlers.model_hub.async_owner import run_owned_in_thread
from core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES
-from core.handlers.model_hub.events import redact_untrusted_text
+from core.handlers.model_hub.events import bounded_diagnostic_detail
from core.handlers.model_hub.json_wire import (
JSONEvent,
JSONPath,
@@ -39,7 +39,6 @@ from core.handlers.model_hub.json_wire import (
project_json_reader,
)
from core.handlers.model_hub.request import FORWARDED_CALLER_HEADERS
-from core.message_output import plain_untrusted_text
from core.os_errors import local_os_errno
from core.handlers.model_hub.stream_wire import (
ErrorEnvelopePath,
@@ -57,8 +56,6 @@ from vibe.model_hub_runtime.state import SourceRecord
_STREAM_CHUNK_BYTES = 64 * 1024
-# Upper bound on upstream error text shown to the user in a terminal message.
-_UPSTREAM_DETAIL_CHARS = 400
# The engine answers a failed upstream connection itself, before any upstream
# response, with exactly its own generic envelope and Go's transport error text.
# That is a network fact, not an upstream server verdict.
@@ -69,9 +66,6 @@ _ENGINE_TRANSPORT_FAILURE = re.compile(
r"http2: (?:client connection lost|server sent GOAWAY)|server closed idle connection|"
r"no such host|network is unreachable|proxyconnect"
)
-# C0/C1 controls and every Unicode Bidi_Control character: either can act on
-# whatever renders the text (a terminal, a log viewer, a browser).
-_CONTROL_CHARACTERS = re.compile(r"[\x00-\x1f\x7f-\x9f\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]")
# This threshold only selects memory or a temporary file; it never rejects or
# truncates upstream response bytes.
_PRELUDE_MEMORY_BYTES = 256 * 1024
@@ -1746,7 +1740,7 @@ def _reduce_protocol_observation(
observation.error_envelope_paths,
)
upstream_detail = (
- _bounded_upstream_detail(observation.error_message)
+ bounded_diagnostic_detail(observation.error_message)
if observation.error_message is not None
else _upstream_error_detail(
observation.error_payload or b"",
@@ -1944,7 +1938,7 @@ def _upstream_error_detail(
if not payload or not project_json_reader(io.BytesIO(payload), message_paths, visit):
return None
for path in message_paths:
- text = _bounded_upstream_detail(values.get(path, ""))
+ text = bounded_diagnostic_detail(values.get(path, ""))
if text:
return text
return None
@@ -1996,18 +1990,6 @@ def _upstream_reset_advice(
return None
-def _bounded_upstream_detail(message: str) -> str | None:
- # Control characters (terminal escapes, BEL) would act on whatever renders
- # the text, a log viewer included; they carry no message content.
- text = " ".join(_CONTROL_CHARACTERS.sub(" ", message).split())
- if not text:
- return None
- text = plain_untrusted_text(redact_untrusted_text(text))
- if len(text) > _UPSTREAM_DETAIL_CHARS:
- text = text[: _UPSTREAM_DETAIL_CHARS - 1].rstrip() + "…"
- return text
-
-
def _safe_error_code(value: object) -> str | None:
if not isinstance(value, str) or value not in UPSTREAM_MACHINE_ERROR_CODES:
return None