Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): remove inference deadlines and preserve recovery (#1939)
Call White0766903d53a0
diff --git a/config/v2_config.py b/config/v2_config.pyindex bef51813d..c5d71f11e 100644--- a/config/v2_config.py+++ b/config/v2_config.py@@ -178,47 +178,14 @@ DEFAULT_HARNESS_RUN_QUEUED_TTL_SECONDS = 1800# three: a session actively recovering its queue keeps touching the row.DEFAULT_HARNESS_RUN_HOLD_TTL_SECONDS = 3600-# Absolute-time backstop for evicting a Codex transport whose turn is stuck-# "active" forever (e.g. the ``codex app-server`` wedged or silently-# disconnected after ``turn/start`` but before ``turn/completed``, so-# ``_active_turns`` is never cleared). Without this, ``evict_idle_transports``-# treats an active turn as an ABSOLUTE veto and the wedged app-server process-# leaks until service restart (mirrors the Claude leak in #622/#623).-#-# A transport with an active turn is force-evicted once it has been idle for-# ``max(idle_timeout * MULTIPLIER, FLOOR_SECONDS)``. Set the multiplier <= 0 to-# disable the backstop entirely.-#-# TRADE-OFF: this cap is driven purely by ``last_activity`` (refreshed on every-# Codex notification), so it CANNOT distinguish a genuinely wedged turn from a-# legitimately long, fully-silent one. A single tool/MCP run or model "thinking"-# phase that emits no notifications for longer than the cap will be misjudged as-# stuck and have its transport torn down. The multiplier defaults higher than a-# typical tool-run assumption, and the floor guarantees a >= 30 min window even-# when ``idle_timeout`` is configured small, to keep that false-positive rare.+# Repair stale Codex adapter-local active flags only after durable ownership+# allows reclamation. Silence never overrides an owned Turn or Activity.+# The repair threshold is max(idle_timeout * multiplier, floor); a nonpositive+# multiplier disables stale-flag repair.DEFAULT_CODEX_STUCK_ACTIVE_IDLE_EVICTION_MULTIPLIER = 3DEFAULT_CODEX_STUCK_ACTIVE_IDLE_EVICTION_FLOOR_SECONDS = 1800-# Absolute-age backstop for idle eviction. A Claude session that is still-# flagged ``active`` (its per-turn receiver never released the flag, e.g. a-# long-lived receiver blocked on ``receive_messages`` with no stream EOF) is-# force-evicted once its ``last_activity`` is older than-# ``max(idle_timeout * STUCK_ACTIVE_IDLE_EVICTION_MULTIPLIER, FLOOR_SECONDS)``.-# This decouples eviction from the receiver's flag-release logic, so a-# stuck-active session can no longer pin its ~220MB ``claude`` subprocess until-# the next service restart. A genuine in-flight turn keeps touching-# ``last_activity`` (assistant/tool messages), so it normally stays well under-# this cap. Set the multiplier to 0 to disable the backstop.-#-# Trade-off: ``last_activity`` is only refreshed when an SDK message arrives.-# Because a stuck (blocked-receiver) session and a session running a single-# silent tool call are indistinguishable from ``last_activity`` alone, a real-# turn whose ONE tool invocation runs silently for longer than-# the cap would be force-evicted mid-turn. The default cap is at least 30min-# because Claude Code's Bash tool caps at 10min, so a single 30min-silent turn-# is not expected in practice; raise the multiplier if your deployment runs-# longer silent tools (e.g. long builds via custom/MCP tools that emit no-# intermediate messages).+# Claude uses the same ownership-first policy for its receiver's active flag.DEFAULT_STUCK_ACTIVE_IDLE_EVICTION_MULTIPLIER = 3DEFAULT_STUCK_ACTIVE_IDLE_EVICTION_FLOOR_SECONDS = 1800DEFAULT_OPENCODE_ERROR_RETRY_LIMIT = 1diff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.pyindex 4979d0157..960a32666 100644--- a/core/handlers/model_hub/adapter.py+++ b/core/handlers/model_hub/adapter.py@@ -10,6 +10,7 @@ orchestrator and land with every affected consumer on the same tested head.from __future__ import annotations+import asynciofrom dataclasses import dataclassfrom enum import Enumfrom typing import Any, AsyncIterator, Callable, Final, Literal, Mapping, Protocol, Sequence@@ -89,6 +90,14 @@ class OriginNotAllowedError(Exception):``RawCallOutcome`` and never triggers fallback."""+class InvokeCancelledError(asyncio.CancelledError):+ """Owner cancellation carrying wire facts observed before transport cleanup."""++ def __init__(self, observed: ProtocolSSEState) -> None:+ super().__init__()+ self.observed = observed++class RawOutcomeKind(str, Enum):SUCCESS = "success"HTTP_ERROR = "http_error"@@ -553,5 +562,7 @@ class EngineAdapter(Protocol):``on_admitted`` exactly once before any network wait. Callback failurereleases the lease without invoking upstream. Transport completion,close, or cancellation releases the lease independently of settlement.+ Cancellation before handle return raises ``InvokeCancelledError`` when+ wire facts were observed, so L2 can meter them without inventing failure."""...diff --git a/core/handlers/model_hub/provenance.py b/core/handlers/model_hub/provenance.pyindex 69a7ac947..6b898dee8 100644--- a/core/handlers/model_hub/provenance.py+++ b/core/handlers/model_hub/provenance.py@@ -35,7 +35,7 @@ from .events import (SOURCE_DETAIL_EVENT_REASONS,event_reason_label,)-from .resolver import ModelHubTurnResolution, source_eligible_for_backend+from .resolver import ModelHubTurnResolution, parse_model_hub_timestamp, source_eligible_for_backendfrom .state_file import write_state_document@@ -446,6 +446,7 @@ def turn_supply_facts(source=", ".join(source.display_name for source in cooling),retry_at=min((source.state.retry_at or "" for source in cooling),+ key=parse_model_hub_timestamp,default="",),blockers=tuple(blockers),@@ -1484,6 +1485,11 @@ class TurnCorrelationRegistry:trace = self._traces.get(turn_id)if trace is None or trace.outcome_frozen:return+ # Admission supersedes earlier supply failures, not their attempt history.+ trace.model_supply_state = None+ trace.blockers = []+ if trace.terminal_outcome is not None and trace.terminal_outcome.outcome in {"no_candidate", "exhausted"}:+ trace.terminal_outcome = Nonetrace.pending_attempts[request_id] = AttemptIdentity(source_id=source_id,resolved_model_id=resolved_model_id,diff --git a/core/handlers/model_hub/resolver.py b/core/handlers/model_hub/resolver.pyindex a3f4546df..8258c5d27 100644--- a/core/handlers/model_hub/resolver.py+++ b/core/handlers/model_hub/resolver.py@@ -4,7 +4,7 @@ from __future__ import annotationsimport refrom dataclasses import dataclass, replace-from datetime import datetime+from datetime import datetime, timezonefrom typing import Literalfrom config.v2_config import (@@ -104,11 +104,18 @@ def source_eligible_for_backend(source: ModelHubSourceConfig, backend: str) -> breturn ModelHubConfig.source_eligible_for_backend(source, backend)+def parse_model_hub_timestamp(value: str) -> datetime:+ """Compare ISO instants, interpreting older naive records as UTC."""++ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))+ return parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed++def source_retry_ready(source: ModelHubSourceConfig, now: datetime | None) -> bool:if source.state.status != "cooldown" or now is None:return Falsetry:- retry_at = datetime.fromisoformat((source.state.retry_at or "").replace("Z", "+00:00"))+ retry_at = parse_model_hub_timestamp(source.state.retry_at or "")except ValueError:return Falsereturn retry_at <= nowdiff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.pyindex 5c8fceee5..c8aca22e1 100644--- a/core/handlers/model_hub/service.py+++ b/core/handlers/model_hub/service.py@@ -53,6 +53,7 @@ from .adapter import (EngineEnsureResult,EngineHealth,EngineStatus,+ InvokeCancelledError,InvokeHandle,OAuthFlowState,OriginNotAllowedError,@@ -122,6 +123,7 @@ from .resolver import (effective_model_route,inspect_exact_hop,matching_v1_model_id as _matching_v1_model_id,+ parse_model_hub_timestamp,resolve_model_hub_turn,source_after_cooldown_recovery,source_eligible_for_backend,@@ -498,21 +500,6 @@ def _source_id() -> str:return f"src_{uuid.uuid4().hex[:12]}"-def _parse_datetime(value: str) -> datetime:- """Parse a timestamp into something comparable with this service's clock.-- Every parsed value is compared against ``self.now()``, which is UTC-aware.- A provider or an older persisted record may still carry a naive ISO string,- and comparing the two raises ``TypeError`` rather than answering the- question — so read a naive timestamp as the UTC it was written as.- """-- parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))- if parsed.tzinfo is None:- return parsed.replace(tzinfo=timezone.utc)- return parsed--def _mask_credential(value: str) -> str:"""Create the one-way display mask frozen by the source contract."""normalized = value.strip()@@ -1674,7 +1661,7 @@ class ModelHubService:if not flow.expires_at_iso or flow.state in {"success", "failed", "cancelled"}:returntry:- expired = _parse_datetime(flow.expires_at_iso) <= self.now()+ expired = parse_model_hub_timestamp(flow.expires_at_iso) <= self.now()except ValueError:returnif expired:@@ -5373,6 +5360,7 @@ class ModelHubService:for item in chain_payload["chain"]if item["retry_at"]),+ key=parse_model_hub_timestamp,default=None,)raise ModelHubError(@@ -6183,7 +6171,7 @@ class ModelHubService:if (source.state.status == "cooldown"and source.state.retry_at is not None- and _parse_datetime(source.state.retry_at) >= retry_at+ and parse_model_hub_timestamp(source.state.retry_at) >= retry_at):return Falseprevious = self._clone_config(config)@@ -6624,23 +6612,17 @@ class ModelHubService:exact_retry: bool = False,on_admitted: Callable[[], None] | None = None,) -> tuple[InvokeHandle, Optional[RawCallOutcome], asyncio.CancelledError | None]:- transport_admitted = False-- def admitted() -> None:- nonlocal transport_admitted- transport_admitted = True- if on_admitted is not None:- on_admitted()+ acquired_handle: InvokeHandle | None = None- async def meter_handle(- handle: InvokeHandle,+ async def meter_observed(+ observed: ProtocolSSEState | None,outcome: RawCallOutcome | None,) -> None:await self._meter_call(source_id=source.id,model_id=model_id,outcome=outcome,- observed=handle.observed,+ observed=observed,)async def meter_available_outcome(@@ -6649,22 +6631,28 @@ class ModelHubService:if handle.stream is not None and not handle.outcome_available:return Noneoutcome = await self._engine_call(handle.outcome())- await meter_handle(handle, outcome)+ await meter_observed(handle.observed, outcome)return outcomeasync def invoke_and_meter_bodyless() -> tuple[InvokeHandle, Optional[RawCallOutcome]]:- handle = await self._invoke_admitted(- source=source,- model_id=model_id,- requested_model_id=requested_model_id,- request=request,- stream=stream,- backend=cast(BackendName, backend),- excluded_source_ids=excluded_source_ids,- supply_channel=supply_channel,- exact_retry=exact_retry,- on_admitted=admitted,- )+ nonlocal acquired_handle+ try:+ handle = await self._invoke_admitted(+ source=source,+ model_id=model_id,+ requested_model_id=requested_model_id,+ request=request,+ stream=stream,+ backend=cast(BackendName, backend),+ excluded_source_ids=excluded_source_ids,+ supply_channel=supply_channel,+ exact_retry=exact_retry,+ on_admitted=on_admitted,+ )+ except InvokeCancelledError as cancelled:+ await meter_observed(cancelled.observed, None)+ raise+ acquired_handle = handleif handle.stream is not None:# The body is the gateway's to forward, so the tokens in it are the# gateway's to meter.@@ -6679,7 +6667,9 @@ class ModelHubService:handle, outcome = await asyncio.shield(attempt_task)except asyncio.CancelledError as caught:cancelled = caught- if not transport_admitted:+ # Upstream inference is cancellable even after transport admission.+ # Only an acquired handle's finite settlement must outlive its caller.+ if acquired_handle is None:attempt_task.cancel()try:handle, outcome = await await_owned_task(attempt_task)@@ -6691,7 +6681,7 @@ class ModelHubService:await handle.close_stream()outcome = await meter_available_outcome(handle)if outcome is None:- await meter_handle(handle, None)+ await meter_observed(handle.observed, None)return outcomecleanup_task = asyncio.create_task(close_and_meter_observed_stream())diff --git a/core/handlers/model_hub/stream_wire.py b/core/handlers/model_hub/stream_wire.pyindex b6b3b142a..d803b89fe 100644--- a/core/handlers/model_hub/stream_wire.py+++ b/core/handlers/model_hub/stream_wire.py@@ -605,6 +605,12 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {("type",),"response.reasoning_summary_text.delta",),+ ProtocolModelOutputEnvelope(+ # https://platform.openai.com/docs/api-reference/responses-streaming/response/reasoning-text/delta+ "response.reasoning_text.delta",+ ("type",),+ "response.reasoning_text.delta",+ ),ProtocolModelOutputEnvelope(# https://platform.openai.com/docs/api-reference/responses-streaming/response/function-call-arguments/delta"response.function_call_arguments.delta",@@ -656,6 +662,13 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {None,require_nonempty=True,),+ ProtocolModelOutputEnvelope(+ # https://api-docs.deepseek.com/guides/reasoning_model+ None,+ ("choices", "*", "delta", "reasoning_content"),+ None,+ require_nonempty=True,+ ),ProtocolModelOutputEnvelope(# https://platform.openai.com/docs/api-reference/chat/create#chat-create-streamNone,diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.pyindex 74b4a4cf6..2a66d4ccb 100644--- a/core/handlers/model_hub/turn_gateway.py+++ b/core/handlers/model_hub/turn_gateway.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport asyncioimport jsonimport logging+import mathimport socketimport tempfilefrom collections import deque@@ -39,6 +40,7 @@ from .provenance import (render_turn_outcome_copy,)from .request import ModelHubRequest+from .resolver import parse_model_hub_timestampfrom .stream_wire import (ProtocolSSEState,ProtocolUsageReport,@@ -829,18 +831,33 @@ class ModelHubTurnGateway:translation = translate_opencode_tool_names(payload)payload = translation.requestexecution.response_tool_aliases = translation.response_aliases- resolved = await self.service.resolve(- backend=backend,- model_id=resolution_model,- request=ModelHubRequest(- payload,- protocol=_REQUEST_PROTOCOLS[endpoint],- headers=protocol_headers,- ),- stream=stream,- supply_channel="hub",- attempt_observer=observe_attempt,- )+ for retry in range(2):+ try:+ resolved = await self.service.resolve(+ backend=backend,+ model_id=resolution_model,+ request=ModelHubRequest(+ payload,+ protocol=_REQUEST_PROTOCOLS[endpoint],+ headers=protocol_headers,+ ),+ stream=stream,+ supply_channel="hub",+ attempt_observer=observe_attempt,+ )+ break+ except ModelHubError as exc:+ delay = self._cooldown_retry_delay(exc.turn_outcome)+ if (+ retry+ or delay is None+ or exc.turn_outcome is None+ or exc.turn_outcome.outcome != "no_candidate"+ ):+ raise+ # Native callers may ignore Retry-After. With no runnable+ # hop, wait for one known recovery before admission.+ await asyncio.sleep(delay)except ModelHubError as exc:turn_outcome = exc.turn_outcomeif turn_outcome is None and exc.code == "engine_down":@@ -1236,7 +1253,27 @@ class ModelHubTurnGateway:turn_outcome,fallback_code=code,)- return self._error_response(status=status, code=code, rendered=rendered)+ delay = self._cooldown_retry_delay(turn_outcome)+ if delay is not None and status == 409:+ status = 503+ return self._error_response(+ status=status,+ code=code,+ rendered=rendered,+ retry_after=math.ceil(delay) if delay is not None else None,+ )++ def _cooldown_retry_delay(+ self,+ turn_outcome: TurnOutcomeProjectionInput | None,+ ) -> float | None:+ if turn_outcome is None or turn_outcome.outcome not in {"no_candidate", "exhausted"}:+ return None+ facts = turn_outcome.supply_facts+ if facts is None or facts.supply_state != "waiting" or not facts.retry_at:+ return None+ retry_at = parse_model_hub_timestamp(facts.retry_at)+ return max(0.0, (retry_at - self._now()).total_seconds())async def _settle_turn_handle(self,@@ -1330,6 +1367,7 @@ class ModelHubTurnGateway:status: int,code: str,rendered: _RenderedTurnOutcome | None = None,+ retry_after: int | None = None,) -> web.Response:language = self._language_provider() or "en"message = rendered.message if rendered is not None else None@@ -1338,6 +1376,9 @@ class ModelHubTurnGateway:message = i18n_t(message_key, language)if message == message_key:message = i18n_t("modelHub.errors.upstream_error", language)+ headers = {"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"}+ if retry_after is not None and status in {429, 503}:+ headers["Retry-After"] = str(max(1, retry_after))return web.json_response({"error": {@@ -1347,5 +1388,5 @@ class ModelHubTurnGateway:}},status=status,- headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},+ headers=headers,)diff --git a/core/handlers/session_handler.py b/core/handlers/session_handler.pyindex 1cbeccc62..10f6a88a2 100644--- a/core/handlers/session_handler.py+++ b/core/handlers/session_handler.py@@ -2433,24 +2433,12 @@ class SessionHandler(BaseHandler):) -> int:"""Disconnect Claude sessions that have been idle beyond the timeout.- A session is normally exempt from eviction while it is flagged- ``active`` (a turn is in flight). That veto is **not** absolute: if the- receiver coroutine never releases the flag (e.g. it stays alive but- blocked on ``receive_messages`` with no stream EOF), the session would- otherwise be pinned forever and its ``claude`` subprocess would survive- until the next service restart. As an independent backstop, a session- that is ``active`` but whose ``last_activity`` is older than- ``max(idle_timeout * stuck_active_multiplier,- stuck_active_floor_seconds)`` is force-evicted regardless of why the- flag was not cleared. A genuine in-flight turn keeps touching- ``last_activity`` via assistant/tool messages, so it normally stays well- under this cap. Pass ``stuck_active_multiplier <= 0`` to disable the- backstop. Caveat: a real turn whose single tool call runs silently for- longer than the cap is indistinguishable from a stuck session and would- be force-evicted — see ``DEFAULT_STUCK_ACTIVE_IDLE_EVICTION_MULTIPLIER``.+ Durable Turn and Activity ownership always vetoes reclamation, even+ during silent inference or tools. The age backstop repairs an+ adapter-local active flag only after durable ownership independently+ allows reclamation. Pass ``stuck_active_multiplier <= 0`` to disable+ that stale-flag repair."""- from core.runtime_ownership import SessionRuntimeDisposition-if idle_timeout <= 0:return 0@@ -2506,13 +2494,11 @@ class SessionHandler(BaseHandler):continueidle_for = now - last_activityif composite_key in self.active_sessions:- # Stuck-active backstop: only evict once well past the cap.- if stuck_threshold is not None and idle_for >= stuck_threshold:- if ownership.disposition in {- SessionRuntimeDisposition.TRANSITIONING,- SessionRuntimeDisposition.UNKNOWN,- }:- continue+ if (+ not ownership.blocks_reclamation+ and stuck_threshold is not None+ and idle_for >= stuck_threshold+ ):expired.append((composite_key, idle_for))continueif not ownership.blocks_reclamation and idle_for >= idle_timeout:@@ -2570,11 +2556,7 @@ class SessionHandler(BaseHandler):allowed = bool(stuck_threshold is not Noneand recheck_idle >= stuck_threshold- and ownership.disposition- not in {- SessionRuntimeDisposition.TRANSITIONING,- SessionRuntimeDisposition.UNKNOWN,- }+ and not ownership.blocks_reclamation)else:allowed = bool(diff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.pyindex 4979d0157..960a32666 100644--- a/docs/plans/model-hub-contracts/adapter-interface.py+++ b/docs/plans/model-hub-contracts/adapter-interface.py@@ -10,6 +10,7 @@ orchestrator and land with every affected consumer on the same tested head.from __future__ import annotations+import asynciofrom dataclasses import dataclassfrom enum import Enumfrom typing import Any, AsyncIterator, Callable, Final, Literal, Mapping, Protocol, Sequence@@ -89,6 +90,14 @@ class OriginNotAllowedError(Exception):``RawCallOutcome`` and never triggers fallback."""+class InvokeCancelledError(asyncio.CancelledError):+ """Owner cancellation carrying wire facts observed before transport cleanup."""++ def __init__(self, observed: ProtocolSSEState) -> None:+ super().__init__()+ self.observed = observed++class RawOutcomeKind(str, Enum):SUCCESS = "success"HTTP_ERROR = "http_error"@@ -553,5 +562,7 @@ class EngineAdapter(Protocol):``on_admitted`` exactly once before any network wait. Callback failurereleases the lease without invoking upstream. Transport completion,close, or cancellation releases the lease independently of settlement.+ Cancellation before handle return raises ``InvokeCancelledError`` when+ wire facts were observed, so L2 can meter them without inventing failure."""...diff --git a/docs/plans/model-hub-inference-deadlines.md b/docs/plans/model-hub-inference-deadlines.mdnew file mode 100644index 000000000..1c0c81abd--- /dev/null+++ b/docs/plans/model-hub-inference-deadlines.md@@ -0,0 +1,212 @@+# Model Hub inference deadlines++## Problem and contract++The September 8, 2026 investigation found that Model Hub reused its 60-second+transport budget for response headers, the first response bytes, first model+output, and completion of buffered inference. A connected, healthy provider can+take longer at any of these stages. Retrying discards its work and can exhaust+the caller's retries without ever allowing the model to finish.++An admitted inference request must remain live until protocol completion, a+concrete transport failure, or cancellation by its owner. Elapsed model time+alone is not failure. This property applies to every supported protocol and to+both streaming and buffered responses. Cancellation must still release the+connection, temporary storage, and source ownership, and a waiting request must+not block unrelated requests.++## Scope++- Remove inference wall-clock deadlines in the engine client.+- Retain bounded connection establishment, management/discovery calls, reading+ an already-failed HTTP response, and resource cleanup.+- Start the finite local buffered-response parsing budget after the complete+ response has arrived, so model execution cannot consume it.+- Audit backend idle reclamation, gateway cancellation, and native caller+ timeouts before closing the change. Record evidence and scope decisions here.+- Treat a known source cooldown as delayed admission, with one cancellable wait+ and a fresh resolution at its existing recovery time. Only retry a request+ that had no runnable candidate; never replay visible output or restart a+ failed attempt inside the same gateway request.+- Recognize Responses reasoning-text and compatible Chat reasoning-content+ deltas as model output using the existing protocol taxonomy.+- Keep production configuration and the running service unchanged during+ development; verify with hermetic tests and local Incus where applicable.++## Validation++Exercise delayed inference at each wire boundary beyond the connection budget,+then assert successful completion or owner-driven cancellation. Preserve error+classification, usage, response replay, source isolation, and shutdown tests.++## Audit inventory++| Boundary | Initial evidence | Decision |+| --- | --- | --- |+| Engine response headers / first bytes / first output | Uses transport timeout | Remove inference deadline |+| Buffered inference completion | Uses transport timeout | Remove inference deadline |+| Buffered local parsing | Shares deadline with body download | Give local parsing its own budget |+| Gateway teardown and cancellation drain | Runs after completion/cancellation | Retain cleanup bounds |+| Codex and Claude active-session reclamation | Age backstop overrides durable active ownership | Require the existing ownership veto at both reclamation checks; retain repair of orphaned local flags |+| Codex SSE idle | Native default is 300,000 ms, after HTTP headers | Retain the native policy; zero expires immediately, not disabled. No arbitrary huge override and no early gateway headers |+| Source cooldown | A waiting route returns 409; Codex does not honor HTTP Retry-After | Delay no-candidate admission until the known retry time once; re-resolve current configuration; use 503 and Retry-After for remaining time-recoverable errors |+| Prelude output classification | Responses reasoning text and Chat reasoning content were treated as metadata | Extend the existing taxonomy and acceptance fixtures |+| Prelude storage | Memory spills at 256 KiB; replay has deliberately no total byte ceiling | Preserve lossless replay and cancellation cleanup, not a new response-size rejection policy |+| Codex local control RPC | 120-second acknowledgement deadline; turns run via notifications | Retain, not an inference deadline; resetting initialization on one slow RPC is a separate transport issue |+| OpenCode active turn | Default duration cap already disabled | No change |+| OpenCode admission and stale overlay cleanup | 120-second acceptance check and 30-second old-process drain | Retain; neither limits an accepted live turn |+| HTTP request body | 16 MiB input cap; oversized requests use aiohttp's 413 | Retain size protection; protocol-shaped 413 rendering is a separate issue |++Base inspected: `6bad0ca137ca9a72678511a094d366aa7e48af36`.++## Ownership decision++The ownership snapshot proves an owner still exists, not that its process is+healthy. The old age backstop has no discriminator between a silent valid turn+and a live-but-deadlocked backend. Narrowing the veto to ignore the resource's+own active Turn would therefore restore the same demonstrated false eviction.+This change deliberately prioritizes owner-controlled termination. Explicit+Stop, native terminal/EOF handling, and reclamation of unowned local flags remain.+A live deadlock with no terminal evidence can remain until explicit Stop; do not+claim that this change introduces automatic deadlock detection. Such detection+needs independent backend evidence, not another elapsed-inference threshold.++## Incident evidence++Local service logs and durable Turn rows were compared for the two reported+Sessions. Times below are UTC+08:00 on September 8, 2026.++- `sesssbbj4mt2y`: forced progress-timeout settlement at 00:22:37, 01:21:07,+ 09:00:12, and 09:50:27, with native reconnect messages before settlement.+- `sesvmgbdub2gp`: the same forced settlement at 01:31:15. A late native 503+ followed at 01:34:35; both durable rows name the same native Turn.+- `sesssbbj4mt2y` likewise recorded a late 503 at 09:57:16 against the native+ Turn forcibly settled at 09:50:27. These rows are not independent upstream+ outages and must not be counted as such.+- Another observed call cooled its sole source at 09:14:52 until 09:15:22,+ while the caller exhausted reconnects by 09:14:56 and failed with 409.+- The generic 409 at 09:58:14 in `sesvmgbdub2gp` has no attributable gateway+ provenance. It must not be asserted to be the cooldown case without evidence.+ Route/configuration conflicts remain terminal; their safeguards are unchanged.++## Native caller evidence++Verified against the installed Codex 0.153.2 and its exact official source tag:++- [HTTP retry implementation](https://github.com/openai/codex/blob/rust-v0.153.2/codex-rs/codex-client/src/retry.rs)+ uses local backoff and does not read Retry-After.+- [Native regression test](https://github.com/openai/codex/blob/rust-v0.153.2/codex-rs/core/tests/suite/retry_after.rs)+ explicitly asserts local backoff despite a Retry-After header.+- [SSE consumer](https://github.com/openai/codex/blob/rust-v0.153.2/codex-rs/codex-api/src/sse/responses.rs)+ applies its idle deadline to the next parsed SSE event, after headers.+ SSE comment heartbeats do not necessarily reset that event-level deadline.+ The gateway still defers headers until model output or a protocol terminal,+ preserving pre-output fallback and HTTP error semantics.++The native SSE cap and a provider's own limits remain outside the removed+Avibe inference deadlines; this PR does not claim end-to-end infinite patience.++## Validation evidence++- Real loopback HTTP tests cover all three protocols, five delayed inference+ boundaries, and completion/cancellation. They assert no timeout or cooldown+ classification for slow successful inference and closed client resources.+- Gateway HTTP tests cover known cooldown, repeated failure, configuration+ changes during the wait, and cancellation for every backend.+- Existing protocol classification, byte replay, usage, teardown, native+ termination, and stale-local-flag repair suites remain required.+- Focused combined local validation: 1,223 passed, 30 subtests passed, one+ previously expected xfail. Changed Python files pass Ruff.+- Production configuration, credentials, and installed service are unchanged.++## Review inventory and scope decisions++The orchestrator fetched every review thread, review verdict, and matching lint+run before this revision. Review `5136689855` examined+`2d0700c64c849ba97e71ec7b4cbc7f9cf9522492` and raised three findings:++| Root-cause class | Finding | Complete correction |+| --- | --- | --- |+| Timestamp compatibility | Valid UTC `Z` strings fail on Python 3.10 | Normalize the offset before parsing; test both accepted UTC forms through source resolution and the gateway |+| Cancellation ownership | Transport admission shields an unbounded upstream wait | Propagate cancellation until a handle is acquired; retain shielding for finite settlement; carry already-observed wire facts on cancellation for usage accounting |+| Retry provenance | A previous no-candidate marker wins over a later admitted attempt | Clear obsolete supply facts and their terminal projection at admission; cover later success, failure, and cancellation |++This is the first findings-bearing head for all three classes; no repeated-class+circuit breaker has tripped. No architecture or persistent data-model rewrite is+required. The adapter's cancellation exception extends `CancelledError` with+already-observed wire facts only; it introduces no source failure, retry, or new+terminal outcome. Its canonical interface and mirror change together with both+consumers. Unbounded inference remains cancellable; finite ledger/resource work+retains its existing owner and limits.++The expanded local regression passed 1,410 tests and 30 subtests, with one+pre-existing xfail; changed Python files pass Ruff and whitespace checks.++The new complete gateway-to-service-to-adapter-to-HTTP test reproduced the+original leak: downstream cancellation returned while the loopback provider+request remained open. It now verifies every wire wait and protocol, transport+lease release, and exactly-once accounting of pre-output usage. Existing tests+for a response beating cancellation now synchronize on actual handle availability,+not mere transport admission.++Local Incus installation, startup, and health verification completed on the first+head, with 803 passing tests, 30 passing subtests, and one existing xfail. Only+the task-specific `avr-wt-model-hub-inference-deadlines` environment was used.+The first lint run's migration-fixture failure is supplied by merged dependency+#1933 (`a58644528deb8365876915904d11fb89df48db11`); integrate it normally before+requesting the next exact-head review and CI.++### Second-head circuit breaker and orchestrator decision++The full paginated inventory contains five findings across two reviewed heads:+`2d0700c64c849ba97e71ec7b4cbc7f9cf9522492` (review `5136689855`) and+`bb0ba11f056084884231c89814084b23c07594ad` (review `5136913971`). The first+head's three findings are listed above. The second head repeats two classes:++First-head threads: `PRRT_kwDOPbFPYs6gEzJI` (timestamp),+`PRRT_kwDOPbFPYs6gEzJM` (cancellation), and `PRRT_kwDOPbFPYs6gEzJN` (provenance).++| Root-cause class | Second-head finding | Full-chain diagnosis |+| --- | --- | --- |+| Timestamp semantics | Thread `PRRT_kwDOPbFPYs6gFJW0`: string ordering selects the wrong cooldown deadline | Validation preserves offsets. Parsing at the gateway alone does not fix selection in `turn_supply_facts`; the probe API has the same string minimum. Resolver readiness, service comparisons, and gateway delay must share timestamp semantics |+| Retry provenance | Thread `PRRT_kwDOPbFPYs6gFJW2`: an earlier exhausted projection overrides Stop after readmission | Admission clears only no-candidate projections and only when a supply marker exists. Exhaustion has no such marker. All supply-level retry projections need the same admission boundary, independently of that marker |++Both repeated classes reached the two-head threshold. The orchestrator paused+editing, inspected all five threads and their consuming paths/tests, and chose+the following smallest complete correction before resuming:++- Move the existing service timestamp parser to the existing resolver module.+ Reuse it for readiness, source/OAuth time comparisons, both earliest-deadline+ selections, and gateway delay. Compare aware instants, preserving the selected+ original string for rendering and compatibility. Keep Python 3.10 `Z` support+ and the existing interpretation of old naive timestamps as UTC.+- On actual attempt admission, retire both `no_candidate` and `exhausted`+ projections independently of supply-marker presence. Preserve failed-attempt+ history, request-specific identities, committed served/nonretryable facts,+ and the Stop/frozen-outcome boundary. No persistent schema or lifecycle rewrite.+- Enumerate every supply-level outcome/variant from the production authority in+ the retry regression, followed by success, exhaustion, terminal failure, or+ Stop. Exercise exhaustion, delayed recovery, readmission, and Stop through real+ gateway HTTP as well. Exercise mixed offsets, fractional timestamps, and route+ permutations through resolution, probe, and gateway admission.+- Keep the resolver-consuming retry test in the existing L3 ownership scope.+ Current-head lint run `34182012889` failed only because its prior placement in+ `test_model_hub_provenance.py` added an unregistered resolver importer (O1).+ Move the test to L3; do not weaken the authority guard or expand lane ownership.++The first lint run `34179699562` belongs to the old head and its dependency+failure is resolved by the already-integrated #1933. The current head has 15+successful checks and the O1 failure plus its failing unit-test aggregate.+Cancellation ownership did not recur on the second reviewed head. The durable+watch and cursor remain unchanged. Production, merge, and deployment stay out+of scope; fresh exact-head review, complete lint, and task-local Incus validation+are still required after this correction.++Before the correction, the added consuming tests reproduced 12 failures across+chronological selection, old naive timestamps, and exhaustion followed by Stop+(including Codex and Claude HTTP requests). Afterward all 75 focused cases pass.+The combined local regression passes 1,492 tests and 30 subtests with one existing+xfail, including the previously failing config/authority closure test. The+authority checker reports no findings; changed Python files pass Ruff and+whitespace checks. The orchestrator inspected the parser/minimum diff and its+probe-to-gateway consumer, and the admission diff and its HTTP Stop consumer.diff --git a/modules/agents/codex/agent.py b/modules/agents/codex/agent.pyindex 32828c885..26c05daf5 100644--- a/modules/agents/codex/agent.py+++ b/modules/agents/codex/agent.py@@ -1345,10 +1345,7 @@ class CodexAgent(BaseAgent):and not has_activeand idle_for >= idle_timeout)- stuck_candidate = bool(stuck_sessions) and ownership.disposition not in {- SessionRuntimeDisposition.TRANSITIONING,- SessionRuntimeDisposition.UNKNOWN,- }+ stuck_candidate = bool(stuck_sessions) and not ownership.blocks_reclamationif not ordinary_candidate and not stuck_candidate:continue@@ -1370,12 +1367,10 @@ class CodexAgent(BaseAgent):now=current_now,cap=stuck_active_cap,)- if ownership.disposition in {- SessionRuntimeDisposition.TRANSITIONING,- SessionRuntimeDisposition.UNKNOWN,- }:- continue- if ownership.blocks_reclamation and not stuck_sessions:+ # Silence cannot revoke a durable Turn or Activity owner.+ # The age backstop only repairs stale adapter-local flags once+ # durable ownership independently allows reclamation.+ if ownership.blocks_reclamation:continuesettled_stuck_sessions: set[str] = set()@@ -1514,13 +1509,11 @@ class CodexAgent(BaseAgent):release(context)def _stuck_active_idle_eviction_cap(self, idle_timeout: float) -> Optional[float]:- """Idle cap after which an *active* transport is force-evicted.+ """Age threshold for repairing an unowned adapter-local active flag.Returns ``None`` when the backstop is disabled (multiplier <= 0), in- which case an active turn remains an absolute veto. Otherwise a- transport with an active turn is force-evicted once it has been idle for- ``max(idle_timeout * multiplier, floor)`` — the floor keeps the window- sane even when ``idle_timeout`` is configured very small.+ which case an active flag remains an absolute veto. Durable ownership+ always vetoes reclamation regardless of this threshold."""multiplier = DEFAULT_CODEX_STUCK_ACTIVE_IDLE_EVICTION_MULTIPLIERif multiplier <= 0:diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yamlindex b27f4571a..01a2fed33 100644--- a/tests/scenarios/INDEX.yaml+++ b/tests/scenarios/INDEX.yaml@@ -149,6 +149,7 @@ capabilities:- tests/test_model_hub_transport_leases.py- tests/test_model_hub_injection.py- tests/test_model_hub_runtime.py+ - tests/test_model_hub_inference_wait.py- tests/test_model_hub_l3.py- tests/test_model_hub_usage.py- tests/test_local_deps.pydiff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 72aa6d732..3d58c8467 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -8,6 +8,7 @@ capability:- tests/test_model_hub_resolution.py- tests/test_model_hub_injection.py- tests/test_model_hub_runtime.py+ - tests/test_model_hub_inference_wait.py- tests/test_model_hub_l3.py- tests/test_model_hub_usage.py- tests/test_local_deps.py@@ -326,6 +327,12 @@ scenarios:kind: lifecyclelayer: unittest: tests/test_local_deps.py::test_reconcile_startup_dependencies_uses_automatic_runtime_admission+ - id: MH-RUNTIME-009+ name: Connected inference remains live until completion, transport failure, or owner cancellation+ status: covered+ kind: timeout_cancel+ layer: contract+ test: tests/test_model_hub_inference_wait.py::test_connected_inference_waits_for_its_owner- id: MH-USAGE-001name: Tokens an upstream reported before the turn ended are still meteredstatus: covereddiff --git a/tests/scenarios/model_hub/observations.yaml b/tests/scenarios/model_hub/observations.yamlindex 6bcd09c00..771d7374a 100644--- a/tests/scenarios/model_hub/observations.yaml+++ b/tests/scenarios/model_hub/observations.yaml@@ -1,6 +1,11 @@version: 2capability: model_hubobservations:+ - id: MH-OBS-008+ source: local-codex-gateway-investigation-2026-09-08+ summary: A shared transport budget also limited response headers, first bytes, first model output, and buffered inference completion. Prior tests explicitly required these premature timeouts. Delayed loopback HTTP fixtures now assert completion and cancellation across all supported protocols independently of the connection budget.+ affects:+ - MH-RUNTIME-009- id: MH-OBS-007source: opencode-1.18.18-codex-0.153.2-local-http-regression-2026-09-06summary: OpenCode inline config deep-merges provider and model headers, allowing native Authorization headers to replace the Hub token even when baseURL and apiKey are pinned. The managed runtime config hook must replace complete Hub provider definitions. Codex native and same-name provider fixtures retained the configured Source credential and upstream model on the wire.diff --git a/tests/test_claude_cli_path.py b/tests/test_claude_cli_path.pyindex f38505ee6..8b4652102 100644--- a/tests/test_claude_cli_path.py+++ b/tests/test_claude_cli_path.py@@ -20,7 +20,7 @@ from config.v2_settings import RoutingSettingsfrom core import git_runtime as git_runtime_modulefrom core.handlers.session_handler import SessionHandlerfrom core.runtime_activation import RuntimeActivationRegistry-from core.runtime_ownership import SessionRuntimeDisposition+from core.runtime_ownership import RuntimeTargetOwnershipSnapshot, SessionRuntimeDispositionfrom modules.claude_sdk_compat import CLAUDE_SDK_MAX_BUFFER_SIZEfrom modules.im import MessageContext@@ -2179,6 +2179,51 @@ def test_session_handler_keeps_active_claude_session(monkeypatch, tmp_path: Pathassert composite_key in controller.claude_sessions[email protected]("ownership_arrives_during_check", [False, True])+def test_silent_owned_claude_turn_survives_both_reclamation_checks(+ monkeypatch: pytest.MonkeyPatch,+ tmp_path: Path,+ ownership_arrives_during_check: bool,+) -> None:+ captured: dict[str, Any] = {}+ monkeypatch.setattr(session_handler_module, "ClaudeAgentOptions", _StubClaudeAgentOptions)+ monkeypatch.setattr(session_handler_module, "ClaudeSDKClient", _disconnect_counting_client(captured))+ controller = _Controller(tmp_path)+ handler = SessionHandler(controller)+ _run_session(handler, MessageContext(user_id="U123", channel_id="C123"))+ composite_key = f"slack_C123:{tmp_path}"+ handler.session_last_activity[composite_key] = 0.0+ handler.active_sessions.add(composite_key)+ monkeypatch.setattr(session_handler_module.time, "monotonic", lambda: 1_000_000.0)++ def snapshot(target, disposition=SessionRuntimeDisposition.ACTIVE):+ return RuntimeTargetOwnershipSnapshot(+ backend="claude",+ resource_key=target.resource_key,+ activity_runtime_keys=(),+ sessions=(),+ sessionless_active_activity_ids=(),+ sessionless_fallback_run_ids=(),+ disposition=disposition,+ )++ controller.runtime_ownership = SimpleNamespace(+ snapshot=snapshot,+ snapshot_many=lambda targets: tuple(+ snapshot(+ target,+ SessionRuntimeDisposition.RECLAIMABLE+ if ownership_arrives_during_check else SessionRuntimeDisposition.ACTIVE,+ )+ for target in targets+ ),+ )+ assert asyncio.run(handler.evict_idle_sessions(600)) == 0+ assert captured["disconnects"] == 0+ assert composite_key in handler.active_sessions+ assert composite_key in controller.claude_sessions++def test_evict_idle_sessions_force_evicts_stuck_active_session(monkeypatch, tmp_path: Path) -> None:"""The active flag is not an absolute veto.diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.pyindex c5f25aa0e..25e217fab 100644--- a/tests/test_codex_agent.py+++ b/tests/test_codex_agent.py@@ -16,6 +16,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))from core.processing_indicator import STOPPED_REACTION_EMOJIfrom core.runtime_activation import RuntimeActivationRegistry+from core.runtime_ownership import RuntimeTargetOwnershipSnapshot, SessionRuntimeDispositionfrom modules.agents.base import BaseAgent as RealBaseAgentfrom modules.agents.codex.transport import CodexRPCError@@ -1272,6 +1273,38 @@ class CodexAgentStopTests(unittest.IsolatedAsyncioTestCase):self.assertIn("/tmp/work", agent._transports)self.assertEqual(cleared_turns, [])+ async def test_silent_owned_codex_turn_survives_both_reclamation_checks(self):+ for ownership_arrives_during_check in (False, True):+ with self.subTest(ownership_arrives_during_check=ownership_arrives_during_check):+ agent, stops, invalidated, cleared = self._make_evict_agent(+ active_turn="turn-1", last_activity=0.0,+ )++ def snapshot(disposition):+ return RuntimeTargetOwnershipSnapshot(+ backend="codex",+ resource_key="/tmp/work",+ activity_runtime_keys=(),+ sessions=(),+ sessionless_active_activity_ids=(),+ sessionless_fallback_run_ids=(),+ disposition=disposition,+ )++ active = snapshot(SessionRuntimeDisposition.ACTIVE)+ snapshots = (+ [snapshot(SessionRuntimeDisposition.RECLAIMABLE), active]+ if ownership_arrives_during_check else [active]+ )+ agent._runtime_ownership_snapshot_for_cwd = Mock(side_effect=snapshots)+ with patch.object(_MODULE.time, "monotonic", return_value=1_000_000.0):+ self.assertEqual(await agent.evict_idle_transports(600), 0)+ self.assertEqual(stops, [])+ self.assertEqual(invalidated, [])+ self.assertEqual(cleared, [])+ self.assertEqual(agent._turn_registry.get_active_turn("session-1"), "turn-1")+ agent.controller.emit_agent_message.assert_not_awaited()+async def test_hfr_143_observable_session_progress_wins_locked_recheck(self):"""HFR-143: attributable progress keeps a productive turn alive."""agent, stop_calls, _invalidated, _cleared = self._make_evict_agent(diff --git a/tests/test_model_hub_inference_wait.py b/tests/test_model_hub_inference_wait.pynew file mode 100644index 000000000..463c56f87--- /dev/null+++ b/tests/test_model_hub_inference_wait.py@@ -0,0 +1,221 @@+"""Connected inference is governed by completion and cancellation, not age."""++from __future__ import annotations++import asyncio+from types import SimpleNamespace++from aiohttp import web+import pytest++from core.handlers.model_hub.adapter import RawOutcomeKind+from core.handlers.model_hub.classification import classify_outcome+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway+from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _service, _source, _usage_of+from vibe.model_hub_runtime import client as client_module+from vibe.model_hub_runtime.adapter import CLIProxyEngineAdapter+from vibe.model_hub_runtime.client import EngineClient, EngineConnection+from vibe.model_hub_runtime.state import EngineStateStore, SourceRecord+++WIRE = {+ "openai_responses": (+ b'event: response.created\ndata: {"type":"response.created"}\n\n',+ b'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"ok"}\n\n',+ b'event: response.completed\ndata: {"type":"response.completed"}\n\n',+ b'{"output":[{"content":[{"type":"output_text","text":"ok"}]}]}',+ ),+ "openai_chat": (+ b'data: {"choices":[{"delta":{"role":"assistant"}}]}\n\n',+ b'data: {"choices":[{"delta":{"content":"ok"}}]}\n\n',+ b"data: [DONE]\n\n",+ b'{"choices":[{"message":{"role":"assistant","content":"ok"}}]}',+ ),+ "anthropic": (+ b'event: message_start\ndata: {"type":"message_start","message":{"role":"assistant","usage":{"input_tokens":41}}}\n\n',+ b'event: content_block_delta\ndata: {"type":"content_block_delta","delta":{"type":"text_delta","text":"ok"}}\n\n',+ b'event: message_stop\ndata: {"type":"message_stop"}\n\n',+ b'{"type":"message","content":[{"type":"text","text":"ok"}]}',+ ),+}++[email protected]("protocol", WIRE)[email protected]("phase", ["headers", "first_byte", "prelude", "stream_body", "buffered_body"])[email protected]("cancel", [False, True], ids=["complete", "cancel"])[email protected]("entrypoint", ["client", "gateway"])+def test_connected_inference_waits_for_its_owner(+ monkeypatch: pytest.MonkeyPatch,+ tmp_path,+ protocol: str,+ phase: str,+ cancel: bool,+ entrypoint: str,+) -> None:+ """MH-RUNTIME-009: connected inference ends by completion or owner cancellation."""++ async def run() -> None:+ blocked = asyncio.Event()+ release = asyncio.Event()+ request_finished = asyncio.Event()+ streaming = phase != "buffered_body"+ metadata, output, terminal, buffered = WIRE[protocol]+ sessions = []+ real_session = client_module.aiohttp.ClientSession++ def track_session(**kwargs):+ timeout = kwargs["timeout"]+ assert timeout.total is None and timeout.sock_read is None+ assert timeout.connect == timeout.sock_connect == 0.02+ session = real_session(**kwargs)+ sessions.append(session)+ return session++ monkeypatch.setattr(client_module.aiohttp, "ClientSession", track_session)++ async def pause(stage: str) -> None:+ if phase == stage:+ blocked.set()+ await release.wait()++ async def respond(request: web.Request) -> web.StreamResponse:+ try:+ await request.json()+ await pause("headers")+ response = web.StreamResponse(+ headers={"Content-Type": "text/event-stream" if streaming else "application/json"},+ )+ await response.prepare(request)+ await pause("first_byte")+ if streaming:+ await response.write(metadata)+ await pause("prelude")+ await response.write(output)+ await pause("stream_body")+ await response.write(terminal)+ else:+ await response.write(buffered[:1])+ await pause("buffered_body")+ await response.write(buffered[1:])+ await response.write_eof()+ return response+ finally:+ request_finished.set()++ app = web.Application()+ app.router.add_post("/{path:.*}", respond)+ runner = web.AppRunner(app, handler_cancellation=True)+ await runner.setup()+ await web.TCPSite(runner, "127.0.0.1", 0).start()+ port = runner.addresses[0][1]+ source = SourceRecord(+ source_id="src_patient1",+ vendor="custom",+ protocol=protocol,+ base_url="https://provider.example.test",+ credential_ref="cred_patient1",+ allowed_origins=(),+ model_ids=("patient-model",),+ prefix="patient",+ )+ client = EngineClient(+ EngineConnection(f"http://127.0.0.1:{port}", "fixture-management", "fixture-gateway"),+ timeout=0.02,+ )+ gateway = None+ adapter = None+ if entrypoint == "gateway":+ backend, endpoint = {+ "openai_responses": ("codex", "responses"),+ "openai_chat": ("opencode", "chat/completions"),+ "anthropic": ("claude", "messages"),+ }[protocol]+ configured = _source(+ source.source_id,+ "Patient inference",+ protocol=protocol,+ vendor="anthropic" if backend == "claude" else "openai",+ )+ state = EngineStateStore(tmp_path / "engine")+ configured.credential_ref = state.store_api_key(+ "fixture-key", vendor=configured.vendor, protocol=protocol,+ )+ service = _service(tmp_path, sources=[configured])+ models = _canonicalize_fixed_test_routes(service)+ state.sync_sources(service._bindings(service.store.load()))+ adapter = CLIProxyEngineAdapter(+ supervisor=SimpleNamespace(client=lambda: client), state_store=state,+ )+ # Installation is not the boundary under test; invocation and leases are real.+ adapter.ensure_installed = service.adapter.ensure_installed+ service.adapter = adapter+ service._engine_synced = True+ gateway = ModelHubTurnGateway(service)+ base_url, token = await gateway.endpoint(+ backend,+ process_scope="/repo",+ turn_id="turn_patient_inference",+ requested_model_id=models.get(backend, "shared-model"),+ resolved_model_id="shared-model",+ source_id=configured.id,+ )++ async def consume() -> bytes:+ if gateway is not None:+ async with real_session(trust_env=False) as downstream:+ async with downstream.post(+ f"{base_url}/v1/{endpoint}",+ json={"model": "shared-model", "stream": streaming},+ headers={"Authorization": f"Bearer {token}"},+ ) as response:+ assert response.status == 200+ return await response.read()+ handle = await client.invoke(source, "patient-model", {}, stream=streaming)+ try:+ assert handle.stream is not None+ body = b"".join([chunk async for chunk in handle.stream])+ outcome = await handle.outcome()+ assert outcome.kind is RawOutcomeKind.SUCCESS+ decision = classify_outcome(outcome)+ assert decision.action == "return"+ assert decision.reason is None and decision.cooldown_seconds == 0+ return body+ finally:+ await handle.close_stream()++ task = asyncio.create_task(consume())+ try:+ await asyncio.wait_for(blocked.wait(), timeout=2)+ # Spend several connection budgets while the server owns the wait.+ done, _pending = await asyncio.wait({task}, timeout=0.1)+ assert not done, "inference was terminated solely because model output was delayed"+ if cancel:+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(task, timeout=2)+ await asyncio.wait_for(request_finished.wait(), timeout=2)+ else:+ release.set()+ assert await asyncio.wait_for(task, timeout=2) == (+ metadata + output + terminal if streaming else buffered+ )+ assert sessions and all(session.closed for session in sessions)+ if adapter is not None:+ assert adapter._active_transports == 0+ assert adapter._transports_idle.is_set()+ await gateway._drain_turn_requests("turn_patient_inference")+ assert service.events.list() == []+ if protocol == "anthropic" and phase in {"prelude", "stream_body"}:+ usage = _usage_of(service, configured.id)+ assert usage["input_tokens"] == 41+ assert usage["requests"] == 1+ finally:+ release.set()+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ if gateway is not None:+ await gateway.close()+ await runner.cleanup()++ asyncio.run(run())diff --git a/tests/test_model_hub_l3.py b/tests/test_model_hub_l3.pyindex 6ef6d83ab..1be5a8daf 100644--- a/tests/test_model_hub_l3.py+++ b/tests/test_model_hub_l3.py@@ -1408,6 +1408,72 @@ def test_stopped_settlement_cannot_erase_committed_served_history(_assert_valid("turn-provenance.schema.json", record)+ ("previous_decision", "variant"),+ [+ (decision, variant)+ for decision, rule in TURN_OUTCOME_RENDERING_AUTHORITY.items()+ if rule.outcome in {"no_candidate", "exhausted"}+ for variant, _key in rule.copy_keys+ ],+)[email protected]("ending", ["served", "exhausted", "failed_terminal", "canceled"])+def test_admitted_retry_supersedes_earlier_supply_failure(tmp_path, previous_decision, variant, ending):+ store = BoundedProvenanceStore(tmp_path / "records.json")+ registry = TurnCorrelationRegistry(store)+ turn_id = _begin_hub_attempt(registry, turn_id="turn-retry")+ previous_failure = _outcome(RawOutcomeKind.HTTP_ERROR, status=503)+ registry.finish_attempt(turn_id, outcome=previous_failure, decision=classify_outcome(previous_failure))+ source = _source(+ "src_recovered01", "Recovering",+ status="cooldown" if variant == "waiting" else "standby",+ retry_at=(NOW + timedelta(seconds=30)).isoformat() if variant == "waiting" else None,+ )+ config = _config([] if previous_decision == "turn.no_candidate.unconfigured" else [source])+ projection = produce_turn_outcome(+ previous_decision,+ config=config,+ resolution=_terminal_resolution_facts(+ config,+ supply_status="degraded" if variant == "waiting_without_retry" else variant,+ structural_reason="route_unconfigured" if previous_decision == "turn.no_candidate.unconfigured" else None,+ next_hop=(source.id, "shared-model") if variant == "waiting_without_retry" else None,+ ),+ )+ if projection.outcome == "no_candidate":+ registry.mark_gateway_no_candidate(+ turn_id, projection.supply_facts.supply_state,+ blockers=(ExactHopBlocker(source.id, "shared-model", "server_error"),),+ )+ registry.record_turn_outcome(turn_id, projection)+ registry.begin_attempt(+ turn_id, source_id=source.id, resolved_model_id="shared-model", channel="hub", via_mapping=False,+ request_id="recovered-request",+ )+ if ending != "canceled":+ outcome = _outcome(+ RawOutcomeKind.SUCCESS if ending == "served" else RawOutcomeKind.HTTP_ERROR,+ status={"served": 200, "exhausted": 503, "failed_terminal": 400}[ending],+ code="invalid_parameter" if ending == "failed_terminal" else None,+ source_id=source.id,+ )+ registry.finish_attempt(+ turn_id, outcome=outcome, decision=classify_outcome(outcome), request_id="recovered-request",+ )+ settled_by = SETTLED_BY_STOPPED if ending == "canceled" else SETTLED_BY_TERMINAL_RESULT+ registry.close_turn_admission(turn_id, settled_by=settled_by)+ registry.settle(turn_id, settled_by=settled_by)+ record = store.get(turn_id)+ assert record["outcome"] == ending+ assert record["model_supply_state"] is None+ assert record["blockers"] == []+ assert record["failed_attempts"][0]["source_id"] == previous_failure.source_id+ assert len(record["failed_attempts"]) == (2 if ending == "exhausted" else 1)+ if ending == "canceled":+ assert record["canceled_attempt"]["source_id"] == source.id+ _assert_valid("turn-provenance.schema.json", record)++def test_stopped_settlement_cannot_erase_committed_protocol_failure(tmp_path: Path,) -> None:@@ -2885,6 +2951,262 @@ def test_gateway_preserves_exhausted_provenance_after_all_hops_fallback(asyncio.run(exercise())[email protected]("already_cooling", [False, True])[email protected]("ending", ["recover", "fail", "config_change", "cancel", "extend_recover", "admitted_cancel"])+ ("backend", "endpoint", "protocol"),+ [+ ("codex", "responses", "openai_responses"),+ ("claude", "messages", "anthropic"),+ ("opencode", "chat/completions", "openai_chat"),+ ],+)+def test_gateway_cooldown_is_one_cancellable_retry(+ tmp_path: Path,+ already_cooling: bool,+ ending: str,+ backend: str,+ endpoint: str,+ protocol: str,+) -> None:+ """MH-RUNTIME-009: known recovery, not caller retry speed, owns cooldown waiting."""++ async def exercise() -> None:+ source = _source(+ "src_retrycool01",+ "Recovering",+ status="cooldown" if already_cooling else "standby",+ retry_at=(NOW + timedelta(seconds=30)).isoformat().replace("+00:00", "Z") if already_cooling else None,+ vendor="anthropic" if backend == "claude" else "openai",+ protocol=protocol,+ )+ failure = _outcome(RawOutcomeKind.HTTP_ERROR, status=503, source_id=source.id)+ final = failure if ending == "fail" else _outcome(RawOutcomeKind.SUCCESS, source_id=source.id)+ service = _service(+ tmp_path,+ sources=[source],+ outcomes=([failure] if not already_cooling else []) + [final],+ )+ fixed = _canonicalize_fixed_test_routes(service)+ model = fixed[backend] if backend in fixed else "shared-model"+ payload = {"model": "shared-model", "stream": False}+ if endpoint == "responses":+ payload["input"] = "ping"+ else:+ payload["messages"] = [{"role": "user", "content": "ping"}]+ clock = {"now": NOW}+ service.now = lambda: clock["now"]+ gateway = ModelHubTurnGateway(service, now=lambda: clock["now"])+ waiting = asyncio.Event()+ release = asyncio.Event()+ wait_ended = asyncio.Event()+ admitted = asyncio.Event()+ upstream_canceled = asyncio.Event()+ delays = []++ async def wait_for_recovery(delay: float) -> None:+ delays.append(delay)+ waiting.set()+ try:+ await release.wait()+ clock["now"] += timedelta(seconds=delay)+ finally:+ wait_ended.set()++ async def invoke_until_stopped(source_id, model_id, request, stream, origin, *, on_admitted=None):+ on_admitted()+ service.adapter.invocations.append((source_id, model_id, origin))+ admitted.set()+ try:+ await asyncio.Future()+ finally:+ upstream_canceled.set()++ base_url, token = await gateway.endpoint(+ backend,+ process_scope="/repo",+ turn_id="turn_cooldown_retry",+ requested_model_id=model,+ resolved_model_id="shared-model",+ source_id=source.id,+ )+ try:+ with patch("core.handlers.model_hub.turn_gateway.asyncio.sleep", side_effect=wait_for_recovery):+ async with aiohttp.ClientSession(trust_env=False) as client:+ if not already_cooling:+ first = await client.post(+ f"{base_url}/v1/{endpoint}",+ json=payload,+ headers={"Authorization": f"Bearer {token}"},+ )+ await first.read()+ assert first.status == 503+ assert first.headers["Retry-After"] == "30"+ if ending == "admitted_cancel":+ service.adapter.invoke = invoke_until_stopped+ request = asyncio.create_task(client.post(+ f"{base_url}/v1/{endpoint}",+ json=payload,+ headers={"Authorization": f"Bearer {token}"},+ ))+ try:+ await asyncio.wait_for(waiting.wait(), timeout=2)+ assert not request.done()+ assert len(service.adapter.invocations) == (0 if already_cooling else 1)+ if ending == "cancel":+ request.cancel()+ with pytest.raises(asyncio.CancelledError):+ await request+ await asyncio.wait_for(wait_ended.wait(), timeout=2)+ elif ending == "admitted_cancel":+ release.set()+ await asyncio.wait_for(admitted.wait(), timeout=2)+ gateway.correlation.close_turn_admission(+ "turn_cooldown_retry", settled_by=SETTLED_BY_STOPPED,+ )+ request.cancel()+ with pytest.raises(asyncio.CancelledError):+ await request+ await asyncio.wait_for(upstream_canceled.wait(), timeout=2)+ else:+ if ending == "config_change":+ service.store.config.sources[0].state = ModelHubSourceStateConfig(+ status="needs_action",+ detail_key="models.source.needs_action.credential_revoked",+ )+ elif ending == "extend_recover":+ service.store.config.sources[0].state.retry_at = (+ NOW + timedelta(seconds=60)+ ).isoformat().replace("+00:00", "Z")+ release.set()+ response = await asyncio.wait_for(request, timeout=2)+ await response.read()+ assert response.status == {+ "recover": 200, "fail": 503, "config_change": 409, "extend_recover": 503,+ }[ending]+ if ending in {"fail", "extend_recover"}:+ assert response.headers["Retry-After"] == "30"+ else:+ assert "Retry-After" not in response.headers+ if ending == "extend_recover":+ clock["now"] += timedelta(seconds=30)+ recovered = await client.post(+ f"{base_url}/v1/{endpoint}",+ json=payload,+ headers={"Authorization": f"Bearer {token}"},+ )+ await recovered.read()+ assert recovered.status == 200+ assert delays == [30.0]+ expected = (0 if already_cooling else 1) + (+ ending in {"recover", "fail", "extend_recover", "admitted_cancel"}+ )+ assert len(service.adapter.invocations) == expected+ finally:+ if not request.done():+ request.cancel()+ await asyncio.gather(request, return_exceptions=True)+ finally:+ release.set()+ await gateway.close()+ if ending in {"extend_recover", "admitted_cancel"} and backend != "opencode":+ gateway.correlation.settle(+ "turn_cooldown_retry",+ settled_by=SETTLED_BY_STOPPED if ending == "admitted_cancel" else SETTLED_BY_TERMINAL_RESULT,+ )+ record = service.provenance.get("turn_cooldown_retry")+ assert record["outcome"] == ("canceled" if ending == "admitted_cancel" else "served")+ assert record["model_supply_state"] is None+ assert record["blockers"] == []+ assert len(record["failed_attempts"]) == (0 if already_cooling else 1)+ if ending == "admitted_cancel":+ assert record["canceled_attempt"]["source_id"] == source.id++ asyncio.run(exercise())++[email protected]("suffix", ["Z", "+00:00", ""])+def test_gateway_parses_supported_cooldown_timestamps_on_python310(tmp_path, suffix):+ class Python310Datetime(datetime):+ @classmethod+ def fromisoformat(cls, value):+ if value.endswith("Z"):+ raise ValueError("Python 3.10 requires a numeric UTC offset")+ return super().fromisoformat(value)++ retry_at = (NOW + timedelta(seconds=30)).replace(tzinfo=None).isoformat() + suffix+ source = _source(+ "src_timestamp01", "Cooling", status="cooldown", retry_at=retry_at,+ )+ service = _service(tmp_path, sources=[source])+ model = _canonicalize_fixed_test_routes(service)["codex"]+ gateway = ModelHubTurnGateway(service, now=lambda: NOW)+ with patch("core.handlers.model_hub.resolver.datetime", Python310Datetime):+ with pytest.raises(ModelHubError) as failed:+ asyncio.run(service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub"))+ assert gateway._cooldown_retry_delay(failed.value.turn_outcome) == 30++[email protected]("reverse_route", [False, True])+ ("early", "late", "delay"),+ [+ ("2026-07-29T17:00:30.125+05:00", "2026-07-29T12:01:00Z", 30.125),+ ("2026-07-29T12:00:30Z", "2026-07-29T05:01:00-07:00", 30),+ ("2026-07-29T12:00:30Z", "2026-07-29T12:00:30.125Z", 30),+ ],+)+def test_cooldown_selection_uses_instants_through_probe_and_gateway(tmp_path, reverse_route, early, late, delay):+ async def exercise():+ first = _source("src_earliest01", "First recovery", status="cooldown", retry_at=early)+ second = _source("src_later0001", "Later recovery", status="cooldown", retry_at=late)+ service = _service(+ tmp_path,+ sources=[second, first] if reverse_route else [first, second],+ outcomes=[_outcome(RawOutcomeKind.SUCCESS, source_id=first.id)],+ )+ model = _canonicalize_fixed_test_routes(service)["codex"]+ clock = {"now": NOW}+ service.now = lambda: clock["now"]+ with pytest.raises(ModelHubError) as probe:+ await service.probe_agent("codex", model)+ assert probe.value.code == "probe_no_candidate"+ assert probe.value.data["supply"]["retry_at"] == early+ with pytest.raises(ModelHubError) as failed:+ await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")+ assert failed.value.turn_outcome.supply_facts.retry_at == early++ gateway = ModelHubTurnGateway(service, now=lambda: clock["now"])+ assert gateway._cooldown_retry_delay(failed.value.turn_outcome) == delay+ delays = []++ async def wait_for_recovery(seconds):+ delays.append(seconds)+ clock["now"] += timedelta(seconds=seconds)++ base_url, token = await gateway.endpoint(+ "codex", process_scope="/repo", turn_id="turn_earliest_recovery",+ requested_model_id=model, resolved_model_id="shared-model", source_id=first.id,+ )+ try:+ with patch("core.handlers.model_hub.turn_gateway.asyncio.sleep", side_effect=wait_for_recovery):+ async with aiohttp.ClientSession(trust_env=False) as client:+ response = await client.post(+ f"{base_url}/v1/responses",+ json={"model": "shared-model", "input": "ping", "stream": False},+ headers={"Authorization": f"Bearer {token}"},+ )+ await response.read()+ assert response.status == 200+ assert delays == [delay]+ assert service.adapter.invocations == [(first.id, "shared-model", "codex")]+ assert service.store.load().sources[0 if reverse_route else 1].state.retry_at == late+ finally:+ await gateway.close()++ asyncio.run(exercise())++def test_gateway_exhaustion_uses_no_time_copy_when_an_earlier_hop_recovers(tmp_path: Path,) -> None:@@ -4090,13 +4412,16 @@ def test_resolver_settles_a_bodyless_attempt_that_beats_cancellation(requested_model = _canonicalize_fixed_test_routes(service)["codex"]invoke_started = asyncio.Event()release_invoke = asyncio.Event()+ handle_ready = asyncio.Event()invoke = service.adapter.invokeasync def blocked_invoke(*args, **kwargs):kwargs.pop("on_admitted")()invoke_started.set()await release_invoke.wait()- return await invoke(*args, **kwargs)+ handle = await invoke(*args, **kwargs)+ handle_ready.set()+ return handleservice.adapter.invoke = blocked_invoketask = asyncio.create_task(@@ -4109,8 +4434,9 @@ def test_resolver_settles_a_bodyless_attempt_that_beats_cancellation())await asyncio.wait_for(invoke_started.wait(), timeout=1)- task.cancel()release_invoke.set()+ await asyncio.wait_for(handle_ready.wait(), timeout=1)+ task.cancel()with pytest.raises(asyncio.CancelledError):await asyncio.wait_for(task, timeout=1)@@ -4142,13 +4468,16 @@ def test_resolver_meters_an_observed_stream_that_beats_cancellation(requested_model = _canonicalize_fixed_test_routes(service)["codex"]invoke_started = asyncio.Event()release_invoke = asyncio.Event()+ handle_ready = asyncio.Event()invoke = service.adapter.invokeasync def blocked_invoke(*args, **kwargs):kwargs.pop("on_admitted")()invoke_started.set()await release_invoke.wait()- return await invoke(*args, **kwargs)+ result = await invoke(*args, **kwargs)+ handle_ready.set()+ return resultservice.adapter.invoke = blocked_invoketask = asyncio.create_task(@@ -4161,8 +4490,9 @@ def test_resolver_meters_an_observed_stream_that_beats_cancellation())await asyncio.wait_for(invoke_started.wait(), timeout=1)- task.cancel()release_invoke.set()+ await asyncio.wait_for(handle_ready.wait(), timeout=1)+ task.cancel()with pytest.raises(asyncio.CancelledError):await asyncio.wait_for(task, timeout=1)diff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.pyindex aeb50fad8..9f9695825 100644--- a/tests/test_model_hub_runtime.py+++ b/tests/test_model_hub_runtime.py@@ -169,7 +169,6 @@ def test_stream_prelude_replays_large_keepalive_history_before_output() -> None:wire_state=state,source=source,model_id="claude-sonnet-4-5",- timeout=1,)payload = b"".join([chunk async for chunk in prelude.chunks()])prelude.close()@@ -570,12 +569,17 @@ def test_engine_error_projection_does_not_materialize_unrelated_payload(assert reads and all(size == client_module._STREAM_CHUNK_BYTES for size in reads)-def test_stream_prelude_uses_one_absolute_pre_output_deadline() -> None:+def test_stream_prelude_retains_metadata_until_model_output() -> None:async def run() -> None:+ output = b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n'+class Content:+ reads = 0+async def read(self, _size: int) -> bytes:- await asyncio.sleep(0.02)- return b": keepalive\n\n"+ self.reads += 1+ await asyncio.sleep(0)+ return b": keepalive\n\n" if self.reads < 3 else outputresponse = SimpleNamespace(content=Content(), status=200)source = SourceRecord(@@ -591,17 +595,22 @@ def test_stream_prelude_uses_one_absolute_pre_output_deadline() -> None:prelude = client_module._StreamPrelude(memory_limit=64)state = client_module.ProtocolSSEState("anthropic")- with pytest.raises(asyncio.TimeoutError):- await client_module._read_stream_prelude(+ try:+ outcome = await client_module._read_stream_prelude(response=response,first=b": first\n\n",prelude=prelude,wire_state=state,source=source,model_id="claude-sonnet-4-5",- timeout=0.01,)- prelude.close()+ assert outcome is None+ assert state.model_output_started+ assert b"".join([chunk async for chunk in prelude.chunks()]) == (+ b": first\n\n" + b": keepalive\n\n" * 2 + output+ )+ finally:+ prelude.close()asyncio.run(run())@@ -6070,39 +6079,21 @@ def test_engine_error_fields_ignore_machine_codes_outside_the_trusted_envelope()assert candidates == ("api_error",)- ("phase", "stream", "expected_kind", "expected_status", "stream_started"),- [- ("first_byte", True, RawOutcomeKind.TIMEOUT, None, False),- ("error_body", True, RawOutcomeKind.HTTP_ERROR, 429, False),- ("non_stream", False, RawOutcomeKind.TIMEOUT, 200, False),- ],-)-def test_engine_client_times_out_before_completion(+def test_engine_client_bounds_reading_an_already_failed_response(monkeypatch: pytest.MonkeyPatch,- phase: str,- stream: bool,- expected_kind: RawOutcomeKind,- expected_status: int | None,- stream_started: bool,) -> None:async def run() -> None:blocked_phase = asyncio.Event()never_release = asyncio.Event()class Content:- reads = 0-async def read(self, _size: int) -> bytes:- self.reads += 1- if phase == "non_stream" and self.reads == 1:- return b"{"blocked_phase.set()await never_release.wait()return b""class Response:- status = 429 if phase == "error_body" else 200+ status = 429content = Content()headers = {"Content-Type": "text/event-stream"}@@ -6134,15 +6125,15 @@ def test_engine_client_times_out_before_completion(source,"model-a",{},- stream=stream,+ stream=True,)assert blocked_phase.is_set()assert handle.stream is Noneoutcome = await handle.outcome()- assert outcome.kind is expected_kind- assert outcome.http_status == expected_status- assert outcome.stream_started is stream_started+ assert outcome.kind is RawOutcomeKind.HTTP_ERROR+ assert outcome.http_status == 429+ assert outcome.stream_started is Falseasyncio.run(run())diff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.pyindex 2a681f827..4e1ff38dc 100644--- a/tests/test_model_hub_structure_guards.py+++ b/tests/test_model_hub_structure_guards.py@@ -183,6 +183,13 @@ MODEL_OUTPUT_ENVELOPE_FIXTURES = ("response.reasoning_summary_text.delta",False,),+ (+ "openai_responses",+ "response.reasoning_text.delta",+ ("type",),+ "response.reasoning_text.delta",+ False,+ ),("openai_responses","response.function_call_arguments.delta",@@ -198,6 +205,7 @@ MODEL_OUTPUT_ENVELOPE_FIXTURES = (False,),("openai_chat", None, ("choices", "*", "delta", "content"), None, True),+ ("openai_chat", None, ("choices", "*", "delta", "reasoning_content"), None, True),("openai_chat", None, ("choices", "*", "delta", "refusal"), None, True),("openai_chat", None, ("choices", "*", "delta", "tool_calls"), None, True),("openai_chat", None, ("choices", "*", "delta", "function_call"), None, True),@@ -1434,6 +1442,20 @@ def test_chat_role_metadata_does_not_cross_the_model_output_boundary() -> None:assert state.model_output_started is False+ ("protocol", "payload"),+ [+ ("openai_responses", {"type": "response.reasoning_text.delta", "delta": "thinking"}),+ ("openai_chat", {"choices": [{"delta": {"reasoning_content": "thinking"}}]}),+ ],+)+def test_reasoning_is_model_output_before_any_answer_text(protocol: str, payload: dict) -> None:+ state = ProtocolSSEState(protocol)+ event = f"event: {payload['type']}\n".encode() if "type" in payload else b""+ state.observe(event + b"data: " + json.dumps(payload).encode() + b"\n\n")+ assert state.model_output_started is True++def test_chat_any_nonempty_choice_crosses_the_model_output_boundary() -> None:state = ProtocolSSEState("openai_chat")state.observe(diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.pyindex 272f85731..2f5fd5892 100644--- a/vibe/model_hub_runtime/client.py+++ b/vibe/model_hub_runtime/client.py@@ -22,6 +22,7 @@ from config.v2_config import normalize_model_hub_base_urlfrom core.handlers.model_hub.adapter import (DiscoveredModel,ENGINE_TRANSPORT_TIMEOUT_SECONDS,+ InvokeCancelledError,RawCallOutcome,RawOutcomeKind,)@@ -384,6 +385,9 @@ class EngineClient:sock_connect=self.timeout,sock_read=None,)+ # Connecting to the local engine is bounded. Once connected, headers+ # and response bytes can wait on upstream inference for any duration;+ # completion, transport failure, or owner cancellation ends that wait.session = aiohttp.ClientSession(timeout=timeout, trust_env=False)response: aiohttp.ClientResponse | None = Nonefirst_received = False@@ -394,7 +398,7 @@ class EngineClient:# Held here rather than inside the prelude reader so every way out of this# call can still see what the wire already reported. A stream that reports# usage before its first model output — Anthropic's `message_start` does —- # and then times out has already been billed for those input tokens.+ # and then fails has already been billed for those input tokens.wire_state: ProtocolSSEState | None = Nonedef ended(outcome: RawCallOutcome) -> EngineInvokeHandle:@@ -411,14 +415,11 @@ class EngineClient:return completed_handle(outcome)try:- response = await asyncio.wait_for(- session.post(- self._url(endpoint),- json=body,- headers=headers,- allow_redirects=False,- ),- timeout=self.timeout,+ response = await session.post(+ self._url(endpoint),+ json=body,+ headers=headers,+ allow_redirects=False,)if response.status >= 300:error_body = _StreamPrelude()@@ -473,10 +474,7 @@ class EngineClient:await session.close()return ended(outcome)- first = await asyncio.wait_for(- response.content.read(_STREAM_CHUNK_BYTES),- timeout=self.timeout,- )+ first = await response.content.read(_STREAM_CHUNK_BYTES)if not first:response.close()await session.close()@@ -493,18 +491,14 @@ class EngineClient:if not stream:buffered_body = _StreamPrelude()prelude = buffered_body- response_deadline = time.monotonic() + self.timeoutawait buffered_body.write_async(first)- await asyncio.wait_for(- _read_response_into(response.content, buffered_body),- timeout=self.timeout,- )+ await _read_response_into(response.content, buffered_body)observation = await _observe_buffered_protocol_response_async(observe_buffered_protocol_response,request_protocol,buffered_body,machine_error_codes=UPSTREAM_MACHINE_ERROR_CODES,- deadline=response_deadline,+ deadline=time.monotonic() + self.timeout,)outcome = _reduce_protocol_observation(observation,@@ -535,7 +529,6 @@ class EngineClient:wire_state=wire_state,source=source,model_id=model_id,- timeout=self.timeout,)model_output_started = wire_state.model_output_startedif prelude_outcome is not None:@@ -588,6 +581,10 @@ class EngineClient:ownership_transferred = Truetransport_transferred = Truereturn handle+ except asyncio.CancelledError as cancelled:+ if wire_state is not None:+ raise InvokeCancelledError(wire_state) from cancelled+ raiseexcept asyncio.TimeoutError:if response is not None:response.close()@@ -1256,7 +1253,6 @@ async def _read_stream_prelude(wire_state: ProtocolSSEState,source: SourceRecord,model_id: str,- timeout: float,) -> RawCallOutcome | None:"""Buffer transport metadata until the sole first-model-output fact.@@ -1266,7 +1262,6 @@ async def _read_stream_prelude("""await _received(first, prelude=prelude, wire_state=wire_state)- deadline = asyncio.get_running_loop().time() + timeoutwhile not wire_state.model_output_started:outcome = _observed_stream_terminal_outcome(wire_state,@@ -1276,13 +1271,7 @@ async def _read_stream_prelude()if outcome is not None:return outcome- remaining = deadline - asyncio.get_running_loop().time()- if remaining <= 0:- raise asyncio.TimeoutError- chunk = await asyncio.wait_for(- response.content.read(_STREAM_CHUNK_BYTES),- timeout=remaining,- )+ chunk = await response.content.read(_STREAM_CHUNK_BYTES)if not chunk:completion = _observed_stream_terminal_outcome(wire_state,