Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): buffer Responses output for one exact overload replay (#3007)
Call White95de1ee3a339
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.pyindex 33a7c5dd7..2b5f61cae 100644--- a/core/handlers/model_hub/service.py+++ b/core/handlers/model_hub/service.py@@ -1034,7 +1034,7 @@ class HandleSettlement:turn_outcome: TurnOutcomeProjectionInput | None-HandleTerminationOrigin = Literal["downstream_cancel", "upstream_terminal"]+HandleTerminationOrigin = Literal["downstream_cancel", "upstream_terminal", "local_delivery_failure"]AttemptObserver = Callable[@@ -10595,12 +10595,14 @@ class ModelHubService:*,termination_origin: HandleTerminationOrigin,record_attempt: Callable[[RawCallOutcome, ResolutionDecision], None],+ retry_buffered_overload: bool = False,) -> HandleSettlement:"""Settle every consumed hub handle before its terminal facts are exposed."""try:return await self._settle_handle_outcome(resolved, outcome, termination_origin=termination_origin, record_attempt=record_attempt,+ retry_buffered_overload=retry_buffered_overload,)finally:if resolved is not None:@@ -10613,21 +10615,27 @@ class ModelHubService:*,termination_origin: HandleTerminationOrigin,record_attempt: Callable[[RawCallOutcome, ResolutionDecision], None],+ retry_buffered_overload: bool,) -> HandleSettlement:- if resolved is not None and resolved.handle is not None:+ if resolved is not None and resolved.handle is not None and not retry_buffered_overload:await self._observe_handle_recovery(resolved.source_id, resolved.settlement_generation, resolved.handle,backend=resolved.backend, model_id=resolved.requested_model_id,)- if termination_origin == "downstream_cancel" and outcome is None:+ if termination_origin in {"downstream_cancel", "local_delivery_failure"} and outcome is None:+ # Closing a still-live producer does not manufacture its terminal.+ # Metering already retained every observed token; release admission+ # without classifying a local disk failure as an upstream refusal.return HandleSettlement(- outcome=outcome,+ outcome=None,decision=None,- turn_outcome=produce_turn_outcome("turn.canceled"),+ turn_outcome=produce_turn_outcome(+ "turn.canceled" if termination_origin == "downstream_cancel" else "turn.engine_down",+ ),)- if termination_origin == "downstream_cancel":- # The gateway selects this origin only after closing the producer.- # A committed outcome at that barrier owns history over cancellation.+ if termination_origin in {"downstream_cancel", "local_delivery_failure"}:+ # A producer result committed during close still owns its Source+ # settlement; a local delivery failure projects separately afterwards.termination_origin = "upstream_terminal"if termination_origin != "upstream_terminal":raise AssertionError("unknown handle termination origin")@@ -10639,6 +10647,25 @@ class ModelHubService:resolved.credential_ref,outcome,)+ # Only the gateway can grant this after matching the exact overload+ # in a fully buffered, uncommitted Responses body. Keep the upstream+ # stream_started fact intact for usage and provenance.+ retry_overload = (+ retry_buffered_overload+ and outcome.kind is RawOutcomeKind.HTTP_ERROR and outcome.http_status == 200+ and outcome.stream_started+ and decision.action == "surface" and decision.reason == "server_error"+ )+ if retry_overload:+ decision = replace(decision, action="fallback", error_code=None)+ elif retry_buffered_overload and resolved.handle is not None:+ # Preserve ordinary first-output recovery evidence, but not for the+ # discarded overload: it must extend the existing failure streak,+ # not briefly announce recovery and reset its own backoff.+ await self._observe_handle_recovery(+ resolved.source_id, resolved.settlement_generation, resolved.handle,+ backend=resolved.backend, model_id=resolved.requested_model_id,+ )record_attempt(outcome, decision)if decision.action == "return":if self._verified_recovery_outcome(outcome):@@ -10662,7 +10689,7 @@ class ModelHubService:decision=decision,),)- if decision.action != "surface":+ if decision.action != "surface" and not retry_overload:raise AssertionError("consumed streams cannot retry or fall through")source_transition_persisted: bool | None = Noneif decision.reason is not None:@@ -10681,6 +10708,10 @@ class ModelHubService:settlement_generation=resolved.settlement_generation,outcome=outcome,)+ if retry_overload:+ # An attempt failed, not the Turn. The resolver still owns its+ # original recovery window, routing, and the eventual terminal.+ return HandleSettlement(outcome=outcome, decision=decision, turn_outcome=None)return HandleSettlement(outcome=outcome,decision=decision,@@ -11673,9 +11704,16 @@ class ModelHubService:)async def resolve_with_recovery(- self, *, recovery_observer: Callable[[dict | None], None] | None = None, **kwargs,+ self, *, recovery_observer: Callable[[dict | None], None] | None = None,+ prepare_response: Callable[[ResolvedInvocation], Awaitable[RawCallOutcome | None]] | None = None,+ **kwargs,) -> ResolvedInvocation:- """One pending model request, including all fallback passes and waits."""+ """One pending model request, including all fallback passes and waits.++ A gateway may buffer and settle a response before committing it. It+ returns only a settled retryable outcome, never an unconsumed handle;+ that replay shares this pending request rather than opening a new window.+ """if self.recovery.window_seconds == 0:# Explicit opt-out for single-walk probes and characterization tests.@@ -11685,7 +11723,17 @@ class ModelHubService:try:while True:try:- return await self.resolve(**kwargs, recovery_request=pending)+ resolved = await self.resolve(**kwargs, recovery_request=pending)+ retry_outcome = await prepare_response(resolved) if prepare_response is not None else None+ if retry_outcome is None:+ return resolved+ pending.start()+ pending.reason = "server_error"+ pending.last_upstream_failure = self._upstream_failure(resolved.source_label, retry_outcome)+ if getattr(kwargs.get("request"), "pinned_hop", None) is not None:+ # Never replay opaque continuation content on a different+ # hop. Its caller owns re-rendering the portable request.+ raise _pin_unavailable()except ModelHubError as exc:projection = exc.turn_outcomeif (diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.pyindex 42dfdd88d..bfd2a1ab3 100644--- a/core/handlers/model_hub/turn_gateway.py+++ b/core/handlers/model_hub/turn_gateway.py@@ -3,6 +3,7 @@from __future__ import annotationsimport asyncio+import errnoimport functoolsimport jsonimport logging@@ -76,6 +77,7 @@ from core.os_errors import local_error_detail_MAX_REQUEST_BYTES: Final = 128 * 1024 * 1024_BUFFERED_RESPONSE_MEMORY_BYTES: Final = 256 * 1024+_MAX_REPLAY_BUFFER_BYTES: Final = 128 * 1024 * 1024_RESPONSE_CHUNK_BYTES: Final = 64 * 1024# Claude Code aborts a stream whose response headers miss its first-byte window# and treats a byte-silent body as stalled. Resolution can outlast both through@@ -243,6 +245,30 @@ class _TurnExecution:# A large buffered JSON body whose headers have already left. A local disk# failure can no longer replace its HTTP status or append an SSE ending.buffered_response: web.StreamResponse | None = None+ # An initial Responses stream is held until its terminal is known. This+ # flag is armed only after the entire body is safely stored, never during+ # cancellation or a failed spool write.+ buffered_stream: BinaryIO | None = None+ retry_buffered_overload: bool = False++ def next_attempt(self) -> None:+ """Retire a fully settled/metered discarded attempt without ending its request."""++ assert self.settlement_task is not None and self.settlement_task.done()+ assert not self.owes_metering and self.stream_response is None+ self.resolved = None+ self.handle = None+ self.settlement_task = None+ self.settlement_origin = None+ self.settlement_recorded = False+ self.terminal_fact_committed = False+ self.rendered_turn_outcome = None+ self.wire_state = None+ self.handle_outcome = None+ self.completed_at = None+ self.usage_write = None+ self.buffered_stream = None+ self.retry_buffered_overload = False@propertydef upstream_observation(self) -> ProtocolSSEState | None:@@ -1143,6 +1169,32 @@ class ModelHubTurnGateway:)protocol = "google" if google_request else _REQUEST_PROTOCOLS[endpoint]+ replayed = False++ def adopt(resolved: ResolvedInvocation) -> None:+ # Own the body before buffering, health writes, or keepalive teardown+ # can be cancelled. Each callback closes only its captured handle.+ execution.resolved = resolved+ if resolved.handle is not None and resolved.handle.stream is not None:+ execution.handle = resolved.handle+ resources.push_async_callback(resolved.handle.close_stream)++ async def prepare_response(resolved: ResolvedInvocation) -> RawCallOutcome | None:+ nonlocal replayed+ adopt(resolved)+ if replayed or resolved.outcome is not None or execution.handle is None:+ return None+ outcome, settlement = await self._buffer_responses_stream(execution, terminalizer, resources)+ if settlement.decision is None or settlement.decision.action != "fallback":+ return None+ # The first partial text/tool calls never reached the consumer. Its+ # attempt has already settled and been metered, independently of+ # whatever the next route resolves to (including a local refusal).+ await run_owned_in_thread(execution.buffered_stream.close)+ replayed = True+ execution.next_attempt()+ return outcome+try:caller_headers = {name.lower(): value for name, value in request.headers.items() if name.lower() in FORWARDED_CALLER_HEADERS@@ -1174,13 +1226,13 @@ class ModelHubTurnGateway:supply_channel="hub",attempt_observer=observe_attempt,recovery_observer=terminalizer.update_recovery,+ prepare_response=(+ prepare_response if stream and protocol == "openai_responses"+ and self.service.recovery.window_seconds > 0 else None+ ),)- # Owned before the keepalive stops, so a cancellation while it- # stops still closes and meters a Source stream already won.- execution.resolved = resolved- if resolved.handle is not None and resolved.handle.stream is not None:- execution.handle = resolved.handle- resources.push_async_callback(resolved.handle.close_stream)+ if execution.resolved is not resolved:+ adopt(resolved)except ModelHubError as exc:if exc.code == PIN_UNAVAILABLE_CODE:# Never shown to the user: Vibey re-renders the request in@@ -1237,7 +1289,7 @@ class ModelHubTurnGateway:response.headers.update(_response_origin_headers(exc))return await self._through_committed_stream(execution, protocol, response)- if execution.handle is not None:+ if execution.handle is not None and execution.settlement_task is None:# The execution boundary must own cleanup before an awaited health# write: cancellation here still settles and meters this handle.await self.service._observe_handle_recovery(@@ -1327,6 +1379,65 @@ class ModelHubTurnGateway:await self._downstream_io(committed.write_eof())return committed+ async def _buffer_responses_stream(+ self,+ execution: _TurnExecution,+ terminalizer: GatewayTurnTerminalizer,+ resources: AsyncExitStack,+ ) -> tuple[RawCallOutcome, HandleSettlement]:+ """Observe and settle one uncommitted stream; no content leaves here."""++ handle = execution.handle+ assert handle is not None and handle.stream is not None+ payload = tempfile.SpooledTemporaryFile(max_size=_BUFFERED_RESPONSE_MEMORY_BYTES)+ execution.buffered_stream = payload+ resources.push_async_callback(run_owned_in_thread, payload.close)+ wire_state = ProtocolSSEState("openai_responses")+ execution.wire_state = wire_state+ buffered_bytes = 0+ try:+ async for chunk in handle.stream:+ await wire_state.observe_async(chunk)+ buffered_bytes += len(chunk)+ if buffered_bytes > _MAX_REPLAY_BUFFER_BYTES:+ raise OSError(errno.EFBIG, "Model Hub response replay buffer limit exceeded")+ await run_owned_in_thread(payload.write, chunk)+ execution.completed_at = self._now()+ # Match the bounded wire facts before the engine's display/code+ # sanitizers can elide an unknown code or normalize message text.+ # A root `type:error` selects the SSE envelope, not an error class.+ error_types = set(wire_state.error_type_candidates)+ if () in wire_state.error_envelope_paths:+ error_types.discard("error")+ execution.retry_buffered_overload = (+ wire_state.terminal_outcome == "failed_terminal"+ and wire_state.model_output_started+ and error_types | set(wire_state.error_code_candidates) == {"server_error"}+ and wire_state.error_message == "Our servers are currently overloaded. Please try again later."+ )+ outcome, settlement, _rendered = await self._settle_metered_turn(+ execution, terminalizer, termination_origin="upstream_terminal", defer_projection=True,+ )+ await run_owned_in_thread(payload.seek, 0)+ except OSError as exc:+ # A local spool failure is never a reason to replay the upstream.+ # Meter/settle its actual observation before reporting delivery loss.+ execution.retry_buffered_overload = False+ await self._settle_metered_turn(+ execution, terminalizer, termination_origin="local_delivery_failure", defer_projection=True,+ )+ raise ModelHubError(+ "engine_down", status=502, turn_outcome=ENGINE_DOWN_TURN_OUTCOME,+ local_error_detail=local_error_detail(exc),+ ) from exc+ assert outcome is not None and settlement is not None+ return outcome, settlement++ @staticmethod+ async def _read_buffered_stream(payload: BinaryIO) -> AsyncIterator[bytes]:+ while chunk := await run_owned_in_thread(payload.read, _RESPONSE_CHUNK_BYTES):+ yield chunk+async def _resolved_response(self,request: web.Request,@@ -1339,9 +1450,9 @@ class ModelHubTurnGateway:) -> web.StreamResponse:if resolved.supply_channel != "hub":return self._error_response(status=409, code="mode_switch_blocked")- # Resolution has crossed the engine's first-output barrier or already- # settled successfully: this request can no longer fail over. The same- # admitted origin is held by its provenance attempt.+ # Resolution crossed the engine's first-output barrier. A Responses+ # overload may have been discarded before this point; only the final+ # admitted producer supplies these downstream headers.origin_headers = _response_origin_headers(resolved)if resolved.outcome is not None:# A call that reached the resolver's own hands has already been@@ -1451,26 +1562,44 @@ class ModelHubTurnGateway:if response is None:response = web.StreamResponse(status=200, headers={**_SSE_RESPONSE_HEADERS, **origin_headers})await self._downstream_io(response.prepare(request))+ buffered = execution.buffered_stream+ # Track what actually leaves, not the already-complete spooled wire. A+ # disk read can fail between frames; only forwarded bytes may determine+ # whether a terminal is still owed or a partial frame needs invalidating.wire_state = ProtocolSSEState(protocol)execution.wire_state = wire_state+ body = self._read_buffered_stream(buffered) if buffered is not None else handle.streamtool_name_rewriter = StreamingToolNameRewriter(execution.response_tool_aliases)- async for chunk in handle.stream:- output_started_before = wire_state.model_output_started- await wire_state.observe_async(chunk)- if wire_state.model_output_started and not output_started_before:- terminalizer.mark_stream_started()- rewritten = tool_name_rewriter.feed(chunk)- if rewritten:- await self._downstream_io(response.write(rewritten))- trailing = tool_name_rewriter.finish()- if trailing:- await self._downstream_io(response.write(trailing))- execution.completed_at = self._now()- _outcome, _settlement, rendered = await self._settle_metered_turn(- execution,- terminalizer,- termination_origin="upstream_terminal",- )+ try:+ async for chunk in body:+ output_started_before = wire_state.model_output_started+ await wire_state.observe_async(chunk)+ if wire_state.model_output_started and not output_started_before:+ terminalizer.mark_stream_started()+ rewritten = tool_name_rewriter.feed(chunk)+ if rewritten:+ await self._downstream_io(response.write(rewritten))+ trailing = tool_name_rewriter.finish()+ if trailing:+ await self._downstream_io(response.write(trailing))+ except OSError as exc:+ if buffered is None or isinstance(exc, _DownstreamDisconnected):+ raise+ # The producer was already settled/metered before local playback.+ # Delivery loss must not turn its success into Source ill health.+ detail = local_error_detail(exc)+ terminalizer.engine_down(local_error_detail=detail)+ rendered = self._commit_and_render_turn_outcome(+ execution, terminalizer, replace(ENGINE_DOWN_TURN_OUTCOME, local_error_detail=detail),+ )+ else:+ if execution.completed_at is None:+ execution.completed_at = self._now()+ _outcome, _settlement, rendered = await self._settle_metered_turn(+ execution,+ terminalizer,+ termination_origin="upstream_terminal",+ )await self._write_stream_terminal_copy(response,protocol,@@ -1791,7 +1920,10 @@ class ModelHubTurnGateway:# settlement/history -> t5 render settlement -> t6 downstream EOF.if handle is not None:await handle.close_stream()- if termination_origin == "downstream_cancel" and handle is not None and handle.outcome_available:+ if (+ termination_origin in {"downstream_cancel", "local_delivery_failure"}+ and handle is not None and handle.outcome_available+ ):# A producer outcome that became available during close owns history;# a downstream write/cancel after that barrier cannot rewrite it.termination_origin = "upstream_terminal"@@ -1815,8 +1947,9 @@ class ModelHubTurnGateway:outcome,termination_origin=termination_origin,record_attempt=record_attempt,+ retry_buffered_overload=execution.retry_buffered_overload,)- if termination_origin == "downstream_cancel":+ if termination_origin in {"downstream_cancel", "local_delivery_failure"}:return settlement.outcome, settlementassert settlement.outcome is not Noneassert settlement.decision is not Nonediff --git a/docs/CLI.md b/docs/CLI.mdindex 514261b9a..73bb5349a 100644--- a/docs/CLI.md+++ b/docs/CLI.md@@ -913,6 +913,26 @@ vibe restart --delay-seconds 60The Model Hub engine process is named `cli-proxy-api` (hyphenated); `pgrep cliproxyapi` therefore always returns no match.+#### Responses streams and overload recovery++With automatic Model Hub recovery enabled, the first OpenAI Responses stream is+buffered before delivery, including compatible custom Sources. You may therefore+see no incremental text, reasoning, or tool calls until that response ends.+If it ends with the exact `server_error` message+`Our servers are currently overloaded. Please try again later.`, Model Hub+discards that partial response and retries once through your configured route,+within the same recovery window. Only the final attempt reaches the Agent.+Other streamed errors and other protocols do not gain post-output retries.++The retry itself streams normally and is not replayed again after output.+Both upstream attempts are counted in usage and may be billed; provider-hosted+tools can also have run on the discarded attempt. This prevents duplicate+**downstream** text/tool calls, not duplicate provider-side execution. No Source+order, credentials, or backend settings are changed by recovery. The initial+response buffer uses at most 256 KiB of RAM before spilling to temporary disk,+with a 128 MiB total cap. A disk error or an oversized response ends locally+without another replay.+## Web UI ControlsThe web UI (`http://127.0.0.1:5123`) provides the same controls:diff --git a/docs/plans/model-hub-retry-experience.md b/docs/plans/model-hub-retry-experience.mdindex d397a6e05..d46654b61 100644--- a/docs/plans/model-hub-retry-experience.md+++ b/docs/plans/model-hub-retry-experience.md@@ -98,6 +98,12 @@ Retries stop once the first user-visible model-output byte is committed,including recognized reasoning and tool-call output. Headers, SSE comments,and protocol metadata are not output. Preserve lossless prelude handling.+The narrow exception in `openai-streamed-overload-retry.md` buffers the initial+Responses stream before downstream commit. Only an exact `server_error` overload+may discard that body and replay once through this same recovery window. It+includes compatible custom Sources; it does not broaden other error/protocol+retry rules or promise exactly-once upstream execution.+Pre-output inference retry does not guarantee exactly-once upstream billing:a dropped connection may hide completed inference. Keep every observed usagereport exactly once locally. Do not extend this policy to replaying upstream@@ -382,7 +388,7 @@ in `tests/scenarios/model_hub/catalog.yaml` during implementation.| Concurrent waiting Turns/backends | One recovery admission per affected Source; backups proceed; waiters do not each increment the streak or allocate a separate probe. || Recovery owner canceled or slow | No lock/transport leaks; canceled owner releases only its own admission; another waiter can proceed; slow connected inference is not killed. || Delayed headers / first output / buffered completion | Success beyond the former transport timeout and beyond the admission window; no health penalty, retry, or duplicate usage caused by elapsed inference time. |-| Stream boundary | Heartbeats/headers alone do not prove recovery; reasoning and tool output do. No Hub replay after output, including engine/transport failure and cancellation races. Valid empty completion also releases recovery admission. |+| Stream boundary | Heartbeats/headers alone do not prove recovery; reasoning and tool output do. Generic Hub replay stops after output. The exact OpenAI Responses overload exception buffers before downstream commit and permits one post-output replay in the same recovery window. Other streamed failures remain terminal. Valid empty completion also releases recovery admission. || Advice and clocks | Missing/malformed/past/large Retry-After, differing UTC offsets, clock shifts, and deterministic jitter; no stale serialized deadline or busy loop. || Stronger blockers and fault scope | Bad credentials, quota requiring action, local engine failure, mixed blockers, and explicit model errors retain correct remedies; waiting cannot hide them or corrupt unrelated Source/model state. || Source replacement / stale settlement | Old success/failure/cleanup cannot mutate the replacement or release its recovery admission; backup success cannot clear primary failure history. |diff --git a/docs/plans/openai-streamed-overload-retry.md b/docs/plans/openai-streamed-overload-retry.mdnew file mode 100644index 000000000..c1f99a6f3--- /dev/null+++ b/docs/plans/openai-streamed-overload-retry.md@@ -0,0 +1,96 @@+# OpenAI streamed overload replay++## Background and goal++The engine hands a Responses stream to the gateway at first model output.+Ordinary recovery therefore cannot replay a later overload without duplicating+text or tool calls. The gateway must hold that output before deciding to replay.+The change belongs to the shared Model Hub, not a Desktop or Agent adapter.++## Change contract++With automatic recovery enabled, the gateway buffers the first streamed+`openai_responses` body before committing response headers or bytes. Once fully+consumed, only this conjunction permits one buffered-output replay:++- an HTTP 200 stream with recognized model output;+- its classified failure is `server_error`, and its wire type/code candidates+ name only `server_error` (excluding the root `type:error` envelope selector);+- its bounded wire message, before display sanitization or whitespace folding,+ is exactly+ `Our servers are currently overloaded. Please try again later.`++This is protocol-based, including OpenAI-compatible custom Sources; it is not a+vendor-label or Desktop switch. Other errors and protocols keep their original+outcomes. Successful Responses bodies are also delayed: the error cannot be+known before reading the terminal. No non-streaming request changes.++The failed body is discarded, and normal configured routing/recovery chooses+the next attempt. A ready backup proceeds immediately; a sole cooling Source+waits for its normal eligibility. This remains one pending inference request:+the original 120-second admission window and half-open ownership never reset.+An already admitted inference may finish beyond that window.++Only one **post-output replay** is allowed per gateway request. The next+resolved stream is forwarded normally, including its partial output if it+fails; it is not buffered/replayed again. Ordinary pre-output retries retain+their existing budget, so total upstream attempts need not be limited to two.+If no retry can be admitted, only the resolver's terminal error is delivered.++## Ownership and implementation++- `resolve_with_recovery` retains its `RecoveryRequest` while a gateway callback+ prepares the resolved response. A callback may return a settled retry outcome+ to resume that same loop; final completion clears progress once.+- The gateway owns the adopted handle and spool before any awaited operation.+ Spool I/O and cleanup run off the controller loop with cancellation-safe+ ownership. RAM is capped at 256 KiB; larger bodies spill to a temporary file.+ The complete replay buffer is capped at 128 MiB; exceeding it is a local+ delivery failure, never permission for another upstream request.+- Service settlement records the discarded overload as a failed fallback+ attempt and applies its normal Source cooldown, but does not project a final+ Turn failure. The actual upstream `stream_started` fact remains true. Hidden+ output from this discarded overload does not clear an existing recovery+ streak or emit a recovery-success event.+- Each upstream attempt is metered once. Only after settlement/metering and+ discard does the gateway clear its attempt-owned state. Cancellation while+ buffering, waiting, or sending the next body still owns cleanup.+- Only the final admitted producer supplies response headers. A pinned Vibey+ continuation is never replayed on another Source: existing `pin_unavailable`+ handback asks its caller to re-render portable content instead.+- A spool failure is a local delivery failure, not replay permission or evidence+ against the provider. Upstream usage is retained. If closing a still-live+ producer yields no protocol terminal, settlement records that local ending+ without fabricating an upstream outcome; admission and transport are released.++## Known by design++- Initial Responses output, reasoning, and client-executed tool calls are not+ visible until the first stream ends. This first-byte latency is the cost of+ preventing duplicate downstream effects; other protocols still stream live.+- Provider-side hosted tools may already have run, and the provider may bill+ both attempts. This does not provide exactly-once upstream execution.+- Each spool uses at most 128 MiB of temporary storage. A full/unreadable+ temporary disk or a response exceeding that cap ends the request locally;+ temporary files are closed on every owned exit.+- No route/config/credential changes, schema migrations, deployment, service+ restart, or live upstream experiment is included.++## Validation++The owner-boundary tests use real loopback HTTP, the production engine client,+and the gateway; all credentials and state are synthetic and test-owned.++- `MH-RETRY-OVERLOAD-001`: exact root/nested error matrix, custom and OpenAI+ Sources, retry success/failure, lossless nonmatches, distinct usage and+ provenance, real Vibey fallback origin and pin handback, other protocols.+- `MH-RETRY-OVERLOAD-002`: withholding output, disk rollover, disconnect during+ buffering/recovery wait, local spool write/read failures, and the size cap.+- `MH-RETRY-OVERLOAD-003`: earlier pre-output refusal and streamed overload share+ one recovery clock and failure streak; expired admission exposes no discarded+ bytes and an overload cannot announce recovery merely by producing hidden text.++The positive regression was run before the fix and failed because the original+partial text/tool calls and overload were returned instead of the retry body.+Adjacent recovery, inference-wait, L3, and Vibey consumer tests are also run.+Exact-head Codex review and CI remain delivery gates.diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yamlindex d091cf2bf..6a6e3c082 100644--- a/tests/scenarios/INDEX.yaml+++ b/tests/scenarios/INDEX.yaml@@ -95,6 +95,8 @@ capabilities:- tests/test_model_hub_retry_advice.py- tests/test_model_hub_retry_policy.py- tests/test_model_hub_retry_lifecycle.py+ - tests/test_model_hub_streamed_overload.py+ - tests/test_model_hub_streamed_overload_lifecycle.py- tests/test_model_hub_recovery_evidence.py- tests/test_model_hub_progress.py- tests/test_model_hub_failure_presentation.pydiff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yamlindex 2fb6dea6d..40698a05b 100644--- a/tests/scenarios/model_hub/catalog.yaml+++ b/tests/scenarios/model_hub/catalog.yaml@@ -36,6 +36,8 @@ capability:- tests/test_model_hub_retry_advice.py- tests/test_model_hub_retry_policy.py- tests/test_model_hub_retry_lifecycle.py+ - tests/test_model_hub_streamed_overload.py+ - tests/test_model_hub_streamed_overload_lifecycle.py- tests/test_model_hub_recovery_evidence.py- tests/test_model_hub_progress.py- tests/test_model_hub_failure_presentation.py@@ -737,6 +739,31 @@ scenarios:kind: recoverylayer: unittest: tests/test_model_hub_retry_policy.py::test_one_window_covers_all_fallback_passes_without_replaying_after_output+ - id: MH-RETRY-OVERLOAD-001+ name: Exact Responses overload discards the first body and replays once with independent accounting+ status: covered+ kind: recovery+ layer: unit+ test:+ - tests/test_model_hub_streamed_overload.py::test_buffered_overload_commits_only_final_attempt+ - tests/test_model_hub_streamed_overload.py::test_other_response_endings_are_lossless_and_never_replayed+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_vibey_overload_uses_backup_without_leaking_origin_or_pinned_content+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_other_protocols_still_stream_before_terminal_and_never_replay+ - id: MH-RETRY-OVERLOAD-002+ name: Buffered Responses withhold output and retain spool and upstream ownership through cancellation+ status: covered+ kind: recovery+ layer: unit+ test:+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_initial_response_waits_for_terminal_and_owns_spool_until_cleanup+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_spool_io_failure_is_local_metered_and_never_replayed+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_cancellation_during_replay_wait_keeps_only_the_metered_failed_attempt+ - id: MH-RETRY-OVERLOAD-003+ name: Buffered overload replay shares the original preoutput recovery admission window+ status: covered+ kind: recovery+ layer: unit+ test: tests/test_model_hub_streamed_overload_lifecycle.py::test_replay_keeps_preoutput_recovery_window- id: MH-RETRY-RECOVERY-001name: Real protocol output or empty completion proves recovery while unknown success does notstatus: covereddiff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.pyindex 2e1b8a2b5..b48e7be2c 100644--- a/tests/test_model_hub_retry_lifecycle.py+++ b/tests/test_model_hub_retry_lifecycle.py@@ -90,9 +90,11 @@ async def gateway_client(service, backend, model):async def loopback_engine(tmp_path, service, respond, *, models=None):"""Use the production client/admission/stream observer against local HTTP."""config = service.store.load()- source = config.sources[0]state = EngineStateStore(tmp_path / "engine")- source.credential_ref = state.store_api_key("fixture", vendor=source.vendor, protocol=source.protocol)+ for source in config.sources:+ source.credential_ref = state.store_api_key(+ "fixture", vendor=source.vendor, protocol=source.protocol, base_url=source.base_url,+ )service.store.save(config)state.sync_sources(service._bindings(config))app = web.Application()diff --git a/tests/test_model_hub_streamed_overload.py b/tests/test_model_hub_streamed_overload.pynew file mode 100644index 000000000..5ee24d52f--- /dev/null+++ b/tests/test_model_hub_streamed_overload.py@@ -0,0 +1,131 @@+"""The real gateway must commit only the winning Responses stream.++Existing recovery tests stop at the first-output barrier. These exercise the+exception after text/tool output, so a classification-only retry (duplicate+output), unmetered replay, or an unbounded replay loop cannot pass.+"""++from __future__ import annotations++import asyncio+import json++from aiohttp import web+import pytest++from core.handlers.model_hub.provenance import SERVED_HOP_HEADER+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT+from tests.test_model_hub_l3 import _source, _usage_of+from tests.test_model_hub_retry_lifecycle import configured_service, gateway_client, loopback_engine+++OVERLOAD = "Our servers are currently overloaded. Please try again later."+++def event(payload):+ return b"event: " + payload["type"].encode() + b"\ndata: " + json.dumps(payload).encode() + b"\n\n"+++def output(label):+ return (+ event({"type": "response.output_text.delta", "delta": label})+ + event({"type": "response.output_item.added", "item": {+ "type": "function_call", "id": label, "call_id": label, "name": "fixture_tool", "arguments": "{}",+ }})+ )+++def ending(*, error=None, message=OVERLOAD, envelope="response.failed", tokens=11, error_type=None):+ usage = {"input_tokens": tokens, "output_tokens": 2}+ if error and envelope == "error":+ return event({"type": "error", "code": error, "message": message})+ status = "failed" if error else "completed"+ return event({"type": f"response.{status}", "response": {+ "status": status, "usage": usage,+ **({"error": {"code": error, "message": message, **({"type": error_type} if error_type else {})}} if error else {}),+ }})++[email protected]("vendor", ["openai", "custom"])[email protected]("envelope", ["response.failed", "error"])[email protected]("second_error", [None, "server_error", "permission_error"])+def test_buffered_overload_commits_only_final_attempt(tmp_path, vendor, envelope, second_error):+ """MH-RETRY-OVERLOAD-001: one replay, isolated bodies and per-call accounting."""+ async def run():+ source = _source("src_buffered01", "Buffered", vendor=vendor)+ source.base_url = "https://fixture.invalid"+ service, clock, models = configured_service(tmp_path, sources=[source])+ calls = []+ first = output("discarded-text-and-tool") + ending(error="server_error", envelope=envelope)+ final = output("final-text-and-tool") + ending(error=second_error, tokens=17)++ async def respond(request):+ calls.append(await request.json())+ assert len(calls) <= 2, "the second streamed failure must not replay"+ return web.Response(+ body=first if len(calls) == 1 else final, content_type="text/event-stream",+ headers={"x-request-id": f"attempt-{len(calls)}"},+ )++ async with loopback_engine(tmp_path, service, respond) as (adapter, _):+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:+ assert response.status == 200+ body = await response.read()+ assert body == final+ assert SERVED_HOP_HEADER not in response.headers # Native callers do not receive Vibey origins.+ await gateway._drain_turn_requests("turn_lifecycle")+ assert len(calls) == 2 and calls[0] == calls[1]+ assert clock.delays == [30]+ assert adapter._active_transports == 0+ usage = _usage_of(service, source.id)+ assert usage["requests"] == 2+ assert usage["input_tokens"] == (28 if envelope == "response.failed" else 17)+ gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)+ record = service.provenance.get("turn_lifecycle")+ assert record["failed_attempts"][0]["reason"] == "server_error"+ assert record["failed_attempts"][0]["response_correlation_ids"] == [+ {"header": "x-request-id", "value": "attempt-1"},+ ]+ if second_error is None:+ assert record["outcome"] == "served"+ assert record["served"]["response_correlation_ids"] == [+ {"header": "x-request-id", "value": "attempt-2"},+ ]+ else:+ assert record["outcome"] != "served"+ assert gateway.correlation.recovery_snapshot("turn_lifecycle") == []+ asyncio.run(run())++[email protected]("error,error_type,message", [+ (None, None, OVERLOAD),+ ("server_error", None, "An internal error occurred."),+ ("server_error", None, OVERLOAD + " extra"),+ ("server_error", None, OVERLOAD.replace("currently overloaded", "currently overloaded")),+ ("permission_error", None, OVERLOAD),+ ("overloaded_error", None, OVERLOAD),+ ("different_vendor_error", "server_error", OVERLOAD),+ ("internal_error", "server_error", OVERLOAD),+])+def test_other_response_endings_are_lossless_and_never_replayed(tmp_path, error, error_type, message):+ """MH-RETRY-OVERLOAD-001: exact typed envelope, not text/code heuristics."""+ async def run():+ service, _clock, models = configured_service(tmp_path)+ body = output("single-attempt") + ending(error=error, message=message, error_type=error_type)+ calls = 0++ async def respond(request):+ nonlocal calls+ calls += 1+ assert calls == 1+ return web.Response(body=body, content_type="text/event-stream")++ async with loopback_engine(tmp_path, service, respond):+ async with gateway_client(service, "codex", models["codex"]) as (_, client, url, headers):+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:+ assert response.status == 200 and await response.read() == body+ if error == "server_error":+ assert service.store.load().sources[0].state.status == "cooldown"+ assert calls == 1+ asyncio.run(run())diff --git a/tests/test_model_hub_streamed_overload_lifecycle.py b/tests/test_model_hub_streamed_overload_lifecycle.pynew file mode 100644index 000000000..04c097875--- /dev/null+++ b/tests/test_model_hub_streamed_overload_lifecycle.py@@ -0,0 +1,314 @@+"""Buffering must preserve caller identity, recovery clocks, and cancellation.++These are distinct from the exact-error matrix: they exercise real in-flight+HTTP ownership and the Vibey pin/fallback boundary, which fixed response bodies+cannot protect.+"""++import asyncio+import json++import aiohttp+from aiohttp import web+import pytest++from core.handlers.model_hub import turn_gateway+from core.handlers.model_hub.provenance import SERVED_HOP_HEADER+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT+from tests.test_model_hub_l3 import _source, _usage_of+from tests.test_model_hub_retry_policy import Clock, WIRE+from tests.test_model_hub_vibey_consumer import _vibey_service+from tests.test_model_hub_retry_lifecycle import configured_service, gateway_client, loopback_engine+from tests.test_model_hub_streamed_overload import OVERLOAD, output, ending, event++[email protected]("pinned", [False, True])+def test_vibey_overload_uses_backup_without_leaking_origin_or_pinned_content(tmp_path, pinned):+ """MH-RETRY-OVERLOAD-001: real Vibey caller, fallback headers and pin handback."""+ async def run():+ primary = _source("src_primary01", "Primary")+ backup = _source("src_backup001", "Backup")+ service = _vibey_service(tmp_path, [primary, backup])+ clock = Clock()+ service.now, service.recovery = clock.now, clock.policy()+ calls = []+ first = output("discarded") + ending(error="server_error")+ final = output("backup") + ending()++ async def respond(request):+ calls.append(await request.json())+ return web.Response(body=first if len(calls) == 1 else final, content_type="text/event-stream")++ async with loopback_engine(tmp_path, service, respond) as (adapter, _):+ gateway = ModelHubTurnGateway(service)+ base, token = await gateway.endpoint(+ "vibey", process_scope="buffered", turn_id="turn_vibey",+ requested_model_id="menu-alias", resolved_model_id="shared-model", source_id=primary.id,+ )+ try:+ headers = {"Authorization": f"Bearer {token}", "x-avibe-turn-id": "turn_vibey"}+ if pinned:+ headers.update({"x-avibe-pin-source": primary.id, "x-avibe-pin-model": "shared-model"})+ async with aiohttp.ClientSession(trust_env=False) as client:+ async with client.post(+ f"{base}/v1/responses", headers=headers,+ json={"model": "shared-model", "stream": True, "input": "fixture prompt"},+ ) as response:+ if pinned:+ assert response.status == 409+ assert (await response.json())["error"]["code"] == "pin_unavailable"+ assert SERVED_HOP_HEADER not in response.headers+ else:+ assert response.status == 200 and await response.read() == final+ assert json.loads(response.headers[SERVED_HOP_HEADER])["source"] == backup.id+ await gateway._drain_turn_requests("turn_vibey")+ assert len(calls) == (1 if pinned else 2)+ assert clock.delays == [] # A ready backup is not the cooling primary.+ assert adapter._active_transports == 0+ assert _usage_of(service, primary.id)["requests"] == 1+ if pinned:+ assert gateway.correlation.terminal_projection("turn_vibey", backend="vibey") is None+ else:+ assert _usage_of(service, backup.id)["requests"] == 1+ gateway.correlation.settle("turn_vibey", settled_by=SETTLED_BY_TERMINAL_RESULT)+ assert service.provenance.get("turn_vibey")["served"]["source_id"] == backup.id+ finally:+ await gateway.close()+ asyncio.run(run())++[email protected]("cancel", [False, True])+def test_initial_response_waits_for_terminal_and_owns_spool_until_cleanup(tmp_path, monkeypatch, cancel):+ """MH-RETRY-OVERLOAD-002: no early downstream bytes, bounded RAM, disconnect cleanup."""+ async def run():+ service, _clock, models = configured_service(tmp_path)+ prefix_sent, release, upstream_closed = (asyncio.Event() for _ in range(3))+ body = output("before-terminal-" + "x" * 300_000)+ files = []+ create = turn_gateway.tempfile.SpooledTemporaryFile++ def track_file(*args, **kwargs):+ payload = create(*args, **kwargs)+ files.append(payload)+ return payload++ monkeypatch.setattr(turn_gateway.tempfile, "SpooledTemporaryFile", track_file)++ async def respond(request):+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})+ await response.prepare(request)+ try:+ await response.write(body)+ prefix_sent.set()+ await release.wait()+ await response.write(ending())+ await response.write_eof()+ return response+ finally:+ upstream_closed.set()++ async with loopback_engine(tmp_path, service, respond) as (adapter, _):+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):+ task = asyncio.create_task(client.post(url, headers=headers, json={"model": "shared-model", "stream": True}))+ try:+ await asyncio.wait_for(prefix_sent.wait(), 2)+ assert not (await asyncio.wait({task}, timeout=0.05))[0], "headers committed before terminal"+ if cancel:+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ await asyncio.wait_for(upstream_closed.wait(), 2)+ else:+ release.set()+ response = await asyncio.wait_for(task, 2)+ assert await response.read() == body + ending()+ await gateway._drain_turn_requests("turn_lifecycle")+ assert adapter._active_transports == 0+ assert files and all(payload.closed for payload in files)+ assert any(payload._rolled for payload in files)+ assert _usage_of(service, service.store.load().sources[0].id)["requests"] == 1+ finally:+ release.set()+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ asyncio.run(run())++[email protected]("late_seconds,expected_calls", [(0, 3), (121, 2)])+def test_replay_keeps_preoutput_recovery_window(tmp_path, late_seconds, expected_calls):+ """MH-RETRY-OVERLOAD-003: an earlier refusal's window must not reset at replay."""+ async def run():+ service, clock, models = configured_service(tmp_path)+ calls = []+ first = output("discarded") + ending(error="server_error")+ final = output("final") + ending()++ async def respond(request):+ calls.append(await request.json())+ if len(calls) == 1:+ return web.json_response({"error": {"type": "server_error"}}, status=503)+ if len(calls) == 2:+ clock.advance(late_seconds)+ else:+ assert not any(event["kind"] == "recover" for event in service.events.list())+ return web.Response(body=first if len(calls) == 2 else final, content_type="text/event-stream")++ async with loopback_engine(tmp_path, service, respond):+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:+ if late_seconds:+ assert response.status == 400+ assert (await response.json())["error"]["code"] == "model_hub_recovery_exhausted"+ else:+ assert response.status == 200 and await response.read() == final+ await gateway._drain_turn_requests("turn_lifecycle")+ assert len(calls) == expected_calls+ assert clock.delays == ([30] if late_seconds else [30, 60])+ assert sum(event["kind"] == "recover" for event in service.events.list()) == (0 if late_seconds else 1)+ assert gateway.correlation.recovery_snapshot("turn_lifecycle") == []+ asyncio.run(run())++[email protected]("backend,protocol", [("claude", "anthropic"), ("opencode", "openai_chat")])+def test_other_protocols_still_stream_before_terminal_and_never_replay(tmp_path, backend, protocol):+ """MH-RETRY-OVERLOAD-001: the exception must not broaden to other wire protocols."""+ async def run():+ source = _source("src_other001", "Other", protocol=protocol, vendor="anthropic" if backend == "claude" else "openai")+ service, clock, models = configured_service(tmp_path, sources=[source])+ prefix = WIRE[protocol][0] + WIRE[protocol][1]+ error = (b'data: ' if protocol == "openai_chat" else b'event: error\ndata: ') + json.dumps({+ "type": "error", "error": {"type": "server_error", "message": OVERLOAD},+ }).encode() + b"\n\n"+ release = asyncio.Event()+ calls = 0++ async def respond(request):+ nonlocal calls+ calls += 1+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})+ await response.prepare(request)+ await response.write(prefix)+ await release.wait()+ await response.write(error)+ return response++ async with loopback_engine(tmp_path, service, respond):+ async with gateway_client(service, backend, models[backend]) as (_, client, url, headers):+ try:+ response = await asyncio.wait_for(client.post(+ url, headers=headers, json={"model": "shared-model", "stream": True},+ ), 2)+ assert await asyncio.wait_for(response.content.readexactly(len(prefix)), 2) == prefix+ release.set()+ assert await response.read() == error+ assert calls == 1 and clock.delays == []+ finally:+ release.set()+ asyncio.run(run())++[email protected]("stage,terminal", [+ ("write", True), ("read", True), ("limit", True),+ ("write", False), ("limit", False),+])+def test_spool_io_failure_is_local_metered_and_never_replayed(tmp_path, monkeypatch, stage, terminal):+ """MH-RETRY-OVERLOAD-002: added disk I/O cannot lose usage or blame the provider."""+ async def run():+ service, clock, models = configured_service(tmp_path)+ original = turn_gateway.tempfile.SpooledTemporaryFile+ files = []+ calls = 0+ upstream_closed = asyncio.Event()+ body = output("answer") + ending() if terminal else event({+ "type": "response.created", "response": {"usage": {"input_tokens": 13, "output_tokens": 1}},+ }) + output("answer")++ def faulty_spool(*args, **kwargs):+ payload = original(*args, **kwargs)+ files.append(payload)++ def fail_io(*_args):+ raise OSError(28 if stage == "write" else 5, "fixture local disk error")++ if stage != "limit":+ setattr(payload, stage, fail_io)+ return payload++ monkeypatch.setattr(turn_gateway.tempfile, "SpooledTemporaryFile", faulty_spool)+ if stage == "limit":+ monkeypatch.setattr(turn_gateway, "_MAX_REPLAY_BUFFER_BYTES", len(body) - 1)++ async def respond(request):+ nonlocal calls+ calls += 1+ if terminal:+ return web.Response(body=body, content_type="text/event-stream")+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})+ await response.prepare(request)+ try:+ await response.write(body)+ # No terminal exists when storage fails. Only gateway teardown+ # may close this still-live upstream, not a fixture timeout.+ await asyncio.Future()+ finally:+ upstream_closed.set()++ async with loopback_engine(tmp_path, service, respond) as (adapter, _):+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:+ response_body = await response.read()+ assert response.status == (200 if stage == "read" else 502)+ assert b"engine_down" in response_body+ await gateway._drain_turn_requests("turn_lifecycle")+ if not terminal:+ await asyncio.wait_for(upstream_closed.wait(), 2)+ assert adapter._active_transports == 0 and clock.delays == [] and calls == 1+ assert files and all(payload.closed for payload in files)+ source = service.store.load().sources[0]+ assert _usage_of(service, source.id)["requests"] == 1+ if not terminal:+ assert _usage_of(service, source.id)["input_tokens"] == 13+ assert source.state.status == "standby"+ gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)+ assert service.provenance.get("turn_lifecycle")["terminal_error"]["reason"] == "engine_down"+ asyncio.run(run())+++def test_cancellation_during_replay_wait_keeps_only_the_metered_failed_attempt(tmp_path):+ """MH-RETRY-OVERLOAD-002: dropping the first body must reset every attempt-owned debt."""+ async def run():+ service, _clock, models = configured_service(tmp_path)+ waiting = asyncio.Event()+ calls = 0++ async def wait_for_cancel(_seconds):+ waiting.set()+ await asyncio.Future()++ service.recovery.sleep = wait_for_cancel++ async def respond(request):+ nonlocal calls+ calls += 1+ return web.Response(body=output("discarded") + ending(error="server_error"), content_type="text/event-stream")++ async with loopback_engine(tmp_path, service, respond) as (adapter, _):+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):+ task = asyncio.create_task(client.post(url, headers=headers, json={"model": "shared-model", "stream": True}))+ try:+ await asyncio.wait_for(waiting.wait(), 2)+ progress = gateway.correlation.recovery_snapshot("turn_lifecycle")+ assert progress and progress[0]["phase"] == "waiting"+ assert gateway.correlation.terminal_projection("turn_lifecycle", backend="codex") is None+ finally:+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ await gateway._drain_turn_requests("turn_lifecycle")+ assert adapter._active_transports == 0 and calls == 1+ source = service.store.load().sources[0]+ assert _usage_of(service, source.id)["requests"] == 1+ assert service.recovery._sources[source.id].owner is None+ assert gateway.correlation.recovery_snapshot("turn_lifecycle") == []+ asyncio.run(run())diff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.pyindex c26cbd748..826b8aac6 100644--- a/tests/test_model_hub_structure_guards.py+++ b/tests/test_model_hub_structure_guards.py@@ -758,7 +758,10 @@ def test_usage_metering_has_one_owner_per_call_population() -> None:# handle. Review 4970...: it used to ask whether the turn was *settled*, which# a forced terminal had already made true for a call the vendor billed.owes = _functions(gateway_tree)["owes_metering"]- readers = (recorder, owes)+ # A buffered overload retires a whole upstream attempt before adopting a+ # second one. That transition owns clearing the prior write handle; the+ # real gateway replay tests prove both calls are still metered once.+ readers = (recorder, owes, _functions(gateway_tree)["next_attempt"])handles = [nodefor node in ast.walk(gateway_tree)@@ -806,7 +809,9 @@ def test_settling_a_turn_is_what_meters_it_for_every_ending_there_will_be() -> Nendings = [node for node in ast.walk(gateway_tree) if _call_name(node) == "_settle_metered_turn"]assert len(endings) >= 2assert all(- any(ending in set(ast.walk(functions[name])) for name in ("_settle_boundary_termination", "_resolved_response"))+ any(ending in set(ast.walk(functions[name])) for name in (+ "_settle_boundary_termination", "_resolved_response", "_buffer_responses_stream",+ ))for ending in endings)@@ -821,20 +826,24 @@ def test_a_metered_row_is_dated_by_the_call_and_not_by_what_followed_it() -> Non# test can make one. What is left to protect is where the instant comes from.gateway_tree = _tree(TURN_GATEWAY)functions = _functions(gateway_tree)- # Captured where the upstream body ends, which is the one function that has a- # body to end. A capture anywhere else is a second answer to when the call- # finished, and the endings are exactly the places that would get it wrong.+ # Captured where the upstream body ends, whether it is forwarded live or+ # spooled before commit. Clearing the completed attempt for a replay is not+ # another timestamp capture; each call retains its own completion instant.captures = [nodefor node in ast.walk(gateway_tree)if isinstance(node, ast.Assign)+ and not (isinstance(node.value, ast.Constant) and node.value.value is None)and any(isinstance(target, ast.Attribute) and target.attr == "completed_at"for target in node.targets)]assert captures- assert all(capture in set(ast.walk(functions["_resolved_response"])) for capture in captures)+ assert all(+ any(capture in set(ast.walk(functions[name])) for name in ("_resolved_response", "_buffer_responses_stream"))+ for capture in captures+ )# And carried from there to the ledger rather than re-read at the write.handoff = next(node for node in ast.walk(functions["_record_usage"]) if _call_name(node) == "record"