Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): buffer Responses output for one exact overload replay (#3007)

Call White95de1ee3a339
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index 33a7c5dd7..2b5f61cae 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -1034,7 +1034,7 @@ class HandleSettlement:
turn_outcome: TurnOutcomeProjectionInput | None
-HandleTerminationOrigin = Literal["downstream_cancel", "upstream_terminal"]
+HandleTerminationOrigin = Literal["downstream_cancel", "upstream_terminal", "local_delivery_failure"]
AttemptObserver = Callable[
@@ -10595,12 +10595,14 @@ class ModelHubService:
*,
termination_origin: HandleTerminationOrigin,
record_attempt: Callable[[RawCallOutcome, ResolutionDecision], None],
+ retry_buffered_overload: bool = False,
) -> HandleSettlement:
"""Settle every consumed hub handle before its terminal facts are exposed."""
try:
return await self._settle_handle_outcome(
resolved, outcome, termination_origin=termination_origin, record_attempt=record_attempt,
+ retry_buffered_overload=retry_buffered_overload,
)
finally:
if resolved is not None:
@@ -10613,21 +10615,27 @@ class ModelHubService:
*,
termination_origin: HandleTerminationOrigin,
record_attempt: Callable[[RawCallOutcome, ResolutionDecision], None],
+ retry_buffered_overload: bool,
) -> HandleSettlement:
- if resolved is not None and resolved.handle is not None:
+ if resolved is not None and resolved.handle is not None and not retry_buffered_overload:
await self._observe_handle_recovery(
resolved.source_id, resolved.settlement_generation, resolved.handle,
backend=resolved.backend, model_id=resolved.requested_model_id,
)
- if termination_origin == "downstream_cancel" and outcome is None:
+ if termination_origin in {"downstream_cancel", "local_delivery_failure"} and outcome is None:
+ # Closing a still-live producer does not manufacture its terminal.
+ # Metering already retained every observed token; release admission
+ # without classifying a local disk failure as an upstream refusal.
return HandleSettlement(
- outcome=outcome,
+ outcome=None,
decision=None,
- turn_outcome=produce_turn_outcome("turn.canceled"),
+ turn_outcome=produce_turn_outcome(
+ "turn.canceled" if termination_origin == "downstream_cancel" else "turn.engine_down",
+ ),
)
- if termination_origin == "downstream_cancel":
- # The gateway selects this origin only after closing the producer.
- # A committed outcome at that barrier owns history over cancellation.
+ if termination_origin in {"downstream_cancel", "local_delivery_failure"}:
+ # A producer result committed during close still owns its Source
+ # settlement; a local delivery failure projects separately afterwards.
termination_origin = "upstream_terminal"
if termination_origin != "upstream_terminal":
raise AssertionError("unknown handle termination origin")
@@ -10639,6 +10647,25 @@ class ModelHubService:
resolved.credential_ref,
outcome,
)
+ # Only the gateway can grant this after matching the exact overload
+ # in a fully buffered, uncommitted Responses body. Keep the upstream
+ # stream_started fact intact for usage and provenance.
+ retry_overload = (
+ retry_buffered_overload
+ and outcome.kind is RawOutcomeKind.HTTP_ERROR and outcome.http_status == 200
+ and outcome.stream_started
+ and decision.action == "surface" and decision.reason == "server_error"
+ )
+ if retry_overload:
+ decision = replace(decision, action="fallback", error_code=None)
+ elif retry_buffered_overload and resolved.handle is not None:
+ # Preserve ordinary first-output recovery evidence, but not for the
+ # discarded overload: it must extend the existing failure streak,
+ # not briefly announce recovery and reset its own backoff.
+ await self._observe_handle_recovery(
+ resolved.source_id, resolved.settlement_generation, resolved.handle,
+ backend=resolved.backend, model_id=resolved.requested_model_id,
+ )
record_attempt(outcome, decision)
if decision.action == "return":
if self._verified_recovery_outcome(outcome):
@@ -10662,7 +10689,7 @@ class ModelHubService:
decision=decision,
),
)
- if decision.action != "surface":
+ if decision.action != "surface" and not retry_overload:
raise AssertionError("consumed streams cannot retry or fall through")
source_transition_persisted: bool | None = None
if decision.reason is not None:
@@ -10681,6 +10708,10 @@ class ModelHubService:
settlement_generation=resolved.settlement_generation,
outcome=outcome,
)
+ if retry_overload:
+ # An attempt failed, not the Turn. The resolver still owns its
+ # original recovery window, routing, and the eventual terminal.
+ return HandleSettlement(outcome=outcome, decision=decision, turn_outcome=None)
return HandleSettlement(
outcome=outcome,
decision=decision,
@@ -11673,9 +11704,16 @@ class ModelHubService:
)
async def resolve_with_recovery(
- self, *, recovery_observer: Callable[[dict | None], None] | None = None, **kwargs,
+ self, *, recovery_observer: Callable[[dict | None], None] | None = None,
+ prepare_response: Callable[[ResolvedInvocation], Awaitable[RawCallOutcome | None]] | None = None,
+ **kwargs,
) -> ResolvedInvocation:
- """One pending model request, including all fallback passes and waits."""
+ """One pending model request, including all fallback passes and waits.
+
+ A gateway may buffer and settle a response before committing it. It
+ returns only a settled retryable outcome, never an unconsumed handle;
+ that replay shares this pending request rather than opening a new window.
+ """
if self.recovery.window_seconds == 0:
# Explicit opt-out for single-walk probes and characterization tests.
@@ -11685,7 +11723,17 @@ class ModelHubService:
try:
while True:
try:
- return await self.resolve(**kwargs, recovery_request=pending)
+ resolved = await self.resolve(**kwargs, recovery_request=pending)
+ retry_outcome = await prepare_response(resolved) if prepare_response is not None else None
+ if retry_outcome is None:
+ return resolved
+ pending.start()
+ pending.reason = "server_error"
+ pending.last_upstream_failure = self._upstream_failure(resolved.source_label, retry_outcome)
+ if getattr(kwargs.get("request"), "pinned_hop", None) is not None:
+ # Never replay opaque continuation content on a different
+ # hop. Its caller owns re-rendering the portable request.
+ raise _pin_unavailable()
except ModelHubError as exc:
projection = exc.turn_outcome
if (
diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.py
index 42dfdd88d..bfd2a1ab3 100644
--- a/core/handlers/model_hub/turn_gateway.py
+++ b/core/handlers/model_hub/turn_gateway.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import errno
import functools
import json
import logging
@@ -76,6 +77,7 @@ from core.os_errors import local_error_detail
_MAX_REQUEST_BYTES: Final = 128 * 1024 * 1024
_BUFFERED_RESPONSE_MEMORY_BYTES: Final = 256 * 1024
+_MAX_REPLAY_BUFFER_BYTES: Final = 128 * 1024 * 1024
_RESPONSE_CHUNK_BYTES: Final = 64 * 1024
# Claude Code aborts a stream whose response headers miss its first-byte window
# and treats a byte-silent body as stalled. Resolution can outlast both through
@@ -243,6 +245,30 @@ class _TurnExecution:
# A large buffered JSON body whose headers have already left. A local disk
# failure can no longer replace its HTTP status or append an SSE ending.
buffered_response: web.StreamResponse | None = None
+ # An initial Responses stream is held until its terminal is known. This
+ # flag is armed only after the entire body is safely stored, never during
+ # cancellation or a failed spool write.
+ buffered_stream: BinaryIO | None = None
+ retry_buffered_overload: bool = False
+
+ def next_attempt(self) -> None:
+ """Retire a fully settled/metered discarded attempt without ending its request."""
+
+ assert self.settlement_task is not None and self.settlement_task.done()
+ assert not self.owes_metering and self.stream_response is None
+ self.resolved = None
+ self.handle = None
+ self.settlement_task = None
+ self.settlement_origin = None
+ self.settlement_recorded = False
+ self.terminal_fact_committed = False
+ self.rendered_turn_outcome = None
+ self.wire_state = None
+ self.handle_outcome = None
+ self.completed_at = None
+ self.usage_write = None
+ self.buffered_stream = None
+ self.retry_buffered_overload = False
@property
def upstream_observation(self) -> ProtocolSSEState | None:
@@ -1143,6 +1169,32 @@ class ModelHubTurnGateway:
)
protocol = "google" if google_request else _REQUEST_PROTOCOLS[endpoint]
+ replayed = False
+
+ def adopt(resolved: ResolvedInvocation) -> None:
+ # Own the body before buffering, health writes, or keepalive teardown
+ # can be cancelled. Each callback closes only its captured handle.
+ execution.resolved = resolved
+ if resolved.handle is not None and resolved.handle.stream is not None:
+ execution.handle = resolved.handle
+ resources.push_async_callback(resolved.handle.close_stream)
+
+ async def prepare_response(resolved: ResolvedInvocation) -> RawCallOutcome | None:
+ nonlocal replayed
+ adopt(resolved)
+ if replayed or resolved.outcome is not None or execution.handle is None:
+ return None
+ outcome, settlement = await self._buffer_responses_stream(execution, terminalizer, resources)
+ if settlement.decision is None or settlement.decision.action != "fallback":
+ return None
+ # The first partial text/tool calls never reached the consumer. Its
+ # attempt has already settled and been metered, independently of
+ # whatever the next route resolves to (including a local refusal).
+ await run_owned_in_thread(execution.buffered_stream.close)
+ replayed = True
+ execution.next_attempt()
+ return outcome
+
try:
caller_headers = {
name.lower(): value for name, value in request.headers.items() if name.lower() in FORWARDED_CALLER_HEADERS
@@ -1174,13 +1226,13 @@ class ModelHubTurnGateway:
supply_channel="hub",
attempt_observer=observe_attempt,
recovery_observer=terminalizer.update_recovery,
+ prepare_response=(
+ prepare_response if stream and protocol == "openai_responses"
+ and self.service.recovery.window_seconds > 0 else None
+ ),
)
- # Owned before the keepalive stops, so a cancellation while it
- # stops still closes and meters a Source stream already won.
- execution.resolved = resolved
- if resolved.handle is not None and resolved.handle.stream is not None:
- execution.handle = resolved.handle
- resources.push_async_callback(resolved.handle.close_stream)
+ if execution.resolved is not resolved:
+ adopt(resolved)
except ModelHubError as exc:
if exc.code == PIN_UNAVAILABLE_CODE:
# Never shown to the user: Vibey re-renders the request in
@@ -1237,7 +1289,7 @@ class ModelHubTurnGateway:
response.headers.update(_response_origin_headers(exc))
return await self._through_committed_stream(execution, protocol, response)
- if execution.handle is not None:
+ if execution.handle is not None and execution.settlement_task is None:
# The execution boundary must own cleanup before an awaited health
# write: cancellation here still settles and meters this handle.
await self.service._observe_handle_recovery(
@@ -1327,6 +1379,65 @@ class ModelHubTurnGateway:
await self._downstream_io(committed.write_eof())
return committed
+ async def _buffer_responses_stream(
+ self,
+ execution: _TurnExecution,
+ terminalizer: GatewayTurnTerminalizer,
+ resources: AsyncExitStack,
+ ) -> tuple[RawCallOutcome, HandleSettlement]:
+ """Observe and settle one uncommitted stream; no content leaves here."""
+
+ handle = execution.handle
+ assert handle is not None and handle.stream is not None
+ payload = tempfile.SpooledTemporaryFile(max_size=_BUFFERED_RESPONSE_MEMORY_BYTES)
+ execution.buffered_stream = payload
+ resources.push_async_callback(run_owned_in_thread, payload.close)
+ wire_state = ProtocolSSEState("openai_responses")
+ execution.wire_state = wire_state
+ buffered_bytes = 0
+ try:
+ async for chunk in handle.stream:
+ await wire_state.observe_async(chunk)
+ buffered_bytes += len(chunk)
+ if buffered_bytes > _MAX_REPLAY_BUFFER_BYTES:
+ raise OSError(errno.EFBIG, "Model Hub response replay buffer limit exceeded")
+ await run_owned_in_thread(payload.write, chunk)
+ execution.completed_at = self._now()
+ # Match the bounded wire facts before the engine's display/code
+ # sanitizers can elide an unknown code or normalize message text.
+ # A root `type:error` selects the SSE envelope, not an error class.
+ error_types = set(wire_state.error_type_candidates)
+ if () in wire_state.error_envelope_paths:
+ error_types.discard("error")
+ execution.retry_buffered_overload = (
+ wire_state.terminal_outcome == "failed_terminal"
+ and wire_state.model_output_started
+ and error_types | set(wire_state.error_code_candidates) == {"server_error"}
+ and wire_state.error_message == "Our servers are currently overloaded. Please try again later."
+ )
+ outcome, settlement, _rendered = await self._settle_metered_turn(
+ execution, terminalizer, termination_origin="upstream_terminal", defer_projection=True,
+ )
+ await run_owned_in_thread(payload.seek, 0)
+ except OSError as exc:
+ # A local spool failure is never a reason to replay the upstream.
+ # Meter/settle its actual observation before reporting delivery loss.
+ execution.retry_buffered_overload = False
+ await self._settle_metered_turn(
+ execution, terminalizer, termination_origin="local_delivery_failure", defer_projection=True,
+ )
+ raise ModelHubError(
+ "engine_down", status=502, turn_outcome=ENGINE_DOWN_TURN_OUTCOME,
+ local_error_detail=local_error_detail(exc),
+ ) from exc
+ assert outcome is not None and settlement is not None
+ return outcome, settlement
+
+ @staticmethod
+ async def _read_buffered_stream(payload: BinaryIO) -> AsyncIterator[bytes]:
+ while chunk := await run_owned_in_thread(payload.read, _RESPONSE_CHUNK_BYTES):
+ yield chunk
+
async def _resolved_response(
self,
request: web.Request,
@@ -1339,9 +1450,9 @@ class ModelHubTurnGateway:
) -> web.StreamResponse:
if resolved.supply_channel != "hub":
return self._error_response(status=409, code="mode_switch_blocked")
- # Resolution has crossed the engine's first-output barrier or already
- # settled successfully: this request can no longer fail over. The same
- # admitted origin is held by its provenance attempt.
+ # Resolution crossed the engine's first-output barrier. A Responses
+ # overload may have been discarded before this point; only the final
+ # admitted producer supplies these downstream headers.
origin_headers = _response_origin_headers(resolved)
if resolved.outcome is not None:
# A call that reached the resolver's own hands has already been
@@ -1451,26 +1562,44 @@ class ModelHubTurnGateway:
if response is None:
response = web.StreamResponse(status=200, headers={**_SSE_RESPONSE_HEADERS, **origin_headers})
await self._downstream_io(response.prepare(request))
+ buffered = execution.buffered_stream
+ # Track what actually leaves, not the already-complete spooled wire. A
+ # disk read can fail between frames; only forwarded bytes may determine
+ # whether a terminal is still owed or a partial frame needs invalidating.
wire_state = ProtocolSSEState(protocol)
execution.wire_state = wire_state
+ body = self._read_buffered_stream(buffered) if buffered is not None else handle.stream
tool_name_rewriter = StreamingToolNameRewriter(execution.response_tool_aliases)
- async for chunk in handle.stream:
- output_started_before = wire_state.model_output_started
- await wire_state.observe_async(chunk)
- if wire_state.model_output_started and not output_started_before:
- terminalizer.mark_stream_started()
- rewritten = tool_name_rewriter.feed(chunk)
- if rewritten:
- await self._downstream_io(response.write(rewritten))
- trailing = tool_name_rewriter.finish()
- if trailing:
- await self._downstream_io(response.write(trailing))
- execution.completed_at = self._now()
- _outcome, _settlement, rendered = await self._settle_metered_turn(
- execution,
- terminalizer,
- termination_origin="upstream_terminal",
- )
+ try:
+ async for chunk in body:
+ output_started_before = wire_state.model_output_started
+ await wire_state.observe_async(chunk)
+ if wire_state.model_output_started and not output_started_before:
+ terminalizer.mark_stream_started()
+ rewritten = tool_name_rewriter.feed(chunk)
+ if rewritten:
+ await self._downstream_io(response.write(rewritten))
+ trailing = tool_name_rewriter.finish()
+ if trailing:
+ await self._downstream_io(response.write(trailing))
+ except OSError as exc:
+ if buffered is None or isinstance(exc, _DownstreamDisconnected):
+ raise
+ # The producer was already settled/metered before local playback.
+ # Delivery loss must not turn its success into Source ill health.
+ detail = local_error_detail(exc)
+ terminalizer.engine_down(local_error_detail=detail)
+ rendered = self._commit_and_render_turn_outcome(
+ execution, terminalizer, replace(ENGINE_DOWN_TURN_OUTCOME, local_error_detail=detail),
+ )
+ else:
+ if execution.completed_at is None:
+ execution.completed_at = self._now()
+ _outcome, _settlement, rendered = await self._settle_metered_turn(
+ execution,
+ terminalizer,
+ termination_origin="upstream_terminal",
+ )
await self._write_stream_terminal_copy(
response,
protocol,
@@ -1791,7 +1920,10 @@ class ModelHubTurnGateway:
# settlement/history -> t5 render settlement -> t6 downstream EOF.
if handle is not None:
await handle.close_stream()
- if termination_origin == "downstream_cancel" and handle is not None and handle.outcome_available:
+ if (
+ termination_origin in {"downstream_cancel", "local_delivery_failure"}
+ and handle is not None and handle.outcome_available
+ ):
# A producer outcome that became available during close owns history;
# a downstream write/cancel after that barrier cannot rewrite it.
termination_origin = "upstream_terminal"
@@ -1815,8 +1947,9 @@ class ModelHubTurnGateway:
outcome,
termination_origin=termination_origin,
record_attempt=record_attempt,
+ retry_buffered_overload=execution.retry_buffered_overload,
)
- if termination_origin == "downstream_cancel":
+ if termination_origin in {"downstream_cancel", "local_delivery_failure"}:
return settlement.outcome, settlement
assert settlement.outcome is not None
assert settlement.decision is not None
diff --git a/docs/CLI.md b/docs/CLI.md
index 514261b9a..73bb5349a 100644
--- a/docs/CLI.md
+++ b/docs/CLI.md
@@ -913,6 +913,26 @@ vibe restart --delay-seconds 60
The Model Hub engine process is named `cli-proxy-api` (hyphenated); `pgrep cliproxyapi` therefore always returns no match.
+#### Responses streams and overload recovery
+
+With automatic Model Hub recovery enabled, the first OpenAI Responses stream is
+buffered before delivery, including compatible custom Sources. You may therefore
+see no incremental text, reasoning, or tool calls until that response ends.
+If it ends with the exact `server_error` message
+`Our servers are currently overloaded. Please try again later.`, Model Hub
+discards that partial response and retries once through your configured route,
+within the same recovery window. Only the final attempt reaches the Agent.
+Other streamed errors and other protocols do not gain post-output retries.
+
+The retry itself streams normally and is not replayed again after output.
+Both upstream attempts are counted in usage and may be billed; provider-hosted
+tools can also have run on the discarded attempt. This prevents duplicate
+**downstream** text/tool calls, not duplicate provider-side execution. No Source
+order, credentials, or backend settings are changed by recovery. The initial
+response buffer uses at most 256 KiB of RAM before spilling to temporary disk,
+with a 128 MiB total cap. A disk error or an oversized response ends locally
+without another replay.
+
## Web UI Controls
The web UI (`http://127.0.0.1:5123`) provides the same controls:
diff --git a/docs/plans/model-hub-retry-experience.md b/docs/plans/model-hub-retry-experience.md
index d397a6e05..d46654b61 100644
--- a/docs/plans/model-hub-retry-experience.md
+++ b/docs/plans/model-hub-retry-experience.md
@@ -98,6 +98,12 @@ Retries stop once the first user-visible model-output byte is committed,
including recognized reasoning and tool-call output. Headers, SSE comments,
and protocol metadata are not output. Preserve lossless prelude handling.
+The narrow exception in `openai-streamed-overload-retry.md` buffers the initial
+Responses stream before downstream commit. Only an exact `server_error` overload
+may discard that body and replay once through this same recovery window. It
+includes compatible custom Sources; it does not broaden other error/protocol
+retry rules or promise exactly-once upstream execution.
+
Pre-output inference retry does not guarantee exactly-once upstream billing:
a dropped connection may hide completed inference. Keep every observed usage
report exactly once locally. Do not extend this policy to replaying upstream
@@ -382,7 +388,7 @@ in `tests/scenarios/model_hub/catalog.yaml` during implementation.
| Concurrent waiting Turns/backends | One recovery admission per affected Source; backups proceed; waiters do not each increment the streak or allocate a separate probe. |
| Recovery owner canceled or slow | No lock/transport leaks; canceled owner releases only its own admission; another waiter can proceed; slow connected inference is not killed. |
| Delayed headers / first output / buffered completion | Success beyond the former transport timeout and beyond the admission window; no health penalty, retry, or duplicate usage caused by elapsed inference time. |
-| Stream boundary | Heartbeats/headers alone do not prove recovery; reasoning and tool output do. No Hub replay after output, including engine/transport failure and cancellation races. Valid empty completion also releases recovery admission. |
+| Stream boundary | Heartbeats/headers alone do not prove recovery; reasoning and tool output do. Generic Hub replay stops after output. The exact OpenAI Responses overload exception buffers before downstream commit and permits one post-output replay in the same recovery window. Other streamed failures remain terminal. Valid empty completion also releases recovery admission. |
| Advice and clocks | Missing/malformed/past/large Retry-After, differing UTC offsets, clock shifts, and deterministic jitter; no stale serialized deadline or busy loop. |
| Stronger blockers and fault scope | Bad credentials, quota requiring action, local engine failure, mixed blockers, and explicit model errors retain correct remedies; waiting cannot hide them or corrupt unrelated Source/model state. |
| Source replacement / stale settlement | Old success/failure/cleanup cannot mutate the replacement or release its recovery admission; backup success cannot clear primary failure history. |
diff --git a/docs/plans/openai-streamed-overload-retry.md b/docs/plans/openai-streamed-overload-retry.md
new file mode 100644
index 000000000..c1f99a6f3
--- /dev/null
+++ b/docs/plans/openai-streamed-overload-retry.md
@@ -0,0 +1,96 @@
+# OpenAI streamed overload replay
+
+## Background and goal
+
+The engine hands a Responses stream to the gateway at first model output.
+Ordinary recovery therefore cannot replay a later overload without duplicating
+text or tool calls. The gateway must hold that output before deciding to replay.
+The change belongs to the shared Model Hub, not a Desktop or Agent adapter.
+
+## Change contract
+
+With automatic recovery enabled, the gateway buffers the first streamed
+`openai_responses` body before committing response headers or bytes. Once fully
+consumed, only this conjunction permits one buffered-output replay:
+
+- an HTTP 200 stream with recognized model output;
+- its classified failure is `server_error`, and its wire type/code candidates
+ name only `server_error` (excluding the root `type:error` envelope selector);
+- its bounded wire message, before display sanitization or whitespace folding,
+ is exactly
+ `Our servers are currently overloaded. Please try again later.`
+
+This is protocol-based, including OpenAI-compatible custom Sources; it is not a
+vendor-label or Desktop switch. Other errors and protocols keep their original
+outcomes. Successful Responses bodies are also delayed: the error cannot be
+known before reading the terminal. No non-streaming request changes.
+
+The failed body is discarded, and normal configured routing/recovery chooses
+the next attempt. A ready backup proceeds immediately; a sole cooling Source
+waits for its normal eligibility. This remains one pending inference request:
+the original 120-second admission window and half-open ownership never reset.
+An already admitted inference may finish beyond that window.
+
+Only one **post-output replay** is allowed per gateway request. The next
+resolved stream is forwarded normally, including its partial output if it
+fails; it is not buffered/replayed again. Ordinary pre-output retries retain
+their existing budget, so total upstream attempts need not be limited to two.
+If no retry can be admitted, only the resolver's terminal error is delivered.
+
+## Ownership and implementation
+
+- `resolve_with_recovery` retains its `RecoveryRequest` while a gateway callback
+ prepares the resolved response. A callback may return a settled retry outcome
+ to resume that same loop; final completion clears progress once.
+- The gateway owns the adopted handle and spool before any awaited operation.
+ Spool I/O and cleanup run off the controller loop with cancellation-safe
+ ownership. RAM is capped at 256 KiB; larger bodies spill to a temporary file.
+ The complete replay buffer is capped at 128 MiB; exceeding it is a local
+ delivery failure, never permission for another upstream request.
+- Service settlement records the discarded overload as a failed fallback
+ attempt and applies its normal Source cooldown, but does not project a final
+ Turn failure. The actual upstream `stream_started` fact remains true. Hidden
+ output from this discarded overload does not clear an existing recovery
+ streak or emit a recovery-success event.
+- Each upstream attempt is metered once. Only after settlement/metering and
+ discard does the gateway clear its attempt-owned state. Cancellation while
+ buffering, waiting, or sending the next body still owns cleanup.
+- Only the final admitted producer supplies response headers. A pinned Vibey
+ continuation is never replayed on another Source: existing `pin_unavailable`
+ handback asks its caller to re-render portable content instead.
+- A spool failure is a local delivery failure, not replay permission or evidence
+ against the provider. Upstream usage is retained. If closing a still-live
+ producer yields no protocol terminal, settlement records that local ending
+ without fabricating an upstream outcome; admission and transport are released.
+
+## Known by design
+
+- Initial Responses output, reasoning, and client-executed tool calls are not
+ visible until the first stream ends. This first-byte latency is the cost of
+ preventing duplicate downstream effects; other protocols still stream live.
+- Provider-side hosted tools may already have run, and the provider may bill
+ both attempts. This does not provide exactly-once upstream execution.
+- Each spool uses at most 128 MiB of temporary storage. A full/unreadable
+ temporary disk or a response exceeding that cap ends the request locally;
+ temporary files are closed on every owned exit.
+- No route/config/credential changes, schema migrations, deployment, service
+ restart, or live upstream experiment is included.
+
+## Validation
+
+The owner-boundary tests use real loopback HTTP, the production engine client,
+and the gateway; all credentials and state are synthetic and test-owned.
+
+- `MH-RETRY-OVERLOAD-001`: exact root/nested error matrix, custom and OpenAI
+ Sources, retry success/failure, lossless nonmatches, distinct usage and
+ provenance, real Vibey fallback origin and pin handback, other protocols.
+- `MH-RETRY-OVERLOAD-002`: withholding output, disk rollover, disconnect during
+ buffering/recovery wait, local spool write/read failures, and the size cap.
+- `MH-RETRY-OVERLOAD-003`: earlier pre-output refusal and streamed overload share
+ one recovery clock and failure streak; expired admission exposes no discarded
+ bytes and an overload cannot announce recovery merely by producing hidden text.
+
+The positive regression was run before the fix and failed because the original
+partial text/tool calls and overload were returned instead of the retry body.
+Adjacent recovery, inference-wait, L3, and Vibey consumer tests are also run.
+Exact-head Codex review and CI remain delivery gates.
diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yaml
index d091cf2bf..6a6e3c082 100644
--- a/tests/scenarios/INDEX.yaml
+++ b/tests/scenarios/INDEX.yaml
@@ -95,6 +95,8 @@ capabilities:
- tests/test_model_hub_retry_advice.py
- tests/test_model_hub_retry_policy.py
- tests/test_model_hub_retry_lifecycle.py
+ - tests/test_model_hub_streamed_overload.py
+ - tests/test_model_hub_streamed_overload_lifecycle.py
- tests/test_model_hub_recovery_evidence.py
- tests/test_model_hub_progress.py
- tests/test_model_hub_failure_presentation.py
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 2fb6dea6d..40698a05b 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -36,6 +36,8 @@ capability:
- tests/test_model_hub_retry_advice.py
- tests/test_model_hub_retry_policy.py
- tests/test_model_hub_retry_lifecycle.py
+ - tests/test_model_hub_streamed_overload.py
+ - tests/test_model_hub_streamed_overload_lifecycle.py
- tests/test_model_hub_recovery_evidence.py
- tests/test_model_hub_progress.py
- tests/test_model_hub_failure_presentation.py
@@ -737,6 +739,31 @@ scenarios:
kind: recovery
layer: unit
test: tests/test_model_hub_retry_policy.py::test_one_window_covers_all_fallback_passes_without_replaying_after_output
+ - id: MH-RETRY-OVERLOAD-001
+ name: Exact Responses overload discards the first body and replays once with independent accounting
+ status: covered
+ kind: recovery
+ layer: unit
+ test:
+ - tests/test_model_hub_streamed_overload.py::test_buffered_overload_commits_only_final_attempt
+ - tests/test_model_hub_streamed_overload.py::test_other_response_endings_are_lossless_and_never_replayed
+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_vibey_overload_uses_backup_without_leaking_origin_or_pinned_content
+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_other_protocols_still_stream_before_terminal_and_never_replay
+ - id: MH-RETRY-OVERLOAD-002
+ name: Buffered Responses withhold output and retain spool and upstream ownership through cancellation
+ status: covered
+ kind: recovery
+ layer: unit
+ test:
+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_initial_response_waits_for_terminal_and_owns_spool_until_cleanup
+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_spool_io_failure_is_local_metered_and_never_replayed
+ - tests/test_model_hub_streamed_overload_lifecycle.py::test_cancellation_during_replay_wait_keeps_only_the_metered_failed_attempt
+ - id: MH-RETRY-OVERLOAD-003
+ name: Buffered overload replay shares the original preoutput recovery admission window
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_streamed_overload_lifecycle.py::test_replay_keeps_preoutput_recovery_window
- id: MH-RETRY-RECOVERY-001
name: Real protocol output or empty completion proves recovery while unknown success does not
status: covered
diff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.py
index 2e1b8a2b5..b48e7be2c 100644
--- a/tests/test_model_hub_retry_lifecycle.py
+++ b/tests/test_model_hub_retry_lifecycle.py
@@ -90,9 +90,11 @@ async def gateway_client(service, backend, model):
async def loopback_engine(tmp_path, service, respond, *, models=None):
"""Use the production client/admission/stream observer against local HTTP."""
config = service.store.load()
- source = config.sources[0]
state = EngineStateStore(tmp_path / "engine")
- source.credential_ref = state.store_api_key("fixture", vendor=source.vendor, protocol=source.protocol)
+ for source in config.sources:
+ source.credential_ref = state.store_api_key(
+ "fixture", vendor=source.vendor, protocol=source.protocol, base_url=source.base_url,
+ )
service.store.save(config)
state.sync_sources(service._bindings(config))
app = web.Application()
diff --git a/tests/test_model_hub_streamed_overload.py b/tests/test_model_hub_streamed_overload.py
new file mode 100644
index 000000000..5ee24d52f
--- /dev/null
+++ b/tests/test_model_hub_streamed_overload.py
@@ -0,0 +1,131 @@
+"""The real gateway must commit only the winning Responses stream.
+
+Existing recovery tests stop at the first-output barrier. These exercise the
+exception after text/tool output, so a classification-only retry (duplicate
+output), unmetered replay, or an unbounded replay loop cannot pass.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+
+from aiohttp import web
+import pytest
+
+from core.handlers.model_hub.provenance import SERVED_HOP_HEADER
+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
+from tests.test_model_hub_l3 import _source, _usage_of
+from tests.test_model_hub_retry_lifecycle import configured_service, gateway_client, loopback_engine
+
+
+OVERLOAD = "Our servers are currently overloaded. Please try again later."
+
+
+def event(payload):
+ return b"event: " + payload["type"].encode() + b"\ndata: " + json.dumps(payload).encode() + b"\n\n"
+
+
+def output(label):
+ return (
+ event({"type": "response.output_text.delta", "delta": label})
+ + event({"type": "response.output_item.added", "item": {
+ "type": "function_call", "id": label, "call_id": label, "name": "fixture_tool", "arguments": "{}",
+ }})
+ )
+
+
+def ending(*, error=None, message=OVERLOAD, envelope="response.failed", tokens=11, error_type=None):
+ usage = {"input_tokens": tokens, "output_tokens": 2}
+ if error and envelope == "error":
+ return event({"type": "error", "code": error, "message": message})
+ status = "failed" if error else "completed"
+ return event({"type": f"response.{status}", "response": {
+ "status": status, "usage": usage,
+ **({"error": {"code": error, "message": message, **({"type": error_type} if error_type else {})}} if error else {}),
+ }})
+
+
[email protected]("vendor", ["openai", "custom"])
[email protected]("envelope", ["response.failed", "error"])
[email protected]("second_error", [None, "server_error", "permission_error"])
+def test_buffered_overload_commits_only_final_attempt(tmp_path, vendor, envelope, second_error):
+ """MH-RETRY-OVERLOAD-001: one replay, isolated bodies and per-call accounting."""
+ async def run():
+ source = _source("src_buffered01", "Buffered", vendor=vendor)
+ source.base_url = "https://fixture.invalid"
+ service, clock, models = configured_service(tmp_path, sources=[source])
+ calls = []
+ first = output("discarded-text-and-tool") + ending(error="server_error", envelope=envelope)
+ final = output("final-text-and-tool") + ending(error=second_error, tokens=17)
+
+ async def respond(request):
+ calls.append(await request.json())
+ assert len(calls) <= 2, "the second streamed failure must not replay"
+ return web.Response(
+ body=first if len(calls) == 1 else final, content_type="text/event-stream",
+ headers={"x-request-id": f"attempt-{len(calls)}"},
+ )
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _):
+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:
+ assert response.status == 200
+ body = await response.read()
+ assert body == final
+ assert SERVED_HOP_HEADER not in response.headers # Native callers do not receive Vibey origins.
+ await gateway._drain_turn_requests("turn_lifecycle")
+ assert len(calls) == 2 and calls[0] == calls[1]
+ assert clock.delays == [30]
+ assert adapter._active_transports == 0
+ usage = _usage_of(service, source.id)
+ assert usage["requests"] == 2
+ assert usage["input_tokens"] == (28 if envelope == "response.failed" else 17)
+ gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ record = service.provenance.get("turn_lifecycle")
+ assert record["failed_attempts"][0]["reason"] == "server_error"
+ assert record["failed_attempts"][0]["response_correlation_ids"] == [
+ {"header": "x-request-id", "value": "attempt-1"},
+ ]
+ if second_error is None:
+ assert record["outcome"] == "served"
+ assert record["served"]["response_correlation_ids"] == [
+ {"header": "x-request-id", "value": "attempt-2"},
+ ]
+ else:
+ assert record["outcome"] != "served"
+ assert gateway.correlation.recovery_snapshot("turn_lifecycle") == []
+ asyncio.run(run())
+
+
[email protected]("error,error_type,message", [
+ (None, None, OVERLOAD),
+ ("server_error", None, "An internal error occurred."),
+ ("server_error", None, OVERLOAD + " extra"),
+ ("server_error", None, OVERLOAD.replace("currently overloaded", "currently overloaded")),
+ ("permission_error", None, OVERLOAD),
+ ("overloaded_error", None, OVERLOAD),
+ ("different_vendor_error", "server_error", OVERLOAD),
+ ("internal_error", "server_error", OVERLOAD),
+])
+def test_other_response_endings_are_lossless_and_never_replayed(tmp_path, error, error_type, message):
+ """MH-RETRY-OVERLOAD-001: exact typed envelope, not text/code heuristics."""
+ async def run():
+ service, _clock, models = configured_service(tmp_path)
+ body = output("single-attempt") + ending(error=error, message=message, error_type=error_type)
+ calls = 0
+
+ async def respond(request):
+ nonlocal calls
+ calls += 1
+ assert calls == 1
+ return web.Response(body=body, content_type="text/event-stream")
+
+ async with loopback_engine(tmp_path, service, respond):
+ async with gateway_client(service, "codex", models["codex"]) as (_, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:
+ assert response.status == 200 and await response.read() == body
+ if error == "server_error":
+ assert service.store.load().sources[0].state.status == "cooldown"
+ assert calls == 1
+ asyncio.run(run())
diff --git a/tests/test_model_hub_streamed_overload_lifecycle.py b/tests/test_model_hub_streamed_overload_lifecycle.py
new file mode 100644
index 000000000..04c097875
--- /dev/null
+++ b/tests/test_model_hub_streamed_overload_lifecycle.py
@@ -0,0 +1,314 @@
+"""Buffering must preserve caller identity, recovery clocks, and cancellation.
+
+These are distinct from the exact-error matrix: they exercise real in-flight
+HTTP ownership and the Vibey pin/fallback boundary, which fixed response bodies
+cannot protect.
+"""
+
+import asyncio
+import json
+
+import aiohttp
+from aiohttp import web
+import pytest
+
+from core.handlers.model_hub import turn_gateway
+from core.handlers.model_hub.provenance import SERVED_HOP_HEADER
+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
+from tests.test_model_hub_l3 import _source, _usage_of
+from tests.test_model_hub_retry_policy import Clock, WIRE
+from tests.test_model_hub_vibey_consumer import _vibey_service
+from tests.test_model_hub_retry_lifecycle import configured_service, gateway_client, loopback_engine
+from tests.test_model_hub_streamed_overload import OVERLOAD, output, ending, event
+
+
[email protected]("pinned", [False, True])
+def test_vibey_overload_uses_backup_without_leaking_origin_or_pinned_content(tmp_path, pinned):
+ """MH-RETRY-OVERLOAD-001: real Vibey caller, fallback headers and pin handback."""
+ async def run():
+ primary = _source("src_primary01", "Primary")
+ backup = _source("src_backup001", "Backup")
+ service = _vibey_service(tmp_path, [primary, backup])
+ clock = Clock()
+ service.now, service.recovery = clock.now, clock.policy()
+ calls = []
+ first = output("discarded") + ending(error="server_error")
+ final = output("backup") + ending()
+
+ async def respond(request):
+ calls.append(await request.json())
+ return web.Response(body=first if len(calls) == 1 else final, content_type="text/event-stream")
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _):
+ gateway = ModelHubTurnGateway(service)
+ base, token = await gateway.endpoint(
+ "vibey", process_scope="buffered", turn_id="turn_vibey",
+ requested_model_id="menu-alias", resolved_model_id="shared-model", source_id=primary.id,
+ )
+ try:
+ headers = {"Authorization": f"Bearer {token}", "x-avibe-turn-id": "turn_vibey"}
+ if pinned:
+ headers.update({"x-avibe-pin-source": primary.id, "x-avibe-pin-model": "shared-model"})
+ async with aiohttp.ClientSession(trust_env=False) as client:
+ async with client.post(
+ f"{base}/v1/responses", headers=headers,
+ json={"model": "shared-model", "stream": True, "input": "fixture prompt"},
+ ) as response:
+ if pinned:
+ assert response.status == 409
+ assert (await response.json())["error"]["code"] == "pin_unavailable"
+ assert SERVED_HOP_HEADER not in response.headers
+ else:
+ assert response.status == 200 and await response.read() == final
+ assert json.loads(response.headers[SERVED_HOP_HEADER])["source"] == backup.id
+ await gateway._drain_turn_requests("turn_vibey")
+ assert len(calls) == (1 if pinned else 2)
+ assert clock.delays == [] # A ready backup is not the cooling primary.
+ assert adapter._active_transports == 0
+ assert _usage_of(service, primary.id)["requests"] == 1
+ if pinned:
+ assert gateway.correlation.terminal_projection("turn_vibey", backend="vibey") is None
+ else:
+ assert _usage_of(service, backup.id)["requests"] == 1
+ gateway.correlation.settle("turn_vibey", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ assert service.provenance.get("turn_vibey")["served"]["source_id"] == backup.id
+ finally:
+ await gateway.close()
+ asyncio.run(run())
+
+
[email protected]("cancel", [False, True])
+def test_initial_response_waits_for_terminal_and_owns_spool_until_cleanup(tmp_path, monkeypatch, cancel):
+ """MH-RETRY-OVERLOAD-002: no early downstream bytes, bounded RAM, disconnect cleanup."""
+ async def run():
+ service, _clock, models = configured_service(tmp_path)
+ prefix_sent, release, upstream_closed = (asyncio.Event() for _ in range(3))
+ body = output("before-terminal-" + "x" * 300_000)
+ files = []
+ create = turn_gateway.tempfile.SpooledTemporaryFile
+
+ def track_file(*args, **kwargs):
+ payload = create(*args, **kwargs)
+ files.append(payload)
+ return payload
+
+ monkeypatch.setattr(turn_gateway.tempfile, "SpooledTemporaryFile", track_file)
+
+ async def respond(request):
+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+ await response.prepare(request)
+ try:
+ await response.write(body)
+ prefix_sent.set()
+ await release.wait()
+ await response.write(ending())
+ await response.write_eof()
+ return response
+ finally:
+ upstream_closed.set()
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _):
+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):
+ task = asyncio.create_task(client.post(url, headers=headers, json={"model": "shared-model", "stream": True}))
+ try:
+ await asyncio.wait_for(prefix_sent.wait(), 2)
+ assert not (await asyncio.wait({task}, timeout=0.05))[0], "headers committed before terminal"
+ if cancel:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ await asyncio.wait_for(upstream_closed.wait(), 2)
+ else:
+ release.set()
+ response = await asyncio.wait_for(task, 2)
+ assert await response.read() == body + ending()
+ await gateway._drain_turn_requests("turn_lifecycle")
+ assert adapter._active_transports == 0
+ assert files and all(payload.closed for payload in files)
+ assert any(payload._rolled for payload in files)
+ assert _usage_of(service, service.store.load().sources[0].id)["requests"] == 1
+ finally:
+ release.set()
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ asyncio.run(run())
+
+
[email protected]("late_seconds,expected_calls", [(0, 3), (121, 2)])
+def test_replay_keeps_preoutput_recovery_window(tmp_path, late_seconds, expected_calls):
+ """MH-RETRY-OVERLOAD-003: an earlier refusal's window must not reset at replay."""
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ calls = []
+ first = output("discarded") + ending(error="server_error")
+ final = output("final") + ending()
+
+ async def respond(request):
+ calls.append(await request.json())
+ if len(calls) == 1:
+ return web.json_response({"error": {"type": "server_error"}}, status=503)
+ if len(calls) == 2:
+ clock.advance(late_seconds)
+ else:
+ assert not any(event["kind"] == "recover" for event in service.events.list())
+ return web.Response(body=first if len(calls) == 2 else final, content_type="text/event-stream")
+
+ async with loopback_engine(tmp_path, service, respond):
+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:
+ if late_seconds:
+ assert response.status == 400
+ assert (await response.json())["error"]["code"] == "model_hub_recovery_exhausted"
+ else:
+ assert response.status == 200 and await response.read() == final
+ await gateway._drain_turn_requests("turn_lifecycle")
+ assert len(calls) == expected_calls
+ assert clock.delays == ([30] if late_seconds else [30, 60])
+ assert sum(event["kind"] == "recover" for event in service.events.list()) == (0 if late_seconds else 1)
+ assert gateway.correlation.recovery_snapshot("turn_lifecycle") == []
+ asyncio.run(run())
+
+
[email protected]("backend,protocol", [("claude", "anthropic"), ("opencode", "openai_chat")])
+def test_other_protocols_still_stream_before_terminal_and_never_replay(tmp_path, backend, protocol):
+ """MH-RETRY-OVERLOAD-001: the exception must not broaden to other wire protocols."""
+ async def run():
+ source = _source("src_other001", "Other", protocol=protocol, vendor="anthropic" if backend == "claude" else "openai")
+ service, clock, models = configured_service(tmp_path, sources=[source])
+ prefix = WIRE[protocol][0] + WIRE[protocol][1]
+ error = (b'data: ' if protocol == "openai_chat" else b'event: error\ndata: ') + json.dumps({
+ "type": "error", "error": {"type": "server_error", "message": OVERLOAD},
+ }).encode() + b"\n\n"
+ release = asyncio.Event()
+ calls = 0
+
+ async def respond(request):
+ nonlocal calls
+ calls += 1
+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+ await response.prepare(request)
+ await response.write(prefix)
+ await release.wait()
+ await response.write(error)
+ return response
+
+ async with loopback_engine(tmp_path, service, respond):
+ async with gateway_client(service, backend, models[backend]) as (_, client, url, headers):
+ try:
+ response = await asyncio.wait_for(client.post(
+ url, headers=headers, json={"model": "shared-model", "stream": True},
+ ), 2)
+ assert await asyncio.wait_for(response.content.readexactly(len(prefix)), 2) == prefix
+ release.set()
+ assert await response.read() == error
+ assert calls == 1 and clock.delays == []
+ finally:
+ release.set()
+ asyncio.run(run())
+
+
[email protected]("stage,terminal", [
+ ("write", True), ("read", True), ("limit", True),
+ ("write", False), ("limit", False),
+])
+def test_spool_io_failure_is_local_metered_and_never_replayed(tmp_path, monkeypatch, stage, terminal):
+ """MH-RETRY-OVERLOAD-002: added disk I/O cannot lose usage or blame the provider."""
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ original = turn_gateway.tempfile.SpooledTemporaryFile
+ files = []
+ calls = 0
+ upstream_closed = asyncio.Event()
+ body = output("answer") + ending() if terminal else event({
+ "type": "response.created", "response": {"usage": {"input_tokens": 13, "output_tokens": 1}},
+ }) + output("answer")
+
+ def faulty_spool(*args, **kwargs):
+ payload = original(*args, **kwargs)
+ files.append(payload)
+
+ def fail_io(*_args):
+ raise OSError(28 if stage == "write" else 5, "fixture local disk error")
+
+ if stage != "limit":
+ setattr(payload, stage, fail_io)
+ return payload
+
+ monkeypatch.setattr(turn_gateway.tempfile, "SpooledTemporaryFile", faulty_spool)
+ if stage == "limit":
+ monkeypatch.setattr(turn_gateway, "_MAX_REPLAY_BUFFER_BYTES", len(body) - 1)
+
+ async def respond(request):
+ nonlocal calls
+ calls += 1
+ if terminal:
+ return web.Response(body=body, content_type="text/event-stream")
+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+ await response.prepare(request)
+ try:
+ await response.write(body)
+ # No terminal exists when storage fails. Only gateway teardown
+ # may close this still-live upstream, not a fixture timeout.
+ await asyncio.Future()
+ finally:
+ upstream_closed.set()
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _):
+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": True}) as response:
+ response_body = await response.read()
+ assert response.status == (200 if stage == "read" else 502)
+ assert b"engine_down" in response_body
+ await gateway._drain_turn_requests("turn_lifecycle")
+ if not terminal:
+ await asyncio.wait_for(upstream_closed.wait(), 2)
+ assert adapter._active_transports == 0 and clock.delays == [] and calls == 1
+ assert files and all(payload.closed for payload in files)
+ source = service.store.load().sources[0]
+ assert _usage_of(service, source.id)["requests"] == 1
+ if not terminal:
+ assert _usage_of(service, source.id)["input_tokens"] == 13
+ assert source.state.status == "standby"
+ gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ assert service.provenance.get("turn_lifecycle")["terminal_error"]["reason"] == "engine_down"
+ asyncio.run(run())
+
+
+def test_cancellation_during_replay_wait_keeps_only_the_metered_failed_attempt(tmp_path):
+ """MH-RETRY-OVERLOAD-002: dropping the first body must reset every attempt-owned debt."""
+ async def run():
+ service, _clock, models = configured_service(tmp_path)
+ waiting = asyncio.Event()
+ calls = 0
+
+ async def wait_for_cancel(_seconds):
+ waiting.set()
+ await asyncio.Future()
+
+ service.recovery.sleep = wait_for_cancel
+
+ async def respond(request):
+ nonlocal calls
+ calls += 1
+ return web.Response(body=output("discarded") + ending(error="server_error"), content_type="text/event-stream")
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _):
+ async with gateway_client(service, "codex", models["codex"]) as (gateway, client, url, headers):
+ task = asyncio.create_task(client.post(url, headers=headers, json={"model": "shared-model", "stream": True}))
+ try:
+ await asyncio.wait_for(waiting.wait(), 2)
+ progress = gateway.correlation.recovery_snapshot("turn_lifecycle")
+ assert progress and progress[0]["phase"] == "waiting"
+ assert gateway.correlation.terminal_projection("turn_lifecycle", backend="codex") is None
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ await gateway._drain_turn_requests("turn_lifecycle")
+ assert adapter._active_transports == 0 and calls == 1
+ source = service.store.load().sources[0]
+ assert _usage_of(service, source.id)["requests"] == 1
+ assert service.recovery._sources[source.id].owner is None
+ assert gateway.correlation.recovery_snapshot("turn_lifecycle") == []
+ asyncio.run(run())
diff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.py
index c26cbd748..826b8aac6 100644
--- a/tests/test_model_hub_structure_guards.py
+++ b/tests/test_model_hub_structure_guards.py
@@ -758,7 +758,10 @@ def test_usage_metering_has_one_owner_per_call_population() -> None:
# handle. Review 4970...: it used to ask whether the turn was *settled*, which
# a forced terminal had already made true for a call the vendor billed.
owes = _functions(gateway_tree)["owes_metering"]
- readers = (recorder, owes)
+ # A buffered overload retires a whole upstream attempt before adopting a
+ # second one. That transition owns clearing the prior write handle; the
+ # real gateway replay tests prove both calls are still metered once.
+ readers = (recorder, owes, _functions(gateway_tree)["next_attempt"])
handles = [
node
for node in ast.walk(gateway_tree)
@@ -806,7 +809,9 @@ def test_settling_a_turn_is_what_meters_it_for_every_ending_there_will_be() -> N
endings = [node for node in ast.walk(gateway_tree) if _call_name(node) == "_settle_metered_turn"]
assert len(endings) >= 2
assert all(
- any(ending in set(ast.walk(functions[name])) for name in ("_settle_boundary_termination", "_resolved_response"))
+ any(ending in set(ast.walk(functions[name])) for name in (
+ "_settle_boundary_termination", "_resolved_response", "_buffer_responses_stream",
+ ))
for ending in endings
)
@@ -821,20 +826,24 @@ def test_a_metered_row_is_dated_by_the_call_and_not_by_what_followed_it() -> Non
# test can make one. What is left to protect is where the instant comes from.
gateway_tree = _tree(TURN_GATEWAY)
functions = _functions(gateway_tree)
- # Captured where the upstream body ends, which is the one function that has a
- # body to end. A capture anywhere else is a second answer to when the call
- # finished, and the endings are exactly the places that would get it wrong.
+ # Captured where the upstream body ends, whether it is forwarded live or
+ # spooled before commit. Clearing the completed attempt for a replay is not
+ # another timestamp capture; each call retains its own completion instant.
captures = [
node
for node in ast.walk(gateway_tree)
if isinstance(node, ast.Assign)
+ and not (isinstance(node.value, ast.Constant) and node.value.value is None)
and any(
isinstance(target, ast.Attribute) and target.attr == "completed_at"
for target in node.targets
)
]
assert captures
- assert all(capture in set(ast.walk(functions["_resolved_response"])) for capture in captures)
+ assert all(
+ any(capture in set(ast.walk(functions[name])) for name in ("_resolved_response", "_buffer_responses_stream"))
+ for capture in captures
+ )
# And carried from there to the ledger rather than re-read at the write.
handoff = next(
node for node in ast.walk(functions["_record_usage"]) if _call_name(node) == "record"