Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): extend Vibey relay prelude patience (#3095)

Call White1cb592485654
diff --git a/docs/plans/model-hub-inference-deadlines.md b/docs/plans/model-hub-inference-deadlines.md
index 0dddabe37..946c5f30a 100644
--- a/docs/plans/model-hub-inference-deadlines.md
+++ b/docs/plans/model-hub-inference-deadlines.md
@@ -287,9 +287,8 @@ was dead or its upstream stuck was not recorded.
- A streamed request to a Source in the table below ends when the engine sends
the gateway no byte for **120 s**. Every byte restarts the bound: data events,
- pings and comments alike. It runs from the moment the request body is written,
- so it covers the wait for response headers too. A paused read, while the
- downstream caller is slower than the engine, does not count.
+ pings and comments alike. Native callers apply it from the moment the
+ request body is written, so it covers the wait for response headers too.
- When the bound runs out, the gateway closes the engine connection. The engine
cancels its upstream request through its request context; measured on the
shipped v8.0.23-3 engine, the upstream saw the disconnect within 0.02 s.
@@ -412,3 +411,16 @@ HTTP status its error handling keys on.
reaches Vibey as `interrupted` rather than `stalled`.
- A persistently silent unmeasured Source still costs about two 600 s attempts
(Model Hub recovery), as before (Vibey's own stall and one continuation).
+
+## Remote relay request-body prelude (2026-10-31)
+
+The 2026-10-31 relay investigation found Anthropic requests that spent nearly
+120 s in the relay's request-body read before the caller cancelled the
+connection. For Vibey, the engine now uses its existing 600 s caller-owned
+budget for the request/response prelude when it exceeds the measured 120 s
+bound. Once the first response byte arrives, measured Anthropic streams retain
+the 120 s per-read idle bound. Native callers retain the 120 s prelude bound.
+
+This is a client-side resilience margin, not proof that a relay is healthy. The
+relay still needs its own request-body idle timeout, phase diagnostics, and
+attempt-ID propagation; those changes are tracked in the relay project.
diff --git a/tests/test_model_hub_inference_wait.py b/tests/test_model_hub_inference_wait.py
index 599258773..e1376f5b3 100644
--- a/tests/test_model_hub_inference_wait.py
+++ b/tests/test_model_hub_inference_wait.py
@@ -73,9 +73,9 @@ def test_connected_inference_waits_for_its_owner(
timeout = kwargs["timeout"]
# Elapsed time never ends inference; only a stream's silence does (MH-RUNTIME-013).
assert timeout.total is None
- # Gateway Sources name their vendor; the bare client fixture is a custom vendor.
- idle_bounded = streaming and entrypoint == "gateway" and protocol in {"anthropic", "openai_responses"}
- assert timeout.sock_read == (client_module._STREAM_IDLE_TIMEOUT_SECONDS if idle_bounded else None)
+ # Measured streams apply their phase-specific bounds around reads rather
+ # than through aiohttp's single session-level sock_read timer.
+ assert timeout.sock_read is None
assert timeout.connect == timeout.sock_connect == CONNECT_TIMEOUT_SECONDS
session = real_session(**kwargs)
sessions.append(session)
diff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.py
index b48e7be2c..5e005cdf2 100644
--- a/tests/test_model_hub_retry_lifecycle.py
+++ b/tests/test_model_hub_retry_lifecycle.py
@@ -1243,9 +1243,10 @@ def test_a_silent_engine_stream_is_cancelled_and_retried_at_its_idle_bound(
@pytest.mark.parametrize(
"origin,vendor,source_protocol,bound",
[
- # Vibey's own bound, which Vibey no longer applies to the wait for response headers.
+ # An unmeasured Source keeps Vibey's caller-owned bound.
("vibey", "xai", "openai_responses", "caller"),
- # A measured Source keeps its shorter bound for every caller.
+ # A measured Anthropic stream keeps its shorter inter-byte bound after
+ # the first response byte; Vibey's larger bound applies only before it.
("vibey", "anthropic", "anthropic", "idle"),
# A native caller keeps its own bound, and an unmeasured Source stays unbounded here.
("codex", "xai", "openai_responses", None),
@@ -1257,10 +1258,9 @@ def test_vibey_hands_its_silence_bound_to_the_engine_read(
"""MH-RUNTIME-013: before model output only the engine read sees the upstream, so it holds Vibey's bound.
The gateway withholds the response, heartbeats included, until the first
- model output, and a model that thinks without streaming its thinking sends
- nothing else for as long as it thinks. Vibey therefore bounds no wait for
- response headers itself; the engine read applies Vibey's silence bound to
- every stream the measured Source table leaves unbounded.
+ model output. Vibey therefore gives the engine its 600-second caller-owned
+ pre-response patience; a measured Source resumes its shorter 120-second
+ inter-byte bound after the first response byte.
"""
idle, caller = 0.3, 0.8
monkeypatch.setattr(engine_client_module, "_STREAM_IDLE_TIMEOUT_SECONDS", idle)
@@ -1307,3 +1307,58 @@ def test_vibey_hands_its_silence_bound_to_the_engine_read(
assert len(stalled_for) == 1 and expected * 0.9 <= stalled_for[0] < min(expected + 0.4, caller * 2)
asyncio.run(run())
+
+
[email protected]("vendor,protocol", [("anthropic", "anthropic"), ("openai", "openai_responses")])
[email protected]("origin", ["vibey", "native"])
[email protected]("phase", ["headers", "first_byte", "split_prelude"])
+def test_measured_stream_prelude_uses_one_caller_budget_until_first_byte(
+ monkeypatch, tmp_path, vendor, protocol, origin, phase,
+):
+ """MH-RUNTIME-013: slow relay preludes use caller patience, not a fresh budget per phase.
+
+ Vibey can wait beyond measured idle for headers or the first body byte;
+ native callers cannot. Splitting the delay across headers and body must
+ not reset the budget. The existing idle test covers later byte gaps.
+ """
+
+ idle, caller = 0.3, 0.8
+ delay = (idle + caller) / 2
+ monkeypatch.setattr(engine_client_module, "_STREAM_IDLE_TIMEOUT_SECONDS", idle)
+ monkeypatch.setattr(engine_adapter_module, "PROVIDER_SILENCE_TIMEOUT_S", caller)
+
+ async def run() -> None:
+ source = _source("src_slowheaders", "Slow prelude", vendor=vendor, protocol=protocol)
+ service, _clock, _models = configured_service(tmp_path, sources=[source])
+ metadata, output, terminal, _buffered = WIRE[protocol]
+
+ async def respond(request: web.Request) -> web.StreamResponse:
+ await request.json()
+ if phase in {"headers", "split_prelude"}:
+ await asyncio.sleep(delay)
+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+ await response.prepare(request)
+ if phase in {"first_byte", "split_prelude"}:
+ await asyncio.sleep(delay)
+ await response.write(metadata + output + terminal)
+ await response.write_eof()
+ return response
+
+ caller_origin = origin if origin == "vibey" else "claude" if protocol == "anthropic" else "codex"
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
+ handle = await asyncio.wait_for(
+ adapter.invoke(source.id, "shared-model", {"stream": True}, True, caller_origin),
+ timeout=3,
+ )
+ body = b"".join([chunk async for chunk in handle.stream]) if handle.stream is not None else b""
+ outcome = await asyncio.wait_for(handle.outcome(), timeout=3)
+ assert adapter._active_transports == 0
+
+ if origin == "vibey" and phase != "split_prelude":
+ assert outcome.kind == RawOutcomeKind.SUCCESS
+ assert body == metadata + output + terminal
+ else:
+ assert outcome.kind == RawOutcomeKind.TIMEOUT
+ assert body == b""
+
+ asyncio.run(run())
diff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.py
index 623100b45..ec90562f4 100644
--- a/tests/test_model_hub_runtime.py
+++ b/tests/test_model_hub_runtime.py
@@ -202,6 +202,7 @@ def test_stream_prelude_replays_large_keepalive_history_before_output() -> None:
wire_state=state,
source=source,
model_id="claude-sonnet-4-5",
+ read_timeout=None,
)
payload = b"".join([chunk async for chunk in prelude.chunks()])
prelude.close()
@@ -639,6 +640,7 @@ def test_stream_prelude_retains_metadata_until_model_output() -> None:
wire_state=state,
source=source,
model_id="claude-sonnet-4-5",
+ read_timeout=None,
)
assert outcome is None
assert state.model_output_started
@@ -6923,12 +6925,13 @@ def test_engine_client_marks_loopback_stream_disconnect_as_engine_down(
) -> None:
async def run() -> None:
class Content:
- async def read(self, _size: int) -> bytes:
- return b'data: {"choices":[{"delta":{"content":"first"}}]}\n\n'
+ reads = 0
- async def iter_chunked(self, _size: int):
+ async def read(self, _size: int) -> bytes:
+ self.reads += 1
+ if self.reads == 1:
+ return b'data: {"choices":[{"delta":{"content":"first"}}]}\n\n'
raise client_module.aiohttp.ClientConnectionError("loopback engine closed")
- yield b""
class Response:
status = 200
@@ -7121,13 +7124,14 @@ def test_engine_client_keeps_served_after_terminal_marker_then_disconnect(
terminal_chunk: bytes,
) -> None:
async def run() -> None:
+ chunks = iter((output_chunk, terminal_chunk))
+
class Content:
async def read(self, _size: int) -> bytes:
- return output_chunk
-
- async def iter_chunked(self, _size: int):
- yield terminal_chunk
- raise client_module.aiohttp.ClientConnectionError("late disconnect")
+ try:
+ return next(chunks)
+ except StopIteration:
+ raise client_module.aiohttp.ClientConnectionError("late disconnect") from None
class Response:
status = 200
@@ -7254,13 +7258,11 @@ def test_engine_client_transparently_forwards_unvalidated_stream_data(
b'event: response.completed\ndata: {"type":"response.completed","sequence_number":2}\n\n'
)
+ chunks = iter((output, invalid_chunk, terminal))
+
class Content:
async def read(self, _size: int) -> bytes:
- return output
-
- async def iter_chunked(self, _size: int):
- yield invalid_chunk
- yield terminal
+ return next(chunks, b"")
class Response:
status = 200
@@ -7453,13 +7455,11 @@ def test_engine_client_recognizes_responses_failure_terminals(
separators=(",", ":"),
).encode()
+ chunks = iter((b"event: " + event_type.encode() + b"\ndata: " + terminal + b"\n\n",))
+
class Content:
async def read(self, _size: int) -> bytes:
- return b"event: " + event_type.encode() + b"\ndata: " + terminal + b"\n\n"
-
- async def iter_chunked(self, _size: int):
- if False:
- yield b""
+ return next(chunks, b"")
class Response:
status = 200
@@ -7609,9 +7609,8 @@ def test_downstream_close_after_chat_finish_reason_does_not_fabricate_success()
wire_state.observe(first)
class Content:
- async def iter_chunked(self, _size: int):
- if False:
- yield b""
+ async def read(self, _size: int) -> bytes:
+ return b""
class Response:
status = 200
@@ -7644,6 +7643,7 @@ def test_downstream_close_after_chat_finish_reason_does_not_fabricate_success()
source=source,
model_id="model-a",
protocol="openai_chat",
+ read_timeout=None,
outcome_future=outcome_future,
wire_state=wire_state,
)
@@ -7663,13 +7663,11 @@ def test_engine_client_keeps_success_after_a_later_complete_frame(
terminal = b'event: response.completed\ndata: {"type":"response.completed"}\n\n'
extra = b'data: {"type":"response.output_text.delta"}\n\n'
+ chunks = iter((first, terminal, extra))
+
class Content:
async def read(self, _size: int) -> bytes:
- return first
-
- async def iter_chunked(self, _size: int):
- yield terminal
- yield extra
+ return next(chunks, b"")
class Response:
status = 200
@@ -8275,6 +8273,90 @@ def test_engine_client_cancellation_closes_pre_handle_resources(
asyncio.run(run())
[email protected]("phase", ["headers", "first_byte", "stream_body"])
+def test_engine_client_owner_cancellation_wins_completed_io(
+ monkeypatch: pytest.MonkeyPatch,
+ phase: str,
+) -> None:
+ """MH-RUNTIME-013: Stop wins a same-loop I/O completion, including Python 3.10/3.11.
+
+ Blocking-read cancellation tests miss wait_for's old completed-inner-task
+ race: it could return that result after the owner's cancellation arrived.
+ """
+
+ async def run() -> None:
+ released = []
+ output = b'event: content_block_delta\ndata: {"type":"content_block_delta"}\n\n'
+ terminal = b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
+
+ class Content:
+ reads = 0
+
+ async def read(self, _size: int) -> bytes:
+ self.reads += 1
+ current = "first_byte" if self.reads == 1 else "stream_body"
+ if phase == current and self.reads <= 2:
+ asyncio.get_running_loop().call_soon(owner.cancel)
+ return output if self.reads == 1 else terminal if self.reads == 2 else b""
+
+ class Response:
+ status = 200
+ content = Content()
+ headers = {"Content-Type": "text/event-stream"}
+ closed = False
+
+ def close(self) -> None:
+ self.closed = True
+
+ response = Response()
+
+ class Session:
+ closed = False
+
+ async def post(self, *_args, **_kwargs):
+ if phase == "headers":
+ asyncio.get_running_loop().call_soon(owner.cancel)
+ return response
+
+ async def close(self) -> None:
+ self.closed = True
+
+ session = Session()
+ monkeypatch.setattr(client_module.aiohttp, "ClientSession", lambda **_: session)
+ source = SourceRecord(
+ source_id="src_fixture123", vendor="anthropic", protocol="anthropic",
+ base_url="https://api.example.test/v1", credential_ref="cred_fixture123",
+ allowed_origins=(), model_ids=("model-a",), prefix="source-fixture123",
+ )
+ client = EngineClient(EngineConnection("http://127.0.0.1:15220", "management", "gateway"))
+
+ async def consume() -> None:
+ handle = await client.invoke(
+ source, "model-a", {}, stream=True, on_transport_done=lambda: released.append(True),
+ )
+ try:
+ assert handle.stream is not None
+ async for _chunk in handle.stream:
+ pass
+ finally:
+ await handle.close_stream()
+
+ owner = asyncio.create_task(consume())
+ try:
+ done, _ = await asyncio.wait({owner}, timeout=1)
+ assert owner in done, "owner cancellation was swallowed and left a live request"
+ with pytest.raises(asyncio.CancelledError):
+ await owner
+ assert session.closed
+ assert phase == "headers" or response.closed
+ assert released
+ finally:
+ owner.cancel()
+ await asyncio.gather(owner, return_exceptions=True)
+
+ asyncio.run(run())
+
+
def test_buffered_projection_drains_before_its_spool_is_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -8357,13 +8439,11 @@ def test_closing_an_unstarted_stream_publishes_an_observed_terminal(
b'"response":{"usage":{"input_tokens":12,"output_tokens":3}}}\n\n'
)
+ chunks = iter((first,))
+
class Content:
async def read(self, _size: int) -> bytes:
- return first
-
- async def iter_chunked(self, _size: int):
- if False:
- yield b""
+ return next(chunks, b"")
class Response:
status = 200
@@ -8520,10 +8600,9 @@ def test_slow_source_prelude_is_bounded_without_blocking_other_routes(
blocked.set()
await never.wait()
return b""
- return fast_output
-
- async def iter_chunked(self, _size: int):
- yield fast_terminal
+ if self.reads == 1:
+ return fast_output
+ return fast_terminal if self.reads == 2 else b""
class Response:
status = 200
diff --git a/vibe/model_hub_runtime/adapter.py b/vibe/model_hub_runtime/adapter.py
index 4fafd4e74..dea1be689 100644
--- a/vibe/model_hub_runtime/adapter.py
+++ b/vibe/model_hub_runtime/adapter.py
@@ -3366,8 +3366,9 @@ class CLIProxyEngineAdapter:
request_headers=getattr(request, "headers", None),
on_transport_done=release,
on_request_sent=request_sent,
- # Vibey waits on the gateway without a bound of its own until the
- # response headers, which the gateway sends at the first model output.
+ # Vibey gives measured streams its larger caller-owned pre-response
+ # patience; the engine resumes the measured inter-byte bound once
+ # the first response byte arrives.
silence_bound_s=PROVIDER_SILENCE_TIMEOUT_S if origin == "vibey" else None,
)
if handle.outcome_available and engine_disowned_call(await handle.outcome()):
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index df82bd7f0..645bb39df 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -104,6 +104,7 @@ _STREAM_IDLE_BOUNDED_SOURCES = frozenset({("anthropic", "anthropic"), ("openai",
_OFFICIAL_BASE_URLS = official_api_key_base_urls()
logger = logging.getLogger(__name__)
_ProjectedJSON = TypeVar("_ProjectedJSON")
+_IOResult = TypeVar("_IOResult")
_ENGINE_HEADER_VALUE_MAX_LENGTH = 128
_RETRY_AFTER_MS_MAX = 365 * 24 * 60 * 60 * 1000
@@ -636,21 +637,32 @@ class EngineClient:
if request_protocol == "anthropic":
headers.setdefault("anthropic-version", "2023-06-01")
+ measured = False
idle_bound: float | None = None
+ first_response_bound: float | None = None
+ manual_read_bound: float | None = None
if stream:
measured = (
request_protocol == source.protocol
and (source.vendor, source.protocol) in _STREAM_IDLE_BOUNDED_SOURCES
)
- # A caller that holds no bound of its own before model output hands
- # it over: until then the gateway withholds the whole response,
- # heartbeats included, so only this read sees the upstream's bytes.
+ # The measured bound protects gaps between response bytes. Vibey has
+ # an explicit 600s caller-owned patience budget; use that larger
+ # budget only until the first response byte so a remote relay can
+ # finish receiving a large request body without weakening the
+ # measured 120s inter-byte protection after the stream starts.
idle_bound = _STREAM_IDLE_TIMEOUT_SECONDS if measured else silence_bound_s
+ if measured:
+ first_response_bound = max(idle_bound or 0.0, silence_bound_s or 0.0)
+ manual_read_bound = idle_bound
timeout = aiohttp.ClientTimeout(
total=None,
connect=self.timeout,
sock_connect=self.timeout,
- sock_read=idle_bound,
+ # Measured streams use explicit per-phase reads below: their first
+ # response may use the caller's larger prelude budget, while later
+ # reads retain the measured inter-byte bound.
+ sock_read=None if measured else idle_bound,
)
# Connecting to the local engine is bounded. Once connected, inference
# can take any duration; completion, transport failure, or owner
@@ -677,6 +689,9 @@ class EngineClient:
trace.on_request_chunk_sent.append(request_sent)
trace_configs.append(trace)
session = aiohttp.ClientSession(timeout=timeout, trust_env=False, trace_configs=trace_configs)
+ prelude_deadline = (
+ time.monotonic() + first_response_bound if first_response_bound is not None else None
+ )
response: aiohttp.ClientResponse | None = None
retry_after: str | None = None
retry_after_ms: int | None = None
@@ -731,12 +746,17 @@ class EngineClient:
))
try:
- response = await session.post(
+ request = session.post(
self._url(endpoint),
data=encoded,
headers=headers,
allow_redirects=False,
)
+ if first_response_bound is None:
+ response = await request
+ else:
+ remaining = max(0.001, (prelude_deadline or time.monotonic()) - time.monotonic())
+ response = await _await_engine_io(request, timeout=remaining)
response_correlation_ids = sanitize_response_correlation_ids(response.headers)
if response.status >= 300:
response_received_at = datetime.now(timezone.utc)
@@ -826,7 +846,12 @@ class EngineClient:
await session.close()
return ended(outcome)
- first = await response.content.read(_STREAM_CHUNK_BYTES)
+ first_read_timeout = (
+ None
+ if prelude_deadline is None
+ else max(0.001, prelude_deadline - time.monotonic())
+ )
+ first = await _read_content_chunk(response.content, timeout=first_read_timeout)
if not first:
response.close()
await session.close()
@@ -890,6 +915,7 @@ class EngineClient:
wire_state=wire_state,
source=source,
model_id=model_id,
+ read_timeout=manual_read_bound,
response_correlation_ids=response_correlation_ids,
)
model_output_started = wire_state.model_output_started
@@ -912,6 +938,7 @@ class EngineClient:
source=source,
model_id=model_id,
protocol=request_protocol,
+ read_timeout=manual_read_bound,
outcome_future=outcome_future,
wire_state=wire_state,
on_transport_done=on_transport_done,
@@ -1801,6 +1828,38 @@ class _ResponseTooLarge(Exception):
"""A response grew past the bound its request protocol sets."""
+async def _await_engine_io(awaitable: Awaitable[_IOResult], *, timeout: float | None) -> _IOResult:
+ """Bound one I/O wait without losing owner cancellation to a completed read.
+
+ Python 3.10/3.11 wait_for can swallow cancellation if its inner task has
+ just completed. wait keeps owner cancellation authoritative; we still own
+ and drain the I/O task before its session or response can be closed.
+ """
+
+ if timeout is None:
+ return await awaitable
+ operation = asyncio.ensure_future(awaitable)
+ try:
+ done, _ = await asyncio.wait({operation}, timeout=timeout)
+ if not done:
+ raise asyncio.TimeoutError
+ return operation.result()
+ except BaseException:
+ operation.cancel()
+ await asyncio.gather(operation, return_exceptions=True)
+ raise
+
+
+async def _read_content_chunk(
+ content: aiohttp.StreamReader,
+ *,
+ timeout: float | None,
+) -> bytes:
+ """Read one engine chunk, optionally applying a caller-owned idle bound."""
+
+ return await _await_engine_io(content.read(_STREAM_CHUNK_BYTES), timeout=timeout)
+
+
async def _read_response_into(
content: aiohttp.StreamReader,
target: _StreamPrelude,
@@ -1824,6 +1883,7 @@ async def _read_stream_prelude(
wire_state: ProtocolSSEState,
source: SourceRecord,
model_id: str,
+ read_timeout: float | None,
response_correlation_ids: tuple[tuple[str, str], ...] = (),
) -> RawCallOutcome | None:
"""Buffer transport metadata until the sole first-model-output fact.
@@ -1843,7 +1903,7 @@ async def _read_stream_prelude(
)
if outcome is not None:
return replace(outcome, response_correlation_ids=response_correlation_ids)
- chunk = await response.content.read(_STREAM_CHUNK_BYTES)
+ chunk = await _read_content_chunk(response.content, timeout=read_timeout)
if not chunk:
completion = _observed_stream_terminal_outcome(
wire_state,
@@ -1896,6 +1956,7 @@ async def _response_stream(
source: SourceRecord,
model_id: str,
protocol: str,
+ read_timeout: float | None,
outcome_future: asyncio.Future[RawCallOutcome],
wire_state: ProtocolSSEState,
response_correlation_ids: tuple[tuple[str, str], ...] = (),
@@ -1906,7 +1967,7 @@ async def _response_stream(
async for chunk in prelude.chunks():
yield chunk
prelude.close()
- async for chunk in response.content.iter_chunked(_STREAM_CHUNK_BYTES):
+ while chunk := await _read_content_chunk(response.content, timeout=read_timeout):
if chunk:
await wire_state.observe_async(chunk)
yield chunk