Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): give buffered response projection its own budget (#2225)

Call White4cebecfab572
diff --git a/docs/plans/model-hub-inference-deadlines.md b/docs/plans/model-hub-inference-deadlines.md
index 2f0178070..6bc2b6cf6 100644
--- a/docs/plans/model-hub-inference-deadlines.md
+++ b/docs/plans/model-hub-inference-deadlines.md
@@ -244,3 +244,21 @@ xfail, including the previously failing config/authority closure test. The
authority checker reports no findings; changed Python files pass Ruff and
whitespace checks. The orchestrator inspected the parser/minimum diff and its
probe-to-gateway consumer, and the admission diff and its HTTP Stop consumer.
+
+## Local projection budget follow-up (2026-09-27)
+
+The audit decision "give local parsing its own budget" was only half applied:
+the buffered success path started its parse deadline after the body arrived but
+sized it with the client's connection budget, and the failed-response path
+shared one deadline between the error-body read and its projection. The
+inference-wait regression uses a 20 ms connection budget, so any scheduling or
+GC pause of that length while parsing an already-received body became a
+`TIMEOUT`, a 1-second source cooldown, and a gateway 503. That is the recurring
+`gateway-complete-buffered_body-openai_responses` CI failure on master.
+
+Both projections now start their own `_LOCAL_PROJECTION_TIMEOUT_SECONDS` budget
+after the body has arrived. The connection bound and the bounded read of an
+already-failed response are unchanged, and production timing is unchanged
+because that budget equals the 60-second transport default. A regression drives
+a projection that outlasts the connection budget and asserts success; the
+projection-budget test asserts the budget is independent of the client timeout.
diff --git a/tests/test_model_hub_runtime.py b/tests/test_model_hub_runtime.py
index c885d6983..2dbf7ec77 100644
--- a/tests/test_model_hub_runtime.py
+++ b/tests/test_model_hub_runtime.py
@@ -7381,7 +7381,7 @@ def test_engine_client_projects_machine_errors_from_large_buffered_bodies(
("status", "expected_kind"),
((200, RawOutcomeKind.TIMEOUT), (503, RawOutcomeKind.HTTP_ERROR)),
)
-def test_buffered_response_projection_uses_the_response_absolute_deadline(
+def test_buffered_response_projection_has_its_own_local_budget(
monkeypatch: pytest.MonkeyPatch,
status: int,
expected_kind: RawOutcomeKind,
@@ -7411,10 +7411,10 @@ def test_buffered_response_projection_uses_the_response_absolute_deadline(
async def close(self) -> None:
return None
- projection_deadlines: list[float] = []
+ projection_budgets: list[float] = []
def project_before_deadline(_reader, _projector, *, deadline):
- projection_deadlines.append(deadline)
+ projection_budgets.append(deadline - time.monotonic())
raise asyncio.TimeoutError("buffered response exceeded its request deadline")
monkeypatch.setattr(client_module.aiohttp, "ClientSession", lambda **_: Session())
@@ -7433,15 +7433,79 @@ def test_buffered_response_projection_uses_the_response_absolute_deadline(
model_ids=("model-a",),
prefix="source-fixture123",
)
- started = time.monotonic()
handle = await EngineClient(
EngineConnection("http://127.0.0.1:15220", "management", "gateway"),
- timeout=1.0,
+ timeout=0.02,
).invoke(source, "model-a", {}, stream=False)
assert (await handle.outcome()).kind is expected_kind
- assert projection_deadlines == pytest.approx([started + 1.0], abs=0.1)
+ # A received body is projected under its own budget, not the connection budget.
+ assert projection_budgets == pytest.approx(
+ [client_module._LOCAL_PROJECTION_TIMEOUT_SECONDS], abs=0.1,
+ )
+
+ asyncio.run(run())
+
+
+def test_buffered_projection_outlasting_the_connection_budget_still_succeeds(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ payload = b'{"output":[{"content":[{"type":"output_text","text":"ok"}]}]}'
+
+ async def run() -> None:
+ class Content:
+ reads = 0
+
+ async def read(self, _size: int) -> bytes:
+ self.reads += 1
+ return payload if self.reads == 1 else b""
+
+ class Response:
+ status = 200
+ content = Content()
+ headers = {"Content-Type": "application/json"}
+
+ def close(self) -> None:
+ return None
+
+ class Session:
+ async def post(self, *_args, **_kwargs):
+ return Response()
+
+ async def close(self) -> None:
+ return None
+
+ real_read = client_module._DeadlineReader.read
+
+ def stalled_read(self, size: int = -1) -> bytes:
+ # A thread-scheduling or GC pause longer than the whole connection budget.
+ time.sleep(0.05)
+ return real_read(self, size)
+
+ monkeypatch.setattr(client_module.aiohttp, "ClientSession", lambda **_: Session())
+ monkeypatch.setattr(client_module._DeadlineReader, "read", stalled_read)
+ source = SourceRecord(
+ source_id="src_fixture123",
+ vendor="custom",
+ protocol="openai_responses",
+ base_url="https://api.example.test/v1",
+ credential_ref="cred_fixture123",
+ allowed_origins=(),
+ model_ids=("model-a",),
+ prefix="source-fixture123",
+ )
+
+ handle = await EngineClient(
+ EngineConnection("http://127.0.0.1:15220", "management", "gateway"),
+ timeout=0.02,
+ ).invoke(source, "model-a", {}, stream=False)
+ try:
+ assert (await handle.outcome()).kind is RawOutcomeKind.SUCCESS
+ assert handle.stream is not None
+ assert b"".join([chunk async for chunk in handle.stream]) == payload
+ finally:
+ await handle.close_stream()
asyncio.run(run())
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index 494d1c9fa..58ca3dd75 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -55,6 +55,9 @@ _UPSTREAM_DETAIL_CHARS = 400
# truncates upstream response bytes.
_PRELUDE_MEMORY_BYTES = 256 * 1024
_ERROR_OBSERVATION_BYTES = 256 * 1024
+# Projecting a response body that has already arrived is local work. It has
+# its own budget so the client's connection budget never limits it.
+_LOCAL_PROJECTION_TIMEOUT_SECONDS = ENGINE_TRANSPORT_TIMEOUT_SECONDS
_OFFICIAL_BASE_URLS = official_api_key_base_urls()
_PROTOCOL_HEADERS = frozenset({"anthropic-beta", "anthropic-version", "openai-beta"})
logger = logging.getLogger(__name__)
@@ -507,7 +510,6 @@ class EngineClient:
if retry_after is not None and (len(retry_after) > 128 or not retry_after.isascii()):
retry_after = None
error_body = _StreamPrelude()
- response_deadline = time.monotonic() + self.timeout
try:
await asyncio.wait_for(
_read_response_into(response.content, error_body),
@@ -518,7 +520,7 @@ class EngineClient:
request_protocol,
error_body,
machine_error_codes=UPSTREAM_MACHINE_ERROR_CODES,
- deadline=response_deadline,
+ deadline=time.monotonic() + _LOCAL_PROJECTION_TIMEOUT_SECONDS,
)
payload = await error_body.prefix_async(_ERROR_OBSERVATION_BYTES)
except (asyncio.TimeoutError, aiohttp.ClientError):
@@ -583,7 +585,7 @@ class EngineClient:
request_protocol,
buffered_body,
machine_error_codes=UPSTREAM_MACHINE_ERROR_CODES,
- deadline=time.monotonic() + self.timeout,
+ deadline=time.monotonic() + _LOCAL_PROJECTION_TIMEOUT_SECONDS,
)
outcome = _reduce_protocol_observation(
observation,