Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Retry advice crosses the adapter boundary without adding retry ownership."""
from __future__ import annotations
import asyncio
import socket
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from aiohttp import web
import pytest
from core.handlers.model_hub.adapter import RawCallOutcome, RawOutcomeKind
from core.handlers.model_hub.retry import retry_after_deadline
from core.handlers.model_hub.stream_wire import ProtocolUsageReport
from vibe.model_hub_runtime import client as client_module
from vibe.model_hub_runtime.client import EngineClient, EngineConnection
from vibe.model_hub_runtime.state import SourceRecord
WIRE = {
"anthropic": (
"/v1/messages",
b'{"type":"message","content":[{"type":"text","text":"ok"}]}',
b'event: content_block_delta\ndata: {"type":"content_block_delta",'
b'"delta":{"type":"text_delta","text":"ok"}}\n\n'
b'event: message_stop\ndata: {"type":"message_stop"}\n\n',
b'event: error\ndata: {"type":"error","error":{"type":"api_error"}}\n\n',
),
"openai_responses": (
"/v1/responses",
b'{"output":[{"content":[{"type":"output_text","text":"ok"}]}]}',
b'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"ok"}\n\n'
b'event: response.completed\ndata: {"type":"response.completed"}\n\n',
b'event: response.failed\ndata: {"type":"response.failed","response":'
b'{"error":{"type":"server_error"}}}\n\n',
),
"openai_chat": (
"/v1/chat/completions",
b'{"choices":[{"message":{"role":"assistant","content":"ok"}}]}',
b'data: {"choices":[{"delta":{"content":"ok"}}]}\n\ndata: [DONE]\n\n',
b'data: {"error":{"type":"server_error"}}\n\n',
),
}
ERROR_BODY = (
b'{"error":{"type":"server_error","message":"private body detail","retry_after":"999"}}'
)
def _source(protocol: str, *, vendor: str = "custom") -> SourceRecord:
return SourceRecord(
source_id="src_retryadvice",
vendor=vendor,
protocol=protocol,
base_url="https://unused.example.test",
credential_ref="cred_retryadvice",
allowed_origins=(),
model_ids=("model-a",),
prefix="retry-advice",
)
@asynccontextmanager
async def _engine(respond, *, timeout: float = 2):
requests = []
async def handle(request: web.Request) -> web.StreamResponse:
requests.append((request.path, await request.json()))
return await respond(request)
app = web.Application()
app.router.add_post("/{path:.*}", handle)
runner = web.AppRunner(app, handler_cancellation=True)
await runner.setup()
try:
await web.TCPSite(runner, "127.0.0.1", 0).start()
port = runner.addresses[0][1]
client = EngineClient(
EngineConnection(f"http://127.0.0.1:{port}", "fixture-management", "fixture-gateway"),
timeout=timeout,
)
yield client, requests
finally:
await runner.cleanup()
async def _invoke(
client: EngineClient,
protocol: str,
stream: bool,
*,
vendor: str = "custom",
request_protocol: str | None = None,
):
cleanups = []
handle = await client.invoke(
_source(protocol, vendor=vendor),
"model-a",
{},
stream=stream,
request_protocol=request_protocol or protocol,
on_transport_done=lambda: cleanups.append(True),
)
try:
body = b"" if handle.stream is None else b"".join([chunk async for chunk in handle.stream])
outcome = await handle.outcome()
finally:
await handle.close_stream()
assert cleanups
return outcome, body
@pytest.mark.parametrize("optional_count", [0, 3])
def test_existing_outcome_constructors_keep_retry_advice_optional(optional_count: int) -> None:
usage = ProtocolUsageReport.of(input_tokens=12, cached_input_tokens=0, output_tokens=3)
optional = ("server_error", ("server_error",), usage)
outcome = RawCallOutcome(
RawOutcomeKind.HTTP_ERROR, 503, None, None, False, "model-a", "src_retryadvice",
*optional[:optional_count],
)
assert outcome.retry_after is None
assert outcome.response_received_at is None
assert outcome.usage == (usage if optional_count else None)
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])
async def test_failed_response_carries_only_allowlisted_correlation_ids(protocol: str, stream: bool) -> None:
async def respond(_request: web.Request) -> web.Response:
return web.Response(
status=400,
body=ERROR_BODY,
headers={
"X-Request-Id": "relay-request-1",
"Request-Id": "provider-request-1",
"Set-Cookie": "session=secret",
"Organization-Id": "org-private",
},
content_type="application/json",
)
async with _engine(respond) as (client, _requests):
outcome, body = await _invoke(client, protocol, stream)
assert outcome.kind is RawOutcomeKind.HTTP_ERROR
assert outcome.response_correlation_ids == (
("request-id", "provider-request-1"),
("x-request-id", "relay-request-1"),
)
assert body == b""
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])
async def test_success_response_carries_its_own_correlation_ids(protocol: str, stream: bool) -> None:
_path, buffered, streamed, _stream_error = WIRE[protocol]
async def respond(_request: web.Request) -> web.Response:
return web.Response(
status=200,
body=streamed if stream else buffered,
headers={"X-Request-Id": "served-request-1"},
content_type="text/event-stream" if stream else "application/json",
)
async with _engine(respond) as (client, _requests):
outcome, _body = await _invoke(client, protocol, stream)
assert outcome.kind is RawOutcomeKind.SUCCESS
assert outcome.response_correlation_ids == (("x-request-id", "served-request-1"),)
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])
@pytest.mark.parametrize(
("advice", "expected"),
[
pytest.param("60", "60", id="delay-seconds"),
pytest.param("0", "0", id="zero"),
pytest.param("Wed, 09 Sep 2026 04:20:00 GMT", "Wed, 09 Sep 2026 04:20:00 GMT", id="http-date"),
pytest.param("Sun, 06 Nov 1994 08:49:37 GMT", "Sun, 06 Nov 1994 08:49:37 GMT", id="past-date"),
pytest.param("-1", "-1", id="negative-left-to-policy"),
pytest.param("1.5", "1.5", id="fraction-left-to-policy"),
pytest.param("not-a-date", "not-a-date", id="malformed-left-to-policy"),
pytest.param("9" * 128, "9" * 128, id="overflow-left-to-policy"),
pytest.param("9" * 129, None, id="oversized-dropped-not-truncated"),
pytest.param("60" + "x" * 127, None, id="oversized-valid-prefix"),
pytest.param("12", None, id="nonascii-digits"),
pytest.param("60秒", None, id="nonascii-suffix"),
pytest.param("", "", id="empty-left-to-policy"),
pytest.param(None, None, id="missing"),
],
)
async def test_failed_http_response_carries_only_bounded_ascii_advice(
protocol: str, stream: bool, advice: str | None, expected: str | None,
) -> None:
async def respond(_request: web.Request) -> web.Response:
headers = {
"Authorization": "private response credential",
"X-Retry-After": "888",
"X-Private-Detail": "private header detail",
}
if advice is not None:
headers["rEtRy-AfTeR"] = advice
return web.Response(status=503, body=ERROR_BODY, headers=headers, content_type="application/json")
async with _engine(respond) as (client, requests):
before = datetime.now(timezone.utc)
outcome, body = await _invoke(client, protocol, stream)
after = datetime.now(timezone.utc)
assert outcome.kind is RawOutcomeKind.HTTP_ERROR
assert outcome.http_status == 503
assert outcome.stream_started is False
assert outcome.retry_after == expected
assert outcome.response_received_at is not None
assert outcome.response_received_at.tzinfo is timezone.utc
assert before <= outcome.response_received_at <= after
assert outcome.error_type == "server_error"
assert outcome.error_candidates == ("server_error",)
assert "private" not in repr(outcome)
assert body == b""
assert requests == [(WIRE[protocol][0], {"model": "retry-advice/model-a", "stream": stream})]
@pytest.mark.parametrize(("status", "error_type"), [(429, "rate_limit_error"), (503, "server_error")])
async def test_only_a_usage_cap_turns_its_body_reset_into_advice(status: int, error_type: str) -> None:
"""Reset fields in any other error body are not retry advice and cannot set a cooldown."""
body = {"error": {"type": error_type, "resets_at": 4102444800, "resets_in_seconds": 600}}
async def respond(_request: web.Request) -> web.Response:
return web.json_response(body, status=status)
async with _engine(respond) as (client, _requests):
outcome, _body = await _invoke(client, "openai_responses", False)
assert (outcome.http_status, outcome.error_type) == (status, error_type)
assert outcome.retry_after is None
@pytest.mark.parametrize(
("raw", "expected"),
[
pytest.param("1500", 1500, id="milliseconds"),
pytest.param("0", 0, id="zero"),
pytest.param("-1", None, id="negative"),
pytest.param("1.5", None, id="fraction"),
pytest.param("12", None, id="nonascii"),
pytest.param("9" * 129, None, id="oversized"),
pytest.param("9" * 128, None, id="overflow"),
pytest.param("", None, id="empty"),
],
)
async def test_retry_after_ms_is_validated_without_becoming_retry_after_seconds(raw: str, expected: int | None) -> None:
async def respond(_request: web.Request) -> web.Response:
return web.Response(
status=429,
body=ERROR_BODY,
headers={"rEtRy-AfTeR-Ms": raw},
content_type="application/json",
)
async with _engine(respond) as (client, _requests):
outcome, _body = await _invoke(client, "anthropic", False)
assert outcome.retry_after is None
assert outcome.retry_after_ms == expected
if expected is not None:
assert outcome.response_received_at is not None
deadline = retry_after_deadline(
None, outcome.response_received_at, milliseconds=expected,
)
assert deadline is not None
assert (deadline - outcome.response_received_at).total_seconds() == expected / 1000
async def test_retry_after_and_retry_after_ms_use_the_later_valid_deadline() -> None:
async def respond(_request: web.Request) -> web.Response:
return web.Response(
status=429,
body=ERROR_BODY,
headers={"Retry-After": "1", "Retry-After-Ms": "2500"},
content_type="application/json",
)
async with _engine(respond) as (client, _requests):
outcome, _body = await _invoke(client, "anthropic", False)
assert outcome.retry_after == "1"
assert outcome.retry_after_ms == 2500
assert outcome.response_received_at is not None
deadline = retry_after_deadline(
outcome.retry_after,
outcome.response_received_at,
milliseconds=outcome.retry_after_ms,
)
assert deadline is not None
assert (deadline - outcome.response_received_at).total_seconds() == 2.5
async def test_failed_claude_response_carries_only_bounded_unified_headers() -> None:
async def respond(_request: web.Request) -> web.Response:
return web.Response(
status=429,
body=ERROR_BODY,
headers={
"Anthropic-Ratelimit-Unified-Status": "rejected",
"Anthropic-Ratelimit-Unified-5h-Reset": "1797000000",
"Anthropic-Ratelimit-Unified-Representative-Claim": "5h",
"Anthropic-Ratelimit-Unified-7d-Status": "bad",
"Anthropic-Ratelimit-Unified-Unknown": "private",
"Set-Cookie": "session=secret",
},
content_type="application/json",
)
async with _engine(respond) as (client, _requests):
outcome, _body = await _invoke(
client,
"anthropic",
False,
vendor="anthropic",
request_protocol="openai_responses",
)
assert outcome.claude_rate_limit_headers == (
("anthropic-ratelimit-unified-5h-reset", "1797000000"),
("anthropic-ratelimit-unified-representative-claim", "5h"),
("anthropic-ratelimit-unified-status", "rejected"),
)
assert "Set-Cookie" not in repr(outcome)
async def test_non_claude_source_does_not_project_unified_headers_for_claude_caller() -> None:
async def respond(_request: web.Request) -> web.Response:
return web.Response(
status=429,
body=ERROR_BODY,
headers={
"Anthropic-Ratelimit-Unified-Status": "rejected",
"Anthropic-Ratelimit-Unified-5h-Reset": "1797000000",
},
content_type="application/json",
)
async with _engine(respond) as (client, _requests):
outcome, _body = await _invoke(
client,
"anthropic",
False,
vendor="custom",
request_protocol="anthropic",
)
assert outcome.claude_rate_limit_headers == ()
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])
@pytest.mark.parametrize("body_mode", ["slow", "timeout", "disconnect", "malformed"])
async def test_retry_advice_survives_error_body_delays_and_failures(
monkeypatch: pytest.MonkeyPatch, protocol: str, stream: bool, body_mode: str,
) -> None:
"""MH-RETRY-ADVICE-001: advice keeps the header receipt clock across body failures."""
receipt = datetime(2026, 9, 9, 4, 15, tzinfo=timezone.utc)
current_time = receipt
clock_samples = []
sampled = asyncio.Event()
release_body = asyncio.Event()
class Clock:
@staticmethod
def now(tz):
assert tz is timezone.utc
clock_samples.append(current_time)
sampled.set()
return current_time
monkeypatch.setattr(client_module, "datetime", Clock)
async def respond(request: web.Request) -> web.StreamResponse:
response = web.StreamResponse(
status=429,
headers={
"Retry-After": "60",
"Content-Type": "application/json",
"Content-Length": str(len(ERROR_BODY)),
},
)
await response.prepare(request)
await release_body.wait()
if body_mode == "timeout":
return response
if body_mode == "disconnect":
await response.write(ERROR_BODY[:5])
assert request.transport is not None
request.transport.close()
else:
await response.write(b"x" * len(ERROR_BODY) if body_mode == "malformed" else ERROR_BODY)
await response.write_eof()
return response
async with _engine(respond, timeout=0.25 if body_mode == "timeout" else 2) as (client, requests):
pending = asyncio.create_task(_invoke(client, protocol, stream))
try:
# The server cannot finish its body until the receipt clock is sampled.
await asyncio.wait_for(sampled.wait(), timeout=2)
assert not pending.done()
current_time += timedelta(minutes=5)
if body_mode != "timeout":
release_body.set()
outcome, body = await asyncio.wait_for(pending, timeout=3)
finally:
release_body.set()
if not pending.done():
pending.cancel()
await asyncio.gather(pending, return_exceptions=True)
assert clock_samples == [receipt]
assert outcome.response_received_at == receipt
assert outcome.retry_after == "60"
assert outcome.kind is RawOutcomeKind.HTTP_ERROR
assert outcome.http_status == 429
assert outcome.stream_started is False
assert outcome.error_type == ("server_error" if body_mode == "slow" else None)
assert body == b""
assert len(requests) == 1
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("stream", [False, True], ids=["buffered", "streaming"])
@pytest.mark.parametrize(
("body_mode", "expected_kind"),
[
("success", RawOutcomeKind.SUCCESS),
("shaped_error", RawOutcomeKind.HTTP_ERROR),
("empty", RawOutcomeKind.NETWORK_ERROR),
],
)
async def test_http_200_never_supplies_retry_advice(
protocol: str, stream: bool, body_mode: str, expected_kind: RawOutcomeKind,
) -> None:
_, buffered, streamed, stream_error = WIRE[protocol]
payload = {
"success": streamed if stream else buffered,
"shaped_error": stream_error if stream else ERROR_BODY,
"empty": b"",
}[body_mode]
async def respond(_request: web.Request) -> web.Response:
return web.Response(
body=payload,
headers={"Retry-After": "60"},
content_type="text/event-stream" if stream else "application/json",
)
async with _engine(respond) as (client, requests):
outcome, body = await _invoke(client, protocol, stream)
assert outcome.kind is expected_kind
assert outcome.http_status == 200
assert outcome.retry_after is None
assert outcome.response_received_at is None
assert body == (payload if body_mode == "success" else b"")
assert len(requests) == 1
async def test_connection_failure_has_no_response_receipt_or_retry_advice() -> None:
# Reserve a local port without listening: no real service can own this target.
with socket.socket() as reserved:
reserved.bind(("127.0.0.1", 0))
port = reserved.getsockname()[1]
client = EngineClient(
EngineConnection(f"http://127.0.0.1:{port}", "fixture-management", "fixture-gateway"),
)
outcome, body = await _invoke(client, "openai_responses", False)
assert outcome.kind is RawOutcomeKind.NETWORK_ERROR
assert outcome.http_status is None
assert outcome.retry_after is None
assert outcome.response_received_at is None
assert body == b""
@pytest.mark.parametrize("protocol", WIRE)
async def test_local_registration_failure_keeps_its_identity_with_http_receipt(protocol: str) -> None:
message = "unknown provider for model retry-advice/model-a"
# CLIProxyAPI's local routing rejection, rebuilt as a Claude error on /v1/messages.
body = (
{"type": "error", "error": {"type": "invalid_request_error", "message": message}}
if protocol == "anthropic"
else {"error": {"message": message, "type": "invalid_request_error", "code": "model_not_found", "param": "model"}}
)
async def respond(_request: web.Request) -> web.Response:
return web.json_response(body, status=400, headers={"Retry-After": "60"})
async with _engine(respond) as (client, requests):
outcome, body = await _invoke(client, protocol, False)
assert outcome.kind is RawOutcomeKind.NETWORK_ERROR
assert outcome.error_code == "engine_down"
assert outcome.http_status == 400
assert outcome.retry_after == "60"
assert outcome.response_received_at is not None
assert outcome.response_received_at.tzinfo is timezone.utc
assert body == b""
assert len(requests) == 1