Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

24.3 KB
Raw Blame
import contextlib
import json
import re
import uuid
from collections.abc import Iterator, Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Final

import httpx
import openai
import pytest
import yaml
from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
from integration._support.responses_stream import healthy_stream
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue

from litellm.proxy import client as proxy_client

pytestmark: Final = [
    pytest.mark.timeout(2 * graceful_stop_seconds() + 120),
    pytest.mark.xdist_group("chat-stream-guardrail-error-events"),
    pytest.mark.shares_worker_state,
]

_MODEL: Final = "stream-error-events"
_FILTER: Final = "keeper-filter"
_SPAN_POST: Final = "span-post-call"
_SPAN_PRE: Final = "span-pre-call"
_BUFFERED: Final = "buffered-post-call"
_RAISING: Final = "raising-post-call"
_REPLACE: Final = "replace-client-error"
_REPLACEMENT_MESSAGE: Final = "Request blocked by policy"
_CRASH_MESSAGE: Final = "Guardrail service crashed while scanning"
_RAISED_BLOCK: Final = "Raised block: policy"
_SPAN: Final = "spanword"
_LEAD: Final = "The lighthouse on the northern cliff has guided ships for two hundred years, and "
_TEXTS: Final[Mapping[str, tuple[str, ...]]] = {
    "clean": ("Hello", " there"),
    "filter-late": (_LEAD, "the keeper lights the lamp at dusk."),
    "filter-early": ("The keeper lights the lamp.", " Ships pass."),
    "span-early": (f"{_SPAN} opens the answer", " and more"),
    "span-late": ("Hello there, ", f"{_SPAN} follows"),
    "buffered": ("The lamp is lit by", " the keeper."),
    "drop": ("Hello", " there"),
}
_BUFFERED_PAUSE_SECONDS: Final = 1.5
_DROP_PAUSE_SECONDS: Final = 0.5
_SCENARIO: Final = re.compile(rb"scenario-([a-z-]+) marker-([0-9a-f]+)")
_MODULE: Final = "stream_error_hooks"
_MODULE_SOURCE: Final = f'''import re

from fastapi import HTTPException

from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_logger import CustomLogger


class SpanGuardrail(CustomGuardrail):
    async def async_post_call_streaming_hook(self, user_api_key_dict, response):
        match = re.search("{_SPAN}", response)
        if match is not None:
            raise HTTPException(status_code=400, detail={{"error": "Content blocked: {_SPAN}", "matched_span": match}})
        return None

    async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
        match = re.search("{_SPAN}", str(data["messages"][-1]["content"]))
        if match is not None:
            raise HTTPException(status_code=400, detail={{"matched_span": match}})
        return data


class BufferingGuardrail(CustomGuardrail):
    async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
        chunks = [chunk async for chunk in response]
        text = "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
        if "keeper" in text:
            raise HTTPException(status_code=400, detail={{"error": "Buffered block: keeper"}})
        for chunk in chunks:
            yield chunk


class RaisingGuardrail(CustomGuardrail):
    async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
        prompt = str(request_data.get("messages") or request_data.get("input"))
        found = re.search("raise-(builtin|http)-after-([0-9]+)", prompt)
        sent = 0
        async for chunk in response:
            if found is not None and sent == int(found.group(2)):
                if found.group(1) == "builtin":
                    raise RuntimeError("{_CRASH_MESSAGE}")
                raise HTTPException(status_code=400, detail={{"error": "{_RAISED_BLOCK}"}})
            yield chunk
            sent += 1


class ReplacesClientErrors(CustomLogger):
    async def async_post_call_failure_hook(
        self, request_data, original_exception, user_api_key_dict, traceback_str=None
    ):
        if "{_REPLACE}" in str(request_data.get("messages") or request_data.get("input")):
            return HTTPException(status_code=403, detail="{_REPLACEMENT_MESSAGE}")
        return None


replacer = ReplacesClientErrors()
'''


def _chunk(identity: str, delta: Mapping[str, JsonValue], finish: str | None = None) -> bytes:
    body: Final = {
        "id": identity,
        "object": "chat.completion.chunk",
        "created": 1,
        "model": "gpt-4o-mini",
        "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish}],
    }
    return b"data: " + json.dumps(body).encode() + b"\n\n"


def _stream(identity: str, pieces: Sequence[str]) -> tuple[bytes, ...]:
    first, *rest = pieces
    return (
        _chunk(identity, {"role": "assistant", "content": first}),
        *(_chunk(identity, {"content": piece}) for piece in rest),
        _chunk(identity, {}, finish="stop"),
        b"data: [DONE]\n\n",
    )


def _respond(request: Request) -> Reply:
    found: Final = _SCENARIO.search(request.body)
    assert found is not None, request.body
    name: Final = found.group(1).decode()
    marker: Final = found.group(2).decode()
    if request.target.endswith("/responses"):
        return Reply(content_type="text/event-stream", chunks=healthy_stream(f"resp_{marker}", "".join(_TEXTS[name])))
    chunks: Final = _stream(f"chatcmpl-{marker}", _TEXTS[name])
    match name:
        case "buffered":
            return Reply(content_type="text/event-stream", chunks=chunks, pause_between_chunks=_BUFFERED_PAUSE_SECONDS)
        case "drop":
            return Reply(
                content_type="text/event-stream",
                chunks=chunks,
                abort_after=2,
                pause_between_chunks=_DROP_PAUSE_SECONDS,
            )
        case _:
            return Reply(content_type="text/event-stream", chunks=chunks)


def _config(upstream_url: str) -> dict[str, JsonValue]:
    base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
    guardrails: Final[tuple[tuple[str, str, str], ...]] = (
        (_SPAN_POST, f"{_MODULE}.SpanGuardrail", "post_call"),
        (_SPAN_PRE, f"{_MODULE}.SpanGuardrail", "pre_call"),
        (_BUFFERED, f"{_MODULE}.BufferingGuardrail", "post_call"),
        (_RAISING, f"{_MODULE}.RaisingGuardrail", "post_call"),
    )
    return {
        **base,
        "litellm_settings": {**object_value(base["litellm_settings"]), "callbacks": [f"{_MODULE}.replacer"]},
        "guardrails": [
            {
                "guardrail_name": _FILTER,
                "litellm_params": {
                    "guardrail": "litellm_content_filter",
                    "mode": "post_call",
                    "default_on": False,
                    "blocked_words": [{"keyword": "keeper", "action": "BLOCK"}],
                },
            },
            *(
                {"guardrail_name": name, "litellm_params": {"guardrail": guardrail, "mode": mode, "default_on": False}}
                for name, guardrail, mode in guardrails
            ),
        ],
        "model_list": [
            {
                "model_name": _MODEL,
                "litellm_params": {
                    "model": "openai/gpt-4o-mini",
                    "api_base": f"{upstream_url}/v1",
                    "api_key": "synthetic-stream-key",
                    "allow_client_keepalive_override": True,
                },
            }
        ],
    }


@dataclass(frozen=True, slots=True)
class Rig:
    proxy: OwnedProxy
    upstream: Wire


@pytest.fixture(scope="module")
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
    with wire_server(_respond) as upstream:
        directory: Final = tmp_path_factory.mktemp("chat-stream-guardrail-error-events")
        (directory / f"{_MODULE}.py").write_text(_MODULE_SOURCE)
        config_path: Final = directory / "chat-stream-guardrail-error-events.yaml"
        config_path.write_text(yaml.safe_dump(_config(upstream.url)))
        with owned_proxy_process(
            session_gateway,
            directory,
            {},
            config=config_path,
            workers=2,
            healthcheck=WorkerHealthcheck.LENIENT,
        ) as proxy:
            yield Rig(proxy, upstream)


def _prompt(scenario: str, marker: str, *directives: str) -> str:
    return " ".join((f"scenario-{scenario}", f"marker-{marker}", *directives))


def _body(
    scenario: str, marker: str, guardrails: Sequence[str], *directives: str, **extra: JsonValue
) -> dict[str, JsonValue]:
    return {
        "model": _MODEL,
        "stream": True,
        "messages": [{"role": "user", "content": _prompt(scenario, marker, *directives)}],
        "guardrails": list(guardrails),
        "num_retries": 0,
        "cache": {"no-cache": True},
        **extra,
    }


def _marker() -> str:
    return uuid.uuid4().hex


def _calls_with(wire: Wire, marker: str) -> int:
    return sum(1 for request in wire.drain() if marker.encode() in request.body)


def _stream_chat(rig: Rig, body: Mapping[str, JsonValue]) -> httpx.Response:
    return rig.proxy.gateway.request("POST", "/v1/chat/completions", body)


def _data_lines(text: str) -> tuple[str, ...]:
    return tuple(line.removeprefix("data: ") for line in text.splitlines() if line.startswith("data: "))


def _payloads(text: str) -> tuple[dict[str, JsonValue], ...]:
    return tuple(JSON_OBJECT.validate_json(line) for line in _data_lines(text) if line != "[DONE]")


def _choices(payloads: Sequence[Mapping[str, JsonValue]]) -> Iterator[dict[str, JsonValue]]:
    for payload in payloads:
        yield from map(object_value, list_value(payload.get("choices") or []))


def _content(payloads: Sequence[Mapping[str, JsonValue]]) -> str:
    return "".join(string_value(object_value(choice["delta"]).get("content") or "") for choice in _choices(payloads))


def _finish_reasons(payloads: Sequence[Mapping[str, JsonValue]]) -> tuple[JsonValue, ...]:
    return tuple(choice.get("finish_reason") for choice in _choices(payloads))


@dataclass(frozen=True, slots=True)
class EndedInBand:
    content: str
    error: dict[str, JsonValue]


def _ended_in_band(response: httpx.Response, status: int) -> EndedInBand:
    assert response.status_code == status, response.text
    assert response.headers["content-type"].startswith("text/event-stream"), response.headers
    lines: Final = _data_lines(response.text)
    assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text
    payloads: Final = _payloads(response.text)
    assert "error" in payloads[-1] and all("error" not in payload for payload in payloads[:-1]), response.text
    assert "stop" not in _finish_reasons(payloads), response.text
    return EndedInBand(_content(payloads[:-1]), object_value(payloads[-1]["error"]))


def test_mid_stream_content_filter_block_ends_with_an_error_event_and_done(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("filter-late", marker, (_FILTER,)))
    ended: Final = _ended_in_band(response, 200)
    assert ended.content and _LEAD.startswith(ended.content), response.text
    assert ended.error["code"] == "400", ended.error
    assert "keeper" in string_value(ended.error["message"]), ended.error
    assert object_value(ended.error["provider_specific_fields"])["keyword"] == "keeper", ended.error
    assert _calls_with(rig.upstream, marker) == 1


def test_mid_stream_content_filter_block_reaches_the_openai_sdk_as_the_block_error(rig: Rig) -> None:
    marker: Final = _marker()
    client: Final = openai.OpenAI(
        base_url=f"{rig.proxy.gateway.client.base_url}/v1", api_key=rig.proxy.gateway.key, max_retries=0
    )
    stream: Final = client.chat.completions.create(
        model=_MODEL,
        messages=[{"role": "user", "content": f"scenario-filter-late marker-{marker}"}],
        stream=True,
        extra_body={"guardrails": [_FILTER], "num_retries": 0, "cache": {"no-cache": True}},
    )
    with pytest.raises(openai.APIError) as raised:
        tuple(stream)
    assert type(raised.value) is openai.APIError, repr(raised.value)
    assert "keeper" in raised.value.message, raised.value.message


def test_early_content_filter_block_keeps_its_400_event_stream_body(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("filter-early", marker, (_FILTER,)))
    ended: Final = _ended_in_band(response, 400)
    assert ended.content == "", response.text
    assert ended.error["code"] == "400", ended.error
    assert object_value(ended.error["provider_specific_fields"])["keyword"] == "keeper", ended.error
    assert _calls_with(rig.upstream, marker) == 1


@pytest.mark.parametrize(
    ("scenario", "status"),
    [pytest.param("span-early", 400, id="first_chunk"), pytest.param("span-late", 200, id="later_chunk")],
)
def test_per_chunk_block_with_an_unencodable_detail_still_sends_the_error_event(
    rig: Rig, scenario: str, status: int
) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body(scenario, marker, (_SPAN_POST,)))
    ended: Final = _ended_in_band(response, status)
    pieces: Final = _TEXTS[scenario]
    seen_by_hook: Final = "".join(pieces[: next(i for i, piece in enumerate(pieces) if _SPAN in piece) + 1])
    assert ended.content == ("" if status == 400 else pieces[0]), response.text
    assert ended.error["code"] == "400", ended.error
    assert ended.error["message"] == f"Content blocked: {_SPAN}", ended.error
    assert object_value(ended.error["provider_specific_fields"])["matched_span"] == str(
        re.search(_SPAN, seen_by_hook)
    ), ended.error


@pytest.mark.parametrize(
    ("scenario", "status"),
    [pytest.param("filter-early", 403, id="before_the_first_chunk"), pytest.param("filter-late", 200, id="mid_stream")],
)
def test_failure_hook_replacement_is_the_error_the_stream_ends_with(rig: Rig, scenario: str, status: int) -> None:
    marker: Final = _marker()
    body: Final = _body(scenario, marker, (_FILTER,))
    replaced: Final = {
        **body,
        "messages": [{"role": "user", "content": f"scenario-{scenario} marker-{marker} {_REPLACE}"}],
    }
    response: Final = _stream_chat(rig, replaced)
    ended: Final = _ended_in_band(response, status)
    assert ended.error["code"] == "403", ended.error
    assert ended.error["message"] == _REPLACEMENT_MESSAGE, ended.error
    assert "keeper" not in json.dumps(ended.error), ended.error


def test_buffered_block_after_keepalive_pings_ends_with_an_error_event_and_done(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("buffered", marker, (_BUFFERED,), keepalive_seconds=1))
    assert response.text.startswith(": ping\n\n"), response.text
    ended: Final = _ended_in_band(response, 200)
    assert ended.content == "", response.text
    assert ended.error["code"] == "400", ended.error
    assert ended.error["message"] == "Buffered block: keeper", ended.error


def test_clean_stream_with_guardrails_on_ends_with_stop_and_one_done(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("clean", marker, (_FILTER, _SPAN_POST)))
    assert response.status_code == 200, response.text
    lines: Final = _data_lines(response.text)
    assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text
    payloads: Final = _payloads(response.text)
    assert all("error" not in payload for payload in payloads), response.text
    assert _content(payloads) == "".join(_TEXTS["clean"]), response.text
    assert "stop" in _finish_reasons(payloads), response.text


def test_gemini_route_provider_failure_mid_stream_sends_an_error_frame_without_done(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = rig.proxy.gateway.request(
        "POST",
        f"/v1beta/models/{_MODEL}:streamGenerateContent",
        {"contents": [{"role": "user", "parts": [{"text": f"scenario-drop marker-{marker}"}]}]},
        params={"alt": "sse"},
    )
    assert response.status_code == 200, response.text
    assert "[DONE]" not in response.text, response.text
    payloads: Final = _payloads(response.text)
    assert "error" in payloads[-1] and all("error" not in payload for payload in payloads[:-1]), response.text
    assert _calls_with(rig.upstream, marker) == 1


def test_mid_stream_block_releases_the_parallel_request_slot_and_logs_one_failure(rig: Rig) -> None:
    marker: Final = _marker()
    with rig.proxy.gateway.scenario() as scenario:
        key: Final = scenario.key(max_parallel_requests=1)
        with rig.proxy.gateway.client.stream(
            "POST",
            "/v1/chat/completions",
            json=_body("filter-late", marker, (_FILTER,)),
            headers={"Authorization": f"Bearer {key}"},
        ) as blocked:
            call_id: Final = blocked.headers["x-litellm-call-id"]
            with contextlib.suppress(httpx.RemoteProtocolError):
                blocked.read()
        follow_up: Final = rig.proxy.gateway.request(
            "POST", "/v1/chat/completions", _body("clean", _marker(), ()), key=key
        )
        assert follow_up.status_code == 200, follow_up.text
        assert _content(_payloads(follow_up.text)) == "".join(_TEXTS["clean"]), follow_up.text
    rows: Final = eventually(
        lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)),
        lambda found: len(found) >= 1,
        seconds=70,
    )
    assert [row["status"] for row in rows] == ["failure"], rows


def test_failure_hook_replacement_hides_a_non_http_failure_mid_stream(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("clean", marker, (_RAISING,), "raise-builtin-after-1", _REPLACE))
    ended: Final = _ended_in_band(response, 200)
    assert ended.content == _TEXTS["clean"][0], response.text
    assert ended.error["code"] == "403", ended.error
    assert ended.error["message"] == _REPLACEMENT_MESSAGE, ended.error
    assert _CRASH_MESSAGE not in response.text, response.text
    assert _calls_with(rig.upstream, marker) == 1


def test_failure_hook_replacement_answers_an_early_non_http_failure_as_a_json_error(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("clean", marker, (_RAISING,), "raise-builtin-after-0", _REPLACE))
    assert response.status_code == 403, response.text
    assert response.headers["content-type"].startswith("application/json"), response.headers
    error: Final = object_value(JSON_OBJECT.validate_json(response.content)["error"])
    assert error["code"] == "403", error
    assert error["message"] == _REPLACEMENT_MESSAGE, error
    assert _CRASH_MESSAGE not in response.text, response.text
    assert _calls_with(rig.upstream, marker) == 1


def test_failure_hook_replacement_ends_a_queue_stream_that_fails_before_the_first_chunk(rig: Rig) -> None:
    marker: Final = _marker()
    body: Final = _body("clean", marker, (_RAISING,), "raise-builtin-after-0", _REPLACE, priority=0)
    response: Final = rig.proxy.gateway.request("POST", "/queue/chat/completions", body)
    ended: Final = _ended_in_band(response, 200)
    assert ended.content == "", response.text
    assert ended.error["code"] == "403", ended.error
    assert ended.error["message"] == _REPLACEMENT_MESSAGE, ended.error
    assert _CRASH_MESSAGE not in response.text, response.text
    assert _calls_with(rig.upstream, marker) == 1


def test_early_non_http_failure_without_a_replacement_keeps_its_json_error_body(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = _stream_chat(rig, _body("clean", marker, (_RAISING,), "raise-builtin-after-0"))
    assert response.status_code == 500, response.text
    assert response.headers["content-type"].startswith("application/json"), response.headers
    error: Final = object_value(JSON_OBJECT.validate_json(response.content)["error"])
    assert error["code"] == "500", error
    assert _CRASH_MESSAGE in string_value(error["message"]), error
    assert _calls_with(rig.upstream, marker) == 1


def _responses_body(marker: str, *directives: str) -> dict[str, JsonValue]:
    return {
        "model": _MODEL,
        "stream": True,
        "input": _prompt("clean", marker, *directives),
        "guardrails": [_RAISING],
        "cache": {"no-cache": True},
    }


def _failed_response_error(response: httpx.Response) -> dict[str, JsonValue]:
    assert response.status_code == 200, response.text
    lines: Final = _data_lines(response.text)
    assert lines[-1] == "[DONE]" and lines.count("[DONE]") == 1, response.text
    events: Final = _payloads(response.text)
    kinds: Final = tuple(string_value(event["type"]) for event in events)
    assert kinds[0] == "response.created" and kinds[-1] == "response.failed", response.text
    assert "response.completed" not in kinds and kinds.count("response.failed") == 1, response.text
    streamed: Final = "".join(
        string_value(event.get("delta") or "") for event in events if event["type"] == "response.output_text.delta"
    )
    assert streamed == "".join(_TEXTS["clean"]), response.text
    created: Final = object_value(events[0]["response"])
    failed: Final = object_value(events[-1]["response"])
    assert failed["status"] == "failed" and failed["id"] == created["id"], failed
    return object_value(failed["error"])


def test_responses_stream_failing_on_a_builtin_exception_ends_with_response_failed(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = rig.proxy.gateway.request(
        "POST", "/v1/responses", _responses_body(marker, "raise-builtin-after-2")
    )
    error: Final = _failed_response_error(response)
    assert error == {"code": "server_error", "message": _CRASH_MESSAGE}, error
    assert _calls_with(rig.upstream, marker) == 1


def test_responses_stream_failure_hook_replacement_is_the_failed_response_error(rig: Rig) -> None:
    marker: Final = _marker()
    response: Final = rig.proxy.gateway.request(
        "POST", "/v1/responses", _responses_body(marker, "raise-http-after-2", _REPLACE)
    )
    error: Final = _failed_response_error(response)
    assert error["code"] == "permission_error", error
    assert _REPLACEMENT_MESSAGE in string_value(error["message"]), error
    assert _RAISED_BLOCK not in response.text, response.text
    assert _calls_with(rig.upstream, marker) == 1


def _until_stream_error(stream: Iterator[Mapping[str, object]]) -> Iterator[Mapping[str, object] | Exception]:
    try:
        yield from stream
    except proxy_client.StreamError as error:
        yield error


def test_python_client_raises_the_block_error_after_the_partial_reply(rig: Rig) -> None:
    marker: Final = _marker()
    with rig.proxy.gateway.scenario() as scenario:
        client: Final = proxy_client.ChatClient(
            str(rig.proxy.gateway.client.base_url), api_key=scenario.key(metadata={"guardrails": [_FILTER]})
        )
        stream: Final = client.completions_stream(
            model=_MODEL, messages=[{"role": "user", "content": _prompt("filter-late", marker)}]
        )
        *chunks, last = tuple(_until_stream_error(stream))
    assert isinstance(last, proxy_client.StreamError), last
    payloads: Final = tuple(JSON_OBJECT.validate_python(chunk) for chunk in chunks)
    assert all("error" not in payload for payload in payloads), payloads
    content: Final = _content(payloads)
    assert content and _LEAD.startswith(content), payloads
    assert "keeper" in str(last), str(last)
    assert JSON_OBJECT.validate_python(last.error)["code"] == "400", last.error
    assert _calls_with(rig.upstream, marker) == 1


def test_non_streaming_pre_call_block_with_an_unencodable_detail_answers_400(rig: Rig) -> None:
    pytest.skip("BUG: a non-JSON value in a guardrail HTTPException detail answers 500 on non-streaming chat")
    marker: Final = _marker()
    content: Final = f"{_SPAN} in the prompt scenario-clean marker-{marker}"
    response: Final = rig.proxy.gateway.request(
        "POST",
        "/v1/chat/completions",
        {
            "model": _MODEL,
            "messages": [{"role": "user", "content": content}],
            "guardrails": [_SPAN_PRE],
            "cache": {"no-cache": True},
        },
    )
    assert response.status_code == 400, response.text
    error: Final = object_value(JSON_OBJECT.validate_json(response.content)["error"])
    assert error["message"] == json.dumps({"matched_span": str(re.search(_SPAN, content))}), error
    assert _calls_with(rig.upstream, marker) == 0