Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

17.8 KB
Raw Blame
from __future__ import annotations

import json
import os
import uuid
from collections.abc import Callable
from typing import Final

import openai
import pytest
from integration._support.client import Gateway, Scenario, object_value, string_value
from integration._support.wire import Reply, Request, Wire, wire_server
from integration._support.worker_readiness import wait_for_worker_readiness
from openai.types.responses import ResponseCompletedEvent
from pydantic import JsonValue, TypeAdapter

_MODEL: Final = "gpt-5"
_API_KEY: Final = "synthetic-responses-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1")))
_ALIASES: Final = ("/v1", "", "/openai/v1")


def _deployments_in_group(gateway: Gateway, group: str) -> int:
    entries: Final = gateway.get("/model/info")["data"]
    assert isinstance(entries, list), entries
    return sum(object_value(entry).get("model_name") == group for entry in entries)


def _wait_for_workers(gateway: Gateway, group: str, deployments: int) -> None:
    wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS)


def model_discovery_reply(model: str) -> Reply:
    return Reply(
        body=json.dumps(
            {"object": "list", "data": [{"id": model, "object": "model", "created": 1, "owned_by": "openai"}]}
        ).encode()
    )


def drain_contract_requests(wire: Wire) -> tuple[Request, ...]:
    requests: Final = wire.drain()
    discovery: Final = tuple(request for request in requests if request.target == "/v1/models")
    assert all(request.method == "GET" and request.body == b"" for request in discovery), requests
    return tuple(request for request in requests if request.target != "/v1/models")


def _response(identity: str, status: str = "completed") -> bytes:
    return json.dumps(
        {
            "id": identity,
            "object": "response",
            "created_at": 1,
            "status": status,
            "model": _MODEL,
            "output": [],
        }
    ).encode()


def _stream(identity: str) -> tuple[bytes, ...]:
    return tuple(
        f"data: {json.dumps(event)}\n\n".encode()
        for event in (
            {"type": "response.created", "response": json.loads(_response(identity, "in_progress"))},
            {"type": "response.completed", "response": json.loads(_response(identity))},
        )
    ) + (b"data: [DONE]\n\n",)


def _register_in_group(scenario: Scenario, wire: Wire, group: str, order: int | None = None) -> str:
    existing: Final = _deployments_in_group(scenario.gateway, group)
    created: Final = scenario.gateway.post(
        "/model/new",
        {
            "model_name": group,
            "litellm_params": {
                "model": f"openai/{_MODEL}",
                "api_key": _API_KEY,
                "api_base": f"{wire.url}/v1",
                **({} if order is None else {"order": order}),
            },
            "model_info": {},
        },
    )
    model_id: Final = string_value(object_value(created["model_info"])["id"])
    scenario.cleanups.callback(scenario.delete_model, model_id)
    _wait_for_workers(scenario.gateway, group, existing + 1)
    return group


def _register(scenario: Scenario, wire: Wire, group: str | None = None) -> str:
    if group is not None:
        return _register_in_group(scenario, wire, group)
    model: Final = scenario.model(
        model=f"openai/{_MODEL}",
        api_key=_API_KEY,
        api_base=f"{wire.url}/v1",
    )
    _wait_for_workers(scenario.gateway, model, 1)
    return model


def _enable_responses_affinity(scenario: Scenario) -> None:
    gateway: Final = scenario.gateway
    settings: Final = object_value(gateway.get("/router/settings")["current_values"])
    current: Final = settings.get("optional_pre_call_checks")
    original: Final = list(current) if isinstance(current, list) else []
    scenario.cleanups.callback(
        gateway.post,
        "/config/update",
        {"router_settings": {"optional_pre_call_checks": original}},
    )
    gateway.post(
        "/config/update",
        {"router_settings": {"optional_pre_call_checks": [*original, "responses_api_deployment_check"]}},
    )


@pytest.mark.parametrize("prefix", _ALIASES)
@pytest.mark.parametrize("stream", (False, True))
def test_previous_response_id_uses_the_raw_id_and_affinity(
    gateway: Gateway,
    prefix: str,
    stream: bool,
) -> None:
    first_raw_id: Final = f"resp_turn_one_{uuid.uuid4().hex}"
    second_raw_id: Final = f"resp_turn_two_{uuid.uuid4().hex}"

    def respond(request: Request) -> Reply:
        if request.method == "GET" and request.target == "/v1/models":
            return model_discovery_reply(_MODEL)
        assert request.method == "POST"
        assert request.target == "/v1/responses"
        assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers)
        body: Final = _JSON_OBJECT.validate_json(request.body)
        if body["input"] == "turn one":
            assert body == {"model": _MODEL, "input": "turn one"}, request.body
            return Reply(body=_response(first_raw_id))
        if stream:
            return Reply(content_type="text/event-stream", chunks=_stream(second_raw_id))
        return Reply(body=_response(second_raw_id))

    def unused(request: Request) -> Reply:
        if request.method == "GET" and request.target == "/v1/models":
            return model_discovery_reply(_MODEL)
        assert request.method == "POST"
        assert request.target == "/v1/responses"
        assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers)
        if stream:
            return Reply(content_type="text/event-stream", chunks=_stream(second_raw_id))
        return Reply(body=_response(second_raw_id))

    with wire_server(respond) as first_wire, wire_server(unused) as second_wire, gateway.scenario() as scenario:
        model: Final = f"integration-{uuid.uuid4().hex}"
        _enable_responses_affinity(scenario)
        assert _register(scenario, first_wire, model) == model
        client: Final = openai.OpenAI(
            base_url=f"{gateway.client.base_url}{prefix}",
            api_key=gateway.key,
            max_retries=0,
        )
        first: Final = client.responses.create(model=model, input="turn one")
        assert first.id.startswith("resp_") and first.id != first_raw_id, first.model_dump_json()
        _register_in_group(scenario, second_wire, model, order=1)
        second_result: Final = client.responses.create(
            model=model,
            input="turn two",
            previous_response_id=first.id,
            stream=stream,
        )
        if stream:
            events: Final = list(second_result)
            completed: Final = events[-1]
            assert isinstance(completed, ResponseCompletedEvent), events
            assert completed.response.id != second_raw_id, completed.model_dump_json()
            assert completed.response.output == [], completed.model_dump_json()
        else:
            assert second_result.id != second_raw_id, second_result.model_dump_json()
            assert second_result.output == [], second_result.model_dump_json()
        assert drain_contract_requests(second_wire) == (), "previous_response_id did not preserve deployment affinity"
        requests: Final = drain_contract_requests(first_wire)
        assert len(requests) == 2, requests
        assert [_JSON_OBJECT.validate_json(request.body) for request in requests] == [
            {"model": _MODEL, "input": "turn one"},
            {
                "model": _MODEL,
                "input": "turn two",
                "previous_response_id": first_raw_id,
                "stream": stream,
            },
        ], [request.body for request in requests]


def _counting_peer(raw_id: str, input_tokens: int) -> Callable[[Request], Reply]:
    def respond(request: Request) -> Reply:
        if request.method == "GET" and request.target == "/v1/models":
            return model_discovery_reply(_MODEL)
        if request.target == "/v1/responses/input_tokens":
            return Reply(body=json.dumps({"object": "response.input_tokens", "input_tokens": input_tokens}).encode())
        return Reply(body=_response(raw_id))

    return respond


def test_input_tokens_count_goes_to_the_deployment_that_served_the_previous_response(gateway: Gateway) -> None:
    raw_id: Final = f"resp_count_affinity_{uuid.uuid4().hex}"
    with (
        wire_server(_counting_peer(raw_id, 37)) as first_wire,
        wire_server(_counting_peer(raw_id, 99)) as second_wire,
        gateway.scenario() as scenario,
    ):
        model: Final = f"integration-{uuid.uuid4().hex}"
        _enable_responses_affinity(scenario)
        assert _register(scenario, first_wire, model) == model
        client: Final = openai.OpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
        first: Final = client.responses.create(model=model, input="turn one")
        _register_in_group(scenario, second_wire, model, order=1)
        counted: Final = client.responses.input_tokens.count(
            model=model, input="turn two", previous_response_id=first.id
        )
        assert counted.input_tokens == 37, counted.model_dump_json()
        assert drain_contract_requests(second_wire) == (), "the count left the previous response's deployment"
        requests: Final = drain_contract_requests(first_wire)
        assert [(request.target, _JSON_OBJECT.validate_json(request.body)) for request in requests] == [
            ("/v1/responses", {"model": _MODEL, "input": "turn one"}),
            ("/v1/responses/input_tokens", {"model": _MODEL, "input": "turn two", "previous_response_id": raw_id}),
        ], [request.body for request in requests]


@pytest.mark.parametrize("prefix", _ALIASES)
def test_codex_responses_request_preserves_all_supported_fields(gateway: Gateway, prefix: str) -> None:
    response_id: Final = f"resp_codex_{uuid.uuid4().hex}"
    input_items: Final = [
        {
            "type": "message",
            "role": "user",
            "content": [{"type": "input_text", "text": "Inspect the requested change."}],
        },
        {
            "type": "reasoning",
            "id": f"rs_{uuid.uuid4().hex}",
            "summary": [{"type": "summary_text", "text": "Plan the inspection"}],
            "encrypted_content": f"gAAAA-codex-encrypted-{uuid.uuid4().hex}",
        },
        {
            "type": "function_call",
            "call_id": "call_lookup",
            "name": "lookup",
            "arguments": '{"query": "diff"}',
        },
        {
            "type": "function_call_output",
            "call_id": "call_lookup",
            "output": '{"rows": 1}',
        },
        {
            "type": "custom_tool_call",
            "call_id": "call_patch",
            "name": "ApplyPatch",
            "input": "*** Begin Patch\n*** End Patch",
        },
        {
            "type": "custom_tool_call_output",
            "call_id": "call_patch",
            "output": "Done",
        },
    ]
    function_tool: Final = {
        "type": "function",
        "name": "lookup",
        "description": "Look up a record",
        "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
        "strict": True,
    }
    custom_tool: Final = {
        "type": "custom",
        "name": "ApplyPatch",
        "description": "Apply a patch",
        "format": {"type": "grammar", "syntax": "lark", "definition": 'start: "ok"'},
    }
    web_search_tool: Final = {"type": "web_search"}
    expected_body: Final = {
        "model": _MODEL,
        "input": input_items,
        "stream": True,
        "include": ["reasoning.encrypted_content"],
        "store": False,
        "prompt_cache_key": "cache-codex-contract",
        "reasoning": {"effort": "high", "summary": "auto"},
        "text": {
            "verbosity": "low",
            "format": {
                "type": "json_schema",
                "name": "patch_summary",
                "strict": True,
                "schema": {
                    "type": "object",
                    "properties": {"summary": {"type": "string"}},
                    "required": ["summary"],
                    "additionalProperties": False,
                },
            },
        },
        "tools": [function_tool, custom_tool, web_search_tool],
        "tool_choice": "auto",
        "parallel_tool_calls": True,
        "instructions": "Return concise structured output.",
    }

    def respond(request: Request) -> Reply:
        if request.method == "GET" and request.target == "/v1/models":
            return model_discovery_reply(_MODEL)
        assert request.method == "POST"
        assert request.target == "/v1/responses"
        assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers)
        _JSON_OBJECT.validate_json(request.body)
        return Reply(content_type="text/event-stream", chunks=_stream(response_id))

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = _register(scenario, wire)
        client: Final = openai.OpenAI(
            base_url=f"{gateway.client.base_url}{prefix}",
            api_key=gateway.key,
            max_retries=0,
        )
        stream: Final = client.responses.create(
            model=model,
            input=input_items,
            stream=True,
            include=["reasoning.encrypted_content"],
            store=False,
            prompt_cache_key="cache-codex-contract",
            reasoning={"effort": "high", "summary": "auto"},
            text={
                "verbosity": "low",
                "format": {
                    "type": "json_schema",
                    "name": "patch_summary",
                    "strict": True,
                    "schema": {
                        "type": "object",
                        "properties": {"summary": {"type": "string"}},
                        "required": ["summary"],
                        "additionalProperties": False,
                    },
                },
            },
            tools=[function_tool, custom_tool, web_search_tool],
            tool_choice="auto",
            parallel_tool_calls=True,
            instructions="Return concise structured output.",
        )
        events: Final = list(stream)
        completed: Final = events[-1]
        assert isinstance(completed, ResponseCompletedEvent), events
        assert completed.response.status == "completed", completed.model_dump_json()
        assert completed.response.output == [], completed.model_dump_json()
        requests: Final = drain_contract_requests(wire)
        assert len(requests) == 1, requests
        assert _JSON_OBJECT.validate_json(requests[0].body) == expected_body, requests[0].body
        assert [request.target for request in requests] == ["/v1/responses"]


@pytest.mark.parametrize("prefix", _ALIASES)
@pytest.mark.parametrize("case", ("mixed", "many"))
def test_responses_metadata_reaches_the_provider_as_typed(
    gateway: Gateway,
    prefix: str,
    case: str,
) -> None:
    metadata: Final = (
        {
            "string_value": "keep this",
            "integer_value": 7,
            "boolean_value": True,
            "nested_value": {"key": "drop this"},
            "second_string": "keep this too",
        }
        if case == "mixed"
        else {f"key_{index:02d}": f"value_{index:02d}" for index in range(17)}
    )
    expected_metadata: Final = dict(
        tuple((key, value) for key, value in metadata.items() if isinstance(value, str))[:16]
    )

    def respond(request: Request) -> Reply:
        if request.method == "GET" and request.target == "/v1/models":
            return model_discovery_reply(_MODEL)
        assert (request.method, request.target) == ("POST", "/v1/responses"), request.target
        return Reply(body=_response(f"resp_metadata_{uuid.uuid4().hex}"))

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = _register(scenario, wire)
        response: Final = gateway.request(
            "POST",
            f"{prefix}/responses",
            {
                "model": model,
                "input": "metadata propagation",
                "metadata": metadata,
            },
        )
        assert response.status_code == 200, response.text
        requests: Final = drain_contract_requests(wire)
        assert len(requests) == 1, requests
        assert _JSON_OBJECT.validate_json(requests[0].body) == {
            "model": _MODEL,
            "input": "metadata propagation",
            "metadata": expected_metadata,
        }, requests[0].body


def test_conversation_is_forwarded_as_a_separate_responses_parameter(gateway: Gateway) -> None:
    pytest.skip("BUG: POST /v1/responses drops the conversation param; upstream receives the body without it")
    conversation: Final = {"id": f"conv_{uuid.uuid4().hex}"}

    def respond(request: Request) -> Reply:
        if request.method == "GET" and request.target == "/v1/models":
            return model_discovery_reply(_MODEL)
        return Reply(body=_response(f"resp_conversation_{uuid.uuid4().hex}"))

    with wire_server(respond) as wire, gateway.scenario() as scenario:
        model: Final = _register(scenario, wire)
        client: Final = openai.OpenAI(
            base_url=f"{gateway.client.base_url}/v1",
            api_key=gateway.key,
            max_retries=0,
        )
        response: Final = client.responses.create(
            model=model,
            input="continue this conversation",
            conversation=conversation,
        )
        assert response.object == "response", response.model_dump_json()
        requests: Final = drain_contract_requests(wire)
        assert [(request.method, request.target) for request in requests] == [("POST", "/v1/responses")], requests
        assert _JSON_OBJECT.validate_json(requests[0].body) == {
            "model": _MODEL,
            "input": "continue this conversation",
            "conversation": conversation,
        }, requests[0].body