The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
from __future__ import annotations
import contextlib
import datetime
import json
import socket
import socketserver
import ssl
import threading
import uuid
from collections.abc import Generator
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from queue import SimpleQueue
from typing import Final
import httpx
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID
from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value
from integration._support.database import read_rows
from integration._support.process import owned_proxy
from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario
from integration._support.wire import Reply, Request, Wire, wire_server
from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse, TextResponse
from pydantic import JsonValue
from litellm.litellm_core_utils.cloud_storage_security import BEDROCK_MANAGED_S3_OUTPUT_PREFIX
OUTPUT_BYTES: Final = 4096
ERROR_BYTES: Final = 512
BEDROCK_MODEL: Final = "bedrock/anthropic.claude-3-haiku-20240307-v1:0"
BEDROCK_MODEL_ID: Final = "anthropic.claude-3-haiku-20240307-v1:0"
BEDROCK_REGION: Final = "us-east-1"
BEDROCK_AUTHORITY: Final = f"bedrock.{BEDROCK_REGION}.amazonaws.com:443"
BEDROCK_BUCKET: Final = "integration-batch-listing-bucket"
BEDROCK_ROLE_ARN: Final = "arn:aws:iam::123456789012:role/integration-batch-role"
BEDROCK_JOB_ARN_PREFIX: Final = f"arn:aws:bedrock:{BEDROCK_REGION}:123456789012:model-invocation-job/"
BEDROCK_LAST_MODIFIED: Final = "Thu, 02 Oct 2025 12:00:00 GMT"
BEDROCK_OUTPUT_CONTENT: Final = b'{"recordId":"req-1","modelOutput":{}}\n'
_BATCH_PROCESSED_SQL: Final = 'SELECT batch_processed FROM "LiteLLM_ManagedObjectTable" WHERE unified_object_id=%s'
@dataclass(frozen=True, slots=True)
class _OpenAIBatch:
model: str
owner_key: str
unrelated_key: str | None
batch_id: str
input_file_id: str
scenario: ScenarioHandle
def _output_content(model: str) -> str:
return (
json.dumps(
{
"id": "batch_req_$REQUEST_ID",
"custom_id": "req-1",
"response": {
"status_code": 200,
"request_id": "$REQUEST_ID",
"body": {
"id": "chatcmpl-$REQUEST_ID",
"object": "chat.completion",
"model": model,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
},
},
"error": None,
},
separators=(",", ":"),
)
+ "\n"
)
def _error_content() -> str:
return (
json.dumps(
{
"id": "batch_req_$REQUEST_ID",
"custom_id": "req-2",
"response": {"status_code": 400, "body": {"error": {"message": "rejected"}}},
"error": {"code": "bad_request", "message": "rejected"},
},
separators=(",", ":"),
)
+ "\n"
)
def _batch_routes(
model: str,
*,
output_file: bool = True,
metadata_fails: bool = False,
) -> RoutedResponse:
completed: Final[dict[str, JsonValue]] = {
"id": "batch-$REQUEST_ID",
"object": "batch",
"endpoint": "/v1/chat/completions",
"errors": None,
"input_file_id": "file-in-$REQUEST_ID",
"completion_window": "24h",
"status": "completed",
"output_file_id": "file-out-$REQUEST_ID" if output_file else None,
"error_file_id": "file-err-$REQUEST_ID",
"created_at": 1,
"in_progress_at": 1,
"completed_at": 1,
"expires_at": 1,
"request_counts": {"total": 1 if not output_file else 2, "completed": 1 if output_file else 0, "failed": 1},
"metadata": None,
}
output_metadata: Final = JsonResponse(
content_type="application/json",
body=(
{"error": "provider metadata unavailable"}
if metadata_fails
else {
"id": "file-out-$REQUEST_ID",
"object": "file",
"purpose": "batch_output",
"bytes": OUTPUT_BYTES,
"created_at": 1,
"filename": "output.jsonl",
"status": "processed",
}
),
status=500 if metadata_fails else 200,
)
return RoutedResponse(
content_type="application/x-routed",
routes={
"POST /files": JsonResponse(
content_type="application/json",
body={
"id": "file-in-$REQUEST_ID",
"object": "file",
"purpose": "batch",
"bytes": 100,
"created_at": 1,
"filename": "input.jsonl",
"status": "processed",
},
),
"POST /batches": JsonResponse(
content_type="application/json",
body={**completed, "status": "validating", "output_file_id": None, "error_file_id": None},
),
"GET /batches/batch-$REQUEST_ID": JsonResponse(content_type="application/json", body=completed),
"GET /files/file-out-$REQUEST_ID": output_metadata,
"GET /files/file-err-$REQUEST_ID": JsonResponse(
content_type="application/json",
body={
"id": "file-err-$REQUEST_ID",
"object": "file",
"purpose": "batch_output",
"bytes": ERROR_BYTES,
"created_at": 1,
"filename": "errors.jsonl",
"status": "processed",
},
),
"GET /files/file-out-$REQUEST_ID/content": TextResponse(
content_type="application/jsonl",
body=_output_content(model),
),
"GET /files/file-err-$REQUEST_ID/content": TextResponse(
content_type="application/jsonl",
body=_error_content(),
),
},
)
def _create_batch(
scenario: Scenario,
routes: RoutedResponse,
*,
unrelated_user: bool = False,
) -> _OpenAIBatch:
handle: Final = register_scenario(f"batch-output-listing-{uuid.uuid4().hex}", routes)
scenario.cleanups.callback(delete_scenario, handle)
model: Final = scenario.model(api_base=handle.api_base())
owner_id: Final = scenario.user(user_role="internal_user")
owner_key: Final = scenario.key(user_id=owner_id, models=[model])
unrelated_key: Final = (
scenario.key(user_id=scenario.user(user_role="internal_user"), models=[model]) if unrelated_user else None
)
input_content: Final = json.dumps(
{
"custom_id": "req-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": 8},
}
).encode()
uploaded: Final = scenario.gateway.request_multipart(
"/v1/files",
{"purpose": "batch", "target_model_names": model},
{"file": ("input.jsonl", input_content, "application/jsonl")},
key=owner_key,
)
assert uploaded.status_code == 200, uploaded.text
input_file_id: Final = string_value(JSON_OBJECT.validate_json(uploaded.content)["id"])
created: Final = scenario.gateway.request(
"POST",
"/v1/batches",
{
"input_file_id": input_file_id,
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"model": model,
},
key=owner_key,
)
assert created.status_code == 200, created.text
batch_id: Final = string_value(JSON_OBJECT.validate_json(created.content)["id"])
return _OpenAIBatch(model, owner_key, unrelated_key, batch_id, input_file_id, handle)
def _retrieve_batch(gateway: Gateway, batch: _OpenAIBatch) -> dict[str, JsonValue]:
response: Final = gateway.request("GET", f"/v1/batches/{batch.batch_id}", key=batch.owner_key)
assert response.status_code == 200, response.text
return JSON_OBJECT.validate_json(response.content)
def _list_files(
gateway: Gateway,
key: str,
*,
purpose: str | None = None,
) -> tuple[dict[str, JsonValue], ...]:
response: Final = gateway.request(
"GET",
"/v1/files",
key=key,
params={"purpose": purpose} if purpose is not None else None,
)
assert response.status_code == 200, response.text
values: Final = JSON_OBJECT.validate_json(response.content)["data"]
assert isinstance(values, list), response.text
return tuple(object_value(value) for value in values)
def _metadata_hit_count(gateway: Gateway, scenario: ScenarioHandle) -> int:
response: Final = httpx.get(
f"{gateway.upstream_url}/__observations",
timeout=15,
trust_env=False,
)
assert response.status_code == 200, response.text
requests: Final = JSON_OBJECT.validate_json(response.content)["requests"]
assert isinstance(requests, list), response.text
return sum(1 for request in requests if _is_metadata_hit(request, scenario.scenario_id))
def _is_metadata_hit(request: JsonValue, scenario_id: str) -> bool:
if not isinstance(request, dict):
return False
path: Final = request.get("path")
return request.get("method") == "GET" and isinstance(path, str) and path.endswith(f"/files/file-out-{scenario_id}")
def test_output_file_lists_after_owner_retrieves_completed_batch(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
batch: Final = _create_batch(scenario, _batch_routes(model="gpt-4o-mini"))
retrieved: Final = _retrieve_batch(gateway, batch)
assert retrieved["status"] == "completed", retrieved
output_id: Final = string_value(retrieved["output_file_id"])
error_id: Final = string_value(retrieved["error_file_id"])
files: Final = _list_files(gateway, batch.owner_key)
output: Final = next((file for file in files if file.get("id") == output_id), None)
assert output is not None, f"Completed batch output {output_id} is absent from GET /v1/files"
assert (output["purpose"], output["bytes"]) == ("batch_output", OUTPUT_BYTES), output
output_files: Final = _list_files(gateway, batch.owner_key, purpose="batch_output")
assert {string_value(file["id"]) for file in output_files} == {output_id, error_id}, output_files
assert all(file["purpose"] == "batch_output" for file in output_files), output_files
input_files: Final = _list_files(gateway, batch.owner_key, purpose="batch")
assert tuple(string_value(file["id"]) for file in input_files) == (batch.input_file_id,), input_files
def test_poller_registers_listable_output_files_without_a_batch_retrieve(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
batch: Final = _create_batch(
scenario,
_batch_routes(model="gpt-4o-mini"),
unrelated_user=True,
)
assert batch.unrelated_key is not None
output_files: Final = eventually(
lambda: _list_files(gateway, batch.owner_key, purpose="batch_output"),
lambda values: len(values) == 2,
seconds=60,
)
assert {file["purpose"] for file in output_files} == {"batch_output"}, output_files
unrelated_files: Final = _list_files(gateway, batch.unrelated_key, purpose="batch_output")
assert unrelated_files == (), unrelated_files
retrieved: Final = _retrieve_batch(gateway, batch)
output_id: Final = string_value(retrieved["output_file_id"])
error_id: Final = string_value(retrieved["error_file_id"])
assert {string_value(file["id"]) for file in output_files} == {output_id, error_id}, output_files
def test_error_file_only_batch_still_lists_its_error_file(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
batch: Final = _create_batch(
scenario,
_batch_routes(model="gpt-4o-mini", output_file=False),
)
error_files: Final = eventually(
lambda: _list_files(gateway, batch.owner_key, purpose="batch_output"),
lambda values: len(values) == 1,
seconds=60,
)
retrieved: Final = _retrieve_batch(gateway, batch)
assert retrieved["output_file_id"] is None, retrieved
error_id: Final = string_value(retrieved["error_file_id"])
assert tuple(string_value(file["id"]) for file in error_files) == (error_id,), error_files
assert error_files[0]["purpose"] == "batch_output", error_files
def test_output_file_lists_with_basic_details_when_provider_file_lookup_fails(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
batch: Final = _create_batch(
scenario,
_batch_routes(model="gpt-4o-mini", metadata_fails=True),
)
retrieved: Final = _retrieve_batch(gateway, batch)
output_id: Final = string_value(retrieved["output_file_id"])
output: Final = next(
(file for file in _list_files(gateway, batch.owner_key) if file.get("id") == output_id),
None,
)
assert output is not None, f"Completed batch output {output_id} is absent after provider metadata failure"
assert (output["purpose"], output["filename"]) == ("batch_output", f"file-out-{batch.scenario.scenario_id}"), (
output
)
assert "litellm_details_fallback" not in output, output
def test_fallback_output_file_details_refresh_once_provider_recovers(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
batch: Final = _create_batch(
scenario,
_batch_routes(model="gpt-4o-mini", metadata_fails=True),
unrelated_user=True,
)
retrieved: Final = _retrieve_batch(gateway, batch)
output_id: Final = string_value(retrieved["output_file_id"])
listed_files: Final = _list_files(gateway, batch.owner_key, purpose="batch_output")
basic_output: Final = next((file for file in listed_files if file.get("id") == output_id), None)
assert basic_output is not None, f"Completed batch output {output_id} is absent after provider metadata failure"
assert (basic_output["purpose"], basic_output["filename"]) == (
"batch_output",
f"file-out-{batch.scenario.scenario_id}",
), basic_output
assert basic_output["bytes"] != OUTPUT_BYTES, basic_output
assert "litellm_details_fallback" not in basic_output, basic_output
processed_batch_rows: Final = eventually(
lambda: read_rows(_BATCH_PROCESSED_SQL, (string_value(retrieved["id"]),)),
lambda rows: len(rows) == 1 and rows[0].get("batch_processed") is True,
seconds=60,
)
assert processed_batch_rows[0]["batch_processed"] is True, processed_batch_rows
metadata_hits_before_recovery: Final = _metadata_hit_count(gateway, batch.scenario)
assert metadata_hits_before_recovery >= 1, "The provider metadata route was not called before recovery"
register_scenario(
batch.scenario.scenario_id,
_batch_routes(model=batch.model),
control_url=batch.scenario.control_url,
)
details_response: Final = gateway.request("GET", f"/v1/files/{output_id}", key=batch.owner_key)
assert details_response.status_code == 200, details_response.text
details: Final = JSON_OBJECT.validate_json(details_response.content)
assert details["bytes"] == OUTPUT_BYTES, details
assert (details["filename"], details["purpose"]) == ("output.jsonl", "batch_output"), details
assert "litellm_details_fallback" not in details, details
assert _metadata_hit_count(gateway, batch.scenario) - metadata_hits_before_recovery == 1
refreshed_files: Final = _list_files(gateway, batch.owner_key, purpose="batch_output")
refreshed_output: Final = next((file for file in refreshed_files if file.get("id") == output_id), None)
assert refreshed_output is not None, f"Refreshed batch output {output_id} is absent from the list"
assert (refreshed_output["bytes"], refreshed_output["filename"]) == (OUTPUT_BYTES, "output.jsonl"), (
refreshed_output
)
assert "litellm_details_fallback" not in refreshed_output, refreshed_output
assert batch.unrelated_key is not None
unrelated_details: Final = gateway.request("GET", f"/v1/files/{output_id}", key=batch.unrelated_key)
assert unrelated_details.status_code == 403, unrelated_details.text
def _tls_context(directory: Path) -> ssl.SSLContext:
key: Final = ec.generate_private_key(ec.SECP256R1())
now: Final = datetime.datetime.now(datetime.timezone.utc)
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, BEDROCK_AUTHORITY.split(":")[0])])
certificate: Final = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(days=1))
.not_valid_after(now + datetime.timedelta(days=1))
.sign(key, hashes.SHA256())
)
certificate_file: Final = directory / "bedrock.pem"
key_file: Final = directory / "bedrock.key"
certificate_file.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
key_file.write_bytes(
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
)
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(certificate_file, key_file)
return context
def _pipe(source: socket.socket, sink: socket.socket) -> None:
with contextlib.suppress(OSError):
for chunk in iter(lambda: source.recv(65536), b""):
sink.sendall(chunk)
with contextlib.suppress(OSError):
sink.shutdown(socket.SHUT_WR)
@dataclass(frozen=True, slots=True)
class _ConnectProxy:
url: str
authorities: SimpleQueue[str]
@contextmanager
def _bedrock_tunnel(destination: Wire) -> Generator[_ConnectProxy, None, None]:
authorities: Final[SimpleQueue[str]] = SimpleQueue()
destination_port: Final = int(destination.url.rsplit(":", 1)[1])
class Tunnel(socketserver.StreamRequestHandler):
rbufsize = 0
request: socket.socket
def handle(self) -> None:
authority: Final = self.rfile.readline().decode().split()[1]
while self.rfile.readline() not in (b"\r\n", b""):
pass
authorities.put(authority)
if authority != BEDROCK_AUTHORITY:
self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n")
return
self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
self.request.settimeout(10)
with socket.create_connection(("127.0.0.1", destination_port), timeout=10) as upstream:
outbound: Final = threading.Thread(target=_pipe, args=(self.request, upstream))
outbound.start()
_pipe(upstream, self.request)
outbound.join(timeout=12)
with socketserver.ThreadingTCPServer(("127.0.0.1", 0), Tunnel) as server:
thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05})
thread.start()
try:
yield _ConnectProxy(f"http://127.0.0.1:{server.server_address[1]}", authorities)
finally:
server.shutdown()
thread.join(timeout=6)
@dataclass(frozen=True, slots=True)
class _BedrockControlPlane:
job_locations: SimpleQueue[tuple[str, str, str]]
job_id: str
job_arn: str
def __call__(self, request: Request) -> Reply:
if request.method == "POST" and request.target == "/model-invocation-job":
body: Final = JSON_OBJECT.validate_json(request.body)
input_config: Final = object_value(object_value(body["inputDataConfig"])["s3InputDataConfig"])
output_config: Final = object_value(object_value(body["outputDataConfig"])["s3OutputDataConfig"])
job_name: Final = string_value(body["jobName"])
self.job_locations.put(
(string_value(input_config["s3Uri"]), string_value(output_config["s3Uri"]), job_name)
)
return Reply(body=json.dumps({"jobArn": self.job_arn}).encode())
if request.method == "GET" and request.target.endswith(self.job_id):
input_uri, output_uri, retrieved_job_name = self.job_locations.get()
self.job_locations.put((input_uri, output_uri, retrieved_job_name))
return Reply(
body=json.dumps(
{
"jobArn": self.job_arn,
"jobName": retrieved_job_name,
"modelId": BEDROCK_MODEL_ID,
"status": "Completed",
"submitTime": 1700000000,
"lastModifiedTime": 1700000001,
"endTime": 1700000002,
"inputDataConfig": {"s3InputDataConfig": {"s3Uri": input_uri}},
"outputDataConfig": {"s3OutputDataConfig": {"s3Uri": output_uri}},
"totalRecordCount": 1,
"successRecordCount": 1,
"errorRecordCount": 0,
}
).encode()
)
return Reply(status=404, body=b'{"message":"not scripted"}')
def _bedrock_s3_peer(request: Request) -> Reply:
if request.method == "PUT" and request.target.startswith(f"/{BEDROCK_BUCKET}/"):
return Reply(body=b"")
if request.method == "GET" and request.target.startswith(f"/{BEDROCK_BUCKET}/{BEDROCK_MANAGED_S3_OUTPUT_PREFIX}"):
if request.headers.get("range") == "bytes=0-0":
return Reply(
status=206,
body=BEDROCK_OUTPUT_CONTENT[:1],
headers={
"Content-Range": f"bytes 0-0/{len(BEDROCK_OUTPUT_CONTENT)}",
"Last-Modified": BEDROCK_LAST_MODIFIED,
},
)
return Reply(status=200, body=BEDROCK_OUTPUT_CONTENT, headers={"Last-Modified": BEDROCK_LAST_MODIFIED})
return Reply(status=404, body=b'{"message":"not scripted"}')
def test_bedrock_batch_output_lists_and_retrieves_details(gateway: Gateway, tmp_path: Path) -> None:
job_id: Final = f"integration-batch-listing-{uuid.uuid4().hex}"
job_arn: Final = BEDROCK_JOB_ARN_PREFIX + job_id
environment: Final = {
"AWS_CA_BUNDLE": str(tmp_path / "bedrock.pem"),
"AWS_EC2_METADATA_DISABLED": "true",
"SSL_VERIFY": "False",
}
with (
wire_server(_bedrock_s3_peer) as s3,
wire_server(_BedrockControlPlane(SimpleQueue(), job_id, job_arn), tls=_tls_context(tmp_path)) as bedrock,
_bedrock_tunnel(bedrock) as tunnel,
owned_proxy(gateway, tmp_path, {**environment, "HTTPS_PROXY": tunnel.url}) as candidate,
candidate.scenario() as scenario,
):
model: Final = scenario.model(
model=BEDROCK_MODEL,
api_key=None,
api_base=None,
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
aws_region_name=BEDROCK_REGION,
s3_bucket_name=BEDROCK_BUCKET,
s3_endpoint_url=s3.url,
aws_batch_role_arn=BEDROCK_ROLE_ARN,
)
owner_id: Final = scenario.user(user_role="internal_user")
owner_key: Final = scenario.key(user_id=owner_id, models=[model])
input_line: Final = {
"custom_id": "req-1",
"method": "POST",
"url": "/v1/chat/completions",
"body": {"model": model, "messages": [{"role": "user", "content": "ping"}], "max_tokens": 8},
}
uploaded: Final = candidate.request_multipart(
"/v1/files",
{"purpose": "batch", "target_model_names": model},
{"file": ("input.jsonl", (json.dumps(input_line) + "\n").encode(), "application/jsonl")},
key=owner_key,
)
assert uploaded.status_code == 200, uploaded.text
created: Final = candidate.request(
"POST",
"/v1/batches",
{
"input_file_id": string_value(JSON_OBJECT.validate_json(uploaded.content)["id"]),
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
},
key=owner_key,
)
assert created.status_code == 200, created.text
batch_id: Final = string_value(JSON_OBJECT.validate_json(created.content)["id"])
retrieved: Final = candidate.request("GET", f"/v1/batches/{batch_id}", key=owner_key)
assert retrieved.status_code == 200, retrieved.text
batch_object: Final = JSON_OBJECT.validate_json(retrieved.content)
assert batch_object["status"] == "completed", retrieved.text
output_id: Final = string_value(batch_object["output_file_id"])
output_files: Final = eventually(
lambda: _list_files(candidate, owner_key, purpose="batch_output"),
lambda values: any(file.get("id") == output_id for file in values),
seconds=30,
)
listed_output: Final = next(file for file in output_files if file.get("id") == output_id)
details: Final = candidate.request("GET", f"/v1/files/{output_id}", key=owner_key)
assert details.status_code == 200, details.text
detail_object: Final = JSON_OBJECT.validate_json(details.content)
assert (detail_object["id"], detail_object["bytes"], detail_object["purpose"]) == (
output_id,
len(BEDROCK_OUTPUT_CONTENT),
"batch_output",
), detail_object
assert listed_output["id"] == output_id, listed_output
s3_requests: Final = s3.drain()
uploads: Final = tuple(request for request in s3_requests if request.method == "PUT")
assert len(uploads) == 1, f"Expected one input S3 upload, saw {[request.target for request in uploads]}"
ranged_metadata: Final = tuple(
request
for request in s3_requests
if request.method == "GET" and request.headers.get("range") == "bytes=0-0"
)
assert ranged_metadata, "Bedrock file metadata retrieval did not issue a ranged S3 GET"
assert all(
request.headers.get("authorization", "").startswith("AWS4-HMAC-SHA256 ") for request in ranged_metadata
), ranged_metadata
authorities: Final = tuple(tunnel.authorities.get_nowait() for _ in range(tunnel.authorities.qsize()))
assert BEDROCK_AUTHORITY in authorities, authorities
bedrock_requests: Final = bedrock.drain()
assert any(request.method == "POST" for request in bedrock_requests), bedrock_requests
assert any(request.method == "GET" for request in bedrock_requests), bedrock_requests
def test_repeated_batch_retrieve_does_not_refetch_saved_file_details(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
batch: Final = _create_batch(scenario, _batch_routes(model="gpt-4o-mini"))
first: Final = _retrieve_batch(gateway, batch)
output_id: Final = string_value(first["output_file_id"])
output_before: Final = next(
(file for file in _list_files(gateway, batch.owner_key) if file.get("id") == output_id),
None,
)
assert output_before is not None, f"Completed batch output {output_id} is absent from GET /v1/files"
hits_before: Final = _metadata_hit_count(gateway, batch.scenario)
assert hits_before >= 1, "The provider file metadata route was not called for the batch output"
second: Final = _retrieve_batch(gateway, batch)
third: Final = _retrieve_batch(gateway, batch)
assert second["status"] == third["status"] == "completed", (second, third)
output_after: Final = next(
(file for file in _list_files(gateway, batch.owner_key) if file.get("id") == output_id),
None,
)
assert output_after == output_before, output_after
additional_hits: Final = _metadata_hit_count(gateway, batch.scenario) - hits_before
assert additional_hits == 0, f"Repeated batch retrieval fetched metadata {additional_hits} more times"