Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

4.4 KB
Raw Blame
from __future__ import annotations

import json
from typing import Final
from urllib.parse import unquote

import httpx
from integration._support.client import Gateway, Scenario
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue

BACKEND: Final = "gemini-embedding-2-preview"
PROJECT: Final = "scripted-project"
LOCATION: Final = "us-central1"
BUCKET: Final = "scripted-bucket"
UPLOAD_PREFIX: Final = f"/upload/storage/v1/b/{BUCKET}/o?uploadType=media&name="
GCS_URI: Final = "gs://scripted-bucket/clips/animals.mp4"
BLOCK: Final[dict[str, JsonValue]] = {
    "type": "file",
    "file": {"file_id": GCS_URI, "video_metadata": {"fps": 1.0, "start_offset": "0s", "end_offset": "3s"}},
}
GCS_PART: Final[dict[str, JsonValue]] = {
    "file_data": {"mime_type": "video/mp4", "file_uri": GCS_URI},
    "video_metadata": {"fps": 1.0, "startOffset": "0s", "endOffset": "3s"},
}
TEXT_PART: Final[dict[str, JsonValue]] = {"text": "a red bus"}


def row(custom_id: str, model: str, elements: JsonValue) -> dict[str, JsonValue]:
    return {
        "custom_id": custom_id,
        "method": "POST",
        "url": "/v1/embeddings",
        "body": {"model": model, "input": elements},
    }


def stored_row(key: str, parts: list[JsonValue]) -> dict[str, JsonValue]:
    return {"key": key, "request": {"content": {"parts": parts}}}


def gcs_peer(request: Request) -> Reply:
    assert request.method == "POST", request.method
    assert request.target.startswith(UPLOAD_PREFIX), request.target
    assert request.headers["authorization"] == "Bearer scripted-token"
    name: Final = unquote(request.target.removeprefix(UPLOAD_PREFIX))
    stored: Final = {
        "kind": "storage#object",
        "id": f"{BUCKET}/{name}/1759950000000000",
        "name": name,
        "bucket": BUCKET,
        "size": str(len(request.body)),
        "timeCreated": "2026-10-08T19:00:00.000Z",
        "contentType": "application/octet-stream",
    }
    return Reply(body=json.dumps(stored).encode())


def vertex_batch_model(gateway: Gateway, scenario: Scenario, url: str) -> str:
    return scenario.model(
        model=f"vertex_ai/{BACKEND}",
        api_key=None,
        api_base=url,
        vertex_project=PROJECT,
        vertex_location=LOCATION,
        vertex_credentials=service_account_json(PROJECT, gateway.upstream_url),
        gcs_bucket_name=BUCKET,
    )


def upload(gateway: Gateway, model: str, rows: tuple[dict[str, JsonValue], ...]) -> httpx.Response:
    jsonl: Final = "\n".join(json.dumps(line) for line in rows) + "\n"
    return gateway.request_multipart(
        "/v1/files",
        {"purpose": "batch", "target_model_names": model},
        {"file": ("in.jsonl", jsonl.encode(), "application/jsonl")},
    )


def test_block_rows_upload_as_embed_content_requests(gateway: Gateway) -> None:
    with wire_server(gcs_peer) as wire, gateway.scenario() as scenario:
        model: Final = vertex_batch_model(gateway, scenario, wire.url)
        rows: Final = (
            row("mixed", model, [BLOCK, "a red bus"]),
            row("bare", model, "a red bus"),
            row("nested", model, [[BLOCK, "a red bus"]]),
        )
        response: Final = upload(gateway, model, rows)
        assert response.status_code == 200, response.text
        assert response.json()["object"] == "file" and response.json()["purpose"] == "batch", response.text
        uploads: Final = wire.drain()
        assert len(uploads) == 1, [upload.target for upload in uploads]
        stored: Final = tuple(json.loads(line) for line in uploads[0].body.decode().splitlines() if line.strip())
        assert stored == (
            stored_row("mixed#0/2", [GCS_PART]),
            stored_row("mixed#1/2", [TEXT_PART]),
            stored_row("bare", [TEXT_PART]),
            stored_row("nested", [GCS_PART, TEXT_PART]),
        )


def test_bare_block_row_fails_the_upload_before_any_storage_write(gateway: Gateway) -> None:
    with wire_server(gcs_peer) as wire, gateway.scenario() as scenario:
        model: Final = vertex_batch_model(gateway, scenario, wire.url)
        response: Final = upload(gateway, model, (row("mixed", model, [BLOCK, "a red bus"]), row("bare", model, BLOCK)))
        assert response.status_code >= 400, response.text
        assert "got dict" in response.text, response.text
        assert wire.drain() == (), "a rejected batch file reached the bucket"