Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

2.9 KB
Raw Blame
from pathlib import Path
from typing import Final

from ...shared.reporting.models import SURFACES
from ...shared.reporting.strategy import (
    CaseDefinition,
    ModuleCaseSpec,
    NotImplementedCaseSpec,
    StrategyDefinition,
)
from .reporting import render_e2e_results
from .runner import run_e2e_cases

CASES: Final[tuple[CaseDefinition, ...]] = (
    CaseDefinition(
        "ocr",
        NotImplementedCaseSpec(reason="OCR has no Python implementation left to compare against."),
        surface="sdk",
    ),
    CaseDefinition(
        "messages",
        NotImplementedCaseSpec(
            reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
        ),
        surface="sdk",
    ),
    CaseDefinition(
        "responses",
        NotImplementedCaseSpec(
            reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
        ),
        surface="sdk",
    ),
    CaseDefinition(
        "count_tokens",
        NotImplementedCaseSpec(reason="No Rust count_tokens parity test is present yet."),
        surface="sdk",
    ),
    CaseDefinition(
        "chat_completions",
        NotImplementedCaseSpec(
            reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
        ),
        surface="sdk",
    ),
    CaseDefinition(
        "transcription",
        NotImplementedCaseSpec(
            reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
        ),
        surface="sdk",
    ),
    CaseDefinition(
        "ocr",
        NotImplementedCaseSpec(reason="No gateway end-to-end OCR parity case is registered."),
        surface="gateway",
    ),
    CaseDefinition(
        "messages",
        NotImplementedCaseSpec(reason="No gateway end-to-end Messages parity case is registered."),
        surface="gateway",
    ),
    CaseDefinition(
        "responses",
        NotImplementedCaseSpec(reason="No gateway end-to-end Responses parity case is registered."),
        surface="gateway",
    ),
    CaseDefinition(
        "count_tokens",
        NotImplementedCaseSpec(reason="No gateway end-to-end token-count parity case is registered."),
        surface="gateway",
    ),
    CaseDefinition(
        "chat_completions",
        NotImplementedCaseSpec(reason="No gateway end-to-end chat parity case is registered."),
        surface="gateway",
    ),
    CaseDefinition(
        "transcription",
        NotImplementedCaseSpec(reason="No gateway end-to-end transcription parity case is registered."),
        surface="gateway",
    ),
)

STRATEGY: Final = StrategyDefinition(
    id="e2e_parity",
    order=10,
    label="End-to-end parity",
    description="Compare observable Python and Rust behavior over generated and recorded inputs.",
    directory=Path(__file__).parent,
    runnable_spec=ModuleCaseSpec,
    cases=CASES,
    run=run_e2e_cases,
    render=render_e2e_results,
    surfaces=SURFACES,
)