Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.7 KB
Raw Blame
from typing import Any
from unittest.mock import AsyncMock, patch

import pytest
from fastapi import HTTPException

from llm_gateway.baseten import (
    BASETEN_MODELS,
    _inject_baseten_params,
    ensure_baseten_configured,
    make_baseten_responses_call,
)
from llm_gateway.config import Settings
from llm_gateway.rate_limiting.cost_refresh import ALIAS_METRIC_LABELS, COST_ALIASES, normalize_metric_labels
from llm_gateway.rate_limiting.model_cost_overrides import MODEL_COST_OVERRIDES

GLM_MODEL = "@cf/zai-org/glm-5.2"
BASETEN_MODEL = "openai/zai-org/GLM-5.2"
DEEPSEEK_MODEL = "deepseek-ai/deepseek-v4-flash-0731"
BASETEN_DEEPSEEK_LITELLM_MODEL = "openai/deepseek-ai/DeepSeek-V4-Flash-0731"
GLM53_MODEL = "zai-org/glm-5.3"
BASETEN_GLM53_LITELLM_MODEL = "openai/zai-org/GLM-5.3"
GLM53_FLASH_MODEL = "zai-org/glm-5.3-flash"
BASETEN_GLM53_FLASH_LITELLM_MODEL = "openai/zai-org/GLM-5.3-Flash"


@pytest.mark.parametrize(
    ("public_model", "litellm_model", "metric_model", "input_cost", "output_cost", "cache_read_cost"),
    [
        (GLM_MODEL, BASETEN_MODEL, "baseten/zai-org/glm-5.2", 1.4e-06, 4.4e-06, 1.4e-07),
        (DEEPSEEK_MODEL, BASETEN_DEEPSEEK_LITELLM_MODEL, f"baseten/{DEEPSEEK_MODEL}", 1.3e-07, 2.6e-07, 2.8e-08),
        (GLM53_MODEL, BASETEN_GLM53_LITELLM_MODEL, f"baseten/{GLM53_MODEL}", 1.4e-06, 4.4e-06, 1.4e-07),
        (GLM53_FLASH_MODEL, BASETEN_GLM53_FLASH_LITELLM_MODEL, f"baseten/{GLM53_FLASH_MODEL}", 1.5e-07, 5e-07, 3e-08),
    ],
)
def test_inject_baseten_params_maps_model_and_pins_api_key(
    public_model: str,
    litellm_model: str,
    metric_model: str,
    input_cost: float,
    output_cost: float,
    cache_read_cost: float,
) -> None:
    kwargs: dict[str, Any] = {
        "model": public_model,
        "headers": {"Host": "attacker.example"},
        "extra_headers": {"Authorization": "Bearer attacker", "X-Other": "no"},
    }

    _inject_baseten_params(kwargs, "https://inference.baseten.co/v1", "test-key")

    assert kwargs["model"] == litellm_model
    assert kwargs["api_base"] == "https://inference.baseten.co/v1"
    assert "headers" not in kwargs
    assert kwargs["extra_headers"] == {"Authorization": "Bearer test-key"}
    assert kwargs["drop_params"] is True
    assert COST_ALIASES[litellm_model] == (metric_model, "openai")
    assert MODEL_COST_OVERRIDES[metric_model]["input_cost_per_token"] == input_cost
    assert MODEL_COST_OVERRIDES[metric_model]["cache_read_input_token_cost"] == cache_read_cost
    assert MODEL_COST_OVERRIDES[metric_model]["output_cost_per_token"] == output_cost
    assert normalize_metric_labels(litellm_model, "openai") == ("baseten", public_model)


@pytest.mark.parametrize("public_model", sorted(BASETEN_MODELS))
def test_every_baseten_model_is_priced_and_labeled(public_model: str) -> None:
    # Derived from BASETEN_MODELS, so a model added there without a price fails here rather than
    # billing nothing: the usage reporter drops any generation whose $ai_total_cost_usd is 0.
    litellm_key = f"openai/{BASETEN_MODELS[public_model]}"
    assert litellm_key in COST_ALIASES or litellm_key in MODEL_COST_OVERRIDES
    provider, metric_model = ALIAS_METRIC_LABELS[litellm_key]
    assert provider == "baseten"
    assert metric_model == public_model


def test_inject_baseten_params_forces_streaming_usage() -> None:
    kwargs: dict[str, Any] = {
        "model": GLM_MODEL,
        "stream": True,
        "stream_options": {"include_usage": False},
    }

    _inject_baseten_params(kwargs, "https://inference.baseten.co/v1", "test-key")

    assert kwargs["stream_options"] == {"include_usage": True, "continuous_usage_stats": True}


@pytest.mark.parametrize(
    ("param", "requested", "sent"),
    [
        ("max_tokens", 1, 2),
        ("max_tokens", 4096, 4096),
        ("max_completion_tokens", 1, 2),
        ("max_output_tokens", 1, 2),
    ],
)
def test_inject_baseten_params_raises_token_limit_to_baseten_minimum(param: str, requested: int, sent: int) -> None:
    kwargs: dict[str, Any] = {"model": GLM_MODEL, param: requested}

    _inject_baseten_params(kwargs, "https://inference.baseten.co/v1", "test-key")

    assert kwargs[param] == sent


def test_ensure_baseten_configured_requires_api_key() -> None:
    with pytest.raises(HTTPException) as exc_info:
        ensure_baseten_configured(Settings())

    assert exc_info.value.status_code == 503


async def test_make_baseten_responses_call_uses_chat_completions_bridge() -> None:
    llm_call = make_baseten_responses_call("https://inference.baseten.co/v1", "test-key")

    with patch("llm_gateway.baseten.litellm.aresponses", new=AsyncMock(return_value="ok")) as mock_aresponses:
        await llm_call(model=GLM_MODEL, input="hi", use_chat_completions_api=False)

    assert mock_aresponses.call_args.kwargs["use_chat_completions_api"] is True