Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch

import litellm
import pytest
from fastapi import HTTPException

from llm_gateway.config import Settings
from llm_gateway.modal import (
    MODAL_ALLOWED_MODELS,
    MODAL_MODELS,
    _inject_modal_params,
    ensure_modal_model_allowed,
    ensure_modal_model_configured,
    make_modal_anthropic_call,
    make_modal_responses_call,
    should_route_glm_to_modal,
)
from llm_gateway.modal_routing import send_modal_anthropic_messages
from llm_gateway.rate_limiting.cost_refresh import ALIAS_METRIC_LABELS, COST_ALIASES
from llm_gateway.rate_limiting.model_cost_overrides import MODEL_COST_OVERRIDES

GLM_MODEL = "@cf/zai-org/glm-5.2"
KIMI_MODEL = "moonshotai/kimi-k3"


def _modal_settings(**overrides: Any) -> Settings:
    base: dict[str, Any] = {
        "modal_api_base": "https://posthog--glm.us-east.modal.direct/v1",
        "modal_kimi_api_base": "https://posthog--kimi.us-west.modal.direct/v1",
        "modal_key": "wk-test",
        "modal_secret": "ws-test",
    }
    base.update(overrides)
    return Settings(**base)


def test_inject_modal_params_maps_model_and_pins_proxy_auth() -> None:
    # Caller-supplied header dicts (`headers` or `extra_headers`, any casing) must never reach the
    # authenticated upstream request — litellm merges both into the outbound headers, and a
    # forwarded Host header would exfiltrate the proxy-token pair.
    kwargs: dict = {
        "model": GLM_MODEL,
        "headers": {"Host": "attacker.example"},
        "extra_headers": {"MODAL-KEY": "attacker", "host": "attacker.example", "X-Other": "no"},
    }
    _inject_modal_params(kwargs, "https://modal.test/v1", "wk", "ws")

    assert kwargs["model"] == "openai/zai-org/GLM-5.2-FP8"
    assert kwargs["api_base"] == "https://modal.test/v1"
    # litellm's OpenAI client rejects a missing api_key even though Modal auth is header-based.
    assert kwargs["api_key"]
    assert "headers" not in kwargs
    assert kwargs["extra_headers"] == {"Modal-Key": "wk", "Modal-Secret": "ws"}
    # Load-bearing: if this fails, the litellm model key is unpriced/unlabeled and cost lookup
    # silently falls to default_fallback_cost_usd (and metrics lose the modal provider label).
    assert kwargs["model"] in COST_ALIASES
    assert kwargs["model"] in ALIAS_METRIC_LABELS


@pytest.mark.parametrize(
    ("initial", "expected"),
    [
        (
            {"stream": True, "stream_options": {"include_usage": False}},
            {"include_usage": True, "continuous_usage_stats": True},
        ),
        ({"stream": True}, {"include_usage": True, "continuous_usage_stats": True}),
        ({}, None),
    ],
)
def test_inject_modal_params_forces_streaming_usage(initial: dict[str, Any], expected: dict[str, bool] | None) -> None:
    kwargs: dict[str, Any] = {"model": KIMI_MODEL, **initial}
    _inject_modal_params(kwargs, "https://modal.test/v1", "wk", "ws")
    assert kwargs.get("stream_options") == expected
    assert kwargs.get("extra_body") == ({"stream_options": expected} if expected is not None else None)


@pytest.mark.parametrize(("initial", "expected"), [({}, True), ({"drop_params": False}, False)])
def test_inject_modal_params_drop_params(initial: dict, expected: bool) -> None:
    # drop_params must default on (the OpenAI-compatible surface 400s on Anthropic-only params)
    # while an explicit caller opt-out keeps its value.
    kwargs: dict = {"model": GLM_MODEL, **initial}
    _inject_modal_params(kwargs, "https://modal.test/v1", "wk", "ws")
    assert kwargs["drop_params"] is expected


def test_modal_model_map_served_names_all_priced_and_labeled() -> None:
    # An unpriced Modal model would bill the flat fallback cost; the label must keep the public id
    # so one model id slices across backends in dashboards.
    for public_id, config in MODAL_MODELS.items():
        litellm_key = f"openai/{config.served_model}"
        assert litellm_key in COST_ALIASES or litellm_key in MODEL_COST_OVERRIDES
        provider, metric_model = ALIAS_METRIC_LABELS[litellm_key]
        assert provider == "modal"
        assert metric_model == public_id


@pytest.mark.parametrize("model", [GLM_MODEL, KIMI_MODEL])
def test_ensure_modal_model_allowed_accepts_mapped_model(model: str) -> None:
    ensure_modal_model_allowed(model)
    assert model in MODAL_ALLOWED_MODELS


def test_kimi_uses_its_endpoint_with_shared_credentials() -> None:
    assert ensure_modal_model_configured(KIMI_MODEL, _modal_settings()) == (
        "https://posthog--kimi.us-west.modal.direct/v1",
        "wk-test",
        "ws-test",
    )


async def test_modal_anthropic_drops_clear_thinking_when_thinking_is_disabled() -> None:
    request = {
        "model": KIMI_MODEL,
        "messages": [{"role": "user", "content": "Hello"}],
        "thinking": {"type": "disabled"},
        "context_management": {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]},
    }

    with patch("llm_gateway.modal_routing.send_modal_request", new=AsyncMock(return_value={})) as send_request:
        await send_modal_anthropic_messages(request, MagicMock(), False, "posthog_code")

    forwarded_request = send_request.call_args.args[0]
    assert "context_management" not in forwarded_request
    assert request["context_management"] == {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]}


@pytest.mark.parametrize("model", ["@cf/moonshotai/kimi-k2.6", "@cf/unknown/model", "zai-org/GLM-5.2-FP8"])
def test_ensure_modal_model_allowed_rejects_unmapped_model(model: str) -> None:
    with pytest.raises(HTTPException) as exc_info:
        ensure_modal_model_allowed(model)
    assert exc_info.value.status_code == 400
    assert exc_info.value.detail["error"]["type"] == "invalid_request_error"
    assert model in exc_info.value.detail["error"]["message"]


@pytest.mark.parametrize(
    ("fraction", "product_fractions", "configured", "expected"),
    [
        (0.0, {}, True, False),
        (1.0, {}, True, True),
        # Missing credentials beat any fraction.
        (1.0, {}, False, False),
        # Per-product override beats the global fraction in both directions.
        (0.0, {"posthog_code": 1.0}, True, True),
        (1.0, {"posthog_code": 0.0}, True, False),
    ],
)
def test_should_route_glm_to_modal(
    fraction: float, product_fractions: dict[str, float], configured: bool, expected: bool
) -> None:
    overrides: dict[str, Any] = {
        "glm_modal_traffic_fraction": fraction,
        "glm_modal_product_traffic_fractions": product_fractions,
    }
    if not configured:
        overrides["modal_key"] = None
    settings = _modal_settings(**overrides)
    assert (
        should_route_glm_to_modal(GLM_MODEL, product="posthog_code", user_key="user-1", settings=settings) is expected
    )


@pytest.mark.parametrize("product", ["twig", "array"])
def test_product_aliases_resolve_in_fraction_lookup(product: str) -> None:
    # Legacy aliases must read posthog_code's per-product fraction, or its ramp/rollback would
    # silently skip alias traffic.
    settings = _modal_settings(
        glm_modal_traffic_fraction=0.0, glm_modal_product_traffic_fractions={"posthog_code": 1.0}
    )
    assert should_route_glm_to_modal(GLM_MODEL, product=product, user_key="user-1", settings=settings) is True


def test_should_route_only_modal_served_models() -> None:
    # kimi is CF-allowlisted but has no Modal-served equivalent; routing it to Modal would 404.
    settings = _modal_settings(glm_modal_traffic_fraction=1.0)
    assert (
        should_route_glm_to_modal("@cf/moonshotai/kimi-k2.6", product="posthog_code", user_key="u", settings=settings)
        is False
    )


def test_partial_fraction_is_sticky_and_monotonic() -> None:
    # Repeat calls must be identical, and a user routed to Modal at fraction f must stay routed at
    # any higher fraction — otherwise each ramp-up step would flap migrated users back.
    fractions = (0.25, 0.5, 0.75)
    settings = [_modal_settings(glm_modal_traffic_fraction=f) for f in fractions]
    for user in ("user-1", "user-2", "user-3", "user-4", "user-5"):
        decisions = [
            should_route_glm_to_modal(GLM_MODEL, product="posthog_code", user_key=user, settings=s) for s in settings
        ]
        assert decisions == sorted(decisions)
        repeat = [
            should_route_glm_to_modal(GLM_MODEL, product="posthog_code", user_key=user, settings=s) for s in settings
        ]
        assert repeat == decisions


def test_partial_fraction_splits_pinned_users() -> None:
    # Pinned user keys with known buckets ("3" -> ~0.25, "2" -> ~0.96) must land on opposite sides
    # of fraction 0.5 — a degenerate bucket (constant 0 or 1) passes the monotonicity test above
    # but fails here.
    settings = _modal_settings(glm_modal_traffic_fraction=0.5)
    assert should_route_glm_to_modal(GLM_MODEL, product="posthog_code", user_key="3", settings=settings) is True
    assert should_route_glm_to_modal(GLM_MODEL, product="posthog_code", user_key="2", settings=settings) is False


async def test_modal_anthropic_stream_usage_survives_messages_bridge_to_acompletion() -> None:
    llm_call = make_modal_anthropic_call("https://modal.test/v1", "wk", "ws")

    with patch("litellm.acompletion", new=AsyncMock(side_effect=RuntimeError("captured"))) as mock_acompletion:
        with pytest.raises(RuntimeError):
            await llm_call(model=GLM_MODEL, max_tokens=16, messages=[{"role": "user", "content": "hi"}], stream=True)

    kwargs = mock_acompletion.call_args.kwargs
    assert kwargs["model"] == "openai/zai-org/GLM-5.2-FP8"
    assert kwargs["stream"] is True
    assert kwargs["extra_body"]["stream_options"] == {"include_usage": True, "continuous_usage_stats": True}


async def test_modal_responses_stream_usage_survives_responses_bridge_to_acompletion() -> None:
    llm_call = make_modal_responses_call("https://modal.test/v1", "wk", "ws")

    with patch("litellm.acompletion", new=AsyncMock(side_effect=RuntimeError("captured"))) as mock_acompletion:
        with pytest.raises(litellm.exceptions.APIConnectionError):
            await llm_call(model=GLM_MODEL, input="hi", stream=True)

    kwargs = mock_acompletion.call_args.kwargs
    assert kwargs["model"] == "zai-org/GLM-5.2-FP8"
    assert kwargs["custom_llm_provider"] == "openai"
    assert kwargs["stream"] is True
    assert kwargs["extra_body"]["stream_options"] == {"include_usage": True, "continuous_usage_stats": True}


async def test_make_modal_responses_call_forces_bridge_and_ignores_smuggled_flag() -> None:
    # vLLM has no /responses route, so the Responses->chat/completions bridge must be forced and a
    # caller-smuggled use_chat_completions_api=False must not escape it (same bug class as CF).
    llm_call = make_modal_responses_call("https://modal.test/v1", "wk", "ws")

    with patch("llm_gateway.modal.litellm.aresponses", new=AsyncMock(return_value="ok")) as mock_aresponses:
        await llm_call(model=GLM_MODEL, input="hi", use_chat_completions_api=False)

    kwargs = mock_aresponses.call_args.kwargs
    assert kwargs["use_chat_completions_api"] is True
    assert kwargs["model"] == "openai/zai-org/GLM-5.2-FP8"
    assert kwargs["api_base"] == "https://modal.test/v1"
    assert kwargs["extra_headers"]["Modal-Key"] == "wk"
    assert kwargs["extra_headers"]["Modal-Secret"] == "ws"
    assert kwargs["input"] == "hi"