Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

20.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Unit tests for the per-session cost calculator: math against
``core/inference/pricing.py`` plus graceful degradation."""

import math

from core.inference.pricing import (
    ANTHROPIC_CACHE_5M_WRITE_MULT,
    ANTHROPIC_CACHE_1H_WRITE_MULT,
    ANTHROPIC_CACHE_READ_MULT,
    ANTHROPIC_FAST_MODE_MULT,
    ANTHROPIC_PRICING,
    OPENAI_CACHE_READ_MULT,
    OPENAI_CONTAINER_USD_PER_HOUR,
    OPENAI_PRICING,
    OPENAI_WEB_SEARCH_USD_PER_1K,
    calculate_cost,
    pricing_snapshot,
)


def _isclose(
    a,
    b,
    tol = 1e-6,
):
    return math.isclose(a, b, rel_tol = tol, abs_tol = tol)


def test_unknown_model_priced_false():
    out = calculate_cost(
        "anthropic",
        "made-up-model-9000",
        {"input_tokens": 100, "output_tokens": 50},
    )
    assert out["priced"] is False
    assert out["total_usd"] == 0.0
    assert out["billable_input_tokens"] == 100
    assert out["billable_output_tokens"] == 50


def test_anthropic_opus_4_7_input_and_output_math():
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {"input_tokens": 1_000_000, "output_tokens": 1_000_000},
    )
    assert _isclose(out["input_usd"], 5.0)
    assert _isclose(out["output_usd"], 25.0)
    assert _isclose(out["total_usd"], 30.0)


def test_anthropic_fast_mode_charges_6x_standard_opus():
    """6x on input + output when ``usage.speed == "fast"``.
    https://platform.claude.com/docs/en/build-with-claude/fast-mode"""
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 1_000_000,
            "output_tokens": 1_000_000,
            "speed": "fast",
        },
    )
    assert _isclose(out["input_usd"], 5.0 * ANTHROPIC_FAST_MODE_MULT)
    assert _isclose(out["output_usd"], 25.0 * ANTHROPIC_FAST_MODE_MULT)
    assert _isclose(out["total_usd"], 30.0 * ANTHROPIC_FAST_MODE_MULT)
    assert "(fast)" in out["model_priced"], out["model_priced"]


def test_anthropic_fast_mode_does_not_affect_standard_speed():
    """``speed: "standard"`` (or missing) keeps the base rates."""
    out_standard = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 1_000_000,
            "output_tokens": 1_000_000,
            "speed": "standard",
        },
    )
    out_missing = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {"input_tokens": 1_000_000, "output_tokens": 1_000_000},
    )
    assert _isclose(out_standard["total_usd"], out_missing["total_usd"])
    assert _isclose(out_standard["total_usd"], 30.0)


def test_anthropic_fast_mode_stacks_with_cache_read_multiplier():
    """Cache multipliers apply on top of fast-mode (per docs)."""
    base = ANTHROPIC_PRICING["claude-opus-4-7"]["input_per_mtok"]
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "cache_read_input_tokens": 1_000_000,
            "speed": "fast",
        },
    )
    expected = base * ANTHROPIC_FAST_MODE_MULT * ANTHROPIC_CACHE_READ_MULT
    assert _isclose(out["cache_read_usd"], expected)


def test_anthropic_cache_5m_and_read_use_correct_multipliers():
    base = ANTHROPIC_PRICING["claude-opus-4-7"]["input_per_mtok"]
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "cache_creation_input_tokens": 1_000_000,
            "cache_read_input_tokens": 1_000_000,
            "cache_creation": {
                "ephemeral_5m_input_tokens": 1_000_000,
                "ephemeral_1h_input_tokens": 0,
            },
        },
    )
    assert _isclose(out["cache_write_usd"], base * ANTHROPIC_CACHE_5M_WRITE_MULT)
    assert _isclose(out["cache_read_usd"], base * ANTHROPIC_CACHE_READ_MULT)
    # billable_input_tokens = input + cache_create + cache_read
    assert out["billable_input_tokens"] == 2_000_000


def test_anthropic_cache_1h_write_uses_2x_multiplier():
    base = ANTHROPIC_PRICING["claude-opus-4-7"]["input_per_mtok"]
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "cache_creation_input_tokens": 1_000_000,
            "cache_read_input_tokens": 0,
            "cache_creation": {
                "ephemeral_5m_input_tokens": 0,
                "ephemeral_1h_input_tokens": 1_000_000,
            },
        },
    )
    assert _isclose(out["cache_write_usd"], base * ANTHROPIC_CACHE_1H_WRITE_MULT)


def test_anthropic_cache_5m_default_when_no_breakdown():
    base = ANTHROPIC_PRICING["claude-opus-4-7"]["input_per_mtok"]
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "cache_creation_input_tokens": 500_000,
        },
    )
    expected = 0.5 * base * ANTHROPIC_CACHE_5M_WRITE_MULT
    assert _isclose(out["cache_write_usd"], expected)


def test_anthropic_web_search_charged_per_thousand():
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "server_tool_use": {"web_search_requests": 250},
        },
    )
    assert _isclose(out["server_tools_usd"], 2.5)  # $10/1000 * 250


def test_anthropic_code_exec_charged_per_hour():
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "server_tool_use": {"code_execution_hours": 2.0},
        },
    )
    assert _isclose(out["server_tools_usd"], 0.10)  # $0.05/hr * 2


def test_anthropic_dated_id_falls_back_to_canonical_prefix():
    out = calculate_cost(
        "anthropic",
        "claude-opus-4-7-20260712",
        {"input_tokens": 1_000_000, "output_tokens": 0},
    )
    assert out["priced"] is True
    assert _isclose(out["input_usd"], 5.0)


def test_openai_gpt55_input_output_math():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {"input_tokens": 200_000, "output_tokens": 50_000},
    )
    assert _isclose(out["input_usd"], 200_000 / 1_000_000.0 * 5.0)
    assert _isclose(out["output_usd"], 50_000 / 1_000_000.0 * 30.0)
    assert _isclose(out["total_usd"], 1.0 + 1.5)


def test_openai_cache_read_subtracted_from_input_at_discount():
    # OpenAI folds cached into input_tokens; subtract and re-bill at 0.1x.
    base = OPENAI_PRICING["gpt-5.5"]["input_per_mtok"]
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 100_000,
            "output_tokens": 0,
            "input_tokens_details": {"cached_tokens": 80_000},
        },
    )
    assert _isclose(out["input_usd"], 20_000 / 1_000_000.0 * base)
    assert _isclose(out["cache_read_usd"], 80_000 / 1_000_000.0 * base * OPENAI_CACHE_READ_MULT)


def test_openai_billable_input_tokens_does_not_double_count_cache_read():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 100_000,
            "output_tokens": 0,
            "input_tokens_details": {"cached_tokens": 80_000},
        },
    )
    assert out["billable_input_tokens"] == 100_000


def test_openai_dated_snapshot_inherits_canonical_pricing():
    out = calculate_cost(
        "openai",
        "gpt-5.5-2026-04-23",
        {"input_tokens": 200_000, "output_tokens": 0},
    )
    assert out["priced"] is True
    assert _isclose(out["input_usd"], 200_000 / 1_000_000.0 * 5.0)


def test_openai_gpt54_family_uses_verified_prices():
    cases = {
        "gpt-5.4": (200_000, 200_000 / 1_000_000.0 * 2.5, 200_000 / 1_000_000.0 * 15.0),
        "gpt-5.4-mini": (1_000_000, 0.75, 4.5),
        "gpt-5.4-nano": (1_000_000, 0.20, 1.25),
        "gpt-5.3-codex": (1_000_000, 1.75, 14.0),
    }
    for model, (in_tokens, exp_in, exp_out) in cases.items():
        out = calculate_cost(
            "openai",
            model,
            {"input_tokens": in_tokens, "output_tokens": in_tokens},
        )
        assert out["priced"] is True, model
        assert _isclose(out["input_usd"], exp_in), model
        assert _isclose(out["output_usd"], exp_out), model


def test_openai_unlisted_model_priced_false_not_zero_default():
    for model in ("o3", "o4-mini", "gpt-4.5", "gpt-4.5-preview"):
        out = calculate_cost(
            "openai",
            model,
            {"input_tokens": 1_000_000, "output_tokens": 1_000_000},
        )
        assert out["priced"] is False, model
        assert out["total_usd"] == 0.0, model
        assert out["billable_input_tokens"] == 1_000_000, model
        assert out["billable_output_tokens"] == 1_000_000, model


def test_anthropic_canonical_4_5_ids_are_priced():
    cases = {
        "claude-opus-4-5": (5.0, 25.0),
        "claude-sonnet-4-5": (3.0, 15.0),
        "claude-haiku-4-5": (1.0, 5.0),
        "claude-opus-4-1": (15.0, 75.0),
    }
    for model, (inp, outp) in cases.items():
        out = calculate_cost(
            "anthropic",
            model,
            {"input_tokens": 1_000_000, "output_tokens": 1_000_000},
        )
        assert out["priced"] is True, model
        assert _isclose(out["input_usd"], inp), model
        assert _isclose(out["output_usd"], outp), model


def test_openai_gpt55_short_context_under_272k_uses_base_rates():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {"input_tokens": 100_000, "output_tokens": 5_000},
    )
    assert _isclose(out["input_usd"], 100_000 / 1_000_000.0 * 5.0)
    assert _isclose(out["output_usd"], 5_000 / 1_000_000.0 * 30.0)
    assert "long-context" not in out["model_priced"], out["model_priced"]


def test_openai_gpt55_long_context_crossover_uses_higher_rates():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {"input_tokens": 300_000, "output_tokens": 10_000},
    )
    assert _isclose(out["input_usd"], 300_000 / 1_000_000.0 * 10.0)
    assert _isclose(out["output_usd"], 10_000 / 1_000_000.0 * 45.0)
    assert "long-context" in out["model_priced"], out["model_priced"]


def test_openai_gpt54_long_context_crossover():
    out = calculate_cost(
        "openai",
        "gpt-5.4",
        {"input_tokens": 500_000, "output_tokens": 20_000},
    )
    assert _isclose(out["input_usd"], 500_000 / 1_000_000.0 * 5.0)
    assert _isclose(out["output_usd"], 20_000 / 1_000_000.0 * 22.5)


def test_openai_gpt54_mini_has_no_long_context_tier():
    out = calculate_cost(
        "openai",
        "gpt-5.4-mini",
        {"input_tokens": 500_000, "output_tokens": 0},
    )
    assert _isclose(out["input_usd"], 500_000 / 1_000_000.0 * 0.75)
    assert "long-context" not in out["model_priced"], out["model_priced"]


def test_openai_web_search_charged_per_thousand():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "openai_tool_use": {"web_search_requests": 250},
        },
    )
    assert _isclose(out["server_tools_usd"], 250 / 1_000.0 * OPENAI_WEB_SEARCH_USD_PER_1K)
    assert _isclose(out["total_usd"], 250 / 1_000.0 * OPENAI_WEB_SEARCH_USD_PER_1K)


def test_openai_container_hours_charged():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 0,
            "output_tokens": 0,
            "openai_tool_use": {"container_hours": 1.5},
        },
    )
    assert _isclose(out["server_tools_usd"], 1.5 * OPENAI_CONTAINER_USD_PER_HOUR)


def test_openai_tool_surcharges_added_to_total():
    out = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 100_000,
            "output_tokens": 5_000,
            "openai_tool_use": {
                "web_search_requests": 3,
                "container_hours": 0.25,
            },
        },
    )
    expected_input = 100_000 / 1_000_000.0 * 5.0
    expected_output = 5_000 / 1_000_000.0 * 30.0
    expected_tools = (
        3 / 1_000.0 * OPENAI_WEB_SEARCH_USD_PER_1K + 0.25 * OPENAI_CONTAINER_USD_PER_HOUR
    )
    assert _isclose(
        out["total_usd"],
        round(expected_input + expected_output + expected_tools, 6),
    )


def test_snapshot_contains_provider_buckets_and_multipliers():
    snap = pricing_snapshot()
    assert set(snap.keys()) == {"anthropic", "openai"}
    a = snap["anthropic"]
    o = snap["openai"]
    assert "models" in a and "claude-opus-4-7" in a["models"]
    assert a["cache_5m_write_mult"] == ANTHROPIC_CACHE_5M_WRITE_MULT
    assert a["cache_1h_write_mult"] == ANTHROPIC_CACHE_1H_WRITE_MULT
    assert a["cache_read_mult"] == ANTHROPIC_CACHE_READ_MULT
    assert a["fast_mode_mult"] == ANTHROPIC_FAST_MODE_MULT
    assert "web_search_usd_per_1k" in a
    assert "code_execution_usd_per_hour" in a
    assert "models" in o and "gpt-5.5" in o["models"]
    assert o["cache_read_mult"] == OPENAI_CACHE_READ_MULT
    assert o["web_search_usd_per_1k"] == OPENAI_WEB_SEARCH_USD_PER_1K
    assert o["container_usd_per_hour"] == OPENAI_CONTAINER_USD_PER_HOUR
    gpt55 = o["models"]["gpt-5.5"]
    assert gpt55["long_context_threshold"] == 272_000
    assert gpt55["long_context_input_per_mtok"] == 10.0
    assert gpt55["long_context_output_per_mtok"] == 45.0


def test_longest_prefix_match_wins_for_dated_mini_snapshot():
    """`gpt-5.4-mini-2026-...` inherits the mini rate, not the shorter
    `gpt-5.4` rate (longest prefix wins)."""
    out = calculate_cost(
        "openai",
        "gpt-5.4-mini-2026-04-23",
        {"input_tokens": 1_000_000, "output_tokens": 0},
    )
    assert out["priced"] is True
    assert _isclose(out["input_usd"], 0.75), out


def test_longest_prefix_match_wins_for_dated_pro_snapshot():
    out = calculate_cost(
        "openai",
        "gpt-5.5-pro-2026-04-23",
        {"input_tokens": 1_000_000, "output_tokens": 0},
    )
    assert out["priced"] is True
    assert _isclose(out["input_usd"], 30.0), out


def test_openai_chat_style_usage_keys_priced_correctly():
    """Chat-style envelope (`prompt_tokens`/`completion_tokens`) must
    produce a non-zero cost (previously silently zeroed)."""
    out = calculate_cost(
        "openai",
        "gpt-5.4-mini",
        {"prompt_tokens": 1_000_000, "completion_tokens": 1_000_000},
    )
    assert _isclose(out["input_usd"], 0.75), out
    assert _isclose(out["output_usd"], 4.5), out


def test_input_tokens_preferred_when_both_keys_present():
    """Raw key wins when both envelope shapes are present."""
    out = calculate_cost(
        "openai",
        "gpt-5.4-mini",
        {
            "input_tokens": 2_000_000,
            "prompt_tokens": 5_000_000,
            "output_tokens": 0,
        },
    )
    assert _isclose(out["input_usd"], 1.50), out


def test_anthropic_chat_style_prompt_tokens_dedupes_cache_buckets():
    """Anthropic chat-style prompt_tokens already folds cache buckets;
    don't double-count billable input."""
    raw = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "input_tokens": 1_000_000,
            "cache_creation_input_tokens": 200_000,
            "cache_read_input_tokens": 500_000,
            "output_tokens": 0,
        },
    )
    chat = calculate_cost(
        "anthropic",
        "claude-opus-4-7",
        {
            "prompt_tokens": 1_700_000,
            "cache_creation_input_tokens": 200_000,
            "cache_read_input_tokens": 500_000,
            "completion_tokens": 0,
        },
    )
    assert _isclose(chat["input_usd"], raw["input_usd"]), (chat, raw)
    assert _isclose(chat["cache_write_usd"], raw["cache_write_usd"]), (chat, raw)
    assert _isclose(chat["cache_read_usd"], raw["cache_read_usd"]), (chat, raw)
    assert _isclose(chat["total_usd"], raw["total_usd"]), (chat, raw)
    assert chat["billable_input_tokens"] == raw["billable_input_tokens"], (chat, raw)


def test_openai_chat_style_prompt_tokens_keeps_cache_read_semantics():
    """OpenAI prompt_tokens includes cache_read like raw input_tokens."""
    raw = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 1_000_000,
            "input_tokens_details": {"cached_tokens": 200_000},
            "output_tokens": 100_000,
        },
    )
    chat = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "prompt_tokens": 1_000_000,
            "cache_read_input_tokens": 200_000,
            "completion_tokens": 100_000,
        },
    )
    assert _isclose(chat["total_usd"], raw["total_usd"]), (chat, raw)


def test_openai_chat_style_envelope_reads_cache_from_prompt_tokens_details():
    """Chat-style envelope ships cached under prompt_tokens_details;
    calculator must honour both that and input_tokens_details."""
    base = OPENAI_PRICING["gpt-5.5"]["input_per_mtok"]
    raw = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "input_tokens": 100_000,
            "input_tokens_details": {"cached_tokens": 80_000},
            "output_tokens": 0,
        },
    )
    chat_style = calculate_cost(
        "openai",
        "gpt-5.5",
        {
            "prompt_tokens": 100_000,
            "prompt_tokens_details": {"cached_tokens": 80_000},
            "completion_tokens": 0,
        },
    )
    assert _isclose(chat_style["input_usd"], raw["input_usd"]), (chat_style, raw)
    assert _isclose(chat_style["cache_read_usd"], raw["cache_read_usd"]), (chat_style, raw)
    assert _isclose(
        chat_style["cache_read_usd"],
        80_000 / 1_000_000.0 * base * OPENAI_CACHE_READ_MULT,
    )


def test_explicit_zero_output_tokens_wins_over_stale_completion_tokens():
    """Explicit ``output_tokens: 0`` beats a stale ``completion_tokens``;
    the previous `or` fallback treated 0 as missing."""
    out = calculate_cost(
        "openai",
        "gpt-4o-mini",
        {
            "input_tokens": 100,
            "output_tokens": 0,
            "completion_tokens": 50,
        },
    )
    assert out["billable_output_tokens"] == 0, out
    assert out["output_usd"] == 0.0, out


def test_openai_gpt56_family_uses_verified_prices():
    """Sol / Terra / Luna, both context tiers, from the published table."""
    expected = {
        "gpt-5.6-sol": (5.0, 30.0, 10.0, 45.0),
        "gpt-5.6-terra": (2.0, 12.0, 4.0, 18.0),
        "gpt-5.6-luna": (0.20, 1.20, 0.40, 1.80),
    }
    for model, (inp, out, long_in, long_out) in expected.items():
        row = OPENAI_PRICING[model]
        assert row["input_per_mtok"] == inp, model
        assert row["output_per_mtok"] == out, model
        assert row["long_context_threshold"] == 272_000, model
        assert row["long_context_input_per_mtok"] == long_in, model
        assert row["long_context_output_per_mtok"] == long_out, model
        short = calculate_cost("openai", model, {"input_tokens": 1000, "output_tokens": 1000})
        assert short["priced"] is True, model
        assert _isclose(short["input_usd"], 1000 / 1_000_000.0 * inp), model
        long = calculate_cost("openai", model, {"input_tokens": 300_000, "output_tokens": 1000})
        assert _isclose(long["input_usd"], 300_000 / 1_000_000.0 * long_in), model
        assert _isclose(long["output_usd"], 1000 / 1_000_000.0 * long_out), model


def test_sonnet_5_bills_the_launch_rate_until_the_cutover():
    """Sonnet 5 ships at $2/$10 through 2026-08-31 and $3/$15 from 2026-09-01.
    Ref: https://platform.claude.com/docs/en/about-claude/pricing"""
    import datetime

    from core.inference.pricing import _launch_prices

    table = ANTHROPIC_PRICING["claude-sonnet-5"]
    assert table["input_per_mtok"] == 3.0 and table["output_per_mtok"] == 15.0

    launch = _launch_prices("anthropic", "claude-sonnet-5", table, today = datetime.date(2026, 8, 31))
    assert launch["input_per_mtok"] == 2.0 and launch["output_per_mtok"] == 10.0
    dated = _launch_prices(
        "anthropic", "claude-sonnet-5-20260629", table, today = datetime.date(2026, 8, 31)
    )
    assert dated["input_per_mtok"] == 2.0

    after = _launch_prices("anthropic", "claude-sonnet-5", table, today = datetime.date(2026, 9, 1))
    assert after == table
    opus = ANTHROPIC_PRICING["claude-opus-5"]
    assert (
        _launch_prices("anthropic", "claude-opus-5", opus, today = datetime.date(2026, 8, 31)) == opus
    )


def test_fast_mode_multiplier_is_2x_for_the_supported_opus_models():
    """Fast mode is Opus 5 and Opus 4.8 only, at $10/$50 against the $5/$25 base."""
    assert ANTHROPIC_FAST_MODE_MULT == 2.0
    for model in ("claude-opus-5", "claude-opus-4-8"):
        usage = {"input_tokens": 1_000_000, "output_tokens": 1_000_000, "speed": "fast"}
        out = calculate_cost("anthropic", model, usage)
        assert _isclose(out["input_usd"], 10.0), model
        assert _isclose(out["output_usd"], 50.0), model