Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

20.0 KB
Raw Blame
"""Tests for the model deprecation helper module.

These tests focus on the helper itself — not on the proxy endpoint or
Slack integration — so they can run without the full proxy stack.
"""

import logging
from datetime import date, datetime, timezone
from unittest.mock import MagicMock



import litellm
from litellm import Router
from litellm._logging import verbose_logger
from litellm.proxy.common_utils.model_deprecation import (
    _classify,
    _parse_deprecation_date,
    collect_model_deprecations,
    format_deprecation_alert_message,
)


def _make_router(deployments):
    router = MagicMock()
    router.get_model_list.return_value = deployments
    return router


class TestParseDeprecationDate:
    def test_should_parse_iso_string(self):
        assert _parse_deprecation_date("2026-12-31") == date(2026, 12, 31)

    def test_should_pass_through_date_object(self):
        d = date(2026, 1, 1)
        assert _parse_deprecation_date(d) == d

    def test_should_return_none_for_documentation_sentinel(self):
        # The JSON map ships a sentinel string under the "sample_spec" key.
        assert (
            _parse_deprecation_date(
                "date when the model becomes deprecated in the format YYYY-MM-DD"
            )
            is None
        )

    def test_should_return_none_for_none(self):
        assert _parse_deprecation_date(None) is None

    def test_should_return_none_for_unsupported_type(self):
        assert _parse_deprecation_date(12345) is None

    def test_should_narrow_datetime_to_date(self):
        assert _parse_deprecation_date(
            datetime(2026, 12, 31, 23, 59, tzinfo=timezone.utc)
        ) == date(2026, 12, 31)


class TestClassify:
    def test_should_classify_past_dates_as_deprecated(self):
        assert _classify(-1, warn_within_days=30) == "deprecated"
        assert _classify(-365, warn_within_days=30) == "deprecated"

    def test_should_classify_inside_window_as_imminent(self):
        assert _classify(0, warn_within_days=30) == "imminent"
        assert _classify(15, warn_within_days=30) == "imminent"
        assert _classify(30, warn_within_days=30) == "imminent"

    def test_should_classify_outside_window_as_upcoming(self):
        assert _classify(31, warn_within_days=30) == "upcoming"
        assert _classify(365, warn_within_days=30) == "upcoming"


class TestCollectModelDeprecations:
    def test_should_return_empty_response_when_router_is_none(self):
        snapshot = collect_model_deprecations(llm_router=None)
        assert snapshot.deprecated == []
        assert snapshot.imminent == []
        assert snapshot.upcoming == []

    def test_should_skip_models_without_deprecation_metadata(self, monkeypatch):
        monkeypatch.setattr(litellm, "model_cost", {})
        router = _make_router(
            [
                {
                    "model_name": "gpt-4o",
                    "litellm_params": {"model": "openai/gpt-4o"},
                    "model_info": {"id": "abc"},
                }
            ]
        )
        snapshot = collect_model_deprecations(llm_router=router)
        assert snapshot.deprecated == []
        assert snapshot.imminent == []
        assert snapshot.upcoming == []

    def test_should_classify_into_three_buckets(self, monkeypatch):
        today = date(2026, 6, 1)
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {
                "deprecated-model": {
                    "deprecation_date": "2026-01-01",
                    "litellm_provider": "openai",
                },
                "imminent-model": {
                    "deprecation_date": "2026-06-15",
                    "litellm_provider": "openai",
                },
                "upcoming-model": {
                    "deprecation_date": "2027-01-01",
                    "litellm_provider": "openai",
                },
            },
        )
        router = _make_router(
            [
                {
                    "model_name": "deprecated-alias",
                    "litellm_params": {"model": "openai/deprecated-model"},
                    "model_info": {"id": "1"},
                },
                {
                    "model_name": "imminent-alias",
                    "litellm_params": {"model": "imminent-model"},
                    "model_info": {"id": "2"},
                },
                {
                    "model_name": "upcoming-alias",
                    "litellm_params": {"model": "openai/upcoming-model"},
                    "model_info": {"id": "3"},
                },
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=today
        )

        assert [m.model_name for m in snapshot.deprecated] == ["deprecated-alias"]
        assert [m.model_name for m in snapshot.imminent] == ["imminent-alias"]
        assert [m.model_name for m in snapshot.upcoming] == ["upcoming-alias"]

        assert snapshot.deprecated[0].days_until_deprecation < 0
        assert snapshot.imminent[0].days_until_deprecation == 14
        assert snapshot.upcoming[0].days_until_deprecation > 30

    def test_should_prefer_explicit_deployment_override(self, monkeypatch):
        today = date(2026, 6, 1)
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {"some-model": {"deprecation_date": "2030-01-01"}},
        )
        router = _make_router(
            [
                {
                    "model_name": "my-alias",
                    "litellm_params": {"model": "some-model"},
                    "model_info": {
                        "id": "x",
                        "deprecation_date": "2026-06-10",
                    },
                }
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=today
        )

        assert len(snapshot.imminent) == 1
        assert snapshot.imminent[0].deprecation_date == date(2026, 6, 10)

    def test_should_dedupe_duplicate_deployments_in_same_group(self, monkeypatch):
        today = date(2026, 6, 1)
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {"shared-model": {"deprecation_date": "2026-06-10"}},
        )
        router = _make_router(
            [
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "shared-model"},
                    "model_info": {"id": "1"},
                },
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "shared-model"},
                    "model_info": {"id": "2"},
                },
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=today
        )

        assert len(snapshot.imminent) == 1

    def test_should_resolve_via_unprefixed_model_name(self, monkeypatch):
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {"gpt-4o": {"deprecation_date": "2026-06-10"}},
        )
        router = _make_router(
            [
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "openai/gpt-4o"},
                    "model_info": {"id": "1"},
                }
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=date(2026, 6, 1)
        )

        assert [m.litellm_model for m in snapshot.imminent] == ["gpt-4o"]

    def test_should_keep_both_dates_when_group_has_conflicting_dates(self, monkeypatch):
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {"shared-model": {"deprecation_date": "2026-06-10"}},
        )
        router = _make_router(
            [
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "shared-model"},
                    "model_info": {"id": "1"},
                },
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "shared-model"},
                    "model_info": {"id": "2", "deprecation_date": "2027-01-01"},
                },
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=date(2026, 6, 1)
        )

        assert len(snapshot.imminent) == 1
        assert len(snapshot.upcoming) == 1

    def test_should_resolve_via_base_model(self, monkeypatch):
        today = date(2026, 6, 1)
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {"base-thing": {"deprecation_date": "2026-06-10"}},
        )
        router = _make_router(
            [
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "azure/some-deployment-name"},
                    "model_info": {"id": "1", "base_model": "base-thing"},
                }
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=today
        )

        assert len(snapshot.imminent) == 1
        assert snapshot.imminent[0].litellm_model == "base-thing"


class TestFormatDeprecationAlertMessage:
    def test_should_return_none_when_nothing_to_alert(self):
        snapshot = collect_model_deprecations(llm_router=None)
        assert format_deprecation_alert_message(snapshot) is None

    def test_should_render_imminent_and_deprecated_sections(self, monkeypatch):
        today = date(2026, 6, 1)
        monkeypatch.setattr(
            litellm,
            "model_cost",
            {
                "dead-model": {
                    "deprecation_date": "2026-01-01",
                    "litellm_provider": "openai",
                },
                "soon-model": {
                    "deprecation_date": "2026-06-15",
                    "litellm_provider": "anthropic",
                },
                "later-model": {
                    "deprecation_date": "2027-01-01",
                    "litellm_provider": "anthropic",
                },
            },
        )
        router = _make_router(
            [
                {
                    "model_name": "dead",
                    "litellm_params": {"model": "dead-model"},
                    "model_info": {"id": "1"},
                },
                {
                    "model_name": "soon",
                    "litellm_params": {"model": "soon-model"},
                    "model_info": {"id": "2"},
                },
                {
                    "model_name": "later",
                    "litellm_params": {"model": "later-model"},
                    "model_info": {"id": "3"},
                },
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=today
        )
        message = format_deprecation_alert_message(snapshot)

        assert message is not None
        assert "Already deprecated" in message
        assert "Deprecating within 30 days" in message
        assert "`dead`" in message
        assert "`soon`" in message
        # Upcoming models must NOT be in the alert (avoid alert fatigue).
        assert "`later`" not in message

    def test_should_neutralize_slack_markup_from_model_metadata(self):
        today = date(2026, 6, 1)
        router = _make_router(
            [
                {
                    "model_name": "<!channel> pwned",
                    "litellm_params": {"model": "openai/whatever"},
                    "model_info": {
                        "id": "1",
                        "deprecation_date": "2026-06-10",
                        "litellm_provider": "<https://evil.example|openai> & co",
                    },
                }
            ]
        )

        snapshot = collect_model_deprecations(
            llm_router=router, warn_within_days=30, today=today
        )
        message = format_deprecation_alert_message(snapshot)

        assert message is not None
        assert "<!channel>" not in message
        assert "<https://evil.example|openai>" not in message
        assert "&lt;!channel&gt; pwned" in message
        assert "&lt;https://evil.example|openai&gt; &amp; co" in message


class TestSuccessorModel:
    def test_should_surface_the_successor_set_on_the_deployment(self, monkeypatch):
        monkeypatch.setattr(litellm, "model_cost", {})
        router = Router(
            model_list=[
                {
                    "model_name": "old",
                    "litellm_params": {"model": "openai/old-model"},
                    "model_info": {"id": "1", "deprecation_date": "2026-06-10", "successor_model": "new"},
                },
                {"model_name": "new", "litellm_params": {"model": "openai/new-model"}, "model_info": {"id": "2"}},
            ]
        )

        snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))

        assert snapshot.imminent[0].successor_model == "new"

    def test_should_name_the_deployment_successor_when_the_date_comes_from_the_cost_map(self, monkeypatch):
        monkeypatch.setattr(litellm, "model_cost", {"old-model": {"deprecation_date": "2026-06-10"}})
        router = Router(
            model_list=[
                {
                    "model_name": "old",
                    "litellm_params": {"model": "openai/old-model"},
                    "model_info": {"id": "1", "successor_model": "my-pick"},
                },
                {"model_name": "my-pick", "litellm_params": {"model": "openai/pick"}, "model_info": {"id": "2"}},
            ]
        )

        snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))

        assert snapshot.imminent[0].successor_model == "my-pick"

    def test_should_ignore_blank_or_non_string_successors(self, monkeypatch):
        monkeypatch.setattr(litellm, "model_cost", {})
        router = _make_router(
            [
                {
                    "model_name": "blank",
                    "litellm_params": {"model": "openai/a"},
                    "model_info": {"id": "1", "deprecation_date": "2026-06-10", "successor_model": "   "},
                },
                {
                    "model_name": "number",
                    "litellm_params": {"model": "openai/b"},
                    "model_info": {"id": "2", "deprecation_date": "2026-06-10", "successor_model": 7},
                },
            ]
        )

        snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))

        assert [m.successor_model for m in snapshot.imminent] == [None, None]

    def test_should_report_successors_as_set_and_warn_when_no_model_in_the_team_scope_matches(
        self, monkeypatch, caplog
    ):
        monkeypatch.setattr(litellm, "model_cost", {})
        router = Router(
            model_list=[
                {
                    "model_name": "model_name_t1_old",
                    "litellm_params": {"model": "openai/team-old"},
                    "model_info": {
                        "id": "1",
                        "team_id": "t1",
                        "team_public_model_name": "team-old",
                        "deprecation_date": "2026-06-10",
                        "successor_model": "team-new",
                    },
                },
                {
                    "model_name": "model_name_t1_new",
                    "litellm_params": {"model": "openai/team-new"},
                    "model_info": {"id": "2", "team_id": "t1", "team_public_model_name": "team-new"},
                },
                {
                    "model_name": "global-old",
                    "litellm_params": {"model": "openai/global-old"},
                    "model_info": {"id": "3", "deprecation_date": "2026-06-10", "successor_model": "team-new"},
                },
                {
                    "model_name": "typo-old",
                    "litellm_params": {"model": "openai/typo-old"},
                    "model_info": {"id": "4", "deprecation_date": "2026-06-10", "successor_model": "gpt-nwe"},
                },
                {
                    "model_name": "wildcard-old",
                    "litellm_params": {"model": "openai/wildcard-old"},
                    "model_info": {"id": "5", "deprecation_date": "2026-06-10", "successor_model": "anthropic/x"},
                },
                {"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*"}, "model_info": {"id": "6"}},
            ]
        )

        with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
            snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))

        assert {m.model_name: m.successor_model for m in snapshot.imminent} == {
            "model_name_t1_old": "team-new",
            "global-old": "team-new",
            "typo-old": "gpt-nwe",
            "wildcard-old": "anthropic/x",
        }
        warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()]
        assert len(warnings) == 2
        assert any("global-old names successor team-new" in warning for warning in warnings)
        assert any("typo-old names successor gpt-nwe" in warning for warning in warnings)

    def test_should_keep_the_first_deployment_and_carry_the_group_successor_when_deduping(self, monkeypatch):
        monkeypatch.setattr(litellm, "model_cost", {})
        router = Router(
            model_list=[
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "openai/a"},
                    "model_info": {"id": "1", "deprecation_date": "2026-06-10"},
                },
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "openai/b"},
                    "model_info": {"id": "2", "deprecation_date": "2026-06-10", "successor_model": "alias-v2"},
                },
                {
                    "model_name": "alias",
                    "litellm_params": {"model": "openai/c"},
                    "model_info": {"id": "3", "deprecation_date": "2026-06-10", "successor_model": "alias-v3"},
                },
                {"model_name": "alias-v2", "litellm_params": {"model": "openai/v2"}, "model_info": {"id": "4"}},
                {"model_name": "alias-v3", "litellm_params": {"model": "openai/v3"}, "model_info": {"id": "5"}},
            ]
        )

        snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))

        assert [(m.litellm_model, m.successor_model) for m in snapshot.imminent] == [("openai/a", "alias-v2")]

    def test_should_name_the_successor_in_the_slack_entry_and_escape_it(self, monkeypatch):
        monkeypatch.setattr(litellm, "model_cost", {})
        router = Router(
            model_list=[
                {
                    "model_name": "old",
                    "litellm_params": {"model": "openai/old-model"},
                    "model_info": {
                        "id": "1",
                        "deprecation_date": "2026-06-10",
                        "successor_model": "<https://evil.example|new>",
                    },
                },
                {
                    "model_name": "<https://evil.example|new>",
                    "litellm_params": {"model": "openai/new-model"},
                    "model_info": {"id": "2"},
                },
                {
                    "model_name": "orphan",
                    "litellm_params": {"model": "openai/orphan-model"},
                    "model_info": {"id": "3", "deprecation_date": "2026-06-10"},
                },
            ]
        )

        snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
        message = format_deprecation_alert_message(snapshot)

        assert message is not None
        assert "migrate to `&lt;https://evil.example|new&gt;`" in message
        assert "<https://evil.example|new>" not in message
        assert message.count("migrate to") == 1