Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

25.3 KB
Raw Blame
"""Auto binding, default resolution and lifecycle regressions against a real DB,
plus smoke tests against the installed xrouter-llm catalog."""

import asyncio
from types import SimpleNamespace

import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import StaticPool

from xagent.core.model.chat.basic.router import (
    _MODALITY_ABILITIES,
    RouterLLM,
    _ResolvedRouterLLM,
)
from xagent.web import models
from xagent.web.api import admin_users, agents
from xagent.web.api import model as model_api
from xagent.web.api import workforces
from xagent.web.auth_dependencies import get_current_user
from xagent.web.models import database
from xagent.web.models.auto_model import AutoModelCandidate, AutoModelConfig
from xagent.web.models.model import Model
from xagent.web.models.user import User, UserDefaultModel, UserModel
from xagent.web.schemas.model import AutoModelConfigUpdate, ModelUpdate
from xagent.web.services import (
    auto_model_service,
    builder_chat_runtime,
    llm_utils,
    model_service,
)
from xagent.web.services.auto_model_service import (
    AutoModelConfigurationError,
    AutoModelService,
)
from xagent.web.services.client_error_messages import CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE
from xagent.web.services.llm_utils import (
    AutoModelUnavailableError,
    UserAwareModelStorage,
)


@pytest.fixture
def env(monkeypatch, tmp_path):
    monkeypatch.setenv("XAGENT_STORAGE_ROOT", str(tmp_path))
    engine = create_engine(
        "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool
    )

    @event.listens_for(engine, "connect")
    def foreign_keys(conn, _):
        conn.execute("PRAGMA foreign_keys=ON")

    models.Base.metadata.create_all(engine)
    factory = sessionmaker(bind=engine, expire_on_commit=False)
    db = factory()

    monkeypatch.setattr(database, "_SessionLocal", factory)
    monkeypatch.setattr(admin_users, "get_session_local", lambda: factory)
    monkeypatch.setattr(builder_chat_runtime, "get_session_local", lambda: factory)
    monkeypatch.setattr(model_service, "_visible_user_ids_hook", None)
    profiles = {
        "text-a": SimpleNamespace(input_modalities=("text",)),
        "text-b": SimpleNamespace(input_modalities=("text",)),
        "image-a": SimpleNamespace(input_modalities=("text", "image")),
    }
    catalog = SimpleNamespace(
        known_model_ids=lambda: tuple(profiles), get=profiles.__getitem__
    )
    monkeypatch.setattr(
        auto_model_service, "load_router_profile_catalog", lambda: catalog
    )
    owner = User(username="owner", password_hash="unused", is_admin=True)
    consumer = User(username="consumer", password_hash="unused", is_admin=False)
    db.add_all([owner, consumer])
    db.commit()
    yield SimpleNamespace(db=db, factory=factory, owner=owner, consumer=consumer)
    db.close()
    engine.dispose()


def target(env, owner, *, name="saved", abilities=None, shared=False):
    row = Model(
        model_id=name,
        category="llm",
        model_provider="openai",
        model_name="gpt-4",
        api_key="fake-test-key",
        base_url="https://example.invalid/v1",
        abilities=abilities or ["chat"],
        is_active=True,
    )
    env.db.add(row)
    env.db.flush()
    env.db.add(
        UserModel(
            user_id=owner.id,
            model_id=row.id,
            is_owner=True,
            can_edit=True,
            can_delete=True,
            is_shared=shared,
        )
    )
    env.db.commit()
    return row


def configure(env, user, bindings, *, default=False):
    return AutoModelService(env.db).upsert_config(
        user_id=user.id,
        request=AutoModelConfigUpdate(
            candidates=[
                {"routing_model_id": profile, "target_model_id": t.id}
                for profile, t in bindings
            ],
            fallback_model_id=bindings[0][1].id,
            set_as_default=default,
        ),
    )


@pytest.fixture
def llm_utils_warnings(monkeypatch):
    """Record the warnings llm_utils logs, bypassing logging configuration.

    Replacing the module's ``logger`` binding keeps the assertion independent
    of whichever logging setup ran first in the test process.
    """
    real = llm_utils.logger
    warnings: list[str] = []

    class RecordingLogger:
        def warning(self, msg, *args, **kwargs):
            warnings.append(msg % args if args else msg)

        def __getattr__(self, name):
            return getattr(real, name)

    monkeypatch.setattr(llm_utils, "logger", RecordingLogger())
    return warnings


def skipped_candidates(warnings):
    return [w for w in warnings if w.startswith("Skipping Auto candidate ")]


def test_candidate_relationship_reads_persisted_position_order(env):
    first = target(env, env.consumer, name="first")
    second = target(env, env.consumer, name="second")
    config = configure(
        env,
        env.consumer,
        [("text-a", first), ("text-b", second)],
    )
    assert config.mode == "routing"
    assert [candidate.position for candidate in config.candidates] == [0, 1]

    config.candidates[0].position = -1
    env.db.flush()
    config.candidates[1].position = 0
    env.db.flush()
    config.candidates[0].position = 1
    env.db.commit()
    env.db.expire_all()

    loaded = env.db.get(AutoModelConfig, config.id)
    assert loaded is not None
    assert [candidate.routing_model_id for candidate in loaded.candidates] == [
        "text-b",
        "text-a",
    ]


def test_create_all_schema_allows_same_profile_for_distinct_targets(env):
    first = target(env, env.consumer, name="first")
    second = target(env, env.consumer, name="second")
    config = configure(env, env.consumer, [("text-a", first)])
    env.db.add(
        AutoModelCandidate(
            config_id=config.id,
            routing_model_id="text-a",
            target_model_id=second.id,
            position=1,
        )
    )
    env.db.commit()
    env.db.expire_all()

    loaded = env.db.get(AutoModelConfig, config.id)
    assert loaded is not None
    assert [
        (candidate.routing_model_id, candidate.target_model_id, candidate.position)
        for candidate in loaded.candidates
    ] == [
        ("text-a", first.id, 0),
        ("text-a", second.id, 1),
    ]


def broken_default(env, slot="general"):
    t = target(env, env.consumer)
    cfg = configure(env, env.consumer, [("text-a", t)])
    env.db.add(
        UserDefaultModel(
            user_id=env.consumer.id, model_id=cfg.router_model_id, config_type=slot
        )
    )
    t.is_active = False
    env.db.commit()
    return cfg


@pytest.mark.parametrize(
    "profile,abilities", [("image-a", ["chat"]), ("text-a", ["chat", "vision"])]
)
def test_rejects_profile_capability_mismatch_without_saving(env, profile, abilities):
    t = target(env, env.consumer, abilities=abilities)
    with pytest.raises(AutoModelConfigurationError, match="input modalities"):
        configure(env, env.consumer, [(profile, t)])
    assert env.db.query(AutoModelConfig).count() == 0


def test_legacy_mismatched_binding_fails_before_provider_call(env, llm_utils_warnings):
    t = target(env, env.consumer)
    cfg = configure(env, env.consumer, [("text-a", t)])
    cfg.candidates[0].routing_model_id = "image-a"
    env.db.commit()
    with pytest.raises(AutoModelUnavailableError, match="no active configured"):
        UserAwareModelStorage(env.db).get_llm_by_id(
            cfg.router_model.model_id, env.consumer.id
        )
    skipped = skipped_candidates(llm_utils_warnings)
    assert len(skipped) == 1
    assert "'image-a'" in skipped[0] and "input modalities" in skipped[0]


@pytest.mark.parametrize("removed_index", [0, 1])
def test_routing_skips_candidate_whose_profile_left_the_catalog(
    env, llm_utils_warnings, removed_index
):
    first = target(env, env.consumer, name="first")
    second = target(env, env.consumer, name="second")
    cfg = configure(
        env, env.consumer, [("text-a", first), ("text-b", second)], default=True
    )
    cfg.candidates[removed_index].routing_model_id = "removed-profile"
    env.db.commit()
    remaining = ("text-a", "text-b")[1 - removed_index]

    llm = UserAwareModelStorage(env.db).get_llm_by_id(
        cfg.router_model.model_id, env.consumer.id
    )

    assert isinstance(llm, RouterLLM)
    assert llm._candidate_models == (remaining,)
    # The first candidate is the saved fallback; skipping it clears the fallback.
    assert llm._fallback_model == (None if removed_index == 0 else remaining)
    assert (
        llm._downstream_resolver(remaining).model_id
        == (("first", "second")[1 - removed_index])
    )
    skipped = skipped_candidates(llm_utils_warnings)
    assert len(skipped) == 1
    assert "'removed-profile'" in skipped[0] and "xrouter-llm catalog" in skipped[0]
    default = model_service.get_default_model(env.consumer.id)
    assert isinstance(default, RouterLLM)
    assert default._candidate_models == (remaining,)


@pytest.mark.parametrize("mismatched_index", [0, 1])
def test_routing_skips_candidate_whose_profile_no_longer_matches_its_target(
    env, llm_utils_warnings, mismatched_index
):
    # Saving rejects this binding, but it can still exist, for example after
    # an xrouter-llm release changes a profile's input modalities.
    first = target(env, env.consumer, name="first")
    second = target(env, env.consumer, name="second")
    cfg = configure(
        env, env.consumer, [("text-a", first), ("text-b", second)], default=True
    )
    cfg.candidates[mismatched_index].routing_model_id = "image-a"
    env.db.commit()
    remaining = ("text-a", "text-b")[1 - mismatched_index]

    llm = UserAwareModelStorage(env.db).get_llm_by_id(
        cfg.router_model.model_id, env.consumer.id
    )

    assert isinstance(llm, RouterLLM)
    assert llm._candidate_models == (remaining,)
    # The first candidate is the saved fallback; skipping it clears the fallback.
    assert llm._fallback_model == (None if mismatched_index == 0 else remaining)
    assert (
        llm._downstream_resolver(remaining).model_id
        == ("first", "second")[1 - mismatched_index]
    )
    skipped = skipped_candidates(llm_utils_warnings)
    assert len(skipped) == 1
    assert "'image-a'" in skipped[0] and "input modalities" in skipped[0]
    default = model_service.get_default_model(env.consumer.id)
    assert isinstance(default, RouterLLM)
    assert default._candidate_models == (remaining,)


@pytest.mark.parametrize(
    "stale_profiles",
    [
        ("removed-profile", "another-removed-profile"),
        ("image-a", "image-a"),
        ("removed-profile", "image-a"),
    ],
    ids=["unknown", "mismatched", "mixed"],
)
def test_routing_with_only_stale_bindings_stays_unavailable(
    env, llm_utils_warnings, stale_profiles
):
    first = target(env, env.consumer, name="first")
    second = target(env, env.consumer, name="second")
    cfg = configure(
        env, env.consumer, [("text-a", first), ("text-b", second)], default=True
    )
    cfg.candidates[0].routing_model_id = stale_profiles[0]
    cfg.candidates[1].routing_model_id = stale_profiles[1]
    env.db.commit()

    with pytest.raises(AutoModelUnavailableError, match="no active configured"):
        UserAwareModelStorage(env.db).get_llm_by_id(
            cfg.router_model.model_id, env.consumer.id
        )
    assert len(skipped_candidates(llm_utils_warnings)) == 2
    with pytest.raises(AutoModelUnavailableError):
        model_service.get_default_model(env.consumer.id)


def test_routing_maps_missing_router_dependency_to_unavailable(env, monkeypatch):
    t = target(env, env.consumer)
    cfg = configure(env, env.consumer, [("text-a", t)], default=True)

    def missing_dependency():
        raise auto_model_service.AutoModelDependencyError(
            "Auto model configuration needs xrouter-llm>=0.3.6"
        )

    monkeypatch.setattr(
        auto_model_service, "load_router_profile_catalog", missing_dependency
    )
    with pytest.raises(AutoModelUnavailableError, match="xrouter-llm"):
        UserAwareModelStorage(env.db).get_llm_by_id(
            cfg.router_model.model_id, env.consumer.id
        )
    with pytest.raises(AutoModelUnavailableError):
        model_service.get_default_model(env.consumer.id)


def test_unknown_profile_has_its_own_configuration_error_type():
    profiles = {"image-a": SimpleNamespace(input_modalities=("text", "image"))}
    catalog = SimpleNamespace(
        known_model_ids=lambda: tuple(profiles), get=profiles.__getitem__
    )
    unknown_error = auto_model_service.UnknownAutoProfileError

    with pytest.raises(unknown_error, match="xrouter-llm catalog") as unknown:
        auto_model_service.validate_candidate_modalities(
            catalog, "removed-profile", ["chat"]
        )
    assert isinstance(unknown.value, AutoModelConfigurationError)

    # A mismatch must stay the parent type, or ability edits would skip it.
    with pytest.raises(
        AutoModelConfigurationError, match="input modalities"
    ) as mismatch:
        auto_model_service.validate_candidate_modalities(catalog, "image-a", ["chat"])
    assert not isinstance(mismatch.value, unknown_error)


def test_sequential_keeps_candidate_whose_profile_left_the_catalog(env):
    first = target(env, env.consumer, name="first")
    second = target(env, env.consumer, name="second")
    cfg = configure(env, env.consumer, [("text-a", first), ("text-b", second)])
    cfg.mode = "sequential_failover"
    cfg.fallback_model_id = None
    cfg.candidates[0].routing_model_id = "removed-profile"
    env.db.commit()

    llm = UserAwareModelStorage(env.db).get_llm_by_id(
        cfg.router_model.model_id, env.consumer.id
    )

    assert isinstance(llm, RouterLLM)
    assert llm._candidate_models == tuple(
        f"candidate:{candidate.id}" for candidate in cfg.candidates
    )


def test_configured_wrapper_never_adds_profile_only_abilities():
    downstream = SimpleNamespace(abilities=["chat"])
    router = RouterLLM(
        candidate_models=["image-a"], downstream_resolver=lambda _: downstream
    )
    resolved = _ResolvedRouterLLM(
        router=router,
        downstream=downstream,
        selected_model="image-a",
        context_window=None,
        input_modalities=("image",),
    )
    assert resolved.abilities == ["chat"]


@pytest.mark.parametrize(
    "changes",
    [
        {"model_name": "gpt-4.1"},
        {"model_provider": "deepseek", "model_name": "deepseek-v4-flash"},
        {"base_url": "https://another.invalid/v1"},
        {"abilities": ["chat", "vision"]},
    ],
)
def test_bound_candidate_identity_and_modalities_cannot_change(env, changes):
    t = target(env, env.consumer)
    configure(env, env.consumer, [("text-a", t)])
    with pytest.raises(HTTPException) as error:
        asyncio.run(
            model_api.update_model(
                t.model_id, ModelUpdate(**changes), db=env.db, user=env.consumer
            )
        )
    assert error.value.status_code == 409
    env.db.rollback()
    assert t.model_name == "gpt-4" and t.abilities == ["chat"]


def test_owner_identity_change_prunes_external_binding(env):
    t = target(env, env.owner, shared=True)
    cfg = configure(env, env.consumer, [("text-a", t)])
    asyncio.run(
        model_api.update_model(
            t.model_id, ModelUpdate(model_name="gpt-4.1"), db=env.db, user=env.owner
        )
    )
    env.db.expire_all()
    assert env.db.query(AutoModelCandidate).filter_by(config_id=cfg.id).count() == 0
    assert env.db.get(AutoModelConfig, cfg.id).fallback_model_id is None


@pytest.mark.parametrize("configured_compact", [False, True])
def test_missing_compact_does_not_resolve_broken_unrelated_general(
    env, configured_compact
):
    broken_default(env)
    healthy = target(env, env.consumer, name="healthy")
    if configured_compact:
        env.db.add(
            UserDefaultModel(
                user_id=env.consumer.id, model_id=healthy.id, config_type="compact"
            )
        )
        env.db.commit()
    resolved = UserAwareModelStorage(env.db).resolve_llms_from_names(
        [healthy.model_id] * 3 + [None], env.consumer.id
    )
    assert all(llm.model_id == healthy.model_id for llm in resolved)
    builder = builder_chat_runtime._load_builder_chat_runtime_inputs_sync(
        user_id=env.consumer.id,
        requested_file_ids=[],
        model_name=healthy.model_id,
        compact_model_name=None,
    )
    assert builder.llm.model_id == builder.compact_llm.model_id == healthy.model_id


@pytest.mark.parametrize(
    "slot,getter",
    [
        ("general", "get_default_model"),
        ("small_fast", "get_fast_model"),
        ("visual", "get_default_vision_model"),
        ("compact", "get_compact_model"),
    ],
)
def test_default_getters_preserve_auto_configuration_error(env, slot, getter):
    broken_default(env, slot)
    with pytest.raises(AutoModelUnavailableError):
        UserAwareModelStorage(env.db).get_configured_defaults(env.consumer.id)
    kwargs = {"db": env.db} if slot == "visual" else {}
    with pytest.raises(AutoModelUnavailableError):
        getattr(model_service, getter)(env.consumer.id, **kwargs)


@pytest.mark.parametrize("surface", ["optimize", "workforce"])
def test_http_entrypoints_map_actual_broken_auto_to_safe_409(env, surface):
    broken_default(env)
    app = FastAPI()
    app.include_router(agents.router if surface == "optimize" else workforces.router)

    def db_override():
        yield env.db

    app.dependency_overrides[agents.get_db] = db_override
    app.dependency_overrides[workforces.get_db] = db_override
    app.dependency_overrides[get_current_user] = lambda: env.consumer
    with TestClient(app) as client:
        if surface == "optimize":
            response = client.post(
                "/api/agents/optimize-instructions",
                json={"instructions": "Write a clear summary."},
            )
        else:
            response = client.post(
                "/api/workforces/from-prompt", json={"prompt": "Write a clear summary."}
            )
    assert response.status_code == 409
    assert response.json()["detail"] == CLIENT_SAFE_AUTO_MODEL_UNAVAILABLE


@pytest.mark.parametrize("retained_grant", [False, True])
def test_admin_delete_prunes_only_inaccessible_bindings(env, retained_grant):
    t = target(env, env.owner, shared=True, abilities=["chat", "tool_calling"])
    cfg = configure(env, env.consumer, [("text-a", t)])
    ids = env.owner.id, env.consumer.id, t.id, cfg.id, cfg.router_model_id
    if retained_grant:
        env.db.add(UserModel(user_id=env.consumer.id, model_id=t.id, is_owner=True))
        env.db.commit()
    env.db.close()
    assert admin_users._delete_user_rows_sync(user_id=ids[0]) is not None
    with env.factory() as db:
        assert db.get(User, ids[0]) is None
        assert db.get(Model, ids[2]) is not None
        assert db.query(AutoModelCandidate).filter_by(config_id=ids[3]).count() == int(
            retained_grant
        )
        assert db.get(AutoModelConfig, ids[3]).fallback_model_id == (
            ids[2] if retained_grant else None
        )
        assert db.get(Model, ids[4]).abilities == (
            ["chat", "tool_calling"] if retained_grant else ["chat"]
        )


def test_compatible_ability_edit_refreshes_auto_without_locking_credentials(env):
    t = target(env, env.consumer, abilities=["chat", "tool_calling"])
    cfg = configure(env, env.consumer, [("text-a", t)])
    asyncio.run(
        model_api.update_model(
            t.model_id,
            ModelUpdate(
                abilities=["chat"], api_key="new-fake-key", base_url=t.base_url
            ),
            db=env.db,
            user=env.consumer,
        )
    )
    env.db.expire_all()
    assert env.db.get(Model, cfg.router_model_id).abilities == ["chat"]
    assert t.api_key == "new-fake-key"


def test_pruning_candidate_recalculates_remaining_abilities(env):
    shared = target(env, env.owner, name="shared", shared=True)
    own = target(env, env.consumer, name="own", abilities=["chat", "tool_calling"])
    cfg = configure(env, env.consumer, [("text-a", shared), ("text-b", own)])
    assert "tool_calling" not in cfg.router_model.abilities
    asyncio.run(
        model_api.update_model(
            shared.model_id,
            ModelUpdate(share_with_users=False),
            db=env.db,
            user=env.owner,
        )
    )
    env.db.expire_all()
    assert env.db.get(Model, cfg.router_model_id).abilities == ["chat", "tool_calling"]


def test_editing_model_bound_to_unknown_profile_is_not_blocked(env):
    t = target(env, env.consumer)
    cfg = configure(env, env.consumer, [("text-a", t)])
    cfg.candidates[0].routing_model_id = "removed-profile"
    env.db.commit()
    # Routing skips this binding while its profile is not in the installed
    # catalog, so the edit is not checked against it.
    asyncio.run(
        model_api.update_model(
            t.model_id,
            ModelUpdate(abilities=["chat", "vision"]),
            db=env.db,
            user=env.consumer,
        )
    )
    env.db.expire_all()
    assert env.db.get(Model, t.id).abilities == ["chat", "vision"]
    with pytest.raises(AutoModelUnavailableError, match="no active configured"):
        UserAwareModelStorage(env.db).get_llm_by_id(
            cfg.router_model.model_id, env.consumer.id
        )


@pytest.mark.parametrize("unknown_first", [True, False])
def test_ability_edit_skips_only_the_unknown_profile_binding(env, unknown_first):
    # Two users bind the same model. A shared model's abilities cannot change
    # at all, so the consumer reaches the owner's model through a direct,
    # unshared grant row instead.
    t = target(env, env.owner)
    env.db.add(UserModel(user_id=env.consumer.id, model_id=t.id, is_owner=True))
    env.db.commit()
    users = [env.consumer, env.owner] if unknown_first else [env.owner, env.consumer]
    configs = {user.id: configure(env, user, [("text-a", t)]) for user in users}
    configs[env.consumer.id].candidates[0].routing_model_id = "removed-profile"
    env.db.commit()
    # update_model walks the bindings in this query's order; pin it so each
    # case really puts the unknown binding where its name says.
    order = [
        candidate.routing_model_id
        for candidate in env.db.query(AutoModelCandidate)
        .filter(AutoModelCandidate.target_model_id == t.id)
        .all()
    ]
    assert order == (
        ["removed-profile", "text-a"]
        if unknown_first
        else ["text-a", "removed-profile"]
    )

    with pytest.raises(HTTPException) as error:
        asyncio.run(
            model_api.update_model(
                t.model_id,
                ModelUpdate(abilities=["chat", "vision"]),
                db=env.db,
                user=env.owner,
            )
        )
    assert error.value.status_code == 409
    assert "input modalities" in str(error.value.detail)
    env.db.rollback()
    assert t.abilities == ["chat"]


def _bundled_auto_router_models() -> tuple[str, ...]:
    pytest.importorskip("xrouter_llm")
    from xrouter_llm import default_routers_dir
    from xrouter_llm.serving import load_router_configs

    configs = load_router_configs(default_routers_dir())
    return tuple(configs[auto_model_service.AUTO_ROUTER_CONFIG_NAME].models)


def test_bundled_auto_router_profiles_pass_candidate_validation():
    """Every profile in xrouter-llm's own ``auto`` router config can be bound.

    Uses the installed catalog, not the stub the ``env`` fixture injects.
    """
    auto_models = _bundled_auto_router_models()
    catalog = auto_model_service.load_router_profile_catalog()
    known = set(catalog.known_model_ids())

    assert auto_models
    assert [profile_id for profile_id in auto_models if profile_id not in known] == []
    for profile_id in auto_models:
        modalities = set(catalog.get(profile_id).input_modalities)
        abilities = [
            ability
            for modality, ability in _MODALITY_ABILITIES.items()
            if modality in modalities
        ]
        auto_model_service.validate_candidate_modalities(catalog, profile_id, abilities)


def test_bundled_catalog_has_the_profiles_saved_auto_configs_rebind_across():
    """Pin the catalog changes at the current xrouter-llm floor that saved Auto
    configurations have to be rebound across; update together with that floor.
    """
    auto_models = set(_bundled_auto_router_models())
    catalog = auto_model_service.load_router_profile_catalog()
    known = set(catalog.known_model_ids())
    removed = {"deepseek/deepseek-v4-flash-0731", "deepseek/deepseek-v4-pro-0813"}
    added = {
        "anthropic/claude-fable-5.1",
        "anthropic/claude-opus-5.5",
        "anthropic/claude-sonnet-5.5",
        "deepseek/deepseek-v4.1-flash",
        "google/gemini-3.8-flash",
        "openai/gpt-6-astra",
        "openai/gpt-6-luna",
        "openai/gpt-6.1-sol",
        "qwen/qwen3.8-max-0902",
        "qwen/qwen3.8-omni-flash",
        "x-ai/grok-4.7",
        "xiaomi/mimo-v2.6-flash",
        "xiaomi/mimo-v2.6-pro",
        "z-ai/glm-5.3",
        "z-ai/glm-5.3-flashx",
    }

    assert known.isdisjoint(removed)
    assert added <= known
    assert set(catalog.get("deepseek/deepseek-v4.1-flash").input_modalities) == {
        "text",
        "image",
    }
    assert auto_models.isdisjoint(
        removed | {"google/gemini-3.5-flash", "google/gemini-3.7-flash"}
    )
    assert {
        "deepseek/deepseek-v4.1-flash",
        "google/gemini-3.8-flash",
        "openai/gpt-6-astra",
        "qwen/qwen3.8-max-0902",
        "qwen/qwen3.8-omni-flash",
        "z-ai/glm-5.3",
    } <= auto_models